Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 19 additions & 2 deletions client/python/USAGE.md
Original file line number Diff line number Diff line change
Expand Up @@ -228,12 +228,29 @@ validation and protobuf conversion internally
### `DenseVector`

```
DenseVector(values: list[float] | tuple[float, ...])
DenseVector(values: list[float] | tuple[float, ...] | array | tensor)
```
- Validates numeric input
- Accepts 1D NumPy arrays and PyTorch, TensorFlow, or JAX tensors
- Rejects arrays and tensors that are not 1D
- Normalizes values to `float`
- Immutable (`frozen=True`)

Input conversion uses the capabilities exposed by the value rather than requiring
NumPy, PyTorch, TensorFlow, or JAX as client dependencies.

```python
vector = DenseVector(numpy_array)

vector.to_list()
vector.to_numpy()
vector.to_torch()
vector.to_tensorflow()
vector.to_jax()
```

Each reverse conversion helper requires its corresponding library to be installed.

---

### `Payload`
Expand Down Expand Up @@ -374,4 +391,4 @@ python -m grpc_tools.protoc \

After running this:
- `vector_db_pb2_grpc.py` and `vector_db_pb2.py` will be updated
- No other client code should need changes
- No other client code should need changes
150 changes: 146 additions & 4 deletions client/python/tests/test_models.py
Original file line number Diff line number Diff line change
@@ -1,29 +1,106 @@
from dataclasses import FrozenInstanceError

import pytest

from vortexdb.grpc import vector_db_pb2
from vortexdb.models import (
ContentType,
DenseVector,
Payload,
Point,
Similarity,
ContentType,
)

from vortexdb.grpc import vector_db_pb2

# DenseVector Tests


def test_dense_vector_valid():
a = [1, 2.5, 3]
v = DenseVector(a)
assert v.values == [1.0, 2.5, 3.0]
assert all(isinstance(value, float) for value in v.values)


def test_dense_vector_accepts_tuple():
v = DenseVector((1, 2, 3))
assert v.values == [1.0, 2.0, 3.0]


def test_dense_vector_accepts_numpy_array():
numpy = pytest.importorskip("numpy")
v = DenseVector(numpy.array([1, 2.5, 3], dtype=numpy.float32))
assert v.values == [1.0, 2.5, 3.0]


def test_dense_vector_accepts_numpy_scalars_in_list():
numpy = pytest.importorskip("numpy")
v = DenseVector([numpy.float32(1.5), numpy.float32(2.5)])
assert v.values == [1.5, 2.5]
assert all(type(value) is float for value in v.values)


def test_dense_vector_accepts_pytorch_tensor():
torch = pytest.importorskip("torch")
v = DenseVector(torch.tensor([1, 2.5, 3]))
assert v.values == [1.0, 2.5, 3.0]


def test_dense_vector_accepts_tensorflow_tensor():
tensorflow = pytest.importorskip("tensorflow")
v = DenseVector(tensorflow.constant([1.0, 2.5, 3.0]))
assert v.values == [1.0, 2.5, 3.0]


def test_dense_vector_accepts_jax_array():
jax_numpy = pytest.importorskip("jax.numpy")
v = DenseVector(jax_numpy.array([1, 2.5, 3]))
assert v.values == [1.0, 2.5, 3.0]


class _TensorLike:
def tolist(self):
return [1, 2.5, 3]


def test_dense_vector_accepts_tensor_with_tolist():
v = DenseVector(_TensorLike())
assert v.values == [1.0, 2.5, 3.0]


class _TensorShape:
rank = 1


class _MaterializedTensor:
def tolist(self):
return [1, 2.5, 3]


class _TensorFlowLike:
shape = _TensorShape()

def numpy(self):
return _MaterializedTensor()


def test_dense_vector_accepts_tensor_with_numpy_conversion():
v = DenseVector(_TensorFlowLike())
assert v.values == [1.0, 2.5, 3.0]


class _JaxLike:
ndim = 1

def __array__(self):
numpy = pytest.importorskip("numpy")
return numpy.array([1, 2.5, 3])


def test_dense_vector_accepts_tensor_with_array_protocol():
v = DenseVector(_JaxLike())
assert v.values == [1.0, 2.5, 3.0]


def test_dense_vector_rejects_empty():
with pytest.raises(ValueError):
DenseVector([])
Expand All @@ -34,9 +111,39 @@ def test_dense_vector_rejects_non_numeric():
DenseVector([1, "a", 3])


@pytest.mark.parametrize("values", [[[1, 2], [3, 4]], [[1, 2, 3]]])
def test_dense_vector_rejects_nested_sequences(values):
with pytest.raises(ValueError, match="one-dimensional"):
DenseVector(values)


def test_dense_vector_rejects_multidimensional_array():
numpy = pytest.importorskip("numpy")
with pytest.raises(ValueError, match="one-dimensional"):
DenseVector(numpy.array([[1, 2], [3, 4]]))


def test_dense_vector_rejects_scalar_array():
numpy = pytest.importorskip("numpy")
with pytest.raises(ValueError, match="one-dimensional"):
DenseVector(numpy.array(1.0))


class _MatrixLike:
ndim = 2

def tolist(self):
return [[1, 2], [3, 4]]


def test_dense_vector_rejects_multidimensional_tensor_like_value():
with pytest.raises(ValueError, match="one-dimensional"):
DenseVector(_MatrixLike())


def test_dense_vector_is_frozen():
v = DenseVector([1, 2, 3])
with pytest.raises(Exception):
with pytest.raises(FrozenInstanceError):
v.values = [4, 5, 6]


Expand All @@ -46,6 +153,41 @@ def test_dense_vector_to_proto():
assert list(proto.values) == [1.0, 2.0, 3.0]


def test_dense_vector_to_numpy():
numpy = pytest.importorskip("numpy")
result = DenseVector([1, 2.5, 3]).to_numpy()
assert isinstance(result, numpy.ndarray)
assert result.ndim == 1
assert numpy.issubdtype(result.dtype, numpy.floating)
assert result.tolist() == [1.0, 2.5, 3.0]


def test_dense_vector_to_torch():
torch = pytest.importorskip("torch")
result = DenseVector([1, 2.5, 3]).to_torch()
assert isinstance(result, torch.Tensor)
assert result.ndim == 1
assert result.dtype.is_floating_point
assert result.tolist() == [1.0, 2.5, 3.0]


def test_dense_vector_to_tensorflow():
tensorflow = pytest.importorskip("tensorflow")
result = DenseVector([1, 2.5, 3]).to_tensorflow()
assert tensorflow.is_tensor(result)
assert result.shape.rank == 1
assert result.dtype.is_floating
assert result.numpy().tolist() == [1.0, 2.5, 3.0]


def test_dense_vector_to_jax():
jax_numpy = pytest.importorskip("jax.numpy")
result = DenseVector([1, 2.5, 3]).to_jax()
assert result.ndim == 1
assert jax_numpy.issubdtype(result.dtype, jax_numpy.floating)
assert result.tolist() == [1.0, 2.5, 3.0]


# Similarity Test


Expand Down
102 changes: 90 additions & 12 deletions client/python/vortexdb/models.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
from dataclasses import dataclass
from enum import Enum
from typing import List
from vortexdb.grpc import vector_db_pb2
from numbers import Real
from typing import Any

from vortexdb.grpc import vector_db_pb2

# I found this to be a good idea, because
# 1. readability
Expand Down Expand Up @@ -43,32 +44,109 @@ def from_proto(value: int) -> "ContentType":
}[value]


# TODO Extend support to other data types than lists or tuples (numpy arrays probably)
# TODO Further compatibility to allow conversions directly to numpy arrays (similar to .to_list())
@dataclass(frozen=True)
class DenseVector:
values: List[float]
values: list[float]

def __post_init__(self):
if not isinstance(self.values, (list, tuple)):
raise TypeError("DenseVector expects a list or tuple of floats")
if isinstance(self.values, (list, tuple)):
normalized_values = list(self.values)
else:
normalized_values = self._array_like_to_list(self.values)

if not self.values:
if not normalized_values:
raise ValueError("DenseVector cannot be empty")

for v in self.values:
if not isinstance(v, (int, float)):
if any(isinstance(value, (list, tuple)) for value in normalized_values):
raise ValueError("DenseVector expects a one-dimensional vector")

for v in normalized_values:
if not isinstance(v, Real):
raise TypeError("DenseVector values must be numeric (int or float)")

# force float normalization
object.__setattr__(self, "values", [float(v) for v in self.values])
object.__setattr__(self, "values", [float(v) for v in normalized_values])

@staticmethod
def _array_like_to_list(values: Any) -> list[Any]:
shape = getattr(values, "shape", None)
rank = getattr(values, "ndim", None)

if rank is None and shape is not None:
rank = getattr(shape, "rank", None)
if rank is None:
try:
rank = len(shape)
except (TypeError, ValueError):
pass

if rank is not None and rank != 1:
raise ValueError("DenseVector expects a one-dimensional vector")

to_list = getattr(values, "tolist", None)
if callable(to_list):
converted = to_list()
else:
to_numpy = getattr(values, "numpy", None)
if callable(to_numpy):
converted = to_numpy()
else:
to_array = getattr(values, "__array__", None)
if not callable(to_array):
raise TypeError("DenseVector could not convert the array or tensor")
converted = to_array()

converted_to_list = getattr(converted, "tolist", None)
if not callable(converted_to_list):
raise TypeError("DenseVector could not convert the array or tensor")
converted = converted_to_list()

if not isinstance(converted, list):
raise TypeError("DenseVector could not convert the array or tensor")
return converted

def to_proto(self) -> vector_db_pb2.DenseVector:
return vector_db_pb2.DenseVector(values=self.values)

def to_list(self) -> list[float]:
return list(self.values)

# Keep framework dependencies optional by importing only when conversion is requested.
def to_numpy(self) -> Any:
try:
import numpy
except ImportError as error:
raise ImportError(
"NumPy is required to convert DenseVector to an array"
) from error
return numpy.asarray(self.values, dtype=float)

def to_torch(self) -> Any:
try:
import torch
except ImportError as error:
raise ImportError(
"PyTorch is required to convert DenseVector to a tensor"
) from error
return torch.tensor(self.values)

def to_tensorflow(self) -> Any:
try:
import tensorflow
except ImportError as error:
raise ImportError(
"TensorFlow is required to convert DenseVector to a tensor"
) from error
return tensorflow.convert_to_tensor(self.values)

def to_jax(self) -> Any:
try:
import jax.numpy
except ImportError as error:
raise ImportError(
"JAX is required to convert DenseVector to an array"
) from error
return jax.numpy.asarray(self.values)


# & Helper Function for Batch of DenseVectors
def to_dense_vectors(arr):
Expand Down
Loading
Loading