Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,7 @@ impl CompareKernel for DecimalByteParts {
.decimal_value()
.vortex_expect("checked for null in entry func");

match decimal_value_wrapper_to_primitive(
rhs_decimal,
lhs.msp().as_primitive_typed().ptype(),
) {
match decimal_value_wrapper_to_primitive(rhs_decimal, lhs.msp().dtype().as_ptype()) {
Ok(value) => {
let encoded_scalar = Scalar::try_new(scalar_type, Some(value))?;
let encoded_const = ConstantArray::new(encoded_scalar, rhs.len());
Expand Down
21 changes: 9 additions & 12 deletions encodings/runend/src/array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,6 @@ use vortex_array::dtype::DType;
use vortex_array::dtype::Nullability;
use vortex_array::dtype::PType;
use vortex_array::legacy_session;
use vortex_array::scalar::PValue;
use vortex_array::search_sorted::SearchSorted;
use vortex_array::search_sorted::SearchSortedSide;
use vortex_array::serde::ArrayChildren;
use vortex_array::smallvec::smallvec;
use vortex_array::validity::Validity;
Expand All @@ -48,6 +45,8 @@ use crate::compress::runend_decode_primitive;
use crate::compress::runend_decode_varbinview;
use crate::compress::runend_encode;
use crate::decompress_bool::runend_decode_bools;
use crate::ops::find_physical_index;
use crate::ops::find_slice_end_index;
use crate::rules::RULES;

/// A [`RunEnd`]-encoded Vortex array.
Expand Down Expand Up @@ -230,17 +229,15 @@ pub trait RunEndArrayExt: TypedArrayRef<RunEnd> {
self.values().dtype()
}

fn find_physical_index(&self, index: usize) -> VortexResult<usize> {
Ok(self
.ends()
.as_primitive_typed()
.search_sorted(
&PValue::from(index + self.offset()),
SearchSortedSide::Right,
)?
.to_ends_index(self.ends().len()))
fn find_physical_index(&self, index: usize, ctx: &mut ExecutionCtx) -> VortexResult<usize> {
find_physical_index(self.ends(), index + self.offset(), ctx)
}

fn find_slice_end_index(&self, index: usize, ctx: &mut ExecutionCtx) -> VortexResult<usize> {
find_slice_end_index(self.ends(), index + self.offset(), ctx)
}
}

impl<T: TypedArrayRef<RunEnd>> RunEndArrayExt for T {}

#[derive(Clone, Debug)]
Expand Down
38 changes: 12 additions & 26 deletions encodings/runend/src/arrow.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,12 @@ use vortex_array::arrays::primitive::PrimitiveArrayExt;
use vortex_array::arrow::FromArrowArray;
use vortex_array::dtype::NativePType;
use vortex_array::legacy_session;
use vortex_array::scalar::PValue;
use vortex_array::search_sorted::SearchSorted;
use vortex_array::search_sorted::SearchSortedSide;
use vortex_array::validity::Validity;
use vortex_buffer::Buffer;
use vortex_error::VortexResult;

use crate::RunEndData;
use crate::ops::find_physical_index;
use crate::ops::find_slice_end_index;

impl<R: RunEndIndexType> FromArrowArray<&RunArray<R>> for RunEndData
Expand All @@ -41,14 +39,13 @@ where
ends.validity()?,
)
.into_array();
let mut ctx = legacy_session().create_execution_ctx();

let (ends_slice, values_slice) = if offset == 0 && len == array.run_ends().max_value() {
(ends_array, values)
} else {
let slice_begin = ends_array
.as_primitive_typed()
.search_sorted(&PValue::from(offset), SearchSortedSide::Right)?
.to_ends_index(ends_array.len());
let slice_end = find_slice_end_index(&ends_array, offset + len)?;
let slice_begin = find_physical_index(&ends_array, offset, &mut ctx)?;
let slice_end = find_slice_end_index(&ends_array, offset + len, &mut ctx)?;

(
ends_array.slice(slice_begin..slice_end)?,
Expand All @@ -57,8 +54,6 @@ where
};

// SAFETY: arrow-rs enforces the RunEndArray invariants, we inherit their guarantees.
// TODO(ctx): trait fixes - FromArrowArray::from_arrow has a fixed signature.
let mut ctx = legacy_session().create_execution_ctx();
RunEndData::validate_parts(&ends_slice, &values_slice, offset, len, &mut ctx)?;
Ok(unsafe { RunEndData::new_unchecked(offset) })
}
Expand Down Expand Up @@ -92,16 +87,15 @@ mod tests {
use vortex_array::dtype::NativePType;
use vortex_array::dtype::Nullability;
use vortex_array::dtype::PType;
use vortex_array::scalar::PValue;
use vortex_array::search_sorted::SearchSorted;
use vortex_array::search_sorted::SearchSortedSide;
use vortex_array::validity::Validity;
use vortex_buffer::Buffer;
use vortex_buffer::buffer;
use vortex_error::VortexResult;
use vortex_session::VortexSession;

use crate::RunEnd;
use crate::RunEndArray;
use crate::ops::find_physical_index;
use crate::ops::find_slice_end_index;

static SESSION: LazyLock<VortexSession> = LazyLock::new(|| {
Expand All @@ -113,7 +107,7 @@ mod tests {
fn decode_run_array<R: RunEndIndexType>(
array: &RunArray<R>,
nullable: bool,
) -> VortexResult<crate::RunEndArray>
) -> VortexResult<RunEndArray>
where
R::Native: NativePType,
{
Expand All @@ -131,28 +125,20 @@ mod tests {
ends.validity()?,
)
.into_array();
let mut ctx = SESSION.create_execution_ctx();
let (ends_slice, values_slice) = if offset == 0 && len == array.run_ends().max_value() {
(ends_array, values)
} else {
let slice_begin = ends_array
.as_primitive_typed()
.search_sorted(&PValue::from(offset), SearchSortedSide::Right)?
.to_ends_index(ends_array.len());
let slice_end = find_slice_end_index(&ends_array, offset + len)?;
let slice_begin = find_physical_index(&ends_array, offset, &mut ctx)?;
let slice_end = find_slice_end_index(&ends_array, offset + len, &mut ctx)?;

(
ends_array.slice(slice_begin..slice_end)?,
values.slice(slice_begin..slice_end)?,
)
};

RunEnd::try_new_offset_length(
ends_slice,
values_slice,
offset,
array.len(),
&mut SESSION.create_execution_ctx(),
)
RunEnd::try_new_offset_length(ends_slice, values_slice, offset, array.len(), &mut ctx)
}

#[test]
Expand Down
4 changes: 2 additions & 2 deletions encodings/runend/src/kernel.rs
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,8 @@ fn slice(
) -> VortexResult<ArrayRef> {
let new_length = range.len();

let slice_begin = array.find_physical_index(range.start)?;
let slice_end = crate::ops::find_slice_end_index(array.ends(), range.end + array.offset())?;
let slice_begin = array.find_physical_index(range.start, ctx)?;
let slice_end = array.find_slice_end_index(range.end, ctx)?;

// If the sliced range contains only a single run, opt to return a ConstantArray.
if slice_begin + 1 == slice_end {
Expand Down
33 changes: 25 additions & 8 deletions encodings/runend/src/ops.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,11 @@
use vortex_array::ArrayRef;
use vortex_array::ArrayView;
use vortex_array::ExecutionCtx;
use vortex_array::scalar::PValue;
use vortex_array::match_each_unsigned_integer_ptype;
use vortex_array::scalar::Scalar;
use vortex_array::search_sorted::SearchResult;
use vortex_array::search_sorted::SearchSorted;
use vortex_array::search_sorted::SearchSortedPrimitiveArray;
use vortex_array::search_sorted::SearchSortedSide;
use vortex_array::vtable::OperationsVTable;
use vortex_error::VortexResult;
Expand All @@ -21,20 +22,24 @@ impl OperationsVTable<RunEnd> for RunEnd {
index: usize,
ctx: &mut ExecutionCtx,
) -> VortexResult<Scalar> {
array
.values()
.execute_scalar(array.find_physical_index(index)?, ctx)
let physical_index = array.find_physical_index(index, ctx)?;
array.values().execute_scalar(physical_index, ctx)
}
}

/// Find the physical offset for and index that would be an end of the slice i.e., one past the last element.
///
/// If the index exists in the array we want to take that position (as we are searching from the right)
/// otherwise we want to take the next one
pub(crate) fn find_slice_end_index(array: &ArrayRef, index: usize) -> VortexResult<usize> {
let result = array
.as_primitive_typed()
.search_sorted(&PValue::from(index), SearchSortedSide::Right)?;
pub(crate) fn find_slice_end_index(
Comment thread
robert3005 marked this conversation as resolved.
array: &ArrayRef,
index: usize,
ctx: &mut ExecutionCtx,
) -> VortexResult<usize> {
let result = match_each_unsigned_integer_ptype!(array.dtype().as_ptype(), |T| {
SearchSortedPrimitiveArray::<T>::new(array, ctx)
.search_sorted(&index, SearchSortedSide::Right)?
});
Ok(match result {
SearchResult::Found(i) => i,
SearchResult::NotFound(i) => {
Expand All @@ -47,6 +52,18 @@ pub(crate) fn find_slice_end_index(array: &ArrayRef, index: usize) -> VortexResu
})
}

pub(crate) fn find_physical_index(
array: &ArrayRef,
Comment thread
robert3005 marked this conversation as resolved.
index: usize,
ctx: &mut ExecutionCtx,
) -> VortexResult<usize> {
match_each_unsigned_integer_ptype!(array.dtype().as_ptype(), |T| {
Ok(SearchSortedPrimitiveArray::<T>::new(array, ctx)
.search_sorted(&index, SearchSortedSide::Right)?
.to_ends_index(array.len()))
})
}

#[cfg(test)]
mod tests {
use std::sync::LazyLock;
Expand Down
9 changes: 7 additions & 2 deletions vortex-array/benches/listview_zip.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,23 +3,28 @@

#![expect(clippy::unwrap_used)]

use std::sync::LazyLock;

use divan::Bencher;
use vortex_array::ArrayRef;
use vortex_array::IntoArray;
use vortex_array::LEGACY_SESSION;
use vortex_array::RecursiveCanonical;
use vortex_array::VortexSessionExecute;
use vortex_array::array_session;
use vortex_array::arrays::BoolArray;
use vortex_array::arrays::ListViewArray;
use vortex_array::builtins::ArrayBuiltins;
use vortex_array::validity::Validity;
use vortex_buffer::BufferMut;
use vortex_mask::Mask;
use vortex_session::VortexSession;

fn main() {
divan::main();
}

static SESSION: LazyLock<VortexSession> = LazyLock::new(array_session);

// Smaller than the value-path benches: listview zip cost is dominated by element concatenation and
// per-list canonicalization. A few thousand lists already exercise the select while keeping each
// case well under a few hundred microseconds under CodSpeed's instruction-count simulation, which
Expand Down Expand Up @@ -50,7 +55,7 @@ fn run(bencher: Bencher, if_true: ArrayRef, if_false: ArrayRef) {
if_true.clone(),
if_false.clone(),
mask.clone().into_array(),
LEGACY_SESSION.create_execution_ctx(),
SESSION.create_execution_ctx(),
)
})
.bench_refs(|(t, f, m, ctx)| {
Expand Down
2 changes: 1 addition & 1 deletion vortex-array/src/arrays/bool/compute/zip.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ impl ZipKernel for Bool {
};

// Null mask entries select `if_false`, matching `Zip`'s SQL ELSE semantics.
let mask = mask.try_to_mask_fill_null_false(ctx)?;
let mask = mask.clone().null_as_false().execute(ctx)?;
let mask_values = match &mask {
// Defer trivial masks to the generic zip, which just casts the surviving side.
Mask::AllTrue(_) | Mask::AllFalse(_) => return Ok(None),
Expand Down
2 changes: 1 addition & 1 deletion vortex-array/src/arrays/listview/compute/zip.rs
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ impl ZipKernel for ListView {
};

// Null mask entries select `if_false`, matching `Zip`'s SQL ELSE semantics.
let mask = mask.try_to_mask_fill_null_false(ctx)?;
let mask = mask.clone().null_as_false().execute(ctx)?;
match &mask {
// Defer the trivial masks to the generic zip, which just casts one side.
Mask::AllTrue(_) | Mask::AllFalse(_) => return Ok(None),
Expand Down
2 changes: 1 addition & 1 deletion vortex-array/src/arrays/primitive/compute/zip.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ impl ZipKernel for Primitive {
}

// Null mask entries select `if_false`, matching `Zip`'s SQL ELSE semantics.
let mask = mask.try_to_mask_fill_null_false(ctx)?;
let mask = mask.clone().null_as_false().execute(ctx)?;
match &mask {
// Defer trivial masks to the generic zip, which just casts the surviving side.
Mask::AllTrue(_) | Mask::AllFalse(_) => return Ok(None),
Expand Down
14 changes: 10 additions & 4 deletions vortex-array/src/arrays/primitive/tests.rs
Original file line number Diff line number Diff line change
@@ -1,33 +1,39 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use std::sync::LazyLock;

use vortex_buffer::buffer;
use vortex_session::VortexSession;

use crate::ArrayRef;
use crate::IntoArray;
use crate::array_session;
use crate::arrays::BoolArray;
use crate::arrays::PrimitiveArray;
use crate::compute::conformance::filter::test_filter_conformance;
use crate::compute::conformance::mask::test_mask_conformance;
use crate::compute::conformance::search_sorted::rstest_reuse::apply;
use crate::compute::conformance::search_sorted::search_sorted_conformance;
use crate::compute::conformance::search_sorted::*;
use crate::scalar::PValue;
use crate::executor::VortexSessionExecute;
use crate::search_sorted::SearchResult;
use crate::search_sorted::SearchSorted;
use crate::search_sorted::SearchSortedPrimitiveArray;
use crate::search_sorted::SearchSortedSide;
use crate::validity::Validity;

static SESSION: LazyLock<VortexSession> = LazyLock::new(array_session);

#[apply(search_sorted_conformance)]
fn test_search_sorted_primitive(
#[case] array: ArrayRef,
#[case] value: i32,
#[case] side: SearchSortedSide,
#[case] expected: SearchResult,
) -> vortex_error::VortexResult<()> {
let res = array
.as_primitive_typed()
.search_sorted(&Some(PValue::from(value)), side)?;
let res = SearchSortedPrimitiveArray::<i32>::new(&array, &mut SESSION.create_execution_ctx())
.search_sorted(&value, side)?;
assert_eq!(res, expected);
Ok(())
}
Expand Down
6 changes: 3 additions & 3 deletions vortex-array/src/arrays/struct_/compute/zip.rs
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ impl ZipKernel for Struct {
let fields = if_true
.iter_unmasked_fields()
.zip(if_false.iter_unmasked_fields())
.map(|(t, f)| ArrayBuiltins::zip(mask, t.clone(), f.clone()))
.map(|(t, f)| mask.zip(t.clone(), f.clone()))
.collect::<VortexResult<Vec<_>>>()?;

let v1 = if_true.validity()?;
Expand All @@ -48,11 +48,11 @@ impl ZipKernel for Struct {
(Validity::AllInvalid, Validity::AllInvalid) => Validity::AllInvalid,

(v1, v2) => {
let mask_mask = mask.try_to_mask_fill_null_false(ctx)?;
let mask_mask = mask.clone().null_as_false().execute(ctx)?;
let v1m = v1.execute_mask(if_true.len(), ctx)?;
let v2m = v2.execute_mask(if_false.len(), ctx)?;

let combined = (v1m.bitand(&mask_mask)).bitor(&v2m.bitand(&mask_mask.not()));
let combined = v1m.bitand(&mask_mask).bitor(&v2m.bitand(&mask_mask.not()));
Validity::from_mask(
combined,
if_true.dtype().nullability() | if_false.dtype().nullability(),
Expand Down
2 changes: 1 addition & 1 deletion vortex-array/src/arrays/varbinview/compute/zip.rs
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ impl ZipKernel for VarBinView {
let true_validity = if_true.varbinview_validity().execute_mask(len, ctx)?;
let false_validity = if_false.varbinview_validity().execute_mask(len, ctx)?;

let mask = mask.try_to_mask_fill_null_false(ctx)?;
let mask = mask.clone().null_as_false().execute(ctx)?;
let if_false_view = if_false;
match mask.slices() {
AllOr::All => push_range(
Expand Down
2 changes: 1 addition & 1 deletion vortex-array/src/compute/conformance/cast.rs
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,7 @@ fn test_cast_to_nullable(array: &ArrayRef) {
}

fn test_cast_from_floating_point_types(array: &ArrayRef) {
let ptype = array.as_primitive_typed().ptype();
let ptype = array.dtype().as_ptype();
test_cast_to_primitive(array, PType::I8, false);
test_cast_to_primitive(array, PType::U8, false);
test_cast_to_primitive(array, PType::I16, false);
Expand Down
1 change: 0 additions & 1 deletion vortex-array/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,6 @@ pub mod stream;
#[cfg(any(test, feature = "_test-harness"))]
pub mod test_harness;
pub mod validity;
pub mod variants;

pub mod flatbuffers {
//! Re-exported autogenerated code from the core Vortex flatbuffer definitions.
Expand Down
Loading
Loading