diff --git a/encodings/experimental/onpair/src/array.rs b/encodings/experimental/onpair/src/array.rs index f9e18c36ac4..106eb409de1 100644 --- a/encodings/experimental/onpair/src/array.rs +++ b/encodings/experimental/onpair/src/array.rs @@ -482,7 +482,8 @@ impl VTable for OnPair { .clone() .execute::(ctx)? .into_array(), - ); + ctx, + )?; return Ok(()); }; diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs index eba5be1b749..c77f75cf0ba 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs @@ -519,7 +519,7 @@ mod test { let mut primitive_builder = PrimitiveBuilder::::with_capacity(chunked.dtype().nullability(), 10 * 100); - primitive_builder.extend_from_array(&chunked); + primitive_builder.extend_from_array(&chunked, &mut ctx)?; let ca_into = primitive_builder.finish(); assert_arrays_eq!(into_ca, ca_into); diff --git a/encodings/fsst/src/array.rs b/encodings/fsst/src/array.rs index 05a07b34c4d..218e8b78413 100644 --- a/encodings/fsst/src/array.rs +++ b/encodings/fsst/src/array.rs @@ -292,7 +292,8 @@ impl VTable for FSST { .clone() .execute::(ctx)? .into_array(), - ); + ctx, + )?; return Ok(()); }; diff --git a/encodings/sparse/src/canonical.rs b/encodings/sparse/src/canonical.rs index dd84123851a..17b4241f293 100644 --- a/encodings/sparse/src/canonical.rs +++ b/encodings/sparse/src/canonical.rs @@ -239,6 +239,7 @@ fn execute_sparse_lists_inner( &mut builder, fill_elements.as_ref(), sparse_idx - next_index, + ctx, ); if patch_valid { @@ -246,7 +247,7 @@ fn execute_sparse_lists_inner( .list_elements_at(patch_idx) .vortex_expect("list_elements_at"); builder - .append_array_as_list(&patch_list) + .append_array_as_list(&patch_list, ctx) .vortex_expect("Failed to append sparse value"); } else { builder.append_null(); @@ -255,7 +256,7 @@ fn execute_sparse_lists_inner( next_index = sparse_idx + 1; } - append_list_fill(&mut builder, fill_elements.as_ref(), len - next_index); + append_list_fill(&mut builder, fill_elements.as_ref(), len - next_index, ctx); builder.finish() } @@ -328,6 +329,7 @@ fn execute_sparse_fixed_size_list_inner( &mut builder, fill_elements.as_ref(), sparse_idx - next_index, + ctx, ); // Append the patch value, handling null patches by appending defaults. @@ -336,7 +338,7 @@ fn execute_sparse_fixed_size_list_inner( .fixed_size_list_elements_at(patch_idx) .vortex_expect("fixed_size_list_elements_at"); builder - .append_array_as_list(&patch_list) + .append_array_as_list(&patch_list, ctx) .vortex_expect("Failed to append sparse fixed-size-list value"); } else { builder.append_null(); @@ -346,7 +348,12 @@ fn execute_sparse_fixed_size_list_inner( } // Fill remaining positions after last patch. - append_fixed_size_list_fill(&mut builder, fill_elements.as_ref(), array_len - next_index); + append_fixed_size_list_fill( + &mut builder, + fill_elements.as_ref(), + array_len - next_index, + ctx, + ); builder.finish_into_fixed_size_list() } @@ -367,11 +374,12 @@ fn append_list_fill( builder: &mut ListViewBuilder, fill_elements: Option<&ArrayRef>, count: usize, + ctx: &mut ExecutionCtx, ) { if let Some(fill_elements) = fill_elements { for _ in 0..count { builder - .append_array_as_list(fill_elements) + .append_array_as_list(fill_elements, ctx) .vortex_expect("Failed to append sparse fill value"); } } else { @@ -383,11 +391,12 @@ fn append_fixed_size_list_fill( builder: &mut FixedSizeListBuilder, fill_elements: Option<&ArrayRef>, count: usize, + ctx: &mut ExecutionCtx, ) { if let Some(fill_elements) = fill_elements { for _ in 0..count { builder - .append_array_as_list(fill_elements) + .append_array_as_list(fill_elements, ctx) .vortex_expect("Failed to append sparse fixed-size-list fill value"); } } else { diff --git a/vortex-array/benches/listview_builder_extend.rs b/vortex-array/benches/listview_builder_extend.rs index 75487564ea8..5aa80fa3faf 100644 --- a/vortex-array/benches/listview_builder_extend.rs +++ b/vortex-array/benches/listview_builder_extend.rs @@ -8,6 +8,8 @@ use std::sync::Arc; use divan::Bencher; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::ListViewArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::builders::ArrayBuilder; @@ -18,6 +20,7 @@ use vortex_array::dtype::Nullability::Nullable; use vortex_array::dtype::PType::I32; use vortex_array::validity::Validity; use vortex_buffer::Buffer; +use vortex_error::VortexExpect; fn main() { divan::main(); @@ -69,7 +72,9 @@ fn extend_from_array_zctl(bencher: Bencher, (num_lists, list_size): (usize, usiz num_lists * list_size, num_lists, ); - builder.extend_from_array(source); + builder + .extend_from_array(source, &mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("extend_from_array"); divan::black_box(builder.finish_into_listview()) }); } @@ -91,7 +96,9 @@ fn extend_from_array_non_zctl_overlapping( num_lists * list_size, num_lists, ); - builder.extend_from_array(source); + builder + .extend_from_array(source, &mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("extend_from_array"); divan::black_box(builder.finish_into_listview()) }); } diff --git a/vortex-array/src/aggregate_fn/fns/uncompressed_size_in_bytes/mod.rs b/vortex-array/src/aggregate_fn/fns/uncompressed_size_in_bytes/mod.rs index 7a77ed9555c..a4a2737c991 100644 --- a/vortex-array/src/aggregate_fn/fns/uncompressed_size_in_bytes/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/uncompressed_size_in_bytes/mod.rs @@ -344,8 +344,11 @@ mod tests { fn materialized_uncompressed_size_in_bytes(array: &ArrayRef) -> u64 { let mut builder = builder_with_capacity(array.dtype(), array.len()); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); unsafe { - builder.extend_from_array_unchecked(array); + builder + .extend_from_array_unchecked(array, &mut ctx) + .unwrap(); } builder.finish().nbytes() } diff --git a/vortex-array/src/array/vtable/mod.rs b/vortex-array/src/array/vtable/mod.rs index 366eed77ad9..c23d8ed09a9 100644 --- a/vortex-array/src/array/vtable/mod.rs +++ b/vortex-array/src/array/vtable/mod.rs @@ -151,7 +151,7 @@ pub trait VTable: 'static + Clone + Sized + Send + Sync + Debug { .clone() .execute::(ctx)? .into_array(); - builder.extend_from_array(&canonical); + builder.extend_from_array(&canonical, ctx)?; Ok(()) } diff --git a/vortex-array/src/arrays/constant/vtable/mod.rs b/vortex-array/src/arrays/constant/vtable/mod.rs index 89c571eb827..2b7b54d6713 100644 --- a/vortex-array/src/arrays/constant/vtable/mod.rs +++ b/vortex-array/src/arrays/constant/vtable/mod.rs @@ -236,7 +236,7 @@ impl VTable for Constant { .clone() .execute::(ctx)? .into_array(); - builder.extend_from_array(&canonical); + builder.extend_from_array(&canonical, ctx)?; } } diff --git a/vortex-array/src/arrays/listview/rebuild.rs b/vortex-array/src/arrays/listview/rebuild.rs index 63ed33c6498..5a62135ae3a 100644 --- a/vortex-array/src/arrays/listview/rebuild.rs +++ b/vortex-array/src/arrays/listview/rebuild.rs @@ -311,7 +311,7 @@ impl ListViewArray { new_offsets.push(n_elements); new_sizes.push(size); - new_elements_builder.extend_from_array(&elements_canonical.slice(start..stop)?); + new_elements_builder.extend_from_array(&elements_canonical.slice(start..stop)?, ctx)?; n_elements += num_traits::cast(size).vortex_expect("Cast failed"); } diff --git a/vortex-array/src/arrays/patched/vtable/mod.rs b/vortex-array/src/arrays/patched/vtable/mod.rs index 4ba5bc542e1..c44332b784b 100644 --- a/vortex-array/src/arrays/patched/vtable/mod.rs +++ b/vortex-array/src/arrays/patched/vtable/mod.rs @@ -198,7 +198,7 @@ impl VTable for Patched { .clone() .execute::(ctx)? .into_array(); - builder.extend_from_array(&canonical); + builder.extend_from_array(&canonical, ctx)?; return Ok(()); } diff --git a/vortex-array/src/arrays/varbinview/compact.rs b/vortex-array/src/arrays/varbinview/compact.rs index 6effc7c656a..a159f9781d0 100644 --- a/vortex-array/src/arrays/varbinview/compact.rs +++ b/vortex-array/src/arrays/varbinview/compact.rs @@ -136,7 +136,10 @@ impl VarBinViewArray { self.len(), buffer_utilization_threshold, ); - builder.extend_from_array(&self.clone().into_array()); + // This array is already canonical, so executing it through the builder is cheap; we only + // need a context to satisfy the `extend_from_array` signature. + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + builder.extend_from_array(&self.clone().into_array(), &mut ctx)?; Ok(builder.finish_into_varbinview()) } } diff --git a/vortex-array/src/builders/bool.rs b/vortex-array/src/builders/bool.rs index fdae7984844..8067c651637 100644 --- a/vortex-array/src/builders/bool.rs +++ b/vortex-array/src/builders/bool.rs @@ -5,23 +5,19 @@ use std::any::Any; use std::mem; use vortex_buffer::BitBufferMut; -use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_mask::Mask; use crate::ArrayRef; +use crate::ExecutionCtx; use crate::IntoArray; -use crate::LEGACY_SESSION; -use crate::VortexSessionExecute; use crate::arrays::BoolArray; use crate::arrays::bool::BoolArrayExt; use crate::builders::ArrayBuilder; use crate::builders::DEFAULT_BUILDER_CAPACITY; use crate::builders::LazyBitBufferBuilder; use crate::canonical::Canonical; -#[expect(deprecated)] -use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::dtype::Nullability; use crate::scalar::Scalar; @@ -115,22 +111,22 @@ impl ArrayBuilder for BoolBuilder { Ok(()) } - unsafe fn extend_from_array_unchecked(&mut self, array: &ArrayRef) { - #[expect(deprecated)] - let bool_array = array.to_bool(); + unsafe fn extend_from_array_unchecked( + &mut self, + array: &ArrayRef, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { + let bool_array = array.clone().execute::(ctx)?; self.inner.append_buffer(&bool_array.to_bit_buffer()); self.nulls.append_validity_mask( &bool_array .as_ref() - .validity() - .vortex_expect("validity_mask") - .execute_mask( - bool_array.as_ref().len(), - &mut LEGACY_SESSION.create_execution_ctx(), - ) - .vortex_expect("Failed to compute validity mask"), + .validity()? + .execute_mask(bool_array.as_ref().len(), ctx)?, ); + + Ok(()) } fn reserve_exact(&mut self, additional: usize) { diff --git a/vortex-array/src/builders/decimal.rs b/vortex-array/src/builders/decimal.rs index f8d438d477b..be04acd627f 100644 --- a/vortex-array/src/builders/decimal.rs +++ b/vortex-array/src/builders/decimal.rs @@ -12,11 +12,8 @@ use vortex_error::vortex_panic; use vortex_mask::Mask; use crate::ArrayRef; +use crate::ExecutionCtx; use crate::IntoArray; -use crate::LEGACY_SESSION; -#[expect(deprecated)] -use crate::ToCanonical as _; -use crate::VortexSessionExecute; use crate::arrays::DecimalArray; use crate::builders::ArrayBuilder; use crate::builders::DEFAULT_BUILDER_CAPACITY; @@ -195,9 +192,12 @@ impl ArrayBuilder for DecimalBuilder { Ok(()) } - unsafe fn extend_from_array_unchecked(&mut self, array: &ArrayRef) { - #[expect(deprecated)] - let decimal_array = array.to_decimal(); + unsafe fn extend_from_array_unchecked( + &mut self, + array: &ArrayRef, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { + let decimal_array = array.clone().execute::(ctx)?; match_each_decimal_value_type!(decimal_array.values_type(), |D| { // Extends the values buffer from another buffer of type D where D can be coerced to the @@ -209,14 +209,11 @@ impl ArrayBuilder for DecimalBuilder { self.nulls.append_validity_mask( &decimal_array .as_ref() - .validity() - .vortex_expect("validity_mask") - .execute_mask( - decimal_array.as_ref().len(), - &mut LEGACY_SESSION.create_execution_ctx(), - ) - .vortex_expect("Failed to compute validity mask"), + .validity()? + .execute_mask(decimal_array.as_ref().len(), ctx)?, ); + + Ok(()) } fn reserve_exact(&mut self, additional: usize) { @@ -328,7 +325,9 @@ mod tests { let i8s = i8s.finish(); let mut i128s = DecimalBuilder::new::(DecimalDType::new(2, 1), false.into()); - i128s.extend_from_array(&i8s); + i128s + .extend_from_array(&i8s, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); let i128s = i128s.finish(); for i in 0..i8s.len() { diff --git a/vortex-array/src/builders/extension.rs b/vortex-array/src/builders/extension.rs index d947877e297..aae352c0740 100644 --- a/vortex-array/src/builders/extension.rs +++ b/vortex-array/src/builders/extension.rs @@ -8,6 +8,7 @@ use vortex_error::vortex_ensure; use vortex_mask::Mask; use crate::ArrayRef; +use crate::ExecutionCtx; use crate::IntoArray; use crate::arrays::ExtensionArray; use crate::arrays::extension::ExtensionArrayExt; @@ -15,8 +16,6 @@ use crate::builders::ArrayBuilder; use crate::builders::DEFAULT_BUILDER_CAPACITY; use crate::builders::builder_with_capacity; use crate::canonical::Canonical; -#[expect(deprecated)] -use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::dtype::extension::ExtDTypeRef; use crate::scalar::ExtScalar; @@ -99,10 +98,14 @@ impl ArrayBuilder for ExtensionBuilder { self.append_value(scalar.as_extension()) } - unsafe fn extend_from_array_unchecked(&mut self, array: &ArrayRef) { - #[expect(deprecated)] - let ext_array = array.to_extension(); - self.storage.extend_from_array(ext_array.storage_array()) + unsafe fn extend_from_array_unchecked( + &mut self, + array: &ArrayRef, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { + let ext_array = array.clone().execute::(ctx)?; + self.storage + .extend_from_array(ext_array.storage_array(), ctx) } fn reserve_exact(&mut self, capacity: usize) { diff --git a/vortex-array/src/builders/fixed_size_list.rs b/vortex-array/src/builders/fixed_size_list.rs index 7a3c3bb068a..033649a02e4 100644 --- a/vortex-array/src/builders/fixed_size_list.rs +++ b/vortex-array/src/builders/fixed_size_list.rs @@ -12,9 +12,8 @@ use vortex_error::vortex_panic; use vortex_mask::Mask; use crate::ArrayRef; +use crate::ExecutionCtx; use crate::IntoArray; -use crate::LEGACY_SESSION; -use crate::VortexSessionExecute; use crate::arrays::FixedSizeListArray; use crate::arrays::fixed_size_list::FixedSizeListArrayExt; use crate::builders::ArrayBuilder; @@ -22,8 +21,6 @@ use crate::builders::DEFAULT_BUILDER_CAPACITY; use crate::builders::LazyBitBufferBuilder; use crate::builders::builder_with_capacity; use crate::canonical::Canonical; -#[expect(deprecated)] -use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::dtype::Nullability; use crate::scalar::ListScalar; @@ -83,7 +80,11 @@ impl FixedSizeListBuilder { /// /// Note that the list entry will be non-null but the elements themselves are allowed to be null /// (only if the elements [`DType`] is nullable, of course). - pub fn append_array_as_list(&mut self, array: &ArrayRef) -> VortexResult<()> { + pub fn append_array_as_list( + &mut self, + array: &ArrayRef, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { vortex_ensure!( array.dtype() == self.element_dtype(), "Array dtype {:?} does not match list element dtype {:?}", @@ -97,7 +98,7 @@ impl FixedSizeListBuilder { self.list_size() ); - self.elements_builder.extend_from_array(array); + self.elements_builder.extend_from_array(array, ctx)?; self.nulls.append_non_null(); Ok(()) @@ -237,21 +238,22 @@ impl ArrayBuilder for FixedSizeListBuilder { /// This will increase the capacity if extending with this `array` would go past the original /// capacity. - unsafe fn extend_from_array_unchecked(&mut self, array: &ArrayRef) { - #[expect(deprecated)] - let fsl = array.to_fixed_size_list(); + unsafe fn extend_from_array_unchecked( + &mut self, + array: &ArrayRef, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { + let fsl = array.clone().execute::(ctx)?; if fsl.is_empty() { - return; + return Ok(()); } - self.elements_builder.extend_from_array(fsl.elements()); - self.nulls.append_validity_mask( - &array - .validity() - .vortex_expect("validity_mask in extend_from_array_unchecked") - .execute_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) - .vortex_expect("Failed to compute validity mask"), - ); + self.elements_builder + .extend_from_array(fsl.elements(), ctx)?; + self.nulls + .append_validity_mask(&array.validity()?.execute_mask(array.len(), ctx)?); + + Ok(()) } fn reserve_exact(&mut self, additional: usize) { @@ -679,8 +681,12 @@ mod tests { let mut builder = FixedSizeListBuilder::with_capacity(dtype, 2, Nullable, 0); let source_array = source.into_array(); - builder.extend_from_array(&source_array); - builder.extend_from_array(&source_array); + builder + .extend_from_array(&source_array, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); + builder + .extend_from_array(&source_array, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); let fsl = builder.finish(); assert_eq!(fsl.len(), 6); @@ -756,8 +762,18 @@ mod tests { let mut builder = FixedSizeListBuilder::with_capacity(dtype, 0, Nullable, 0); - builder.extend_from_array(&source1.into_array()); - builder.extend_from_array(&source2.into_array()); + builder + .extend_from_array( + &source1.into_array(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); + builder + .extend_from_array( + &source2.into_array(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let fsl = builder.finish(); assert_eq!(fsl.len(), 5); @@ -833,7 +849,12 @@ mod tests { .unwrap(); // Extend with empty array (should be no-op). - builder.extend_from_array(&source.into_array()); + builder + .extend_from_array( + &source.into_array(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let fsl = builder.finish(); assert_eq!(fsl.len(), 1); @@ -871,7 +892,12 @@ mod tests { Validity::AllValid, 1, ); - builder.extend_from_array(&source.into_array()); + builder + .extend_from_array( + &source.into_array(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let fsl = builder.finish(); assert_eq!(fsl.len(), 6); @@ -1002,7 +1028,9 @@ mod tests { // Append a primitive array as a single list entry. let arr1 = buffer![1i32, 2, 3].into_array(); - builder.append_array_as_list(&arr1).unwrap(); + builder + .append_array_as_list(&arr1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); // Interleave with a list scalar. builder @@ -1018,7 +1046,9 @@ mod tests { // Append another primitive array as a single list entry. let arr2 = buffer![4i32, 5, 6].into_array(); - builder.append_array_as_list(&arr2).unwrap(); + builder + .append_array_as_list(&arr2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); // Interleave with another list scalar. builder @@ -1048,11 +1078,19 @@ mod tests { let mut builder = FixedSizeListBuilder::with_capacity(Arc::clone(&dtype), 3, NonNullable, 10); let wrong_dtype_arr = buffer![1i64, 2, 3].into_array(); - assert!(builder.append_array_as_list(&wrong_dtype_arr).is_err()); + assert!( + builder + .append_array_as_list(&wrong_dtype_arr, &mut LEGACY_SESSION.create_execution_ctx()) + .is_err() + ); // Test length mismatch error. let mut builder = FixedSizeListBuilder::with_capacity(dtype, 3, NonNullable, 10); let wrong_len_arr = buffer![1i32, 2].into_array(); - assert!(builder.append_array_as_list(&wrong_len_arr).is_err()); + assert!( + builder + .append_array_as_list(&wrong_len_arr, &mut LEGACY_SESSION.create_execution_ctx()) + .is_err() + ); } } diff --git a/vortex-array/src/builders/list.rs b/vortex-array/src/builders/list.rs index c7b506f228a..3c9ff6bbbb9 100644 --- a/vortex-array/src/builders/list.rs +++ b/vortex-array/src/builders/list.rs @@ -13,10 +13,11 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::Canonical; +use crate::ExecutionCtx; use crate::IntoArray; -use crate::LEGACY_SESSION; -use crate::VortexSessionExecute; use crate::arrays::ListArray; +use crate::arrays::ListViewArray; +use crate::arrays::PrimitiveArray; use crate::arrays::listview::ListViewArrayExt; use crate::builders::ArrayBuilder; use crate::builders::DEFAULT_BUILDER_CAPACITY; @@ -95,7 +96,11 @@ impl ListBuilder { /// /// Note that the list entry will be non-null but the elements themselves are allowed to be null /// (only if the elements [`DType`] in nullable, of course). - pub fn append_array_as_list(&mut self, array: &ArrayRef) -> VortexResult<()> { + pub fn append_array_as_list( + &mut self, + array: &ArrayRef, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { vortex_ensure!( array.dtype() == self.element_dtype(), "Array dtype {:?} does not match list element dtype {:?}", @@ -103,7 +108,7 @@ impl ListBuilder { self.element_dtype() ); - self.elements_builder.extend_from_array(array); + self.elements_builder.extend_from_array(array, ctx)?; self.nulls.append_non_null(); self.offsets_builder.append_value( O::from_usize(self.elements_builder.len()) @@ -217,35 +222,33 @@ impl ArrayBuilder for ListBuilder { self.append_value(scalar.as_list()) } - unsafe fn extend_from_array_unchecked(&mut self, array: &ArrayRef) { - #[expect(deprecated)] - let list = array.to_listview(); + unsafe fn extend_from_array_unchecked( + &mut self, + array: &ArrayRef, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { + let list = array.clone().execute::(ctx)?; if list.is_empty() { - return; + return Ok(()); } // Append validity information. - self.nulls.append_validity_mask( - &array - .validity() - .vortex_expect("validity_mask in extend_from_array_unchecked") - .execute_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) - .vortex_expect("Failed to compute validity mask"), - ); + self.nulls + .append_validity_mask(&array.validity()?.execute_mask(array.len(), ctx)?); // Note that `ListViewArray` has `n` offsets and sizes, not `n+1` offsets like `ListArray`. let elements = list.elements(); - #[expect(deprecated)] - let offsets = list.offsets().to_primitive(); - #[expect(deprecated)] - let sizes = list.sizes().to_primitive(); + let offsets = list.offsets().clone().execute::(ctx)?; + let sizes = list.sizes().clone().execute::(ctx)?; fn extend_inner( builder: &mut ListBuilder, new_elements: &ArrayRef, new_offsets: &[OffsetType], new_sizes: &[SizeType], - ) where + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> + where O: IntegerPType, OffsetType: IntegerPType, SizeType: IntegerPType, @@ -263,10 +266,10 @@ impl ArrayBuilder for ListBuilder { let size: usize = new_sizes[i].as_(); if size > 0 { - let list_elements = new_elements - .slice(offset..offset + size) - .vortex_expect("list builder slice"); - builder.elements_builder.extend_from_array(&list_elements); + let list_elements = new_elements.slice(offset..offset + size)?; + builder + .elements_builder + .extend_from_array(&list_elements, ctx)?; curr_offset += size; } @@ -279,6 +282,8 @@ impl ArrayBuilder for ListBuilder { // SAFETY: We have initialized all `num_lists` values, and since the `offsets` array is // non-nullable, we are done. unsafe { offsets_range.finish() }; + + Ok(()) } match_each_integer_ptype!(offsets.ptype(), |OffsetType| { @@ -288,6 +293,7 @@ impl ArrayBuilder for ListBuilder { elements, offsets.as_slice::(), sizes.as_slice::(), + ctx, ) }) }) @@ -454,10 +460,14 @@ mod tests { let mut ctx = LEGACY_SESSION.create_execution_ctx(); let mut builder = ListBuilder::::with_capacity(Arc::new(I32.into()), Nullable, 18, 9); - builder.extend_from_array(&list); - builder.extend_from_array(&list); - builder.extend_from_array(&list.slice(0..0).unwrap()); - builder.extend_from_array(&list.slice(1..3).unwrap()); + builder.extend_from_array(&list, &mut ctx).unwrap(); + builder.extend_from_array(&list, &mut ctx).unwrap(); + builder + .extend_from_array(&list.slice(0..0).unwrap(), &mut ctx) + .unwrap(); + builder + .extend_from_array(&list.slice(1..3).unwrap(), &mut ctx) + .unwrap(); #[expect(deprecated)] let expected = ListArray::from_iter_opt_slow::( @@ -641,7 +651,9 @@ mod tests { // Append a primitive array as a single list entry. let arr1 = buffer![1i32, 2, 3].into_array(); - builder.append_array_as_list(&arr1).unwrap(); + builder + .append_array_as_list(&arr1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); // Interleave with a list scalar. builder @@ -657,11 +669,15 @@ mod tests { // Append another primitive array as a single list entry. let arr2 = buffer![4i32, 5].into_array(); - builder.append_array_as_list(&arr2).unwrap(); + builder + .append_array_as_list(&arr2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); // Append an empty array as a single list entry (empty list). let arr3 = buffer![0i32; 0].into_array(); - builder.append_array_as_list(&arr3).unwrap(); + builder + .append_array_as_list(&arr3, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); // Interleave with another list scalar (empty list). builder @@ -686,6 +702,10 @@ mod tests { // Test dtype mismatch error. let mut builder = ListBuilder::::with_capacity(dtype, NonNullable, 20, 10); let wrong_dtype_arr = buffer![1i64, 2, 3].into_array(); - assert!(builder.append_array_as_list(&wrong_dtype_arr).is_err()); + assert!( + builder + .append_array_as_list(&wrong_dtype_arr, &mut LEGACY_SESSION.create_execution_ctx()) + .is_err() + ); } } diff --git a/vortex-array/src/builders/listview.rs b/vortex-array/src/builders/listview.rs index 4c350f1888d..7d40a756da1 100644 --- a/vortex-array/src/builders/listview.rs +++ b/vortex-array/src/builders/listview.rs @@ -20,8 +20,7 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::Canonical; -use crate::LEGACY_SESSION; -use crate::VortexSessionExecute; +use crate::ExecutionCtx; use crate::array::IntoArray; use crate::arrays::ListViewArray; use crate::arrays::PrimitiveArray; @@ -117,7 +116,11 @@ impl ListViewBuilder { /// /// Note that the list entry will be non-null but the elements themselves are allowed to be null /// (only if the elements [`DType`] is nullable, of course). - pub fn append_array_as_list(&mut self, array: &ArrayRef) -> VortexResult<()> { + pub fn append_array_as_list( + &mut self, + array: &ArrayRef, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { vortex_ensure!( array.dtype() == self.element_dtype(), "Array dtype {:?} does not match list element dtype {:?}", @@ -135,7 +138,7 @@ impl ListViewBuilder { "appending this list would cause an offset overflow" ); - self.elements_builder.extend_from_array(array); + self.elements_builder.extend_from_array(array, ctx)?; self.nulls.append_non_null(); self.offsets_builder.append_value( @@ -291,41 +294,30 @@ impl ArrayBuilder for ListViewBuilder { self.append_value(list_scalar) } - unsafe fn extend_from_array_unchecked(&mut self, array: &ArrayRef) { - // TODO: The `ArrayBuilder` trait does not thread an `ExecutionCtx` through its extend - // methods, so we are forced to mint a fresh `LEGACY_SESSION` context here on every call - // (which for chunked input means once per chunk). Once the trait carries a `&mut - // ExecutionCtx`, the caller's session should be reused instead. - let mut ctx = LEGACY_SESSION.create_execution_ctx(); - - let listview = array - .clone() - .execute::(&mut ctx) - .vortex_expect("failed to execute array into ListViewArray in extend_from_array"); + unsafe fn extend_from_array_unchecked( + &mut self, + array: &ArrayRef, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { + let listview = array.clone().execute::(ctx)?; if listview.is_empty() { - return; + return Ok(()); } // Normalize to an exact zero-copy-to-list layout and then bulk append. This avoids the // very expensive scalar_at-per-list path for overlapping / out-of-order list views. - let listview = listview - .rebuild(ListViewRebuildMode::MakeExact, &mut ctx) - .vortex_expect("ListViewArray::rebuild(MakeExact) failed in extend_from_array"); + let listview = listview.rebuild(ListViewRebuildMode::MakeExact, ctx)?; debug_assert!(listview.is_zero_copy_to_list()); - self.nulls.append_validity_mask( - &array - .validity() - .vortex_expect("validity_mask in extend_from_array_unchecked") - .execute_mask(array.len(), &mut ctx) - .vortex_expect("Failed to compute validity mask"), - ); + self.nulls + .append_validity_mask(&array.validity()?.execute_mask(array.len(), ctx)?); // Bulk append the new elements (which should have no gaps or overlaps). let old_elements_len = self.elements_builder.len(); self.elements_builder .reserve_exact(listview.elements().len()); - self.elements_builder.extend_from_array(listview.elements()); + self.elements_builder + .extend_from_array(listview.elements(), ctx)?; let new_elements_len = self.elements_builder.len(); // Reserve enough space for the new views. @@ -337,11 +329,8 @@ impl ArrayBuilder for ListViewBuilder { let cast_sizes = listview .sizes() .clone() - .cast(self.sizes_builder.dtype().clone()) - .vortex_expect( - "was somehow unable to cast the new sizes to the type of the builder sizes", - ); - self.sizes_builder.extend_from_array(&cast_sizes); + .cast(self.sizes_builder.dtype().clone())?; + self.sizes_builder.extend_from_array(&cast_sizes, ctx)?; // Now we need to adjust all of the offsets by adding the current number of elements in the // builder. @@ -349,11 +338,7 @@ impl ArrayBuilder for ListViewBuilder { let uninit_range = self.offsets_builder.uninit_range(extend_length); // This should be cheap because we didn't compress after rebuilding. - let new_offsets = listview - .offsets() - .clone() - .execute::(&mut ctx) - .vortex_expect("failed to execute list view offsets into a PrimitiveArray"); + let new_offsets = listview.offsets().clone().execute::(ctx)?; match_each_integer_ptype!(new_offsets.ptype(), |A| { adjust_and_extend_offsets::( @@ -362,7 +347,9 @@ impl ArrayBuilder for ListViewBuilder { old_elements_len, new_elements_len, ); - }) + }); + + Ok(()) } fn reserve_exact(&mut self, capacity: usize) { @@ -662,7 +649,12 @@ mod tests { .unwrap(); // Extend from the ListArray. - builder.extend_from_array(&source.into_array()); + builder + .extend_from_array( + &source.into_array(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); // Extend from empty array (should be no-op). let empty_source = ListArray::from_iter_opt_slow::>( @@ -670,7 +662,12 @@ mod tests { Arc::new(I32.into()), ) .unwrap(); - builder.extend_from_array(&empty_source.into_array()); + builder + .extend_from_array( + &empty_source.into_array(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let listview = builder.finish_into_listview(); assert_eq!(listview.len(), 4); @@ -725,7 +722,12 @@ mod tests { let mut builder = ListViewBuilder::::with_capacity(Arc::clone(&dtype), Nullable, 0, 0); - builder.extend_from_array(&source.into_array()); + builder + .extend_from_array( + &source.into_array(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let listview = builder.finish_into_listview(); assert_eq!(listview.len(), 3); @@ -778,7 +780,9 @@ mod tests { // Append a primitive array as a single list entry. let arr1 = buffer![1i32, 2, 3].into_array(); - builder.append_array_as_list(&arr1).unwrap(); + builder + .append_array_as_list(&arr1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); // Interleave with a list scalar. builder @@ -794,11 +798,15 @@ mod tests { // Append another primitive array as a single list entry. let arr2 = buffer![4i32, 5].into_array(); - builder.append_array_as_list(&arr2).unwrap(); + builder + .append_array_as_list(&arr2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); // Append an empty array as a single list entry (empty list). let arr3 = buffer![0i32; 0].into_array(); - builder.append_array_as_list(&arr3).unwrap(); + builder + .append_array_as_list(&arr3, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); // Interleave with another list scalar. builder @@ -829,6 +837,10 @@ mod tests { // Test dtype mismatch error. let mut builder = ListViewBuilder::::with_capacity(dtype, NonNullable, 20, 10); let wrong_dtype_arr = buffer![1i64, 2, 3].into_array(); - assert!(builder.append_array_as_list(&wrong_dtype_arr).is_err()); + assert!( + builder + .append_array_as_list(&wrong_dtype_arr, &mut LEGACY_SESSION.create_execution_ctx()) + .is_err() + ); } } diff --git a/vortex-array/src/builders/mod.rs b/vortex-array/src/builders/mod.rs index e9f86fa4332..15b508a403b 100644 --- a/vortex-array/src/builders/mod.rs +++ b/vortex-array/src/builders/mod.rs @@ -34,10 +34,11 @@ use std::any::Any; use std::sync::Arc; use vortex_error::VortexResult; -use vortex_error::vortex_panic; +use vortex_error::vortex_bail; use vortex_mask::Mask; use crate::ArrayRef; +use crate::ExecutionCtx; use crate::canonical::Canonical; use crate::dtype::DType; use crate::match_each_decimal_value_type; @@ -155,18 +156,28 @@ pub trait ArrayBuilder: Send { /// The inner part of `extend_from_array`. /// + /// Canonicalizing the array (and computing its validity mask) is fallible and is performed + /// using the provided [`ExecutionCtx`], so this method returns a [`VortexResult`]. + /// /// # Safety /// /// The array that must have an equal [`DType`] to the array builder's `DType` (with nullability /// superset semantics). - unsafe fn extend_from_array_unchecked(&mut self, array: &ArrayRef); + unsafe fn extend_from_array_unchecked( + &mut self, + array: &ArrayRef, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()>; /// Extends the array with the provided array, canonicalizing if necessary. /// /// Implementors must validate that the passed in [`ArrayRef`] has the correct [`DType`]. - fn extend_from_array(&mut self, array: &ArrayRef) { + /// + /// Canonicalizing the array (and computing its validity mask) is fallible and is performed + /// using the provided [`ExecutionCtx`], so this method returns a [`VortexResult`]. + fn extend_from_array(&mut self, array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<()> { if !self.dtype().eq_with_nullability_superset(array.dtype()) { - vortex_panic!( + vortex_bail!( "tried to extend a builder with `DType` {} with an array with `DType {}", self.dtype(), array.dtype() @@ -174,7 +185,7 @@ pub trait ArrayBuilder: Send { } // SAFETY: We checked that the array had a valid `DType` above. - unsafe { self.extend_from_array_unchecked(array) } + unsafe { self.extend_from_array_unchecked(array, ctx) } } /// Allocate space for extra `additional` items diff --git a/vortex-array/src/builders/null.rs b/vortex-array/src/builders/null.rs index 45b6e7ed974..79bcf49722d 100644 --- a/vortex-array/src/builders/null.rs +++ b/vortex-array/src/builders/null.rs @@ -8,6 +8,7 @@ use vortex_error::vortex_ensure; use vortex_mask::Mask; use crate::ArrayRef; +use crate::ExecutionCtx; use crate::IntoArray; use crate::arrays::NullArray; use crate::builders::ArrayBuilder; @@ -69,8 +70,13 @@ impl ArrayBuilder for NullBuilder { Ok(()) } - unsafe fn extend_from_array_unchecked(&mut self, array: &ArrayRef) { + unsafe fn extend_from_array_unchecked( + &mut self, + array: &ArrayRef, + _ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { self.append_nulls(array.len()); + Ok(()) } fn reserve_exact(&mut self, _additional: usize) {} diff --git a/vortex-array/src/builders/primitive.rs b/vortex-array/src/builders/primitive.rs index c4b5abfb083..7724e75c3af 100644 --- a/vortex-array/src/builders/primitive.rs +++ b/vortex-array/src/builders/primitive.rs @@ -5,22 +5,18 @@ use std::any::Any; use std::mem::MaybeUninit; use vortex_buffer::BufferMut; -use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_mask::Mask; use crate::ArrayRef; +use crate::ExecutionCtx; use crate::IntoArray; -use crate::LEGACY_SESSION; -use crate::VortexSessionExecute; use crate::arrays::PrimitiveArray; use crate::builders::ArrayBuilder; use crate::builders::DEFAULT_BUILDER_CAPACITY; use crate::builders::LazyBitBufferBuilder; use crate::canonical::Canonical; -#[expect(deprecated)] -use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::dtype::NativePType; use crate::dtype::Nullability; @@ -177,9 +173,12 @@ impl ArrayBuilder for PrimitiveBuilder { Ok(()) } - unsafe fn extend_from_array_unchecked(&mut self, array: &ArrayRef) { - #[expect(deprecated)] - let array = array.to_primitive(); + unsafe fn extend_from_array_unchecked( + &mut self, + array: &ArrayRef, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { + let array = array.clone().execute::(ctx)?; // This should be checked in `extend_from_array` but we can check it again. debug_assert_eq!( @@ -192,14 +191,11 @@ impl ArrayBuilder for PrimitiveBuilder { self.nulls.append_validity_mask( &array .as_ref() - .validity() - .vortex_expect("validity_mask") - .execute_mask( - array.as_ref().len(), - &mut LEGACY_SESSION.create_execution_ctx(), - ) - .vortex_expect("Failed to compute validity mask"), + .validity()? + .execute_mask(array.as_ref().len(), ctx)?, ); + + Ok(()) } fn reserve_exact(&mut self, additional: usize) { @@ -370,6 +366,8 @@ mod tests { use vortex_error::VortexExpect; use super::*; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; use crate::assert_arrays_eq; /// REGRESSION TEST: This test verifies that multiple sequential ranges have correct offsets. diff --git a/vortex-array/src/builders/struct_.rs b/vortex-array/src/builders/struct_.rs index d396cf186a0..aa8fc99b32e 100644 --- a/vortex-array/src/builders/struct_.rs +++ b/vortex-array/src/builders/struct_.rs @@ -12,9 +12,8 @@ use vortex_error::vortex_panic; use vortex_mask::Mask; use crate::ArrayRef; +use crate::ExecutionCtx; use crate::IntoArray; -use crate::LEGACY_SESSION; -use crate::VortexSessionExecute; use crate::arrays::StructArray; use crate::arrays::struct_::StructArrayExt; use crate::builders::ArrayBuilder; @@ -22,8 +21,6 @@ use crate::builders::DEFAULT_BUILDER_CAPACITY; use crate::builders::LazyBitBufferBuilder; use crate::builders::builder_with_capacity; use crate::canonical::Canonical; -#[expect(deprecated)] -use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::dtype::Nullability; use crate::dtype::StructFields; @@ -168,24 +165,24 @@ impl ArrayBuilder for StructBuilder { self.append_value(scalar.as_struct()) } - unsafe fn extend_from_array_unchecked(&mut self, array: &ArrayRef) { - #[expect(deprecated)] - let array = array.to_struct(); + unsafe fn extend_from_array_unchecked( + &mut self, + array: &ArrayRef, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { + let array = array.clone().execute::(ctx)?; for (a, builder) in array .iter_unmasked_fields() .zip_eq(self.builders.iter_mut()) { - builder.extend_from_array(a); + builder.extend_from_array(a, ctx)?; } - self.nulls.append_validity_mask( - &array - .validity() - .vortex_expect("validity_mask") - .execute_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) - .vortex_expect("Failed to compute validity mask"), - ); + self.nulls + .append_validity_mask(&array.validity()?.execute_mask(array.len(), ctx)?); + + Ok(()) } fn reserve_exact(&mut self, capacity: usize) { diff --git a/vortex-array/src/builders/varbinview.rs b/vortex-array/src/builders/varbinview.rs index 392bef59997..e2389914582 100644 --- a/vortex-array/src/builders/varbinview.rs +++ b/vortex-array/src/builders/varbinview.rs @@ -19,17 +19,14 @@ use vortex_utils::aliases::hash_map::Entry; use vortex_utils::aliases::hash_map::HashMap; use crate::ArrayRef; +use crate::ExecutionCtx; use crate::IntoArray; -use crate::LEGACY_SESSION; -use crate::VortexSessionExecute; use crate::arrays::VarBinViewArray; use crate::arrays::varbinview::build_views::BinaryView; use crate::arrays::varbinview::compact::BufferUtilization; use crate::builders::ArrayBuilder; use crate::builders::LazyBitBufferBuilder; use crate::canonical::Canonical; -#[expect(deprecated)] -use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::scalar::Scalar; @@ -294,16 +291,15 @@ impl ArrayBuilder for VarBinViewBuilder { Ok(()) } - unsafe fn extend_from_array_unchecked(&mut self, array: &ArrayRef) { - #[expect(deprecated)] - let array = array.to_varbinview(); + unsafe fn extend_from_array_unchecked( + &mut self, + array: &ArrayRef, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { + let array = array.clone().execute::(ctx)?; self.flush_in_progress(); - let mask = array - .validity() - .vortex_expect("validity_mask") - .execute_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) - .vortex_expect("Failed to compute validity mask"); + let mask = array.validity()?.execute_mask(array.len(), ctx)?; self.push_only_validity_mask(&mask); @@ -346,6 +342,8 @@ impl ArrayBuilder for VarBinViewBuilder { } }, } + + Ok(()) } fn reserve_exact(&mut self, additional: usize) { @@ -890,7 +888,9 @@ mod tests { let mut builder = VarBinViewBuilder::with_capacity(DType::Utf8(Nullability::Nullable), 10); builder.append_value("Hello1"); - builder.extend_from_array(&array); + builder + .extend_from_array(&array, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); builder.append_nulls(2); builder.append_value("Hello3");