diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 43f329d85c3d5b..728bc3366e33bf 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1206,6 +1206,15 @@ DEFINE_Validator(variant_max_json_key_length, DEFINE_Validator(variant_storage_parse_mode, [](const int config) -> bool { return config >= 0 && config <= 2; }); +// Lance uses one BE-wide session so metadata/index caches and the optional Foyer data-file cache +// can be shared by all Lance dataset readers. +DEFINE_Int64(lance_index_cache_size_bytes, "6442450944"); // 6GB +DEFINE_Int64(lance_metadata_cache_size_bytes, "1073741824"); // 1GB +DEFINE_Bool(enable_lance_data_cache, "true"); +DEFINE_String(lance_data_cache_path, "${DORIS_HOME}/lance_data_cache"); +DEFINE_Int64(lance_data_cache_disk_capacity_bytes, "107374182400"); // 100GB +DEFINE_Int64(lance_data_cache_read_block_size_bytes, "1048576"); // 1MB + // block file cache DEFINE_Bool(enable_file_cache, "false"); // ATTENTION: For test only. Keep this enabled in production. diff --git a/be/src/common/config.h b/be/src/common/config.h index 1e50e539d96bed..f6a969111039b2 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1249,6 +1249,14 @@ DECLARE_Bool(enable_debug_points); DECLARE_Int32(pipeline_executor_size); DECLARE_Int32(blocking_pipeline_executor_size); +// Lance shared session and optional Foyer data-file cache. +DECLARE_Int64(lance_index_cache_size_bytes); +DECLARE_Int64(lance_metadata_cache_size_bytes); +DECLARE_Bool(enable_lance_data_cache); +DECLARE_String(lance_data_cache_path); +DECLARE_Int64(lance_data_cache_disk_capacity_bytes); +DECLARE_Int64(lance_data_cache_read_block_size_bytes); + // block file cache DECLARE_Bool(enable_file_cache); DECLARE_mBool(enable_file_cache_write_from_s3_file_writer); diff --git a/be/src/format_v2/lance/lance_reader_helper.cpp b/be/src/format_v2/lance/lance_reader_helper.cpp new file mode 100644 index 00000000000000..446a4b40179e4d --- /dev/null +++ b/be/src/format_v2/lance/lance_reader_helper.cpp @@ -0,0 +1,321 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/lance/lance_reader_helper.h" + +#include +#include +#include +#include + +#include +#include + +#include "common/logging.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_factory.hpp" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nothing.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_struct.h" + +namespace doris::format::lance { +namespace { + +constexpr std::string_view ARROW_EXTENSION_NAME = "ARROW:extension:name"; + +int arrow_time_precision(arrow::TimeUnit::type unit) { + switch (unit) { + case arrow::TimeUnit::SECOND: + return 0; + case arrow::TimeUnit::MILLI: + return 3; + case arrow::TimeUnit::MICRO: + case arrow::TimeUnit::NANO: + return 6; + } + return 6; +} + +Status check_arrow_field_semantics(const std::shared_ptr& field) { + if (field->HasMetadata()) { + const auto extension_name = field->metadata()->Get(ARROW_EXTENSION_NAME); + if (extension_name.ok() && !extension_name.ValueUnsafe().empty()) { + return Status::NotSupported( + "unsupported Lance Arrow extension type '{}' for field '{}'", + extension_name.ValueUnsafe(), field->name()); + } + } + if (field->type()->id() == arrow::Type::DICTIONARY) { + return Status::NotSupported("unsupported Lance Arrow dictionary type for field '{}': {}", + field->name(), field->type()->ToString()); + } + return Status::OK(); +} + +Status arrow_field_to_doris_type(const std::shared_ptr& field, + DataTypePtr* doris_type) { + RETURN_IF_ERROR(check_arrow_field_semantics(field)); + const auto& arrow_type = field->type(); + const auto nullable_primitive = [&](PrimitiveType type, int precision = 0, int scale = 0, + int len = -1) { + *doris_type = + DataTypeFactory::instance().create_data_type(type, true, precision, scale, len); + return Status::OK(); + }; + + switch (arrow_type->id()) { + case arrow::Type::BOOL: + return nullable_primitive(TYPE_BOOLEAN); + case arrow::Type::INT8: + return nullable_primitive(TYPE_TINYINT); + case arrow::Type::UINT8: + case arrow::Type::INT16: + return nullable_primitive(TYPE_SMALLINT); + case arrow::Type::UINT16: + case arrow::Type::INT32: + return nullable_primitive(TYPE_INT); + case arrow::Type::UINT32: + case arrow::Type::INT64: + return nullable_primitive(TYPE_BIGINT); + case arrow::Type::UINT64: + return nullable_primitive(TYPE_LARGEINT); + case arrow::Type::HALF_FLOAT: + case arrow::Type::FLOAT: + return nullable_primitive(TYPE_FLOAT); + case arrow::Type::DOUBLE: + return nullable_primitive(TYPE_DOUBLE); + case arrow::Type::STRING: + case arrow::Type::LARGE_STRING: + return nullable_primitive(TYPE_STRING); + case arrow::Type::BINARY: + case arrow::Type::LARGE_BINARY: + return nullable_primitive(TYPE_VARBINARY, 0, 0, std::numeric_limits::max()); + case arrow::Type::FIXED_SIZE_BINARY: { + const auto binary = std::static_pointer_cast(arrow_type); + return nullable_primitive(TYPE_VARBINARY, 0, 0, binary->byte_width()); + } + case arrow::Type::DATE32: + case arrow::Type::DATE64: + return nullable_primitive(TYPE_DATEV2); + case arrow::Type::TIME32: + case arrow::Type::TIME64: { + const auto time = std::static_pointer_cast(arrow_type); + return nullable_primitive(TYPE_TIMEV2, 0, arrow_time_precision(time->unit())); + } + case arrow::Type::TIMESTAMP: { + const auto timestamp = std::static_pointer_cast(arrow_type); + const auto doris_type = timestamp->timezone().empty() ? TYPE_DATETIMEV2 : TYPE_TIMESTAMPTZ; + return nullable_primitive(doris_type, 0, arrow_time_precision(timestamp->unit())); + } + case arrow::Type::DECIMAL128: + case arrow::Type::DECIMAL256: { + const auto decimal = std::static_pointer_cast(arrow_type); + const int precision = decimal->precision(); + const int scale = decimal->scale(); + if (precision <= 0 || precision > arrow::Decimal256Type::kMaxPrecision || scale < 0 || + scale > precision) { + return Status::NotSupported( + "unsupported Lance Arrow decimal type for field '{}': precision={}, scale={}", + field->name(), precision, scale); + } + const PrimitiveType doris_decimal_type = precision <= 9 ? TYPE_DECIMAL32 + : precision <= 18 ? TYPE_DECIMAL64 + : precision <= 38 ? TYPE_DECIMAL128I + : TYPE_DECIMAL256; + return nullable_primitive(doris_decimal_type, precision, scale); + } + case arrow::Type::LIST: + case arrow::Type::LARGE_LIST: + case arrow::Type::FIXED_SIZE_LIST: { + const auto list = std::static_pointer_cast(arrow_type); + DataTypePtr value_type; + RETURN_IF_ERROR(arrow_field_to_doris_type(list->value_field(), &value_type)); + *doris_type = make_nullable(std::make_shared(value_type)); + return Status::OK(); + } + case arrow::Type::MAP: { + const auto map = std::static_pointer_cast(arrow_type); + RETURN_IF_ERROR(check_arrow_field_semantics(map->value_field())); + DataTypePtr key_type; + DataTypePtr item_type; + RETURN_IF_ERROR(arrow_field_to_doris_type(map->key_field(), &key_type)); + RETURN_IF_ERROR(arrow_field_to_doris_type(map->item_field(), &item_type)); + *doris_type = make_nullable(std::make_shared(key_type, item_type)); + return Status::OK(); + } + case arrow::Type::STRUCT: { + const auto struct_type = std::static_pointer_cast(arrow_type); + DataTypes field_types; + Strings field_names; + field_types.reserve(struct_type->num_fields()); + field_names.reserve(struct_type->num_fields()); + for (const auto& child : struct_type->fields()) { + DataTypePtr field_type; + RETURN_IF_ERROR(arrow_field_to_doris_type(child, &field_type)); + field_types.emplace_back(std::move(field_type)); + field_names.emplace_back(child->name()); + } + *doris_type = make_nullable(std::make_shared(field_types, field_names)); + return Status::OK(); + } + default: + return Status::NotSupported("unsupported Lance Arrow type: {}", arrow_type->ToString()); + } +} + +} // namespace + +void LanceDatasetDeleter::operator()(LanceDataset* dataset) const { + lance_dataset_close(dataset); +} + +void LanceScannerDeleter::operator()(LanceScanner* scanner) const { + lance_scanner_close(scanner); +} + +void LanceBatchDeleter::operator()(LanceBatch* batch) const { + lance_batch_free(batch); +} + +size_t lance_vector_element_width(TVectorElementType::type type) { + switch (type) { + case TVectorElementType::FLOAT16: + return sizeof(uint16_t); + case TVectorElementType::FLOAT32: + return sizeof(float); + case TVectorElementType::FLOAT64: + return sizeof(double); + case TVectorElementType::UINT8: + case TVectorElementType::INT8: + return sizeof(uint8_t); + } + return 0; +} + +Status parse_fragment_ids(const TLanceFileDesc& lance_params, std::vector* fragment_ids) { + DORIS_CHECK(fragment_ids != nullptr); + fragment_ids->clear(); + if (!lance_params.__isset.fragment_ids || lance_params.fragment_ids.empty()) { + return Status::OK(); + } + fragment_ids->reserve(lance_params.fragment_ids.size()); + for (const auto fragment_id : lance_params.fragment_ids) { + if (fragment_id < 0) { + return Status::InvalidArgument("Lance fragment id must be non-negative: {}", + fragment_id); + } + fragment_ids->emplace_back(static_cast(fragment_id)); + } + return Status::OK(); +} + +Status parse_index_segment_uuids(const TLanceFileDesc& lance_params, + std::vector* segment_uuids, size_t* segment_count) { + DORIS_CHECK(segment_uuids != nullptr); + DORIS_CHECK(segment_count != nullptr); + segment_uuids->clear(); + *segment_count = 0; + if (!lance_params.__isset.index_segment_uuids || lance_params.index_segment_uuids.empty()) { + return Status::OK(); + } + constexpr size_t UUID_SIZE = 16; + if (lance_params.index_segment_uuids.size() > std::numeric_limits::max() / UUID_SIZE) { + return Status::InvalidArgument("too many Lance index segment UUIDs"); + } + segment_uuids->reserve(lance_params.index_segment_uuids.size() * UUID_SIZE); + for (const auto& uuid : lance_params.index_segment_uuids) { + if (uuid.size() != UUID_SIZE) { + return Status::InvalidArgument("Lance index segment UUID must contain 16 bytes, got {}", + uuid.size()); + } + segment_uuids->insert(segment_uuids->end(), uuid.begin(), uuid.end()); + } + *segment_count = lance_params.index_segment_uuids.size(); + return Status::OK(); +} + +Status convert_arrow_schema_to_doris(const std::shared_ptr& arrow_schema, + std::vector* column_names, + std::vector* column_types) { + DORIS_CHECK(arrow_schema != nullptr); + DORIS_CHECK(column_names != nullptr); + DORIS_CHECK(column_types != nullptr); + + std::vector parsed_names; + std::vector parsed_types; + parsed_names.reserve(arrow_schema->num_fields()); + parsed_types.reserve(arrow_schema->num_fields()); + std::unordered_set unique_names; + unique_names.reserve(arrow_schema->num_fields()); + for (const auto& field : arrow_schema->fields()) { + if (!unique_names.emplace(field->name()).second) { + return Status::InvalidArgument("duplicate Lance schema column: {}", field->name()); + } + DataTypePtr doris_type; + const auto type_status = arrow_field_to_doris_type(field, &doris_type); + if (type_status.is()) { + parsed_types.emplace_back(std::make_shared()); + } else { + RETURN_IF_ERROR(type_status); + DORIS_CHECK(doris_type != nullptr); + parsed_types.emplace_back(std::move(doris_type)); + } + parsed_names.emplace_back(field->name()); + } + *column_names = std::move(parsed_names); + *column_types = std::move(parsed_types); + return Status::OK(); +} + +Status build_lance_storage_options(const TFileScanRangeParams* scan_params, + std::vector* options) { + DORIS_CHECK(options != nullptr); + options->clear(); + if (scan_params == nullptr || !scan_params->__isset.lance_scan_params || + !scan_params->lance_scan_params.__isset.lance_storage_options) { + return Status::OK(); + } + const auto& storage_options = scan_params->lance_scan_params.lance_storage_options; + options->reserve(storage_options.size() * 2); + for (const auto& [key, value] : storage_options) { + // Both values cross a C-string boundary. Reject embedded NULs instead of silently opening + // a different dataset configuration from the one validated and used by the FE. + if (key.find('\0') != std::string::npos || value.find('\0') != std::string::npos) { + return Status::InvalidArgument( + "Lance storage option '{}' contains a NUL and cannot reach lance-c", + key.substr(0, key.find('\0'))); + } + options->emplace_back(key); + options->emplace_back(value); + } + return Status::OK(); +} + +Status lance_error(std::string_view operation) { + const char* raw_message = lance_last_error_message(); + std::string message = raw_message == nullptr ? "" : raw_message; + if (raw_message != nullptr) { + lance_free_string(raw_message); + } + if (message.empty()) { + message = fmt::format("error_code={}", static_cast(lance_last_error_code())); + } + return Status::InternalError("{} failed: {}", operation, message); +} + +} // namespace doris::format::lance diff --git a/be/src/format_v2/lance/lance_reader_helper.h b/be/src/format_v2/lance/lance_reader_helper.h new file mode 100644 index 00000000000000..689e4f4fd9f492 --- /dev/null +++ b/be/src/format_v2/lance/lance_reader_helper.h @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "gen_cpp/PlanNodes_types.h" + +struct LanceBatch; +struct LanceDataset; +struct LanceScanner; + +namespace arrow { +class Schema; +} // namespace arrow + +namespace doris::format::lance { + +inline constexpr std::string_view LANCE_DISTANCE_COLUMN = "_distance"; +inline constexpr std::string_view LANCE_SCORE_COLUMN = "_score"; +inline constexpr std::string_view LANCE_ROW_ID_COLUMN = "_rowid"; +inline constexpr const char* LANCE_READER_PROFILE = "LanceReader"; + +struct LanceDatasetDeleter { + void operator()(LanceDataset* dataset) const; +}; + +struct LanceScannerDeleter { + void operator()(LanceScanner* scanner) const; +}; + +struct LanceBatchDeleter { + void operator()(LanceBatch* batch) const; +}; + +size_t lance_vector_element_width(TVectorElementType::type type); + +// Validate and convert the fragment and index-segment identifiers carried by the FE into the +// unsigned and packed representations expected by lance-c. +Status parse_fragment_ids(const TLanceFileDesc& lance_params, std::vector* fragment_ids); +Status parse_index_segment_uuids(const TLanceFileDesc& lance_params, + std::vector* segment_uuids, size_t* segment_count); + +// Convert every top-level field without discarding unsupported columns. Malformed schemas still +// return an error and leave both output vectors unchanged. DataTypeNothing is the local sentinel +// for a valid Arrow field whose logical type Doris does not support. +Status convert_arrow_schema_to_doris(const std::shared_ptr& arrow_schema, + std::vector* column_names, + std::vector* column_types); + +// The FE sends storage options in Lance's own vocabulary. Preserve the key-value sequence exactly +// while validating that every value can cross the C-string boundary into lance-c. +Status build_lance_storage_options(const TFileScanRangeParams* scan_params, + std::vector* options); + +// Copy and release lance-c's thread-local error message before returning a Doris status. +Status lance_error(std::string_view operation); + +} // namespace doris::format::lance diff --git a/be/src/format_v2/lance/lance_runtime_filter_helper.cpp b/be/src/format_v2/lance/lance_runtime_filter_helper.cpp new file mode 100644 index 00000000000000..26b7c6402e78c7 --- /dev/null +++ b/be/src/format_v2/lance/lance_runtime_filter_helper.cpp @@ -0,0 +1,363 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/lance/lance_runtime_filter_helper.h" + +#include + +#include +#include +#include +#include +#include + +#include "common/logging.h" +#include "core/data_type/data_type_nullable.h" +#include "core/field.h" +#include "exprs/hybrid_set.h" +#include "exprs/runtime_filter_expr.h" +#include "exprs/vdirect_in_predicate.h" +#include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" +#include "exprs/vslot_ref.h" +#include "format/format_common.h" +#include "runtime/runtime_profile.h" + +namespace doris::format::lance { +namespace { + +constexpr std::string_view LANCE_RUNTIME_FILTER_CACHE_KEY_PREFIX = "lance-runtime-filter-sql:"; + +std::string format_filter_ids(const std::vector& filter_ids) { + std::string result; + for (const auto filter_id : filter_ids) { + if (!result.empty()) { + result.append(","); + } + result.append(std::to_string(filter_id)); + } + return result; +} + +std::string quote_sql_identifier(std::string_view identifier) { + // Lance SQL uses backticks for delimited identifiers. Escape an embedded backtick by doubling + // it, matching the SQL parser's quoted-identifier syntax. + std::string quoted("`"); + quoted.reserve(identifier.size() + 2); + for (const char ch : identifier) { + if (ch == '`') { + quoted.append("``"); + } else { + quoted.push_back(ch); + } + } + quoted.push_back('`'); + return quoted; +} + +const RuntimeFilterExpr* get_runtime_filter(const VExprContextSPtr& conjunct) { + if (conjunct == nullptr || conjunct->root() == nullptr) { + return nullptr; + } + return dynamic_cast(conjunct->root().get()); +} + +void append_sql_conjunct(std::string_view conjunct, std::string* expression) { + if (!expression->empty()) { + expression->append(" AND "); + } + expression->append(conjunct); +} + +std::string lowercase_ascii(std::string value) { + std::ranges::transform(value, value.begin(), [](const unsigned char ch) { + return static_cast(std::tolower(ch)); + }); + return value; +} + +std::string quote_string_value(std::string_view value) { + std::string quoted("'"); + quoted.reserve(value.size() + 2); + for (const char ch : value) { + if (ch == '\'') { + quoted.append("''"); + } else { + quoted.push_back(ch); + } + } + quoted.push_back('\''); + return quoted; +} + +std::optional to_lance_sql_literal(const VLiteral& literal) { + const auto type = remove_nullable(literal.get_data_type()); + auto options = DataTypeSerDe::get_default_format_options(); + auto timezone = cctz::utc_time_zone(); + options.timezone = &timezone; + const auto value = literal.value(options); + switch (type->get_primitive_type()) { + case TYPE_BOOLEAN: { + const auto normalized = lowercase_ascii(value); + if (normalized == "0" || normalized == "false") { + return "FALSE"; + } + if (normalized == "1" || normalized == "true") { + return "TRUE"; + } + return std::nullopt; + } + case TYPE_TINYINT: + case TYPE_SMALLINT: + case TYPE_INT: + case TYPE_BIGINT: + case TYPE_LARGEINT: + return value; + case TYPE_FLOAT: + case TYPE_DOUBLE: { + const auto normalized = lowercase_ascii(value); + if (normalized.find("nan") != std::string::npos || + normalized.find("inf") != std::string::npos) { + return std::nullopt; + } + return value; + } + case TYPE_DECIMALV2: + case TYPE_DECIMAL32: + case TYPE_DECIMAL64: + case TYPE_DECIMAL128I: + case TYPE_DECIMAL256: + return value; + case TYPE_CHAR: + case TYPE_VARCHAR: + case TYPE_STRING: + return quote_string_value(value); + case TYPE_DATE: + case TYPE_DATEV2: + return "DATE " + quote_string_value(value); + case TYPE_DATETIME: + case TYPE_DATETIMEV2: + return "TIMESTAMP " + quote_string_value(value); + default: + return std::nullopt; + } +} + +template +std::optional in_value_to_lance_sql_literal(const void* raw_value, + const DataTypePtr& data_type) { + if (raw_value == nullptr || data_type == nullptr) { + return std::nullopt; + } + Field field; + if constexpr (is_string_type(PT)) { + const auto* value = static_cast(raw_value); + using CppType = typename PrimitiveTypeTraits::CppType; + field = Field::create_field(CppType(value->data, value->size)); + } else { + using CppType = typename PrimitiveTypeTraits::CppType; + field = Field::create_field(*static_cast(raw_value)); + } + return to_lance_sql_literal(VLiteral(data_type, field)); +} + +std::optional in_value_to_lance_sql_literal(PrimitiveType primitive_type, + const void* raw_value, + const DataTypePtr& data_type) { +#define DISPATCH_IN_VALUE(TYPE) \ + case TYPE: \ + return in_value_to_lance_sql_literal(raw_value, data_type) + switch (primitive_type) { + DISPATCH_IN_VALUE(TYPE_BOOLEAN); + DISPATCH_IN_VALUE(TYPE_TINYINT); + DISPATCH_IN_VALUE(TYPE_SMALLINT); + DISPATCH_IN_VALUE(TYPE_INT); + DISPATCH_IN_VALUE(TYPE_BIGINT); + DISPATCH_IN_VALUE(TYPE_LARGEINT); + DISPATCH_IN_VALUE(TYPE_FLOAT); + DISPATCH_IN_VALUE(TYPE_DOUBLE); + DISPATCH_IN_VALUE(TYPE_DATE); + DISPATCH_IN_VALUE(TYPE_DATETIME); + DISPATCH_IN_VALUE(TYPE_DATEV2); + DISPATCH_IN_VALUE(TYPE_DATETIMEV2); + DISPATCH_IN_VALUE(TYPE_CHAR); + DISPATCH_IN_VALUE(TYPE_VARCHAR); + DISPATCH_IN_VALUE(TYPE_STRING); + DISPATCH_IN_VALUE(TYPE_DECIMALV2); + DISPATCH_IN_VALUE(TYPE_DECIMAL32); + DISPATCH_IN_VALUE(TYPE_DECIMAL64); + DISPATCH_IN_VALUE(TYPE_DECIMAL128I); + DISPATCH_IN_VALUE(TYPE_DECIMAL256); + default: + return std::nullopt; + } +#undef DISPATCH_IN_VALUE +} + +std::optional build_in_filter_sql(const VDirectInPredicate& predicate) { + if (predicate.get_num_children() != 1) { + return std::nullopt; + } + const auto slot = std::dynamic_pointer_cast(predicate.get_child(0)); + const auto values = predicate.get_set_func(); + if (slot == nullptr || slot->data_type() == nullptr || values == nullptr || + values->contain_null() || values->size() == 0) { + return std::nullopt; + } + + const auto data_type = remove_nullable(slot->data_type()); + std::string expression("(" + quote_sql_identifier(slot->column_name()) + " IN ("); + auto* iterator = values->begin(); + bool first_value = true; + while (iterator != nullptr && iterator->has_next()) { + auto value = in_value_to_lance_sql_literal(data_type->get_primitive_type(), + iterator->get_value(), data_type); + if (!value.has_value()) { + return std::nullopt; + } + if (!first_value) { + expression.append(", "); + } + expression.append(*value); + first_value = false; + iterator->next(); + } + if (first_value) { + return std::nullopt; + } + expression.append("))"); + return expression; +} + +std::optional build_range_filter_sql(const VExpr& predicate) { + if ((predicate.op() != TExprOpcode::GE && predicate.op() != TExprOpcode::LE) || + predicate.get_num_children() != 2) { + return std::nullopt; + } + const auto slot = std::dynamic_pointer_cast(predicate.get_child(0)); + const auto literal = std::dynamic_pointer_cast(predicate.get_child(1)); + if (slot == nullptr || literal == nullptr) { + return std::nullopt; + } + const auto sql_literal = to_lance_sql_literal(*literal); + if (!sql_literal.has_value()) { + return std::nullopt; + } + const auto* sql_operator = predicate.op() == TExprOpcode::GE ? ">=" : "<="; + return "(" + quote_sql_identifier(slot->column_name()) + " " + sql_operator + " " + + *sql_literal + ")"; +} + +std::optional runtime_filter_to_lance_sql(const RuntimeFilterExpr& runtime_filter) { + const auto impl = runtime_filter.get_impl(); + if (impl == nullptr) { + return std::nullopt; + } + if (const auto* in_predicate = dynamic_cast(impl.get()); + in_predicate != nullptr) { + return build_in_filter_sql(*in_predicate); + } + return build_range_filter_sql(*impl); +} + +std::shared_ptr build_runtime_filter_sql( + const VExprContextSPtrs& conjuncts) { + auto result = std::make_shared(); + std::set seen_filter_ids; + std::set pushed_filter_ids; + for (const auto& conjunct : conjuncts) { + const auto* runtime_filter = get_runtime_filter(conjunct); + if (runtime_filter == nullptr) { + continue; + } + const auto filter_id = runtime_filter->filter_id(); + seen_filter_ids.emplace(filter_id); + + const auto expression = runtime_filter_to_lance_sql(*runtime_filter); + if (!expression.has_value()) { + continue; + } + append_sql_conjunct(*expression, &result->expression); + pushed_filter_ids.emplace(filter_id); + } + + result->pushable_filter_ids.assign(pushed_filter_ids.begin(), pushed_filter_ids.end()); + for (const auto filter_id : seen_filter_ids) { + if (!pushed_filter_ids.contains(filter_id)) { + result->skipped_filter_ids.emplace_back(filter_id); + } + } + return result; +} + +std::optional build_cache_key(const VExprContextSPtrs& conjuncts) { + // This cache is scoped to one FileScanLocalState. An RF is immutable after it is published, so + // the sorted RF IDs uniquely identify the snapshot shared by its parallel scanners. + std::set filter_ids; + for (const auto& conjunct : conjuncts) { + if (const auto* runtime_filter = get_runtime_filter(conjunct); runtime_filter != nullptr) { + filter_ids.emplace(runtime_filter->filter_id()); + } + } + if (filter_ids.empty()) { + return std::nullopt; + } + + std::string key(LANCE_RUNTIME_FILTER_CACHE_KEY_PREFIX); + for (const auto filter_id : filter_ids) { + key.append(std::to_string(filter_id)).append(","); + } + return key; +} + +} // namespace + +std::shared_ptr get_or_create_lance_runtime_filter_sql( + const VExprContextSPtrs& conjuncts, ShardedKVCache* cache) { + const auto cache_key = build_cache_key(conjuncts); + if (!cache_key.has_value()) { + return nullptr; + } + if (cache == nullptr) { + return build_runtime_filter_sql(conjuncts); + } + + auto* cached = cache->get>( + *cache_key, [&]() -> std::shared_ptr* { + return new std::shared_ptr( + build_runtime_filter_sql(conjuncts)); + }); + return cached == nullptr ? nullptr : *cached; +} + +void record_lance_runtime_filter_pushdown(RuntimeProfile* profile, + const LanceRuntimeFilterSql& runtime_filter_sql) { + DORIS_CHECK(profile != nullptr); + const auto pushed_ids = format_filter_ids(runtime_filter_sql.pushable_filter_ids); + const auto skipped_ids = format_filter_ids(runtime_filter_sql.skipped_filter_ids); + + if (!pushed_ids.empty()) { + profile->add_info_string("LanceRuntimeFilterPushedIds", pushed_ids); + } + if (!skipped_ids.empty()) { + profile->add_info_string("LanceRuntimeFilterSkippedIds", skipped_ids); + } + VLOG_DEBUG << "Lance runtime filter pushdown: pushed_ids=[" << pushed_ids << "], skipped_ids=[" + << skipped_ids << "]"; +} + +} // namespace doris::format::lance diff --git a/be/src/format_v2/lance/lance_runtime_filter_helper.h b/be/src/format_v2/lance/lance_runtime_filter_helper.h new file mode 100644 index 00000000000000..c40631bfdbcc09 --- /dev/null +++ b/be/src/format_v2/lance/lance_runtime_filter_helper.h @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "exprs/vexpr_fwd.h" + +namespace doris { + +class ShardedKVCache; +class RuntimeProfile; + +namespace format::lance { + +struct LanceRuntimeFilterSql { + std::string expression; + std::vector pushable_filter_ids; + std::vector skipped_filter_ids; +}; + +// Build one immutable SQL snapshot for all supported Doris runtime filters. When cache is non-null, +// equivalent RF snapshots from parallel Lance readers in the same FileScanLocalState share the +// conversion result. The returned snapshot also identifies RFs that cannot be represented exactly +// by Lance SQL. A null result means that conjuncts contain no Doris runtime filter. +std::shared_ptr get_or_create_lance_runtime_filter_sql( + const VExprContextSPtrs& conjuncts, ShardedKVCache* cache); + +void record_lance_runtime_filter_pushdown(RuntimeProfile* profile, + const LanceRuntimeFilterSql& runtime_filter_sql); + +} // namespace format::lance +} // namespace doris diff --git a/be/src/format_v2/lance/lance_session_manager.cpp b/be/src/format_v2/lance/lance_session_manager.cpp new file mode 100644 index 00000000000000..c64c3c281e7ff0 --- /dev/null +++ b/be/src/format_v2/lance/lance_session_manager.cpp @@ -0,0 +1,118 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/lance/lance_session_manager.h" + +#include + +#include +#include + +#include "common/config.h" +#include "common/logging.h" +#include "format_v2/lance/lance_reader_helper.h" + +namespace doris::format::lance { +namespace { + +LanceSessionManager::Config load_lance_session_config() { + return { + .lance_index_cache_size_bytes = config::lance_index_cache_size_bytes, + .lance_metadata_cache_size_bytes = config::lance_metadata_cache_size_bytes, + .enable_lance_data_cache = config::enable_lance_data_cache, + .lance_data_cache_path = config::lance_data_cache_path, + .lance_data_cache_disk_capacity_bytes = config::lance_data_cache_disk_capacity_bytes, + .lance_data_cache_read_block_size_bytes = + config::lance_data_cache_read_block_size_bytes, + }; +} + +} // namespace + +LanceSessionManager& LanceSessionManager::instance() { + // Function-local static initialization is thread safe. Cache configuration is process scoped, + // so changing it requires a BE restart. + static LanceSessionManager manager(load_lance_session_config()); + return manager; +} + +LanceSessionManager::LanceSessionManager(Config config) : _config(std::move(config)) { + LOG(INFO) << "Creating BE-wide Lance session manager: lance_index_cache_size_bytes=" + << _config.lance_index_cache_size_bytes + << ", lance_metadata_cache_size_bytes=" + << _config.lance_metadata_cache_size_bytes + << ", enable_lance_data_cache=" << _config.enable_lance_data_cache + << ", lance_data_cache_path=" << _config.lance_data_cache_path + << ", lance_data_cache_disk_capacity_bytes=" + << _config.lance_data_cache_disk_capacity_bytes + << ", lance_data_cache_read_block_size_bytes=" + << _config.lance_data_cache_read_block_size_bytes + << ", foyer_memory_capacity_bytes=" + << _config.lance_data_cache_read_block_size_bytes; +} + +LanceSessionManager::~LanceSessionManager() { + lance_session_close(_session); +} + +Status LanceSessionManager::_initialize() { + if (_config.enable_lance_data_cache) { + const LanceDataCacheOptions data_cache_options { + .directory = _config.lance_data_cache_path.c_str(), + // Foyer's HybridCache requires a memory tier. Keep it at the minimum useful + // capacity of exactly one range-cache block; entries use WriteOnInsertion and + // are persisted to the disk tier immediately. + .memory_capacity_bytes = + static_cast(_config.lance_data_cache_read_block_size_bytes), + .disk_capacity_bytes = + static_cast(_config.lance_data_cache_disk_capacity_bytes), + .read_block_size_bytes = + static_cast(_config.lance_data_cache_read_block_size_bytes), + }; + _session = lance_session_new_with_data_cache( + static_cast(_config.lance_index_cache_size_bytes), + static_cast(_config.lance_metadata_cache_size_bytes), + &data_cache_options); + } else { + _session = lance_session_new( + static_cast(_config.lance_index_cache_size_bytes), + static_cast(_config.lance_metadata_cache_size_bytes)); + } + if (_session == nullptr) { + return lance_error("create shared Lance session"); + } + return Status::OK(); +} + +Status LanceSessionManager::open_dataset(const char* uri, const char* const* storage_options, + uint64_t version, LanceDataset** dataset) { + if (uri == nullptr || dataset == nullptr) { + return Status::InvalidArgument("Lance dataset URI and output must not be null"); + } + *dataset = nullptr; + + std::call_once(_initialize_once, [this] { _initialize_status = _initialize(); }); + RETURN_IF_ERROR(_initialize_status); + + *dataset = lance_dataset_open_with_session(uri, storage_options, version, _session); + if (*dataset == nullptr) { + return lance_error("open Lance dataset with shared session"); + } + return Status::OK(); +} + +} // namespace doris::format::lance diff --git a/be/src/format_v2/lance/lance_session_manager.h b/be/src/format_v2/lance/lance_session_manager.h new file mode 100644 index 00000000000000..905eb78b1af30d --- /dev/null +++ b/be/src/format_v2/lance/lance_session_manager.h @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" + +struct LanceDataset; +struct LanceSession; + +namespace doris::format::lance { + +// Owns the single Lance session shared by all queries in one BE process. The session always owns +// Lance's metadata/index caches and optionally installs the Foyer data-file cache. Readers only +// open datasets through this class and do not depend on the selected data-cache implementation. +class LanceSessionManager final { +public: + struct Config { + int64_t lance_index_cache_size_bytes = 0; + int64_t lance_metadata_cache_size_bytes = 0; + bool enable_lance_data_cache = false; + std::string lance_data_cache_path; + int64_t lance_data_cache_disk_capacity_bytes = 0; + int64_t lance_data_cache_read_block_size_bytes = 0; + }; + + static LanceSessionManager& instance(); + + // The explicit configuration constructor keeps the process-global config out of focused + // manager tests. Production readers use instance(). + explicit LanceSessionManager(Config config); + ~LanceSessionManager(); + + LanceSessionManager(const LanceSessionManager&) = delete; + LanceSessionManager& operator=(const LanceSessionManager&) = delete; + + Status open_dataset(const char* uri, const char* const* storage_options, uint64_t version, + LanceDataset** dataset); + +private: + Status _initialize(); + + Config _config; + std::once_flag _initialize_once; + LanceSession* _session = nullptr; + Status _initialize_status = Status::OK(); +}; + +} // namespace doris::format::lance diff --git a/be/src/format_v2/table/lance_reader.cpp b/be/src/format_v2/table/lance_reader.cpp index e8ad0c5229aca3..950b16bce292e8 100644 --- a/be/src/format_v2/table/lance_reader.cpp +++ b/be/src/format_v2/table/lance_reader.cpp @@ -21,236 +21,27 @@ #include #include #include -#include #include +#include #include #include #include #include +#include #include "common/consts.h" #include "common/logging.h" #include "core/column/column_nullable.h" #include "core/column/column_string.h" -#include "core/data_type/data_type_array.h" -#include "core/data_type/data_type_factory.hpp" -#include "core/data_type/data_type_map.h" -#include "core/data_type/data_type_nothing.h" -#include "core/data_type/data_type_struct.h" #include "exec/common/endian.h" +#include "format_v2/lance/lance_reader_helper.h" +#include "format_v2/lance/lance_runtime_filter_helper.h" +#include "format_v2/lance/lance_session_manager.h" #include "runtime/file_scan_profile.h" #include "storage/utils.h" namespace doris::format::lance { -namespace { - -struct LanceDatasetDeleter { - void operator()(LanceDataset* dataset) const { lance_dataset_close(dataset); } -}; - -struct LanceScannerDeleter { - void operator()(LanceScanner* scanner) const { lance_scanner_close(scanner); } -}; - -struct LanceBatchDeleter { - void operator()(LanceBatch* batch) const { lance_batch_free(batch); } -}; - -constexpr std::string_view DISTANCE_COLUMN = "_distance"; -constexpr std::string_view ROW_ID_COLUMN = "_rowid"; -constexpr std::string_view ARROW_EXTENSION_NAME = "ARROW:extension:name"; -constexpr const char* LANCE_READER_PROFILE = "LanceReader"; - -size_t vector_element_width(TVectorElementType::type type) { - switch (type) { - case TVectorElementType::FLOAT16: - return sizeof(uint16_t); - case TVectorElementType::FLOAT32: - return sizeof(float); - case TVectorElementType::FLOAT64: - return sizeof(double); - case TVectorElementType::UINT8: - case TVectorElementType::INT8: - return sizeof(uint8_t); - } - return 0; -} - -int arrow_time_precision(arrow::TimeUnit::type unit) { - switch (unit) { - case arrow::TimeUnit::SECOND: - return 0; - case arrow::TimeUnit::MILLI: - return 3; - case arrow::TimeUnit::MICRO: - case arrow::TimeUnit::NANO: - return 6; - } - return 6; -} - -Status check_arrow_field_semantics(const std::shared_ptr& field) { - if (field->HasMetadata()) { - const auto extension_name = field->metadata()->Get(ARROW_EXTENSION_NAME); - if (extension_name.ok() && !extension_name.ValueUnsafe().empty()) { - return Status::NotSupported( - "unsupported Lance Arrow extension type '{}' for field '{}'", - extension_name.ValueUnsafe(), field->name()); - } - } - if (field->type()->id() == arrow::Type::DICTIONARY) { - return Status::NotSupported("unsupported Lance Arrow dictionary type for field '{}': {}", - field->name(), field->type()->ToString()); - } - return Status::OK(); -} - -Status arrow_field_to_doris_type(const std::shared_ptr& field, - DataTypePtr* doris_type) { - RETURN_IF_ERROR(check_arrow_field_semantics(field)); - const auto& arrow_type = field->type(); - const auto nullable_primitive = [&](PrimitiveType type, int precision = 0, int scale = 0, - int len = -1) { - *doris_type = - DataTypeFactory::instance().create_data_type(type, true, precision, scale, len); - return Status::OK(); - }; - - switch (arrow_type->id()) { - case arrow::Type::BOOL: - return nullable_primitive(TYPE_BOOLEAN); - case arrow::Type::INT8: - return nullable_primitive(TYPE_TINYINT); - case arrow::Type::UINT8: - case arrow::Type::INT16: - return nullable_primitive(TYPE_SMALLINT); - case arrow::Type::UINT16: - case arrow::Type::INT32: - return nullable_primitive(TYPE_INT); - case arrow::Type::UINT32: - case arrow::Type::INT64: - return nullable_primitive(TYPE_BIGINT); - case arrow::Type::UINT64: - return nullable_primitive(TYPE_LARGEINT); - case arrow::Type::HALF_FLOAT: - case arrow::Type::FLOAT: - return nullable_primitive(TYPE_FLOAT); - case arrow::Type::DOUBLE: - return nullable_primitive(TYPE_DOUBLE); - case arrow::Type::STRING: - case arrow::Type::LARGE_STRING: - return nullable_primitive(TYPE_STRING); - case arrow::Type::BINARY: - case arrow::Type::LARGE_BINARY: - return nullable_primitive(TYPE_VARBINARY, 0, 0, std::numeric_limits::max()); - case arrow::Type::FIXED_SIZE_BINARY: { - const auto binary = std::static_pointer_cast(arrow_type); - return nullable_primitive(TYPE_VARBINARY, 0, 0, binary->byte_width()); - } - case arrow::Type::DATE32: - case arrow::Type::DATE64: - return nullable_primitive(TYPE_DATEV2); - case arrow::Type::TIME32: - case arrow::Type::TIME64: { - const auto time = std::static_pointer_cast(arrow_type); - return nullable_primitive(TYPE_TIMEV2, 0, arrow_time_precision(time->unit())); - } - case arrow::Type::TIMESTAMP: { - const auto timestamp = std::static_pointer_cast(arrow_type); - const auto doris_type = timestamp->timezone().empty() ? TYPE_DATETIMEV2 : TYPE_TIMESTAMPTZ; - return nullable_primitive(doris_type, 0, arrow_time_precision(timestamp->unit())); - } - case arrow::Type::DECIMAL128: - case arrow::Type::DECIMAL256: { - const auto decimal = std::static_pointer_cast(arrow_type); - const int precision = decimal->precision(); - const int scale = decimal->scale(); - if (precision <= 0 || precision > arrow::Decimal256Type::kMaxPrecision || scale < 0 || - scale > precision) { - return Status::NotSupported( - "unsupported Lance Arrow decimal type for field '{}': precision={}, scale={}", - field->name(), precision, scale); - } - const PrimitiveType doris_decimal_type = precision <= 9 ? TYPE_DECIMAL32 - : precision <= 18 ? TYPE_DECIMAL64 - : precision <= 38 ? TYPE_DECIMAL128I - : TYPE_DECIMAL256; - return nullable_primitive(doris_decimal_type, precision, scale); - } - case arrow::Type::LIST: - case arrow::Type::LARGE_LIST: - case arrow::Type::FIXED_SIZE_LIST: { - const auto list = std::static_pointer_cast(arrow_type); - DataTypePtr value_type; - RETURN_IF_ERROR(arrow_field_to_doris_type(list->value_field(), &value_type)); - *doris_type = make_nullable(std::make_shared(value_type)); - return Status::OK(); - } - case arrow::Type::MAP: { - const auto map = std::static_pointer_cast(arrow_type); - RETURN_IF_ERROR(check_arrow_field_semantics(map->value_field())); - DataTypePtr key_type; - DataTypePtr item_type; - RETURN_IF_ERROR(arrow_field_to_doris_type(map->key_field(), &key_type)); - RETURN_IF_ERROR(arrow_field_to_doris_type(map->item_field(), &item_type)); - *doris_type = make_nullable(std::make_shared(key_type, item_type)); - return Status::OK(); - } - case arrow::Type::STRUCT: { - const auto struct_type = std::static_pointer_cast(arrow_type); - DataTypes field_types; - Strings field_names; - field_types.reserve(struct_type->num_fields()); - field_names.reserve(struct_type->num_fields()); - for (const auto& child : struct_type->fields()) { - DataTypePtr field_type; - RETURN_IF_ERROR(arrow_field_to_doris_type(child, &field_type)); - field_types.emplace_back(std::move(field_type)); - field_names.emplace_back(child->name()); - } - *doris_type = make_nullable(std::make_shared(field_types, field_names)); - return Status::OK(); - } - default: - return Status::NotSupported("unsupported Lance Arrow type: {}", arrow_type->ToString()); - } -} - -} // namespace - -Status convert_arrow_schema_to_doris(const std::shared_ptr& arrow_schema, - std::vector* column_names, - std::vector* column_types) { - DORIS_CHECK(arrow_schema != nullptr); - DORIS_CHECK(column_names != nullptr); - DORIS_CHECK(column_types != nullptr); - - std::vector parsed_names; - std::vector parsed_types; - parsed_names.reserve(arrow_schema->num_fields()); - parsed_types.reserve(arrow_schema->num_fields()); - std::unordered_set unique_names; - unique_names.reserve(arrow_schema->num_fields()); - for (const auto& field : arrow_schema->fields()) { - if (!unique_names.emplace(field->name()).second) { - return Status::InvalidArgument("duplicate Lance schema column: {}", field->name()); - } - DataTypePtr doris_type; - const auto type_status = arrow_field_to_doris_type(field, &doris_type); - if (type_status.is()) { - parsed_types.emplace_back(std::make_shared()); - } else { - RETURN_IF_ERROR(type_status); - DORIS_CHECK(doris_type != nullptr); - parsed_types.emplace_back(std::move(doris_type)); - } - parsed_names.emplace_back(field->name()); - } - *column_names = std::move(parsed_names); - *column_types = std::move(parsed_types); - return Status::OK(); -} LanceTableReader::~LanceTableReader() { static_cast(close()); @@ -265,7 +56,7 @@ Status LanceTableReader::fetch_schema(const TFileRangeDesc& range, } const auto& params = range.table_format_params.lance_params; std::vector storage_options; - RETURN_IF_ERROR(_storage_options(&scan_params, &storage_options)); + RETURN_IF_ERROR(build_lance_storage_options(&scan_params, &storage_options)); std::vector storage_option_ptrs; storage_option_ptrs.reserve(storage_options.size() + 1); for (const auto& option : storage_options) { @@ -278,12 +69,12 @@ Status LanceTableReader::fetch_schema(const TFileRangeDesc& range, storage_options.empty() ? nullptr : storage_option_ptrs.data(), static_cast(params.version))); if (dataset == nullptr) { - return _lance_error("open Lance dataset for schema"); + return lance_error("open Lance dataset for schema"); } ArrowSchema arrow_schema {}; if (lance_dataset_schema(dataset.get(), &arrow_schema) != 0) { - return _lance_error("get Lance dataset schema"); + return lance_error("get Lance dataset schema"); } auto imported_schema = arrow::ImportSchema(&arrow_schema); if (!imported_schema.ok()) { @@ -303,6 +94,7 @@ Status LanceTableReader::init(TableReadOptions&& options) { DORIS_CHECK(_runtime_state != nullptr); DORIS_CHECK(_scanner_profile != nullptr); DORIS_CHECK(_scan_params != nullptr); + RETURN_IF_ERROR(_resolve_search_kind()); _ctz = _runtime_state->timezone_obj(); const auto& lance_scan_params = _scan_params->lance_scan_params; @@ -322,6 +114,12 @@ Status LanceTableReader::init(TableReadOptions&& options) { TUnit::UNIT, LANCE_READER_PROFILE, 1); _execution_bytes_read = ADD_CHILD_COUNTER_WITH_LEVEL( _scanner_profile, "LanceExecutionIOBytesRead", TUnit::BYTES, LANCE_READER_PROFILE, 1); + _data_cache_bytes_read_from_cache = ADD_CHILD_COUNTER_WITH_LEVEL( + _scanner_profile, "LanceDataCacheBytesReadFromCache", TUnit::BYTES, + LANCE_READER_PROFILE, 1); + _data_cache_bytes_read_from_remote = ADD_CHILD_COUNTER_WITH_LEVEL( + _scanner_profile, "LanceDataCacheBytesReadFromRemote", TUnit::BYTES, + LANCE_READER_PROFILE, 1); _index_partition_cache_miss_loads = ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "LanceIndexPartitionCacheMissLoads", TUnit::UNIT, LANCE_READER_PROFILE, 1); @@ -358,16 +156,42 @@ Status LanceTableReader::init(TableReadOptions&& options) { ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "LanceIVFPartitionRankingTime", LANCE_READER_PROFILE, 1)}, }; - _vector_search = _scan_params->__isset.lance_scan_params && - lance_scan_params.__isset.external_search_request; - if (_vector_search) { + if (_search_kind != SearchKind::NORMAL) { RETURN_IF_ERROR(_validate_external_search_request()); const auto& request = lance_scan_params.external_search_request; - const auto& vector = request.search_query.vector_search; - _scanner_profile->add_info_string("LanceTopK", std::to_string(vector.top_k)); - _scanner_profile->add_info_string("LanceOffset", std::to_string(vector.offset)); - _scanner_profile->add_info_string("LanceTopKPlusOffset", - std::to_string(vector.top_k + vector.offset)); + int64_t top_k; + int64_t offset; + if (_search_kind == SearchKind::VECTOR) { + const auto& vector = request.search_query.vector_search; + top_k = vector.top_k; + offset = vector.offset; + _scanner_profile->add_info_string("LanceSearchType", "VECTOR"); + } else { + DORIS_CHECK(_search_kind == SearchKind::FULL_TEXT); + const auto& full_text = request.search_query.full_text_search; + top_k = full_text.top_k; + offset = full_text.offset; + _scanner_profile->add_info_string("LanceSearchType", "FULL_TEXT"); + _scanner_profile->add_info_string( + "LanceFtsCoverageMode", + full_text.coverage_mode == TFtsCoverageMode::STRICT ? "STRICT" : "INDEX_ONLY"); + if (full_text.query_type == TFtsQueryType::MATCH) { + _scanner_profile->add_info_string("LanceFtsQueryType", "MATCH"); + _scanner_profile->add_info_string( + "LanceFtsMatchOperator", + full_text.match_operator == TFtsMatchOperator::AND ? "AND" : "OR"); + _scanner_profile->add_info_string( + "LanceFtsMaxFuzzyDistance", + std::to_string(full_text.max_fuzzy_distance)); + } else { + _scanner_profile->add_info_string("LanceFtsQueryType", "PHRASE"); + _scanner_profile->add_info_string("LanceFtsPhraseSlop", + std::to_string(full_text.phrase_slop)); + } + } + _scanner_profile->add_info_string("LanceTopK", std::to_string(top_k)); + _scanner_profile->add_info_string("LanceOffset", std::to_string(offset)); + _scanner_profile->add_info_string("LanceTopKPlusOffset", std::to_string(top_k + offset)); _planned_index_segment_count = ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "LancePlannedIndexSegmentCount", TUnit::UNIT, LANCE_READER_PROFILE, 1); @@ -395,9 +219,9 @@ Status LanceTableReader::init(TableReadOptions&& options) { return Status::InvalidArgument("Lance projected column '{}' has no type", column.name); } if (column.name.starts_with(BeConsts::GLOBAL_ROWID_COL)) { - if (!_vector_search) { + if (_search_kind == SearchKind::NORMAL) { return Status::NotSupported( - "Lance global row id is currently supported only for vector search"); + "Lance global row id is currently supported only for external search"); } if (_global_rowid_output_idx.has_value()) { return Status::InvalidArgument("duplicate Lance global row id projected column: {}", @@ -414,12 +238,19 @@ Status LanceTableReader::init(TableReadOptions&& options) { if (!_output_name_to_idx.emplace(column.name, idx).second) { return Status::InvalidArgument("duplicate Lance projected column: {}", column.name); } - if (_vector_search && column.name == DISTANCE_COLUMN) { + if (_search_kind == SearchKind::VECTOR && column.name == LANCE_DISTANCE_COLUMN) { const auto distance_type = remove_nullable(column.type); if (distance_type->get_primitive_type() != TYPE_FLOAT) { return Status::InvalidArgument( "Lance vector search column '{}' must have Doris FLOAT type, but was {}", - DISTANCE_COLUMN, column.type->get_name()); + LANCE_DISTANCE_COLUMN, column.type->get_name()); + } + } else if (_search_kind == SearchKind::FULL_TEXT && column.name == LANCE_SCORE_COLUMN) { + const auto score_type = remove_nullable(column.type); + if (score_type->get_primitive_type() != TYPE_FLOAT) { + return Status::InvalidArgument( + "Lance full-text search column '{}' must have Doris FLOAT type, but was {}", + LANCE_SCORE_COLUMN, column.type->get_name()); } } } @@ -429,6 +260,7 @@ Status LanceTableReader::init(TableReadOptions&& options) { Status LanceTableReader::prepare_split(const SplitReadOptions& options) { _close_scanner(); _eof = false; + _runtime_filter_cache = options.cache; RETURN_IF_ERROR(TableReader::prepare_split(options)); // Lance does not currently provide metadata aggregate pushdown. Do not let a generic @@ -485,7 +317,7 @@ Status LanceTableReader::get_block(Block* block, bool* eos) { break; } if (scan_status != 0 || raw_batch == nullptr) { - return _lance_error("read next Lance batch"); + return lance_error("read next Lance batch"); } std::unique_ptr batch(raw_batch); @@ -533,7 +365,9 @@ Status LanceTableReader::read_by_row_ids(const TFileRangeDesc& range, } SCOPED_TIMER(_row_id_fetch_total_time); - RETURN_IF_ERROR(_ensure_dataset_open(range)); + // Phase-two row fetch does not execute FTS, so a reader created only for take_rows must not + // collect query-specific global statistics. + RETURN_IF_ERROR(_ensure_dataset_open(range, false)); std::vector columns; columns.reserve(_projected_columns.size() + 1); for (const auto& column : _projected_columns) { @@ -552,7 +386,7 @@ Status LanceTableReader::read_by_row_ids(const TFileRangeDesc& range, if (stream.release != nullptr) { stream.release(&stream); } - return _lance_error("take Lance rows by row id"); + return lance_error("take Lance rows by row id"); } auto imported_reader = arrow::ImportRecordBatchReader(&stream); if (!imported_reader.ok()) { @@ -608,8 +442,31 @@ Status LanceTableReader::close() { return TableReader::close(); } +Status LanceTableReader::_resolve_search_kind() { + DORIS_CHECK(_scan_params != nullptr); + _search_kind = SearchKind::NORMAL; + if (!_scan_params->__isset.lance_scan_params) { + return Status::OK(); + } + const auto& lance_scan_params = _scan_params->lance_scan_params; + if (!lance_scan_params.__isset.external_search_request) { + return Status::OK(); + } + const auto& request = lance_scan_params.external_search_request; + if (!request.__isset.search_query) { + return Status::InvalidArgument("external search request requires search_query"); + } + const bool has_vector = request.search_query.__isset.vector_search; + const bool has_full_text = request.search_query.__isset.full_text_search; + if (has_vector == has_full_text) { + return Status::InvalidArgument("external search query must set exactly one search kind"); + } + _search_kind = has_vector ? SearchKind::VECTOR : SearchKind::FULL_TEXT; + return Status::OK(); +} + Status LanceTableReader::_validate_external_search_request() const { - // FE validates requests produced by vector_search(), but this reader consumes a deserialized + // FE validates requests produced by the search TVFs, but this reader consumes a deserialized // Thrift boundary. Recheck structural invariants and values used for allocation, pointer // arithmetic, C-string calls, and narrowing conversions before accessing them below. DORIS_CHECK(_scan_params != nullptr); @@ -618,7 +475,7 @@ Status LanceTableReader::_validate_external_search_request() const { DORIS_CHECK(lance_scan_params.__isset.external_search_request); if (lance_scan_params.__isset.lance_substrait_filter) { return Status::InvalidArgument( - "Lance vector search cannot combine its pre-search filter with " + "Lance external search cannot combine its pre-search filter with " "lance_substrait_filter"); } @@ -627,58 +484,122 @@ Status LanceTableReader::_validate_external_search_request() const { return Status::NotSupported("unsupported external search schema version: {}", request.schema_version); } - if (!request.__isset.search_query) { - return Status::InvalidArgument("external search request requires search_query"); - } - - const bool has_vector = request.search_query.__isset.vector_search; - const bool has_full_text = request.search_query.__isset.full_text_search; - if (has_vector == has_full_text) { - return Status::InvalidArgument("external search query must set exactly one search kind"); - } - if (has_full_text) { - return Status::NotSupported("Lance Format V2 reader does not yet support full-text search"); - } - - const auto& vector = request.search_query.vector_search; - if (!vector.__isset.column || vector.column.empty() || - vector.column.find('\0') != std::string::npos) { - return Status::InvalidArgument("Lance vector search requires a non-empty column"); - } - if (!vector.__isset.query_vector) { - return Status::InvalidArgument("Lance vector search requires a query vector"); - } - const auto& query_vector = vector.query_vector; - if (!query_vector.__isset.element_type || !query_vector.__isset.dimension || - !query_vector.__isset.values) { - return Status::InvalidArgument( - "Lance query vector requires element_type, dimension, and values"); - } - if (query_vector.dimension <= 0) { - return Status::InvalidArgument("Lance query vector dimension must be positive: {}", - query_vector.dimension); - } - const auto element_width = vector_element_width(query_vector.element_type); - if (element_width == 0) { - return Status::NotSupported("unsupported Lance query vector element type: {}", - static_cast(query_vector.element_type)); - } - const auto dimension = static_cast(query_vector.dimension); - if (dimension > std::numeric_limits::max() / element_width || - query_vector.values.size() != dimension * element_width) { - return Status::InvalidArgument( - "Lance query vector byte size {} does not match dimension {} and element width {}", - query_vector.values.size(), dimension, element_width); - } - if (!vector.__isset.top_k || vector.top_k <= 0) { - return Status::InvalidArgument("Lance vector search top_k must be positive"); - } - if (!vector.__isset.offset || vector.offset < 0) { - return Status::InvalidArgument("Lance vector search offset must be non-negative"); - } + DORIS_CHECK(request.__isset.search_query); + DORIS_CHECK(_search_kind != SearchKind::NORMAL); constexpr auto UINT32_MAX_VALUE = static_cast(std::numeric_limits::max()); - if (vector.offset > UINT32_MAX_VALUE || vector.top_k > UINT32_MAX_VALUE - vector.offset) { - return Status::InvalidArgument("Lance vector search top_k + offset exceeds uint32 range"); + if (_search_kind == SearchKind::VECTOR) { + const auto& vector = request.search_query.vector_search; + if (!vector.__isset.column || vector.column.empty() || + vector.column.find('\0') != std::string::npos) { + return Status::InvalidArgument("Lance vector search requires a non-empty column"); + } + if (!vector.__isset.query_vector) { + return Status::InvalidArgument("Lance vector search requires a query vector"); + } + const auto& query_vector = vector.query_vector; + if (!query_vector.__isset.element_type || !query_vector.__isset.dimension || + !query_vector.__isset.values) { + return Status::InvalidArgument( + "Lance query vector requires element_type, dimension, and values"); + } + if (query_vector.dimension <= 0) { + return Status::InvalidArgument("Lance query vector dimension must be positive: {}", + query_vector.dimension); + } + const auto element_width = lance_vector_element_width(query_vector.element_type); + if (element_width == 0) { + return Status::NotSupported("unsupported Lance query vector element type: {}", + static_cast(query_vector.element_type)); + } + const auto dimension = static_cast(query_vector.dimension); + if (dimension > std::numeric_limits::max() / element_width || + query_vector.values.size() != dimension * element_width) { + return Status::InvalidArgument( + "Lance query vector byte size {} does not match dimension {} and element width " + "{}", + query_vector.values.size(), dimension, element_width); + } + if (!vector.__isset.top_k || vector.top_k <= 0) { + return Status::InvalidArgument("Lance vector search top_k must be positive"); + } + if (!vector.__isset.offset || vector.offset < 0) { + return Status::InvalidArgument("Lance vector search offset must be non-negative"); + } + if (vector.offset > UINT32_MAX_VALUE || vector.top_k > UINT32_MAX_VALUE - vector.offset) { + return Status::InvalidArgument( + "Lance vector search top_k + offset exceeds uint32 range"); + } + } else { + DORIS_CHECK(_search_kind == SearchKind::FULL_TEXT); + const auto& full_text = request.search_query.full_text_search; + if (!full_text.__isset.column || full_text.column.empty() || + full_text.column.find('\0') != std::string::npos) { + return Status::InvalidArgument("Lance full-text search requires a non-empty column"); + } + if (!full_text.__isset.query || full_text.query.empty() || + full_text.query.find('\0') != std::string::npos) { + return Status::InvalidArgument("Lance full-text search requires a non-empty query"); + } + if (!full_text.__isset.top_k || full_text.top_k <= 0) { + return Status::InvalidArgument("Lance full-text search top_k must be positive"); + } + if (!full_text.__isset.offset || full_text.offset < 0) { + return Status::InvalidArgument("Lance full-text search offset must be non-negative"); + } + if (full_text.offset > UINT32_MAX_VALUE || + full_text.top_k > UINT32_MAX_VALUE - full_text.offset) { + return Status::InvalidArgument( + "Lance full-text search top_k + offset exceeds uint32 range"); + } + if (!full_text.__isset.coverage_mode || + (full_text.coverage_mode != TFtsCoverageMode::STRICT && + full_text.coverage_mode != TFtsCoverageMode::INDEX_ONLY)) { + return Status::InvalidArgument( + "Lance full-text search requires STRICT or INDEX_ONLY coverage_mode"); + } + if (full_text.__isset.global_statistics && full_text.global_statistics.empty()) { + return Status::InvalidArgument( + "Lance full-text search global_statistics must not be empty when set"); + } + if (!full_text.__isset.query_type || + (full_text.query_type != TFtsQueryType::MATCH && + full_text.query_type != TFtsQueryType::PHRASE)) { + return Status::InvalidArgument( + "Lance full-text search requires MATCH or PHRASE query_type"); + } + if (full_text.query_type == TFtsQueryType::MATCH) { + if (!full_text.__isset.match_operator || + (full_text.match_operator != TFtsMatchOperator::OR && + full_text.match_operator != TFtsMatchOperator::AND)) { + return Status::InvalidArgument( + "Lance MATCH query requires OR or AND match_operator"); + } + if (!full_text.__isset.max_fuzzy_distance || full_text.max_fuzzy_distance < 0) { + return Status::InvalidArgument( + "Lance MATCH query max_fuzzy_distance must be non-negative"); + } + if (full_text.max_fuzzy_distance != 0) { + return Status::NotSupported( + "Lance prepared FTS does not yet support max_fuzzy_distance={}", + full_text.max_fuzzy_distance); + } + if (full_text.__isset.phrase_slop) { + return Status::InvalidArgument("Lance MATCH query cannot set phrase_slop"); + } + } else { + if (!full_text.__isset.phrase_slop || full_text.phrase_slop < 0) { + return Status::InvalidArgument( + "Lance PHRASE query phrase_slop must be non-negative"); + } + if (full_text.__isset.match_operator || full_text.__isset.max_fuzzy_distance) { + return Status::InvalidArgument( + "Lance PHRASE query cannot set MATCH-only parameters"); + } + } + if (request.__isset.vector_search_options) { + return Status::InvalidArgument( + "Lance full-text search cannot set vector_search_options"); + } } if (request.__isset.search_filter) { @@ -687,22 +608,16 @@ Status LanceTableReader::_validate_external_search_request() const { return Status::InvalidArgument( "external search filter requires format and non-empty payload"); } - switch (filter.format) { - case TSearchFilterFormat::SQL: - if (filter.payload.find('\0') != std::string::npos) { - return Status::InvalidArgument( - "Lance SQL search filter contains an embedded NUL byte"); - } - break; - case TSearchFilterFormat::SUBSTRAIT: - break; - default: + if (filter.format != TSearchFilterFormat::SQL) { return Status::NotSupported("unsupported external search filter format: {}", static_cast(filter.format)); } + if (filter.payload.find('\0') != std::string::npos) { + return Status::InvalidArgument("Lance SQL search filter contains an embedded NUL byte"); + } } - if (request.__isset.vector_search_options) { + if (_search_kind == SearchKind::VECTOR && request.__isset.vector_search_options) { const auto& options = request.vector_search_options; if (options.__isset.nprobes && options.nprobes <= 0) { return Status::InvalidArgument("Lance nprobes must be positive"); @@ -717,7 +632,8 @@ Status LanceTableReader::_validate_external_search_request() const { return Status::OK(); } -Status LanceTableReader::_ensure_dataset_open(const TFileRangeDesc& range) { +Status LanceTableReader::_ensure_dataset_open(const TFileRangeDesc& range, + bool prepare_fts_context) { DatasetKey key; RETURN_IF_ERROR(_dataset_key(range, &key)); if (_dataset == nullptr) { @@ -727,6 +643,10 @@ Status LanceTableReader::_ensure_dataset_open(const TFileRangeDesc& range) { return Status::InvalidArgument( "Lance reader cannot mix dataset snapshots or storage options"); } + if (_search_kind == SearchKind::FULL_TEXT && prepare_fts_context && + _fts_query_context == nullptr) { + RETURN_IF_ERROR(_prepare_fts_query_context()); + } return Status::OK(); } @@ -740,12 +660,45 @@ Status LanceTableReader::_open_dataset(const DatasetKey& key) { { SCOPED_TIMER(_dataset_open_time); - _dataset = lance_dataset_open( - key.uri.c_str(), key.storage_options.empty() ? nullptr : storage_option_ptrs.data(), - static_cast(key.version)); + RETURN_IF_ERROR(LanceSessionManager::instance().open_dataset( + key.uri.c_str(), + key.storage_options.empty() ? nullptr : storage_option_ptrs.data(), + static_cast(key.version), &_dataset)); } - if (_dataset == nullptr) { - return _lance_error("open Lance dataset"); + return Status::OK(); +} + +Status LanceTableReader::_prepare_fts_query_context() { + DORIS_CHECK(_dataset != nullptr); + DORIS_CHECK(_fts_query_context == nullptr); + DORIS_CHECK(_scan_params != nullptr); + const auto& full_text = + _scan_params->lance_scan_params.external_search_request.search_query.full_text_search; + if (full_text.__isset.global_statistics) { + return Status::NotSupported( + "Lance FE-provided FTS global statistics require a lance-c consumer API"); + } + const auto coverage_mode = full_text.coverage_mode == TFtsCoverageMode::STRICT + ? LANCE_FTS_COVERAGE_STRICT + : LANCE_FTS_COVERAGE_INDEX_ONLY; + // Keep statistics preparation at the reader/scanner lifetime today. A future FE-provided + // opaque statistics payload should enter through this boundary and create the same context, + // leaving segment-scoped scanner execution unchanged. + if (full_text.query_type == TFtsQueryType::MATCH) { + const auto match_operator = full_text.match_operator == TFtsMatchOperator::AND + ? LANCE_FTS_MATCH_OPERATOR_AND + : LANCE_FTS_MATCH_OPERATOR_OR; + _fts_query_context = lance_dataset_prepare_fts_match_query( + _dataset, full_text.column.c_str(), full_text.query.c_str(), match_operator, + static_cast(full_text.max_fuzzy_distance), coverage_mode); + } else { + DORIS_CHECK(full_text.query_type == TFtsQueryType::PHRASE); + _fts_query_context = lance_dataset_prepare_fts_phrase_query( + _dataset, full_text.column.c_str(), full_text.query.c_str(), + full_text.phrase_slop, coverage_mode); + } + if (_fts_query_context == nullptr) { + return lance_error("prepare Lance FTS query context"); } return Status::OK(); } @@ -761,26 +714,32 @@ Status LanceTableReader::_open_scanner(const TFileRangeDesc& range) { const auto& column = _projected_columns[idx]; columns.emplace_back(column.name.c_str()); } - if (_vector_search && columns.empty()) { + if (_search_kind != SearchKind::NORMAL && columns.empty()) { // Keep an explicit empty user projection from becoming `nullptr`, which means all dataset - // columns to lance-c. nearest() already returns this optional system column. - columns.emplace_back(DISTANCE_COLUMN.data()); + // columns to lance-c. Search execution already returns its generated result column. + columns.emplace_back(_search_kind == SearchKind::VECTOR ? LANCE_DISTANCE_COLUMN.data() + : LANCE_SCORE_COLUMN.data()); } columns.emplace_back(nullptr); const auto& lance_scan_params = _scan_params->lance_scan_params; - const char* sql_filter = nullptr; - if (_vector_search) { + std::string sql_filter; + std::shared_ptr runtime_filter_sql; + if (_search_kind == SearchKind::NORMAL) { + runtime_filter_sql = + get_or_create_lance_runtime_filter_sql(_conjuncts, _runtime_filter_cache); + } else { const auto& request = lance_scan_params.external_search_request; if (request.__isset.search_filter && request.search_filter.format == TSearchFilterFormat::SQL) { - sql_filter = request.search_filter.payload.c_str(); + sql_filter = request.search_filter.payload; } } LanceScanner* scanner = - lance_scanner_new(_dataset, columns.size() == 1 ? nullptr : columns.data(), sql_filter); + lance_scanner_new(_dataset, columns.size() == 1 ? nullptr : columns.data(), + sql_filter.empty() ? nullptr : sql_filter.c_str()); if (scanner == nullptr) { - return _lance_error("create Lance scanner"); + return lance_error("create Lance scanner"); } std::unique_ptr scanner_guard(scanner); const auto collect_scan_statistics = [](void* callback_ctx, @@ -788,114 +747,93 @@ Status LanceTableReader::_open_scanner(const TFileRangeDesc& range) { LanceTableReader::_collect_scan_statistics(callback_ctx, statistics); }; if (lance_scanner_set_statistics_callback(scanner, collect_scan_statistics, this) != 0) { - return _lance_error("set Lance scanner statistics callback"); + return lance_error("set Lance scanner statistics callback"); } if (_global_rowid_output_idx.has_value() && lance_scanner_with_row_id(scanner, true) != 0) { - return _lance_error("enable Lance row id output"); + return lance_error("enable Lance row id output"); } - if (_scan_params->__isset.lance_scan_params && - lance_scan_params.__isset.lance_substrait_filter && - !lance_scan_params.lance_substrait_filter.empty()) { - const auto& filter = lance_scan_params.lance_substrait_filter; - if (lance_scanner_set_substrait_filter( - scanner, reinterpret_cast(filter.data()), filter.size()) != 0) { - return _lance_error("set Lance Substrait filter"); - } + if (lance_scan_params.__isset.lance_substrait_filter && + lance_scanner_set_substrait_filter( + scanner, + reinterpret_cast(lance_scan_params.lance_substrait_filter.data()), + lance_scan_params.lance_substrait_filter.size()) != 0) { + return lance_error("set Lance Substrait filter"); } - if (_vector_search) { - const auto& request = lance_scan_params.external_search_request; - if (request.__isset.search_filter && - request.search_filter.format == TSearchFilterFormat::SUBSTRAIT) { - const auto& filter = request.search_filter.payload; - if (lance_scanner_set_substrait_filter(scanner, - reinterpret_cast(filter.data()), - filter.size()) != 0) { - return _lance_error("set Lance vector search Substrait filter"); - } + if (runtime_filter_sql != nullptr && !runtime_filter_sql->expression.empty()) { + if (lance_scanner_additional_sql_filter(scanner, runtime_filter_sql->expression.c_str()) != + 0) { + return lance_error("set Lance additional SQL filter"); } + record_lance_runtime_filter_pushdown(_scanner_profile, *runtime_filter_sql); } const auto batch_size = _batch_size > 0 ? _batch_size : _runtime_state->batch_size(); if (lance_scanner_set_batch_size(scanner, static_cast(batch_size)) != 0) { - return _lance_error("set Lance scanner batch size"); + return lance_error("set Lance scanner batch size"); } const auto& lance_params = range.table_format_params.lance_params; - if (lance_params.__isset.fragment_ids && !lance_params.fragment_ids.empty()) { - const auto& thrift_ids = lance_params.fragment_ids; - std::vector fragment_ids; - fragment_ids.reserve(thrift_ids.size()); - for (const auto fragment_id : thrift_ids) { - fragment_ids.emplace_back(static_cast(fragment_id)); - } - if (lance_scanner_set_fragment_ids(scanner, fragment_ids.data(), fragment_ids.size()) != - 0) { - return _lance_error("set Lance scanner fragment ids"); - } - } - if (lance_params.__isset.index_segment_uuids && !lance_params.index_segment_uuids.empty()) { - if (!_vector_search) { - return Status::InvalidArgument( - "Lance index segments are only supported for vector search splits"); - } - constexpr size_t UUID_SIZE = 16; - if (lance_params.index_segment_uuids.size() > - std::numeric_limits::max() / UUID_SIZE) { - return Status::InvalidArgument("too many Lance index segment UUIDs"); - } - std::vector segment_uuids; - segment_uuids.reserve(lance_params.index_segment_uuids.size() * UUID_SIZE); - for (const auto& uuid : lance_params.index_segment_uuids) { - if (uuid.size() != UUID_SIZE) { - return Status::InvalidArgument( - "Lance index segment UUID must contain 16 bytes, got {}", uuid.size()); - } - segment_uuids.insert(segment_uuids.end(), uuid.begin(), uuid.end()); - } - if (lance_scanner_set_index_segments(scanner, segment_uuids.data(), - lance_params.index_segment_uuids.size()) != 0) { - return _lance_error("set Lance scanner index segments"); - } - } - // Ordinary scans may carry a pushed-down LIMIT. The FE only sets it when all predicates are - // pushed into Lance, so the scanner can safely stop after `limit` rows. Vector search manages - // its own top_k limit in _configure_vector_search, so skip it here. - if (!_vector_search && lance_params.__isset.limit && lance_params.limit > 0) { - if (lance_scanner_set_limit(scanner, lance_params.limit) != 0) { - return _lance_error("set Lance scanner limit"); - } - } - if (_vector_search) { - // Distributed vector search always restricts each scanner to an explicit fragment set. - // Tell Lance that this fragment scan is the input to nearest() before installing the - // query. The same prefilter path also applies the TVF search filter, when present. - if (lance_scanner_set_prefilter(scanner, true) != 0) { - return _lance_error("enable Lance vector prefilter"); - } - RETURN_IF_ERROR(_configure_vector_search(scanner)); - const int64_t fragment_count = - lance_params.__isset.fragment_ids - ? static_cast(lance_params.fragment_ids.size()) - : 0; - if (lance_params.__isset.index_segment_uuids && !lance_params.index_segment_uuids.empty()) { - COUNTER_UPDATE(_planned_index_segment_count, - static_cast(lance_params.index_segment_uuids.size())); - COUNTER_UPDATE(_planned_indexed_fragment_count, fragment_count); - } else { - COUNTER_UPDATE(_planned_flat_search_fragment_count, fragment_count); - } + switch (_search_kind) { + case SearchKind::NORMAL: + RETURN_IF_ERROR(_configure_normal_scan(scanner, lance_params)); + break; + case SearchKind::VECTOR: + RETURN_IF_ERROR(_configure_vector_search(scanner, lance_params)); + break; + case SearchKind::FULL_TEXT: + RETURN_IF_ERROR(_configure_full_text_search(scanner, lance_params)); + break; } _scanner = scanner_guard.release(); _scanner_batch_size = batch_size; return Status::OK(); } -Status LanceTableReader::_configure_vector_search(LanceScanner* scanner) const { +Status LanceTableReader::_configure_normal_scan(LanceScanner* scanner, + const TLanceFileDesc& lance_params) const { + DORIS_CHECK(scanner != nullptr); + std::vector fragment_ids; + RETURN_IF_ERROR(parse_fragment_ids(lance_params, &fragment_ids)); + if (!fragment_ids.empty() && + lance_scanner_set_fragment_ids(scanner, fragment_ids.data(), fragment_ids.size()) != 0) { + return lance_error("set Lance scanner fragment ids"); + } + if (lance_params.__isset.index_segment_uuids && !lance_params.index_segment_uuids.empty()) { + return Status::InvalidArgument("normal Lance scan cannot contain index segment UUIDs"); + } + // FE sets this only when every predicate has been pushed into Lance. + if (lance_params.__isset.limit && lance_params.limit > 0 && + lance_scanner_set_limit(scanner, lance_params.limit) != 0) { + return lance_error("set Lance scanner limit"); + } + return Status::OK(); +} + +Status LanceTableReader::_configure_vector_search(LanceScanner* scanner, + const TLanceFileDesc& lance_params) const { DORIS_CHECK(scanner != nullptr); DORIS_CHECK(_scan_params != nullptr); DORIS_CHECK(_scan_params->__isset.lance_scan_params); + std::vector fragment_ids; + RETURN_IF_ERROR(parse_fragment_ids(lance_params, &fragment_ids)); + if (!fragment_ids.empty() && + lance_scanner_set_fragment_ids(scanner, fragment_ids.data(), fragment_ids.size()) != 0) { + return lance_error("set Lance vector scanner fragment ids"); + } + std::vector segment_uuids; + size_t segment_count = 0; + RETURN_IF_ERROR(parse_index_segment_uuids(lance_params, &segment_uuids, &segment_count)); + if (segment_count > 0 && + lance_scanner_set_index_segments(scanner, segment_uuids.data(), segment_count) != 0) { + return lance_error("set Lance vector scanner index segments"); + } + // Fragment-scoped nearest queries require prefiltering before installing the query. The same + // path applies the TVF search filter, when present. + if (lance_scanner_set_prefilter(scanner, true) != 0) { + return lance_error("enable Lance vector prefilter"); + } const auto& lance_scan_params = _scan_params->lance_scan_params; DORIS_CHECK(lance_scan_params.__isset.external_search_request); const auto& request = lance_scan_params.external_search_request; @@ -908,7 +846,7 @@ Status LanceTableReader::_configure_vector_search(LanceScanner* scanner) const { const auto set_nearest = [&](const void* values, LanceDataType type) -> Status { if (lance_scanner_nearest(scanner, vector.column.c_str(), values, dimension, type, candidate_k) != 0) { - return _lance_error("set Lance nearest query"); + return lance_error("set Lance nearest query"); } return Status::OK(); }; @@ -977,7 +915,7 @@ Status LanceTableReader::_configure_vector_search(LanceScanner* scanner) const { static_cast(vector.metric)); } if (lance_scanner_set_metric(scanner, metric) != 0) { - return _lance_error("set Lance vector metric"); + return lance_error("set Lance vector metric"); } } @@ -985,31 +923,71 @@ Status LanceTableReader::_configure_vector_search(LanceScanner* scanner) const { const auto& options = request.vector_search_options; if (options.__isset.nprobes && lance_scanner_set_nprobes(scanner, static_cast(options.nprobes)) != 0) { - return _lance_error("set Lance vector nprobes"); + return lance_error("set Lance vector nprobes"); } if (options.__isset.refine_factor && lance_scanner_set_refine_factor(scanner, static_cast(options.refine_factor)) != 0) { - return _lance_error("set Lance vector refine factor"); + return lance_error("set Lance vector refine factor"); } if (options.__isset.ef && lance_scanner_set_ef(scanner, static_cast(options.ef)) != 0) { - return _lance_error("set Lance vector ef"); + return lance_error("set Lance vector ef"); } if (options.__isset.use_index && lance_scanner_set_use_index(scanner, options.use_index) != 0) { - return _lance_error("set Lance vector use_index"); + return lance_error("set Lance vector use_index"); } } if (lance_scanner_set_offset(scanner, vector.offset) != 0) { - return _lance_error("set Lance vector offset"); + return lance_error("set Lance vector offset"); } if (lance_scanner_set_limit(scanner, vector.top_k) != 0) { - return _lance_error("set Lance vector result limit"); + return lance_error("set Lance vector result limit"); + } + const auto fragment_count = static_cast(fragment_ids.size()); + if (segment_count > 0) { + COUNTER_UPDATE(_planned_index_segment_count, static_cast(segment_count)); + COUNTER_UPDATE(_planned_indexed_fragment_count, fragment_count); + } else { + COUNTER_UPDATE(_planned_flat_search_fragment_count, fragment_count); } return Status::OK(); } +Status LanceTableReader::_configure_full_text_search(LanceScanner* scanner, + const TLanceFileDesc& lance_params) const { + DORIS_CHECK(scanner != nullptr); + DORIS_CHECK(_fts_query_context != nullptr); + DORIS_CHECK(_scan_params != nullptr); + // FTS fragment IDs describe the selected segment's coverage for planning and profiling. They + // are not installed as a generic fragment filter because lance-c rejects combining one with a + // prepared FTS context; the segment UUID is the execution boundary. + std::vector fragment_ids; + RETURN_IF_ERROR(parse_fragment_ids(lance_params, &fragment_ids)); + std::vector segment_uuids; + size_t segment_count = 0; + RETURN_IF_ERROR(parse_index_segment_uuids(lance_params, &segment_uuids, &segment_count)); + if (segment_count == 0) { + return Status::InvalidArgument( + "Lance full-text search split requires at least one FTS index segment UUID"); + } + const auto& full_text = + _scan_params->lance_scan_params.external_search_request.search_query.full_text_search; + if (lance_scanner_set_fts_query_context(scanner, _fts_query_context) != 0) { + return lance_error("attach Lance FTS query context"); + } + if (lance_scanner_set_fts_index_segments(scanner, segment_uuids.data(), segment_count) != 0) { + return lance_error("set Lance FTS scanner index segments"); + } + if (lance_scanner_set_limit(scanner, full_text.top_k + full_text.offset) != 0) { + return lance_error("set Lance FTS scanner candidate limit"); + } + COUNTER_UPDATE(_planned_index_segment_count, static_cast(segment_count)); + COUNTER_UPDATE(_planned_indexed_fragment_count, static_cast(fragment_ids.size())); + return Status::OK(); +} + void LanceTableReader::_collect_scan_statistics(void* callback_ctx, const void* opaque_statistics) { const auto* statistics = static_cast(opaque_statistics); if (callback_ctx == nullptr || statistics == nullptr) { @@ -1095,12 +1073,47 @@ void LanceTableReader::_close_scanner() { } void LanceTableReader::_close_dataset() { + if (_fts_query_context != nullptr) { + lance_fts_query_context_close(_fts_query_context); + _fts_query_context = nullptr; + } if (_dataset != nullptr) { + _collect_data_cache_statistics(); lance_dataset_close(_dataset); _dataset = nullptr; } } +void LanceTableReader::_collect_data_cache_statistics() { + if (_dataset == nullptr) { + return; + } + + LanceDataCacheStatistics statistics {}; + if (lance_dataset_get_data_cache_statistics(_dataset, &statistics) != 0) { + const auto status = lance_error("get Lance data cache statistics"); + LOG(WARNING) << "Failed to collect Lance data cache statistics: " << status.to_string(); + return; + } + + const auto set_counter = [](RuntimeProfile::Counter* counter, uint64_t value, + std::string_view metric_name) { + if (counter == nullptr) { + return; + } + if (value > static_cast(std::numeric_limits::max())) { + LOG(WARNING) << "Ignoring Lance data cache metric '" << metric_name << "' with value " + << value << " because it exceeds INT64_MAX"; + return; + } + COUNTER_SET(counter, static_cast(value)); + }; + set_counter(_data_cache_bytes_read_from_cache, statistics.bytes_read_from_cache, + "bytes_read_from_cache"); + set_counter(_data_cache_bytes_read_from_remote, statistics.bytes_read_from_remote, + "bytes_read_from_remote"); +} + Status LanceTableReader::_fill_block_from_lance_batch(LanceBatch* batch, Block* block, size_t* rows) { DORIS_CHECK(batch != nullptr); @@ -1109,7 +1122,7 @@ Status LanceTableReader::_fill_block_from_lance_batch(LanceBatch* batch, Block* ArrowArray array {}; ArrowSchema schema {}; if (lance_batch_to_arrow(batch, &array, &schema) != 0) { - return _lance_error("export Lance batch to Arrow"); + return lance_error("export Lance batch to Arrow"); } auto result = arrow::ImportRecordBatch(&array, &schema); if (!result.ok()) { @@ -1176,11 +1189,12 @@ Status LanceTableReader::_fill_block_from_record_batch( auto& columns = columns_guard.mutable_columns(); for (int arrow_idx = 0; arrow_idx < record_batch->num_columns(); ++arrow_idx) { const auto& field = record_batch->schema()->field(arrow_idx); - if (field->name() == ROW_ID_COLUMN && _global_rowid_output_idx.has_value()) { + if (field->name() == LANCE_ROW_ID_COLUMN && _global_rowid_output_idx.has_value()) { const auto output_idx = *_global_rowid_output_idx; const auto& output_name = _projected_columns[output_idx].name; if (!materialized_columns.emplace(output_name).second) { - return Status::InternalError("Lance returned duplicate column '{}'", ROW_ID_COLUMN); + return Status::InternalError("Lance returned duplicate column '{}'", + LANCE_ROW_ID_COLUMN); } RETURN_IF_ERROR( _append_global_row_ids(record_batch->column(arrow_idx), columns[output_idx])); @@ -1188,9 +1202,10 @@ Status LanceTableReader::_fill_block_from_record_batch( } const auto output_it = _output_name_to_idx.find(field->name()); if (output_it == _output_name_to_idx.end()) { - if (_vector_search && field->name() == DISTANCE_COLUMN) { - // Lance currently auto-projects _distance for nearest queries. It is valid for - // Doris slot pruning to omit that optional result column. + if ((_search_kind == SearchKind::VECTOR && field->name() == LANCE_DISTANCE_COLUMN) || + (_search_kind == SearchKind::FULL_TEXT && field->name() == LANCE_SCORE_COLUMN)) { + // Lance auto-projects the generated search result column. It is valid for Doris + // slot pruning to omit that optional result column. continue; } return Status::InternalError("Lance returned unknown column '{}'", field->name()); @@ -1219,53 +1234,11 @@ Status LanceTableReader::_fill_block_from_record_batch( return Status::OK(); } -// The FE sends these already in Lance's own vocabulary, merged from the catalog properties and -// from whatever the namespace vended. Re-encoding them here would drop every option this list did -// not anticipate, so they are handed to lance-c as they arrive. -Status LanceTableReader::_storage_options(const TFileScanRangeParams* scan_params, - std::vector* options) { - options->clear(); - if (scan_params == nullptr || !scan_params->__isset.lance_scan_params || - !scan_params->lance_scan_params.__isset.lance_storage_options) { - return Status::OK(); - } - const auto& storage_options = scan_params->lance_scan_params.lance_storage_options; - options->reserve(storage_options.size() * 2); - for (const auto& [key, value] : storage_options) { - // These become C strings below, so a NUL would truncate the option here while the FE went - // on using the whole thing, and the two halves would open the dataset with different - // configuration. The FE rejects these on both paths it builds options from - its own - // storage configuration and what a namespace vends - so this is the last line of defence, - // for an FE that predates those checks. Dropping one here instead of failing would just - // recreate the divergence it exists to prevent. - if (key.find('\0') != std::string::npos || value.find('\0') != std::string::npos) { - return Status::InvalidArgument( - "Lance storage option '{}' contains a NUL and cannot reach lance-c", - key.substr(0, key.find('\0'))); - } - options->emplace_back(key); - options->emplace_back(value); - } - return Status::OK(); -} - Status LanceTableReader::_dataset_key(const TFileRangeDesc& range, DatasetKey* key) const { const auto& params = range.table_format_params.lance_params; key->uri = params.dataset_uri; key->version = params.version; - return _storage_options(_scan_params, &key->storage_options); -} - -Status LanceTableReader::_lance_error(std::string_view operation) { - const char* raw_message = lance_last_error_message(); - std::string message = raw_message == nullptr ? "" : raw_message; - if (raw_message != nullptr) { - lance_free_string(raw_message); - } - if (message.empty()) { - message = fmt::format("error_code={}", static_cast(lance_last_error_code())); - } - return Status::InternalError("{} failed: {}", operation, message); + return build_lance_storage_options(_scan_params, &key->storage_options); } } // namespace doris::format::lance diff --git a/be/src/format_v2/table/lance_reader.h b/be/src/format_v2/table/lance_reader.h index 892aaf518e5d83..cb7a0f4e7cdac4 100644 --- a/be/src/format_v2/table/lance_reader.h +++ b/be/src/format_v2/table/lance_reader.h @@ -34,23 +34,20 @@ struct LanceBatch; struct LanceDataset; +struct LanceFtsQueryContext; struct LanceScanner; +namespace doris { +class ShardedKVCache; +} + namespace arrow { class Array; class RecordBatch; -class Schema; } // namespace arrow namespace doris::format::lance { -// Convert every top-level field without discarding unsupported columns. Malformed schemas still -// return an error and leave both output vectors unchanged. DataTypeNothing is the local sentinel -// for a valid Arrow field whose logical type Doris does not support. -Status convert_arrow_schema_to_doris(const std::shared_ptr& arrow_schema, - std::vector* column_names, - std::vector* column_types); - // A FORMAT_LANCE table reader. Unlike file formats such as Parquet, a Lance split is not a // physical-file range. It either selects fragments from a fixed snapshot or scans the whole // latest snapshot, so the dataset is owned by this table reader and each split owns its scanner. @@ -83,11 +80,20 @@ class LanceTableReader final : public TableReader { bool operator==(const DatasetKey&) const = default; }; + Status _resolve_search_kind(); Status _validate_external_search_request() const; - Status _ensure_dataset_open(const TFileRangeDesc& range); + Status _ensure_dataset_open(const TFileRangeDesc& range, bool prepare_fts_context = true); Status _open_dataset(const DatasetKey& key); + Status _prepare_fts_query_context(); Status _open_scanner(const TFileRangeDesc& range); - Status _configure_vector_search(LanceScanner* scanner) const; + Status _configure_normal_scan(LanceScanner* scanner, const TLanceFileDesc& lance_params) const; + Status _configure_vector_search(LanceScanner* scanner, + const TLanceFileDesc& lance_params) const; + Status _configure_full_text_search(LanceScanner* scanner, + const TLanceFileDesc& lance_params) const; + // Collect the cumulative statistics owned by this dataset handle. The Lance-C API returns an + // absolute snapshot, so this method replaces rather than increments the profile counters. + void _collect_data_cache_statistics(); // Keep lance-c's anonymous statistics typedef out of this header. _open_scanner installs the // strongly typed C callback adapter before forwarding the borrowed value here. static void _collect_scan_statistics(void* callback_ctx, const void* opaque_statistics); @@ -98,13 +104,11 @@ class LanceTableReader final : public TableReader { Block* block, size_t* rows); Status _append_global_row_ids(const std::shared_ptr& row_ids, MutableColumnPtr& output_column) const; - static Status _storage_options(const TFileScanRangeParams* scan_params, - std::vector* options); Status _dataset_key(const TFileRangeDesc& range, DatasetKey* key) const; - static Status _lance_error(std::string_view operation); LanceDataset* _dataset = nullptr; LanceScanner* _scanner = nullptr; + ShardedKVCache* _runtime_filter_cache = nullptr; std::optional _opened_dataset_key; std::unordered_map _output_name_to_idx; std::optional _global_rowid_output_idx; @@ -122,11 +126,15 @@ class LanceTableReader final : public TableReader { RuntimeProfile::Counter* _execution_iops = nullptr; RuntimeProfile::Counter* _execution_requests = nullptr; RuntimeProfile::Counter* _execution_bytes_read = nullptr; + RuntimeProfile::Counter* _data_cache_bytes_read_from_cache = nullptr; + RuntimeProfile::Counter* _data_cache_bytes_read_from_remote = nullptr; RuntimeProfile::Counter* _index_partition_cache_miss_loads = nullptr; RuntimeProfile::Counter* _index_comparisons = nullptr; std::unordered_map _lance_count_metrics; std::unordered_map _lance_time_metrics; - bool _vector_search = false; + LanceFtsQueryContext* _fts_query_context = nullptr; + enum class SearchKind { NORMAL, VECTOR, FULL_TEXT }; + SearchKind _search_kind = SearchKind::NORMAL; bool _eof = false; }; diff --git a/be/test/format_v2/lance/lance_runtime_filter_helper_test.cpp b/be/test/format_v2/lance/lance_runtime_filter_helper_test.cpp new file mode 100644 index 00000000000000..cbe7669817b81d --- /dev/null +++ b/be/test/format_v2/lance/lance_runtime_filter_helper_test.cpp @@ -0,0 +1,201 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/lance/lance_runtime_filter_helper.h" + +#include + +#include +#include +#include +#include +#include + +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/field.h" +#include "exprs/create_predicate_function.h" +#include "exprs/hybrid_set.h" +#include "exprs/runtime_filter_expr.h" +#include "exprs/vbloom_predicate.h" +#include "exprs/vdirect_in_predicate.h" +#include "exprs/vectorized_fn_call.h" +#include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" +#include "exprs/vslot_ref.h" +#include "format/format_common.h" +#include "runtime/runtime_profile.h" + +namespace doris::format::lance { +namespace { + +TExprNode runtime_in_node() { + TExprNode node; + node.__set_type(std::make_shared()->to_thrift()); + node.__set_node_type(TExprNodeType::IN_PRED); + node.in_predicate.__set_is_not_in(false); + node.__set_opcode(TExprOpcode::FILTER_IN); + node.__set_is_nullable(false); + return node; +} + +VExprContextSPtr wrap_runtime_filter(VExprSPtr impl, const TExprNode& node, int filter_id) { + return VExprContext::create_shared( + RuntimeFilterExpr::create_shared(node, std::move(impl), 0.0, false, filter_id)); +} + +VExprContextSPtr int64_runtime_in(std::string column_name, std::vector values, + int filter_id) { + std::shared_ptr filter(create_set(TYPE_BIGINT, false)); + for (const auto value : values) { + filter->insert(&value); + } + auto node = runtime_in_node(); + auto predicate = VDirectInPredicate::create_shared(node, std::move(filter), true); + predicate->add_child(VSlotRef::create_shared( + 0, 0, -1, make_nullable(std::make_shared()), std::move(column_name))); + return wrap_runtime_filter(std::move(predicate), node, filter_id); +} + +VExprContextSPtr string_runtime_in(std::string column_name, const std::string& value, + int filter_id) { + std::shared_ptr filter(create_set(TYPE_STRING, false)); + StringRef value_ref(value.data(), value.size()); + filter->insert(&value_ref); + auto node = runtime_in_node(); + auto predicate = VDirectInPredicate::create_shared(node, std::move(filter), true); + predicate->add_child(VSlotRef::create_shared( + 0, 0, -1, make_nullable(std::make_shared()), std::move(column_name))); + return wrap_runtime_filter(std::move(predicate), node, filter_id); +} + +VExprContextSPtr int64_runtime_range(std::string column_name, TExprOpcode::type opcode, + int64_t value, int filter_id) { + const auto value_type = std::make_shared(); + const auto nullable_value_type = make_nullable(value_type); + const auto result_type = make_nullable(std::make_shared()); + + TFunctionName function_name; + function_name.__set_function_name(opcode == TExprOpcode::GE ? "ge" : "le"); + TFunction function; + function.__set_name(function_name); + function.__set_binary_type(TFunctionBinaryType::BUILTIN); + function.__set_arg_types({nullable_value_type->to_thrift(), value_type->to_thrift()}); + function.__set_ret_type(result_type->to_thrift()); + function.__set_has_var_args(false); + + TExprNode predicate_node; + predicate_node.__set_node_type(TExprNodeType::BINARY_PRED); + predicate_node.__set_opcode(opcode); + predicate_node.__set_type(result_type->to_thrift()); + predicate_node.__set_fn(function); + predicate_node.__set_num_children(2); + predicate_node.__set_is_nullable(true); + auto predicate = VectorizedFnCall::create_shared(predicate_node); + predicate->add_child( + VSlotRef::create_shared(0, 0, -1, nullable_value_type, std::move(column_name))); + predicate->add_child( + VLiteral::create_shared(value_type, Field::create_field(value))); + + TExprNode wrapper_node; + wrapper_node.__set_type(std::make_shared()->to_thrift()); + wrapper_node.__set_is_nullable(false); + return wrap_runtime_filter(std::move(predicate), wrapper_node, filter_id); +} + +VExprContextSPtr unsupported_bloom_runtime_filter(std::string column_name, int filter_id) { + auto node = runtime_in_node(); + node.__set_node_type(TExprNodeType::BLOOM_PRED); + node.__set_opcode(TExprOpcode::RT_FILTER); + auto predicate = VBloomPredicate::create_shared(node); + predicate->add_child(VSlotRef::create_shared( + 0, 0, -1, make_nullable(std::make_shared()), std::move(column_name))); + return wrap_runtime_filter(std::move(predicate), node, filter_id); +} + +TEST(LanceRuntimeFilterHelperTest, ConvertsSupportedFiltersToLanceSql) { + const VExprContextSPtrs conjuncts { + int64_runtime_in("order`key", {7}, 3), + string_runtime_in("author", "O'Reilly", 5), + int64_runtime_range("score", TExprOpcode::GE, 10, 7), + int64_runtime_range("score", TExprOpcode::LE, 20, 7), + }; + + const auto result = get_or_create_lance_runtime_filter_sql(conjuncts, nullptr); + ASSERT_NE(result, nullptr); + EXPECT_EQ( + "(`order``key` IN (7)) AND (`author` IN ('O''Reilly')) AND " + "(`score` >= 10) AND (`score` <= 20)", + result->expression); + EXPECT_EQ((std::vector {3, 5, 7}), result->pushable_filter_ids); + EXPECT_TRUE(result->skipped_filter_ids.empty()); +} + +TEST(LanceRuntimeFilterHelperTest, RecordsUnsupportedRuntimeFilters) { + const VExprContextSPtrs conjuncts { + int64_runtime_in("id", {2}, 3), + unsupported_bloom_runtime_filter("id", 8), + }; + + const auto result = get_or_create_lance_runtime_filter_sql(conjuncts, nullptr); + ASSERT_NE(result, nullptr); + EXPECT_EQ("(`id` IN (2))", result->expression); + EXPECT_EQ((std::vector {3}), result->pushable_filter_ids); + EXPECT_EQ((std::vector {8}), result->skipped_filter_ids); + + RuntimeProfile profile("lance_runtime_filter_profile"); + record_lance_runtime_filter_pushdown(&profile, *result); + ASSERT_NE(profile.get_info_string("LanceRuntimeFilterPushedIds"), nullptr); + EXPECT_EQ("3", *profile.get_info_string("LanceRuntimeFilterPushedIds")); + ASSERT_NE(profile.get_info_string("LanceRuntimeFilterSkippedIds"), nullptr); + EXPECT_EQ("8", *profile.get_info_string("LanceRuntimeFilterSkippedIds")); +} + +TEST(LanceRuntimeFilterHelperTest, IgnoresNonRuntimeFilterConjuncts) { + const VExprContextSPtrs conjuncts {VExprContext::create_shared(VSlotRef::create_shared( + 0, 0, -1, std::make_shared(), "ordinary_column"))}; + + EXPECT_EQ(nullptr, get_or_create_lance_runtime_filter_sql(conjuncts, nullptr)); +} + +TEST(LanceRuntimeFilterHelperTest, ReusesSnapshotAcrossParallelReaders) { + ShardedKVCache cache(2); + const VExprContextSPtrs first_conjuncts { + int64_runtime_in("id", {2}, 12), + int64_runtime_range("score", TExprOpcode::GE, 10, 13), + }; + const VExprContextSPtrs reordered_conjuncts { + int64_runtime_range("score", TExprOpcode::GE, 10, 13), + int64_runtime_in("id", {2}, 12), + }; + + const auto first = get_or_create_lance_runtime_filter_sql(first_conjuncts, &cache); + const auto reused = get_or_create_lance_runtime_filter_sql(reordered_conjuncts, &cache); + ASSERT_NE(first, nullptr); + ASSERT_NE(reused, nullptr); + EXPECT_EQ(first.get(), reused.get()); + EXPECT_EQ("(`id` IN (2)) AND (`score` >= 10)", reused->expression); + + const auto different = + get_or_create_lance_runtime_filter_sql({int64_runtime_in("id", {2}, 14)}, &cache); + ASSERT_NE(different, nullptr); + EXPECT_NE(first.get(), different.get()); +} + +} // namespace +} // namespace doris::format::lance diff --git a/be/test/format_v2/lance/lance_session_manager_test.cpp b/be/test/format_v2/lance/lance_session_manager_test.cpp new file mode 100644 index 00000000000000..3421c5e9ad02cf --- /dev/null +++ b/be/test/format_v2/lance/lance_session_manager_test.cpp @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/lance/lance_session_manager.h" + +#include +#include + +#include +#include +#include +#include +#include + +namespace doris::format::lance { +namespace { + +using LanceDatasetPtr = std::unique_ptr; + +std::filesystem::path lance_fixture_path() { + return std::filesystem::path(__FILE__).parent_path().parent_path() / + "table/lance/data/all_types.lance"; +} + +std::filesystem::path unique_cache_path() { + const auto suffix = std::chrono::steady_clock::now().time_since_epoch().count(); + return std::filesystem::temp_directory_path() / + ("doris_lance_foyer_cache_" + std::to_string(suffix)); +} + +TEST(LanceSessionManagerTest, SessionAndDataCacheConfigurationsAreIndependent) { + LanceSessionManager::Config config { + .lance_index_cache_size_bytes = 0, + .lance_metadata_cache_size_bytes = 0, + .enable_lance_data_cache = false, + // These are deliberately invalid and must be ignored while the data cache is off. + .lance_data_cache_path = "", + .lance_data_cache_disk_capacity_bytes = -1, + .lance_data_cache_read_block_size_bytes = -1, + }; + LanceSessionManager manager(std::move(config)); + LanceDataset* raw_dataset = nullptr; + ASSERT_TRUE(manager + .open_dataset(lance_fixture_path().c_str(), nullptr, 0, &raw_dataset) + .ok()); + LanceDatasetPtr dataset(raw_dataset, lance_dataset_close); + ASSERT_NE(dataset, nullptr); +} + +TEST(LanceSessionManagerTest, CreatesFoyerBackedSession) { + const auto cache_path = unique_cache_path(); + std::filesystem::create_directories(cache_path); + const auto cleanup = [&cache_path] { + std::error_code error; + std::filesystem::remove_all(cache_path, error); + }; + + LanceSessionManager::Config config { + .lance_index_cache_size_bytes = 0, + .lance_metadata_cache_size_bytes = 0, + .enable_lance_data_cache = true, + .lance_data_cache_path = cache_path.string(), + .lance_data_cache_disk_capacity_bytes = 32 * 1024 * 1024, + .lance_data_cache_read_block_size_bytes = 64 * 1024, + }; + { + LanceSessionManager manager(std::move(config)); + LanceDataset* raw_dataset = nullptr; + const auto status = manager.open_dataset(lance_fixture_path().c_str(), nullptr, 0, + &raw_dataset); + LanceDatasetPtr dataset(raw_dataset, lance_dataset_close); + EXPECT_TRUE(status.ok()) << status; + EXPECT_NE(dataset, nullptr); + } + cleanup(); +} + +} // namespace +} // namespace doris::format::lance diff --git a/be/test/format_v2/table/lance/data/fts_indexed.lance/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/metadata.lance b/be/test/format_v2/table/lance/data/fts_indexed.lance/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/metadata.lance new file mode 100644 index 00000000000000..340a57964dc091 Binary files /dev/null and b/be/test/format_v2/table/lance/data/fts_indexed.lance/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/metadata.lance differ diff --git a/be/test/format_v2/table/lance/data/fts_indexed.lance/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/part_0_docs.lance b/be/test/format_v2/table/lance/data/fts_indexed.lance/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/part_0_docs.lance new file mode 100644 index 00000000000000..9b336efbb78855 Binary files /dev/null and b/be/test/format_v2/table/lance/data/fts_indexed.lance/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/part_0_docs.lance differ diff --git a/be/test/format_v2/table/lance/data/fts_indexed.lance/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/part_0_invert.lance b/be/test/format_v2/table/lance/data/fts_indexed.lance/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/part_0_invert.lance new file mode 100644 index 00000000000000..056f31f2d9346f Binary files /dev/null and b/be/test/format_v2/table/lance/data/fts_indexed.lance/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/part_0_invert.lance differ diff --git a/be/test/format_v2/table/lance/data/fts_indexed.lance/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/part_0_tokens.lance b/be/test/format_v2/table/lance/data/fts_indexed.lance/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/part_0_tokens.lance new file mode 100644 index 00000000000000..f265ddd2b8838c Binary files /dev/null and b/be/test/format_v2/table/lance/data/fts_indexed.lance/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/part_0_tokens.lance differ diff --git a/be/test/format_v2/table/lance/data/fts_indexed.lance/_transactions/0-4a942168-96d8-4f07-bce0-e3049c8ea6c5.txn b/be/test/format_v2/table/lance/data/fts_indexed.lance/_transactions/0-4a942168-96d8-4f07-bce0-e3049c8ea6c5.txn new file mode 100644 index 00000000000000..65272d53b4f7b4 Binary files /dev/null and b/be/test/format_v2/table/lance/data/fts_indexed.lance/_transactions/0-4a942168-96d8-4f07-bce0-e3049c8ea6c5.txn differ diff --git a/be/test/format_v2/table/lance/data/fts_indexed.lance/_transactions/1-b311fe36-9353-4f41-ae19-5054d4a73009.txn b/be/test/format_v2/table/lance/data/fts_indexed.lance/_transactions/1-b311fe36-9353-4f41-ae19-5054d4a73009.txn new file mode 100644 index 00000000000000..be61bddcadfc85 Binary files /dev/null and b/be/test/format_v2/table/lance/data/fts_indexed.lance/_transactions/1-b311fe36-9353-4f41-ae19-5054d4a73009.txn differ diff --git a/be/test/format_v2/table/lance/data/fts_indexed.lance/_transactions/2-76e69d8c-04b7-402e-8620-7f9a772a2224.txn b/be/test/format_v2/table/lance/data/fts_indexed.lance/_transactions/2-76e69d8c-04b7-402e-8620-7f9a772a2224.txn new file mode 100644 index 00000000000000..e9216b8131e17a Binary files /dev/null and b/be/test/format_v2/table/lance/data/fts_indexed.lance/_transactions/2-76e69d8c-04b7-402e-8620-7f9a772a2224.txn differ diff --git a/be/test/format_v2/table/lance/data/fts_indexed.lance/_versions/18446744073709551612.manifest b/be/test/format_v2/table/lance/data/fts_indexed.lance/_versions/18446744073709551612.manifest new file mode 100644 index 00000000000000..83edf4ee407b29 Binary files /dev/null and b/be/test/format_v2/table/lance/data/fts_indexed.lance/_versions/18446744073709551612.manifest differ diff --git a/be/test/format_v2/table/lance/data/fts_indexed.lance/_versions/18446744073709551613.manifest b/be/test/format_v2/table/lance/data/fts_indexed.lance/_versions/18446744073709551613.manifest new file mode 100644 index 00000000000000..19ab9f8f79ef41 Binary files /dev/null and b/be/test/format_v2/table/lance/data/fts_indexed.lance/_versions/18446744073709551613.manifest differ diff --git a/be/test/format_v2/table/lance/data/fts_indexed.lance/_versions/18446744073709551614.manifest b/be/test/format_v2/table/lance/data/fts_indexed.lance/_versions/18446744073709551614.manifest new file mode 100644 index 00000000000000..f89780183298b8 Binary files /dev/null and b/be/test/format_v2/table/lance/data/fts_indexed.lance/_versions/18446744073709551614.manifest differ diff --git a/be/test/format_v2/table/lance/data/fts_indexed.lance/_versions/latest_version_hint.json b/be/test/format_v2/table/lance/data/fts_indexed.lance/_versions/latest_version_hint.json new file mode 100644 index 00000000000000..f93d3984472d99 --- /dev/null +++ b/be/test/format_v2/table/lance/data/fts_indexed.lance/_versions/latest_version_hint.json @@ -0,0 +1 @@ +{"version":3} \ No newline at end of file diff --git a/be/test/format_v2/table/lance/data/fts_indexed.lance/data/000100011001100000000001ee52754ad1a1da89dd0efa4379.lance b/be/test/format_v2/table/lance/data/fts_indexed.lance/data/000100011001100000000001ee52754ad1a1da89dd0efa4379.lance new file mode 100644 index 00000000000000..5079d336a0c6b0 Binary files /dev/null and b/be/test/format_v2/table/lance/data/fts_indexed.lance/data/000100011001100000000001ee52754ad1a1da89dd0efa4379.lance differ diff --git a/be/test/format_v2/table/lance/data/fts_indexed.lance/data/100101010010010100010000fd779442f589c03379a2858ea8.lance b/be/test/format_v2/table/lance/data/fts_indexed.lance/data/100101010010010100010000fd779442f589c03379a2858ea8.lance new file mode 100644 index 00000000000000..3714a20bae4cfa Binary files /dev/null and b/be/test/format_v2/table/lance/data/fts_indexed.lance/data/100101010010010100010000fd779442f589c03379a2858ea8.lance differ diff --git a/be/test/format_v2/table/lance_reader_test.cpp b/be/test/format_v2/table/lance_reader_test.cpp index 9a2e802217eeb4..0eceb92128da0b 100644 --- a/be/test/format_v2/table/lance_reader_test.cpp +++ b/be/test/format_v2/table/lance_reader_test.cpp @@ -57,7 +57,14 @@ #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_varbinary.h" #include "exec/common/endian.h" +#include "exprs/create_predicate_function.h" +#include "exprs/hybrid_set.h" +#include "exprs/runtime_filter_expr.h" +#include "exprs/vdirect_in_predicate.h" #include "exprs/vexpr.h" +#include "exprs/vexpr_context.h" +#include "exprs/vslot_ref.h" +#include "format_v2/lance/lance_reader_helper.h" #include "runtime/runtime_profile.h" #include "runtime/runtime_state.h" #include "storage/utils.h" @@ -166,6 +173,26 @@ ColumnDefinition projected_column(std::string name, PrimitiveType type, bool nul DataTypeFactory::instance().create_data_type(type, nullable)); } +VExprContextSPtr create_int64_runtime_in_conjunct(std::string column_name, + const std::vector& values, + int filter_id) { + std::shared_ptr filter(create_set(TYPE_BIGINT, false)); + for (const auto value : values) { + filter->insert(&value); + } + TExprNode node; + node.__set_type(std::make_shared()->to_thrift()); + node.__set_node_type(TExprNodeType::IN_PRED); + node.in_predicate.__set_is_not_in(false); + node.__set_opcode(TExprOpcode::FILTER_IN); + node.__set_is_nullable(false); + auto predicate = VDirectInPredicate::create_shared(node, std::move(filter), true); + predicate->add_child(VSlotRef::create_shared( + 0, 0, -1, make_nullable(std::make_shared()), std::move(column_name))); + return VExprContext::create_shared( + RuntimeFilterExpr::create_shared(node, std::move(predicate), 0.0, false, filter_id)); +} + void add_output_columns(Block* block, const Columns& columns) { for (const auto& column : columns) { block->insert({column.type->create_column(), column.type, column.name}); @@ -249,6 +276,94 @@ TFileScanRangeParams make_float32_vector_search_params( return scan_params; } +TFileScanRangeParams make_full_text_search_params( + std::string query_text, int64_t top_k, int64_t offset, + TFtsCoverageMode::type coverage_mode = TFtsCoverageMode::STRICT, + std::optional filter = std::nullopt, + TFtsMatchOperator::type match_operator = TFtsMatchOperator::OR, + int32_t max_fuzzy_distance = 0) { + TFullTextSearchParams full_text_params; + full_text_params.__set_column("body"); + full_text_params.__set_query(std::move(query_text)); + full_text_params.__set_top_k(top_k); + full_text_params.__set_offset(offset); + full_text_params.__set_coverage_mode(coverage_mode); + full_text_params.__set_query_type(TFtsQueryType::MATCH); + full_text_params.__set_match_operator(match_operator); + full_text_params.__set_max_fuzzy_distance(max_fuzzy_distance); + + TExternalSearchQuery query; + query.__set_full_text_search(std::move(full_text_params)); + TExternalSearchRequest request; + request.__set_schema_version(1); + request.__set_search_query(std::move(query)); + if (filter.has_value()) { + TSearchFilter search_filter; + search_filter.__set_format(TSearchFilterFormat::SQL); + search_filter.__set_payload(*filter); + request.__set_search_filter(std::move(search_filter)); + } + + TLanceScanParams lance_scan_params; + lance_scan_params.__set_external_search_request(std::move(request)); + TFileScanRangeParams scan_params; + scan_params.__set_lance_scan_params(std::move(lance_scan_params)); + return scan_params; +} + +TFileScanRangeParams make_phrase_search_params( + std::string query_text, int64_t top_k, int64_t offset, int32_t slop, + TFtsCoverageMode::type coverage_mode = TFtsCoverageMode::STRICT, + std::optional filter = std::nullopt) { + auto scan_params = make_full_text_search_params(std::move(query_text), top_k, offset, + coverage_mode, std::move(filter)); + auto& full_text = scan_params.lance_scan_params.external_search_request.search_query + .full_text_search; + full_text.__set_query_type(TFtsQueryType::PHRASE); + full_text.__isset.match_operator = false; + full_text.__isset.max_fuzzy_distance = false; + full_text.__set_phrase_slop(slop); + return scan_params; +} + +Status get_index_segment_uuids(const std::filesystem::path& dataset_uri, const char* index_name, + std::vector* encoded_uuids) { + std::unique_ptr dataset( + lance_dataset_open(dataset_uri.c_str(), nullptr, 0), lance_dataset_close); + if (dataset == nullptr) { + return Status::InternalError("Failed to open Lance fixture: {}", dataset_uri.string()); + } + const auto segment_count = lance_dataset_index_segment_count(dataset.get(), index_name); + if (segment_count == 0) { + return Status::InternalError("Lance fixture index '{}' has no segments", index_name); + } + std::vector raw_uuids(segment_count * 16); + uint64_t actual_count = 0; + if (lance_dataset_index_segments(dataset.get(), index_name, raw_uuids.data(), segment_count, + &actual_count) != 0 || + actual_count != segment_count) { + return Status::InternalError("Failed to enumerate Lance fixture index '{}' segments", + index_name); + } + encoded_uuids->clear(); + encoded_uuids->reserve(segment_count); + for (size_t index = 0; index < segment_count; ++index) { + encoded_uuids->emplace_back(reinterpret_cast(raw_uuids.data() + index * 16), + 16); + } + return Status::OK(); +} + +TFileRangeDesc make_full_text_search_range(const std::filesystem::path& dataset_uri, + const LanceFixtureInfo& fixture, + const char* index_name) { + auto range = make_lance_range(dataset_uri, fixture.version, fixture.fragment_ids); + std::vector segment_uuids; + EXPECT_TRUE(get_index_segment_uuids(dataset_uri, index_name, &segment_uuids).ok()); + range.table_format_params.lance_params.__set_index_segment_uuids(std::move(segment_uuids)); + return range; +} + TEST(LanceTableReaderVectorSearchTest, RejectsMalformedVectorPayloadBeforeReadingIt) { const Columns columns { projected_column("row_id", TYPE_BIGINT, false), @@ -268,6 +383,259 @@ TEST(LanceTableReaderVectorSearchTest, RejectsMalformedVectorPayloadBeforeReadin EXPECT_NE(status.to_string().find("query vector byte size"), std::string::npos); } +TEST(LanceTableReaderFullTextSearchTest, ValidatesRequestAndScoreTypeBeforeDatasetAccess) { + const Columns columns { + projected_column("row_id", TYPE_BIGINT, false), + projected_column("_score", TYPE_FLOAT, true), + }; + TQueryGlobals query_globals; + RuntimeState state(query_globals); + RuntimeProfile valid_profile("lance_fts_valid_request"); + auto valid_params = make_full_text_search_params("lance", 4, 1); + LanceTableReader valid_reader; + ASSERT_TRUE(init_reader(&valid_reader, columns, &state, &valid_profile, &valid_params).ok()); + ASSERT_NE(valid_profile.get_info_string("LanceSearchType"), nullptr); + EXPECT_EQ("FULL_TEXT", *valid_profile.get_info_string("LanceSearchType")); + ASSERT_NE(valid_profile.get_info_string("LanceFtsCoverageMode"), nullptr); + EXPECT_EQ("STRICT", *valid_profile.get_info_string("LanceFtsCoverageMode")); + ASSERT_NE(valid_profile.get_info_string("LanceFtsQueryType"), nullptr); + EXPECT_EQ("MATCH", *valid_profile.get_info_string("LanceFtsQueryType")); + ASSERT_NE(valid_profile.get_info_string("LanceFtsMatchOperator"), nullptr); + EXPECT_EQ("OR", *valid_profile.get_info_string("LanceFtsMatchOperator")); + ASSERT_NE(valid_profile.get_info_string("LanceFtsMaxFuzzyDistance"), nullptr); + EXPECT_EQ("0", *valid_profile.get_info_string("LanceFtsMaxFuzzyDistance")); + ASSERT_NE(valid_profile.get_info_string("LanceTopKPlusOffset"), nullptr); + EXPECT_EQ("5", *valid_profile.get_info_string("LanceTopKPlusOffset")); + + RuntimeProfile empty_query_profile("lance_fts_empty_query"); + auto empty_query_params = make_full_text_search_params("", 4, 0); + LanceTableReader empty_query_reader; + const auto empty_query_status = init_reader(&empty_query_reader, columns, &state, + &empty_query_profile, &empty_query_params); + EXPECT_FALSE(empty_query_status.ok()); + EXPECT_NE(empty_query_status.to_string().find("non-empty query"), std::string::npos); + + RuntimeProfile coverage_profile("lance_fts_invalid_coverage"); + auto invalid_coverage_params = + make_full_text_search_params("lance", 4, 0, static_cast(99)); + LanceTableReader coverage_reader; + const auto coverage_status = init_reader(&coverage_reader, columns, &state, &coverage_profile, + &invalid_coverage_params); + EXPECT_FALSE(coverage_status.ok()); + EXPECT_NE(coverage_status.to_string().find("STRICT or INDEX_ONLY"), std::string::npos); + + const Columns wrong_score_columns { + projected_column("row_id", TYPE_BIGINT, false), + projected_column("_score", TYPE_DOUBLE, true), + }; + RuntimeProfile score_profile("lance_fts_invalid_score_type"); + auto score_params = make_full_text_search_params("lance", 4, 0); + LanceTableReader score_reader; + const auto score_status = + init_reader(&score_reader, wrong_score_columns, &state, &score_profile, &score_params); + EXPECT_FALSE(score_status.ok()); + EXPECT_NE(score_status.to_string().find("must have Doris FLOAT type"), std::string::npos); +} + +TEST(LanceTableReaderFullTextSearchTest, ValidatesQuerySpecificParameters) { + const Columns columns { + projected_column("row_id", TYPE_BIGINT, false), + projected_column("_score", TYPE_FLOAT, true), + }; + TQueryGlobals query_globals; + RuntimeState state(query_globals); + + RuntimeProfile phrase_profile("lance_fts_phrase_request"); + auto phrase_params = make_phrase_search_params("lance search", 4, 0, 1); + LanceTableReader phrase_reader; + ASSERT_TRUE(init_reader(&phrase_reader, columns, &state, &phrase_profile, &phrase_params).ok()); + ASSERT_NE(phrase_profile.get_info_string("LanceFtsQueryType"), nullptr); + EXPECT_EQ("PHRASE", *phrase_profile.get_info_string("LanceFtsQueryType")); + ASSERT_NE(phrase_profile.get_info_string("LanceFtsPhraseSlop"), nullptr); + EXPECT_EQ("1", *phrase_profile.get_info_string("LanceFtsPhraseSlop")); + + RuntimeProfile fuzzy_profile("lance_fts_fuzzy_request"); + auto fuzzy_params = make_full_text_search_params( + "lance", 4, 0, TFtsCoverageMode::STRICT, std::nullopt, + TFtsMatchOperator::OR, 1); + LanceTableReader fuzzy_reader; + const auto fuzzy_status = + init_reader(&fuzzy_reader, columns, &state, &fuzzy_profile, &fuzzy_params); + EXPECT_FALSE(fuzzy_status.ok()); + EXPECT_NE(fuzzy_status.to_string().find("does not yet support max_fuzzy_distance=1"), + std::string::npos); + + RuntimeProfile match_slop_profile("lance_fts_match_with_slop"); + auto match_slop_params = make_full_text_search_params("lance", 4, 0); + match_slop_params.lance_scan_params.external_search_request.search_query.full_text_search + .__set_phrase_slop(1); + LanceTableReader match_slop_reader; + const auto match_slop_status = init_reader(&match_slop_reader, columns, &state, + &match_slop_profile, &match_slop_params); + EXPECT_FALSE(match_slop_status.ok()); + EXPECT_NE(match_slop_status.to_string().find("MATCH query cannot set phrase_slop"), + std::string::npos); + + RuntimeProfile negative_slop_profile("lance_fts_negative_phrase_slop"); + auto negative_slop_params = make_phrase_search_params("lance search", 4, 0, -1); + LanceTableReader negative_slop_reader; + const auto negative_slop_status = init_reader(&negative_slop_reader, columns, &state, + &negative_slop_profile, &negative_slop_params); + EXPECT_FALSE(negative_slop_status.ok()); + EXPECT_NE(negative_slop_status.to_string().find("phrase_slop must be non-negative"), + std::string::npos); +} + +std::vector> read_full_text_search_rows(LanceTableReader* reader, + Block* block) { + std::vector> rows; + bool eos = false; + while (!eos) { + EXPECT_TRUE(reader->get_block(block, &eos).ok()); + if (eos) { + continue; + } + const auto& row_ids = assert_cast(*block->get_by_position(0).column); + const auto& scores = assert_cast(*block->get_by_position(1).column); + const auto& score_values = assert_cast(scores.get_nested_column()); + for (size_t row = 0; row < block->rows(); ++row) { + EXPECT_EQ(0, scores.get_null_map_data()[row]); + rows.emplace_back(row_ids.get_data()[row], score_values.get_data()[row]); + } + } + return rows; +} + +TEST(LanceTableReaderFullTextSearchTest, SearchesIndexedSnapshotWithOptionalScoreProjection) { + const std::filesystem::path dataset_uri = + "./be/test/format_v2/table/lance/data/fts_indexed.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(dataset_uri, &fixture).ok()); + auto range = make_full_text_search_range(dataset_uri, fixture, "body_fts"); + + const Columns scored_columns { + projected_column("row_id", TYPE_BIGINT, false), + projected_column("_score", TYPE_FLOAT, true), + }; + TQueryGlobals query_globals; + RuntimeState state(query_globals); + RuntimeProfile scored_profile("lance_fts_indexed_fixture"); + auto scored_params = make_full_text_search_params("lance", 4, 0); + LanceTableReader scored_reader; + ASSERT_TRUE(init_reader(&scored_reader, scored_columns, &state, &scored_profile, &scored_params) + .ok()); + ASSERT_TRUE(prepare_range(&scored_reader, range).ok()); + Block scored_block; + add_output_columns(&scored_block, scored_columns); + const auto scored_rows = read_full_text_search_rows(&scored_reader, &scored_block); + ASSERT_EQ(4U, scored_rows.size()); + EXPECT_EQ((std::vector {3, 2, 1, 7}), + (std::vector {scored_rows[0].first, scored_rows[1].first, + scored_rows[2].first, scored_rows[3].first})); + for (size_t index = 0; index < scored_rows.size(); ++index) { + EXPECT_GT(scored_rows[index].second, 0.0F); + if (index > 0) { + EXPECT_GT(scored_rows[index - 1].second, scored_rows[index].second); + } + } + EXPECT_TRUE(scored_reader.close().ok()); + + // Lance returns _score for ordering even when Doris does not materialize it. The reader must + // ignore that generated column while still rejecting any unexpected _rowid output. + const Columns row_id_columns {projected_column("row_id", TYPE_BIGINT, false)}; + RuntimeProfile row_id_profile("lance_fts_without_score_projection_fixture"); + auto row_id_params = make_full_text_search_params("lance", 2, 0); + LanceTableReader row_id_reader; + ASSERT_TRUE(init_reader(&row_id_reader, row_id_columns, &state, &row_id_profile, &row_id_params) + .ok()); + ASSERT_TRUE(prepare_range(&row_id_reader, range).ok()); + Block row_id_block; + add_output_columns(&row_id_block, row_id_columns); + std::vector row_ids; + bool eos = false; + while (!eos) { + ASSERT_TRUE(row_id_reader.get_block(&row_id_block, &eos).ok()); + if (!eos) { + const auto& values = + assert_cast(*row_id_block.get_by_position(0).column); + row_ids.insert(row_ids.end(), values.get_data().begin(), values.get_data().end()); + } + } + EXPECT_EQ((std::vector {3, 2}), row_ids); + EXPECT_TRUE(row_id_reader.close().ok()); +} + +TEST(LanceTableReaderFullTextSearchTest, SupportsMatchAndPhraseQueries) { + const std::filesystem::path dataset_uri = + "./be/test/format_v2/table/lance/data/fts_indexed.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(dataset_uri, &fixture).ok()); + auto range = make_full_text_search_range(dataset_uri, fixture, "body_fts"); + + const Columns columns { + projected_column("row_id", TYPE_BIGINT, false), + projected_column("_score", TYPE_FLOAT, true), + }; + TQueryGlobals query_globals; + RuntimeState state(query_globals); + + RuntimeProfile match_and_profile("lance_fts_match_and"); + auto match_and_params = make_full_text_search_params( + "lance storage", 10, 0, TFtsCoverageMode::STRICT, std::nullopt, + TFtsMatchOperator::AND); + LanceTableReader match_and_reader; + ASSERT_TRUE(init_reader(&match_and_reader, columns, &state, &match_and_profile, + &match_and_params) + .ok()); + ASSERT_TRUE(prepare_range(&match_and_reader, range).ok()); + Block match_and_block; + add_output_columns(&match_and_block, columns); + const auto match_and_rows = + read_full_text_search_rows(&match_and_reader, &match_and_block); + ASSERT_EQ(1U, match_and_rows.size()); + EXPECT_EQ(7, match_and_rows[0].first); + EXPECT_TRUE(match_and_reader.close().ok()); + + RuntimeProfile phrase_profile("lance_fts_phrase_exact"); + auto phrase_params = make_phrase_search_params("lance search", 10, 0, 0); + LanceTableReader phrase_reader; + ASSERT_TRUE(init_reader(&phrase_reader, columns, &state, &phrase_profile, &phrase_params).ok()); + ASSERT_TRUE(prepare_range(&phrase_reader, range).ok()); + Block phrase_block; + add_output_columns(&phrase_block, columns); + const auto phrase_rows = read_full_text_search_rows(&phrase_reader, &phrase_block); + std::vector phrase_row_ids; + phrase_row_ids.reserve(phrase_rows.size()); + for (const auto& [row_id, score] : phrase_rows) { + EXPECT_GT(score, 0.0F); + phrase_row_ids.emplace_back(row_id); + } + std::sort(phrase_row_ids.begin(), phrase_row_ids.end()); + EXPECT_EQ((std::vector {1, 2, 3}), phrase_row_ids); + EXPECT_TRUE(phrase_reader.close().ok()); + + RuntimeProfile phrase_slop_profile("lance_fts_phrase_slop"); + auto phrase_slop_params = make_phrase_search_params("lance engine", 10, 0, 1); + LanceTableReader phrase_slop_reader; + ASSERT_TRUE(init_reader(&phrase_slop_reader, columns, &state, &phrase_slop_profile, + &phrase_slop_params) + .ok()); + ASSERT_TRUE(prepare_range(&phrase_slop_reader, range).ok()); + Block phrase_slop_block; + add_output_columns(&phrase_slop_block, columns); + const auto phrase_slop_rows = + read_full_text_search_rows(&phrase_slop_reader, &phrase_slop_block); + std::vector phrase_slop_row_ids; + phrase_slop_row_ids.reserve(phrase_slop_rows.size()); + for (const auto& [row_id, score] : phrase_slop_rows) { + EXPECT_GT(score, 0.0F); + phrase_slop_row_ids.emplace_back(row_id); + } + std::sort(phrase_slop_row_ids.begin(), phrase_slop_row_ids.end()); + EXPECT_EQ((std::vector {1, 2, 3}), phrase_slop_row_ids); + EXPECT_TRUE(phrase_slop_reader.close().ok()); +} + TEST(LanceTableReaderVectorSearchTest, RejectsMalformedIndexSegmentUuid) { const std::filesystem::path dataset_uri = "./be/test/format_v2/table/lance/data/all_types.lance"; @@ -465,6 +833,8 @@ TEST(LanceTableReaderVectorSearchTest, SearchesMultipleFragmentSplits) { "LanceExecutionIOOps", "LanceExecutionIORequests", "LanceExecutionIOBytesRead", + "LanceDataCacheBytesReadFromCache", + "LanceDataCacheBytesReadFromRemote", "LanceIndexPartitionCacheMissLoads", "LanceIndexComparisons", "LanceFragmentsScanned", @@ -587,7 +957,9 @@ TEST(LanceTableReaderVectorSearchTest, ReturnsStableGlobalRowIdsAndFetchesPayloa EXPECT_NE(fetch_profile.get_counter("LanceRowIdFetchTotalTime"), nullptr); expect_lance_profile_hierarchy(&fetch_profile, {"LanceDatasetOpenTime", "LanceRowIdTakeReadTime", - "LanceArrowToDorisBlockTime", "LanceRowIdFetchTotalTime"}); + "LanceArrowToDorisBlockTime", "LanceRowIdFetchTotalTime", + "LanceDataCacheBytesReadFromCache", + "LanceDataCacheBytesReadFromRemote"}); EXPECT_TRUE(payload_reader.close().ok()); } @@ -635,7 +1007,7 @@ TEST(LanceTableReaderVectorSearchTest, ReadsOnlyGlobalRowIdVirtualColumn) { EXPECT_TRUE(reader.close().ok()); } -TEST(LanceTableReaderFilterTest, PushesFilterOnNonProjectedColumn) { +TEST(LanceTableReaderFilterTest, CombinesStaticSubstraitFilterWithRuntimeFilter) { const std::filesystem::path dataset_uri = "./be/test/format_v2/table/lance/data/all_types.lance"; LanceFixtureInfo fixture; @@ -712,6 +1084,85 @@ TEST(LanceTableReaderFilterTest, PushesFilterOnNonProjectedColumn) { std::ranges::sort(labels); EXPECT_EQ((std::vector {"extra", "mixed"}), labels); EXPECT_TRUE(reader.close().ok()); + + // The static Substrait prefilter and a later runtime filter must both remain active. Static + // row_id >= 3 yields {3, 4}; runtime row_id IN (2, 4) yields {2, 4}; their intersection is {4}. + std::string combined_substrait_filter; + ASSERT_TRUE(base64_decode(substrait_filter_base64, &combined_substrait_filter)); + TLanceScanParams combined_lance_scan_params; + combined_lance_scan_params.__set_lance_substrait_filter(std::move(combined_substrait_filter)); + TFileScanRangeParams combined_scan_params; + combined_scan_params.__set_lance_scan_params(std::move(combined_lance_scan_params)); + const Columns row_id_columns {projected_column("row_id", TYPE_BIGINT, false)}; + RuntimeProfile combined_profile("lance_substrait_and_runtime_filter_fixture"); + const auto combined_runtime_filter = create_int64_runtime_in_conjunct("row_id", {2, 4}, 42); + + LanceTableReader combined_reader; + ASSERT_TRUE(init_reader(&combined_reader, row_id_columns, &state, &combined_profile, + &combined_scan_params, {combined_runtime_filter}) + .ok()); + ASSERT_TRUE(prepare_fixture(&combined_reader, dataset_uri, fixture, fixture.fragment_ids).ok()); + + Block combined_block; + add_output_columns(&combined_block, row_id_columns); + std::vector combined_row_ids; + eos = false; + while (!eos) { + ASSERT_TRUE(combined_reader.get_block(&combined_block, &eos).ok()); + if (eos) { + continue; + } + const auto& row_ids = + assert_cast(*combined_block.get_by_position(0).column); + combined_row_ids.insert(combined_row_ids.end(), row_ids.get_data().begin(), + row_ids.get_data().end()); + } + EXPECT_EQ((std::vector {4}), combined_row_ids); + ASSERT_NE(combined_profile.get_info_string("LanceRuntimeFilterPushedIds"), nullptr); + EXPECT_EQ("42", *combined_profile.get_info_string("LanceRuntimeFilterPushedIds")); + EXPECT_TRUE(combined_reader.close().ok()); +} + +TEST(LanceTableReaderFilterTest, PushesRuntimeInFilterIntoLanceScanner) { + const std::filesystem::path dataset_uri = + "./be/test/format_v2/table/lance/data/all_types.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(dataset_uri, &fixture).ok()); + + const Columns columns {projected_column("row_id", TYPE_BIGINT, false)}; + TQueryOptions query_options; + query_options.__set_batch_size(4); + TQueryGlobals query_globals; + RuntimeState state(query_globals); + state.set_query_options(query_options); + RuntimeProfile profile("lance_runtime_filter_pushdown_fixture"); + TFileScanRangeParams scan_params; + const auto runtime_filter = create_int64_runtime_in_conjunct("row_id", {2, 4}, 41); + + LanceTableReader reader; + ASSERT_TRUE( + init_reader(&reader, columns, &state, &profile, &scan_params, {runtime_filter}).ok()); + ASSERT_TRUE(prepare_fixture(&reader, dataset_uri, fixture, fixture.fragment_ids).ok()); + + Block block; + add_output_columns(&block, columns); + std::vector actual_row_ids; + bool eos = false; + while (!eos) { + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + if (eos) { + continue; + } + const auto& row_ids = assert_cast(*block.get_by_position(0).column); + actual_row_ids.insert(actual_row_ids.end(), row_ids.get_data().begin(), + row_ids.get_data().end()); + } + std::ranges::sort(actual_row_ids); + EXPECT_EQ((std::vector {2, 4}), actual_row_ids); + ASSERT_NE(profile.get_info_string("LanceRuntimeFilterPushedIds"), nullptr); + EXPECT_EQ("41", *profile.get_info_string("LanceRuntimeFilterPushedIds")); + EXPECT_EQ(profile.get_info_string("LanceRuntimeFilterSkippedIds"), nullptr); + EXPECT_TRUE(reader.close().ok()); } TEST(LanceTableReaderFilterTest, LeavesResidualPredicatesToScanner) { diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/lance_build_preinstalled_catalog.py b/docker/thirdparties/docker-compose/iceberg/scripts/lance_build_preinstalled_catalog.py index 9721074094f4b3..cb7d79df9341b8 100644 --- a/docker/thirdparties/docker-compose/iceberg/scripts/lance_build_preinstalled_catalog.py +++ b/docker/thirdparties/docker-compose/iceberg/scripts/lance_build_preinstalled_catalog.py @@ -28,7 +28,7 @@ The generated catalog contains: - __manifest Directory Namespace V2 manifest table (with its scalar indexes). - all_types.lance The pre-existing compatibility-mode root table, re-registered as-is. - - The `doris` namespace with one indexed vector table per cell of the + - The `doris` namespace with two full-text-search fixtures, one indexed vector table per cell of the algorithm x element type x metric matrix (hash-prefixed directories), listed in VECTOR_TABLES below; BREADTH_TABLE, one table carrying the remaining cells at plan level; and NESTED_TABLE, a nested-field scalar index fixture used by the SHOW INDEX / @@ -445,6 +445,41 @@ def _digest_of(vector) -> str: NESTED_COLUMN = "attributes.`child.with.dot`" NESTED_ROWS = 16 +# Full-text-search fixtures. full_text_search indexes both fragments. The partial table +# deliberately appends its second fragment after index creation so Doris can exercise the +# STRICT and INDEX_ONLY coverage modes against the same committed index segment. +FTS_TABLE = "full_text_search" +FTS_PARTIAL_TABLE = "full_text_search_partial" +FTS_INDEX_NAME = "body_fts" +FTS_PARTIAL_INDEX_NAME = "body_fts_partial" +FTS_ROWS = ( + (1, "Lance search", "lance search engine", "tech"), + (2, "Lance search twice", "lance lance search engine", "tech"), + (3, "Lance search three times", "lance lance lance search engine", "tech"), + (4, "Vector search", "vector search engine", "vector"), + (5, "Full text search", "full text search engine", "search"), + (6, "Doris lakehouse", "apache doris lakehouse", "database"), + (7, "Lance storage", "lance columnar storage format analytics", "storage"), + (8, "Unrelated", "unrelated document", "other"), +) +FTS_PARTIAL_ROWS = ( + (101, "lance indexed document", "indexed"), + (102, "another lance indexed document", "indexed"), + (103, "unrelated indexed document", "indexed"), + (104, "lance appended after indexing", "unindexed"), + (105, "second lance appended after indexing", "unindexed"), +) +FTS_INDEX_PARAMS = { + "base_tokenizer": "simple", + "language": "English", + "max_token_length": 40, + "lower_case": True, + "stem": False, + "remove_stop_words": False, + "ascii_folding": False, + "with_position": True, +} + def buildable_combos(): """Every (element type, metric, algorithm) cell the embedded Lance actually accepts. @@ -697,6 +732,77 @@ def create_vector_table(namespace, table_name: str, spec: dict) -> str: return location +def make_fts_table(rows, *, include_title: bool) -> pa.Table: + if include_title: + schema = pa.schema( + [ + pa.field("row_id", pa.int64(), nullable=False), + pa.field("title", pa.string(), nullable=False), + pa.field("body", pa.string(), nullable=False), + pa.field("category", pa.string(), nullable=False), + ] + ) + columns = list(zip(*rows)) + return pa.Table.from_arrays( + [ + pa.array(columns[0], type=pa.int64()), + pa.array(columns[1], type=pa.string()), + pa.array(columns[2], type=pa.string()), + pa.array(columns[3], type=pa.string()), + ], + schema=schema, + ) + + schema = pa.schema( + [ + pa.field("row_id", pa.int64(), nullable=False), + pa.field("body", pa.string(), nullable=False), + pa.field("category", pa.string(), nullable=False), + ] + ) + columns = list(zip(*rows)) + return pa.Table.from_arrays( + [ + pa.array(columns[0], type=pa.int64()), + pa.array(columns[1], type=pa.string()), + pa.array(columns[2], type=pa.string()), + ], + schema=schema, + ) + + +def create_fts_table(namespace, table_name: str, rows, *, indexed_rows: int, + index_name: str, include_title: bool) -> str: + first = make_fts_table(rows[:indexed_rows], include_title=include_title) + buffer = io.BytesIO() + with ipc.new_stream(buffer, first.schema) as writer: + writer.write_table(first) + response = namespace.create_table( + CreateTableRequest(id=[NAMESPACE, table_name]), buffer.getvalue() + ) + location = response.location + dataset = lance.dataset(location) + if table_name == FTS_TABLE: + # Make the fully indexed fixture cover two physical fragments. + lance.write_dataset( + make_fts_table(rows[indexed_rows:], include_title=include_title), + location, + mode="append", + ) + dataset = lance.dataset(location) + dataset.create_scalar_index( + "body", "INVERTED", name=index_name, **FTS_INDEX_PARAMS + ) + if table_name == FTS_PARTIAL_TABLE: + # Keep these rows outside the committed segment for coverage-mode tests. + lance.write_dataset( + make_fts_table(rows[indexed_rows:], include_title=include_title), + location, + mode="append", + ) + return location + + def make_nested_fragment_table(row_offset_start: int, row_offset_end: int) -> pa.Table: offsets = list(range(row_offset_start, row_offset_end)) attributes = pa.StructArray.from_arrays( @@ -794,6 +900,22 @@ def build(root: Path, all_types_source: Path) -> None: RegisterTableRequest(id=["all_types"], location=ALL_TYPES_DIR) ) namespace.create_namespace(CreateNamespaceRequest(id=[NAMESPACE])) + create_fts_table( + namespace, + FTS_TABLE, + FTS_ROWS, + indexed_rows=4, + index_name=FTS_INDEX_NAME, + include_title=True, + ) + create_fts_table( + namespace, + FTS_PARTIAL_TABLE, + FTS_PARTIAL_ROWS, + indexed_rows=3, + index_name=FTS_PARTIAL_INDEX_NAME, + include_title=False, + ) for table_name, spec in VECTOR_TABLES.items(): location = create_vector_table(namespace, table_name, spec) # DirectoryNamespace.create_table_index exists but raises UnsupportedOperationError, @@ -1431,11 +1553,43 @@ def check_nested_dataset(location: str): assert probe == [7], f"{NESTED_TABLE}: BTREE probe returned {probe}" +def check_fts_dataset(location: str, *, table_name: str, index_name: str, + expected_rows: int, expected_indexed_fragments: int) -> None: + dataset = lance.dataset(location) + assert dataset.count_rows() == expected_rows, ( + f"{table_name}: expected {expected_rows} rows" + ) + fragments = dataset.get_fragments() + assert len(fragments) == 2, f"{table_name}: expected 2 fragments" + indices = {index["name"]: index for index in dataset.list_indices()} + index = indices.get(index_name) + assert index is not None, f"{table_name}: missing FTS index {index_name}: {indices}" + assert index["type"] == "Inverted", ( + f"{table_name}: {index_name} has type {index['type']}, expected Inverted" + ) + indexed_fragments = set(index["fragment_ids"]) + assert len(indexed_fragments) == expected_indexed_fragments, ( + f"{table_name}: {index_name} covers fragments {sorted(indexed_fragments)}, expected " + f"{expected_indexed_fragments} fragments" + ) + if table_name == FTS_TABLE: + ranked = dataset.scanner( + columns=["row_id", "_score"], + full_text_query={"query": "lance", "columns": ["body"]}, + limit=4, + ).to_table() + assert ranked["row_id"].to_pylist() == [3, 2, 1, 7], ( + f"{table_name}: indexed BM25 probe returned {ranked.to_pydict()}" + ) + + def check_catalog(root: Path) -> None: check_data_shapes() namespace = lance_namespace.connect("dir", {"root": str(root)}) tables = namespace.list_tables(ListTablesRequest(id=[NAMESPACE])) - expected_tables = sorted([*VECTOR_TABLES, BREADTH_TABLE, NESTED_TABLE]) + expected_tables = sorted( + [*VECTOR_TABLES, BREADTH_TABLE, NESTED_TABLE, FTS_TABLE, FTS_PARTIAL_TABLE] + ) assert sorted(tables.tables) == expected_tables, ( f"unexpected {NAMESPACE} tables: {tables.tables}" ) @@ -1490,6 +1644,25 @@ def check_catalog(root: Path) -> None: nested_path = Path(nested.location.removeprefix("file://")) assert nested_path.is_dir(), f"{NESTED_TABLE} location missing: {nested.location}" check_nested_dataset(nested.location) + + full_fts = namespace.describe_table(DescribeTableRequest(id=[NAMESPACE, FTS_TABLE])) + check_fts_dataset( + full_fts.location, + table_name=FTS_TABLE, + index_name=FTS_INDEX_NAME, + expected_rows=len(FTS_ROWS), + expected_indexed_fragments=2, + ) + partial_fts = namespace.describe_table( + DescribeTableRequest(id=[NAMESPACE, FTS_PARTIAL_TABLE]) + ) + check_fts_dataset( + partial_fts.location, + table_name=FTS_PARTIAL_TABLE, + index_name=FTS_PARTIAL_INDEX_NAME, + expected_rows=len(FTS_PARTIAL_ROWS), + expected_indexed_fragments=1, + ) print(f"self-check OK: {root}") diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/metadata.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/metadata.lance new file mode 100644 index 00000000000000..340a57964dc091 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/metadata.lance differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/part_0_docs.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/part_0_docs.lance new file mode 100644 index 00000000000000..9b336efbb78855 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/part_0_docs.lance differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/part_0_invert.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/part_0_invert.lance new file mode 100644 index 00000000000000..056f31f2d9346f Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/part_0_invert.lance differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/part_0_tokens.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/part_0_tokens.lance new file mode 100644 index 00000000000000..f265ddd2b8838c Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/part_0_tokens.lance differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/_transactions/0-4a942168-96d8-4f07-bce0-e3049c8ea6c5.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/_transactions/0-4a942168-96d8-4f07-bce0-e3049c8ea6c5.txn new file mode 100644 index 00000000000000..65272d53b4f7b4 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/_transactions/0-4a942168-96d8-4f07-bce0-e3049c8ea6c5.txn differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/_transactions/1-b311fe36-9353-4f41-ae19-5054d4a73009.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/_transactions/1-b311fe36-9353-4f41-ae19-5054d4a73009.txn new file mode 100644 index 00000000000000..be61bddcadfc85 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/_transactions/1-b311fe36-9353-4f41-ae19-5054d4a73009.txn differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/_transactions/2-76e69d8c-04b7-402e-8620-7f9a772a2224.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/_transactions/2-76e69d8c-04b7-402e-8620-7f9a772a2224.txn new file mode 100644 index 00000000000000..e9216b8131e17a Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/_transactions/2-76e69d8c-04b7-402e-8620-7f9a772a2224.txn differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/_versions/18446744073709551612.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/_versions/18446744073709551612.manifest new file mode 100644 index 00000000000000..83edf4ee407b29 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/_versions/18446744073709551612.manifest differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/_versions/18446744073709551613.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/_versions/18446744073709551613.manifest new file mode 100644 index 00000000000000..19ab9f8f79ef41 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/_versions/18446744073709551613.manifest differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/_versions/18446744073709551614.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/_versions/18446744073709551614.manifest new file mode 100644 index 00000000000000..f89780183298b8 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/_versions/18446744073709551614.manifest differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/_versions/latest_version_hint.json b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/_versions/latest_version_hint.json new file mode 100644 index 00000000000000..f93d3984472d99 --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/_versions/latest_version_hint.json @@ -0,0 +1 @@ +{"version":3} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/data/000100011001100000000001ee52754ad1a1da89dd0efa4379.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/data/000100011001100000000001ee52754ad1a1da89dd0efa4379.lance new file mode 100644 index 00000000000000..5079d336a0c6b0 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/data/000100011001100000000001ee52754ad1a1da89dd0efa4379.lance differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/data/100101010010010100010000fd779442f589c03379a2858ea8.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/data/100101010010010100010000fd779442f589c03379a2858ea8.lance new file mode 100644 index 00000000000000..3714a20bae4cfa Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/14d4e11d_doris$full_text_search/data/100101010010010100010000fd779442f589c03379a2858ea8.lance differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/8831a412-b4a7-4975-950c-8fe62bf16ef7/bitmap_page_lookup.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/19cc56b4-0a64-4d27-a659-61c9404f2b4e/bitmap_page_lookup.lance similarity index 80% rename from docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/8831a412-b4a7-4975-950c-8fe62bf16ef7/bitmap_page_lookup.lance rename to docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/19cc56b4-0a64-4d27-a659-61c9404f2b4e/bitmap_page_lookup.lance index 2efd05d5ca4493..94c34d8fb83014 100644 Binary files a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/8831a412-b4a7-4975-950c-8fe62bf16ef7/bitmap_page_lookup.lance and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/19cc56b4-0a64-4d27-a659-61c9404f2b4e/bitmap_page_lookup.lance differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/3d69787d-d9aa-4cc5-8358-cd34c85bd6f1/page_data.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/3d69787d-d9aa-4cc5-8358-cd34c85bd6f1/page_data.lance new file mode 100644 index 00000000000000..d9b756b383036c Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/3d69787d-d9aa-4cc5-8358-cd34c85bd6f1/page_data.lance differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/5b68f7cb-d3d6-404d-9fe4-2fab2d4c2c3e/page_lookup.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/3d69787d-d9aa-4cc5-8358-cd34c85bd6f1/page_lookup.lance similarity index 100% rename from docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/5b68f7cb-d3d6-404d-9fe4-2fab2d4c2c3e/page_lookup.lance rename to docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/3d69787d-d9aa-4cc5-8358-cd34c85bd6f1/page_lookup.lance diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/5b68f7cb-d3d6-404d-9fe4-2fab2d4c2c3e/page_data.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/5b68f7cb-d3d6-404d-9fe4-2fab2d4c2c3e/page_data.lance deleted file mode 100644 index 4028aef2a37423..00000000000000 Binary files a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/5b68f7cb-d3d6-404d-9fe4-2fab2d4c2c3e/page_data.lance and /dev/null differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/6cd7ad50-bc0e-484d-87b1-f9b3ac35ce9b/bitmap_page_lookup.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/6cd7ad50-bc0e-484d-87b1-f9b3ac35ce9b/bitmap_page_lookup.lance new file mode 100644 index 00000000000000..1fd15b9ca22f57 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/6cd7ad50-bc0e-484d-87b1-f9b3ac35ce9b/bitmap_page_lookup.lance differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/b2a0b46c-741e-4884-8b1b-15aa5e389eb0/bitmap_page_lookup.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/b2a0b46c-741e-4884-8b1b-15aa5e389eb0/bitmap_page_lookup.lance deleted file mode 100644 index 7b80eb38797d6f..00000000000000 Binary files a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/b2a0b46c-741e-4884-8b1b-15aa5e389eb0/bitmap_page_lookup.lance and /dev/null differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_transactions/28-39e6dfbf-f1c5-4b1e-808a-ee342f74383f.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_transactions/28-39e6dfbf-f1c5-4b1e-808a-ee342f74383f.txn deleted file mode 100644 index 93f477daf06232..00000000000000 Binary files a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_transactions/28-39e6dfbf-f1c5-4b1e-808a-ee342f74383f.txn and /dev/null differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_transactions/33-54b88aa6-136b-41a5-9652-3a7132712a1a.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_transactions/33-54b88aa6-136b-41a5-9652-3a7132712a1a.txn new file mode 100644 index 00000000000000..b3102ead4af799 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_transactions/33-54b88aa6-136b-41a5-9652-3a7132712a1a.txn differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_versions/18446744073709551581.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_versions/18446744073709551581.manifest new file mode 100644 index 00000000000000..6cf3b6446df8a8 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_versions/18446744073709551581.manifest differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_versions/18446744073709551586.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_versions/18446744073709551586.manifest deleted file mode 100644 index e5cef20ebaa120..00000000000000 Binary files a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_versions/18446744073709551586.manifest and /dev/null differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_versions/latest_version_hint.json b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_versions/latest_version_hint.json index c6ccfd69ce22cd..6e754b99aaa8aa 100644 --- a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_versions/latest_version_hint.json +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_versions/latest_version_hint.json @@ -1 +1 @@ -{"version":29} \ No newline at end of file +{"version":34} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/data/10010110000000010010101183fee04e8eb3b452a26bcf00a5.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/data/10010110000000010010101183fee04e8eb3b452a26bcf00a5.lance new file mode 100644 index 00000000000000..4cbe1bdceff686 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/data/10010110000000010010101183fee04e8eb3b452a26bcf00a5.lance differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/data/01101101000001110001101007a96744fd8fa4f4791c8ccd2a.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/data/1001111110010100110110006d24fe4da58277b24ffe1d6a0e.lance similarity index 60% rename from docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/data/01101101000001110001101007a96744fd8fa4f4791c8ccd2a.lance rename to docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/data/1001111110010100110110006d24fe4da58277b24ffe1d6a0e.lance index 5c95966b4be9f9..ac42f15f2dab96 100644 Binary files a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/data/01101101000001110001101007a96744fd8fa4f4791c8ccd2a.lance and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/data/1001111110010100110110006d24fe4da58277b24ffe1d6a0e.lance differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/data/110001010111011100000100d7d3884108b45243313e881513.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/data/110001010111011100000100d7d3884108b45243313e881513.lance deleted file mode 100644 index 92532de4d5a022..00000000000000 Binary files a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/data/110001010111011100000100d7d3884108b45243313e881513.lance and /dev/null differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/_indices/998a5a77-b8d4-4318-abad-3fc2341fc194/metadata.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/_indices/998a5a77-b8d4-4318-abad-3fc2341fc194/metadata.lance new file mode 100644 index 00000000000000..af3dad059c929c Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/_indices/998a5a77-b8d4-4318-abad-3fc2341fc194/metadata.lance differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/_indices/998a5a77-b8d4-4318-abad-3fc2341fc194/part_0_docs.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/_indices/998a5a77-b8d4-4318-abad-3fc2341fc194/part_0_docs.lance new file mode 100644 index 00000000000000..5dce8a68b3a07d Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/_indices/998a5a77-b8d4-4318-abad-3fc2341fc194/part_0_docs.lance differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/_indices/998a5a77-b8d4-4318-abad-3fc2341fc194/part_0_invert.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/_indices/998a5a77-b8d4-4318-abad-3fc2341fc194/part_0_invert.lance new file mode 100644 index 00000000000000..2d4ccf3f952ecb Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/_indices/998a5a77-b8d4-4318-abad-3fc2341fc194/part_0_invert.lance differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/_indices/998a5a77-b8d4-4318-abad-3fc2341fc194/part_0_tokens.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/_indices/998a5a77-b8d4-4318-abad-3fc2341fc194/part_0_tokens.lance new file mode 100644 index 00000000000000..e730e8ed9d19fb Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/_indices/998a5a77-b8d4-4318-abad-3fc2341fc194/part_0_tokens.lance differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/_transactions/0-1bba5bff-2ab6-434b-8eaa-822470b46ed7.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/_transactions/0-1bba5bff-2ab6-434b-8eaa-822470b46ed7.txn new file mode 100644 index 00000000000000..8cb78392d3a659 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/_transactions/0-1bba5bff-2ab6-434b-8eaa-822470b46ed7.txn differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/_transactions/1-0a6db125-38e9-4ae5-94d5-5972747763df.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/_transactions/1-0a6db125-38e9-4ae5-94d5-5972747763df.txn new file mode 100644 index 00000000000000..2611a3256d7a42 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/_transactions/1-0a6db125-38e9-4ae5-94d5-5972747763df.txn differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/_transactions/2-315f4e75-d073-4bd2-a75d-f04c8347628b.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/_transactions/2-315f4e75-d073-4bd2-a75d-f04c8347628b.txn new file mode 100644 index 00000000000000..b87214d155d2c6 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/_transactions/2-315f4e75-d073-4bd2-a75d-f04c8347628b.txn differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/_versions/18446744073709551612.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/_versions/18446744073709551612.manifest new file mode 100644 index 00000000000000..e025943666c9df Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/_versions/18446744073709551612.manifest differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/_versions/18446744073709551613.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/_versions/18446744073709551613.manifest new file mode 100644 index 00000000000000..691897dc756c03 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/_versions/18446744073709551613.manifest differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/_versions/18446744073709551614.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/_versions/18446744073709551614.manifest new file mode 100644 index 00000000000000..7de391aba49a31 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/_versions/18446744073709551614.manifest differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/_versions/latest_version_hint.json b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/_versions/latest_version_hint.json new file mode 100644 index 00000000000000..f93d3984472d99 --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/_versions/latest_version_hint.json @@ -0,0 +1 @@ +{"version":3} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/data/00101001111000100000100086dae842f7bee179e75e92939e.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/data/00101001111000100000100086dae842f7bee179e75e92939e.lance new file mode 100644 index 00000000000000..9588683b1be0f9 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/data/00101001111000100000100086dae842f7bee179e75e92939e.lance differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/data/001100101100101101000001aafd3f464c855149c5211be544.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/data/001100101100101101000001aafd3f464c855149c5211be544.lance new file mode 100644 index 00000000000000..d561f04f549448 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/d9b832a2_doris$full_text_search_partial/data/001100101100101101000001aafd3f464c855149c5211be544.lance differ diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java index 06b7242d161875..741710801f3b23 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.trees.expressions.functions.table.File; import org.apache.doris.nereids.trees.expressions.functions.table.Frontends; import org.apache.doris.nereids.trees.expressions.functions.table.FrontendsDisks; +import org.apache.doris.nereids.trees.expressions.functions.table.FullTextSearch; import org.apache.doris.nereids.trees.expressions.functions.table.GroupCommit; import org.apache.doris.nereids.trees.expressions.functions.table.Hdfs; import org.apache.doris.nereids.trees.expressions.functions.table.Http; @@ -77,7 +78,8 @@ public class BuiltinTableValuedFunctions implements FunctionHelper { tableValued(ParquetKvMetadata.class, "parquet_kv_metadata"), tableValued(ParquetBloomProbe.class, "parquet_bloom_probe"), tableValued(CdcStream.class, "cdc_stream"), - tableValued(VectorSearch.class, "vector_search") + tableValued(VectorSearch.class, "vector_search"), + tableValued(FullTextSearch.class, "full_text_search") ); public static final BuiltinTableValuedFunctions INSTANCE = new BuiltinTableValuedFunctions(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java index c2fa3d1a6ee8a4..33ebc39a30ef85 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java @@ -291,7 +291,7 @@ public LanceTableMetadata loadTableMetadata(String dbName, String tableName) { return loadTableMetadata(dbName, tableName, Optional.empty(), false); } - public LanceTableMetadata loadTableMetadataForVectorSearch(String dbName, String tableName) { + public LanceTableMetadata loadTableMetadataForSearch(String dbName, String tableName) { return loadTableMetadata(dbName, tableName, Optional.empty(), true); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalTable.java index acba68b63fcbe5..0d1f124fce6f88 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalTable.java @@ -66,8 +66,8 @@ public LanceTableMetadata loadMetadata() { return ((LanceExternalCatalog) catalog).loadTableMetadata(db.getRemoteName(), remoteName); } - public LanceTableMetadata loadMetadataForVectorSearch() { - return ((LanceExternalCatalog) catalog).loadTableMetadataForVectorSearch( + public LanceTableMetadata loadMetadataForSearch() { + return ((LanceExternalCatalog) catalog).loadTableMetadataForSearch( db.getRemoteName(), remoteName); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexSegmentInfo.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexSegmentInfo.java index 7fb373f9246035..f0f1667cbee573 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexSegmentInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexSegmentInfo.java @@ -17,27 +17,32 @@ package org.apache.doris.datasource.lance; +import org.lance.index.IndexType; + import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.UUID; -/** Immutable metadata for one physical segment of a logical Lance vector index. */ +/** Immutable metadata for one physical segment of a logical Lance search index. */ public final class LanceIndexSegmentInfo { private final UUID uuid; private final String indexName; private final List fieldIds; private final List fragmentIds; + private final IndexType indexType; private final String metric; public LanceIndexSegmentInfo(UUID uuid, String indexName, List fieldIds, - List fragmentIds, String metric) { + List fragmentIds, IndexType indexType, String metric) { this.uuid = uuid; this.indexName = indexName; this.fieldIds = Collections.unmodifiableList(new ArrayList<>(fieldIds)); this.fragmentIds = fragmentIds == null ? null : Collections.unmodifiableList(new ArrayList<>(fragmentIds)); + this.indexType = Objects.requireNonNull(indexType, "indexType must not be null"); this.metric = metric; } @@ -56,14 +61,26 @@ public List getFieldIds() { /** * Returns the fragment bitmap recorded in the manifest. * - *

Legacy index segments may not have a bitmap. Callers must not infer coverage from the - * segment's dataset version in that case. + *

Index segments without a fragment bitmap have unknown coverage. Callers must not infer + * coverage from the segment's dataset version in that case. */ public Optional> getFragmentIds() { return Optional.ofNullable(fragmentIds); } - /** Returns the normalized Lance metric name, or an empty optional for legacy metadata. */ + public IndexType getIndexType() { + return indexType; + } + + public boolean isVectorIndex() { + return indexType.getValue() >= IndexType.VECTOR.getValue(); + } + + public boolean isFullTextIndex() { + return indexType == IndexType.INVERTED; + } + + /** Returns the normalized Lance metric name when the index metadata supplies one. */ public Optional getMetric() { return Optional.ofNullable(metric); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceMetadataLoader.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceMetadataLoader.java index 9c4803af7100d6..a3c27e7b6bbdf6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceMetadataLoader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceMetadataLoader.java @@ -72,7 +72,7 @@ public static LanceTableMetadata loadLatest(String datasetUri, datasetUri, lanceStorageOptions, OptionalLong.empty(), allocator, false); } - /** Loads the latest fixed snapshot together with vector index segment coverage. */ + /** Loads the latest fixed snapshot together with search-index segment coverage. */ public static LanceTableMetadata loadLatestWithIndexSegments( String datasetUri, Map lanceStorageOptions, BufferAllocator allocator) throws Exception { return loadInternal( @@ -109,7 +109,7 @@ private static LanceTableMetadata loadInternal(String datasetUri, Map lanceFieldIds = loadIndexSegments ? loadTopLevelFieldIds(dataset) : Collections.emptyMap(); List indexSegments = loadIndexSegments - ? loadVectorIndexSegments(dataset) : Collections.emptyList(); + ? loadSearchIndexSegments(dataset) : Collections.emptyList(); return loadIndexSegments ? LanceTableMetadata.withIndexSegments(datasetUri, resolvedVersion, dataset.getSchema(), fragments, lanceFieldIds, @@ -134,12 +134,13 @@ private static Map loadTopLevelFieldIds(Dataset dataset) { return result; } - private static List loadVectorIndexSegments(Dataset dataset) { + private static List loadSearchIndexSegments(Dataset dataset) { List result = new ArrayList<>(); for (IndexDescription description : dataset.describeIndices()) { String metric = parseMetric(description.getDetailsJson()); for (Index segment : description.getSegments()) { - if (segment.indexType() == null || segment.indexType().getValue() < 100) { + if (segment.indexType() == null || (segment.indexType().getValue() < 100 + && segment.indexType() != org.lance.index.IndexType.INVERTED)) { continue; } List fragmentIds = segment.fragments() @@ -152,7 +153,7 @@ private static List loadVectorIndexSegments(Dataset datas }) .orElse(null); result.add(new LanceIndexSegmentInfo(segment.uuid(), description.getName(), - description.getFieldIds(), fragmentIds, metric)); + description.getFieldIds(), fragmentIds, segment.indexType(), metric)); } } return result; @@ -166,8 +167,8 @@ private static String parseMetric(String detailsJson) { JsonNode metric = JsonUtil.readTree(detailsJson).get("metric_type"); return metric == null || !metric.isTextual() ? null : metric.asText().toUpperCase(); } catch (RuntimeException e) { - // Index details are optional compatibility metadata. An unknown legacy encoding should - // disable metric-sensitive segment planning rather than prevent ordinary table access. + // Index details are optional metadata. Malformed details disable metric-sensitive + // segment planning rather than preventing ordinary table access. return null; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/IndexSegmentSplitPlan.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/IndexSegmentSplitPlan.java index c8c61fa46a33d1..b890242ddff16c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/IndexSegmentSplitPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/IndexSegmentSplitPlan.java @@ -26,7 +26,7 @@ import java.util.Set; import java.util.UUID; -/** Builds vector-search splits from physical Lance index segments and unindexed fragments. */ +/** Builds external-search splits from physical Lance index segments and optional fragments. */ final class IndexSegmentSplitPlan { private final String datasetUri; private final long version; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java index 394cb6ebf068a1..626fe08870102a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java @@ -39,10 +39,14 @@ import org.apache.doris.thrift.TExternalSearchRequest; import org.apache.doris.thrift.TFileFormatType; import org.apache.doris.thrift.TFileRangeDesc; +import org.apache.doris.thrift.TFtsCoverageMode; +import org.apache.doris.thrift.TFtsQueryType; +import org.apache.doris.thrift.TFullTextSearchParams; import org.apache.doris.thrift.TLanceFileDesc; import org.apache.doris.thrift.TLanceScanParams; import org.apache.doris.thrift.TTableFormatFileDesc; import org.apache.doris.thrift.TVectorMetric; +import org.apache.doris.thrift.TVectorSearchOptions; import org.apache.doris.thrift.TVectorSearchParams; import java.nio.ByteBuffer; @@ -61,14 +65,22 @@ * Keeping them in one node prevents those common parts from drifting apart. The search request is * also an explicit mode marker. Ordinary scans are split by fragment. Indexed vector searches are * split by physical index segment, with uncovered fragments retained as flat-search fallbacks. - * Each search split produces local candidates; a Doris TopN above this scan merges them into the - * requested snapshot-wide result. + * Full-text searches are split only by committed inverted-index segments, with coverage governed + * by the request's STRICT or INDEX_ONLY mode. Each search split produces local candidates; a Doris + * TopN above this scan merges them into the requested snapshot-wide result. */ public class LanceScanNode extends FileQueryScanNode { + private enum SearchKind { + NORMAL, + VECTOR, + FULL_TEXT + } + private LanceExternalTable lanceTable; private LanceTableMetadata plannedMetadata; - private int vectorFieldId = -1; - private TExternalSearchRequest externalSearchRequest; + private final int searchFieldId; + private final TExternalSearchRequest externalSearchRequest; + private final SearchKind searchKind; private byte[] lanceSubstraitFilter = new byte[0]; private String lancePushdownPredicate = ""; private long plannedVersion = -1; @@ -81,38 +93,45 @@ public LanceScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckColum SessionVariable sessionVariable, ScanContext scanContext) { super(id, desc, "LANCE_SCAN_NODE", StatisticalType.LANCE_SCAN_NODE, scanContext, needCheckColumnPriv, sessionVariable); + this.searchFieldId = -1; + this.externalSearchRequest = null; + this.searchKind = SearchKind.NORMAL; } /** * Creates the search mode of this node. * *

The tuple descriptor belongs to a FunctionGenTable and contains generated columns such as - * {@code _distance}. Therefore the real Lance table and the metadata snapshot selected while - * analyzing the TVF must be passed separately. + * {@code _distance} or {@code _score}. Therefore the real Lance table and the metadata + * snapshot selected while analyzing the TVF must be passed separately. */ - public static LanceScanNode forVectorSearch(PlanNodeId id, TupleDescriptor desc, - LanceExternalTable lanceTable, LanceTableMetadata plannedMetadata, int vectorFieldId, + public static LanceScanNode forExternalSearch(PlanNodeId id, TupleDescriptor desc, + LanceExternalTable lanceTable, LanceTableMetadata plannedMetadata, int searchFieldId, TExternalSearchRequest externalSearchRequest, SessionVariable sessionVariable) { - return new LanceScanNode(id, desc, lanceTable, plannedMetadata, vectorFieldId, + return new LanceScanNode(id, desc, lanceTable, plannedMetadata, searchFieldId, externalSearchRequest, sessionVariable); } private LanceScanNode(PlanNodeId id, TupleDescriptor desc, LanceExternalTable lanceTable, - LanceTableMetadata plannedMetadata, int vectorFieldId, + LanceTableMetadata plannedMetadata, int searchFieldId, TExternalSearchRequest externalSearchRequest, SessionVariable sessionVariable) { super(id, desc, "LANCE_SCAN_NODE", StatisticalType.LANCE_SCAN_NODE, ScanContext.builder().clusterName(sessionVariable.resolveCloudClusterName()).build(), false, sessionVariable); this.lanceTable = lanceTable; this.plannedMetadata = plannedMetadata; - this.vectorFieldId = vectorFieldId; + this.searchFieldId = searchFieldId; + if (externalSearchRequest == null) { + throw new IllegalArgumentException("Lance external search request must not be null"); + } this.externalSearchRequest = externalSearchRequest.deepCopy(); + this.searchKind = resolveSearchKind(this.externalSearchRequest); } @Override protected void doInitialize() throws UserException { List sourceColumns; - if (isExternalSearch()) { + if (searchKind != SearchKind.NORMAL) { sourceColumns = desc.getTable().getColumns(); } else { lanceTable = (LanceExternalTable) desc.getTable(); @@ -123,11 +142,11 @@ protected void doInitialize() throws UserException { super.doInitialize(); ExternalUtil.initSchemaInfo(params, -1L, sourceColumns); - if (isExternalSearch()) { + if (searchKind != SearchKind.NORMAL) { // Search output comes from the FunctionGenTable because it adds generated columns such - // as _distance. The real Lance table is still retained for storage and metadata access. + // as _distance or _score. The real Lance table is retained for storage and metadata. getOrCreateLanceScanParams() - .setExternalSearchRequest(createFragmentSearchRequest(externalSearchRequest)); + .setExternalSearchRequest(createSplitSearchRequest()); } } @@ -153,9 +172,9 @@ private boolean canPushDownLimit() { @Override protected void convertPredicate() { - if (isExternalSearch()) { + if (searchKind != SearchKind.NORMAL) { // The TVF "filter" property is already serialized in externalSearchRequest and is - // evaluated by Lance before vector search. Outer WHERE conjuncts have different + // evaluated by Lance before candidate search. Outer WHERE conjuncts have different // semantics: keep them as Doris scan residuals. Each fragment first returns its Lance // ANN candidates, then Doris evaluates these conjuncts before the local/global TopN. } else { @@ -186,20 +205,31 @@ public List getSplits(int numBackends) throws UserException { LanceTableMetadata metadata = plannedMetadata; plannedVersion = metadata.getVersion(); plannedFragments = metadata.getFragments().size(); - plannedUnindexedFragments = isExternalSearch() ? plannedFragments : 0; + plannedUnindexedFragments = searchKind == SearchKind.NORMAL ? 0 : plannedFragments; plannedIndexSegments = 0; plannedIndexFragments = 0; - if (isExternalSearch() && plannedVersion <= 0) { + if (searchKind != SearchKind.NORMAL && plannedVersion <= 0) { throw new UserException( - "Lance vector search requires a fixed positive dataset version"); + "Lance external search requires a fixed positive dataset version"); } Map visibleFragments = getVisibleFragments(metadata); - if (isExternalSearch() && shouldUseIndex()) { - Optional> indexSplits = createIndexSegmentSplits(metadata, visibleFragments); - if (indexSplits.isPresent()) { - return indexSplits.get(); - } + switch (searchKind) { + case FULL_TEXT: + return createFullTextIndexSegmentSplits(metadata, visibleFragments); + case VECTOR: + if (isVectorIndexEnabled()) { + Optional> indexSplits = createVectorIndexSegmentSplits( + metadata, visibleFragments); + if (indexSplits.isPresent()) { + return indexSplits.get(); + } + } + break; + case NORMAL: + break; + default: + throw new IllegalStateException("Unsupported Lance search kind " + searchKind); } return createFragmentSplits(metadata, visibleFragments); } @@ -236,25 +266,25 @@ private List createFragmentSplits(LanceTableMetadata metadata, return splits; } - private Optional> createIndexSegmentSplits(LanceTableMetadata metadata, + private Optional> createVectorIndexSegmentSplits(LanceTableMetadata metadata, Map visibleFragments) throws UserException { if (metadata.getIndexSegments().isEmpty()) { return Optional.empty(); } TVectorSearchParams vectorSearchParam = externalSearchRequest.getSearchQuery().getVectorSearch(); - if (vectorFieldId < 0) { + if (searchFieldId < 0) { throw new UserException("Lance vector column '" + vectorSearchParam.getColumn() + "' has no field ID in the Lance schema"); } - List matchingSegments = selectIndexSegments( - metadata.getIndexSegments(), vectorFieldId); + List matchingSegments = selectVectorIndexSegments( + metadata.getIndexSegments(), searchFieldId); if (matchingSegments.isEmpty() || !metricMatches(vectorSearchParam, matchingSegments)) { return Optional.empty(); } Optional indexPlan = planIndexSegments( - metadata, matchingSegments, visibleFragments); + metadata, matchingSegments, visibleFragments, false); if (!indexPlan.isPresent()) { return Optional.empty(); } @@ -266,12 +296,44 @@ private Optional> createIndexSegmentSplits(LanceTableMetadata metada return Optional.of(plan.buildSplits()); } - private static List selectIndexSegments( - List indexSegments, int vectorFieldId) { + private List createFullTextIndexSegmentSplits(LanceTableMetadata metadata, + Map visibleFragments) throws UserException { + TFullTextSearchParams fullText = + externalSearchRequest.getSearchQuery().getFullTextSearch(); + if (searchFieldId < 0) { + throw new UserException("Lance full-text column '" + fullText.getColumn() + + "' has no field ID in the Lance schema"); + } + List matchingSegments = selectFullTextIndexSegments( + metadata.getIndexSegments(), searchFieldId, fullText.getColumn()); + if (matchingSegments.isEmpty()) { + throw new UserException("No committed Lance FTS index exists for column '" + + fullText.getColumn() + "' at dataset version " + metadata.getVersion()); + } + IndexSegmentSplitPlan plan = planIndexSegments( + metadata, matchingSegments, visibleFragments, true) + .orElseThrow(() -> new UserException("Lance FTS index for column '" + + fullText.getColumn() + "' has no visible indexed fragments at dataset version " + + metadata.getVersion())); + plannedIndexSegments = plan.splitCount(); + plannedIndexFragments = plan.indexSegmentFragmentCount(); + plannedUnindexedFragments = plannedFragments - plannedIndexFragments; + if (fullText.getCoverageMode() == TFtsCoverageMode.STRICT + && plannedUnindexedFragments != 0) { + throw new UserException("Lance FTS coverage_mode=STRICT requires every fragment at " + + "dataset version " + metadata.getVersion() + " to be indexed; column '" + + fullText.getColumn() + "' has " + plannedUnindexedFragments + + " unindexed fragments. Rebuild the index or use coverage_mode=index_only"); + } + return plan.buildSplits(); + } + + private static List selectVectorIndexSegments( + List indexSegments, int fieldId) { List selectedSegments = new ArrayList<>(); String selectedIndexName = null; for (LanceIndexSegmentInfo segment : indexSegments) { - if (!segment.getFieldIds().contains(vectorFieldId)) { + if (!segment.isVectorIndex() || !segment.getFieldIds().contains(fieldId)) { continue; } if (selectedIndexName == null) { @@ -284,19 +346,52 @@ private static List selectIndexSegments( return selectedSegments; } + private static List selectFullTextIndexSegments( + List indexSegments, int fieldId, String column) + throws UserException { + List selectedSegments = new ArrayList<>(); + String selectedIndexName = null; + for (LanceIndexSegmentInfo segment : indexSegments) { + if (!segment.isFullTextIndex() || !segment.getFieldIds().contains(fieldId)) { + continue; + } + if (selectedIndexName == null) { + selectedIndexName = segment.getIndexName(); + } else if (!selectedIndexName.equals(segment.getIndexName())) { + throw new UserException("Multiple Lance FTS indexes exist for column '" + column + + "'; distributed FTS requires one unambiguous logical index"); + } + selectedSegments.add(segment); + } + return selectedSegments; + } + private static Optional planIndexSegments( LanceTableMetadata metadata, List indexSegments, - Map visibleFragments) { + Map visibleFragments, + boolean requireKnownCoverage) throws UserException { IndexSegmentSplitPlan plan = new IndexSegmentSplitPlan( metadata.getDatasetUri(), metadata.getVersion(), indexSegments.size()); for (LanceIndexSegmentInfo segment : indexSegments) { Optional> segmentFragments = segment.getFragmentIds(); if (!segmentFragments.isPresent()) { + if (requireKnownCoverage) { + throw new UserException("Lance FTS segment " + segment.getUuid() + + " has no fragment coverage metadata"); + } return Optional.empty(); } List visibleIndexSegmentFragmentIds = effectiveFragmentIds( segmentFragments.get(), visibleFragments); + if (requireKnownCoverage) { + for (Long fragmentId : visibleIndexSegmentFragmentIds) { + if (plan.isCoveredByIndexSegment(fragmentId)) { + throw new UserException("Lance FTS fragment " + fragmentId + + " is covered by multiple physical index segments"); + } + } + } if (!visibleIndexSegmentFragmentIds.isEmpty()) { plan.addIndexSegmentSplit( segment.getUuid(), visibleIndexSegmentFragmentIds, @@ -339,10 +434,13 @@ private static void appendUnindexedFragmentSplits(IndexSegmentSplitPlan plan, } } - private boolean shouldUseIndex() { - return !externalSearchRequest.isSetVectorSearchOptions() - || !externalSearchRequest.getVectorSearchOptions().isSetUseIndex() - || externalSearchRequest.getVectorSearchOptions().isUseIndex(); + private boolean isVectorIndexEnabled() { + // default use_index is true + if (!externalSearchRequest.isSetVectorSearchOptions()) { + return true; + } + TVectorSearchOptions options = externalSearchRequest.getVectorSearchOptions(); + return !options.isSetUseIndex() || options.isUseIndex(); } private static boolean metricMatches(TVectorSearchParams vector, @@ -372,11 +470,15 @@ protected void setScanParams(TFileRangeDesc rangeDesc, Split split) { if (lanceSplit.getFragmentIds().isEmpty()) { throw new IllegalArgumentException("Lance scan split must contain fragments"); } - if (!isExternalSearch() && (lanceSplit.getFragmentIds().size() != 1 + if (searchKind == SearchKind.NORMAL && (lanceSplit.getFragmentIds().size() != 1 || lanceSplit.hasIndexSegmentUuids())) { throw new IllegalArgumentException( "Ordinary Lance scan split must contain one fragment and no index segment"); } + if (searchKind == SearchKind.FULL_TEXT && !lanceSplit.hasIndexSegmentUuids()) { + throw new IllegalArgumentException( + "Lance full-text search split must contain an FTS index segment"); + } lanceParams.setFragmentIds(lanceSplit.getFragmentIds()); if (lanceSplit.hasIndexSegmentUuids()) { List uuids = new ArrayList<>(lanceSplit.getIndexSegmentUuids().size()); @@ -390,8 +492,8 @@ protected void setScanParams(TFileRangeDesc rangeDesc, Split split) { lanceParams.setIndexSegmentUuids(uuids); } // Push LIMIT into each ordinary fragment scanner only when it is safe to truncate that - // fragment early. Vector search uses its own per-split candidate bound. - if (!isExternalSearch() && canPushDownLimit()) { + // fragment early. External searches use their own per-split candidate bound. + if (searchKind == SearchKind.NORMAL && canPushDownLimit()) { lanceParams.setLimit(getLimit()); } @@ -413,7 +515,7 @@ protected List getPathPartitionKeys() { @Override protected TableIf getTargetTable() { - if (isExternalSearch()) { + if (searchKind != SearchKind.NORMAL) { // In search mode desc.getTable() is a FunctionGenTable, but default-value expressions // and storage access still belong to the underlying Lance table. return lanceTable; @@ -432,13 +534,39 @@ protected Map getLocationProperties() { @Override public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { StringBuilder result = new StringBuilder(super.getNodeExplainString(prefix, detailLevel)); - if (isExternalSearch()) { - TVectorSearchParams vector = externalSearchRequest.getSearchQuery().getVectorSearch(); - result.append(prefix).append("externalSearchType=VECTOR\n"); - result.append(prefix).append("lanceVectorColumn=").append(vector.getColumn()).append("\n"); - result.append(prefix).append("lanceMetric=") - .append(vector.isSetMetric() ? metricName(vector.getMetric()) : "default") - .append("\n"); + if (searchKind != SearchKind.NORMAL) { + if (searchKind == SearchKind.VECTOR) { + TVectorSearchParams vector = + externalSearchRequest.getSearchQuery().getVectorSearch(); + result.append(prefix).append("externalSearchType=VECTOR\n"); + result.append(prefix).append("lanceVectorColumn=") + .append(vector.getColumn()).append("\n"); + result.append(prefix).append("lanceMetric=") + .append(vector.isSetMetric() ? metricName(vector.getMetric()) : "default") + .append("\n"); + } else { + if (searchKind != SearchKind.FULL_TEXT) { + throw new IllegalStateException("Unsupported Lance search kind " + searchKind); + } + TFullTextSearchParams fullText = + externalSearchRequest.getSearchQuery().getFullTextSearch(); + result.append(prefix).append("externalSearchType=FULL_TEXT\n"); + result.append(prefix).append("lanceFullTextColumn=") + .append(fullText.getColumn()).append("\n"); + result.append(prefix).append("lanceFtsCoverageMode=") + .append(fullText.getCoverageMode()).append("\n"); + result.append(prefix).append("lanceFtsQueryType=") + .append(fullText.getQueryType()).append("\n"); + if (fullText.getQueryType() == TFtsQueryType.MATCH) { + result.append(prefix).append("lanceFtsMatchOperator=") + .append(fullText.getMatchOperator()).append("\n"); + result.append(prefix).append("lanceFtsMaxFuzzyDistance=") + .append(fullText.getMaxFuzzyDistance()).append("\n"); + } else { + result.append(prefix).append("lanceFtsPhraseSlop=") + .append(fullText.getPhraseSlop()).append("\n"); + } + } result.append(prefix).append("lanceVersion=") .append(plannedMetadata.getVersion()).append("\n"); result.append(prefix).append("lanceSearchFragments=") @@ -465,19 +593,40 @@ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { return result.toString(); } - private boolean isExternalSearch() { - return externalSearchRequest != null; + TExternalSearchRequest createSplitSearchRequest() { + TExternalSearchRequest splitRequest = externalSearchRequest.deepCopy(); + // Every split must retain enough rows for the later global OFFSET/LIMIT. Applying the + // logical offset independently inside each split could discard rows that belong to the + // snapshot-wide result. + switch (searchKind) { + case VECTOR: + TVectorSearchParams vector = splitRequest.getSearchQuery().getVectorSearch(); + vector.setTopK(vector.getTopK() + vector.getOffset()); + vector.setOffset(0); + break; + case FULL_TEXT: + TFullTextSearchParams fullText = splitRequest.getSearchQuery().getFullTextSearch(); + fullText.setTopK(fullText.getTopK() + fullText.getOffset()); + fullText.setOffset(0); + break; + case NORMAL: + default: + throw new IllegalStateException("Cannot create a search split for " + searchKind); + } + return splitRequest; } - static TExternalSearchRequest createFragmentSearchRequest(TExternalSearchRequest searchRequest) { - TExternalSearchRequest fragmentRequest = searchRequest.deepCopy(); - TVectorSearchParams vector = fragmentRequest.getSearchQuery().getVectorSearch(); - // Every fragment must retain enough rows for the later global OFFSET/LIMIT. Applying the - // logical offset independently inside each fragment could discard rows that belong to the - // snapshot-wide result. - vector.setTopK(vector.getTopK() + vector.getOffset()); - vector.setOffset(0); - return fragmentRequest; + private static SearchKind resolveSearchKind(TExternalSearchRequest searchRequest) { + if (!searchRequest.isSetSearchQuery()) { + throw new IllegalArgumentException("Lance external search request requires search_query"); + } + boolean hasVector = searchRequest.getSearchQuery().isSetVectorSearch(); + boolean hasFullText = searchRequest.getSearchQuery().isSetFullTextSearch(); + if (hasVector == hasFullText) { + throw new IllegalArgumentException( + "Lance external search query must set exactly one search kind"); + } + return hasVector ? SearchKind.VECTOR : SearchKind.FULL_TEXT; } private static String metricName(TVectorMetric metric) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java index feed5e456ac663..702bc48614712f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java @@ -38,6 +38,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalTVFRelation; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; import org.apache.doris.qe.SessionVariable; +import org.apache.doris.tablefunction.FullTextSearchTableValuedFunction; import org.apache.doris.tablefunction.VectorSearchTableValuedFunction; import com.google.common.collect.ImmutableSet; @@ -129,7 +130,7 @@ boolean checkRelationTableSupportedType(PhysicalCatalogRelation relation) { } boolean checkTVFRelationTableSupportedType(PhysicalTVFRelation tvfRelation) { - if (isVectorSearch(tvfRelation)) { + if (isLanceExternalSearch(tvfRelation)) { return true; } @@ -146,8 +147,10 @@ boolean checkTVFRelationTableSupportedType(PhysicalTVFRelation tvfRelation) { return false; } - private boolean isVectorSearch(PhysicalTVFRelation tvfRelation) { - return VectorSearchTableValuedFunction.NAME.equals(tvfRelation.getFunction().getName()); + private boolean isLanceExternalSearch(PhysicalTVFRelation tvfRelation) { + String functionName = tvfRelation.getFunction().getName(); + return VectorSearchTableValuedFunction.NAME.equals(functionName) + || FullTextSearchTableValuedFunction.NAME.equals(functionName); } @Override @@ -189,7 +192,7 @@ public Optional visitPhysicalTVFRelation( PhysicalTVFRelation tvfRelation, ProbeContext context) { // The first Lance implementation fetches top-level columns by row ID. Keep nested // sub-column projections in the search phase until take_rows supports access paths. - if (isVectorSearch(tvfRelation) && context.slot.hasSubColPath()) { + if (isLanceExternalSearch(tvfRelation) && context.slot.hasSubColPath()) { return Optional.empty(); } if (checkTVFRelationTableSupportedType(tvfRelation) && tvfRelation.getOutput().contains(context.slot) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java index 41cce5539a30d3..d145e09c875e4e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java @@ -68,6 +68,7 @@ import org.apache.doris.nereids.trees.expressions.functions.generator.Unnest; import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; import org.apache.doris.nereids.trees.expressions.functions.scalar.GroupingScalarFunction; +import org.apache.doris.nereids.trees.expressions.functions.table.FullTextSearch; import org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction; import org.apache.doris.nereids.trees.expressions.functions.table.VectorSearch; import org.apache.doris.nereids.trees.expressions.literal.IntegerLikeLiteral; @@ -116,6 +117,7 @@ import org.apache.doris.nereids.util.TypeCoercionUtils; import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.SqlModeHelper; +import org.apache.doris.tablefunction.FullTextSearchTableValuedFunction; import org.apache.doris.tablefunction.VectorSearchTableValuedFunction; import com.google.common.base.Joiner; @@ -1788,7 +1790,8 @@ private Plan bindTableValuedFunction(MatchingContext ctx) { TableValuedFunction tableValuedFunction = (TableValuedFunction) bindResult.first; LogicalTVFRelation relation = new LogicalTVFRelation( unboundTVFRelation.getRelationId(), tableValuedFunction, ImmutableList.of()); - if (!(tableValuedFunction instanceof VectorSearch)) { + if (!(tableValuedFunction instanceof VectorSearch) + && !(tableValuedFunction instanceof FullTextSearch)) { return relation; } @@ -1796,17 +1799,31 @@ private Plan bindTableValuedFunction(MatchingContext ctx) { // relation with a Doris TopN to merge them into the snapshot-wide result. The predicate // pushdown rules move an outer WHERE below this synthetic TopN, where it is evaluated as // a Doris scan residual after each fragment's Lance search and before the global TopN. - VectorSearchTableValuedFunction vectorSearch = - (VectorSearchTableValuedFunction) tableValuedFunction.getCatalogFunction(); - Slot distance = relation.getOutput().stream() - .filter(slot -> slot.getName().equalsIgnoreCase( - VectorSearchTableValuedFunction.DISTANCE_COLUMN)) + if (tableValuedFunction instanceof VectorSearch) { + VectorSearchTableValuedFunction vectorSearch = + (VectorSearchTableValuedFunction) tableValuedFunction.getCatalogFunction(); + Slot distance = requireSearchResultSlot( + relation, VectorSearchTableValuedFunction.DISTANCE_COLUMN, + VectorSearchTableValuedFunction.NAME); + return new LogicalTopN<>(ImmutableList.of(new OrderKey(distance, true, false)), + vectorSearch.getTopK(), vectorSearch.getOffset(), relation); + } + FullTextSearchTableValuedFunction fullTextSearch = + (FullTextSearchTableValuedFunction) tableValuedFunction.getCatalogFunction(); + Slot score = requireSearchResultSlot( + relation, FullTextSearchTableValuedFunction.SCORE_COLUMN, + FullTextSearchTableValuedFunction.NAME); + return new LogicalTopN<>(ImmutableList.of(new OrderKey(score, false, false)), + fullTextSearch.getTopK(), fullTextSearch.getOffset(), relation); + } + + private Slot requireSearchResultSlot( + LogicalTVFRelation relation, String column, String functionName) { + return relation.getOutput().stream() + .filter(slot -> slot.getName().equalsIgnoreCase(column)) .findFirst() - .orElseThrow(() -> new AnalysisException("vector_search() output is missing '" - + VectorSearchTableValuedFunction.DISTANCE_COLUMN + "'")); - OrderKey distanceAscending = new OrderKey(distance, true, false); - return new LogicalTopN<>(ImmutableList.of(distanceAscending), - vectorSearch.getTopK(), vectorSearch.getOffset(), relation); + .orElseThrow(() -> new AnalysisException(functionName + "() output is missing '" + + column + "'")); } private void checkIfOutputAliasNameDuplicatedForGroupBy(Collection expressions, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughVectorSearchTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughVectorSearchTopN.java index f6c7feed6dbb7a..73f65e5a5ed8be 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughVectorSearchTopN.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughVectorSearchTopN.java @@ -23,24 +23,25 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalTopN; +import org.apache.doris.tablefunction.FullTextSearchTableValuedFunction; import org.apache.doris.tablefunction.VectorSearchTableValuedFunction; /** - * Move an outer vector_search WHERE predicate below its Doris merge TopN. + * Move an outer Lance external-search WHERE predicate below its Doris merge TopN. * - *

The TopN immediately above a vector_search TVF is added by {@code BindExpression} to merge + *

The TopN immediately above a search TVF is added by {@code BindExpression} to merge * the candidates returned by all Lance fragment scans. The SQL WHERE predicate must therefore be * evaluated below this TopN so it can become a residual conjunct on the Doris Lance scan node: * * 

  * Filter                         TopN
  *   TopN            ->            Filter
- *     vector_search                 vector_search
+ *     search TVF                    search TVF
  *
* - *

This remains a postfilter relative to Lance nearest(): every fragment first returns its ANN - * candidates, and Doris filters those candidates before the local/global TopN. It is deliberately - * not converted into the Lance prefilter carried by the TVF's {@code filter} property. + *

This remains a postfilter relative to the Lance search: every split first returns candidates, + * and Doris filters those candidates before the local/global TopN. It is deliberately not + * converted into the Lance prefilter carried by the TVF's {@code filter} property. */ public class PushDownFilterThroughVectorSearchTopN extends OneRewriteRuleFactory { @Override @@ -48,8 +49,9 @@ public Rule build() { return logicalFilter(logicalTopN(logicalTVFRelation())) .then(filter -> { LogicalTopN topN = filter.child(); - if (!VectorSearchTableValuedFunction.NAME.equals( - topN.child().getFunction().getName())) { + String functionName = topN.child().getFunction().getName(); + if (!VectorSearchTableValuedFunction.NAME.equals(functionName) + && !FullTextSearchTableValuedFunction.NAME.equals(functionName)) { return null; } LogicalFilter scanFilter = new LogicalFilter<>( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/FullTextSearch.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/FullTextSearch.java new file mode 100644 index 00000000000000..55b297ae6688c7 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/FullTextSearch.java @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.table; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Properties; +import org.apache.doris.nereids.types.coercion.AnyDataType; +import org.apache.doris.tablefunction.FullTextSearchTableValuedFunction; +import org.apache.doris.tablefunction.TableValuedFunctionIf; + +import java.util.Map; + +/** Lance full_text_search relation TVF. */ +public class FullTextSearch extends TableValuedFunction { + public FullTextSearch(Properties properties) { + super(FullTextSearchTableValuedFunction.NAME, properties); + } + + @Override + public FunctionSignature customSignature() { + return FunctionSignature.of(AnyDataType.INSTANCE_WITHOUT_INDEX, getArgumentsTypes()); + } + + @Override + protected TableValuedFunctionIf toCatalogFunction() { + try { + Map arguments = getTVFProperties().getMap(); + return new FullTextSearchTableValuedFunction(arguments); + } catch (Throwable t) { + throw new AnalysisException("Can not build full_text_search(): " + t.getMessage(), t); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/FullTextSearchTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/FullTextSearchTableValuedFunction.java new file mode 100644 index 00000000000000..3acf3adbd5f1d9 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/FullTextSearchTableValuedFunction.java @@ -0,0 +1,177 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.tablefunction; + +import org.apache.doris.common.AnalysisException; +import org.apache.doris.datasource.lance.LanceTableMetadata; +import org.apache.doris.thrift.TExternalSearchQuery; +import org.apache.doris.thrift.TExternalSearchRequest; +import org.apache.doris.thrift.TFtsCoverageMode; +import org.apache.doris.thrift.TFtsMatchOperator; +import org.apache.doris.thrift.TFtsQueryType; +import org.apache.doris.thrift.TFullTextSearchParams; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableSet; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; + +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** Relation TVF for a fixed-snapshot distributed Lance full-text search. */ +public class FullTextSearchTableValuedFunction extends LanceExternalSearchTableValuedFunction { + public static final String NAME = "full_text_search"; + public static final String SCORE_COLUMN = "_score"; + + private static final String QUERY = "query"; + private static final String COVERAGE_MODE = "coverage_mode"; + private static final String QUERY_TYPE = "query_type"; + private static final String OPERATOR = "operator"; + private static final String MAX_FUZZY_DISTANCE = "max_fuzzy_distance"; + private static final String SLOP = "slop"; + private static final Set PROPERTIES = ImmutableSet.of( + TABLE, COLUMN, QUERY, TOP_K, OFFSET, FILTER, COVERAGE_MODE, + QUERY_TYPE, OPERATOR, MAX_FUZZY_DISTANCE, SLOP); + + public FullTextSearchTableValuedFunction(Map properties) + throws AnalysisException { + super(prepare(properties)); + } + + private static PreparedSearch prepare(Map properties) + throws AnalysisException { + Map params = normalizeProperties(properties, PROPERTIES, NAME); + CommonSearch common = prepareCommon(params, NAME, + "FullTextSearchTableValuedFunction", "full-text search", true); + + Field field = findStringField(common.metadata(), required(params, COLUMN, NAME)); + int fieldId = requireLanceFieldId(common.metadata(), field, "full-text"); + String query = required(params, QUERY, NAME); + if (query.indexOf('\0') >= 0) { + throw new AnalysisException("'query' must not contain an embedded NUL byte"); + } + + TFtsQueryType queryType = parseQueryType(params.getOrDefault(QUERY_TYPE, "match")); + TFullTextSearchParams fullTextParams = new TFullTextSearchParams() + .setColumn(field.getName()) + .setQuery(query) + .setTopK(common.topK()) + .setOffset(common.offset()) + .setQueryType(queryType) + .setCoverageMode(parseCoverageMode( + params.getOrDefault(COVERAGE_MODE, "strict"))); + if (queryType == TFtsQueryType.MATCH) { + if (params.containsKey(SLOP)) { + throw new AnalysisException("'slop' is only valid for query_type=phrase"); + } + int maxFuzzyDistance = (int) parseLong( + params.getOrDefault(MAX_FUZZY_DISTANCE, "0"), + MAX_FUZZY_DISTANCE, 0, Integer.MAX_VALUE); + if (maxFuzzyDistance != 0) { + throw new AnalysisException("'max_fuzzy_distance' must be 0 because the bundled " + + "Lance-C does not yet support prepared fuzzy matching"); + } + fullTextParams.setMatchOperator( + parseMatchOperator(params.getOrDefault(OPERATOR, "or"))) + .setMaxFuzzyDistance(maxFuzzyDistance); + } else { + if (params.containsKey(OPERATOR)) { + throw new AnalysisException("'operator' is only valid for query_type=match"); + } + if (params.containsKey(MAX_FUZZY_DISTANCE)) { + throw new AnalysisException( + "'max_fuzzy_distance' is only valid for query_type=match"); + } + int slop = (int) parseLong( + params.getOrDefault(SLOP, "0"), SLOP, 0, Integer.MAX_VALUE); + fullTextParams.setPhraseSlop(slop); + } + TExternalSearchRequest searchRequest = new TExternalSearchRequest() + .setSchemaVersion(1) + .setSearchQuery(TExternalSearchQuery.full_text_search(fullTextParams)); + return prepareSearch( + common, fieldId, searchRequest, SCORE_COLUMN, "full-text search"); + } + + @VisibleForTesting + static Field findStringField(LanceTableMetadata metadata, String column) + throws AnalysisException { + Field match = null; + for (Field field : metadata.getSchema().getFields()) { + if (field.getName().equalsIgnoreCase(column)) { + if (match != null) { + throw new AnalysisException("Lance full-text column '" + column + + "' is ambiguous under case-insensitive matching"); + } + match = field; + } + } + if (match == null) { + throw new AnalysisException("Lance full-text column '" + column + "' does not exist"); + } + ArrowType.ArrowTypeID typeId = match.getType().getTypeID(); + if (typeId != ArrowType.ArrowTypeID.Utf8 + && typeId != ArrowType.ArrowTypeID.LargeUtf8) { + throw new AnalysisException("Lance full-text column '" + match.getName() + + "' must be STRING"); + } + return match; + } + + @VisibleForTesting + static TFtsCoverageMode parseCoverageMode(String value) throws AnalysisException { + switch (value.trim().toLowerCase(Locale.ROOT)) { + case "strict": + return TFtsCoverageMode.STRICT; + case "index_only": + case "index-only": + return TFtsCoverageMode.INDEX_ONLY; + default: + throw new AnalysisException("Unsupported FTS coverage_mode '" + value + + "': expected strict or index_only"); + } + } + + @VisibleForTesting + static TFtsQueryType parseQueryType(String value) throws AnalysisException { + switch (value.trim().toLowerCase(Locale.ROOT)) { + case "match": + return TFtsQueryType.MATCH; + case "phrase": + return TFtsQueryType.PHRASE; + default: + throw new AnalysisException("Unsupported FTS query_type '" + value + + "': expected match or phrase"); + } + } + + @VisibleForTesting + static TFtsMatchOperator parseMatchOperator(String value) throws AnalysisException { + switch (value.trim().toLowerCase(Locale.ROOT)) { + case "or": + return TFtsMatchOperator.OR; + case "and": + return TFtsMatchOperator.AND; + default: + throw new AnalysisException("Unsupported FTS operator '" + value + + "': expected 'or' or 'and'"); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LanceExternalSearchTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LanceExternalSearchTableValuedFunction.java new file mode 100644 index 00000000000000..856ee108fc6c9a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LanceExternalSearchTableValuedFunction.java @@ -0,0 +1,360 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.tablefunction; + +import org.apache.doris.analysis.TableName; +import org.apache.doris.analysis.TupleDescriptor; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; +import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.datasource.lance.LanceExternalCatalog; +import org.apache.doris.datasource.lance.LanceExternalTable; +import org.apache.doris.datasource.lance.LanceTableMetadata; +import org.apache.doris.datasource.lance.LanceTypeConverter; +import org.apache.doris.datasource.lance.source.LanceScanNode; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.analyzer.UnboundSlot; +import org.apache.doris.nereids.exceptions.ParseException; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.planner.PlanNodeId; +import org.apache.doris.planner.ScanNode; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; +import org.apache.doris.thrift.TExternalSearchRequest; +import org.apache.doris.thrift.TSearchFilter; +import org.apache.doris.thrift.TSearchFilterFormat; + +import org.apache.arrow.vector.types.pojo.Field; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.OptionalInt; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; + +/** Common immutable planning state and validation for Lance external-search relation TVFs. */ +abstract class LanceExternalSearchTableValuedFunction extends TableValuedFunctionIf { + protected static final String TABLE = "table"; + protected static final String COLUMN = "column"; + protected static final String TOP_K = "top_k"; + protected static final String OFFSET = "offset"; + protected static final String FILTER = "filter"; + + private static final String FULLY_QUALIFIED_TABLE_NAME_ERROR = + "'table' must be a fully qualified catalog.database.table name"; + private static final long UINT32_MAX = 0xFFFF_FFFFL; + + private final String displayName; + private final TableName sourceTableName; + private final LanceExternalTable sourceTable; + private final LanceTableMetadata metadata; + private final int fieldId; + private final TExternalSearchRequest searchRequest; + private final List columns; + private final long topK; + private final long offset; + + protected LanceExternalSearchTableValuedFunction(PreparedSearch prepared) { + CommonSearch common = prepared.common; + this.displayName = common.displayName; + this.sourceTableName = common.sourceTableName; + this.sourceTable = common.sourceTable; + this.metadata = common.metadata; + this.fieldId = prepared.fieldId; + this.searchRequest = prepared.searchRequest.deepCopy(); + this.columns = Collections.unmodifiableList(new ArrayList<>(prepared.columns)); + this.topK = common.topK; + this.offset = common.offset; + } + + public final LanceExternalTable getSourceTable() { + return sourceTable; + } + + public final LanceTableMetadata getMetadata() { + return metadata; + } + + public final TExternalSearchRequest getSearchRequest() { + return searchRequest.deepCopy(); + } + + public final long getTopK() { + return topK; + } + + public final long getOffset() { + return offset; + } + + @Override + public final String getTableName() { + return displayName + "<" + sourceTableName + ">"; + } + + @Override + public final List getTableColumns() { + return columns; + } + + @Override + public final ScanNode getScanNode(PlanNodeId id, TupleDescriptor desc, SessionVariable sv) { + return LanceScanNode.forExternalSearch( + id, desc, sourceTable, metadata, fieldId, searchRequest, sv); + } + + protected static Map normalizeProperties(Map properties, + Set allowedProperties, String functionName) throws AnalysisException { + Map normalized = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + for (Map.Entry entry : properties.entrySet()) { + String key = entry.getKey().toLowerCase(Locale.ROOT); + if (!allowedProperties.contains(key)) { + throw new AnalysisException("'" + entry.getKey() + + "' is an invalid property for " + functionName + "()"); + } + if (normalized.put(key, entry.getValue()) != null) { + throw new AnalysisException( + "Duplicate " + functionName + "() property '" + key + "'"); + } + } + return normalized; + } + + protected static String required(Map params, String key, String functionName) + throws AnalysisException { + String value = params.get(key); + if (value == null || value.trim().isEmpty()) { + throw new AnalysisException( + "Missing required " + functionName + "() property '" + key + "'"); + } + return value.trim(); + } + + protected static CommonSearch prepareCommon(Map params, String functionName, + String displayName, String searchDescription, boolean loadIndexMetadata) + throws AnalysisException { + TableName sourceTableName = parseTableName(required(params, TABLE, functionName)); + LanceExternalTable sourceTable = findLanceExternalTable(sourceTableName); + LanceTableMetadata metadata; + try { + metadata = loadIndexMetadata + ? sourceTable.loadMetadataForSearch() : sourceTable.loadMetadata(); + } catch (RuntimeException e) { + throw new AnalysisException("Failed to load Lance metadata for " + searchDescription + + " on " + sourceTableName + ": " + e.getMessage(), e); + } + if (metadata.getVersion() <= 0) { + throw new AnalysisException("Lance " + searchDescription + + " requires a fixed positive dataset version"); + } + + long topK = parseLong(params.getOrDefault(TOP_K, "10"), TOP_K, 1, Long.MAX_VALUE); + long offset = parseLong(params.getOrDefault(OFFSET, "0"), OFFSET, 0, Long.MAX_VALUE); + if (offset > UINT32_MAX || topK > UINT32_MAX - offset) { + throw new AnalysisException("'top_k + offset' must not exceed " + UINT32_MAX); + } + return new CommonSearch(params, displayName, sourceTableName, sourceTable, metadata, + topK, offset); + } + + protected static PreparedSearch prepareSearch(CommonSearch common, int fieldId, + TExternalSearchRequest searchRequest, String resultColumn, String searchDescription) + throws AnalysisException { + if (common.params.containsKey(FILTER)) { + searchRequest.setSearchFilter(new TSearchFilter() + .setFormat(TSearchFilterFormat.SQL) + .setPayload(validateAndEncodeSqlFilter(common.params.get(FILTER)))); + } + List columns = buildOutputColumns( + common.metadata, resultColumn, searchDescription); + return new PreparedSearch(common, fieldId, searchRequest, columns); + } + + protected static int requireLanceFieldId(LanceTableMetadata metadata, Field field, + String searchDescription) throws AnalysisException { + OptionalInt fieldId = metadata.getLanceFieldId(field.getName()); + if (!fieldId.isPresent()) { + throw new AnalysisException("Lance " + searchDescription + " column '" + + field.getName() + "' has no field ID in the Lance schema"); + } + return fieldId.getAsInt(); + } + + protected static List buildOutputColumns(LanceTableMetadata metadata, + String resultColumn, String searchDescription) throws AnalysisException { + List result = new ArrayList<>(metadata.getSchema().getFields().size() + 1); + Set fieldNames = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + int position = 0; + for (Field field : metadata.getSchema().getFields()) { + if (!fieldNames.add(field.getName())) { + throw new AnalysisException("Duplicate Lance schema column under " + + "case-insensitive matching: '" + field.getName() + "'"); + } + if (field.getName().startsWith(Column.GLOBAL_ROWID_COL)) { + throw new AnalysisException("Lance table contains column '" + field.getName() + + "' using reserved Doris internal column prefix '" + + Column.GLOBAL_ROWID_COL + "'"); + } + if (field.getName().equalsIgnoreCase(resultColumn)) { + throw new AnalysisException("Lance table already contains reserved " + + searchDescription + " column '" + resultColumn + "'"); + } + String comment = field.getMetadata() == null + ? null : field.getMetadata().get("comment"); + Type type; + try { + type = LanceTypeConverter.toDorisType(field); + } catch (RuntimeException e) { + throw new AnalysisException("Invalid Lance type for column '" + field.getName() + + "': " + e.getMessage(), e); + } + result.add(new Column(field.getName(), type, false, null, + field.isNullable(), comment, true, position++)); + } + result.add(new Column(resultColumn, Type.FLOAT, false, null, + true, null, true, position)); + return result; + } + + protected static TableName parseTableName(String value) throws AnalysisException { + Expression expression; + try { + expression = new NereidsParser().parseExpression(value); + } catch (ParseException e) { + throw new AnalysisException(FULLY_QUALIFIED_TABLE_NAME_ERROR, e); + } + if (!(expression instanceof UnboundSlot)) { + throw new AnalysisException(FULLY_QUALIFIED_TABLE_NAME_ERROR); + } + List names = ((UnboundSlot) expression).getNameParts(); + if (names.size() != 3) { + throw new AnalysisException(FULLY_QUALIFIED_TABLE_NAME_ERROR); + } + return new TableName(names.get(0), names.get(1), names.get(2)); + } + + protected static LanceExternalTable findLanceExternalTable(TableName tableName) + throws AnalysisException { + ConnectContext context = ConnectContext.get(); + if (!Env.getCurrentEnv().getAccessManager() + .checkTblPriv(context, tableName, PrivPredicate.SELECT)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "SELECT", + context.getQualifiedUser(), context.getRemoteIP(), + tableName.getDb() + ": " + tableName.getTbl()); + } + CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(tableName.getCtl()); + if (!(catalog instanceof LanceExternalCatalog)) { + throw new AnalysisException("Catalog '" + tableName.getCtl() + + "' is not a Lance catalog"); + } + TableIf table = catalog.getDbOrAnalysisException(tableName.getDb()) + .getTableOrAnalysisException(tableName.getTbl()); + if (!(table instanceof LanceExternalTable)) { + throw new AnalysisException("Table '" + tableName + "' is not a Lance table"); + } + return (LanceExternalTable) table; + } + + protected static byte[] validateAndEncodeSqlFilter(String filter) throws AnalysisException { + if (filter == null || filter.trim().isEmpty()) { + throw new AnalysisException("'filter' must not be empty"); + } + if (filter.indexOf('\0') >= 0) { + throw new AnalysisException("'filter' must not contain an embedded NUL byte"); + } + return filter.getBytes(StandardCharsets.UTF_8); + } + + protected static long parseLong(String value, String property, long min, long max) + throws AnalysisException { + try { + long parsed = Long.parseLong(value); + if (parsed < min || parsed > max) { + throw new AnalysisException("'" + property + "' must be between " + + min + " and " + max); + } + return parsed; + } catch (NumberFormatException e) { + throw new AnalysisException("'" + property + "' must be an integer", e); + } + } + + protected static final class CommonSearch { + private final Map params; + private final String displayName; + private final TableName sourceTableName; + private final LanceExternalTable sourceTable; + private final LanceTableMetadata metadata; + private final long topK; + private final long offset; + + private CommonSearch(Map params, String displayName, + TableName sourceTableName, LanceExternalTable sourceTable, + LanceTableMetadata metadata, long topK, long offset) { + this.params = Collections.unmodifiableMap(new TreeMap<>(params)); + this.displayName = displayName; + this.sourceTableName = sourceTableName; + this.sourceTable = sourceTable; + this.metadata = metadata; + this.topK = topK; + this.offset = offset; + } + + protected Map params() { + return params; + } + + protected LanceTableMetadata metadata() { + return metadata; + } + + protected long topK() { + return topK; + } + + protected long offset() { + return offset; + } + } + + protected static final class PreparedSearch { + private final CommonSearch common; + private final int fieldId; + private final TExternalSearchRequest searchRequest; + private final List columns; + + private PreparedSearch(CommonSearch common, int fieldId, + TExternalSearchRequest searchRequest, List columns) { + this.common = common; + this.fieldId = fieldId; + this.searchRequest = searchRequest; + this.columns = columns; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java index 7a7569583d5283..79c10b465bee27 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java @@ -104,6 +104,8 @@ public static TableValuedFunctionIf getTableFunction(String funcName, Map PROPERTIES = ImmutableSet.of( TABLE, COLUMN, QUERY_VECTOR, TOP_K, OFFSET, METRIC, FILTER, NPROBES, REFINE_FACTOR, EF, USE_INDEX); - private final TableName sourceTableName; - private final LanceExternalTable sourceTable; - private final LanceTableMetadata metadata; - private final int vectorFieldId; - private final List columns; - private final TExternalSearchRequest searchRequest; - public VectorSearchTableValuedFunction(Map properties) throws AnalysisException { - Map params = normalizeProperties(properties); - sourceTableName = parseTableName(required(params, TABLE)); - sourceTable = findLanceExternalTable(sourceTableName); + super(prepare(properties)); + } + + private static PreparedSearch prepare(Map properties) + throws AnalysisException { + Map params = normalizeProperties(properties, PROPERTIES, NAME); boolean useIndex = !params.containsKey(USE_INDEX) || parseBoolean(params.get(USE_INDEX), USE_INDEX); - try { - metadata = useIndex - ? sourceTable.loadMetadataForVectorSearch() : sourceTable.loadMetadata(); - } catch (RuntimeException e) { - throw new AnalysisException("Failed to load Lance metadata for vector search on " - + sourceTableName + ": " + e.getMessage(), e); - } - if (metadata.getVersion() <= 0) { - throw new AnalysisException("Lance vector search requires a fixed positive dataset version"); - } + CommonSearch common = prepareCommon(params, NAME, + "VectorSearchTableValuedFunction", "vector search", useIndex); Field vectorField = LanceVectorQuery.findVectorColumnField( - metadata.getSchema(), required(params, COLUMN)); - vectorFieldId = useIndex ? requireLanceFieldId(metadata, vectorField) : -1; + common.metadata().getSchema(), required(params, COLUMN, NAME)); + int vectorFieldId = useIndex + ? requireLanceFieldId(common.metadata(), vectorField) : -1; TSearchVector queryVector = LanceVectorQuery.parseAndEncodeQueryVector( - vectorField, required(params, QUERY_VECTOR)); - long topK = parseLong(params.getOrDefault(TOP_K, "10"), TOP_K, 1, Long.MAX_VALUE); - long offset = parseLong(params.getOrDefault(OFFSET, "0"), OFFSET, 0, Long.MAX_VALUE); - if (offset > UINT32_MAX || topK > UINT32_MAX - offset) { - throw new AnalysisException("'top_k + offset' must not exceed " + UINT32_MAX); - } + vectorField, required(params, QUERY_VECTOR, NAME)); TVectorSearchParams vectorParams = new TVectorSearchParams() .setColumn(vectorField.getName()) .setQueryVector(queryVector) - .setTopK(topK) - .setOffset(offset); + .setTopK(common.topK()) + .setOffset(common.offset()); if (params.containsKey(METRIC)) { vectorParams.setMetric(parseMetric(params.get(METRIC))); } - searchRequest = new TExternalSearchRequest() + TExternalSearchRequest searchRequest = new TExternalSearchRequest() .setSchemaVersion(1) .setSearchQuery(TExternalSearchQuery.vector_search(vectorParams)); - if (params.containsKey(FILTER)) { - searchRequest.setSearchFilter(new TSearchFilter() - .setFormat(TSearchFilterFormat.SQL) - .setPayload(validateAndEncodeSqlFilter(params.get(FILTER)))); + TVectorSearchOptions vectorSearchOptions = buildVectorSearchOptions(params, useIndex); + if (vectorSearchOptions != null) { + searchRequest.setVectorSearchOptions(vectorSearchOptions); } + return prepareSearch( + common, vectorFieldId, searchRequest, DISTANCE_COLUMN, "vector search"); + } - TVectorSearchOptions vectorSearchOptions = new TVectorSearchOptions(); - boolean hasVectorSearchOptions = false; + private static TVectorSearchOptions buildVectorSearchOptions( + Map params, boolean useIndex) throws AnalysisException { + TVectorSearchOptions options = new TVectorSearchOptions(); + boolean configured = false; if (params.containsKey(NPROBES)) { - vectorSearchOptions.setNprobes(parsePositiveInt(params.get(NPROBES), NPROBES)); - hasVectorSearchOptions = true; + options.setNprobes(parsePositiveInt(params.get(NPROBES), NPROBES)); + configured = true; } if (params.containsKey(REFINE_FACTOR)) { - vectorSearchOptions.setRefineFactor( + options.setRefineFactor( parsePositiveInt(params.get(REFINE_FACTOR), REFINE_FACTOR)); - hasVectorSearchOptions = true; + configured = true; } if (params.containsKey(EF)) { - vectorSearchOptions.setEf(parsePositiveInt(params.get(EF), EF)); - hasVectorSearchOptions = true; + options.setEf(parsePositiveInt(params.get(EF), EF)); + configured = true; } if (params.containsKey(USE_INDEX)) { - vectorSearchOptions.setUseIndex(useIndex); - hasVectorSearchOptions = true; - } - if (hasVectorSearchOptions) { - searchRequest.setVectorSearchOptions(vectorSearchOptions); - } - columns = buildOutputColumns(metadata); - } - - public LanceExternalTable getSourceTable() { - return sourceTable; - } - - public LanceTableMetadata getMetadata() { - return metadata; - } - - public TExternalSearchRequest getSearchRequest() { - return searchRequest.deepCopy(); - } - - public long getTopK() { - return searchRequest.getSearchQuery().getVectorSearch().getTopK(); - } - - public long getOffset() { - return searchRequest.getSearchQuery().getVectorSearch().getOffset(); - } - - @Override - public String getTableName() { - return "VectorSearchTableValuedFunction<" + sourceTableName + ">"; - } - - @Override - public List getTableColumns() { - return columns; - } - - @Override - public ScanNode getScanNode(PlanNodeId id, TupleDescriptor desc, SessionVariable sv) { - return LanceScanNode.forVectorSearch(id, desc, sourceTable, metadata, - vectorFieldId, searchRequest, sv); - } - - private static Map normalizeProperties(Map properties) - throws AnalysisException { - Map normalized = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); - for (Map.Entry entry : properties.entrySet()) { - String key = entry.getKey().toLowerCase(Locale.ROOT); - if (!PROPERTIES.contains(key)) { - throw new AnalysisException("'" + entry.getKey() - + "' is an invalid property for vector_search()"); - } - if (normalized.put(key, entry.getValue()) != null) { - throw new AnalysisException("Duplicate vector_search() property '" + key + "'"); - } - } - return normalized; - } - - private static String required(Map params, String key) - throws AnalysisException { - String value = params.get(key); - if (value == null || value.trim().isEmpty()) { - throw new AnalysisException("Missing required vector_search() property '" + key + "'"); + options.setUseIndex(useIndex); + configured = true; } - return value.trim(); - } - - @VisibleForTesting - static TableName parseTableName(String value) throws AnalysisException { - Expression expression; - try { - expression = new NereidsParser().parseExpression(value); - } catch (ParseException e) { - throw new AnalysisException(FULLY_QUALIFIED_TABLE_NAME_ERROR, e); - } - if (!(expression instanceof UnboundSlot)) { - throw new AnalysisException(FULLY_QUALIFIED_TABLE_NAME_ERROR); - } - List names = ((UnboundSlot) expression).getNameParts(); - if (names.size() != 3) { - throw new AnalysisException(FULLY_QUALIFIED_TABLE_NAME_ERROR); - } - return new TableName(names.get(0), names.get(1), names.get(2)); - } - - private static LanceExternalTable findLanceExternalTable(TableName tableName) - throws AnalysisException { - ConnectContext context = ConnectContext.get(); - if (!Env.getCurrentEnv().getAccessManager() - .checkTblPriv(context, tableName, PrivPredicate.SELECT)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "SELECT", - context.getQualifiedUser(), context.getRemoteIP(), - tableName.getDb() + ": " + tableName.getTbl()); - } - CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(tableName.getCtl()); - if (!(catalog instanceof LanceExternalCatalog)) { - throw new AnalysisException("Catalog '" + tableName.getCtl() - + "' is not a Lance catalog"); - } - TableIf table = catalog.getDbOrAnalysisException(tableName.getDb()) - .getTableOrAnalysisException(tableName.getTbl()); - if (!(table instanceof LanceExternalTable)) { - throw new AnalysisException("Table '" + tableName + "' is not a Lance table"); - } - return (LanceExternalTable) table; + return configured ? options : null; } @VisibleForTesting static List buildOutputColumns(LanceTableMetadata metadata) throws AnalysisException { - List result = new ArrayList<>(metadata.getSchema().getFields().size() + 1); - Set fieldNames = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); - int position = 0; - for (Field field : metadata.getSchema().getFields()) { - if (!fieldNames.add(field.getName())) { - throw new AnalysisException("Duplicate Lance schema column under " - + "case-insensitive matching: '" + field.getName() + "'"); - } - if (field.getName().startsWith(Column.GLOBAL_ROWID_COL)) { - throw new AnalysisException("Lance table contains column '" + field.getName() - + "' using reserved Doris internal column prefix '" - + Column.GLOBAL_ROWID_COL + "'"); - } - if (field.getName().equalsIgnoreCase(DISTANCE_COLUMN)) { - throw new AnalysisException("Lance table already contains reserved vector search " - + "column '" + DISTANCE_COLUMN + "'"); - } - String comment = field.getMetadata() == null - ? null : field.getMetadata().get("comment"); - Type type; - try { - type = LanceTypeConverter.toDorisType(field); - } catch (RuntimeException e) { - throw new AnalysisException("Invalid Lance type for column '" + field.getName() - + "': " + e.getMessage(), e); - } - result.add(new Column(field.getName(), type, false, null, - field.isNullable(), comment, true, position++)); - } - result.add(new Column(DISTANCE_COLUMN, Type.FLOAT, false, null, - true, null, true, position)); - return result; + return buildOutputColumns(metadata, DISTANCE_COLUMN, "vector search"); } @VisibleForTesting static int requireLanceFieldId(LanceTableMetadata metadata, Field field) throws AnalysisException { - OptionalInt fieldId = metadata.getLanceFieldId(field.getName()); - if (!fieldId.isPresent()) { - throw new AnalysisException("Lance vector column '" + field.getName() - + "' has no field ID in the Lance schema"); - } - return fieldId.getAsInt(); - } - - @VisibleForTesting - static byte[] validateAndEncodeSqlFilter(String filter) throws AnalysisException { - if (filter == null || filter.trim().isEmpty()) { - throw new AnalysisException("'filter' must not be empty"); - } - if (filter.indexOf('\0') >= 0) { - throw new AnalysisException("'filter' must not contain an embedded NUL byte"); - } - return filter.getBytes(StandardCharsets.UTF_8); - } - - private static long parseLong(String value, String property, long min, long max) - throws AnalysisException { - try { - long parsed = Long.parseLong(value); - if (parsed < min || parsed > max) { - throw new AnalysisException("'" + property + "' must be between " - + min + " and " + max); - } - return parsed; - } catch (NumberFormatException e) { - throw new AnalysisException("'" + property + "' must be an integer", e); - } + return requireLanceFieldId(metadata, field, "vector"); } private static int parsePositiveInt(String value, String property) diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java index 2fb3b7c390df6f..ddffce15bdf293 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java @@ -30,6 +30,10 @@ import org.apache.doris.thrift.TExternalSearchQuery; import org.apache.doris.thrift.TExternalSearchRequest; import org.apache.doris.thrift.TFileRangeDesc; +import org.apache.doris.thrift.TFtsCoverageMode; +import org.apache.doris.thrift.TFtsMatchOperator; +import org.apache.doris.thrift.TFtsQueryType; +import org.apache.doris.thrift.TFullTextSearchParams; import org.apache.doris.thrift.TVectorMetric; import org.apache.doris.thrift.TVectorSearchOptions; import org.apache.doris.thrift.TVectorSearchParams; @@ -39,6 +43,7 @@ import org.apache.arrow.vector.types.pojo.Schema; import org.junit.Assert; import org.junit.Test; +import org.lance.index.IndexType; import java.nio.ByteBuffer; import java.util.Arrays; @@ -203,9 +208,11 @@ public void testExternalSearchUsesOneSplitPerIndexSegmentAndKeepsUnindexedFragme Collections.singletonMap("vector", 9), Arrays.asList( new LanceIndexSegmentInfo(firstSegment, "vector_idx", - Collections.singletonList(9), Arrays.asList(1L, 2L), "L2"), + Collections.singletonList(9), Arrays.asList(1L, 2L), + IndexType.VECTOR, "L2"), new LanceIndexSegmentInfo(secondSegment, "vector_idx", - Collections.singletonList(9), Arrays.asList(3L, 4L), "L2")), + Collections.singletonList(9), Arrays.asList(3L, 4L), + IndexType.VECTOR, "L2")), Collections.emptyMap()); LanceScanNode node = newSearchNode(metadata, vectorSearchRequest(5, 0)); @@ -238,7 +245,8 @@ public void testExternalSearchUseIndexFalseKeepsFragmentSplits() throws Exceptio Collections.emptyMap(), Collections.singletonList( new LanceIndexSegmentInfo(UUID.randomUUID(), "vector_idx", - Collections.singletonList(9), Arrays.asList(1L, 2L), "L2")), + Collections.singletonList(9), Arrays.asList(1L, 2L), + IndexType.VECTOR, "L2")), Collections.emptyMap()); TExternalSearchRequest request = vectorSearchRequest(5, 0); request.setVectorSearchOptions(new TVectorSearchOptions().setUseIndex(false)); @@ -251,6 +259,116 @@ public void testExternalSearchUseIndexFalseKeepsFragmentSplits() throws Exceptio assertSplit(splits.get(1), 2, 8, 88); } + @Test + public void testFullTextSearchUsesOneSplitPerCommittedIndexSegment() throws Exception { + UUID firstSegment = UUID.fromString("11111111-2222-3333-4444-555555555555"); + UUID secondSegment = UUID.fromString("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"); + LanceTableMetadata metadata = LanceTableMetadata.withIndexSegments( + "s3://bucket/table.lance", + 42, + fullTextSchema(), + Arrays.asList( + new LanceFragmentInfo(1, 8, 8), + new LanceFragmentInfo(2, 7, 7), + new LanceFragmentInfo(3, 6, 6)), + Collections.singletonMap("body", 7), + Arrays.asList( + new LanceIndexSegmentInfo(firstSegment, "body_fts", + Collections.singletonList(7), Arrays.asList(1L, 2L), + IndexType.INVERTED, null), + new LanceIndexSegmentInfo(secondSegment, "body_fts", + Collections.singletonList(7), Collections.singletonList(3L), + IndexType.INVERTED, null)), + Collections.emptyMap()); + TExternalSearchRequest request = fullTextSearchRequest( + 5, 2, TFtsCoverageMode.STRICT); + LanceScanNode node = newSearchNode(metadata, request); + + List splits = node.getSplits(8); + + Assert.assertEquals(2, splits.size()); + assertIndexSplit(splits.get(0), firstSegment, Arrays.asList(1L, 2L), 15, 100); + assertIndexSplit(splits.get(1), secondSegment, Collections.singletonList(3L), 15, 40); + + TFileRangeDesc range = new TFileRangeDesc(); + node.setScanParams(range, splits.get(1)); + Assert.assertEquals(Collections.singletonList(3L), range.getTableFormatParams() + .getLanceParams().getFragmentIds()); + ByteBuffer encodedUuid = range.getTableFormatParams().getLanceParams() + .getIndexSegmentUuids().get(0).duplicate(); + Assert.assertEquals(secondSegment.getMostSignificantBits(), encodedUuid.getLong()); + Assert.assertEquals(secondSegment.getLeastSignificantBits(), encodedUuid.getLong()); + + TExternalSearchRequest splitRequest = node.createSplitSearchRequest(); + Assert.assertEquals(7, + splitRequest.getSearchQuery().getFullTextSearch().getTopK()); + Assert.assertEquals(0, + splitRequest.getSearchQuery().getFullTextSearch().getOffset()); + Assert.assertEquals(5, request.getSearchQuery().getFullTextSearch().getTopK()); + Assert.assertEquals(2, request.getSearchQuery().getFullTextSearch().getOffset()); + Assert.assertEquals(TFtsQueryType.MATCH, + splitRequest.getSearchQuery().getFullTextSearch().getQueryType()); + Assert.assertEquals(TFtsMatchOperator.OR, + splitRequest.getSearchQuery().getFullTextSearch().getMatchOperator()); + Assert.assertEquals(0, + splitRequest.getSearchQuery().getFullTextSearch().getMaxFuzzyDistance()); + } + + @Test + public void testFullTextSplitRequestPreservesPhraseQuery() { + TExternalSearchRequest logicalRequest = phraseSearchRequest(5, 2, 1); + LanceScanNode node = LanceScanNode.forExternalSearch( + new PlanNodeId(0), new TupleDescriptor(new TupleId(0)), null, + null, -1, logicalRequest, new SessionVariable()); + + TExternalSearchRequest splitRequest = node.createSplitSearchRequest(); + + TFullTextSearchParams splitFullText = + splitRequest.getSearchQuery().getFullTextSearch(); + Assert.assertEquals(7, splitFullText.getTopK()); + Assert.assertEquals(0, splitFullText.getOffset()); + Assert.assertEquals(TFtsQueryType.PHRASE, splitFullText.getQueryType()); + Assert.assertEquals(1, splitFullText.getPhraseSlop()); + Assert.assertFalse(splitFullText.isSetMatchOperator()); + Assert.assertFalse(splitFullText.isSetMaxFuzzyDistance()); + + TFullTextSearchParams logicalFullText = + logicalRequest.getSearchQuery().getFullTextSearch(); + Assert.assertEquals(5, logicalFullText.getTopK()); + Assert.assertEquals(2, logicalFullText.getOffset()); + } + + @Test + public void testFullTextSearchCoverageModesHandleUnindexedFragments() throws Exception { + UUID segment = UUID.fromString("11111111-2222-3333-4444-555555555555"); + LanceTableMetadata metadata = LanceTableMetadata.withIndexSegments( + "s3://bucket/table.lance", + 42, + fullTextSchema(), + Arrays.asList( + new LanceFragmentInfo(1, 8, 8), + new LanceFragmentInfo(2, 7, 7), + new LanceFragmentInfo(3, 6, 6)), + Collections.singletonMap("body", 7), + Collections.singletonList( + new LanceIndexSegmentInfo(segment, "body_fts", + Collections.singletonList(7), Arrays.asList(1L, 2L), + IndexType.INVERTED, null)), + Collections.emptyMap()); + + LanceScanNode strictNode = newSearchNode(metadata, + fullTextSearchRequest(10, 0, TFtsCoverageMode.STRICT)); + UserException strictFailure = Assert.assertThrows(UserException.class, + () -> strictNode.getSplits(2)); + Assert.assertTrue(strictFailure.getMessage().contains("1 unindexed fragments")); + + LanceScanNode indexOnlyNode = newSearchNode(metadata, + fullTextSearchRequest(10, 0, TFtsCoverageMode.INDEX_ONLY)); + List indexOnlySplits = indexOnlyNode.getSplits(2); + Assert.assertEquals(1, indexOnlySplits.size()); + assertIndexSplit(indexOnlySplits.get(0), segment, Arrays.asList(1L, 2L), 15, 100); + } + @Test public void testExternalSearchFallsBackToFragmentSplitsForMetricMismatch() throws Exception { LanceTableMetadata metadata = LanceTableMetadata.withIndexSegments( @@ -263,7 +381,8 @@ public void testExternalSearchFallsBackToFragmentSplitsForMetricMismatch() throw Collections.singletonMap("vector", 9), Collections.singletonList( new LanceIndexSegmentInfo(UUID.randomUUID(), "vector_idx", - Collections.singletonList(9), Arrays.asList(1L, 2L), "L2")), + Collections.singletonList(9), Arrays.asList(1L, 2L), + IndexType.VECTOR, "L2")), Collections.emptyMap()); TExternalSearchRequest request = vectorSearchRequest(5, 0); request.getSearchQuery().getVectorSearch().setMetric(TVectorMetric.COSINE); @@ -286,7 +405,8 @@ public void testExternalSearchRejectsMissingFieldIdForIndexSegmentPlanning() { Collections.emptyMap(), Collections.singletonList( new LanceIndexSegmentInfo(UUID.randomUUID(), "vector_idx", - Collections.singletonList(9), Collections.singletonList(1L), "L2")), + Collections.singletonList(9), Collections.singletonList(1L), + IndexType.VECTOR, "L2")), Collections.emptyMap()); LanceScanNode node = newSearchNode(metadata, vectorSearchRequest(5, 0)); @@ -297,14 +417,16 @@ public void testExternalSearchRejectsMissingFieldIdForIndexSegmentPlanning() { } @Test - public void testFragmentSearchRetainsTopKPlusOffsetCandidates() { + public void testSplitSearchRetainsTopKPlusOffsetCandidates() { TExternalSearchRequest logicalRequest = vectorSearchRequest(5, 2); + LanceScanNode node = LanceScanNode.forExternalSearch( + new PlanNodeId(0), new TupleDescriptor(new TupleId(0)), null, + null, -1, logicalRequest, new SessionVariable()); - TExternalSearchRequest fragmentRequest = - LanceScanNode.createFragmentSearchRequest(logicalRequest); + TExternalSearchRequest splitRequest = node.createSplitSearchRequest(); - Assert.assertEquals(7, fragmentRequest.getSearchQuery().getVectorSearch().getTopK()); - Assert.assertEquals(0, fragmentRequest.getSearchQuery().getVectorSearch().getOffset()); + Assert.assertEquals(7, splitRequest.getSearchQuery().getVectorSearch().getTopK()); + Assert.assertEquals(0, splitRequest.getSearchQuery().getVectorSearch().getOffset()); Assert.assertEquals(5, logicalRequest.getSearchQuery().getVectorSearch().getTopK()); Assert.assertEquals(2, logicalRequest.getSearchQuery().getVectorSearch().getOffset()); } @@ -330,11 +452,13 @@ private static LanceScanNode newNode() { private static LanceScanNode newSearchNode( LanceTableMetadata metadata, TExternalSearchRequest request) { - String vectorColumn = request.getSearchQuery().getVectorSearch().getColumn(); - int vectorFieldId = metadata.getLanceFieldId(vectorColumn).orElse(-1); - return LanceScanNode.forVectorSearch( + String searchColumn = request.getSearchQuery().isSetVectorSearch() + ? request.getSearchQuery().getVectorSearch().getColumn() + : request.getSearchQuery().getFullTextSearch().getColumn(); + int searchFieldId = metadata.getLanceFieldId(searchColumn).orElse(-1); + return LanceScanNode.forExternalSearch( new PlanNodeId(0), new TupleDescriptor(new TupleId(0)), null, - metadata, vectorFieldId, request, new SessionVariable()); + metadata, searchFieldId, request, new SessionVariable()); } private static void setMetadata(LanceScanNode node, LanceTableMetadata metadata) throws Exception { @@ -365,6 +489,11 @@ private static Schema vectorSchema() { Field.nullable("vector", ArrowType.Utf8.INSTANCE))); } + private static Schema fullTextSchema() { + return new Schema(Collections.singletonList( + Field.nullable("body", ArrowType.Utf8.INSTANCE))); + } + private static void assertInvalidSplit(Runnable action, String expectedMessage) { try { action.run(); @@ -382,4 +511,32 @@ private static TExternalSearchRequest vectorSearchRequest(long topK, long offset return new TExternalSearchRequest() .setSearchQuery(TExternalSearchQuery.vector_search(vector)); } + + private static TExternalSearchRequest fullTextSearchRequest( + long topK, long offset, TFtsCoverageMode coverageMode) { + TFullTextSearchParams fullText = new TFullTextSearchParams() + .setColumn("body") + .setQuery("lance") + .setTopK(topK) + .setOffset(offset) + .setCoverageMode(coverageMode) + .setQueryType(TFtsQueryType.MATCH) + .setMatchOperator(TFtsMatchOperator.OR) + .setMaxFuzzyDistance(0); + return new TExternalSearchRequest() + .setSearchQuery(TExternalSearchQuery.full_text_search(fullText)); + } + + private static TExternalSearchRequest phraseSearchRequest(long topK, long offset, int slop) { + TFullTextSearchParams fullText = new TFullTextSearchParams() + .setColumn("body") + .setQuery("lance search") + .setTopK(topK) + .setOffset(offset) + .setCoverageMode(TFtsCoverageMode.STRICT) + .setQueryType(TFtsQueryType.PHRASE) + .setPhraseSlop(slop); + return new TExternalSearchRequest() + .setSearchQuery(TExternalSearchQuery.full_text_search(fullText)); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/FullTextSearchTableValuedFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/FullTextSearchTableValuedFunctionTest.java new file mode 100644 index 00000000000000..10ab0f241100d3 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/FullTextSearchTableValuedFunctionTest.java @@ -0,0 +1,127 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.tablefunction; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.datasource.lance.LanceTableMetadata; +import org.apache.doris.thrift.TFtsCoverageMode; +import org.apache.doris.thrift.TFtsMatchOperator; +import org.apache.doris.thrift.TFtsQueryType; + +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public class FullTextSearchTableValuedFunctionTest { + @Test + public void testParseCoverageMode() throws Exception { + Assert.assertEquals(TFtsCoverageMode.STRICT, + FullTextSearchTableValuedFunction.parseCoverageMode(" STRICT ")); + Assert.assertEquals(TFtsCoverageMode.INDEX_ONLY, + FullTextSearchTableValuedFunction.parseCoverageMode("index_only")); + Assert.assertEquals(TFtsCoverageMode.INDEX_ONLY, + FullTextSearchTableValuedFunction.parseCoverageMode("INDEX-ONLY")); + + AnalysisException invalid = Assert.assertThrows(AnalysisException.class, + () -> FullTextSearchTableValuedFunction.parseCoverageMode("flat")); + Assert.assertTrue(invalid.getMessage().contains("strict or index_only")); + } + + @Test + public void testParseQueryTypeAndMatchOperator() throws Exception { + Assert.assertEquals(TFtsQueryType.MATCH, + FullTextSearchTableValuedFunction.parseQueryType(" MATCH ")); + Assert.assertEquals(TFtsQueryType.PHRASE, + FullTextSearchTableValuedFunction.parseQueryType("phrase")); + Assert.assertEquals(TFtsMatchOperator.OR, + FullTextSearchTableValuedFunction.parseMatchOperator(" OR ")); + Assert.assertEquals(TFtsMatchOperator.AND, + FullTextSearchTableValuedFunction.parseMatchOperator("and")); + + AnalysisException invalidType = Assert.assertThrows(AnalysisException.class, + () -> FullTextSearchTableValuedFunction.parseQueryType("boolean")); + Assert.assertTrue(invalidType.getMessage().contains("match or phrase")); + + AnalysisException invalidOperator = Assert.assertThrows(AnalysisException.class, + () -> FullTextSearchTableValuedFunction.parseMatchOperator("xor")); + Assert.assertTrue(invalidOperator.getMessage().contains("'or' or 'and'")); + } + + @Test + public void testResolveStringColumnCaseInsensitively() throws Exception { + Field rowId = Field.notNullable("row_id", new ArrowType.Int(64, true)); + Field body = Field.nullable("Body", ArrowType.LargeUtf8.INSTANCE); + LanceTableMetadata metadata = metadata(rowId, body); + + Assert.assertSame(body, + FullTextSearchTableValuedFunction.findStringField(metadata, "body")); + } + + @Test + public void testRejectMissingNonStringAndAmbiguousColumns() { + LanceTableMetadata metadata = metadata( + Field.notNullable("row_id", new ArrowType.Int(64, true)), + Field.nullable("body", ArrowType.Utf8.INSTANCE)); + AnalysisException missing = Assert.assertThrows(AnalysisException.class, + () -> FullTextSearchTableValuedFunction.findStringField(metadata, "title")); + Assert.assertTrue(missing.getMessage().contains("does not exist")); + + LanceTableMetadata nonString = metadata( + Field.notNullable("body", new ArrowType.Int(64, true))); + AnalysisException wrongType = Assert.assertThrows(AnalysisException.class, + () -> FullTextSearchTableValuedFunction.findStringField(nonString, "body")); + Assert.assertTrue(wrongType.getMessage().contains("must be STRING")); + + LanceTableMetadata ambiguous = metadata( + Field.nullable("Body", ArrowType.Utf8.INSTANCE), + Field.nullable("body", ArrowType.Utf8.INSTANCE)); + AnalysisException duplicate = Assert.assertThrows(AnalysisException.class, + () -> FullTextSearchTableValuedFunction.findStringField(ambiguous, "BODY")); + Assert.assertTrue(duplicate.getMessage().contains("ambiguous")); + } + + @Test + public void testBuildOutputColumnsAddsNullableFloatScore() throws Exception { + LanceTableMetadata metadata = metadata( + Field.notNullable("row_id", new ArrowType.Int(64, true)), + Field.nullable("body", ArrowType.Utf8.INSTANCE)); + + List columns = FullTextSearchTableValuedFunction.buildOutputColumns( + metadata, FullTextSearchTableValuedFunction.SCORE_COLUMN, "full-text search"); + + Assert.assertEquals(3, columns.size()); + Column score = columns.get(2); + Assert.assertEquals(FullTextSearchTableValuedFunction.SCORE_COLUMN, score.getName()); + Assert.assertEquals(Type.FLOAT, score.getType()); + Assert.assertTrue(score.isAllowNull()); + } + + private static LanceTableMetadata metadata(Field... fields) { + return LanceTableMetadata.withoutIndexSegments( + "s3://bucket/table.lance", 42, new Schema(Arrays.asList(fields)), + Collections.emptyList(), Collections.emptyMap()); + } +} diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index 15701eb34d75b0..5ae5251f54f849 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -478,13 +478,48 @@ struct TVectorSearchParams { 5: optional TVectorMetric metric } -// Logical parameters for one full-text query. `query` initially carries the backend query string; -// richer structured query forms can be added as new fields without changing this basic contract. +enum TFtsCoverageMode { + STRICT, + INDEX_ONLY +} + +enum TFtsQueryType { + MATCH, + PHRASE +} + +enum TFtsMatchOperator { + OR, + AND +} + +// Logical parameters for one full-text query. MATCH combines analyzed terms with match_operator; +// PHRASE searches the analyzed terms in order and permits phrase_slop intervening positions. struct TFullTextSearchParams { 1: optional string column 2: optional string query 3: optional i64 top_k 4: optional i64 offset + // STRICT requires the selected FTS index to cover the complete pinned snapshot. INDEX_ONLY + // searches and scores only fragments covered by committed FTS index segments. + 5: optional TFtsCoverageMode coverage_mode + // Reserved for distributed FTS. Every BE may search a different index-segment subset, so + // statistics prepared independently on each BE would produce BM25 scores that are not + // comparable during the final TopK merge. The payload must be prepared once for the exact + // snapshot, logical FTS index, segment set, and query (including its final fuzzy vocabulary), + // then validated and installed unchanged on every BE scanner. + // + // Doris currently leaves this field unset and BE rejects a set value because Lance/lance-c + // does not yet provide the complete producer and consumer contract. Track the upstream work + // at https://github.com/lance-format/lance/issues/8937. + 6: optional binary global_statistics + 7: optional TFtsQueryType query_type + // MATCH-only parameters. max_fuzzy_distance is reserved as zero until the bundled Lance-C + // supports one canonical fuzzy vocabulary across all prepared index segments. + 8: optional TFtsMatchOperator match_operator + 9: optional i32 max_fuzzy_distance + // PHRASE-only parameter. Zero requires an exact phrase. + 10: optional i32 phrase_slop } enum TSearchFilterFormat { @@ -534,8 +569,9 @@ struct TLanceFileDesc { // most this many rows; the upper LIMIT operator still enforces the global bound. // Only set for ordinary scans whose predicates are fully pushed into Lance. 4: optional i64 limit - // Physical vector-index segments assigned to this distributed search split. Each value is one - // UUID encoded as 16 bytes in RFC 4122 order. Unset for ordinary and unindexed-fragment scans. + // Physical vector or FTS index segments assigned to this distributed search split. Each value + // is one UUID encoded as 16 bytes in RFC 4122 order. Unset for ordinary and vector + // unindexed-fragment scans. 5: optional list index_segment_uuids } @@ -544,8 +580,8 @@ struct TLanceScanParams { // ScanNode level so it is not serialized once per fragment split. 1: optional binary lance_substrait_filter // Provider-independent search request. Set at ScanNode level so all ranges use the same logical - // query. Lance vector search uses one range per fragment and Doris merges the split-local - // candidates. + // query. Lance external search uses one range per fragment or physical index segment, and Doris + // merges the split-local candidates. 2: optional TExternalSearchRequest external_search_request // Lance-native storage options, handed to lance-c untranslated. The namespace protocol treats // storage_options as opaque configuration passed directly to Lance, so any key vocabulary the diff --git a/regression-test/data/external_table_p0/lance/test_lance_full_text_search.out b/regression-test/data/external_table_p0/lance/test_lance_full_text_search.out new file mode 100644 index 00000000000000..31f0f62d9034e1 --- /dev/null +++ b/regression-test/data/external_table_p0/lance/test_lance_full_text_search.out @@ -0,0 +1,36 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !fts_ranked -- +3 1.0073528 +2 0.9261318 +1 0.74574697 +7 0.6000379 + +-- !fts_match_or -- +1 +2 +3 +7 + +-- !fts_match_and -- +7 + +-- !fts_phrase_exact -- +1 +2 +3 + +-- !fts_phrase_slop -- +1 +2 +3 + +-- !fts_offset -- +2 +1 + +-- !fts_filter -- +7 + +-- !fts_index_only -- +101 +102 diff --git a/regression-test/data/external_table_p0/lance/test_lance_runtime_filter_pushdown.out b/regression-test/data/external_table_p0/lance/test_lance_runtime_filter_pushdown.out new file mode 100644 index 00000000000000..d22abdc0d07eda --- /dev/null +++ b/regression-test/data/external_table_p0/lance/test_lance_runtime_filter_pushdown.out @@ -0,0 +1,18 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !runtime_filter_and_substrait -- +7 10 +9 100 + +-- !two_phase_vector_search -- +1 even item-0001 0.0 +2 odd item-0002 16.0 +3 even item-0003 64.0 +4 odd item-0004 144.0 +5 even item-0005 256.0 + +-- !explicit_join_vector_search -- +1 even item-0001 0.0 +2 odd item-0002 16.0 +3 even item-0003 64.0 +4 odd item-0004 144.0 +5 even item-0005 256.0 diff --git a/regression-test/suites/external_table_p0/lance/test_lance_full_text_search.groovy b/regression-test/suites/external_table_p0/lance/test_lance_full_text_search.groovy new file mode 100644 index 00000000000000..a2f9901cb42637 --- /dev/null +++ b/regression-test/suites/external_table_p0/lance/test_lance_full_text_search.groovy @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_lance_full_text_search", "p0,external") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable Lance full-text search test because the Iceberg MinIO environment is disabled.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String catalogName = "test_lance_full_text_search" + String fullTable = "${catalogName}.doris.full_text_search" + String partialTable = "${catalogName}.doris.full_text_search_partial" + + def search = { String table, String query, String topK, String offset = "0", + String coverageMode = "strict", String extra = "" -> + """full_text_search( + "table"="${table}", + "column"="body", + "query"="${query}", + "top_k"="${topK}", + "offset"="${offset}", + "coverage_mode"="${coverageMode}"${extra})""" + } + + sql "DROP CATALOG IF EXISTS `${catalogName}`" + try { + sql """ + CREATE CATALOG `${catalogName}` PROPERTIES ( + "type" = "lance", + "lance.catalog.type" = "filesystem", + "warehouse" = "s3://warehouse/lance", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.region" = "us-east-1", + "use_path_style" = "true" + ) + """ + sql "SET enable_file_scanner_v2 = true" + + // The full corpus is indexed across multiple fragments. The plan must use FTS index + // segment splits, and strict coverage must see no unindexed fragment. + explain { + sql """SELECT row_id, _score + FROM ${search(fullTable, "lance", "4")} + ORDER BY _score DESC, row_id""" + contains "externalSearchType=FULL_TEXT" + contains "lanceFullTextColumn=body" + contains "lanceFtsCoverageMode=STRICT" + contains "lanceFtsQueryType=MATCH" + contains "lanceFtsMatchOperator=OR" + contains "lanceSearchUnindexedFragments=0" + contains "lanceSearchIndexSegments=" + } + + qt_fts_ranked """ + SELECT row_id, _score + FROM ${search(fullTable, "lance", "4")} + ORDER BY _score DESC, row_id + """ + + // MATCH defaults to OR, while AND requires every analyzed query term to be present. + qt_fts_match_or """ + SELECT row_id + FROM ${search(fullTable, "lance storage", "10", "0", "strict", + ', "query_type"="match", "operator"="or"')} + ORDER BY row_id + """ + qt_fts_match_and """ + SELECT row_id + FROM ${search(fullTable, "lance storage", "10", "0", "strict", + ', "query_type"="match", "operator"="and"')} + ORDER BY row_id + """ + + // This fixture stores token positions. Exact Phrase matches adjacent terms, while slop=1 + // permits the intervening "search" token in "lance search engine". + qt_fts_phrase_exact """ + SELECT row_id + FROM ${search(fullTable, "lance search", "10", "0", "strict", + ', "query_type"="phrase", "slop"="0"')} + ORDER BY row_id + """ + qt_fts_phrase_slop """ + SELECT row_id + FROM ${search(fullTable, "lance engine", "10", "0", "strict", + ', "query_type"="phrase", "slop"="1"')} + ORDER BY row_id + """ + + // The wire contract already reserves fuzzy distance, but the bundled Lance revision does + // not yet provide one canonical fuzzy vocabulary across prepared index segments. + test { + sql """SELECT row_id + FROM ${search(fullTable, "lance", "10", "0", "strict", + ', "max_fuzzy_distance"="1"')} + ORDER BY row_id""" + exception "does not yet support prepared fuzzy matching" + } + + // OFFSET belongs to the snapshot-wide TopN. It must not be applied independently by + // every physical FTS index split. + qt_fts_offset """ + SELECT row_id + FROM ${search(fullTable, "lance", "2", "1")} + ORDER BY _score DESC, row_id + """ + + // The TVF filter runs inside Lance before FTS candidates are selected. + qt_fts_filter """ + SELECT row_id + FROM ${search(fullTable, "lance", "4", "0", "strict", + ', "filter"="category = \'storage\'"')} + ORDER BY _score DESC, row_id + """ + + // The second table has a committed FTS index plus one fragment containing two rows + // appended afterwards. STRICT is the safe default and rejects the incomplete snapshot. + test { + sql """SELECT row_id + FROM ${search(partialTable, "lance", "10")} + ORDER BY _score DESC, row_id""" + exception "unindexed fragments" + } + + // INDEX_ONLY searches exactly the fragments covered by the selected FTS segment. The + // two appended rows also contain "lance" but must not leak into the result. + explain { + sql """SELECT row_id + FROM ${search(partialTable, "lance", "10", "0", "index_only")} + ORDER BY _score DESC, row_id""" + contains "externalSearchType=FULL_TEXT" + contains "lanceFtsCoverageMode=INDEX_ONLY" + contains "lanceSearchUnindexedFragments=1" + } + qt_fts_index_only """ + SELECT row_id + FROM ${search(partialTable, "lance", "10", "0", "index_only")} + ORDER BY _score DESC, row_id + """ + } finally { + // sql "DROP CATALOG IF EXISTS `${catalogName}`" + } +} diff --git a/regression-test/suites/external_table_p0/lance/test_lance_runtime_filter_pushdown.groovy b/regression-test/suites/external_table_p0/lance/test_lance_runtime_filter_pushdown.groovy new file mode 100644 index 00000000000000..870f51cd7b2b9a --- /dev/null +++ b/regression-test/suites/external_table_p0/lance/test_lance_runtime_filter_pushdown.groovy @@ -0,0 +1,158 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_lance_runtime_filter_pushdown", "p0,external") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable Lance runtime-filter test because the Iceberg MinIO environment is disabled.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String catalogName = "test_lance_runtime_filter_pushdown" + String buildTable = "test_lance_runtime_filter_build" + String internalDb = context.dbName + String lanceTable = "`${catalogName}`.`doris`.`predicate_pushdown`" + + sql "SWITCH internal" + sql "USE `${internalDb}`" + sql "DROP TABLE IF EXISTS `${buildTable}`" + sql "DROP CATALOG IF EXISTS `${catalogName}`" + + try { + sql """ + CREATE TABLE `${buildTable}` ( + id BIGINT NOT NULL + ) ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ) + """ + sql "INSERT INTO `${buildTable}` VALUES (4), (7), (9), (11)" + + sql """ + CREATE CATALOG `${catalogName}` PROPERTIES ( + "type" = "lance", + "lance.catalog.type" = "filesystem", + "warehouse" = "s3://warehouse/lance", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.region" = "us-east-1", + "use_path_style" = "true" + ) + """ + + sql "SET enable_file_scanner_v2 = true" + sql "SET enable_sql_cache = false" + sql "SET enable_query_cache = false" + sql "SET runtime_filter_mode = 'GLOBAL'" + sql "SET runtime_filter_type = 'IN'" + sql "SET runtime_filter_wait_infinitely = true" + sql "SET enable_runtime_filter_prune = false" + + String query = """ + SELECT /*+ leading(l broadcast b) */ l.row_id, l.int64_value + FROM ${lanceTable} l + INNER JOIN `internal`.`${internalDb}`.`${buildTable}` b + ON l.row_id = b.id + WHERE l.int64_value >= 10 + ORDER BY l.row_id + """ + + // The regular WHERE predicate is converted to the primary Substrait filter. The join + // produces an IN runtime filter on row_id, which must be attached to the Lance scan. + explain { + sql "verbose ${query}" + check { explainString -> + assertTrue(explainString.contains("VLANCE_SCAN_NODE")) + assertTrue(explainString.contains("lancePushdownPredicate=")) + assertTrue(explainString.contains("int64_value")) + assertTrue(explainString.contains("runtime filters: RF")) + assertTrue(explainString.contains("-> row_id")) + return true + } + } + + // Build-side IDs are 4, 7, 9 and 11. The static Lance predicate keeps only rows whose + // int64_value is at least 10, so the intersection is exactly rows 7 and 9. In particular, + // this catches an implementation that replaces the Substrait filter with the later RF. + qt_runtime_filter_and_substrait "${query}" + + // Compare Doris's dedicated two-phase row-id fetch with the equivalent SQL formulation: + // first produce narrow ANN candidates, then broadcast them and fetch payload columns from + // a normal Lance scan. Both queries must return the same ordered rows and distances. + String vectorTable = "${catalogName}.doris.vs_ivf_pq_f32" + String qualifiedVectorTable = "`${catalogName}`.`doris`.`vs_ivf_pq_f32`" + String headQuery = "[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]" + String vectorSearch = """vector_search( + "table"="${vectorTable}", + "column"="embedding", + "query_vector"="${headQuery}", + "top_k"="5", + "metric"="l2", + "nprobes"="4", + "refine_factor"="10", + "use_index"="true")""" + String twoPhaseQuery = """ + SELECT row_id, category, label, _distance + FROM ${vectorSearch} + ORDER BY _distance, row_id + """ + + sql "SET topn_lazy_materialization_threshold = 1024" + explain { + sql "verbose ${twoPhaseQuery}" + contains "VMaterializeNode" + contains "__DORIS_GLOBAL_ROWID_COL__vector_search" + } + qt_two_phase_vector_search "${twoPhaseQuery}" + + String explicitJoinQuery = """ + WITH candidates AS ( + SELECT row_id, _distance + FROM ${vectorSearch} + ORDER BY _distance, row_id + LIMIT 5 + ) + SELECT w.row_id, w.category, w.label, c._distance + FROM ${qualifiedVectorTable} w + JOIN [broadcast] candidates c ON w.row_id = c.row_id + ORDER BY c._distance, c.row_id + """ + + sql "SET topn_lazy_materialization_threshold = -1" + sql "SET disable_join_reorder = true" + explain { + sql "verbose ${explicitJoinQuery}" + contains "VHASH JOIN" + contains "JOIN(BROADCAST)" + contains "runtime filters: RF" + contains "-> row_id" + notContains "VMaterializeNode" + } + qt_explicit_join_vector_search "${explicitJoinQuery}" + } finally { + // sql "SWITCH internal" + // sql "USE `${internalDb}`" + // sql "DROP TABLE IF EXISTS `${buildTable}`" + // sql "DROP CATALOG IF EXISTS `${catalogName}`" + } +} diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index bb9c5928809170..721960d72783ad 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -774,12 +774,15 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " PAIMON_CPP " ]]; then echo "Finished patching ${PAIMON_CPP_SOURCE}" fi -# Patch lance-c with the scan execution statistics API from upstream PR #64. +# Apply Doris lance-c patches. if [[ " ${TP_ARCHIVES[*]} " =~ " LANCE_C " ]]; then - if [[ "${LANCE_C_SOURCE}" == "lance-c-0.1.7" ]]; then + if [[ "${LANCE_C_SOURCE}" == "lance-c-0.1.9" ]]; then cd "${TP_SOURCE_DIR}/${LANCE_C_SOURCE}" if [[ ! -f "${PATCHED_MARK}" ]]; then - patch -p1 <"${TP_PATCH_DIR}/lance-c-0.1.7-pr-64.patch" + patch --batch --forward --reject-file=- --fuzz=0 --no-backup-if-mismatch -s \ + -p1 <"${TP_PATCH_DIR}/lance-c-0.1.9-pr-73.patch" + patch --batch --forward --reject-file=- --fuzz=0 --no-backup-if-mismatch -s \ + -p1 <"${TP_PATCH_DIR}/lance-c-0.1.9-pr-74.patch" touch "${PATCHED_MARK}" fi cd - diff --git a/thirdparty/patches/lance-c-0.1.7-pr-64.patch b/thirdparty/patches/lance-c-0.1.7-pr-64.patch deleted file mode 100644 index 17f91bc7c318ad..00000000000000 --- a/thirdparty/patches/lance-c-0.1.7-pr-64.patch +++ /dev/null @@ -1,1522 +0,0 @@ -From e3320c1e7d5c72234b3b44e9e7e9a93a72fe488c Mon Sep 17 00:00:00 2001 -From: zhangstar333 -Date: Mon, 24 Aug 2026 16:37:33 +0800 -Subject: [PATCH 1/3] update - ---- - include/lance/lance.h | 73 ++++++++++++++++ - include/lance/lance.hpp | 10 +++ - src/scanner.rs | 179 +++++++++++++++++++++++++++++++++++++- - tests/c_api_test.rs | 188 +++++++++++++++++++++++++++++++++++++++- - 4 files changed, 448 insertions(+), 2 deletions(-) - -diff --git a/include/lance/lance.h b/include/lance/lance.h -index 986905b..c6c3985 100644 ---- a/include/lance/lance.h -+++ b/include/lance/lance.h -@@ -863,6 +863,79 @@ int32_t lance_scanner_set_substrait_filter( - size_t len - ); - -+/** Type of a dynamically named scan metric. */ -+typedef enum { -+ LANCE_SCAN_METRIC_COUNT = 0, -+ LANCE_SCAN_METRIC_TIME_NANOSECONDS = 1, -+} LanceScanMetricKind; -+ -+/** -+ * Borrowed view of one dynamically named scan metric. -+ * -+ * `name` is not NUL-terminated. `name` and this structure are valid only for -+ * the duration of the LanceScanStatisticsCallback invocation. -+ */ -+typedef struct { -+ const char* name; -+ size_t name_len; -+ LanceScanMetricKind kind; -+ uint64_t value; -+} LanceScanMetric; -+ -+/** -+ * Borrowed view of the execution statistics for one finalized scan. -+ * -+ * The fixed fields are stable summary metrics. `metrics` contains additional -+ * implementation-specific counters and timings. Those names are not a stable -+ * API and are intended for diagnostics and profiles. Dynamic metrics are -+ * best-effort and may be omitted if they cannot be materialized. `metrics` is -+ * NULL when `metrics_len` is zero. -+ */ -+typedef struct { -+ uint64_t iops; -+ uint64_t requests; -+ uint64_t bytes_read; -+ uint64_t indices_loaded; -+ uint64_t index_partitions_loaded; -+ uint64_t index_comparisons; -+ const LanceScanMetric* metrics; -+ size_t metrics_len; -+} LanceScanStatistics; -+ -+/** -+ * Receives scan statistics when a stream reaches EOF, fails, or is released. -+ * -+ * The statistics and all nested pointers are borrowed and valid only for the -+ * duration of this call. The callback may run on the thread that consumes or -+ * releases the scan stream and must therefore be thread-safe. It must return -+ * normally without throwing an exception or unwinding, and must not call any -+ * `lance_scanner_*` function with the originating scanner. -+ * -+ * Scan statistics are diagnostic and best-effort. The callback must handle its -+ * own errors and must not use them to abort or throw across this FFI boundary. -+ */ -+typedef void (*LanceScanStatisticsCallback)( -+ void* callback_ctx, -+ const LanceScanStatistics* statistics -+); -+ -+/** -+ * Register the execution-statistics callback for this scanner. -+ * -+ * Must be called before starting the scan; registering after the scan starts -+ * returns an error. `callback` must not be NULL. `callback_ctx` may be NULL. A -+ * non-NULL `callback_ctx` must remain valid, and `callback` must remain valid, -+ * until the stream reaches EOF, fails, or is released. Replaces a previously -+ * registered callback. -+ * -+ * @return 0 on success, -1 on error -+ */ -+int32_t lance_scanner_set_statistics_callback( -+ LanceScanner* scanner, -+ LanceScanStatisticsCallback callback, -+ void* callback_ctx -+); -+ - /** Close and free a scanner handle. */ - void lance_scanner_close(LanceScanner* scanner); - -diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp -index afce358..9440a23 100644 ---- a/include/lance/lance.hpp -+++ b/include/lance/lance.hpp -@@ -1127,6 +1127,16 @@ class Scanner { - return substrait_filter(bytes.data(), bytes.size()); - } - -+ /// Register a callback for scan execution statistics before starting the scan. -+ /// The callback may run on the thread that consumes or releases the stream. It -+ /// must be thread-safe, must not throw, and must not re-enter the originating -+ /// scanner. A non-null callback context must outlive the exported stream. -+ Scanner& statistics_callback(LanceScanStatisticsCallback callback, void* callback_ctx) { -+ if (lance_scanner_set_statistics_callback(handle_.get(), callback, callback_ctx) != 0) -+ check_error(); -+ return *this; -+ } -+ - /// Restrict the next k-NN query to a subset of vector index segments. - /// Pass `len` 16-byte UUIDs concatenated as a single byte buffer - /// (total bytes = `len * 16`). Pass len=0 (and any pointer) to clear. -diff --git a/src/scanner.rs b/src/scanner.rs -index f44b82f..d95089e 100644 ---- a/src/scanner.rs -+++ b/src/scanner.rs -@@ -14,7 +14,9 @@ use arrow::ffi_stream::FFI_ArrowArrayStream; - use arrow_schema::SchemaRef; - use futures::{FutureExt, Stream, StreamExt}; - use lance::Dataset; --use lance::dataset::scanner::DatasetRecordBatchStream; -+use lance::dataset::scanner::{ -+ DatasetRecordBatchStream, ExecutionStatsCallback, ExecutionSummaryCounts, -+}; - use lance_core::Result; - use lance_index::scalar::FullTextSearchQuery; - use lance_io::stream::RecordBatchStream; -@@ -69,6 +71,8 @@ pub struct LanceScanner { - // the spawned async task can poison the handle from outside this call - // frame via `poison_flag()`. - poisoned: Arc, -+ scan_statistics_callback: Option, -+ scan_started: AtomicBool, - // Materialized on first iteration call - stream: Option>>, - #[allow(dead_code)] -@@ -122,6 +126,8 @@ impl LanceScanner { - prefilter: false, - fts_query: None, - poisoned: Arc::new(AtomicBool::new(false)), -+ scan_statistics_callback: None, -+ scan_started: AtomicBool::new(false), - stream: None, - schema: None, - } -@@ -157,6 +163,7 @@ impl LanceScanner { - - /// Build the underlying Scanner and open a stream. - fn materialize_stream(&mut self) -> Result<()> { -+ self.scan_started.store(true, Ordering::Release); - let mut scanner = self.dataset.scan(); - if let Some(cols) = &self.columns { - scanner.project(cols)?; -@@ -212,6 +219,9 @@ impl LanceScanner { - if let Some(fts) = &self.fts_query { - scanner.full_text_search(fts.clone())?; - } -+ if let Some(callback) = &self.scan_statistics_callback { -+ scanner.scan_stats_callback(callback.clone()); -+ } - let stream = block_on(scanner.try_into_stream())?; - self.schema = Some(stream.schema()); - self.stream = Some(Box::pin(stream)); -@@ -220,6 +230,7 @@ impl LanceScanner { - - /// Build a Scanner (without materializing) and return it. - fn build_scanner(&self) -> Result { -+ self.scan_started.store(true, Ordering::Release); - let mut scanner = self.dataset.scan(); - if let Some(cols) = &self.columns { - scanner.project(cols)?; -@@ -274,10 +285,122 @@ impl LanceScanner { - if let Some(fts) = &self.fts_query { - scanner.full_text_search(fts.clone())?; - } -+ if let Some(callback) = &self.scan_statistics_callback { -+ scanner.scan_stats_callback(callback.clone()); -+ } - Ok(scanner) - } - } - -+/// Type of a dynamically named scan metric. -+#[repr(i32)] -+#[derive(Clone, Copy, Debug, PartialEq, Eq)] -+pub enum LanceScanMetricKind { -+ /// Monotonically accumulated counter. -+ Count = 0, -+ /// Accumulated duration in nanoseconds. -+ TimeNanoseconds = 1, -+} -+ -+/// Borrowed view of one dynamically named scan metric. -+/// -+/// `name` is not NUL-terminated. Both `name` and this structure are valid only -+/// for the duration of the scan statistics callback. -+#[repr(C)] -+#[derive(Clone, Copy, Debug)] -+pub struct LanceScanMetric { -+ pub name: *const c_char, -+ pub name_len: usize, -+ pub kind: LanceScanMetricKind, -+ pub value: u64, -+} -+ -+/// Borrowed view of the execution statistics for one completed scan. -+/// -+/// The fixed fields are stable summary metrics. `metrics` contains additional -+/// implementation-specific counters and timings and is valid only for the -+/// duration of the callback. -+#[repr(C)] -+#[derive(Clone, Copy, Debug)] -+pub struct LanceScanStatistics { -+ pub iops: u64, -+ pub requests: u64, -+ pub bytes_read: u64, -+ pub indices_loaded: u64, -+ pub index_partitions_loaded: u64, -+ pub index_comparisons: u64, -+ pub metrics: *const LanceScanMetric, -+ pub metrics_len: usize, -+} -+ -+/// Callback invoked when a scan stream reaches EOF, fails, or is released. -+/// -+/// The callback is an FFI boundary and must return normally without unwinding -+/// or throwing an exception. It must not call back into `lance_scanner_*` with -+/// the originating scanner. -+pub type LanceScanStatisticsCallback = -+ Option; -+ -+struct SendScanStatisticsCallback { -+ callback: unsafe extern "C" fn(*mut c_void, *const LanceScanStatistics), -+ ctx: *mut c_void, -+} -+ -+// SAFETY: The C API requires the callback and its context to remain valid and -+// safe to invoke from the thread that consumes or releases the scan stream. -+unsafe impl Send for SendScanStatisticsCallback {} -+unsafe impl Sync for SendScanStatisticsCallback {} -+ -+impl SendScanStatisticsCallback { -+ fn invoke(&self, counts: &ExecutionSummaryCounts) { -+ // Dynamic profile metrics are best-effort. Use fallible reservation so -+ // allocation failure omits them instead of aborting the embedding process. -+ let mut metrics = Vec::new(); -+ if let Some(metrics_len) = counts.all_counts.len().checked_add(counts.all_times.len()) -+ && metrics.try_reserve_exact(metrics_len).is_ok() -+ { -+ metrics.extend( -+ counts -+ .all_counts -+ .iter() -+ .map(|(name, value)| LanceScanMetric { -+ name: name.as_ptr().cast(), -+ name_len: name.len(), -+ kind: LanceScanMetricKind::Count, -+ value: *value as u64, -+ }), -+ ); -+ metrics.extend( -+ counts -+ .all_times -+ .iter() -+ .map(|(name, value)| LanceScanMetric { -+ name: name.as_ptr().cast(), -+ name_len: name.len(), -+ kind: LanceScanMetricKind::TimeNanoseconds, -+ value: *value as u64, -+ }), -+ ); -+ } -+ -+ let statistics = LanceScanStatistics { -+ iops: counts.iops as u64, -+ requests: counts.requests as u64, -+ bytes_read: counts.bytes_read as u64, -+ indices_loaded: counts.indices_loaded as u64, -+ index_partitions_loaded: counts.parts_loaded as u64, -+ index_comparisons: counts.index_comparisons as u64, -+ metrics: if metrics.is_empty() { -+ ptr::null() -+ } else { -+ metrics.as_ptr() -+ }, -+ metrics_len: metrics.len(), -+ }; -+ unsafe { (self.callback)(self.ctx, &statistics) }; -+ } -+} -+ - // --------------------------------------------------------------------------- - // Poison check shared by all `lance_scanner_*` entry points - // --------------------------------------------------------------------------- -@@ -529,6 +652,60 @@ unsafe fn scanner_set_substrait_filter_inner( - Ok(0) - } - -+/// Register a callback that receives execution statistics when the scan stream -+/// reaches EOF, fails, or is released. -+/// -+/// The callback and `callback_ctx` must remain valid until the scan stream is -+/// finalized. Metric names and arrays passed to the callback are borrowed and -+/// must be copied if the caller needs to retain them. The callback must be -+/// thread-safe, must return normally without unwinding or throwing an -+/// exception, and must not call `lance_scanner_*` with the originating scanner. -+#[unsafe(no_mangle)] -+pub unsafe extern "C" fn lance_scanner_set_statistics_callback( -+ scanner: *mut LanceScanner, -+ callback: LanceScanStatisticsCallback, -+ callback_ctx: *mut c_void, -+) -> i32 { -+ scanner_poison_check!(scanner, -1); -+ ffi_try!( -+ unsafe { scanner_set_statistics_callback_inner(scanner, callback, callback_ctx) }, -+ neg -+ ) -+} -+ -+unsafe fn scanner_set_statistics_callback_inner( -+ scanner: *mut LanceScanner, -+ callback: LanceScanStatisticsCallback, -+ callback_ctx: *mut c_void, -+) -> Result { -+ if scanner.is_null() { -+ return Err(lance_core::Error::invalid_input_source( -+ "scanner is NULL".into(), -+ )); -+ } -+ let Some(callback) = callback else { -+ return Err(lance_core::Error::invalid_input_source( -+ "statistics callback is NULL".into(), -+ )); -+ }; -+ -+ let s = unsafe { &mut *scanner }; -+ if s.scan_started.load(Ordering::Acquire) { -+ return Err(lance_core::Error::invalid_input_source( -+ "statistics callback must be registered before the scan starts".into(), -+ )); -+ } -+ -+ let callback = SendScanStatisticsCallback { -+ callback, -+ ctx: callback_ctx, -+ }; -+ s.scan_statistics_callback = Some(Arc::new(move |counts: &ExecutionSummaryCounts| { -+ callback.invoke(counts); -+ })); -+ Ok(0) -+} -+ - /// Close and free a scanner handle. - /// - /// Best-effort (issue #61): this drops a possibly-live -diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs -index d6cd928..db35fea 100644 ---- a/tests/c_api_test.rs -+++ b/tests/c_api_test.rs -@@ -6,7 +6,7 @@ - //! These tests call the `extern "C"` functions directly from Rust, - //! validating the C API contract without needing a C compiler. - --use std::ffi::{CString, c_char}; -+use std::ffi::{CString, c_char, c_void}; - use std::process::Command; - use std::ptr; - use std::sync::Arc; -@@ -99,6 +99,58 @@ fn c_str(s: &str) -> CString { - CString::new(s).unwrap() - } - -+#[derive(Default)] -+struct CapturedScanStatistics { -+ calls: usize, -+ iops: u64, -+ requests: u64, -+ bytes_read: u64, -+ indices_loaded: u64, -+ index_partitions_loaded: u64, -+ index_comparisons: u64, -+ metrics: Vec<(String, LanceScanMetricKind, u64)>, -+} -+ -+unsafe extern "C" fn capture_scan_statistics( -+ callback_ctx: *mut c_void, -+ statistics: *const LanceScanStatistics, -+) { -+ assert!(!callback_ctx.is_null()); -+ assert!(!statistics.is_null()); -+ let captured = unsafe { &mut *callback_ctx.cast::() }; -+ let statistics = unsafe { &*statistics }; -+ let metrics = if statistics.metrics_len == 0 { -+ &[] -+ } else { -+ assert!(!statistics.metrics.is_null()); -+ unsafe { std::slice::from_raw_parts(statistics.metrics, statistics.metrics_len) } -+ }; -+ -+ captured.calls += 1; -+ captured.iops = statistics.iops; -+ captured.requests = statistics.requests; -+ captured.bytes_read = statistics.bytes_read; -+ captured.indices_loaded = statistics.indices_loaded; -+ captured.index_partitions_loaded = statistics.index_partitions_loaded; -+ captured.index_comparisons = statistics.index_comparisons; -+ captured.metrics = metrics -+ .iter() -+ .map(|metric| { -+ let name = if metric.name_len == 0 { -+ &[] -+ } else { -+ assert!(!metric.name.is_null()); -+ unsafe { std::slice::from_raw_parts(metric.name.cast::(), metric.name_len) } -+ }; -+ ( -+ std::str::from_utf8(name).unwrap().to_owned(), -+ metric.kind, -+ metric.value, -+ ) -+ }) -+ .collect(); -+} -+ - /// Helper: build a tiny dataset whose `value` column is nullable AND contains - /// at least one NULL. Used by tests that need to exercise upstream's - /// nullability-tightening pre-scan failure path. -@@ -284,6 +336,140 @@ fn test_scanner_to_arrow_stream() { - unsafe { lance_dataset_close(ds) }; - } - -+#[test] -+fn test_scanner_statistics_callback_with_next_multi_fragment() { -+ let (_tmp, uri) = create_multi_fragment_dataset(); -+ let c_uri = c_str(&uri); -+ let ds = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) }; -+ assert!(!ds.is_null()); -+ assert_eq!(unsafe { lance_dataset_fragment_count(ds) }, 2); -+ -+ let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) }; -+ assert!(!scanner.is_null()); -+ let mut captured = CapturedScanStatistics::default(); -+ assert_eq!( -+ unsafe { -+ lance_scanner_set_statistics_callback( -+ scanner, -+ Some(capture_scan_statistics), -+ (&mut captured as *mut CapturedScanStatistics).cast(), -+ ) -+ }, -+ 0 -+ ); -+ -+ loop { -+ let mut batch = ptr::null_mut(); -+ match unsafe { lance_scanner_next(scanner, &mut batch) } { -+ 0 => unsafe { lance_batch_free(batch) }, -+ 1 => break, -+ status => panic!("scanner_next returned error: {status}"), -+ } -+ } -+ -+ assert_eq!(captured.calls, 1); -+ assert!(captured.bytes_read > 0); -+ assert!(captured.requests > 0); -+ assert!(captured.metrics.iter().all(|(name, _, _)| !name.is_empty())); -+ -+ unsafe { lance_scanner_close(scanner) }; -+ unsafe { lance_dataset_close(ds) }; -+} -+ -+#[test] -+fn test_scanner_statistics_callback_with_arrow_stream() { -+ let (_tmp, uri) = create_test_dataset(); -+ let c_uri = c_str(&uri); -+ let ds = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) }; -+ assert!(!ds.is_null()); -+ -+ let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) }; -+ assert!(!scanner.is_null()); -+ let mut captured = CapturedScanStatistics::default(); -+ assert_eq!( -+ unsafe { -+ lance_scanner_set_statistics_callback( -+ scanner, -+ Some(capture_scan_statistics), -+ (&mut captured as *mut CapturedScanStatistics).cast(), -+ ) -+ }, -+ 0 -+ ); -+ -+ let mut stream = FFI_ArrowArrayStream::empty(); -+ assert_eq!(unsafe { lance_scanner_to_arrow_stream(scanner, &mut stream) }, 0); -+ let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut stream) }.unwrap(); -+ assert_eq!(reader.map(|batch| batch.unwrap().num_rows()).sum::(), 5); -+ assert_eq!(captured.calls, 1); -+ assert!(captured.bytes_read > 0); -+ -+ unsafe { lance_scanner_close(scanner) }; -+ unsafe { lance_dataset_close(ds) }; -+} -+ -+#[test] -+fn test_scanner_statistics_callback_rejects_null_inputs() { -+ assert_eq!( -+ unsafe { -+ lance_scanner_set_statistics_callback( -+ ptr::null_mut(), -+ Some(capture_scan_statistics), -+ ptr::null_mut(), -+ ) -+ }, -+ -1 -+ ); -+ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); -+ -+ let (_tmp, uri) = create_test_dataset(); -+ let c_uri = c_str(&uri); -+ let ds = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) }; -+ let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) }; -+ assert_eq!( -+ unsafe { lance_scanner_set_statistics_callback(scanner, None, ptr::null_mut()) }, -+ -1 -+ ); -+ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); -+ -+ unsafe { lance_scanner_close(scanner) }; -+ unsafe { lance_dataset_close(ds) }; -+} -+ -+#[test] -+fn test_scanner_statistics_callback_rejects_registration_after_scan_started() { -+ let (_tmp, uri) = create_test_dataset(); -+ let c_uri = c_str(&uri); -+ let ds = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) }; -+ assert!(!ds.is_null()); -+ let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) }; -+ assert!(!scanner.is_null()); -+ -+ let mut batch = ptr::null_mut(); -+ assert_eq!(unsafe { lance_scanner_next(scanner, &mut batch) }, 0); -+ assert!(!batch.is_null()); -+ unsafe { lance_batch_free(batch) }; -+ -+ let mut captured = CapturedScanStatistics::default(); -+ assert_eq!( -+ unsafe { -+ lance_scanner_set_statistics_callback( -+ scanner, -+ Some(capture_scan_statistics), -+ (&mut captured as *mut CapturedScanStatistics).cast(), -+ ) -+ }, -+ -1 -+ ); -+ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); -+ let error = take_last_error_message(); -+ assert!(error.contains("before the scan starts"), "{error}"); -+ -+ unsafe { lance_scanner_close(scanner) }; -+ assert_eq!(captured.calls, 0); -+ unsafe { lance_dataset_close(ds) }; -+} -+ - #[test] - fn test_scanner_with_filter() { - let (_tmp, uri) = create_test_dataset(); - -From 8e6e92140031c06184da079b8f6c194799c0de88 Mon Sep 17 00:00:00 2001 -From: zhangstar333 -Date: Mon, 24 Aug 2026 17:04:51 +0800 -Subject: [PATCH 2/3] formatter - ---- - include/lance/lance.h | 23 ++++--- - include/lance/lance.hpp | 9 +-- - src/scanner.rs | 23 ++++--- - tests/c_api_test.rs | 134 +++++++++++++++++++++++++++++++++++++++- - 4 files changed, 165 insertions(+), 24 deletions(-) - -diff --git a/include/lance/lance.h b/include/lance/lance.h -index c6c3985..1a6822b 100644 ---- a/include/lance/lance.h -+++ b/include/lance/lance.h -@@ -883,7 +883,7 @@ typedef struct { - } LanceScanMetric; - - /** -- * Borrowed view of the execution statistics for one finalized scan. -+ * Borrowed view of the execution statistics for one fully consumed scan. - * - * The fixed fields are stable summary metrics. `metrics` contains additional - * implementation-specific counters and timings. Those names are not a stable -@@ -903,13 +903,13 @@ typedef struct { - } LanceScanStatistics; - - /** -- * Receives scan statistics when a stream reaches EOF, fails, or is released. -+ * Receives scan statistics after a stream is fully consumed to EOF. - * - * The statistics and all nested pointers are borrowed and valid only for the -- * duration of this call. The callback may run on the thread that consumes or -- * releases the scan stream and must therefore be thread-safe. It must return -- * normally without throwing an exception or unwinding, and must not call any -- * `lance_scanner_*` function with the originating scanner. -+ * duration of this call. The callback may run on the thread that observes EOF -+ * and must therefore be thread-safe. It must return normally without throwing -+ * an exception or unwinding, and must not call any `lance_scanner_*` function -+ * with the originating scanner. - * - * Scan statistics are diagnostic and best-effort. The callback must handle its - * own errors and must not use them to abort or throw across this FFI boundary. -@@ -925,8 +925,15 @@ typedef void (*LanceScanStatisticsCallback)( - * Must be called before starting the scan; registering after the scan starts - * returns an error. `callback` must not be NULL. `callback_ctx` may be NULL. A - * non-NULL `callback_ctx` must remain valid, and `callback` must remain valid, -- * until the stream reaches EOF, fails, or is released. Replaces a previously -- * registered callback. -+ * until the callback returns or, if the callback has not run, until the owning -+ * scan stream is released. For `lance_scanner_next` and -+ * `lance_scanner_poll_next`, the scanner owns the stream. For an exported -+ * ArrowArrayStream, the Arrow stream owns it independently of the scanner. -+ * -+ * The callback is invoked exactly once when the stream is fully consumed to -+ * EOF. It is not guaranteed to run if execution fails, the scan is cancelled, -+ * or the scanner / ArrowArrayStream is released before EOF. Replaces a -+ * previously registered callback. - * - * @return 0 on success, -1 on error - */ -diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp -index 9440a23..4268801 100644 ---- a/include/lance/lance.hpp -+++ b/include/lance/lance.hpp -@@ -1127,10 +1127,11 @@ class Scanner { - return substrait_filter(bytes.data(), bytes.size()); - } - -- /// Register a callback for scan execution statistics before starting the scan. -- /// The callback may run on the thread that consumes or releases the stream. It -- /// must be thread-safe, must not throw, and must not re-enter the originating -- /// scanner. A non-null callback context must outlive the exported stream. -+ /// Register a callback for scan statistics after successful full exhaustion. -+ /// The callback is not guaranteed on error, cancellation, or early release. It -+ /// may run on the thread that observes EOF, must be thread-safe, must not throw, -+ /// and must not re-enter the originating scanner. The callback and a non-null -+ /// context must remain valid until the callback returns or the stream is released. - Scanner& statistics_callback(LanceScanStatisticsCallback callback, void* callback_ctx) { - if (lance_scanner_set_statistics_callback(handle_.get(), callback, callback_ctx) != 0) - check_error(); -diff --git a/src/scanner.rs b/src/scanner.rs -index d95089e..414c269 100644 ---- a/src/scanner.rs -+++ b/src/scanner.rs -@@ -315,7 +315,7 @@ pub struct LanceScanMetric { - pub value: u64, - } - --/// Borrowed view of the execution statistics for one completed scan. -+/// Borrowed view of the execution statistics for one fully consumed scan. - /// - /// The fixed fields are stable summary metrics. `metrics` contains additional - /// implementation-specific counters and timings and is valid only for the -@@ -333,7 +333,7 @@ pub struct LanceScanStatistics { - pub metrics_len: usize, - } - --/// Callback invoked when a scan stream reaches EOF, fails, or is released. -+/// Callback invoked after a scan stream is fully consumed to EOF. - /// - /// The callback is an FFI boundary and must return normally without unwinding - /// or throwing an exception. It must not call back into `lance_scanner_*` with -@@ -347,7 +347,7 @@ struct SendScanStatisticsCallback { - } - - // SAFETY: The C API requires the callback and its context to remain valid and --// safe to invoke from the thread that consumes or releases the scan stream. -+// safe to invoke from the thread that observes the scan stream's EOF. - unsafe impl Send for SendScanStatisticsCallback {} - unsafe impl Sync for SendScanStatisticsCallback {} - -@@ -652,14 +652,17 @@ unsafe fn scanner_set_substrait_filter_inner( - Ok(0) - } - --/// Register a callback that receives execution statistics when the scan stream --/// reaches EOF, fails, or is released. -+/// Register a callback that receives execution statistics after the scan stream -+/// is fully consumed to EOF. - /// --/// The callback and `callback_ctx` must remain valid until the scan stream is --/// finalized. Metric names and arrays passed to the callback are borrowed and --/// must be copied if the caller needs to retain them. The callback must be --/// thread-safe, must return normally without unwinding or throwing an --/// exception, and must not call `lance_scanner_*` with the originating scanner. -+/// The callback is not guaranteed to run if execution fails, the scan is -+/// cancelled, or the scanner / exported Arrow stream is released before EOF. -+/// The callback and `callback_ctx` must remain valid until the callback returns -+/// or, if it has not run, until the owning scan stream is released. Metric names -+/// and arrays passed to the callback are borrowed and must be copied if the -+/// caller needs to retain them. The callback must be thread-safe, must return -+/// normally without unwinding or throwing an exception, and must not call -+/// `lance_scanner_*` with the originating scanner. - #[unsafe(no_mangle)] - pub unsafe extern "C" fn lance_scanner_set_statistics_callback( - scanner: *mut LanceScanner, -diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs -index db35fea..c1bb71d 100644 ---- a/tests/c_api_test.rs -+++ b/tests/c_api_test.rs -@@ -372,7 +372,43 @@ fn test_scanner_statistics_callback_with_next_multi_fragment() { - assert!(captured.requests > 0); - assert!(captured.metrics.iter().all(|(name, _, _)| !name.is_empty())); - -+ let mut batch = ptr::null_mut(); -+ assert_eq!(unsafe { lance_scanner_next(scanner, &mut batch) }, 1); -+ assert!(batch.is_null()); -+ assert_eq!(captured.calls, 1, "callback must run exactly once"); -+ -+ unsafe { lance_scanner_close(scanner) }; -+ unsafe { lance_dataset_close(ds) }; -+} -+ -+#[test] -+fn test_scanner_statistics_callback_not_called_on_early_scanner_close() { -+ let (_tmp, uri) = create_test_dataset(); -+ let c_uri = c_str(&uri); -+ let ds = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) }; -+ assert!(!ds.is_null()); -+ -+ let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) }; -+ assert!(!scanner.is_null()); -+ let mut captured = CapturedScanStatistics::default(); -+ assert_eq!( -+ unsafe { -+ lance_scanner_set_statistics_callback( -+ scanner, -+ Some(capture_scan_statistics), -+ (&mut captured as *mut CapturedScanStatistics).cast(), -+ ) -+ }, -+ 0 -+ ); -+ -+ let mut batch = ptr::null_mut(); -+ assert_eq!(unsafe { lance_scanner_next(scanner, &mut batch) }, 0); -+ assert!(!batch.is_null()); -+ unsafe { lance_batch_free(batch) }; -+ - unsafe { lance_scanner_close(scanner) }; -+ assert_eq!(captured.calls, 0); - unsafe { lance_dataset_close(ds) }; - } - -@@ -398,9 +434,15 @@ fn test_scanner_statistics_callback_with_arrow_stream() { - ); - - let mut stream = FFI_ArrowArrayStream::empty(); -- assert_eq!(unsafe { lance_scanner_to_arrow_stream(scanner, &mut stream) }, 0); -+ assert_eq!( -+ unsafe { lance_scanner_to_arrow_stream(scanner, &mut stream) }, -+ 0 -+ ); - let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut stream) }.unwrap(); -- assert_eq!(reader.map(|batch| batch.unwrap().num_rows()).sum::(), 5); -+ assert_eq!( -+ reader.map(|batch| batch.unwrap().num_rows()).sum::(), -+ 5 -+ ); - assert_eq!(captured.calls, 1); - assert!(captured.bytes_read > 0); - -@@ -408,6 +450,74 @@ fn test_scanner_statistics_callback_with_arrow_stream() { - unsafe { lance_dataset_close(ds) }; - } - -+#[test] -+fn test_scanner_statistics_callback_not_called_on_early_arrow_stream_release() { -+ let (_tmp, uri) = create_test_dataset(); -+ let c_uri = c_str(&uri); -+ let ds = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) }; -+ assert!(!ds.is_null()); -+ -+ let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) }; -+ assert!(!scanner.is_null()); -+ let mut captured = CapturedScanStatistics::default(); -+ assert_eq!( -+ unsafe { -+ lance_scanner_set_statistics_callback( -+ scanner, -+ Some(capture_scan_statistics), -+ (&mut captured as *mut CapturedScanStatistics).cast(), -+ ) -+ }, -+ 0 -+ ); -+ -+ let mut stream = FFI_ArrowArrayStream::empty(); -+ assert_eq!( -+ unsafe { lance_scanner_to_arrow_stream(scanner, &mut stream) }, -+ 0 -+ ); -+ let mut reader = unsafe { ArrowArrayStreamReader::from_raw(&mut stream) }.unwrap(); -+ assert!(reader.next().unwrap().is_ok()); -+ drop(reader); -+ -+ assert_eq!(captured.calls, 0); -+ unsafe { lance_scanner_close(scanner) }; -+ assert_eq!(captured.calls, 0); -+ unsafe { lance_dataset_close(ds) }; -+} -+ -+#[test] -+fn test_scanner_statistics_callback_not_called_on_materialization_error() { -+ let (_tmp, uri) = create_test_dataset(); -+ let c_uri = c_str(&uri); -+ let ds = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) }; -+ assert!(!ds.is_null()); -+ -+ let bad_filter = c_str("NOT A VALID >>> FILTER ???"); -+ let scanner = unsafe { lance_scanner_new(ds, ptr::null(), bad_filter.as_ptr()) }; -+ assert!(!scanner.is_null()); -+ let mut captured = CapturedScanStatistics::default(); -+ assert_eq!( -+ unsafe { -+ lance_scanner_set_statistics_callback( -+ scanner, -+ Some(capture_scan_statistics), -+ (&mut captured as *mut CapturedScanStatistics).cast(), -+ ) -+ }, -+ 0 -+ ); -+ -+ let mut batch = ptr::null_mut(); -+ assert_eq!(unsafe { lance_scanner_next(scanner, &mut batch) }, -1); -+ assert!(batch.is_null()); -+ assert_eq!(captured.calls, 0); -+ -+ unsafe { lance_scanner_close(scanner) }; -+ assert_eq!(captured.calls, 0); -+ unsafe { lance_dataset_close(ds) }; -+} -+ - #[test] - fn test_scanner_statistics_callback_rejects_null_inputs() { - assert_eq!( -@@ -1367,6 +1477,17 @@ fn test_poll_next_basic() { - let c_uri = c_str(&uri_clone); - let ds = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) }; - let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) }; -+ let mut captured = CapturedScanStatistics::default(); -+ assert_eq!( -+ unsafe { -+ lance_scanner_set_statistics_callback( -+ scanner, -+ Some(capture_scan_statistics), -+ (&mut captured as *mut CapturedScanStatistics).cast(), -+ ) -+ }, -+ 0 -+ ); - - use std::sync::atomic::{AtomicBool, Ordering}; - static WOKE: AtomicBool = AtomicBool::new(false); -@@ -1401,6 +1522,15 @@ fn test_poll_next_basic() { - assert!(iterations < 1000, "poll loop should not spin forever"); - } - assert_eq!(total_rows, 5); -+ assert_eq!(captured.calls, 1); -+ -+ let mut batch: *mut LanceBatch = ptr::null_mut(); -+ assert_eq!( -+ unsafe { lance_scanner_poll_next(scanner, test_waker, ptr::null_mut(), &mut batch) }, -+ LancePollStatus::Finished -+ ); -+ assert!(batch.is_null()); -+ assert_eq!(captured.calls, 1, "callback must run exactly once"); - - unsafe { lance_scanner_close(scanner) }; - unsafe { lance_dataset_close(ds) }; - -From fa168ef99951d0396e50c060e540dee93e14e2be Mon Sep 17 00:00:00 2001 -From: zhangstar333 -Date: Mon, 24 Aug 2026 20:47:41 +0800 -Subject: [PATCH 3/3] update - ---- - include/lance/lance.h | 55 ++++++++--- - include/lance/lance.hpp | 22 ++++- - src/scanner.rs | 51 +++++++--- - tests/c_api_test.rs | 188 +++++++++++++++++++++++++++++++------ - tests/cpp/test_c_api.c | 39 +++++++- - tests/cpp/test_cpp_api.cpp | 28 +++++- - 6 files changed, 319 insertions(+), 64 deletions(-) - -diff --git a/include/lance/lance.h b/include/lance/lance.h -index 1a6822b..5b12f3d 100644 ---- a/include/lance/lance.h -+++ b/include/lance/lance.h -@@ -873,7 +873,8 @@ typedef enum { - * Borrowed view of one dynamically named scan metric. - * - * `name` is not NUL-terminated. `name` and this structure are valid only for -- * the duration of the LanceScanStatisticsCallback invocation. -+ * the duration of the LanceScanStatisticsCallback invocation. Metric order is -+ * unspecified. - */ - typedef struct { - const char* name; -@@ -905,11 +906,22 @@ typedef struct { - /** - * Receives scan statistics after a stream is fully consumed to EOF. - * -- * The statistics and all nested pointers are borrowed and valid only for the -- * duration of this call. The callback may run on the thread that observes EOF -- * and must therefore be thread-safe. It must return normally without throwing -- * an exception or unwinding, and must not call any `lance_scanner_*` function -- * with the originating scanner. -+ * `statistics` is non-NULL. It and all nested pointers are borrowed and valid -+ * only for the duration of this call. The callback may run on the thread that -+ * observes EOF and must therefore be thread-safe. It must return normally -+ * without throwing an exception or unwinding, and must not call any -+ * `lance_scanner_*` function with the originating scanner. -+ * -+ * From callback entry until the enclosing operation that observes EOF has -+ * returned to its caller, the callback must not directly or indirectly cause -+ * `get_schema`, `get_next`, `get_last_error`, or `release` to be called on any -+ * ArrowArrayStream derived from the originating scanner, nor cause such a -+ * stream to be moved, destroyed, or otherwise accessed. This includes signaling -+ * or scheduling another thread to act based only on callback completion: the -+ * callback returns before the enclosing stream operation does. Such interaction -+ * is reentrant and has undefined behavior. Normal access may resume only after -+ * the enclosing ArrowArrayStream `get_next`, `lance_scanner_next`, or -+ * `lance_scanner_poll_next` call returns to its caller. - * - * Scan statistics are diagnostic and best-effort. The callback must handle its - * own errors and must not use them to abort or throw across this FFI boundary. -@@ -925,15 +937,27 @@ typedef void (*LanceScanStatisticsCallback)( - * Must be called before starting the scan; registering after the scan starts - * returns an error. `callback` must not be NULL. `callback_ctx` may be NULL. A - * non-NULL `callback_ctx` must remain valid, and `callback` must remain valid, -- * until the callback returns or, if the callback has not run, until the owning -- * scan stream is released. For `lance_scanner_next` and -- * `lance_scanner_poll_next`, the scanner owns the stream. For an exported -- * ArrowArrayStream, the Arrow stream owns it independently of the scanner. -- * -- * The callback is invoked exactly once when the stream is fully consumed to -- * EOF. It is not guaranteed to run if execution fails, the scan is cancelled, -- * or the scanner / ArrowArrayStream is released before EOF. Replaces a -- * previously registered callback. -+ * until all of the following are true: the scanner is closed, every in-flight -+ * `lance_scanner_scan_async` call has delivered its completion callback, and -+ * every ArrowArrayStream derived from the scanner has been released. The -+ * registration remains installed after a callback returns and applies to -+ * streams created later from the same scanner. For `lance_scanner_next` and -+ * `lance_scanner_poll_next`, the scanner owns the stream. Exported and -+ * asynchronous ArrowArrayStreams own their registrations independently of the -+ * scanner and may invoke the callback after the scanner is closed. Concurrent -+ * streams may invoke the callback concurrently. -+ * -+ * The callback is invoked exactly once for each derived stream that is fully -+ * consumed to EOF. It is not guaranteed to run for a stream if execution fails, -+ * the scan is cancelled, or the scanner / ArrowArrayStream is released before -+ * EOF. Before scanning starts, a new registration replaces the previous one; -+ * after a successful replacement, the previous callback and context are no -+ * longer retained and may be retired. -+ * -+ * From callback entry until the enclosing EOF-observing operation returns, the -+ * callback must not directly or indirectly cause interaction with any -+ * ArrowArrayStream derived from this scanner; see LanceScanStatisticsCallback -+ * for the complete reentrancy restriction. - * - * @return 0 on success, -1 on error - */ -@@ -950,6 +974,7 @@ void lance_scanner_close(LanceScanner* scanner); - - /** - * Materialize the scan as an ArrowArrayStream (blocking). -+ * The scanner remains valid, and each call creates an independent stream. - * - * Reading the exported stream may surface a mid-iteration panic as one - * error through the Arrow C stream contract (nonzero get_next plus -diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp -index 4268801..8aa97e2 100644 ---- a/include/lance/lance.hpp -+++ b/include/lance/lance.hpp -@@ -1128,10 +1128,22 @@ class Scanner { - } - - /// Register a callback for scan statistics after successful full exhaustion. -- /// The callback is not guaranteed on error, cancellation, or early release. It -- /// may run on the thread that observes EOF, must be thread-safe, must not throw, -- /// and must not re-enter the originating scanner. The callback and a non-null -- /// context must remain valid until the callback returns or the stream is released. -+ /// The registration applies to every stream derived from this scanner, including -+ /// concurrent streams and streams created after an earlier callback returns. The -+ /// callback is not guaranteed on error, cancellation, or early release. It may -+ /// run on the thread that observes EOF, must be thread-safe, must not throw, and -+ /// must not re-enter the originating scanner. The callback and a non-null context -+ /// must remain valid until the scanner is closed, all async scan requests have -+ /// delivered their completion callbacks, and all derived streams are released. -+ /// From callback entry until the enclosing operation that observes EOF has -+ /// returned to its caller, the callback must not directly or indirectly cause -+ /// any ArrowArrayStream derived from this Scanner to be accessed, called, -+ /// released, moved, or destroyed. This includes signaling or scheduling another -+ /// thread to act based only on callback completion: the callback returns before -+ /// the enclosing stream operation does. Such interaction is reentrant and has -+ /// undefined behavior. Normal access may resume only after the enclosing -+ /// ArrowArrayStream `get_next`, `lance_scanner_next`, or -+ /// `lance_scanner_poll_next` call returns. - Scanner& statistics_callback(LanceScanStatisticsCallback callback, void* callback_ctx) { - if (lance_scanner_set_statistics_callback(handle_.get(), callback, callback_ctx) != 0) - check_error(); -@@ -1153,7 +1165,7 @@ class Scanner { - return index_segments(reinterpret_cast(uuids.data()), uuids.size()); - } - -- /// Materialize the scan as an ArrowArrayStream (blocking). -+ /// Materialize an independent ArrowArrayStream (blocking). The scanner remains valid. - void to_arrow_stream(ArrowArrayStream* out) { - if (lance_scanner_to_arrow_stream(handle_.get(), out) != 0) - check_error(); -diff --git a/src/scanner.rs b/src/scanner.rs -index 414c269..ef9d290 100644 ---- a/src/scanner.rs -+++ b/src/scanner.rs -@@ -305,7 +305,7 @@ pub enum LanceScanMetricKind { - /// Borrowed view of one dynamically named scan metric. - /// - /// `name` is not NUL-terminated. Both `name` and this structure are valid only --/// for the duration of the scan statistics callback. -+/// for the duration of the scan statistics callback. Metric order is unspecified. - #[repr(C)] - #[derive(Clone, Copy, Debug)] - pub struct LanceScanMetric { -@@ -318,8 +318,10 @@ pub struct LanceScanMetric { - /// Borrowed view of the execution statistics for one fully consumed scan. - /// - /// The fixed fields are stable summary metrics. `metrics` contains additional --/// implementation-specific counters and timings and is valid only for the --/// duration of the callback. -+/// implementation-specific counters and timings whose names are not a stable API -+/// and are intended only for diagnostics and profiles. Dynamic metrics are -+/// best-effort and may be omitted if they cannot be materialized. `metrics` is -+/// null when `metrics_len` is zero and is valid only for the callback duration. - #[repr(C)] - #[derive(Clone, Copy, Debug)] - pub struct LanceScanStatistics { -@@ -333,13 +335,20 @@ pub struct LanceScanStatistics { - pub metrics_len: usize, - } - --/// Callback invoked after a scan stream is fully consumed to EOF. -+/// Callback invoked once for each derived scan stream that is fully consumed to EOF. - /// - /// The callback is an FFI boundary and must return normally without unwinding - /// or throwing an exception. It must not call back into `lance_scanner_*` with --/// the originating scanner. -+/// the originating scanner. From callback entry until the enclosing operation -+/// that observes EOF has returned to its caller, the callback must not directly -+/// or indirectly cause any Arrow C stream derived from the originating scanner to -+/// be called, released, moved, destroyed, or otherwise accessed. This includes -+/// signaling or scheduling another thread to act based only on callback completion: -+/// the callback returns before the enclosing stream operation does. Such interaction -+/// is reentrant and has undefined behavior. Normal access may resume only after the -+/// enclosing `get_next`, `lance_scanner_next`, or `lance_scanner_poll_next` returns. - pub type LanceScanStatisticsCallback = -- Option; -+ Option; - - struct SendScanStatisticsCallback { - callback: unsafe extern "C" fn(*mut c_void, *const LanceScanStatistics), -@@ -347,7 +356,9 @@ struct SendScanStatisticsCallback { - } - - // SAFETY: The C API requires the callback and its context to remain valid and --// safe to invoke from the thread that observes the scan stream's EOF. -+// safe to invoke until the scanner is closed, every in-flight asynchronous scan -+// has delivered its completion callback, and every derived stream has been -+// released. Concurrent derived streams may invoke the callback concurrently. - unsafe impl Send for SendScanStatisticsCallback {} - unsafe impl Sync for SendScanStatisticsCallback {} - -@@ -657,12 +668,24 @@ unsafe fn scanner_set_substrait_filter_inner( - /// - /// The callback is not guaranteed to run if execution fails, the scan is - /// cancelled, or the scanner / exported Arrow stream is released before EOF. --/// The callback and `callback_ctx` must remain valid until the callback returns --/// or, if it has not run, until the owning scan stream is released. Metric names --/// and arrays passed to the callback are borrowed and must be copied if the --/// caller needs to retain them. The callback must be thread-safe, must return --/// normally without unwinding or throwing an exception, and must not call --/// `lance_scanner_*` with the originating scanner. -+/// The registration applies to every stream derived from this scanner, including -+/// streams created after an earlier callback has returned. The callback and -+/// `callback_ctx` must remain valid until the scanner is closed, every in-flight -+/// asynchronous scan has delivered its completion callback, and every derived -+/// stream has been released. -+/// Metric names and arrays passed to the callback are borrowed and must be copied -+/// if the caller needs to retain them. The callback must be thread-safe, must -+/// return normally without unwinding or throwing an exception, and must not call -+/// `lance_scanner_*` with the originating scanner. From callback entry until the -+/// enclosing operation that observes EOF has returned to its caller, the callback -+/// must not directly or indirectly cause any Arrow C stream derived from that -+/// scanner to be called, released, moved, destroyed, or otherwise accessed. This -+/// includes signaling or scheduling another thread to act based only on callback -+/// completion: the callback returns before the enclosing stream operation does. -+/// Such interaction is reentrant and has undefined behavior. Normal access may -+/// resume only after the enclosing `get_next`, `lance_scanner_next`, or -+/// `lance_scanner_poll_next` returns. Replacing the registration before scanning -+/// starts immediately releases the previous registration. - #[unsafe(no_mangle)] - pub unsafe extern "C" fn lance_scanner_set_statistics_callback( - scanner: *mut LanceScanner, -@@ -732,7 +755,7 @@ pub unsafe extern "C" fn lance_scanner_close(scanner: *mut LanceScanner) { - /// Materialize the scan as an Arrow C Data Interface `ArrowArrayStream`. - /// - /// This is the preferred API for simple integrations — blocks the calling thread. --/// The scanner is consumed by this call and should not be used afterward (close it). -+/// The scanner remains valid and may be used to create additional streams. - /// - /// The exported stream is panic-guarded (issue #61): a panic during export - /// poisons the scanner — this call returns -1 with `LANCE_ERR_PANIC`, and -diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs -index c1bb71d..daa7425 100644 ---- a/tests/c_api_test.rs -+++ b/tests/c_api_test.rs -@@ -10,6 +10,7 @@ use std::ffi::{CString, c_char, c_void}; - use std::process::Command; - use std::ptr; - use std::sync::Arc; -+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering as AtomicOrdering}; - - use arrow::ffi::from_ffi; - use arrow::ffi::{FFI_ArrowArray, FFI_ArrowSchema}; -@@ -151,6 +152,29 @@ unsafe extern "C" fn capture_scan_statistics( - .collect(); - } - -+#[derive(Default)] -+struct AtomicScanStatisticsCapture { -+ calls: AtomicUsize, -+ invalid_statistics: AtomicBool, -+} -+ -+unsafe extern "C" fn capture_scan_statistics_atomically( -+ callback_ctx: *mut c_void, -+ statistics: *const LanceScanStatistics, -+) { -+ if callback_ctx.is_null() { -+ return; -+ } -+ let captured = unsafe { &*callback_ctx.cast::() }; -+ if statistics.is_null() { -+ captured -+ .invalid_statistics -+ .store(true, AtomicOrdering::SeqCst); -+ return; -+ } -+ captured.calls.fetch_add(1, AtomicOrdering::SeqCst); -+} -+ - /// Helper: build a tiny dataset whose `value` column is nullable AND contains - /// at least one NULL. Used by tests that need to exercise upstream's - /// nullability-tightening pre-scan failure path. -@@ -413,7 +437,7 @@ fn test_scanner_statistics_callback_not_called_on_early_scanner_close() { - } - - #[test] --fn test_scanner_statistics_callback_with_arrow_stream() { -+fn test_scanner_statistics_callback_applies_to_reused_scanner() { - let (_tmp, uri) = create_test_dataset(); - let c_uri = c_str(&uri); - let ds = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) }; -@@ -433,20 +457,87 @@ fn test_scanner_statistics_callback_with_arrow_stream() { - 0 - ); - -- let mut stream = FFI_ArrowArrayStream::empty(); -+ let mut first_stream = FFI_ArrowArrayStream::empty(); - assert_eq!( -- unsafe { lance_scanner_to_arrow_stream(scanner, &mut stream) }, -+ unsafe { lance_scanner_to_arrow_stream(scanner, &mut first_stream) }, - 0 - ); -- let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut stream) }.unwrap(); -+ let first_reader = unsafe { ArrowArrayStreamReader::from_raw(&mut first_stream) }.unwrap(); - assert_eq!( -- reader.map(|batch| batch.unwrap().num_rows()).sum::(), -+ first_reader -+ .map(|batch| batch.unwrap().num_rows()) -+ .sum::(), - 5 - ); - assert_eq!(captured.calls, 1); -- assert!(captured.bytes_read > 0); - -+ let mut second_stream = FFI_ArrowArrayStream::empty(); -+ assert_eq!( -+ unsafe { lance_scanner_to_arrow_stream(scanner, &mut second_stream) }, -+ 0 -+ ); - unsafe { lance_scanner_close(scanner) }; -+ -+ let second_reader = unsafe { ArrowArrayStreamReader::from_raw(&mut second_stream) }.unwrap(); -+ assert_eq!( -+ second_reader -+ .map(|batch| batch.unwrap().num_rows()) -+ .sum::(), -+ 5 -+ ); -+ assert_eq!(captured.calls, 2); -+ -+ unsafe { lance_dataset_close(ds) }; -+} -+ -+#[test] -+fn test_scanner_statistics_callback_supports_concurrent_exported_streams() { -+ struct SendableArrowStream(FFI_ArrowArrayStream); -+ unsafe impl Send for SendableArrowStream {} -+ -+ fn consume_stream(mut stream: SendableArrowStream) -> usize { -+ let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut stream.0) }.unwrap(); -+ reader.map(|batch| batch.unwrap().num_rows()).sum() -+ } -+ -+ let (_tmp, uri) = create_test_dataset(); -+ let c_uri = c_str(&uri); -+ let ds = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) }; -+ assert!(!ds.is_null()); -+ -+ let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) }; -+ assert!(!scanner.is_null()); -+ let captured = Arc::new(AtomicScanStatisticsCapture::default()); -+ assert_eq!( -+ unsafe { -+ lance_scanner_set_statistics_callback( -+ scanner, -+ Some(capture_scan_statistics_atomically), -+ Arc::as_ptr(&captured).cast_mut().cast(), -+ ) -+ }, -+ 0 -+ ); -+ -+ let mut first_stream = FFI_ArrowArrayStream::empty(); -+ let mut second_stream = FFI_ArrowArrayStream::empty(); -+ assert_eq!( -+ unsafe { lance_scanner_to_arrow_stream(scanner, &mut first_stream) }, -+ 0 -+ ); -+ assert_eq!( -+ unsafe { lance_scanner_to_arrow_stream(scanner, &mut second_stream) }, -+ 0 -+ ); -+ unsafe { lance_scanner_close(scanner) }; -+ -+ let first = std::thread::spawn(move || consume_stream(SendableArrowStream(first_stream))); -+ let second = std::thread::spawn(move || consume_stream(SendableArrowStream(second_stream))); -+ assert_eq!(first.join().unwrap(), 5); -+ assert_eq!(second.join().unwrap(), 5); -+ assert_eq!(captured.calls.load(AtomicOrdering::SeqCst), 2); -+ assert!(!captured.invalid_statistics.load(AtomicOrdering::SeqCst)); -+ - unsafe { lance_dataset_close(ds) }; - } - -@@ -546,6 +637,55 @@ fn test_scanner_statistics_callback_rejects_null_inputs() { - unsafe { lance_dataset_close(ds) }; - } - -+#[test] -+fn test_scanner_statistics_callback_replaces_registration_before_scan() { -+ let (_tmp, uri) = create_test_dataset(); -+ let c_uri = c_str(&uri); -+ let ds = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) }; -+ assert!(!ds.is_null()); -+ let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) }; -+ assert!(!scanner.is_null()); -+ -+ let mut replaced = CapturedScanStatistics::default(); -+ let mut active = CapturedScanStatistics::default(); -+ assert_eq!( -+ unsafe { -+ lance_scanner_set_statistics_callback( -+ scanner, -+ Some(capture_scan_statistics), -+ (&mut replaced as *mut CapturedScanStatistics).cast(), -+ ) -+ }, -+ 0 -+ ); -+ assert_eq!( -+ unsafe { -+ lance_scanner_set_statistics_callback( -+ scanner, -+ Some(capture_scan_statistics), -+ (&mut active as *mut CapturedScanStatistics).cast(), -+ ) -+ }, -+ 0 -+ ); -+ -+ let mut stream = FFI_ArrowArrayStream::empty(); -+ assert_eq!( -+ unsafe { lance_scanner_to_arrow_stream(scanner, &mut stream) }, -+ 0 -+ ); -+ let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut stream) }.unwrap(); -+ assert_eq!( -+ reader.map(|batch| batch.unwrap().num_rows()).sum::(), -+ 5 -+ ); -+ assert_eq!(replaced.calls, 0); -+ assert_eq!(active.calls, 1); -+ -+ unsafe { lance_scanner_close(scanner) }; -+ unsafe { lance_dataset_close(ds) }; -+} -+ - #[test] - fn test_scanner_statistics_callback_rejects_registration_after_scan_started() { - let (_tmp, uri) = create_test_dataset(); -@@ -814,6 +954,17 @@ fn test_scanner_scan_async() { - - let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) }; - assert!(!scanner.is_null()); -+ let captured = Arc::new(AtomicScanStatisticsCapture::default()); -+ assert_eq!( -+ unsafe { -+ lance_scanner_set_statistics_callback( -+ scanner, -+ Some(capture_scan_statistics_atomically), -+ Arc::as_ptr(&captured).cast_mut().cast(), -+ ) -+ }, -+ 0 -+ ); - - // Synchronization primitive for the async callback. - struct CallbackResult { -@@ -845,6 +996,7 @@ fn test_scanner_scan_async() { - on_complete, - Arc::as_ptr(&pair_clone) as *mut std::ffi::c_void, - ); -+ lance_scanner_close(scanner); - } - - // Wait for callback. -@@ -861,8 +1013,9 @@ fn test_scanner_scan_async() { - let reader = unsafe { ArrowArrayStreamReader::from_raw(ffi_stream) }.unwrap(); - let total_rows: usize = reader.map(|r| r.unwrap().num_rows()).sum(); - assert_eq!(total_rows, 5); -+ assert_eq!(captured.calls.load(AtomicOrdering::SeqCst), 1); -+ assert!(!captured.invalid_statistics.load(AtomicOrdering::SeqCst)); - -- unsafe { lance_scanner_close(scanner) }; - unsafe { lance_dataset_close(ds) }; - } - -@@ -1477,18 +1630,6 @@ fn test_poll_next_basic() { - let c_uri = c_str(&uri_clone); - let ds = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) }; - let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) }; -- let mut captured = CapturedScanStatistics::default(); -- assert_eq!( -- unsafe { -- lance_scanner_set_statistics_callback( -- scanner, -- Some(capture_scan_statistics), -- (&mut captured as *mut CapturedScanStatistics).cast(), -- ) -- }, -- 0 -- ); -- - use std::sync::atomic::{AtomicBool, Ordering}; - static WOKE: AtomicBool = AtomicBool::new(false); - unsafe extern "C" fn test_waker(_ctx: *mut std::ffi::c_void) { -@@ -1522,15 +1663,6 @@ fn test_poll_next_basic() { - assert!(iterations < 1000, "poll loop should not spin forever"); - } - assert_eq!(total_rows, 5); -- assert_eq!(captured.calls, 1); -- -- let mut batch: *mut LanceBatch = ptr::null_mut(); -- assert_eq!( -- unsafe { lance_scanner_poll_next(scanner, test_waker, ptr::null_mut(), &mut batch) }, -- LancePollStatus::Finished -- ); -- assert!(batch.is_null()); -- assert_eq!(captured.calls, 1, "callback must run exactly once"); - - unsafe { lance_scanner_close(scanner) }; - unsafe { lance_dataset_close(ds) }; -diff --git a/tests/cpp/test_c_api.c b/tests/cpp/test_c_api.c -index b3b78f0..4499cb9 100644 ---- a/tests/cpp/test_c_api.c -+++ b/tests/cpp/test_c_api.c -@@ -39,6 +39,36 @@ - } \ - } while (0) - -+typedef struct { -+ uint64_t calls; -+ uint64_t bytes_read; -+ int invalid; -+} ScanStatisticsCapture; -+ -+static void capture_scan_statistics( -+ void *callback_ctx, -+ const LanceScanStatistics *statistics -+) { -+ if (callback_ctx == NULL) return; -+ ScanStatisticsCapture *captured = (ScanStatisticsCapture *)callback_ctx; -+ if (statistics == NULL || -+ (statistics->metrics_len > 0 && statistics->metrics == NULL)) { -+ captured->invalid = 1; -+ return; -+ } -+ for (size_t i = 0; i < statistics->metrics_len; ++i) { -+ const LanceScanMetric *metric = &statistics->metrics[i]; -+ if ((metric->name_len > 0 && metric->name == NULL) || -+ (metric->kind != LANCE_SCAN_METRIC_COUNT && -+ metric->kind != LANCE_SCAN_METRIC_TIME_NANOSECONDS)) { -+ captured->invalid = 1; -+ return; -+ } -+ } -+ captured->calls += 1; -+ captured->bytes_read = statistics->bytes_read; -+} -+ - static void test_open_and_metadata(const char *uri) { - printf(" test_open_and_metadata... "); - -@@ -84,10 +114,14 @@ static void test_scan(const char *uri) { - /* Full scan via ArrowArrayStream */ - LanceScanner *scanner = lance_scanner_new(ds, NULL, NULL); - ASSERT(scanner != NULL, "scanner creation failed"); -+ ScanStatisticsCapture captured = {0}; -+ int32_t rc = lance_scanner_set_statistics_callback( -+ scanner, capture_scan_statistics, &captured); -+ ASSERT(rc == 0, "statistics callback registration failed"); - - struct ArrowArrayStream stream; - memset(&stream, 0, sizeof(stream)); -- int32_t rc = lance_scanner_to_arrow_stream(scanner, &stream); -+ rc = lance_scanner_to_arrow_stream(scanner, &stream); - ASSERT(rc == 0, "to_arrow_stream failed"); - - /* Read schema from stream */ -@@ -113,6 +147,9 @@ static void test_scan(const char *uri) { - } - - ASSERT(total_rows == expected_rows, "row count mismatch"); -+ ASSERT(captured.calls == 1, "statistics callback count mismatch"); -+ ASSERT(captured.bytes_read > 0, "statistics should report bytes read"); -+ ASSERT(captured.invalid == 0, "statistics callback received invalid data"); - printf("rows=%llu... ", (unsigned long long)total_rows); - - if (stream.release) stream.release(&stream); -diff --git a/tests/cpp/test_cpp_api.cpp b/tests/cpp/test_cpp_api.cpp -index f8ae701..3293bfb 100644 ---- a/tests/cpp/test_cpp_api.cpp -+++ b/tests/cpp/test_cpp_api.cpp -@@ -25,6 +25,25 @@ - #define TEST(name) printf(" %s... ", #name) - #define PASS() printf("OK\n") - -+struct ScanStatisticsCapture { -+ uint64_t calls = 0; -+ uint64_t bytes_read = 0; -+ bool invalid = false; -+}; -+ -+static void capture_scan_statistics( -+ void* callback_ctx, -+ const LanceScanStatistics* statistics) noexcept { -+ if (!callback_ctx) return; -+ auto* captured = static_cast(callback_ctx); -+ if (!statistics || (statistics->metrics_len > 0 && !statistics->metrics)) { -+ captured->invalid = true; -+ return; -+ } -+ captured->calls += 1; -+ captured->bytes_read = statistics->bytes_read; -+} -+ - static void test_dataset_open(const std::string& uri) { - TEST(test_dataset_open); - -@@ -70,7 +89,11 @@ static void test_scanner_fluent(const std::string& uri) { - - // Fluent builder pattern. - auto scanner = ds.scan(); -- scanner.limit(5).offset(0).batch_size(2); -+ ScanStatisticsCapture captured; -+ scanner.limit(5) -+ .offset(0) -+ .batch_size(2) -+ .statistics_callback(capture_scan_statistics, &captured); - - ArrowArrayStream stream; - memset(&stream, 0, sizeof(stream)); -@@ -89,6 +112,9 @@ static void test_scanner_fluent(const std::string& uri) { - } - - assert(total == 5); -+ assert(captured.calls == 1); -+ assert(captured.bytes_read > 0); -+ assert(!captured.invalid); - printf("rows=%llu... ", (unsigned long long)total); - - if (stream.release) stream.release(&stream); diff --git a/thirdparty/patches/lance-c-0.1.9-pr-73.patch b/thirdparty/patches/lance-c-0.1.9-pr-73.patch new file mode 100644 index 00000000000000..7d5eb45ad6ce7d --- /dev/null +++ b/thirdparty/patches/lance-c-0.1.9-pr-73.patch @@ -0,0 +1,2317 @@ +From a4c71309ddb76ad79808e3e8f7797bcd9bfc174a Mon Sep 17 00:00:00 2001 +From: zhangstar333 +Date: Thu, 3 Sep 2026 13:00:43 +0800 +Subject: [PATCH] foyer + +--- + Cargo.lock | 205 ++++++ + Cargo.toml | 4 + + README.md | 22 + + include/lance/lance.h | 63 ++ + include/lance/lance.hpp | 29 + + src/data_cache.rs | 68 ++ + src/dataset.rs | 11 + + src/foyer_data_cache.rs | 1215 ++++++++++++++++++++++++++++++++++++ + src/lib.rs | 4 + + src/restore.rs | 8 + + src/session.rs | 13 +- + src/writer.rs | 1 + + tests/c_api_test.rs | 221 +++++++ + tests/cpp/test_c_api.c | 32 + + tests/cpp/test_cpp_api.cpp | 23 + + 15 files changed, 1918 insertions(+), 1 deletion(-) + create mode 100644 src/data_cache.rs + create mode 100644 src/foyer_data_cache.rs + +diff --git a/Cargo.lock b/Cargo.lock +index 60c1caf..85536f5 100644 +--- a/Cargo.lock ++++ b/Cargo.lock +@@ -434,6 +434,16 @@ dependencies = [ + "loom", + ] + ++[[package]] ++name = "asyncband" ++version = "0.6.7" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "94a214ba60d6231afd0e805e3c27c45a1626d9debaa5a5061c45a1ea1b2f1ed0" ++dependencies = [ ++ "hashbrown 0.17.1", ++ "slab", ++] ++ + [[package]] + name = "atoi" + version = "2.0.0" +@@ -1267,6 +1277,17 @@ version = "0.8.7" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + ++[[package]] ++name = "core_affinity" ++version = "0.8.3" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "a034b3a7b624016c6e13f5df875747cc25f884156aad2abd12b6c46797971342" ++dependencies = [ ++ "libc", ++ "num_cpus", ++ "winapi", ++] ++ + [[package]] + name = "countio" + version = "0.3.0" +@@ -2112,6 +2133,12 @@ dependencies = [ + "url", + ] + ++[[package]] ++name = "datasketches" ++version = "0.3.0" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "46c4cf71a36b46dcfc00e5014c0c20ccad2b1b6a008304d7d57d2749b2d41b3d" ++ + [[package]] + name = "der" + version = "0.7.10" +@@ -2298,6 +2325,16 @@ version = "0.2.3" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55" + ++[[package]] ++name = "fastant" ++version = "0.1.11" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "2e825441bfb2d831c47c97d05821552db8832479f44c571b97fededbf0099c07" ++dependencies = [ ++ "small_ctor", ++ "web-time", ++] ++ + [[package]] + name = "fastrand" + version = "2.3.0" +@@ -2369,6 +2406,127 @@ dependencies = [ + "percent-encoding", + ] + ++[[package]] ++name = "foyer" ++version = "0.22.4" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "cab5c4bac30455a0dbc4c858436fb440d51bc6543daafbdf35c5e6ca94c0afd7" ++dependencies = [ ++ "anyhow", ++ "asyncband", ++ "equivalent", ++ "foyer-common", ++ "foyer-memory", ++ "foyer-storage", ++ "foyer-tokio", ++ "futures-util", ++ "mixtrics", ++ "pin-project", ++ "serde", ++ "tracing", ++] ++ ++[[package]] ++name = "foyer-common" ++version = "0.22.4" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "e3634c6f3da978b6cae98d54367a2342041d3a4786b20c0384164cc7fce94787" ++dependencies = [ ++ "anyhow", ++ "bytes", ++ "cfg-if 1.0.4", ++ "foyer-tokio", ++ "mixtrics", ++ "parking_lot", ++ "pin-project", ++ "twox-hash", ++] ++ ++[[package]] ++name = "foyer-intrusive-collections" ++version = "0.10.0-dev" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "6e4fee46bea69e0596130e3210e65d3424e0ac1e6df3bde6636304bdf1ca4a3b" ++dependencies = [ ++ "memoffset", ++] ++ ++[[package]] ++name = "foyer-memory" ++version = "0.22.4" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "5349a61af676b3275bfef7a5fceceeadf6b0a7dd9ec14ba0edd78e1ff771d101" ++dependencies = [ ++ "anyhow", ++ "asyncband", ++ "bitflags", ++ "datasketches", ++ "equivalent", ++ "foyer-common", ++ "foyer-intrusive-collections", ++ "foyer-tokio", ++ "futures-util", ++ "hashbrown 0.17.1", ++ "itertools 0.15.0", ++ "mixtrics", ++ "parking_lot", ++ "paste", ++ "pin-project", ++ "serde", ++ "tracing", ++] ++ ++[[package]] ++name = "foyer-storage" ++version = "0.22.4" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "49cfcce3c10a1f2ac65bfcf0bd85501462d454c594c8a16a4a27cce773599381" ++dependencies = [ ++ "allocator-api2", ++ "anyhow", ++ "asyncband", ++ "bytes", ++ "core_affinity", ++ "equivalent", ++ "fastant", ++ "foyer-common", ++ "foyer-memory", ++ "foyer-tokio", ++ "fs4", ++ "futures-core", ++ "futures-util", ++ "hashbrown 0.17.1", ++ "io-uring", ++ "itertools 0.15.0", ++ "libc", ++ "lz4", ++ "parking_lot", ++ "pin-project", ++ "rand 0.10.1", ++ "tracing", ++ "twox-hash", ++ "zstd", ++] ++ ++[[package]] ++name = "foyer-tokio" ++version = "0.22.4" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "6b7315103199f3415a010befd6dd5a1c8e7082fbc49c0194931e65629995d9d1" ++dependencies = [ ++ "tokio", ++] ++ ++[[package]] ++name = "fs4" ++version = "0.13.1" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4" ++dependencies = [ ++ "rustix", ++ "windows-sys 0.59.0", ++] ++ + [[package]] + name = "fs_extra" + version = "1.3.0" +@@ -3392,6 +3550,15 @@ dependencies = [ + "either", + ] + ++[[package]] ++name = "itertools" ++version = "0.15.0" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" ++dependencies = [ ++ "either", ++] ++ + [[package]] + name = "itoa" + version = "1.0.18" +@@ -3703,8 +3870,11 @@ dependencies = [ + "arrow", + "arrow-array", + "arrow-schema", ++ "async-trait", ++ "bytes", + "chrono", + "datafusion", ++ "foyer", + "futures", + "half", + "lance", +@@ -3718,6 +3888,7 @@ dependencies = [ + "lance-table", + "libc", + "log", ++ "object_store", + "pin-project", + "prost", + "snafu", +@@ -4399,6 +4570,15 @@ version = "2.8.0" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + ++[[package]] ++name = "memoffset" ++version = "0.9.1" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" ++dependencies = [ ++ "autocfg", ++] ++ + [[package]] + name = "mime" + version = "0.3.17" +@@ -4436,6 +4616,16 @@ dependencies = [ + "windows-sys 0.61.2", + ] + ++[[package]] ++name = "mixtrics" ++version = "0.2.5" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "2c46b5adfb7a3ae4996d327a5bdc90e78fec025806dd312bdbe6f07a755e0ec9" ++dependencies = [ ++ "itertools 0.15.0", ++ "parking_lot", ++] ++ + [[package]] + name = "moka" + version = "0.12.15" +@@ -6547,6 +6737,12 @@ version = "0.4.12" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + ++[[package]] ++name = "small_ctor" ++version = "0.1.2" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "88414a5ca1f85d82cc34471e975f0f74f6aa54c40f062efa42c0080e7f763f81" ++ + [[package]] + name = "smallvec" + version = "1.15.1" +@@ -7831,6 +8027,15 @@ dependencies = [ + "windows-targets 0.52.6", + ] + ++[[package]] ++name = "windows-sys" ++version = "0.59.0" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" ++dependencies = [ ++ "windows-targets 0.52.6", ++] ++ + [[package]] + name = "windows-sys" + version = "0.60.2" +diff --git a/Cargo.toml b/Cargo.toml +index d072a5d..342928c 100644 +--- a/Cargo.toml ++++ b/Cargo.toml +@@ -30,6 +30,8 @@ datafusion = { version = "54.0.0", default-features = false } + arrow = { version = "58.0.0", features = ["prettyprint", "ffi"] } + arrow-array = "58.0.0" + arrow-schema = "58.0.0" ++async-trait = "0.1" ++bytes = "1" + # Direct to name `chrono::TimeDelta` (the field type of lance's public + # `AutoCleanupParams`) and `chrono::DateTime`/`Utc` (index metadata + # timestamps); already in the graph transitively via lance. +@@ -37,8 +39,10 @@ chrono = { version = "0.4", default-features = false } + half = "2" + tokio = { version = "1", features = ["rt-multi-thread", "sync"] } + futures = "0.3" ++foyer = "=0.22.4" + log = "0.4" + libc = "0.2" ++object_store = "0.13.2" + pin-project = "1.0" + prost = "0.14" + snafu = "0.9" +diff --git a/README.md b/README.md +index 2056671..d9a5f2b 100644 +--- a/README.md ++++ b/README.md +@@ -68,6 +68,7 @@ Based on the [liblance RFC](https://github.com/lance-format/lance/discussions/60 + | [x] | Async scan | Callback-based `lance_scanner_scan_async()` for non-blocking scans | + | [x] | Dataset metadata | `lance_dataset_version()`, `lance_dataset_count_rows()`, `lance_dataset_latest_version()` | + | [x] | Filter pushdown | `lance_scanner_set_substrait_filter()` accepts a serialized Substrait `ExtendedExpression`; `lance_scanner_additional_sql_filter()` adds SQL predicates with AND before scanning starts | ++| [x] | Data-file cache | Optional Foyer memory/disk cache for immutable `data/*.lance` reads | + + ## Building + +@@ -197,6 +198,27 @@ auto ds = lance::Dataset::open_with_session(session, "data.lance"); + auto stats = session.cache_stats(); + ``` + ++To add a process-local memory/disk cache for remote Lance data-file reads, ++create the session with Foyer configuration. The cache is deliberately narrow: ++whole-object, single-range, and batched range reads of direct `data/*.lance` ++children are cached. Conditional and versioned reads, plus manifests, deletion ++files, and index files, keep using Lance's normal paths. Use one shared session ++for datasets that share the cache directory. ++ ++```cpp ++lance::DataCacheOptions data_cache{ ++ "/var/cache/my-service/lance", ++ 512ULL * 1024 * 1024, // memory tier ++ 100ULL * 1024 * 1024 * 1024, // disk tier ++ 1ULL * 1024 * 1024, // range-cache block ++}; ++lance::Session session( ++ 6ULL * 1024 * 1024 * 1024, ++ 1ULL * 1024 * 1024 * 1024, ++ data_cache); ++auto ds = lance::Dataset::open_with_session(session, "s3://bucket/data.lance"); ++``` ++ + ### Open at a specific version + + `lance_dataset_open` takes a `version` argument — `0` means the latest, any +diff --git a/include/lance/lance.h b/include/lance/lance.h +index 3bf291f..e4c593e 100644 +--- a/include/lance/lance.h ++++ b/include/lance/lance.h +@@ -206,6 +206,36 @@ typedef struct LanceSessionCacheStats { + uint64_t metadata_cache_size_bytes; + } LanceSessionCacheStats; + ++/** ++ * Configuration for the optional Foyer cache of immutable Lance data files. ++ * ++ * Whole-object, single-range, and batched range reads of direct ++ * `data/.lance` children are cached. Conditional and versioned reads, ++ * plus metadata, deletion files, and index files, continue to use Lance's normal ++ * paths. ++ */ ++typedef struct LanceDataCacheOptions { ++ const char* directory; ++ /** Maximum raw data bytes retained by Foyer's in-memory tier. */ ++ uint64_t memory_capacity_bytes; ++ /** Maximum bytes allocated to Foyer's disk tier. */ ++ uint64_t disk_capacity_bytes; ++ /** Data-file range cache unit, in bytes. */ ++ uint64_t read_block_size_bytes; ++} LanceDataCacheOptions; ++ ++/** ++ * Cumulative Foyer data-cache statistics for one opened dataset handle. ++ * ++ * Successful reads are accumulated. Both fields measure bytes returned to the ++ * dataset reader. Their sum is the logical data-file range bytes observed by ++ * the Foyer wrapper; block-aligned origin read amplification is not included. ++ */ ++typedef struct LanceDataCacheStatistics { ++ uint64_t bytes_read_from_cache; ++ uint64_t bytes_read_from_remote; ++} LanceDataCacheStatistics; ++ + /** + * Create a session that can share metadata and index caches across datasets. + * +@@ -217,6 +247,25 @@ LanceSession* lance_session_new( + uint64_t metadata_cache_size_bytes + ); + ++/** ++ * Create a shared Lance session with a Foyer data-file cache. ++ * ++ * `data_cache_options` and its `directory` field must not be NULL. The cache ++ * directory and all capacities are process configuration and remain owned by ++ * the caller; their values are copied during this call. ++ * ++ * `read_block_size_bytes` must be a non-zero multiple of 4096. The memory ++ * capacity must hold at least one read block. The disk capacity must be a ++ * multiple of 4096 and hold at least two read blocks. ++ * ++ * @return Session handle, or NULL on error ++ */ ++LanceSession* lance_session_new_with_data_cache( ++ uint64_t index_cache_size_bytes, ++ uint64_t metadata_cache_size_bytes, ++ const LanceDataCacheOptions* data_cache_options ++); ++ + /** + * Close a session handle. Safe to call with NULL. Datasets previously opened + * with the session remain valid and retain the shared cache state. +@@ -273,6 +322,20 @@ LanceDataset* lance_dataset_open_with_session( + const LanceSession* session + ); + ++/** ++ * Copy this dataset handle's cumulative data-cache statistics. ++ * ++ * A dataset not opened with a data cache reports all-zero statistics. The ++ * snapshot belongs only to this dataset handle; the underlying cache may ++ * still be shared by other datasets through a session. ++ * ++ * @return 0 on success, -1 on error ++ */ ++int32_t lance_dataset_get_data_cache_statistics( ++ const LanceDataset* dataset, ++ LanceDataCacheStatistics* out_statistics ++); ++ + /** Close and free a dataset handle. Safe to call with NULL. */ + void lance_dataset_close(LanceDataset* dataset); + +diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp +index 6cf245f..c1102e4 100644 +--- a/include/lance/lance.hpp ++++ b/include/lance/lance.hpp +@@ -171,6 +171,13 @@ struct SqlColumn { + + // ─── Shared Session ────────────────────────────────────────────────────────── + ++struct DataCacheOptions { ++ std::string directory; ++ uint64_t memory_capacity_bytes; ++ uint64_t disk_capacity_bytes; ++ uint64_t read_block_size_bytes; ++}; ++ + class Session { + Handle handle_; + +@@ -180,6 +187,21 @@ class Session { + if (!handle_) check_error(); + } + ++ Session(uint64_t index_cache_size_bytes, ++ uint64_t metadata_cache_size_bytes, ++ const DataCacheOptions& data_cache_options) { ++ LanceDataCacheOptions options{ ++ data_cache_options.directory.c_str(), ++ data_cache_options.memory_capacity_bytes, ++ data_cache_options.disk_capacity_bytes, ++ data_cache_options.read_block_size_bytes, ++ }; ++ handle_ = Handle( ++ lance_session_new_with_data_cache( ++ index_cache_size_bytes, metadata_cache_size_bytes, &options)); ++ if (!handle_) check_error(); ++ } ++ + LanceSessionCacheStats cache_stats() const { + LanceSessionCacheStats stats{}; + if (lance_session_get_cache_stats(handle_.get(), &stats) != 0) +@@ -260,6 +282,13 @@ class Dataset { + return Dataset(ds); + } + ++ LanceDataCacheStatistics data_cache_statistics() const { ++ LanceDataCacheStatistics statistics{}; ++ if (lance_dataset_get_data_cache_statistics(handle_.get(), &statistics) != 0) ++ check_error(); ++ return statistics; ++ } ++ + /// Write an Arrow record batch stream to a Lance dataset and return the + /// open dataset at the committed version. + /// +diff --git a/src/data_cache.rs b/src/data_cache.rs +new file mode 100644 +index 0000000..430b4c8 +--- /dev/null ++++ b/src/data_cache.rs +@@ -0,0 +1,68 @@ ++// SPDX-License-Identifier: Apache-2.0 ++// SPDX-FileCopyrightText: Copyright The Lance Authors ++ ++//! Private bridge between a shared data-cache backend and dataset handles. ++ ++use std::fmt::Debug; ++use std::sync::Arc; ++ ++use lance::Dataset; ++use lance_core::Result; ++ ++use crate::dataset::LanceDataset; ++use crate::error::ffi_try; ++ ++/// Data-cache statistics owned by one opened dataset. ++#[repr(C)] ++#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] ++pub struct LanceDataCacheStatistics { ++ /// Requested bytes returned from usable data-cache entries. ++ pub bytes_read_from_cache: u64, ++ /// Requested bytes returned after a data-cache miss or fallback. ++ pub bytes_read_from_remote: u64, ++} ++ ++pub(crate) trait DatasetDataCache: Debug + Send + Sync { ++ fn snapshot(&self) -> LanceDataCacheStatistics; ++ ++ fn attach_fresh(&self, dataset: Dataset) -> (Dataset, Arc); ++} ++ ++pub(crate) trait DataCacheFactory: Debug + Send + Sync { ++ fn attach(&self, dataset: Dataset) -> (Dataset, Arc); ++} ++ ++/// Copy this dataset handle's cumulative data-cache statistics into ++/// `out_statistics`. ++/// ++/// A dataset not opened with a data cache reports all-zero statistics. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_dataset_get_data_cache_statistics( ++ dataset: *const LanceDataset, ++ out_statistics: *mut LanceDataCacheStatistics, ++) -> i32 { ++ ffi_try!( ++ unsafe { dataset_get_data_cache_statistics_inner(dataset, out_statistics) }, ++ neg ++ ) ++} ++ ++unsafe fn dataset_get_data_cache_statistics_inner( ++ dataset: *const LanceDataset, ++ out_statistics: *mut LanceDataCacheStatistics, ++) -> Result { ++ if dataset.is_null() || out_statistics.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "dataset and out_statistics must not be NULL".into(), ++ )); ++ } ++ let dataset = unsafe { &*dataset }; ++ let statistics = dataset ++ .data_cache ++ .as_ref() ++ .map_or_else(LanceDataCacheStatistics::default, |cache| cache.snapshot()); ++ unsafe { ++ std::ptr::write_unaligned(out_statistics, statistics); ++ } ++ Ok(0) ++} +diff --git a/src/dataset.rs b/src/dataset.rs +index cc1f87c..76fd39e 100644 +--- a/src/dataset.rs ++++ b/src/dataset.rs +@@ -14,6 +14,7 @@ use lance::Dataset; + use lance::dataset::builder::DatasetBuilder; + use lance_core::Result; + ++use crate::data_cache::DatasetDataCache; + use crate::error::{ffi_try, swallow_unwind}; + use crate::helpers; + use crate::runtime::block_on; +@@ -23,6 +24,7 @@ use crate::stream_guard::guarded_ffi_stream_from_reader; + /// Opaque handle representing an opened Lance dataset. + pub struct LanceDataset { + pub(crate) inner: RwLock>, ++ pub(crate) data_cache: Option>, + } + + impl LanceDataset { +@@ -182,8 +184,16 @@ unsafe fn open_dataset_inner( + } + + let dataset = block_on(builder.load())?; ++ let (dataset, data_cache) = ++ if let Some(factory) = session.and_then(|session| session.data_cache_factory.clone()) { ++ let (dataset, data_cache) = factory.attach(dataset); ++ (dataset, Some(data_cache)) ++ } else { ++ (dataset, None) ++ }; + let handle = LanceDataset { + inner: RwLock::new(Arc::new(dataset)), ++ data_cache, + }; + Ok(Box::into_raw(Box::new(handle))) + } +@@ -519,6 +529,7 @@ mod tests { + .unwrap(); + let handle = LanceDataset { + inner: RwLock::new(Arc::new(dataset)), ++ data_cache: None, + }; + (tmp, handle) + } +diff --git a/src/foyer_data_cache.rs b/src/foyer_data_cache.rs +new file mode 100644 +index 0000000..9a10e43 +--- /dev/null ++++ b/src/foyer_data_cache.rs +@@ -0,0 +1,1215 @@ ++// SPDX-License-Identifier: Apache-2.0 ++// SPDX-FileCopyrightText: Copyright The Lance Authors ++ ++//! Foyer-backed cache for immutable Lance data-file reads. ++ ++use std::collections::{BTreeMap, HashMap}; ++use std::ffi::c_char; ++use std::fmt::{Debug, Display, Formatter}; ++use std::ops::Range; ++use std::path::Path as FsPath; ++use std::sync::atomic::{AtomicU64, Ordering}; ++use std::sync::{Arc, Mutex, Weak}; ++ ++use async_trait::async_trait; ++use bytes::{Bytes, BytesMut}; ++use foyer::{ ++ BlockEngineConfig, DeviceBuilder, FsDeviceBuilder, HybridCache, HybridCacheBuilder, ++ HybridCachePolicy, PsyncIoEngineConfig, ++}; ++use futures::stream::BoxStream; ++use lance_io::object_store::WrappingObjectStore; ++use object_store::path::Path; ++use object_store::{ ++ CopyOptions, GetOptions, GetResult, GetResultPayload, ListResult, MultipartUpload, ObjectMeta, ++ ObjectStore, ObjectStoreExt, PutMultipartOptions, PutOptions, PutPayload, PutResult, ++ RenameOptions, Result, ++}; ++ ++use crate::data_cache::{DataCacheFactory, DatasetDataCache, LanceDataCacheStatistics}; ++use crate::error::ffi_try; ++use crate::helpers; ++use crate::runtime::block_on; ++use crate::session::{LanceSession, session_new_with_data_cache_factory}; ++ ++const CACHE_KEY_VERSION: &str = "lance-data-v1"; ++const FOYER_PAGE_SIZE: usize = 4096; ++ ++/// Configuration for the optional Foyer data-file cache. ++#[repr(C)] ++#[derive(Clone, Copy, Debug)] ++pub struct LanceDataCacheOptions { ++ /// UTF-8 directory used by Foyer for persistent cache storage. ++ pub directory: *const c_char, ++ /// Maximum data bytes retained by Foyer's in-memory tier. ++ pub memory_capacity_bytes: u64, ++ /// Maximum bytes retained by Foyer's disk tier. ++ pub disk_capacity_bytes: u64, ++ /// Read/cache unit. Must be a non-zero multiple of 4096. ++ pub read_block_size_bytes: u64, ++} ++ ++/// Create a shared Lance session with a Foyer cache for immutable data-file ++/// whole-object, single-range, and batched range reads. ++/// ++/// `data_cache_options` and its `directory` field must not be NULL. The ++/// pointed-to values are copied before this function returns. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_session_new_with_data_cache( ++ index_cache_size_bytes: u64, ++ metadata_cache_size_bytes: u64, ++ data_cache_options: *const LanceDataCacheOptions, ++) -> *mut LanceSession { ++ ffi_try!( ++ unsafe { ++ session_new_with_data_cache_inner( ++ index_cache_size_bytes, ++ metadata_cache_size_bytes, ++ data_cache_options, ++ ) ++ }, ++ null ++ ) ++} ++ ++unsafe fn session_new_with_data_cache_inner( ++ index_cache_size_bytes: u64, ++ metadata_cache_size_bytes: u64, ++ data_cache_options: *const LanceDataCacheOptions, ++) -> lance_core::Result<*mut LanceSession> { ++ if data_cache_options.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "data_cache_options must not be NULL".into(), ++ )); ++ } ++ let options = unsafe { &*data_cache_options }; ++ let directory = unsafe { helpers::parse_c_string(options.directory)? }.ok_or_else(|| { ++ lance_core::Error::invalid_input_source( ++ "data_cache_options.directory must not be NULL".into(), ++ ) ++ })?; ++ if directory.is_empty() { ++ return Err(lance_core::Error::invalid_input_source( ++ "data_cache_options.directory must not be empty".into(), ++ )); ++ } ++ ++ let memory_capacity = u64_to_usize(options.memory_capacity_bytes, "memory_capacity_bytes")?; ++ let disk_capacity = u64_to_usize(options.disk_capacity_bytes, "disk_capacity_bytes")?; ++ let read_block_size = u64_to_usize(options.read_block_size_bytes, "read_block_size_bytes")?; ++ validate_data_cache_sizes(memory_capacity, disk_capacity, read_block_size)?; ++ ++ let data_cache = block_on(FoyerDataCache::try_new( ++ FsPath::new(&directory), ++ memory_capacity, ++ disk_capacity, ++ read_block_size, ++ )) ++ .map_err(|error| { ++ lance_core::Error::io(format!( ++ "failed to initialize Foyer data cache at {directory:?}: {error}" ++ )) ++ })?; ++ session_new_with_data_cache_factory( ++ index_cache_size_bytes, ++ metadata_cache_size_bytes, ++ Some(Arc::new(data_cache)), ++ ) ++} ++ ++fn validate_data_cache_sizes( ++ memory_capacity: usize, ++ disk_capacity: usize, ++ read_block_size: usize, ++) -> lance_core::Result<()> { ++ if read_block_size == 0 || !read_block_size.is_multiple_of(FOYER_PAGE_SIZE) { ++ return Err(lance_core::Error::invalid_input_source( ++ format!( ++ "read_block_size_bytes={read_block_size} must be a non-zero multiple of {FOYER_PAGE_SIZE}" ++ ) ++ .into(), ++ )); ++ } ++ if memory_capacity < read_block_size { ++ return Err(lance_core::Error::invalid_input_source( ++ format!( ++ "memory_capacity_bytes={memory_capacity} must be at least read_block_size_bytes={read_block_size}" ++ ) ++ .into(), ++ )); ++ } ++ let minimum_disk_capacity = read_block_size.checked_mul(2).ok_or_else(|| { ++ lance_core::Error::invalid_input_source( ++ format!("read_block_size_bytes={read_block_size} is too large").into(), ++ ) ++ })?; ++ if !disk_capacity.is_multiple_of(FOYER_PAGE_SIZE) || disk_capacity < minimum_disk_capacity { ++ return Err(lance_core::Error::invalid_input_source( ++ format!( ++ "disk_capacity_bytes={disk_capacity} must be a multiple of {FOYER_PAGE_SIZE} and at least twice read_block_size_bytes={read_block_size}" ++ ) ++ .into(), ++ )); ++ } ++ Ok(()) ++} ++ ++fn u64_to_usize(value: u64, field: &'static str) -> lance_core::Result { ++ usize::try_from(value).map_err(|_| { ++ lance_core::Error::invalid_input_source( ++ format!("{field}={value} exceeds usize::MAX on this target").into(), ++ ) ++ }) ++} ++ ++/// Process-local owner of a Foyer hybrid cache. ++#[derive(Clone)] ++pub(crate) struct FoyerDataCache { ++ cache: HybridCache, ++ read_block_size: usize, ++ wrapped_stores: Arc>>, ++} ++ ++struct WrappedStore { ++ wrapper: Weak, ++ origin: Weak, ++} ++ ++impl Debug for FoyerDataCache { ++ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { ++ f.debug_struct("FoyerDataCache") ++ .field("read_block_size", &self.read_block_size) ++ .finish_non_exhaustive() ++ } ++} ++ ++impl FoyerDataCache { ++ pub(crate) async fn try_new( ++ directory: &FsPath, ++ memory_capacity: usize, ++ disk_capacity: usize, ++ read_block_size: usize, ++ ) -> std::result::Result { ++ let engine_block_size = read_block_size ++ .checked_mul(2) ++ .ok_or_else(|| foyer::Error::new(foyer::ErrorKind::Config, "read block size overflow"))? ++ .max(FOYER_PAGE_SIZE); ++ let memory_shards = (memory_capacity / read_block_size).clamp(1, 8); ++ let device = FsDeviceBuilder::new(directory) ++ .with_capacity(disk_capacity) ++ .build()?; ++ let engine = BlockEngineConfig::new(device).with_block_size(engine_block_size); ++ let cache = HybridCacheBuilder::new() ++ .with_name("lance_data") ++ .with_policy(HybridCachePolicy::WriteOnInsertion) ++ // Entries are already sent to storage on insertion. Avoid making ++ // the last Dataset drop wait for a final full-memory flush. ++ .with_flush_on_close(false) ++ .memory(memory_capacity) ++ .with_shards(memory_shards) ++ .with_weighter(|_key: &String, value: &Bytes| value.len().max(1)) ++ .storage() ++ .with_io_engine_config(PsyncIoEngineConfig::new()) ++ .with_engine_config(engine) ++ .build() ++ .await?; ++ Ok(Self { ++ cache, ++ read_block_size, ++ wrapped_stores: Arc::new(Mutex::new(HashMap::new())), ++ }) ++ } ++ ++ fn is_cacheable_data_file(location: &Path) -> bool { ++ let mut parts = location.as_ref().rsplit('/'); ++ matches!( ++ (parts.next(), parts.next()), ++ (Some(file), Some("data")) if file.ends_with(".lance") ++ ) ++ } ++ ++ fn key(&self, store_prefix: &str, location: &Path, block_index: u64) -> String { ++ format!( ++ "{CACHE_KEY_VERSION}\0{}\0{store_prefix}\0{}\0{block_index}", ++ self.read_block_size, ++ location.as_ref() ++ ) ++ } ++ ++ fn size_key(&self, store_prefix: &str, location: &Path) -> String { ++ format!( ++ "{CACHE_KEY_VERSION}\0{}\0{store_prefix}\0{}\0size", ++ self.read_block_size, ++ location.as_ref() ++ ) ++ } ++ ++ fn create_scope(&self) -> Arc { ++ Arc::new(DatasetFoyerDataCache { ++ cache: self.clone(), ++ statistics: Arc::new(FoyerDataCacheStatistics::default()), ++ }) ++ } ++ ++ fn unwrap_store(&self, store: Arc) -> Arc { ++ let identity = Arc::as_ptr(&store) as *const () as usize; ++ let origin = { ++ let mut wrapped_stores = self ++ .wrapped_stores ++ .lock() ++ .unwrap_or_else(|error| error.into_inner()); ++ let origin = wrapped_stores.get(&identity).and_then(|entry| { ++ let wrapper = entry.wrapper.upgrade()?; ++ if Arc::ptr_eq(&wrapper, &store) { ++ entry.origin.upgrade() ++ } else { ++ None ++ } ++ }); ++ if origin.is_none() { ++ wrapped_stores.remove(&identity); ++ } ++ origin ++ }; ++ match origin { ++ Some(origin) => origin, ++ None => store, ++ } ++ } ++ ++ fn remember_wrapper(&self, wrapper: &Arc, origin: &Arc) { ++ let identity = Arc::as_ptr(wrapper) as *const () as usize; ++ self.wrapped_stores ++ .lock() ++ .unwrap_or_else(|error| error.into_inner()) ++ .insert( ++ identity, ++ WrappedStore { ++ wrapper: Arc::downgrade(wrapper), ++ origin: Arc::downgrade(origin), ++ }, ++ ); ++ } ++ ++ fn forget_wrapper(&self, identity: usize) { ++ self.wrapped_stores ++ .lock() ++ .unwrap_or_else(|error| error.into_inner()) ++ .remove(&identity); ++ } ++} ++ ++#[derive(Debug, Default)] ++struct FoyerDataCacheStatistics { ++ bytes_read_from_cache: AtomicU64, ++ bytes_read_from_remote: AtomicU64, ++} ++ ++impl FoyerDataCacheStatistics { ++ fn record(&self, bytes_read_from_cache: u64, bytes_read_from_remote: u64) { ++ self.bytes_read_from_cache ++ .fetch_add(bytes_read_from_cache, Ordering::Relaxed); ++ self.bytes_read_from_remote ++ .fetch_add(bytes_read_from_remote, Ordering::Relaxed); ++ } ++ ++ fn snapshot(&self) -> LanceDataCacheStatistics { ++ LanceDataCacheStatistics { ++ bytes_read_from_cache: self.bytes_read_from_cache.load(Ordering::Relaxed), ++ bytes_read_from_remote: self.bytes_read_from_remote.load(Ordering::Relaxed), ++ } ++ } ++} ++ ++impl DataCacheFactory for FoyerDataCache { ++ fn attach(&self, dataset: lance::Dataset) -> (lance::Dataset, Arc) { ++ let scope = self.create_scope(); ++ let wrapper: Arc = scope.clone(); ++ let dataset = dataset.with_object_store_wrappers([wrapper]); ++ (dataset, scope) ++ } ++} ++ ++#[derive(Debug)] ++struct DatasetFoyerDataCache { ++ cache: FoyerDataCache, ++ statistics: Arc, ++} ++ ++impl DatasetDataCache for DatasetFoyerDataCache { ++ fn snapshot(&self) -> LanceDataCacheStatistics { ++ self.statistics.snapshot() ++ } ++ ++ fn attach_fresh(&self, dataset: lance::Dataset) -> (lance::Dataset, Arc) { ++ self.cache.attach(dataset) ++ } ++} ++ ++impl WrappingObjectStore for DatasetFoyerDataCache { ++ fn wrap(&self, store_prefix: &str, original: Arc) -> Arc { ++ // A derived Dataset can already contain this cache wrapper. Resolve ++ // that exact wrapper back to its origin before attaching fresh ++ // dataset-scoped counters. ++ let original = self.cache.unwrap_store(original); ++ let reader = DataCacheReader { ++ cache: self.cache.clone(), ++ store_prefix: store_prefix.to_owned(), ++ original: original.clone(), ++ statistics: self.statistics.clone(), ++ }; ++ let cached_store = ++ Arc::new_cyclic(|weak: &Weak| DataCacheObjectStore { ++ reader, ++ identity: weak.as_ptr() as usize, ++ }); ++ let wrapped: Arc = cached_store.clone(); ++ self.cache.remember_wrapper(&wrapped, &original); ++ wrapped ++ } ++} ++ ++#[derive(Debug)] ++struct DataCacheObjectStore { ++ reader: DataCacheReader, ++ identity: usize, ++} ++ ++#[derive(Clone, Debug)] ++struct DataCacheReader { ++ cache: FoyerDataCache, ++ store_prefix: String, ++ original: Arc, ++ statistics: Arc, ++} ++ ++impl Drop for DataCacheObjectStore { ++ fn drop(&mut self) { ++ self.reader.cache.forget_wrapper(self.identity); ++ } ++} ++ ++impl Display for DataCacheObjectStore { ++ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { ++ write!(f, "FoyerDataCache({})", self.reader.original) ++ } ++} ++ ++impl DataCacheObjectStore { ++ fn is_cache_safe_get(options: &GetOptions) -> bool { ++ !options.head ++ && options.if_match.is_none() ++ && options.if_none_match.is_none() ++ && options.if_modified_since.is_none() ++ && options.if_unmodified_since.is_none() ++ && options.version.is_none() ++ && options.extensions.is_empty() ++ } ++ ++ async fn cached_get(&self, location: &Path, options: GetOptions) -> Result { ++ // Fetch metadata separately so the returned GetResult retains the origin's identity while ++ // its payload uses the same block cache as get_ranges(). This also provides the object size ++ // needed to resolve bounded, offset, and suffix ranges. ++ let GetResult { ++ meta: metadata, ++ attributes, ++ .. ++ } = self ++ .reader ++ .original ++ .get_opts( ++ location, ++ GetOptions { ++ head: true, ++ ..Default::default() ++ }, ++ ) ++ .await?; ++ let object_size = metadata.size; ++ self.reader.cache.cache.insert( ++ self.reader ++ .cache ++ .size_key(&self.reader.store_prefix, location), ++ Bytes::copy_from_slice(&object_size.to_le_bytes()), ++ ); ++ ++ let range = match options.range.clone() { ++ Some(requested) => match requested.as_range(object_size) { ++ Ok(range) if !range.is_empty() => range, ++ // Preserve the origin's exact error for invalid or empty ranges. ++ _ => return self.reader.original.get_opts(location, options).await, ++ }, ++ None => 0..object_size, ++ }; ++ ++ let reader = self.reader.clone(); ++ let stream_location = location.clone(); ++ let stream_range = range.clone(); ++ let stream = futures::stream::try_unfold( ++ (reader, stream_location, stream_range), ++ |(reader, location, remaining)| async move { ++ if remaining.is_empty() { ++ return Ok(None); ++ } ++ ++ // Yield no more than the remainder of one cache block. The next block is not ++ // requested until the consumer polls again, so cancellation drops the pending ++ // range without downloading or retaining the rest of the object. ++ let block_size = reader.cache.read_block_size as u64; ++ let bytes_to_boundary = block_size - remaining.start % block_size; ++ let end = remaining ++ .start ++ .saturating_add(bytes_to_boundary) ++ .min(remaining.end); ++ let chunk_range = remaining.start..end; ++ let chunk = reader ++ .cached_ranges(&location, std::slice::from_ref(&chunk_range)) ++ .await? ++ .into_iter() ++ .next() ++ .ok_or_else(|| cache_error(format!("missing get result for {location}")))?; ++ Ok(Some((chunk, (reader, location, end..remaining.end)))) ++ }, ++ ); ++ let payload = GetResultPayload::Stream(Box::pin(stream)); ++ Ok(GetResult { ++ payload, ++ meta: metadata, ++ range, ++ attributes, ++ }) ++ } ++} ++ ++impl DataCacheReader { ++ async fn read_origin_ranges( ++ &self, ++ location: &Path, ++ ranges: &[Range], ++ ) -> Result> { ++ let bytes = self.original.get_ranges(location, ranges).await?; ++ self.statistics.record(0, total_bytes(&bytes)); ++ Ok(bytes) ++ } ++ ++ async fn object_size(&self, location: &Path) -> Result { ++ let key = self.cache.size_key(&self.store_prefix, location); ++ match self.cache.cache.get(&key).await { ++ Ok(Some(entry)) => match entry.value().as_ref().try_into() { ++ Ok(bytes) => return Ok(u64::from_le_bytes(bytes)), ++ Err(_) => log::warn!( ++ "Foyer data-cache size entry was malformed for {location}; refreshing it" ++ ), ++ }, ++ Ok(None) => {} ++ Err(error) => { ++ log::warn!("Foyer data-cache size lookup failed for {location}: {error}"); ++ } ++ } ++ ++ let size = self.original.head(location).await?.size; ++ self.cache ++ .cache ++ .insert(key, Bytes::copy_from_slice(&size.to_le_bytes())); ++ Ok(size) ++ } ++ ++ async fn cached_ranges(&self, location: &Path, ranges: &[Range]) -> Result> { ++ if ranges.is_empty() { ++ return Ok(Vec::new()); ++ } ++ if ranges.iter().any(|range| range.start >= range.end) { ++ return self.read_origin_ranges(location, ranges).await; ++ } ++ ++ let object_size = self.object_size(location).await?; ++ if ranges.iter().any(|range| range.start >= object_size) { ++ // Preserve the origin's exact error for ranges that start at or ++ // beyond EOF. ++ return self.read_origin_ranges(location, ranges).await; ++ } ++ let readable_ranges = ranges ++ .iter() ++ .map(|range| range.start..range.end.min(object_size)) ++ .collect::>(); ++ ++ let block_size = self.cache.read_block_size as u64; ++ let mut blocks = BTreeMap::>::new(); ++ for range in &readable_ranges { ++ let first = range.start / block_size; ++ let last = (range.end - 1) / block_size; ++ for block_index in first..=last { ++ blocks.entry(block_index).or_default(); ++ } ++ } ++ ++ for (block_index, block) in &mut blocks { ++ let key = self.cache.key(&self.store_prefix, location, *block_index); ++ match self.cache.cache.get(&key).await { ++ Ok(Some(entry)) => *block = Some(entry.value().clone()), ++ Ok(None) => {} ++ Err(error) => { ++ // Cache availability must not affect query correctness. ++ log::warn!("Foyer data-cache lookup failed for {location}: {error}"); ++ } ++ } ++ } ++ ++ let (bytes_read_from_cache, bytes_read_from_remote) = ++ requested_bytes_by_cache_status(&readable_ranges, block_size, &blocks); ++ let missing: Vec = blocks ++ .iter() ++ .filter_map(|(block_index, value)| value.is_none().then_some(*block_index)) ++ .collect(); ++ let mut runs = Vec::>::new(); ++ for block_index in missing { ++ let start = block_index ++ .checked_mul(block_size) ++ .ok_or_else(|| cache_error("data-cache block offset overflow"))?; ++ let end = start ++ .checked_add(block_size) ++ .ok_or_else(|| cache_error("data-cache block end overflow"))? ++ .min(object_size); ++ match runs.last_mut() { ++ Some(run) if run.end == start => run.end = end, ++ _ => runs.push(start..end), ++ } ++ } ++ ++ if !runs.is_empty() { ++ // Fetch all contiguous miss runs together so a large Lance read is ++ // not expanded into one remote request per cache block. ++ let fetched = self.original.get_ranges(location, &runs).await?; ++ for (run, bytes) in runs.into_iter().zip(fetched) { ++ let first_block = run.start / block_size; ++ for (offset, chunk) in bytes.chunks(self.cache.read_block_size).enumerate() { ++ let block_index = first_block + offset as u64; ++ let value = Bytes::copy_from_slice(chunk); ++ let key = self.cache.key(&self.store_prefix, location, block_index); ++ self.cache.cache.insert(key, value.clone()); ++ if let Some(block) = blocks.get_mut(&block_index) { ++ *block = Some(value); ++ } ++ } ++ } ++ } ++ ++ let assembled = readable_ranges ++ .iter() ++ .map(|range| assemble_range(location, range, block_size, &blocks)) ++ .collect::>>(); ++ match assembled { ++ Ok(bytes) => { ++ self.statistics ++ .record(bytes_read_from_cache, bytes_read_from_remote); ++ Ok(bytes) ++ } ++ Err(error) => { ++ // A malformed or incomplete cached entry must never turn a ++ // valid source read into a query failure. ++ log::warn!( ++ "Foyer data-cache entry was unusable for {location}; bypassing cache: {error}" ++ ); ++ let bytes = self.original.get_ranges(location, ranges).await?; ++ self.statistics.record(0, total_bytes(&bytes)); ++ Ok(bytes) ++ } ++ } ++ } ++} ++ ++fn total_bytes(ranges: &[Bytes]) -> u64 { ++ ranges.iter().fold(0_u64, |total, bytes| { ++ total.saturating_add(bytes.len() as u64) ++ }) ++} ++ ++fn requested_bytes_by_cache_status( ++ ranges: &[Range], ++ block_size: u64, ++ blocks: &BTreeMap>, ++) -> (u64, u64) { ++ let mut hit_bytes = 0_u64; ++ let mut miss_bytes = 0_u64; ++ for range in ranges { ++ let mut start = range.start; ++ while start < range.end { ++ let block_index = start / block_size; ++ let block_start = block_index * block_size; ++ let end = range.end.min(block_start.saturating_add(block_size)); ++ let bytes = end - start; ++ if blocks ++ .get(&block_index) ++ .is_some_and(|block| block.is_some()) ++ { ++ hit_bytes = hit_bytes.saturating_add(bytes); ++ } else { ++ miss_bytes = miss_bytes.saturating_add(bytes); ++ } ++ start = end; ++ } ++ } ++ (hit_bytes, miss_bytes) ++} ++ ++fn assemble_range( ++ location: &Path, ++ range: &Range, ++ block_size: u64, ++ blocks: &BTreeMap>, ++) -> Result { ++ if range.is_empty() { ++ return Ok(Bytes::new()); ++ } ++ let first = range.start / block_size; ++ let last = (range.end - 1) / block_size; ++ if first == last { ++ let block = blocks ++ .get(&first) ++ .and_then(Option::as_ref) ++ .ok_or_else(|| cache_error(format!("missing block {first} for {location}")))?; ++ let block_start = first * block_size; ++ let start = usize::try_from(range.start - block_start) ++ .map_err(|_| cache_error("data-cache slice start exceeds usize::MAX"))?; ++ let end = usize::try_from((range.end - block_start).min(block_size)) ++ .map_err(|_| cache_error("data-cache slice end exceeds usize::MAX"))? ++ .min(block.len()); ++ if start >= block.len() { ++ return Err(cache_error(format!( ++ "short data-cache block {first} for {location}: need {start}..{end}, got {} bytes", ++ block.len() ++ ))); ++ } ++ return Ok(block.slice(start..end)); ++ } ++ ++ let requested_len = usize::try_from(range.end - range.start) ++ .map_err(|_| cache_error(format!("range {range:?} for {location} exceeds usize::MAX")))?; ++ let mut output = BytesMut::with_capacity(requested_len); ++ for block_index in first..=last { ++ let block = blocks ++ .get(&block_index) ++ .and_then(Option::as_ref) ++ .ok_or_else(|| cache_error(format!("missing block {block_index} for {location}")))?; ++ let block_start = block_index * block_size; ++ let start = usize::try_from(range.start.saturating_sub(block_start)) ++ .map_err(|_| cache_error("data-cache slice start exceeds usize::MAX"))?; ++ let end_in_block = range.end.saturating_sub(block_start).min(block_size); ++ let end = usize::try_from(end_in_block) ++ .map_err(|_| cache_error("data-cache slice end exceeds usize::MAX"))?; ++ if start >= block.len() { ++ if !output.is_empty() { ++ break; ++ } ++ return Err(cache_error(format!( ++ "short data-cache block {block_index} for {location}: need {start}..{end}, got {} bytes", ++ block.len() ++ ))); ++ } ++ let actual_end = end.min(block.len()); ++ output.extend_from_slice(&block[start..actual_end]); ++ if actual_end < end { ++ break; ++ } ++ } ++ Ok(output.freeze()) ++} ++ ++fn cache_error(message: impl Into) -> object_store::Error { ++ object_store::Error::Generic { ++ store: "foyer_data_cache", ++ source: Box::new(std::io::Error::other(message.into())), ++ } ++} ++ ++#[async_trait] ++#[deny(clippy::missing_trait_methods)] ++impl ObjectStore for DataCacheObjectStore { ++ async fn put_opts( ++ &self, ++ location: &Path, ++ payload: PutPayload, ++ opts: PutOptions, ++ ) -> Result { ++ self.reader.original.put_opts(location, payload, opts).await ++ } ++ ++ async fn put_multipart_opts( ++ &self, ++ location: &Path, ++ opts: PutMultipartOptions, ++ ) -> Result> { ++ self.reader ++ .original ++ .put_multipart_opts(location, opts) ++ .await ++ } ++ ++ async fn get_opts(&self, location: &Path, options: GetOptions) -> Result { ++ if FoyerDataCache::is_cacheable_data_file(location) && Self::is_cache_safe_get(&options) { ++ self.cached_get(location, options).await ++ } else { ++ self.reader.original.get_opts(location, options).await ++ } ++ } ++ ++ async fn get_ranges(&self, location: &Path, ranges: &[Range]) -> Result> { ++ if FoyerDataCache::is_cacheable_data_file(location) { ++ self.reader.cached_ranges(location, ranges).await ++ } else { ++ self.reader.original.get_ranges(location, ranges).await ++ } ++ } ++ ++ fn delete_stream( ++ &self, ++ locations: BoxStream<'static, Result>, ++ ) -> BoxStream<'static, Result> { ++ self.reader.original.delete_stream(locations) ++ } ++ ++ fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result> { ++ self.reader.original.list(prefix) ++ } ++ ++ fn list_with_offset( ++ &self, ++ prefix: Option<&Path>, ++ offset: &Path, ++ ) -> BoxStream<'static, Result> { ++ self.reader.original.list_with_offset(prefix, offset) ++ } ++ ++ async fn list_with_delimiter(&self, prefix: Option<&Path>) -> Result { ++ self.reader.original.list_with_delimiter(prefix).await ++ } ++ ++ async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> Result<()> { ++ self.reader.original.copy_opts(from, to, options).await ++ } ++ ++ async fn rename_opts(&self, from: &Path, to: &Path, options: RenameOptions) -> Result<()> { ++ self.reader.original.rename_opts(from, to, options).await ++ } ++} ++ ++#[cfg(test)] ++mod tests { ++ use std::sync::mpsc; ++ use std::time::Duration; ++ ++ use futures::StreamExt; ++ use lance_io::object_store::ChainedWrappingObjectStore; ++ use object_store::GetRange; ++ use object_store::memory::InMemory; ++ ++ use super::*; ++ ++ fn wrap_for_test( ++ cache: &FoyerDataCache, ++ original: Arc, ++ ) -> (Arc, Arc) { ++ let scope = cache.create_scope(); ++ (scope.wrap("memory://test", original), scope) ++ } ++ ++ #[tokio::test] ++ async fn caches_only_immutable_data_file_ranges() { ++ let directory = tempfile::tempdir().unwrap(); ++ let cache = FoyerDataCache::try_new(directory.path(), 256 * 1024, 1024 * 1024, 64 * 1024) ++ .await ++ .unwrap(); ++ let original = Arc::new(InMemory::new()); ++ let data_path = Path::from("table.lance/data/part-0.lance"); ++ let second_data_path = Path::from("table.lance/data/part-1.lance"); ++ let manifest_path = Path::from("table.lance/_versions/1.manifest"); ++ let data = Bytes::from((0..200_000).map(|value| value as u8).collect::>()); ++ let second_data = Bytes::from_static(b"second fragment"); ++ original.put(&data_path, data.clone().into()).await.unwrap(); ++ original ++ .put(&second_data_path, second_data.clone().into()) ++ .await ++ .unwrap(); ++ original ++ .put(&manifest_path, Bytes::from_static(b"manifest").into()) ++ .await ++ .unwrap(); ++ ++ let (wrapped, statistics) = wrap_for_test(&cache, original.clone()); ++ let ranges = vec![10..90_000, 65_000..140_000, 190_000..220_000]; ++ let second_data_range = 0..15; ++ let first = wrapped.get_ranges(&data_path, &ranges).await.unwrap(); ++ assert_eq!(first[0], data.slice(10..90_000)); ++ assert_eq!(first[1], data.slice(65_000..140_000)); ++ assert_eq!(first[2], data.slice(190_000..200_000)); ++ assert_eq!( ++ wrapped ++ .get_ranges(&second_data_path, std::slice::from_ref(&second_data_range)) ++ .await ++ .unwrap(), ++ vec![second_data.clone()] ++ ); ++ ++ assert_eq!( ++ statistics.snapshot(), ++ LanceDataCacheStatistics { ++ bytes_read_from_cache: 0, ++ bytes_read_from_remote: 175_005, ++ } ++ ); ++ ++ original.delete(&data_path).await.unwrap(); ++ original.delete(&second_data_path).await.unwrap(); ++ let second = wrapped.get_ranges(&data_path, &ranges).await.unwrap(); ++ assert_eq!(second, first); ++ assert_eq!( ++ wrapped ++ .get_ranges(&second_data_path, std::slice::from_ref(&second_data_range)) ++ .await ++ .unwrap(), ++ vec![second_data] ++ ); ++ ++ assert_eq!( ++ statistics.snapshot(), ++ LanceDataCacheStatistics { ++ bytes_read_from_cache: 175_005, ++ bytes_read_from_remote: 175_005, ++ } ++ ); ++ ++ assert_eq!( ++ wrapped.get_range(&manifest_path, 0..8).await.unwrap(), ++ Bytes::from_static(b"manifest") ++ ); ++ original.delete(&manifest_path).await.unwrap(); ++ assert!(wrapped.get_range(&manifest_path, 0..8).await.is_err()); ++ } ++ ++ #[tokio::test] ++ async fn caches_small_data_file_whole_object_reads() { ++ let directory = tempfile::tempdir().unwrap(); ++ let cache = FoyerDataCache::try_new(directory.path(), 128 * 1024, 1024 * 1024, 64 * 1024) ++ .await ++ .unwrap(); ++ let original = Arc::new(InMemory::new()); ++ let data_path = Path::from("table.lance/data/small.lance"); ++ let data = Bytes::from(vec![7; 42_000]); ++ original.put(&data_path, data.clone().into()).await.unwrap(); ++ ++ let (wrapped, statistics) = wrap_for_test(&cache, original); ++ let first = wrapped ++ .get(&data_path) ++ .await ++ .unwrap() ++ .bytes() ++ .await ++ .unwrap(); ++ assert_eq!(first, data); ++ assert_eq!( ++ statistics.snapshot(), ++ LanceDataCacheStatistics { ++ bytes_read_from_cache: 0, ++ bytes_read_from_remote: 42_000, ++ } ++ ); ++ ++ let second = wrapped ++ .get(&data_path) ++ .await ++ .unwrap() ++ .bytes() ++ .await ++ .unwrap(); ++ assert_eq!(second, data); ++ assert_eq!( ++ statistics.snapshot(), ++ LanceDataCacheStatistics { ++ bytes_read_from_cache: 42_000, ++ bytes_read_from_remote: 42_000, ++ } ++ ); ++ } ++ ++ #[tokio::test] ++ async fn streams_large_data_file_gets_with_bounded_read_ahead() { ++ let directory = tempfile::tempdir().unwrap(); ++ let block_size = 64 * 1024; ++ let cache = FoyerDataCache::try_new(directory.path(), 512 * 1024, 1024 * 1024, block_size) ++ .await ++ .unwrap(); ++ let original = Arc::new(InMemory::new()); ++ let data_path = Path::from("table.lance/data/large.lance"); ++ let data = Bytes::from(vec![7; 4 * block_size]); ++ original.put(&data_path, data.clone().into()).await.unwrap(); ++ ++ let (wrapped, statistics) = wrap_for_test(&cache, original); ++ let result = wrapped.get(&data_path).await.unwrap(); ++ assert_eq!(statistics.snapshot(), LanceDataCacheStatistics::default()); ++ ++ let mut stream = result.into_stream(); ++ let first = stream.next().await.unwrap().unwrap(); ++ assert_eq!(first, data.slice(..block_size)); ++ assert_eq!( ++ statistics.snapshot(), ++ LanceDataCacheStatistics { ++ bytes_read_from_cache: 0, ++ bytes_read_from_remote: block_size as u64, ++ } ++ ); ++ ++ drop(stream); ++ assert_eq!( ++ statistics.snapshot().bytes_read_from_remote, ++ block_size as u64 ++ ); ++ } ++ ++ #[tokio::test] ++ async fn caches_single_data_file_range_reads() { ++ let directory = tempfile::tempdir().unwrap(); ++ let cache = FoyerDataCache::try_new(directory.path(), 512 * 1024, 1024 * 1024, 64 * 1024) ++ .await ++ .unwrap(); ++ let original = Arc::new(InMemory::new()); ++ let data = Bytes::from((0..100_000).map(|value| value as u8).collect::>()); ++ let cases = [ ++ ( ++ Path::from("table.lance/data/bounded.lance"), ++ GetRange::Bounded(1_000..2_000), ++ 1_000..2_000, ++ ), ++ ( ++ Path::from("table.lance/data/offset.lance"), ++ GetRange::Offset(90_000), ++ 90_000..100_000, ++ ), ++ ( ++ Path::from("table.lance/data/suffix.lance"), ++ GetRange::Suffix(500), ++ 99_500..100_000, ++ ), ++ ]; ++ for (path, _, _) in &cases { ++ original.put(path, data.clone().into()).await.unwrap(); ++ } ++ ++ let (wrapped, statistics) = wrap_for_test(&cache, original); ++ for (path, requested, expected_range) in cases { ++ let expected = data.slice(expected_range.start as usize..expected_range.end as usize); ++ let before = statistics.snapshot(); ++ let first = wrapped ++ .get_opts(&path, GetOptions::new().with_range(Some(requested.clone()))) ++ .await ++ .unwrap() ++ .bytes() ++ .await ++ .unwrap(); ++ assert_eq!(first, expected); ++ assert_eq!( ++ statistics.snapshot().bytes_read_from_remote, ++ before.bytes_read_from_remote + expected.len() as u64 ++ ); ++ ++ let second = wrapped ++ .get_opts(&path, GetOptions::new().with_range(Some(requested))) ++ .await ++ .unwrap() ++ .bytes() ++ .await ++ .unwrap(); ++ assert_eq!(second, expected); ++ assert_eq!( ++ statistics.snapshot().bytes_read_from_cache, ++ before.bytes_read_from_cache + expected.len() as u64 ++ ); ++ } ++ ++ let before = statistics.snapshot(); ++ let conditional = GetOptions::new().with_if_match(Some("wrong-etag")); ++ assert!( ++ wrapped ++ .get_opts(&Path::from("table.lance/data/bounded.lance"), conditional) ++ .await ++ .is_err() ++ ); ++ assert_eq!(statistics.snapshot(), before); ++ ++ assert!( ++ wrapped ++ .get_range( ++ &Path::from("table.lance/data/bounded.lance"), ++ 100_000..100_001 ++ ) ++ .await ++ .is_err() ++ ); ++ assert_eq!(statistics.snapshot(), before); ++ } ++ ++ #[tokio::test] ++ async fn recovers_cached_data_from_disk() { ++ let directory = tempfile::tempdir().unwrap(); ++ let original = Arc::new(InMemory::new()); ++ let data_path = Path::from("table.lance/data/part-0.lance"); ++ let data = Bytes::from((0..100_000).map(|value| value as u8).collect::>()); ++ original.put(&data_path, data.clone().into()).await.unwrap(); ++ ++ let cache = FoyerDataCache::try_new(directory.path(), 128 * 1024, 1024 * 1024, 64 * 1024) ++ .await ++ .unwrap(); ++ let (wrapped, _) = wrap_for_test(&cache, original.clone()); ++ let requested_range = 10..90_000; ++ assert_eq!( ++ wrapped ++ .get_ranges(&data_path, std::slice::from_ref(&requested_range)) ++ .await ++ .unwrap(), ++ vec![data.slice(10..90_000)] ++ ); ++ drop(wrapped); ++ cache.cache.close().await.unwrap(); ++ drop(cache); ++ ++ original.delete(&data_path).await.unwrap(); ++ let recovered = ++ FoyerDataCache::try_new(directory.path(), 128 * 1024, 1024 * 1024, 64 * 1024) ++ .await ++ .unwrap(); ++ let (wrapped, statistics) = wrap_for_test(&recovered, original); ++ assert_eq!( ++ wrapped ++ .get_ranges(&data_path, std::slice::from_ref(&requested_range)) ++ .await ++ .unwrap(), ++ vec![data.slice(10..90_000)] ++ ); ++ assert_eq!(statistics.snapshot().bytes_read_from_cache, 89_990); ++ assert_eq!(statistics.snapshot().bytes_read_from_remote, 0); ++ } ++ ++ #[tokio::test] ++ async fn dataset_scopes_share_cache_without_sharing_statistics() { ++ let directory = tempfile::tempdir().unwrap(); ++ let cache = FoyerDataCache::try_new(directory.path(), 128 * 1024, 1024 * 1024, 64 * 1024) ++ .await ++ .unwrap(); ++ let original = Arc::new(InMemory::new()); ++ let data_path = Path::from("table.lance/data/part-0.lance"); ++ let data = Bytes::from((0..100_000).map(|value| value as u8).collect::>()); ++ original.put(&data_path, data.clone().into()).await.unwrap(); ++ ++ let source_scope = cache.create_scope(); ++ let source_store = source_scope.wrap("memory://test", original.clone()); ++ ++ // A restored Dataset is derived from an already-wrapped source ++ // Dataset. The fresh scope must unwrap to the registered origin rather ++ // than nesting over the source scope. ++ let restored_scope = cache.create_scope(); ++ let restored_store = restored_scope.wrap("memory://test", source_store.clone()); ++ let requested_range = 10..90_000; ++ assert_eq!( ++ restored_store ++ .get_ranges(&data_path, std::slice::from_ref(&requested_range)) ++ .await ++ .unwrap(), ++ vec![data.slice(10..90_000)] ++ ); ++ assert_eq!(source_scope.snapshot(), LanceDataCacheStatistics::default()); ++ assert_eq!(restored_scope.snapshot().bytes_read_from_remote, 89_990); ++ ++ original.delete(&data_path).await.unwrap(); ++ assert_eq!( ++ source_store ++ .get_ranges(&data_path, std::slice::from_ref(&requested_range)) ++ .await ++ .unwrap(), ++ vec![data.slice(10..90_000)] ++ ); ++ assert_eq!(source_scope.snapshot().bytes_read_from_cache, 89_990); ++ assert_eq!(restored_scope.snapshot().bytes_read_from_remote, 89_990); ++ ++ drop(restored_store); ++ drop(source_store); ++ assert!(cache.wrapped_stores.lock().unwrap().is_empty()); ++ } ++ ++ #[tokio::test] ++ async fn chained_scopes_drop_intermediate_store_without_deadlocking() { ++ let directory = tempfile::tempdir().unwrap(); ++ let cache = FoyerDataCache::try_new(directory.path(), 128 * 1024, 1024 * 1024, 64 * 1024) ++ .await ++ .unwrap(); ++ let first: Arc = cache.create_scope(); ++ let second: Arc = cache.create_scope(); ++ let chained = ChainedWrappingObjectStore::new(vec![first, second]); ++ let original: Arc = Arc::new(InMemory::new()); ++ let (sender, receiver) = mpsc::channel(); ++ ++ let thread = std::thread::spawn(move || { ++ sender ++ .send(chained.wrap("memory://test", original)) ++ .unwrap(); ++ }); ++ let wrapped = receiver ++ .recv_timeout(Duration::from_secs(2)) ++ .expect("chained cache wrappers deadlocked while dropping the intermediate store"); ++ thread.join().unwrap(); ++ ++ assert_eq!(cache.wrapped_stores.lock().unwrap().len(), 1); ++ drop(wrapped); ++ assert!(cache.wrapped_stores.lock().unwrap().is_empty()); ++ } ++ ++ #[tokio::test] ++ async fn truncates_ranges_at_eof_before_enumerating_cache_blocks() { ++ let directory = tempfile::tempdir().unwrap(); ++ let cache = FoyerDataCache::try_new(directory.path(), 128 * 1024, 1024 * 1024, 64 * 1024) ++ .await ++ .unwrap(); ++ let original = Arc::new(InMemory::new()); ++ let data_path = Path::from("table.lance/data/part-0.lance"); ++ let data = Bytes::from((0..100_000).map(|value| value as u8).collect::>()); ++ original.put(&data_path, data.into()).await.unwrap(); ++ ++ let requested_range = 99_990..300_000; ++ let expected = original ++ .get_ranges(&data_path, std::slice::from_ref(&requested_range)) ++ .await ++ .unwrap(); ++ let (wrapped, statistics) = wrap_for_test(&cache, original.clone()); ++ let actual = wrapped ++ .get_ranges(&data_path, std::slice::from_ref(&requested_range)) ++ .await ++ .unwrap(); ++ assert_eq!(actual, expected); ++ assert_eq!(statistics.snapshot().bytes_read_from_remote, 10); ++ ++ original.delete(&data_path).await.unwrap(); ++ assert_eq!( ++ wrapped ++ .get_ranges(&data_path, std::slice::from_ref(&requested_range)) ++ .await ++ .unwrap(), ++ expected ++ ); ++ assert_eq!(statistics.snapshot().bytes_read_from_cache, 10); ++ assert_eq!(statistics.snapshot().bytes_read_from_remote, 10); ++ } ++ ++ #[test] ++ fn recognizes_only_direct_data_children() { ++ assert!(FoyerDataCache::is_cacheable_data_file(&Path::from( ++ "dataset/data/part.lance" ++ ))); ++ assert!(!FoyerDataCache::is_cacheable_data_file(&Path::from( ++ "dataset/data/nested/part.lance" ++ ))); ++ assert!(!FoyerDataCache::is_cacheable_data_file(&Path::from( ++ "dataset/indices/index.lance" ++ ))); ++ assert!(!FoyerDataCache::is_cacheable_data_file(&Path::from( ++ "dataset/_versions/1.manifest" ++ ))); ++ } ++} +diff --git a/src/lib.rs b/src/lib.rs +index 8b212f5..923528a 100644 +--- a/src/lib.rs ++++ b/src/lib.rs +@@ -25,11 +25,13 @@ mod alter_columns; + mod async_dispatcher; + mod batch; + mod compact; ++mod data_cache; + mod data_statistics; + mod dataset; + mod delete; + mod drop_columns; + mod error; ++mod foyer_data_cache; + mod fragment_writer; + mod fts_query; + mod helpers; +@@ -51,6 +53,7 @@ pub use add_columns::*; + pub use alter_columns::*; + pub use batch::*; + pub use compact::*; ++pub use data_cache::{LanceDataCacheStatistics, lance_dataset_get_data_cache_statistics}; + pub use data_statistics::*; + pub use dataset::*; + pub use delete::*; +@@ -58,6 +61,7 @@ pub use drop_columns::*; + pub use error::{ + LanceErrorCode, lance_free_string, lance_last_error_code, lance_last_error_message, + }; ++pub use foyer_data_cache::{LanceDataCacheOptions, lance_session_new_with_data_cache}; + pub use fragment_writer::*; + pub use fts_query::*; + pub use index::*; +diff --git a/src/restore.rs b/src/restore.rs +index 7804b55..fa2d26d 100644 +--- a/src/restore.rs ++++ b/src/restore.rs +@@ -65,8 +65,16 @@ unsafe fn restore_inner(dataset: *const LanceDataset, version: u64) -> Result<*m + Ok::<_, lance_core::Error>(checked_out) + })?; + ++ let (restored, data_cache) = if let Some(data_cache) = &ds.data_cache { ++ let (restored, data_cache) = data_cache.attach_fresh(restored); ++ (restored, Some(data_cache)) ++ } else { ++ (restored, None) ++ }; ++ + let handle = LanceDataset { + inner: RwLock::new(Arc::new(restored)), ++ data_cache, + }; + Ok(Box::into_raw(Box::new(handle))) + } +diff --git a/src/session.rs b/src/session.rs +index 60a1623..9ed8cfe 100644 +--- a/src/session.rs ++++ b/src/session.rs +@@ -8,12 +8,14 @@ use std::sync::Arc; + use lance::session::Session; + use lance_core::Result; + ++use crate::data_cache::DataCacheFactory; + use crate::error::{ffi_try, swallow_unwind}; + use crate::runtime::block_on; + +-/// Opaque handle for sharing Lance metadata and index caches across datasets. ++/// Opaque handle for shared Lance caches across datasets. + pub struct LanceSession { + pub(crate) inner: Arc, ++ pub(crate) data_cache_factory: Option>, + } + + /// Snapshot of a session's metadata and index cache statistics. +@@ -47,6 +49,14 @@ pub extern "C" fn lance_session_new( + fn session_new_inner( + index_cache_size_bytes: u64, + metadata_cache_size_bytes: u64, ++) -> Result<*mut LanceSession> { ++ session_new_with_data_cache_factory(index_cache_size_bytes, metadata_cache_size_bytes, None) ++} ++ ++pub(crate) fn session_new_with_data_cache_factory( ++ index_cache_size_bytes: u64, ++ metadata_cache_size_bytes: u64, ++ data_cache_factory: Option>, + ) -> Result<*mut LanceSession> { + let index_cache_size_bytes = u64_to_usize(index_cache_size_bytes, "index_cache_size_bytes")?; + let metadata_cache_size_bytes = +@@ -58,6 +68,7 @@ fn session_new_inner( + ); + Ok(Box::into_raw(Box::new(LanceSession { + inner: Arc::new(session), ++ data_cache_factory, + }))) + } + +diff --git a/src/writer.rs b/src/writer.rs +index 1971510..ba51c87 100644 +--- a/src/writer.rs ++++ b/src/writer.rs +@@ -282,6 +282,7 @@ unsafe fn write_dataset_inner( + if !out_dataset.is_null() { + let handle = LanceDataset { + inner: RwLock::new(Arc::new(dataset)), ++ data_cache: None, + }; + // SAFETY: `out_dataset` is non-NULL (checked above) and the caller + // guarantees it points to caller-owned, writable storage of size +diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs +index 8805764..b4313f4 100644 +--- a/tests/c_api_test.rs ++++ b/tests/c_api_test.rs +@@ -96,10 +96,83 @@ fn create_large_dataset(num_rows: i32) -> (tempfile::TempDir, String) { + (tmp, uri) + } + ++/// Helper: create two fragments large enough for Lance's batched range-read ++/// path, which is the path wrapped by the Foyer data cache. ++fn create_large_multi_fragment_dataset(num_rows_per_fragment: i32) -> (tempfile::TempDir, String) { ++ let (tmp, uri) = create_large_dataset(num_rows_per_fragment); ++ let schema = Arc::new(Schema::new(vec![ ++ Field::new("id", DataType::Int32, false), ++ Field::new("value", DataType::Float32, true), ++ Field::new("label", DataType::Utf8, true), ++ ])); ++ let ids: Vec = (num_rows_per_fragment..2 * num_rows_per_fragment).collect(); ++ let values: Vec = ids.iter().map(|id| *id as f32 * 0.5).collect(); ++ let labels: Vec = ids.iter().map(|id| format!("row_{id}")).collect(); ++ let label_refs: Vec<&str> = labels.iter().map(String::as_str).collect(); ++ let batch = RecordBatch::try_new( ++ schema.clone(), ++ vec![ ++ Arc::new(Int32Array::from(ids)), ++ Arc::new(Float32Array::from(values)), ++ Arc::new(StringArray::from(label_refs)), ++ ], ++ ) ++ .unwrap(); ++ ++ lance_c::runtime::block_on(async { ++ let mut dataset = Dataset::open(&uri).await.unwrap(); ++ dataset ++ .append( ++ arrow::record_batch::RecordBatchIterator::new(vec![Ok(batch)], schema), ++ None, ++ ) ++ .await ++ .unwrap(); ++ }); ++ ++ (tmp, uri) ++} ++ + fn c_str(s: &str) -> CString { + CString::new(s).unwrap() + } + ++fn file_object_store_uri(path: &str) -> CString { ++ let path = path.replace('\\', "/"); ++ let leading_slash = if path.starts_with('/') { "" } else { "/" }; ++ c_str(&format!("file-object-store://{leading_slash}{path}")) ++} ++ ++fn create_data_cache_session() -> (tempfile::TempDir, *mut LanceSession) { ++ let directory = tempfile::tempdir().unwrap(); ++ let c_directory = c_str(directory.path().to_str().unwrap()); ++ let options = LanceDataCacheOptions { ++ directory: c_directory.as_ptr(), ++ memory_capacity_bytes: 8 * 1024 * 1024, ++ disk_capacity_bytes: 32 * 1024 * 1024, ++ read_block_size_bytes: 64 * 1024, ++ }; ++ let session = unsafe { lance_session_new_with_data_cache(0, 16 * 1024 * 1024, &options) }; ++ assert!(!session.is_null(), "data-cache session should be created"); ++ (directory, session) ++} ++ ++fn data_cache_statistics(dataset: *const LanceDataset) -> LanceDataCacheStatistics { ++ let mut statistics = LanceDataCacheStatistics::default(); ++ assert_eq!( ++ unsafe { lance_dataset_get_data_cache_statistics(dataset, &mut statistics) }, ++ 0 ++ ); ++ statistics ++} ++ ++fn scanned_row_count(dataset: *const LanceDataset) -> usize { ++ scan_all_rows(dataset) ++ .iter() ++ .map(RecordBatch::num_rows) ++ .sum() ++} ++ + #[derive(Default)] + struct CapturedScanStatistics { + calls: usize, +@@ -313,6 +386,120 @@ fn test_shared_session_rejects_null_inputs() { + } + } + ++#[test] ++fn test_session_with_data_cache_serves_repeated_scan() { ++ let (tmp, uri) = create_large_multi_fragment_dataset(10_000); ++ let c_uri = file_object_store_uri(&uri); ++ let (_cache_directory, session) = create_data_cache_session(); ++ ++ let dataset = ++ unsafe { lance_dataset_open_with_session(c_uri.as_ptr(), ptr::null(), 0, session) }; ++ assert!(!dataset.is_null(), "dataset open should succeed"); ++ ++ assert_eq!(data_cache_statistics(dataset), Default::default()); ++ assert_eq!(scanned_row_count(dataset), 20_000); ++ let first_statistics = data_cache_statistics(dataset); ++ assert!(first_statistics.bytes_read_from_remote > 0); ++ ++ let cached_dataset = ++ unsafe { lance_dataset_open_with_session(c_uri.as_ptr(), ptr::null(), 0, session) }; ++ assert!( ++ !cached_dataset.is_null(), ++ "second dataset open should succeed" ++ ); ++ unsafe { lance_session_close(session) }; ++ ++ for entry in std::fs::read_dir(tmp.path().join("large_ds/data")).unwrap() { ++ std::fs::remove_file(entry.unwrap().path()).unwrap(); ++ } ++ assert_eq!(scanned_row_count(cached_dataset), 20_000); ++ let cached_statistics = data_cache_statistics(cached_dataset); ++ assert!(cached_statistics.bytes_read_from_cache > 0); ++ assert_eq!(cached_statistics.bytes_read_from_remote, 0); ++ ++ assert_eq!(data_cache_statistics(dataset), first_statistics); ++ ++ unsafe { lance_dataset_close(cached_dataset) }; ++ unsafe { lance_dataset_close(dataset) }; ++} ++ ++#[test] ++fn test_dataset_data_cache_statistics_validates_inputs_and_defaults_to_zero() { ++ let (_tmp, uri) = create_test_dataset(); ++ let c_uri = c_str(&uri); ++ let dataset = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) }; ++ assert!(!dataset.is_null()); ++ ++ let mut statistics = LanceDataCacheStatistics::default(); ++ assert_eq!( ++ unsafe { lance_dataset_get_data_cache_statistics(dataset, &mut statistics) }, ++ 0 ++ ); ++ assert_eq!(statistics, LanceDataCacheStatistics::default()); ++ assert_eq!( ++ unsafe { lance_dataset_get_data_cache_statistics(ptr::null(), &mut statistics) }, ++ -1 ++ ); ++ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); ++ assert_eq!( ++ unsafe { lance_dataset_get_data_cache_statistics(dataset, ptr::null_mut()) }, ++ -1 ++ ); ++ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); ++ ++ unsafe { lance_dataset_close(dataset) }; ++} ++ ++#[test] ++fn test_session_with_data_cache_rejects_invalid_options() { ++ let session = unsafe { lance_session_new_with_data_cache(0, 0, ptr::null()) }; ++ assert!(session.is_null()); ++ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); ++ ++ let null_directory = LanceDataCacheOptions { ++ directory: ptr::null(), ++ memory_capacity_bytes: 128 * 1024, ++ disk_capacity_bytes: 1024 * 1024, ++ read_block_size_bytes: 64 * 1024, ++ }; ++ let session = unsafe { lance_session_new_with_data_cache(0, 0, &null_directory) }; ++ assert!(session.is_null()); ++ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); ++ ++ let empty_directory = c_str(""); ++ let empty_directory_options = LanceDataCacheOptions { ++ directory: empty_directory.as_ptr(), ++ memory_capacity_bytes: 128 * 1024, ++ disk_capacity_bytes: 1024 * 1024, ++ read_block_size_bytes: 64 * 1024, ++ }; ++ let session = unsafe { lance_session_new_with_data_cache(0, 0, &empty_directory_options) }; ++ assert!(session.is_null()); ++ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); ++ ++ let cache_directory = tempfile::tempdir().unwrap(); ++ let c_cache_directory = c_str(cache_directory.path().to_str().unwrap()); ++ let unaligned_block = LanceDataCacheOptions { ++ directory: c_cache_directory.as_ptr(), ++ memory_capacity_bytes: 128 * 1024, ++ disk_capacity_bytes: 1024 * 1024, ++ read_block_size_bytes: 65_535, ++ }; ++ let session = unsafe { lance_session_new_with_data_cache(0, 0, &unaligned_block) }; ++ assert!(session.is_null()); ++ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); ++ ++ let zero_capacity = LanceDataCacheOptions { ++ directory: c_cache_directory.as_ptr(), ++ memory_capacity_bytes: 0, ++ disk_capacity_bytes: 0, ++ read_block_size_bytes: 64 * 1024, ++ }; ++ let session = unsafe { lance_session_new_with_data_cache(0, 0, &zero_capacity) }; ++ assert!(session.is_null()); ++ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); ++} ++ + #[test] + fn test_open_nonexistent() { + let c_uri = c_str("memory://nonexistent_dataset_xyz"); +@@ -2787,6 +2974,40 @@ fn test_dataset_restore_to_prior_version() { + unsafe { lance_dataset_close(ds) }; + } + ++#[test] ++fn test_restored_handle_has_independent_data_cache_statistics() { ++ let (_tmp, uri) = create_large_multi_fragment_dataset(10_000); ++ let c_uri = file_object_store_uri(&uri); ++ let (_cache_directory, session) = create_data_cache_session(); ++ let source = ++ unsafe { lance_dataset_open_with_session(c_uri.as_ptr(), ptr::null(), 0, session) }; ++ assert!(!source.is_null()); ++ ++ assert_eq!(scanned_row_count(source), 20_000); ++ let source_statistics = data_cache_statistics(source); ++ assert!(source_statistics.bytes_read_from_remote > 0); ++ ++ let restored = unsafe { lance_dataset_restore(source, 1) }; ++ assert!(!restored.is_null()); ++ assert_eq!(data_cache_statistics(restored), Default::default()); ++ let source_statistics_after_restore = data_cache_statistics(source); ++ ++ assert_eq!(scanned_row_count(restored), 10_000); ++ let restored_statistics = data_cache_statistics(restored); ++ assert!(restored_statistics.bytes_read_from_cache > 0); ++ ++ assert_eq!( ++ data_cache_statistics(source), ++ source_statistics_after_restore ++ ); ++ ++ unsafe { ++ lance_session_close(session); ++ lance_dataset_close(restored); ++ lance_dataset_close(source); ++ } ++} ++ + #[test] + fn test_dataset_restore_to_current_latest_writes_new_manifest() { + // Restoring to the current latest still writes a new manifest. The +diff --git a/tests/cpp/test_c_api.c b/tests/cpp/test_c_api.c +index c49ecfa..dd674eb 100644 +--- a/tests/cpp/test_c_api.c ++++ b/tests/cpp/test_c_api.c +@@ -126,6 +126,37 @@ static void test_shared_session(const char *uri) { + (unsigned long long)stats.metadata_cache_entries); + } + ++static void test_data_cache_session(const char *uri, const char *write_uri) { ++ printf(" test_data_cache_session... "); ++ ++ char cache_directory[4096]; ++ int path_len = snprintf(cache_directory, sizeof(cache_directory), ++ "%s_foyer_cache", write_uri); ++ ASSERT(path_len > 0 && (size_t)path_len < sizeof(cache_directory), ++ "cache directory path is too long"); ++ LanceDataCacheOptions options = { ++ .directory = cache_directory, ++ .memory_capacity_bytes = 128 * 1024, ++ .disk_capacity_bytes = 1024 * 1024, ++ .read_block_size_bytes = 64 * 1024, ++ }; ++ LanceSession *session = ++ lance_session_new_with_data_cache(0, 16 * 1024 * 1024, &options); ++ ASSERT(session != NULL, "data-cache session creation failed"); ++ ++ LanceDataset *ds = lance_dataset_open_with_session(uri, NULL, 0, session); ++ ASSERT(ds != NULL, "data-cache session dataset open failed"); ++ LanceDataCacheStatistics statistics; ++ memset(&statistics, 0, sizeof(statistics)); ++ ASSERT(lance_dataset_get_data_cache_statistics(ds, &statistics) == 0, ++ "data-cache dataset statistics failed"); ++ lance_session_close(session); ++ ASSERT(lance_dataset_count_rows(ds) > 0, ++ "dataset should remain valid after data-cache session close"); ++ lance_dataset_close(ds); ++ printf("OK\n"); ++} ++ + static void test_scan(const char *uri) { + printf(" test_scan... "); + +@@ -973,6 +1004,7 @@ int main(int argc, char **argv) { + + test_open_and_metadata(uri); + test_shared_session(uri); ++ test_data_cache_session(uri, write_uri); + test_scan(uri); + test_scan_with_limit(uri); + test_versions(uri); +diff --git a/tests/cpp/test_cpp_api.cpp b/tests/cpp/test_cpp_api.cpp +index 17b1ab6..e1aadbd 100644 +--- a/tests/cpp/test_cpp_api.cpp ++++ b/tests/cpp/test_cpp_api.cpp +@@ -99,6 +99,28 @@ static void test_shared_session(const std::string& uri) { + PASS(); + } + ++static void test_data_cache_session(const std::string& uri, ++ const std::string& write_uri) { ++ TEST(test_data_cache_session); ++ ++ lance::DataCacheOptions options{ ++ write_uri + "_foyer_cache", ++ 128 * 1024, ++ 1024 * 1024, ++ 64 * 1024, ++ }; ++ auto session = std::make_unique( ++ 0, 16 * 1024 * 1024, options); ++ auto ds = lance::Dataset::open_with_session(*session, uri); ++ auto statistics = ds.data_cache_statistics(); ++ assert(statistics.bytes_read_from_cache == 0); ++ assert(statistics.bytes_read_from_remote == 0); ++ session.reset(); ++ assert(ds.count_rows() > 0); ++ ++ PASS(); ++} ++ + static void test_dataset_schema(const std::string& uri) { + TEST(test_dataset_schema); + +@@ -922,6 +944,7 @@ int main(int argc, char** argv) { + + test_dataset_open(uri); + test_shared_session(uri); ++ test_data_cache_session(uri, write_uri); + test_dataset_schema(uri); + test_scanner_fluent(uri); + test_scanner_async_stream_ownership(uri); diff --git a/thirdparty/patches/lance-c-0.1.9-pr-74.patch b/thirdparty/patches/lance-c-0.1.9-pr-74.patch new file mode 100644 index 00000000000000..24c6d33457d84c --- /dev/null +++ b/thirdparty/patches/lance-c-0.1.9-pr-74.patch @@ -0,0 +1,1159 @@ +From b07f970bf2cf3f983cc6043bc72a0fe444c0fd4c Mon Sep 17 00:00:00 2001 +From: zhangstar333 +Date: Thu, 3 Sep 2026 17:16:34 +0800 +Subject: [PATCH 1/2] fts + +--- + include/lance/lance.h | 74 ++++++++--- + include/lance/lance.hpp | 44 +++++-- + src/fts_query.rs | 235 +++++++++++++++++++++++++++++------ + src/scanner.rs | 72 ++++++++--- + tests/c_api_test.rs | 264 +++++++++++++++++++++++++++++++++++++--- + 5 files changed, 595 insertions(+), 94 deletions(-) + +diff --git a/include/lance/lance.h b/include/lance/lance.h +index 3bf291f..0c76edc 100644 +--- a/include/lance/lance.h ++++ b/include/lance/lance.h +@@ -1695,39 +1695,85 @@ typedef enum { + LANCE_FTS_COVERAGE_INDEX_ONLY = 1, + } LanceFtsCoverageMode; + ++/** How the analyzed terms of one Match query are combined. */ ++typedef enum { ++ /** At least one analyzed term must match. */ ++ LANCE_FTS_MATCH_OPERATOR_OR = 0, ++ /** Every analyzed term must match. */ ++ LANCE_FTS_MATCH_OPERATOR_AND = 1, ++} LanceFtsMatchOperator; ++ ++/** ++ * Prepare an OR Match query context for one column. ++ * ++ * @deprecated Use lance_dataset_prepare_fts_match_query() to select the Match ++ * operator explicitly. This compatibility API is equivalent to ++ * LANCE_FTS_MATCH_OPERATOR_OR. ++ */ ++LanceFtsQueryContext* lance_dataset_prepare_fts_query( ++ const LanceDataset* dataset, ++ const char* column, ++ const char* query, ++ uint32_t max_fuzzy_distance, ++ int32_t coverage_mode ++); ++ + /** +- * Prepare an immutable, process-local FTS query context for one column. ++ * Prepare an immutable, process-local Match query context for one column. + * + * Preparation pins the dataset handle's current snapshot, enumerates all + * committed FTS segments for `column`, checks fragment coverage, opens those +- * segments, and computes one query-specific global BM25 scorer across their +- * indexed documents. The context can then be shared by any number of scanners +- * created from the exact same process-local dataset snapshot. It has no +- * serialization or cross-process transport format. Reopening the same URI and +- * manifest version creates a different identity and cannot reuse the context, +- * because storage options and object-store endpoints may differ. ++ * segments, and prepares one global BM25 scorer across their indexed ++ * documents. `match_operator` supports both AND and OR. ++ * ++ * The context can be shared by scanners created from the exact same ++ * process-local dataset snapshot. It has no serialization or cross-process ++ * transport format. Reopening the same URI and manifest version creates a ++ * different identity and cannot reuse the context because storage options and ++ * object-store endpoints may differ. + * + * In LANCE_FTS_COVERAGE_INDEX_ONLY mode, unindexed fragments are allowed and + * excluded from both the scorer corpus and query results. In STRICT mode any + * unindexed fragment makes this call fail. + * +- * Prepared contexts currently support exact Match queries only. +- * `max_fuzzy_distance` must be zero because fuzzy execution requires its +- * canonical expanded vocabulary to be prepared together with the scorer. +- * This restriction does not apply to lance_scanner_full_text_search(). +- * +- * @param max_fuzzy_distance Must be zero for prepared query contexts. ++ * @param match_operator Fixed-width LanceFtsMatchOperator discriminant. ++ * @param max_fuzzy_distance Reserved for prepared fuzzy matching and currently ++ * must be 0. The parameter is retained so enabling ++ * canonical cross-segment fuzzy vocabulary injection ++ * later does not require another C ABI change. + * @param coverage_mode Fixed-width LanceFtsCoverageMode discriminant. + * @return Context handle on success, or NULL on error. + */ +-LanceFtsQueryContext* lance_dataset_prepare_fts_query( ++LanceFtsQueryContext* lance_dataset_prepare_fts_match_query( + const LanceDataset* dataset, + const char* column, + const char* query, ++ int32_t match_operator, + uint32_t max_fuzzy_distance, + int32_t coverage_mode + ); + ++/** ++ * Prepare an immutable, process-local Phrase query context for one column. ++ * ++ * The selected FTS index must store token positions. `slop == 0` requires an ++ * exact phrase; a positive value permits that many intervening positions. ++ * Dataset identity, coverage, sharing, and segment-scoped execution follow the ++ * same contract as lance_dataset_prepare_fts_match_query(). ++ * ++ * @param slop Maximum number of intervening token positions permitted between ++ * adjacent phrase terms. ++ * @param coverage_mode Fixed-width LanceFtsCoverageMode discriminant. ++ * @return Context handle on success, or NULL on error. ++ */ ++LanceFtsQueryContext* lance_dataset_prepare_fts_phrase_query( ++ const LanceDataset* dataset, ++ const char* column, ++ const char* query, ++ uint32_t slop, ++ int32_t coverage_mode ++); ++ + /** + * Close a context handle. NULL-safe. Scanners that already attached this + * context retain shared ownership and remain valid. +diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp +index 6cf245f..973216a 100644 +--- a/include/lance/lance.hpp ++++ b/include/lance/lance.hpp +@@ -128,6 +128,11 @@ enum class FtsCoverageMode : int32_t { + IndexOnly = LANCE_FTS_COVERAGE_INDEX_ONLY, + }; + ++enum class FtsMatchOperator : int32_t { ++ Or = LANCE_FTS_MATCH_OPERATOR_OR, ++ And = LANCE_FTS_MATCH_OPERATOR_AND, ++}; ++ + /// Tunable parameters for Dataset::write. Numeric fields default-out via 0; + /// `data_storage_version` defaults out via `std::nullopt`. + /// +@@ -764,18 +769,43 @@ class Dataset { + /// Create a Scanner builder for this dataset. + Scanner scan() const; + +- /// Prepare a query-specific global BM25 scorer over the committed FTS +- /// segments of this pinned snapshot. IndexOnly permits unindexed fragments; +- /// Strict rejects them. Prepared contexts currently require +- /// `max_fuzzy_distance == 0`. The context can only be attached to scanners +- /// created from this exact process-local dataset snapshot. ++ /// Compatibility wrapper for an OR Match query. ++ [[deprecated("Use prepare_fts_match_query() to select the Match operator")]] + FtsQueryContext prepare_fts_query( + const std::string& column, + const std::string& query, + uint32_t max_fuzzy_distance = 0, + FtsCoverageMode coverage_mode = FtsCoverageMode::Strict) const { +- auto* context = lance_dataset_prepare_fts_query( +- handle_.get(), column.c_str(), query.c_str(), max_fuzzy_distance, ++ return prepare_fts_match_query(column, query, FtsMatchOperator::Or, ++ max_fuzzy_distance, coverage_mode); ++ } ++ ++ /// Prepare a Match query with a global BM25 scorer. AND and OR are ++ /// supported. `max_fuzzy_distance` is reserved and currently must be zero; ++ /// keeping it here avoids another API change when canonical cross-segment ++ /// fuzzy vocabulary injection becomes available. ++ FtsQueryContext prepare_fts_match_query( ++ const std::string& column, ++ const std::string& query, ++ FtsMatchOperator match_operator = FtsMatchOperator::Or, ++ uint32_t max_fuzzy_distance = 0, ++ FtsCoverageMode coverage_mode = FtsCoverageMode::Strict) const { ++ auto* context = lance_dataset_prepare_fts_match_query( ++ handle_.get(), column.c_str(), query.c_str(), ++ static_cast(match_operator), max_fuzzy_distance, ++ static_cast(coverage_mode)); ++ if (!context) check_error(); ++ return FtsQueryContext(context); ++ } ++ ++ /// Prepare a Phrase query. Its FTS index must store token positions. ++ FtsQueryContext prepare_fts_phrase_query( ++ const std::string& column, ++ const std::string& query, ++ uint32_t slop = 0, ++ FtsCoverageMode coverage_mode = FtsCoverageMode::Strict) const { ++ auto* context = lance_dataset_prepare_fts_phrase_query( ++ handle_.get(), column.c_str(), query.c_str(), slop, + static_cast(coverage_mode)); + if (!context) check_error(); + return FtsQueryContext(context); +diff --git a/src/fts_query.rs b/src/fts_query.rs +index cd194c7..3bf7f3e 100644 +--- a/src/fts_query.rs ++++ b/src/fts_query.rs +@@ -14,7 +14,9 @@ use lance_core::{Error, Result}; + use lance_index::IndexCriteria; + use lance_index::metrics::NoOpMetricsCollector; + use lance_index::scalar::FullTextSearchQuery; +-use lance_index::scalar::inverted::query::{FtsQuery, collect_query_tokens}; ++use lance_index::scalar::inverted::query::{ ++ FtsQuery, MatchQuery, Operator, PhraseQuery, collect_query_tokens, ++}; + use lance_index::scalar::inverted::{InvertedIndex, MemBM25Scorer, build_global_bm25_scorer}; + use lance_table::format::IndexMetadata; + use uuid::Uuid; +@@ -48,12 +50,53 @@ impl TryFrom for LanceFtsCoverageMode { + } + } + ++/// Operator used to combine the analyzed terms of a Match query. ++#[repr(i32)] ++#[derive(Clone, Copy, Debug, PartialEq, Eq)] ++pub enum LanceFtsMatchOperator { ++ /// At least one analyzed term must match. ++ Or = 0, ++ /// Every analyzed term must match. ++ And = 1, ++} ++ ++impl TryFrom for LanceFtsMatchOperator { ++ type Error = Error; ++ ++ fn try_from(value: i32) -> Result { ++ match value { ++ 0 => Ok(Self::Or), ++ 1 => Ok(Self::And), ++ _ => Err(Error::invalid_input(format!( ++ "invalid match_operator {value}; expected 0 (OR) or 1 (AND)" ++ ))), ++ } ++ } ++} ++ ++impl From for Operator { ++ fn from(value: LanceFtsMatchOperator) -> Self { ++ match value { ++ LanceFtsMatchOperator::Or => Self::Or, ++ LanceFtsMatchOperator::And => Self::And, ++ } ++ } ++} ++ ++/// Query-specific state that must be shared by every segment-scoped scan. ++pub(crate) enum PreparedFtsQuery { ++ /// Exact Match queries share one corpus-wide scorer. ++ Match(Arc), ++ /// Phrase does not expand terms, so a shared global scorer is sufficient. ++ Phrase(Arc), ++} ++ + /// Rust-owned immutable state behind [`LanceFtsQueryContext`]. + pub(crate) struct FtsQueryContextInner { + pub(crate) dataset: Arc, + pub(crate) query: FullTextSearchQuery, + pub(crate) segments: Vec, +- pub(crate) scorer: Arc, ++ pub(crate) prepared: PreparedFtsQuery, + } + + impl FtsQueryContextInner { +@@ -87,7 +130,7 @@ fn invalid_input(message: impl Into) -> Error { + async fn prepare_fts_query_context( + dataset: Arc, + column: String, +- query_text: String, ++ query: FullTextSearchQuery, + coverage_mode: LanceFtsCoverageMode, + ) -> Result { + let logical_index = dataset +@@ -193,34 +236,78 @@ async fn prepare_fts_query_context( + ))); + } + +- let query = FullTextSearchQuery::new(query_text).with_column(column.clone())?; +- let match_query = match &query.query { +- FtsQuery::Match(query) => query, ++ let prepared = match &query.query { ++ FtsQuery::Match(match_query) => { ++ let mut tokenizer = indices[0].tokenizer(); ++ let query_tokens = collect_query_tokens(&match_query.terms, &mut tokenizer); ++ let params = query ++ .params() ++ .with_fuzziness(match_query.fuzziness) ++ .with_max_expansions(match_query.max_expansions) ++ .with_prefix_length(match_query.prefix_length); ++ PreparedFtsQuery::Match(Arc::new( ++ build_global_bm25_scorer(&indices, &query_tokens, ¶ms).await?, ++ )) ++ } ++ FtsQuery::Phrase(phrase_query) => { ++ if !expected_params.has_positions() { ++ return Err(invalid_input(format!( ++ "FTS index '{}' for column '{column}' does not store token positions required by Phrase queries; recreate the index with positions enabled", ++ logical_index.name ++ ))); ++ } ++ let mut tokenizer = indices[0].tokenizer(); ++ let query_tokens = collect_query_tokens(&phrase_query.terms, &mut tokenizer); ++ let params = query.params().with_phrase_slop(Some(phrase_query.slop)); ++ PreparedFtsQuery::Phrase(Arc::new( ++ build_global_bm25_scorer(&indices, &query_tokens, ¶ms).await?, ++ )) ++ } + _ => { + return Err(Error::internal( +- "prepared FTS query unexpectedly produced a non-Match query".to_string(), ++ "prepared FTS query must be a single-column Match or Phrase query".to_string(), + )); + } + }; +- let mut tokenizer = indices[0].tokenizer(); +- let query_tokens = collect_query_tokens(&match_query.terms, &mut tokenizer); +- let params = query +- .params() +- .with_fuzziness(match_query.fuzziness) +- .with_max_expansions(match_query.max_expansions) +- .with_prefix_length(match_query.prefix_length); +- let scorer = Arc::new(build_global_bm25_scorer(&indices, &query_tokens, ¶ms).await?); + + Ok(FtsQueryContextInner { + dataset, + query, + segments, +- scorer, ++ prepared, + }) + } + +-/// Prepare a process-local global BM25 scorer and the committed segment list +-/// for one single-column Match query against the dataset's pinned snapshot. ++unsafe fn parse_query_inputs( ++ dataset: *const LanceDataset, ++ column: *const c_char, ++ query: *const c_char, ++ coverage_mode: i32, ++) -> Result<(Arc, String, String, LanceFtsCoverageMode)> { ++ if dataset.is_null() || column.is_null() || query.is_null() { ++ return Err(invalid_input("dataset, column, and query must not be NULL")); ++ } ++ let column = unsafe { helpers::parse_c_string(column)? } ++ .filter(|value| !value.is_empty()) ++ .ok_or_else(|| invalid_input("column must not be empty"))? ++ .to_string(); ++ let query = unsafe { helpers::parse_c_string(query)? } ++ .filter(|value| !value.is_empty()) ++ .ok_or_else(|| invalid_input("query must not be empty"))? ++ .to_string(); ++ let coverage_mode = LanceFtsCoverageMode::try_from(coverage_mode)?; ++ let snapshot = unsafe { &*dataset }.snapshot(); ++ Ok((snapshot, column, query, coverage_mode)) ++} ++ ++fn into_context(inner: FtsQueryContextInner) -> *mut LanceFtsQueryContext { ++ Box::into_raw(Box::new(LanceFtsQueryContext { ++ inner: Arc::new(inner), ++ })) ++} ++ ++/// Compatibility API for an OR Match query. ++#[deprecated(note = "use lance_dataset_prepare_fts_match_query to select the Match operator")] + #[unsafe(no_mangle)] + pub unsafe extern "C" fn lance_dataset_prepare_fts_query( + dataset: *const LanceDataset, +@@ -231,46 +318,120 @@ pub unsafe extern "C" fn lance_dataset_prepare_fts_query( + ) -> *mut LanceFtsQueryContext { + ffi_try!( + unsafe { +- prepare_fts_query_inner(dataset, column, query, max_fuzzy_distance, coverage_mode) ++ prepare_fts_match_query_inner( ++ dataset, ++ column, ++ query, ++ LanceFtsMatchOperator::Or as i32, ++ max_fuzzy_distance, ++ coverage_mode, ++ ) ++ }, ++ null ++ ) ++} ++ ++/// Prepare a process-local Match query context. AND and OR are supported. ++/// `max_fuzzy_distance` is retained for the future prepared-fuzzy path but ++/// must be zero with the currently pinned Lance revision. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_dataset_prepare_fts_match_query( ++ dataset: *const LanceDataset, ++ column: *const c_char, ++ query: *const c_char, ++ match_operator: i32, ++ max_fuzzy_distance: u32, ++ coverage_mode: i32, ++) -> *mut LanceFtsQueryContext { ++ ffi_try!( ++ unsafe { ++ prepare_fts_match_query_inner( ++ dataset, ++ column, ++ query, ++ match_operator, ++ max_fuzzy_distance, ++ coverage_mode, ++ ) + }, + null + ) + } + +-unsafe fn prepare_fts_query_inner( ++unsafe fn prepare_fts_match_query_inner( + dataset: *const LanceDataset, + column: *const c_char, + query: *const c_char, ++ match_operator: i32, + max_fuzzy_distance: u32, + coverage_mode: i32, + ) -> Result<*mut LanceFtsQueryContext> { +- if dataset.is_null() || column.is_null() || query.is_null() { +- return Err(invalid_input("dataset, column, and query must not be NULL")); +- } +- let column = unsafe { helpers::parse_c_string(column)? } +- .filter(|value| !value.is_empty()) +- .ok_or_else(|| invalid_input("column must not be empty"))? +- .to_string(); +- let query = unsafe { helpers::parse_c_string(query)? } +- .filter(|value| !value.is_empty()) +- .ok_or_else(|| invalid_input("query must not be empty"))? +- .to_string(); +- let coverage_mode = LanceFtsCoverageMode::try_from(coverage_mode)?; ++ let (snapshot, column, query_text, coverage_mode) = ++ unsafe { parse_query_inputs(dataset, column, query, coverage_mode)? }; ++ let operator: Operator = LanceFtsMatchOperator::try_from(match_operator)?.into(); ++ // The parameter remains in the public API so callers do not need another ++ // ABI change when Lance-C moves to a Lance revision that can inject the ++ // same canonical fuzzy vocabulary into every segment-scoped scan. The ++ // pinned Lance revision can share only the scorer, so accepting fuzzy here ++ // would allow different segments to choose different capped expansions. + if max_fuzzy_distance != 0 { + return Err(invalid_input(format!( +- "max_fuzzy_distance must be 0 for prepared FTS query contexts, got {max_fuzzy_distance}; fuzzy queries require a canonical prepared BM25 vocabulary" ++ "max_fuzzy_distance must be 0 for prepared FTS with the pinned Lance revision, got {max_fuzzy_distance}; the parameter is reserved until canonical fuzzy vocabulary injection is available" + ))); + } +- let snapshot = unsafe { &*dataset }.snapshot(); ++ let query = FullTextSearchQuery::new_query( ++ MatchQuery::new(query_text) ++ .with_column(Some(column.clone())) ++ .with_operator(operator) ++ .with_fuzziness(Some(0)) ++ .into(), ++ ); + let inner = block_on(prepare_fts_query_context( + snapshot, + column, + query, + coverage_mode, + ))?; +- Ok(Box::into_raw(Box::new(LanceFtsQueryContext { +- inner: Arc::new(inner), +- }))) ++ Ok(into_context(inner)) ++} ++ ++/// Prepare a process-local Phrase query context. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_dataset_prepare_fts_phrase_query( ++ dataset: *const LanceDataset, ++ column: *const c_char, ++ query: *const c_char, ++ slop: u32, ++ coverage_mode: i32, ++) -> *mut LanceFtsQueryContext { ++ ffi_try!( ++ unsafe { prepare_fts_phrase_query_inner(dataset, column, query, slop, coverage_mode) }, ++ null ++ ) ++} ++ ++unsafe fn prepare_fts_phrase_query_inner( ++ dataset: *const LanceDataset, ++ column: *const c_char, ++ query: *const c_char, ++ slop: u32, ++ coverage_mode: i32, ++) -> Result<*mut LanceFtsQueryContext> { ++ let (snapshot, column, query_text, coverage_mode) = ++ unsafe { parse_query_inputs(dataset, column, query, coverage_mode)? }; ++ let query = FullTextSearchQuery::new_query( ++ PhraseQuery::new(query_text) ++ .with_column(Some(column.clone())) ++ .with_slop(slop) ++ .into(), ++ ); ++ let inner = block_on(prepare_fts_query_context( ++ snapshot, ++ column, ++ query, ++ coverage_mode, ++ ))?; ++ Ok(into_context(inner)) + } + + /// Close a context handle. NULL-safe. Scanners that already attached the +diff --git a/src/scanner.rs b/src/scanner.rs +index 0c29b17..cbf1b13 100644 +--- a/src/scanner.rs ++++ b/src/scanner.rs +@@ -18,7 +18,7 @@ use lance::Dataset; + use lance::dataset::scanner::{ + DatasetRecordBatchStream, ExecutionStatsCallback, ExecutionSummaryCounts, + }; +-use lance::io::exec::fts::{FlatMatchQueryExec, MatchQueryExec}; ++use lance::io::exec::fts::{FlatMatchQueryExec, MatchQueryExec, PhraseQueryExec}; + use lance_core::Result; + use lance_index::scalar::FullTextSearchQuery; + use lance_io::stream::RecordBatchStream; +@@ -33,7 +33,8 @@ use crate::error::{ + panic_payload_message, set_lance_error, set_last_error, swallow_unwind, + }; + use crate::fts_query::{ +- FtsQueryContextInner, LanceFtsQueryContext, clone_context, parse_segment_uuids, ++ FtsQueryContextInner, LanceFtsQueryContext, PreparedFtsQuery, clone_context, ++ parse_segment_uuids, + }; + use crate::helpers; + use crate::runtime::{RT, block_on}; +@@ -328,17 +329,17 @@ impl PreparedScanner { + let (plan, rewritten) = rewrite_prepared_fts_plan( + plan, + &distributed_fts.segments, +- &distributed_fts.context.scorer, ++ &distributed_fts.context.prepared, + selected_segments_have_current_fragments, + )?; +- if rewritten.match_query_execs > 1 ++ if rewritten.indexed_query_execs > 1 + || rewritten.flat_match_query_execs > 1 +- || rewritten.match_query_execs + rewritten.flat_match_query_execs == 0 +- || (selected_segments_have_current_fragments && rewritten.match_query_execs != 1) ++ || rewritten.indexed_query_execs + rewritten.flat_match_query_execs == 0 ++ || (selected_segments_have_current_fragments && rewritten.indexed_query_execs != 1) + { + return Err(lance_core::Error::internal(format!( +- "unexpected prepared FTS plan for selected segments with current fragment coverage {selected_segments_have_current_fragments}: rewrote {} MatchQueryExec node(s) and removed {} FlatMatchQueryExec node(s)", +- rewritten.match_query_execs, rewritten.flat_match_query_execs ++ "unexpected prepared FTS plan for selected segments with current fragment coverage {selected_segments_have_current_fragments}: rewrote {} indexed FTS query node(s) and removed {} FlatMatchQueryExec node(s)", ++ rewritten.indexed_query_execs, rewritten.flat_match_query_execs + ))); + } + let stream = lance_datafusion::exec::execute_plan( +@@ -420,14 +421,14 @@ fn segments_have_current_fragments( + + #[derive(Default)] + struct PreparedFtsPlanRewriteCounts { +- match_query_execs: usize, ++ indexed_query_execs: usize, + flat_match_query_execs: usize, + } + + fn rewrite_prepared_fts_plan( + plan: Arc, + segments: &[IndexMetadata], +- scorer: &Arc, ++ prepared: &PreparedFtsQuery, + selected_segments_have_current_fragments: bool, + ) -> Result<(Arc, PreparedFtsPlanRewriteCounts)> { + // Lance's ordinary FTS planner adds a flat-search branch for fragments not +@@ -438,7 +439,7 @@ fn rewrite_prepared_fts_plan( + return Ok(( + Arc::new(EmptyExec::new(plan.schema())), + PreparedFtsPlanRewriteCounts { +- match_query_execs: 0, ++ indexed_query_execs: 0, + flat_match_query_execs: 1, + }, + )); +@@ -454,11 +455,11 @@ fn rewrite_prepared_fts_plan( + let (new_child, child_rewritten) = rewrite_prepared_fts_plan( + Arc::clone(child), + segments, +- scorer, ++ prepared, + selected_segments_have_current_fragments, + )?; + new_children.push(new_child); +- rewritten.match_query_execs += child_rewritten.match_query_execs; ++ rewritten.indexed_query_execs += child_rewritten.indexed_query_execs; + rewritten.flat_match_query_execs += child_rewritten.flat_match_query_execs; + } + plan.with_new_children(new_children).map_err(|error| { +@@ -469,10 +470,15 @@ fn rewrite_prepared_fts_plan( + }; + + if let Some(exec) = rebuilt.downcast_ref::() { +- rewritten.match_query_execs += 1; ++ rewritten.indexed_query_execs += 1; + if !selected_segments_have_current_fragments { + return Ok((Arc::new(EmptyExec::new(rebuilt.schema())), rewritten)); + } ++ let PreparedFtsQuery::Match(scorer) = prepared else { ++ return Err(lance_core::Error::internal( ++ "prepared Phrase state cannot be attached to MatchQueryExec".to_string(), ++ )); ++ }; + let replacement = MatchQueryExec::new_with_segments( + Arc::clone(exec.dataset()), + exec.query().clone(), +@@ -483,6 +489,26 @@ fn rewrite_prepared_fts_plan( + .with_base_scorer(Arc::clone(scorer)); + return Ok((Arc::new(replacement), rewritten)); + } ++ if let Some(exec) = rebuilt.downcast_ref::() { ++ rewritten.indexed_query_execs += 1; ++ if !selected_segments_have_current_fragments { ++ return Ok((Arc::new(EmptyExec::new(rebuilt.schema())), rewritten)); ++ } ++ let PreparedFtsQuery::Phrase(scorer) = prepared else { ++ return Err(lance_core::Error::internal( ++ "prepared Match state cannot be attached to PhraseQueryExec".to_string(), ++ )); ++ }; ++ let replacement = PhraseQueryExec::new_with_segments( ++ Arc::clone(exec.dataset()), ++ exec.query().clone(), ++ exec.params().clone(), ++ exec.prefilter_source().clone(), ++ segments.to_vec(), ++ ) ++ .with_base_scorer(Arc::clone(scorer)); ++ return Ok((Arc::new(replacement), rewritten)); ++ } + Ok((rebuilt, rewritten)) + } + +@@ -2152,7 +2178,8 @@ mod tests { + use crate::dataset::{lance_dataset_close, lance_dataset_open}; + use crate::error::{lance_last_error_code, lance_last_error_message}; + use crate::fts_query::{ +- LanceFtsCoverageMode, lance_dataset_prepare_fts_query, lance_fts_query_context_close, ++ LanceFtsCoverageMode, LanceFtsMatchOperator, lance_dataset_prepare_fts_match_query, ++ lance_fts_query_context_close, + }; + use std::ffi::{CStr, CString}; + use std::sync::atomic::{AtomicI32, AtomicUsize}; +@@ -2276,10 +2303,11 @@ mod tests { + let column = CString::new("name").unwrap(); + let query = CString::new("a").unwrap(); + let context = unsafe { +- lance_dataset_prepare_fts_query( ++ lance_dataset_prepare_fts_match_query( + dataset, + column.as_ptr(), + query.as_ptr(), ++ LanceFtsMatchOperator::Or as i32, + 0, + LanceFtsCoverageMode::IndexOnly as i32, + ) +@@ -2293,7 +2321,6 @@ mod tests { + let prepared = unsafe { &*scanner }.build_scanner().unwrap(); + let distributed = prepared.distributed_fts.as_ref().unwrap(); + let segments = distributed.segments.clone(); +- let scorer = Arc::clone(&distributed.context.scorer); + let plan = block_on(prepared.scanner.create_plan()).unwrap(); + assert_eq!( + prepared_fts_plan_shape(&plan), +@@ -2303,9 +2330,14 @@ mod tests { + + let has_current_fragments = + segments_have_current_fragments(&distributed.context.dataset, &segments).unwrap(); +- let (rewritten, counts) = +- rewrite_prepared_fts_plan(plan, &segments, &scorer, has_current_fragments).unwrap(); +- assert_eq!(counts.match_query_execs, 1); ++ let (rewritten, counts) = rewrite_prepared_fts_plan( ++ plan, ++ &segments, ++ &distributed.context.prepared, ++ has_current_fragments, ++ ) ++ .unwrap(); ++ assert_eq!(counts.indexed_query_execs, 1); + assert_eq!(counts.flat_match_query_execs, 1); + assert_eq!(prepared_fts_plan_shape(&rewritten), (1, 0, 0)); + +diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs +index 8805764..a3e4ef7 100644 +--- a/tests/c_api_test.rs ++++ b/tests/c_api_test.rs +@@ -5762,6 +5762,203 @@ fn load_fts_segment_uuids(uri: &str, column: &str) -> Vec<[u8; 16]> { + }) + } + ++#[test] ++#[allow(deprecated)] ++fn test_prepared_fts_match_phrase_and_legacy_compatibility() { ++ let tmp = tempfile::tempdir().unwrap(); ++ let uri = tmp ++ .path() ++ .join("prepared_fts_queries") ++ .to_str() ++ .unwrap() ++ .to_string(); ++ let schema = Arc::new(Schema::new(vec![ ++ Field::new("id", DataType::Int32, false), ++ Field::new("text", DataType::Utf8, false), ++ ])); ++ let batch = RecordBatch::try_new( ++ schema.clone(), ++ vec![ ++ Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5])), ++ Arc::new(StringArray::from(vec![ ++ "quick brown fox", ++ "quick blue fox", ++ "slow brown fox", ++ "quik brown fox", ++ "quick red brown fox", ++ ])), ++ ], ++ ) ++ .unwrap(); ++ lance_c::runtime::block_on(async { ++ Dataset::write( ++ arrow::record_batch::RecordBatchIterator::new(vec![Ok(batch)], schema), ++ &uri, ++ None, ++ ) ++ .await ++ .unwrap(); ++ }); ++ ++ let uri_c = c_str(&uri); ++ let column = c_str("text"); ++ let index_params = ++ c_str(r#"{"base_tokenizer":"simple","language":"English","with_position":true}"#); ++ let dataset = unsafe { lance_dataset_open(uri_c.as_ptr(), ptr::null(), 0) }; ++ assert_eq!( ++ unsafe { ++ lance_dataset_create_scalar_index( ++ dataset, ++ column.as_ptr(), ++ ptr::null(), ++ LanceScalarIndexType::Inverted as i32, ++ index_params.as_ptr(), ++ false, ++ ) ++ }, ++ 0 ++ ); ++ ++ let query = c_str("quick brown"); ++ let exact_or = unsafe { ++ lance_dataset_prepare_fts_match_query( ++ dataset, ++ column.as_ptr(), ++ query.as_ptr(), ++ LanceFtsMatchOperator::Or as i32, ++ 0, ++ LanceFtsCoverageMode::Strict as i32, ++ ) ++ }; ++ assert!(!exact_or.is_null()); ++ assert_eq!(collect_context_fts_scores(dataset, exact_or, None).len(), 5); ++ unsafe { lance_fts_query_context_close(exact_or) }; ++ ++ let legacy_or = unsafe { ++ lance_dataset_prepare_fts_query( ++ dataset, ++ column.as_ptr(), ++ query.as_ptr(), ++ 0, ++ LanceFtsCoverageMode::Strict as i32, ++ ) ++ }; ++ assert!(!legacy_or.is_null()); ++ assert_eq!( ++ collect_context_fts_scores(dataset, legacy_or, None).len(), ++ 5 ++ ); ++ unsafe { lance_fts_query_context_close(legacy_or) }; ++ ++ let exact_and = unsafe { ++ lance_dataset_prepare_fts_match_query( ++ dataset, ++ column.as_ptr(), ++ query.as_ptr(), ++ LanceFtsMatchOperator::And as i32, ++ 0, ++ LanceFtsCoverageMode::Strict as i32, ++ ) ++ }; ++ assert!(!exact_and.is_null()); ++ let exact_and_scores = collect_context_fts_scores(dataset, exact_and, None); ++ let mut exact_and_ids = exact_and_scores.keys().copied().collect::>(); ++ exact_and_ids.sort_unstable(); ++ assert_eq!(exact_and_ids, vec![1, 5]); ++ unsafe { lance_fts_query_context_close(exact_and) }; ++ ++ let fuzzy_and = unsafe { ++ lance_dataset_prepare_fts_match_query( ++ dataset, ++ column.as_ptr(), ++ query.as_ptr(), ++ LanceFtsMatchOperator::And as i32, ++ 1, ++ LanceFtsCoverageMode::Strict as i32, ++ ) ++ }; ++ assert!(fuzzy_and.is_null()); ++ let message = take_last_error_message(); ++ assert!( ++ message.contains("max_fuzzy_distance must be 0"), ++ "{message}" ++ ); ++ ++ let phrase = unsafe { ++ lance_dataset_prepare_fts_phrase_query( ++ dataset, ++ column.as_ptr(), ++ query.as_ptr(), ++ 0, ++ LanceFtsCoverageMode::Strict as i32, ++ ) ++ }; ++ assert!(!phrase.is_null(), "{}", take_last_error_message()); ++ let phrase_scores = collect_context_fts_scores(dataset, phrase, None); ++ assert_eq!(phrase_scores.keys().copied().collect::>(), vec![1]); ++ unsafe { lance_fts_query_context_close(phrase) }; ++ ++ let phrase_with_slop = unsafe { ++ lance_dataset_prepare_fts_phrase_query( ++ dataset, ++ column.as_ptr(), ++ query.as_ptr(), ++ 1, ++ LanceFtsCoverageMode::Strict as i32, ++ ) ++ }; ++ assert!(!phrase_with_slop.is_null(), "{}", take_last_error_message()); ++ let phrase_with_slop_scores = collect_context_fts_scores(dataset, phrase_with_slop, None); ++ let mut phrase_with_slop_ids = phrase_with_slop_scores.keys().copied().collect::>(); ++ phrase_with_slop_ids.sort_unstable(); ++ assert_eq!(phrase_with_slop_ids, vec![1, 5]); ++ unsafe { lance_fts_query_context_close(phrase_with_slop) }; ++ ++ unsafe { lance_dataset_close(dataset) }; ++} ++ ++#[test] ++fn test_prepared_fts_phrase_requires_positions() { ++ let (_tmp, uri) = create_test_dataset(); ++ let uri_c = c_str(&uri); ++ let column = c_str("name"); ++ let query = c_str("alice smith"); ++ let index_params = c_str(r#"{"base_tokenizer":"simple","language":"English"}"#); ++ let dataset = unsafe { lance_dataset_open(uri_c.as_ptr(), ptr::null(), 0) }; ++ assert_eq!( ++ unsafe { ++ lance_dataset_create_scalar_index( ++ dataset, ++ column.as_ptr(), ++ ptr::null(), ++ LanceScalarIndexType::Inverted as i32, ++ index_params.as_ptr(), ++ false, ++ ) ++ }, ++ 0 ++ ); ++ ++ let context = unsafe { ++ lance_dataset_prepare_fts_phrase_query( ++ dataset, ++ column.as_ptr(), ++ query.as_ptr(), ++ 0, ++ LanceFtsCoverageMode::Strict as i32, ++ ) ++ }; ++ assert!(context.is_null()); ++ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); ++ let message = take_last_error_message(); ++ assert!( ++ message.contains("does not store token positions"), ++ "{message}" ++ ); ++ ++ unsafe { lance_dataset_close(dataset) }; ++} ++ + #[test] + fn test_prepared_fts_row_id_output_is_explicit() { + let (_tmp, uri) = create_test_dataset(); +@@ -5785,10 +5982,11 @@ fn test_prepared_fts_row_id_output_is_explicit() { + 0 + ); + let context = unsafe { +- lance_dataset_prepare_fts_query( ++ lance_dataset_prepare_fts_match_query( + dataset, + column.as_ptr(), + query.as_ptr(), ++ LanceFtsMatchOperator::Or as i32, + 0, + LanceFtsCoverageMode::Strict as i32, + ) +@@ -5880,10 +6078,11 @@ fn test_prepare_fts_query_index_only_allows_unindexed_fragment() { + + let dataset = unsafe { lance_dataset_open(uri_c.as_ptr(), ptr::null(), 0) }; + let strict = unsafe { +- lance_dataset_prepare_fts_query( ++ lance_dataset_prepare_fts_match_query( + dataset, + column.as_ptr(), + query.as_ptr(), ++ LanceFtsMatchOperator::Or as i32, + 0, + LanceFtsCoverageMode::Strict as i32, + ) +@@ -5898,10 +6097,11 @@ fn test_prepare_fts_query_index_only_allows_unindexed_fragment() { + assert!(message.contains("unindexed fragments"), "{message}"); + + let context = unsafe { +- lance_dataset_prepare_fts_query( ++ lance_dataset_prepare_fts_match_query( + dataset, + column.as_ptr(), + query.as_ptr(), ++ LanceFtsMatchOperator::Or as i32, + 0, + LanceFtsCoverageMode::IndexOnly as i32, + ) +@@ -5976,10 +6176,11 @@ fn test_prepared_fts_index_only_empty_segment_returns_empty_shard() { + let query = c_str("alice"); + let dataset = unsafe { lance_dataset_open(uri_c.as_ptr(), ptr::null(), 0) }; + let context = unsafe { +- lance_dataset_prepare_fts_query( ++ lance_dataset_prepare_fts_match_query( + dataset, + column.as_ptr(), + query.as_ptr(), ++ LanceFtsMatchOperator::Or as i32, + 0, + LanceFtsCoverageMode::IndexOnly as i32, + ) +@@ -6075,10 +6276,11 @@ fn test_prepared_fts_global_scorer_is_shared_across_segment_splits() { + + let dataset = unsafe { lance_dataset_open(uri_c.as_ptr(), ptr::null(), 0) }; + let context = unsafe { +- lance_dataset_prepare_fts_query( ++ lance_dataset_prepare_fts_match_query( + dataset, + column.as_ptr(), + query.as_ptr(), ++ LanceFtsMatchOperator::Or as i32, + 0, + LanceFtsCoverageMode::Strict as i32, + ) +@@ -6186,7 +6388,7 @@ fn test_prepared_fts_global_scorer_is_shared_across_segment_splits() { + } + + #[test] +-fn test_prepare_fts_query_rejects_null_empty_invalid_mode_and_fuzzy() { ++fn test_prepare_fts_queries_reject_invalid_inputs() { + let (_tmp, uri) = create_test_dataset(); + let uri_c = c_str(&uri); + let dataset = unsafe { lance_dataset_open(uri_c.as_ptr(), ptr::null(), 0) }; +@@ -6196,10 +6398,11 @@ fn test_prepare_fts_query_rejects_null_empty_invalid_mode_and_fuzzy() { + + assert!( + unsafe { +- lance_dataset_prepare_fts_query( ++ lance_dataset_prepare_fts_match_query( + ptr::null(), + column.as_ptr(), + query.as_ptr(), ++ LanceFtsMatchOperator::Or as i32, + 0, + LanceFtsCoverageMode::Strict as i32, + ) +@@ -6208,10 +6411,11 @@ fn test_prepare_fts_query_rejects_null_empty_invalid_mode_and_fuzzy() { + ); + assert!( + unsafe { +- lance_dataset_prepare_fts_query( ++ lance_dataset_prepare_fts_match_query( + dataset, + empty.as_ptr(), + query.as_ptr(), ++ LanceFtsMatchOperator::Or as i32, + 0, + LanceFtsCoverageMode::Strict as i32, + ) +@@ -6219,20 +6423,39 @@ fn test_prepare_fts_query_rejects_null_empty_invalid_mode_and_fuzzy() { + .is_null() + ); + assert!( +- unsafe { lance_dataset_prepare_fts_query(dataset, column.as_ptr(), empty.as_ptr(), 0, 0) } +- .is_null() ++ unsafe { ++ lance_dataset_prepare_fts_match_query( ++ dataset, ++ column.as_ptr(), ++ empty.as_ptr(), ++ LanceFtsMatchOperator::Or as i32, ++ 0, ++ LanceFtsCoverageMode::Strict as i32, ++ ) ++ } ++ .is_null() + ); + assert!( +- unsafe { lance_dataset_prepare_fts_query(dataset, column.as_ptr(), query.as_ptr(), 0, 99) } +- .is_null() ++ unsafe { ++ lance_dataset_prepare_fts_match_query( ++ dataset, ++ column.as_ptr(), ++ query.as_ptr(), ++ LanceFtsMatchOperator::Or as i32, ++ 0, ++ 99, ++ ) ++ } ++ .is_null() + ); + assert!( + unsafe { +- lance_dataset_prepare_fts_query( ++ lance_dataset_prepare_fts_match_query( + dataset, + column.as_ptr(), + query.as_ptr(), +- 1, ++ 99, ++ 0, + LanceFtsCoverageMode::Strict as i32, + ) + } +@@ -6244,9 +6467,18 @@ fn test_prepare_fts_query_rejects_null_empty_invalid_mode_and_fuzzy() { + .to_string_lossy() + .into_owned() + }; ++ assert!(message.contains("invalid match_operator"), "{message}"); + assert!( +- message.contains("max_fuzzy_distance must be 0"), +- "{message}" ++ unsafe { ++ lance_dataset_prepare_fts_phrase_query( ++ dataset, ++ column.as_ptr(), ++ ptr::null(), ++ 0, ++ LanceFtsCoverageMode::Strict as i32, ++ ) ++ } ++ .is_null() + ); + let scanner = unsafe { lance_scanner_new(dataset, ptr::null(), ptr::null()) }; + assert_eq!( + +From 9bb38749c9de185664511def3cbeb699ee60a38b Mon Sep 17 00:00:00 2001 +From: zhangstar333 +Date: Thu, 3 Sep 2026 17:49:02 +0800 +Subject: [PATCH 2/2] update slop i32 + +--- + include/lance/lance.h | 6 +++--- + include/lance/lance.hpp | 5 +++-- + src/fts_query.rs | 6 ++++-- + tests/c_api_test.rs | 14 ++++++++++++++ + 4 files changed, 24 insertions(+), 7 deletions(-) + +diff --git a/include/lance/lance.h b/include/lance/lance.h +index 0c76edc..25c0430 100644 +--- a/include/lance/lance.h ++++ b/include/lance/lance.h +@@ -1761,8 +1761,8 @@ LanceFtsQueryContext* lance_dataset_prepare_fts_match_query( + * Dataset identity, coverage, sharing, and segment-scoped execution follow the + * same contract as lance_dataset_prepare_fts_match_query(). + * +- * @param slop Maximum number of intervening token positions permitted between +- * adjacent phrase terms. ++ * @param slop Maximum non-negative number of intervening token positions ++ * permitted between adjacent phrase terms. + * @param coverage_mode Fixed-width LanceFtsCoverageMode discriminant. + * @return Context handle on success, or NULL on error. + */ +@@ -1770,7 +1770,7 @@ LanceFtsQueryContext* lance_dataset_prepare_fts_phrase_query( + const LanceDataset* dataset, + const char* column, + const char* query, +- uint32_t slop, ++ int32_t slop, + int32_t coverage_mode + ); + +diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp +index 973216a..1e69a06 100644 +--- a/include/lance/lance.hpp ++++ b/include/lance/lance.hpp +@@ -798,11 +798,12 @@ class Dataset { + return FtsQueryContext(context); + } + +- /// Prepare a Phrase query. Its FTS index must store token positions. ++ /// Prepare a Phrase query. Its FTS index must store token positions and ++ /// slop must be non-negative. + FtsQueryContext prepare_fts_phrase_query( + const std::string& column, + const std::string& query, +- uint32_t slop = 0, ++ int32_t slop = 0, + FtsCoverageMode coverage_mode = FtsCoverageMode::Strict) const { + auto* context = lance_dataset_prepare_fts_phrase_query( + handle_.get(), column.c_str(), query.c_str(), slop, +diff --git a/src/fts_query.rs b/src/fts_query.rs +index 3bf7f3e..c9d3a43 100644 +--- a/src/fts_query.rs ++++ b/src/fts_query.rs +@@ -401,7 +401,7 @@ pub unsafe extern "C" fn lance_dataset_prepare_fts_phrase_query( + dataset: *const LanceDataset, + column: *const c_char, + query: *const c_char, +- slop: u32, ++ slop: i32, + coverage_mode: i32, + ) -> *mut LanceFtsQueryContext { + ffi_try!( +@@ -414,9 +414,11 @@ unsafe fn prepare_fts_phrase_query_inner( + dataset: *const LanceDataset, + column: *const c_char, + query: *const c_char, +- slop: u32, ++ slop: i32, + coverage_mode: i32, + ) -> Result<*mut LanceFtsQueryContext> { ++ let slop = u32::try_from(slop) ++ .map_err(|_| invalid_input(format!("slop must be non-negative, got {slop}")))?; + let (snapshot, column, query_text, coverage_mode) = + unsafe { parse_query_inputs(dataset, column, query, coverage_mode)? }; + let query = FullTextSearchQuery::new_query( +diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs +index a3e4ef7..9411e15 100644 +--- a/tests/c_api_test.rs ++++ b/tests/c_api_test.rs +@@ -5914,6 +5914,20 @@ fn test_prepared_fts_match_phrase_and_legacy_compatibility() { + assert_eq!(phrase_with_slop_ids, vec![1, 5]); + unsafe { lance_fts_query_context_close(phrase_with_slop) }; + ++ let negative_phrase_slop = unsafe { ++ lance_dataset_prepare_fts_phrase_query( ++ dataset, ++ column.as_ptr(), ++ query.as_ptr(), ++ -1, ++ LanceFtsCoverageMode::Strict as i32, ++ ) ++ }; ++ assert!(negative_phrase_slop.is_null()); ++ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); ++ let message = take_last_error_message(); ++ assert!(message.contains("slop must be non-negative"), "{message}"); ++ + unsafe { lance_dataset_close(dataset) }; + } + diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index 219bdef38b03a8..9592b84e602579 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -552,10 +552,10 @@ PUGIXML_SOURCE=pugixml-1.15 PUGIXML_MD5SUM="3b894c29455eb33a40b165c6e2de5895" # lance-c -LANCE_C_DOWNLOAD="https://github.com/lance-format/lance-c/archive/refs/tags/v0.1.7.tar.gz" -LANCE_C_NAME="lance-c-v0.1.7.tar.gz" -LANCE_C_SOURCE="lance-c-0.1.7" -LANCE_C_MD5SUM="15ef7cd20a2e1606384251cb2d41d42f" +LANCE_C_DOWNLOAD="https://github.com/lance-format/lance-c/archive/refs/tags/v0.1.9.tar.gz" +LANCE_C_NAME="lance-c-v0.1.9.tar.gz" +LANCE_C_SOURCE="lance-c-0.1.9" +LANCE_C_MD5SUM="7138ed44e92d4bc91d5b522a6b92ed64" # all thirdparties which need to be downloaded is set in array TP_ARCHIVES export TP_ARCHIVES=(