From fcdfb9276e6987cd5f8c6ae8a680934a402e29a0 Mon Sep 17 00:00:00 2001 From: Shenghui Wu <793703860@qq.com> Date: Fri, 30 May 2025 14:55:55 +0800 Subject: [PATCH 01/58] *: support tici scan operator (#10200) (cherry picked from commit 240a9ee790c400c895a103036ad746f3686c137a) --- .gitmodules | 3 + contrib/CMakeLists.txt | 2 + contrib/tici | 1 + contrib/tici-search-lib/CMakeLists.txt | 30 ++++ dbms/CMakeLists.txt | 5 + dbms/src/Common/TiFlashMetrics.h | 3 +- dbms/src/Debug/dbgQueryExecutor.cpp | 26 ++- dbms/src/Flash/BatchCoprocessorHandler.cpp | 1 + dbms/src/Flash/Coprocessor/DAGContext.cpp | 2 + dbms/src/Flash/Coprocessor/DAGContext.h | 3 + .../Flash/Coprocessor/GenSchemaAndColumn.cpp | 13 ++ .../Flash/Coprocessor/GenSchemaAndColumn.h | 1 + dbms/src/Flash/Coprocessor/ShardInfo.h | 142 ++++++++++++++++ .../Coprocessor/StorageTantivyInterpreter.h | 58 +++++++ dbms/src/Flash/Coprocessor/TiCIScan.cpp | 38 +++++ dbms/src/Flash/Coprocessor/TiCIScan.h | 50 ++++++ .../Coprocessor/collectOutputFieldTypes.cpp | 19 +++ dbms/src/Flash/CoprocessorHandler.cpp | 1 + dbms/src/Flash/Planner/PhysicalPlan.cpp | 15 ++ dbms/src/Flash/Planner/PhysicalPlan.h | 1 + dbms/src/Flash/Planner/PlanType.h | 1 + .../Flash/Planner/Plans/PhysicalTiCIScan.cpp | 143 ++++++++++++++++ .../Flash/Planner/Plans/PhysicalTiCIScan.h | 64 +++++++ .../ExecutorStatisticsCollector.cpp | 9 +- .../Flash/Statistics/traverseExecutors.cpp | 6 +- dbms/src/Operators/TantivyReaderSourceOp.cpp | 94 +++++++++++ dbms/src/Operators/TantivyReaderSourceOp.h | 63 +++++++ dbms/src/Server/CMakeLists.txt | 4 + dbms/src/Storages/StorageTantivy.cpp | 84 ++++++++++ dbms/src/Storages/StorageTantivy.h | 78 +++++++++ .../src/Storages/Tantivy/TantivyInputStream.h | 158 ++++++++++++++++++ .../Storages/tests/gtest_filter_parser.cpp | 2 +- .../tests/gtests_parse_push_down_filter.cpp | 2 +- dbms/src/TestUtils/MPPTaskTestUtils.cpp | 2 +- rust-toolchain.toml | 5 +- 35 files changed, 1111 insertions(+), 18 deletions(-) create mode 160000 contrib/tici create mode 100644 contrib/tici-search-lib/CMakeLists.txt create mode 100644 dbms/src/Flash/Coprocessor/ShardInfo.h create mode 100644 dbms/src/Flash/Coprocessor/StorageTantivyInterpreter.h create mode 100644 dbms/src/Flash/Coprocessor/TiCIScan.cpp create mode 100644 dbms/src/Flash/Coprocessor/TiCIScan.h create mode 100644 dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.cpp create mode 100644 dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.h create mode 100644 dbms/src/Operators/TantivyReaderSourceOp.cpp create mode 100644 dbms/src/Operators/TantivyReaderSourceOp.h create mode 100644 dbms/src/Storages/StorageTantivy.cpp create mode 100644 dbms/src/Storages/StorageTantivy.h create mode 100644 dbms/src/Storages/Tantivy/TantivyInputStream.h mode change 120000 => 100644 rust-toolchain.toml diff --git a/.gitmodules b/.gitmodules index 5f5aa5e778e..54744c7c28b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -155,3 +155,6 @@ [submodule "contrib/highfive"] path = contrib/highfive url = https://github.com/BlueBrain/HighFive +[submodule "contrib/tici"] + path = contrib/tici + url = git@github.com:pingcap-inc/tici.git diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index b9d21c95f6f..f9ab57e3c80 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -207,3 +207,5 @@ if (ENABLE_TESTS AND NOT CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG") add_subdirectory(hdf5-cmake) add_subdirectory(highfive-cmake) endif () + +add_subdirectory(tici-search-lib) \ No newline at end of file diff --git a/contrib/tici b/contrib/tici new file mode 160000 index 00000000000..73ab58467f9 --- /dev/null +++ b/contrib/tici @@ -0,0 +1 @@ +Subproject commit 73ab58467f97240423c6b390204b49ac02c9aa41 diff --git a/contrib/tici-search-lib/CMakeLists.txt b/contrib/tici-search-lib/CMakeLists.txt new file mode 100644 index 00000000000..6406964c644 --- /dev/null +++ b/contrib/tici-search-lib/CMakeLists.txt @@ -0,0 +1,30 @@ +set(TICI_PROJECT_DIR "${CMAKE_SOURCE_DIR}/contrib/tici/components/searchlib") +set(TICI_LIB_NAME "tici_search_lib") +set(TICI_LIB "${CMAKE_CURRENT_BINARY_DIR}/release/${CMAKE_STATIC_LIBRARY_PREFIX}${TICI_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}") + +file(GLOB LIB_SOURCE_FILES "${TICI_PROJECT_DIR}/src/*") +file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/cxxbridge) + +add_custom_command( + OUTPUT ${TICI_LIB} + COMMAND cargo build -p tici-search-lib --release --target-dir ${CMAKE_CURRENT_BINARY_DIR} --manifest-path ${TICI_PROJECT_DIR}/Cargo.toml + WORKING_DIRECTORY ${TICI_PROJECT_DIR} + DEPENDS ${LIB_SOURCE_FILES} + COMMENT "Build Rust lib"${CMAKE_CURRENT_BINARY_DIR} +) + +add_custom_target(rustbuild ALL DEPENDS ${TICI_LIB}) +add_library(tici_search_lib_static STATIC IMPORTED GLOBAL) +set_target_properties(tici_search_lib_static PROPERTIES + IMPORTED_LOCATION ${TICI_LIB} +) + +add_dependencies(tici_search_lib_static rustbuild) +target_include_directories(tici_search_lib_static INTERFACE + ${CMAKE_CURRENT_BINARY_DIR}/cxxbridge) + +add_library(tici_search_lib SHARED "${TiFlash_SOURCE_DIR}/libs/libclara-cmake/dummy.cpp") +target_link_libraries(tici_search_lib PRIVATE "$") +target_include_directories(tici_search_lib INTERFACE + ${CMAKE_CURRENT_BINARY_DIR}/cxxbridge) + diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index dec2aa38795..3353ad7d30c 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -224,6 +224,7 @@ target_link_libraries (dbms absl::synchronization tiflash_contrib::aws_s3 tiflash_vector_search + tici_search_lib etcdpb tiflash_parsers @@ -373,6 +374,10 @@ if (ENABLE_TESTS) DESTINATION ".") endif () + install (TARGETS tici_search_lib + COMPONENT tiflash-gtest + DESTINATION ".") + target_compile_options(gtests_dbms PRIVATE -Wno-unknown-pragmas -Wno-deprecated-copy) add_check(gtests_dbms) diff --git a/dbms/src/Common/TiFlashMetrics.h b/dbms/src/Common/TiFlashMetrics.h index 6aa58cf56dd..3e78fab1398 100644 --- a/dbms/src/Common/TiFlashMetrics.h +++ b/dbms/src/Common/TiFlashMetrics.h @@ -98,7 +98,8 @@ static_assert(RAFT_REGION_BIG_WRITE_THRES * 4 < RAFT_REGION_BIG_WRITE_MAX, "Inva F(type_partition_ts, {"type", "partition_table_scan"}), \ F(type_window, {"type", "window"}), \ F(type_window_sort, {"type", "window_sort"}), \ - F(type_expand, {"type", "expand"})) \ + F(type_expand, {"type", "expand"}), \ + F(type_tici, {"type", "tici_scan"})) \ M(tiflash_memory_exceed_quota_count, "Total number of cases where memory exceeds quota", Counter) \ M(tiflash_coprocessor_request_duration_seconds, \ "Bucketed histogram of request duration", \ diff --git a/dbms/src/Debug/dbgQueryExecutor.cpp b/dbms/src/Debug/dbgQueryExecutor.cpp index f3906a56a33..5ccebe0f8bc 100644 --- a/dbms/src/Debug/dbgQueryExecutor.cpp +++ b/dbms/src/Debug/dbgQueryExecutor.cpp @@ -399,8 +399,17 @@ tipb::SelectResponse executeDAGRequest( region_id, RegionInfo(region_id, region_version, region_conf_version, std::move(key_ranges), nullptr)); - DAGContext - dag_context(dag_request, std::move(tables_regions_info), NullspaceID, "", DAGRequestKind::Cop, "", 0, "", log); + DAGContext dag_context( + dag_request, + std::move(tables_regions_info), + QueryShardInfos(), + NullspaceID, + "", + DAGRequestKind::Cop, + "", + 0, + "", + log); context.setDAGContext(&dag_context); DAGDriver driver(context, start_ts, DEFAULT_UNSPECIFIED_SCHEMA_VERSION, &dag_response, true); @@ -434,8 +443,17 @@ bool runAndCompareDagReq( region_id, RegionInfo(region_id, region->version(), region->confVer(), std::move(key_ranges), nullptr)); - DAGContext - dag_context(dag_request, std::move(tables_regions_info), NullspaceID, "", DAGRequestKind::Cop, "", 0, "", log); + DAGContext dag_context( + dag_request, + std::move(tables_regions_info), + QueryShardInfos(), + NullspaceID, + "", + DAGRequestKind::Cop, + "", + 0, + "", + log); context.setDAGContext(&dag_context); DAGDriver driver(context, properties.start_ts, DEFAULT_UNSPECIFIED_SCHEMA_VERSION, &dag_response, true); diff --git a/dbms/src/Flash/BatchCoprocessorHandler.cpp b/dbms/src/Flash/BatchCoprocessorHandler.cpp index 7a432732815..eeed4f93c64 100644 --- a/dbms/src/Flash/BatchCoprocessorHandler.cpp +++ b/dbms/src/Flash/BatchCoprocessorHandler.cpp @@ -79,6 +79,7 @@ grpc::Status BatchCoprocessorHandler::execute() DAGContext dag_context( dag_request, std::move(tables_regions_info), + QueryShardInfos::create(cop_request->table_shard_infos()), RequestUtils::deriveKeyspaceID(cop_request->context()), cop_context.db_context.getClientInfo().current_address.toString(), DAGRequestKind::BatchCop, diff --git a/dbms/src/Flash/Coprocessor/DAGContext.cpp b/dbms/src/Flash/Coprocessor/DAGContext.cpp index 746406ce6b9..05f7d89db08 100644 --- a/dbms/src/Flash/Coprocessor/DAGContext.cpp +++ b/dbms/src/Flash/Coprocessor/DAGContext.cpp @@ -49,6 +49,7 @@ bool strictSqlMode(UInt64 sql_mode) DAGContext::DAGContext( tipb::DAGRequest & dag_request_, TablesRegionsInfo && tables_regions_info_, + QueryShardInfos && query_shard_infos_, KeyspaceID keyspace_id_, const String & tidb_host_, DAGRequestKind kind_, @@ -65,6 +66,7 @@ DAGContext::DAGContext( , kind(kind_) , is_root_mpp_task(false) , tables_regions_info(std::move(tables_regions_info_)) + , query_shard_infos(std::move(query_shard_infos_)) , log(std::move(log_)) , operator_spill_contexts(std::make_shared()) , flags(dag_request->flags()) diff --git a/dbms/src/Flash/Coprocessor/DAGContext.h b/dbms/src/Flash/Coprocessor/DAGContext.h index f790b1a49f6..31782106d89 100644 --- a/dbms/src/Flash/Coprocessor/DAGContext.h +++ b/dbms/src/Flash/Coprocessor/DAGContext.h @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -155,6 +156,7 @@ class DAGContext DAGContext( tipb::DAGRequest & dag_request_, TablesRegionsInfo && tables_regions_info_, + QueryShardInfos && query_shard_infos_, KeyspaceID keyspace_id_, const String & tidb_host_, DAGRequestKind cop_kind_, @@ -380,6 +382,7 @@ class DAGContext TablesRegionsInfo tables_regions_info; // part of regions_for_local_read + regions_for_remote_read, only used for batch-cop RegionInfoList retry_regions; + QueryShardInfos query_shard_infos; LoggerPtr log; diff --git a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp index d7df2262cfb..dd28ae86143 100644 --- a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp +++ b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp @@ -91,6 +91,19 @@ NamesAndTypes genNamesAndTypes(const TiDBTableScan & table_scan, const StringRef return genNamesAndTypes(table_scan.getColumns(), column_prefix); } +NamesAndTypes genNamesAndTypesForTiCI(const TiDB::ColumnInfos & column_infos, const StringRef & column_prefix) +{ + NamesAndTypes names_and_types; + names_and_types.reserve(column_infos.size()); + for (const auto & column_info : column_infos) + { + names_and_types.emplace_back( + fmt::format("{}_{}", column_prefix, column_info.id), + getDataTypeByColumnInfoForComputingLayer(column_info)); + } + return names_and_types; +} + std::tuple genColumnDefinesForDisaggregatedRead(const TiDBTableScan & table_scan) { auto column_defines = std::make_shared(); diff --git a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.h b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.h index 4c579662912..d9bd51c095a 100644 --- a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.h +++ b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.h @@ -30,6 +30,7 @@ String genNameForExchangeReceiver(Int32 col_index); NamesAndTypes genNamesAndTypes(const TiDBTableScan & table_scan, const StringRef & column_prefix); NamesAndTypes genNamesAndTypes(const TiDB::ColumnInfos & column_infos, const StringRef & column_prefix); +NamesAndTypes genNamesAndTypesForTiCI(const TiDB::ColumnInfos & column_infos, const StringRef & column_prefix); ColumnsWithTypeAndName getColumnWithTypeAndName(const NamesAndTypes & names_and_types); NamesAndTypes toNamesAndTypes(const DAGSchema & dag_schema); diff --git a/dbms/src/Flash/Coprocessor/ShardInfo.h b/dbms/src/Flash/Coprocessor/ShardInfo.h new file mode 100644 index 00000000000..f75ea71517a --- /dev/null +++ b/dbms/src/Flash/Coprocessor/ShardInfo.h @@ -0,0 +1,142 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#include +#include +#pragma GCC diagnostic pop + +namespace DB +{ + +struct ShardInfo +{ + UInt64 shard_id; + UInt64 shard_epoch; + using KeyRanges = std::vector>; + KeyRanges key_ranges; + + explicit ShardInfo(const coprocessor::ShardInfo & info) + : shard_id(info.shard_id()) + , shard_epoch(info.shard_epoch()) + , key_ranges(genCopKeyRange(info.ranges())) + {} + + String toString() const + { + std::ostringstream sb; + sb << "ShardID: " << shard_id << ", ShardEpoch: " << shard_epoch << ", KeyRanges: "; + for (const auto & range : key_ranges) + { + sb << "[" << range.first->toString() << ", " << range.second->toString() << ") "; + } + return sb.str(); + } +}; + +using ShardInfoMap = std::unordered_map; +using ShardInfoList = std::vector; + +class TableShardInfos +{ +public: + TableShardInfos() = default; + + static TableShardInfos create(const coprocessor::TableShardInfos & table_shard_infos) + { + TableShardInfos infos; + infos.executor_id = table_shard_infos.executor_id(); + for (const auto & info : table_shard_infos.shard_infos()) + { + ShardInfo shard_info(info); + infos.shard_info_list.push_back(shard_info); + } + return infos; + } + + String toString() const + { + std::ostringstream sb; + sb << "TableShardInfos: ExecutorID: " << executor_id << ", ShardInfos: ["; + for (const auto & shard_info : shard_info_list) + { + sb << shard_info.toString() << ", "; + } + sb << "]"; + return sb.str(); + } + + String executor_id; + ShardInfoList shard_info_list; +}; + +using TableShardInfoMap = std::unordered_map; +using TableShardInfoList = std::vector; + +class QueryShardInfos +{ +public: + QueryShardInfos() = default; + + static QueryShardInfos create(const google::protobuf::RepeatedPtrField & shard_infos) + { + QueryShardInfos query_shard_infos; + for (const auto & shard_info : shard_infos) + { + auto table_shard_info = TableShardInfos::create(shard_info); + query_shard_infos.table_shard_info_list.push_back(table_shard_info); + } + return query_shard_infos; + } + + TableShardInfos getTableShardInfosByExecutorID(String executor_id) const + { + for (const auto & table_shard_info : table_shard_info_list) + { + if (table_shard_info.executor_id == executor_id) + return table_shard_info; + } + throw Exception("No TableShardInfo found for executor ID: " + executor_id, ErrorCodes::LOGICAL_ERROR); + } + + + String toString() const + { + std::ostringstream sb; + sb << "QueryShardInfos: ["; + for (const auto & shard_info : table_shard_info_list) + { + sb << shard_info.toString() << ", "; + } + sb << "]"; + return sb.str(); + } + + size_t size() const { return table_shard_info_list.size(); } + + TableShardInfoList table_shard_info_list; +}; + +} // namespace DB diff --git a/dbms/src/Flash/Coprocessor/StorageTantivyInterpreter.h b/dbms/src/Flash/Coprocessor/StorageTantivyInterpreter.h new file mode 100644 index 00000000000..aef274fb259 --- /dev/null +++ b/dbms/src/Flash/Coprocessor/StorageTantivyInterpreter.h @@ -0,0 +1,58 @@ +// Copyright 2025 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +class StorageTantivyIterpreter +{ +public: + StorageTantivyIterpreter(Context & context_, const TiCIScan & tici_scan_, size_t max_streams_) + : context(context_) + , storage(std::make_unique(context_, tici_scan_)) + , max_streams(max_streams_) + , log(Logger::get(context.getDAGContext()->log ? context.getDAGContext()->log->identifier() : "")) + , tici_scan(tici_scan_) + {} + + void execute(PipelineExecutorContext & exec_context, PipelineExecGroupBuilder & group_builder) + { + storage->read(exec_context, group_builder, Names(), SelectQueryInfo(), context, 0, max_streams); + } + + // Members will be transferred to DAGQueryBlockInterpreter after execute + std::unique_ptr analyzer; + +private: + Context & context; + std::unique_ptr storage; + size_t max_streams; + + LoggerPtr log; + const TiCIScan tici_scan; +}; +} // namespace DB diff --git a/dbms/src/Flash/Coprocessor/TiCIScan.cpp b/dbms/src/Flash/Coprocessor/TiCIScan.cpp new file mode 100644 index 00000000000..958fd90129e --- /dev/null +++ b/dbms/src/Flash/Coprocessor/TiCIScan.cpp @@ -0,0 +1,38 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include + +#include +namespace DB +{ +TiCIScan::TiCIScan(const tipb::Executor * tici_scan_, const String & executor_id_, const DAGContext & dag_context) + : tici_scan(tici_scan_) + , executor_id(executor_id_) + , table_id(tici_scan->idx_scan().table_id()) + , index_id(tici_scan->idx_scan().index_id()) + , return_columns(TiDB::toTiDBColumnInfos(tici_scan->idx_scan().columns())) + , query_columns(TiDB::toTiDBColumnInfos(tici_scan->idx_scan().fts_query_info().columns())) + , query_type(tici_scan->idx_scan().fts_query_info().query_type()) + , shard_infos(dag_context.query_shard_infos.getTableShardInfosByExecutorID(tici_scan_->executor_id())) + , query_json_str(tici_scan->idx_scan().fts_query_info().query_text()) + , limit(tici_scan->idx_scan().fts_query_info().top_k()) +{ + RUNTIME_ASSERT(tici_scan->idx_scan().fts_query_info().query_func() == tipb::ScalarFuncSig::FTSMatchWord); +} +} // namespace DB diff --git a/dbms/src/Flash/Coprocessor/TiCIScan.h b/dbms/src/Flash/Coprocessor/TiCIScan.h new file mode 100644 index 00000000000..d63761525fc --- /dev/null +++ b/dbms/src/Flash/Coprocessor/TiCIScan.h @@ -0,0 +1,50 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +namespace DB +{ +class DAGContext; + +class TiCIScan +{ +public: + TiCIScan(const tipb::Executor * tici_scan_, const String & executor_id_, const DAGContext & dag_context); + explicit TiCIScan(const tipb::Executor * tici_scan_); + + const TiDB::ColumnInfos & getQueryColumns() const { return query_columns; } + const TiDB::ColumnInfos & getReturnColumns() const { return return_columns; } + const TableShardInfos & getShardInfos() const { return shard_infos; } + const int & getTableId() const { return table_id; } + const int & getIndexId() const { return index_id; } + const std::string & getQuery() const { return query_json_str; } + const int & getLimit() const { return limit; } + +private: + const tipb::Executor * tici_scan; + [[maybe_unused]] String executor_id; + const int table_id; + const int index_id; + TiDB::ColumnInfos return_columns; + TiDB::ColumnInfos query_columns; + [[maybe_unused]] tipb::FTSQueryType query_type; + const TableShardInfos shard_infos; + std::string query_json_str; + const int limit; +}; +} // namespace DB diff --git a/dbms/src/Flash/Coprocessor/collectOutputFieldTypes.cpp b/dbms/src/Flash/Coprocessor/collectOutputFieldTypes.cpp index 6c9ec8d1ed0..f655733d84f 100644 --- a/dbms/src/Flash/Coprocessor/collectOutputFieldTypes.cpp +++ b/dbms/src/Flash/Coprocessor/collectOutputFieldTypes.cpp @@ -17,6 +17,7 @@ #include #include #include +#include namespace DB { @@ -103,6 +104,22 @@ bool collectForTableScan(std::vector & output_field_types, cons return false; } +bool collectForIndexScan(std::vector & output_field_types, const tipb::IndexScan & tici_scan) +{ + for (const auto & ci : tici_scan.columns()) + { + tipb::FieldType field_type; + field_type.set_tp(ci.tp()); + field_type.set_flag(ci.flag()); + field_type.set_flen(ci.columnlen()); + field_type.set_decimal(ci.decimal()); + for (const auto & elem : ci.elems()) + field_type.add_elems(elem); + output_field_types.push_back(field_type); + } + return false; +} + bool collectForExpand2(std::vector & output_field_types, const tipb::Expand2 & expand2) { // just collect from the level one. @@ -256,6 +273,8 @@ bool collectForExecutor(std::vector & output_field_types, const return collectForExpand(output_field_types, executor); case tipb::ExecType::TypeExpand2: return collectForExpand2(output_field_types, executor.expand2()); + case tipb::ExecType::TypeIndexScan: + return collectForIndexScan(output_field_types, executor.idx_scan()); default: return true; } diff --git a/dbms/src/Flash/CoprocessorHandler.cpp b/dbms/src/Flash/CoprocessorHandler.cpp index 77a11ceab0e..c92a3ddd5e4 100644 --- a/dbms/src/Flash/CoprocessorHandler.cpp +++ b/dbms/src/Flash/CoprocessorHandler.cpp @@ -151,6 +151,7 @@ grpc::Status CoprocessorHandler::execute() DAGContext dag_context( dag_request, std::move(tables_regions_info), + QueryShardInfos::create(cop_request->table_shard_infos()), RequestUtils::deriveKeyspaceID(cop_request->context()), cop_context.db_context.getClientInfo().current_address.toString(), kind, diff --git a/dbms/src/Flash/Planner/PhysicalPlan.cpp b/dbms/src/Flash/Planner/PhysicalPlan.cpp index 72f50aad69c..2c0759978a7 100644 --- a/dbms/src/Flash/Planner/PhysicalPlan.cpp +++ b/dbms/src/Flash/Planner/PhysicalPlan.cpp @@ -12,10 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include #include +#include #include #include #include @@ -33,12 +35,14 @@ #include #include #include +#include #include #include #include #include #include #include +#include namespace DB { @@ -89,6 +93,14 @@ void PhysicalPlan::buildTableScan(const String & executor_id, const tipb::Execut dagContext().table_scan_executor_id = executor_id; } +void PhysicalPlan::buildTiCIScan(const String & executor_id, const tipb::Executor * executor) +{ + RUNTIME_ASSERT(executor->idx_scan().has_fts_query_info()); + TiCIScan tici_scan(executor, executor_id, dagContext()); + pushBack(PhysicalTiCIScan::build(executor_id, log, tici_scan)); + dagContext().table_scan_executor_id = executor_id; +} + void PhysicalPlan::build(const tipb::Executor * executor) { RUNTIME_CHECK(executor); @@ -198,6 +210,9 @@ void PhysicalPlan::build(const tipb::Executor * executor) GET_METRIC(tiflash_coprocessor_executor_count, type_partition_ts).Increment(); buildTableScan(executor_id, executor); break; + case tipb::ExecType::TypeIndexScan: + buildTiCIScan(executor_id, executor); + break; case tipb::ExecType::TypeJoin: { GET_METRIC(tiflash_coprocessor_executor_count, type_join).Increment(); diff --git a/dbms/src/Flash/Planner/PhysicalPlan.h b/dbms/src/Flash/Planner/PhysicalPlan.h index 8c54295a750..a9ae47c5fb5 100644 --- a/dbms/src/Flash/Planner/PhysicalPlan.h +++ b/dbms/src/Flash/Planner/PhysicalPlan.h @@ -61,6 +61,7 @@ class PhysicalPlan DAGContext & dagContext() const; void buildTableScan(const String & executor_id, const tipb::Executor * executor); + void buildTiCIScan(const String & executor_id, const tipb::Executor * executor); private: std::vector cur_plan_nodes{}; diff --git a/dbms/src/Flash/Planner/PlanType.h b/dbms/src/Flash/Planner/PlanType.h index 37422ab8fb0..a99c93992f3 100644 --- a/dbms/src/Flash/Planner/PlanType.h +++ b/dbms/src/Flash/Planner/PlanType.h @@ -42,6 +42,7 @@ struct PlanType JoinBuild = 17, JoinProbe = 18, GetResult = 19, + TiCiScan = 20, }; PlanTypeEnum enum_value; diff --git a/dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.cpp b/dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.cpp new file mode 100644 index 00000000000..eff4c847efe --- /dev/null +++ b/dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.cpp @@ -0,0 +1,143 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace +{ +NamesWithAliases buildTableScanProjectionCols( + Int64 logical_table_id, + const NamesAndTypes & schema, + const Block & storage_header) +{ + if (unlikely(schema.size() != storage_header.columns())) + throw TiFlashException( + fmt::format( + "The tidb table scan schema size {} is different from the tiflash storage schema size {}, table id is " + "{}", + schema.size(), + storage_header.columns(), + logical_table_id), + Errors::Planner::BadRequest); + NamesWithAliases schema_project_cols; + for (size_t i = 0; i < schema.size(); ++i) + { + const auto & table_scan_col_name = schema[i].name; + const auto & table_scan_col_type = schema[i].type; + const auto & storage_col_name = storage_header.getColumnsWithTypeAndName()[i].name; + const auto & storage_col_type = storage_header.getColumnsWithTypeAndName()[i].type; + if (unlikely(!table_scan_col_type->equals(*storage_col_type))) + throw TiFlashException( + fmt::format( + R"(The data type {} from tidb table scan schema is different from the data type {} from tiflash storage schema, + table id is {}, + column index is {}, + column name from tidb table scan is {}, + column name from tiflash storage is {})", + table_scan_col_type->getName(), + storage_col_type->getName(), + logical_table_id, + i, + table_scan_col_name, + storage_col_name), + Errors::Planner::BadRequest); + schema_project_cols.emplace_back(storage_col_name, table_scan_col_name); + } + return schema_project_cols; +} +} // namespace + +PhysicalTiCIScan::PhysicalTiCIScan( + const String & executor_id_, + const NamesAndTypes & schema_, + const String & req_id, + const TiCIScan & tici_scan_, + const Block & sample_block_) + : PhysicalLeaf(executor_id_, PlanType::TableScan, schema_, FineGrainedShuffle{}, req_id) + , tici_scan(tici_scan_) + , sample_block(sample_block_) +{} + +PhysicalPlanNodePtr PhysicalTiCIScan::build( + const String & executor_id, + const LoggerPtr & log, + const TiCIScan & tici_scan) +{ + auto schema = genNamesAndTypes(tici_scan.getReturnColumns(), "tici_scan"); + auto physical_tici_scan + = std::make_shared(executor_id, schema, log->identifier(), tici_scan, Block(schema)); + return physical_tici_scan; +} + +void PhysicalTiCIScan::buildPipeline( + PipelineBuilder & builder, + Context & context, + PipelineExecutorContext & exec_context) +{ + // For building PipelineExec in compile time. + StorageTantivyIterpreter storage_interpreter(context, tici_scan, context.getMaxStreams()); + storage_interpreter.execute(exec_context, pipeline_exec_builder); + buildProjection(exec_context, pipeline_exec_builder); + + PhysicalPlanNode::buildPipeline(builder, context, exec_context); +} + +void PhysicalTiCIScan::finalizeImpl(const Names & parent_require) +{ + FinalizeHelper::checkSchemaContainsParentRequire(schema, parent_require); +} + +const Block & PhysicalTiCIScan::getSampleBlock() const +{ + return sample_block; +} + +void PhysicalTiCIScan::buildPipelineExecGroupImpl( + PipelineExecutorContext & /*exec_status*/, + PipelineExecGroupBuilder & group_builder, + Context & /*context*/, + size_t /*concurrency*/) +{ + assert(group_builder.empty()); + group_builder = std::move(pipeline_exec_builder); +} + +void PhysicalTiCIScan::buildProjection(PipelineExecutorContext & exec_context, PipelineExecGroupBuilder & group_builder) +{ + auto header = group_builder.getCurrentHeader(); + const auto & schema_project_cols = buildTableScanProjectionCols(tici_scan.getTableId(), schema, header); + + /// In order to keep TransformOp's schema consistent with PhysicalPlan's schema. + /// It is worth noting that the column uses the name as the unique identifier in the Block, so the column name must also be consistent. + ExpressionActionsPtr schema_actions = PhysicalPlanHelper::newActions(header); + schema_actions->add(ExpressionAction::project(schema_project_cols)); + executeExpression(exec_context, group_builder, schema_actions, log); +} +} // namespace DB diff --git a/dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.h b/dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.h new file mode 100644 index 00000000000..90ec286cee4 --- /dev/null +++ b/dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.h @@ -0,0 +1,64 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include + +namespace DB +{ +class PhysicalTiCIScan : public PhysicalLeaf +{ +public: + static PhysicalPlanNodePtr build(const String & executor_id, const LoggerPtr & log, const TiCIScan & tici_scan); + + PhysicalTiCIScan( + const String & executor_id_, + const NamesAndTypes & schema_, + const String & req_id, + const TiCIScan & tici_scan_, + const Block & sample_block_); + + void finalizeImpl(const Names & parent_require) override; + + const Block & getSampleBlock() const override; + + bool setFilterConditions(const String & filter_executor_id, const tipb::Selection & selection); + + bool hasFilterConditions() const; + + const String & getFilterConditionsId() const; + + void buildPipeline(PipelineBuilder & builder, Context & context, PipelineExecutorContext & exec_context) override; + +private: + void buildPipelineExecGroupImpl( + PipelineExecutorContext & /*exec_status*/, + PipelineExecGroupBuilder & group_builder, + Context & /*context*/, + size_t /*concurrency*/) override; + void buildProjection(PipelineExecutorContext & exec_context, PipelineExecGroupBuilder & group_builder); + FilterConditions filter_conditions; + + TiCIScan tici_scan; + + Block sample_block; + + PipelineExecGroupBuilder pipeline_exec_builder; +}; +} // namespace DB diff --git a/dbms/src/Flash/Statistics/ExecutorStatisticsCollector.cpp b/dbms/src/Flash/Statistics/ExecutorStatisticsCollector.cpp index f2fc9e4fde0..bb11d9badb8 100644 --- a/dbms/src/Flash/Statistics/ExecutorStatisticsCollector.cpp +++ b/dbms/src/Flash/Statistics/ExecutorStatisticsCollector.cpp @@ -96,11 +96,7 @@ void ExecutorStatisticsCollector::initialize(DAGContext * dag_context_) TopNStatistics, WindowStatistics, ExpandStatistics>(&executor)) - { - throw TiFlashException( - fmt::format("Unknown executor type, executor_id: {}", executor.executor_id()), - Errors::Coprocessor::Internal); - } + {} return true; }); @@ -118,7 +114,7 @@ void ExecutorStatisticsCollector::fillChildren() assert(child.has_executor_id()); children.push_back(child.executor_id()); }); - profiles[executor.executor_id()]->setChildren(children); + //profiles[executor.executor_id()]->setChildren(children); return true; }); } @@ -233,6 +229,7 @@ void ExecutorStatisticsCollector::fillLocalExecutionSummaries(tipb::SelectRespon } } } + return; RUNTIME_CHECK_MSG(target_executor_summary, "cannot find executor summary to put ru consumption"); RUNTIME_CHECK_MSG( local_ru->SerializeToString(target_executor_summary->mutable_ru_consumption()), diff --git a/dbms/src/Flash/Statistics/traverseExecutors.cpp b/dbms/src/Flash/Statistics/traverseExecutors.cpp index c2d217df255..55f749877b1 100644 --- a/dbms/src/Flash/Statistics/traverseExecutors.cpp +++ b/dbms/src/Flash/Statistics/traverseExecutors.cpp @@ -14,6 +14,7 @@ #include #include +#include namespace DB { @@ -23,11 +24,10 @@ Children getChildren(const tipb::Executor & executor) { case tipb::ExecType::TypeTableScan: case tipb::ExecType::TypePartitionTableScan: + case tipb::ExecType::TypeIndexScan: return {}; case tipb::ExecType::TypeJoin: return {&executor.join().children(0), &executor.join().children(1)}; - case tipb::ExecType::TypeIndexScan: - throw TiFlashException("IndexScan is not supported", Errors::Coprocessor::Unimplemented); case tipb::ExecType::TypeSelection: return Children{&executor.selection().child()}; case tipb::ExecType::TypeAggregation: @@ -57,4 +57,4 @@ Children getChildren(const tipb::Executor & executor) throw TiFlashException("Should not reach here", Errors::Coprocessor::Internal); } } -} // namespace DB \ No newline at end of file +} // namespace DB diff --git a/dbms/src/Operators/TantivyReaderSourceOp.cpp b/dbms/src/Operators/TantivyReaderSourceOp.cpp new file mode 100644 index 00000000000..0796c295e9e --- /dev/null +++ b/dbms/src/Operators/TantivyReaderSourceOp.cpp @@ -0,0 +1,94 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +TantivyReaderSourceOp::TantivyReaderSourceOp( + PipelineExecutorContext & exec_context_, + const String & req_id, + const Int64 & table_id, + const Int64 & index_id, + const TableShardInfos & query_shard_infos, + const NamesAndTypes & query_columns, + const NamesAndTypes & return_columns, + const String & query_json_str, + const UInt64 & limit) + : SourceOp(exec_context_, req_id) +{ + setHeader(Block(return_columns)); + + input = std::make_shared( + log, + table_id, + index_id, + query_shard_infos, + query_columns, + return_columns, + query_json_str, + limit); +} + +String TantivyReaderSourceOp::getName() const +{ + return "TantivyReaderSourceOp"; +} + +void TantivyReaderSourceOp::operatePrefixImpl() +{ + LOG_DEBUG(log, "start reading from TantivyReaderSourceOp"); +} + +void TantivyReaderSourceOp::operateSuffixImpl() +{ + LOG_DEBUG(log, "finish read {} rows from TantivyReaderSourceOp", total_rows); +} + +Block TantivyReaderSourceOp::popFromBlockQueue() +{ + assert(!block_queue.empty()); + Block block = std::move(block_queue.front()); + block_queue.pop(); + return block; +} + +OperatorStatus TantivyReaderSourceOp::readImpl(Block & block) +{ + if (!block_queue.empty()) + { + block = popFromBlockQueue(); + return OperatorStatus::HAS_OUTPUT; + } + + assert(block_queue.empty()); + for (;;) + { + auto tmp = input->readImpl(); + block_queue.push(tmp); + total_rows += tmp.rows(); + if (!tmp) + break; + } + block = popFromBlockQueue(); + return OperatorStatus::HAS_OUTPUT; +} + +} // namespace DB diff --git a/dbms/src/Operators/TantivyReaderSourceOp.h b/dbms/src/Operators/TantivyReaderSourceOp.h new file mode 100644 index 00000000000..8287e24f372 --- /dev/null +++ b/dbms/src/Operators/TantivyReaderSourceOp.h @@ -0,0 +1,63 @@ +// Copyright 2023 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +class TantivyReaderSourceOp : public SourceOp +{ +public: + TantivyReaderSourceOp( + PipelineExecutorContext & exec_context_, + const String & req_id, + const Int64 & table_id, + const Int64 & index_id, + const TableShardInfos & query_shard_infos, + const NamesAndTypes & query_columns, + const NamesAndTypes & return_columns, + const String & query_json_str, + const UInt64 & limit); + + String getName() const override; + +protected: + void operatePrefixImpl() override; + void operateSuffixImpl() override; + + OperatorStatus readImpl(Block & block) override; + + IOProfileInfoPtr getIOProfileInfo() const override { return io_profile_info; } + +private: + Block popFromBlockQueue(); + +private: + std::queue block_queue; + + UInt64 total_rows{}; + IOProfileInfoPtr io_profile_info; + + std::shared_ptr input; +}; +} // namespace DB diff --git a/dbms/src/Server/CMakeLists.txt b/dbms/src/Server/CMakeLists.txt index 74fa754a468..887f2d75079 100644 --- a/dbms/src/Server/CMakeLists.txt +++ b/dbms/src/Server/CMakeLists.txt @@ -128,6 +128,10 @@ if (OS_LINUX) DESTINATION ".") endif() + install (TARGETS tici_search_lib + COMPONENT tiflash-release + DESTINATION ".") + if (NOT (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG" OR SAN_DEBUG)) install (SCRIPT ${TiFlash_SOURCE_DIR}/cmake/tiflash_linux_post_install.cmake COMPONENT tiflash-release) endif() diff --git a/dbms/src/Storages/StorageTantivy.cpp b/dbms/src/Storages/StorageTantivy.cpp new file mode 100644 index 00000000000..48d71399a27 --- /dev/null +++ b/dbms/src/Storages/StorageTantivy.cpp @@ -0,0 +1,84 @@ +// Copyright 2025 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace DB +{ + +StorageTantivy::StorageTantivy(Context & context_, const TiCIScan & tici_scan_) + : IStorage() + , tici_scan(tici_scan_) + , context(context_) + , log(Logger::get(context_.getDAGContext()->log ? context_.getDAGContext()->log->identifier() : "")) +{} + +BlockInputStreams StorageTantivy::read( + const Names &, + const SelectQueryInfo &, + const Context &, + QueryProcessingStage::Enum &, + size_t, + unsigned int) +{ + return {}; +} + +void StorageTantivy::read( + PipelineExecutorContext & exec_status, + PipelineExecGroupBuilder & group_builder, + [[maybe_unused]] const Names & column_names, + [[maybe_unused]] const SelectQueryInfo & info, + [[maybe_unused]] const Context & context, + [[maybe_unused]] size_t max_block_size, + [[maybe_unused]] unsigned num_streams) +{ + auto query_columns = genNamesAndTypesForTiCI(tici_scan.getQueryColumns(), "column"); + auto return_columns = genNamesAndTypesForTiCI(tici_scan.getReturnColumns(), "column"); + + group_builder.addConcurrency(std::make_unique( + exec_status, + log->identifier(), + tici_scan.getTableId(), + tici_scan.getIndexId(), + tici_scan.getShardInfos(), + query_columns, + return_columns, + tici_scan.getQuery(), + tici_scan.getLimit())); +} + +} // namespace DB diff --git a/dbms/src/Storages/StorageTantivy.h b/dbms/src/Storages/StorageTantivy.h new file mode 100644 index 00000000000..b487549012a --- /dev/null +++ b/dbms/src/Storages/StorageTantivy.h @@ -0,0 +1,78 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#include +#include +#pragma GCC diagnostic pop + +#include +#include +namespace DB +{ +class StorageTantivy : public IStorage +{ +public: + StorageTantivy(Context & context_, const TiCIScan & tici_scan_); + + std::string getName() const override { return "StorageTantivy"; } + + std::string getTableName() const override { return "StorageTantivy"; } + + BlockInputStreams read( + const Names & column_names, + const SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum & processed_stage, + size_t max_block_size, + unsigned num_streams) override; + + + void read( + PipelineExecutorContext & exec_status, + PipelineExecGroupBuilder & group_builder, + [[maybe_unused]] const Names & column_names, + [[maybe_unused]] const SelectQueryInfo & info, + [[maybe_unused]] const Context & context, + [[maybe_unused]] size_t max_block_size, + [[maybe_unused]] unsigned num_streams) override; + // Members will be transferred to DAGQueryBlockInterpreter after execute + std::unique_ptr analyzer; + +private: + const TiCIScan tici_scan; + [[maybe_unused]] Context & context; + LoggerPtr log; +}; +} // namespace DB diff --git a/dbms/src/Storages/Tantivy/TantivyInputStream.h b/dbms/src/Storages/Tantivy/TantivyInputStream.h new file mode 100644 index 00000000000..c267177a7f5 --- /dev/null +++ b/dbms/src/Storages/Tantivy/TantivyInputStream.h @@ -0,0 +1,158 @@ +// Copyright 2025 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB::TS +{ +class TantivyInputStream : public IProfilingBlockInputStream +{ + static constexpr auto NAME = "TantivyInputStream"; + +public: + TantivyInputStream( + LoggerPtr log_, + Int64 table_id_, + Int64 index_id_, + TableShardInfos query_shard_infos_, + NamesAndTypes query_columns_, + NamesAndTypes return_columns_, + String query_json_str_, + UInt64 limit_) + : log(log_) + , table_id(table_id_) + , index_id(index_id_) + , query_shard_infos(query_shard_infos_) + , query_columns(query_columns_) + , return_columns(return_columns_) + , query_json_str(query_json_str_) + , limit(limit_) + {} + + String getName() const override { return NAME; } + + Block getHeader() const override { return header; } + + Block readImpl() override + { + if (done) + { + return {}; + } + + Block ret = readFromS3(processed_shard); + processed_shard++; + done = processed_shard >= query_shard_infos.shard_info_list.size(); + return ret; + } + +protected: + Block readFromS3(size_t processing) + { + auto query_fields = getFields(query_columns); + auto return_fields = getFields(return_columns); + auto & shard_info = query_shard_infos.shard_info_list[processing]; + LOG_INFO(log, "Processing shard: {}, shard info: {}", processing, shard_info.toString()); + auto shard_id = shard_info.shard_id; + auto key_ranges = getKeyRanges(shard_info.key_ranges); + + auto search_param = SearchParam{static_cast(limit)}; + rust::Vec documents = search( + table_id, + index_id, + shard_id, + key_ranges.first, + key_ranges.second, + query_fields, + return_fields, + query_json_str, + search_param); + + Block res(return_columns); + int i = 0; + for (auto & name_and_type : return_columns) + { + auto col = res.getByName(name_and_type.name).column->assumeMutable(); + if (removeNullable(name_and_type.type)->isStringOrFixedString()) + { + for (auto & doc : documents) + { + col->insert(Field(String(doc.fieldValues[i].string_value.c_str()))); + } + } + if (removeNullable(name_and_type.type)->isInteger()) + { + for (auto & doc : documents) + { + col->insert(Field(doc.fieldValues[i].int_value)); + } + } + i++; + } + return res; + } + +private: + Block header; + bool done = false; + LoggerPtr log; + Int64 table_id; + Int64 index_id; + TableShardInfos query_shard_infos; + NamesAndTypes query_columns; + NamesAndTypes return_columns; + String query_json_str; + UInt64 limit; + + size_t processed_shard = 0; + + rust::Vec getFields(NamesAndTypes & columns) + { + rust::Vec fields; + for (auto & name_and_type : columns) + { + LOG_INFO(log, "name: {}, type: {}", name_and_type.name, name_and_type.type->getName()); + fields.push_back(name_and_type.name); + } + return fields; + } + + static std::pair, rust::Vec> getKeyRanges( + const std::vector> & key_ranges) + { + rust::Vec start_key_ranges; + rust::Vec end_key_ranges; + for (const auto & range : key_ranges) + { + start_key_ranges.push_back(range.first->toString()); + end_key_ranges.push_back(range.second->toString()); + } + return std::make_pair(std::move(start_key_ranges), std::move(end_key_ranges)); + } +}; + +} // namespace DB::TS diff --git a/dbms/src/Storages/tests/gtest_filter_parser.cpp b/dbms/src/Storages/tests/gtest_filter_parser.cpp index 2e6b35c1a9a..85548ba9aac 100644 --- a/dbms/src/Storages/tests/gtest_filter_parser.cpp +++ b/dbms/src/Storages/tests/gtest_filter_parser.cpp @@ -84,7 +84,7 @@ DM::RSOperatorPtr FilterParserTest::generateRsOperator( [&](const String &, const String &) { return table_info; }, getDAGProperties("")); auto & dag_request = *query_tasks[0].dag_request; - DAGContext dag_context(dag_request, {}, NullspaceID, "", DAGRequestKind::Cop, "", 0, "", log); + DAGContext dag_context(dag_request, {}, QueryShardInfos(), NullspaceID, "", DAGRequestKind::Cop, "", 0, "", log); ctx->setDAGContext(&dag_context); // Don't care about regions information in this test google::protobuf::RepeatedPtrField conditions; diff --git a/dbms/src/Storages/tests/gtests_parse_push_down_filter.cpp b/dbms/src/Storages/tests/gtests_parse_push_down_filter.cpp index 3655be161ab..7fe6c346f61 100644 --- a/dbms/src/Storages/tests/gtests_parse_push_down_filter.cpp +++ b/dbms/src/Storages/tests/gtests_parse_push_down_filter.cpp @@ -76,7 +76,7 @@ DM::PushDownFilterPtr generatePushDownFilter( getDAGProperties("")); auto & dag_request = *query_tasks[0].dag_request; auto log = Logger::get(); - DAGContext dag_context(dag_request, {}, NullspaceID, "", DAGRequestKind::Cop, "", 0, "", log); + DAGContext dag_context(dag_request, {}, QueryShardInfos(), NullspaceID, "", DAGRequestKind::Cop, "", 0, "", log); ctx.setDAGContext(&dag_context); // Don't care about regions information in this test google::protobuf::RepeatedPtrField empty_condition; diff --git a/dbms/src/TestUtils/MPPTaskTestUtils.cpp b/dbms/src/TestUtils/MPPTaskTestUtils.cpp index d718a4d37cb..e2103b7f26e 100644 --- a/dbms/src/TestUtils/MPPTaskTestUtils.cpp +++ b/dbms/src/TestUtils/MPPTaskTestUtils.cpp @@ -166,7 +166,7 @@ ColumnsWithTypeAndName MPPTaskTestUtils::executeCoprocessorTask(std::shared_ptr< auto * data = req->mutable_data(); dag_request->AppendToString(data); - DAGContext dag_context(*dag_request, {}, NullspaceID, "", DAGRequestKind::Cop, "", 0, "", Logger::get()); + DAGContext dag_context(*dag_request, {}, {}, NullspaceID, "", DAGRequestKind::Cop, "", 0, "", Logger::get()); TiFlashTestEnv::getGlobalContext(test_meta.context_idx).setDAGContext(&dag_context); TiFlashTestEnv::getGlobalContext(test_meta.context_idx).setCopTest(); diff --git a/rust-toolchain.toml b/rust-toolchain.toml deleted file mode 120000 index 82e224aad22..00000000000 --- a/rust-toolchain.toml +++ /dev/null @@ -1 +0,0 @@ -contrib/tiflash-proxy/rust-toolchain.toml \ No newline at end of file diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 00000000000..4e8563be36b --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "1.85.0" +components = ["rustfmt", "clippy", "rust-src", "rust-analyzer"] +profile = "minimal" From c84e208b80efa9f0506ac0d139bcda37829c0cd8 Mon Sep 17 00:00:00 2001 From: Shenghui Wu <793703860@qq.com> Date: Tue, 3 Jun 2025 16:53:11 +0800 Subject: [PATCH 02/58] chore: update kvproto,tici submodule and insert Null to missing field (#10216) (cherry picked from commit d670d429323b1fbc9378422230c978ff893b59f5) --- contrib/tici | 2 +- dbms/src/Server/Server.cpp | 3 ++ .../src/Storages/Tantivy/TantivyInputStream.h | 29 ++++++++++++++++--- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/contrib/tici b/contrib/tici index 73ab58467f9..cec1fcd7ad2 160000 --- a/contrib/tici +++ b/contrib/tici @@ -1 +1 @@ -Subproject commit 73ab58467f97240423c6b390204b49ac02c9aa41 +Subproject commit cec1fcd7ad2c3f369f1e432f7cf0bc63fcf3a44e diff --git a/dbms/src/Server/Server.cpp b/dbms/src/Server/Server.cpp index 01cdb742d0a..d2aca6a01da 100644 --- a/dbms/src/Server/Server.cpp +++ b/dbms/src/Server/Server.cpp @@ -101,6 +101,7 @@ #include #include #include +#include #include #include @@ -1024,6 +1025,7 @@ try auto schema_cache_size = config().getInt("schema_cache_size", 10000); global_context->initializeSharedBlockSchemas(schema_cache_size); + start_reader_server("reader.toml"); // Load remaining databases loadMetadata(*global_context); LOG_DEBUG(log, "Load metadata done."); @@ -1098,6 +1100,7 @@ try wn_ps->waitUntilInitedFromRemoteStore(); } + { TCPServersHolder tcp_http_servers_holder( *this, diff --git a/dbms/src/Storages/Tantivy/TantivyInputStream.h b/dbms/src/Storages/Tantivy/TantivyInputStream.h index c267177a7f5..160be3d1302 100644 --- a/dbms/src/Storages/Tantivy/TantivyInputStream.h +++ b/dbms/src/Storages/Tantivy/TantivyInputStream.h @@ -93,25 +93,46 @@ class TantivyInputStream : public IProfilingBlockInputStream search_param); Block res(return_columns); - int i = 0; + if (documents.empty()) + { + return res; + } for (auto & name_and_type : return_columns) { + int idx = -1; + for (size_t j = 0; j < documents[0].fieldValues.size(); j++) + { + if (documents[0].fieldValues[j].field_name == name_and_type.name) + { + idx = j; + break; + } + } + if (idx == -1) + { + for (size_t j = 0; j < documents.size(); j++) + { + // Insert default value for missing fields + res.getByName(name_and_type.name).column->assumeMutable()->insertDefault(); + } + continue; + } + auto col = res.getByName(name_and_type.name).column->assumeMutable(); if (removeNullable(name_and_type.type)->isStringOrFixedString()) { for (auto & doc : documents) { - col->insert(Field(String(doc.fieldValues[i].string_value.c_str()))); + col->insert(Field(String(doc.fieldValues[idx].string_value.c_str()))); } } if (removeNullable(name_and_type.type)->isInteger()) { for (auto & doc : documents) { - col->insert(Field(doc.fieldValues[i].int_value)); + col->insert(Field(doc.fieldValues[idx].int_value)); } } - i++; } return res; } From 3a403b36c38664bf69b78607a9421f2a09f96e54 Mon Sep 17 00:00:00 2001 From: Yu Jin Date: Fri, 6 Jun 2025 17:24:28 +0800 Subject: [PATCH 03/58] make tici-search-lib compile compatible with mac (#10229) (cherry picked from commit 87be1f9f82f7af6e5aae9641966f9d7e088373b1) --- contrib/tici-search-lib/CMakeLists.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/contrib/tici-search-lib/CMakeLists.txt b/contrib/tici-search-lib/CMakeLists.txt index 6406964c644..b783f0ff6f8 100644 --- a/contrib/tici-search-lib/CMakeLists.txt +++ b/contrib/tici-search-lib/CMakeLists.txt @@ -25,6 +25,14 @@ target_include_directories(tici_search_lib_static INTERFACE add_library(tici_search_lib SHARED "${TiFlash_SOURCE_DIR}/libs/libclara-cmake/dummy.cpp") target_link_libraries(tici_search_lib PRIVATE "$") +if(APPLE) + target_link_libraries(tici_search_lib PRIVATE + "-framework Security" + "-framework CoreFoundation" + "-framework IOKit" + ) +endif() + target_include_directories(tici_search_lib INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/cxxbridge) From 6d4180ec80de88dce9d995238cef040296da5f25 Mon Sep 17 00:00:00 2001 From: Shenghui Wu <793703860@qq.com> Date: Tue, 24 Jun 2025 11:39:45 +0800 Subject: [PATCH 04/58] tici: Using u8 slice instead of String for KeyRanges (#10250) (cherry picked from commit 9cd39c3c20b354be13e6b0622c0a3e04af561713) --- contrib/tici-search-lib/CMakeLists.txt | 2 +- dbms/src/Flash/Coprocessor/ShardInfo.h | 6 ++--- .../src/Storages/Tantivy/TantivyInputStream.h | 23 +++++++++++-------- rust-toolchain.toml | 2 +- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/contrib/tici-search-lib/CMakeLists.txt b/contrib/tici-search-lib/CMakeLists.txt index b783f0ff6f8..74e8a6ce40c 100644 --- a/contrib/tici-search-lib/CMakeLists.txt +++ b/contrib/tici-search-lib/CMakeLists.txt @@ -1,4 +1,4 @@ -set(TICI_PROJECT_DIR "${CMAKE_SOURCE_DIR}/contrib/tici/components/searchlib") +set(TICI_PROJECT_DIR "${CMAKE_SOURCE_DIR}/contrib/tici") set(TICI_LIB_NAME "tici_search_lib") set(TICI_LIB "${CMAKE_CURRENT_BINARY_DIR}/release/${CMAKE_STATIC_LIBRARY_PREFIX}${TICI_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}") diff --git a/dbms/src/Flash/Coprocessor/ShardInfo.h b/dbms/src/Flash/Coprocessor/ShardInfo.h index f75ea71517a..648a5d988b0 100644 --- a/dbms/src/Flash/Coprocessor/ShardInfo.h +++ b/dbms/src/Flash/Coprocessor/ShardInfo.h @@ -35,13 +35,13 @@ struct ShardInfo { UInt64 shard_id; UInt64 shard_epoch; - using KeyRanges = std::vector>; + using KeyRanges = google::protobuf::RepeatedPtrField; KeyRanges key_ranges; explicit ShardInfo(const coprocessor::ShardInfo & info) : shard_id(info.shard_id()) , shard_epoch(info.shard_epoch()) - , key_ranges(genCopKeyRange(info.ranges())) + , key_ranges(info.ranges()) {} String toString() const @@ -50,7 +50,7 @@ struct ShardInfo sb << "ShardID: " << shard_id << ", ShardEpoch: " << shard_epoch << ", KeyRanges: "; for (const auto & range : key_ranges) { - sb << "[" << range.first->toString() << ", " << range.second->toString() << ") "; + sb << "[" << range.start() << ", " << range.end() << ") "; } return sb.str(); } diff --git a/dbms/src/Storages/Tantivy/TantivyInputStream.h b/dbms/src/Storages/Tantivy/TantivyInputStream.h index 160be3d1302..678d2f2e27f 100644 --- a/dbms/src/Storages/Tantivy/TantivyInputStream.h +++ b/dbms/src/Storages/Tantivy/TantivyInputStream.h @@ -85,8 +85,7 @@ class TantivyInputStream : public IProfilingBlockInputStream table_id, index_id, shard_id, - key_ranges.first, - key_ranges.second, + key_ranges, query_fields, return_fields, query_json_str, @@ -162,17 +161,23 @@ class TantivyInputStream : public IProfilingBlockInputStream return fields; } - static std::pair, rust::Vec> getKeyRanges( - const std::vector> & key_ranges) + static rust::Vec<::Range> getKeyRanges(ShardInfo::KeyRanges & key_ranges) { - rust::Vec start_key_ranges; - rust::Vec end_key_ranges; + rust::Vec<::Range> res; for (const auto & range : key_ranges) { - start_key_ranges.push_back(range.first->toString()); - end_key_ranges.push_back(range.second->toString()); + rust::Slice<::std::uint8_t const> start( + reinterpret_cast(range.start().c_str()), + range.start().size()); + rust::Slice<::std::uint8_t const> end( + reinterpret_cast(range.end().c_str()), + range.end().size()); + res.push_back({ + .start = std::move(start), + .end = std::move(end), + }); } - return std::make_pair(std::move(start_key_ranges), std::move(end_key_ranges)); + return res; } }; diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 4e8563be36b..14ba487c8e7 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] -channel = "1.85.0" +channel = "nightly-2025-02-28" components = ["rustfmt", "clippy", "rust-src", "rust-analyzer"] profile = "minimal" From 40fbc3bf8c058c68f12cd8ed39c7c3b2f0c09098 Mon Sep 17 00:00:00 2001 From: jinhelin Date: Fri, 27 Jun 2025 13:18:00 +0800 Subject: [PATCH 05/58] tici: Pass the read_ts parameter to searchlib. (#10263) (cherry picked from commit 06dcac5324c30c703907740c379c8321c6080722) --- dbms/src/Operators/TantivyReaderSourceOp.cpp | 6 ++++-- dbms/src/Operators/TantivyReaderSourceOp.h | 3 ++- dbms/src/Storages/StorageTantivy.cpp | 6 ++++-- dbms/src/Storages/Tantivy/TantivyInputStream.h | 8 ++++++-- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/dbms/src/Operators/TantivyReaderSourceOp.cpp b/dbms/src/Operators/TantivyReaderSourceOp.cpp index 0796c295e9e..295d3cb29dd 100644 --- a/dbms/src/Operators/TantivyReaderSourceOp.cpp +++ b/dbms/src/Operators/TantivyReaderSourceOp.cpp @@ -31,7 +31,8 @@ TantivyReaderSourceOp::TantivyReaderSourceOp( const NamesAndTypes & query_columns, const NamesAndTypes & return_columns, const String & query_json_str, - const UInt64 & limit) + const UInt64 & limit, + const UInt64 & read_ts) : SourceOp(exec_context_, req_id) { setHeader(Block(return_columns)); @@ -44,7 +45,8 @@ TantivyReaderSourceOp::TantivyReaderSourceOp( query_columns, return_columns, query_json_str, - limit); + limit, + read_ts); } String TantivyReaderSourceOp::getName() const diff --git a/dbms/src/Operators/TantivyReaderSourceOp.h b/dbms/src/Operators/TantivyReaderSourceOp.h index 8287e24f372..9de0f53d12a 100644 --- a/dbms/src/Operators/TantivyReaderSourceOp.h +++ b/dbms/src/Operators/TantivyReaderSourceOp.h @@ -37,7 +37,8 @@ class TantivyReaderSourceOp : public SourceOp const NamesAndTypes & query_columns, const NamesAndTypes & return_columns, const String & query_json_str, - const UInt64 & limit); + const UInt64 & limit, + const UInt64 & read_ts); String getName() const override; diff --git a/dbms/src/Storages/StorageTantivy.cpp b/dbms/src/Storages/StorageTantivy.cpp index 48d71399a27..acb19851139 100644 --- a/dbms/src/Storages/StorageTantivy.cpp +++ b/dbms/src/Storages/StorageTantivy.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -61,7 +62,7 @@ void StorageTantivy::read( PipelineExecutorContext & exec_status, PipelineExecGroupBuilder & group_builder, [[maybe_unused]] const Names & column_names, - [[maybe_unused]] const SelectQueryInfo & info, + const SelectQueryInfo & info, [[maybe_unused]] const Context & context, [[maybe_unused]] size_t max_block_size, [[maybe_unused]] unsigned num_streams) @@ -78,7 +79,8 @@ void StorageTantivy::read( query_columns, return_columns, tici_scan.getQuery(), - tici_scan.getLimit())); + tici_scan.getLimit(), + info.mvcc_query_info->start_ts)); } } // namespace DB diff --git a/dbms/src/Storages/Tantivy/TantivyInputStream.h b/dbms/src/Storages/Tantivy/TantivyInputStream.h index 678d2f2e27f..4aab407ab3a 100644 --- a/dbms/src/Storages/Tantivy/TantivyInputStream.h +++ b/dbms/src/Storages/Tantivy/TantivyInputStream.h @@ -42,7 +42,8 @@ class TantivyInputStream : public IProfilingBlockInputStream NamesAndTypes query_columns_, NamesAndTypes return_columns_, String query_json_str_, - UInt64 limit_) + UInt64 limit_, + UInt64 read_ts_) : log(log_) , table_id(table_id_) , index_id(index_id_) @@ -51,6 +52,7 @@ class TantivyInputStream : public IProfilingBlockInputStream , return_columns(return_columns_) , query_json_str(query_json_str_) , limit(limit_) + , read_ts(read_ts_) {} String getName() const override { return NAME; } @@ -89,7 +91,8 @@ class TantivyInputStream : public IProfilingBlockInputStream query_fields, return_fields, query_json_str, - search_param); + search_param, + read_ts); Block res(return_columns); if (documents.empty()) @@ -147,6 +150,7 @@ class TantivyInputStream : public IProfilingBlockInputStream NamesAndTypes return_columns; String query_json_str; UInt64 limit; + UInt64 read_ts; size_t processed_shard = 0; From cad3d061f9d427d5239f9cd3bfbd7dc64a38b2ce Mon Sep 17 00:00:00 2001 From: jinhelin Date: Fri, 27 Jun 2025 17:40:39 +0800 Subject: [PATCH 06/58] tici: Fix mvcc_query_info may be null in StorageTantivy (#10269) (cherry picked from commit 4e92c4a82cd758dcff2096c6baf8fb7bc9373d47) --- dbms/src/Storages/StorageTantivy.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/StorageTantivy.cpp b/dbms/src/Storages/StorageTantivy.cpp index acb19851139..c45265b637c 100644 --- a/dbms/src/Storages/StorageTantivy.cpp +++ b/dbms/src/Storages/StorageTantivy.cpp @@ -62,7 +62,7 @@ void StorageTantivy::read( PipelineExecutorContext & exec_status, PipelineExecGroupBuilder & group_builder, [[maybe_unused]] const Names & column_names, - const SelectQueryInfo & info, + [[maybe_unused]] const SelectQueryInfo & info, [[maybe_unused]] const Context & context, [[maybe_unused]] size_t max_block_size, [[maybe_unused]] unsigned num_streams) @@ -80,7 +80,7 @@ void StorageTantivy::read( return_columns, tici_scan.getQuery(), tici_scan.getLimit(), - info.mvcc_query_info->start_ts)); + /*read_ts*/ 0)); // TODO: currently, info.mvcc_query_info is null. } } // namespace DB From 715261d15129a513d3b7d88a3ebaed4ac51adfed Mon Sep 17 00:00:00 2001 From: jinhelin Date: Wed, 9 Jul 2025 12:06:18 +0800 Subject: [PATCH 07/58] contrib: update tici (#10279) (cherry picked from commit 389704232df72e689ae47f624f07df5c7b0107aa) --- dbms/src/Server/Server.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/dbms/src/Server/Server.cpp b/dbms/src/Server/Server.cpp index d2aca6a01da..676307843ac 100644 --- a/dbms/src/Server/Server.cpp +++ b/dbms/src/Server/Server.cpp @@ -1025,7 +1025,16 @@ try auto schema_cache_size = config().getInt("schema_cache_size", 10000); global_context->initializeSharedBlockSchemas(schema_cache_size); - start_reader_server("reader.toml"); + if (config().getBool("tici.enable", false)) + { + auto config_path = config().getString("tici.config_path", "reader.toml"); + start_reader_server(config_path); + } + else + { + LOG_INFO(log, "tici is disabled"); + } + // Load remaining databases loadMetadata(*global_context); LOG_DEBUG(log, "Load metadata done."); From fdbb0f82734cc2d33b488b11a224338698a2d0bb Mon Sep 17 00:00:00 2001 From: Shenghui Wu <793703860@qq.com> Date: Thu, 24 Jul 2025 15:20:52 +0800 Subject: [PATCH 08/58] cop: support remote read for tici query (#10297) (cherry picked from commit c102ffb0b37fa716423ed08bc61cd063b17f37c1) --- dbms/src/Flash/Coprocessor/DAGContext.h | 1 + dbms/src/Flash/Coprocessor/DAGDriver.cpp | 12 ++ dbms/src/Flash/Coprocessor/RemoteRequest.cpp | 75 +++++++ dbms/src/Flash/Coprocessor/RemoteRequest.h | 12 ++ dbms/src/Flash/Coprocessor/ShardInfo.h | 2 + .../Coprocessor/StorageTantivyInterpreter.cpp | 186 ++++++++++++++++++ .../Coprocessor/StorageTantivyInterpreter.h | 21 +- dbms/src/Flash/Coprocessor/TiCIScan.cpp | 6 + dbms/src/Flash/Coprocessor/TiCIScan.h | 3 + dbms/src/Operators/TantivyReaderSourceOp.cpp | 2 +- dbms/src/Operators/TantivyReaderSourceOp.h | 2 +- dbms/src/Storages/StorageTantivy.cpp | 41 +++- dbms/src/Storages/StorageTantivy.h | 7 + .../src/Storages/Tantivy/TantivyInputStream.h | 24 ++- 14 files changed, 379 insertions(+), 15 deletions(-) create mode 100644 dbms/src/Flash/Coprocessor/StorageTantivyInterpreter.cpp diff --git a/dbms/src/Flash/Coprocessor/DAGContext.h b/dbms/src/Flash/Coprocessor/DAGContext.h index 31782106d89..fb76f220e90 100644 --- a/dbms/src/Flash/Coprocessor/DAGContext.h +++ b/dbms/src/Flash/Coprocessor/DAGContext.h @@ -383,6 +383,7 @@ class DAGContext // part of regions_for_local_read + regions_for_remote_read, only used for batch-cop RegionInfoList retry_regions; QueryShardInfos query_shard_infos; + ShardInfoList retry_shards; LoggerPtr log; diff --git a/dbms/src/Flash/Coprocessor/DAGDriver.cpp b/dbms/src/Flash/Coprocessor/DAGDriver.cpp index daa760cc3c1..61514bb422c 100644 --- a/dbms/src/Flash/Coprocessor/DAGDriver.cpp +++ b/dbms/src/Flash/Coprocessor/DAGDriver.cpp @@ -221,6 +221,18 @@ try batch_cop_writer->Write(response); } + if (!dag_context.retry_shards.empty()) + { + coprocessor::BatchResponse response; + for (const auto & shard : dag_context.retry_shards) + { + auto * retry_shard = response.add_retry_shards(); + retry_shard->set_shard_id(shard.shard_id); + retry_shard->set_shard_epoch(shard.shard_epoch); + } + batch_cop_writer->Write(response); + } + auto streaming_writer = std::make_shared(batch_cop_writer); TiDB::TiDBCollators collators; auto response_writer = std::make_unique>( diff --git a/dbms/src/Flash/Coprocessor/RemoteRequest.cpp b/dbms/src/Flash/Coprocessor/RemoteRequest.cpp index 637203a728c..3840fadb7c3 100644 --- a/dbms/src/Flash/Coprocessor/RemoteRequest.cpp +++ b/dbms/src/Flash/Coprocessor/RemoteRequest.cpp @@ -16,7 +16,10 @@ #include #include #include +#include #include +#include +#include #include #include @@ -103,6 +106,52 @@ RemoteRequest RemoteRequest::build( return {std::move(dag_req), std::move(schema), std::move(key_ranges), connection_id, connection_alias}; } +RemoteRequest RemoteRequest::build( + const ShardInfoList & shard_infos, + DAGContext & dag_context, + const TiCIScan & tici_scan, + UInt64 connection_id, + const String & connection_alias, + const LoggerPtr & log) +{ + LOG_INFO(log, "{}", printShards(shard_infos, tici_scan.getTableId(), tici_scan.getIndexId())); + DAGSchema schema; + tipb::DAGRequest dag_req; + { + tipb::Executor * ts_exec = dag_req.mutable_root_executor(); + ts_exec->set_tp(tipb::ExecType::TypeIndexScan); + ts_exec->set_executor_id(tici_scan.getTiCIScan()->executor_id()); + auto * mutable_tici_scan = ts_exec->mutable_idx_scan(); + tici_scan.constructTiCIScanForRemoteRead(mutable_tici_scan); + + const auto & return_columns = tici_scan.getReturnColumns(); + auto names_and_types = genNamesAndTypesForTiCI(return_columns, "column"); + for (size_t i = 0; i < return_columns.size(); ++i) + { + const auto & col = return_columns[i]; + schema.emplace_back(std::make_pair(names_and_types[i].name, col)); + dag_req.add_output_offsets(i); + } + dag_req.set_encode_type(tipb::EncodeType::TypeCHBlock); + dag_req.set_force_encode_type(true); + } + /// do not collect execution summaries because in this case because the execution summaries + /// will be collected by CoprocessorBlockInputStream. + /// Otherwise rows in execution summary of table scan will be double. + dag_req.set_collect_execution_summaries(false); + dag_req.set_flags(dag_context.getFlags()); + dag_req.set_sql_mode(dag_context.getSQLMode()); + dag_req.set_div_precision_increment(dag_context.getDivPrecisionIncrement()); + const auto & original_dag_req = *dag_context.dag_request; + if (original_dag_req.has_time_zone_name() && !original_dag_req.time_zone_name().empty()) + dag_req.set_time_zone_name(original_dag_req.time_zone_name()); + if (original_dag_req.has_time_zone_offset()) + dag_req.set_time_zone_offset(original_dag_req.time_zone_offset()); + + std::vector key_ranges = buildKeyRanges(shard_infos); + return {std::move(dag_req), std::move(schema), std::move(key_ranges), connection_id, connection_alias}; +} + std::vector RemoteRequest::buildKeyRanges(const RegionRetryList & retry_regions) { std::vector key_ranges; @@ -115,6 +164,18 @@ std::vector RemoteRequest::buildKeyRanges(const return key_ranges; } +std::vector RemoteRequest::buildKeyRanges(const ShardInfoList & retry_shards) +{ + std::vector key_ranges; + for (const auto & shard : retry_shards) + { + for (const auto & range : shard.key_ranges) + key_ranges.emplace_back(range.start(), range.end()); + } + sort(key_ranges.begin(), key_ranges.end()); + return key_ranges; +} + std::string RemoteRequest::printRetryRegions(const RegionRetryList & retry_regions, TableID table_id) { FmtBuffer buffer; @@ -128,4 +189,18 @@ std::string RemoteRequest::printRetryRegions(const RegionRetryList & retry_regio return buffer.toString(); } +std::string RemoteRequest::printShards(const ShardInfoList & shards, Int64 table_id, Int64 index_id) +{ + FmtBuffer buffer; + buffer.fmtAppend("Start to build remote request for {} shards (", shards.size()); + buffer.joinStr( + shards.cbegin(), + shards.cend(), + [](const auto & shard, FmtBuffer & fb) { fb.fmtAppend("{}", shard.getID()); }, + ","); + buffer.fmtAppend(") for table {} and index {}", table_id, index_id); + return buffer.toString(); +} + + } // namespace DB diff --git a/dbms/src/Flash/Coprocessor/RemoteRequest.h b/dbms/src/Flash/Coprocessor/RemoteRequest.h index b107d7890a0..13639eebd01 100644 --- a/dbms/src/Flash/Coprocessor/RemoteRequest.h +++ b/dbms/src/Flash/Coprocessor/RemoteRequest.h @@ -16,6 +16,8 @@ #include #include +#include +#include #include #include @@ -59,8 +61,18 @@ struct RemoteRequest UInt64 connection_id, const String & connection_alias, const LoggerPtr & log); + + static RemoteRequest build( + const ShardInfoList & shard_infos, + DAGContext & dag_context, + const TiCIScan & tici_scan, + UInt64 connection_id, + const String & connection_alias, + const LoggerPtr & log); static std::vector buildKeyRanges(const RegionRetryList & retry_regions); + static std::vector buildKeyRanges(const ShardInfoList & retry_shards); static std::string printRetryRegions(const RegionRetryList & retry_regions, TableID table_id); + static std::string printShards(const ShardInfoList & shards, Int64 table_id, Int64 index_id); tipb::DAGRequest dag_request; DAGSchema schema; diff --git a/dbms/src/Flash/Coprocessor/ShardInfo.h b/dbms/src/Flash/Coprocessor/ShardInfo.h index 648a5d988b0..3e1576cf37c 100644 --- a/dbms/src/Flash/Coprocessor/ShardInfo.h +++ b/dbms/src/Flash/Coprocessor/ShardInfo.h @@ -54,6 +54,8 @@ struct ShardInfo } return sb.str(); } + + UInt64 getID() const { return shard_id; } }; using ShardInfoMap = std::unordered_map; diff --git a/dbms/src/Flash/Coprocessor/StorageTantivyInterpreter.cpp b/dbms/src/Flash/Coprocessor/StorageTantivyInterpreter.cpp new file mode 100644 index 00000000000..a740f7502be --- /dev/null +++ b/dbms/src/Flash/Coprocessor/StorageTantivyInterpreter.cpp @@ -0,0 +1,186 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +void StorageTantivyIterpreter::execute(PipelineExecutorContext & exec_context, PipelineExecGroupBuilder & group_builder) +{ + storage->splitRemoteReadAndLocalRead(); + // local_read + storage->read(exec_context, group_builder, Names(), SelectQueryInfo(), context, 0, max_streams); + // remote_read + auto remote_shard_infos = storage->getRemoteShardInfos(); + if (!remote_shard_infos.empty()) + { + if (context.getDAGContext()->isCop()) + throw RegionException({}, RegionException::RegionReadStatus::NOT_FOUND, "shard not found"); + context.getDAGContext()->retry_shards.insert( + context.getDAGContext()->retry_shards.end(), + remote_shard_infos.begin(), + remote_shard_infos.end()); + } + + auto remote_request = buildRemoteRequests(remote_shard_infos); + if (!remote_request.empty()) + { + buildRemoteExec(exec_context, group_builder, remote_request); + } +} + +std::vector StorageTantivyIterpreter::buildRemoteRequests(ShardInfoList & remote_shard_infos) +{ + std::vector remote_requests; + if (remote_shard_infos.empty()) + { + return remote_requests; + } + remote_requests.push_back(RemoteRequest::build( + remote_shard_infos, + *context.getDAGContext(), + tici_scan, + context.getDAGContext()->getConnectionID(), + context.getDAGContext()->getConnectionAlias(), + log)); + return remote_requests; +} + +void StorageTantivyIterpreter::buildRemoteExec( + PipelineExecutorContext & exec_context, + PipelineExecGroupBuilder & group_builder, + const std::vector & remote_requests) +{ + auto coprocessor_reader = buildCoprocessorReader(remote_requests); + size_t concurrent_num = coprocessor_reader->enableCopStream() ? context.getSettingsRef().max_threads.get() + : coprocessor_reader->getConcurrency(); + + for (size_t i = 0; i < concurrent_num; ++i) + group_builder.addConcurrency( + std::make_unique(exec_context, log->identifier(), coprocessor_reader)); + + LOG_DEBUG(log, "remote sourceOps built"); +} + +CoprocessorReaderPtr StorageTantivyIterpreter::buildCoprocessorReader( + const std::vector & remote_requests) +{ + std::vector all_tasks = buildCopTasks(remote_requests); + if (all_tasks.empty()) + { + throw TiFlashException( + "No coprocessor tasks built for remote read, please check remote request", + Errors::Coprocessor::BadRequest); + } + const DAGSchema & schema = remote_requests[0].schema; + pingcap::kv::Cluster * cluster = tmt.getKVCluster(); + bool has_enforce_encode_type = true; + pingcap::kv::LabelFilter tiflash_label_filter = pingcap::kv::labelFilterNoTiFlashWriteNode; + + size_t concurrent_num = std::min(context.getSettingsRef().max_threads, all_tasks.size()); + size_t queue_size = context.getSettingsRef().remote_read_queue_size > 0 + ? context.getSettingsRef().remote_read_queue_size.get() + : concurrent_num * 4; + bool enable_cop_stream = context.getSettingsRef().enable_cop_stream_for_remote_read; + UInt64 cop_timeout = context.getSettingsRef().cop_timeout_for_remote_read; + String store_zone_label; + auto kv_store = tmt.getKVStore(); + if likely (kv_store) + { + for (int i = 0; i < kv_store->getStoreMeta().labels_size(); ++i) + { + if (kv_store->getStoreMeta().labels().at(i).key() == "zone") + { + store_zone_label = kv_store->getStoreMeta().labels().at(i).value(); + break; + } + } + } + auto coprocessor_reader = std::make_shared( + schema, + cluster, + std::move(all_tasks), + has_enforce_encode_type, + concurrent_num, + enable_cop_stream, + queue_size, + cop_timeout, + tiflash_label_filter, + log->identifier(), + store_zone_label); + context.getDAGContext()->addCoprocessorReader(coprocessor_reader); + + return coprocessor_reader; +} + +std::vector StorageTantivyIterpreter::buildCopTasks( + const std::vector & remote_requests) +{ + std::vector all_tasks; + + pingcap::kv::Cluster * cluster = tmt.getKVCluster(); + for (const auto & remote_request : remote_requests) + { + pingcap::coprocessor::RequestPtr req = std::make_shared(); + remote_request.dag_request.SerializeToString(&(req->data)); + req->tp = pingcap::coprocessor::ReqType::DAG; + req->start_ts = context.getSettingsRef().read_tso; + req->schema_version = context.getSettingsRef().schema_version; + req->resource_group_name = (*context.getDAGContext()).getResourceGroupName(); + + pingcap::kv::Backoffer bo(pingcap::kv::copBuildTaskMaxBackoff); + pingcap::kv::StoreType store_type = pingcap::kv::StoreType::TiFlash; + std::multimap meta_data; + meta_data.emplace("is_remote_read", "true"); + + auto tasks = pingcap::coprocessor::buildCopTaskForFullText( + bo, + cluster, + remote_request.key_ranges, + req, + store_type, + (*context.getDAGContext()).getKeyspaceID(), + remote_request.connection_id, + remote_request.connection_alias, + &Poco::Logger::get("pingcap/coprocessor"), + std::move(meta_data), + [&] {}, + tici_scan.getTableId(), + tici_scan.getIndexId(), + tici_scan.getTiCIScan()->executor_id()); + all_tasks.insert(all_tasks.end(), tasks.begin(), tasks.end()); + } + return all_tasks; +} + +} // namespace DB diff --git a/dbms/src/Flash/Coprocessor/StorageTantivyInterpreter.h b/dbms/src/Flash/Coprocessor/StorageTantivyInterpreter.h index aef274fb259..0d4f47c0805 100644 --- a/dbms/src/Flash/Coprocessor/StorageTantivyInterpreter.h +++ b/dbms/src/Flash/Coprocessor/StorageTantivyInterpreter.h @@ -15,13 +15,19 @@ #pragma once #include +#include #include #include #include #include #include +#include +#include #include #include +#include +#include +#include #include #include @@ -35,14 +41,20 @@ class StorageTantivyIterpreter : context(context_) , storage(std::make_unique(context_, tici_scan_)) , max_streams(max_streams_) + , tmt(context.getTMTContext()) , log(Logger::get(context.getDAGContext()->log ? context.getDAGContext()->log->identifier() : "")) , tici_scan(tici_scan_) {} - void execute(PipelineExecutorContext & exec_context, PipelineExecGroupBuilder & group_builder) - { - storage->read(exec_context, group_builder, Names(), SelectQueryInfo(), context, 0, max_streams); - } + void execute(PipelineExecutorContext & exec_context, PipelineExecGroupBuilder & group_builder); + + std::vector buildRemoteRequests(ShardInfoList & remote_shard_infos); + void buildRemoteExec( + PipelineExecutorContext & exec_context, + PipelineExecGroupBuilder & group_builder, + const std::vector & remote_requests); + CoprocessorReaderPtr buildCoprocessorReader(const std::vector & remote_requests); + std::vector buildCopTasks(const std::vector & remote_requests); // Members will be transferred to DAGQueryBlockInterpreter after execute std::unique_ptr analyzer; @@ -52,6 +64,7 @@ class StorageTantivyIterpreter std::unique_ptr storage; size_t max_streams; + TMTContext & tmt; LoggerPtr log; const TiCIScan tici_scan; }; diff --git a/dbms/src/Flash/Coprocessor/TiCIScan.cpp b/dbms/src/Flash/Coprocessor/TiCIScan.cpp index 958fd90129e..b9e7a597d0a 100644 --- a/dbms/src/Flash/Coprocessor/TiCIScan.cpp +++ b/dbms/src/Flash/Coprocessor/TiCIScan.cpp @@ -35,4 +35,10 @@ TiCIScan::TiCIScan(const tipb::Executor * tici_scan_, const String & executor_id { RUNTIME_ASSERT(tici_scan->idx_scan().fts_query_info().query_func() == tipb::ScalarFuncSig::FTSMatchWord); } + +void TiCIScan::constructTiCIScanForRemoteRead(tipb::IndexScan * tipb_index_scan) const +{ + assert(tipb_index_scan != nullptr); + *tipb_index_scan = tici_scan->idx_scan(); +} // namespace DB } // namespace DB diff --git a/dbms/src/Flash/Coprocessor/TiCIScan.h b/dbms/src/Flash/Coprocessor/TiCIScan.h index d63761525fc..66d5afa70cf 100644 --- a/dbms/src/Flash/Coprocessor/TiCIScan.h +++ b/dbms/src/Flash/Coprocessor/TiCIScan.h @@ -34,6 +34,9 @@ class TiCIScan const int & getIndexId() const { return index_id; } const std::string & getQuery() const { return query_json_str; } const int & getLimit() const { return limit; } + const tipb::Executor * getTiCIScan() const { return tici_scan; } + + void constructTiCIScanForRemoteRead(tipb::IndexScan * tipb_index_scan) const; private: const tipb::Executor * tici_scan; diff --git a/dbms/src/Operators/TantivyReaderSourceOp.cpp b/dbms/src/Operators/TantivyReaderSourceOp.cpp index 295d3cb29dd..236f745f12c 100644 --- a/dbms/src/Operators/TantivyReaderSourceOp.cpp +++ b/dbms/src/Operators/TantivyReaderSourceOp.cpp @@ -27,7 +27,7 @@ TantivyReaderSourceOp::TantivyReaderSourceOp( const String & req_id, const Int64 & table_id, const Int64 & index_id, - const TableShardInfos & query_shard_infos, + const ShardInfoList & query_shard_infos, const NamesAndTypes & query_columns, const NamesAndTypes & return_columns, const String & query_json_str, diff --git a/dbms/src/Operators/TantivyReaderSourceOp.h b/dbms/src/Operators/TantivyReaderSourceOp.h index 9de0f53d12a..24da1cef327 100644 --- a/dbms/src/Operators/TantivyReaderSourceOp.h +++ b/dbms/src/Operators/TantivyReaderSourceOp.h @@ -33,7 +33,7 @@ class TantivyReaderSourceOp : public SourceOp const String & req_id, const Int64 & table_id, const Int64 & index_id, - const TableShardInfos & query_shard_infos, + const ShardInfoList & query_shard_infos, const NamesAndTypes & query_columns, const NamesAndTypes & return_columns, const String & query_json_str, diff --git a/dbms/src/Storages/StorageTantivy.cpp b/dbms/src/Storages/StorageTantivy.cpp index c45265b637c..118944a25bf 100644 --- a/dbms/src/Storages/StorageTantivy.cpp +++ b/dbms/src/Storages/StorageTantivy.cpp @@ -34,7 +34,10 @@ #include #include #include +#include +#include +#include #include namespace DB @@ -70,12 +73,13 @@ void StorageTantivy::read( auto query_columns = genNamesAndTypesForTiCI(tici_scan.getQueryColumns(), "column"); auto return_columns = genNamesAndTypesForTiCI(tici_scan.getReturnColumns(), "column"); + // local read group_builder.addConcurrency(std::make_unique( exec_status, log->identifier(), tici_scan.getTableId(), tici_scan.getIndexId(), - tici_scan.getShardInfos(), + local_read, query_columns, return_columns, tici_scan.getQuery(), @@ -83,4 +87,39 @@ void StorageTantivy::read( /*read_ts*/ 0)); // TODO: currently, info.mvcc_query_info is null. } +void StorageTantivy::splitRemoteReadAndLocalRead() +{ + const auto & all = tici_scan.getShardInfos().shard_info_list; + ShardInfoList local_shard_infos; + ShardInfoList remote_shard_infos; + ::rust::Vec<::Shard> shards; + auto index_id = tici_scan.getIndexId(); + auto table_id = tici_scan.getTableId(); + for (const auto & shard_info : all) + { + shards.push_back(::Shard{ + .table_id = table_id, + .index_id = index_id, + .shard_id = shard_info.shard_id, + .shard_epoch = shard_info.shard_epoch, + }); + } + + auto result = check_shards(shards); + for (size_t i = 0; i < result.size(); ++i) + { + const auto & is_local = result[i]; + if (is_local) + { + local_shard_infos.push_back(all[i]); + } + else + { + remote_shard_infos.push_back(all[i]); + } + } + remote_read = remote_shard_infos; + local_read = local_shard_infos; +} + } // namespace DB diff --git a/dbms/src/Storages/StorageTantivy.h b/dbms/src/Storages/StorageTantivy.h index b487549012a..b2be6580d49 100644 --- a/dbms/src/Storages/StorageTantivy.h +++ b/dbms/src/Storages/StorageTantivy.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -70,9 +71,15 @@ class StorageTantivy : public IStorage // Members will be transferred to DAGQueryBlockInterpreter after execute std::unique_ptr analyzer; + void splitRemoteReadAndLocalRead(); + ShardInfoList getRemoteShardInfos() const { return remote_read; } + private: const TiCIScan tici_scan; [[maybe_unused]] Context & context; LoggerPtr log; + + ShardInfoList remote_read; + ShardInfoList local_read; }; } // namespace DB diff --git a/dbms/src/Storages/Tantivy/TantivyInputStream.h b/dbms/src/Storages/Tantivy/TantivyInputStream.h index 4aab407ab3a..8baf05b7c9f 100644 --- a/dbms/src/Storages/Tantivy/TantivyInputStream.h +++ b/dbms/src/Storages/Tantivy/TantivyInputStream.h @@ -38,7 +38,7 @@ class TantivyInputStream : public IProfilingBlockInputStream LoggerPtr log_, Int64 table_id_, Int64 index_id_, - TableShardInfos query_shard_infos_, + ShardInfoList query_shard_infos_, NamesAndTypes query_columns_, NamesAndTypes return_columns_, String query_json_str_, @@ -65,10 +65,16 @@ class TantivyInputStream : public IProfilingBlockInputStream { return {}; } + if (processed_shard >= query_shard_infos.size()) + { + LOG_INFO(log, "empty shard info, returning empty block"); + done = true; + return {}; + } Block ret = readFromS3(processed_shard); processed_shard++; - done = processed_shard >= query_shard_infos.shard_info_list.size(); + done = processed_shard >= query_shard_infos.size(); return ret; } @@ -77,16 +83,18 @@ class TantivyInputStream : public IProfilingBlockInputStream { auto query_fields = getFields(query_columns); auto return_fields = getFields(return_columns); - auto & shard_info = query_shard_infos.shard_info_list[processing]; + auto & shard_info = query_shard_infos[processing]; LOG_INFO(log, "Processing shard: {}, shard info: {}", processing, shard_info.toString()); - auto shard_id = shard_info.shard_id; auto key_ranges = getKeyRanges(shard_info.key_ranges); auto search_param = SearchParam{static_cast(limit)}; rust::Vec documents = search( - table_id, - index_id, - shard_id, + { + .table_id = table_id, + .index_id = index_id, + .shard_id = shard_info.shard_id, + .shard_epoch = shard_info.shard_epoch, + }, key_ranges, query_fields, return_fields, @@ -145,7 +153,7 @@ class TantivyInputStream : public IProfilingBlockInputStream LoggerPtr log; Int64 table_id; Int64 index_id; - TableShardInfos query_shard_infos; + ShardInfoList query_shard_infos; NamesAndTypes query_columns; NamesAndTypes return_columns; String query_json_str; From e539cb546f28df66704ecc44b559a895ac872684 Mon Sep 17 00:00:00 2001 From: Shenghui Wu <793703860@qq.com> Date: Wed, 30 Jul 2025 18:02:25 +0800 Subject: [PATCH 09/58] tici: using etcd client to get tici meta service address (#10319) Signed-off-by: wshwsh12 <793703860@qq.com> (cherry picked from commit 47c7bd764e2ae9b7d3381cd13305873f463c9df0) --- dbms/src/Storages/KVStore/TMTContext.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/dbms/src/Storages/KVStore/TMTContext.cpp b/dbms/src/Storages/KVStore/TMTContext.cpp index 0adec2569fe..09523210ca2 100644 --- a/dbms/src/Storages/KVStore/TMTContext.cpp +++ b/dbms/src/Storages/KVStore/TMTContext.cpp @@ -166,6 +166,25 @@ TMTContext::TMTContext( { startMonitorMPPTaskThread(mpp_task_manager); etcd_client = Etcd::Client::create(cluster->pd_client, cluster_config); + + if (!cluster->pd_client->isMock()) + { + auto [kv, status] = etcd_client.get()->leader("/tici/metaservice/election"); + if (!status.ok()) + { + LOG_WARNING( + Logger::get(), + "Failed to get tici meta service leader from etcd, code={} msg={}", + magic_enum::enum_name(status.error_code()), + status.error_message()); + } + else + { + LOG_INFO(Logger::get(), "Get tici meta service leader from etcd: {}", kv.value()); + cluster->shard_cache = std::make_unique( + kv.value()); // Use the leader address as shard cache address + } + } } void TMTContext::initS3GCManager(const TiFlashRaftProxyHelper * proxy_helper) From 3c1630154e59ebd2bc2bf93ee759c1628e9b9c0d Mon Sep 17 00:00:00 2001 From: jinhelin Date: Fri, 1 Aug 2025 17:14:18 +0800 Subject: [PATCH 10/58] contrib: Update tici (#10320) (cherry picked from commit a5afcd7aba4a860c398ca287c73e1293837bbccb) --- dbms/src/Server/Server.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/dbms/src/Server/Server.cpp b/dbms/src/Server/Server.cpp index 676307843ac..96dc8600f63 100644 --- a/dbms/src/Server/Server.cpp +++ b/dbms/src/Server/Server.cpp @@ -1027,7 +1027,6 @@ try if (config().getBool("tici.enable", false)) { - auto config_path = config().getString("tici.config_path", "reader.toml"); start_reader_server(config_path); } else From 181e349e399dadc0bf03c14e397e9a5e679f4e06 Mon Sep 17 00:00:00 2001 From: Shenghui Wu <793703860@qq.com> Date: Thu, 7 Aug 2025 16:30:54 +0800 Subject: [PATCH 11/58] tici: support complex expr for tici searchlib (#10326) (cherry picked from commit 0857aa9fe82e48c93e716faa3ea6246b9b474a1b) --- dbms/src/Flash/Coprocessor/TiCIScan.cpp | 9 +-- dbms/src/Flash/Coprocessor/TiCIScan.h | 9 ++- dbms/src/Operators/TantivyReaderSourceOp.cpp | 17 +--- dbms/src/Operators/TantivyReaderSourceOp.h | 5 +- dbms/src/Storages/StorageTantivy.cpp | 6 +- .../src/Storages/Tantivy/TantivyInputStream.h | 81 ++++++++++++++++--- 6 files changed, 87 insertions(+), 40 deletions(-) diff --git a/dbms/src/Flash/Coprocessor/TiCIScan.cpp b/dbms/src/Flash/Coprocessor/TiCIScan.cpp index b9e7a597d0a..209cd0e5d0e 100644 --- a/dbms/src/Flash/Coprocessor/TiCIScan.cpp +++ b/dbms/src/Flash/Coprocessor/TiCIScan.cpp @@ -27,18 +27,15 @@ TiCIScan::TiCIScan(const tipb::Executor * tici_scan_, const String & executor_id , table_id(tici_scan->idx_scan().table_id()) , index_id(tici_scan->idx_scan().index_id()) , return_columns(TiDB::toTiDBColumnInfos(tici_scan->idx_scan().columns())) - , query_columns(TiDB::toTiDBColumnInfos(tici_scan->idx_scan().fts_query_info().columns())) , query_type(tici_scan->idx_scan().fts_query_info().query_type()) , shard_infos(dag_context.query_shard_infos.getTableShardInfosByExecutorID(tici_scan_->executor_id())) - , query_json_str(tici_scan->idx_scan().fts_query_info().query_text()) , limit(tici_scan->idx_scan().fts_query_info().top_k()) -{ - RUNTIME_ASSERT(tici_scan->idx_scan().fts_query_info().query_func() == tipb::ScalarFuncSig::FTSMatchWord); -} +{} void TiCIScan::constructTiCIScanForRemoteRead(tipb::IndexScan * tipb_index_scan) const { assert(tipb_index_scan != nullptr); *tipb_index_scan = tici_scan->idx_scan(); -} // namespace DB +} + } // namespace DB diff --git a/dbms/src/Flash/Coprocessor/TiCIScan.h b/dbms/src/Flash/Coprocessor/TiCIScan.h index 66d5afa70cf..1629e0e4f18 100644 --- a/dbms/src/Flash/Coprocessor/TiCIScan.h +++ b/dbms/src/Flash/Coprocessor/TiCIScan.h @@ -27,27 +27,28 @@ class TiCIScan TiCIScan(const tipb::Executor * tici_scan_, const String & executor_id_, const DAGContext & dag_context); explicit TiCIScan(const tipb::Executor * tici_scan_); - const TiDB::ColumnInfos & getQueryColumns() const { return query_columns; } const TiDB::ColumnInfos & getReturnColumns() const { return return_columns; } const TableShardInfos & getShardInfos() const { return shard_infos; } const int & getTableId() const { return table_id; } const int & getIndexId() const { return index_id; } - const std::string & getQuery() const { return query_json_str; } const int & getLimit() const { return limit; } const tipb::Executor * getTiCIScan() const { return tici_scan; } void constructTiCIScanForRemoteRead(tipb::IndexScan * tipb_index_scan) const; + const ::google::protobuf::RepeatedPtrField<::tipb::Expr> & getMatchExpr() const + { + return tici_scan->idx_scan().fts_query_info().match_expr(); + } + private: const tipb::Executor * tici_scan; [[maybe_unused]] String executor_id; const int table_id; const int index_id; TiDB::ColumnInfos return_columns; - TiDB::ColumnInfos query_columns; [[maybe_unused]] tipb::FTSQueryType query_type; const TableShardInfos shard_infos; - std::string query_json_str; const int limit; }; } // namespace DB diff --git a/dbms/src/Operators/TantivyReaderSourceOp.cpp b/dbms/src/Operators/TantivyReaderSourceOp.cpp index 236f745f12c..943e2d9ce7f 100644 --- a/dbms/src/Operators/TantivyReaderSourceOp.cpp +++ b/dbms/src/Operators/TantivyReaderSourceOp.cpp @@ -28,25 +28,16 @@ TantivyReaderSourceOp::TantivyReaderSourceOp( const Int64 & table_id, const Int64 & index_id, const ShardInfoList & query_shard_infos, - const NamesAndTypes & query_columns, const NamesAndTypes & return_columns, - const String & query_json_str, const UInt64 & limit, - const UInt64 & read_ts) + const UInt64 & read_ts, + const google::protobuf::RepeatedPtrField & expr) : SourceOp(exec_context_, req_id) { setHeader(Block(return_columns)); - input = std::make_shared( - log, - table_id, - index_id, - query_shard_infos, - query_columns, - return_columns, - query_json_str, - limit, - read_ts); + input = std::make_shared< + TS::TantivyInputStream>(log, table_id, index_id, query_shard_infos, return_columns, limit, read_ts, expr); } String TantivyReaderSourceOp::getName() const diff --git a/dbms/src/Operators/TantivyReaderSourceOp.h b/dbms/src/Operators/TantivyReaderSourceOp.h index 24da1cef327..81614beb36b 100644 --- a/dbms/src/Operators/TantivyReaderSourceOp.h +++ b/dbms/src/Operators/TantivyReaderSourceOp.h @@ -34,11 +34,10 @@ class TantivyReaderSourceOp : public SourceOp const Int64 & table_id, const Int64 & index_id, const ShardInfoList & query_shard_infos, - const NamesAndTypes & query_columns, const NamesAndTypes & return_columns, - const String & query_json_str, const UInt64 & limit, - const UInt64 & read_ts); + const UInt64 & read_ts, + const google::protobuf::RepeatedPtrField & expr); String getName() const override; diff --git a/dbms/src/Storages/StorageTantivy.cpp b/dbms/src/Storages/StorageTantivy.cpp index 118944a25bf..6430ae16a87 100644 --- a/dbms/src/Storages/StorageTantivy.cpp +++ b/dbms/src/Storages/StorageTantivy.cpp @@ -70,7 +70,6 @@ void StorageTantivy::read( [[maybe_unused]] size_t max_block_size, [[maybe_unused]] unsigned num_streams) { - auto query_columns = genNamesAndTypesForTiCI(tici_scan.getQueryColumns(), "column"); auto return_columns = genNamesAndTypesForTiCI(tici_scan.getReturnColumns(), "column"); // local read @@ -80,11 +79,10 @@ void StorageTantivy::read( tici_scan.getTableId(), tici_scan.getIndexId(), local_read, - query_columns, return_columns, - tici_scan.getQuery(), tici_scan.getLimit(), - /*read_ts*/ 0)); // TODO: currently, info.mvcc_query_info is null. + /*read_ts*/ 0, + tici_scan.getMatchExpr())); } void StorageTantivy::splitRemoteReadAndLocalRead() diff --git a/dbms/src/Storages/Tantivy/TantivyInputStream.h b/dbms/src/Storages/Tantivy/TantivyInputStream.h index 8baf05b7c9f..eb4a9cb8bf9 100644 --- a/dbms/src/Storages/Tantivy/TantivyInputStream.h +++ b/dbms/src/Storages/Tantivy/TantivyInputStream.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -39,20 +40,18 @@ class TantivyInputStream : public IProfilingBlockInputStream Int64 table_id_, Int64 index_id_, ShardInfoList query_shard_infos_, - NamesAndTypes query_columns_, NamesAndTypes return_columns_, - String query_json_str_, UInt64 limit_, - UInt64 read_ts_) + UInt64 read_ts_, + google::protobuf::RepeatedPtrField match_expr_) : log(log_) , table_id(table_id_) , index_id(index_id_) , query_shard_infos(query_shard_infos_) - , query_columns(query_columns_) , return_columns(return_columns_) - , query_json_str(query_json_str_) , limit(limit_) , read_ts(read_ts_) + , match_expr(match_expr_) {} String getName() const override { return NAME; } @@ -81,13 +80,13 @@ class TantivyInputStream : public IProfilingBlockInputStream protected: Block readFromS3(size_t processing) { - auto query_fields = getFields(query_columns); auto return_fields = getFields(return_columns); auto & shard_info = query_shard_infos[processing]; LOG_INFO(log, "Processing shard: {}, shard info: {}", processing, shard_info.toString()); auto key_ranges = getKeyRanges(shard_info.key_ranges); auto search_param = SearchParam{static_cast(limit)}; + auto expr = tipbToTiCIExpr(match_expr); rust::Vec documents = search( { .table_id = table_id, @@ -96,9 +95,8 @@ class TantivyInputStream : public IProfilingBlockInputStream .shard_epoch = shard_info.shard_epoch, }, key_ranges, - query_fields, return_fields, - query_json_str, + expr, search_param, read_ts); @@ -154,11 +152,10 @@ class TantivyInputStream : public IProfilingBlockInputStream Int64 table_id; Int64 index_id; ShardInfoList query_shard_infos; - NamesAndTypes query_columns; NamesAndTypes return_columns; - String query_json_str; UInt64 limit; UInt64 read_ts; + const google::protobuf::RepeatedPtrField match_expr; size_t processed_shard = 0; @@ -191,6 +188,70 @@ class TantivyInputStream : public IProfilingBlockInputStream } return res; } + + ::Expr tipbToTiCIExpr(google::protobuf::RepeatedPtrField exprs) + { + if (exprs.empty()) + { + throw std::runtime_error("Empty match expression"); + } + ::Expr ret = tipbToTiCIExpr(exprs[0]); + for (auto i = 1; i < exprs.size(); ++i) + { + auto child = tipbToTiCIExpr(exprs[i]); + ret = { + .tp = tipb::ExprType::ScalarFunc, + .children = {ret, child}, + .sig = tipb::ScalarFuncSig::LogicalAnd, + }; + } + return ret; + } + + ::Expr tipbToTiCIExpr(const tipb::Expr & expr) + { + ::Expr ret; + switch (expr.tp()) + { + case tipb::ExprType::ScalarFunc: + { + ret.tp = tipb::ExprType::ScalarFunc; + for (const auto & child : expr.children()) + { + ret.children.push_back(tipbToTiCIExpr(child)); + } + switch (expr.sig()) + { + case tipb::ScalarFuncSig::FTSMatchWord: + case tipb::ScalarFuncSig::FTSMatchPrefix: + case tipb::ScalarFuncSig::LogicalAnd: + case tipb::ScalarFuncSig::LogicalOr: + ret.sig = expr.sig(); + break; + default: + throw std::runtime_error("Unsupported expression sig"); + } + + return ret; + } + case tipb::ExprType::ColumnRef: + { + ret.tp = tipb::ExprType::ColumnRef; + auto id = decodeDAGInt64(expr.val()); + LOG_INFO(log, "column_{}", std::to_string(id)); + ret.val = "column_" + std::to_string(id); + return ret; + } + case tipb::ExprType::String: + { + ret.tp = tipb::ExprType::String; + ret.val = expr.val(); + return ret; + } + default: + throw std::runtime_error("Unsupported expression type"); + } + } }; } // namespace DB::TS From 3526279f561e9306d527896c4f924b931df5cc9b Mon Sep 17 00:00:00 2001 From: jinhelin Date: Tue, 12 Aug 2025 11:39:27 +0800 Subject: [PATCH 12/58] *: Add metrics for tici (#10339) (cherry picked from commit b4c8f4dc804fee63d6c2cd3235a04fb3681d4913) --- dbms/src/Server/MetricsPrometheus.cpp | 47 +++++++++++++++++++++++---- dbms/src/Server/MetricsPrometheus.h | 2 -- 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/dbms/src/Server/MetricsPrometheus.cpp b/dbms/src/Server/MetricsPrometheus.cpp index 045ad4b4f14..5b8a4461c0f 100644 --- a/dbms/src/Server/MetricsPrometheus.cpp +++ b/dbms/src/Server/MetricsPrometheus.cpp @@ -36,9 +36,9 @@ #include #include #include -#include #include #include +#include namespace DB { @@ -98,11 +98,29 @@ class MetricHandler : public Poco::Net::HTTPRequestHandler { auto metrics = collectMetrics(); auto serializer = std::unique_ptr{new prometheus::TextSerializer()}; - String body = serializer->Serialize(metrics); + auto body = concatTextMetrics(serializer->Serialize(metrics), gather_prometheus_metrics()); response.sendBuffer(body.data(), body.size()); } private: + static String concatTextMetrics(String && tiflash_metrics, ::rust::Vec<::std::uint8_t> && tici_metrics) + { + if (tiflash_metrics.empty()) + return String{tici_metrics.begin(), tici_metrics.end()}; + if (tici_metrics.empty()) + return tiflash_metrics; + + if (tiflash_metrics.back() != '\n') + tiflash_metrics.push_back('\n'); + + tiflash_metrics.insert(tiflash_metrics.end(), tici_metrics.begin(), tici_metrics.end()); + + if (tiflash_metrics.back() != '\n') + tiflash_metrics.push_back('\n'); + + return tiflash_metrics; + } + std::vector collectMetrics() const { auto collected_metrics = std::vector{}; @@ -152,7 +170,7 @@ class MetricHandlerFactory : public Poco::Net::HTTPRequestHandlerFactory std::vector> collectables; }; -std::shared_ptr getHTTPServer( +std::shared_ptr getSecureHTTPServer( Context & global_context, std::vector> collectables, const String & address) @@ -184,6 +202,19 @@ std::shared_ptr getHTTPServer( return server; } +std::shared_ptr getHTTPServer( + std::vector> collectables, + const String & address) +{ + Poco::Net::ServerSocket socket; + Poco::Net::HTTPServerParams::Ptr http_params = new Poco::Net::HTTPServerParams; + Poco::Net::SocketAddress addr = Poco::Net::SocketAddress(address); + socket.bind(addr, true); + socket.listen(); + auto server = std::make_shared(new MetricHandlerFactory(collectables), socket, http_params); + return server; +} + constexpr Int64 MILLISECOND = 1000; constexpr Int64 INIT_DELAY = 5; @@ -272,7 +303,7 @@ MetricsPrometheus::MetricsPrometheus(Context & context, const AsynchronousMetric std::vector> collectables{ tiflash_metrics.registry, tiflash_metrics.process_collector}; - server = getHTTPServer(context, collectables, addr); + server = getSecureHTTPServer(context, collectables, addr); server->start(); LOG_INFO( log, @@ -282,9 +313,11 @@ MetricsPrometheus::MetricsPrometheus(Context & context, const AsynchronousMetric } else { - exposer = std::make_shared(addr); - exposer->RegisterCollectable(tiflash_metrics.registry); - exposer->RegisterCollectable(tiflash_metrics.process_collector); + std::vector> collectables{ + tiflash_metrics.registry, + tiflash_metrics.process_collector}; + server = getHTTPServer(collectables, addr); + server->start(); LOG_INFO( log, "Enable prometheus pull mode; Listen Host = {}, Metrics Port = {}", diff --git a/dbms/src/Server/MetricsPrometheus.h b/dbms/src/Server/MetricsPrometheus.h index 29aaa7154cd..7b743aefff8 100644 --- a/dbms/src/Server/MetricsPrometheus.h +++ b/dbms/src/Server/MetricsPrometheus.h @@ -17,7 +17,6 @@ #include #include #include -#include #include namespace DB @@ -54,7 +53,6 @@ class MetricsPrometheus int metrics_interval; std::shared_ptr gateway; - std::shared_ptr exposer; std::shared_ptr server; }; From 2b4b9359132bb1165e6e18f6c53fe7cc48efb8d1 Mon Sep 17 00:00:00 2001 From: Lloyd-Pottiger <60744015+Lloyd-Pottiger@users.noreply.github.com> Date: Fri, 15 Aug 2025 17:02:57 +0800 Subject: [PATCH 13/58] *: fix MacOS build (#10349) Signed-off-by: Lloyd-Pottiger (cherry picked from commit 5cf82e3721c80d0adfff71a7af49829cf0c4099d) --- dbms/src/Server/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dbms/src/Server/CMakeLists.txt b/dbms/src/Server/CMakeLists.txt index 887f2d75079..4d8120db369 100644 --- a/dbms/src/Server/CMakeLists.txt +++ b/dbms/src/Server/CMakeLists.txt @@ -148,6 +148,10 @@ elseif(APPLE) COMPONENT tiflash-release DESTINATION "." PRE_INCLUDE_REGEXES ".*proxy.*") + + install (TARGETS tici_search_lib + COMPONENT tiflash-release + DESTINATION ".") endif () include(CMakePrintHelpers) From c9b1911776939b774eb0fcf8bc9f29977855cb6a Mon Sep 17 00:00:00 2001 From: Lloyd-Pottiger <60744015+Lloyd-Pottiger@users.noreply.github.com> Date: Mon, 18 Aug 2025 17:53:50 +0800 Subject: [PATCH 14/58] tici: reduce log (#10351) Signed-off-by: Lloyd-Pottiger (cherry picked from commit 93c7f972fe1d1347c353b5357607c3a26bc9f59e) --- .../src/Storages/Tantivy/TantivyInputStream.h | 49 ++++++++++++------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/dbms/src/Storages/Tantivy/TantivyInputStream.h b/dbms/src/Storages/Tantivy/TantivyInputStream.h index eb4a9cb8bf9..2508669a9a1 100644 --- a/dbms/src/Storages/Tantivy/TantivyInputStream.h +++ b/dbms/src/Storages/Tantivy/TantivyInputStream.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -51,8 +52,17 @@ class TantivyInputStream : public IProfilingBlockInputStream , return_columns(return_columns_) , limit(limit_) , read_ts(read_ts_) - , match_expr(match_expr_) - {} + { + FmtBuffer buf; + buf.joinStr( + return_columns.begin(), + return_columns.end(), + [](const auto & nt, FmtBuffer & fb) { fb.fmtAppend("{}:{}", nt.name, nt.type->getName()); }, + ", "); + auto [expr, cids] = tipbToTiCIExpr(match_expr_); + match_expr = std::move(expr); + LOG_INFO(log, "columns: [{}], match columns: {}", buf.toString(), cids); + } String getName() const override { return NAME; } @@ -86,7 +96,6 @@ class TantivyInputStream : public IProfilingBlockInputStream auto key_ranges = getKeyRanges(shard_info.key_ranges); auto search_param = SearchParam{static_cast(limit)}; - auto expr = tipbToTiCIExpr(match_expr); rust::Vec documents = search( { .table_id = table_id, @@ -96,7 +105,7 @@ class TantivyInputStream : public IProfilingBlockInputStream }, key_ranges, return_fields, - expr, + match_expr, search_param, read_ts); @@ -155,16 +164,15 @@ class TantivyInputStream : public IProfilingBlockInputStream NamesAndTypes return_columns; UInt64 limit; UInt64 read_ts; - const google::protobuf::RepeatedPtrField match_expr; + ::Expr match_expr; size_t processed_shard = 0; - rust::Vec getFields(NamesAndTypes & columns) + static rust::Vec getFields(NamesAndTypes & columns) { rust::Vec fields; for (auto & name_and_type : columns) { - LOG_INFO(log, "name: {}, type: {}", name_and_type.name, name_and_type.type->getName()); fields.push_back(name_and_type.name); } return fields; @@ -189,36 +197,40 @@ class TantivyInputStream : public IProfilingBlockInputStream return res; } - ::Expr tipbToTiCIExpr(google::protobuf::RepeatedPtrField exprs) + std::tuple<::Expr, std::vector> tipbToTiCIExpr(google::protobuf::RepeatedPtrField exprs) { if (exprs.empty()) { throw std::runtime_error("Empty match expression"); } - ::Expr ret = tipbToTiCIExpr(exprs[0]); + auto [ret, cids] = tipbToTiCIExpr(exprs[0]); for (auto i = 1; i < exprs.size(); ++i) { - auto child = tipbToTiCIExpr(exprs[i]); + auto [child_expr, child_cids] = tipbToTiCIExpr(exprs[i]); ret = { .tp = tipb::ExprType::ScalarFunc, - .children = {ret, child}, + .children = {ret, child_expr}, .sig = tipb::ScalarFuncSig::LogicalAnd, }; + cids.insert(cids.end(), child_cids.begin(), child_cids.end()); } - return ret; + return {ret, cids}; } - ::Expr tipbToTiCIExpr(const tipb::Expr & expr) + std::tuple<::Expr, std::vector> tipbToTiCIExpr(const tipb::Expr & expr) { ::Expr ret; switch (expr.tp()) { case tipb::ExprType::ScalarFunc: { + std::vector children_cids; ret.tp = tipb::ExprType::ScalarFunc; for (const auto & child : expr.children()) { - ret.children.push_back(tipbToTiCIExpr(child)); + auto [child_expr, child_cids] = tipbToTiCIExpr(child); + ret.children.push_back(child_expr); + children_cids.insert(children_cids.end(), child_cids.begin(), child_cids.end()); } switch (expr.sig()) { @@ -232,21 +244,20 @@ class TantivyInputStream : public IProfilingBlockInputStream throw std::runtime_error("Unsupported expression sig"); } - return ret; + return {ret, children_cids}; } case tipb::ExprType::ColumnRef: { ret.tp = tipb::ExprType::ColumnRef; auto id = decodeDAGInt64(expr.val()); - LOG_INFO(log, "column_{}", std::to_string(id)); - ret.val = "column_" + std::to_string(id); - return ret; + ret.val = fmt::format("column_{}", id); + return {ret, {id}}; } case tipb::ExprType::String: { ret.tp = tipb::ExprType::String; ret.val = expr.val(); - return ret; + return {ret, {}}; } default: throw std::runtime_error("Unsupported expression type"); From 5b5e890cacba97b1bc57433b588b938392ac5b19 Mon Sep 17 00:00:00 2001 From: jinhelin Date: Mon, 25 Aug 2025 17:23:51 +0800 Subject: [PATCH 15/58] tici: Update tici and pass `read_tso` to tici (#10362) (cherry picked from commit 4404720d5b88a66e227d9e8aef95994510a4d862) --- dbms/src/Storages/StorageTantivy.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/StorageTantivy.cpp b/dbms/src/Storages/StorageTantivy.cpp index 6430ae16a87..155278fede3 100644 --- a/dbms/src/Storages/StorageTantivy.cpp +++ b/dbms/src/Storages/StorageTantivy.cpp @@ -66,7 +66,7 @@ void StorageTantivy::read( PipelineExecGroupBuilder & group_builder, [[maybe_unused]] const Names & column_names, [[maybe_unused]] const SelectQueryInfo & info, - [[maybe_unused]] const Context & context, + const Context & context, [[maybe_unused]] size_t max_block_size, [[maybe_unused]] unsigned num_streams) { @@ -81,7 +81,7 @@ void StorageTantivy::read( local_read, return_columns, tici_scan.getLimit(), - /*read_ts*/ 0, + context.getSettingsRef().read_tso, tici_scan.getMatchExpr())); } From 14e263ce62529644ada3d62b18e31300cb5f900f Mon Sep 17 00:00:00 2001 From: Shenghui Wu <793703860@qq.com> Date: Tue, 2 Sep 2025 19:42:34 +0800 Subject: [PATCH 16/58] tici: support pushdown not expr into tici searchlib (#10393) (cherry picked from commit 48faed1cc2ca37ccb8d4321868afa61f562b9220) --- dbms/src/Storages/Tantivy/TantivyInputStream.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dbms/src/Storages/Tantivy/TantivyInputStream.h b/dbms/src/Storages/Tantivy/TantivyInputStream.h index 2508669a9a1..9bc30f6e867 100644 --- a/dbms/src/Storages/Tantivy/TantivyInputStream.h +++ b/dbms/src/Storages/Tantivy/TantivyInputStream.h @@ -238,6 +238,8 @@ class TantivyInputStream : public IProfilingBlockInputStream case tipb::ScalarFuncSig::FTSMatchPrefix: case tipb::ScalarFuncSig::LogicalAnd: case tipb::ScalarFuncSig::LogicalOr: + case tipb::ScalarFuncSig::UnaryNotInt: + case tipb::ScalarFuncSig::UnaryNotReal: ret.sig = expr.sig(); break; default: From 8270bd10ce7b5093e95832322d3d3e5397f02012 Mon Sep 17 00:00:00 2001 From: Lloyd-Pottiger <60744015+Lloyd-Pottiger@users.noreply.github.com> Date: Fri, 5 Sep 2025 17:16:54 +0800 Subject: [PATCH 17/58] tici: update tici commit & refine log (#10407) Signed-off-by: Lloyd-Pottiger (cherry picked from commit 104d87ff55b6d63b6c84ba503a8e53964215a403) --- dbms/src/Operators/TantivyReaderSourceOp.cpp | 4 ++-- dbms/src/Operators/TantivyReaderSourceOp.h | 2 ++ dbms/src/Storages/Tantivy/TantivyInputStream.h | 6 +++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/dbms/src/Operators/TantivyReaderSourceOp.cpp b/dbms/src/Operators/TantivyReaderSourceOp.cpp index 943e2d9ce7f..987d6eb2f63 100644 --- a/dbms/src/Operators/TantivyReaderSourceOp.cpp +++ b/dbms/src/Operators/TantivyReaderSourceOp.cpp @@ -47,12 +47,12 @@ String TantivyReaderSourceOp::getName() const void TantivyReaderSourceOp::operatePrefixImpl() { - LOG_DEBUG(log, "start reading from TantivyReaderSourceOp"); + watcher.start(); } void TantivyReaderSourceOp::operateSuffixImpl() { - LOG_DEBUG(log, "finish read {} rows from TantivyReaderSourceOp", total_rows); + LOG_DEBUG(log, "TantivyReaderSourceOp read {} rows, took {} ms", total_rows, watcher.elapsedMilliseconds()); } Block TantivyReaderSourceOp::popFromBlockQueue() diff --git a/dbms/src/Operators/TantivyReaderSourceOp.h b/dbms/src/Operators/TantivyReaderSourceOp.h index 81614beb36b..8277104a4a6 100644 --- a/dbms/src/Operators/TantivyReaderSourceOp.h +++ b/dbms/src/Operators/TantivyReaderSourceOp.h @@ -59,5 +59,7 @@ class TantivyReaderSourceOp : public SourceOp IOProfileInfoPtr io_profile_info; std::shared_ptr input; + Stopwatch watcher; }; + } // namespace DB diff --git a/dbms/src/Storages/Tantivy/TantivyInputStream.h b/dbms/src/Storages/Tantivy/TantivyInputStream.h index 9bc30f6e867..913a49bd8d4 100644 --- a/dbms/src/Storages/Tantivy/TantivyInputStream.h +++ b/dbms/src/Storages/Tantivy/TantivyInputStream.h @@ -61,7 +61,7 @@ class TantivyInputStream : public IProfilingBlockInputStream ", "); auto [expr, cids] = tipbToTiCIExpr(match_expr_); match_expr = std::move(expr); - LOG_INFO(log, "columns: [{}], match columns: {}", buf.toString(), cids); + LOG_DEBUG(log, "columns: [{}], match columns: {}", buf.toString(), cids); } String getName() const override { return NAME; } @@ -76,7 +76,7 @@ class TantivyInputStream : public IProfilingBlockInputStream } if (processed_shard >= query_shard_infos.size()) { - LOG_INFO(log, "empty shard info, returning empty block"); + LOG_DEBUG(log, "empty shard info, returning empty block"); done = true; return {}; } @@ -92,7 +92,7 @@ class TantivyInputStream : public IProfilingBlockInputStream { auto return_fields = getFields(return_columns); auto & shard_info = query_shard_infos[processing]; - LOG_INFO(log, "Processing shard: {}, shard info: {}", processing, shard_info.toString()); + LOG_DEBUG(log, "Processing shard: {}, shard info: {}", processing, shard_info.toString()); auto key_ranges = getKeyRanges(shard_info.key_ranges); auto search_param = SearchParam{static_cast(limit)}; From 38cba05c35797c9bc7f2ff9521dad7150fbfd5a8 Mon Sep 17 00:00:00 2001 From: Shenghui Wu <793703860@qq.com> Date: Wed, 17 Sep 2025 17:43:07 +0800 Subject: [PATCH 18/58] tici: support count pushdown to tici searchlib (#10428) (cherry picked from commit b2fa044a0c15404847c12ee2c758d594f42d47e9) --- dbms/src/Flash/Coprocessor/TiCIScan.h | 5 +++ .../Flash/Planner/Plans/PhysicalAggregation.h | 8 +++++ .../Flash/Planner/Plans/PhysicalTiCIScan.cpp | 2 +- .../Flash/Planner/Plans/PhysicalTiCIScan.h | 2 ++ dbms/src/Flash/Planner/optimize.cpp | 36 ++++++++++++++++++- dbms/src/Operators/TantivyReaderSourceOp.cpp | 15 ++++++-- dbms/src/Operators/TantivyReaderSourceOp.h | 3 +- dbms/src/Storages/StorageTantivy.cpp | 3 +- .../src/Storages/Tantivy/TantivyInputStream.h | 34 +++++++++++++++--- 9 files changed, 97 insertions(+), 11 deletions(-) diff --git a/dbms/src/Flash/Coprocessor/TiCIScan.h b/dbms/src/Flash/Coprocessor/TiCIScan.h index 1629e0e4f18..52c1f2ddeee 100644 --- a/dbms/src/Flash/Coprocessor/TiCIScan.h +++ b/dbms/src/Flash/Coprocessor/TiCIScan.h @@ -41,6 +41,10 @@ class TiCIScan return tici_scan->idx_scan().fts_query_info().match_expr(); } + bool isCount() const { return is_count_agg; } + + void setIsCountAgg(bool v) { is_count_agg = v; } + private: const tipb::Executor * tici_scan; [[maybe_unused]] String executor_id; @@ -50,5 +54,6 @@ class TiCIScan [[maybe_unused]] tipb::FTSQueryType query_type; const TableShardInfos shard_infos; const int limit; + bool is_count_agg = false; }; } // namespace DB diff --git a/dbms/src/Flash/Planner/Plans/PhysicalAggregation.h b/dbms/src/Flash/Planner/Plans/PhysicalAggregation.h index 6be21ded21b..b15f0a28e53 100644 --- a/dbms/src/Flash/Planner/Plans/PhysicalAggregation.h +++ b/dbms/src/Flash/Planner/Plans/PhysicalAggregation.h @@ -66,6 +66,14 @@ class PhysicalAggregation : public PhysicalUnary const Block & getSampleBlock() const override; + bool isCountNotNullableColumnWithoutGroupbyKey() const + { + // Only single count(const/not-null column) without any group by key pattern can be optimized by TiCI + // TiFlash will convert count(const/not-null column) to count(), so the argument_names is empty. + return aggregate_descriptions.size() == 1 && aggregate_descriptions[0].function->getName() == "count" + && aggregate_descriptions[0].argument_names.empty() && aggregation_keys.empty(); + } + private: void buildBlockInputStreamImpl(DAGPipeline & pipeline, Context & context, size_t max_streams) override; diff --git a/dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.cpp b/dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.cpp index eff4c847efe..125997e137a 100644 --- a/dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.cpp +++ b/dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.cpp @@ -80,7 +80,7 @@ PhysicalTiCIScan::PhysicalTiCIScan( const String & req_id, const TiCIScan & tici_scan_, const Block & sample_block_) - : PhysicalLeaf(executor_id_, PlanType::TableScan, schema_, FineGrainedShuffle{}, req_id) + : PhysicalLeaf(executor_id_, PlanType::TiCiScan, schema_, FineGrainedShuffle{}, req_id) , tici_scan(tici_scan_) , sample_block(sample_block_) {} diff --git a/dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.h b/dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.h index 90ec286cee4..6842915958f 100644 --- a/dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.h +++ b/dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.h @@ -34,6 +34,8 @@ class PhysicalTiCIScan : public PhysicalLeaf const TiCIScan & tici_scan_, const Block & sample_block_); + void setIsCountAgg(bool v) { tici_scan.setIsCountAgg(v); } + void finalizeImpl(const Names & parent_require) override; const Block & getSampleBlock() const override; diff --git a/dbms/src/Flash/Planner/optimize.cpp b/dbms/src/Flash/Planner/optimize.cpp index 216dab04296..56bd50088b1 100644 --- a/dbms/src/Flash/Planner/optimize.cpp +++ b/dbms/src/Flash/Planner/optimize.cpp @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include +#include #include #include @@ -38,10 +40,42 @@ class FinalizeRule : public Rule static RulePtr create() { return std::make_shared(); } }; +class TiCICountAggOptimizeRule : public Rule +{ +public: + PhysicalPlanNodePtr apply(const Context & context, PhysicalPlanNodePtr plan, const LoggerPtr & logger) override + { + if (plan->tp() == PlanType::Aggregation) + { + auto agg = std::static_pointer_cast(plan); + if (agg->isCountNotNullableColumnWithoutGroupbyKey()) + { + auto child = agg->children(0); + if (child->tp() == PlanType::TiCiScan) + { + auto tici_scan = std::static_pointer_cast(child); + tici_scan->setIsCountAgg(true); + } + } + } + if (plan->childrenSize() != 0) + { + for (size_t i = 0; i < plan->childrenSize(); ++i) + { + apply(context, plan->children(i), logger); + } + } + + return plan; + } + + static RulePtr create() { return std::make_shared(); } +}; + PhysicalPlanNodePtr optimize(const Context & context, PhysicalPlanNodePtr plan, const LoggerPtr & log) { RUNTIME_CHECK(plan); - static std::vector rules{FinalizeRule::create()}; + static std::vector rules{TiCICountAggOptimizeRule::create(), FinalizeRule::create()}; for (const auto & rule : rules) { plan = rule->apply(context, plan, log); diff --git a/dbms/src/Operators/TantivyReaderSourceOp.cpp b/dbms/src/Operators/TantivyReaderSourceOp.cpp index 987d6eb2f63..e98795f81e4 100644 --- a/dbms/src/Operators/TantivyReaderSourceOp.cpp +++ b/dbms/src/Operators/TantivyReaderSourceOp.cpp @@ -31,13 +31,22 @@ TantivyReaderSourceOp::TantivyReaderSourceOp( const NamesAndTypes & return_columns, const UInt64 & limit, const UInt64 & read_ts, - const google::protobuf::RepeatedPtrField & expr) + const google::protobuf::RepeatedPtrField & expr, + bool is_count) : SourceOp(exec_context_, req_id) { setHeader(Block(return_columns)); - input = std::make_shared< - TS::TantivyInputStream>(log, table_id, index_id, query_shard_infos, return_columns, limit, read_ts, expr); + input = std::make_shared( + log, + table_id, + index_id, + query_shard_infos, + return_columns, + limit, + read_ts, + expr, + is_count); } String TantivyReaderSourceOp::getName() const diff --git a/dbms/src/Operators/TantivyReaderSourceOp.h b/dbms/src/Operators/TantivyReaderSourceOp.h index 8277104a4a6..8a39d1c22a5 100644 --- a/dbms/src/Operators/TantivyReaderSourceOp.h +++ b/dbms/src/Operators/TantivyReaderSourceOp.h @@ -37,7 +37,8 @@ class TantivyReaderSourceOp : public SourceOp const NamesAndTypes & return_columns, const UInt64 & limit, const UInt64 & read_ts, - const google::protobuf::RepeatedPtrField & expr); + const google::protobuf::RepeatedPtrField & expr, + bool is_count); String getName() const override; diff --git a/dbms/src/Storages/StorageTantivy.cpp b/dbms/src/Storages/StorageTantivy.cpp index 155278fede3..13798bf1ca1 100644 --- a/dbms/src/Storages/StorageTantivy.cpp +++ b/dbms/src/Storages/StorageTantivy.cpp @@ -82,7 +82,8 @@ void StorageTantivy::read( return_columns, tici_scan.getLimit(), context.getSettingsRef().read_tso, - tici_scan.getMatchExpr())); + tici_scan.getMatchExpr(), + tici_scan.isCount())); } void StorageTantivy::splitRemoteReadAndLocalRead() diff --git a/dbms/src/Storages/Tantivy/TantivyInputStream.h b/dbms/src/Storages/Tantivy/TantivyInputStream.h index 913a49bd8d4..f559d386444 100644 --- a/dbms/src/Storages/Tantivy/TantivyInputStream.h +++ b/dbms/src/Storages/Tantivy/TantivyInputStream.h @@ -44,7 +44,8 @@ class TantivyInputStream : public IProfilingBlockInputStream NamesAndTypes return_columns_, UInt64 limit_, UInt64 read_ts_, - google::protobuf::RepeatedPtrField match_expr_) + google::protobuf::RepeatedPtrField match_expr_, + bool is_count) : log(log_) , table_id(table_id_) , index_id(index_id_) @@ -52,6 +53,7 @@ class TantivyInputStream : public IProfilingBlockInputStream , return_columns(return_columns_) , limit(limit_) , read_ts(read_ts_) + , is_count(is_count) { FmtBuffer buf; buf.joinStr( @@ -81,14 +83,14 @@ class TantivyInputStream : public IProfilingBlockInputStream return {}; } - Block ret = readFromS3(processed_shard); + Block ret = readFromS3(processed_shard, is_count); processed_shard++; done = processed_shard >= query_shard_infos.size(); return ret; } protected: - Block readFromS3(size_t processing) + Block readFromS3(size_t processing, bool is_count) { auto return_fields = getFields(return_columns); auto & shard_info = query_shard_infos[processing]; @@ -96,7 +98,10 @@ class TantivyInputStream : public IProfilingBlockInputStream auto key_ranges = getKeyRanges(shard_info.key_ranges); auto search_param = SearchParam{static_cast(limit)}; - rust::Vec documents = search( + if (is_count) + return_fields = {}; + + SearchResult search_result = search( { .table_id = table_id, .index_id = index_id, @@ -110,6 +115,26 @@ class TantivyInputStream : public IProfilingBlockInputStream read_ts); Block res(return_columns); + if (is_count) + { + if (search_result.count == 0) + { + return res; + } + // Construct const columns with search_result.count items, only used by count(*) + for (auto & name_and_type : return_columns) + { + auto type_default_value = name_and_type.type->getDefault(); + auto col = name_and_type.type->createColumnConst( + static_cast(search_result.count), + type_default_value); + res.getByName(name_and_type.name).column = std::move(col); + } + return res; + } + + + auto documents = search_result.rows; if (documents.empty()) { return res; @@ -165,6 +190,7 @@ class TantivyInputStream : public IProfilingBlockInputStream UInt64 limit; UInt64 read_ts; ::Expr match_expr; + bool is_count; size_t processed_shard = 0; From 57b26d77d70b29f14749a85c057de13727d05eea Mon Sep 17 00:00:00 2001 From: Lloyd-Pottiger <60744015+Lloyd-Pottiger@users.noreply.github.com> Date: Sun, 28 Sep 2025 19:10:34 +0800 Subject: [PATCH 19/58] *: update TiCI (#10461) Signed-off-by: Lloyd-Pottiger (cherry picked from commit 7c1c0478aa3c7f1ebe9607641c9fd2a1cd97926e) --- dbms/src/Server/Server.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/dbms/src/Server/Server.cpp b/dbms/src/Server/Server.cpp index 96dc8600f63..985e9e5c09e 100644 --- a/dbms/src/Server/Server.cpp +++ b/dbms/src/Server/Server.cpp @@ -1025,13 +1025,12 @@ try auto schema_cache_size = config().getInt("schema_cache_size", 10000); global_context->initializeSharedBlockSchemas(schema_cache_size); - if (config().getBool("tici.enable", false)) + if (auto addr = config().getString("tici.reader_node.addr", ""); !addr.empty()) { - start_reader_server(config_path); - } - else - { - LOG_INFO(log, "tici is disabled"); + auto service_addr = config().getString("flash.service_addr"); + auto pd_addr = config().getString("raft.pd_addr"); + start_reader_server(config_path, addr, service_addr, pd_addr); + LOG_INFO(log, "tici is enabled, reader_node addr: {}", addr); } // Load remaining databases From fdb2b81a68625e134bfc3b54c7f7e5d41c757b46 Mon Sep 17 00:00:00 2001 From: jinhelin Date: Wed, 22 Oct 2025 12:34:38 +0800 Subject: [PATCH 20/58] contrib: Update tici (#10490) (cherry picked from commit 41c765c2640cc3034ee5d0f1aa327fd0501da478) --- dbms/src/Storages/Tantivy/TantivyInputStream.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbms/src/Storages/Tantivy/TantivyInputStream.h b/dbms/src/Storages/Tantivy/TantivyInputStream.h index f559d386444..3e3b97d54b8 100644 --- a/dbms/src/Storages/Tantivy/TantivyInputStream.h +++ b/dbms/src/Storages/Tantivy/TantivyInputStream.h @@ -165,7 +165,8 @@ class TantivyInputStream : public IProfilingBlockInputStream { for (auto & doc : documents) { - col->insert(Field(String(doc.fieldValues[idx].string_value.c_str()))); + const auto & v = doc.fieldValues[idx].string_value; + col->insert(Field(String(v.begin(), v.end()))); } } if (removeNullable(name_and_type.type)->isInteger()) From ccf28c2b17dc6d071046dc745366fa60b785e37e Mon Sep 17 00:00:00 2001 From: Shenghui Wu <793703860@qq.com> Date: Thu, 13 Nov 2025 20:30:23 +0800 Subject: [PATCH 21/58] tici: support pushdown cmp function (#10542) (cherry picked from commit d39ad69f5bbfde28b55c5a3351bcb308f839e086) --- .../src/Storages/Tantivy/TantivyInputStream.h | 62 +++++++++++++++++-- 1 file changed, 58 insertions(+), 4 deletions(-) diff --git a/dbms/src/Storages/Tantivy/TantivyInputStream.h b/dbms/src/Storages/Tantivy/TantivyInputStream.h index 3e3b97d54b8..312973308bb 100644 --- a/dbms/src/Storages/Tantivy/TantivyInputStream.h +++ b/dbms/src/Storages/Tantivy/TantivyInputStream.h @@ -267,6 +267,36 @@ class TantivyInputStream : public IProfilingBlockInputStream case tipb::ScalarFuncSig::LogicalOr: case tipb::ScalarFuncSig::UnaryNotInt: case tipb::ScalarFuncSig::UnaryNotReal: + case tipb::ScalarFuncSig::EQInt: + case tipb::ScalarFuncSig::NEInt: + case tipb::ScalarFuncSig::LTInt: + case tipb::ScalarFuncSig::LEInt: + case tipb::ScalarFuncSig::GTInt: + case tipb::ScalarFuncSig::GEInt: + case tipb::ScalarFuncSig::EQString: + case tipb::ScalarFuncSig::NEString: + case tipb::ScalarFuncSig::LTString: + case tipb::ScalarFuncSig::LEString: + case tipb::ScalarFuncSig::GTString: + case tipb::ScalarFuncSig::GEString: + case tipb::ScalarFuncSig::EQReal: + case tipb::ScalarFuncSig::NEReal: + case tipb::ScalarFuncSig::LTReal: + case tipb::ScalarFuncSig::LEReal: + case tipb::ScalarFuncSig::GTReal: + case tipb::ScalarFuncSig::GEReal: + case tipb::ScalarFuncSig::EQTime: + case tipb::ScalarFuncSig::NETime: + case tipb::ScalarFuncSig::LTTime: + case tipb::ScalarFuncSig::LETime: + case tipb::ScalarFuncSig::GTTime: + case tipb::ScalarFuncSig::GETime: + case tipb::ScalarFuncSig::EQDecimal: + case tipb::ScalarFuncSig::NEDecimal: + case tipb::ScalarFuncSig::LTDecimal: + case tipb::ScalarFuncSig::LEDecimal: + case tipb::ScalarFuncSig::GTDecimal: + case tipb::ScalarFuncSig::GEDecimal: ret.sig = expr.sig(); break; default: @@ -277,15 +307,39 @@ class TantivyInputStream : public IProfilingBlockInputStream } case tipb::ExprType::ColumnRef: { - ret.tp = tipb::ExprType::ColumnRef; + ret.tp = expr.tp(); auto id = decodeDAGInt64(expr.val()); - ret.val = fmt::format("column_{}", id); + auto str = fmt::format("column_{}", id); + std::copy(str.begin(), str.end(), std::back_inserter(ret.val)); return {ret, {id}}; } case tipb::ExprType::String: + case tipb::ExprType::Int64: + case tipb::ExprType::Uint64: + case tipb::ExprType::Float32: + case tipb::ExprType::Float64: + case tipb::ExprType::MysqlTime: { - ret.tp = tipb::ExprType::String; - ret.val = expr.val(); + ret.tp = expr.tp(); + std::copy(expr.val().begin(), expr.val().end(), std::back_inserter(ret.val)); + return {ret, {}}; + } + case tipb::ExprType::MysqlDecimal: + { + ret.tp = expr.tp(); + auto field = decodeDAGDecimal(expr.val()); + String str; + if (field.getType() == Field::Types::Decimal32) + str = field.get>().toString(); + else if (field.getType() == Field::Types::Decimal64) + str = field.get>().toString(); + else if (field.getType() == Field::Types::Decimal128) + str = field.get>().toString(); + else if (field.getType() == Field::Types::Decimal256) + str = field.get>().toString(); + else + throw TiFlashException("Not decimal literal" + expr.DebugString(), Errors::Coprocessor::BadRequest); + std::copy(str.begin(), str.end(), std::back_inserter(ret.val)); return {ret, {}}; } default: From 9770c013331b8e7abacf48d839ad57e443db4d35 Mon Sep 17 00:00:00 2001 From: Shenghui Wu <793703860@qq.com> Date: Mon, 24 Nov 2025 12:40:34 +0800 Subject: [PATCH 22/58] tici: support keyspaceid in read path (#10528) (cherry picked from commit cd1d271f2afb60c14d6db9f63a0b5967a7480bd9) --- dbms/src/Flash/BatchCoprocessorHandler.cpp | 3 ++- dbms/src/Flash/Coprocessor/TiCIScan.cpp | 1 + dbms/src/Flash/Coprocessor/TiCIScan.h | 2 ++ dbms/src/Flash/CoprocessorHandler.cpp | 3 ++- dbms/src/Operators/TantivyReaderSourceOp.cpp | 2 ++ dbms/src/Operators/TantivyReaderSourceOp.h | 1 + dbms/src/Storages/StorageTantivy.cpp | 1 + dbms/src/Storages/Tantivy/TantivyInputStream.h | 4 ++++ 8 files changed, 15 insertions(+), 2 deletions(-) diff --git a/dbms/src/Flash/BatchCoprocessorHandler.cpp b/dbms/src/Flash/BatchCoprocessorHandler.cpp index eeed4f93c64..b9b4102ffcf 100644 --- a/dbms/src/Flash/BatchCoprocessorHandler.cpp +++ b/dbms/src/Flash/BatchCoprocessorHandler.cpp @@ -52,8 +52,9 @@ grpc::Status BatchCoprocessorHandler::execute() try { + bool is_tici = cop_request->table_shard_infos_size() != 0; RUNTIME_CHECK_MSG( - !cop_context.db_context.getSharedContextDisagg()->isDisaggregatedComputeMode(), + !cop_context.db_context.getSharedContextDisagg()->isDisaggregatedComputeMode() || is_tici, "cannot run cop or batchCop request on tiflash_compute node"); switch (cop_request->tp()) diff --git a/dbms/src/Flash/Coprocessor/TiCIScan.cpp b/dbms/src/Flash/Coprocessor/TiCIScan.cpp index 209cd0e5d0e..e8e9573b42a 100644 --- a/dbms/src/Flash/Coprocessor/TiCIScan.cpp +++ b/dbms/src/Flash/Coprocessor/TiCIScan.cpp @@ -24,6 +24,7 @@ namespace DB TiCIScan::TiCIScan(const tipb::Executor * tici_scan_, const String & executor_id_, const DAGContext & dag_context) : tici_scan(tici_scan_) , executor_id(executor_id_) + , keyspace_id(dag_context.getKeyspaceID()) , table_id(tici_scan->idx_scan().table_id()) , index_id(tici_scan->idx_scan().index_id()) , return_columns(TiDB::toTiDBColumnInfos(tici_scan->idx_scan().columns())) diff --git a/dbms/src/Flash/Coprocessor/TiCIScan.h b/dbms/src/Flash/Coprocessor/TiCIScan.h index 52c1f2ddeee..dd7f9574d43 100644 --- a/dbms/src/Flash/Coprocessor/TiCIScan.h +++ b/dbms/src/Flash/Coprocessor/TiCIScan.h @@ -29,6 +29,7 @@ class TiCIScan const TiDB::ColumnInfos & getReturnColumns() const { return return_columns; } const TableShardInfos & getShardInfos() const { return shard_infos; } + const uint & getKeyspaceID() const { return keyspace_id; } const int & getTableId() const { return table_id; } const int & getIndexId() const { return index_id; } const int & getLimit() const { return limit; } @@ -48,6 +49,7 @@ class TiCIScan private: const tipb::Executor * tici_scan; [[maybe_unused]] String executor_id; + const uint keyspace_id; const int table_id; const int index_id; TiDB::ColumnInfos return_columns; diff --git a/dbms/src/Flash/CoprocessorHandler.cpp b/dbms/src/Flash/CoprocessorHandler.cpp index c92a3ddd5e4..ded93355d40 100644 --- a/dbms/src/Flash/CoprocessorHandler.cpp +++ b/dbms/src/Flash/CoprocessorHandler.cpp @@ -99,8 +99,9 @@ grpc::Status CoprocessorHandler::execute() try { + bool is_tici = cop_request->table_shard_infos_size() != 0; RUNTIME_CHECK_MSG( - !cop_context.db_context.getSharedContextDisagg()->isDisaggregatedComputeMode(), + !cop_context.db_context.getSharedContextDisagg()->isDisaggregatedComputeMode() || is_tici, "cannot run cop or batchCop request on tiflash_compute node"); switch (cop_request->tp()) diff --git a/dbms/src/Operators/TantivyReaderSourceOp.cpp b/dbms/src/Operators/TantivyReaderSourceOp.cpp index e98795f81e4..6fc42ee152a 100644 --- a/dbms/src/Operators/TantivyReaderSourceOp.cpp +++ b/dbms/src/Operators/TantivyReaderSourceOp.cpp @@ -25,6 +25,7 @@ namespace DB TantivyReaderSourceOp::TantivyReaderSourceOp( PipelineExecutorContext & exec_context_, const String & req_id, + const UInt32 & keyspace_id, const Int64 & table_id, const Int64 & index_id, const ShardInfoList & query_shard_infos, @@ -39,6 +40,7 @@ TantivyReaderSourceOp::TantivyReaderSourceOp( input = std::make_shared( log, + keyspace_id, table_id, index_id, query_shard_infos, diff --git a/dbms/src/Operators/TantivyReaderSourceOp.h b/dbms/src/Operators/TantivyReaderSourceOp.h index 8a39d1c22a5..1f6bbf4a7a7 100644 --- a/dbms/src/Operators/TantivyReaderSourceOp.h +++ b/dbms/src/Operators/TantivyReaderSourceOp.h @@ -31,6 +31,7 @@ class TantivyReaderSourceOp : public SourceOp TantivyReaderSourceOp( PipelineExecutorContext & exec_context_, const String & req_id, + const UInt32 & keyspace_id, const Int64 & table_id, const Int64 & index_id, const ShardInfoList & query_shard_infos, diff --git a/dbms/src/Storages/StorageTantivy.cpp b/dbms/src/Storages/StorageTantivy.cpp index 13798bf1ca1..988ef66e32a 100644 --- a/dbms/src/Storages/StorageTantivy.cpp +++ b/dbms/src/Storages/StorageTantivy.cpp @@ -76,6 +76,7 @@ void StorageTantivy::read( group_builder.addConcurrency(std::make_unique( exec_status, log->identifier(), + tici_scan.getKeyspaceID(), tici_scan.getTableId(), tici_scan.getIndexId(), local_read, diff --git a/dbms/src/Storages/Tantivy/TantivyInputStream.h b/dbms/src/Storages/Tantivy/TantivyInputStream.h index 312973308bb..2c56f3e0982 100644 --- a/dbms/src/Storages/Tantivy/TantivyInputStream.h +++ b/dbms/src/Storages/Tantivy/TantivyInputStream.h @@ -38,6 +38,7 @@ class TantivyInputStream : public IProfilingBlockInputStream public: TantivyInputStream( LoggerPtr log_, + UInt32 keyspace_id_, Int64 table_id_, Int64 index_id_, ShardInfoList query_shard_infos_, @@ -47,6 +48,7 @@ class TantivyInputStream : public IProfilingBlockInputStream google::protobuf::RepeatedPtrField match_expr_, bool is_count) : log(log_) + , keyspace_id(keyspace_id_) , table_id(table_id_) , index_id(index_id_) , query_shard_infos(query_shard_infos_) @@ -103,6 +105,7 @@ class TantivyInputStream : public IProfilingBlockInputStream SearchResult search_result = search( { + .keyspace_id = keyspace_id, .table_id = table_id, .index_id = index_id, .shard_id = shard_info.shard_id, @@ -184,6 +187,7 @@ class TantivyInputStream : public IProfilingBlockInputStream Block header; bool done = false; LoggerPtr log; + UInt32 keyspace_id; Int64 table_id; Int64 index_id; ShardInfoList query_shard_infos; From 0caa9f6aad5a425fc891f1e29f9765536ac35be8 Mon Sep 17 00:00:00 2001 From: Shenghui Wu <793703860@qq.com> Date: Tue, 25 Nov 2025 20:35:11 +0800 Subject: [PATCH 23/58] tici: support read datetime and timestamp (#10559) (cherry picked from commit f842b09fe89f849e602ecd0f6886b7fb93d4ca1a) --- .../Coprocessor/DAGExpressionAnalyzer.cpp | 26 ++++++++ .../Flash/Coprocessor/DAGExpressionAnalyzer.h | 6 ++ dbms/src/Operators/TantivyReaderSourceOp.cpp | 6 +- dbms/src/Operators/TantivyReaderSourceOp.h | 3 +- dbms/src/Storages/StorageTantivy.cpp | 34 +++++++++- dbms/src/Storages/StorageTantivy.h | 2 + .../src/Storages/Tantivy/TantivyInputStream.h | 62 ++++++++++++++++--- 7 files changed, 126 insertions(+), 13 deletions(-) diff --git a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.cpp b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.cpp index 095d2fc8813..8a52b970d5c 100644 --- a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.cpp +++ b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.cpp @@ -1227,6 +1227,32 @@ bool DAGExpressionAnalyzer::appendExtraCastsAfterTS( return true; } +bool DAGExpressionAnalyzer::appendExtraCastsAfterTiCI( + ExpressionActionsChain & chain, + const std::vector & may_need_add_cast_column, + const TiCIScan & tici_scan) +{ + auto & step = initAndGetLastStep(chain); + auto & actions = step.actions; + + auto [has_cast, casted_columns] + = buildExtraCastsAfterTS(actions, may_need_add_cast_column, tici_scan.getReturnColumns()); + + if (!has_cast) + return false; + + // Add a projection to replace the original columns with the casted columns. + NamesWithAliases project_cols; + for (size_t i = 0; i < may_need_add_cast_column.size(); ++i) + project_cols.emplace_back(casted_columns[i], source_columns[i].name); + actions->add(ExpressionAction::project(project_cols)); + + for (auto & col : source_columns) + step.required_output.push_back(col.name); + + return true; +} + String DAGExpressionAnalyzer::appendDurationCast( const String & fsp_expr, const String & dur_expr, diff --git a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.h b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.h index 452c2b0814f..f283c32be8e 100644 --- a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.h +++ b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -113,6 +114,11 @@ class DAGExpressionAnalyzer : private boost::noncopyable const std::vector & may_need_add_cast_column, const TiDBTableScan & table_scan); + bool appendExtraCastsAfterTiCI( + ExpressionActionsChain & chain, + const std::vector & may_need_add_cast_column, + const TiCIScan & tici_scan); + /// return true if some actions is needed bool appendJoinKeyAndJoinFilters( ExpressionActionsChain & chain, diff --git a/dbms/src/Operators/TantivyReaderSourceOp.cpp b/dbms/src/Operators/TantivyReaderSourceOp.cpp index 6fc42ee152a..61e587ee02c 100644 --- a/dbms/src/Operators/TantivyReaderSourceOp.cpp +++ b/dbms/src/Operators/TantivyReaderSourceOp.cpp @@ -33,7 +33,8 @@ TantivyReaderSourceOp::TantivyReaderSourceOp( const UInt64 & limit, const UInt64 & read_ts, const google::protobuf::RepeatedPtrField & expr, - bool is_count) + bool is_count, + const TimezoneInfo & timezone_info) : SourceOp(exec_context_, req_id) { setHeader(Block(return_columns)); @@ -48,7 +49,8 @@ TantivyReaderSourceOp::TantivyReaderSourceOp( limit, read_ts, expr, - is_count); + is_count, + timezone_info); } String TantivyReaderSourceOp::getName() const diff --git a/dbms/src/Operators/TantivyReaderSourceOp.h b/dbms/src/Operators/TantivyReaderSourceOp.h index 1f6bbf4a7a7..21574fe382d 100644 --- a/dbms/src/Operators/TantivyReaderSourceOp.h +++ b/dbms/src/Operators/TantivyReaderSourceOp.h @@ -39,7 +39,8 @@ class TantivyReaderSourceOp : public SourceOp const UInt64 & limit, const UInt64 & read_ts, const google::protobuf::RepeatedPtrField & expr, - bool is_count); + bool is_count, + const TimezoneInfo & timezone_info); String getName() const override; diff --git a/dbms/src/Storages/StorageTantivy.cpp b/dbms/src/Storages/StorageTantivy.cpp index 988ef66e32a..47035a72b3e 100644 --- a/dbms/src/Storages/StorageTantivy.cpp +++ b/dbms/src/Storages/StorageTantivy.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -44,8 +45,7 @@ namespace DB { StorageTantivy::StorageTantivy(Context & context_, const TiCIScan & tici_scan_) - : IStorage() - , tici_scan(tici_scan_) + : tici_scan(tici_scan_) , context(context_) , log(Logger::get(context_.getDAGContext()->log ? context_.getDAGContext()->log->identifier() : "")) {} @@ -84,7 +84,35 @@ void StorageTantivy::read( tici_scan.getLimit(), context.getSettingsRef().read_tso, tici_scan.getMatchExpr(), - tici_scan.isCount())); + tici_scan.isCount(), + context.getTimezoneInfo())); + + executeCastAfterTiCIScan(exec_status, group_builder); +} + +void StorageTantivy::executeCastAfterTiCIScan( + PipelineExecutorContext & exec_status, + PipelineExecGroupBuilder & group_builder) +{ + // execute timezone cast or duration cast if needed for local tici scan + DAGExpressionAnalyzer analyzer{group_builder.getCurrentHeader(), context}; + ExpressionActionsChain chain; + std::vector may_need_add_cast_column; + for (size_t i = 0; i < tici_scan.getReturnColumns().size(); ++i) + may_need_add_cast_column.push_back(true); + if (analyzer.appendExtraCastsAfterTiCI(chain, may_need_add_cast_column, tici_scan)) + { + ExpressionActionsPtr extra_cast = chain.getLastActions(); + assert(extra_cast); + chain.finalize(); + chain.clear(); + for (size_t i = 0; i < group_builder.concurrency(); ++i) + { + auto & builder = group_builder.getCurBuilder(i); + builder.appendTransformOp( + std::make_unique(exec_status, log->identifier(), extra_cast)); + } + } } void StorageTantivy::splitRemoteReadAndLocalRead() diff --git a/dbms/src/Storages/StorageTantivy.h b/dbms/src/Storages/StorageTantivy.h index b2be6580d49..d817707b4e1 100644 --- a/dbms/src/Storages/StorageTantivy.h +++ b/dbms/src/Storages/StorageTantivy.h @@ -68,6 +68,8 @@ class StorageTantivy : public IStorage [[maybe_unused]] const Context & context, [[maybe_unused]] size_t max_block_size, [[maybe_unused]] unsigned num_streams) override; + + void executeCastAfterTiCIScan(PipelineExecutorContext & exec_status, PipelineExecGroupBuilder & group_builder); // Members will be transferred to DAGQueryBlockInterpreter after execute std::unique_ptr analyzer; diff --git a/dbms/src/Storages/Tantivy/TantivyInputStream.h b/dbms/src/Storages/Tantivy/TantivyInputStream.h index 2c56f3e0982..fc5cfbd7743 100644 --- a/dbms/src/Storages/Tantivy/TantivyInputStream.h +++ b/dbms/src/Storages/Tantivy/TantivyInputStream.h @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -31,6 +32,19 @@ namespace DB::TS { + +// convert literal value from timezone specified in cop request to UTC in-place +inline UInt64 convertPackedU64WithTimezone(UInt64 from_time, const TimezoneInfo & timezone_info) +{ + static const auto & time_zone_utc = DateLUT::instance("UTC"); + UInt64 result_time = from_time; + if (timezone_info.is_name_based) + convertTimeZone(from_time, result_time, *timezone_info.timezone, time_zone_utc); + else if (timezone_info.timezone_offset != 0) + convertTimeZoneByOffset(from_time, result_time, false, timezone_info.timezone_offset); + return result_time; +} + class TantivyInputStream : public IProfilingBlockInputStream { static constexpr auto NAME = "TantivyInputStream"; @@ -46,7 +60,8 @@ class TantivyInputStream : public IProfilingBlockInputStream UInt64 limit_, UInt64 read_ts_, google::protobuf::RepeatedPtrField match_expr_, - bool is_count) + bool is_count, + const TimezoneInfo & timezone_info_) : log(log_) , keyspace_id(keyspace_id_) , table_id(table_id_) @@ -56,6 +71,7 @@ class TantivyInputStream : public IProfilingBlockInputStream , limit(limit_) , read_ts(read_ts_) , is_count(is_count) + , timezone_info(timezone_info_) { FmtBuffer buf; buf.joinStr( @@ -179,6 +195,14 @@ class TantivyInputStream : public IProfilingBlockInputStream col->insert(Field(doc.fieldValues[idx].int_value)); } } + if (removeNullable(name_and_type.type)->isDateOrDateTime()) + { + for (auto & doc : documents) + { + auto t = static_cast(doc.fieldValues[idx].int_value); + col->insert(Field(t)); + } + } } return res; } @@ -196,6 +220,7 @@ class TantivyInputStream : public IProfilingBlockInputStream UInt64 read_ts; ::Expr match_expr; bool is_count; + const TimezoneInfo & timezone_info; size_t processed_shard = 0; @@ -248,6 +273,7 @@ class TantivyInputStream : public IProfilingBlockInputStream return {ret, cids}; } + std::tuple<::Expr, std::vector> tipbToTiCIExpr(const tipb::Expr & expr) { ::Expr ret; @@ -289,12 +315,6 @@ class TantivyInputStream : public IProfilingBlockInputStream case tipb::ScalarFuncSig::LEReal: case tipb::ScalarFuncSig::GTReal: case tipb::ScalarFuncSig::GEReal: - case tipb::ScalarFuncSig::EQTime: - case tipb::ScalarFuncSig::NETime: - case tipb::ScalarFuncSig::LTTime: - case tipb::ScalarFuncSig::LETime: - case tipb::ScalarFuncSig::GTTime: - case tipb::ScalarFuncSig::GETime: case tipb::ScalarFuncSig::EQDecimal: case tipb::ScalarFuncSig::NEDecimal: case tipb::ScalarFuncSig::LTDecimal: @@ -303,6 +323,33 @@ class TantivyInputStream : public IProfilingBlockInputStream case tipb::ScalarFuncSig::GEDecimal: ret.sig = expr.sig(); break; + case tipb::ScalarFuncSig::EQTime: + case tipb::ScalarFuncSig::NETime: + case tipb::ScalarFuncSig::LTTime: + case tipb::ScalarFuncSig::LETime: + case tipb::ScalarFuncSig::GTTime: + case tipb::ScalarFuncSig::GETime: + { + ret.sig = expr.sig(); + size_t col_idx = 0, val_idx = 1; + if (isColumnExpr(expr.children(1))) + std::swap(col_idx, val_idx); + if (expr.children(col_idx).field_type().tp() == TiDB::TypeTimestamp) + { + const auto & child_expr = expr.children(val_idx); + if (isLiteralExpr(child_expr)) + { + UInt64 val = decodeDAGUInt64(child_expr.val()); + val = convertPackedU64WithTimezone(val, timezone_info); + WriteBufferFromOwnString ss; + encodeDAGUInt64(val, ss); + ret.children[val_idx].val.clear(); + auto str = ss.releaseStr(); + std::copy(str.begin(), str.end(), std::back_inserter(ret.children[val_idx].val)); + } + } + break; + } default: throw std::runtime_error("Unsupported expression sig"); } @@ -352,4 +399,5 @@ class TantivyInputStream : public IProfilingBlockInputStream } }; + } // namespace DB::TS From 5aee2019216cb609b0d416420c012bfeec809ad8 Mon Sep 17 00:00:00 2001 From: Shenghui Wu <793703860@qq.com> Date: Wed, 26 Nov 2025 15:52:02 +0800 Subject: [PATCH 24/58] tici: supprot in expr push down to tici (#10564) (cherry picked from commit b66d0c85a8d6f14962094ed04956bfb06e73e674) --- .../src/Storages/Tantivy/TantivyInputStream.h | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/dbms/src/Storages/Tantivy/TantivyInputStream.h b/dbms/src/Storages/Tantivy/TantivyInputStream.h index fc5cfbd7743..70fe1a5e843 100644 --- a/dbms/src/Storages/Tantivy/TantivyInputStream.h +++ b/dbms/src/Storages/Tantivy/TantivyInputStream.h @@ -321,6 +321,10 @@ class TantivyInputStream : public IProfilingBlockInputStream case tipb::ScalarFuncSig::LEDecimal: case tipb::ScalarFuncSig::GTDecimal: case tipb::ScalarFuncSig::GEDecimal: + case tipb::ScalarFuncSig::InInt: + case tipb::ScalarFuncSig::InString: + case tipb::ScalarFuncSig::InReal: + case tipb::ScalarFuncSig::InDecimal: ret.sig = expr.sig(); break; case tipb::ScalarFuncSig::EQTime: @@ -350,6 +354,35 @@ class TantivyInputStream : public IProfilingBlockInputStream } break; } + case tipb::ScalarFuncSig::InTime: + { + ret.sig = expr.sig(); + if (expr.children(0).field_type().tp() == TiDB::TypeTimestamp) + { + for (int val_idx = 1; val_idx < expr.children_size(); ++val_idx) + { + const auto & child_expr = expr.children(val_idx); + if (isLiteralExpr(child_expr)) + { + UInt64 val = decodeDAGUInt64(child_expr.val()); + val = convertPackedU64WithTimezone(val, timezone_info); + WriteBufferFromOwnString ss; + encodeDAGUInt64(val, ss); + ret.children[val_idx].val.clear(); + auto str = ss.releaseStr(); + std::copy(str.begin(), str.end(), std::back_inserter(ret.children[val_idx].val)); + } + else + { + throw TiFlashException( + "InTime only support literal values", + Errors::Coprocessor::BadRequest); + } + } + } + break; + } + default: throw std::runtime_error("Unsupported expression sig"); } From 7c3c64bdef605969d3a86eeeb69126d0de5067d4 Mon Sep 17 00:00:00 2001 From: Calvin Neo Date: Fri, 28 Nov 2025 12:20:44 +0800 Subject: [PATCH 25/58] searchlib: fix compile under arm arch (#10571) Signed-off-by: Calvin Neo (cherry picked from commit d361a8e2e4fc64ce8ec33de144afba30062e4b08) --- contrib/tici-search-lib/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/tici-search-lib/CMakeLists.txt b/contrib/tici-search-lib/CMakeLists.txt index 74e8a6ce40c..413a1886d2c 100644 --- a/contrib/tici-search-lib/CMakeLists.txt +++ b/contrib/tici-search-lib/CMakeLists.txt @@ -24,6 +24,8 @@ target_include_directories(tici_search_lib_static INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/cxxbridge) add_library(tici_search_lib SHARED "${TiFlash_SOURCE_DIR}/libs/libclara-cmake/dummy.cpp") +target_compile_options(tici_search_lib PRIVATE -pthread) +target_link_options(tici_search_lib PRIVATE -pthread) target_link_libraries(tici_search_lib PRIVATE "$") if(APPLE) target_link_libraries(tici_search_lib PRIVATE From 043181d4750507110a43ff6202e2c6e7050ff1f6 Mon Sep 17 00:00:00 2001 From: jinhelin Date: Wed, 26 Nov 2025 21:40:08 +0800 Subject: [PATCH 26/58] tici: Support starting tici reader based on tici port and tiflash address (#10567) (cherry picked from commit bd2cd6d5368b5d45b9d31f494020f65088e49c73) --- dbms/src/Server/Server.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/dbms/src/Server/Server.cpp b/dbms/src/Server/Server.cpp index 985e9e5c09e..4355812b030 100644 --- a/dbms/src/Server/Server.cpp +++ b/dbms/src/Server/Server.cpp @@ -1025,12 +1025,14 @@ try auto schema_cache_size = config().getInt("schema_cache_size", 10000); global_context->initializeSharedBlockSchemas(schema_cache_size); - if (auto addr = config().getString("tici.reader_node.addr", ""); !addr.empty()) + auto tici_reader_addr = config().getString("tici.reader_node.addr", ""); + auto tici_reader_port = config().getInt("tici.reader_node.port", 0); + if (!tici_reader_addr.empty() || tici_reader_port > 0) { auto service_addr = config().getString("flash.service_addr"); auto pd_addr = config().getString("raft.pd_addr"); - start_reader_server(config_path, addr, service_addr, pd_addr); - LOG_INFO(log, "tici is enabled, reader_node addr: {}", addr); + start_reader_server(config_path, tici_reader_addr, service_addr, pd_addr); + LOG_INFO(log, "tici is enabled, reader_node addr: {}, port: {}", tici_reader_addr, tici_reader_port); } // Load remaining databases From c6f2228338c8944f847fa93db76b94bb3014d536 Mon Sep 17 00:00:00 2001 From: jinhelin Date: Tue, 2 Dec 2025 10:39:31 +0800 Subject: [PATCH 27/58] TiCI: Make starting reader server non-blocking (#10584) (cherry picked from commit a03c329ab9ff6546f2848fbe1569eb4fcba3d7f3) --- dbms/src/Server/Server.cpp | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/dbms/src/Server/Server.cpp b/dbms/src/Server/Server.cpp index 4355812b030..87ad5b7d6b0 100644 --- a/dbms/src/Server/Server.cpp +++ b/dbms/src/Server/Server.cpp @@ -1025,16 +1025,6 @@ try auto schema_cache_size = config().getInt("schema_cache_size", 10000); global_context->initializeSharedBlockSchemas(schema_cache_size); - auto tici_reader_addr = config().getString("tici.reader_node.addr", ""); - auto tici_reader_port = config().getInt("tici.reader_node.port", 0); - if (!tici_reader_addr.empty() || tici_reader_port > 0) - { - auto service_addr = config().getString("flash.service_addr"); - auto pd_addr = config().getString("raft.pd_addr"); - start_reader_server(config_path, tici_reader_addr, service_addr, pd_addr); - LOG_INFO(log, "tici is enabled, reader_node addr: {}, port: {}", tici_reader_addr, tici_reader_port); - } - // Load remaining databases loadMetadata(*global_context); LOG_DEBUG(log, "Load metadata done."); @@ -1261,6 +1251,18 @@ try proxy_machine.runKVStore(tmt_context); + auto tici_reader_addr = config().getString("tici.reader_node.addr", ""); + auto tici_reader_port = config().getInt("tici.reader_node.port", 0); + if (!tici_reader_addr.empty() || tici_reader_port > 0) + { + Stopwatch watch; + auto service_addr = config().getString("flash.service_addr"); + auto pd_addr = config().getString("raft.pd_addr"); + LOG_INFO(log, "TiCI starting, addr={}, port={}", tici_reader_addr, tici_reader_port); + start_reader_server(config_path, tici_reader_addr, service_addr, pd_addr); + LOG_INFO(log, "TiCI started, cost={}s", watch.elapsedSeconds()); + } + try { // Bind CPU affinity after all threads started. From e51d7641433ed9cd7704ad69cb7fca1cee025008 Mon Sep 17 00:00:00 2001 From: Shenghui Wu <793703860@qq.com> Date: Tue, 2 Dec 2025 14:40:44 +0800 Subject: [PATCH 28/58] tici: support for concurrent processing of shard requests to improve overall throughput (#10583) (cherry picked from commit 6ea4a8b3f06e649c0a41568a07124a3fae92baf8) --- dbms/src/Operators/TantivyReaderSourceOp.cpp | 61 ++-- dbms/src/Operators/TantivyReaderSourceOp.h | 18 +- dbms/src/Storages/StorageTantivy.cpp | 18 +- dbms/src/Storages/StorageTantivy.h | 2 +- .../src/Storages/Tantivy/TantivyInputStream.h | 226 +----------- dbms/src/Storages/Tantivy/TiCIReadTaskPool.h | 339 ++++++++++++++++++ 6 files changed, 398 insertions(+), 266 deletions(-) create mode 100644 dbms/src/Storages/Tantivy/TiCIReadTaskPool.h diff --git a/dbms/src/Operators/TantivyReaderSourceOp.cpp b/dbms/src/Operators/TantivyReaderSourceOp.cpp index 61e587ee02c..3073aeb5709 100644 --- a/dbms/src/Operators/TantivyReaderSourceOp.cpp +++ b/dbms/src/Operators/TantivyReaderSourceOp.cpp @@ -25,32 +25,12 @@ namespace DB TantivyReaderSourceOp::TantivyReaderSourceOp( PipelineExecutorContext & exec_context_, const String & req_id, - const UInt32 & keyspace_id, - const Int64 & table_id, - const Int64 & index_id, - const ShardInfoList & query_shard_infos, - const NamesAndTypes & return_columns, - const UInt64 & limit, - const UInt64 & read_ts, - const google::protobuf::RepeatedPtrField & expr, - bool is_count, - const TimezoneInfo & timezone_info) + const TS::TiCIReadTaskPoolPtr task_pool_, + const NamesAndTypes & return_columns) : SourceOp(exec_context_, req_id) + , task_pool(task_pool_) { setHeader(Block(return_columns)); - - input = std::make_shared( - log, - keyspace_id, - table_id, - index_id, - query_shard_infos, - return_columns, - limit, - read_ts, - expr, - is_count, - timezone_info); } String TantivyReaderSourceOp::getName() const @@ -78,23 +58,36 @@ Block TantivyReaderSourceOp::popFromBlockQueue() OperatorStatus TantivyReaderSourceOp::readImpl(Block & block) { - if (!block_queue.empty()) + if (unlikely(done)) { - block = popFromBlockQueue(); return OperatorStatus::HAS_OUTPUT; } - assert(block_queue.empty()); - for (;;) + while (true) { - auto tmp = input->readImpl(); - block_queue.push(tmp); - total_rows += tmp.rows(); - if (!tmp) - break; + while (!cur_stream) + { + auto task = task_pool->getNextTask(); + if (!task) + { + done = true; + return OperatorStatus::HAS_OUTPUT; + } + cur_stream = task_pool->buildInputStream(task); + } + Block res = cur_stream->read(); + if (res) + { + total_rows += res.rows(); + block.swap(res); + return OperatorStatus::HAS_OUTPUT; + } + else + { + cur_stream = {}; + continue; + } } - block = popFromBlockQueue(); - return OperatorStatus::HAS_OUTPUT; } } // namespace DB diff --git a/dbms/src/Operators/TantivyReaderSourceOp.h b/dbms/src/Operators/TantivyReaderSourceOp.h index 21574fe382d..bd704515a01 100644 --- a/dbms/src/Operators/TantivyReaderSourceOp.h +++ b/dbms/src/Operators/TantivyReaderSourceOp.h @@ -15,10 +15,12 @@ #pragma once #include +#include #include #include #include #include +#include #include #include @@ -31,16 +33,8 @@ class TantivyReaderSourceOp : public SourceOp TantivyReaderSourceOp( PipelineExecutorContext & exec_context_, const String & req_id, - const UInt32 & keyspace_id, - const Int64 & table_id, - const Int64 & index_id, - const ShardInfoList & query_shard_infos, - const NamesAndTypes & return_columns, - const UInt64 & limit, - const UInt64 & read_ts, - const google::protobuf::RepeatedPtrField & expr, - bool is_count, - const TimezoneInfo & timezone_info); + TS::TiCIReadTaskPoolPtr task_pool_, + const NamesAndTypes & return_columns); String getName() const override; @@ -60,8 +54,10 @@ class TantivyReaderSourceOp : public SourceOp UInt64 total_rows{}; IOProfileInfoPtr io_profile_info; + TS::TiCIReadTaskPoolPtr task_pool; + bool done = false; - std::shared_ptr input; + BlockInputStreamPtr cur_stream; Stopwatch watcher; }; diff --git a/dbms/src/Storages/StorageTantivy.cpp b/dbms/src/Storages/StorageTantivy.cpp index 47035a72b3e..724825cb677 100644 --- a/dbms/src/Storages/StorageTantivy.cpp +++ b/dbms/src/Storages/StorageTantivy.cpp @@ -68,14 +68,12 @@ void StorageTantivy::read( [[maybe_unused]] const SelectQueryInfo & info, const Context & context, [[maybe_unused]] size_t max_block_size, - [[maybe_unused]] unsigned num_streams) + unsigned num_streams) { auto return_columns = genNamesAndTypesForTiCI(tici_scan.getReturnColumns(), "column"); - // local read - group_builder.addConcurrency(std::make_unique( - exec_status, - log->identifier(), + auto tici_task_pool = std::make_shared( + log, tici_scan.getKeyspaceID(), tici_scan.getTableId(), tici_scan.getIndexId(), @@ -85,7 +83,15 @@ void StorageTantivy::read( context.getSettingsRef().read_tso, tici_scan.getMatchExpr(), tici_scan.isCount(), - context.getTimezoneInfo())); + context.getTimezoneInfo()); + + num_streams = std::min(num_streams, local_read.size()); + // local read + for (size_t i = 0; i < num_streams; ++i) + { + group_builder.addConcurrency( + std::make_unique(exec_status, log->identifier(), tici_task_pool, return_columns)); + } executeCastAfterTiCIScan(exec_status, group_builder); } diff --git a/dbms/src/Storages/StorageTantivy.h b/dbms/src/Storages/StorageTantivy.h index d817707b4e1..40174614d1a 100644 --- a/dbms/src/Storages/StorageTantivy.h +++ b/dbms/src/Storages/StorageTantivy.h @@ -67,7 +67,7 @@ class StorageTantivy : public IStorage [[maybe_unused]] const SelectQueryInfo & info, [[maybe_unused]] const Context & context, [[maybe_unused]] size_t max_block_size, - [[maybe_unused]] unsigned num_streams) override; + unsigned num_streams) override; void executeCastAfterTiCIScan(PipelineExecutorContext & exec_status, PipelineExecGroupBuilder & group_builder); // Members will be transferred to DAGQueryBlockInterpreter after execute diff --git a/dbms/src/Storages/Tantivy/TantivyInputStream.h b/dbms/src/Storages/Tantivy/TantivyInputStream.h index 70fe1a5e843..5dbd1e7277e 100644 --- a/dbms/src/Storages/Tantivy/TantivyInputStream.h +++ b/dbms/src/Storages/Tantivy/TantivyInputStream.h @@ -55,34 +55,23 @@ class TantivyInputStream : public IProfilingBlockInputStream UInt32 keyspace_id_, Int64 table_id_, Int64 index_id_, - ShardInfoList query_shard_infos_, + ShardInfo query_shard_info_, NamesAndTypes return_columns_, UInt64 limit_, UInt64 read_ts_, - google::protobuf::RepeatedPtrField match_expr_, - bool is_count, - const TimezoneInfo & timezone_info_) + ::Expr match_expr_, + bool is_count) : log(log_) , keyspace_id(keyspace_id_) , table_id(table_id_) , index_id(index_id_) - , query_shard_infos(query_shard_infos_) + , query_shard_info(query_shard_info_) , return_columns(return_columns_) , limit(limit_) , read_ts(read_ts_) + , match_expr(match_expr_) , is_count(is_count) - , timezone_info(timezone_info_) - { - FmtBuffer buf; - buf.joinStr( - return_columns.begin(), - return_columns.end(), - [](const auto & nt, FmtBuffer & fb) { fb.fmtAppend("{}:{}", nt.name, nt.type->getName()); }, - ", "); - auto [expr, cids] = tipbToTiCIExpr(match_expr_); - match_expr = std::move(expr); - LOG_DEBUG(log, "columns: [{}], match columns: {}", buf.toString(), cids); - } + {} String getName() const override { return NAME; } @@ -94,25 +83,17 @@ class TantivyInputStream : public IProfilingBlockInputStream { return {}; } - if (processed_shard >= query_shard_infos.size()) - { - LOG_DEBUG(log, "empty shard info, returning empty block"); - done = true; - return {}; - } - - Block ret = readFromS3(processed_shard, is_count); - processed_shard++; - done = processed_shard >= query_shard_infos.size(); + Block ret = readFromS3(is_count); + done = true; return ret; } protected: - Block readFromS3(size_t processing, bool is_count) + Block readFromS3(bool is_count) { auto return_fields = getFields(return_columns); - auto & shard_info = query_shard_infos[processing]; - LOG_DEBUG(log, "Processing shard: {}, shard info: {}", processing, shard_info.toString()); + auto shard_info = query_shard_info; + LOG_DEBUG(log, "shard info: {}", shard_info.toString()); auto key_ranges = getKeyRanges(shard_info.key_ranges); auto search_param = SearchParam{static_cast(limit)}; @@ -214,15 +195,12 @@ class TantivyInputStream : public IProfilingBlockInputStream UInt32 keyspace_id; Int64 table_id; Int64 index_id; - ShardInfoList query_shard_infos; + ShardInfo query_shard_info; NamesAndTypes return_columns; UInt64 limit; UInt64 read_ts; ::Expr match_expr; bool is_count; - const TimezoneInfo & timezone_info; - - size_t processed_shard = 0; static rust::Vec getFields(NamesAndTypes & columns) { @@ -252,185 +230,5 @@ class TantivyInputStream : public IProfilingBlockInputStream } return res; } - - std::tuple<::Expr, std::vector> tipbToTiCIExpr(google::protobuf::RepeatedPtrField exprs) - { - if (exprs.empty()) - { - throw std::runtime_error("Empty match expression"); - } - auto [ret, cids] = tipbToTiCIExpr(exprs[0]); - for (auto i = 1; i < exprs.size(); ++i) - { - auto [child_expr, child_cids] = tipbToTiCIExpr(exprs[i]); - ret = { - .tp = tipb::ExprType::ScalarFunc, - .children = {ret, child_expr}, - .sig = tipb::ScalarFuncSig::LogicalAnd, - }; - cids.insert(cids.end(), child_cids.begin(), child_cids.end()); - } - return {ret, cids}; - } - - - std::tuple<::Expr, std::vector> tipbToTiCIExpr(const tipb::Expr & expr) - { - ::Expr ret; - switch (expr.tp()) - { - case tipb::ExprType::ScalarFunc: - { - std::vector children_cids; - ret.tp = tipb::ExprType::ScalarFunc; - for (const auto & child : expr.children()) - { - auto [child_expr, child_cids] = tipbToTiCIExpr(child); - ret.children.push_back(child_expr); - children_cids.insert(children_cids.end(), child_cids.begin(), child_cids.end()); - } - switch (expr.sig()) - { - case tipb::ScalarFuncSig::FTSMatchWord: - case tipb::ScalarFuncSig::FTSMatchPrefix: - case tipb::ScalarFuncSig::LogicalAnd: - case tipb::ScalarFuncSig::LogicalOr: - case tipb::ScalarFuncSig::UnaryNotInt: - case tipb::ScalarFuncSig::UnaryNotReal: - case tipb::ScalarFuncSig::EQInt: - case tipb::ScalarFuncSig::NEInt: - case tipb::ScalarFuncSig::LTInt: - case tipb::ScalarFuncSig::LEInt: - case tipb::ScalarFuncSig::GTInt: - case tipb::ScalarFuncSig::GEInt: - case tipb::ScalarFuncSig::EQString: - case tipb::ScalarFuncSig::NEString: - case tipb::ScalarFuncSig::LTString: - case tipb::ScalarFuncSig::LEString: - case tipb::ScalarFuncSig::GTString: - case tipb::ScalarFuncSig::GEString: - case tipb::ScalarFuncSig::EQReal: - case tipb::ScalarFuncSig::NEReal: - case tipb::ScalarFuncSig::LTReal: - case tipb::ScalarFuncSig::LEReal: - case tipb::ScalarFuncSig::GTReal: - case tipb::ScalarFuncSig::GEReal: - case tipb::ScalarFuncSig::EQDecimal: - case tipb::ScalarFuncSig::NEDecimal: - case tipb::ScalarFuncSig::LTDecimal: - case tipb::ScalarFuncSig::LEDecimal: - case tipb::ScalarFuncSig::GTDecimal: - case tipb::ScalarFuncSig::GEDecimal: - case tipb::ScalarFuncSig::InInt: - case tipb::ScalarFuncSig::InString: - case tipb::ScalarFuncSig::InReal: - case tipb::ScalarFuncSig::InDecimal: - ret.sig = expr.sig(); - break; - case tipb::ScalarFuncSig::EQTime: - case tipb::ScalarFuncSig::NETime: - case tipb::ScalarFuncSig::LTTime: - case tipb::ScalarFuncSig::LETime: - case tipb::ScalarFuncSig::GTTime: - case tipb::ScalarFuncSig::GETime: - { - ret.sig = expr.sig(); - size_t col_idx = 0, val_idx = 1; - if (isColumnExpr(expr.children(1))) - std::swap(col_idx, val_idx); - if (expr.children(col_idx).field_type().tp() == TiDB::TypeTimestamp) - { - const auto & child_expr = expr.children(val_idx); - if (isLiteralExpr(child_expr)) - { - UInt64 val = decodeDAGUInt64(child_expr.val()); - val = convertPackedU64WithTimezone(val, timezone_info); - WriteBufferFromOwnString ss; - encodeDAGUInt64(val, ss); - ret.children[val_idx].val.clear(); - auto str = ss.releaseStr(); - std::copy(str.begin(), str.end(), std::back_inserter(ret.children[val_idx].val)); - } - } - break; - } - case tipb::ScalarFuncSig::InTime: - { - ret.sig = expr.sig(); - if (expr.children(0).field_type().tp() == TiDB::TypeTimestamp) - { - for (int val_idx = 1; val_idx < expr.children_size(); ++val_idx) - { - const auto & child_expr = expr.children(val_idx); - if (isLiteralExpr(child_expr)) - { - UInt64 val = decodeDAGUInt64(child_expr.val()); - val = convertPackedU64WithTimezone(val, timezone_info); - WriteBufferFromOwnString ss; - encodeDAGUInt64(val, ss); - ret.children[val_idx].val.clear(); - auto str = ss.releaseStr(); - std::copy(str.begin(), str.end(), std::back_inserter(ret.children[val_idx].val)); - } - else - { - throw TiFlashException( - "InTime only support literal values", - Errors::Coprocessor::BadRequest); - } - } - } - break; - } - - default: - throw std::runtime_error("Unsupported expression sig"); - } - - return {ret, children_cids}; - } - case tipb::ExprType::ColumnRef: - { - ret.tp = expr.tp(); - auto id = decodeDAGInt64(expr.val()); - auto str = fmt::format("column_{}", id); - std::copy(str.begin(), str.end(), std::back_inserter(ret.val)); - return {ret, {id}}; - } - case tipb::ExprType::String: - case tipb::ExprType::Int64: - case tipb::ExprType::Uint64: - case tipb::ExprType::Float32: - case tipb::ExprType::Float64: - case tipb::ExprType::MysqlTime: - { - ret.tp = expr.tp(); - std::copy(expr.val().begin(), expr.val().end(), std::back_inserter(ret.val)); - return {ret, {}}; - } - case tipb::ExprType::MysqlDecimal: - { - ret.tp = expr.tp(); - auto field = decodeDAGDecimal(expr.val()); - String str; - if (field.getType() == Field::Types::Decimal32) - str = field.get>().toString(); - else if (field.getType() == Field::Types::Decimal64) - str = field.get>().toString(); - else if (field.getType() == Field::Types::Decimal128) - str = field.get>().toString(); - else if (field.getType() == Field::Types::Decimal256) - str = field.get>().toString(); - else - throw TiFlashException("Not decimal literal" + expr.DebugString(), Errors::Coprocessor::BadRequest); - std::copy(str.begin(), str.end(), std::back_inserter(ret.val)); - return {ret, {}}; - } - default: - throw std::runtime_error("Unsupported expression type"); - } - } }; - - } // namespace DB::TS diff --git a/dbms/src/Storages/Tantivy/TiCIReadTaskPool.h b/dbms/src/Storages/Tantivy/TiCIReadTaskPool.h new file mode 100644 index 00000000000..edb25f5f323 --- /dev/null +++ b/dbms/src/Storages/Tantivy/TiCIReadTaskPool.h @@ -0,0 +1,339 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include + +namespace DB::TS +{ +struct TiCIReadTask +{ +public: + explicit TiCIReadTask(const ShardInfo & shard_info_) + : shard_info(shard_info_) + {} + + void initInputStream( + LoggerPtr log_, + UInt32 keyspace_id_, + Int64 table_id_, + Int64 index_id_, + ShardInfo query_shard_info_, + NamesAndTypes return_columns_, + UInt64 limit_, + UInt64 read_ts_, + ::Expr match_expr_, + bool is_count) + { + input_stream = std::make_shared( + log_, + keyspace_id_, + table_id_, + index_id_, + query_shard_info_, + return_columns_, + limit_, + read_ts_, + match_expr_, + is_count); + } + + bool isInitialized() const { return input_stream != nullptr; } + + BlockInputStreamPtr getInputStream() const + { + RUNTIME_CHECK(input_stream != nullptr); + return input_stream; + } + ShardInfo getShardInfo() const { return shard_info; } + +private: + ShardInfo shard_info; + BlockInputStreamPtr input_stream; +}; + +using TiCIReadTaskPtr = std::shared_ptr; + +struct TiCIReadTaskPool +{ +public: + using TiCIReadTasks = std::vector>; + + TiCIReadTaskPool( + LoggerPtr log_, + UInt32 keyspace_id_, + Int64 table_id_, + Int64 index_id_, + const ShardInfoList & shard_infos, + NamesAndTypes return_columns_, + UInt64 limit_, + UInt64 read_ts_, + google::protobuf::RepeatedPtrField match_expr_, + bool is_count, + const TimezoneInfo & timezone_info_) + : log(log_) + , keyspace_id(keyspace_id_) + , table_id(table_id_) + , index_id(index_id_) + , return_columns(return_columns_) + , limit(limit_) + , read_ts(read_ts_) + , is_count(is_count) + { + for (const auto & shard_info : shard_infos) + { + tasks.emplace_back(std::make_shared(shard_info)); + } + FmtBuffer buf; + buf.joinStr( + return_columns.begin(), + return_columns.end(), + [](const auto & nt, FmtBuffer & fb) { fb.fmtAppend("{}:{}", nt.name, nt.type->getName()); }, + ", "); + auto [expr, cids] = tipbToTiCIExpr(match_expr_, timezone_info_); + match_expr = std::move(expr); + LOG_DEBUG(log, "columns: [{}], match columns: {}", buf.toString(), cids); + } + + TiCIReadTaskPtr getNextTask() + { + std::lock_guard lock(mutex); + if (tasks.empty()) + return nullptr; + TiCIReadTaskPtr task = tasks.back(); + tasks.pop_back(); + return task; + } + + BlockInputStreamPtr buildInputStream(TiCIReadTaskPtr & task) + { + RUNTIME_CHECK(task != nullptr); + if (!task->isInitialized()) + { + task->initInputStream( + log, + keyspace_id, + table_id, + index_id, + task->getShardInfo(), + return_columns, + limit, + read_ts, + match_expr, + is_count); + } + return task->getInputStream(); + } + +private: + mutable std::mutex mutex; + TiCIReadTasks tasks; + + LoggerPtr log; + UInt32 keyspace_id; + Int64 table_id; + Int64 index_id; + NamesAndTypes return_columns; + UInt64 limit; + UInt64 read_ts; + ::Expr match_expr; + bool is_count; + + static std::tuple<::Expr, std::vector> tipbToTiCIExpr( + const tipb::Expr & expr, + const TimezoneInfo & timezone_info) + { + ::Expr ret; + switch (expr.tp()) + { + case tipb::ExprType::ScalarFunc: + { + std::vector children_cids; + ret.tp = tipb::ExprType::ScalarFunc; + for (const auto & child : expr.children()) + { + auto [child_expr, child_cids] = tipbToTiCIExpr(child, timezone_info); + ret.children.push_back(child_expr); + children_cids.insert(children_cids.end(), child_cids.begin(), child_cids.end()); + } + switch (expr.sig()) + { + case tipb::ScalarFuncSig::FTSMatchWord: + case tipb::ScalarFuncSig::FTSMatchPrefix: + case tipb::ScalarFuncSig::LogicalAnd: + case tipb::ScalarFuncSig::LogicalOr: + case tipb::ScalarFuncSig::UnaryNotInt: + case tipb::ScalarFuncSig::UnaryNotReal: + case tipb::ScalarFuncSig::EQInt: + case tipb::ScalarFuncSig::NEInt: + case tipb::ScalarFuncSig::LTInt: + case tipb::ScalarFuncSig::LEInt: + case tipb::ScalarFuncSig::GTInt: + case tipb::ScalarFuncSig::GEInt: + case tipb::ScalarFuncSig::EQString: + case tipb::ScalarFuncSig::NEString: + case tipb::ScalarFuncSig::LTString: + case tipb::ScalarFuncSig::LEString: + case tipb::ScalarFuncSig::GTString: + case tipb::ScalarFuncSig::GEString: + case tipb::ScalarFuncSig::EQReal: + case tipb::ScalarFuncSig::NEReal: + case tipb::ScalarFuncSig::LTReal: + case tipb::ScalarFuncSig::LEReal: + case tipb::ScalarFuncSig::GTReal: + case tipb::ScalarFuncSig::GEReal: + case tipb::ScalarFuncSig::EQDecimal: + case tipb::ScalarFuncSig::NEDecimal: + case tipb::ScalarFuncSig::LTDecimal: + case tipb::ScalarFuncSig::LEDecimal: + case tipb::ScalarFuncSig::GTDecimal: + case tipb::ScalarFuncSig::GEDecimal: + case tipb::ScalarFuncSig::InInt: + case tipb::ScalarFuncSig::InString: + case tipb::ScalarFuncSig::InReal: + case tipb::ScalarFuncSig::InDecimal: + ret.sig = expr.sig(); + break; + case tipb::ScalarFuncSig::EQTime: + case tipb::ScalarFuncSig::NETime: + case tipb::ScalarFuncSig::LTTime: + case tipb::ScalarFuncSig::LETime: + case tipb::ScalarFuncSig::GTTime: + case tipb::ScalarFuncSig::GETime: + { + ret.sig = expr.sig(); + size_t col_idx = 0, val_idx = 1; + if (isColumnExpr(expr.children(1))) + std::swap(col_idx, val_idx); + if (expr.children(col_idx).field_type().tp() == TiDB::TypeTimestamp) + { + const auto & child_expr = expr.children(val_idx); + if (isLiteralExpr(child_expr)) + { + UInt64 val = decodeDAGUInt64(child_expr.val()); + val = convertPackedU64WithTimezone(val, timezone_info); + WriteBufferFromOwnString ss; + encodeDAGUInt64(val, ss); + ret.children[val_idx].val.clear(); + auto str = ss.releaseStr(); + std::copy(str.begin(), str.end(), std::back_inserter(ret.children[val_idx].val)); + } + } + break; + } + case tipb::ScalarFuncSig::InTime: + { + ret.sig = expr.sig(); + if (expr.children(0).field_type().tp() == TiDB::TypeTimestamp) + { + for (int val_idx = 1; val_idx < expr.children_size(); ++val_idx) + { + const auto & child_expr = expr.children(val_idx); + if (isLiteralExpr(child_expr)) + { + UInt64 val = decodeDAGUInt64(child_expr.val()); + val = convertPackedU64WithTimezone(val, timezone_info); + WriteBufferFromOwnString ss; + encodeDAGUInt64(val, ss); + ret.children[val_idx].val.clear(); + auto str = ss.releaseStr(); + std::copy(str.begin(), str.end(), std::back_inserter(ret.children[val_idx].val)); + } + else + { + throw TiFlashException( + "InTime only support literal values", + Errors::Coprocessor::BadRequest); + } + } + } + break; + } + default: + throw std::runtime_error("Unsupported expression sig"); + } + + return {ret, children_cids}; + } + case tipb::ExprType::ColumnRef: + { + ret.tp = expr.tp(); + auto id = decodeDAGInt64(expr.val()); + auto str = fmt::format("column_{}", id); + std::copy(str.begin(), str.end(), std::back_inserter(ret.val)); + return {ret, {id}}; + } + case tipb::ExprType::String: + case tipb::ExprType::Int64: + case tipb::ExprType::Uint64: + case tipb::ExprType::Float32: + case tipb::ExprType::Float64: + case tipb::ExprType::MysqlTime: + { + ret.tp = expr.tp(); + std::copy(expr.val().begin(), expr.val().end(), std::back_inserter(ret.val)); + return {ret, {}}; + } + case tipb::ExprType::MysqlDecimal: + { + ret.tp = expr.tp(); + auto field = decodeDAGDecimal(expr.val()); + String str; + if (field.getType() == Field::Types::Decimal32) + str = field.get>().toString(); + else if (field.getType() == Field::Types::Decimal64) + str = field.get>().toString(); + else if (field.getType() == Field::Types::Decimal128) + str = field.get>().toString(); + else if (field.getType() == Field::Types::Decimal256) + str = field.get>().toString(); + else + throw TiFlashException("Not decimal literal" + expr.DebugString(), Errors::Coprocessor::BadRequest); + std::copy(str.begin(), str.end(), std::back_inserter(ret.val)); + return {ret, {}}; + } + default: + throw std::runtime_error("Unsupported expression type"); + } + } + + static std::tuple<::Expr, std::vector> tipbToTiCIExpr( + google::protobuf::RepeatedPtrField exprs, + const TimezoneInfo & tz) + { + if (exprs.empty()) + { + throw std::runtime_error("Empty match expression"); + } + auto [ret, cids] = tipbToTiCIExpr(exprs[0], tz); + for (auto i = 1; i < exprs.size(); ++i) + { + auto [child_expr, child_cids] = tipbToTiCIExpr(exprs[i], tz); + ret = { + .tp = tipb::ExprType::ScalarFunc, + .children = {ret, child_expr}, + .sig = tipb::ScalarFuncSig::LogicalAnd, + }; + cids.insert(cids.end(), child_cids.begin(), child_cids.end()); + } + return {ret, cids}; + } +}; + +using TiCIReadTaskPoolPtr = std::shared_ptr; + +} // namespace DB::TS From 503cea8ef616388cb4a4204b2d1cf27455f7bdb9 Mon Sep 17 00:00:00 2001 From: Shenghui Wu <793703860@qq.com> Date: Wed, 3 Dec 2025 19:12:32 +0800 Subject: [PATCH 29/58] tici: Fix error Check !cur_group.empty() failed (#10589) (cherry picked from commit 02878ec9c33b1b0edfe9f599d853321a5a89349c) --- dbms/src/Storages/StorageTantivy.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dbms/src/Storages/StorageTantivy.cpp b/dbms/src/Storages/StorageTantivy.cpp index 724825cb677..b7051731750 100644 --- a/dbms/src/Storages/StorageTantivy.cpp +++ b/dbms/src/Storages/StorageTantivy.cpp @@ -85,7 +85,7 @@ void StorageTantivy::read( tici_scan.isCount(), context.getTimezoneInfo()); - num_streams = std::min(num_streams, local_read.size()); + num_streams = std::max(1, std::min(num_streams, local_read.size())); // local read for (size_t i = 0; i < num_streams; ++i) { @@ -100,6 +100,8 @@ void StorageTantivy::executeCastAfterTiCIScan( PipelineExecutorContext & exec_status, PipelineExecGroupBuilder & group_builder) { + if (group_builder.concurrency() == 0) + return; // execute timezone cast or duration cast if needed for local tici scan DAGExpressionAnalyzer analyzer{group_builder.getCurrentHeader(), context}; ExpressionActionsChain chain; From adbba63bb47621b461d61a6456e1ff453322bc7d Mon Sep 17 00:00:00 2001 From: ChangRui-Ryan Date: Mon, 15 Dec 2025 14:41:58 +0800 Subject: [PATCH 30/58] tici: optimize remote read for count (#10590) (cherry picked from commit 4593a5a05410b70e386d257e3a77706c89087486) --- dbms/src/Flash/Coprocessor/RemoteRequest.cpp | 52 ++++++++++++++----- dbms/src/Flash/Coprocessor/TiCIScan.h | 9 ++++ dbms/src/Flash/Planner/PhysicalPlanNode.h | 2 + dbms/src/Flash/Planner/Plans/PhysicalBinary.h | 4 +- dbms/src/Flash/Planner/Plans/PhysicalLeaf.h | 6 ++- .../Flash/Planner/Plans/PhysicalTiCIScan.cpp | 3 +- .../Flash/Planner/Plans/PhysicalTiCIScan.h | 11 +++- dbms/src/Flash/Planner/Plans/PhysicalUnary.h | 8 ++- dbms/src/Flash/Planner/optimize.cpp | 11 +++- dbms/src/Storages/StorageTantivy.cpp | 15 ++++-- .../src/Storages/Tantivy/TantivyInputStream.h | 18 ++----- 11 files changed, 101 insertions(+), 38 deletions(-) diff --git a/dbms/src/Flash/Coprocessor/RemoteRequest.cpp b/dbms/src/Flash/Coprocessor/RemoteRequest.cpp index 3840fadb7c3..f634463ea73 100644 --- a/dbms/src/Flash/Coprocessor/RemoteRequest.cpp +++ b/dbms/src/Flash/Coprocessor/RemoteRequest.cpp @@ -118,20 +118,48 @@ RemoteRequest RemoteRequest::build( DAGSchema schema; tipb::DAGRequest dag_req; { - tipb::Executor * ts_exec = dag_req.mutable_root_executor(); - ts_exec->set_tp(tipb::ExecType::TypeIndexScan); - ts_exec->set_executor_id(tici_scan.getTiCIScan()->executor_id()); - auto * mutable_tici_scan = ts_exec->mutable_idx_scan(); - tici_scan.constructTiCIScanForRemoteRead(mutable_tici_scan); - - const auto & return_columns = tici_scan.getReturnColumns(); - auto names_and_types = genNamesAndTypesForTiCI(return_columns, "column"); - for (size_t i = 0; i < return_columns.size(); ++i) + tipb::Executor * tici_scan_exec; + NamesAndTypes names_and_types; + if (tici_scan.isCount()) { - const auto & col = return_columns[i]; - schema.emplace_back(std::make_pair(names_and_types[i].name, col)); - dag_req.add_output_offsets(i); + auto * root_exec = dag_req.mutable_root_executor(); + root_exec->set_tp(tipb::ExecType::TypeAggregation); + root_exec->set_executor_id(tici_scan.getCountAggExecutorId()); + auto * new_aggregation = root_exec->mutable_aggregation(); + tipb::Expr * agg_func = new_aggregation->add_agg_func(); + agg_func->set_tp(tipb::ExprType::Count); + auto * ft = agg_func->mutable_field_type(); + // "count" always returns a NOT NULL INT64 column + ft->set_tp(TiDB::TypeLongLong); + ft->set_flag(TiDB::ColumnFlagNotNull); + tici_scan_exec = new_aggregation->mutable_child(); + + names_and_types = tici_scan.getNamesAndTypes(); + TiDB::ColumnInfo ci; + // "count" always returns a NOT NULL INT64 column + ci.tp = TiDB::TypeLongLong; + ci.setNotNullFlag(); + schema.emplace_back(std::make_pair(names_and_types[0].name, std::move(ci))); + dag_req.add_output_offsets(0); + } + else + { + const auto & return_columns = tici_scan.getReturnColumns(); + tici_scan_exec = dag_req.mutable_root_executor(); + names_and_types = genNamesAndTypesForTiCI(return_columns, "column"); + for (size_t i = 0; i < return_columns.size(); ++i) + { + const auto & col = return_columns[i]; + schema.emplace_back(std::make_pair(names_and_types[i].name, col)); + dag_req.add_output_offsets(i); + } } + + tici_scan_exec->set_tp(tipb::ExecType::TypeIndexScan); + tici_scan_exec->set_executor_id(tici_scan.getTiCIScan()->executor_id()); + auto * mutable_tici_scan = tici_scan_exec->mutable_idx_scan(); + tici_scan.constructTiCIScanForRemoteRead(mutable_tici_scan); + dag_req.set_encode_type(tipb::EncodeType::TypeCHBlock); dag_req.set_force_encode_type(true); } diff --git a/dbms/src/Flash/Coprocessor/TiCIScan.h b/dbms/src/Flash/Coprocessor/TiCIScan.h index dd7f9574d43..5f33fbb3cb6 100644 --- a/dbms/src/Flash/Coprocessor/TiCIScan.h +++ b/dbms/src/Flash/Coprocessor/TiCIScan.h @@ -27,7 +27,10 @@ class TiCIScan TiCIScan(const tipb::Executor * tici_scan_, const String & executor_id_, const DAGContext & dag_context); explicit TiCIScan(const tipb::Executor * tici_scan_); + // Do not use getReturnColumns when is_count_agg is true const TiDB::ColumnInfos & getReturnColumns() const { return return_columns; } + void setNamesAndTypes(const NamesAndTypes & new_names_and_types) { names_and_types = new_names_and_types; } + const NamesAndTypes getNamesAndTypes() const { return names_and_types; } const TableShardInfos & getShardInfos() const { return shard_infos; } const uint & getKeyspaceID() const { return keyspace_id; } const int & getTableId() const { return table_id; } @@ -46,6 +49,10 @@ class TiCIScan void setIsCountAgg(bool v) { is_count_agg = v; } + void setCountAggExecutorId(const String & v) { count_agg_executor_id = v; } + + const String & getCountAggExecutorId() const { return count_agg_executor_id; } + private: const tipb::Executor * tici_scan; [[maybe_unused]] String executor_id; @@ -53,9 +60,11 @@ class TiCIScan const int table_id; const int index_id; TiDB::ColumnInfos return_columns; + NamesAndTypes names_and_types; [[maybe_unused]] tipb::FTSQueryType query_type; const TableShardInfos shard_infos; const int limit; bool is_count_agg = false; + String count_agg_executor_id; }; } // namespace DB diff --git a/dbms/src/Flash/Planner/PhysicalPlanNode.h b/dbms/src/Flash/Planner/PhysicalPlanNode.h index ea33264eef0..e6f7ebe7396 100644 --- a/dbms/src/Flash/Planner/PhysicalPlanNode.h +++ b/dbms/src/Flash/Planner/PhysicalPlanNode.h @@ -57,6 +57,8 @@ class PhysicalPlanNode : public std::enable_shared_from_this virtual PhysicalPlanNodePtr children(size_t /*i*/) const = 0; + virtual void setChild(size_t i, PhysicalPlanNodePtr new_child) = 0; + const PlanType & tp() const { return type; } const String & execId() const { return executor_id; } diff --git a/dbms/src/Flash/Planner/Plans/PhysicalBinary.h b/dbms/src/Flash/Planner/Plans/PhysicalBinary.h index 6a78cfa6660..1886a3b39d1 100644 --- a/dbms/src/Flash/Planner/Plans/PhysicalBinary.h +++ b/dbms/src/Flash/Planner/Plans/PhysicalBinary.h @@ -48,7 +48,9 @@ class PhysicalBinary : public PhysicalPlanNode return i == 0 ? left : right; } - size_t childrenSize() const override { return 2; }; + void setChild(size_t, PhysicalPlanNodePtr) override { throw Exception("Unsupport"); } + + size_t childrenSize() const override { return 2; } protected: PhysicalPlanNodePtr left; diff --git a/dbms/src/Flash/Planner/Plans/PhysicalLeaf.h b/dbms/src/Flash/Planner/Plans/PhysicalLeaf.h index 549953930d2..ade58afbdcd 100644 --- a/dbms/src/Flash/Planner/Plans/PhysicalLeaf.h +++ b/dbms/src/Flash/Planner/Plans/PhysicalLeaf.h @@ -39,6 +39,8 @@ class PhysicalLeaf : public PhysicalPlanNode throw TiFlashException("the children size of PhysicalLeaf is zero", Errors::Planner::Internal); } - size_t childrenSize() const override { return 0; }; + void setChild(size_t, PhysicalPlanNodePtr) override { throw Exception("Unsupport"); } + + size_t childrenSize() const override { return 0; } }; -} // namespace DB \ No newline at end of file +} // namespace DB diff --git a/dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.cpp b/dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.cpp index 125997e137a..f00be52f077 100644 --- a/dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.cpp +++ b/dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.cpp @@ -104,7 +104,8 @@ void PhysicalTiCIScan::buildPipeline( // For building PipelineExec in compile time. StorageTantivyIterpreter storage_interpreter(context, tici_scan, context.getMaxStreams()); storage_interpreter.execute(exec_context, pipeline_exec_builder); - buildProjection(exec_context, pipeline_exec_builder); + if (!tici_scan.isCount()) + buildProjection(exec_context, pipeline_exec_builder); PhysicalPlanNode::buildPipeline(builder, context, exec_context); } diff --git a/dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.h b/dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.h index 6842915958f..10d161611cc 100644 --- a/dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.h +++ b/dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -34,8 +35,6 @@ class PhysicalTiCIScan : public PhysicalLeaf const TiCIScan & tici_scan_, const Block & sample_block_); - void setIsCountAgg(bool v) { tici_scan.setIsCountAgg(v); } - void finalizeImpl(const Names & parent_require) override; const Block & getSampleBlock() const override; @@ -48,6 +47,14 @@ class PhysicalTiCIScan : public PhysicalLeaf void buildPipeline(PipelineBuilder & builder, Context & context, PipelineExecutorContext & exec_context) override; + void setCountAgg(std::shared_ptr agg) + { + tici_scan.setIsCountAgg(true); + schema = agg->getSchema(); + tici_scan.setNamesAndTypes(agg->getSchema()); + tici_scan.setCountAggExecutorId(agg->execId()); + } + private: void buildPipelineExecGroupImpl( PipelineExecutorContext & /*exec_status*/, diff --git a/dbms/src/Flash/Planner/Plans/PhysicalUnary.h b/dbms/src/Flash/Planner/Plans/PhysicalUnary.h index fada29a645c..6695e4e947f 100644 --- a/dbms/src/Flash/Planner/Plans/PhysicalUnary.h +++ b/dbms/src/Flash/Planner/Plans/PhysicalUnary.h @@ -46,7 +46,13 @@ class PhysicalUnary : public PhysicalPlanNode return child; } - size_t childrenSize() const override { return 1; }; + void setChild(size_t i, PhysicalPlanNodePtr new_child) override + { + RUNTIME_CHECK_MSG(i == 0, "child_index({}) shouldn't >= childrenSize({})", i, childrenSize()); + child = std::move(new_child); + } + + size_t childrenSize() const override { return 1; } protected: PhysicalPlanNodePtr child; diff --git a/dbms/src/Flash/Planner/optimize.cpp b/dbms/src/Flash/Planner/optimize.cpp index 56bd50088b1..8978f7d69e7 100644 --- a/dbms/src/Flash/Planner/optimize.cpp +++ b/dbms/src/Flash/Planner/optimize.cpp @@ -54,7 +54,9 @@ class TiCICountAggOptimizeRule : public Rule if (child->tp() == PlanType::TiCiScan) { auto tici_scan = std::static_pointer_cast(child); - tici_scan->setIsCountAgg(true); + tici_scan->setCountAgg(agg); + // return tici_scan directly to replace agg + return tici_scan; } } } @@ -62,7 +64,12 @@ class TiCICountAggOptimizeRule : public Rule { for (size_t i = 0; i < plan->childrenSize(); ++i) { - apply(context, plan->children(i), logger); + auto new_child = apply(context, plan->children(i), logger); + if (new_child != plan->children(i)) + { + // replace child with the PhysicalPlanNodePtr returned by apply + plan->setChild(i, new_child); + } } } diff --git a/dbms/src/Storages/StorageTantivy.cpp b/dbms/src/Storages/StorageTantivy.cpp index b7051731750..7437dfb8602 100644 --- a/dbms/src/Storages/StorageTantivy.cpp +++ b/dbms/src/Storages/StorageTantivy.cpp @@ -70,7 +70,16 @@ void StorageTantivy::read( [[maybe_unused]] size_t max_block_size, unsigned num_streams) { - auto return_columns = genNamesAndTypesForTiCI(tici_scan.getReturnColumns(), "column"); + NamesAndTypes return_columns; + if (tici_scan.isCount()) + { + RUNTIME_CHECK_MSG(tici_scan.getNamesAndTypes().size() == 1, "count search should return one column"); + return_columns = tici_scan.getNamesAndTypes(); + } + else + { + return_columns = genNamesAndTypesForTiCI(tici_scan.getReturnColumns(), "column"); + } auto tici_task_pool = std::make_shared( log, @@ -92,8 +101,8 @@ void StorageTantivy::read( group_builder.addConcurrency( std::make_unique(exec_status, log->identifier(), tici_task_pool, return_columns)); } - - executeCastAfterTiCIScan(exec_status, group_builder); + if (!tici_scan.isCount()) + executeCastAfterTiCIScan(exec_status, group_builder); } void StorageTantivy::executeCastAfterTiCIScan( diff --git a/dbms/src/Storages/Tantivy/TantivyInputStream.h b/dbms/src/Storages/Tantivy/TantivyInputStream.h index 5dbd1e7277e..97f94cf7fca 100644 --- a/dbms/src/Storages/Tantivy/TantivyInputStream.h +++ b/dbms/src/Storages/Tantivy/TantivyInputStream.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -117,23 +118,12 @@ class TantivyInputStream : public IProfilingBlockInputStream Block res(return_columns); if (is_count) { - if (search_result.count == 0) - { - return res; - } - // Construct const columns with search_result.count items, only used by count(*) - for (auto & name_and_type : return_columns) - { - auto type_default_value = name_and_type.type->getDefault(); - auto col = name_and_type.type->createColumnConst( - static_cast(search_result.count), - type_default_value); - res.getByName(name_and_type.name).column = std::move(col); - } + RUNTIME_CHECK_MSG(return_columns.size() == 1, "count search should return one column"); + auto & column = res.getByPosition(0).column->assumeMutableRef(); + column.insert(Int64(search_result.count)); return res; } - auto documents = search_result.rows; if (documents.empty()) { From 72449bed6540c254b91e532fe9b781a5d6a4c7a9 Mon Sep 17 00:00:00 2001 From: jinhelin Date: Wed, 14 Jan 2026 12:07:49 +0800 Subject: [PATCH 31/58] Server: Fix config name of tici (#10661) (cherry picked from commit 609a88b808ef2dc6c19e5149dfd6ad73add69490) --- dbms/src/Server/Server.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/src/Server/Server.cpp b/dbms/src/Server/Server.cpp index 87ad5b7d6b0..ea75f2000a9 100644 --- a/dbms/src/Server/Server.cpp +++ b/dbms/src/Server/Server.cpp @@ -1251,8 +1251,8 @@ try proxy_machine.runKVStore(tmt_context); - auto tici_reader_addr = config().getString("tici.reader_node.addr", ""); - auto tici_reader_port = config().getInt("tici.reader_node.port", 0); + auto tici_reader_addr = config().getString("tici.reader-node.addr", ""); + auto tici_reader_port = config().getInt("tici.reader-node.port", 0); if (!tici_reader_addr.empty() || tici_reader_port > 0) { Stopwatch watch; From 64b9f741de4bceb4965810768a20bc42207c5604 Mon Sep 17 00:00:00 2001 From: jinhelin Date: Thu, 29 Jan 2026 20:14:59 +0800 Subject: [PATCH 32/58] tici: Support read null value from tici (#10693) (cherry picked from commit 32ee6f35f43bb0699b575be015b4196357ff0919) --- .../src/Storages/Tantivy/TantivyInputStream.h | 45 ++++++++++++++++--- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/dbms/src/Storages/Tantivy/TantivyInputStream.h b/dbms/src/Storages/Tantivy/TantivyInputStream.h index 97f94cf7fca..47e023e341a 100644 --- a/dbms/src/Storages/Tantivy/TantivyInputStream.h +++ b/dbms/src/Storages/Tantivy/TantivyInputStream.h @@ -151,29 +151,64 @@ class TantivyInputStream : public IProfilingBlockInputStream } auto col = res.getByName(name_and_type.name).column->assumeMutable(); + bool has_null = false; if (removeNullable(name_and_type.type)->isStringOrFixedString()) { for (auto & doc : documents) { - const auto & v = doc.fieldValues[idx].string_value; - col->insert(Field(String(v.begin(), v.end()))); + const auto & field_value = doc.fieldValues[idx]; + if (field_value.is_null) + { + has_null = true; + col->insert(Field()); + } + else + { + const auto & v = field_value.string_value; + col->insert(Field(String(v.begin(), v.end()))); + } } } if (removeNullable(name_and_type.type)->isInteger()) { for (auto & doc : documents) { - col->insert(Field(doc.fieldValues[idx].int_value)); + const auto & field_value = doc.fieldValues[idx]; + if (field_value.is_null) + { + has_null = true; + col->insert(Field()); + } + else + { + col->insert(Field(field_value.int_value)); + } } } if (removeNullable(name_and_type.type)->isDateOrDateTime()) { for (auto & doc : documents) { - auto t = static_cast(doc.fieldValues[idx].int_value); - col->insert(Field(t)); + const auto & field_value = doc.fieldValues[idx]; + if (field_value.is_null) + { + has_null = true; + col->insert(Field()); + } + else + { + auto t = static_cast(field_value.int_value); + col->insert(Field(t)); + } } } + if (has_null) + { + RUNTIME_CHECK_MSG( + col->isColumnNullable(), + "column {} is not nullable, but got null value from TiCI", + name_and_type.name); + } } return res; } From 46298d80cdd50d98313cf5e3c626e1455ac30bf2 Mon Sep 17 00:00:00 2001 From: Shenghui Wu <793703860@qq.com> Date: Mon, 2 Feb 2026 14:07:00 +0800 Subject: [PATCH 33/58] tici : bump tipb and add FTS_MATCH_PHRASE support for TiCI pushdown (#10692) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR updates the tipb submodule to include the newly added FTSMatchPhrase scalar function signature, and wires it through TiFlash’s TiCI pushdown path. - Bump contrib/tipb to 1852f9829ce3d3962895fec43f908b31fbdc58fb - Extend TiCI expression conversion to accept tipb::ScalarFuncSig::FTSMatchPhrase (alongside existing FTSMatchWord / FTSMatchPrefix) so it can be sent to TiCI via FTSQueryInfo.match_expr. (cherry picked from commit 882aea931e16e2fa9ddaad2bef9987af4a13a434) --- contrib/tipb | 2 +- dbms/src/Storages/Tantivy/TiCIReadTaskPool.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/tipb b/contrib/tipb index 0607513e7fa..1852f9829ce 160000 --- a/contrib/tipb +++ b/contrib/tipb @@ -1 +1 @@ -Subproject commit 0607513e7fa40f3b564f0b269da76c1c8dc90032 +Subproject commit 1852f9829ce3d3962895fec43f908b31fbdc58fb diff --git a/dbms/src/Storages/Tantivy/TiCIReadTaskPool.h b/dbms/src/Storages/Tantivy/TiCIReadTaskPool.h index edb25f5f323..84fe7b99067 100644 --- a/dbms/src/Storages/Tantivy/TiCIReadTaskPool.h +++ b/dbms/src/Storages/Tantivy/TiCIReadTaskPool.h @@ -174,6 +174,7 @@ struct TiCIReadTaskPool { case tipb::ScalarFuncSig::FTSMatchWord: case tipb::ScalarFuncSig::FTSMatchPrefix: + case tipb::ScalarFuncSig::FTSMatchPhrase: case tipb::ScalarFuncSig::LogicalAnd: case tipb::ScalarFuncSig::LogicalOr: case tipb::ScalarFuncSig::UnaryNotInt: From 8d65aae14bd1a67fad6fa335411421fd6d5247e2 Mon Sep 17 00:00:00 2001 From: ChangRui-Ryan Date: Fri, 6 Feb 2026 12:05:38 +0800 Subject: [PATCH 34/58] tici: support general topn (#10706) (cherry picked from commit d41da1a3369bcebe0fee267d5a3898a60673ce82) --- dbms/src/Flash/Coprocessor/TiCIScan.cpp | 7 ++++++ dbms/src/Flash/Coprocessor/TiCIScan.h | 6 +++++ dbms/src/Storages/StorageTantivy.cpp | 2 ++ .../src/Storages/Tantivy/TantivyInputStream.h | 23 ++++++++++++++++++- dbms/src/Storages/Tantivy/TiCIReadTaskPool.h | 12 ++++++++++ 5 files changed, 49 insertions(+), 1 deletion(-) diff --git a/dbms/src/Flash/Coprocessor/TiCIScan.cpp b/dbms/src/Flash/Coprocessor/TiCIScan.cpp index e8e9573b42a..1e65076e67a 100644 --- a/dbms/src/Flash/Coprocessor/TiCIScan.cpp +++ b/dbms/src/Flash/Coprocessor/TiCIScan.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include +#include #include #include #include @@ -31,6 +32,12 @@ TiCIScan::TiCIScan(const tipb::Executor * tici_scan_, const String & executor_id , query_type(tici_scan->idx_scan().fts_query_info().query_type()) , shard_infos(dag_context.query_shard_infos.getTableShardInfosByExecutorID(tici_scan_->executor_id())) , limit(tici_scan->idx_scan().fts_query_info().top_k()) + , sort_column_ids( + tici_scan->idx_scan().fts_query_info().sort_column_ids().begin(), + tici_scan->idx_scan().fts_query_info().sort_column_ids().end()) + , sort_column_asc( + tici_scan->idx_scan().fts_query_info().sort_column_asc().begin(), + tici_scan->idx_scan().fts_query_info().sort_column_asc().end()) {} void TiCIScan::constructTiCIScanForRemoteRead(tipb::IndexScan * tipb_index_scan) const diff --git a/dbms/src/Flash/Coprocessor/TiCIScan.h b/dbms/src/Flash/Coprocessor/TiCIScan.h index 5f33fbb3cb6..a9229dcfb98 100644 --- a/dbms/src/Flash/Coprocessor/TiCIScan.h +++ b/dbms/src/Flash/Coprocessor/TiCIScan.h @@ -53,6 +53,10 @@ class TiCIScan const String & getCountAggExecutorId() const { return count_agg_executor_id; } + const std::vector & getSortColumnIds() const { return sort_column_ids; } + + const std::vector & getSortColumnAsc() const { return sort_column_asc; } + private: const tipb::Executor * tici_scan; [[maybe_unused]] String executor_id; @@ -64,6 +68,8 @@ class TiCIScan [[maybe_unused]] tipb::FTSQueryType query_type; const TableShardInfos shard_infos; const int limit; + std::vector sort_column_ids; + std::vector sort_column_asc; bool is_count_agg = false; String count_agg_executor_id; }; diff --git a/dbms/src/Storages/StorageTantivy.cpp b/dbms/src/Storages/StorageTantivy.cpp index 7437dfb8602..5fd41726d0b 100644 --- a/dbms/src/Storages/StorageTantivy.cpp +++ b/dbms/src/Storages/StorageTantivy.cpp @@ -89,6 +89,8 @@ void StorageTantivy::read( local_read, return_columns, tici_scan.getLimit(), + tici_scan.getSortColumnIds(), + tici_scan.getSortColumnAsc(), context.getSettingsRef().read_tso, tici_scan.getMatchExpr(), tici_scan.isCount(), diff --git a/dbms/src/Storages/Tantivy/TantivyInputStream.h b/dbms/src/Storages/Tantivy/TantivyInputStream.h index 47e023e341a..0b38d533c7f 100644 --- a/dbms/src/Storages/Tantivy/TantivyInputStream.h +++ b/dbms/src/Storages/Tantivy/TantivyInputStream.h @@ -59,6 +59,8 @@ class TantivyInputStream : public IProfilingBlockInputStream ShardInfo query_shard_info_, NamesAndTypes return_columns_, UInt64 limit_, + std::vector sort_column_ids_, + std::vector sort_column_asc_, UInt64 read_ts_, ::Expr match_expr_, bool is_count) @@ -69,6 +71,8 @@ class TantivyInputStream : public IProfilingBlockInputStream , query_shard_info(query_shard_info_) , return_columns(return_columns_) , limit(limit_) + , sort_column_ids(sort_column_ids_) + , sort_column_asc(sort_column_asc_) , read_ts(read_ts_) , match_expr(match_expr_) , is_count(is_count) @@ -97,7 +101,22 @@ class TantivyInputStream : public IProfilingBlockInputStream LOG_DEBUG(log, "shard info: {}", shard_info.toString()); auto key_ranges = getKeyRanges(shard_info.key_ranges); - auto search_param = SearchParam{static_cast(limit)}; + rust::Vec tici_sort_column_names; + for (const auto & sort_column_id : sort_column_ids) + { + tici_sort_column_names.push_back(rust::String("column_" + std::to_string(sort_column_id))); + } + rust::Vec tici_sort_column_asc; + for (const auto & asc : sort_column_asc) + { + tici_sort_column_asc.push_back(asc); + } + + SearchParam search_param{ + .limit = static_cast(limit), + .sort_field_names = std::move(tici_sort_column_names), + .is_asc = std::move(tici_sort_column_asc), + }; if (is_count) return_fields = {}; @@ -223,6 +242,8 @@ class TantivyInputStream : public IProfilingBlockInputStream ShardInfo query_shard_info; NamesAndTypes return_columns; UInt64 limit; + std::vector sort_column_ids; + std::vector sort_column_asc; UInt64 read_ts; ::Expr match_expr; bool is_count; diff --git a/dbms/src/Storages/Tantivy/TiCIReadTaskPool.h b/dbms/src/Storages/Tantivy/TiCIReadTaskPool.h index 84fe7b99067..0c75dd42a7e 100644 --- a/dbms/src/Storages/Tantivy/TiCIReadTaskPool.h +++ b/dbms/src/Storages/Tantivy/TiCIReadTaskPool.h @@ -35,6 +35,8 @@ struct TiCIReadTask ShardInfo query_shard_info_, NamesAndTypes return_columns_, UInt64 limit_, + std::vector sort_column_ids_, + std::vector sort_column_asc_, UInt64 read_ts_, ::Expr match_expr_, bool is_count) @@ -47,6 +49,8 @@ struct TiCIReadTask query_shard_info_, return_columns_, limit_, + sort_column_ids_, + sort_column_asc_, read_ts_, match_expr_, is_count); @@ -81,6 +85,8 @@ struct TiCIReadTaskPool const ShardInfoList & shard_infos, NamesAndTypes return_columns_, UInt64 limit_, + std::vector sort_column_ids_, + std::vector sort_column_asc_, UInt64 read_ts_, google::protobuf::RepeatedPtrField match_expr_, bool is_count, @@ -91,6 +97,8 @@ struct TiCIReadTaskPool , index_id(index_id_) , return_columns(return_columns_) , limit(limit_) + , sort_column_ids(sort_column_ids_) + , sort_column_asc(sort_column_asc_) , read_ts(read_ts_) , is_count(is_count) { @@ -132,6 +140,8 @@ struct TiCIReadTaskPool task->getShardInfo(), return_columns, limit, + sort_column_ids, + sort_column_asc, read_ts, match_expr, is_count); @@ -149,6 +159,8 @@ struct TiCIReadTaskPool Int64 index_id; NamesAndTypes return_columns; UInt64 limit; + std::vector sort_column_ids; + std::vector sort_column_asc; UInt64 read_ts; ::Expr match_expr; bool is_count; From 58bde3695a945b2b7cd672ccca9cab02f1ea1b63 Mon Sep 17 00:00:00 2001 From: Shenghui Wu <793703860@qq.com> Date: Tue, 10 Feb 2026 15:12:19 +0800 Subject: [PATCH 35/58] tici: support return version (#10457) (cherry picked from commit 76a5c3ee406636b5af73c48de0adea2cbe732d9b) --- dbms/src/Storages/Tantivy/TantivyInputStream.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/dbms/src/Storages/Tantivy/TantivyInputStream.h b/dbms/src/Storages/Tantivy/TantivyInputStream.h index 0b38d533c7f..e9cc291b838 100644 --- a/dbms/src/Storages/Tantivy/TantivyInputStream.h +++ b/dbms/src/Storages/Tantivy/TantivyInputStream.h @@ -50,6 +50,8 @@ class TantivyInputStream : public IProfilingBlockInputStream { static constexpr auto NAME = "TantivyInputStream"; + static constexpr auto version_column_name = "column_-1024"; + public: TantivyInputStream( LoggerPtr log_, @@ -161,6 +163,15 @@ class TantivyInputStream : public IProfilingBlockInputStream } if (idx == -1) { + if (name_and_type.name == version_column_name) + { + auto col = res.getByName(name_and_type.name).column->assumeMutable(); + for (auto & doc : documents) + { + col->insert(Field(doc.version)); + } + continue; + } for (size_t j = 0; j < documents.size(); j++) { // Insert default value for missing fields From 54c60d1b94b24bbfbd066f1bbadaa6bf56a9f138 Mon Sep 17 00:00:00 2001 From: JaySon Date: Fri, 13 Feb 2026 09:41:31 +0800 Subject: [PATCH 36/58] tici: Log down shard keyrange in hex format (#10718) Signed-off-by: JaySon-Huang (cherry picked from commit 7267fb32ca3443651c9834784c0a8961761ed32a) --- dbms/src/Flash/Coprocessor/ShardInfo.h | 55 +++++++++++++++----------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/dbms/src/Flash/Coprocessor/ShardInfo.h b/dbms/src/Flash/Coprocessor/ShardInfo.h index 3e1576cf37c..b06c6201b08 100644 --- a/dbms/src/Flash/Coprocessor/ShardInfo.h +++ b/dbms/src/Flash/Coprocessor/ShardInfo.h @@ -21,7 +21,6 @@ #include #include -#include #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" #include @@ -46,13 +45,19 @@ struct ShardInfo String toString() const { - std::ostringstream sb; - sb << "ShardID: " << shard_id << ", ShardEpoch: " << shard_epoch << ", KeyRanges: "; - for (const auto & range : key_ranges) - { - sb << "[" << range.start() << ", " << range.end() << ") "; - } - return sb.str(); + FmtBuffer buf; + buf.fmtAppend("ShardID: {}, ShardEpoch: {}, KeyRanges: ", shard_id, shard_epoch); + buf.joinStr( + key_ranges.begin(), + key_ranges.end(), + [](const coprocessor::KeyRange & range, FmtBuffer & fb) { + fb.fmtAppend( + "[{}, {})", + Redact::keyToDebugString(range.start().data(), range.start().size()), + Redact::keyToDebugString(range.end().data(), range.end().size())); + }, + " "); + return buf.toString(); } UInt64 getID() const { return shard_id; } @@ -80,14 +85,15 @@ class TableShardInfos String toString() const { - std::ostringstream sb; - sb << "TableShardInfos: ExecutorID: " << executor_id << ", ShardInfos: ["; - for (const auto & shard_info : shard_info_list) - { - sb << shard_info.toString() << ", "; - } - sb << "]"; - return sb.str(); + FmtBuffer buf; + buf.fmtAppend("ExecutorID: {}, ShardInfos: [", executor_id); + buf.joinStr( + shard_info_list.begin(), + shard_info_list.end(), + [](const ShardInfo & shard_info, FmtBuffer & fb) { fb.append(shard_info.toString()); }, + " "); + buf.append("]"); + return buf.toString(); } String executor_id; @@ -126,14 +132,15 @@ class QueryShardInfos String toString() const { - std::ostringstream sb; - sb << "QueryShardInfos: ["; - for (const auto & shard_info : table_shard_info_list) - { - sb << shard_info.toString() << ", "; - } - sb << "]"; - return sb.str(); + FmtBuffer buf; + buf.fmtAppend("QueryShardInfos: ["); + buf.joinStr( + table_shard_info_list.begin(), + table_shard_info_list.end(), + [](const TableShardInfos & table_shard_info, FmtBuffer & fb) { fb.append(table_shard_info.toString()); }, + ", "); + buf.append("]"); + return buf.toString(); } size_t size() const { return table_shard_info_list.size(); } From 799bbc6fc8b5949da6be88cf618ed05ac10df6fe Mon Sep 17 00:00:00 2001 From: Shenghui Wu Date: Fri, 13 Feb 2026 19:33:29 +0800 Subject: [PATCH 37/58] tici: add some log and metrics for debug (#10668) Co-authored-by: JaySon (cherry picked from commit 18e88cb85129ba3bd5b2a01f0d049d48d93d8eac) --- .../Coprocessor/StorageTantivyInterpreter.cpp | 31 +++++++++++++++++-- dbms/src/Flash/Planner/PhysicalPlan.cpp | 13 ++++++++ .../Flash/Planner/Plans/PhysicalTiCIScan.cpp | 7 +++++ 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/dbms/src/Flash/Coprocessor/StorageTantivyInterpreter.cpp b/dbms/src/Flash/Coprocessor/StorageTantivyInterpreter.cpp index a740f7502be..bbd534d2d6b 100644 --- a/dbms/src/Flash/Coprocessor/StorageTantivyInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/StorageTantivyInterpreter.cpp @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include @@ -40,8 +41,22 @@ void StorageTantivyIterpreter::execute(PipelineExecutorContext & exec_context, P storage->splitRemoteReadAndLocalRead(); // local_read storage->read(exec_context, group_builder, Names(), SelectQueryInfo(), context, 0, max_streams); + // remote_read + const auto & executor_id = tici_scan.getTiCIScan()->executor_id(); auto remote_shard_infos = storage->getRemoteShardInfos(); + const auto total_shards = tici_scan.getShardInfos().shard_info_list.size(); + const auto remote_shards = remote_shard_infos.size(); + const auto local_shards = total_shards >= remote_shards ? (total_shards - remote_shards) : 0; + LOG_INFO( + log, + "tici split shards: executor_id={} total={} local={} remote={} local_concurrency={} start_ts={}", + executor_id, + total_shards, + local_shards, + remote_shards, + group_builder.concurrency(), + context.getSettingsRef().read_tso); if (!remote_shard_infos.empty()) { if (context.getDAGContext()->isCop()) @@ -55,7 +70,17 @@ void StorageTantivyIterpreter::execute(PipelineExecutorContext & exec_context, P auto remote_request = buildRemoteRequests(remote_shard_infos); if (!remote_request.empty()) { - buildRemoteExec(exec_context, group_builder, remote_request); + PipelineExecGroupBuilder remote_builder; + buildRemoteExec(exec_context, remote_builder, remote_request); + if (!remote_builder.empty()) + LOG_INFO( + log, + "tici remote sourceOps built: executor_id={} remote_requests={} concurrency={} start_ts={}", + executor_id, + remote_request.size(), + remote_builder.concurrency(), + context.getSettingsRef().read_tso); + group_builder.merge(std::move(remote_builder)); } } @@ -174,12 +199,14 @@ std::vector StorageTantivyIterpreter::buildCopTas remote_request.connection_alias, &Poco::Logger::get("pingcap/coprocessor"), std::move(meta_data), - [&] {}, + [&] { GET_METRIC(tiflash_coprocessor_request_count, type_remote_read_sent).Increment(); }, tici_scan.getTableId(), tici_scan.getIndexId(), tici_scan.getTiCIScan()->executor_id()); all_tasks.insert(all_tasks.end(), tasks.begin(), tasks.end()); } + GET_METRIC(tiflash_coprocessor_request_count, type_remote_read_constructed) + .Increment(static_cast(all_tasks.size())); return all_tasks; } diff --git a/dbms/src/Flash/Planner/PhysicalPlan.cpp b/dbms/src/Flash/Planner/PhysicalPlan.cpp index 2c0759978a7..b614c7d61d8 100644 --- a/dbms/src/Flash/Planner/PhysicalPlan.cpp +++ b/dbms/src/Flash/Planner/PhysicalPlan.cpp @@ -97,6 +97,18 @@ void PhysicalPlan::buildTiCIScan(const String & executor_id, const tipb::Executo { RUNTIME_ASSERT(executor->idx_scan().has_fts_query_info()); TiCIScan tici_scan(executor, executor_id, dagContext()); + LOG_INFO( + log, + "tici scan: keyspace_id={} table_id={} index_id={} limit={} shard_count={} match_expr_size={} query_type={} " + "start_ts={}", + tici_scan.getKeyspaceID(), + tici_scan.getTableId(), + tici_scan.getIndexId(), + tici_scan.getLimit(), + tici_scan.getShardInfos().shard_info_list.size(), + tici_scan.getMatchExpr().size(), + tipb::FTSQueryType_Name(executor->idx_scan().fts_query_info().query_type()), + context.getSettingsRef().read_tso); pushBack(PhysicalTiCIScan::build(executor_id, log, tici_scan)); dagContext().table_scan_executor_id = executor_id; } @@ -211,6 +223,7 @@ void PhysicalPlan::build(const tipb::Executor * executor) buildTableScan(executor_id, executor); break; case tipb::ExecType::TypeIndexScan: + GET_METRIC(tiflash_coprocessor_executor_count, type_tici).Increment(); buildTiCIScan(executor_id, executor); break; case tipb::ExecType::TypeJoin: diff --git a/dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.cpp b/dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.cpp index f00be52f077..894977e3e33 100644 --- a/dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.cpp +++ b/dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.cpp @@ -101,6 +101,13 @@ void PhysicalTiCIScan::buildPipeline( Context & context, PipelineExecutorContext & exec_context) { + LOG_INFO( + log, + "build tici pipeline: executor_id={} is_count={} max_streams={} start_ts={}", + executor_id, + tici_scan.isCount(), + context.getMaxStreams(), + context.getSettingsRef().read_tso); // For building PipelineExec in compile time. StorageTantivyIterpreter storage_interpreter(context, tici_scan, context.getMaxStreams()); storage_interpreter.execute(exec_context, pipeline_exec_builder); From a246027584adc1d448fa468a4a8dd69e70cf716c Mon Sep 17 00:00:00 2001 From: Shenghui Wu <793703860@qq.com> Date: Fri, 13 Feb 2026 23:29:56 +0800 Subject: [PATCH 38/58] *: reuse split snapshot and reduce shard FFI checks (#10717) ref pingcap-inc/tici#749 storage: reuse shard snapshot from split and reduce FFI checks - call `check_shards_and_acquire_snapshot` once in split stage - use returned `Vec` to split local vs remote reads - reuse the split-stage snapshot in `read()` instead of reacquiring - update `contrib/tici` submodule to include: - atomic snapshot + hit bitmap API - regression test `test_snapshot_query_after_remove` (cherry picked from commit ceb3b83d1cc76292c4e98b7a463f42e0c085ee17) --- dbms/src/Storages/StorageTantivy.cpp | 17 +++++++++++++---- dbms/src/Storages/StorageTantivy.h | 3 +++ dbms/src/Storages/Tantivy/TantivyInputStream.h | 7 ++++++- dbms/src/Storages/Tantivy/TiCIReadTaskPool.h | 14 ++++++++++---- 4 files changed, 32 insertions(+), 9 deletions(-) diff --git a/dbms/src/Storages/StorageTantivy.cpp b/dbms/src/Storages/StorageTantivy.cpp index 5fd41726d0b..fd066742c61 100644 --- a/dbms/src/Storages/StorageTantivy.cpp +++ b/dbms/src/Storages/StorageTantivy.cpp @@ -81,6 +81,10 @@ void StorageTantivy::read( return_columns = genNamesAndTypesForTiCI(tici_scan.getReturnColumns(), "column"); } + RUNTIME_CHECK(local_shards_snapshot.has_value()); + auto shards_snapshot = std::move(*local_shards_snapshot); + local_shards_snapshot.reset(); + auto tici_task_pool = std::make_shared( log, tici_scan.getKeyspaceID(), @@ -94,7 +98,8 @@ void StorageTantivy::read( context.getSettingsRef().read_tso, tici_scan.getMatchExpr(), tici_scan.isCount(), - context.getTimezoneInfo()); + context.getTimezoneInfo(), + std::move(shards_snapshot)); num_streams = std::max(1, std::min(num_streams, local_read.size())); // local read @@ -145,6 +150,7 @@ void StorageTantivy::splitRemoteReadAndLocalRead() for (const auto & shard_info : all) { shards.push_back(::Shard{ + .keyspace_id = tici_scan.getKeyspaceID(), .table_id = table_id, .index_id = index_id, .shard_id = shard_info.shard_id, @@ -152,10 +158,13 @@ void StorageTantivy::splitRemoteReadAndLocalRead() }); } - auto result = check_shards(shards); - for (size_t i = 0; i < result.size(); ++i) + ::rust::Vec local_results; + local_shards_snapshot = check_shards_and_acquire_snapshot(shards, local_results); + RUNTIME_CHECK(local_shards_snapshot.has_value()); + RUNTIME_CHECK(local_results.size() == all.size()); + for (size_t i = 0; i < all.size(); ++i) { - const auto & is_local = result[i]; + const auto is_local = local_results[i]; if (is_local) { local_shard_infos.push_back(all[i]); diff --git a/dbms/src/Storages/StorageTantivy.h b/dbms/src/Storages/StorageTantivy.h index 40174614d1a..c019d9f0fa5 100644 --- a/dbms/src/Storages/StorageTantivy.h +++ b/dbms/src/Storages/StorageTantivy.h @@ -29,8 +29,10 @@ #include #include #include +#include #include +#include #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" @@ -83,5 +85,6 @@ class StorageTantivy : public IStorage ShardInfoList remote_read; ShardInfoList local_read; + std::optional> local_shards_snapshot; }; } // namespace DB diff --git a/dbms/src/Storages/Tantivy/TantivyInputStream.h b/dbms/src/Storages/Tantivy/TantivyInputStream.h index e9cc291b838..df593e218ea 100644 --- a/dbms/src/Storages/Tantivy/TantivyInputStream.h +++ b/dbms/src/Storages/Tantivy/TantivyInputStream.h @@ -65,7 +65,8 @@ class TantivyInputStream : public IProfilingBlockInputStream std::vector sort_column_asc_, UInt64 read_ts_, ::Expr match_expr_, - bool is_count) + bool is_count, + std::shared_ptr> shards_snapshot_) : log(log_) , keyspace_id(keyspace_id_) , table_id(table_id_) @@ -78,6 +79,7 @@ class TantivyInputStream : public IProfilingBlockInputStream , read_ts(read_ts_) , match_expr(match_expr_) , is_count(is_count) + , shards_snapshot(std::move(shards_snapshot_)) {} String getName() const override { return NAME; } @@ -122,7 +124,9 @@ class TantivyInputStream : public IProfilingBlockInputStream if (is_count) return_fields = {}; + RUNTIME_CHECK(shards_snapshot != nullptr); SearchResult search_result = search( + **shards_snapshot, { .keyspace_id = keyspace_id, .table_id = table_id, @@ -258,6 +262,7 @@ class TantivyInputStream : public IProfilingBlockInputStream UInt64 read_ts; ::Expr match_expr; bool is_count; + std::shared_ptr> shards_snapshot; static rust::Vec getFields(NamesAndTypes & columns) { diff --git a/dbms/src/Storages/Tantivy/TiCIReadTaskPool.h b/dbms/src/Storages/Tantivy/TiCIReadTaskPool.h index 0c75dd42a7e..7d6df3745c1 100644 --- a/dbms/src/Storages/Tantivy/TiCIReadTaskPool.h +++ b/dbms/src/Storages/Tantivy/TiCIReadTaskPool.h @@ -39,7 +39,8 @@ struct TiCIReadTask std::vector sort_column_asc_, UInt64 read_ts_, ::Expr match_expr_, - bool is_count) + bool is_count, + const std::shared_ptr> & shards_snapshot_) { input_stream = std::make_shared( log_, @@ -53,7 +54,8 @@ struct TiCIReadTask sort_column_asc_, read_ts_, match_expr_, - is_count); + is_count, + shards_snapshot_); } bool isInitialized() const { return input_stream != nullptr; } @@ -90,7 +92,8 @@ struct TiCIReadTaskPool UInt64 read_ts_, google::protobuf::RepeatedPtrField match_expr_, bool is_count, - const TimezoneInfo & timezone_info_) + const TimezoneInfo & timezone_info_, + rust::Box shards_snapshot_) : log(log_) , keyspace_id(keyspace_id_) , table_id(table_id_) @@ -101,6 +104,7 @@ struct TiCIReadTaskPool , sort_column_asc(sort_column_asc_) , read_ts(read_ts_) , is_count(is_count) + , shards_snapshot(std::make_shared>(std::move(shards_snapshot_))) { for (const auto & shard_info : shard_infos) { @@ -144,7 +148,8 @@ struct TiCIReadTaskPool sort_column_asc, read_ts, match_expr, - is_count); + is_count, + shards_snapshot); } return task->getInputStream(); } @@ -164,6 +169,7 @@ struct TiCIReadTaskPool UInt64 read_ts; ::Expr match_expr; bool is_count; + std::shared_ptr> shards_snapshot; static std::tuple<::Expr, std::vector> tipbToTiCIExpr( const tipb::Expr & expr, From ae1e36c14149409e958cdd3a4b2e76068dcb8199 Mon Sep 17 00:00:00 2001 From: ChangRui-Ryan Date: Mon, 23 Mar 2026 10:37:09 +0800 Subject: [PATCH 39/58] tici: support mpp join (#10753) (cherry picked from commit e402b34e4ad95c51f4c8460f38d3e5d99faf4db4) --- dbms/src/Flash/Mpp/MPPTask.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/dbms/src/Flash/Mpp/MPPTask.cpp b/dbms/src/Flash/Mpp/MPPTask.cpp index 7d388dd73ac..8d8e22db868 100644 --- a/dbms/src/Flash/Mpp/MPPTask.cpp +++ b/dbms/src/Flash/Mpp/MPPTask.cpp @@ -448,6 +448,7 @@ void MPPTask::prepare(const mpp::DispatchTaskRequest & task_request) dag_context = std::make_unique(dag_req, task_request.meta(), is_root_mpp_task); dag_context->log = log; dag_context->tables_regions_info = std::move(tables_regions_info); + dag_context->query_shard_infos = QueryShardInfos::create(task_request.table_shard_infos()); dag_context->tidb_host = context->getClientInfo().current_address.toString(); context->setDAGContext(dag_context.get()); From 8964aaebeb3ab61b56710d112615a74ff02fe3ce Mon Sep 17 00:00:00 2001 From: Shenghui Wu <793703860@qq.com> Date: Mon, 20 Apr 2026 20:24:21 +0800 Subject: [PATCH 40/58] tici: consume columnar search results in tiflash (#10802) tici: consume columnar search results in tiflash tiflash: update contrib/tici to latest master Simplify Tantivy nullable handling Restore Tantivy formatting (cherry picked from commit 9bba333a4135a2964530458e551e5da2db8718c2) --- .../src/Storages/Tantivy/TantivyInputStream.h | 374 +++++++++++++----- 1 file changed, 281 insertions(+), 93 deletions(-) diff --git a/dbms/src/Storages/Tantivy/TantivyInputStream.h b/dbms/src/Storages/Tantivy/TantivyInputStream.h index df593e218ea..5b081351ddf 100644 --- a/dbms/src/Storages/Tantivy/TantivyInputStream.h +++ b/dbms/src/Storages/Tantivy/TantivyInputStream.h @@ -14,11 +14,20 @@ #pragma once +#include +#include +#include +#include #include +#include #include #include #include #include +#include +#include +#include +#include #include #include #include @@ -31,6 +40,10 @@ #include #include +#include +#include +#include + namespace DB::TS { @@ -50,8 +63,6 @@ class TantivyInputStream : public IProfilingBlockInputStream { static constexpr auto NAME = "TantivyInputStream"; - static constexpr auto version_column_name = "column_-1024"; - public: TantivyInputStream( LoggerPtr log_, @@ -110,6 +121,7 @@ class TantivyInputStream : public IProfilingBlockInputStream { tici_sort_column_names.push_back(rust::String("column_" + std::to_string(sort_column_id))); } + rust::Vec tici_sort_column_asc; for (const auto & asc : sort_column_asc) { @@ -149,101 +161,55 @@ class TantivyInputStream : public IProfilingBlockInputStream return res; } - auto documents = search_result.rows; - if (documents.empty()) - { + RUNTIME_CHECK_MSG( + search_result.result_type == result_type_rows(), + "expected row search result, got type {}", + search_result.result_type); + + const auto row_count = static_cast(search_result.row_count); + if (row_count == 0) return res; + + auto name_to_pos = buildColumnPositionMap(return_columns); + std::vector filled(return_columns.size(), false); + + for (const auto & column_data : search_result.i64_columns) + { + installIntegerLikeColumn( + res, + name_to_pos, + filled, + return_columns, + column_data.col_name, + column_data.values, + column_data.null_map); } - for (auto & name_and_type : return_columns) + for (const auto & column_data : search_result.u64_columns) { - int idx = -1; - for (size_t j = 0; j < documents[0].fieldValues.size(); j++) - { - if (documents[0].fieldValues[j].field_name == name_and_type.name) - { - idx = j; - break; - } - } - if (idx == -1) - { - if (name_and_type.name == version_column_name) - { - auto col = res.getByName(name_and_type.name).column->assumeMutable(); - for (auto & doc : documents) - { - col->insert(Field(doc.version)); - } - continue; - } - for (size_t j = 0; j < documents.size(); j++) - { - // Insert default value for missing fields - res.getByName(name_and_type.name).column->assumeMutable()->insertDefault(); - } - continue; - } - - auto col = res.getByName(name_and_type.name).column->assumeMutable(); - bool has_null = false; - if (removeNullable(name_and_type.type)->isStringOrFixedString()) - { - for (auto & doc : documents) - { - const auto & field_value = doc.fieldValues[idx]; - if (field_value.is_null) - { - has_null = true; - col->insert(Field()); - } - else - { - const auto & v = field_value.string_value; - col->insert(Field(String(v.begin(), v.end()))); - } - } - } - if (removeNullable(name_and_type.type)->isInteger()) - { - for (auto & doc : documents) - { - const auto & field_value = doc.fieldValues[idx]; - if (field_value.is_null) - { - has_null = true; - col->insert(Field()); - } - else - { - col->insert(Field(field_value.int_value)); - } - } - } - if (removeNullable(name_and_type.type)->isDateOrDateTime()) - { - for (auto & doc : documents) - { - const auto & field_value = doc.fieldValues[idx]; - if (field_value.is_null) - { - has_null = true; - col->insert(Field()); - } - else - { - auto t = static_cast(field_value.int_value); - col->insert(Field(t)); - } - } - } - if (has_null) - { - RUNTIME_CHECK_MSG( - col->isColumnNullable(), - "column {} is not nullable, but got null value from TiCI", - name_and_type.name); - } + installIntegerLikeColumn( + res, + name_to_pos, + filled, + return_columns, + column_data.col_name, + column_data.values, + column_data.null_map); + } + for (const auto & column_data : search_result.f64_columns) + { + installFloatColumn(res, name_to_pos, filled, return_columns, column_data); + } + for (const auto & column_data : search_result.bytes_columns) + { + installBytesColumn(res, name_to_pos, filled, return_columns, column_data); + } + + for (size_t i = 0; i < return_columns.size(); ++i) + { + if (!filled[i]) + fillDefaultColumn(res.getByPosition(i), row_count); } + return res; } @@ -264,6 +230,228 @@ class TantivyInputStream : public IProfilingBlockInputStream bool is_count; std::shared_ptr> shards_snapshot; + static std::unordered_map buildColumnPositionMap(const NamesAndTypes & columns) + { + std::unordered_map positions; + positions.reserve(columns.size()); + for (size_t i = 0; i < columns.size(); ++i) + positions.emplace(columns[i].name, i); + return positions; + } + + static ColumnUInt8::MutablePtr buildNullMapColumn(size_t row_count, const rust::Vec<::std::uint8_t> & null_map) + { + auto null_map_column = ColumnUInt8::create(row_count, 0); + if (null_map.size() == 0) + return null_map_column; + + RUNTIME_CHECK_MSG( + null_map.size() == row_count, + "null map size mismatch, expect {}, got {}", + row_count, + null_map.size()); + auto & dst = null_map_column->getData(); + if (row_count != 0) + std::memcpy(dst.data(), null_map.data(), row_count * sizeof(UInt8)); + return null_map_column; + } + + static void fillDefaultColumn(ColumnWithTypeAndName & column, size_t row_count) + { + auto mutable_column = column.type->createColumn(); + mutable_column->insertManyDefaults(row_count); + column.column = std::move(mutable_column); + } + + template + static MutableColumnPtr buildNumericColumn( + const NameAndTypePair & name_and_type, + const rust::Vec & values, + const rust::Vec<::std::uint8_t> & null_map) + { + const auto row_count = values.size(); + auto nested_column = ColumnVector::create(row_count); + auto & data = nested_column->getData(); + const bool has_null_map = null_map.size() != 0; + + if (has_null_map) + { + RUNTIME_CHECK_MSG( + null_map.size() == row_count, + "null map size mismatch for column {}, expect {}, got {}", + name_and_type.name, + row_count, + null_map.size()); + } + + if constexpr (std::is_same_v) + { + if (row_count != 0) + std::memcpy(data.data(), values.data(), row_count * sizeof(TargetType)); + } + else + { + for (size_t i = 0; i < row_count; ++i) + data[i] = static_cast(values[i]); + } + + if (name_and_type.type->isNullable()) + return ColumnNullable::create(std::move(nested_column), buildNullMapColumn(row_count, null_map)); + return nested_column; + } + + template + static MutableColumnPtr buildIntegerLikeColumn( + const NameAndTypePair & name_and_type, + const rust::Vec & values, + const rust::Vec<::std::uint8_t> & null_map) + { + const auto nested_type = removeNullable(name_and_type.type); + if (typeid_cast(nested_type.get())) + return buildNumericColumn(name_and_type, values, null_map); + if (typeid_cast(nested_type.get())) + return buildNumericColumn(name_and_type, values, null_map); + if (typeid_cast(nested_type.get())) + return buildNumericColumn(name_and_type, values, null_map); + if (typeid_cast(nested_type.get())) + return buildNumericColumn(name_and_type, values, null_map); + if (typeid_cast(nested_type.get())) + return buildNumericColumn(name_and_type, values, null_map); + if (typeid_cast(nested_type.get())) + return buildNumericColumn(name_and_type, values, null_map); + if (typeid_cast(nested_type.get())) + return buildNumericColumn(name_and_type, values, null_map); + if (typeid_cast(nested_type.get())) + return buildNumericColumn(name_and_type, values, null_map); + if (typeid_cast(nested_type.get())) + return buildNumericColumn(name_and_type, values, null_map); + if (typeid_cast(nested_type.get())) + return buildNumericColumn(name_and_type, values, null_map); + if (typeid_cast(nested_type.get())) + return buildNumericColumn(name_and_type, values, null_map); + if (typeid_cast(nested_type.get())) + return buildNumericColumn(name_and_type, values, null_map); + + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "unsupported integer-like target type {} for column {}", + nested_type->getName(), + name_and_type.name); + } + + static MutableColumnPtr buildFloatColumn( + const NameAndTypePair & name_and_type, + const rust::Vec & values, + const rust::Vec<::std::uint8_t> & null_map) + { + const auto nested_type = removeNullable(name_and_type.type); + if (typeid_cast(nested_type.get())) + return buildNumericColumn(name_and_type, values, null_map); + if (typeid_cast(nested_type.get())) + return buildNumericColumn(name_and_type, values, null_map); + + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "unsupported float target type {} for column {}", + nested_type->getName(), + name_and_type.name); + } + + static MutableColumnPtr buildBytesColumn(const NameAndTypePair & name_and_type, const BytesColumnData & column_data) + { + const auto row_count = column_data.offsets.size(); + if (column_data.offsets.size() != 0) + { + RUNTIME_CHECK_MSG( + column_data.offsets[column_data.offsets.size() - 1] == column_data.chars.size(), + "string offsets and chars size mismatch for column {}", + name_and_type.name); + } + + if (removeNullable(name_and_type.type)->isString()) + { + auto nested_column = ColumnString::create(); + auto & chars = nested_column->getChars(); + auto & offsets = nested_column->getOffsets(); + + chars.resize(column_data.chars.size()); + if (!column_data.chars.empty()) + std::memcpy(chars.data(), column_data.chars.data(), column_data.chars.size()); + + offsets.resize(row_count); + for (size_t i = 0; i < row_count; ++i) + offsets[i] = static_cast(column_data.offsets[i]); + + if (name_and_type.type->isNullable()) + return ColumnNullable::create( + std::move(nested_column), + buildNullMapColumn(row_count, column_data.null_map)); + return nested_column; + } + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "unsupported bytes target type {} for column {}", + name_and_type.type->getName(), + name_and_type.name); + } + + template + static void installIntegerLikeColumn( + Block & res, + const std::unordered_map & name_to_pos, + std::vector & filled, + const NamesAndTypes & columns, + const rust::String & col_name, + const rust::Vec & values, + const rust::Vec<::std::uint8_t> & null_map) + { + const String column_name(col_name); + const auto it = name_to_pos.find(column_name); + RUNTIME_CHECK_MSG(it != name_to_pos.end(), "unexpected column {} returned from TiCI", column_name); + + const auto pos = it->second; + RUNTIME_CHECK_MSG(!filled[pos], "duplicate column {} returned from TiCI", column_name); + auto & result_column = res.getByPosition(pos); + result_column.column = buildIntegerLikeColumn(columns[pos], values, null_map); + filled[pos] = true; + } + + static void installFloatColumn( + Block & res, + const std::unordered_map & name_to_pos, + std::vector & filled, + const NamesAndTypes & columns, + const F64ColumnData & column_data) + { + const String column_name(column_data.col_name); + const auto it = name_to_pos.find(column_name); + RUNTIME_CHECK_MSG(it != name_to_pos.end(), "unexpected column {} returned from TiCI", column_name); + + const auto pos = it->second; + RUNTIME_CHECK_MSG(!filled[pos], "duplicate column {} returned from TiCI", column_name); + auto & result_column = res.getByPosition(pos); + result_column.column = buildFloatColumn(columns[pos], column_data.values, column_data.null_map); + filled[pos] = true; + } + + static void installBytesColumn( + Block & res, + const std::unordered_map & name_to_pos, + std::vector & filled, + const NamesAndTypes & columns, + const BytesColumnData & column_data) + { + const String column_name(column_data.col_name); + const auto it = name_to_pos.find(column_name); + RUNTIME_CHECK_MSG(it != name_to_pos.end(), "unexpected column {} returned from TiCI", column_name); + + const auto pos = it->second; + RUNTIME_CHECK_MSG(!filled[pos], "duplicate column {} returned from TiCI", column_name); + auto & result_column = res.getByPosition(pos); + result_column.column = buildBytesColumn(columns[pos], column_data); + filled[pos] = true; + } + static rust::Vec getFields(NamesAndTypes & columns) { rust::Vec fields; From 2537e93203b715076511ef38dce97abe6c5f9de3 Mon Sep 17 00:00:00 2001 From: jinhelin Date: Tue, 21 Apr 2026 18:45:48 +0800 Subject: [PATCH 41/58] contrib: update tici and remove unused table_id (#10806) (cherry picked from commit 9f74b4593d89ba4cdebc344a888ec76f6b313d91) --- dbms/src/Storages/StorageTantivy.cpp | 2 -- dbms/src/Storages/Tantivy/TantivyInputStream.h | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/dbms/src/Storages/StorageTantivy.cpp b/dbms/src/Storages/StorageTantivy.cpp index fd066742c61..84883892680 100644 --- a/dbms/src/Storages/StorageTantivy.cpp +++ b/dbms/src/Storages/StorageTantivy.cpp @@ -146,12 +146,10 @@ void StorageTantivy::splitRemoteReadAndLocalRead() ShardInfoList remote_shard_infos; ::rust::Vec<::Shard> shards; auto index_id = tici_scan.getIndexId(); - auto table_id = tici_scan.getTableId(); for (const auto & shard_info : all) { shards.push_back(::Shard{ .keyspace_id = tici_scan.getKeyspaceID(), - .table_id = table_id, .index_id = index_id, .shard_id = shard_info.shard_id, .shard_epoch = shard_info.shard_epoch, diff --git a/dbms/src/Storages/Tantivy/TantivyInputStream.h b/dbms/src/Storages/Tantivy/TantivyInputStream.h index 5b081351ddf..8340f9d810f 100644 --- a/dbms/src/Storages/Tantivy/TantivyInputStream.h +++ b/dbms/src/Storages/Tantivy/TantivyInputStream.h @@ -141,7 +141,6 @@ class TantivyInputStream : public IProfilingBlockInputStream **shards_snapshot, { .keyspace_id = keyspace_id, - .table_id = table_id, .index_id = index_id, .shard_id = shard_info.shard_id, .shard_epoch = shard_info.shard_epoch, @@ -218,7 +217,7 @@ class TantivyInputStream : public IProfilingBlockInputStream bool done = false; LoggerPtr log; UInt32 keyspace_id; - Int64 table_id; + [[maybe_unused]] Int64 table_id; Int64 index_id; ShardInfo query_shard_info; NamesAndTypes return_columns; From 6ffeecf72a81424e47a00c77c310473822f7b547 Mon Sep 17 00:00:00 2001 From: Calvin Neo Date: Thu, 23 Apr 2026 11:02:42 +0800 Subject: [PATCH 42/58] metrics: improve getGCSafePointWithRetry metrics (#10810) close pingcap/tiflash#10808 Signed-off-by: Calvin Neo (cherry picked from commit f99700d1e43d7d55e4ed68c357862dab3a3d26a0) --- dbms/src/Common/TiFlashMetrics.h | 13 +++++++ dbms/src/Debug/dbgFuncSchema.cpp | 7 +++- dbms/src/Interpreters/Settings.h | 1 + .../DeltaMerge/DeltaMergeStore_InternalBg.cpp | 3 +- .../KVStore/MultiRaft/PrehandleSnapshot.cpp | 3 +- .../KVStore/TiKVHelpers/PDTiKVClient.h | 38 ++++++++++++++++--- dbms/src/Storages/StorageDeltaMerge.cpp | 3 +- dbms/src/TiDB/Schema/SchemaSyncService.cpp | 8 +++- 8 files changed, 65 insertions(+), 11 deletions(-) diff --git a/dbms/src/Common/TiFlashMetrics.h b/dbms/src/Common/TiFlashMetrics.h index 3e78fab1398..06bddee9a09 100644 --- a/dbms/src/Common/TiFlashMetrics.h +++ b/dbms/src/Common/TiFlashMetrics.h @@ -189,6 +189,19 @@ static_assert(RAFT_REGION_BIG_WRITE_THRES * 4 < RAFT_REGION_BIG_WRITE_MAX, "Inva Histogram, \ F(type_sync_schema_apply_duration, {{"type", "sync_schema_duration"}}, ExpBuckets{0.001, 2, 20}), \ F(type_sync_table_schema_apply_duration, {{"type", "sync_table_schema_duration"}}, ExpBuckets{0.001, 2, 20})) \ + M(tiflash_gc_safepoint_request_count, \ + "GC safepoint request events", \ + Counter, \ + F(type_get_gc_state, {{"type", "get_gc_state"}}), \ + F(type_zero_gc_safe_point, {{"type", "zero_gc_safe_point"}}), \ + F(type_pd_response_error, {{"type", "pd_response_error"}}), \ + F(type_request_exception, {{"type", "request_exception"}}), \ + F(type_backoff_error, {{"type", "backoff_error"}})) \ + M(tiflash_gc_safepoint_backoff_count, \ + "Bucketed histogram of GC safepoint request backoff count per call", \ + Histogram, \ + F(type_success, {{"type", "success"}}, ExpBuckets{1, 2, 8}), \ + F(type_failure, {{"type", "failure"}}, ExpBuckets{1, 2, 8})) \ M(tiflash_raft_read_index_count, "Total number of raft read index", Counter) \ M(tiflash_stale_read_count, "Total number of stale read", Counter) \ M(tiflash_raft_read_index_duration_seconds, \ diff --git a/dbms/src/Debug/dbgFuncSchema.cpp b/dbms/src/Debug/dbgFuncSchema.cpp index c310b09ead8..0a726f679e1 100644 --- a/dbms/src/Debug/dbgFuncSchema.cpp +++ b/dbms/src/Debug/dbgFuncSchema.cpp @@ -183,7 +183,12 @@ void dbgFuncGcSchemas(Context & context, const ASTs & args, DBGInvoker::Printer Timestamp gc_safe_point = 0; bool ignore_remain_regions = false; if (args.empty()) - gc_safe_point = PDClientHelper::getGCSafePointWithRetry(context.getTMTContext().getPDClient(), NullspaceID); + gc_safe_point = PDClientHelper::getGCSafePointWithRetry( + context.getTMTContext().getPDClient(), + NullspaceID, + true, + 30, + context.getSettingsRef().safe_point_get_max_backoff_ms); if (!args.empty()) gc_safe_point = safeGet(typeid_cast(*args[0]).value); if (args.size() >= 2) diff --git a/dbms/src/Interpreters/Settings.h b/dbms/src/Interpreters/Settings.h index 4c2e5dbeca4..8798ed1c71b 100644 --- a/dbms/src/Interpreters/Settings.h +++ b/dbms/src/Interpreters/Settings.h @@ -53,6 +53,7 @@ struct Settings M(SettingUInt64, mpp_task_running_timeout, DEFAULT_MPP_TASK_RUNNING_TIMEOUT, "mpp task max time that running without any progress.") \ M(SettingUInt64, mpp_task_waiting_timeout, DEFAULT_MPP_TASK_WAITING_TIMEOUT, "mpp task max time that waiting first data block from source input stream.") \ M(SettingInt64, safe_point_update_interval_seconds, 1, "The interval in seconds to update safe point from PD.") \ + M(SettingInt64, safe_point_get_max_backoff_ms, 120000, "Max total backoff sleep time in milliseconds when fetching GC safe point from PD.") \ M(SettingUInt64, max_block_size, DEFAULT_BLOCK_SIZE, "Maximum block rows for reading") \ M(SettingUInt64, max_block_bytes, DEFAULT_BLOCK_BYTES, "Maximum block bytes for reading") \ M(SettingUInt64, max_insert_block_size, DEFAULT_INSERT_BLOCK_SIZE, "The maximum block size for insertion, if we control the creation of blocks for insertion.") \ diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore_InternalBg.cpp b/dbms/src/Storages/DeltaMerge/DeltaMergeStore_InternalBg.cpp index 7e3d62e2aa3..1760c744bd5 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore_InternalBg.cpp +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore_InternalBg.cpp @@ -373,7 +373,8 @@ bool DeltaMergeStore::updateGCSafePoint() pd_client, keyspace_id, /* ignore_cache= */ false, - global_context.getSettingsRef().safe_point_update_interval_seconds); + global_context.getSettingsRef().safe_point_update_interval_seconds, + global_context.getSettingsRef().safe_point_get_max_backoff_ms); latest_gc_safe_point.store(safe_point, std::memory_order_release); return true; } diff --git a/dbms/src/Storages/KVStore/MultiRaft/PrehandleSnapshot.cpp b/dbms/src/Storages/KVStore/MultiRaft/PrehandleSnapshot.cpp index 427516b9bc3..81c7420971a 100644 --- a/dbms/src/Storages/KVStore/MultiRaft/PrehandleSnapshot.cpp +++ b/dbms/src/Storages/KVStore/MultiRaft/PrehandleSnapshot.cpp @@ -716,7 +716,8 @@ PrehandleResult KVStore::preHandleSSTsToDTFiles( pd_client, keyspace_id, /* ignore_cache= */ false, - context.getSettingsRef().safe_point_update_interval_seconds); + context.getSettingsRef().safe_point_update_interval_seconds, + context.getSettingsRef().safe_point_get_max_backoff_ms); } auto opt = DM::SSTFilesToBlockInputStreamOpts{ diff --git a/dbms/src/Storages/KVStore/TiKVHelpers/PDTiKVClient.h b/dbms/src/Storages/KVStore/TiKVHelpers/PDTiKVClient.h index 7cfcfc86d15..24dc1723268 100644 --- a/dbms/src/Storages/KVStore/TiKVHelpers/PDTiKVClient.h +++ b/dbms/src/Storages/KVStore/TiKVHelpers/PDTiKVClient.h @@ -26,11 +26,13 @@ #include #include +#include #include #include #include #include +#include #include #include @@ -71,11 +73,10 @@ struct KeyspaceGCInfo struct PDClientHelper { - static constexpr int get_safepoint_maxtime = 120000; // 120s. waiting pd recover. - // 10 seconds timeout for getting TSO // https://github.com/pingcap/tidb/blob/069631e2ecfedc000ffb92c67207bea81380f020/pkg/store/mockstore/unistore/pd/client.go#L256-L276 static constexpr int get_tso_maxtime = 10'000; + static constexpr int get_safepoint_maxtime = 120'000; static bool enable_safepoint_v2; @@ -116,8 +117,17 @@ struct PDClientHelper const pingcap::pd::ClientPtr & pd_client, KeyspaceID keyspace_id, bool ignore_cache = true, - Int64 safe_point_update_interval_seconds = 30) + Int64 safe_point_update_interval_seconds = 30, + Int64 safe_point_get_max_backoff_ms = 120000) { + UInt64 backoff_count = 0; + auto observe_backoff_count = [&](bool success) { + if (success) + GET_METRIC(tiflash_gc_safepoint_backoff_count, type_success).Observe(backoff_count); + else + GET_METRIC(tiflash_gc_safepoint_backoff_count, type_failure).Observe(backoff_count); + }; + // If keyspace id is `NullspaceID` it need to use safe point v1. if (enable_safepoint_v2 && keyspace_id != NullspaceID) { @@ -136,23 +146,41 @@ struct PDClientHelper const auto min_interval = std::max(static_cast(1), safe_point_update_interval_seconds); // at least one second if (duration.count() < min_interval) + { + observe_backoff_count(true); return cached_gc_safe_point; + } } - pingcap::kv::Backoffer bo(get_safepoint_maxtime); + pingcap::kv::Backoffer bo(std::max(static_cast(0), safe_point_get_max_backoff_ms)); for (;;) { + bool has_pd_response_error = false; try { + GET_METRIC(tiflash_gc_safepoint_request_count, type_get_gc_state).Increment(); auto safe_point = pd_client->getGCSafePoint(); cached_gc_safe_point = safe_point; LOG_TRACE(Logger::get(), "use safe point v1, gc_safe_point={}", safe_point); safe_point_last_update_time = std::chrono::steady_clock::now(); + observe_backoff_count(true); return safe_point; } catch (pingcap::Exception & e) { - bo.backoff(pingcap::kv::boPDRPC, e); + if (!has_pd_response_error) + GET_METRIC(tiflash_gc_safepoint_request_count, type_request_exception).Increment(); + try + { + ++backoff_count; + bo.backoff(pingcap::kv::boPDRPC, e); + } + catch (pingcap::Exception &) + { + GET_METRIC(tiflash_gc_safepoint_request_count, type_backoff_error).Increment(); + observe_backoff_count(false); + throw; + } } } } diff --git a/dbms/src/Storages/StorageDeltaMerge.cpp b/dbms/src/Storages/StorageDeltaMerge.cpp index cecbfaa353a..b1447bd9513 100644 --- a/dbms/src/Storages/StorageDeltaMerge.cpp +++ b/dbms/src/Storages/StorageDeltaMerge.cpp @@ -697,7 +697,8 @@ void checkStartTs(UInt64 start_ts, const Context & context, const String & req_i pd_client, keyspace_id, /* ignore_cache= */ false, - context.getSettingsRef().safe_point_update_interval_seconds); + context.getSettingsRef().safe_point_update_interval_seconds, + context.getSettingsRef().safe_point_get_max_backoff_ms); if (start_ts < safe_point) { throw TiFlashException( diff --git a/dbms/src/TiDB/Schema/SchemaSyncService.cpp b/dbms/src/TiDB/Schema/SchemaSyncService.cpp index 01a614616d9..34e9d59dbea 100644 --- a/dbms/src/TiDB/Schema/SchemaSyncService.cpp +++ b/dbms/src/TiDB/Schema/SchemaSyncService.cpp @@ -101,8 +101,12 @@ void SchemaSyncService::addKeyspaceGCTasks() GET_METRIC(tiflash_schema_trigger_count, type_timer).Increment(); stage = "GC"; - auto gc_safe_point - = PDClientHelper::getGCSafePointWithRetry(context.getTMTContext().getPDClient(), keyspace); + auto gc_safe_point = PDClientHelper::getGCSafePointWithRetry( + context.getTMTContext().getPDClient(), + keyspace, + true, + 30, + context.getSettingsRef().safe_point_get_max_backoff_ms); done_anything = gc(gc_safe_point, keyspace); return done_anything; From 488b30f87bae9b0cb96c5f6b0fad61579cb62e63 Mon Sep 17 00:00:00 2001 From: Shenghui Wu <793703860@qq.com> Date: Thu, 30 Apr 2026 17:40:54 +0800 Subject: [PATCH 43/58] flash: handle TiCI estimate count RPC (#10826) flash: handle GetEstimateTiCICount RPC for TiCI fulltext stats Signed-off-by: wshwsh12 <793703860@qq.com> (cherry picked from commit 4d9b25a7c5c8542acbd1b48c486ffff98934476b) --- dbms/src/Flash/EstimateTiCICountHandler.cpp | 133 +++++++++ dbms/src/Flash/EstimateTiCICountHandler.h | 40 +++ dbms/src/Flash/FlashService.cpp | 15 + dbms/src/Flash/FlashService.h | 4 + .../src/Storages/Tantivy/TantivyInputStream.h | 34 +-- dbms/src/Storages/Tantivy/TiCIReadTaskPool.h | 184 +------------ dbms/src/Storages/Tantivy/TiCIRequestUtils.h | 256 ++++++++++++++++++ 7 files changed, 451 insertions(+), 215 deletions(-) create mode 100644 dbms/src/Flash/EstimateTiCICountHandler.cpp create mode 100644 dbms/src/Flash/EstimateTiCICountHandler.h create mode 100644 dbms/src/Storages/Tantivy/TiCIRequestUtils.h diff --git a/dbms/src/Flash/EstimateTiCICountHandler.cpp b/dbms/src/Flash/EstimateTiCICountHandler.cpp new file mode 100644 index 00000000000..8caeddb6491 --- /dev/null +++ b/dbms/src/Flash/EstimateTiCICountHandler.cpp @@ -0,0 +1,133 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace +{ +TimezoneInfo buildEstimateTimezoneInfo(const coprocessor::TiCIEstimateCountRequest & request) +{ + TimezoneInfo timezone_info; + if (!request.time_zone_name().empty()) + timezone_info.resetByTimezoneName(request.time_zone_name()); + else + timezone_info.resetByTimezoneOffset(request.time_zone_offset()); + return timezone_info; +} + +tipb::FTSQueryInfo parseEstimateQueryInfo(const coprocessor::TiCIEstimateCountRequest & request) +{ + tipb::FTSQueryInfo query_info; + if (!query_info.ParseFromString(request.fts_query_info())) + throw TiFlashException("Failed to parse fts_query_info", Errors::Coprocessor::BadRequest); + if (query_info.match_expr_size() == 0) + throw TiFlashException("Empty TiCI estimate query expression", Errors::Coprocessor::BadRequest); + return query_info; +} + +rust::Vec<::ShardWithRange> buildEstimateShardRanges(const coprocessor::TiCIEstimateCountRequest & request) +{ + rust::Vec<::ShardWithRange> shards; + for (const auto & shard_info : request.shard_infos()) + { + shards.push_back({ + .shard_id = shard_info.shard_id(), + .ranges = TS::getKeyRanges(shard_info.ranges()), + }); + } + return shards; +} +} // namespace + +EstimateTiCICountHandler::EstimateTiCICountHandler( + const coprocessor::TiCIEstimateCountRequest * request_, + coprocessor::TiCIEstimateCountResponse * response_, + const String & identifier) + : request(request_) + , response(response_) + , log(Logger::get(identifier)) +{} + +grpc::Status EstimateTiCICountHandler::execute() +{ + try + { + if (request->shard_infos_size() == 0) + return grpc::Status::OK; + + const auto keyspace_id = RequestUtils::deriveKeyspaceID(request->context()); + const auto fts_query_info = parseEstimateQueryInfo(*request); + const auto timezone_info = buildEstimateTimezoneInfo(*request); + auto [query, column_ids] = TS::tipbToTiCIExpr(fts_query_info.match_expr(), timezone_info); + (void)column_ids; + + auto shard_ranges = buildEstimateShardRanges(*request); + const auto estimate_result = estimate_count(keyspace_id, shard_ranges, query); + response->set_est_count(estimate_result.estimated_total_count); + LOG_DEBUG( + log, + "GetEstimateTiCICount done, est_count={}, input_shards={}, available_shards={}, sampled_shards={}", + response->est_count(), + request->shard_infos_size(), + estimate_result.available_shards, + estimate_result.sampled_shards); + } + catch (const TiFlashException & e) + { + LOG_WARNING( + log, + "GetEstimateTiCICount failed with TiFlash exception: {}\n{}", + e.displayText(), + e.getStackTrace().toString()); + response->set_other_error(e.standardText()); + } + catch (const Exception & e) + { + LOG_WARNING( + log, + "GetEstimateTiCICount failed with DB exception: {}\n{}", + e.message(), + e.getStackTrace().toString()); + response->set_other_error(e.message()); + } + catch (const pingcap::Exception & e) + { + LOG_WARNING(log, "GetEstimateTiCICount failed with KV exception: {}", e.message()); + response->set_other_error(e.message()); + } + catch (const std::exception & e) + { + LOG_WARNING(log, "GetEstimateTiCICount failed: {}", e.what()); + response->set_other_error(e.what()); + } + catch (...) + { + LOG_WARNING(log, "GetEstimateTiCICount failed with unknown exception"); + response->set_other_error("other exception"); + } + + return grpc::Status::OK; +} + +} // namespace DB diff --git a/dbms/src/Flash/EstimateTiCICountHandler.h b/dbms/src/Flash/EstimateTiCICountHandler.h new file mode 100644 index 00000000000..6d116401f5f --- /dev/null +++ b/dbms/src/Flash/EstimateTiCICountHandler.h @@ -0,0 +1,40 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include + +namespace DB +{ +class EstimateTiCICountHandler +{ +public: + EstimateTiCICountHandler( + const coprocessor::TiCIEstimateCountRequest * request_, + coprocessor::TiCIEstimateCountResponse * response_, + const String & identifier); + + grpc::Status execute(); + +private: + const coprocessor::TiCIEstimateCountRequest * request; + coprocessor::TiCIEstimateCountResponse * response; + const LoggerPtr log; +}; + +} // namespace DB diff --git a/dbms/src/Flash/FlashService.cpp b/dbms/src/Flash/FlashService.cpp index 019386c56a6..0bbf984584b 100644 --- a/dbms/src/Flash/FlashService.cpp +++ b/dbms/src/Flash/FlashService.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -882,6 +883,20 @@ grpc::Status FlashService::Compact( return manual_compact_manager->handleRequest(request, response); } +grpc::Status FlashService::GetEstimateTiCICount( + grpc::ServerContext * grpc_context, + const coprocessor::TiCIEstimateCountRequest * request, + coprocessor::TiCIEstimateCountResponse * response) +{ + CPUAffinityManager::getInstance().bindSelfGrpcThread(); + auto check_result = checkGrpcContext(grpc_context); + if (!check_result.ok()) + return check_result; + + EstimateTiCICountHandler handler(request, response, log->identifier()); + return handler.execute(); +} + grpc::Status FlashService::tryAddLock( grpc::ServerContext * grpc_context, const disaggregated::TryAddLockRequest * request, diff --git a/dbms/src/Flash/FlashService.h b/dbms/src/Flash/FlashService.h index 0c87fa5493a..d4323131a6e 100644 --- a/dbms/src/Flash/FlashService.h +++ b/dbms/src/Flash/FlashService.h @@ -100,6 +100,10 @@ class FlashService const kvrpcpb::CompactRequest * request, kvrpcpb::CompactResponse * response) override; + grpc::Status GetEstimateTiCICount( + grpc::ServerContext * grpc_context, + const coprocessor::TiCIEstimateCountRequest * request, + coprocessor::TiCIEstimateCountResponse * response) override; // For S3 Lock Service grpc::Status tryAddLock( diff --git a/dbms/src/Storages/Tantivy/TantivyInputStream.h b/dbms/src/Storages/Tantivy/TantivyInputStream.h index 8340f9d810f..e842317d7fb 100644 --- a/dbms/src/Storages/Tantivy/TantivyInputStream.h +++ b/dbms/src/Storages/Tantivy/TantivyInputStream.h @@ -30,9 +30,8 @@ #include #include #include -#include -#include #include +#include #include #include #include @@ -47,18 +46,6 @@ namespace DB::TS { -// convert literal value from timezone specified in cop request to UTC in-place -inline UInt64 convertPackedU64WithTimezone(UInt64 from_time, const TimezoneInfo & timezone_info) -{ - static const auto & time_zone_utc = DateLUT::instance("UTC"); - UInt64 result_time = from_time; - if (timezone_info.is_name_based) - convertTimeZone(from_time, result_time, *timezone_info.timezone, time_zone_utc); - else if (timezone_info.timezone_offset != 0) - convertTimeZoneByOffset(from_time, result_time, false, timezone_info.timezone_offset); - return result_time; -} - class TantivyInputStream : public IProfilingBlockInputStream { static constexpr auto NAME = "TantivyInputStream"; @@ -460,24 +447,5 @@ class TantivyInputStream : public IProfilingBlockInputStream } return fields; } - - static rust::Vec<::Range> getKeyRanges(ShardInfo::KeyRanges & key_ranges) - { - rust::Vec<::Range> res; - for (const auto & range : key_ranges) - { - rust::Slice<::std::uint8_t const> start( - reinterpret_cast(range.start().c_str()), - range.start().size()); - rust::Slice<::std::uint8_t const> end( - reinterpret_cast(range.end().c_str()), - range.end().size()); - res.push_back({ - .start = std::move(start), - .end = std::move(end), - }); - } - return res; - } }; } // namespace DB::TS diff --git a/dbms/src/Storages/Tantivy/TiCIReadTaskPool.h b/dbms/src/Storages/Tantivy/TiCIReadTaskPool.h index 7d6df3745c1..11174dd45b1 100644 --- a/dbms/src/Storages/Tantivy/TiCIReadTaskPool.h +++ b/dbms/src/Storages/Tantivy/TiCIReadTaskPool.h @@ -17,6 +17,7 @@ #include #include #include +#include namespace DB::TS { @@ -116,7 +117,7 @@ struct TiCIReadTaskPool return_columns.end(), [](const auto & nt, FmtBuffer & fb) { fb.fmtAppend("{}:{}", nt.name, nt.type->getName()); }, ", "); - auto [expr, cids] = tipbToTiCIExpr(match_expr_, timezone_info_); + auto [expr, cids] = TS::tipbToTiCIExpr(match_expr_, timezone_info_); match_expr = std::move(expr); LOG_DEBUG(log, "columns: [{}], match columns: {}", buf.toString(), cids); } @@ -170,187 +171,6 @@ struct TiCIReadTaskPool ::Expr match_expr; bool is_count; std::shared_ptr> shards_snapshot; - - static std::tuple<::Expr, std::vector> tipbToTiCIExpr( - const tipb::Expr & expr, - const TimezoneInfo & timezone_info) - { - ::Expr ret; - switch (expr.tp()) - { - case tipb::ExprType::ScalarFunc: - { - std::vector children_cids; - ret.tp = tipb::ExprType::ScalarFunc; - for (const auto & child : expr.children()) - { - auto [child_expr, child_cids] = tipbToTiCIExpr(child, timezone_info); - ret.children.push_back(child_expr); - children_cids.insert(children_cids.end(), child_cids.begin(), child_cids.end()); - } - switch (expr.sig()) - { - case tipb::ScalarFuncSig::FTSMatchWord: - case tipb::ScalarFuncSig::FTSMatchPrefix: - case tipb::ScalarFuncSig::FTSMatchPhrase: - case tipb::ScalarFuncSig::LogicalAnd: - case tipb::ScalarFuncSig::LogicalOr: - case tipb::ScalarFuncSig::UnaryNotInt: - case tipb::ScalarFuncSig::UnaryNotReal: - case tipb::ScalarFuncSig::EQInt: - case tipb::ScalarFuncSig::NEInt: - case tipb::ScalarFuncSig::LTInt: - case tipb::ScalarFuncSig::LEInt: - case tipb::ScalarFuncSig::GTInt: - case tipb::ScalarFuncSig::GEInt: - case tipb::ScalarFuncSig::EQString: - case tipb::ScalarFuncSig::NEString: - case tipb::ScalarFuncSig::LTString: - case tipb::ScalarFuncSig::LEString: - case tipb::ScalarFuncSig::GTString: - case tipb::ScalarFuncSig::GEString: - case tipb::ScalarFuncSig::EQReal: - case tipb::ScalarFuncSig::NEReal: - case tipb::ScalarFuncSig::LTReal: - case tipb::ScalarFuncSig::LEReal: - case tipb::ScalarFuncSig::GTReal: - case tipb::ScalarFuncSig::GEReal: - case tipb::ScalarFuncSig::EQDecimal: - case tipb::ScalarFuncSig::NEDecimal: - case tipb::ScalarFuncSig::LTDecimal: - case tipb::ScalarFuncSig::LEDecimal: - case tipb::ScalarFuncSig::GTDecimal: - case tipb::ScalarFuncSig::GEDecimal: - case tipb::ScalarFuncSig::InInt: - case tipb::ScalarFuncSig::InString: - case tipb::ScalarFuncSig::InReal: - case tipb::ScalarFuncSig::InDecimal: - ret.sig = expr.sig(); - break; - case tipb::ScalarFuncSig::EQTime: - case tipb::ScalarFuncSig::NETime: - case tipb::ScalarFuncSig::LTTime: - case tipb::ScalarFuncSig::LETime: - case tipb::ScalarFuncSig::GTTime: - case tipb::ScalarFuncSig::GETime: - { - ret.sig = expr.sig(); - size_t col_idx = 0, val_idx = 1; - if (isColumnExpr(expr.children(1))) - std::swap(col_idx, val_idx); - if (expr.children(col_idx).field_type().tp() == TiDB::TypeTimestamp) - { - const auto & child_expr = expr.children(val_idx); - if (isLiteralExpr(child_expr)) - { - UInt64 val = decodeDAGUInt64(child_expr.val()); - val = convertPackedU64WithTimezone(val, timezone_info); - WriteBufferFromOwnString ss; - encodeDAGUInt64(val, ss); - ret.children[val_idx].val.clear(); - auto str = ss.releaseStr(); - std::copy(str.begin(), str.end(), std::back_inserter(ret.children[val_idx].val)); - } - } - break; - } - case tipb::ScalarFuncSig::InTime: - { - ret.sig = expr.sig(); - if (expr.children(0).field_type().tp() == TiDB::TypeTimestamp) - { - for (int val_idx = 1; val_idx < expr.children_size(); ++val_idx) - { - const auto & child_expr = expr.children(val_idx); - if (isLiteralExpr(child_expr)) - { - UInt64 val = decodeDAGUInt64(child_expr.val()); - val = convertPackedU64WithTimezone(val, timezone_info); - WriteBufferFromOwnString ss; - encodeDAGUInt64(val, ss); - ret.children[val_idx].val.clear(); - auto str = ss.releaseStr(); - std::copy(str.begin(), str.end(), std::back_inserter(ret.children[val_idx].val)); - } - else - { - throw TiFlashException( - "InTime only support literal values", - Errors::Coprocessor::BadRequest); - } - } - } - break; - } - default: - throw std::runtime_error("Unsupported expression sig"); - } - - return {ret, children_cids}; - } - case tipb::ExprType::ColumnRef: - { - ret.tp = expr.tp(); - auto id = decodeDAGInt64(expr.val()); - auto str = fmt::format("column_{}", id); - std::copy(str.begin(), str.end(), std::back_inserter(ret.val)); - return {ret, {id}}; - } - case tipb::ExprType::String: - case tipb::ExprType::Int64: - case tipb::ExprType::Uint64: - case tipb::ExprType::Float32: - case tipb::ExprType::Float64: - case tipb::ExprType::MysqlTime: - { - ret.tp = expr.tp(); - std::copy(expr.val().begin(), expr.val().end(), std::back_inserter(ret.val)); - return {ret, {}}; - } - case tipb::ExprType::MysqlDecimal: - { - ret.tp = expr.tp(); - auto field = decodeDAGDecimal(expr.val()); - String str; - if (field.getType() == Field::Types::Decimal32) - str = field.get>().toString(); - else if (field.getType() == Field::Types::Decimal64) - str = field.get>().toString(); - else if (field.getType() == Field::Types::Decimal128) - str = field.get>().toString(); - else if (field.getType() == Field::Types::Decimal256) - str = field.get>().toString(); - else - throw TiFlashException("Not decimal literal" + expr.DebugString(), Errors::Coprocessor::BadRequest); - std::copy(str.begin(), str.end(), std::back_inserter(ret.val)); - return {ret, {}}; - } - default: - throw std::runtime_error("Unsupported expression type"); - } - } - - static std::tuple<::Expr, std::vector> tipbToTiCIExpr( - google::protobuf::RepeatedPtrField exprs, - const TimezoneInfo & tz) - { - if (exprs.empty()) - { - throw std::runtime_error("Empty match expression"); - } - auto [ret, cids] = tipbToTiCIExpr(exprs[0], tz); - for (auto i = 1; i < exprs.size(); ++i) - { - auto [child_expr, child_cids] = tipbToTiCIExpr(exprs[i], tz); - ret = { - .tp = tipb::ExprType::ScalarFunc, - .children = {ret, child_expr}, - .sig = tipb::ScalarFuncSig::LogicalAnd, - }; - cids.insert(cids.end(), child_cids.begin(), child_cids.end()); - } - return {ret, cids}; - } }; using TiCIReadTaskPoolPtr = std::shared_ptr; diff --git a/dbms/src/Storages/Tantivy/TiCIRequestUtils.h b/dbms/src/Storages/Tantivy/TiCIRequestUtils.h new file mode 100644 index 00000000000..07553dea0b4 --- /dev/null +++ b/dbms/src/Storages/Tantivy/TiCIRequestUtils.h @@ -0,0 +1,256 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace DB::TS +{ + +// Convert literal value from timezone specified in cop request to UTC in-place. +inline UInt64 convertPackedU64WithTimezone(UInt64 from_time, const TimezoneInfo & timezone_info) +{ + static const auto & time_zone_utc = DateLUT::instance("UTC"); + UInt64 result_time = from_time; + if (timezone_info.is_name_based) + convertTimeZone(from_time, result_time, *timezone_info.timezone, time_zone_utc); + else if (timezone_info.timezone_offset != 0) + convertTimeZoneByOffset(from_time, result_time, false, timezone_info.timezone_offset); + return result_time; +} + +inline rust::Vec<::Range> getKeyRanges(const ShardInfo::KeyRanges & key_ranges) +{ + rust::Vec<::Range> res; + for (const auto & range : key_ranges) + { + rust::Slice<::std::uint8_t const> start( + reinterpret_cast(range.start().c_str()), + range.start().size()); + rust::Slice<::std::uint8_t const> end( + reinterpret_cast(range.end().c_str()), + range.end().size()); + res.push_back({ + .start = std::move(start), + .end = std::move(end), + }); + } + return res; +} + +inline std::tuple<::Expr, std::vector> tipbToTiCIExpr( + const tipb::Expr & expr, + const TimezoneInfo & timezone_info) +{ + ::Expr ret; + switch (expr.tp()) + { + case tipb::ExprType::ScalarFunc: + { + std::vector children_cids; + ret.tp = tipb::ExprType::ScalarFunc; + for (const auto & child : expr.children()) + { + auto [child_expr, child_cids] = tipbToTiCIExpr(child, timezone_info); + ret.children.push_back(child_expr); + children_cids.insert(children_cids.end(), child_cids.begin(), child_cids.end()); + } + switch (expr.sig()) + { + case tipb::ScalarFuncSig::FTSMatchWord: + case tipb::ScalarFuncSig::FTSMatchPrefix: + case tipb::ScalarFuncSig::FTSMatchPhrase: + case tipb::ScalarFuncSig::LogicalAnd: + case tipb::ScalarFuncSig::LogicalOr: + case tipb::ScalarFuncSig::UnaryNotInt: + case tipb::ScalarFuncSig::UnaryNotReal: + case tipb::ScalarFuncSig::EQInt: + case tipb::ScalarFuncSig::NEInt: + case tipb::ScalarFuncSig::LTInt: + case tipb::ScalarFuncSig::LEInt: + case tipb::ScalarFuncSig::GTInt: + case tipb::ScalarFuncSig::GEInt: + case tipb::ScalarFuncSig::EQString: + case tipb::ScalarFuncSig::NEString: + case tipb::ScalarFuncSig::LTString: + case tipb::ScalarFuncSig::LEString: + case tipb::ScalarFuncSig::GTString: + case tipb::ScalarFuncSig::GEString: + case tipb::ScalarFuncSig::EQReal: + case tipb::ScalarFuncSig::NEReal: + case tipb::ScalarFuncSig::LTReal: + case tipb::ScalarFuncSig::LEReal: + case tipb::ScalarFuncSig::GTReal: + case tipb::ScalarFuncSig::GEReal: + case tipb::ScalarFuncSig::EQDecimal: + case tipb::ScalarFuncSig::NEDecimal: + case tipb::ScalarFuncSig::LTDecimal: + case tipb::ScalarFuncSig::LEDecimal: + case tipb::ScalarFuncSig::GTDecimal: + case tipb::ScalarFuncSig::GEDecimal: + case tipb::ScalarFuncSig::InInt: + case tipb::ScalarFuncSig::InString: + case tipb::ScalarFuncSig::InReal: + case tipb::ScalarFuncSig::InDecimal: + ret.sig = expr.sig(); + break; + case tipb::ScalarFuncSig::EQTime: + case tipb::ScalarFuncSig::NETime: + case tipb::ScalarFuncSig::LTTime: + case tipb::ScalarFuncSig::LETime: + case tipb::ScalarFuncSig::GTTime: + case tipb::ScalarFuncSig::GETime: + { + ret.sig = expr.sig(); + size_t col_idx = 0, val_idx = 1; + if (isColumnExpr(expr.children(1))) + std::swap(col_idx, val_idx); + if (expr.children(col_idx).field_type().tp() == TiDB::TypeTimestamp) + { + const auto & child_expr = expr.children(val_idx); + if (isLiteralExpr(child_expr)) + { + UInt64 val = decodeDAGUInt64(child_expr.val()); + val = convertPackedU64WithTimezone(val, timezone_info); + WriteBufferFromOwnString ss; + encodeDAGUInt64(val, ss); + ret.children[val_idx].val.clear(); + auto str = ss.releaseStr(); + std::copy(str.begin(), str.end(), std::back_inserter(ret.children[val_idx].val)); + } + } + break; + } + case tipb::ScalarFuncSig::InTime: + { + ret.sig = expr.sig(); + if (expr.children(0).field_type().tp() == TiDB::TypeTimestamp) + { + for (int val_idx = 1; val_idx < expr.children_size(); ++val_idx) + { + const auto & child_expr = expr.children(val_idx); + if (isLiteralExpr(child_expr)) + { + UInt64 val = decodeDAGUInt64(child_expr.val()); + val = convertPackedU64WithTimezone(val, timezone_info); + WriteBufferFromOwnString ss; + encodeDAGUInt64(val, ss); + ret.children[val_idx].val.clear(); + auto str = ss.releaseStr(); + std::copy(str.begin(), str.end(), std::back_inserter(ret.children[val_idx].val)); + } + else + { + throw TiFlashException("InTime only support literal values", Errors::Coprocessor::BadRequest); + } + } + } + break; + } + default: + throw std::runtime_error(fmt::format( + "Unsupported expression sig: tp={}, sig={}, expr={}", + static_cast(expr.tp()), + static_cast(expr.sig()), + expr.DebugString())); + } + + return {ret, children_cids}; + } + case tipb::ExprType::ColumnRef: + { + ret.tp = expr.tp(); + auto id = decodeDAGInt64(expr.val()); + auto str = fmt::format("column_{}", id); + std::copy(str.begin(), str.end(), std::back_inserter(ret.val)); + return {ret, {id}}; + } + case tipb::ExprType::String: + case tipb::ExprType::Int64: + case tipb::ExprType::Uint64: + case tipb::ExprType::Float32: + case tipb::ExprType::Float64: + case tipb::ExprType::MysqlTime: + { + ret.tp = expr.tp(); + std::copy(expr.val().begin(), expr.val().end(), std::back_inserter(ret.val)); + return {ret, {}}; + } + case tipb::ExprType::MysqlDecimal: + { + ret.tp = expr.tp(); + auto field = decodeDAGDecimal(expr.val()); + String str; + if (field.getType() == Field::Types::Decimal32) + str = field.get>().toString(); + else if (field.getType() == Field::Types::Decimal64) + str = field.get>().toString(); + else if (field.getType() == Field::Types::Decimal128) + str = field.get>().toString(); + else if (field.getType() == Field::Types::Decimal256) + str = field.get>().toString(); + else + throw TiFlashException("Not decimal literal: " + expr.DebugString(), Errors::Coprocessor::BadRequest); + std::copy(str.begin(), str.end(), std::back_inserter(ret.val)); + return {ret, {}}; + } + default: + throw std::runtime_error(fmt::format( + "Unsupported expression type: tp={}, expr={}", + static_cast(expr.tp()), + expr.DebugString())); + } +} + +inline std::tuple<::Expr, std::vector> tipbToTiCIExpr( + const google::protobuf::RepeatedPtrField & exprs, + const TimezoneInfo & tz) +{ + if (exprs.empty()) + throw std::runtime_error("Empty match expression"); + auto [ret, cids] = tipbToTiCIExpr(exprs[0], tz); + for (auto i = 1; i < exprs.size(); ++i) + { + auto [child_expr, child_cids] = tipbToTiCIExpr(exprs[i], tz); + ret = { + .tp = tipb::ExprType::ScalarFunc, + .children = {ret, child_expr}, + .sig = tipb::ScalarFuncSig::LogicalAnd, + }; + cids.insert(cids.end(), child_cids.begin(), child_cids.end()); + } + return {ret, cids}; +} + +} // namespace DB::TS From 851f49f9267379623375a178f4dd6ad841520786 Mon Sep 17 00:00:00 2001 From: Calvin Neo Date: Fri, 8 May 2026 12:49:20 +0800 Subject: [PATCH 44/58] *: reduce getGCState freq (#10817) (#10836) (cherry picked from commit a7d3668d6e46366570e60af4deedafb1bda74ef6) --- dbms/src/Common/TiFlashMetrics.h | 1 + dbms/src/Debug/dbgFuncSchema.cpp | 1 - .../dbgKVStore/dbgFuncMockRaftSnapshot.cpp | 4 +- .../DeltaMerge/DeltaMergeStore_InternalBg.cpp | 1 - .../KVStore/MultiRaft/PrehandleSnapshot.cpp | 1 - .../KVStore/TiKVHelpers/PDTiKVClient.h | 50 +++++++++++++++---- dbms/src/Storages/StorageDeltaMerge.cpp | 7 ++- dbms/src/TiDB/Schema/SchemaSyncService.cpp | 1 - 8 files changed, 48 insertions(+), 18 deletions(-) diff --git a/dbms/src/Common/TiFlashMetrics.h b/dbms/src/Common/TiFlashMetrics.h index 06bddee9a09..b658bfcaea1 100644 --- a/dbms/src/Common/TiFlashMetrics.h +++ b/dbms/src/Common/TiFlashMetrics.h @@ -194,6 +194,7 @@ static_assert(RAFT_REGION_BIG_WRITE_THRES * 4 < RAFT_REGION_BIG_WRITE_MAX, "Inva Counter, \ F(type_get_gc_state, {{"type", "get_gc_state"}}), \ F(type_zero_gc_safe_point, {{"type", "zero_gc_safe_point"}}), \ + F(type_rewind, {{"type", "rewind"}}), \ F(type_pd_response_error, {{"type", "pd_response_error"}}), \ F(type_request_exception, {{"type", "request_exception"}}), \ F(type_backoff_error, {{"type", "backoff_error"}})) \ diff --git a/dbms/src/Debug/dbgFuncSchema.cpp b/dbms/src/Debug/dbgFuncSchema.cpp index 0a726f679e1..51a246a0131 100644 --- a/dbms/src/Debug/dbgFuncSchema.cpp +++ b/dbms/src/Debug/dbgFuncSchema.cpp @@ -186,7 +186,6 @@ void dbgFuncGcSchemas(Context & context, const ASTs & args, DBGInvoker::Printer gc_safe_point = PDClientHelper::getGCSafePointWithRetry( context.getTMTContext().getPDClient(), NullspaceID, - true, 30, context.getSettingsRef().safe_point_get_max_backoff_ms); if (!args.empty()) diff --git a/dbms/src/Debug/dbgKVStore/dbgFuncMockRaftSnapshot.cpp b/dbms/src/Debug/dbgKVStore/dbgFuncMockRaftSnapshot.cpp index baefef60223..12dbb5dda1b 100644 --- a/dbms/src/Debug/dbgKVStore/dbgFuncMockRaftSnapshot.cpp +++ b/dbms/src/Debug/dbgKVStore/dbgFuncMockRaftSnapshot.cpp @@ -547,8 +547,8 @@ RegionPtrWithBlock::CachePtr GenRegionPreDecodeBlockData(const RegionPtr & regio gc_safe_point = PDClientHelper::getGCSafePointWithRetry( pd_client, keyspace_id, - false, - context.getSettingsRef().safe_point_update_interval_seconds); + context.getSettingsRef().safe_point_update_interval_seconds, + context.getSettingsRef().safe_point_get_max_backoff_ms); } /** * In 5.0.1, feature `compaction filter` is enabled by default. Under such feature tikv will do gc in write & default cf individually. diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore_InternalBg.cpp b/dbms/src/Storages/DeltaMerge/DeltaMergeStore_InternalBg.cpp index 1760c744bd5..f48b7f0e10d 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore_InternalBg.cpp +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore_InternalBg.cpp @@ -372,7 +372,6 @@ bool DeltaMergeStore::updateGCSafePoint() auto safe_point = PDClientHelper::getGCSafePointWithRetry( pd_client, keyspace_id, - /* ignore_cache= */ false, global_context.getSettingsRef().safe_point_update_interval_seconds, global_context.getSettingsRef().safe_point_get_max_backoff_ms); latest_gc_safe_point.store(safe_point, std::memory_order_release); diff --git a/dbms/src/Storages/KVStore/MultiRaft/PrehandleSnapshot.cpp b/dbms/src/Storages/KVStore/MultiRaft/PrehandleSnapshot.cpp index 81c7420971a..5d062deccde 100644 --- a/dbms/src/Storages/KVStore/MultiRaft/PrehandleSnapshot.cpp +++ b/dbms/src/Storages/KVStore/MultiRaft/PrehandleSnapshot.cpp @@ -715,7 +715,6 @@ PrehandleResult KVStore::preHandleSSTsToDTFiles( gc_safepoint = PDClientHelper::getGCSafePointWithRetry( pd_client, keyspace_id, - /* ignore_cache= */ false, context.getSettingsRef().safe_point_update_interval_seconds, context.getSettingsRef().safe_point_get_max_backoff_ms); } diff --git a/dbms/src/Storages/KVStore/TiKVHelpers/PDTiKVClient.h b/dbms/src/Storages/KVStore/TiKVHelpers/PDTiKVClient.h index 24dc1723268..443c45a88d1 100644 --- a/dbms/src/Storages/KVStore/TiKVHelpers/PDTiKVClient.h +++ b/dbms/src/Storages/KVStore/TiKVHelpers/PDTiKVClient.h @@ -46,6 +46,14 @@ namespace FailPoints extern const char force_pd_grpc_error[]; } // namespace FailPoints +enum class GCSafepointFetchStrategy +{ + // Query paths consume the last value observed by a non-query caller. + CacheOnly, + // Non-query paths may refresh the value from PD when needed. + UpdateCacheIfNeeded, +}; + struct KeyspaceGCInfo { DB::Timestamp ks_gc_sp{}; @@ -116,9 +124,9 @@ struct PDClientHelper static Timestamp getGCSafePointWithRetry( const pingcap::pd::ClientPtr & pd_client, KeyspaceID keyspace_id, - bool ignore_cache = true, Int64 safe_point_update_interval_seconds = 30, - Int64 safe_point_get_max_backoff_ms = 120000) + Int64 safe_point_get_max_backoff_ms = 120000, + GCSafepointFetchStrategy fetch_strategy = GCSafepointFetchStrategy::UpdateCacheIfNeeded) { UInt64 backoff_count = 0; auto observe_backoff_count = [&](bool success) { @@ -128,16 +136,33 @@ struct PDClientHelper GET_METRIC(tiflash_gc_safepoint_backoff_count, type_failure).Observe(backoff_count); }; - // If keyspace id is `NullspaceID` it need to use safe point v1. + if (fetch_strategy == GCSafepointFetchStrategy::CacheOnly) + { + if (enable_safepoint_v2 && keyspace_id != NullspaceID) + { + auto cached = getKeyspaceGCSafepoint(keyspace_id); + observe_backoff_count(true); + return cached.ks_gc_sp; + } + observe_backoff_count(true); + return cached_gc_safe_point; + } + + // If keyspace id is `NullspaceID` it needs to use safe point v1. if (enable_safepoint_v2 && keyspace_id != NullspaceID) { auto gc_safe_point - = getGCSafePointV2WithRetry(pd_client, keyspace_id, ignore_cache, safe_point_update_interval_seconds); + = getGCSafePointV2WithRetry( + pd_client, + keyspace_id, + false, + safe_point_update_interval_seconds, + safe_point_get_max_backoff_ms); LOG_TRACE(Logger::get(), "use safe point v2, keyspace={} gc_safe_point={}", keyspace_id, gc_safe_point); return gc_safe_point; } - if (!ignore_cache) + if (safe_point_update_interval_seconds > 0) { // In case we cost too much to update safe point from PD. auto now = std::chrono::steady_clock::now(); @@ -160,7 +185,8 @@ struct PDClientHelper { GET_METRIC(tiflash_gc_safepoint_request_count, type_get_gc_state).Increment(); auto safe_point = pd_client->getGCSafePoint(); - cached_gc_safe_point = safe_point; + cached_gc_safe_point.store( + std::max(cached_gc_safe_point.load(), safe_point), std::memory_order_release); LOG_TRACE(Logger::get(), "use safe point v1, gc_safe_point={}", safe_point); safe_point_last_update_time = std::chrono::steady_clock::now(); observe_backoff_count(true); @@ -189,7 +215,8 @@ struct PDClientHelper const pingcap::pd::ClientPtr & pd_client, KeyspaceID keyspace_id, bool ignore_cache = false, - Int64 safe_point_update_interval_seconds = 30) + Int64 safe_point_update_interval_seconds = 30, + Int64 safe_point_get_max_backoff_ms = 120000) { if (!ignore_cache) { @@ -207,14 +234,14 @@ struct PDClientHelper } } - pingcap::kv::Backoffer bo(get_safepoint_maxtime); + pingcap::kv::Backoffer bo(std::max(static_cast(0), safe_point_get_max_backoff_ms)); for (;;) { try { auto ks_gc_sp = pd_client->getGCSafePointV2(keyspace_id); updateKeyspaceGCSafepointMap(keyspace_id, ks_gc_sp); - return ks_gc_sp; + return getKeyspaceGCSafepoint(keyspace_id).ks_gc_sp; } catch (pingcap::Exception & e) { @@ -227,7 +254,10 @@ struct PDClientHelper { std::unique_lock lock(ks_gc_sp_mutex); KeyspaceGCInfo new_keyspace_gc_info; - new_keyspace_gc_info.ks_gc_sp = ks_gc_sp; + const auto iter = ks_gc_sp_map.find(keyspace_id); + new_keyspace_gc_info.ks_gc_sp = iter == ks_gc_sp_map.end() + ? ks_gc_sp + : std::max(iter->second.ks_gc_sp, ks_gc_sp); new_keyspace_gc_info.ks_gc_sp_update_time = std::chrono::steady_clock::now(); ks_gc_sp_map[keyspace_id] = new_keyspace_gc_info; } diff --git a/dbms/src/Storages/StorageDeltaMerge.cpp b/dbms/src/Storages/StorageDeltaMerge.cpp index b1447bd9513..089f87a360d 100644 --- a/dbms/src/Storages/StorageDeltaMerge.cpp +++ b/dbms/src/Storages/StorageDeltaMerge.cpp @@ -693,12 +693,15 @@ void checkStartTs(UInt64 start_ts, const Context & context, const String & req_i auto pd_client = tmt.getPDClient(); if (unlikely(pd_client->isMock())) return; + // Query paths only consume the last safepoint observed by TiFlash. Cache + // advancement is still handled by the existing non-query callers so that + // PD traffic does not scale with read concurrency. auto safe_point = PDClientHelper::getGCSafePointWithRetry( pd_client, keyspace_id, - /* ignore_cache= */ false, context.getSettingsRef().safe_point_update_interval_seconds, - context.getSettingsRef().safe_point_get_max_backoff_ms); + context.getSettingsRef().safe_point_get_max_backoff_ms, + GCSafepointFetchStrategy::CacheOnly); if (start_ts < safe_point) { throw TiFlashException( diff --git a/dbms/src/TiDB/Schema/SchemaSyncService.cpp b/dbms/src/TiDB/Schema/SchemaSyncService.cpp index 34e9d59dbea..872c4675731 100644 --- a/dbms/src/TiDB/Schema/SchemaSyncService.cpp +++ b/dbms/src/TiDB/Schema/SchemaSyncService.cpp @@ -104,7 +104,6 @@ void SchemaSyncService::addKeyspaceGCTasks() auto gc_safe_point = PDClientHelper::getGCSafePointWithRetry( context.getTMTContext().getPDClient(), keyspace, - true, 30, context.getSettingsRef().safe_point_get_max_backoff_ms); done_anything = gc(gc_safe_point, keyspace); From 3721a20777d7e1e0434f29c9d8b7b4dedd69f3b5 Mon Sep 17 00:00:00 2001 From: jinhelin Date: Sun, 10 May 2026 11:55:11 +0800 Subject: [PATCH 45/58] contrib: update tici (#10840) (cherry picked from commit a443b89c58378f69880b7ee345106ce654f74c12) --- tests/docker/util.sh | 69 ++++++++++++++++++++++++++++++++++++ tests/fullstack-test/run.sh | 5 +-- tests/fullstack-test2/run.sh | 6 ++-- tests/tidb-ci/run.sh | 12 +++---- 4 files changed, 81 insertions(+), 11 deletions(-) diff --git a/tests/docker/util.sh b/tests/docker/util.sh index cb050a8cda3..97749c6f520 100644 --- a/tests/docker/util.sh +++ b/tests/docker/util.sh @@ -104,6 +104,73 @@ function clean_data_log() { rm -rf ./data ./log } +# The release-8.5 test scripts invoke docker-compose directly. Keep the +# helper usable when they are run outside the newer CI bootstrap that exports +# COMPOSE. +if [[ -z "${COMPOSE:-}" ]]; then + export COMPOSE="docker-compose" +fi + +# feature/fts TiFlash is 8.5.x, while CI may pair it with master PD/TiKV/TiDB. +# Pin PD's compatibility gate before TiFlash registers itself to PD. +function set_pd_cluster_version_for_tiflash() { + local cluster_version="${TIFLASH_TEST_PD_CLUSTER_VERSION:-8.5.6}" + if [[ -z "${cluster_version}" || "${cluster_version}" == "0" || "${cluster_version}" == "false" ]]; then + return + fi + + cluster_version="${cluster_version#v}" + if [[ ! "${cluster_version}" =~ ^[0-9]+\.[0-9]+\.[0-9]+([.-][0-9A-Za-z.-]+)?$ ]]; then + echo "Invalid TIFLASH_TEST_PD_CLUSTER_VERSION: ${TIFLASH_TEST_PD_CLUSTER_VERSION}" >&2 + exit 1 + fi + + local timeout="${TIFLASH_TEST_PD_CLUSTER_VERSION_TIMEOUT:-60}" + local failed='true' + + echo "=> set PD cluster-version to ${cluster_version} for TiFlash test" + + for (( i = 0; i < "${timeout}"; i++ )); do + local output='' + if output=$(${COMPOSE} "$@" exec -T pd0 /pd-ctl -u http://127.0.0.1:2379 config set cluster-version "${cluster_version}" 2>&1); then + echo "${output}" + if echo "${output}" | grep -q "Success!"; then + local current_version='' + current_version=$(${COMPOSE} "$@" exec -T pd0 /pd-ctl -u http://127.0.0.1:2379 config show cluster-version 2>&1 || true) + echo "${current_version}" + if echo "${current_version}" | grep -Fq "${cluster_version}"; then + failed='false' + break + fi + fi + else + echo "${output}" + fi + + if [ $((${i} % 10)) = 0 ] && [ ${i} -ge 10 ]; then + echo " #${i} waiting for PD to set cluster-version" + fi + + sleep 1 + done + + if [ "${failed}" == 'true' ]; then + echo " can not set PD cluster-version" >&2 + exit 1 + fi +} + +function start_cluster_with_tiflash() { + local compose_args=() + for compose_file in "$@"; do + compose_args+=("-f" "${compose_file}") + done + + ${COMPOSE} "${compose_args[@]}" up -d pd0 tikv0 tidb0 + set_pd_cluster_version_for_tiflash "${compose_args[@]}" + ${COMPOSE} "${compose_args[@]}" up -d tiflash0 +} + function check_env() { local cur_dir=$(pwd) if [[ ! -d ${cur_dir}/../../tests/.build/tiflash ]]; then @@ -117,4 +184,6 @@ export -f wait_env export -f wait_tiflash_env export -f set_branch export -f clean_data_log +export -f set_pd_cluster_version_for_tiflash +export -f start_cluster_with_tiflash export -f check_env diff --git a/tests/fullstack-test/run.sh b/tests/fullstack-test/run.sh index 08428dfc476..88a9fa4d20b 100755 --- a/tests/fullstack-test/run.sh +++ b/tests/fullstack-test/run.sh @@ -25,14 +25,15 @@ check_env docker-compose -f cluster.yaml -f tiflash-dt.yaml down clean_data_log -docker-compose -f cluster.yaml -f tiflash-dt.yaml up -d +# run fullstack-tests +start_cluster_with_tiflash cluster.yaml tiflash-dt.yaml wait_env docker-compose -f cluster.yaml -f tiflash-dt.yaml exec -T tiflash0 bash -c 'cd /tests ; ./run-test.sh fullstack-test' docker-compose -f cluster.yaml -f tiflash-dt.yaml down clean_data_log -docker-compose -f cluster.yaml -f tiflash-dt-sync-grpc.yaml up -d +start_cluster_with_tiflash cluster.yaml tiflash-dt-sync-grpc.yaml wait_env docker-compose -f cluster.yaml -f tiflash-dt-sync-grpc.yaml exec -T tiflash0 bash -c 'cd /tests ; ./run-test.sh fullstack-test/mpp' diff --git a/tests/fullstack-test2/run.sh b/tests/fullstack-test2/run.sh index 1936852fd69..f00c793b000 100755 --- a/tests/fullstack-test2/run.sh +++ b/tests/fullstack-test2/run.sh @@ -26,7 +26,7 @@ docker-compose -f cluster.yaml -f tiflash-dt.yaml down clean_data_log # FIXME: now vector does not support run with encryption-at-rest enabled -docker-compose -f cluster.yaml -f tiflash-dt-disable-encrypt.yaml up -d +start_cluster_with_tiflash cluster.yaml tiflash-dt-disable-encrypt.yaml wait_env docker-compose -f cluster.yaml -f tiflash-dt-disable-encrypt.yaml exec -T tiflash0 bash -c 'cd /tests ; ./run-test.sh fullstack-test-vector' @@ -34,14 +34,14 @@ docker-compose -f cluster.yaml -f tiflash-dt-disable-encrypt.yaml down clean_data_log -docker-compose -f cluster.yaml -f tiflash-dt.yaml up -d +start_cluster_with_tiflash cluster.yaml tiflash-dt.yaml wait_env docker-compose -f cluster.yaml -f tiflash-dt.yaml exec -T tiflash0 bash -c 'cd /tests ; ./run-test.sh fullstack-test2 true && ./run-test.sh fullstack-test-dt' docker-compose -f cluster.yaml -f tiflash-dt.yaml down clean_data_log -docker-compose -f cluster.yaml -f tiflash-dt-disable-local-tunnel.yaml up -d +start_cluster_with_tiflash cluster.yaml tiflash-dt-disable-local-tunnel.yaml wait_env docker-compose -f cluster.yaml -f tiflash-dt-disable-local-tunnel.yaml exec -T tiflash0 bash -c 'cd /tests ; ./run-test.sh fullstack-test/mpp' diff --git a/tests/tidb-ci/run.sh b/tests/tidb-ci/run.sh index c590575fa2d..955da500676 100755 --- a/tests/tidb-ci/run.sh +++ b/tests/tidb-ci/run.sh @@ -25,7 +25,7 @@ check_env docker-compose -f cluster.yaml -f tiflash-dt.yaml down clean_data_log -docker-compose -f cluster.yaml -f tiflash-dt.yaml up -d +start_cluster_with_tiflash cluster.yaml tiflash-dt.yaml wait_env docker-compose -f cluster.yaml -f tiflash-dt.yaml exec -T tiflash0 bash -c 'cd /tests ; ./run-test.sh tidb-ci/fail-point-tests && ./run-test.sh tidb-ci/fullstack-test true && ./run-test.sh tidb-ci/fullstack-test-dt' @@ -36,7 +36,7 @@ clean_data_log docker-compose -f cluster_new_collation.yaml -f tiflash-dt.yaml down clean_data_log -docker-compose -f cluster_new_collation.yaml -f tiflash-dt.yaml up -d +start_cluster_with_tiflash cluster_new_collation.yaml tiflash-dt.yaml wait_env docker-compose -f cluster_new_collation.yaml -f tiflash-dt.yaml exec -T tiflash0 bash -c 'cd /tests ; ./run-test.sh tidb-ci/new_collation_fullstack' @@ -47,7 +47,7 @@ clean_data_log docker-compose -f cluster_disable_new_collation.yaml -f tiflash-dt.yaml down clean_data_log -docker-compose -f cluster_disable_new_collation.yaml -f tiflash-dt.yaml up -d +start_cluster_with_tiflash cluster_disable_new_collation.yaml tiflash-dt.yaml wait_env docker-compose -f cluster_disable_new_collation.yaml -f tiflash-dt.yaml exec -T tiflash0 bash -c 'cd /tests ; ./run-test.sh tidb-ci/disable_new_collation_fullstack' @@ -55,15 +55,15 @@ docker-compose -f cluster_disable_new_collation.yaml -f tiflash-dt.yaml down clean_data_log # run force_enable_lm tests -docker-compose -f cluster.yaml -f tiflash-dt-force-enable-lm.yaml up -d +start_cluster_with_tiflash cluster.yaml tiflash-dt-force-enable-lm.yaml wait_env docker-compose -f cluster.yaml -f tiflash-dt-force-enable-lm.yaml exec -T tiflash0 bash -c 'cd /tests ; ./run-test.sh tidb-ci/force_enable_lm' docker-compose -f cluster.yaml -f tiflash-dt-force-enable-lm.yaml down clean_data_log -# run lighweight compression tests -docker-compose -f cluster.yaml -f tiflash-dt-lightweight-compression.yaml up -d +# run lightweight compression tests +start_cluster_with_tiflash cluster.yaml tiflash-dt-lightweight-compression.yaml wait_env docker-compose -f cluster.yaml -f tiflash-dt-lightweight-compression.yaml exec -T tiflash0 bash -c 'cd /tests ; ./run-test.sh tidb-ci/lightweight_compression' From 77df106c0de9f90ea646f41a9c5cdbc0c2504f58 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Wed, 2 Sep 2026 19:45:39 +0800 Subject: [PATCH 46/58] contrib: pin final FTS dependency snapshot for release-8.5 Pin the FTS backport dependencies to the audited release-8.5 snapshot: - tici: 437cbf0d02146a754b7bc20d1ce5ec0841fb9364 - client-c: 80276b5f98c72badcf3b8a48e793d7e56d7fc31a - kvproto: 811b5283cd8a7f8b15be83eb483cb07f1db68639 - tipb: 1852f9829ce3d3962895fec43f908b31fbdc58fb This collapses the 48 pin-only commits while retaining their final dependency state. --- contrib/client-c | 2 +- contrib/kvproto | 2 +- contrib/tici | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/client-c b/contrib/client-c index e20fd904f60..80276b5f98c 160000 --- a/contrib/client-c +++ b/contrib/client-c @@ -1 +1 @@ -Subproject commit e20fd904f603d11b70a0817d5c0126d99cf48db7 +Subproject commit 80276b5f98c72badcf3b8a48e793d7e56d7fc31a diff --git a/contrib/kvproto b/contrib/kvproto index bdcab1db9be..811b5283cd8 160000 --- a/contrib/kvproto +++ b/contrib/kvproto @@ -1 +1 @@ -Subproject commit bdcab1db9bef9fb72278d4884733b0f565f65d6a +Subproject commit 811b5283cd8a7f8b15be83eb483cb07f1db68639 diff --git a/contrib/tici b/contrib/tici index cec1fcd7ad2..437cbf0d021 160000 --- a/contrib/tici +++ b/contrib/tici @@ -1 +1 @@ -Subproject commit cec1fcd7ad2c3f369f1e432f7cf0bc63fcf3a44e +Subproject commit 437cbf0d02146a754b7bc20d1ce5ec0841fb9364 From c87864f6b6da0cabd0f7f77da7bcd4ae95a7e057 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Wed, 2 Sep 2026 19:46:38 +0800 Subject: [PATCH 47/58] style: clean FTS backport whitespace --- contrib/CMakeLists.txt | 2 +- contrib/tici-search-lib/CMakeLists.txt | 7 +++---- dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.cpp | 8 ++++---- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index f9ab57e3c80..8d0e708b421 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -208,4 +208,4 @@ if (ENABLE_TESTS AND NOT CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG") add_subdirectory(highfive-cmake) endif () -add_subdirectory(tici-search-lib) \ No newline at end of file +add_subdirectory(tici-search-lib) diff --git a/contrib/tici-search-lib/CMakeLists.txt b/contrib/tici-search-lib/CMakeLists.txt index 413a1886d2c..16cdb407700 100644 --- a/contrib/tici-search-lib/CMakeLists.txt +++ b/contrib/tici-search-lib/CMakeLists.txt @@ -1,16 +1,16 @@ set(TICI_PROJECT_DIR "${CMAKE_SOURCE_DIR}/contrib/tici") -set(TICI_LIB_NAME "tici_search_lib") +set(TICI_LIB_NAME "tici_search_lib") set(TICI_LIB "${CMAKE_CURRENT_BINARY_DIR}/release/${CMAKE_STATIC_LIBRARY_PREFIX}${TICI_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}") file(GLOB LIB_SOURCE_FILES "${TICI_PROJECT_DIR}/src/*") file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/cxxbridge) add_custom_command( - OUTPUT ${TICI_LIB} + OUTPUT ${TICI_LIB} COMMAND cargo build -p tici-search-lib --release --target-dir ${CMAKE_CURRENT_BINARY_DIR} --manifest-path ${TICI_PROJECT_DIR}/Cargo.toml WORKING_DIRECTORY ${TICI_PROJECT_DIR} DEPENDS ${LIB_SOURCE_FILES} - COMMENT "Build Rust lib"${CMAKE_CURRENT_BINARY_DIR} + COMMENT "Build Rust lib" ) add_custom_target(rustbuild ALL DEPENDS ${TICI_LIB}) @@ -37,4 +37,3 @@ endif() target_include_directories(tici_search_lib INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/cxxbridge) - diff --git a/dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.cpp b/dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.cpp index 894977e3e33..55288c74391 100644 --- a/dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.cpp +++ b/dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.cpp @@ -56,10 +56,10 @@ NamesWithAliases buildTableScanProjectionCols( if (unlikely(!table_scan_col_type->equals(*storage_col_type))) throw TiFlashException( fmt::format( - R"(The data type {} from tidb table scan schema is different from the data type {} from tiflash storage schema, - table id is {}, - column index is {}, - column name from tidb table scan is {}, + R"(The data type {} from tidb table scan schema is different from the data type {} from tiflash storage schema, + table id is {}, + column index is {}, + column name from tidb table scan is {}, column name from tiflash storage is {})", table_scan_col_type->getName(), storage_col_type->getName(), From f710e7f535953d3ea532c75b5a92959c6b094343 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Wed, 2 Sep 2026 19:54:07 +0800 Subject: [PATCH 48/58] fix: make TiCI safepoint cache reads side-effect free --- dbms/src/Storages/KVStore/TiKVHelpers/PDTiKVClient.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dbms/src/Storages/KVStore/TiKVHelpers/PDTiKVClient.h b/dbms/src/Storages/KVStore/TiKVHelpers/PDTiKVClient.h index 443c45a88d1..65c18f0c5fa 100644 --- a/dbms/src/Storages/KVStore/TiKVHelpers/PDTiKVClient.h +++ b/dbms/src/Storages/KVStore/TiKVHelpers/PDTiKVClient.h @@ -35,6 +35,8 @@ #include #include #include +#include +#include using TimePoint = std::atomic>; @@ -265,7 +267,8 @@ struct PDClientHelper static KeyspaceGCInfo getKeyspaceGCSafepoint(KeyspaceID keyspace_id) { std::shared_lock lock(ks_gc_sp_mutex); - return ks_gc_sp_map[keyspace_id]; + const auto iter = ks_gc_sp_map.find(keyspace_id); + return iter == ks_gc_sp_map.end() ? KeyspaceGCInfo{} : iter->second; } static void removeKeyspaceGCSafepoint(KeyspaceID keyspace_id) From 2ccd3d8bc8d3f79b3fee18e713ce6a999d6c7aa1 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Wed, 2 Sep 2026 19:57:12 +0800 Subject: [PATCH 49/58] build: add TiCI wrapper translation unit The release-8.5 tree does not contain the libclara helper introduced on master. Keep only the empty translation unit required by contrib/tici-search-lib. --- libs/libclara-cmake/dummy.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 libs/libclara-cmake/dummy.cpp diff --git a/libs/libclara-cmake/dummy.cpp b/libs/libclara-cmake/dummy.cpp new file mode 100644 index 00000000000..ceee79041cd --- /dev/null +++ b/libs/libclara-cmake/dummy.cpp @@ -0,0 +1,15 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file is intentionally left blank. From 7ebc2643d468f701d3f75fa5b18c5b62e647dc27 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Wed, 2 Sep 2026 20:00:37 +0800 Subject: [PATCH 50/58] fix: refresh TiCI count sample schema --- dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.h | 1 + 1 file changed, 1 insertion(+) diff --git a/dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.h b/dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.h index 10d161611cc..3c4f798d322 100644 --- a/dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.h +++ b/dbms/src/Flash/Planner/Plans/PhysicalTiCIScan.h @@ -51,6 +51,7 @@ class PhysicalTiCIScan : public PhysicalLeaf { tici_scan.setIsCountAgg(true); schema = agg->getSchema(); + sample_block = Block(schema); tici_scan.setNamesAndTypes(agg->getSchema()); tici_scan.setCountAggExecutorId(agg->execId()); } From 3bc30c8a7cb51a2b7e33a08fa06bd18a12417fd9 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Wed, 2 Sep 2026 20:02:24 +0800 Subject: [PATCH 51/58] fix: keep GC safepoint cache monotonic --- .../Storages/KVStore/TiKVHelpers/PDTiKVClient.h | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/dbms/src/Storages/KVStore/TiKVHelpers/PDTiKVClient.h b/dbms/src/Storages/KVStore/TiKVHelpers/PDTiKVClient.h index 65c18f0c5fa..f2ae15a31b7 100644 --- a/dbms/src/Storages/KVStore/TiKVHelpers/PDTiKVClient.h +++ b/dbms/src/Storages/KVStore/TiKVHelpers/PDTiKVClient.h @@ -187,12 +187,17 @@ struct PDClientHelper { GET_METRIC(tiflash_gc_safepoint_request_count, type_get_gc_state).Increment(); auto safe_point = pd_client->getGCSafePoint(); - cached_gc_safe_point.store( - std::max(cached_gc_safe_point.load(), safe_point), std::memory_order_release); - LOG_TRACE(Logger::get(), "use safe point v1, gc_safe_point={}", safe_point); + const auto cached_safe_point = cached_gc_safe_point.load(std::memory_order_acquire); + if (safe_point < cached_safe_point) + GET_METRIC(tiflash_gc_safepoint_request_count, type_rewind).Increment(); + const auto merged_safe_point = std::max(cached_safe_point, safe_point); + cached_gc_safe_point.store(merged_safe_point, std::memory_order_release); + if (merged_safe_point == 0) + GET_METRIC(tiflash_gc_safepoint_request_count, type_zero_gc_safe_point).Increment(); + LOG_TRACE(Logger::get(), "use safe point v1, gc_safe_point={}", merged_safe_point); safe_point_last_update_time = std::chrono::steady_clock::now(); observe_backoff_count(true); - return safe_point; + return merged_safe_point; } catch (pingcap::Exception & e) { @@ -257,11 +262,15 @@ struct PDClientHelper std::unique_lock lock(ks_gc_sp_mutex); KeyspaceGCInfo new_keyspace_gc_info; const auto iter = ks_gc_sp_map.find(keyspace_id); + if (iter != ks_gc_sp_map.end() && ks_gc_sp < iter->second.ks_gc_sp) + GET_METRIC(tiflash_gc_safepoint_request_count, type_rewind).Increment(); new_keyspace_gc_info.ks_gc_sp = iter == ks_gc_sp_map.end() ? ks_gc_sp : std::max(iter->second.ks_gc_sp, ks_gc_sp); new_keyspace_gc_info.ks_gc_sp_update_time = std::chrono::steady_clock::now(); ks_gc_sp_map[keyspace_id] = new_keyspace_gc_info; + if (new_keyspace_gc_info.ks_gc_sp == 0) + GET_METRIC(tiflash_gc_safepoint_request_count, type_zero_gc_safe_point).Increment(); } static KeyspaceGCInfo getKeyspaceGCSafepoint(KeyspaceID keyspace_id) From 59083d2d174c66beee3fe72ba90ee8f50f03070e Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Wed, 2 Sep 2026 20:33:59 +0800 Subject: [PATCH 52/58] fix: expose TiCI input stream header --- dbms/src/Storages/Tantivy/TantivyInputStream.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbms/src/Storages/Tantivy/TantivyInputStream.h b/dbms/src/Storages/Tantivy/TantivyInputStream.h index e842317d7fb..c8932c16559 100644 --- a/dbms/src/Storages/Tantivy/TantivyInputStream.h +++ b/dbms/src/Storages/Tantivy/TantivyInputStream.h @@ -78,6 +78,7 @@ class TantivyInputStream : public IProfilingBlockInputStream , match_expr(match_expr_) , is_count(is_count) , shards_snapshot(std::move(shards_snapshot_)) + , header(return_columns) {} String getName() const override { return NAME; } @@ -200,7 +201,6 @@ class TantivyInputStream : public IProfilingBlockInputStream } private: - Block header; bool done = false; LoggerPtr log; UInt32 keyspace_id; @@ -215,6 +215,7 @@ class TantivyInputStream : public IProfilingBlockInputStream ::Expr match_expr; bool is_count; std::shared_ptr> shards_snapshot; + Block header; static std::unordered_map buildColumnPositionMap(const NamesAndTypes & columns) { From bdf221cfd83b16e51da0a016122df810ce0a3c97 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Wed, 2 Sep 2026 20:35:55 +0800 Subject: [PATCH 53/58] fix: make TiCI stream count type safe --- dbms/src/Storages/StorageTantivy.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/StorageTantivy.cpp b/dbms/src/Storages/StorageTantivy.cpp index 84883892680..60febd59fec 100644 --- a/dbms/src/Storages/StorageTantivy.cpp +++ b/dbms/src/Storages/StorageTantivy.cpp @@ -37,6 +37,7 @@ #include #include +#include #include #include #include @@ -101,9 +102,9 @@ void StorageTantivy::read( context.getTimezoneInfo(), std::move(shards_snapshot)); - num_streams = std::max(1, std::min(num_streams, local_read.size())); + const auto stream_count = std::max(1, std::min(num_streams, local_read.size())); // local read - for (size_t i = 0; i < num_streams; ++i) + for (size_t i = 0; i < stream_count; ++i) { group_builder.addConcurrency( std::make_unique(exec_status, log->identifier(), tici_task_pool, return_columns)); From 6bb099082aa6ba97d7a687b3a4ec50d8c5182509 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Thu, 3 Sep 2026 09:54:30 +0800 Subject: [PATCH 54/58] style: format GC safepoint helper --- .../KVStore/TiKVHelpers/PDTiKVClient.h | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/dbms/src/Storages/KVStore/TiKVHelpers/PDTiKVClient.h b/dbms/src/Storages/KVStore/TiKVHelpers/PDTiKVClient.h index f2ae15a31b7..c4f057d902e 100644 --- a/dbms/src/Storages/KVStore/TiKVHelpers/PDTiKVClient.h +++ b/dbms/src/Storages/KVStore/TiKVHelpers/PDTiKVClient.h @@ -153,13 +153,12 @@ struct PDClientHelper // If keyspace id is `NullspaceID` it needs to use safe point v1. if (enable_safepoint_v2 && keyspace_id != NullspaceID) { - auto gc_safe_point - = getGCSafePointV2WithRetry( - pd_client, - keyspace_id, - false, - safe_point_update_interval_seconds, - safe_point_get_max_backoff_ms); + auto gc_safe_point = getGCSafePointV2WithRetry( + pd_client, + keyspace_id, + false, + safe_point_update_interval_seconds, + safe_point_get_max_backoff_ms); LOG_TRACE(Logger::get(), "use safe point v2, keyspace={} gc_safe_point={}", keyspace_id, gc_safe_point); return gc_safe_point; } @@ -264,9 +263,8 @@ struct PDClientHelper const auto iter = ks_gc_sp_map.find(keyspace_id); if (iter != ks_gc_sp_map.end() && ks_gc_sp < iter->second.ks_gc_sp) GET_METRIC(tiflash_gc_safepoint_request_count, type_rewind).Increment(); - new_keyspace_gc_info.ks_gc_sp = iter == ks_gc_sp_map.end() - ? ks_gc_sp - : std::max(iter->second.ks_gc_sp, ks_gc_sp); + new_keyspace_gc_info.ks_gc_sp + = iter == ks_gc_sp_map.end() ? ks_gc_sp : std::max(iter->second.ks_gc_sp, ks_gc_sp); new_keyspace_gc_info.ks_gc_sp_update_time = std::chrono::steady_clock::now(); ks_gc_sp_map[keyspace_id] = new_keyspace_gc_info; if (new_keyspace_gc_info.ks_gc_sp == 0) From 08189f8089c44f9ffe6a5ec6fd99e91c0e59a47a Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Thu, 3 Sep 2026 09:56:43 +0800 Subject: [PATCH 55/58] fix: include cargo in TiCI rust toolchain --- rust-toolchain.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 14ba487c8e7..b038ec228eb 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] channel = "nightly-2025-02-28" -components = ["rustfmt", "clippy", "rust-src", "rust-analyzer"] +components = ["cargo", "rustfmt", "clippy", "rust-src", "rust-analyzer"] profile = "minimal" From a5ac28a5afbc300c363a870cf047181afda41f1a Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Thu, 3 Sep 2026 11:24:04 +0800 Subject: [PATCH 56/58] build: serialize TiCI Rust target setup --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 43f4b80053c..05b2b7f1cd2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -415,6 +415,6 @@ include (cmake/print_include_directories.cmake) # Another way to do so is to use JOB_POOL with single concurrency. However, it is Ninja specific and it seems to have bugs # with a large number of threads. include (cmake/sequential.cmake) -build_sequentially (process_metrics symbolization tiflash_proxy) +build_sequentially (process_metrics rustbuild symbolization tiflash_proxy) print_flags () From 6cbd060fe268e126ecb5295d7aacb386fabdc40f Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Thu, 3 Sep 2026 17:25:39 +0800 Subject: [PATCH 57/58] fix: use release-8.5 compatible FTS tipb --- contrib/tipb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/tipb b/contrib/tipb index 1852f9829ce..4768c71070e 160000 --- a/contrib/tipb +++ b/contrib/tipb @@ -1 +1 @@ -Subproject commit 1852f9829ce3d3962895fec43f908b31fbdc58fb +Subproject commit 4768c71070e41ca8d986cfde0d8e1aadebff7c77 From d7746eaf93df9f512e5ba02b4d75f9901702f223 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Thu, 3 Sep 2026 19:14:57 +0800 Subject: [PATCH 58/58] fix: adapt Rpc error message calls --- dbms/src/Debug/dbgQueryExecutor.cpp | 5 ++++- dbms/src/Flash/Mpp/MPPTask.cpp | 3 ++- dbms/src/Storages/S3/S3Common.cpp | 5 ++++- dbms/src/Storages/StorageDisaggregatedRemote.cpp | 2 +- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/dbms/src/Debug/dbgQueryExecutor.cpp b/dbms/src/Debug/dbgQueryExecutor.cpp index 5ccebe0f8bc..9f046f94a4a 100644 --- a/dbms/src/Debug/dbgQueryExecutor.cpp +++ b/dbms/src/Debug/dbgQueryExecutor.cpp @@ -270,7 +270,10 @@ BlockInputStreamPtr executeMPPQuery(Context & context, const DAGProperties & pro mpp::DispatchTaskResponse resp; auto status = rpc.call(&client_context, req, &resp); if (!status.ok()) - throw Exception("Meet grpc error while dispatch mpp task: " + rpc.errMsg(status)); + { + std::string extra_msg = "addr: " + Debug::LOCAL_HOST; + throw Exception("Meet grpc error while dispatch mpp task: " + rpc.errMsg(status, extra_msg)); + } if (resp.has_error()) throw Exception("Meet error while dispatch mpp task: " + resp.error().msg()); } diff --git a/dbms/src/Flash/Mpp/MPPTask.cpp b/dbms/src/Flash/Mpp/MPPTask.cpp index 8d8e22db868..ffee389a783 100644 --- a/dbms/src/Flash/Mpp/MPPTask.cpp +++ b/dbms/src/Flash/Mpp/MPPTask.cpp @@ -723,7 +723,8 @@ void MPPTask::reportStatus(const String & err_msg) auto rpc_status = rpc.call(&client_context, req, &resp); if (!rpc_status.ok()) { - throw Exception(rpc.errMsg(rpc_status)); + auto extra_msg = "addr: " + meta.coordinator_address(); + throw Exception(rpc.errMsg(rpc_status, extra_msg)); } if (resp.has_error()) { diff --git a/dbms/src/Storages/S3/S3Common.cpp b/dbms/src/Storages/S3/S3Common.cpp index 3aa63c111e3..e7f3a2f6498 100644 --- a/dbms/src/Storages/S3/S3Common.cpp +++ b/dbms/src/Storages/S3/S3Common.cpp @@ -255,7 +255,10 @@ disaggregated::GetDisaggConfigResponse getDisaggConfigFromDisaggWriteNodes( disaggregated::GetDisaggConfigResponse resp; auto status = rpc.call(&client_context, req, &resp); if (!status.ok()) - throw Exception(rpc.errMsg(status)); + { + std::string extra_msg = "addr: " + send_address; + throw Exception(rpc.errMsg(status, extra_msg)); + } RUNTIME_CHECK(resp.has_s3_config(), resp.ShortDebugString()); diff --git a/dbms/src/Storages/StorageDisaggregatedRemote.cpp b/dbms/src/Storages/StorageDisaggregatedRemote.cpp index e9125bfad03..df2755921b2 100644 --- a/dbms/src/Storages/StorageDisaggregatedRemote.cpp +++ b/dbms/src/Storages/StorageDisaggregatedRemote.cpp @@ -259,7 +259,7 @@ void StorageDisaggregated::buildReadTaskForWriteNode( ErrorCodes::UNKNOWN_EXCEPTION, "EstablishDisaggTask failed, wn_address={}, errmsg={}, {}", req->address(), - rpc.errMsg(status), + rpc.errMsg(status, ""), log->identifier()); const DM::DisaggTaskId snapshot_id(resp.snapshot_id());