Antalya 26.3: Bump to v26.3.13.31-lts

ci/defs/job_configs.py

@@ -149,7 +149,7 @@
             "./ci/jobs/scripts/docker_in_docker.sh",
         ],
     ),
-    run_in_docker=f"altinityinfra/integration-tests-runner+root+--memory={LIMITED_MEM}+--privileged+--dns-search='.'+--security-opt seccomp=unconfined+--cap-add=SYS_PTRACE+{docker_sock_mount}+--volume=clickhouse_integration_tests_volume:/var/lib/docker+--cgroupns=host+--env=CLICKHOUSE_TEST_STAT_URL=$CLICKHOUSE_TEST_STAT_URL+--env=CLICKHOUSE_TEST_STAT_LOGIN=$CLICKHOUSE_TEST_STAT_LOGIN+--env=CLICKHOUSE_TEST_STAT_PASSWORD=$CLICKHOUSE_TEST_STAT_PASSWORD",
+    run_in_docker=f"altinityinfra/integration-tests-runner+root+--memory={LIMITED_MEM}+--privileged+--dns-search='.'+--security-opt seccomp=unconfined+--cap-add=SYS_PTRACE+{docker_sock_mount}+--volume=clickhouse_integration_tests_volume:/var/lib/docker+--cgroupns=host+--ulimit nofile=262144:262144+--env=CLICKHOUSE_TEST_STAT_URL=$CLICKHOUSE_TEST_STAT_URL+--env=CLICKHOUSE_TEST_STAT_LOGIN=$CLICKHOUSE_TEST_STAT_LOGIN+--env=CLICKHOUSE_TEST_STAT_PASSWORD=$CLICKHOUSE_TEST_STAT_PASSWORD",
 )
 
 

cmake/autogenerated_versions.txt

@@ -2,11 +2,11 @@
 
 # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION,
 # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes.
-SET(VERSION_REVISION 54519)
+SET(VERSION_REVISION 54520)
 SET(VERSION_MAJOR 26)
 SET(VERSION_MINOR 3)
-SET(VERSION_PATCH 12)
-SET(VERSION_GITHASH fa3aa24e79104d29f8bfde078fc586ec6ac3a565)
+SET(VERSION_PATCH 13)
+SET(VERSION_GITHASH d23c7536b980c34b39c850b08ef23c509f06aaaa)
 SET(VERSION_DESCRIBE v26.3.12.20001.altinityantalya)
 SET(VERSION_STRING 26.3.12.20001.altinityantalya)
 # end of autochange

contrib/mongo-c-driver-cmake/CMakeLists.txt

@@ -5,13 +5,13 @@ if(NOT USE_MONGODB)
 endif()
 
 set(libbson_VERSION_MAJOR 2)
-set(libbson_VERSION_MINOR 2)
-set(libbson_VERSION_PATCH 2)
-set(libbson_VERSION 2.2.2)
+set(libbson_VERSION_MINOR 3)
+set(libbson_VERSION_PATCH 0)
+set(libbson_VERSION 2.3.0)
 set(libmongoc_VERSION_MAJOR 2)
-set(libmongoc_VERSION_MINOR 2)
-set(libmongoc_VERSION_PATCH 2)
-set(libmongoc_VERSION 2.2.2)
+set(libmongoc_VERSION_MINOR 3)
+set(libmongoc_VERSION_PATCH 0)
+set(libmongoc_VERSION 2.3.0)
 
 set(LIBBSON_SOURCES_ROOT "${ClickHouse_SOURCE_DIR}/contrib/mongo-c-driver/src")
 set(LIBBSON_SOURCE_DIR "${LIBBSON_SOURCES_ROOT}/libbson/src")

docs/en/sql-reference/statements/system.md

@@ -730,7 +730,7 @@ SYSTEM STOP VIEWS
 
 Enable periodic refreshing for the given view or all refreshable views. No immediate refresh is triggered.
 
-If the view is in a Replicated or Shared database, `START VIEW` undoes the effect of `STOP VIEW`, and `START REPLICATED VIEW` undoes the effect of `STOP REPLICATED VIEW`.
+If the view is in a Replicated or Shared database, `START VIEW` undoes the effect of `STOP VIEW`, and `START REPLICATED VIEW` undoes the effect of `STOP REPLICATED VIEW`. `START VIEW` also undoes the effect of `PAUSE VIEW`.
 
 ```sql
 SYSTEM START VIEW [db.]name
@@ -739,7 +739,26 @@ SYSTEM START VIEW [db.]name
 SYSTEM START VIEWS
 ```
 
-### SYSTEM CANCEL VIEW {#cancel-view}
+### SYSTEM PAUSE VIEW, PAUSE VIEWS {#pause-view-pause-views}
+
+Disable periodic refreshing of the given view or all refreshable views.
+Unlike `SYSTEM STOP VIEW`, `SYSTEM PAUSE VIEW` does not interrupt a refresh that is already in progress: the running refresh is allowed to finish, and only subsequent refreshes are prevented.
+
+Undo with `SYSTEM START VIEW` or `SYSTEM START VIEWS`.
+
+:::note
+The paused state does not persist across server restarts. After a restart, views will resume their configured refresh schedules.
+In Replicated or Shared databases, `SYSTEM PAUSE VIEW` only affects the current replica.
+:::
+
+```sql
+SYSTEM PAUSE VIEW [db.]name
+```
+```sql
+SYSTEM PAUSE VIEWS
+```
+
+### SYSTEM REFRESH VIEW {#refresh-view}
 
 If there's a refresh in progress for the given view on the current replica, interrupt and cancel it. Otherwise do nothing.
 

src/Access/Common/AccessType.h

@@ -358,7 +358,7 @@ enum class AccessType : uint8_t
     M(SYSTEM_SWARM, "SYSTEM STOP SWARM MODE, SYSTEM START SWARM MODE, STOP SWARM MODE, START SWARM MODE", GLOBAL, SYSTEM) \
     M(SYSTEM_PULLING_REPLICATION_LOG, "SYSTEM STOP PULLING REPLICATION LOG, SYSTEM START PULLING REPLICATION LOG", TABLE, SYSTEM) \
     M(SYSTEM_CLEANUP, "SYSTEM STOP CLEANUP, SYSTEM START CLEANUP", TABLE, SYSTEM) \
-    M(SYSTEM_VIEWS, "SYSTEM REFRESH VIEW, SYSTEM START VIEWS, SYSTEM STOP VIEWS, SYSTEM START VIEW, SYSTEM STOP VIEW, SYSTEM CANCEL VIEW, REFRESH VIEW, START VIEWS, STOP VIEWS, START VIEW, STOP VIEW, CANCEL VIEW", VIEW, SYSTEM) \
+    M(SYSTEM_VIEWS, "SYSTEM REFRESH VIEW, SYSTEM START VIEWS, SYSTEM STOP VIEWS, SYSTEM START VIEW, SYSTEM STOP VIEW, SYSTEM PAUSE VIEWS, SYSTEM PAUSE VIEW, SYSTEM CANCEL VIEW, REFRESH VIEW, START VIEWS, STOP VIEWS, START VIEW, STOP VIEW, PAUSE VIEWS, PAUSE VIEW, CANCEL VIEW", VIEW, SYSTEM) \
     M(SYSTEM_DISTRIBUTED_SENDS, "SYSTEM STOP DISTRIBUTED SENDS, SYSTEM START DISTRIBUTED SENDS, STOP DISTRIBUTED SENDS, START DISTRIBUTED SENDS", TABLE, SYSTEM_SENDS) \
     M(SYSTEM_REPLICATED_SENDS, "SYSTEM STOP REPLICATED SENDS, SYSTEM START REPLICATED SENDS, STOP REPLICATED SENDS, START REPLICATED SENDS", TABLE, SYSTEM_SENDS) \
     M(SYSTEM_SENDS, "SYSTEM STOP SENDS, SYSTEM START SENDS, STOP SENDS, START SENDS", GROUP, SYSTEM) \

src/AggregateFunctions/AggregateFunctionSingleValueOrNull.cpp

@@ -92,7 +92,7 @@ struct AggregateFunctionSingleValueOrNullData
         {
             ColumnNullable & col = typeid_cast<ColumnNullable &>(to);
             col.getNullMapColumn().insertDefault();
-            data().insertResultInto(col.getNestedColumn(), result_type);
+            data().insertResultInto(col.getNestedColumn(), removeNullable(result_type));
         }
     }
 };
@@ -169,7 +169,7 @@ class AggregateFunctionSingleValueOrNull final
 
     void deserialize(AggregateDataPtr place, ReadBuffer & buf, std::optional<size_t> /* version */, Arena * arena) const override
     {
-        data(place).read(buf, *serialization, result_type, arena);
+        data(place).read(buf, *serialization, value_type, arena);
     }
 
     bool allocatesMemoryInArena() const override { return singleValueTypeAllocatesMemoryInArena(value_type->getTypeId()); }

src/AggregateFunctions/Combinators/AggregateFunctionOrFill.h

@@ -310,6 +310,42 @@ class AggregateFunctionOrFill final : public IAggregateFunctionHelper<AggregateF
                     nested_function->insertResultInto(place, to, arena);
             }
         }
+        else if (nested_function->isState())
+        {
+            /// Mirror the flag-set branch for State-nested combinators: routing
+            /// flag-unset rows through `to.insertDefault()` would call
+            /// `ColumnAggregateFunction::ensureOwnership()` on the inner column and
+            /// reset its `src`, leaving subsequent flag-set rows pushing externally-
+            /// owned state pointers without `src` protection (double-destroy under
+            /// `MemorySanitizer`, issue #105462). The state at `place` is already
+            /// default-initialized by `create()` above, so it is safe to forward.
+            if constexpr (UseNull)
+            {
+                if (!result_is_nullable || inner_nullable)
+                {
+                    if constexpr (merge)
+                        nested_function->insertMergeResultInto(place, to, arena);
+                    else
+                        nested_function->insertResultInto(place, to, arena);
+                }
+                else
+                {
+                    ColumnNullable & col = typeid_cast<ColumnNullable &>(to);
+                    col.getNullMapColumn().getData().push_back(static_cast<UInt8>(1));
+                    if constexpr (merge)
+                        nested_function->insertMergeResultInto(place, col.getNestedColumn(), arena);
+                    else
+                        nested_function->insertResultInto(place, col.getNestedColumn(), arena);
+                }
+            }
+            else
+            {
+                if constexpr (merge)
+                    nested_function->insertMergeResultInto(place, to, arena);
+                else
+                    nested_function->insertResultInto(place, to, arena);
+            }
+        }
         else
             to.insertDefault();
     }

src/AggregateFunctions/UniquesHashSet.h

@@ -2,6 +2,7 @@
 
 #include <math.h>
 
+#include <base/defines.h>
 #include <base/types.h>
 
 #include <IO/WriteBuffer.h>
@@ -465,6 +466,9 @@ class UniquesHashSet : private HashTableAllocatorWithStackMemory<(1ULL << UNIQUE
         if (m_size > UNIQUES_HASH_MAX_SIZE)
             throw Poco::Exception("Cannot write UniquesHashSet: too large size_degree.");
 
+        /// A null `buf` here would indicate upstream state corruption (e.g. a double-destroyed state).
+        chassert(buf);
+
         DB::writeBinaryLittleEndian(skip_degree, wb);
         DB::writeVarUInt(m_size, wb);
 

src/Analyzer/FunctionNode.cpp

@@ -286,7 +286,8 @@ ASTPtr FunctionNode::toASTImpl(const ConvertToASTOptions & options) const
     /// tuple, and adding a type may significantly increase query size.
     /// It should be safe because set type for `column IN tuple` is deduced from `column` type.
     if (isNameOfInFunction(function_name) && argument_nodes.size() > 1 && argument_nodes[1]->getNodeType() == QueryTreeNodeType::CONSTANT
-        && !static_cast<const ConstantNode *>(argument_nodes[1].get())->hasSourceExpression())
+        && !static_cast<const ConstantNode *>(argument_nodes[1].get())->hasSourceExpression()
+        && !isArray(argument_nodes[1]->getResultType()))
     {
         auto expression_list_ast = make_intrusive<ASTExpressionList>();
 

src/Backups/BackupEntriesCollector.cpp

@@ -408,7 +408,7 @@ void BackupEntriesCollector::gatherDatabasesMetadata()
 
             case ASTBackupQuery::ElementType::ALL:
             {
-                for (const auto & [database_name, database] : DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = true}))
+                for (const auto & [database_name, database] : DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_remote_databases = true}))
                 {
                     if (!element.except_databases.contains(database_name))
                     {

src/Client/BuzzHouse/Generator/SessionSettings.cpp

@@ -1206,7 +1206,7 @@ static std::unordered_map<String, CHSetting> serverSettings2 = {
     {"shared_merge_tree_sync_parts_on_partition_operations", trueOrFalseSettingNoOracle},
     {"short_circuit_function_evaluation_for_nulls", trueOrFalseSetting},
     {"short_circuit_function_evaluation_for_nulls_threshold", probRangeSetting},
-    {"show_data_lake_catalogs_in_system_tables", trueOrFalseSettingNoOracle},
+    {"show_remote_databases_in_system_tables", trueOrFalseSettingNoOracle},
     {"show_create_query_identifier_quoting_rule",
      CHSetting(
          [](RandomGenerator & rg, FuzzConfig &)

src/Client/ClientBase.cpp

@@ -1378,6 +1378,15 @@ void ClientBase::processOrdinaryQuery(String query, ASTPtr parsed_query)
 /// Also checks if query execution should be cancelled.
 void ClientBase::receiveResult(ASTPtr parsed_query, Int32 signals_before_stop, bool partial_result_on_first_cancel)
 {
+    /// The connection may already be torn down — e.g. `sendQuery` failed to
+    /// write the query to a server that died mid-transfer and called
+    /// `disconnect()`, after which `processOrdinaryQuery` calls us in a
+    /// best-effort attempt to drain remaining data. With the receive side
+    /// gone there is nothing to poll for; bail out so we don't call
+    /// `connection->poll()` (and friends) on a disconnected connection.
+    if (!connection->isConnected())
+        return;
+
     // TODO: get the poll_interval from commandline.
     const auto receive_timeout = connection_parameters.timeouts.receive_timeout;
     constexpr size_t default_poll_interval = 1000000; /// in microseconds

src/Client/Connection.cpp

@@ -1335,12 +1335,15 @@ std::optional<Poco::Net::SocketAddress> Connection::getResolvedAddress() const
 
 bool Connection::poll(size_t timeout_microseconds)
 {
+    ensureConnected();
     return in->poll(timeout_microseconds);
 }
 
 
 bool Connection::hasReadPendingData() const
 {
+    if (!in)
+        return false;
     return last_input_packet_type.has_value() || in->hasBufferedData();
 }
 
@@ -1369,20 +1372,26 @@ UInt64 Connection::receivePacketType()
     if (last_input_packet_type)
         return *last_input_packet_type;
 
-    UInt64 type;
+    ensureConnected();
+
+    UInt64 type = 0;
     readVarUInt(type, *in);
     return last_input_packet_type.emplace(type);
 }
 
+void Connection::ensureConnected() const
+{
+    /// We are trying to send something to already disconnected connection,
+    /// this means that we continue using Connection after exception.
+    if (!in)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Connection to {} is terminated", getDescription());
+}
 
 Packet Connection::receivePacket()
 {
     try
     {
-        /// We are trying to send something to already disconnected connection,
-        /// this means that we continue using Connection after exception.
-        if (!in)
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Connection to {} is terminated", getDescription());
+        ensureConnected();
 
         Packet res;
 

src/Client/Connection.h

@@ -344,6 +344,8 @@ class Connection : public IServerConnection
     void initBlockLogsInput();
     void initBlockProfileEventsInput();
 
+    void ensureConnected() const;
+
     [[noreturn]] void throwUnexpectedPacket(UInt64 packet_type, const char * expected, TimeoutSetter * timeout_setter = nullptr);
 };
 

src/Client/LineReader.cpp

@@ -8,8 +8,6 @@
 #include <cstring>
 #include <unistd.h>
 #include <poll.h>
-#include <sys/time.h>
-#include <sys/types.h>
 
 
 #pragma clang diagnostic ignored "-Wreserved-identifier"
@@ -59,11 +57,8 @@ namespace DB
 /// Allows delaying the start of query execution until the entirety of query is inserted.
 bool LineReader::hasInputData() const
 {
-    timeval timeout = {0, 0};
-    fd_set fds{};
-    FD_ZERO(&fds);
-    FD_SET(in_fd, &fds);
-    return select(1, &fds, nullptr, nullptr, &timeout) == 1;
+    pollfd pfd{.fd = in_fd, .events = POLLIN, .revents = 0};
+    return poll(&pfd, 1, 0) == 1 && (pfd.revents & POLLIN);
 }
 
 replxx::Replxx::completions_t LineReader::Suggest::getCompletions(const String & prefix, size_t prefix_length, const char * word_break_characters)

src/Common/FailPoint.cpp

@@ -62,6 +62,7 @@ static struct InitFiu
     ONCE(rmt_lightweight_update_sleep_after_block_allocation) \
     ONCE(rmt_merge_task_sleep_in_prepare) \
     ONCE(s3_read_buffer_throw_expired_token) \
+    ONCE(s3_send_request_throw_expired_token) \
     ONCE(distributed_cache_fail_request_in_the_middle_of_request) \
     ONCE(object_storage_queue_fail_commit_once) \
     ONCE(distributed_cache_fail_continue_request) \
@@ -168,7 +169,9 @@ static struct InitFiu
     PAUSEABLE_ONCE(drop_database_before_exclusive_ddl_lock) \
     REGULAR(storage_merge_tree_background_schedule_merge_fail) \
     REGULAR(patch_parts_reverse_column_order) \
-    REGULAR(wide_part_writer_fail_in_add_streams)
+    REGULAR(wide_part_writer_fail_in_add_streams) \
+    REGULAR(compact_part_writer_fail_in_add_streams) \
+    REGULAR(query_metric_log_delay_collect)
 
 namespace FailPoints
 {

src/Common/QueryFuzzer.cpp

@@ -3649,6 +3649,7 @@ void QueryFuzzer::fuzz(ASTPtr & ast)
                 Type::START_REPLICATED_VIEW,
                 Type::STOP_VIEW,
                 Type::STOP_REPLICATED_VIEW,
+                Type::PAUSE_VIEW,
                 Type::CANCEL_VIEW,
                 Type::WAIT_VIEW,
             };

src/Core/Settings.cpp

@@ -6993,9 +6993,9 @@ Query Iceberg table using the snapshot that was current at a specific timestamp.
     DECLARE(Int64, iceberg_snapshot_id, 0, R"(
 Query Iceberg table using the specific snapshot id.
 )", 0) \
-    DECLARE(Bool, show_data_lake_catalogs_in_system_tables, false, R"(
-Enables showing data lake catalogs in system tables.
-)", 0) \
+    DECLARE_WITH_ALIAS(Bool, show_remote_databases_in_system_tables, false, R"(
+Enables showing remote databases (data lake catalogs, MySQL, PostgreSQL) in system tables.
+)", 0, show_data_lake_catalogs_in_system_tables) \
     DECLARE(Bool, delta_lake_enable_expression_visitor_logging, false, R"(
 Enables Test level logs of DeltaLake expression visitor. These logs can be too verbose even for test logging.
 )", 0) \

src/Core/SettingsChangesHistory.cpp

@@ -81,6 +81,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory()
             {"max_skip_unavailable_shards_num", 0, 0, "New setting to limit the number of shards that can be silently skipped when skip_unavailable_shards is enabled."},
             {"max_skip_unavailable_shards_ratio", 0, 0, "New setting to limit the ratio of shards that can be silently skipped when skip_unavailable_shards is enabled."},
             {"allow_experimental_database_s3_tables", false, false, "New setting to enable experimental database S3 tables (AWS Iceberg REST catalog)."},
+            {"show_remote_databases_in_system_tables", false, false, "Renamed from `show_data_lake_catalogs_in_system_tables` and broadened to also hide `MySQL` and `PostgreSQL` databases from `system.tables`, `system.columns` and `system.completions` by default, since enumerating their tables requires expensive remote calls. Users who relied on the previous behavior must set this setting to `true`. The old name is kept as an alias."},
         });
         addSettingsChanges(settings_changes_history, "26.2",
         {
@@ -104,7 +105,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory()
             {"use_page_cache_for_local_disks", false, false, "New setting to use userspace page cache for local disks"},
             {"use_page_cache_for_object_storage", false, false, "New setting to use userspace page cache for object storage table functions"},
             {"use_statistics_cache", false, true, "Enable statistics cache"},
-            {"apply_row_policy_after_final", false, true, "Enabling apply_row_policy_after_final by default, as if was in 25.8 before #87303"},
+            {"apply_row_policy_after_final", true, true, "Enabling apply_row_policy_after_final by default, as if was in 25.8 before #87303"},
             {"ignore_format_null_for_explain", false, true, "FORMAT Null is now ignored for EXPLAIN queries by default"},
             {"input_format_connection_handling", false, false, "New setting to allow parsing and processing remaining data in the buffer if the connection closes unexpectedly"},
             {"input_format_max_block_wait_ms", 0, 0, "New setting to limit maximum wait time in milliseconds before a block is emitted by input format"},
@@ -193,7 +194,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory()
             {"aggregate_function_input_format", "state", "state", "New setting to control AggregateFunction input format during INSERT operations. Setting Value set to state by default"},
             {"delta_lake_snapshot_start_version", -1, -1, "New setting."},
             {"delta_lake_snapshot_end_version", -1, -1, "New setting."},
-            {"apply_row_policy_after_final", false, false, "New setting to control if row policies and PREWHERE are applied after FINAL processing for *MergeTree tables"},
+            {"apply_row_policy_after_final", true, true, "New setting to control if row policies and PREWHERE are applied after FINAL processing for *MergeTree tables"},
             {"apply_prewhere_after_final", false, false, "New setting. When enabled, PREWHERE conditions are applied after FINAL processing."},
             {"compatibility_s3_presigned_url_query_in_path", false, false, "New setting."},
             {"serialize_string_in_memory_with_zero_byte", true, true, "New setting"},