ClickHouse 25.8 LTS

.gitmodules

@@ -30,7 +30,8 @@
 	url = https://github.com/google/re2
 [submodule "contrib/mariadb-connector-c"]
 	path = contrib/mariadb-connector-c
-	url = https://github.com/ClickHouse/mariadb-connector-c
+	url = https://github.com/aiven/mariadb-connector-c
+	branch = aiven/clickhouse-v25.8.12.129
 [submodule "contrib/jemalloc"]
 	path = contrib/jemalloc
 	url = https://github.com/jemalloc/jemalloc
@@ -87,7 +88,8 @@
 	url = https://github.com/ClickHouse/grpc
 [submodule "contrib/aws"]
 	path = contrib/aws
-	url = https://github.com/ClickHouse/aws-sdk-cpp
+	url = https://github.com/aiven/aws-sdk-cpp
+	branch = aiven/clickhouse-v25.8.12.129
 [submodule "contrib/aws-c-event-stream"]
 	path = contrib/aws-c-event-stream
 	url = https://github.com/awslabs/aws-c-event-stream
@@ -214,7 +216,8 @@
 	url = https://github.com/ClickHouse/hive-metastore
 [submodule "contrib/azure"]
 	path = contrib/azure
-	url = https://github.com/ClickHouse/azure-sdk-for-cpp
+	url = https://github.com/aiven/azure-sdk-for-cpp
+	branch = aiven/clickhouse-v25.8.12.129
 [submodule "contrib/minizip-ng"]
 	path = contrib/minizip-ng
 	url = https://github.com/zlib-ng/minizip-ng

cmake/cpu_features.cmake

@@ -104,7 +104,7 @@ elseif (ARCH_AARCH64)
         # jscvt fcma lrcpc dcpop sha3 sm3 sm4 asimddp sha512 sve asimdfhm dit uscat ilrcpc flagm ssbs paca pacg dcpodp svei8mm svebf16 i8mm
         # bf16 dgh rng")
         execute_process(
-            COMMAND grep -P "^(?=.*atomic)(?=.*ssbs)" /proc/cpuinfo
+            COMMAND grep -P "^(?=.*atomic)" /proc/cpuinfo
             OUTPUT_VARIABLE FLAGS)
         if (NOT FLAGS)
             MESSAGE(FATAL_ERROR "The build machine does not satisfy the minimum CPU requirements, try to run cmake with -DNO_ARMV81_OR_HIGHER=1")

contrib/curl-cmake/CMakeLists.txt

@@ -1,4 +1,5 @@
 option (ENABLE_CURL "Enable curl" ${ENABLE_LIBRARIES})
+option (ENABLE_IPV6 "Enable IPv6" 1)
 
 if (NOT ENABLE_CURL)
     message(STATUS "Not using curl")
@@ -192,6 +193,7 @@ target_compile_definitions (_curl PRIVATE
     BUILDING_LIBCURL
     CURL_HIDDEN_SYMBOLS
     libcurl_EXPORTS
+    ENABLE_IPV6
     OPENSSL_NO_ENGINE
     CURL_OS="${CMAKE_SYSTEM_NAME}"
 )

docs/en/operations/system-tables/metrics.md

@@ -628,6 +628,10 @@ Number of data parts checking for consistency
 
 Number of data parts being fetched from replica
 
+### ReplicatedQueuesTotalSize
+
+Total number of items contained in all nodes' replicated queues.
+
 ### ReplicatedSend {#replicatedsend}
 
 Number of data parts being sent to replicas

src/Access/AccessBackup.cpp

@@ -561,7 +561,8 @@ void restoreAccessEntitiesFromBackup(
         LOG_TRACE(log, "{}: Adding with UUID {}", AccessEntityTypeInfo::get(type).formatEntityNameWithType(name), id);
 
         UUID existing_id;
-        if (destination_access_storage.insert(id, entity, replace_if_exists, throw_if_exists, &existing_id))
+        auto check_func = [](const AccessEntityPtr &){}; // No check needed during restore
+        if (destination_access_storage.insert(id, entity, check_func, replace_if_exists, throw_if_exists, &existing_id))
         {
             LOG_TRACE(log, "{}: Added successfully", AccessEntityTypeInfo::get(type).formatEntityNameWithType(name));
             restored_ids.emplace(id);

src/Access/AccessControl.cpp

@@ -551,19 +551,19 @@ scope_guard AccessControl::subscribeForChanges(const std::vector<UUID> & ids, co
     return changes_notifier->subscribeForChanges(ids, handler);
 }
 
-bool AccessControl::insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id)
+bool AccessControl::insertImpl(const UUID & id, const AccessEntityPtr & entity, const CheckFunc & check_func, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id)
 {
-    if (MultipleAccessStorage::insertImpl(id, entity, replace_if_exists, throw_if_exists, conflicting_id))
+    if (MultipleAccessStorage::insertImpl(id, entity, check_func, replace_if_exists, throw_if_exists, conflicting_id))
     {
         changes_notifier->sendNotifications();
         return true;
     }
     return false;
 }
 
-bool AccessControl::removeImpl(const UUID & id, bool throw_if_not_exists)
+bool AccessControl::removeImpl(const UUID & id, const CheckFunc & check_func, bool throw_if_not_exists)
 {
-    bool removed = MultipleAccessStorage::removeImpl(id, throw_if_not_exists);
+    bool removed = MultipleAccessStorage::removeImpl(id, check_func, throw_if_not_exists);
     if (removed)
         changes_notifier->sendNotifications();
     return removed;
@@ -666,6 +666,23 @@ void AccessControl::setDefaultProfileName(const String & default_profile_name)
     settings_profiles_cache->setDefaultProfileName(default_profile_name);
 }
 
+std::optional<UUID> AccessControl::getDefaultProfileId() const
+{
+    return settings_profiles_cache->getDefaultProfileId();
+}
+
+
+bool AccessControl::isDefaultProfileOrDescendant(const UUID & profile_id) const
+{
+    return settings_profiles_cache->isDefaultProfileOrDescendant(profile_id);
+}
+
+
+bool AccessControl::isExpectedProfileOrDescendant(const UUID & profile_id, const UUID & expected_id) const
+{
+    return settings_profiles_cache->isExpectedProfileOrDescendant(profile_id, expected_id);
+}
+
 
 void AccessControl::setCustomSettingsPrefixes(const Strings & prefixes)
 {

src/Access/AccessControl.h

@@ -139,6 +139,10 @@ class AccessControl : public MultipleAccessStorage
     /// The default profile's settings are always applied before any other profile's.
     void setDefaultProfileName(const String & default_profile_name);
 
+    std::optional<UUID> getDefaultProfileId() const;
+    bool isDefaultProfileOrDescendant(const UUID & profile_id) const;
+    bool isExpectedProfileOrDescendant(const UUID & profile_id, const UUID & expected_id) const;
+
     /// Sets prefixes which should be used for custom settings.
     /// This function also enables custom prefixes to be used.
     void setCustomSettingsPrefixes(const Strings & prefixes);
@@ -264,8 +268,8 @@ class AccessControl : public MultipleAccessStorage
     class CustomSettingsPrefixes;
     class PasswordComplexityRules;
 
-    bool insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id) override;
-    bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
+    bool insertImpl(const UUID & id, const AccessEntityPtr & entity, const CheckFunc & check_func, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id) override;
+    bool removeImpl(const UUID & id, const CheckFunc & check_func, bool throw_if_not_exists) override;
     bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;
 
     std::unique_ptr<ContextAccessCache> context_access_cache;

src/Access/Common/AccessType.h

@@ -284,6 +284,7 @@ enum class AccessType : uint8_t
     M(ALTER_SETTINGS_PROFILE, "ALTER PROFILE", GLOBAL, ACCESS_MANAGEMENT) \
     M(DROP_SETTINGS_PROFILE, "DROP PROFILE", GLOBAL, ACCESS_MANAGEMENT) \
     M(ALLOW_SQL_SECURITY_NONE, "CREATE SQL SECURITY NONE, ALLOW SQL SECURITY NONE, SQL SECURITY NONE, SECURITY NONE", GLOBAL, ACCESS_MANAGEMENT) \
+    M(PROTECTED_ACCESS_MANAGEMENT, "", GLOBAL, ACCESS_MANAGEMENT) \
     M(SHOW_USERS, "SHOW CREATE USER", GLOBAL, SHOW_ACCESS) \
     M(SHOW_ROLES, "SHOW CREATE ROLE", GLOBAL, SHOW_ACCESS) \
     M(SHOW_ROW_POLICIES, "SHOW POLICIES, SHOW CREATE ROW POLICY, SHOW CREATE POLICY", TABLE, SHOW_ACCESS) \

src/Access/DiskAccessStorage.cpp

@@ -415,7 +415,7 @@ void DiskAccessStorage::setAllInMemory(const std::vector<std::pair<UUID, AccessE
 
     /// Insert or update entities.
     for (const auto & [id, entity] : entities_without_conflicts)
-        insertNoLock(id, entity, /* replace_if_exists = */ true, /* throw_if_exists = */ false, /* conflicting_id = */ nullptr, /* write_on_disk= */ false);
+        insertNoLock(id, entity, CheckFunc{}, /* replace_if_exists = */ true, /* throw_if_exists = */ false, /* conflicting_id = */ nullptr, /* write_on_disk= */ false);
 }
 
 void DiskAccessStorage::removeAllExceptInMemory(const boost::container::flat_set<UUID> & ids_to_keep)
@@ -425,7 +425,7 @@ void DiskAccessStorage::removeAllExceptInMemory(const boost::container::flat_set
         const auto & id = it->first;
         ++it; /// We must go to the next element in the map `entries_by_id` here because otherwise removeNoLock() can invalidate our iterator.
         if (!ids_to_keep.contains(id))
-            (void)removeNoLock(id, /* throw_if_not_exists */ true, /* write_on_disk= */ false);
+            (void)removeNoLock(id, CheckFunc{}, /* throw_if_not_exists */ true, /* write_on_disk= */ false);
     }
 }
 
@@ -504,15 +504,18 @@ std::optional<std::pair<String, AccessEntityType>> DiskAccessStorage::readNameWi
 }
 
 
-bool DiskAccessStorage::insertImpl(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id)
+bool DiskAccessStorage::insertImpl(const UUID & id, const AccessEntityPtr & new_entity, const CheckFunc & check_func, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id)
 {
     std::lock_guard lock{mutex};
-    return insertNoLock(id, new_entity, replace_if_exists, throw_if_exists, conflicting_id, /* write_on_disk = */ true);
+    return insertNoLock(id, new_entity, check_func, replace_if_exists, throw_if_exists, conflicting_id, /* write_on_disk = */ true);
 }
 
 
-bool DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id, bool write_on_disk)
+bool DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, const CheckFunc & check_func, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id, bool write_on_disk)
 {
+    if (check_func)
+        check_func(new_entity);
+
     const String & name = new_entity->getName();
     AccessEntityType type = new_entity->getType();
 
@@ -565,7 +568,7 @@ bool DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & ne
     if (name_collision && (id_by_name != id))
     {
         assert(replace_if_exists);
-        removeNoLock(id_by_name, /* throw_if_not_exists= */ false, write_on_disk); // NOLINT
+        removeNoLock(id_by_name, CheckFunc{}, /* throw_if_not_exists= */ false, write_on_disk); // NOLINT
     }
 
     if (id_collision)
@@ -590,7 +593,7 @@ bool DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & ne
             return true;
         }
 
-        removeNoLock(id, /* throw_if_not_exists= */ false, write_on_disk); // NOLINT
+        removeNoLock(id, CheckFunc{}, /* throw_if_not_exists= */ false, write_on_disk); // NOLINT
     }
 
     /// Do insertion.
@@ -609,14 +612,14 @@ bool DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & ne
 }
 
 
-bool DiskAccessStorage::removeImpl(const UUID & id, bool throw_if_not_exists)
+bool DiskAccessStorage::removeImpl(const UUID & id, const CheckFunc & check_func, bool throw_if_not_exists)
 {
     std::lock_guard lock{mutex};
-    return removeNoLock(id, throw_if_not_exists, /* write_on_disk= */ true);
+    return removeNoLock(id, check_func, throw_if_not_exists, /* write_on_disk= */ true);
 }
 
 
-bool DiskAccessStorage::removeNoLock(const UUID & id, bool throw_if_not_exists, bool write_on_disk)
+bool DiskAccessStorage::removeNoLock(const UUID & id, const CheckFunc & check_func, bool throw_if_not_exists, bool write_on_disk)
 {
     auto it = entries_by_id.find(id);
     if (it == entries_by_id.end())
@@ -627,6 +630,9 @@ bool DiskAccessStorage::removeNoLock(const UUID & id, bool throw_if_not_exists,
             return false;
     }
 
+    if (check_func && it->second.entity)
+        check_func(it->second.entity);
+
     Entry & entry = it->second;
     AccessEntityType type = entry.type;
 

src/Access/DiskAccessStorage.h

@@ -40,8 +40,8 @@ class DiskAccessStorage : public IAccessStorage
     std::vector<UUID> findAllImpl(AccessEntityType type) const override;
     AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override;
     std::optional<std::pair<String, AccessEntityType>> readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const override;
-    bool insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id) override;
-    bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
+    bool insertImpl(const UUID & id, const AccessEntityPtr & entity, const CheckFunc & check_func, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id) override;
+    bool removeImpl(const UUID & id, const CheckFunc & check_func, bool throw_if_not_exists) override;
     bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;
 
     bool readLists() TSA_REQUIRES(mutex);
@@ -54,9 +54,9 @@ class DiskAccessStorage : public IAccessStorage
     void listsWritingThreadFunc() TSA_NO_THREAD_SAFETY_ANALYSIS;
     void stopListsWritingThread();
 
-    bool insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id, bool write_on_disk) TSA_REQUIRES(mutex);
+    bool insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, const CheckFunc & check_func, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id, bool write_on_disk) TSA_REQUIRES(mutex);
     bool updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists, bool write_on_disk) TSA_REQUIRES(mutex);
-    bool removeNoLock(const UUID & id, bool throw_if_not_exists, bool write_on_disk) TSA_REQUIRES(mutex);
+    bool removeNoLock(const UUID & id, const CheckFunc & check_func, bool throw_if_not_exists, bool write_on_disk) TSA_REQUIRES(mutex);
 
     AccessEntityPtr readAccessEntityFromDisk(const UUID & id) const;
     void writeAccessEntityToDisk(const UUID & id, const IAccessEntity & entity) const;

src/Access/HTTPAuthClient.h

@@ -39,7 +39,10 @@ class HTTPAuthClient
 
     Result authenticateRequest(Poco::Net::HTTPRequest & request) const
     {
-        auto session = makeHTTPSession(HTTPConnectionGroupType::HTTP, uri, timeouts);
+        // Note: makeHTTPSession signature was updated to support custom CA certificates for S3.
+        // This call site was already using makeHTTPSession, but needed to be updated to match the new signature.
+        // For HTTP authentication requests, we pass an empty context (default) since we don't need custom CA certificates.
+        auto session = makeHTTPSession(HTTPConnectionGroupType::HTTP, uri, timeouts, {}, nullptr, {});
         Poco::Net::HTTPResponse response;
 
         auto milliseconds_to_wait = retry_initial_backoff_ms;

src/Access/IAccessEntity.h

@@ -31,6 +31,8 @@ struct IAccessEntity
     virtual void setName(const String & name_) { name = name_; }
     const String & getName() const { return name; }
 
+    virtual bool isProtected() const { return false; }
+
     friend bool operator ==(const IAccessEntity & lhs, const IAccessEntity & rhs) { return lhs.equal(rhs); }
     friend bool operator !=(const IAccessEntity & lhs, const IAccessEntity & rhs) { return !(lhs == rhs); }
 

src/Access/IAccessStorage.cpp

@@ -210,7 +210,12 @@ std::optional<UUID> IAccessStorage::insert(const AccessEntityPtr & entity, bool
 
 bool IAccessStorage::insert(const DB::UUID & id, const DB::AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id)
 {
-    return insertImpl(id, entity, replace_if_exists, throw_if_exists, conflicting_id);
+    return insert(id, entity, CheckFunc{}, replace_if_exists, throw_if_exists, conflicting_id);
+}
+
+bool IAccessStorage::insert(const DB::UUID & id, const DB::AccessEntityPtr & entity, const CheckFunc & check_func, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id)
+{
+    return insertImpl(id, entity, check_func, replace_if_exists, throw_if_exists, conflicting_id);
 }
 
 
@@ -304,7 +309,7 @@ std::vector<UUID> IAccessStorage::insertOrReplace(const std::vector<AccessEntity
 }
 
 
-bool IAccessStorage::insertImpl(const UUID &, const AccessEntityPtr & entity, bool, bool, UUID *)
+bool IAccessStorage::insertImpl(const UUID &, const AccessEntityPtr & entity, const CheckFunc &, bool, bool, UUID *)
 {
     if (isReadOnly())
         throwReadonlyCannotInsert(entity->getType(), entity->getName());
@@ -314,7 +319,12 @@ bool IAccessStorage::insertImpl(const UUID &, const AccessEntityPtr & entity, bo
 
 bool IAccessStorage::remove(const UUID & id, bool throw_if_not_exists)
 {
-    return removeImpl(id, throw_if_not_exists);
+    return remove(id, CheckFunc{}, throw_if_not_exists);
+}
+
+bool IAccessStorage::remove(const UUID & id, const CheckFunc & check_func, bool throw_if_not_exists)
+{
+    return removeImpl(id, check_func, throw_if_not_exists);
 }
 
 
@@ -395,7 +405,7 @@ std::vector<UUID> IAccessStorage::tryRemove(const std::vector<UUID> & ids)
 }
 
 
-bool IAccessStorage::removeImpl(const UUID & id, bool throw_if_not_exists)
+bool IAccessStorage::removeImpl(const UUID & id, const CheckFunc &, bool throw_if_not_exists)
 {
     if (isReadOnly(id))
     {