From 26f0b8b79ae29819f5fb4a55559ffdded5e5e3d2 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Tue, 2 Jun 2026 11:10:26 +0200 Subject: [PATCH 1/2] Cherry-pick of https://github.com/Altinity/ClickHouse/pull/1859 with unresolved conflict markers (resolution in next commit) --- Original cherry-pick message follows: Merge pull request #1859 from Altinity/feat/antalya-26.3/90740 Antalya 26.3: support external paths in Iceberg tables # Conflicts: # src/IO/S3/URI.cpp # src/IO/S3/URI.h # src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp # src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.h # src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp # src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.h # src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp # src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h # src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp # src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp # src/Storages/ObjectStorage/DataLakes/Iceberg/PositionDeleteTransform.cpp # src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h # src/Storages/ObjectStorage/StorageObjectStorageSource.cpp # src/Storages/ObjectStorage/StorageObjectStorageSource.h # src/Storages/ObjectStorage/StorageObjectStorageStableTaskDistributor.cpp # src/Storages/ObjectStorage/Utils.h --- src/Core/ProtocolDefines.h | 3 +- src/Core/Settings.cpp | 3 + src/Core/SettingsChangesHistory.cpp | 1 + src/Databases/DataLake/DatabaseDataLake.cpp | 4 +- src/IO/S3/URI.cpp | 18 +- src/IO/S3/URI.h | 4 + src/Interpreters/ClusterFunctionReadTask.cpp | 6 +- src/Interpreters/IcebergMetadataLog.cpp | 6 +- .../Common/AvroForIcebergDeserializer.cpp | 9 +- .../DataLakes/IDataLakeMetadata.h | 3 + .../DataLakes/Iceberg/Compaction.cpp | 104 ++- .../DataLakes/Iceberg/Compaction.h | 2 + .../Iceberg/IcebergDataObjectInfo.cpp | 54 +- .../DataLakes/Iceberg/IcebergDataObjectInfo.h | 42 +- .../DataLakes/Iceberg/IcebergIterator.cpp | 75 ++- .../DataLakes/Iceberg/IcebergIterator.h | 17 +- .../DataLakes/Iceberg/IcebergMetadata.cpp | 38 +- .../DataLakes/Iceberg/IcebergMetadata.h | 8 + .../DataLakes/Iceberg/IcebergPath.cpp | 6 + .../DataLakes/Iceberg/IcebergPath.h | 6 + .../DataLakes/Iceberg/IcebergWrites.cpp | 1 + .../Iceberg/ManifestFileIterator.cpp | 5 + .../DataLakes/Iceberg/Mutations.cpp | 633 ++++++++++++++++++ .../DataLakes/Iceberg/Mutations.h | 1 + .../Iceberg/PositionDeleteTransform.cpp | 17 +- .../Iceberg/PositionDeleteTransform.h | 22 +- .../Iceberg/StatelessMetadataFileGetter.cpp | 36 +- .../Iceberg/StatelessMetadataFileGetter.h | 10 +- .../ObjectStorage/DataLakes/Iceberg/Utils.cpp | 30 +- .../ObjectStorage/DataLakes/Iceberg/Utils.h | 22 + .../StorageObjectStorageSource.cpp | 91 ++- .../StorageObjectStorageSource.h | 14 + ...rageObjectStorageStableTaskDistributor.cpp | 60 ++ src/Storages/ObjectStorage/Utils.cpp | 531 +++++++++++++++ src/Storages/ObjectStorage/Utils.h | 53 ++ src/Storages/StorageURL.cpp | 6 +- .../__init__.py | 0 .../configs/config.d/cluster.xml | 20 + .../configs/config.d/named_collections.xml | 15 + .../configs/config.d/query_log.xml | 6 + .../configs/users.d/users.xml | 9 + .../test_storage_iceberg_multistorage/test.py | 426 ++++++++++++ .../test_array_evolved_with_struct.py | 2 +- .../metadata/v1.metadata.json | 4 +- .../metadata/v1.metadata.json | 4 +- .../metadata/v1.metadata.json | 4 +- 46 files changed, 2318 insertions(+), 113 deletions(-) create mode 100644 tests/integration/test_storage_iceberg_multistorage/__init__.py create mode 100644 tests/integration/test_storage_iceberg_multistorage/configs/config.d/cluster.xml create mode 100644 tests/integration/test_storage_iceberg_multistorage/configs/config.d/named_collections.xml create mode 100644 tests/integration/test_storage_iceberg_multistorage/configs/config.d/query_log.xml create mode 100644 tests/integration/test_storage_iceberg_multistorage/configs/users.d/users.xml create mode 100644 tests/integration/test_storage_iceberg_multistorage/test.py diff --git a/src/Core/ProtocolDefines.h b/src/Core/ProtocolDefines.h index ec5b8bd015cb..57ccd8f64dde 100644 --- a/src/Core/ProtocolDefines.h +++ b/src/Core/ProtocolDefines.h @@ -39,7 +39,8 @@ static constexpr auto DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_ICEBERG_META static constexpr auto DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_FILE_BUCKETS_INFO = 4; static constexpr auto DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_EXCLUDED_ROWS = 5; static constexpr auto DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_ICEBERG_FILE_STATS = 6; -static constexpr auto DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION = DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_ICEBERG_FILE_STATS; +static constexpr auto DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_ICEBERG_ABSOLUTE_PATH = 7; +static constexpr auto DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION = DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_ICEBERG_ABSOLUTE_PATH; static constexpr auto DATA_LAKE_TABLE_STATE_SNAPSHOT_PROTOCOL_VERSION = 1; diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 99ed9098617a..649252ff6164 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -641,6 +641,9 @@ Use multiple threads for azure multipart upload. )", 0) \ DECLARE(Bool, s3_throw_on_zero_files_match, false, R"( Throw an error, when ListObjects request cannot match any files +)", 0) \ + DECLARE(Bool, s3_propagate_credentials_to_other_storages, false, R"( +Credentials from the base storage are always propagated to secondary object storages when endpoints match. When this setting is enabled, credentials are also propagated when endpoints differ, including less secure connections (for example, from `https` to plain `http`). )", 0) \ DECLARE(Bool, hdfs_throw_on_zero_files_match, false, R"( Throw an error if matched zero files according to glob expansion rules. diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 8c1dc4774a75..242adf009e03 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -159,6 +159,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"max_rand_distribution_parameter", 1e6, 1e6, "New setting to limit shape parameters in random distribution functions, preventing hangs with extreme inputs."}, {"optimize_truncate_order_by_after_group_by_keys", false, true, "Remove trailing ORDER BY elements once all GROUP BY keys are covered in the ORDER BY prefix."}, {"use_statistics_for_part_pruning", false, true, "New setting to use statistics for part pruning during query execution."}, + {"s3_propagate_credentials_to_other_storages", false, false, "New setting"}, {"http_max_fields", 1000000, 1000, "Reduce default to limit pre-authentication memory usage by HTTP connections."}, {"http_max_field_name_size", 131072, 4096, "Reduce default to limit pre-authentication memory usage by HTTP connections."}, {"http_max_request_header_size", 0, 10485760, "New setting to limit total HTTP request header size before authentication."}, diff --git a/src/Databases/DataLake/DatabaseDataLake.cpp b/src/Databases/DataLake/DatabaseDataLake.cpp index 12fbb051ba4a..3b5735c2ed43 100644 --- a/src/Databases/DataLake/DatabaseDataLake.cpp +++ b/src/Databases/DataLake/DatabaseDataLake.cpp @@ -593,7 +593,9 @@ StoragePtr DatabaseDataLake::tryGetTableImpl(const String & name, ContextPtr con LOG_DEBUG(log, "Has no credentials"); } } - else if (!lightweight && table_metadata.requiresCredentials() && std::find(vended_credentials_catalogs.begin(), vended_credentials_catalogs.end(), catalog->getCatalogType()) == vended_credentials_catalogs.end()) + else if (!lightweight && table_metadata.requiresCredentials() + && std::find(vended_credentials_catalogs.begin(), vended_credentials_catalogs.end(), catalog->getCatalogType()) == vended_credentials_catalogs.end() + && table_metadata.getStorageType() != DatabaseDataLakeStorageType::Local) { throw Exception( ErrorCodes::BAD_ARGUMENTS, diff --git a/src/IO/S3/URI.cpp b/src/IO/S3/URI.cpp index dd5429dcf55f..bb2bdfe472f8 100644 --- a/src/IO/S3/URI.cpp +++ b/src/IO/S3/URI.cpp @@ -17,10 +17,16 @@ namespace DB struct URIConverter { +<<<<<<< HEAD static void modifyURI(Poco::URI & uri, NameToNameMap mapper) +======= + static void modifyURI(Poco::URI & uri, std::unordered_map mapper, bool enable_url_encoding = true) +>>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) { Macros macros({{"bucket", uri.getHost()}}); - uri = macros.expand(mapper[uri.getScheme()]).empty() ? uri : Poco::URI(macros.expand(mapper[uri.getScheme()]) + uri.getPathAndQuery()); + uri = macros.expand(mapper[uri.getScheme()]).empty() + ? uri + : Poco::URI(macros.expand(mapper[uri.getScheme()]) + uri.getPathAndQuery(), enable_url_encoding); } }; @@ -32,7 +38,11 @@ namespace ErrorCodes namespace S3 { +<<<<<<< HEAD URI::URI(const std::string & uri_, bool allow_archive_path_syntax, bool keep_presigned_query_parameters, S3UriStyle uri_style) +======= +URI::URI(const std::string & uri_, bool allow_archive_path_syntax, bool keep_presigned_query_parameters, bool enable_url_encoding) +>>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) { /// Case when AWS Private Link Interface is being used /// E.g. (bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w.s3.us-east-1.vpce.amazonaws.com/bucket-name/key) @@ -44,9 +54,9 @@ URI::URI(const std::string & uri_, bool allow_archive_path_syntax, bool keep_pre else uri_str = uri_; - uri = Poco::URI(uri_str); + uri = Poco::URI(uri_str, enable_url_encoding); /// Keep a copy of how Poco parsed the original string before any mapping - Poco::URI original_uri(uri_str); + Poco::URI original_uri(uri_str, enable_url_encoding); bool looks_like_presigned = false; for (const auto & [qk, qv] : original_uri.getQueryParameters()) { @@ -91,7 +101,7 @@ URI::URI(const std::string & uri_, bool allow_archive_path_syntax, bool keep_pre } if (!mapper.empty()) - URIConverter::modifyURI(uri, mapper); + URIConverter::modifyURI(uri, mapper, enable_url_encoding); } storage_name = "S3"; diff --git a/src/IO/S3/URI.h b/src/IO/S3/URI.h index 64b4def76744..15dac27f771c 100644 --- a/src/IO/S3/URI.h +++ b/src/IO/S3/URI.h @@ -41,7 +41,11 @@ struct URI const std::string & uri_, bool allow_archive_path_syntax = false, bool keep_presigned_query_parameters = true, +<<<<<<< HEAD S3UriStyle uri_style = S3UriStyle::AUTO); +======= + bool enable_url_encoding = true); +>>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) void addRegionToURI(const std::string & region); static void validateBucket(const std::string & bucket, const Poco::URI & uri); diff --git a/src/Interpreters/ClusterFunctionReadTask.cpp b/src/Interpreters/ClusterFunctionReadTask.cpp index f3cfad1fba31..b6476630ca49 100644 --- a/src/Interpreters/ClusterFunctionReadTask.cpp +++ b/src/Interpreters/ClusterFunctionReadTask.cpp @@ -35,10 +35,8 @@ ClusterFunctionReadTaskResponse::ClusterFunctionReadTaskResponse(ObjectInfoPtr o data_lake_metadata = object->data_lake_metadata.value(); #if USE_AVRO - if (std::dynamic_pointer_cast(object)) - { - iceberg_info = dynamic_cast(*object).info; - } + if (auto iceberg_object = std::dynamic_pointer_cast(object)) + iceberg_info = iceberg_object->info; #endif const bool send_over_whole_archive = !context->getSettingsRef()[Setting::cluster_function_process_archive_on_multiple_nodes]; diff --git a/src/Interpreters/IcebergMetadataLog.cpp b/src/Interpreters/IcebergMetadataLog.cpp index 9536bc4ae96b..22ca3b2d7a88 100644 --- a/src/Interpreters/IcebergMetadataLog.cpp +++ b/src/Interpreters/IcebergMetadataLog.cpp @@ -105,12 +105,16 @@ void insertRowToLogTable( throw Exception(ErrorCodes::BAD_ARGUMENTS, "Iceberg metadata log table is not configured"); } + String normalized_table_path = table_path; + while (normalized_table_path.size() > 1 && normalized_table_path.back() == '/') + normalized_table_path.pop_back(); + iceberg_metadata_log->add( DB::IcebergMetadataLogElement{ .current_time = spec.tv_sec, .query_id = local_context->getCurrentQueryId(), .content_type = row_log_level, - .table_path = table_path, + .table_path = normalized_table_path, .file_path = file_path.serialize(), .metadata_content = row, .row_in_file = row_in_file, diff --git a/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp b/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp index 4517834d0dec..9dcea53add2e 100644 --- a/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp @@ -157,8 +157,7 @@ ParsedManifestFileEntryPtr AvroForIcebergDeserializer::createParsedManifestFileE } } - - const auto file_path_key = IcebergPathFromMetadata::deserialize( + const auto file_path_from_metadata = IcebergPathFromMetadata::deserialize( getValueFromRowByName(row_index, c_data_file_file_path, TypeIndex::String).safeGet()); /// NOTE: This is weird, because in manifest file partition looks like this: /// { @@ -247,7 +246,7 @@ ParsedManifestFileEntryPtr AvroForIcebergDeserializer::createParsedManifestFileE case FileContentType::DATA: { return std::make_shared( FileContentType::DATA, - file_path_key, + file_path_from_metadata, row_index, status, sequence_number, @@ -294,7 +293,7 @@ ParsedManifestFileEntryPtr AvroForIcebergDeserializer::createParsedManifestFileE } return std::make_shared( FileContentType::POSITION_DELETE, - file_path_key, + file_path_from_metadata, row_index, status, sequence_number, @@ -325,7 +324,7 @@ ParsedManifestFileEntryPtr AvroForIcebergDeserializer::createParsedManifestFileE c_data_file_equality_ids); return std::make_shared( FileContentType::EQUALITY_DELETE, - file_path_key, + file_path_from_metadata, row_index, status, sequence_number, diff --git a/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h b/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h index 389232fb1d16..48494736b0ce 100644 --- a/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h @@ -56,6 +56,9 @@ class IDataLakeMetadata : boost::noncopyable virtual bool operator==(const IDataLakeMetadata & other) const = 0; + /// Returns the full table location URI (e.g. `s3a://bucket/prefix/table/`) + virtual std::string getTableLocation() const { return {}; } + /// Return iterator to `data files`. using FileProgressCallback = std::function; virtual ObjectIterator iterate( diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp index 66f07c521b27..ff7fc2298c70 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -80,6 +81,9 @@ struct Plan std::unordered_map> manifest_list_to_manifest_files; std::unordered_map>> snapshot_id_to_data_files; std::unordered_map> path_to_data_file; + /// Raw paths of every file referenced by the snapshots being compacted, used at cleanup + /// time to also remove files that live outside the base object_storage. + std::unordered_set referenced_file_paths; FileNamesGenerator generator; Poco::JSON::Object::Ptr initial_metadata_object; @@ -121,6 +125,7 @@ static Plan getPlan( const DataLakeStorageSettings & data_lake_settings, const PersistentTableComponents & persistent_table_components, ObjectStoragePtr object_storage, + SecondaryStorages & secondary_storages, const String & write_format, ContextPtr context, CompressionMethod compression_method) @@ -163,14 +168,16 @@ static Plan getPlan( std::unordered_map> manifest_files; for (const auto & snapshot : snapshots_info) { - auto manifest_list = getManifestList(object_storage, persistent_table_components, context, snapshot.manifest_list_path, log); + plan.referenced_file_paths.insert(snapshot.manifest_list_path); + auto manifest_list = getManifestList(object_storage, persistent_table_components, context, snapshot.manifest_list_path, log, secondary_storages); for (const auto & manifest_file : manifest_list) { plan.manifest_list_to_manifest_files[snapshot.manifest_list_path].push_back(manifest_file.manifest_file_path); if (!plan.manifest_file_to_first_snapshot.contains(manifest_file.manifest_file_path)) plan.manifest_file_to_first_snapshot[manifest_file.manifest_file_path] = snapshot.snapshot_id; + plan.referenced_file_paths.insert(manifest_file.manifest_file_path); auto files_handle = getManifestFileEntriesHandle( - object_storage, persistent_table_components, context, log, manifest_file, static_cast(current_schema_id)); + object_storage, persistent_table_components, context, log, manifest_file, static_cast(current_schema_id), secondary_storages); if (!manifest_files.contains(manifest_file.manifest_file_path)) { @@ -179,28 +186,39 @@ static Plan getPlan( } manifest_files[manifest_file.manifest_file_path]->manifest_lists_path.push_back(snapshot.manifest_list_path); for (const auto & pos_delete_file : files_handle.getFilesWithoutDeleted(FileContentType::POSITION_DELETE)) + { all_positional_delete_files.push_back(pos_delete_file); + plan.referenced_file_paths.insert(pos_delete_file->parsed_entry->file_path_key); + } for (const auto & data_file : files_handle.getFilesWithoutDeleted(FileContentType::DATA)) { + plan.referenced_file_paths.insert(data_file->parsed_entry->file_path_key); auto partition_index = plan.partition_encoder.encodePartition(data_file->parsed_entry->partition_key_value); if (plan.partitions.size() <= partition_index) plan.partitions.push_back({}); + const auto & raw_metadata_path = data_file->parsed_entry->file_path_key.serialize(); + auto [resolved_storage, resolved_key] = resolveObjectStorageForPath( + persistent_table_components.table_location, + raw_metadata_path, object_storage, secondary_storages, context, + persistent_table_components.path_resolver); + IcebergDataObjectInfoPtr data_object_info = std::make_shared( - data_file, persistent_table_components.path_resolver.resolve(data_file->parsed_entry->file_path_key), 0); + data_file, raw_metadata_path, 0, resolved_storage, resolved_key); std::shared_ptr data_file_ptr; - if (!plan.path_to_data_file.contains(manifest_file.manifest_file_path)) + auto path_identifier = Iceberg::IcebergPathFromMetadata::makeStorageIdentity(resolved_storage, resolved_key); + if (!plan.path_to_data_file.contains(path_identifier)) { data_file_ptr = std::make_shared(DataFilePlan{ .data_object_info = data_object_info, .manifest_list = manifest_files[manifest_file.manifest_file_path], .patched_path = plan.generator.generateDataFileName()}); - plan.path_to_data_file[manifest_file.manifest_file_path] = data_file_ptr; + plan.path_to_data_file[path_identifier] = data_file_ptr; } else { - data_file_ptr = plan.path_to_data_file[manifest_file.manifest_file_path]; + data_file_ptr = plan.path_to_data_file[path_identifier]; } plan.partitions[partition_index].push_back(data_file_ptr); plan.snapshot_id_to_data_files[snapshot.snapshot_id].push_back(plan.partitions[partition_index].back()); @@ -219,7 +237,7 @@ static Plan getPlan( { if (data_file->data_object_info->info.sequence_number <= delete_file->sequence_number) data_file->data_object_info->addPositionDeleteObject( - delete_file, persistent_table_components.path_resolver.resolve(delete_file->parsed_entry->file_path_key)); + delete_file, delete_file->parsed_entry->file_path_key.serialize()); } } plan.history = std::move(snapshots_info); @@ -235,7 +253,8 @@ static void writeDataFiles( const std::optional & format_settings, ContextPtr context, const String & write_format, - CompressionMethod write_compression_method) + CompressionMethod write_compression_method, + std::shared_ptr secondary_storages) { ColumnMapperPtr column_mapper; { @@ -261,10 +280,15 @@ static void writeDataFiles( format_settings, // todo make compaction using same FormatParserSharedResources std::make_shared(context->getSettingsRef(), 1), - context); + context, + path_resolver, + secondary_storages); - RelativePathWithMetadata relative_path(data_file->data_object_info->getPath()); - auto read_buffer = createReadBuffer(relative_path, object_storage, context, getLogger("IcebergCompaction")); + ObjectStoragePtr storage_to_use = data_file->data_object_info->getResolvedStorage(); + if (!storage_to_use) + storage_to_use = object_storage; + RelativePathWithMetadata object_info(data_file->data_object_info->getPath()); + auto read_buffer = createReadBuffer(object_info, storage_to_use, context, getLogger("IcebergCompaction")); const Settings & settings = context->getSettingsRef(); auto parser_shared_resources = std::make_shared( @@ -478,6 +502,7 @@ static void writeMetadataFiles( { manifest_entry->patched_path = plan.generator.generateManifestEntryName(); manifest_file_renamings[manifest_entry->path] = manifest_entry->patched_path; + auto buffer_manifest_entry = object_storage->writeObject( StoredObject(path_resolver.resolve(manifest_entry->patched_path)), WriteMode::Rewrite, @@ -601,22 +626,55 @@ static void writeMetadataFiles( } } +<<<<<<< HEAD static std::vector getOldFiles(ObjectStoragePtr object_storage, const String & table_path) +======= +/// Files to delete after compaction: a base-storage directory listing under `metadata/` and +/// `data/` (covers historical metadata.json and any orphan files on the base storage), plus +/// any paths from the compacted snapshots that resolve to a secondary storage. +std::vector> getOldFiles( + ObjectStoragePtr object_storage, + SecondaryStorages & secondary_storages, + ContextPtr context, + const PersistentTableComponents & persistent_table_components, + const Plan & plan) +>>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) { - auto metadata_files = listFiles(*object_storage, table_path, "metadata", ""); - auto data_files = listFiles(*object_storage, table_path, "data", ""); + std::vector> result; + + for (auto && file : listFiles(*object_storage, persistent_table_components.table_path, "metadata", "")) + result.emplace_back(object_storage, std::move(file)); + for (auto && file : listFiles(*object_storage, persistent_table_components.table_path, "data", "")) + result.emplace_back(object_storage, std::move(file)); + + for (const auto & raw_path : plan.referenced_file_paths) + { + auto [storage_to_use, key_in_storage] = resolveObjectStorageForPath( + persistent_table_components.table_location, + raw_path.serialize(), + object_storage, + secondary_storages, + context, + persistent_table_components.path_resolver); - for (auto && data_file : data_files) - metadata_files.push_back(data_file); + if (storage_to_use.get() != object_storage.get()) + result.emplace_back(std::move(storage_to_use), std::move(key_in_storage)); + } - return metadata_files; + return result; } +<<<<<<< HEAD static void clearOldFiles(ObjectStoragePtr object_storage, const std::vector & old_files) +======= +void clearOldFiles(const std::vector> & old_files) +>>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) { - for (const auto & metadata_file : old_files) + auto log = getLogger("IcebergCompaction"); + for (const auto & [storage, key] : old_files) { - object_storage->removeObjectIfExists(StoredObject(metadata_file)); + LOG_DEBUG(log, "Removing old file during compaction: storage={}, key={}", storage->getDescription(), key); + storage->removeObjectIfExists(StoredObject(key)); } } @@ -624,6 +682,7 @@ void compactIcebergTable( IcebergHistory snapshots_info, const PersistentTableComponents & persistent_table_components, ObjectStoragePtr object_storage_, + std::shared_ptr secondary_storages_, const DataLakeStorageSettings & data_lake_settings, const std::optional & format_settings_, SharedHeader sample_block_, @@ -637,12 +696,14 @@ void compactIcebergTable( data_lake_settings, persistent_table_components, object_storage_, + *secondary_storages_, write_format, context_, persistent_table_components.metadata_compression_method); if (plan.need_optimize) { - auto old_files = getOldFiles(object_storage_, persistent_table_components.table_path); + auto old_files = getOldFiles( + object_storage_, *secondary_storages_, context_, persistent_table_components, plan); writeDataFiles( plan, sample_block_, @@ -651,9 +712,10 @@ void compactIcebergTable( format_settings_, context_, write_format, - persistent_table_components.metadata_compression_method); + persistent_table_components.metadata_compression_method, + secondary_storages_); writeMetadataFiles(plan, persistent_table_components.path_resolver, object_storage_, context_, sample_block_, write_format, persistent_table_components.table_path); - clearOldFiles(object_storage_, old_files); + clearOldFiles(old_files); } } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.h index 0916002f99f3..302bc3d30e69 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.h @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB::Iceberg @@ -15,6 +16,7 @@ void compactIcebergTable( IcebergHistory snapshots_info, const PersistentTableComponents & persistent_table_components, DB::ObjectStoragePtr object_storage_, + std::shared_ptr secondary_storages_, const DataLakeStorageSettings & data_lake_settings, const std::optional & format_settings_, DB::SharedHeader sample_block_, diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.cpp index 4e161ca8c863..5eb3f8501350 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.cpp @@ -1,3 +1,4 @@ +#include #include "config.h" #include @@ -11,15 +12,18 @@ #include #include +#include #include #include #include +#include namespace DB::ErrorCodes { extern const int NOT_IMPLEMENTED; extern const int UNKNOWN_PROTOCOL; +extern const int PROTOCOL_VERSION_MISMATCH; } @@ -53,10 +57,11 @@ String computePartitionId(const Row & partition_key_value) #if USE_AVRO IcebergDataObjectInfo::IcebergDataObjectInfo( - Iceberg::ProcessedManifestFileEntryPtr data_manifest_file_entry_, const String & resolved_storage_path_, Int32 schema_id_relevant_to_iterator_) - : ObjectInfo(RelativePathWithMetadata(resolved_storage_path_)) + Iceberg::ProcessedManifestFileEntryPtr data_manifest_file_entry_, const String & metadata_path_, Int32 schema_id_relevant_to_iterator_, ObjectStoragePtr resolved_storage_, const String & resolved_key_) + : ObjectInfo(RelativePathWithMetadata(resolved_key_.empty() ? metadata_path_ : resolved_key_)) , info{ data_manifest_file_entry_->parsed_entry->file_path_key, + metadata_path_, data_manifest_file_entry_->resolved_schema_id, schema_id_relevant_to_iterator_, data_manifest_file_entry_->sequence_number, @@ -67,7 +72,11 @@ IcebergDataObjectInfo::IcebergDataObjectInfo( /* equality_deletes_objects */ {}, data_manifest_file_entry_->parsed_entry->record_count, data_manifest_file_entry_->parsed_entry->file_size_in_bytes} + , resolved_storage(std::move(resolved_storage_)) { + /// resolved_storage and resolved_key must be provided together or neither must be provided + /// (default-constructed, meaning the path has not been resolved yet). + chassert(resolved_key_.empty() == (resolved_storage == nullptr)); } IcebergDataObjectInfo::IcebergDataObjectInfo(const RelativePathWithMetadata & path_) @@ -86,13 +95,15 @@ std::shared_ptr IcebergDataObjectInfo::getPositionDeleteTransf const SharedHeader & header, const std::optional & format_settings, FormatParserSharedResourcesPtr parser_shared_resources, - ContextPtr context_) + ContextPtr context_, + const Iceberg::IcebergPathResolver & path_resolver, + std::shared_ptr secondary_storages) { IcebergDataObjectInfoPtr self = shared_from_this(); if (!context_->getSettingsRef()[Setting::use_roaring_bitmap_iceberg_positional_deletes].value) - return std::make_shared(header, self, object_storage, format_settings, parser_shared_resources, context_); + return std::make_shared(header, self, object_storage, format_settings, parser_shared_resources, context_, path_resolver, secondary_storages); else - return std::make_shared(header, self, object_storage, format_settings, parser_shared_resources, context_); + return std::make_shared(header, self, object_storage, format_settings, parser_shared_resources, context_, path_resolver, secondary_storages); } void IcebergDataObjectInfo::addPositionDeleteObject(Iceberg::ProcessedManifestFileEntryPtr position_delete_object, const String & resolved_storage_path) @@ -123,7 +134,30 @@ void IcebergDataObjectInfo::addEqualityDeleteObject(const Iceberg::ProcessedMani void IcebergObjectSerializableInfo::serializeForClusterFunctionProtocol(WriteBuffer & out, size_t protocol_version) const { checkVersion(protocol_version); + + if (requires_external_storage && protocol_version < DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_ICEBERG_ABSOLUTE_PATH) + { + throw Exception( + ErrorCodes::PROTOCOL_VERSION_MISMATCH, + "Iceberg data file '{}' is outside of the table location, " + "worker needs to have protocol version >= {}, but has {}. ", + data_object_file_metadata_path, + DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_ICEBERG_ABSOLUTE_PATH, + protocol_version); + } + + auto path_for_protocol = [&](const String & path) -> String + { + if (protocol_version < DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_ICEBERG_ABSOLUTE_PATH) + return SchemeAuthorityKey(path).key; + return path; + }; + writeStringBinary(data_object_file_path_key.serialize(), out); + if (protocol_version >= DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_ICEBERG_ABSOLUTE_PATH) + { + writeStringBinary(data_object_file_metadata_path, out); + } writeVarInt(underlying_format_read_schema_id, out); writeVarInt(schema_id_relevant_to_iterator, out); writeVarInt(sequence_number, out); @@ -132,12 +166,12 @@ void IcebergObjectSerializableInfo::serializeForClusterFunctionProtocol(WriteBuf writeVarUInt(position_deletes_objects.size(), out); for (const auto & pos_delete_obj : position_deletes_objects) { - writeStringBinary(pos_delete_obj.file_path, out); + writeStringBinary(path_for_protocol(pos_delete_obj.file_path), out); writeStringBinary(pos_delete_obj.file_format, out); if (pos_delete_obj.reference_data_file_path.has_value()) { writeVarUInt(1, out); - writeStringBinary(pos_delete_obj.reference_data_file_path.value(), out); + writeStringBinary(path_for_protocol(pos_delete_obj.reference_data_file_path.value()), out); } else { @@ -149,7 +183,7 @@ void IcebergObjectSerializableInfo::serializeForClusterFunctionProtocol(WriteBuf writeVarUInt(equality_deletes_objects.size(), out); for (const auto & eq_delete_obj : equality_deletes_objects) { - writeStringBinary(eq_delete_obj.file_path, out); + writeStringBinary(path_for_protocol(eq_delete_obj.file_path), out); writeStringBinary(eq_delete_obj.file_format, out); writeVarInt(eq_delete_obj.schema_id, out); if (eq_delete_obj.equality_ids.has_value()) @@ -198,6 +232,10 @@ void IcebergObjectSerializableInfo::deserializeForClusterFunctionProtocol(ReadBu readStringBinary(raw_path, in); data_object_file_path_key = IcebergPathFromMetadata::deserialize(std::move(raw_path)); } + if (protocol_version >= DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_ICEBERG_ABSOLUTE_PATH) + { + readStringBinary(data_object_file_metadata_path, in); + } readVarInt(underlying_format_read_schema_id, in); readVarInt(schema_id_relevant_to_iterator, in); readVarInt(sequence_number, in); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.h index 6763d1013201..28cca2f78264 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.h @@ -21,9 +21,19 @@ String computePartitionId(const Row & partition_key_value); struct IcebergObjectSerializableInfo { IcebergPathFromMetadata data_object_file_path_key; +<<<<<<< HEAD Int32 underlying_format_read_schema_id{}; Int32 schema_id_relevant_to_iterator{}; Int64 sequence_number{}; +======= + /// Raw path string as written in the Iceberg manifest, preserved as-is (may be a full URI like + /// `s3://bucket/...` or a relative path). Used for the `_path` virtual column and as a stable + /// task identifier. Not a canonicalised storage key — see `IcebergPathResolver::resolve` for that. + String data_object_file_metadata_path; + Int32 underlying_format_read_schema_id; + Int32 schema_id_relevant_to_iterator; + Int64 sequence_number; +>>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) String file_format; String manifest_file; String partition_id; @@ -32,6 +42,9 @@ struct IcebergObjectSerializableInfo std::optional record_count; std::optional file_size_in_bytes; + /// Set to true by the coordinator when the file is outside of the table location + bool requires_external_storage = false; + void serializeForClusterFunctionProtocol(WriteBuffer & out, size_t protocol_version) const; void deserializeForClusterFunctionProtocol(ReadBuffer & in, size_t protocol_version); @@ -44,6 +57,7 @@ struct IcebergObjectSerializableInfo #if USE_AVRO #include +#include #include @@ -60,7 +74,14 @@ struct IcebergDataObjectInfo : public ObjectInfo, std::enable_shared_from_this & format_settings, FormatParserSharedResourcesPtr parser_shared_resources, - ContextPtr context_); + ContextPtr context_, + const Iceberg::IcebergPathResolver & path_resolver, + std::shared_ptr secondary_storages); std::optional getFileFormat() const override { return info.file_format; } @@ -83,8 +106,23 @@ struct IcebergDataObjectInfo : public ObjectInfo, std::enable_shared_from_this getMetadataPath() const + { + if (info.data_object_file_metadata_path.empty()) + return std::nullopt; + return info.data_object_file_metadata_path; + } + + ObjectStoragePtr getResolvedStorage() const { return resolved_storage; } + + void setResolvedStorage(ObjectStoragePtr storage) { resolved_storage = std::move(storage); } + void addEqualityDeleteObject(const Iceberg::ProcessedManifestFileEntryPtr & equality_delete_object, const String & resolved_storage_path); Iceberg::IcebergObjectSerializableInfo info; + +private: + /// For files located in a different storage than the table's main storage + ObjectStoragePtr resolved_storage; }; using IcebergDataObjectInfoPtr = std::shared_ptr; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp index 39cc26a5fd6a..8ea7eddad3a8 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp @@ -44,6 +44,7 @@ #include #include #include +#include #include @@ -185,7 +186,8 @@ std::optional SingleThreadIcebergKeysIterator::ne local_context, log, manifest_list_entry.manifest_file_path, - manifest_list_entry.manifest_file_byte_size); + manifest_list_entry.manifest_file_byte_size, + *secondary_storages); current_manifest_file_iterator = Iceberg::ManifestFileIterator::create( manifest_file_cacheable_part.deserializer, @@ -212,7 +214,8 @@ SingleThreadIcebergKeysIterator::SingleThreadIcebergKeysIterator( const ActionsDAG * filter_dag_, Iceberg::TableStateSnapshotPtr table_snapshot_, Iceberg::IcebergDataSnapshotPtr data_snapshot_, - PersistentTableComponents persistent_components_) + PersistentTableComponents persistent_components_, + std::shared_ptr secondary_storages_) : object_storage(object_storage_) , filter_dag( [&]() -> std::shared_ptr @@ -235,6 +238,7 @@ SingleThreadIcebergKeysIterator::SingleThreadIcebergKeysIterator( , data_snapshot(data_snapshot_) , persistent_components(persistent_components_) , log(getLogger("IcebergIterator")) + , secondary_storages(secondary_storages_) , manifest_file_content_type(manifest_file_content_type_) { } @@ -246,10 +250,12 @@ IcebergIterator::IcebergIterator( IDataLakeMetadata::FileProgressCallback callback_, Iceberg::TableStateSnapshotPtr table_snapshot_, Iceberg::IcebergDataSnapshotPtr data_snapshot_, - PersistentTableComponents persistent_components_) + PersistentTableComponents persistent_components_, + std::shared_ptr secondary_storages_) : logger(getLogger("IcebergIterator")) , filter_dag(filter_dag_ ? std::make_shared(filter_dag_->clone()) : nullptr) , object_storage(std::move(object_storage_)) + , local_context(local_context_) , table_state_snapshot(table_snapshot_) , persistent_components(persistent_components_) , data_files_iterator( @@ -259,7 +265,8 @@ IcebergIterator::IcebergIterator( filter_dag.get(), table_snapshot_, data_snapshot_, - persistent_components_) + persistent_components_, + secondary_storages_) , deletes_iterator( object_storage, local_context_, @@ -267,9 +274,15 @@ IcebergIterator::IcebergIterator( filter_dag.get(), table_snapshot_, data_snapshot_, - persistent_components_) + persistent_components_, + secondary_storages_) , blocking_queue(100) , callback(std::move(callback_)) +<<<<<<< HEAD +======= + , table_schema_id(table_snapshot_->schema_id) + , secondary_storages(secondary_storages_) +>>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) { auto delete_file = deletes_iterator.next(); while (delete_file.has_value()) @@ -328,11 +341,17 @@ ObjectInfoPtr IcebergIterator::next(size_t) Iceberg::ProcessedManifestFileEntryPtr manifest_file_entry; if (blocking_queue.pop(manifest_file_entry)) { - IcebergDataObjectInfoPtr object_info - = std::make_shared( - manifest_file_entry, - persistent_components.path_resolver.resolve(manifest_file_entry->parsed_entry->file_path_key), - table_state_snapshot->schema_id); + const auto & raw_metadata_path = manifest_file_entry->parsed_entry->file_path_key.serialize(); + auto [storage_to_use, resolved_key] = resolveObjectStorageForPath( + persistent_components.table_location, raw_metadata_path, + object_storage, *secondary_storages, local_context, + persistent_components.path_resolver); + + IcebergDataObjectInfoPtr object_info = std::make_shared( + manifest_file_entry, raw_metadata_path, table_state_snapshot->schema_id, storage_to_use, resolved_key); + + object_info->info.requires_external_storage = (storage_to_use != object_storage); + for (const auto & position_delete : defineDeletesSpan(manifest_file_entry, position_deletes_files, /* is_equality_delete */ false, logger)) { @@ -368,7 +387,7 @@ ObjectInfoPtr IcebergIterator::next(size_t) lower.has_value() ? lower->serialize() : "[no lower bound]", upper.has_value() ? upper->serialize() : "[no upper bound]"); object_info->addPositionDeleteObject( - position_delete, persistent_components.path_resolver.resolve(position_delete->parsed_entry->file_path_key)); + position_delete, position_delete->parsed_entry->file_path_key.serialize()); } } @@ -385,7 +404,7 @@ ObjectInfoPtr IcebergIterator::next(size_t) defineDeletesSpan(manifest_file_entry, equality_deletes_files, /* is_equality_delete */ true, logger)) { object_info->addEqualityDeleteObject( - equality_delete, persistent_components.path_resolver.resolve(equality_delete->parsed_entry->file_path_key)); + equality_delete, equality_delete->parsed_entry->file_path_key.serialize()); } if (!object_info->info.equality_deletes_objects.empty()) @@ -397,6 +416,38 @@ ObjectInfoPtr IcebergIterator::next(size_t) object_info->info.data_object_file_path_key); } +<<<<<<< HEAD +======= + object_info->relative_path_with_metadata.setFileMetaInfo(std::make_shared( + *persistent_components.schema_processor, + table_schema_id, /// current schema id to use current column names + manifest_file_entry->resolved_schema_id, /// file's schema id to interpret value_bounds bytes + manifest_file_entry->parsed_entry->columns_infos, + manifest_file_entry->parsed_entry->value_bounds)); + + if (!object_info->info.requires_external_storage) + { + auto resolves_to_external_storage = [&](const String & file_path) + { + auto [del_storage, del_key] = resolveObjectStorageForPath( + persistent_components.table_location, file_path, object_storage, *secondary_storages, local_context, + persistent_components.path_resolver); + return del_storage != object_storage; + }; + auto any_external = [&](const auto & delete_objects) + { + for (const auto & del : delete_objects) + if (resolves_to_external_storage(del.file_path)) + return true; + return false; + }; + + object_info->info.requires_external_storage = + any_external(object_info->info.position_deletes_objects) + || any_external(object_info->info.equality_deletes_objects); + } + +>>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) ProfileEvents::increment(ProfileEvents::IcebergMetadataReturnedObjectInfos); if (callback) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.h index 09c21b8ec292..3f71a6111abd 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.h @@ -27,6 +27,7 @@ #include #include #include +#include namespace DB { @@ -44,7 +45,8 @@ class SingleThreadIcebergKeysIterator const ActionsDAG * filter_dag_, TableStateSnapshotPtr table_snapshot_, IcebergDataSnapshotPtr data_snapshot_, - PersistentTableComponents persistent_components); + PersistentTableComponents persistent_components, + std::shared_ptr secondary_storages_); std::optional next(); @@ -57,6 +59,8 @@ class SingleThreadIcebergKeysIterator PersistentTableComponents persistent_components; LoggerPtr log; + std::shared_ptr secondary_storages; + size_t manifest_file_index = 0; Iceberg::ManifestIteratorPtr current_manifest_file_iterator; @@ -75,7 +79,12 @@ class IcebergIterator : public IObjectIterator IDataLakeMetadata::FileProgressCallback callback_, Iceberg::TableStateSnapshotPtr table_snapshot_, Iceberg::IcebergDataSnapshotPtr data_snapshot_, +<<<<<<< HEAD Iceberg::PersistentTableComponents persistent_components); +======= + Iceberg::PersistentTableComponents persistent_components_, + std::shared_ptr secondary_storages_); +>>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) ObjectInfoPtr next(size_t) override; @@ -86,6 +95,7 @@ class IcebergIterator : public IObjectIterator LoggerPtr logger; std::shared_ptr filter_dag; ObjectStoragePtr object_storage; + ContextPtr local_context; const Iceberg::TableStateSnapshotPtr table_state_snapshot; Iceberg::PersistentTableComponents persistent_components; Iceberg::SingleThreadIcebergKeysIterator data_files_iterator; @@ -97,6 +107,11 @@ class IcebergIterator : public IObjectIterator std::vector equality_deletes_files; std::exception_ptr exception; std::mutex exception_mutex; +<<<<<<< HEAD +======= + Int32 table_schema_id; + std::shared_ptr secondary_storages; // Sometimes data or manifests can be located on another storage +>>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) }; } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp index 82adf64ab122..26e7d5e70659 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp @@ -226,7 +226,12 @@ IcebergMetadata::IcebergMetadata( ContextPtr context_) : log(getLogger("IcebergMetadata")) , object_storage(std::move(object_storage_)) +<<<<<<< HEAD , persistent_components(std::move(persistent_components_)) +======= + , secondary_storages(std::make_shared()) + , persistent_components(initializePersistentTableComponents(configuration_, cache_ptr, context_)) +>>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) , data_lake_settings(configuration_->getDataLakeSettings()) , write_format(configuration_->format) { @@ -283,8 +288,16 @@ void IcebergMetadata::backgroundMetadataPrefetcherThread() for (const auto & entry : actual_data_snapshot->manifest_list_entries) { /// second, we fetch, parse and cache each manifest file +<<<<<<< HEAD auto manifest_file_ptr = getManifestFileEntriesHandle( object_storage, persistent_components, ctx, log, entry, actual_table_state_snapshot.schema_id); +======= + auto manifest_file_ptr = Iceberg::getManifestFile( + object_storage, persistent_components, ctx, log, + entry.manifest_file_path, + entry.manifest_file_byte_size, + *secondary_storages); +>>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) } } @@ -414,7 +427,7 @@ IcebergDataSnapshotPtr IcebergMetadata::createIcebergDataSnapshotFromSnapshotJSO return std::make_shared( - getManifestList(object_storage, persistent_components, local_context, manifest_list_file_path, log), + getManifestList(object_storage, persistent_components, local_context, manifest_list_file_path, log, *secondary_storages), snapshot_id, schema_id, total_rows, @@ -443,6 +456,7 @@ bool IcebergMetadata::optimize( snapshots_info, persistent_components, object_storage, + secondary_storages, data_lake_settings, format_settings, sample_block, @@ -1029,7 +1043,7 @@ bool IcebergMetadata::isDataSortedBySortingKey(StorageMetadataPtr storage_metada for (const auto & manifest_list_entry : data_snapshot->manifest_list_entries) { auto files_handle = getManifestFileEntriesHandle( - object_storage, persistent_components, context, log, manifest_list_entry, table_state_snapshot->schema_id); + object_storage, persistent_components, context, log, manifest_list_entry, table_state_snapshot->schema_id, *secondary_storages); if (!files_handle.areAllDataFilesSortedBySortOrderID(sorting_key.sort_order_id.value())) return false; @@ -1060,7 +1074,7 @@ std::optional IcebergMetadata::totalRows(ContextPtr local_context) const for (const auto & manifest_list_entry : actual_data_snapshot->manifest_list_entries) { auto manifest_file_ptr = getManifestFileEntriesHandle( - object_storage, persistent_components, local_context, log, manifest_list_entry, actual_table_state_snapshot.schema_id); + object_storage, persistent_components, local_context, log, manifest_list_entry, actual_table_state_snapshot.schema_id, *secondary_storages); auto data_count = manifest_file_ptr.getRowsCountInAllFilesExcludingDeleted(FileContentType::DATA); auto position_deletes_count = manifest_file_ptr.getRowsCountInAllFilesExcludingDeleted(FileContentType::POSITION_DELETE); if (!data_count.has_value() || !position_deletes_count.has_value()) @@ -1089,7 +1103,7 @@ std::optional IcebergMetadata::totalBytes(ContextPtr local_context) cons for (const auto & manifest_list_entry : actual_data_snapshot->manifest_list_entries) { auto manifest_file_ptr = getManifestFileEntriesHandle( - object_storage, persistent_components, local_context, log, manifest_list_entry, actual_table_state_snapshot.schema_id); + object_storage, persistent_components, local_context, log, manifest_list_entry, actual_table_state_snapshot.schema_id, *secondary_storages); auto count = manifest_file_ptr.getBytesCountInAllDataFilesExcludingDeleted(); if (!count.has_value()) return {}; @@ -1125,7 +1139,8 @@ ObjectIterator IcebergMetadata::iterate( callback, iceberg_table_state, getRelevantDataSnapshotFromTableStateSnapshot(*iceberg_table_state, local_context), - persistent_components); + persistent_components, + secondary_storages); } NamesAndTypesList IcebergMetadata::getTableSchema(ContextPtr local_context) const @@ -1182,7 +1197,7 @@ void IcebergMetadata::addDeleteTransformers( { builder.addSimpleTransform( [&](const SharedHeader & header) - { return iceberg_object_info->getPositionDeleteTransformer(object_storage, header, format_settings, parser_shared_resources, local_context); }); + { return iceberg_object_info->getPositionDeleteTransformer(object_storage, header, format_settings, parser_shared_resources, local_context, persistent_components.path_resolver, secondary_storages); }); } const auto & delete_files = iceberg_object_info->info.equality_deletes_objects; LOG_DEBUG(log, "Constructing filter transform for equality delete, there are {} delete files", delete_files.size()); @@ -1192,9 +1207,14 @@ void IcebergMetadata::addDeleteTransformers( { /// get header of delete file Block delete_file_header; - RelativePathWithMetadata delete_file_object(delete_file.file_path); + + auto [delete_storage_to_use, resolved_delete_key] = resolveObjectStorageForPath( + persistent_components.table_location, delete_file.file_path, object_storage, *secondary_storages, local_context, + persistent_components.path_resolver); + + RelativePathWithMetadata delete_file_object(resolved_delete_key); { - auto schema_read_buffer = createReadBuffer(delete_file_object, object_storage, local_context, log); + auto schema_read_buffer = createReadBuffer(delete_file_object, delete_storage_to_use, local_context, log); auto schema_reader = FormatFactory::instance().getSchemaReader(delete_file.file_format, *schema_read_buffer, local_context); auto columns_with_names = schema_reader->readSchema(); ColumnsWithTypeAndName initial_header_data; @@ -1221,7 +1241,7 @@ void IcebergMetadata::addDeleteTransformers( } /// Then we read the content of the delete file. auto mutable_columns_for_set = block_for_set.cloneEmptyColumns(); - std::unique_ptr data_read_buffer = createReadBuffer(delete_file_object, object_storage, local_context, log); + std::unique_ptr data_read_buffer = createReadBuffer(delete_file_object, delete_storage_to_use, local_context, log); CompressionMethod compression_method = chooseCompressionMethod(delete_file.file_path, "auto"); auto delete_format = FormatFactory::instance().getInput( delete_file.file_format, diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h index ee7b86b47791..dfc24a6ea984 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h @@ -28,6 +28,7 @@ #include #include #include +#include namespace DB { @@ -139,6 +140,8 @@ class IcebergMetadata : public IDataLakeMetadata CompressionMethod getCompressionMethod() const { return persistent_components.metadata_compression_method; } + std::string getTableLocation() const override { return persistent_components.table_location; } + bool optimize(const StorageMetadataPtr & metadata_snapshot, ContextPtr context, const std::optional & format_settings) override; bool supportsDelete() const override { return true; } void mutate( @@ -200,7 +203,12 @@ class IcebergMetadata : public IDataLakeMetadata LoggerPtr log; const ObjectStoragePtr object_storage; +<<<<<<< HEAD const DB::Iceberg::PersistentTableComponents persistent_components; +======= + mutable std::shared_ptr secondary_storages; + DB::Iceberg::PersistentTableComponents persistent_components; +>>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) const DataLakeStorageSettings & data_lake_settings; const String write_format; BackgroundSchedulePoolTaskHolder background_metadata_prefetch_task; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergPath.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergPath.cpp index 833d5f179917..71f55cd6add3 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergPath.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergPath.cpp @@ -1,6 +1,7 @@ #include #include +#include #include namespace DB::ErrorCodes @@ -11,6 +12,11 @@ extern const int BAD_ARGUMENTS; namespace DB::Iceberg { +IcebergPathFromMetadata IcebergPathFromMetadata::makeStorageIdentity(const ObjectStoragePtr & storage, const String & key) +{ + return IcebergPathFromMetadata(storage->getDescription() + '\0' + storage->getObjectsNamespace() + '\0' + key); +} + // This function is used to get the file path inside the directory which corresponds to Iceberg table from the full blob path which is written in manifest and metadata files. // For example, if the full blob path is s3://bucket/table_name/data/00000-1-1234567890.avro, the function will return table_name/data/00000-1-1234567890.avro // Common path should end with "" or "/". diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergPath.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergPath.h index e3e970ce12bf..26cb3a564d0f 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergPath.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergPath.h @@ -3,6 +3,8 @@ #include #include +#include + namespace DB { class FileNamesGenerator; @@ -33,6 +35,10 @@ class IcebergPathFromMetadata /// Also needed to get the file which corresponds to a line in the Chunk when used for position-delete algorithms. static IcebergPathFromMetadata deserialize(String path_) { return IcebergPathFromMetadata(std::move(path_)); } + /// Identity of the physical object a path resolves to, as the triple (storage description, namespace, key). + /// Lets paths spelled differently (s3:// vs s3a:// vs https) but pointing at the same object compare equal. + static IcebergPathFromMetadata makeStorageIdentity(const ObjectStoragePtr & storage, const String & key); + /// Extract the raw path string for writing into Iceberg metadata files, /// serialization, cache keys, virtual column values, etc. const String & serialize() const { return raw_path; } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp index 9529764f1fdd..6e121eb27fa1 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp index d21cf0de238c..70bfefe03f4d 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -394,9 +395,13 @@ ProcessedManifestFileEntryPtr ManifestFileIterator::processRow(size_t row_index) return nullptr; } +<<<<<<< HEAD /// Compute inherited/resolved fields Int64 resolved_snapshot_id = 0; +======= + Int64 resolved_snapshot_id; +>>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) if (parsed_entry->parsed_snapshot_id.has_value()) { resolved_snapshot_id = *parsed_entry->parsed_snapshot_id; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp index 8a04b5c83b3e..8a256f10985e 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp @@ -832,9 +832,642 @@ void alter( auto hint_path = filename_generator.generateVersionHint(); +<<<<<<< HEAD const bool catalog_writes_metadata_file = catalog && catalog->isTransactional(); if (!catalog_writes_metadata_file && !writeMetadataFileAndVersionHint( +======= + if (i == MAX_TRANSACTION_RETRIES) + throw Exception(ErrorCodes::LIMIT_EXCEEDED, "Too many unsuccessed retries to alter iceberg table"); +} + +/// Table-level snapshot retention policy read from Iceberg table properties. +struct RetentionPolicy +{ + Int32 min_snapshots_to_keep = Iceberg::default_min_snapshots_to_keep; + Int64 max_snapshot_age_ms = Iceberg::default_max_snapshot_age_ms; + Int64 max_ref_age_ms = Iceberg::default_max_ref_age_ms; +}; + +static RetentionPolicy readRetentionPolicy(const Poco::JSON::Object::Ptr & metadata, ContextPtr context, const ExpireSnapshotsOptions & options) +{ + RetentionPolicy policy; + const auto & settings = context->getSettingsRef(); + Int64 min_keep_from_settings = settings[Setting::iceberg_expire_default_min_snapshots_to_keep].value; + Int64 max_snapshot_age_from_settings = settings[Setting::iceberg_expire_default_max_snapshot_age_ms].value; + Int64 max_ref_age_from_settings = settings[Setting::iceberg_expire_default_max_ref_age_ms].value; + + if (min_keep_from_settings <= 0 || min_keep_from_settings > std::numeric_limits::max()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "iceberg_expire_default_min_snapshots_to_keep must be in range [1, {}], got {}", + std::numeric_limits::max(), + min_keep_from_settings); + if (max_snapshot_age_from_settings < 0) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "iceberg_expire_default_max_snapshot_age_ms must be non-negative, got {}", + max_snapshot_age_from_settings); + if (max_ref_age_from_settings < 0) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "iceberg_expire_default_max_ref_age_ms must be non-negative, got {}", + max_ref_age_from_settings); + + policy.min_snapshots_to_keep = static_cast(min_keep_from_settings); + policy.max_snapshot_age_ms = max_snapshot_age_from_settings; + policy.max_ref_age_ms = max_ref_age_from_settings; + + if (metadata->has(Iceberg::f_properties)) + { + auto props = metadata->getObject(Iceberg::f_properties); + if (props->has(Iceberg::f_min_snapshots_to_keep)) + policy.min_snapshots_to_keep = std::stoi(props->getValue(Iceberg::f_min_snapshots_to_keep)); + if (props->has(Iceberg::f_max_snapshot_age_ms)) + policy.max_snapshot_age_ms = std::stoll(props->getValue(Iceberg::f_max_snapshot_age_ms)); + if (props->has(Iceberg::f_max_ref_age_ms)) + policy.max_ref_age_ms = std::stoll(props->getValue(Iceberg::f_max_ref_age_ms)); + } + + /// Per-invocation overrides (only affect table-level defaults, not per-ref overrides). + if (options.retain_last.has_value()) + policy.min_snapshots_to_keep = *options.retain_last; + if (options.retention_period_ms.has_value()) + policy.max_snapshot_age_ms = *options.retention_period_ms; + + return policy; +} + +/// Snapshot parent graph built from metadata, used for branch ancestor traversal. +class SnapshotGraph +{ +public: + explicit SnapshotGraph(const Poco::JSON::Array::Ptr & snapshots) + { + for (UInt32 i = 0; i < snapshots->size(); ++i) + { + auto snapshot = snapshots->getObject(i); + Int64 snap_id = snapshot->getValue(Iceberg::f_metadata_snapshot_id); + timestamps[snap_id] = snapshot->getValue(Iceberg::f_timestamp_ms); + if (snapshot->has(Iceberg::f_parent_snapshot_id) && !snapshot->isNull(Iceberg::f_parent_snapshot_id)) + parent_chain[snap_id] = snapshot->getValue(Iceberg::f_parent_snapshot_id); + } + } + + bool hasSnapshot(Int64 snap_id) const { return timestamps.contains(snap_id); } + + Int64 getTimestamp(Int64 snap_id) const { return timestamps.at(snap_id); } + + std::optional getParent(Int64 snap_id) const + { + auto it = parent_chain.find(snap_id); + return it != parent_chain.end() ? std::optional(it->second) : std::nullopt; + } + + /// Retain ancestors from head_id while min-keep or max-age is satisfied. + void walkBranchAncestors(Int64 now_ms, Int64 head_id, Int32 min_keep, Int64 max_age_ms, std::set & retained) const + { + Int64 walk_id = head_id; + Int32 count = 0; + while (hasSnapshot(walk_id)) + { + bool within_min_keep = (count < min_keep); + bool within_max_age = (now_ms - getTimestamp(walk_id) <= max_age_ms); + if (!within_min_keep && !within_max_age) + break; + retained.insert(walk_id); + ++count; + auto parent = getParent(walk_id); + if (!parent) + break; + walk_id = *parent; + } + } + +private: + std::unordered_map parent_chain; + std::unordered_map timestamps; +}; + +/// Apply Iceberg Snapshot Retention Policy. Returns (retained IDs, expired ref names). +static std::pair, Strings> applyRetentionPolicy( + const Poco::JSON::Object::Ptr & metadata, + Int64 current_snapshot_id, + const SnapshotGraph & graph, + const RetentionPolicy & policy, + Int64 now_ms) +{ + std::set retained; + Strings expired_ref_names; + bool main_branch_walked = false; + if (metadata->has(Iceberg::f_refs)) + { + auto refs = metadata->getObject(Iceberg::f_refs); + for (const auto & ref_name : refs->getNames()) + { + auto ref_obj = refs->getObject(ref_name); + Int64 ref_snap_id = ref_obj->getValue(Iceberg::f_metadata_snapshot_id); + String ref_type = ref_obj->getValue(Iceberg::f_type); + + Int64 ref_max_ref_age = ref_obj->has(Iceberg::f_ref_max_ref_age_ms) + ? ref_obj->getValue(Iceberg::f_ref_max_ref_age_ms) + : policy.max_ref_age_ms; + + bool is_main = (ref_name == Iceberg::f_main); + + if (!is_main && !graph.hasSnapshot(ref_snap_id)) + { + LOG_WARNING(getLogger("IcebergExpireSnapshots"), + "Removing invalid ref {}: snapshot {} does not exist", ref_name, ref_snap_id); + expired_ref_names.push_back(ref_name); + continue; + } + + bool ref_expired = !is_main && (now_ms - graph.getTimestamp(ref_snap_id)) > ref_max_ref_age; + + if (ref_expired) + { + expired_ref_names.push_back(ref_name); + continue; + } + + if (ref_type == Iceberg::f_branch) + { + Int32 min_keep = ref_obj->has(Iceberg::f_ref_min_snapshots_to_keep) + ? ref_obj->getValue(Iceberg::f_ref_min_snapshots_to_keep) + : policy.min_snapshots_to_keep; + Int64 max_age = ref_obj->has(Iceberg::f_ref_max_snapshot_age_ms) + ? ref_obj->getValue(Iceberg::f_ref_max_snapshot_age_ms) + : policy.max_snapshot_age_ms; + graph.walkBranchAncestors(now_ms, ref_snap_id, min_keep, max_age, retained); + if (is_main) + main_branch_walked = true; + } + else if (ref_type == Iceberg::f_tag) + { + retained.insert(ref_snap_id); + } + else + { + UNREACHABLE(); + } + } + } + + if (!main_branch_walked) + graph.walkBranchAncestors(now_ms, current_snapshot_id, policy.min_snapshots_to_keep, policy.max_snapshot_age_ms, retained); + + return {retained, expired_ref_names}; +} + +static void collectAllFilePaths( + const Iceberg::ManifestFileIterator::ManifestFileEntriesHandle & entries_handle, + std::set & out) +{ + for (const auto & entry : entries_handle.getFilesWithoutDeleted(FileContentType::DATA)) + out.insert(entry->parsed_entry->file_path_key); + for (const auto & entry : entries_handle.getFilesWithoutDeleted(FileContentType::POSITION_DELETE)) + out.insert(entry->parsed_entry->file_path_key); + for (const auto & entry : entries_handle.getFilesWithoutDeleted(FileContentType::EQUALITY_DELETE)) + out.insert(entry->parsed_entry->file_path_key); +} + +/// Collect all file paths (manifest lists, manifests, data/delete files) +/// referenced by retained snapshots. +/// +/// NOTE: We only collect files with status ADDED/EXISTING (via getFilesWithoutDeleted). +/// Files with status DELETED are being removed by that snapshot and don't need retention +/// from it. A DELETED entry's data file was ADDED in an earlier snapshot — if that snapshot +/// is retained, the file is in the retained set from there; if expired, it will be collected +/// for cleanup from that snapshot's ADDED/EXISTING entries. +/// +/// TODO: To handle partially-failed prior expire_snapshots (where the ADDED snapshot +/// was removed but its data files were not cleaned up), we could also traverse DELETED +/// entries in expired manifests. This requires extending ManifestFileIterator to expose +/// DELETED entries. +static void collectRetainedFiles( + const Poco::JSON::Array::Ptr & retained_snapshots, + ObjectStoragePtr object_storage, + PersistentTableComponents & persistent_table_components, + ContextPtr context, + LoggerPtr log, + Int32 current_schema_id, + std::set & retained_manifest_paths, + std::set & retained_data_file_paths, + std::set & retained_manifest_list_paths) +{ + SecondaryStorages secondary_storages; + for (UInt32 i = 0; i < retained_snapshots->size(); ++i) + { + auto snapshot = retained_snapshots->getObject(i); + if (!snapshot->has(Iceberg::f_manifest_list)) + continue; + + auto manifest_list_path = IcebergPathFromMetadata::deserialize(snapshot->getValue(Iceberg::f_manifest_list)); + retained_manifest_list_paths.insert(manifest_list_path); + + auto manifest_keys = getManifestList(object_storage, persistent_table_components, context, manifest_list_path, log, secondary_storages); + + for (const auto & mf_key : manifest_keys) + { + retained_manifest_paths.insert(mf_key.manifest_file_path); + auto entries_handle = getManifestFileEntriesHandle( + object_storage, persistent_table_components, context, log, + mf_key, current_schema_id, secondary_storages); + collectAllFilePaths(entries_handle, retained_data_file_paths); + } + } +} + +struct ExpiredFiles +{ + std::vector all_paths; + Int64 data_files = 0; + Int64 position_delete_files = 0; + Int64 equality_delete_files = 0; + Int64 manifest_files = 0; + Int64 manifest_lists = 0; +}; + +/// Collect files from expired snapshots that are not referenced by any retained snapshot. +static ExpiredFiles collectExpiredFiles( + const std::vector & expired_manifest_list_paths, + const std::set & retained_manifest_list_paths, + const std::set & retained_manifest_paths, + const std::set & retained_data_file_paths, + ObjectStoragePtr object_storage, + PersistentTableComponents & persistent_table_components, + ContextPtr context, + LoggerPtr log, + Int32 current_schema_id) +{ + ExpiredFiles result; + SecondaryStorages secondary_storages; + std::set seen_expired_manifest_list_paths; + std::set seen_expired_manifest_paths; + for (const auto & manifest_list_path : expired_manifest_list_paths) + { + if (retained_manifest_list_paths.contains(manifest_list_path)) + continue; + + if (seen_expired_manifest_list_paths.contains(manifest_list_path)) + continue; + + ManifestFileCacheKeys manifest_keys; + try + { + manifest_keys = getManifestList(object_storage, persistent_table_components, context, manifest_list_path, log, secondary_storages); + } + catch (...) + { + LOG_WARNING(log, "Failed to read manifest list {}, skipping", manifest_list_path); + continue; + } + + for (const auto & mf_key : manifest_keys) + { + if (retained_manifest_paths.contains(mf_key.manifest_file_path)) + continue; + + if (seen_expired_manifest_paths.contains(mf_key.manifest_file_path)) + continue; + + try + { + auto entries_handle = getManifestFileEntriesHandle( + object_storage, persistent_table_components, context, log, + mf_key, current_schema_id, secondary_storages); + + for (const auto & entry : entries_handle.getFilesWithoutDeleted(FileContentType::DATA)) + if (!retained_data_file_paths.contains(entry->parsed_entry->file_path_key)) + { + result.all_paths.push_back(entry->parsed_entry->file_path_key); + ++result.data_files; + } + for (const auto & entry : entries_handle.getFilesWithoutDeleted(FileContentType::POSITION_DELETE)) + if (!retained_data_file_paths.contains(entry->parsed_entry->file_path_key)) + { + result.all_paths.push_back(entry->parsed_entry->file_path_key); + ++result.position_delete_files; + } + for (const auto & entry : entries_handle.getFilesWithoutDeleted(FileContentType::EQUALITY_DELETE)) + if (!retained_data_file_paths.contains(entry->parsed_entry->file_path_key)) + { + result.all_paths.push_back(entry->parsed_entry->file_path_key); + ++result.equality_delete_files; + } + } + catch (...) + { + LOG_WARNING(log, "Failed to read manifest file {}, skipping", mf_key.manifest_file_path); + continue; + } + + seen_expired_manifest_paths.insert(mf_key.manifest_file_path); + result.all_paths.push_back(mf_key.manifest_file_path); + ++result.manifest_files; + } + + seen_expired_manifest_list_paths.insert(manifest_list_path); + result.all_paths.push_back(manifest_list_path); + ++result.manifest_lists; + } + return result; +} + +/// Trim snapshot-log to the suffix of entries referencing only retained snapshots. +static void trimSnapshotLog( + Poco::JSON::Object::Ptr metadata, + const std::set & expired_snapshot_ids) +{ + if (!metadata->has(Iceberg::f_snapshot_log)) + return; + + auto snapshot_log = metadata->get(Iceberg::f_snapshot_log).extract(); + Int32 suffix_start = static_cast(snapshot_log->size()); + for (Int32 j = static_cast(snapshot_log->size()) - 1; j >= 0; --j) + { + auto entry = snapshot_log->getObject(static_cast(j)); + Int64 snap_id = entry->getValue(Iceberg::f_metadata_snapshot_id); + if (expired_snapshot_ids.contains(snap_id)) + break; + suffix_start = j; + } + Poco::JSON::Array::Ptr retained_log = new Poco::JSON::Array; + for (UInt32 j = static_cast(suffix_start); j < snapshot_log->size(); ++j) + retained_log->add(snapshot_log->getObject(j)); + metadata->set(Iceberg::f_snapshot_log, retained_log); +} + +struct SnapshotPartition +{ + Poco::JSON::Array::Ptr retained_snapshots = new Poco::JSON::Array; + std::set expired_snapshot_ids; + std::vector expired_manifest_list_paths; +}; + +/// Split snapshots into retained and expired. +/// A snapshot is retained if the retention policy selected it, or if the +/// user-provided fuse timestamp protects it (snapshot newer than fuse). +static SnapshotPartition partitionSnapshots( + const Poco::JSON::Array::Ptr & snapshots, + const std::set & retention_retained_ids, + std::optional expire_before_ms) +{ + SnapshotPartition result; + for (UInt32 i = 0; i < snapshots->size(); ++i) + { + auto snapshot = snapshots->getObject(i); + Int64 snap_id = snapshot->getValue(Iceberg::f_metadata_snapshot_id); + Int64 snap_ts = snapshot->getValue(Iceberg::f_timestamp_ms); + + bool is_retained_by_policy = retention_retained_ids.contains(snap_id); + bool is_protected_by_fuse = expire_before_ms.has_value() && (snap_ts >= *expire_before_ms); + + if (is_retained_by_policy || is_protected_by_fuse) + { + result.retained_snapshots->add(snapshot); + } + else + { + result.expired_snapshot_ids.insert(snap_id); + if (snapshot->has(Iceberg::f_manifest_list)) + result.expired_manifest_list_paths.push_back( + Iceberg::IcebergPathFromMetadata::deserialize(snapshot->getValue(Iceberg::f_manifest_list))); + } + } + return result; +} + +static SnapshotPartition partitionSnapshotsByIds( + const Poco::JSON::Object::Ptr & metadata, + const Poco::JSON::Array::Ptr & snapshots, + const std::vector & snapshot_ids, + Int64 current_snapshot_id, + std::optional expire_before_ms) +{ + std::unordered_set requested_ids(snapshot_ids.begin(), snapshot_ids.end()); + std::unordered_set existing_ids; + std::unordered_set ref_protected_ids; + SnapshotPartition result; + + if (metadata->has(Iceberg::f_refs)) + { + auto refs = metadata->getObject(Iceberg::f_refs); + for (const auto & ref_name : refs->getNames()) + { + auto ref = refs->getObject(ref_name); + if (ref->has(Iceberg::f_metadata_snapshot_id)) + ref_protected_ids.insert(ref->getValue(Iceberg::f_metadata_snapshot_id)); + } + } + + ref_protected_ids.insert(current_snapshot_id); + + for (UInt32 i = 0; i < snapshots->size(); ++i) + { + auto snapshot = snapshots->getObject(i); + Int64 snap_id = snapshot->getValue(Iceberg::f_metadata_snapshot_id); + Int64 snap_ts = snapshot->getValue(Iceberg::f_timestamp_ms); + + existing_ids.insert(snap_id); + bool requested = requested_ids.contains(snap_id); + bool is_protected_by_fuse = expire_before_ms.has_value() && (snap_ts >= *expire_before_ms); + + if (requested && ref_protected_ids.contains(snap_id)) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "expire_snapshots cannot expire snapshot {} because it is referenced by current snapshot, branch, or tag", + snap_id); + + if (requested && !is_protected_by_fuse) + { + result.expired_snapshot_ids.insert(snap_id); + if (snapshot->has(Iceberg::f_manifest_list)) + result.expired_manifest_list_paths.push_back(Iceberg::IcebergPathFromMetadata::deserialize(snapshot->getValue(Iceberg::f_manifest_list))); + } + else + { + result.retained_snapshots->add(snapshot); + } + } + + for (Int64 requested_id : requested_ids) + { + if (!existing_ids.contains(requested_id)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "expire_snapshots snapshot id {} does not exist", requested_id); + } + + return result; +} + +/// Mutate metadata: remove expired refs, update snapshots, trim log, bump timestamp. +static void updateMetadataForExpiration( + Poco::JSON::Object::Ptr metadata, + const Strings & expired_ref_names, + const Poco::JSON::Array::Ptr & retained_snapshots, + const std::set & expired_snapshot_ids) +{ + for (const auto & ref_name : expired_ref_names) + metadata->getObject(Iceberg::f_refs)->remove(ref_name); + + metadata->set(Iceberg::f_snapshots, retained_snapshots); + trimSnapshotLog(metadata, expired_snapshot_ids); + + auto now = std::chrono::system_clock::now(); + auto ms = duration_cast(now.time_since_epoch()); + metadata->set(Iceberg::f_last_updated_ms, ms.count()); +} + +static void deleteExpiredFiles( + const std::vector & files_to_delete, + const Iceberg::IcebergPathResolver & path_resolver, + ObjectStoragePtr object_storage, + LoggerPtr log) +{ + for (const auto & file_path : files_to_delete) + { + try + { + object_storage->removeObjectIfExists(StoredObject(path_resolver.resolve(file_path))); + LOG_DEBUG(log, "Deleted expired file {}", file_path); + } + catch (...) + { + LOG_WARNING(log, "Failed to delete file {}: {}", file_path, getCurrentExceptionMessage(false)); + } + } +} + +/// Expire old Iceberg snapshots following the spec's Snapshot Retention Policy. +/// +/// The process: +/// 1. Read retention policy from table properties (with spec defaults). +/// 2. Build the snapshot parent graph and determine which snapshots to retain +/// based on branch/tag refs and their min-snapshots-to-keep / max-snapshot-age-ms. +/// 3. If the caller provided expire_before_ms, it acts as an additional safety +/// fuse — snapshots newer than this timestamp are never expired regardless +/// of retention policy. +/// 4. Collect files exclusively owned by expired snapshots and delete them. +/// 5. Write updated metadata with optimistic concurrency (retry on conflict). +ExpireSnapshotsResult expireSnapshots( + const ExpireSnapshotsOptions & options, + ContextPtr context, + ObjectStoragePtr object_storage, + const DataLakeStorageSettings & data_lake_settings, + PersistentTableComponents & persistent_table_components, + const String & write_format, + std::shared_ptr catalog, + const String & table_name) +{ + auto common_path = persistent_table_components.table_path; + if (!common_path.starts_with('/')) + common_path = "/" + common_path; + + int max_retries = MAX_TRANSACTION_RETRIES; + while (--max_retries > 0) + { + FileNamesGenerator filename_generator(persistent_table_components.path_resolver.getTableLocation(), false, CompressionMethod::None, write_format); + auto log = getLogger("IcebergExpireSnapshots"); + auto [last_version, metadata_path, compression_method] = getLatestOrExplicitMetadataFileAndVersion( + object_storage, + persistent_table_components.table_path, + data_lake_settings, + persistent_table_components.metadata_cache, + context, + log.get(), + persistent_table_components.table_uuid, + persistent_table_components.metadata_compression_method, + /* force_fetch_latest_metadata */ true, + /* ignore_explicit_metadata_file_path */ true); + + filename_generator.setVersion(last_version + 1); + filename_generator.setCompressionMethod(compression_method); + + auto metadata = getMetadataJSONObject( + metadata_path, + object_storage, + persistent_table_components.metadata_cache, + context, + log, + compression_method, + persistent_table_components.table_uuid); + + if (metadata->getValue(f_format_version) < 2) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "expire_snapshots is supported only for the second version of iceberg format"); + + if (!metadata->has(Iceberg::f_current_snapshot_id)) + { + LOG_INFO(log, "No snapshots to expire (table has no current snapshot)"); + return {.dry_run = options.dry_run}; + } + + Int64 current_snapshot_id = metadata->getValue(Iceberg::f_current_snapshot_id); + if (current_snapshot_id < 0) + { + LOG_INFO(log, "No snapshots to expire (table has no current snapshot)"); + return {.dry_run = options.dry_run}; + } + + auto snapshots = metadata->get(Iceberg::f_snapshots).extract(); + auto now_ms = duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); + + Strings expired_ref_names; + SnapshotPartition partition; + if (options.snapshot_ids.has_value()) + { + partition = partitionSnapshotsByIds(metadata, snapshots, *options.snapshot_ids, current_snapshot_id, options.expire_before_ms); + } + else + { + auto policy = readRetentionPolicy(metadata, context, options); + SnapshotGraph graph(snapshots); + auto [retention_retained_ids, retention_expired_ref_names] = applyRetentionPolicy(metadata, current_snapshot_id, graph, policy, now_ms); + expired_ref_names = std::move(retention_expired_ref_names); + partition = partitionSnapshots(snapshots, retention_retained_ids, options.expire_before_ms); + } + + if (partition.expired_snapshot_ids.empty()) + { + LOG_INFO(log, "No snapshots to expire"); + return {.dry_run = options.dry_run}; + } + LOG_INFO(log, "Expiring {} snapshots", partition.expired_snapshot_ids.size()); + + Int32 current_schema_id = metadata->getValue(Iceberg::f_current_schema_id); + + std::set retained_manifest_paths; + std::set retained_data_file_paths; + std::set retained_manifest_list_paths; + collectRetainedFiles( + partition.retained_snapshots, object_storage, persistent_table_components, context, log, + current_schema_id, retained_manifest_paths, retained_data_file_paths, retained_manifest_list_paths); + auto expired_files = collectExpiredFiles( + partition.expired_manifest_list_paths, retained_manifest_list_paths, retained_manifest_paths, retained_data_file_paths, + object_storage, persistent_table_components, context, log, current_schema_id); + + if (options.dry_run) + { + LOG_INFO(log, "Dry-run mode: skip metadata commit and file deletion"); + return ExpireSnapshotsResult{ + .deleted_data_files_count = expired_files.data_files, + .deleted_position_delete_files_count = expired_files.position_delete_files, + .deleted_equality_delete_files_count = expired_files.equality_delete_files, + .deleted_manifest_files_count = expired_files.manifest_files, + .deleted_manifest_lists_count = expired_files.manifest_lists, + .dry_run = true, + }; + } + + updateMetadataForExpiration(metadata, expired_ref_names, partition.retained_snapshots, partition.expired_snapshot_ids); + + std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + Poco::JSON::Stringifier::stringify(metadata, oss, 4); + std::string json_representation = removeEscapedSlashes(oss.str()); + auto metadata_info = filename_generator.generateMetadataPathWithInfo(); + auto hint_path = filename_generator.generateVersionHint(); + if (!writeMetadataFileAndVersionHint( +>>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) persistent_table_components.path_resolver, metadata_info, json_representation, diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.h index 1dbe55e0fb11..be97315ee9c3 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.h @@ -16,6 +16,7 @@ #include #include #include +#include namespace DB::Iceberg { diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/PositionDeleteTransform.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/PositionDeleteTransform.cpp index ddb913f9464b..4c044e1bcf9c 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/PositionDeleteTransform.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/PositionDeleteTransform.cpp @@ -74,12 +74,17 @@ void IcebergPositionDeleteTransform::initializeDeleteSources() { continue; } +<<<<<<< HEAD +======= +>>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) - auto object_path = position_deletes_object.file_path; - auto object_metadata = object_storage->getObjectMetadata(object_path, /*with_tags=*/ false); - auto object_info = RelativePathWithMetadata{object_path, object_metadata}; + auto [delete_storage_to_use, resolved_key] = resolveObjectStorageForPath( + path_resolver.getTableLocation(), position_deletes_object.file_path, object_storage, *secondary_storages, context, + path_resolver); + auto object_metadata = delete_storage_to_use->getObjectMetadata(resolved_key, /*with_tags=*/ false); + RelativePathWithMetadata object_info(resolved_key, object_metadata); String format = position_deletes_object.file_format; if (boost::to_lower_copy(format) != "parquet") @@ -87,7 +92,7 @@ void IcebergPositionDeleteTransform::initializeDeleteSources() Block initial_header; { - std::unique_ptr read_buf_schema = createReadBuffer(object_info, object_storage, context, log); + std::unique_ptr read_buf_schema = createReadBuffer(object_info, delete_storage_to_use, context, log); auto schema_reader = FormatFactory::instance().getSchemaReader(format, *read_buf_schema, context); auto columns_with_names = schema_reader->readSchema(); ColumnsWithTypeAndName initial_header_data; @@ -98,9 +103,9 @@ void IcebergPositionDeleteTransform::initializeDeleteSources() initial_header = Block(initial_header_data); } - CompressionMethod compression_method = chooseCompressionMethod(object_path, "auto"); + CompressionMethod compression_method = chooseCompressionMethod(resolved_key, "auto"); - delete_read_buffers.push_back(createReadBuffer(object_info, object_storage, context, log)); + delete_read_buffers.push_back(createReadBuffer(object_info, delete_storage_to_use, context, log)); auto syntax_result = TreeRewriter(context).analyze(where_ast, initial_header.getNamesAndTypesList()); ExpressionAnalyzer analyzer(where_ast, syntax_result, context); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/PositionDeleteTransform.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/PositionDeleteTransform.h index f1738b3828b5..dcc0a5461b1f 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/PositionDeleteTransform.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/PositionDeleteTransform.h @@ -8,6 +8,7 @@ #include #include #include +#include namespace DB::Iceberg { @@ -28,7 +29,9 @@ class IcebergPositionDeleteTransform : public ISimpleTransform ObjectStoragePtr object_storage_, const std::optional & format_settings_, FormatParserSharedResourcesPtr parser_shared_resources_, - ContextPtr context_) + ContextPtr context_, + const IcebergPathResolver & path_resolver_, + std::shared_ptr secondary_storages_) : ISimpleTransform(header_, header_, false) , header(header_) , iceberg_object_info(iceberg_object_info_) @@ -36,6 +39,8 @@ class IcebergPositionDeleteTransform : public ISimpleTransform , format_settings(format_settings_) , context(context_) , parser_shared_resources(parser_shared_resources_) + , path_resolver(path_resolver_) + , secondary_storages(std::move(secondary_storages_)) { initializeDeleteSources(); } @@ -62,6 +67,9 @@ class IcebergPositionDeleteTransform : public ISimpleTransform ContextPtr context; FormatParserSharedResourcesPtr parser_shared_resources; + const IcebergPathResolver path_resolver; + std::shared_ptr secondary_storages; + /// We need to keep the read buffers alive since the delete_sources depends on them. std::vector> delete_read_buffers; std::vector> delete_sources; @@ -78,8 +86,10 @@ class IcebergBitmapPositionDeleteTransform final : public IcebergPositionDeleteT ObjectStoragePtr object_storage_, const std::optional & format_settings_, FormatParserSharedResourcesPtr parser_shared_resources_, - ContextPtr context_) - : IcebergPositionDeleteTransform(header_, iceberg_object_info_, object_storage_, format_settings_, parser_shared_resources_, context_) + ContextPtr context_, + const IcebergPathResolver & path_resolver_, + std::shared_ptr secondary_storages_) + : IcebergPositionDeleteTransform(header_, iceberg_object_info_, object_storage_, format_settings_, parser_shared_resources_, context_, path_resolver_, std::move(secondary_storages_)) { initialize(); } @@ -104,8 +114,10 @@ class IcebergStreamingPositionDeleteTransform final : public IcebergPositionDele ObjectStoragePtr object_storage_, const std::optional & format_settings_, FormatParserSharedResourcesPtr parser_shared_resources_, - ContextPtr context_) - : IcebergPositionDeleteTransform(header_, iceberg_object_info_, object_storage_, format_settings_, parser_shared_resources_, context_) + ContextPtr context_, + const IcebergPathResolver & path_resolver_, + std::shared_ptr secondary_storages_) + : IcebergPositionDeleteTransform(header_, iceberg_object_info_, object_storage_, format_settings_, parser_shared_resources_, context_, path_resolver_, std::move(secondary_storages_)) { initialize(); } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/StatelessMetadataFileGetter.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/StatelessMetadataFileGetter.cpp index 045470229dc9..d6ed24a6d0f8 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/StatelessMetadataFileGetter.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/StatelessMetadataFileGetter.cpp @@ -70,7 +70,8 @@ Iceberg::ManifestFileCacheableInfo getManifestFile( ContextPtr local_context, LoggerPtr log, const IcebergPathFromMetadata & filename, - size_t bytes_size) + size_t bytes_size, + SecondaryStorages & secondary_storages) { auto log_level = local_context->getSettingsRef()[Setting::iceberg_metadata_log_level].value; @@ -79,15 +80,19 @@ Iceberg::ManifestFileCacheableInfo getManifestFile( auto create_fn = [&, use_iceberg_metadata_cache]() { - RelativePathWithMetadata manifest_object_info(persistent_table_components.path_resolver.resolve(filename)); + auto [storage_to_use, resolved_key_in_storage] = resolveObjectStorageForPath( + persistent_table_components.table_location, filename.serialize(), object_storage, secondary_storages, local_context, + persistent_table_components.path_resolver); + + RelativePathWithMetadata manifest_object_info(resolved_key_in_storage); auto read_settings = local_context->getReadSettings(); /// Do not utilize filesystem cache if more precise cache enabled if (use_iceberg_metadata_cache) read_settings.enable_filesystem_cache = false; - auto buffer = createReadBuffer(manifest_object_info, object_storage, local_context, log, read_settings); - auto manifest_file_deserializer = std::make_unique( + auto buffer = createReadBuffer(manifest_object_info, storage_to_use, local_context, log, read_settings); + auto manifest_file_deserializer = std::make_shared( std::move(buffer), filename, getFormatSettings(local_context)); return Iceberg::ManifestFileCacheableInfo{std::move(manifest_file_deserializer), bytes_size}; @@ -108,7 +113,8 @@ Iceberg::ManifestFileIterator::ManifestFileEntriesHandle getManifestFileEntriesH ContextPtr local_context, LoggerPtr log, const ManifestFileCacheKey & cache_key, - Int32 table_snapshot_schema_id) + Int32 table_snapshot_schema_id, + SecondaryStorages & secondary_storages) { auto cacheable_info = getManifestFile( object_storage, @@ -116,7 +122,8 @@ Iceberg::ManifestFileIterator::ManifestFileEntriesHandle getManifestFileEntriesH local_context, log, cache_key.manifest_file_path, - static_cast(cache_key.manifest_file_byte_size)); + cache_key.manifest_file_byte_size, + secondary_storages); auto iterator = Iceberg::ManifestFileIterator::create( cacheable_info.deserializer, @@ -141,7 +148,8 @@ ManifestFileCacheKeys getManifestList( const PersistentTableComponents & persistent_table_components, ContextPtr local_context, const IcebergPathFromMetadata & filename, - LoggerPtr log) + LoggerPtr log, + SecondaryStorages & secondary_storages) { IcebergMetadataLogLevel log_level = local_context->getSettingsRef()[Setting::iceberg_metadata_log_level].value; @@ -150,14 +158,18 @@ ManifestFileCacheKeys getManifestList( auto create_fn = [&, use_iceberg_metadata_cache]() { - RelativePathWithMetadata object_info(persistent_table_components.path_resolver.resolve(filename)); + auto [storage_to_use, key_in_storage] = resolveObjectStorageForPath( + persistent_table_components.table_location, filename.serialize(), object_storage, secondary_storages, local_context, + persistent_table_components.path_resolver); + + RelativePathWithMetadata object_info(key_in_storage); auto read_settings = local_context->getReadSettings(); /// Do not utilize filesystem cache if more precise cache enabled if (use_iceberg_metadata_cache) read_settings.enable_filesystem_cache = false; - auto manifest_list_buf = createReadBuffer(object_info, object_storage, local_context, log, read_settings); + auto manifest_list_buf = createReadBuffer(object_info, storage_to_use, local_context, log, read_settings); AvroForIcebergDeserializer manifest_list_deserializer(std::move(manifest_list_buf), filename, getFormatSettings(local_context)); /// The manifest list's own Avro metadata governs how it is parsed. A table whose @@ -172,7 +184,7 @@ ManifestFileCacheKeys getManifestList( local_context, manifest_list_deserializer.getMetadataContent(), DB::IcebergMetadataLogLevel::ManifestListMetadata, - persistent_table_components.path_resolver.getTableRoot(), + persistent_table_components.table_path, filename, std::nullopt, std::nullopt); @@ -209,13 +221,13 @@ ManifestFileCacheKeys getManifestList( manifest_list_deserializer.getValueFromRowByName(i, f_content, TypeIndex::Int32).safeGet()); } manifest_file_cache_keys.emplace_back( - manifest_file_name, manifest_length, added_sequence_number, added_snapshot_id.safeGet(), content_type); + manifest_file_name, static_cast(manifest_length), added_sequence_number, added_snapshot_id.safeGet(), content_type); insertRowToLogTable( local_context, manifest_list_deserializer.getContent(i), DB::IcebergMetadataLogLevel::ManifestListEntry, - persistent_table_components.path_resolver.getTableRoot(), + persistent_table_components.table_path, filename, i, std::nullopt); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/StatelessMetadataFileGetter.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/StatelessMetadataFileGetter.h index 2bb2edfdff13..a3e57a7a302b 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/StatelessMetadataFileGetter.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/StatelessMetadataFileGetter.h @@ -16,6 +16,7 @@ #include #include +#include namespace DB::Iceberg { @@ -26,7 +27,8 @@ Iceberg::ManifestFileCacheableInfo getManifestFile( ContextPtr local_context, LoggerPtr log, const IcebergPathFromMetadata & filename, - size_t bytes_size); + size_t bytes_size, + SecondaryStorages & secondary_storages); /// Creates a fully initialized ManifestFileIterator from a cache key. /// All entries are drained so that aggregate methods (e.g. getRowsCountInAllFilesExcludingDeleted) @@ -37,7 +39,8 @@ Iceberg::ManifestFileIterator::ManifestFileEntriesHandle getManifestFileEntriesH ContextPtr local_context, LoggerPtr log, const ManifestFileCacheKey & cache_key, - Int32 table_snapshot_schema_id); + Int32 table_snapshot_schema_id, + SecondaryStorages & secondary_storages); ManifestFileCacheKeys getManifestList( @@ -45,7 +48,8 @@ ManifestFileCacheKeys getManifestList( const PersistentTableComponents & persistent_table_components, ContextPtr local_context, const IcebergPathFromMetadata & filename, - LoggerPtr log); + LoggerPtr log, + SecondaryStorages & secondary_storages); } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp index f7a3f164ac1d..5bba27100f25 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp @@ -40,12 +40,13 @@ #include #include #include +#include #if USE_AVRO #include -#include #include +#include #include #include #include @@ -104,7 +105,6 @@ static constexpr size_t MAX_LIST_RETRIES = 5; namespace DB::Iceberg { - using namespace DB; /// Best-effort heuristic based on ClickHouse naming conventions. @@ -1454,3 +1454,29 @@ void forEachAvroEntry( } #endif + +namespace DB +{ + +ObjectStoragePtr getResolvedStorageFromObjectInfo([[maybe_unused]] const ObjectInfoPtr & object_info, const ObjectStoragePtr & default_storage) +{ +#if USE_AVRO + if (auto iceberg_info = std::dynamic_pointer_cast(object_info)) + { + if (auto resolved = iceberg_info->getResolvedStorage()) + return resolved; + } +#endif + return default_storage; +} + +std::optional getMetadataPathFromObjectInfo([[maybe_unused]] const ObjectInfoPtr & object_info) +{ +#if USE_AVRO + if (auto iceberg_info = std::dynamic_pointer_cast(object_info)) + return iceberg_info->getMetadataPath(); +#endif + return std::nullopt; +} + +} diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h index 43d2c040ad59..eac24bc2ee28 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h @@ -1,9 +1,15 @@ #pragma once +<<<<<<< HEAD #include "config.h" #if USE_AVRO +======= +#include +#include +#include "config.h" +>>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) #include #include #include @@ -16,6 +22,22 @@ #include #include +<<<<<<< HEAD +======= + +namespace DB +{ +struct ObjectInfo; +using ObjectInfoPtr = std::shared_ptr; + +/// These functions are always available; they return fallback values when USE_AVRO is not defined +ObjectStoragePtr getResolvedStorageFromObjectInfo([[maybe_unused]] const ObjectInfoPtr & object_info, const ObjectStoragePtr & default_storage); +std::optional getMetadataPathFromObjectInfo([[maybe_unused]] const ObjectInfoPtr & object_info); +} + +#if USE_AVRO + +>>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) #include #include #include diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index d004243b3131..0e69558dd65c 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -51,6 +51,11 @@ #include #include #include +<<<<<<< HEAD +======= +#include +#include +>>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) #if ENABLE_DISTRIBUTED_CACHE #include #include @@ -202,11 +207,17 @@ std::shared_ptr StorageObjectStorageSource::createFileIterator( { const bool expect_whole_archive = !local_context->getSettingsRef()[Setting::cluster_function_process_archive_on_multiple_nodes]; + /// Use the full table location URI (e.g. `s3a://bucket/prefix/table/`) when available + std::string table_location = configuration->getPathForRead().path; + if (auto * metadata = configuration->getExternalMetadata()) + table_location = metadata->getTableLocation(); + auto distributed_iterator = std::make_unique( local_context->getClusterFunctionReadTaskCallback(), local_context->getSettingsRef()[Setting::max_threads], /*is_archive_=*/is_archive && !expect_whole_archive, object_storage, + table_location, local_context); if (is_archive && expect_whole_archive) @@ -427,6 +438,8 @@ Chunk StorageObjectStorageSource::generate() read_context); } + std::string path_for_virtual_column = getMetadataPathFromObjectInfo(object_info).value_or(path); + const String * iceberg_metadata_file_path = nullptr; #if USE_AVRO if (const auto * iceberg_info = dynamic_cast(object_info.get())) @@ -437,8 +450,12 @@ Chunk StorageObjectStorageSource::generate() chunk, read_from_format_info.requested_virtual_columns, { +<<<<<<< HEAD .path = path, .storage_id = storage_snapshot->storage.getStorageID(), +======= + .path = path_for_virtual_column, +>>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) .size = object_info->isArchive() ? object_info->fileSizeInArchive() : object_metadata->size_bytes, .filename = &filename, .last_modified = object_metadata->last_modified, @@ -670,16 +687,18 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade bool with_tags = read_from_format_info.requested_virtual_columns.contains("_tags"); const auto & path = object_info->isArchive() ? object_info->getPathToArchive() : object_info->getPath(); + ObjectStoragePtr storage_to_use = getResolvedStorageFromObjectInfo(object_info, object_storage); + if (query_settings.ignore_non_existent_file) { - auto metadata = object_storage->tryGetObjectMetadata(path, with_tags); + auto metadata = storage_to_use->tryGetObjectMetadata(path, with_tags); if (!metadata) return {}; object_info->setObjectMetadata(metadata.value()); } else - object_info->setObjectMetadata(object_storage->getObjectMetadata(path, with_tags)); + object_info->setObjectMetadata(storage_to_use->getObjectMetadata(path, with_tags)); } if (query_settings.skip_empty_files && object_info->getObjectMetadata()->size_bytes == 0 @@ -786,9 +805,18 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade } else { +<<<<<<< HEAD ProfileEvents::increment(ProfileEvents::ObjectStorageReadObjects); compression_method = chooseCompressionMethod(object_info->getFileName(), configuration->compression_method); read_buf = createReadBuffer(object_info->relative_path_with_metadata, object_storage, context_, log); +======= + compression_method = chooseCompressionMethod(object_info->getFileName(), configuration->getCompressionMethod()); + read_buf = createReadBuffer( + object_info->relative_path_with_metadata, + getResolvedStorageFromObjectInfo(object_info, object_storage), + context_, + log); +>>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) } Block initial_header = read_from_format_info.format_header; @@ -1567,11 +1595,13 @@ StorageObjectStorageSource::ReadTaskIterator::ReadTaskIterator( size_t max_threads_count, bool is_archive_, ObjectStoragePtr object_storage_, + const std::string & table_location_, ContextPtr context_) : WithContext(context_) , callback(callback_) , is_archive(is_archive_) , object_storage(object_storage_) + , table_location(table_location_) { ThreadPool pool( CurrentMetrics::StorageObjectStorageThreads, @@ -1597,8 +1627,35 @@ StorageObjectStorageSource::ReadTaskIterator::ReadTaskIterator( { auto object = object_future.get(); if (object) + { + resolveIcebergObjectStorageIfNeeded(object); buffer.push_back(object); + } + } +} + +void StorageObjectStorageSource::ReadTaskIterator::resolveIcebergObjectStorageIfNeeded([[maybe_unused]] const ObjectInfoPtr & object) +{ +#if USE_AVRO + /// For Iceberg objects, resolve the storage from the raw metadata path + auto iceberg_info = std::dynamic_pointer_cast(object); + if (!iceberg_info || iceberg_info->getResolvedStorage()) + return; + + auto metadata_path = iceberg_info->getMetadataPath(); + if (!metadata_path) + return; + + /// Only secondary-storage files need resolving here (an ObjectStorage can't be shipped over the + /// wire); base-storage files keep the coordinator's key. + if (auto resolved = tryResolveObjectStorageForPath( + table_location, *metadata_path, object_storage, secondary_storages, getContext()); + resolved && resolved->first != object_storage) + { + iceberg_info->setResolvedStorage(resolved->first); + iceberg_info->relative_path_with_metadata.relative_path = resolved->second; } +#endif } ObjectInfoPtr StorageObjectStorageSource::ReadTaskIterator::next(size_t) @@ -1608,14 +1665,27 @@ ObjectInfoPtr StorageObjectStorageSource::ReadTaskIterator::next(size_t) ObjectInfoPtr object_info; if (current_index >= buffer.size()) { +<<<<<<< HEAD auto task = callback(); if (auto query_status = getContext()->getProcessListElement()) query_status->checkTimeLimit(); if (!task || task->isEmpty()) +======= + if (!getContext()->isSwarmModeEnabled()) + { + LOG_DEBUG(getLogger("StorageObjectStorageSource"), "STOP SWARM MODE called, stop getting new tasks"); + return nullptr; + } + + auto raw = callback(); + if (!raw || raw->isEmpty()) +>>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) return nullptr; - object_info = task->getObjectInfo(); + + object_info = raw->getObjectInfo(); + resolveIcebergObjectStorageIfNeeded(object_info); } else { @@ -1710,7 +1780,10 @@ StorageObjectStorageSource::ArchiveIterator::createArchiveReader(ObjectInfoPtr o /* path_to_archive */ object_info->getPath(), /* archive_read_function */ [=, this]() - { return createReadBuffer(object_info->relative_path_with_metadata, object_storage, getContext(), log); }, + { + auto storage = getResolvedStorageFromObjectInfo(object_info, object_storage); + return createReadBuffer(object_info->relative_path_with_metadata, storage, getContext(), log); + }, /* archive_size */ size); } @@ -1732,7 +1805,10 @@ ObjectInfoPtr StorageObjectStorageSource::ArchiveIterator::next(size_t processor } if (!archive_object->getObjectMetadata()) - archive_object->setObjectMetadata(object_storage->getObjectMetadata(archive_object->getPath(), /*with_tags=*/ false)); + { + ObjectStoragePtr storage_to_use = getResolvedStorageFromObjectInfo(archive_object, object_storage); + archive_object->setObjectMetadata(storage_to_use->getObjectMetadata(archive_object->getPath(), /*with_tags=*/ false)); + } archive_reader = createArchiveReader(archive_object); file_enumerator = archive_reader->firstFile(); @@ -1758,7 +1834,10 @@ ObjectInfoPtr StorageObjectStorageSource::ArchiveIterator::next(size_t processor return {}; if (!archive_object->getObjectMetadata()) - archive_object->setObjectMetadata(object_storage->getObjectMetadata(archive_object->getPath(), /*with_tags=*/ false)); + { + ObjectStoragePtr storage_to_use = getResolvedStorageFromObjectInfo(archive_object, object_storage); + archive_object->setObjectMetadata(storage_to_use->getObjectMetadata(archive_object->getPath(), /*with_tags=*/ false)); + } archive_reader = createArchiveReader(archive_object); if (!archive_reader->fileExists(path_in_archive)) diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.h b/src/Storages/ObjectStorage/StorageObjectStorageSource.h index 3e72bbd15b08..c76d6ff86699 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.h +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.h @@ -11,6 +11,11 @@ #include #include #include +<<<<<<< HEAD +======= +#include +#include +>>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) #include namespace DB @@ -160,6 +165,7 @@ class StorageObjectStorageSource::ReadTaskIterator : public IObjectIterator, pri size_t max_threads_count, bool is_archive_, ObjectStoragePtr object_storage_, + const std::string & table_location_, ContextPtr context_); ObjectInfoPtr next(size_t) override; @@ -169,11 +175,19 @@ class StorageObjectStorageSource::ReadTaskIterator : public IObjectIterator, pri private: ObjectInfoPtr createObjectInfoInArchive(const std::string & path_to_archive, const std::string & path_in_archive); + /// For Iceberg objects: resolve which storage the file lives in (possibly a secondary storage) + /// from the raw metadata path and record it on the object. No-op for non-Iceberg objects. + void resolveIcebergObjectStorageIfNeeded(const ObjectInfoPtr & object); + ClusterFunctionReadTaskCallback callback; ObjectInfos buffer; std::atomic_size_t index = 0; bool is_archive; ObjectStoragePtr object_storage; + std::string table_location; +#if USE_AVRO + SecondaryStorages secondary_storages; /// For Iceberg: cache of storages for external file locations +#endif /// path_to_archive -> archive reader. std::unordered_map> archive_readers; std::mutex archive_readers_mutex; diff --git a/src/Storages/ObjectStorage/StorageObjectStorageStableTaskDistributor.cpp b/src/Storages/ObjectStorage/StorageObjectStorageStableTaskDistributor.cpp index 5a54601b171c..1f5003be3baf 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageStableTaskDistributor.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageStableTaskDistributor.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include #include #include @@ -191,4 +193,62 @@ ObjectInfoPtr StorageObjectStorageStableTaskDistributor::getAnyUnprocessedFile(s return {}; } +<<<<<<< HEAD +======= +void StorageObjectStorageStableTaskDistributor::saveLastNodeActivity(size_t number_of_current_replica) +{ + Poco::Timestamp now; + std::lock_guard lock(mutex); + last_node_activity[number_of_current_replica] = now; +} + +void StorageObjectStorageStableTaskDistributor::rescheduleTasksFromReplica(size_t number_of_current_replica) +{ + LOG_INFO(log, "Replica {} is marked as lost, tasks are returned to queue", number_of_current_replica); + std::lock_guard lock(mutex); + + auto processed_file_list_ptr = replica_to_files_to_be_processed.find(number_of_current_replica); + if (processed_file_list_ptr == replica_to_files_to_be_processed.end()) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Replica number {} was marked as lost already", + number_of_current_replica + ); + + if (replica_to_files_to_be_processed.size() < 2) + throw Exception( + ErrorCodes::CANNOT_READ_ALL_DATA, + "All replicas were marked as lost" + ); + + auto files = std::move(processed_file_list_ptr->second); + replica_to_files_to_be_processed.erase(number_of_current_replica); + for (const auto & file : files) + { + auto file_identifier = getFileIdentifier(file); + auto file_replica_idx = getReplicaForFile(file_identifier); + unprocessed_files.emplace(file_identifier, std::make_pair(file, file_replica_idx)); + connection_to_files[file_replica_idx].push_back(file); + } +} + +String StorageObjectStorageStableTaskDistributor::getFileIdentifier(ObjectInfoPtr file_object, bool write_to_log) const +{ + if (send_over_whole_archive && file_object->isArchive()) + { + auto file_identifier = file_object->getPathOrPathToArchiveIfArchive(); + if (write_to_log) + { + LOG_TEST(log, "Will send over the whole archive {} to replicas. " + "This will be suboptimal, consider turning on " + "cluster_function_process_archive_on_multiple_nodes setting", file_identifier); + } + return file_identifier; + } + /// Prefer the original Iceberg metadata path (possibly absolute / on another storage) when available, + /// so the same file is identified consistently across replicas. + return getMetadataPathFromObjectInfo(file_object).value_or(file_object->getIdentifier()); +} + +>>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) } diff --git a/src/Storages/ObjectStorage/Utils.cpp b/src/Storages/ObjectStorage/Utils.cpp index 8fa02319c9ac..d1681a4e08fa 100644 --- a/src/Storages/ObjectStorage/Utils.cpp +++ b/src/Storages/ObjectStorage/Utils.cpp @@ -18,6 +18,26 @@ #include #include #include +#include +#include +#include +#include +#include +#include +#include +#if USE_AWS_S3 +#include +#endif +#if USE_AVRO +#include +#endif +#if USE_AZURE_BLOB_STORAGE +#include +#endif +#if USE_HDFS +#include +#endif + namespace DB { @@ -27,6 +47,138 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; extern const int LOGICAL_ERROR; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int PATH_ACCESS_DENIED; +} + +namespace +{ + +#if USE_AVRO +std::string normalizeScheme(const std::string & scheme) +{ + auto scheme_lowercase = Poco::toLower(scheme); + + if (scheme_lowercase == "s3a" || scheme_lowercase == "s3n" || scheme_lowercase == "gs" || scheme_lowercase == "gcs" || scheme_lowercase == "oss") + scheme_lowercase = "s3"; + else if (scheme_lowercase == "wasb" || scheme_lowercase == "wasbs" || scheme_lowercase == "abfss") + scheme_lowercase = "abfs"; + + return scheme_lowercase; +} + +std::string factoryTypeForScheme(const std::string & normalized_scheme) +{ + if (normalized_scheme == "s3") return "s3"; + if (normalized_scheme == "abfs") return "azure"; + if (normalized_scheme == "hdfs") return "hdfs"; + if (normalized_scheme == "file") return "local"; + return ""; +} + +#if USE_AWS_S3 +/// For s3:// URIs (generic), bucket needs to match. +/// For explicit http(s):// URIs, both bucket and endpoint must match. +bool s3URIMatches(const S3::URI & target_uri, const std::string & base_bucket, const std::string & base_endpoint, const std::string & target_scheme_normalized) +{ + bool bucket_matches = (target_uri.bucket == base_bucket); + bool endpoint_matches = (target_uri.endpoint == base_endpoint); + bool is_generic_s3_uri = (target_scheme_normalized == "s3"); + return bucket_matches && (endpoint_matches || is_generic_s3_uri); +} + +bool sameEndpoint(const std::string & a, const std::string & b) +{ + SchemeAuthorityKey pa(a); + SchemeAuthorityKey pb(b); + if (pa.authority.empty() || pb.authority.empty()) + return false; + return pa.scheme == pb.scheme && pa.authority == pb.authority; +} +#endif +std::pair getOrCreateStorageAndKey( + const std::string & cache_key, + const std::string & key_to_use, + const std::string & storage_type, + SecondaryStorages & secondary_storages, + const ContextPtr & context, + std::function configure_fn) +{ + std::lock_guard lock(secondary_storages.mutex); + if (auto it = secondary_storages.storages.find(cache_key); it != secondary_storages.storages.end()) + return {it->second, key_to_use}; + + Poco::AutoPtr cfg(new Poco::Util::MapConfiguration); + const std::string config_prefix = "object_storages." + cache_key; + + cfg->setString(config_prefix + ".object_storage_type", storage_type); + + configure_fn(*cfg, config_prefix); + + /// Create under lock to avoid duplicate creation and wasted work + ObjectStoragePtr storage = ObjectStorageFactory::instance().create(cache_key, *cfg, config_prefix, context, /*skip_access_check*/ true); + + secondary_storages.storages.emplace(cache_key, storage); + return {storage, key_to_use}; +} + +bool isAbsolutePath(const std::string & path) +{ + if (!path.empty() && (path.front() == '/' || path.find("://") != std::string_view::npos)) + return true; + + return false; +} + +#endif // USE_AVRO + +} + +SchemeAuthorityKey::SchemeAuthorityKey(const std::string & uri) +{ + if (uri.empty()) + return; + + if (auto scheme_sep = uri.find("://"); scheme_sep != std::string_view::npos) + { + scheme = Poco::toLower(uri.substr(0, scheme_sep)); + auto rest = uri.substr(scheme_sep + 3); // skip :// + + // authority is up to next '/' + auto slash = rest.find('/'); + if (slash == std::string_view::npos) + { + /// Bad URI: missing path component after authority. + /// Exception will be thrown when looking up non-existing object in the storage, so we can just return here. + authority = std::string(rest); + key = "/"; + return; + } + authority = std::string(rest.substr(0, slash)); + /// For file:// URIs, the path is absolute, so we need to keep the leading '/' + /// e.g. file:///home/user/data -> scheme="file", authority="", key="/home/user/data" + if (scheme == "file") + key = std::string(rest.substr(slash)); + else + key = std::string(rest.substr(++slash)); + return; + } + + /// Check for scheme:/path (common for file: https://datatracker.ietf.org/doc/html/rfc8089#appendix-B) + if (auto colon = uri.find(':'); colon != std::string_view::npos && colon > 0) + { + auto after_colon = uri.substr(colon + 1); + + if (!after_colon.empty() && after_colon[0] == '/') + { + scheme = Poco::toLower(uri.substr(0, colon)); + authority = ""; // No authority + key = std::string(after_colon); + return; + } + } + + // Relative path (paths starting with '/' without a scheme are now handled by the caller) + key = std::string(uri); } namespace DataLakeStorageSetting @@ -348,5 +500,384 @@ extern const SettingsUInt64 max_download_buffer_size; extern const SettingsBool use_cache_for_count_from_files; extern const SettingsString filesystem_cache_name; extern const SettingsUInt64 filesystem_cache_boundary_alignment; +extern const SettingsBool s3_propagate_credentials_to_other_storages; +} + +#if USE_AVRO +/// Resolve an absolute metadata path directly to its (object storage, key) by parsing the URI. +/// The storage may be `base_storage` or a secondary one. Returns std::nullopt for paths that must +/// instead go through `path_resolver`: relative paths and bare local-fs absolute base paths. +std::optional> tryResolveObjectStorageForPath( + const std::string & table_location, + const std::string & path, + const DB::ObjectStoragePtr & base_storage, + SecondaryStorages & secondary_storages, + const DB::ContextPtr & context) +{ + if (!isAbsolutePath(path)) + return std::nullopt; // Relative path always belongs to base storage + + auto ensure_local_path_inside_user_files = [&](const std::string & local_path) + { + const auto target_path = std::filesystem::path(local_path).lexically_normal(); + const auto user_files_path = std::filesystem::path(context->getUserFilesPath()).lexically_normal(); + + if (user_files_path.empty() || !fileOrSymlinkPathStartsWith(target_path.string(), user_files_path.string())) + throw DB::Exception( + DB::ErrorCodes::PATH_ACCESS_DENIED, + "File URI '{}' is outside of allowed `user_files` path '{}'", + local_path, + user_files_path.string()); + }; + + SchemeAuthorityKey table_location_decomposed{table_location}; + SchemeAuthorityKey target_decomposed{path}; + + if (target_decomposed.scheme.empty() && target_decomposed.key.starts_with('/')) + { + if (base_storage->getType() == ObjectStorageType::Local) + ensure_local_path_inside_user_files(target_decomposed.key); + + return std::nullopt; + } + + const std::string base_scheme_normalized = normalizeScheme(table_location_decomposed.scheme); + const std::string target_scheme_normalized = normalizeScheme(target_decomposed.scheme); + + /// `file://` paths must stay inside `user_files`. + /// Without this check, metadata could drive reads from arbitrary local paths. + if (target_scheme_normalized == "file") + { + ensure_local_path_inside_user_files(target_decomposed.key); + } + + // For S3 URIs, use S3::URI to properly handle all kinds of URIs, e.g. https://s3.amazonaws.com/bucket/... == s3://bucket/... + #if USE_AWS_S3 + if (target_scheme_normalized == "s3" || target_scheme_normalized == "https" || target_scheme_normalized == "http") + { + std::string normalized_path = path; + if (target_decomposed.scheme == "s3a" || target_decomposed.scheme == "s3n" || target_decomposed.scheme == "oss") + { + normalized_path = "s3://" + target_decomposed.authority + "/" + target_decomposed.key; + } + else if (target_decomposed.scheme == "gcs") + { + normalized_path = "gs://" + target_decomposed.authority + "/" + target_decomposed.key; + } + /// Paths from metadata already have correct encoding; disable Poco::URI + /// percent-decoding so that keys like `col=12%3A00%3A00` are preserved as-is. + S3::URI s3_uri(normalized_path, /*allow_archive_path_syntax*/ false, + /*keep_presigned_query_parameters*/ true, + /*enable_url_encoding*/ false); + + std::string key_to_use = s3_uri.key; + + bool use_base_storage = false; + if (base_storage->getType() == ObjectStorageType::S3) + { + if (auto s3_storage = std::dynamic_pointer_cast(base_storage)) + { + const std::string base_bucket = s3_storage->getObjectsNamespace(); + const std::string base_endpoint = s3_storage->getDescription(); + + if (s3URIMatches(s3_uri, base_bucket, base_endpoint, target_scheme_normalized)) + use_base_storage = true; + } + } + + if (!use_base_storage && (base_scheme_normalized == "s3" || base_scheme_normalized == "https" || base_scheme_normalized == "http")) + { + std::string normalized_table_location = table_location; + if (table_location_decomposed.scheme == "s3a" || table_location_decomposed.scheme == "s3n" || table_location_decomposed.scheme == "oss") + { + normalized_table_location = "s3://" + table_location_decomposed.authority + "/" + table_location_decomposed.key; + } + else if (table_location_decomposed.scheme == "gcs") + { + normalized_table_location = "gs://" + table_location_decomposed.authority + "/" + table_location_decomposed.key; + } + S3::URI base_s3_uri(normalized_table_location, /*allow_archive_path_syntax*/ false, + /*keep_presigned_query_parameters*/ true, + /*enable_url_encoding*/ false); + + if (s3URIMatches(s3_uri, base_s3_uri.bucket, base_s3_uri.endpoint, target_scheme_normalized)) + { + /// The path points inside the table location but is spelled with a different + /// scheme/bucket than the base storage (e.g. a Spark-relocated table whose + /// metadata `location` is `s3a://spark-bucket/...`). The real object lives in the + /// base storage at `table_root` + suffix, which only `IcebergPathResolver::resolve` + /// can compute, so defer to it instead of using the raw metadata key. + return std::nullopt; + } + } + + if (use_base_storage) + return std::make_pair(base_storage, key_to_use); + + /// Construct the endpoint for this storage, then build the cache key from it. + /// A generic `s3://bucket/...` inherits one from the base storage. + const bool endpoint_explicit = (target_decomposed.scheme == "http" || target_decomposed.scheme == "https"); + + std::string endpoint_to_use; + + if (endpoint_explicit) + { + endpoint_to_use = s3_uri.endpoint.empty() + ? ("https://" + s3_uri.bucket + ".s3.amazonaws.com") + : s3_uri.endpoint; + } + else + { + std::string base_endpoint; + if (base_storage->getType() == ObjectStorageType::S3) + base_endpoint = base_storage->getDescription(); + + if (!base_endpoint.empty()) + { + if (base_endpoint.find(".s3.") != std::string::npos && base_endpoint.find(".amazonaws.com") != std::string::npos) + { + /// AWS-style: https://oldbucket.s3.us-east-1.amazonaws.com -> https://newbucket.s3.us-east-1.amazonaws.com + size_t s3_pos = base_endpoint.find(".s3."); + size_t scheme_end = base_endpoint.find("://"); + if (scheme_end != std::string::npos) + { + std::string scheme = base_endpoint.substr(0, scheme_end + 3); + std::string suffix = base_endpoint.substr(s3_pos); + + /// Trim path after endpoint + size_t slash_pos = suffix.find('/', 1); + if (slash_pos != std::string::npos) + suffix = suffix.substr(0, slash_pos); + endpoint_to_use = scheme + s3_uri.bucket + suffix; + } + } + else + { + /// Path-style (e.g. minio): http://host:port/oldbucket -> http://host:port/newbucket + size_t scheme_end = base_endpoint.find("://"); + if (scheme_end != std::string::npos) + { + size_t path_start = base_endpoint.find('/', scheme_end + 3); + if (path_start != std::string::npos) + base_endpoint = base_endpoint.substr(0, path_start); + } + if (!base_endpoint.empty() && base_endpoint.back() == '/') + base_endpoint.pop_back(); + endpoint_to_use = base_endpoint + "/" + s3_uri.bucket; + } + } + + /// Fallback: base storage is not S3 + if (endpoint_to_use.empty()) + { + endpoint_to_use = s3_uri.endpoint.empty() + ? ("https://" + s3_uri.bucket + ".s3.amazonaws.com") + : s3_uri.endpoint; + } + } + + /// Include credential-propagation flag in the cache key: `configure_fn` runs only on miss, + /// so different per-query values of `s3_propagate_credentials_to_other_storages` must not share an entry. + const bool propagate_creds = context->getSettingsRef()[Setting::s3_propagate_credentials_to_other_storages]; + const std::string storage_cache_key = "s3://" + s3_uri.bucket + "@" + endpoint_to_use + + "#propagate=" + (propagate_creds ? "1" : "0"); + + return getOrCreateStorageAndKey( + storage_cache_key, + key_to_use, + "s3", + secondary_storages, + context, + [&](Poco::Util::MapConfiguration & cfg, const std::string & config_prefix) + { + cfg.setString(config_prefix + ".endpoint", endpoint_to_use); + + /// Copy credentials from base storage when the endpoint is the same or + /// `s3_propagate_credentials_to_other_storages` is enabled. + if (base_storage->getType() == ObjectStorageType::S3 + && (context->getSettingsRef()[Setting::s3_propagate_credentials_to_other_storages] + || sameEndpoint(base_storage->getDescription(), endpoint_to_use))) + { + if (auto s3_storage = std::dynamic_pointer_cast(base_storage)) + { + if (auto s3_client = s3_storage->tryGetS3StorageClient()) + { + const auto credentials = s3_client->getCredentials(); + const String & access_key_id = credentials.GetAWSAccessKeyId(); + const String & secret_access_key = credentials.GetAWSSecretKey(); + const String & session_token = credentials.GetSessionToken(); + const String & region = s3_client->getRegion(); + + if (!access_key_id.empty()) + cfg.setString(config_prefix + ".access_key_id", access_key_id); + if (!secret_access_key.empty()) + cfg.setString(config_prefix + ".secret_access_key", secret_access_key); + if (!session_token.empty()) + cfg.setString(config_prefix + ".session_token", session_token); + if (!region.empty()) + cfg.setString(config_prefix + ".region", region); + } + } + } + }); + } + #endif + + #if USE_HDFS + if (target_scheme_normalized == "hdfs") + { + bool use_base_storage = false; + + // Check if base_storage matches (only if it's HDFS) + if (base_storage->getType() == ObjectStorageType::HDFS) + { + if (auto hdfs_storage = std::dynamic_pointer_cast(base_storage)) + { + const std::string base_url = hdfs_storage->getDescription(); + // Extract endpoint from base URL (hdfs://namenode:port/path -> hdfs://namenode:port) + std::string base_endpoint; + if (auto pos = base_url.find('/', base_url.find("//") + 2); pos != std::string::npos) + base_endpoint = base_url.substr(0, pos); + else + base_endpoint = base_url; + + // For HDFS, compare endpoints (namenode addresses) + std::string target_endpoint = target_scheme_normalized + "://" + target_decomposed.authority; + + if (base_endpoint == target_endpoint) + use_base_storage = true; + + // Also check if table_location matches + if (!use_base_storage && base_scheme_normalized == "hdfs") + { + if (table_location_decomposed.authority == target_decomposed.authority) + use_base_storage = true; + } + } + } + + if (use_base_storage) + return std::make_pair(base_storage, target_decomposed.key); + } + #endif + + /// Fallback for schemes not handled above (e.g., abfs, file) + if (base_scheme_normalized == target_scheme_normalized && table_location_decomposed.authority == target_decomposed.authority) + return std::make_pair(base_storage, target_decomposed.key); + + const std::string type_for_factory = factoryTypeForScheme(target_scheme_normalized); + if (type_for_factory.empty()) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Unsupported storage scheme '{}' in path '{}'", target_scheme_normalized, path); + + /// For `file://` URIs the authority is always empty, so using just `"file://"` as the + /// cache key would cause every directory to share a single `LocalObjectStorage` instance + /// whose root (`key_prefix`) is set to the parent directory of the first file ever seen. + /// To avoid this, include the parent directory of the target file in the cache key so that + /// each directory gets its own storage instance with the correct root. + std::string file_dir_path; // only set for file:// URIs + std::string cache_key; + if (target_scheme_normalized == "file") + { + std::filesystem::path fs_path(target_decomposed.key); + file_dir_path = fs_path.parent_path().string(); + if (file_dir_path.empty() || file_dir_path == "/") + file_dir_path = "/"; + else if (file_dir_path.back() != '/') + file_dir_path += '/'; + cache_key = "file://" + file_dir_path; + } + else + { + cache_key = target_scheme_normalized + "://" + target_decomposed.authority; + } + + /// Handle storage types that need new storage creation + return getOrCreateStorageAndKey( + cache_key, + target_decomposed.key, + type_for_factory, + secondary_storages, + context, + [&](Poco::Util::MapConfiguration & cfg, const std::string & config_prefix) + { + if (target_scheme_normalized == "file") + { + cfg.setString(config_prefix + ".path", file_dir_path); + } + else if (target_scheme_normalized == "abfs") + { + std::string container_name; + std::string account_name; + const auto & authority = target_decomposed.authority; + + auto at_pos = authority.find('@'); + if (at_pos != std::string::npos) + { + container_name = authority.substr(0, at_pos); + account_name = authority.substr(at_pos + 1); + /// Remove .dfs.core.windows.net suffix if present + auto suffix_pos = account_name.find('.'); + if (suffix_pos != std::string::npos) + account_name = account_name.substr(0, suffix_pos); + } + else + container_name = authority; + + cfg.setString(config_prefix + ".container_name", container_name); + if (!account_name.empty()) + cfg.setString(config_prefix + ".account_name", account_name); + +#if USE_AZURE_BLOB_STORAGE + /// Copy credentials from base Azure storage if available + if (base_storage->getType() == ObjectStorageType::Azure) + { + if (auto azure_storage = std::dynamic_pointer_cast(base_storage)) + { + const auto & conn_params = azure_storage->getConnectionParameters(); + const auto & auth_method = azure_storage->getAzureBlobStorageAuthMethod(); + + if (std::holds_alternative(auth_method)) + { + cfg.setString(config_prefix + ".connection_string", + std::get(auth_method).toUnderType()); + } + else + { + const auto & endpoint = conn_params.endpoint; + if (!endpoint.storage_account_url.empty()) + cfg.setString(config_prefix + ".storage_account_url", endpoint.storage_account_url); + if (account_name.empty() && !endpoint.account_name.empty()) + cfg.setString(config_prefix + ".account_name", endpoint.account_name); + } + } + } +#endif + } + else if (target_scheme_normalized == "hdfs") + { + // HDFS endpoint must end with '/' + auto endpoint = target_scheme_normalized + "://" + target_decomposed.authority; + if (!endpoint.empty() && endpoint.back() != '/') + endpoint.push_back('/'); + cfg.setString(config_prefix + ".endpoint", endpoint); + } + }); +} + +std::pair resolveObjectStorageForPath( + const std::string & table_location, + const std::string & path, + const DB::ObjectStoragePtr & base_storage, + SecondaryStorages & secondary_storages, + const DB::ContextPtr & context, + const Iceberg::IcebergPathResolver & path_resolver) +{ + if (auto resolved = tryResolveObjectStorageForPath(table_location, path, base_storage, secondary_storages, context)) + return *resolved; + /// Relative paths only: map via path_resolver (table_location -> table_root translation). + return {base_storage, path_resolver.resolve(Iceberg::IcebergPathFromMetadata::deserialize(path))}; } + +#endif + } diff --git a/src/Storages/ObjectStorage/Utils.h b/src/Storages/ObjectStorage/Utils.h index 931ebcbed9ac..ee201c06cd0b 100644 --- a/src/Storages/ObjectStorage/Utils.h +++ b/src/Storages/ObjectStorage/Utils.h @@ -4,11 +4,38 @@ #include #include +#include +#include +#include + namespace DB { class IObjectStorage; +#if USE_AVRO +/// Thread-safe wrapper for secondary object storages map +/// (now only used for Iceberg) +struct SecondaryStorages +{ + mutable std::mutex mutex; + std::map storages; +}; +#endif + +// A URI split into components +// s3://bucket/a/b -> scheme="s3", authority="bucket", path="/a/b" +// file:///var/x -> scheme="file", authority="", path="/var/x" +// /abs/p -> scheme="", authority="", path="/abs/p" +struct SchemeAuthorityKey +{ + explicit SchemeAuthorityKey(const std::string & uri); + + std::string scheme; + std::string authority; + std::string key; +}; + std::optional checkAndGetNewFileOnInsertIfNeeded( const IObjectStorage & object_storage, const StorageObjectStorageConfiguration & configuration, @@ -65,9 +92,35 @@ struct ParseFromDiskResult ParseFromDiskResult parseFromDisk(ASTs args, bool with_structure, ContextPtr context, const fs::path & prefix); +<<<<<<< HEAD void expandPaimonKeeperMacrosIfNeeded( const StorageFactory::Arguments & args, const DataLakeStorageSettingsPtr & storage_settings); +======= +#if USE_AVRO +namespace Iceberg { class IcebergPathResolver; } + +/// Resolve an absolute metadata path directly to its (object storage, key) by parsing the URI. +/// The storage may be `base_storage` or a secondary one. Returns std::nullopt for paths that must +/// instead go through `path_resolver`: relative paths and bare local-fs absolute base paths. +std::optional> tryResolveObjectStorageForPath( + const std::string & table_location, + const std::string & path, + const DB::ObjectStoragePtr & base_storage, + SecondaryStorages & secondary_storages, + const DB::ContextPtr & context); + +/// Resolve a metadata path to (object storage, key) for reading. Absolute paths resolve directly via +/// `tryResolveObjectStorageForPath`; relative paths are mapped via `path_resolver`. +std::pair resolveObjectStorageForPath( + const std::string & table_location, + const std::string & path, + const DB::ObjectStoragePtr & base_storage, + SecondaryStorages & secondary_storages, + const DB::ContextPtr & context, + const Iceberg::IcebergPathResolver & path_resolver); +#endif +>>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) } diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 9c061ab0cf80..e6fb71a2a74f 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -830,10 +830,10 @@ std::function IStorageURLBase::getReadPOSTDataCallback( namespace { - class ReadBufferIterator : public IReadBufferIterator, WithContext + class StorageURLReadBufferIterator : public IReadBufferIterator, WithContext { public: - ReadBufferIterator( + StorageURLReadBufferIterator( const std::vector & urls_to_check_, std::optional format_, const CompressionMethod & compression_method_, @@ -1063,7 +1063,7 @@ std::pair IStorageURLBase::getTableStructureAndForma else urls_to_check = {uri}; - ReadBufferIterator read_buffer_iterator(urls_to_check, format, compression_method, headers, format_settings, context); + StorageURLReadBufferIterator read_buffer_iterator(urls_to_check, format, compression_method, headers, format_settings, context); if (format) return {readSchemaFromFormat(*format, format_settings, read_buffer_iterator, context), *format}; return detectFormatAndReadSchema(format_settings, read_buffer_iterator, context); diff --git a/tests/integration/test_storage_iceberg_multistorage/__init__.py b/tests/integration/test_storage_iceberg_multistorage/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_storage_iceberg_multistorage/configs/config.d/cluster.xml b/tests/integration/test_storage_iceberg_multistorage/configs/config.d/cluster.xml new file mode 100644 index 000000000000..54c08b27abe8 --- /dev/null +++ b/tests/integration/test_storage_iceberg_multistorage/configs/config.d/cluster.xml @@ -0,0 +1,20 @@ + + + + + + node1 + 9000 + + + node2 + 9000 + + + node3 + 9000 + + + + + diff --git a/tests/integration/test_storage_iceberg_multistorage/configs/config.d/named_collections.xml b/tests/integration/test_storage_iceberg_multistorage/configs/config.d/named_collections.xml new file mode 100644 index 000000000000..516e4ba63a3a --- /dev/null +++ b/tests/integration/test_storage_iceberg_multistorage/configs/config.d/named_collections.xml @@ -0,0 +1,15 @@ + + + + http://minio1:9001/root/ + minio + ClickHouse_Minio_P@ssw0rd + + + devstoreaccount1 + Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== + + + + + diff --git a/tests/integration/test_storage_iceberg_multistorage/configs/config.d/query_log.xml b/tests/integration/test_storage_iceberg_multistorage/configs/config.d/query_log.xml new file mode 100644 index 000000000000..a63e91f41fbc --- /dev/null +++ b/tests/integration/test_storage_iceberg_multistorage/configs/config.d/query_log.xml @@ -0,0 +1,6 @@ + + + system + query_log
+ + diff --git a/tests/integration/test_storage_iceberg_multistorage/configs/users.d/users.xml b/tests/integration/test_storage_iceberg_multistorage/configs/users.d/users.xml new file mode 100644 index 000000000000..4b6ba057ecb1 --- /dev/null +++ b/tests/integration/test_storage_iceberg_multistorage/configs/users.d/users.xml @@ -0,0 +1,9 @@ + + + + + default + 1 + + + diff --git a/tests/integration/test_storage_iceberg_multistorage/test.py b/tests/integration/test_storage_iceberg_multistorage/test.py new file mode 100644 index 000000000000..5b477055b2ab --- /dev/null +++ b/tests/integration/test_storage_iceberg_multistorage/test.py @@ -0,0 +1,426 @@ +import pytest +import pyspark +import os +import shutil +import tempfile +import json +import avro.datafile +import avro.io + +from helpers.cluster import ClickHouseCluster +from helpers.s3_tools import ( + LocalUploader, + S3Uploader, + AzureUploader, + LocalDownloader, + S3Downloader, + prepare_s3_bucket, +) +from helpers.iceberg_utils import ( + get_uuid_str, + default_upload_directory, + default_download_directory, +) + +def get_spark(): + builder = ( + pyspark.sql.SparkSession.builder.appName("test_storage_iceberg_multistorage") + .config( + "spark.sql.catalog.spark_catalog", + "org.apache.iceberg.spark.SparkSessionCatalog", + ) + .config("spark.sql.catalog.local", "org.apache.iceberg.spark.SparkCatalog") + .config("spark.sql.catalog.spark_catalog.type", "hadoop") + .config("spark.sql.catalog.spark_catalog.warehouse", "/var/lib/clickhouse/user_files/iceberg_data") + .config( + "spark.sql.extensions", + "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions", + ) + .master("local") + ) + return builder.getOrCreate() + + +@pytest.fixture(scope="package") +def started_cluster(): + try: + cluster = ClickHouseCluster(__file__, with_spark=True) + cluster.add_instance( + "node1", + main_configs=[ + "configs/config.d/query_log.xml", + "configs/config.d/cluster.xml", + "configs/config.d/named_collections.xml", + ], + user_configs=["configs/users.d/users.xml"], + with_minio=True, + with_azurite=True, + stay_alive=True, + ) + + cluster.start() + + prepare_s3_bucket(cluster) + + cluster.spark_session = get_spark() + + cluster.default_s3_uploader = S3Uploader(cluster.minio_client, cluster.minio_bucket) + cluster.default_s3_downloader = S3Downloader(cluster.minio_client, cluster.minio_bucket) + + cluster.azure_container_name = "mycontainer" + cluster.blob_service_client.create_container(cluster.azure_container_name) + cluster.default_azure_uploader = AzureUploader(cluster.blob_service_client, cluster.azure_container_name) + + cluster.default_local_uploader = LocalUploader(cluster.instances["node1"]) + cluster.default_local_downloader = LocalDownloader(cluster.instances["node1"]) + + # Create extra S3 buckets for test_four_different_locations + for i in range(1, 4): + bucket_name = f"{cluster.minio_bucket}-storage{i}" + if not cluster.minio_client.bucket_exists(bucket_name): + cluster.minio_client.make_bucket(bucket_name) + + yield cluster + + finally: + cluster.shutdown() + + +def modify_avro_file(avro_path: str, field_path: list, modifier_func) -> None: + """ + Modify a field in an AVRO file, preserving the rest of it as is. + + field_path: list of keys to navigate to the field + modifier_func: function that takes old value and returns new value + """ + with open(avro_path, 'rb') as f: + reader = avro.datafile.DataFileReader(f, avro.io.DatumReader()) + schema = reader.datum_reader.writers_schema + # Preserve all file metadata (partition-spec, format-version, etc.) + metadata = dict(reader.meta) + records = list(reader) + reader.close() + + for record in records: + obj = record + for key in field_path[:-1]: + if obj is None or key not in obj: + break + obj = obj[key] + else: + if obj and field_path[-1] in obj: + obj[field_path[-1]] = modifier_func(obj[field_path[-1]]) + + with open(avro_path, 'wb') as f: + writer = avro.datafile.DataFileWriter(f, avro.io.DatumWriter(), schema) + for key, value in metadata.items(): + if not key.startswith('avro.'): + writer.set_meta(key, value) + for record in records: + writer.append(record) + writer.close() + + +def get_absolute_path(storage_type: str, cluster, relative_path: str) -> str: + """Convert relative path to absolute path for given storage type.""" + relative_path = relative_path.lstrip("/") + + if storage_type == "s3": + return f"s3a://{cluster.minio_bucket}/{relative_path}" + elif storage_type.startswith("s3:"): # s3:bucket_name format + bucket = storage_type.split(":")[1] + return f"s3a://{bucket}/{relative_path}" + elif storage_type == "azure": + return f"abfs://{cluster.azure_container_name}@{cluster.azurite_account}/{relative_path}" + elif storage_type.startswith("azure:"): # azure:container_name format + container = storage_type.split(":")[1] + return f"abfs://{container}@{cluster.azurite_account}/{relative_path}" + elif storage_type == "local": + return f"file:///{relative_path}" + else: + raise ValueError(f"Unknown storage type: {storage_type}") + + +def get_uploader(storage_type: str, cluster): + if storage_type == "s3": + return cluster.default_s3_uploader + elif storage_type.startswith("s3:"): + bucket = storage_type.split(":")[1] + return S3Uploader(cluster.minio_client, bucket) + elif storage_type == "azure": + return cluster.default_azure_uploader + elif storage_type.startswith("azure:"): + container = storage_type.split(":")[1] + return AzureUploader(cluster.blob_service_client, container) + elif storage_type == "local": + return cluster.default_local_uploader + else: + raise ValueError(f"Unknown storage type: {storage_type}") + + +def get_table_function(metadata_storage: str): + if metadata_storage == "s3" or metadata_storage.startswith("s3:"): + return "icebergS3" + elif metadata_storage == "azure" or metadata_storage.startswith("azure:"): + return "icebergAzure" + elif metadata_storage == "local": + return "icebergLocal" + else: + raise ValueError(f"Unknown storage type: {metadata_storage}") + + +def get_query_args(metadata_storage: str, cluster, table_path: str): + """Get query arguments for the iceberg table function.""" + minio_url = f"http://{cluster.minio_host}:{cluster.minio_port}" + if metadata_storage == "s3": + return f"s3, filename='{table_path}/', format=Parquet, url='{minio_url}/{cluster.minio_bucket}/'" + elif metadata_storage.startswith("s3:"): + bucket = metadata_storage.split(":")[1] + return f"s3, filename='{table_path}/', format=Parquet, url='{minio_url}/{bucket}/'" + elif metadata_storage == "azure": + return f"azure, container='{cluster.azure_container_name}', storage_account_url='{cluster.env_variables['AZURITE_STORAGE_ACCOUNT_URL']}', blob_path='{table_path}/', format=Parquet" + elif metadata_storage.startswith("azure:"): + container = metadata_storage.split(":")[1] + return f"azure, container='{container}', storage_account_url='{cluster.env_variables['AZURITE_STORAGE_ACCOUNT_URL']}', blob_path='{table_path}/', format=Parquet" + elif metadata_storage == "local": + return f"local, path='/{table_path}', format=Parquet" + else: + raise ValueError(f"Unknown storage type: {metadata_storage}") + + +def find_files(directory: str, suffix: str) -> list: + """Find files ending with given suffix.""" + result = [] + for root, _, files in os.walk(directory): + for f in files: + if f.endswith(suffix): + result.append(os.path.join(root, f)) + return result + + +def path_modifier(old_path: str, new_storage: str, cluster, base_path: str): + """Create a new absolute path for a different storage location.""" + # Extract just the filename/relative portion + if "://" in old_path: + # Parse out the path part after protocol://bucket/ + parts = old_path.split("/") + # Find where the actual path starts (after bucket) + for i, part in enumerate(parts): + if base_path.split("/")[0] in part or "var" in part: + relative = "/".join(parts[i:]) + break + else: + relative = parts[-1] + else: + relative = old_path.lstrip("/") + + return get_absolute_path(new_storage, cluster, relative) + + +# ============================================================================= +# Tests +# ============================================================================= + +STORAGE_TYPES = ["s3", "azure", "local"] + +def _get_type_family(t): + if t.startswith("s3"): + return "s3" + elif t.startswith("azure"): + return "azure" + return t + +def _generate_valid_combinations(): + """ + Generate valid storage combinations. + Rule: all components must be same type family as metadata, OR local. + Local doesn't need credentials, so S3+local and Azure+local work. + But S3+Azure doesn't work (credentials aren't interchangeable). + """ + combinations = [] + for metadata in STORAGE_TYPES: + main_family = _get_type_family(metadata) + for manifest_list in STORAGE_TYPES: + if _get_type_family(manifest_list) not in (main_family, "local"): + continue + for manifest in STORAGE_TYPES: + if _get_type_family(manifest) not in (main_family, "local"): + continue + for data in STORAGE_TYPES: + if _get_type_family(data) not in (main_family, "local"): + continue + combinations.append((metadata, manifest_list, manifest, data)) + return combinations + +VALID_COMBINATIONS = _generate_valid_combinations() + +@pytest.mark.parametrize("metadata_storage,manifest_list_storage,manifest_storage,data_storage", VALID_COMBINATIONS) +def test_multi_storage_combinations(started_cluster, metadata_storage, manifest_list_storage, manifest_storage, data_storage): + """ + Test Iceberg table with all components in different storage locations. + """ + instance = started_cluster.instances["node1"] + spark = started_cluster.spark_session + + TABLE_NAME = f"test_combo_{get_uuid_str()}" + + spark.sql(f"CREATE TABLE {TABLE_NAME} (id INT, value STRING) USING iceberg OPTIONS('format-version'='2')") + spark.sql(f"INSERT INTO {TABLE_NAME} VALUES (1, 'alpha'), (2, 'beta'), (3, 'gamma')") + + # Upload to default S3 first + default_upload_directory(started_cluster, "s3", f"/iceberg_data/default/{TABLE_NAME}/", f"/iceberg_data/default/{TABLE_NAME}/") + + # Download all files + temp_dir = tempfile.mkdtemp() + host_path = os.path.join(temp_dir, TABLE_NAME) + os.makedirs(host_path, exist_ok=True) + + default_download_directory(started_cluster, "s3", f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}/", host_path) + + base_path = f"var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}" + metadata_dir = os.path.join(host_path, "metadata") + data_dir = os.path.join(host_path, "data") + + # Step 1: Modify manifest files to point to data_storage + manifest_files = [f for f in find_files(metadata_dir, ".avro") if not os.path.basename(f).startswith("snap-")] + for mf in manifest_files: + modify_avro_file(mf, ["data_file", "file_path"], + lambda p: path_modifier(p, data_storage, started_cluster, base_path)) + + # Step 2: Modify manifest-list files to point to manifest_storage + manifest_list_files = [f for f in find_files(metadata_dir, ".avro") if os.path.basename(f).startswith("snap-")] + for ml in manifest_list_files: + modify_avro_file(ml, ["manifest_path"], + lambda p: path_modifier(p, manifest_storage, started_cluster, base_path)) + + # Step 3: Modify metadata.json to point to manifest_list_storage + for mj in find_files(metadata_dir, ".metadata.json"): + with open(mj, 'r') as f: + data = json.load(f) + + data["location"] = get_absolute_path(metadata_storage, started_cluster, base_path) + + # Update snapshot manifest-list paths + if "snapshots" in data: + for snap in data["snapshots"]: + if "manifest-list" in snap: + snap["manifest-list"] = path_modifier(snap["manifest-list"], manifest_list_storage, started_cluster, base_path) + + with open(mj, 'w') as f: + json.dump(data, f, indent=2) + + # Step 4: Upload to respective storages + # Metadata files (*.metadata.json, version-hint.text) + meta_uploader = get_uploader(metadata_storage, started_cluster) + for f in find_files(metadata_dir, ".metadata.json") + find_files(metadata_dir, "version-hint.text"): + rel = os.path.relpath(f, host_path) + meta_uploader.upload_file(f, f"{base_path}/{rel}") + + # Manifest-list files + ml_uploader = get_uploader(manifest_list_storage, started_cluster) + for f in manifest_list_files: + rel = os.path.relpath(f, host_path) + ml_uploader.upload_file(f, f"{base_path}/{rel}") + + # Manifest files + m_uploader = get_uploader(manifest_storage, started_cluster) + for f in manifest_files: + rel = os.path.relpath(f, host_path) + m_uploader.upload_file(f, f"{base_path}/{rel}") + + # Data files + d_uploader = get_uploader(data_storage, started_cluster) + if os.path.exists(data_dir): + for f in find_files(data_dir, ".parquet"): + rel = os.path.relpath(f, host_path) + d_uploader.upload_file(f, f"{base_path}/{rel}") + + shutil.rmtree(temp_dir) + + func = get_table_function(metadata_storage) + args = get_query_args(metadata_storage, started_cluster, base_path) + + assert instance.query(f"SELECT * FROM {func}({args}) ORDER BY id") == "1\talpha\n2\tbeta\n3\tgamma\n" + + +# S3 is the primary use case for cross-bucket access. +# Azure cross-container: not supported (account_key not extractable from credential object). +def test_four_different_s3_buckets(started_cluster): + """S3: each component in a different bucket (metadata, manifest-list, manifest, data).""" + instance = started_cluster.instances["node1"] + spark = started_cluster.spark_session + + TABLE_NAME = f"test_four_buckets_{get_uuid_str()}" + buckets = [ + started_cluster.minio_bucket, + f"{started_cluster.minio_bucket}-storage1", + f"{started_cluster.minio_bucket}-storage2", + f"{started_cluster.minio_bucket}-storage3", + ] + + metadata_storage = f"s3:{buckets[0]}" + manifest_list_storage = f"s3:{buckets[1]}" + manifest_storage = f"s3:{buckets[2]}" + data_storage = f"s3:{buckets[3]}" + + uploaders = {f"s3:{b}": S3Uploader(started_cluster.minio_client, b) for b in buckets} + + spark.sql(f"CREATE TABLE {TABLE_NAME} (id INT, name STRING, score INT) USING iceberg OPTIONS('format-version'='2')") + spark.sql(f"INSERT INTO {TABLE_NAME} VALUES (1, 'Alice', 100), (2, 'Bob', 85), (3, 'Carol', 92)") + + default_upload_directory(started_cluster, "s3", f"/iceberg_data/default/{TABLE_NAME}/", f"/iceberg_data/default/{TABLE_NAME}/") + + temp_dir = tempfile.mkdtemp() + host_path = os.path.join(temp_dir, TABLE_NAME) + os.makedirs(host_path, exist_ok=True) + + default_download_directory(started_cluster, "s3", f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}/", host_path) + + base_path = f"var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}" + metadata_dir = os.path.join(host_path, "metadata") + data_dir = os.path.join(host_path, "data") + + manifest_files = [f for f in find_files(metadata_dir, ".avro") if not os.path.basename(f).startswith("snap-")] + for mf in manifest_files: + modify_avro_file(mf, ["data_file", "file_path"], + lambda p: path_modifier(p, data_storage, started_cluster, base_path)) + + manifest_list_files = [f for f in find_files(metadata_dir, ".avro") if os.path.basename(f).startswith("snap-")] + for ml in manifest_list_files: + modify_avro_file(ml, ["manifest_path"], + lambda p: path_modifier(p, manifest_storage, started_cluster, base_path)) + + for mj in find_files(metadata_dir, ".metadata.json"): + with open(mj, 'r') as f: + data = json.load(f) + data["location"] = get_absolute_path(metadata_storage, started_cluster, base_path) + if "snapshots" in data: + for snap in data["snapshots"]: + if "manifest-list" in snap: + snap["manifest-list"] = path_modifier(snap["manifest-list"], manifest_list_storage, started_cluster, base_path) + with open(mj, 'w') as f: + json.dump(data, f, indent=2) + + for f in find_files(metadata_dir, ".metadata.json") + find_files(metadata_dir, "version-hint.text"): + rel = os.path.relpath(f, host_path) + uploaders[metadata_storage].upload_file(f, f"{base_path}/{rel}") + + for f in manifest_list_files: + rel = os.path.relpath(f, host_path) + uploaders[manifest_list_storage].upload_file(f, f"{base_path}/{rel}") + + for f in manifest_files: + rel = os.path.relpath(f, host_path) + uploaders[manifest_storage].upload_file(f, f"{base_path}/{rel}") + + if os.path.exists(data_dir): + for f in find_files(data_dir, ".parquet"): + rel = os.path.relpath(f, host_path) + uploaders[data_storage].upload_file(f, f"{base_path}/{rel}") + + shutil.rmtree(temp_dir) + + minio_url = f"http://{started_cluster.minio_host}:{started_cluster.minio_port}" + result = instance.query(f"SELECT * FROM icebergS3(s3, filename='{base_path}/', format=Parquet, url='{minio_url}/{buckets[0]}/') ORDER BY id") + + assert result == "1\tAlice\t100\n2\tBob\t85\n3\tCarol\t92\n" \ No newline at end of file diff --git a/tests/integration/test_storage_iceberg_schema_evolution/test_array_evolved_with_struct.py b/tests/integration/test_storage_iceberg_schema_evolution/test_array_evolved_with_struct.py index 5cb1c02a0c07..9a60da2b2301 100644 --- a/tests/integration/test_storage_iceberg_schema_evolution/test_array_evolved_with_struct.py +++ b/tests/integration/test_storage_iceberg_schema_evolution/test_array_evolved_with_struct.py @@ -55,7 +55,7 @@ def execute_spark_query(query: str): execute_spark_query( f""" - INSERT INTO {TABLE_NAME} VALUES (ARRAY(named_struct('name', 'Singapore', 'zip', 12345), named_struct('name', 'Moscow', 'zip', 54321)), ARRAY(1,2)); + INSERT INTO {TABLE_NAME} VALUES (ARRAY(named_struct('city', 'Singapore', 'zip', 12345), named_struct('city', 'Moscow', 'zip', 54321)), ARRAY(1,2)); """ ) diff --git a/tests/queries/0_stateless/data_minio/field_ids_complex_test/metadata/v1.metadata.json b/tests/queries/0_stateless/data_minio/field_ids_complex_test/metadata/v1.metadata.json index 8d367d20f041..a983881af8f0 100644 --- a/tests/queries/0_stateless/data_minio/field_ids_complex_test/metadata/v1.metadata.json +++ b/tests/queries/0_stateless/data_minio/field_ids_complex_test/metadata/v1.metadata.json @@ -1,7 +1,7 @@ { "format-version" : 2, "table-uuid" : "d4b695ca-ceeb-4537-8a2a-eee90dc6e313", - "location" : "s3a://test/field_ids_struct_test/metadata/field_ids_complex_test", + "location" : "s3a://test/field_ids_complex_test", "last-sequence-number" : 1, "last-updated-ms" : 1757661733693, "last-column-id" : 9, @@ -96,7 +96,7 @@ "total-position-deletes" : "0", "total-equality-deletes" : "0" }, - "manifest-list" : "s3a://test/field_ids_struct_test/metadata/field_ids_complex_test/metadata/snap-607752583403487091-1-140c8dff-1d83-4841-bc40-9aa85205b555.avro", + "manifest-list" : "s3a://test/field_ids_complex_test/metadata/snap-607752583403487091-1-140c8dff-1d83-4841-bc40-9aa85205b555.avro", "schema-id" : 0 } ], "statistics" : [ ], diff --git a/tests/queries/0_stateless/data_minio/field_ids_struct_test/metadata/v1.metadata.json b/tests/queries/0_stateless/data_minio/field_ids_struct_test/metadata/v1.metadata.json index 2d149abb44e7..d6c9079228ac 100644 --- a/tests/queries/0_stateless/data_minio/field_ids_struct_test/metadata/v1.metadata.json +++ b/tests/queries/0_stateless/data_minio/field_ids_struct_test/metadata/v1.metadata.json @@ -1,7 +1,7 @@ { "format-version" : 2, "table-uuid" : "149ecc15-7afc-4311-86b3-3a4c8d4ec08e", - "location" : "s3a://test/field_ids_struct_test/metadata/field_ids_struct_test", + "location" : "s3a://test/field_ids_struct_test", "last-sequence-number" : 1, "last-updated-ms" : 1753959190403, "last-column-id" : 6, @@ -84,7 +84,7 @@ "total-position-deletes" : "0", "total-equality-deletes" : "0" }, - "manifest-list" : "s3a://test/field_ids_struct_test/metadata/field_ids_struct_test/metadata/snap-2512638186869817292-1-ec467367-15a4-4610-8ea8-cf76797afb03.avro", + "manifest-list" : "s3a://test/field_ids_struct_test/metadata/snap-2512638186869817292-1-ec467367-15a4-4610-8ea8-cf76797afb03.avro", "schema-id" : 0 } ], "statistics" : [ ], diff --git a/tests/queries/0_stateless/data_minio/field_ids_table_test/metadata/v1.metadata.json b/tests/queries/0_stateless/data_minio/field_ids_table_test/metadata/v1.metadata.json index 32225eb618ad..1ddc3492cc82 100644 --- a/tests/queries/0_stateless/data_minio/field_ids_table_test/metadata/v1.metadata.json +++ b/tests/queries/0_stateless/data_minio/field_ids_table_test/metadata/v1.metadata.json @@ -1,7 +1,7 @@ { "format-version" : 2, "table-uuid" : "8f1f9ae2-18bb-421e-b640-ec2f85e67bce", - "location" : "s3a://test/field_ids_table_test/metadata/field_ids_table_test", + "location" : "s3a://test/field_ids_table_test", "last-sequence-number" : 1, "last-updated-ms" : 1752481476160, "last-column-id" : 1, @@ -56,7 +56,7 @@ "total-position-deletes" : "0", "total-equality-deletes" : "0" }, - "manifest-list" : "s3a://test/field_ids_table_test/metadata/field_ids_table_test/metadata/snap-2811410366534688344-1-3b002f99-b012-4041-9a97-db477fcc7115.avro", + "manifest-list" : "s3a://test/field_ids_table_test/metadata/snap-2811410366534688344-1-3b002f99-b012-4041-9a97-db477fcc7115.avro", "schema-id" : 0 } ], "statistics" : [ ], From 2585ca3a852e184a66fe3334b42f7245b893c859 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:24:45 +0200 Subject: [PATCH 2/2] Resolve conflicts in cherry-pick of #1859 Kept antalya-26.6 shapes and re-applied the PR's changes on top: - S3::URI: appended `enable_url_encoding` after the base-branch `S3UriStyle uri_style` parameter and updated the two new call sites in Storages/ObjectStorage/Utils.cpp accordingly. - Iceberg/Compaction.cpp: kept `static` linkage that antalya-26.6 gives getOldFiles()/clearOldFiles() while taking the PR's new signatures. - IcebergIterator: dropped the PR's context-only `table_schema_id` / `setFileMetaInfo(DataFileMetaInfo...)` lines which do not exist on antalya-26.6; kept only `secondary_storages` and the requires_external_storage computation. - IcebergMetadata: kept `std::move(persistent_components_)` ctor and the `getManifestFileEntriesHandle` prefetch call of antalya-26.6, adding `*secondary_storages`; kept `const persistent_components`. - collectRetainedFiles()/collectExpiredFiles() moved out of Iceberg/Mutations.cpp into Iceberg/ExpireSnapshotsExecute.cpp on antalya-26.6: applied the PR's SecondaryStorages plumbing there, and likewise for the parallel helper in Iceberg/SnapshotFilesTraversal.cpp and for IcebergMetadata::getFilesForManifest, since getManifestList()/getManifestFileEntriesHandle() now require the parameter. - StorageObjectStorageStableTaskDistributor: getFileIdentifier() does not exist on antalya-26.6 (the identifier is computed inline), so the PR's "prefer the Iceberg metadata path" change was applied at the two inline sites. - StorageObjectStorageSource: kept base-branch `.storage_id` virtual column field, ProfileEvents counter, `configuration->compression_method` spelling and the process-list time-limit check; took the PR's `path_for_virtual_column`, resolved-storage read buffer and `resolveIcebergObjectStorageIfNeeded()`. - SettingsChangesHistory: uncommented the existing `s3_propagate_credentials_to_other_storages` row in place and dropped the cherry-pick's duplicate row. --- src/Core/SettingsChangesHistory.cpp | 3 +- src/IO/S3/URI.cpp | 12 +- src/IO/S3/URI.h | 5 +- .../DataLakes/Iceberg/Compaction.cpp | 12 +- .../Iceberg/ExpireSnapshotsExecute.cpp | 10 +- .../DataLakes/Iceberg/IcebergDataObjectInfo.h | 12 +- .../DataLakes/Iceberg/IcebergIterator.cpp | 14 - .../DataLakes/Iceberg/IcebergIterator.h | 10 +- .../DataLakes/Iceberg/IcebergMetadata.cpp | 18 +- .../DataLakes/Iceberg/IcebergMetadata.h | 6 +- .../Iceberg/ManifestFileIterator.cpp | 6 - .../DataLakes/Iceberg/Mutations.cpp | 633 ------------------ .../Iceberg/PositionDeleteTransform.cpp | 4 - .../Iceberg/SnapshotFilesTraversal.cpp | 5 +- .../ObjectStorage/DataLakes/Iceberg/Utils.h | 10 - .../StorageObjectStorageSource.cpp | 28 +- .../StorageObjectStorageSource.h | 4 - ...rageObjectStorageStableTaskDistributor.cpp | 68 +- src/Storages/ObjectStorage/Utils.cpp | 2 + src/Storages/ObjectStorage/Utils.h | 3 - 20 files changed, 35 insertions(+), 830 deletions(-) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 242adf009e03..379fb8f8a0e3 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -159,7 +159,6 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"max_rand_distribution_parameter", 1e6, 1e6, "New setting to limit shape parameters in random distribution functions, preventing hangs with extreme inputs."}, {"optimize_truncate_order_by_after_group_by_keys", false, true, "Remove trailing ORDER BY elements once all GROUP BY keys are covered in the ORDER BY prefix."}, {"use_statistics_for_part_pruning", false, true, "New setting to use statistics for part pruning during query execution."}, - {"s3_propagate_credentials_to_other_storages", false, false, "New setting"}, {"http_max_fields", 1000000, 1000, "Reduce default to limit pre-authentication memory usage by HTTP connections."}, {"http_max_field_name_size", 131072, 4096, "Reduce default to limit pre-authentication memory usage by HTTP connections."}, {"http_max_request_header_size", 0, 10485760, "New setting to limit total HTTP request header size before authentication."}, @@ -269,7 +268,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() addSettingsChanges(settings_changes_history, "26.1.3.20001.altinityantalya", { // {"iceberg_partition_timezone", "", "", "New setting."}, - // {"s3_propagate_credentials_to_other_storages", false, false, "New setting"}, + {"s3_propagate_credentials_to_other_storages", false, false, "New setting"}, // {"export_merge_tree_part_filename_pattern", "", "{part_name}_{checksum}", "New setting"}, // {"use_parquet_metadata_cache", false, true, "Enables cache of parquet file metadata."}, // {"input_format_parquet_use_metadata_cache", true, false, "Obsolete. No-op"}, // https://github.com/Altinity/ClickHouse/pull/586 diff --git a/src/IO/S3/URI.cpp b/src/IO/S3/URI.cpp index bb2bdfe472f8..7164b5d3c877 100644 --- a/src/IO/S3/URI.cpp +++ b/src/IO/S3/URI.cpp @@ -17,11 +17,7 @@ namespace DB struct URIConverter { -<<<<<<< HEAD - static void modifyURI(Poco::URI & uri, NameToNameMap mapper) -======= - static void modifyURI(Poco::URI & uri, std::unordered_map mapper, bool enable_url_encoding = true) ->>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) + static void modifyURI(Poco::URI & uri, NameToNameMap mapper, bool enable_url_encoding = true) { Macros macros({{"bucket", uri.getHost()}}); uri = macros.expand(mapper[uri.getScheme()]).empty() @@ -38,11 +34,7 @@ namespace ErrorCodes namespace S3 { -<<<<<<< HEAD -URI::URI(const std::string & uri_, bool allow_archive_path_syntax, bool keep_presigned_query_parameters, S3UriStyle uri_style) -======= -URI::URI(const std::string & uri_, bool allow_archive_path_syntax, bool keep_presigned_query_parameters, bool enable_url_encoding) ->>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) +URI::URI(const std::string & uri_, bool allow_archive_path_syntax, bool keep_presigned_query_parameters, S3UriStyle uri_style, bool enable_url_encoding) { /// Case when AWS Private Link Interface is being used /// E.g. (bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w.s3.us-east-1.vpce.amazonaws.com/bucket-name/key) diff --git a/src/IO/S3/URI.h b/src/IO/S3/URI.h index 15dac27f771c..241d75a647a3 100644 --- a/src/IO/S3/URI.h +++ b/src/IO/S3/URI.h @@ -41,11 +41,8 @@ struct URI const std::string & uri_, bool allow_archive_path_syntax = false, bool keep_presigned_query_parameters = true, -<<<<<<< HEAD - S3UriStyle uri_style = S3UriStyle::AUTO); -======= + S3UriStyle uri_style = S3UriStyle::AUTO, bool enable_url_encoding = true); ->>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) void addRegionToURI(const std::string & region); static void validateBucket(const std::string & bucket, const Poco::URI & uri); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp index ff7fc2298c70..4853c51f1a1e 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp @@ -626,19 +626,15 @@ static void writeMetadataFiles( } } -<<<<<<< HEAD -static std::vector getOldFiles(ObjectStoragePtr object_storage, const String & table_path) -======= /// Files to delete after compaction: a base-storage directory listing under `metadata/` and /// `data/` (covers historical metadata.json and any orphan files on the base storage), plus /// any paths from the compacted snapshots that resolve to a secondary storage. -std::vector> getOldFiles( +static std::vector> getOldFiles( ObjectStoragePtr object_storage, SecondaryStorages & secondary_storages, ContextPtr context, const PersistentTableComponents & persistent_table_components, const Plan & plan) ->>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) { std::vector> result; @@ -664,11 +660,7 @@ std::vector> getOldFiles( return result; } -<<<<<<< HEAD -static void clearOldFiles(ObjectStoragePtr object_storage, const std::vector & old_files) -======= -void clearOldFiles(const std::vector> & old_files) ->>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) +static void clearOldFiles(const std::vector> & old_files) { auto log = getLogger("IcebergCompaction"); for (const auto & [storage, key] : old_files) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ExpireSnapshotsExecute.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ExpireSnapshotsExecute.cpp index c026aae02891..5ececa31a5ff 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ExpireSnapshotsExecute.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ExpireSnapshotsExecute.cpp @@ -366,6 +366,7 @@ void collectRetainedFiles( std::set & retained_data_file_paths, std::set & retained_manifest_list_paths) { + SecondaryStorages secondary_storages; for (UInt32 i = 0; i < retained_snapshots->size(); ++i) { auto snapshot = retained_snapshots->getObject(i); @@ -375,14 +376,14 @@ void collectRetainedFiles( auto manifest_list_path = IcebergPathFromMetadata::deserialize(snapshot->getValue(Iceberg::f_manifest_list)); retained_manifest_list_paths.insert(manifest_list_path); - auto manifest_keys = getManifestList(object_storage, persistent_table_components, context, manifest_list_path, log); + auto manifest_keys = getManifestList(object_storage, persistent_table_components, context, manifest_list_path, log, secondary_storages); for (const auto & manifest_entry : manifest_keys) { retained_manifest_paths.insert(manifest_entry.manifest_file_path); auto entries_handle = getManifestFileEntriesHandle( object_storage, persistent_table_components, context, log, - manifest_entry, current_schema_id); + manifest_entry, current_schema_id, secondary_storages); collectAllFilePaths(entries_handle, retained_data_file_paths); } } @@ -410,6 +411,7 @@ ExpiredFiles collectExpiredFiles( Int32 current_schema_id) { ExpiredFiles result; + SecondaryStorages secondary_storages; std::set seen_expired_manifest_list_paths; std::set seen_expired_manifest_paths; for (const auto & manifest_list_path : expired_manifest_list_paths) @@ -423,7 +425,7 @@ ExpiredFiles collectExpiredFiles( ManifestFileCacheKeys manifest_keys; try { - manifest_keys = getManifestList(object_storage, persistent_table_components, context, manifest_list_path, log); + manifest_keys = getManifestList(object_storage, persistent_table_components, context, manifest_list_path, log, secondary_storages); } catch (...) { @@ -443,7 +445,7 @@ ExpiredFiles collectExpiredFiles( { auto entries_handle = getManifestFileEntriesHandle( object_storage, persistent_table_components, context, log, - manifest_entry, current_schema_id); + manifest_entry, current_schema_id, secondary_storages); for (const auto & entry : entries_handle.getFilesWithoutDeleted(FileContentType::DATA)) if (!retained_data_file_paths.contains(entry->parsed_entry->file_path_key)) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.h index 28cca2f78264..1d31aa51aaf8 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.h @@ -21,19 +21,13 @@ String computePartitionId(const Row & partition_key_value); struct IcebergObjectSerializableInfo { IcebergPathFromMetadata data_object_file_path_key; -<<<<<<< HEAD - Int32 underlying_format_read_schema_id{}; - Int32 schema_id_relevant_to_iterator{}; - Int64 sequence_number{}; -======= /// Raw path string as written in the Iceberg manifest, preserved as-is (may be a full URI like /// `s3://bucket/...` or a relative path). Used for the `_path` virtual column and as a stable /// task identifier. Not a canonicalised storage key — see `IcebergPathResolver::resolve` for that. String data_object_file_metadata_path; - Int32 underlying_format_read_schema_id; - Int32 schema_id_relevant_to_iterator; - Int64 sequence_number; ->>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) + Int32 underlying_format_read_schema_id{}; + Int32 schema_id_relevant_to_iterator{}; + Int64 sequence_number{}; String file_format; String manifest_file; String partition_id; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp index 8ea7eddad3a8..1e3960bc2edb 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp @@ -278,11 +278,7 @@ IcebergIterator::IcebergIterator( secondary_storages_) , blocking_queue(100) , callback(std::move(callback_)) -<<<<<<< HEAD -======= - , table_schema_id(table_snapshot_->schema_id) , secondary_storages(secondary_storages_) ->>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) { auto delete_file = deletes_iterator.next(); while (delete_file.has_value()) @@ -416,15 +412,6 @@ ObjectInfoPtr IcebergIterator::next(size_t) object_info->info.data_object_file_path_key); } -<<<<<<< HEAD -======= - object_info->relative_path_with_metadata.setFileMetaInfo(std::make_shared( - *persistent_components.schema_processor, - table_schema_id, /// current schema id to use current column names - manifest_file_entry->resolved_schema_id, /// file's schema id to interpret value_bounds bytes - manifest_file_entry->parsed_entry->columns_infos, - manifest_file_entry->parsed_entry->value_bounds)); - if (!object_info->info.requires_external_storage) { auto resolves_to_external_storage = [&](const String & file_path) @@ -447,7 +434,6 @@ ObjectInfoPtr IcebergIterator::next(size_t) || any_external(object_info->info.equality_deletes_objects); } ->>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) ProfileEvents::increment(ProfileEvents::IcebergMetadataReturnedObjectInfos); if (callback) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.h index 3f71a6111abd..36542ac96dee 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.h @@ -79,12 +79,8 @@ class IcebergIterator : public IObjectIterator IDataLakeMetadata::FileProgressCallback callback_, Iceberg::TableStateSnapshotPtr table_snapshot_, Iceberg::IcebergDataSnapshotPtr data_snapshot_, -<<<<<<< HEAD - Iceberg::PersistentTableComponents persistent_components); -======= - Iceberg::PersistentTableComponents persistent_components_, + Iceberg::PersistentTableComponents persistent_components, std::shared_ptr secondary_storages_); ->>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) ObjectInfoPtr next(size_t) override; @@ -107,11 +103,7 @@ class IcebergIterator : public IObjectIterator std::vector equality_deletes_files; std::exception_ptr exception; std::mutex exception_mutex; -<<<<<<< HEAD -======= - Int32 table_schema_id; std::shared_ptr secondary_storages; // Sometimes data or manifests can be located on another storage ->>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) }; } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp index 26e7d5e70659..71ceb528382d 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp @@ -226,12 +226,8 @@ IcebergMetadata::IcebergMetadata( ContextPtr context_) : log(getLogger("IcebergMetadata")) , object_storage(std::move(object_storage_)) -<<<<<<< HEAD - , persistent_components(std::move(persistent_components_)) -======= , secondary_storages(std::make_shared()) - , persistent_components(initializePersistentTableComponents(configuration_, cache_ptr, context_)) ->>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) + , persistent_components(std::move(persistent_components_)) , data_lake_settings(configuration_->getDataLakeSettings()) , write_format(configuration_->format) { @@ -288,16 +284,8 @@ void IcebergMetadata::backgroundMetadataPrefetcherThread() for (const auto & entry : actual_data_snapshot->manifest_list_entries) { /// second, we fetch, parse and cache each manifest file -<<<<<<< HEAD auto manifest_file_ptr = getManifestFileEntriesHandle( - object_storage, persistent_components, ctx, log, entry, actual_table_state_snapshot.schema_id); -======= - auto manifest_file_ptr = Iceberg::getManifestFile( - object_storage, persistent_components, ctx, log, - entry.manifest_file_path, - entry.manifest_file_byte_size, - *secondary_storages); ->>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) + object_storage, persistent_components, ctx, log, entry, actual_table_state_snapshot.schema_id, *secondary_storages); } } @@ -1006,7 +994,7 @@ IcebergMetadata::IcebergFiles IcebergMetadata::getFilesForManifest( const auto & manifest_list_entry = data_snapshot->manifest_list_entries[manifest_index]; auto handle = getManifestFileEntriesHandle( - object_storage, persistent_components, local_context, log, manifest_list_entry, table_state.schema_id); + object_storage, persistent_components, local_context, log, manifest_list_entry, table_state.schema_id, *secondary_storages); IcebergFiles result; for (auto content_type : {FileContentType::DATA, FileContentType::POSITION_DELETE, FileContentType::EQUALITY_DELETE}) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h index dfc24a6ea984..ad9564c7364c 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h @@ -203,12 +203,8 @@ class IcebergMetadata : public IDataLakeMetadata LoggerPtr log; const ObjectStoragePtr object_storage; -<<<<<<< HEAD - const DB::Iceberg::PersistentTableComponents persistent_components; -======= mutable std::shared_ptr secondary_storages; - DB::Iceberg::PersistentTableComponents persistent_components; ->>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) + const DB::Iceberg::PersistentTableComponents persistent_components; const DataLakeStorageSettings & data_lake_settings; const String write_format; BackgroundSchedulePoolTaskHolder background_metadata_prefetch_task; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp index 70bfefe03f4d..d05e6ebb0062 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp @@ -395,13 +395,7 @@ ProcessedManifestFileEntryPtr ManifestFileIterator::processRow(size_t row_index) return nullptr; } -<<<<<<< HEAD - /// Compute inherited/resolved fields - Int64 resolved_snapshot_id = 0; -======= - Int64 resolved_snapshot_id; ->>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) if (parsed_entry->parsed_snapshot_id.has_value()) { resolved_snapshot_id = *parsed_entry->parsed_snapshot_id; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp index 8a256f10985e..8a04b5c83b3e 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp @@ -832,642 +832,9 @@ void alter( auto hint_path = filename_generator.generateVersionHint(); -<<<<<<< HEAD const bool catalog_writes_metadata_file = catalog && catalog->isTransactional(); if (!catalog_writes_metadata_file && !writeMetadataFileAndVersionHint( -======= - if (i == MAX_TRANSACTION_RETRIES) - throw Exception(ErrorCodes::LIMIT_EXCEEDED, "Too many unsuccessed retries to alter iceberg table"); -} - -/// Table-level snapshot retention policy read from Iceberg table properties. -struct RetentionPolicy -{ - Int32 min_snapshots_to_keep = Iceberg::default_min_snapshots_to_keep; - Int64 max_snapshot_age_ms = Iceberg::default_max_snapshot_age_ms; - Int64 max_ref_age_ms = Iceberg::default_max_ref_age_ms; -}; - -static RetentionPolicy readRetentionPolicy(const Poco::JSON::Object::Ptr & metadata, ContextPtr context, const ExpireSnapshotsOptions & options) -{ - RetentionPolicy policy; - const auto & settings = context->getSettingsRef(); - Int64 min_keep_from_settings = settings[Setting::iceberg_expire_default_min_snapshots_to_keep].value; - Int64 max_snapshot_age_from_settings = settings[Setting::iceberg_expire_default_max_snapshot_age_ms].value; - Int64 max_ref_age_from_settings = settings[Setting::iceberg_expire_default_max_ref_age_ms].value; - - if (min_keep_from_settings <= 0 || min_keep_from_settings > std::numeric_limits::max()) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "iceberg_expire_default_min_snapshots_to_keep must be in range [1, {}], got {}", - std::numeric_limits::max(), - min_keep_from_settings); - if (max_snapshot_age_from_settings < 0) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "iceberg_expire_default_max_snapshot_age_ms must be non-negative, got {}", - max_snapshot_age_from_settings); - if (max_ref_age_from_settings < 0) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "iceberg_expire_default_max_ref_age_ms must be non-negative, got {}", - max_ref_age_from_settings); - - policy.min_snapshots_to_keep = static_cast(min_keep_from_settings); - policy.max_snapshot_age_ms = max_snapshot_age_from_settings; - policy.max_ref_age_ms = max_ref_age_from_settings; - - if (metadata->has(Iceberg::f_properties)) - { - auto props = metadata->getObject(Iceberg::f_properties); - if (props->has(Iceberg::f_min_snapshots_to_keep)) - policy.min_snapshots_to_keep = std::stoi(props->getValue(Iceberg::f_min_snapshots_to_keep)); - if (props->has(Iceberg::f_max_snapshot_age_ms)) - policy.max_snapshot_age_ms = std::stoll(props->getValue(Iceberg::f_max_snapshot_age_ms)); - if (props->has(Iceberg::f_max_ref_age_ms)) - policy.max_ref_age_ms = std::stoll(props->getValue(Iceberg::f_max_ref_age_ms)); - } - - /// Per-invocation overrides (only affect table-level defaults, not per-ref overrides). - if (options.retain_last.has_value()) - policy.min_snapshots_to_keep = *options.retain_last; - if (options.retention_period_ms.has_value()) - policy.max_snapshot_age_ms = *options.retention_period_ms; - - return policy; -} - -/// Snapshot parent graph built from metadata, used for branch ancestor traversal. -class SnapshotGraph -{ -public: - explicit SnapshotGraph(const Poco::JSON::Array::Ptr & snapshots) - { - for (UInt32 i = 0; i < snapshots->size(); ++i) - { - auto snapshot = snapshots->getObject(i); - Int64 snap_id = snapshot->getValue(Iceberg::f_metadata_snapshot_id); - timestamps[snap_id] = snapshot->getValue(Iceberg::f_timestamp_ms); - if (snapshot->has(Iceberg::f_parent_snapshot_id) && !snapshot->isNull(Iceberg::f_parent_snapshot_id)) - parent_chain[snap_id] = snapshot->getValue(Iceberg::f_parent_snapshot_id); - } - } - - bool hasSnapshot(Int64 snap_id) const { return timestamps.contains(snap_id); } - - Int64 getTimestamp(Int64 snap_id) const { return timestamps.at(snap_id); } - - std::optional getParent(Int64 snap_id) const - { - auto it = parent_chain.find(snap_id); - return it != parent_chain.end() ? std::optional(it->second) : std::nullopt; - } - - /// Retain ancestors from head_id while min-keep or max-age is satisfied. - void walkBranchAncestors(Int64 now_ms, Int64 head_id, Int32 min_keep, Int64 max_age_ms, std::set & retained) const - { - Int64 walk_id = head_id; - Int32 count = 0; - while (hasSnapshot(walk_id)) - { - bool within_min_keep = (count < min_keep); - bool within_max_age = (now_ms - getTimestamp(walk_id) <= max_age_ms); - if (!within_min_keep && !within_max_age) - break; - retained.insert(walk_id); - ++count; - auto parent = getParent(walk_id); - if (!parent) - break; - walk_id = *parent; - } - } - -private: - std::unordered_map parent_chain; - std::unordered_map timestamps; -}; - -/// Apply Iceberg Snapshot Retention Policy. Returns (retained IDs, expired ref names). -static std::pair, Strings> applyRetentionPolicy( - const Poco::JSON::Object::Ptr & metadata, - Int64 current_snapshot_id, - const SnapshotGraph & graph, - const RetentionPolicy & policy, - Int64 now_ms) -{ - std::set retained; - Strings expired_ref_names; - bool main_branch_walked = false; - if (metadata->has(Iceberg::f_refs)) - { - auto refs = metadata->getObject(Iceberg::f_refs); - for (const auto & ref_name : refs->getNames()) - { - auto ref_obj = refs->getObject(ref_name); - Int64 ref_snap_id = ref_obj->getValue(Iceberg::f_metadata_snapshot_id); - String ref_type = ref_obj->getValue(Iceberg::f_type); - - Int64 ref_max_ref_age = ref_obj->has(Iceberg::f_ref_max_ref_age_ms) - ? ref_obj->getValue(Iceberg::f_ref_max_ref_age_ms) - : policy.max_ref_age_ms; - - bool is_main = (ref_name == Iceberg::f_main); - - if (!is_main && !graph.hasSnapshot(ref_snap_id)) - { - LOG_WARNING(getLogger("IcebergExpireSnapshots"), - "Removing invalid ref {}: snapshot {} does not exist", ref_name, ref_snap_id); - expired_ref_names.push_back(ref_name); - continue; - } - - bool ref_expired = !is_main && (now_ms - graph.getTimestamp(ref_snap_id)) > ref_max_ref_age; - - if (ref_expired) - { - expired_ref_names.push_back(ref_name); - continue; - } - - if (ref_type == Iceberg::f_branch) - { - Int32 min_keep = ref_obj->has(Iceberg::f_ref_min_snapshots_to_keep) - ? ref_obj->getValue(Iceberg::f_ref_min_snapshots_to_keep) - : policy.min_snapshots_to_keep; - Int64 max_age = ref_obj->has(Iceberg::f_ref_max_snapshot_age_ms) - ? ref_obj->getValue(Iceberg::f_ref_max_snapshot_age_ms) - : policy.max_snapshot_age_ms; - graph.walkBranchAncestors(now_ms, ref_snap_id, min_keep, max_age, retained); - if (is_main) - main_branch_walked = true; - } - else if (ref_type == Iceberg::f_tag) - { - retained.insert(ref_snap_id); - } - else - { - UNREACHABLE(); - } - } - } - - if (!main_branch_walked) - graph.walkBranchAncestors(now_ms, current_snapshot_id, policy.min_snapshots_to_keep, policy.max_snapshot_age_ms, retained); - - return {retained, expired_ref_names}; -} - -static void collectAllFilePaths( - const Iceberg::ManifestFileIterator::ManifestFileEntriesHandle & entries_handle, - std::set & out) -{ - for (const auto & entry : entries_handle.getFilesWithoutDeleted(FileContentType::DATA)) - out.insert(entry->parsed_entry->file_path_key); - for (const auto & entry : entries_handle.getFilesWithoutDeleted(FileContentType::POSITION_DELETE)) - out.insert(entry->parsed_entry->file_path_key); - for (const auto & entry : entries_handle.getFilesWithoutDeleted(FileContentType::EQUALITY_DELETE)) - out.insert(entry->parsed_entry->file_path_key); -} - -/// Collect all file paths (manifest lists, manifests, data/delete files) -/// referenced by retained snapshots. -/// -/// NOTE: We only collect files with status ADDED/EXISTING (via getFilesWithoutDeleted). -/// Files with status DELETED are being removed by that snapshot and don't need retention -/// from it. A DELETED entry's data file was ADDED in an earlier snapshot — if that snapshot -/// is retained, the file is in the retained set from there; if expired, it will be collected -/// for cleanup from that snapshot's ADDED/EXISTING entries. -/// -/// TODO: To handle partially-failed prior expire_snapshots (where the ADDED snapshot -/// was removed but its data files were not cleaned up), we could also traverse DELETED -/// entries in expired manifests. This requires extending ManifestFileIterator to expose -/// DELETED entries. -static void collectRetainedFiles( - const Poco::JSON::Array::Ptr & retained_snapshots, - ObjectStoragePtr object_storage, - PersistentTableComponents & persistent_table_components, - ContextPtr context, - LoggerPtr log, - Int32 current_schema_id, - std::set & retained_manifest_paths, - std::set & retained_data_file_paths, - std::set & retained_manifest_list_paths) -{ - SecondaryStorages secondary_storages; - for (UInt32 i = 0; i < retained_snapshots->size(); ++i) - { - auto snapshot = retained_snapshots->getObject(i); - if (!snapshot->has(Iceberg::f_manifest_list)) - continue; - - auto manifest_list_path = IcebergPathFromMetadata::deserialize(snapshot->getValue(Iceberg::f_manifest_list)); - retained_manifest_list_paths.insert(manifest_list_path); - - auto manifest_keys = getManifestList(object_storage, persistent_table_components, context, manifest_list_path, log, secondary_storages); - - for (const auto & mf_key : manifest_keys) - { - retained_manifest_paths.insert(mf_key.manifest_file_path); - auto entries_handle = getManifestFileEntriesHandle( - object_storage, persistent_table_components, context, log, - mf_key, current_schema_id, secondary_storages); - collectAllFilePaths(entries_handle, retained_data_file_paths); - } - } -} - -struct ExpiredFiles -{ - std::vector all_paths; - Int64 data_files = 0; - Int64 position_delete_files = 0; - Int64 equality_delete_files = 0; - Int64 manifest_files = 0; - Int64 manifest_lists = 0; -}; - -/// Collect files from expired snapshots that are not referenced by any retained snapshot. -static ExpiredFiles collectExpiredFiles( - const std::vector & expired_manifest_list_paths, - const std::set & retained_manifest_list_paths, - const std::set & retained_manifest_paths, - const std::set & retained_data_file_paths, - ObjectStoragePtr object_storage, - PersistentTableComponents & persistent_table_components, - ContextPtr context, - LoggerPtr log, - Int32 current_schema_id) -{ - ExpiredFiles result; - SecondaryStorages secondary_storages; - std::set seen_expired_manifest_list_paths; - std::set seen_expired_manifest_paths; - for (const auto & manifest_list_path : expired_manifest_list_paths) - { - if (retained_manifest_list_paths.contains(manifest_list_path)) - continue; - - if (seen_expired_manifest_list_paths.contains(manifest_list_path)) - continue; - - ManifestFileCacheKeys manifest_keys; - try - { - manifest_keys = getManifestList(object_storage, persistent_table_components, context, manifest_list_path, log, secondary_storages); - } - catch (...) - { - LOG_WARNING(log, "Failed to read manifest list {}, skipping", manifest_list_path); - continue; - } - - for (const auto & mf_key : manifest_keys) - { - if (retained_manifest_paths.contains(mf_key.manifest_file_path)) - continue; - - if (seen_expired_manifest_paths.contains(mf_key.manifest_file_path)) - continue; - - try - { - auto entries_handle = getManifestFileEntriesHandle( - object_storage, persistent_table_components, context, log, - mf_key, current_schema_id, secondary_storages); - - for (const auto & entry : entries_handle.getFilesWithoutDeleted(FileContentType::DATA)) - if (!retained_data_file_paths.contains(entry->parsed_entry->file_path_key)) - { - result.all_paths.push_back(entry->parsed_entry->file_path_key); - ++result.data_files; - } - for (const auto & entry : entries_handle.getFilesWithoutDeleted(FileContentType::POSITION_DELETE)) - if (!retained_data_file_paths.contains(entry->parsed_entry->file_path_key)) - { - result.all_paths.push_back(entry->parsed_entry->file_path_key); - ++result.position_delete_files; - } - for (const auto & entry : entries_handle.getFilesWithoutDeleted(FileContentType::EQUALITY_DELETE)) - if (!retained_data_file_paths.contains(entry->parsed_entry->file_path_key)) - { - result.all_paths.push_back(entry->parsed_entry->file_path_key); - ++result.equality_delete_files; - } - } - catch (...) - { - LOG_WARNING(log, "Failed to read manifest file {}, skipping", mf_key.manifest_file_path); - continue; - } - - seen_expired_manifest_paths.insert(mf_key.manifest_file_path); - result.all_paths.push_back(mf_key.manifest_file_path); - ++result.manifest_files; - } - - seen_expired_manifest_list_paths.insert(manifest_list_path); - result.all_paths.push_back(manifest_list_path); - ++result.manifest_lists; - } - return result; -} - -/// Trim snapshot-log to the suffix of entries referencing only retained snapshots. -static void trimSnapshotLog( - Poco::JSON::Object::Ptr metadata, - const std::set & expired_snapshot_ids) -{ - if (!metadata->has(Iceberg::f_snapshot_log)) - return; - - auto snapshot_log = metadata->get(Iceberg::f_snapshot_log).extract(); - Int32 suffix_start = static_cast(snapshot_log->size()); - for (Int32 j = static_cast(snapshot_log->size()) - 1; j >= 0; --j) - { - auto entry = snapshot_log->getObject(static_cast(j)); - Int64 snap_id = entry->getValue(Iceberg::f_metadata_snapshot_id); - if (expired_snapshot_ids.contains(snap_id)) - break; - suffix_start = j; - } - Poco::JSON::Array::Ptr retained_log = new Poco::JSON::Array; - for (UInt32 j = static_cast(suffix_start); j < snapshot_log->size(); ++j) - retained_log->add(snapshot_log->getObject(j)); - metadata->set(Iceberg::f_snapshot_log, retained_log); -} - -struct SnapshotPartition -{ - Poco::JSON::Array::Ptr retained_snapshots = new Poco::JSON::Array; - std::set expired_snapshot_ids; - std::vector expired_manifest_list_paths; -}; - -/// Split snapshots into retained and expired. -/// A snapshot is retained if the retention policy selected it, or if the -/// user-provided fuse timestamp protects it (snapshot newer than fuse). -static SnapshotPartition partitionSnapshots( - const Poco::JSON::Array::Ptr & snapshots, - const std::set & retention_retained_ids, - std::optional expire_before_ms) -{ - SnapshotPartition result; - for (UInt32 i = 0; i < snapshots->size(); ++i) - { - auto snapshot = snapshots->getObject(i); - Int64 snap_id = snapshot->getValue(Iceberg::f_metadata_snapshot_id); - Int64 snap_ts = snapshot->getValue(Iceberg::f_timestamp_ms); - - bool is_retained_by_policy = retention_retained_ids.contains(snap_id); - bool is_protected_by_fuse = expire_before_ms.has_value() && (snap_ts >= *expire_before_ms); - - if (is_retained_by_policy || is_protected_by_fuse) - { - result.retained_snapshots->add(snapshot); - } - else - { - result.expired_snapshot_ids.insert(snap_id); - if (snapshot->has(Iceberg::f_manifest_list)) - result.expired_manifest_list_paths.push_back( - Iceberg::IcebergPathFromMetadata::deserialize(snapshot->getValue(Iceberg::f_manifest_list))); - } - } - return result; -} - -static SnapshotPartition partitionSnapshotsByIds( - const Poco::JSON::Object::Ptr & metadata, - const Poco::JSON::Array::Ptr & snapshots, - const std::vector & snapshot_ids, - Int64 current_snapshot_id, - std::optional expire_before_ms) -{ - std::unordered_set requested_ids(snapshot_ids.begin(), snapshot_ids.end()); - std::unordered_set existing_ids; - std::unordered_set ref_protected_ids; - SnapshotPartition result; - - if (metadata->has(Iceberg::f_refs)) - { - auto refs = metadata->getObject(Iceberg::f_refs); - for (const auto & ref_name : refs->getNames()) - { - auto ref = refs->getObject(ref_name); - if (ref->has(Iceberg::f_metadata_snapshot_id)) - ref_protected_ids.insert(ref->getValue(Iceberg::f_metadata_snapshot_id)); - } - } - - ref_protected_ids.insert(current_snapshot_id); - - for (UInt32 i = 0; i < snapshots->size(); ++i) - { - auto snapshot = snapshots->getObject(i); - Int64 snap_id = snapshot->getValue(Iceberg::f_metadata_snapshot_id); - Int64 snap_ts = snapshot->getValue(Iceberg::f_timestamp_ms); - - existing_ids.insert(snap_id); - bool requested = requested_ids.contains(snap_id); - bool is_protected_by_fuse = expire_before_ms.has_value() && (snap_ts >= *expire_before_ms); - - if (requested && ref_protected_ids.contains(snap_id)) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "expire_snapshots cannot expire snapshot {} because it is referenced by current snapshot, branch, or tag", - snap_id); - - if (requested && !is_protected_by_fuse) - { - result.expired_snapshot_ids.insert(snap_id); - if (snapshot->has(Iceberg::f_manifest_list)) - result.expired_manifest_list_paths.push_back(Iceberg::IcebergPathFromMetadata::deserialize(snapshot->getValue(Iceberg::f_manifest_list))); - } - else - { - result.retained_snapshots->add(snapshot); - } - } - - for (Int64 requested_id : requested_ids) - { - if (!existing_ids.contains(requested_id)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "expire_snapshots snapshot id {} does not exist", requested_id); - } - - return result; -} - -/// Mutate metadata: remove expired refs, update snapshots, trim log, bump timestamp. -static void updateMetadataForExpiration( - Poco::JSON::Object::Ptr metadata, - const Strings & expired_ref_names, - const Poco::JSON::Array::Ptr & retained_snapshots, - const std::set & expired_snapshot_ids) -{ - for (const auto & ref_name : expired_ref_names) - metadata->getObject(Iceberg::f_refs)->remove(ref_name); - - metadata->set(Iceberg::f_snapshots, retained_snapshots); - trimSnapshotLog(metadata, expired_snapshot_ids); - - auto now = std::chrono::system_clock::now(); - auto ms = duration_cast(now.time_since_epoch()); - metadata->set(Iceberg::f_last_updated_ms, ms.count()); -} - -static void deleteExpiredFiles( - const std::vector & files_to_delete, - const Iceberg::IcebergPathResolver & path_resolver, - ObjectStoragePtr object_storage, - LoggerPtr log) -{ - for (const auto & file_path : files_to_delete) - { - try - { - object_storage->removeObjectIfExists(StoredObject(path_resolver.resolve(file_path))); - LOG_DEBUG(log, "Deleted expired file {}", file_path); - } - catch (...) - { - LOG_WARNING(log, "Failed to delete file {}: {}", file_path, getCurrentExceptionMessage(false)); - } - } -} - -/// Expire old Iceberg snapshots following the spec's Snapshot Retention Policy. -/// -/// The process: -/// 1. Read retention policy from table properties (with spec defaults). -/// 2. Build the snapshot parent graph and determine which snapshots to retain -/// based on branch/tag refs and their min-snapshots-to-keep / max-snapshot-age-ms. -/// 3. If the caller provided expire_before_ms, it acts as an additional safety -/// fuse — snapshots newer than this timestamp are never expired regardless -/// of retention policy. -/// 4. Collect files exclusively owned by expired snapshots and delete them. -/// 5. Write updated metadata with optimistic concurrency (retry on conflict). -ExpireSnapshotsResult expireSnapshots( - const ExpireSnapshotsOptions & options, - ContextPtr context, - ObjectStoragePtr object_storage, - const DataLakeStorageSettings & data_lake_settings, - PersistentTableComponents & persistent_table_components, - const String & write_format, - std::shared_ptr catalog, - const String & table_name) -{ - auto common_path = persistent_table_components.table_path; - if (!common_path.starts_with('/')) - common_path = "/" + common_path; - - int max_retries = MAX_TRANSACTION_RETRIES; - while (--max_retries > 0) - { - FileNamesGenerator filename_generator(persistent_table_components.path_resolver.getTableLocation(), false, CompressionMethod::None, write_format); - auto log = getLogger("IcebergExpireSnapshots"); - auto [last_version, metadata_path, compression_method] = getLatestOrExplicitMetadataFileAndVersion( - object_storage, - persistent_table_components.table_path, - data_lake_settings, - persistent_table_components.metadata_cache, - context, - log.get(), - persistent_table_components.table_uuid, - persistent_table_components.metadata_compression_method, - /* force_fetch_latest_metadata */ true, - /* ignore_explicit_metadata_file_path */ true); - - filename_generator.setVersion(last_version + 1); - filename_generator.setCompressionMethod(compression_method); - - auto metadata = getMetadataJSONObject( - metadata_path, - object_storage, - persistent_table_components.metadata_cache, - context, - log, - compression_method, - persistent_table_components.table_uuid); - - if (metadata->getValue(f_format_version) < 2) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "expire_snapshots is supported only for the second version of iceberg format"); - - if (!metadata->has(Iceberg::f_current_snapshot_id)) - { - LOG_INFO(log, "No snapshots to expire (table has no current snapshot)"); - return {.dry_run = options.dry_run}; - } - - Int64 current_snapshot_id = metadata->getValue(Iceberg::f_current_snapshot_id); - if (current_snapshot_id < 0) - { - LOG_INFO(log, "No snapshots to expire (table has no current snapshot)"); - return {.dry_run = options.dry_run}; - } - - auto snapshots = metadata->get(Iceberg::f_snapshots).extract(); - auto now_ms = duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); - - Strings expired_ref_names; - SnapshotPartition partition; - if (options.snapshot_ids.has_value()) - { - partition = partitionSnapshotsByIds(metadata, snapshots, *options.snapshot_ids, current_snapshot_id, options.expire_before_ms); - } - else - { - auto policy = readRetentionPolicy(metadata, context, options); - SnapshotGraph graph(snapshots); - auto [retention_retained_ids, retention_expired_ref_names] = applyRetentionPolicy(metadata, current_snapshot_id, graph, policy, now_ms); - expired_ref_names = std::move(retention_expired_ref_names); - partition = partitionSnapshots(snapshots, retention_retained_ids, options.expire_before_ms); - } - - if (partition.expired_snapshot_ids.empty()) - { - LOG_INFO(log, "No snapshots to expire"); - return {.dry_run = options.dry_run}; - } - LOG_INFO(log, "Expiring {} snapshots", partition.expired_snapshot_ids.size()); - - Int32 current_schema_id = metadata->getValue(Iceberg::f_current_schema_id); - - std::set retained_manifest_paths; - std::set retained_data_file_paths; - std::set retained_manifest_list_paths; - collectRetainedFiles( - partition.retained_snapshots, object_storage, persistent_table_components, context, log, - current_schema_id, retained_manifest_paths, retained_data_file_paths, retained_manifest_list_paths); - auto expired_files = collectExpiredFiles( - partition.expired_manifest_list_paths, retained_manifest_list_paths, retained_manifest_paths, retained_data_file_paths, - object_storage, persistent_table_components, context, log, current_schema_id); - - if (options.dry_run) - { - LOG_INFO(log, "Dry-run mode: skip metadata commit and file deletion"); - return ExpireSnapshotsResult{ - .deleted_data_files_count = expired_files.data_files, - .deleted_position_delete_files_count = expired_files.position_delete_files, - .deleted_equality_delete_files_count = expired_files.equality_delete_files, - .deleted_manifest_files_count = expired_files.manifest_files, - .deleted_manifest_lists_count = expired_files.manifest_lists, - .dry_run = true, - }; - } - - updateMetadataForExpiration(metadata, expired_ref_names, partition.retained_snapshots, partition.expired_snapshot_ids); - - std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM - Poco::JSON::Stringifier::stringify(metadata, oss, 4); - std::string json_representation = removeEscapedSlashes(oss.str()); - auto metadata_info = filename_generator.generateMetadataPathWithInfo(); - auto hint_path = filename_generator.generateVersionHint(); - if (!writeMetadataFileAndVersionHint( ->>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) persistent_table_components.path_resolver, metadata_info, json_representation, diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/PositionDeleteTransform.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/PositionDeleteTransform.cpp index 4c044e1bcf9c..d7a4afac7f8b 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/PositionDeleteTransform.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/PositionDeleteTransform.cpp @@ -74,10 +74,6 @@ void IcebergPositionDeleteTransform::initializeDeleteSources() { continue; } -<<<<<<< HEAD - -======= ->>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) auto [delete_storage_to_use, resolved_key] = resolveObjectStorageForPath( path_resolver.getTableLocation(), position_deletes_object.file_path, object_storage, *secondary_storages, context, diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/SnapshotFilesTraversal.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/SnapshotFilesTraversal.cpp index 572bf495340e..59ba5b8ccb50 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/SnapshotFilesTraversal.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/SnapshotFilesTraversal.cpp @@ -25,6 +25,7 @@ SnapshotReferencedFiles collectSnapshotReferencedFiles( Int32 current_schema_id) { SnapshotReferencedFiles files; + SecondaryStorages secondary_storages; for (UInt32 i = 0; i < snapshots->size(); ++i) { @@ -36,14 +37,14 @@ SnapshotReferencedFiles collectSnapshotReferencedFiles( files.manifest_list_paths.insert(manifest_list_path); auto manifest_keys = getManifestList( - object_storage, persistent_table_components, context, manifest_list_path, log); + object_storage, persistent_table_components, context, manifest_list_path, log, secondary_storages); for (const auto & manifest_entry : manifest_keys) { files.manifest_paths.insert(manifest_entry.manifest_file_path); auto entries_handle = getManifestFileEntriesHandle( - object_storage, persistent_table_components, context, log, manifest_entry, current_schema_id); + object_storage, persistent_table_components, context, log, manifest_entry, current_schema_id, secondary_storages); for (const auto & entry : entries_handle.getFilesWithoutDeleted(FileContentType::DATA)) files.data_file_paths.insert(entry->parsed_entry->file_path_key); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h index eac24bc2ee28..7e5a508d103e 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h @@ -1,15 +1,8 @@ #pragma once -<<<<<<< HEAD -#include "config.h" - -#if USE_AVRO - -======= #include #include #include "config.h" ->>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) #include #include #include @@ -22,8 +15,6 @@ #include #include -<<<<<<< HEAD -======= namespace DB { @@ -37,7 +28,6 @@ std::optional getMetadataPathFromObjectInfo([[maybe_unused]] const Objec #if USE_AVRO ->>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) #include #include #include diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index 0e69558dd65c..74fc72040e87 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -51,11 +51,7 @@ #include #include #include -<<<<<<< HEAD -======= -#include #include ->>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) #if ENABLE_DISTRIBUTED_CACHE #include #include @@ -450,12 +446,8 @@ Chunk StorageObjectStorageSource::generate() chunk, read_from_format_info.requested_virtual_columns, { -<<<<<<< HEAD - .path = path, - .storage_id = storage_snapshot->storage.getStorageID(), -======= .path = path_for_virtual_column, ->>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) + .storage_id = storage_snapshot->storage.getStorageID(), .size = object_info->isArchive() ? object_info->fileSizeInArchive() : object_metadata->size_bytes, .filename = &filename, .last_modified = object_metadata->last_modified, @@ -805,18 +797,13 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade } else { -<<<<<<< HEAD ProfileEvents::increment(ProfileEvents::ObjectStorageReadObjects); compression_method = chooseCompressionMethod(object_info->getFileName(), configuration->compression_method); - read_buf = createReadBuffer(object_info->relative_path_with_metadata, object_storage, context_, log); -======= - compression_method = chooseCompressionMethod(object_info->getFileName(), configuration->getCompressionMethod()); read_buf = createReadBuffer( object_info->relative_path_with_metadata, getResolvedStorageFromObjectInfo(object_info, object_storage), context_, log); ->>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) } Block initial_header = read_from_format_info.format_header; @@ -1665,23 +1652,12 @@ ObjectInfoPtr StorageObjectStorageSource::ReadTaskIterator::next(size_t) ObjectInfoPtr object_info; if (current_index >= buffer.size()) { -<<<<<<< HEAD - auto task = callback(); + auto raw = callback(); if (auto query_status = getContext()->getProcessListElement()) query_status->checkTimeLimit(); - if (!task || task->isEmpty()) -======= - if (!getContext()->isSwarmModeEnabled()) - { - LOG_DEBUG(getLogger("StorageObjectStorageSource"), "STOP SWARM MODE called, stop getting new tasks"); - return nullptr; - } - - auto raw = callback(); if (!raw || raw->isEmpty()) ->>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) return nullptr; object_info = raw->getObjectInfo(); diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.h b/src/Storages/ObjectStorage/StorageObjectStorageSource.h index c76d6ff86699..dcb72d335a02 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.h +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.h @@ -11,11 +11,7 @@ #include #include #include -<<<<<<< HEAD -======= #include -#include ->>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) #include namespace DB diff --git a/src/Storages/ObjectStorage/StorageObjectStorageStableTaskDistributor.cpp b/src/Storages/ObjectStorage/StorageObjectStorageStableTaskDistributor.cpp index 1f5003be3baf..7a13e7e1da3a 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageStableTaskDistributor.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageStableTaskDistributor.cpp @@ -83,7 +83,11 @@ ObjectInfoPtr StorageObjectStorageStableTaskDistributor::getPreQueuedFile(size_t auto next_file = files.back(); files.pop_back(); - auto file_identifier = send_over_whole_archive ? next_file->getPathOrPathToArchiveIfArchive() : next_file->getIdentifier(); + /// Prefer the original Iceberg metadata path (possibly absolute / on another storage) when available, + /// so the same file is identified consistently across replicas. + auto file_identifier = send_over_whole_archive + ? next_file->getPathOrPathToArchiveIfArchive() + : getMetadataPathFromObjectInfo(next_file).value_or(next_file->getIdentifier()); auto it = unprocessed_files.find(file_identifier); if (it == unprocessed_files.end()) continue; @@ -137,7 +141,9 @@ ObjectInfoPtr StorageObjectStorageStableTaskDistributor::getMatchingFileFromIter } else { - file_identifier = object_info->getIdentifier(); + /// Prefer the original Iceberg metadata path (possibly absolute / on another storage) when available, + /// so the same file is identified consistently across replicas. + file_identifier = getMetadataPathFromObjectInfo(object_info).value_or(object_info->getIdentifier()); } size_t file_replica_idx = getReplicaForFile(file_identifier); @@ -193,62 +199,4 @@ ObjectInfoPtr StorageObjectStorageStableTaskDistributor::getAnyUnprocessedFile(s return {}; } -<<<<<<< HEAD -======= -void StorageObjectStorageStableTaskDistributor::saveLastNodeActivity(size_t number_of_current_replica) -{ - Poco::Timestamp now; - std::lock_guard lock(mutex); - last_node_activity[number_of_current_replica] = now; -} - -void StorageObjectStorageStableTaskDistributor::rescheduleTasksFromReplica(size_t number_of_current_replica) -{ - LOG_INFO(log, "Replica {} is marked as lost, tasks are returned to queue", number_of_current_replica); - std::lock_guard lock(mutex); - - auto processed_file_list_ptr = replica_to_files_to_be_processed.find(number_of_current_replica); - if (processed_file_list_ptr == replica_to_files_to_be_processed.end()) - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Replica number {} was marked as lost already", - number_of_current_replica - ); - - if (replica_to_files_to_be_processed.size() < 2) - throw Exception( - ErrorCodes::CANNOT_READ_ALL_DATA, - "All replicas were marked as lost" - ); - - auto files = std::move(processed_file_list_ptr->second); - replica_to_files_to_be_processed.erase(number_of_current_replica); - for (const auto & file : files) - { - auto file_identifier = getFileIdentifier(file); - auto file_replica_idx = getReplicaForFile(file_identifier); - unprocessed_files.emplace(file_identifier, std::make_pair(file, file_replica_idx)); - connection_to_files[file_replica_idx].push_back(file); - } -} - -String StorageObjectStorageStableTaskDistributor::getFileIdentifier(ObjectInfoPtr file_object, bool write_to_log) const -{ - if (send_over_whole_archive && file_object->isArchive()) - { - auto file_identifier = file_object->getPathOrPathToArchiveIfArchive(); - if (write_to_log) - { - LOG_TEST(log, "Will send over the whole archive {} to replicas. " - "This will be suboptimal, consider turning on " - "cluster_function_process_archive_on_multiple_nodes setting", file_identifier); - } - return file_identifier; - } - /// Prefer the original Iceberg metadata path (possibly absolute / on another storage) when available, - /// so the same file is identified consistently across replicas. - return getMetadataPathFromObjectInfo(file_object).value_or(file_object->getIdentifier()); -} - ->>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) } diff --git a/src/Storages/ObjectStorage/Utils.cpp b/src/Storages/ObjectStorage/Utils.cpp index d1681a4e08fa..757131412660 100644 --- a/src/Storages/ObjectStorage/Utils.cpp +++ b/src/Storages/ObjectStorage/Utils.cpp @@ -568,6 +568,7 @@ std::optional> tryResolveObjectStor /// percent-decoding so that keys like `col=12%3A00%3A00` are preserved as-is. S3::URI s3_uri(normalized_path, /*allow_archive_path_syntax*/ false, /*keep_presigned_query_parameters*/ true, + /*uri_style*/ S3UriStyle::AUTO, /*enable_url_encoding*/ false); std::string key_to_use = s3_uri.key; @@ -598,6 +599,7 @@ std::optional> tryResolveObjectStor } S3::URI base_s3_uri(normalized_table_location, /*allow_archive_path_syntax*/ false, /*keep_presigned_query_parameters*/ true, + /*uri_style*/ S3UriStyle::AUTO, /*enable_url_encoding*/ false); if (s3URIMatches(s3_uri, base_s3_uri.bucket, base_s3_uri.endpoint, target_scheme_normalized)) diff --git a/src/Storages/ObjectStorage/Utils.h b/src/Storages/ObjectStorage/Utils.h index ee201c06cd0b..9a9cbb00dfd4 100644 --- a/src/Storages/ObjectStorage/Utils.h +++ b/src/Storages/ObjectStorage/Utils.h @@ -92,12 +92,10 @@ struct ParseFromDiskResult ParseFromDiskResult parseFromDisk(ASTs args, bool with_structure, ContextPtr context, const fs::path & prefix); -<<<<<<< HEAD void expandPaimonKeeperMacrosIfNeeded( const StorageFactory::Arguments & args, const DataLakeStorageSettingsPtr & storage_settings); -======= #if USE_AVRO namespace Iceberg { class IcebergPathResolver; } @@ -121,6 +119,5 @@ std::pair resolveObjectStorageForPath( const DB::ContextPtr & context, const Iceberg::IcebergPathResolver & path_resolver); #endif ->>>>>>> 6a83f974ee6 (Merge pull request #1859 from Altinity/feat/antalya-26.3/90740) }