From 1715051300ab9ea124f04636d293dbf3309c8620 Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Sat, 8 Aug 2026 23:47:12 +0200 Subject: [PATCH 1/2] Port of iceberg compaction from upstream --- .../table-engines/integrations/iceberg.md | 23 +- src/Core/Settings.cpp | 5 + src/Core/SettingsChangesHistory.cpp | 40 + src/Interpreters/InterpreterOptimizeQuery.cpp | 30 + src/Parsers/ASTOptimizeQuery.cpp | 3 + src/Parsers/ASTOptimizeQuery.h | 4 +- src/Parsers/CommonParsers.h | 1 + src/Parsers/ParserOptimizeQuery.cpp | 6 + .../Common/AvroForIcebergDeserializer.cpp | 13 + .../DataLakes/Iceberg/Compaction.cpp | 685 ++++++- .../DataLakes/Iceberg/Compaction.h | 10 + .../DataLakes/Iceberg/Constant.h | 8 + .../DataLakes/Iceberg/IcebergMetadata.cpp | 37 + .../DataLakes/Iceberg/IcebergMetadata.h | 5 + .../Iceberg/IcebergMetadataFilesCache.h | 2 + .../DataLakes/Iceberg/IcebergWrites.cpp | 302 ++- .../DataLakes/Iceberg/IcebergWrites.h | 48 +- .../DataLakes/Iceberg/ManifestFile.h | 3 + .../DataLakes/Iceberg/MetadataGenerator.cpp | 179 +- .../DataLakes/Iceberg/MetadataGenerator.h | 10 +- .../Iceberg/StatelessMetadataFileGetter.cpp | 11 +- .../ObjectStorage/StorageObjectStorage.h | 2 + .../integration/test_database_iceberg/test.py | 103 + .../test_manifest_compaction.py | 1761 +++++++++++++++++ .../test_storage_iceberg_with_trino/test.py | 73 + 25 files changed, 3282 insertions(+), 82 deletions(-) create mode 100644 tests/integration/test_storage_iceberg_with_spark/test_manifest_compaction.py diff --git a/docs/en/engines/table-engines/integrations/iceberg.md b/docs/en/engines/table-engines/integrations/iceberg.md index 7a93d3f18c35..9b431506b9c4 100644 --- a/docs/en/engines/table-engines/integrations/iceberg.md +++ b/docs/en/engines/table-engines/integrations/iceberg.md @@ -1,6 +1,6 @@ --- -description: 'This engine provides a read-only integration with existing Apache Iceberg - tables in Amazon S3, Azure, HDFS and locally stored tables.' +description: 'This engine provides a read-only data integration with existing Apache Iceberg + tables in Amazon S3, Azure, HDFS and locally stored tables, plus experimental metadata-maintenance writes.' sidebar_label: 'Iceberg' sidebar_position: 90 slug: /engines/table-engines/integrations/iceberg @@ -16,7 +16,7 @@ The Iceberg Table Engine is available but may have limitations. ClickHouse wasn' For optimal compatibility, we suggest using the Iceberg Table Function while we continue to improve support for the Iceberg Table Engine. ::: -This engine provides a read-only integration with existing Apache [Iceberg](https://iceberg.apache.org/) tables in Amazon S3, Azure, HDFS and locally stored tables. +This engine provides a read-only *data* integration with existing Apache [Iceberg](https://iceberg.apache.org/) tables in Amazon S3, Azure, HDFS and locally stored tables. ## Create table {#create-table} @@ -125,6 +125,23 @@ ClickHouse supports partition pruning during SELECT queries for Iceberg tables, ClickHouse supports time travel for Iceberg tables, allowing you to query historical data with a specific timestamp or snapshot ID. +## Manifest file compaction {#manifest-compaction} + +Over time, frequent writes to an Iceberg table can accumulate a large number of small manifest files in the current snapshot's manifest list. A long manifest list slows down query planning, because every manifest file has to be read to discover the data files. ClickHouse can compact these manifest files into fewer, larger ones using the `OPTIMIZE TABLE ... MANIFEST` statement: + +```sql +OPTIMIZE TABLE example_table MANIFEST SETTINGS allow_experimental_iceberg_compaction = 1; +``` + +This produces a new snapshot (a `replace` operation) that references the same data files through a consolidated set of manifest files. No data files are rewritten and no rows are added, deleted, or deduplicated — only the manifest layer is rearranged. + +### Requirements and behavior {#manifest-compaction-behavior} + +- The feature is experimental and gated behind the `allow_experimental_iceberg_compaction` setting. The statement throws an exception if the setting is not enabled. +- Compaction is only attempted when the number of manifest files in the current snapshot's manifest list exceeds the threshold given by the `iceberg_manifest_min_count_to_compact` setting (default `30`). If the current count is less than or equal to the threshold, compaction is skipped and no new snapshot is created. Set the threshold lower to compact more eagerly. +- `OPTIMIZE TABLE ... MANIFEST` is supported only for Iceberg tables. Running it against any other table engine throws an exception. +- `OPTIMIZE TABLE ... MANIFEST` is supported only for Iceberg format-version 2 tables. Running it against a format-version 1 table throws an exception, and so does running it against a format-version 3 table, because the v3 row-lineage `first_row_id` metadata is not yet round-tripped through the manifest rewrite. + ## Processing of tables with deleted rows {#deleted-rows} ClickHouse supports reading Iceberg tables that use the following deletion methods: diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 7f5f7a2a00ff..512ab5391ad5 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -8190,6 +8190,11 @@ Allow to clean up old data files during Iceberg compaction. )", EXPERIMENTAL) \ DECLARE(Bool, allow_experimental_iceberg_compaction, false, R"( Allow to explicitly use 'OPTIMIZE' for iceberg tables. +)", EXPERIMENTAL) \ + DECLARE(UInt64, iceberg_manifest_min_count_to_compact, 30, R"( +Minimum number of manifest files required to trigger manifest-only compaction via OPTIMIZE TABLE ... MANIFEST. +If the current number of manifest files is less than or equal to this threshold, compaction is skipped. +Requires allow_experimental_iceberg_compaction to be enabled. )", EXPERIMENTAL) \ DECLARE(Bool, allow_iceberg_remove_orphan_files, false, R"( Allow to use 'ALTER TABLE ... EXECUTE remove_orphan_files()' for iceberg tables. diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index c2b62c209863..7b298977e07d 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -43,6 +43,46 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() { {"analyzer_compatibility_allow_non_aggregate_in_having", false, false, "New compatibility setting. When enabled, the new analyzer mimics the legacy `HAVING`-to-`WHERE` rewrite for non-aggregate AND-conjuncts instead of raising `NOT_AN_AGGREGATE`."}, {"reserve_memory", 0, 0, "New setting to reserve memory for specific workload before starting a query."}, + {"optimize_or_like_chain", false, true, "Enable by default: optimize OR chains of LIKE/ILIKE/match into multiSearchAny (pure-substring patterns) or multiMatchAny (other patterns, when Hyperscan/Vectorscan is permitted); when neither fast path applies the original OR chain is kept unchanged."}, + {"optimize_or_like_chain_min_patterns", 0, 10, "New setting controlling the minimum number of non-pure-substring LIKE/ILIKE/match branches (sharing the same LHS expression) required for optimize_or_like_chain to rewrite a chain into multiMatchAny. Shorter chains are kept as-is because the multiMatchAny (Hyperscan) rewrite only becomes faster than short-circuit OR evaluation from about nine branches."}, + {"optimize_or_like_chain_min_substrings", 0, 4, "New setting controlling the minimum number of pure-substring (%needle%) LIKE/ILIKE branches (sharing the same LHS expression) required for optimize_or_like_chain to rewrite a chain into multiSearchAny."}, + {"input_format_arrow_use_native_reader", false, true, "New setting to use the native ClickHouse reader for the Arrow and ArrowStream formats instead of the Apache Arrow library."}, + {"output_format_arrow_use_native_writer", false, true, "New setting to use the native ClickHouse writer for the Arrow and ArrowStream formats instead of the Apache Arrow library."}, + {"allow_minmax_index_for_json", true, false, "Forbid creating minmax skip index on JSON columns by default because the index serialization cannot handle heterogeneous Field values"}, + {"s3_allow_server_credentials_in_user_queries", true, false, "New setting to block S3 access from user SQL from resolving the server's own ambient credentials (environment/IMDS/IRSA/instance-profile/AWS-config-file/role_arn-STS/GCP-OAuth-metadata). The previous behavior (allowed) is restored with compatibility settings."}, + {"query_plan_merge_expression_into_join", false, true, "New setting. Allow to merge Expression step into JOIN step during join reordering optimization."}, + {"skip_unavailable_shards_mode", "unavailable_or_table_missing", "unavailable_or_table_missing", "New setting to control which exceptions from a remote shard are ignored when `skip_unavailable_shards` is enabled. The default matches the historical behavior: a shard whose table is missing is treated as unavailable."}, + {"use_text_index_tokens_cache", false, true, "Enabled the text index tokens cache globally."}, + {"use_text_index_header_cache", false, true, "Enabled the text index header cache globally."}, + {"optimize_aggregation_in_order_limit", false, true, "New setting to push the `LIMIT` into aggregation-in-order for early termination when the `ORDER BY` is a prefix of the `GROUP BY` sort description."}, + {"explain_query_plan_default", "legacy", "pretty", "From 26.7, `EXPLAIN PLAN` defaults to `actions=1, compact=1, pretty=1`. Set this to `legacy` to restore the pre-26.7 output."}, + {"format_geojson_validate_geometry", true, true, "New setting that controls whether the GeoJSON format enforces RFC 7946 geometry validity (minimum points per line and ring, ring closure, non-empty multi-geometries) when reading and writing"}, + {"use_partition_minmax_for_primary_key_pruning", false, true, "New setting to use the part's partition minmax to prune more granules during primary key analysis for `MergeTree` tables, when a primary key column is also an input column of the partition key."}, + {"allow_delta_lake_writes", false, false, "Added an alias for setting `allow_experimental_delta_lake_writes`, which was moved to Beta."}, + {"allow_experimental_delta_lake_writes", false, false, "Delta Lake writes were moved to Beta."}, + {"input_format_parquet_dictionary_filter_push_down", 0, 1024 * 1024, "New setting enabling Parquet row-group pruning based on dictionary page contents (reader v3). The value is the maximum dictionary page size in bytes for which the optimization applies; 0 (the previous behavior) disables it."}, + {"compile_regular_expressions", false, true, "New setting to enable JIT compilation of simple regular expressions in functions like `match` and `extract`."}, + {"min_count_to_compile_regular_expression", 3, 3, "New setting controlling how many times a regular expression must be used before it is JIT-compiled."}, + {"allow_aggregate_partitions_independently", false, true, "Enable independent per-partition aggregation by default when the partition key suits the GROUP BY key. The existing runtime heuristics in `ReadFromMergeTree::requestOutputEachPartitionThroughSeparatePortForAggregation` already skip the optimization when the partition layout is unfavorable (too few partitions, too many partitions, or significantly skewed partition sizes), so enabling the setting is safe in the cases where it would otherwise be a no-op."}, + {"text_index_lazy_intersection_density_threshold", 0.2, 0.2, "Renamed from `text_index_density_threshold` (kept as an alias); selects the posting list intersection algorithm in lazy posting list apply mode."}, + {"allow_experimental_text_index_lazy_apply", false, true, "Lazy posting list apply mode for the text index is no longer experimental; the setting is now obsolete and has no effect (lazy mode is selected via `text_index_posting_list_apply_mode = 'lazy'`)."}, + {"allow_experimental_url_wildcard_from_index_pages", false, false, "New setting to enable expanding wildcards in the `url` table function by listing HTTP index pages."}, + {"url_wildcard_max_directories_to_read", 100000, 100000, "New setting to limit the number of directories read when expanding wildcards in the `url` table function."}, + {"allow_experimental_eval_table_function", false, false, "New setting to enable the experimental table function `eval`."}, + {"output_format_csv_header_serialize_tuple_into_separate_columns", false, true, "New setting. When output_format_csv_serialize_tuple_into_separate_columns is enabled, the CSVWithNames/CSVWithNamesAndTypes header now flattens Tuple columns into their leaf fields so the header width matches the data. Set to false to restore the previous single-name header."}, + {"reader_executor_use_long_connections", false, false, "New experimental ReaderExecutor setting (off by default): reuse a held source connection across sequential windows."}, + {"reader_executor_min_bytes_for_seek", 2097152, 2097152, "New experimental ReaderExecutor setting: forward-gap bound for bridging on a held source connection."}, + {"reader_executor_max_tail_for_drain", 1048576, 1048576, "New experimental ReaderExecutor setting: drain bound for completing a dropped long connection."}, + {"precise_float_parsing", false, true, "Use the precise (closest-representable) float parsing algorithm by default, now that it is faster than the previous fast algorithm. Set to false to restore the pre-26.7 fast-but-less-accurate parsing in conversion functions."}, + {"optimize_and_compare_chain_max_hash_work", 0, 5'000'000, "New setting that bounds the work of the `optimize_and_compare_chain` optimization (measured in query-tree nodes hashed) so it cannot dominate analysis of queries with very many or very large `AND`-chains of comparisons. The previous value `0` (unlimited) reproduces the pre-26.7 behavior where the optimization was uncapped, so `compatibility` set to an earlier version keeps deriving transitive predicates without a budget. Set to `0` to disable the budget."}, + {"iceberg_manifest_min_count_to_compact", 30, 30, "New setting to control manifest compaction for Iceberg tables."}, + {"show_remote_databases_in_system_tables", true, true, "New setting to control whether `MySQL` and `PostgreSQL` databases are shown in `system.tables`, `system.columns` and `system.completions`."}, + {"use_constant_folding_in_index_analysis", false, false, "New setting to fold partition-level constants into the filter predicate per part during MergeTree index analysis, improving pruning for filters whose branches depend on partition values."}, + {"join_runtime_filter_size_from_hash_table_stats", false, true, "Use hash table size statistics collected from previous executions to size the JOIN runtime filter. When disabled, fall back to the fixed `join_runtime_bloom_filter_bytes`."}, + }); + + addSettingsChanges(settings_changes_history, "26.6", + { {"output_format_image_width", 1024, 1024, "New setting controlling the width of the output image for image output formats such as PNG."}, {"output_format_image_height", 1024, 1024, "New setting controlling the height of the output image for image output formats such as PNG."}, {"output_format_image_terminal_mode", "", "", "New setting controlling whether image output formats such as PNG are rendered directly to the terminal using an inline image protocol."}, diff --git a/src/Interpreters/InterpreterOptimizeQuery.cpp b/src/Interpreters/InterpreterOptimizeQuery.cpp index 981a7ece9de5..93a5958a20c7 100644 --- a/src/Interpreters/InterpreterOptimizeQuery.cpp +++ b/src/Interpreters/InterpreterOptimizeQuery.cpp @@ -1,3 +1,5 @@ +#include "config.h" + #include #include #include @@ -10,6 +12,11 @@ #include #include #include +#include + +#if USE_AVRO +#include +#endif #include @@ -22,6 +29,7 @@ namespace ErrorCodes { extern const int BAD_ARGUMENTS; extern const int THERE_IS_NO_COLUMN; + extern const int NOT_IMPLEMENTED; } @@ -44,6 +52,28 @@ BlockIO InterpreterOptimizeQuery::execute() auto metadata_snapshot = table->getInMemoryMetadataPtr(getContext(), false); auto storage_snapshot = table->getStorageSnapshot(metadata_snapshot, getContext()); + /// Handle OPTIMIZE TABLE ... MANIFEST for Iceberg tables + if (ast.manifest) + { + if (ast.final || ast.partition || ast.deduplicate || ast.cleanup || ast.dry_run) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "OPTIMIZE MANIFEST is incompatible with FINAL, PARTITION, DEDUPLICATE, CLEANUP, and DRY RUN options"); + +#if USE_AVRO + auto * object_storage_table = dynamic_cast(table.get()); + if (!object_storage_table) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "OPTIMIZE MANIFEST is only supported for Iceberg tables"); + + auto * iceberg_metadata = dynamic_cast(object_storage_table->getExternalMetadata(getContext())); + if (!iceberg_metadata) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "OPTIMIZE MANIFEST is only supported for Iceberg tables"); + + iceberg_metadata->optimizeManifestFiles(metadata_snapshot, getContext(), object_storage_table->getCatalog(), table_id); + return {}; +#else + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "OPTIMIZE MANIFEST is only supported for Iceberg tables"); +#endif + } + // Empty list of names means we deduplicate by all columns, but user can explicitly state which columns to use. Names column_names; if (ast.deduplicate_by_columns) diff --git a/src/Parsers/ASTOptimizeQuery.cpp b/src/Parsers/ASTOptimizeQuery.cpp index ee22604c5066..cc6c7bbda7ad 100644 --- a/src/Parsers/ASTOptimizeQuery.cpp +++ b/src/Parsers/ASTOptimizeQuery.cpp @@ -45,6 +45,9 @@ void ASTOptimizeQuery::formatQueryImpl(WriteBuffer & ostr, const FormatSettings if (cleanup) ostr << " CLEANUP"; + if (manifest) + ostr << " MANIFEST"; + if (deduplicate_by_columns) { ostr << " BY "; diff --git a/src/Parsers/ASTOptimizeQuery.h b/src/Parsers/ASTOptimizeQuery.h index 9d927d8b82f0..205474474243 100644 --- a/src/Parsers/ASTOptimizeQuery.h +++ b/src/Parsers/ASTOptimizeQuery.h @@ -27,10 +27,12 @@ class ASTOptimizeQuery : public ASTQueryWithTableAndOutput, public ASTQueryWithO bool dry_run = false; /// List of part names for DRY RUN (ASTExpressionList of ASTLiteral strings) ASTPtr parts_list; + /// Compact manifests only (for Iceberg tables) + bool manifest = false; /** Get the text that identifies this element. */ String getID(char delim) const override { - return "OptimizeQuery" + (delim + getDatabase()) + delim + getTable() + (final ? "_final" : "") + (deduplicate ? "_deduplicate" : "") + (cleanup ? "_cleanup" : "") + (dry_run ? "_dry_run" : ""); + return "OptimizeQuery" + (delim + getDatabase()) + delim + getTable() + (final ? "_final" : "") + (deduplicate ? "_deduplicate" : "") + (cleanup ? "_cleanup" : "") + (dry_run ? "_dry_run" : "") + (manifest ? "_manifest" : ""); } ASTPtr clone() const override diff --git a/src/Parsers/CommonParsers.h b/src/Parsers/CommonParsers.h index 707a85f581ce..e2bab768e56a 100644 --- a/src/Parsers/CommonParsers.h +++ b/src/Parsers/CommonParsers.h @@ -325,6 +325,7 @@ namespace DB MR_MACROS(LIVE, "LIVE") \ MR_MACROS(LOCAL, "LOCAL") \ MR_MACROS(M, "M") \ + MR_MACROS(MANIFEST, "MANIFEST") \ MR_MACROS(MASTER_THREAD, "MASTER THREAD") \ MR_MACROS(MATCH, "MATCH") \ MR_MACROS(MATERIALIZE_COLUMN, "MATERIALIZE COLUMN") \ diff --git a/src/Parsers/ParserOptimizeQuery.cpp b/src/Parsers/ParserOptimizeQuery.cpp index 2c8f64da48a5..f5ab4d3031c4 100644 --- a/src/Parsers/ParserOptimizeQuery.cpp +++ b/src/Parsers/ParserOptimizeQuery.cpp @@ -33,6 +33,7 @@ bool ParserOptimizeQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte ParserKeyword s_force(Keyword::FORCE); ParserKeyword s_deduplicate(Keyword::DEDUPLICATE); ParserKeyword s_cleanup(Keyword::CLEANUP); + ParserKeyword s_manifest(Keyword::MANIFEST); ParserKeyword s_by(Keyword::BY); ParserToken s_dot(TokenType::Dot); ParserIdentifier name_p(true); @@ -46,6 +47,7 @@ bool ParserOptimizeQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte bool final = false; bool deduplicate = false; bool cleanup = false; + bool manifest = false; String cluster_str; if (!s_optimize_table.ignore(pos, expected)) @@ -90,6 +92,9 @@ bool ParserOptimizeQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte if (s_cleanup.ignore(pos, expected)) cleanup = true; + if (s_manifest.ignore(pos, expected)) + manifest = true; + ASTPtr deduplicate_by_columns; if (deduplicate && s_by.ignore(pos, expected)) { @@ -111,6 +116,7 @@ bool ParserOptimizeQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte query->deduplicate = deduplicate; query->deduplicate_by_columns = deduplicate_by_columns; query->cleanup = cleanup; + query->manifest = manifest; query->database = database; query->table = table; diff --git a/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp b/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp index 4517834d0dec..a4c7715b04fa 100644 --- a/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp @@ -157,6 +157,16 @@ ParsedManifestFileEntryPtr AvroForIcebergDeserializer::createParsedManifestFileE } } + /// `file_sequence_number` can differ from the data `sequence_number` and, like it, is inherited from the + /// manifest's sequence number when null. Keep it raw here; the inherited value is resolved by the caller. + std::optional file_sequence_number; + + if (format_version > 1 && hasPath(f_file_sequence_number)) + { + const auto file_sequence_number_value = getValueFromRowByName(row_index, f_file_sequence_number); + if (!file_sequence_number_value.isNull()) + file_sequence_number = file_sequence_number_value.safeGet(); + } const auto file_path_key = IcebergPathFromMetadata::deserialize( getValueFromRowByName(row_index, c_data_file_file_path, TypeIndex::String).safeGet()); @@ -251,6 +261,7 @@ ParsedManifestFileEntryPtr AvroForIcebergDeserializer::createParsedManifestFileE row_index, status, sequence_number, + file_sequence_number, snapshot_id, partition_key_value, columns_infos, @@ -298,6 +309,7 @@ ParsedManifestFileEntryPtr AvroForIcebergDeserializer::createParsedManifestFileE row_index, status, sequence_number, + file_sequence_number, snapshot_id, partition_key_value, columns_infos, @@ -329,6 +341,7 @@ ParsedManifestFileEntryPtr AvroForIcebergDeserializer::createParsedManifestFileE row_index, status, sequence_number, + file_sequence_number, snapshot_id, partition_key_value, columns_infos, diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp index 66f07c521b27..d7b878ad0124 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp @@ -1,16 +1,20 @@ +#include #include #include +#include #include #include #include #include #include +#include #include #include #include #include #include #include +#include #include #include #include @@ -29,6 +33,7 @@ #include #include #include +#include #include #if USE_AVRO @@ -36,12 +41,26 @@ namespace DB::ErrorCodes { extern const int BAD_ARGUMENTS; + extern const int LOGICAL_ERROR; + extern const int ICEBERG_SPECIFICATION_VIOLATION; extern const int NOT_IMPLEMENTED; } +namespace DB::Setting +{ + extern const SettingsUInt64 iceberg_manifest_min_count_to_compact; +} + +namespace DB::DataLakeStorageSetting +{ + extern const DataLakeStorageSettingsBool iceberg_use_version_hint; +} + namespace DB::Iceberg { +static constexpr size_t MAX_COMPACTION_RETRIES = 100; + using namespace DB; struct ManifestFilePlan @@ -68,8 +87,7 @@ struct DataFilePlan UInt64 new_bytes_count = 0; }; -/// Plan of compaction consists of information about all data files and what delete files should be applied for them. -/// Also it contains some other information about previous metadata. +/// Compaction plan: all data files, the delete files applied to them, and prior metadata. struct Plan { bool need_optimize = false; @@ -116,6 +134,44 @@ struct Plan } partition_encoder; }; +/// Cheap pre-check for `compactIcebergManifests`: read just the current manifest list and report whether its entry count exceeds `threshold`. +static bool isCurrentManifestListAboveThreshold( + Poco::JSON::Object::Ptr metadata_object, + const PersistentTableComponents & persistent_table_components, + ObjectStoragePtr object_storage, + ContextPtr context, + size_t threshold) +{ + LoggerPtr log = getLogger("IcebergCompaction::isCurrentManifestListAboveThreshold"); + + if (!metadata_object->has(Iceberg::f_current_snapshot_id)) + return false; + Int64 current_snapshot_id = metadata_object->getValue(Iceberg::f_current_snapshot_id); + if (current_snapshot_id < 0) + return false; + + String current_manifest_list_path; + auto snapshots = metadata_object->get(Iceberg::f_snapshots).extract(); + for (size_t i = 0; i < snapshots->size(); ++i) + { + const auto snapshot = snapshots->getObject(static_cast(i)); + if (snapshot->getValue(Iceberg::f_metadata_snapshot_id) == current_snapshot_id) + { + current_manifest_list_path = snapshot->getValue(Iceberg::f_manifest_list); + break; + } + } + if (current_manifest_list_path.empty()) + return false; + + auto filename = IcebergPathFromMetadata::deserialize(current_manifest_list_path); + RelativePathWithMetadata object_info(persistent_table_components.path_resolver.resolve(filename)); + auto manifest_list_buf = createReadBuffer(object_info, object_storage, context, log); + AvroForIcebergDeserializer manifest_list_deserializer( + std::move(manifest_list_buf), filename, getFormatSettings(context)); + return manifest_list_deserializer.rows() > threshold; +} + static Plan getPlan( IcebergHistory snapshots_info, const DataLakeStorageSettings & data_lake_settings, @@ -214,7 +270,6 @@ static Plan getPlan( if (partition_index >= plan.partitions.size()) continue; - std::vector result_delete_files; for (auto & data_file : plan.partitions[partition_index]) { if (data_file->data_object_info->info.sequence_number <= delete_file->sequence_number) @@ -320,8 +375,7 @@ static void writeDataFiles( auto file_bytes = write_buffer->count(); if (file_bytes == 0 && !data_file->patched_path.empty()) { - /// Some storage backends (e.g. Azure) don't track bytes in the write buffer. - /// Fall back to querying the actual object size. + /// Some storage backends (e.g. Azure) don't track bytes in the write buffer; query the object size. auto obj_metadata = object_storage->getObjectMetadata(path_resolver.resolve(data_file->patched_path), /*with_tags=*/false); file_bytes = obj_metadata.size_bytes; } @@ -329,6 +383,534 @@ static void writeDataFiles( } } +static bool writeConsolidatedManifestFile( + int metadata_version, + Poco::JSON::Object::Ptr metadata_object, + const PersistentTableComponents & persistent_table_components, + ObjectStoragePtr object_storage, ContextPtr context, + SharedHeader sample_block_, + String write_format, + CompressionMethod compression_method, + const DataLakeStorageSettings & data_lake_settings, + std::shared_ptr catalog, + const StorageID & table_id) +{ + auto log = getLogger("IcebergManifestConsolidation"); + + // Derive current snapshot info directly from the metadata file. + if (!metadata_object->has(Iceberg::f_current_snapshot_id)) + { + LOG_INFO(log, "No current snapshot found, skipping manifest consolidation"); + return true; + } + Int64 current_snapshot_id_val = metadata_object->getValue(Iceberg::f_current_snapshot_id); + if (current_snapshot_id_val < 0) + { + LOG_INFO(log, "No current snapshot found, skipping manifest consolidation"); + return true; + } + + Int64 current_snapshot_id = current_snapshot_id_val; + String current_manifest_list_path; + + { + auto snapshots = metadata_object->get(Iceberg::f_snapshots).extract(); + for (size_t i = 0; i < snapshots->size(); ++i) + { + const auto snapshot = snapshots->getObject(static_cast(i)); + if (snapshot->getValue(Iceberg::f_metadata_snapshot_id) == current_snapshot_id) + { + current_manifest_list_path = snapshot->getValue(Iceberg::f_manifest_list); + break; + } + } + } + + if (current_manifest_list_path.empty()) + { + LOG_INFO(log, "No current snapshot found, skipping manifest consolidation"); + return true; + } + + LOG_INFO(log, "Writing consolidated manifest file from current snapshot {}", current_snapshot_id); + + auto current_schema_id = metadata_object->getValue(Iceberg::f_current_schema_id); + Poco::JSON::Object::Ptr current_schema; + auto schemas = metadata_object->getArray(Iceberg::f_schemas); + for (size_t i = 0; i < schemas->size(); ++i) + { + if (schemas->getObject(static_cast(i))->getValue(Iceberg::f_schema_id) == current_schema_id) + { + current_schema = schemas->getObject(static_cast(i)); + break; + } + } + + if (!current_schema) + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Iceberg metadata does not contain a schema entry matching current-schema-id {}", + current_schema_id); + + auto partitions_specs = metadata_object->getArray(f_partition_specs); + + /// After partition evolution each manifest must be rewritten under the spec its source files used; resolve and cache spec info per spec-id. + struct ResolvedPartitionSpec + { + Poco::JSON::Object::Ptr spec; + std::vector partition_columns; + DataTypes partition_types; + }; + std::unordered_map resolved_specs; + auto resolve_partition_spec = [&](Int32 spec_id) -> const ResolvedPartitionSpec & + { + if (auto it = resolved_specs.find(spec_id); it != resolved_specs.end()) + return it->second; + + Poco::JSON::Object::Ptr spec; + for (UInt32 i = 0; i < partitions_specs->size(); ++i) + { + auto candidate = partitions_specs->getObject(i); + if (candidate->getValue(Iceberg::f_spec_id) == spec_id) + { + spec = candidate; + break; + } + } + if (!spec) + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Iceberg metadata does not contain a partition spec entry matching spec-id {}", + spec_id); + + ResolvedPartitionSpec resolved; + resolved.spec = spec; + auto spec_fields = spec->getArray(f_fields); + + /// Partition field names and the schema source-ids they transform. + std::vector source_ids; + for (UInt32 i = 0; i < spec_fields->size(); ++i) + { + auto spec_field = spec_fields->getObject(i); + resolved.partition_columns.push_back(spec_field->getValue(f_name)); + source_ids.push_back(spec_field->getValue(Iceberg::f_source_id)); + } + + /// Derive partition value types from a schema that defines every source column the spec references, preferring the current schema then any historical one; register all schemas first so they can be queried by id. + for (UInt32 i = 0; i < schemas->size(); ++i) + persistent_table_components.schema_processor->addIcebergTableSchema(schemas->getObject(i)); + + auto build_sample_block = [&](Int32 schema_id) -> std::optional + { + auto fields_characteristics + = persistent_table_components.schema_processor->tryGetFieldsCharacteristics(schema_id, source_ids); + /// A short result means this schema does not define every partition source column. + if (fields_characteristics.size() != source_ids.size()) + return std::nullopt; + Block block; + for (const auto & name_and_type : fields_characteristics) + block.insert(ColumnWithTypeAndName(name_and_type.type, name_and_type.name)); + return block; + }; + + Int32 schema_id_for_spec = static_cast(current_schema_id); + std::optional spec_sample_block = build_sample_block(schema_id_for_spec); + if (!spec_sample_block) + { + for (UInt32 i = 0; i < schemas->size(); ++i) + { + Int32 candidate_id = schemas->getObject(i)->getValue(Iceberg::f_schema_id); + if (candidate_id == schema_id_for_spec) + continue; + spec_sample_block = build_sample_block(candidate_id); + if (spec_sample_block) + { + schema_id_for_spec = candidate_id; + break; + } + } + } + if (!spec_sample_block) + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "No Iceberg schema defines all source columns referenced by partition spec {}", + spec_id); + + auto schema_for_spec = persistent_table_components.schema_processor->getIcebergTableSchemaById(schema_id_for_spec); + auto shared_sample_block = std::make_shared(std::move(*spec_sample_block)); + resolved.partition_types + = ChunkPartitioner(spec_fields, schema_for_spec->getArray(Iceberg::f_fields), context, shared_sample_block).getResultTypes(); + + return resolved_specs.emplace(spec_id, std::move(resolved)).first->second; + }; + + /// Return the raw metadata schema object for a given schema-id, used as the verbatim Avro `schema` header of a rewritten manifest so its data-file bounds resolve under the same schema the files were written with. + auto get_schema_object_by_id = [&](Int32 schema_id) -> Poco::JSON::Object::Ptr + { + for (UInt32 i = 0; i < schemas->size(); ++i) + if (schemas->getObject(i)->getValue(Iceberg::f_schema_id) == schema_id) + return schemas->getObject(i); + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Iceberg metadata does not contain a schema entry matching schema-id {}", + schema_id); + }; + + // Collect data files grouped by (partition spec-id, partition key) + struct PartitionData + { + /// The partition spec the source files were written with; the rewritten manifest reuses it. + Int32 partition_spec_id = 0; + /// The schema the source files were written under; files of different schemas are grouped separately so each rewritten manifest's `schema` header matches all its entries. + Int32 schema_id = 0; + Row partition_values; + std::vector file_paths; + /// Parallel to file_paths: {record_count, file_size_in_bytes} from the source manifest entry. + std::vector> file_metrics; + /// Parallel to file_paths: the original file_format, preserved so a rewrite never relabels the file's format. + std::vector file_formats; + /// Parallel to file_paths: the source file's per-column statistics, preserved across the rewrite. + std::vector file_statistics; + /// Parallel to file_paths: the source file's sort_order_id, preserved so the rewrite keeps sortedness. + std::vector> file_sort_order_ids; + /// Parallel to file_paths: the source entry's lineage, preserved so each file is emitted as an EXISTING entry retaining its lineage. + std::vector file_entry_lineage; + + explicit PartitionData(Poco::JSON::Array::Ptr /*schema*/) + {} + }; + + auto schema_fields = current_schema->getArray(Iceberg::f_fields); + + std::unordered_map partitions_map; + + // Collect live data files from the current snapshot only; iterating older snapshots would resurrect deleted files. + size_t total_data_files = 0; + // Only data manifests are consolidated; delete-file manifests are carried forward unchanged so deleted rows do not reappear. + size_t num_data_manifests = 0; + std::unordered_set delete_manifest_paths; + + auto current_manifest_list = getManifestList( + object_storage, persistent_table_components, context, IcebergPathFromMetadata::deserialize(current_manifest_list_path), log); + + for (const auto & manifest_file : current_manifest_list) + { + if (manifest_file.content_type == ManifestFileContentType::DELETE) + { + delete_manifest_paths.insert(manifest_file.manifest_file_path.serialize()); + continue; + } + ++num_data_manifests; + const Int32 source_partition_spec_id = manifest_file.partition_spec_id; + + /// A manifest-only rewrite cannot round-trip per-file `key_metadata` (data-file encryption keys), so reject rather than silently dropping it and making an encrypted table unreadable. + { + RelativePathWithMetadata key_metadata_object_info(persistent_table_components.path_resolver.resolve(manifest_file.manifest_file_path)); + auto key_metadata_buf = createReadBuffer(key_metadata_object_info, object_storage, context, log); + AvroForIcebergDeserializer key_metadata_deserializer(std::move(key_metadata_buf), manifest_file.manifest_file_path, getFormatSettings(context)); + if (key_metadata_deserializer.hasPath(c_data_file_key_metadata)) + { + for (size_t row = 0; row < key_metadata_deserializer.rows(); ++row) + if (!key_metadata_deserializer.getValueFromRowByName(row, c_data_file_key_metadata).isNull()) + throw Exception( + ErrorCodes::NOT_IMPLEMENTED, + "OPTIMIZE TABLE ... MANIFEST is not supported for Iceberg tables with per-file key_metadata " + "(encrypted data files): preserving the encryption metadata across a manifest rewrite is not implemented"); + } + } + + auto files_handle = getManifestFileEntriesHandle( + object_storage, persistent_table_components, context, log, manifest_file, static_cast(current_schema_id)); + + for (const auto & data_file : files_handle.getFilesWithoutDeleted(FileContentType::DATA)) + { + // Group by source spec-id AND source schema-id so files of different specs or schemas are never merged into one manifest (a manifest carries a single spec and one `schema` header); FieldVisitorDump's type tag prevents UInt64/Int64 collisions. + const Int32 source_schema_id = data_file->resolved_schema_id; + String partition_key = std::to_string(source_partition_spec_id) + "|" + std::to_string(source_schema_id) + "|"; + FieldVisitorDump dump_visitor; + for (const auto & val : data_file->parsed_entry->partition_key_value) + partition_key += applyVisitor(dump_visitor, val) + "|"; + + if (!partitions_map.contains(partition_key)) + partitions_map.emplace(partition_key, PartitionData(schema_fields)); + + auto & pd = partitions_map.at(partition_key); + pd.partition_spec_id = source_partition_spec_id; + pd.schema_id = source_schema_id; + pd.partition_values = data_file->parsed_entry->partition_key_value; + // A single manifest file should not list the same data file twice + if (std::find(pd.file_paths.begin(), pd.file_paths.end(), data_file->parsed_entry->file_path_key) == pd.file_paths.end()) + { + pd.file_paths.push_back(data_file->parsed_entry->file_path_key); + pd.file_metrics.emplace_back(data_file->parsed_entry->record_count, data_file->parsed_entry->file_size_in_bytes); + pd.file_formats.push_back(data_file->parsed_entry->file_format); + pd.file_sort_order_ids.push_back(data_file->parsed_entry->sort_order_id); + + /// Preserve the entry's lineage, resolving inherited (null) snapshot-id and sequence numbers from the manifest, since EXISTING entries require them non-null. + DataFileEntryLineage lineage; + lineage.added_snapshot_id = data_file->parsed_entry->parsed_snapshot_id; + if (!lineage.added_snapshot_id.has_value()) + lineage.added_snapshot_id = manifest_file.added_snapshot_id; + lineage.sequence_number = data_file->parsed_entry->parsed_sequence_number; + if (!lineage.sequence_number.has_value()) + lineage.sequence_number = manifest_file.added_sequence_number; + /// `file_sequence_number` is preserved separately: it can differ from the data sequence number and, when null, inherits the manifest's sequence number. + lineage.file_sequence_number = data_file->parsed_entry->parsed_file_sequence_number; + if (!lineage.file_sequence_number.has_value()) + lineage.file_sequence_number = manifest_file.added_sequence_number; + pd.file_entry_lineage.push_back(lineage); + + /// Carry the source file's per-column stats over verbatim, keeping bounds as the raw serialized bytes so they round-trip. + DataFileColumnStatistics stats; + for (const auto & [field_id, col_info] : data_file->parsed_entry->columns_infos) + { + if (col_info.bytes_size.has_value()) + stats.column_sizes.emplace_back(field_id, *col_info.bytes_size); + if (col_info.rows_count.has_value()) + stats.value_counts.emplace_back(field_id, *col_info.rows_count); + if (col_info.nulls_count.has_value()) + stats.null_value_counts.emplace_back(field_id, *col_info.nulls_count); + } + for (const auto & [field_id, bounds] : data_file->parsed_entry->value_bounds) + { + if (!bounds.first.isNull()) + stats.lower_bounds.emplace_back(field_id, bounds.first.safeGet()); + if (!bounds.second.isNull()) + stats.upper_bounds.emplace_back(field_id, bounds.second.safeGet()); + } + pd.file_statistics.push_back(std::move(stats)); + + ++total_data_files; + } + } + } + + /// Data manifests already optimally consolidated (at most one per partition): rewriting cannot reduce the count, so report success. + if (partitions_map.size() >= num_data_manifests) + { + LOG_INFO(log, "Manifests already optimally consolidated ({} data manifests, {} unique partitions); nothing to do", + num_data_manifests, partitions_map.size()); + return true; + } + + const auto & path_resolver = persistent_table_components.path_resolver; + + // Create file name generator for new metadata files + FileNamesGenerator generator( + path_resolver.getTableLocation(), + false, + compression_method, + write_format); + generator.setVersion(metadata_version + 1); + + MetadataGenerator metadata_generator(metadata_object); + auto generated_metadata_info = generator.generateMetadataPathWithInfo(); + + // Manifest-only rewrite: use a snapshot type that carries all total-* counters forward unchanged, since passing deltas would inflate the totals. + auto new_snapshot = metadata_generator.generateManifestOnlySnapshot( + generator, + generated_metadata_info.path, + current_snapshot_id); + + // Write one manifest file per (partition spec, partition value) group. + std::vector consolidated_manifest_paths; + std::vector manifest_entry_sizes; + /// Parallel to consolidated_manifest_paths: existing (not added) file/row counts, since the referenced data files already exist. + std::vector existing_entry_counts; + /// Parallel to consolidated_manifest_paths: each manifest's partition spec-id. + std::vector entry_partition_spec_ids; + /// Parallel to consolidated_manifest_paths: each manifest's partition fields (value + type), used to recompute the manifest-list `partitions` summary. + std::vector>> entry_partition_summaries; + + /// Cleanup for both commit conflict and exceptions; paths are tracked before writeObject so partially-created objects are removed (removeObjectIfExists tolerates missing objects). + auto cleanup = [&]() + { + for (const auto & mp : consolidated_manifest_paths) + { + try + { + object_storage->removeObjectIfExists(StoredObject(path_resolver.resolve(mp))); + } + catch (...) + { + tryLogCurrentException(log, "Failed to remove orphaned manifest file during cleanup"); + } + } + try + { + object_storage->removeObjectIfExists(StoredObject(path_resolver.resolve(new_snapshot.manifest_list_path))); + } + catch (...) + { + tryLogCurrentException(log, "Failed to remove orphaned manifest list during cleanup"); + } + }; + + try + { + for (auto & [partition_key, pd] : partitions_map) + { + auto manifest_path = generator.generateManifestEntryName(); + auto storage_manifest_path = path_resolver.resolve(manifest_path); + LOG_INFO(log, "Creating manifest file for partition '{}': {} ({} data files)", + partition_key, storage_manifest_path, pd.file_paths.size()); + + /// Track the path before writeObject so `cleanup` removes any object created even if a later step throws. + consolidated_manifest_paths.push_back(manifest_path); + + auto buffer_manifest = object_storage->writeObject( + StoredObject(storage_manifest_path), + WriteMode::Rewrite, + std::nullopt, + DBMS_DEFAULT_BUFFER_SIZE, + context->getWriteSettings()); + + std::vector file_row_counts; + std::vector file_byte_counts; + file_row_counts.reserve(pd.file_metrics.size()); + file_byte_counts.reserve(pd.file_metrics.size()); + Int64 manifest_existing_rows = 0; + for (const auto & [record_count, file_size_in_bytes] : pd.file_metrics) + { + file_row_counts.push_back(static_cast(record_count)); + file_byte_counts.push_back(static_cast(file_size_in_bytes)); + manifest_existing_rows += record_count; + } + + /// Lowest data sequence number across this manifest's files; files keep their original sequence numbers, so min_sequence_number must reflect that minimum. + Int64 manifest_min_sequence_number = std::numeric_limits::max(); + for (const auto & lineage : pd.file_entry_lineage) + manifest_min_sequence_number = std::min(manifest_min_sequence_number, lineage.sequence_number.value_or(0)); + + existing_entry_counts.push_back( + {static_cast(pd.file_paths.size()), manifest_existing_rows, manifest_min_sequence_number}); + + /// Rewrite this manifest under the partition spec its source files used, not the default. + const auto & resolved_spec = resolve_partition_spec(pd.partition_spec_id); + entry_partition_spec_ids.push_back(pd.partition_spec_id); + + /// All files in this manifest share one partition value, so the summary's lower/upper bounds are exactly that value. + std::vector> partition_summary; + for (size_t i = 0; i < resolved_spec.partition_types.size(); ++i) + { + Field partition_value = i < pd.partition_values.size() ? pd.partition_values[i] : Field{}; + partition_summary.emplace_back(partition_value, resolved_spec.partition_types[i]); + } + entry_partition_summaries.push_back(std::move(partition_summary)); + + generateManifestFile( + metadata_object, + resolved_spec.partition_columns, + pd.partition_values, + resolved_spec.partition_types, + pd.file_paths, + file_row_counts, + file_byte_counts, + std::nullopt, + sample_block_, + new_snapshot.snapshot, + write_format, + resolved_spec.spec, + pd.partition_spec_id, + *buffer_manifest, + Iceberg::FileContentType::DATA, + /* user_defined_sequence_number */ std::nullopt, + /* data_file_formats */ pd.file_formats, + /* per_file_statistics */ pd.file_statistics, + /* data_file_sort_order_ids */ pd.file_sort_order_ids, + /* per_file_entry_lineage */ pd.file_entry_lineage, + /* schema_to_serialize */ get_schema_object_by_id(pd.schema_id)); + + buffer_manifest->finalize(); + Int64 manifest_size = buffer_manifest->count(); + if (manifest_size == 0) + manifest_size = object_storage->getObjectMetadata(storage_manifest_path, /*with_tags=*/false).size_bytes; + manifest_entry_sizes.push_back(manifest_size); + } + + // Create manifest list pointing to all per-partition manifest files + auto storage_manifest_list_path = path_resolver.resolve(new_snapshot.manifest_list_path); + LOG_INFO(log, "Creating manifest list with {} partition manifest(s): {}", + consolidated_manifest_paths.size(), storage_manifest_list_path); + + auto buffer_manifest_list = object_storage->writeObject( + StoredObject(storage_manifest_list_path), + WriteMode::Rewrite, + std::nullopt, + DBMS_DEFAULT_BUFFER_SIZE, + context->getWriteSettings()); + + generateManifestList( + path_resolver, + metadata_object, + object_storage, + context, + consolidated_manifest_paths, + new_snapshot.snapshot, + manifest_entry_sizes, + *buffer_manifest_list, + Iceberg::FileContentType::DATA, + false, + /* per_entry_content_types */ {}, + existing_entry_counts, + /* carry_forward_manifest_paths */ delete_manifest_paths, + /* entry_partition_spec_ids */ entry_partition_spec_ids, + /* entry_partition_summaries */ entry_partition_summaries); + buffer_manifest_list->finalize(); + + // Commit: write metadata file with If-None-Match + ETag-based CAS version hint; returns false if another writer claimed this version, so the caller retries. + { + std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + Poco::JSON::Stringifier::stringify(metadata_object, oss, 4); + std::string json_representation = removeEscapedSlashes(oss.str()); + + auto hint_path = generator.generateVersionHint(); + LOG_INFO(log, "Committing metadata file: {}", + path_resolver.resolve(generated_metadata_info.path)); + + /// A transactional catalog is the source of truth for the current metadata; for it the storage-side + /// version hint is irrelevant, so only write the metadata file + version hint when no such catalog owns the table. + const bool catalog_writes_metadata_file = catalog && catalog->isTransactional(); + if (!catalog_writes_metadata_file + && !writeMetadataFileAndVersionHint( + path_resolver, + generated_metadata_info, + json_representation, + hint_path, + object_storage, + context, + data_lake_settings[DataLakeStorageSetting::iceberg_use_version_hint])) + { + LOG_INFO(log, "Metadata commit conflict detected, cleaning up temporary files"); + cleanup(); + return false; + } + + /// Advance the catalog pointer to the new metadata so catalog-based readers see the compacted snapshot. + if (catalog) + { + auto catalog_filename = path_resolver.resolveForCatalog(generated_metadata_info.path); + const auto & [namespace_name, table_name] = DataLake::parseTableName(table_id.getTableName()); + if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot.snapshot)) + { + LOG_INFO(log, "Metadata commit conflict detected via catalog, cleaning up temporary files"); + cleanup(); + return false; + } + } + } + } + catch (...) + { + cleanup(); + throw; + } + + LOG_INFO(log, "Successfully created {} partition manifest file(s) covering {} data files", + consolidated_manifest_paths.size(), total_data_files); + return true; +} + namespace { @@ -436,7 +1018,6 @@ static void writeMetadataFiles( { std::unordered_map, std::unordered_set> grouped_by_manifest_files_result; std::unordered_map, size_t> grouped_by_manifest_files_partitions; - std::unordered_map, size_t> partition_values; std::unordered_map> patched_path_to_data_file; for (const auto & [_, data_file] : plan.path_to_data_file) @@ -449,7 +1030,6 @@ static void writeMetadataFiles( { grouped_by_manifest_files_partitions[data_file->manifest_list] = i; grouped_by_manifest_files_result[data_file->manifest_list].insert(data_file->patched_path); - partition_values[data_file->manifest_list] = i; } } @@ -620,6 +1200,97 @@ static void clearOldFiles(ObjectStoragePtr object_storage, const std::vector catalog, + const StorageID & table_id) +{ + auto log = getLogger("IcebergManifestCompaction"); + LOG_INFO(log, "Starting manifest-only compaction for Iceberg table"); + + const size_t min_count_to_compact = context_->getSettingsRef()[DB::Setting::iceberg_manifest_min_count_to_compact]; + + for (size_t attempt = 0; attempt < MAX_COMPACTION_RETRIES; ++attempt) + { + if (attempt > 0) + LOG_INFO(log, "Retrying manifest compaction (attempt {}/{})", attempt + 1, MAX_COMPACTION_RETRIES); + + const auto [metadata_version, metadata_file_path, _] = getLatestOrExplicitMetadataFileAndVersion( + object_storage_, + persistent_table_components.table_path, + data_lake_settings, + persistent_table_components.metadata_cache, + context_, + log.get(), + persistent_table_components.table_uuid, + persistent_table_components.metadata_compression_method, + /* force_fetch_latest_metadata */ true, + /* ignore_explicit_metadata_file_path */ true); + + auto metadata_object = getMetadataJSONObject( + metadata_file_path, + object_storage_, + persistent_table_components.metadata_cache, + context_, + log, + persistent_table_components.metadata_compression_method, + persistent_table_components.table_uuid); + + /// Validate the format version on the freshly-fetched metadata (before the threshold early-return), since the table may have been upgraded to v3 by another writer after this table object was created. + const Int32 format_version = metadata_object->getValue(Iceberg::f_format_version); + if (format_version < 2) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "OPTIMIZE TABLE ... MANIFEST is supported only for Iceberg format_version 2."); + if (format_version >= 3) + throw Exception( + ErrorCodes::NOT_IMPLEMENTED, + "OPTIMIZE TABLE ... MANIFEST is not yet supported for Iceberg format-version 3: " + "row-lineage 'first_row_id' round-trip is not implemented"); + + /// Cheap pre-check: read just the current manifest list to decide whether the table is above the configured threshold. + if (!isCurrentManifestListAboveThreshold( + metadata_object, persistent_table_components, object_storage_, context_, min_count_to_compact)) + { + LOG_INFO(log, "Manifest compaction is not needed (manifest list is within threshold {})", + min_count_to_compact); + return; + } + + if (writeConsolidatedManifestFile( + metadata_version, + metadata_object, + persistent_table_components, + object_storage_, + context_, + sample_block_, + write_format, + persistent_table_components.metadata_compression_method, + data_lake_settings, + catalog, + table_id)) + { + // Invalidate metadata cache so the next reader picks up the new state + if (persistent_table_components.metadata_cache) + { + persistent_table_components.metadata_cache->remove(persistent_table_components.table_path); + if (persistent_table_components.table_uuid) + persistent_table_components.metadata_cache->remove(*persistent_table_components.table_uuid); + } + LOG_INFO(log, "Successfully compacted manifest list"); + return; + } + } + + throw Exception(ErrorCodes::LOGICAL_ERROR, "Manifest compaction failed to commit after {} attempts", + MAX_COMPACTION_RETRIES); +} + void compactIcebergTable( IcebergHistory snapshots_info, const PersistentTableComponents & persistent_table_components, diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.h index 0916002f99f3..725d7449331a 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.h @@ -21,5 +21,15 @@ void compactIcebergTable( DB::ContextPtr context_, const String & write_format); +void compactIcebergManifests( + const PersistentTableComponents & persistent_table_components, + DB::ObjectStoragePtr object_storage_, + const DataLakeStorageSettings & data_lake_settings, + DB::SharedHeader sample_block_, + DB::ContextPtr context_, + const String & write_format, + std::shared_ptr catalog, + const StorageID & table_id); + #endif } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Constant.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/Constant.h index b87ff718e172..6c480c7e92c8 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Constant.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Constant.h @@ -66,6 +66,7 @@ DEFINE_ICEBERG_FIELD(record_count); DEFINE_ICEBERG_FIELD(file_path); DEFINE_ICEBERG_FIELD(file_format); DEFINE_ICEBERG_FIELD(file_size_in_bytes); +DEFINE_ICEBERG_FIELD(sort_order_id); DEFINE_ICEBERG_FIELD(refs); DEFINE_ICEBERG_FIELD(branch); DEFINE_ICEBERG_FIELD(tag); @@ -81,10 +82,16 @@ DEFINE_ICEBERG_FIELD(statistics); DEFINE_ICEBERG_FIELD(properties); DEFINE_ICEBERG_FIELD(owner); DEFINE_ICEBERG_FIELD(column_sizes); +DEFINE_ICEBERG_FIELD(value_counts); DEFINE_ICEBERG_FIELD(null_value_counts); DEFINE_ICEBERG_FIELD(lower_bounds); DEFINE_ICEBERG_FIELD(upper_bounds); DEFINE_ICEBERG_FIELD(partitions); +/// Fields of a manifest-list `partitions` field_summary record. +DEFINE_ICEBERG_FIELD(contains_null); +DEFINE_ICEBERG_FIELD(contains_nan); +DEFINE_ICEBERG_FIELD(lower_bound); +DEFINE_ICEBERG_FIELD(upper_bound); DEFINE_ICEBERG_FIELD(key_metadata); DEFINE_ICEBERG_FIELD(replace); @@ -179,6 +186,7 @@ DEFINE_ICEBERG_FIELD_COMPOUND(data_file, referenced_data_file); DEFINE_ICEBERG_FIELD_COMPOUND(data_file, sort_order_id); DEFINE_ICEBERG_FIELD_COMPOUND(data_file, record_count); DEFINE_ICEBERG_FIELD_COMPOUND(data_file, file_size_in_bytes); +DEFINE_ICEBERG_FIELD_COMPOUND(data_file, key_metadata); /// Fallback defaults for snapshot retention policy when table properties are absent. /// These values follow the Java reference implementation; the Iceberg spec does not diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp index a7bd62ccd110..71df1671aaa8 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp @@ -458,6 +458,43 @@ bool IcebergMetadata::optimize( } } +bool IcebergMetadata::optimizeManifestFiles( + const StorageMetadataPtr & metadata_snapshot, + ContextPtr context, + std::shared_ptr catalog, + const StorageID & storage_id) +{ + if (context->getSettingsRef()[Setting::allow_experimental_iceberg_compaction]) + { + /// Reject manifest compaction on format-version 3: the writer does not yet round-trip the row-lineage `first_row_id`, so a rewrite would drop row ids (fail-close). + if (persistent_components.format_version >= 3) + throw Exception( + ErrorCodes::NOT_IMPLEMENTED, + "OPTIMIZE TABLE ... MANIFEST is not yet supported for Iceberg format-version 3: " + "row-lineage 'first_row_id' round-trip is not implemented"); + + const auto sample_block = std::make_shared(metadata_snapshot->getSampleBlock()); + + // Perform manifest-only compaction using the current snapshot from the metadata file + compactIcebergManifests( + persistent_components, + object_storage, + data_lake_settings, + sample_block, + context, + write_format, + catalog, + storage_id); + + return true; + } + else + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, "Enable 'allow_experimental_iceberg_compaction' setting to call OPTIMIZE TABLE ... MANIFEST for iceberg tables."); + } +} + std::pair IcebergMetadata::getStateImpl(const ContextPtr & local_context, Poco::JSON::Object::Ptr metadata_object) const { diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h index 96d683476294..03e062f714cd 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h @@ -140,6 +140,11 @@ class IcebergMetadata : public IDataLakeMetadata CompressionMethod getCompressionMethod() const { return persistent_components.metadata_compression_method; } bool optimize(const StorageMetadataPtr & metadata_snapshot, ContextPtr context, const std::optional & format_settings) override; + bool optimizeManifestFiles( + const StorageMetadataPtr & metadata_snapshot, + ContextPtr context, + std::shared_ptr catalog, + const StorageID & storage_id); bool supportsDelete() const override { return true; } void mutate( const MutationCommands & commands, diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadataFilesCache.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadataFilesCache.h index 11b5f7233088..32fcc97d435d 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadataFilesCache.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadataFilesCache.h @@ -59,6 +59,8 @@ struct ManifestFileCacheKey Int64 added_sequence_number; Int64 added_snapshot_id; Iceberg::ManifestFileContentType content_type; + /// Partition spec the manifest was written with, needed to rewrite each manifest under its own spec during compaction after partition evolution. + Int32 partition_spec_id; }; using ManifestFileCacheKeys = std::vector; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp index 7a267e3b2b6d..c5770881ae1a 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp @@ -50,10 +50,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -111,8 +113,7 @@ namespace FailPoints static constexpr auto MAX_TRANSACTION_RETRIES = 100; // NOLINTBEGIN(clang-analyzer-core.uninitialized.UndefReturn) -// We work a lot with avro library. Clang analyzer is about GenericDatum structure. It thinks that value in generic datum can be uninitialized. -// No idea why +// Clang analyzer wrongly thinks the avro GenericDatum value can be uninitialized. namespace { @@ -140,6 +141,22 @@ bool canDumpIcebergStats(const Field & field, DataTypePtr type) } } +/// Whether a float/double partition value is NaN, which the manifest-list partition summary records via `contains_nan` rather than as ordered lower/upper bounds. +bool isNaNPartitionValue(const Field & field, DataTypePtr type) +{ + switch (type->getTypeId()) + { + case TypeIndex::Nullable: + return !field.isNull() + && isNaNPartitionValue(field, assert_cast(type.get())->getNestedType()); + case TypeIndex::Float32: + case TypeIndex::Float64: + return !field.isNull() && std::isnan(field.safeGet()); + default: + return false; + } +} + template std::vector dumpValue(T value) { @@ -204,6 +221,14 @@ std::vector dumpFieldToBytes(const Field & field, DataTypePtr type) return dumpValue(field.safeGet()); case TypeIndex::Time64: return dumpValue(getTimeValueInMicroseconds(field, type)); + case TypeIndex::UInt8: + case TypeIndex::Int8: + case TypeIndex::UInt16: + case TypeIndex::Int16: + case TypeIndex::UInt32: + return dumpValue(static_cast(applyVisitor(FieldVisitorConvertToNumber(), field))); + case TypeIndex::UInt64: + return dumpValue(applyVisitor(FieldVisitorConvertToNumber(), field)); case TypeIndex::DateTime64: return dumpValue(field.safeGet().getValue().value); case TypeIndex::String: @@ -267,6 +292,13 @@ String removeEscapedSlashes(const String & json_str) return result; } +String stringifyJSON(const Poco::Dynamic::Var & json, unsigned indent) +{ + std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + Poco::JSON::Stringifier::stringify(json, oss, indent); + return removeEscapedSlashes(oss.str()); +} + static void extendSchemaForPartitions( String & schema, const std::vector & partition_columns, @@ -321,6 +353,43 @@ static void extendSchemaForPartitions( } } +namespace +{ +void setVersionedField(avro::GenericRecord & rec, const auto & value, const String & field_name) +{ + size_t field_index = rec.fieldIndex(field_name); + const avro::NodePtr & field_schema = rec.schema()->leafAt(static_cast(field_index)); + + if (field_schema->type() == avro::AVRO_UNION) + { + avro::GenericUnion field(field_schema); + field.selectBranch(1); + field.datum() = avro::GenericDatum(value); + rec.fieldAt(field_index) = avro::GenericDatum(field_schema, field); + } + else + { + rec.fieldAt(field_index) = avro::GenericDatum(value); + } +} + +Poco::JSON::Object::Ptr getCurrentSchema(const Poco::JSON::Object::Ptr & metadata) +{ + Int32 current_schema_id = metadata->getValue(Iceberg::f_current_schema_id); + auto schemas = metadata->getArray(Iceberg::f_schemas); + for (size_t i = 0; i < schemas->size(); ++i) + { + auto schema = schemas->getObject(static_cast(i)); + if (schema->getValue(Iceberg::f_schema_id) == current_schema_id) + return schema; + } + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Not found schema with current-schema-id {} in the schemas list", + current_schema_id); +} +} + void generateManifestFile( Poco::JSON::Object::Ptr metadata, const std::vector & partition_columns, @@ -337,8 +406,25 @@ void generateManifestFile( Int64 partition_spec_id, WriteBuffer & buf, Iceberg::FileContentType content_type, - std::optional user_defined_sequence_number) + std::optional user_defined_sequence_number, + const std::vector & data_file_formats, + const std::vector & per_file_statistics, + const std::vector> & data_file_sort_order_ids, + const std::vector & per_file_entry_lineage, + Poco::JSON::Object::Ptr schema_to_serialize) { + chassert( + data_file_formats.empty() || data_file_formats.size() == data_file_names.size(), + "data_file_formats size does not match number of data files"); + chassert( + per_file_statistics.empty() || per_file_statistics.size() == data_file_names.size(), + "per_file_statistics size does not match number of data files"); + chassert( + data_file_sort_order_ids.empty() || data_file_sort_order_ids.size() == data_file_names.size(), + "data_file_sort_order_ids size does not match number of data files"); + chassert( + per_file_entry_lineage.empty() || per_file_entry_lineage.size() == data_file_names.size(), + "per_file_entry_lineage size does not match number of data files"); Int32 version = metadata->getValue(Iceberg::f_format_version); String schema_representation; if (version == 1) @@ -356,10 +442,8 @@ void generateManifestFile( if (root_schema->type() != avro::AVRO_RECORD) throw Exception(ErrorCodes::LOGICAL_ERROR, "Iceberg manifest file schema must be record"); - std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM - int current_schema_id = metadata->getValue(Iceberg::f_current_schema_id); - Poco::JSON::Stringifier::stringify(metadata->getArray(Iceberg::f_schemas)->getObject(current_schema_id), oss, 4); - std::string json_representation = removeEscapedSlashes(oss.str()); + Poco::JSON::Object::Ptr schema_object_to_write = schema_to_serialize ? schema_to_serialize : getCurrentSchema(metadata); + std::string json_representation = stringifyJSON(schema_object_to_write, 4); auto adapter = std::make_unique(buf); avro::DataFileWriter writer(std::move(adapter), schema); @@ -377,8 +461,16 @@ void generateManifestFile( avro::GenericDatum manifest_datum(root_schema); avro::GenericRecord & manifest = manifest_datum.value(); - manifest.field(Iceberg::f_status) = avro::GenericDatum(1); - Int64 snapshot_id = new_snapshot->getValue(Iceberg::f_metadata_snapshot_id); + /// A metadata-only rewrite (non-empty per_file_entry_lineage) writes each entry as EXISTING, keeping the snapshot-id and sequence number that originally added the file rather than re-stamping it as ADDED. + const DataFileEntryLineage * entry_lineage + = per_file_entry_lineage.empty() ? nullptr : &per_file_entry_lineage[file_idx]; + + manifest.field(Iceberg::f_status) + = avro::GenericDatum(entry_lineage ? static_cast(ManifestEntryStatus::EXISTING) + : static_cast(ManifestEntryStatus::ADDED)); + Int64 snapshot_id = (entry_lineage && entry_lineage->added_snapshot_id) + ? *entry_lineage->added_snapshot_id + : new_snapshot->getValue(Iceberg::f_metadata_snapshot_id); auto set_versioned_field = [&](const auto & value, const String & field_name) { @@ -404,36 +496,59 @@ void generateManifestFile( if (version > 1) { - Int64 sequence_number = user_defined_sequence_number.value_or(new_snapshot->getValue(Iceberg::f_metadata_sequence_number)); - - set_versioned_field(sequence_number, Iceberg::f_sequence_number); - set_versioned_field(sequence_number, Iceberg::f_file_sequence_number); + Int64 sequence_number = (entry_lineage && entry_lineage->sequence_number) + ? *entry_lineage->sequence_number + : user_defined_sequence_number.value_or(new_snapshot->getValue(Iceberg::f_metadata_sequence_number)); + + /// A manifest-only rewrite preserves the source entry's `file_sequence_number`, which can differ from the data + /// `sequence_number`; for a genuinely new file there is no lineage and it equals the data sequence number. + Int64 file_sequence_number = (entry_lineage && entry_lineage->file_sequence_number) + ? *entry_lineage->file_sequence_number + : sequence_number; + + setVersionedField(manifest, sequence_number, Iceberg::f_sequence_number); + setVersionedField(manifest, file_sequence_number, Iceberg::f_file_sequence_number); } avro::GenericRecord & data_file = manifest.field(Iceberg::f_data_file).value(); if (version > 1) data_file.field(Iceberg::f_content) = avro::GenericDatum(static_cast(content_type)); data_file.field(Iceberg::f_file_path) = avro::GenericDatum(data_file_name.serialize()); - data_file.field(Iceberg::f_file_format) = avro::GenericDatum(format); + data_file.field(Iceberg::f_file_format) + = avro::GenericDatum(data_file_formats.empty() ? format : data_file_formats[file_idx]); - if (data_file_statistics) + /// Writes (field-id, value) pairs into the union-typed `field_name` array of the data_file record. + auto set_fields = [&]( + const std::vector> & statistics, const std::string & field_name, U && dump_function) { - auto set_fields = [&]( - const std::vector> & statistics, const std::string & field_name, U && dump_function) + auto & data_file_record = data_file.field(field_name); + data_file_record.selectBranch(1); + auto & record_values = data_file_record.value(); + auto schema_element = record_values.schema()->leafAt(0); + for (const auto & [field_id, value] : statistics) { - auto & data_file_record = data_file.field(field_name); - data_file_record.selectBranch(1); - auto & record_values = data_file_record.value(); - auto schema_element = record_values.schema()->leafAt(0); - for (const auto & [field_id, value] : statistics) - { - avro::GenericDatum record_datum(schema_element); - auto & record = record_datum.value(); - record.field(Iceberg::f_key) = static_cast(field_id); - record.field(Iceberg::f_value) = dump_function(field_id, value); - record_values.value().push_back(record_datum); - } - }; + avro::GenericDatum record_datum(schema_element); + auto & record = record_datum.value(); + record.field(Iceberg::f_key) = static_cast(field_id); + record.field(Iceberg::f_value) = dump_function(field_id, value); + record_values.value().push_back(record_datum); + } + }; + if (!per_file_statistics.empty()) + { + /// Manifest-only rewrite: carry over the source file's column stats verbatim. + const auto & stats = per_file_statistics[file_idx]; + /// Bounds are raw bytes; convert to std::vector to produce an Avro `bytes` datum. + auto to_bytes = [](Int32, const String & value) + { return std::vector(value.begin(), value.end()); }; + set_fields(stats.column_sizes, Iceberg::f_column_sizes, [](Int32, Int64 value) { return value; }); + set_fields(stats.value_counts, Iceberg::f_value_counts, [](Int32, Int64 value) { return value; }); + set_fields(stats.null_value_counts, Iceberg::f_null_value_counts, [](Int32, Int64 value) { return value; }); + set_fields(stats.lower_bounds, Iceberg::f_lower_bounds, to_bytes); + set_fields(stats.upper_bounds, Iceberg::f_upper_bounds, to_bytes); + } + else if (data_file_statistics) + { auto statistics = data_file_statistics->getColumnSizes(); set_fields(statistics, Iceberg::f_column_sizes, [](size_t, size_t value) { return static_cast(value); }); @@ -461,11 +576,19 @@ void generateManifestFile( } data_file.field(Iceberg::f_record_count) = avro::GenericDatum(static_cast(data_file_row_counts[file_idx])); data_file.field(Iceberg::f_file_size_in_bytes) = avro::GenericDatum(static_cast(data_file_byte_counts[file_idx])); + + /// Preserve the source file's sort_order_id. + if (!data_file_sort_order_ids.empty() && data_file_sort_order_ids[file_idx].has_value()) + { + auto & sort_order_field = data_file.field(Iceberg::f_sort_order_id); + sort_order_field.selectBranch(1); + sort_order_field.value() = *data_file_sort_order_ids[file_idx]; + } + avro::GenericRecord & partition_record = data_file.field("partition").value(); for (size_t i = 0; i < partition_columns.size(); ++i) { - /// Build the Avro datum that holds the actual partition value (without - /// the surrounding union). Throws on an unsupported value type. + /// Build the Avro datum holding the partition value; throws on an unsupported type. auto make_value_datum = [&]() -> avro::GenericDatum { auto partition_time_type = getTimeTypeOrNull(partition_types[i]); @@ -498,10 +621,7 @@ void generateManifestFile( if (is_nullable_partition) { - /// Nullable partition columns are encoded as Avro `["null", T]` - /// unions. NULL selects branch 0; a concrete value selects branch 1. - /// See issue #105852: before this change, NULL partition values were - /// silently written as 0 because the schema was non-nullable. + /// Nullable partition columns are Avro `["null", T]` unions: NULL is branch 0, a value is branch 1. size_t field_index = 0; if (!partition_record.schema()->nameIndex(partition_columns[i], field_index)) throw Exception( @@ -549,8 +669,27 @@ void generateManifestList( const std::vector & manifest_entry_sizes, WriteBuffer & buf, Iceberg::FileContentType content_type, - bool use_previous_snapshots) + bool use_previous_snapshots, + const std::vector & per_entry_content_types, + const std::vector & existing_entry_counts, + const std::unordered_set & carry_forward_manifest_paths, + const std::vector & entry_partition_spec_ids, + const std::vector>> & entry_partition_summaries) { + chassert( + per_entry_content_types.empty() || per_entry_content_types.size() == manifest_entry_names.size(), + "per_entry_content_types size does not match number of manifest entries"); + chassert( + entry_partition_spec_ids.empty() || entry_partition_spec_ids.size() == manifest_entry_names.size(), + "entry_partition_spec_ids size does not match number of manifest entries"); + chassert( + entry_partition_summaries.empty() || entry_partition_summaries.size() == manifest_entry_names.size(), + "entry_partition_summaries size does not match number of manifest entries"); + /// When provided, existing_entry_counts marks a manifest-only rewrite and supplies per-entry counts. + chassert( + existing_entry_counts.empty() || existing_entry_counts.size() == manifest_entry_names.size(), + "existing_entry_counts size does not match number of manifest entries"); + const bool manifest_only_rewrite = !existing_entry_counts.empty(); Int32 version = metadata->getValue(Iceberg::f_format_version); String schema_representation; if (version == 1) @@ -569,14 +708,22 @@ void generateManifestList( avro::GenericDatum entry_datum(schema.root()); avro::GenericRecord & entry = entry_datum.value(); + const Iceberg::FileContentType entry_content + = per_entry_content_types.empty() ? content_type : per_entry_content_types[entry_idx]; + entry.field(Iceberg::f_manifest_path) = manifest_entry_names[entry_idx].serialize(); entry.field(Iceberg::f_manifest_length) = manifest_entry_sizes[entry_idx]; - entry.field(Iceberg::f_partition_spec_id) = metadata->getValue(Iceberg::f_default_spec_id); + entry.field(Iceberg::f_partition_spec_id) = entry_partition_spec_ids.empty() + ? metadata->getValue(Iceberg::f_default_spec_id) + : entry_partition_spec_ids[entry_idx]; if (version > 1) { - entry.field(Iceberg::f_content) = static_cast(content_type); - entry.field(Iceberg::f_sequence_number) = new_snapshot->getValue(Iceberg::f_metadata_sequence_number); - entry.field(Iceberg::f_min_sequence_number) = new_snapshot->getValue(Iceberg::f_metadata_sequence_number); + entry.field(Iceberg::f_content) = static_cast(entry_content); + /// For a manifest-only rewrite, min_sequence_number is the per-manifest minimum of the preserved original sequence numbers. + const Int64 new_sequence_number = new_snapshot->getValue(Iceberg::f_metadata_sequence_number); + entry.field(Iceberg::f_sequence_number) = new_sequence_number; + entry.field(Iceberg::f_min_sequence_number) + = manifest_only_rewrite ? existing_entry_counts[entry_idx].min_sequence_number : new_sequence_number; } auto set_versioned_field = [&](const auto & value, const String & field_name) @@ -601,6 +748,59 @@ void generateManifestList( }; entry.field(Iceberg::f_added_snapshot_id) = new_snapshot->getValue(Iceberg::f_metadata_snapshot_id); auto summary = new_snapshot->getObject(Iceberg::f_summary); + if (manifest_only_rewrite) + { + /// Manifest-only rewrite (`replace`): data files already existed, so they are reported as existing, not added. + const auto & counts = existing_entry_counts[entry_idx]; + setVersionedField(entry, 0, Iceberg::f_added_files_count); + setVersionedField(entry, counts.existing_files_count, Iceberg::f_existing_files_count); + setVersionedField(entry, 0, Iceberg::f_deleted_files_count); + setVersionedField(entry, 0, Iceberg::f_added_rows_count); + setVersionedField(entry, counts.existing_rows_count, Iceberg::f_existing_rows_count); + setVersionedField(entry, 0, Iceberg::f_deleted_rows_count); + + /// Recompute the `partitions` summary so pruning bounds survive the rewrite (lower_bound == upper_bound per field). + if (!entry_partition_summaries.empty()) + { + auto & partitions_field = entry.field(Iceberg::f_partitions); + partitions_field.selectBranch(1); + auto & summaries = partitions_field.value(); + auto summary_schema = summaries.schema()->leafAt(0); + for (const auto & [partition_value, partition_type] : entry_partition_summaries[entry_idx]) + { + avro::GenericDatum summary_datum(summary_schema); + auto & summary_record = summary_datum.value(); + const bool is_null = partition_value.isNull(); + summary_record.field(Iceberg::f_contains_null) = avro::GenericDatum(is_null); + if (!is_null) + { + if (isNaNPartitionValue(partition_value, partition_type)) + { + /// NaN float/double partition value: record it via `contains_nan` instead of publishing the NaN bytes as ordered bounds. + auto & contains_nan = summary_record.field(Iceberg::f_contains_nan); + contains_nan.selectBranch(1); + contains_nan.value() = true; + } + else if (canDumpIcebergStats(partition_value, partition_type)) + { + auto bound = dumpFieldToBytes(partition_value, partition_type); + auto & lower = summary_record.field(Iceberg::f_lower_bound); + lower.selectBranch(1); + lower.value>() = bound; + auto & upper = summary_record.field(Iceberg::f_upper_bound); + upper.selectBranch(1); + upper.value>() = bound; + } + /// else: a partition type whose bounds we cannot serialize (e.g. Decimal); leave the bounds null, matching the data-file statistics path. + } + summaries.value().push_back(summary_datum); + } + } + + writer.write(entry_datum); + continue; + } + if (version == 1) { set_versioned_field(1, Iceberg::f_added_files_count); @@ -638,7 +838,8 @@ void generateManifestList( writer.write(entry_datum); } - if (use_previous_snapshots) + /// Copy entries from the parent snapshot's manifest list: `use_previous_snapshots` copies all, `carry_forward_manifest_paths` copies only the listed manifests. + if (use_previous_snapshots || !carry_forward_manifest_paths.empty()) { auto parent_snapshot_id = new_snapshot->getValue(Iceberg::f_parent_snapshot_id); auto snapshots = metadata->getArray(Iceberg::f_snapshots); @@ -654,15 +855,16 @@ void generateManifestList( [&](const avro::GenericDatum & datum) { const avro::GenericRecord & old_entry = datum.value(); + /// When a path filter is supplied, copy only the matching entries. + if (!carry_forward_manifest_paths.empty() + && !carry_forward_manifest_paths.contains(old_entry.field(Iceberg::f_manifest_path).value())) + return; avro::GenericDatum new_datum(schema.root()); avro::GenericRecord & new_entry = new_datum.value(); new_entry.field(f_manifest_path) = old_entry.field(Iceberg::f_manifest_path); new_entry.field(f_manifest_length) = old_entry.field(Iceberg::f_manifest_length); new_entry.field(f_partition_spec_id) = old_entry.field(Iceberg::f_partition_spec_id); - /// In some version, iceberg-spark has changed the type of field `f_added_snapshot_id` - /// from 'null, long' to 'long'. See https://github.com/apache/iceberg/pull/11626. - /// Just in case that we read the old type 'null, long', we do this conversion: read every field - /// and write it again with new, correct schema. + /// iceberg-spark changed `f_added_snapshot_id` from 'null, long' to 'long' (apache/iceberg#11626); rewrite with the new schema in case we read the old type. if (old_entry.hasField(Iceberg::f_added_snapshot_id)) { const avro::GenericDatum & old_added_snapshot_id_entry = old_entry.field(Iceberg::f_added_snapshot_id); @@ -700,7 +902,8 @@ void generateManifestList( add_field_to_datum(Iceberg::f_existing_rows_count); add_field_to_datum(Iceberg::f_deleted_rows_count); add_field_to_datum(Iceberg::f_key_metadata); - if (version == 2) + /// v2 and v3 share the manifest-list schema, so these fields exist for both. + if (version > 1) { add_field_to_datum(Iceberg::f_content); add_field_to_datum(Iceberg::f_sequence_number); @@ -1172,10 +1375,7 @@ bool IcebergStorageSink::initializeMetadata() } } - /// If there's an active metadata cache, we can't just cache 'our' written version as - /// latest, because it could've been overwritten by a concurrent catalog update. - /// We safely invalidate the cache, and the very next reader gets the most up-to-date - /// latest version. See `PersistentTableComponents::invalidateMetadataCache`. + /// Invalidate the cache so the next reader gets the latest version, which a concurrent catalog update may have changed. persistent_table_components.invalidateMetadataCache(); } catch (...) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.h index f25c77baef8d..b505486b7fc2 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -44,6 +45,25 @@ namespace DB String removeEscapedSlashes(const String & json_str); +String stringifyJSON(const Poco::Dynamic::Var & json, unsigned indent = 0); + +/// Per-file column statistics carried over verbatim from a source manifest entry during a manifest-only rewrite. +struct DataFileColumnStatistics +{ + std::vector> column_sizes; + std::vector> value_counts; + std::vector> null_value_counts; + std::vector> lower_bounds; + std::vector> upper_bounds; +}; + +/// Per-file manifest-entry lineage (`added_snapshot_id`, data `sequence_number` and `file_sequence_number`) carried over for a manifest-only rewrite. +struct DataFileEntryLineage +{ + std::optional added_snapshot_id; + std::optional sequence_number; + std::optional file_sequence_number; +}; void generateManifestFile( Poco::JSON::Object::Ptr metadata, const std::vector & partition_columns, @@ -60,7 +80,26 @@ void generateManifestFile( Int64 partition_spec_id, WriteBuffer & buf, Iceberg::FileContentType content_type, - std::optional user_defined_sequence_number = std::nullopt); + std::optional user_defined_sequence_number = std::nullopt, + /// Optional per-file formats parallel to `data_file_names`; when non-empty each entry's original `file_format` is preserved, else `format` is used. + const std::vector & data_file_formats = {}, + /// Optional per-file column statistics parallel to `data_file_names`; when non-empty each entry's stats come from the matching element, else `data_file_statistics` is used. + const std::vector & per_file_statistics = {}, + /// Optional per-file `sort_order_id` parallel to `data_file_names`; when set it is written back to preserve sortedness, else the field is left null. + const std::vector> & data_file_sort_order_ids = {}, + /// Optional per-file manifest-entry lineage parallel to `data_file_names`; when non-empty entries are written as EXISTING keeping their original snapshot-id and sequence number, else as ADDED by the new snapshot. + const std::vector & per_file_entry_lineage = {}, + /// Optional schema to serialize into the manifest's Avro `schema` header; when null the table's current schema is used. + Poco::JSON::Object::Ptr schema_to_serialize = nullptr); + +/// Per manifest-list entry existing-file/existing-row counts for a manifest-only rewrite, where every referenced data file already existed. +struct ManifestListEntryExistingCounts +{ + Int64 existing_files_count = 0; + Int64 existing_rows_count = 0; + /// Minimum data sequence number across the entries in this manifest, used as the manifest-list `min_sequence_number`. + Int64 min_sequence_number = 0; +}; void generateManifestList( const Iceberg::IcebergPathResolver & path_resolver, @@ -72,7 +111,12 @@ void generateManifestList( const std::vector & manifest_entry_sizes, WriteBuffer & buf, Iceberg::FileContentType content_type, - bool use_previous_snapshots = true); + bool use_previous_snapshots = true, + const std::vector & per_entry_content_types = {}, + const std::vector & existing_entry_counts = {}, + const std::unordered_set & carry_forward_manifest_paths = {}, + const std::vector & entry_partition_spec_ids = {}, + const std::vector>> & entry_partition_summaries = {}); class IcebergStorageSink final : public SinkToStorage { diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h index f7c2ced00bf3..1368c71d0fcb 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h @@ -76,6 +76,7 @@ struct ParsedManifestFileEntry : boost::noncopyable ManifestEntryStatus status; std::optional parsed_sequence_number; + std::optional parsed_file_sequence_number; std::optional parsed_snapshot_id; DB::Row partition_key_value; @@ -100,6 +101,7 @@ struct ParsedManifestFileEntry : boost::noncopyable Int64 row_number_, ManifestEntryStatus status_, std::optional written_sequence_number_, + std::optional written_file_sequence_number_, std::optional written_snapshot_id_, DB::Row partition_key_value_, std::unordered_map columns_infos_, @@ -116,6 +118,7 @@ struct ParsedManifestFileEntry : boost::noncopyable , row_number(row_number_) , status(status_) , parsed_sequence_number(written_sequence_number_) + , parsed_file_sequence_number(written_file_sequence_number_) , parsed_snapshot_id(written_snapshot_id_) , partition_key_value(std::move(partition_key_value_)) , columns_infos(std::move(columns_infos_)) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp index 85f5127c21c4..30590c18dff4 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -18,6 +19,7 @@ namespace DB::ErrorCodes { extern const int BAD_ARGUMENTS; + extern const int ICEBERG_SPECIFICATION_VIOLATION; } @@ -37,6 +39,59 @@ Poco::JSON::Object::Ptr deepCopy(Poco::JSON::Object::Ptr obj) return result.extract(); } +/// Read a numeric `total-*` field from the parent snapshot's summary, returning std::nullopt when absent or null. +std::optional readParentTotal(Poco::JSON::Object::Ptr parent_snapshot, const char * field_name) +{ + if (!parent_snapshot || !parent_snapshot->has(Iceberg::f_summary)) + return std::nullopt; + auto parent_summary = parent_snapshot->getObject(Iceberg::f_summary); + if (!parent_summary || !parent_summary->has(field_name) || parent_summary->isNull(field_name)) + return std::nullopt; + return parse(parent_summary->getValue(field_name)); +} + +/// Write the standard `total-*` counters into `summary` by adding each per-field delta to the corresponding parent value. +void setSnapshotTotals( + Poco::JSON::Object::Ptr summary, + Poco::JSON::Object::Ptr parent_snapshot, + Int64 added_records, + Int64 added_files_size, + Int64 added_data_files, + Int64 added_delete_files, + Int64 added_position_deletes, + Int64 added_equality_deletes) +{ + /// Data totals (records, files size, data files) describe the whole table state. + auto set_data_total = [&](const char * field_name, Int64 added) + { + /// No parent snapshot: this is the base snapshot, so its total is exactly what it adds. + if (!parent_snapshot) + { + summary->set(field_name, std::to_string(added)); + return; + } + /// The parent omits this data total, so the new table-wide total cannot be derived: fail the rewrite instead of corrupting the summary. + auto parent_value = readParentTotal(parent_snapshot, field_name); + if (!parent_value.has_value()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Cannot derive Iceberg snapshot total '{}': the parent snapshot's summary omits it", + field_name); + summary->set(field_name, std::to_string(*parent_value + added)); + }; + /// Delete-family totals: a missing parent counter means "none", so treating it as 0 is safe. + auto set_delete_total = [&](const char * field_name, Int64 added) + { + summary->set(field_name, std::to_string(readParentTotal(parent_snapshot, field_name).value_or(0) + added)); + }; + set_data_total(Iceberg::f_total_records, added_records); + set_data_total(Iceberg::f_total_files_size, added_files_size); + set_data_total(Iceberg::f_total_data_files, added_data_files); + set_delete_total(Iceberg::f_total_delete_files, added_delete_files); + set_delete_total(Iceberg::f_total_position_deletes, added_position_deletes); + set_delete_total(Iceberg::f_total_equality_deletes, added_equality_deletes); +} + bool checkValidSchemaEvolution(Poco::Dynamic::Var old_type, Poco::Dynamic::Var new_type) { if (old_type.isString() && new_type.isString() && old_type.extract() == new_type.extract()) @@ -80,8 +135,7 @@ MetadataGenerator::MetadataGenerator(Poco::JSON::Object::Ptr metadata_object_) Int64 MetadataGenerator::getMaxSequenceNumber() { - /// Use the authoritative top-level field per Iceberg V2 spec. - /// Iterating snapshots is unreliable when catalogs prune snapshot history. + /// Use the authoritative top-level field per Iceberg V2 spec, since iterating snapshots is unreliable when history is pruned. if (metadata_object->has(Iceberg::f_last_sequence_number)) return metadata_object->getValue(Iceberg::f_last_sequence_number); @@ -163,18 +217,15 @@ MetadataGenerator::NextMetadataResult MetadataGenerator::generateNextMetadata( summary->set(Iceberg::f_changed_partition_count, std::to_string(num_partitions)); } - auto sum_with_parent_snapshot = [&](const char * field_name, Int64 snapshot_value) - { - Int64 prev_value = parent_snapshot ? parse(parent_snapshot->getObject(Iceberg::f_summary)->getValue(field_name)) : 0; - summary->set(field_name, std::to_string(prev_value + snapshot_value)); - }; - - sum_with_parent_snapshot(Iceberg::f_total_records, added_records); - sum_with_parent_snapshot(Iceberg::f_total_files_size, added_files_size); - sum_with_parent_snapshot(Iceberg::f_total_data_files, added_files); - sum_with_parent_snapshot(Iceberg::f_total_delete_files, added_delete_files); - sum_with_parent_snapshot(Iceberg::f_total_position_deletes, num_deleted_rows); - sum_with_parent_snapshot(Iceberg::f_total_equality_deletes, 0); + setSnapshotTotals( + summary, + parent_snapshot, + /*added_records=*/added_records, + /*added_files_size=*/added_files_size, + /*added_data_files=*/added_files, + /*added_delete_files=*/added_delete_files, + /*added_position_deletes=*/num_deleted_rows, + /*added_equality_deletes=*/0); new_snapshot->set(Iceberg::f_summary, summary); new_snapshot->set(Iceberg::f_schema_id, metadata_object->getValue(Iceberg::f_current_schema_id)); @@ -236,6 +287,106 @@ MetadataGenerator::NextMetadataResult MetadataGenerator::generateNextMetadata( return {new_snapshot, manifest_list_path}; } +MetadataGenerator::NextMetadataResult MetadataGenerator::generateManifestOnlySnapshot( + FileNamesGenerator & generator, + const Iceberg::IcebergPathFromMetadata & metadata_file_path, + Int64 parent_snapshot_id) +{ + int format_version = metadata_object->getValue(Iceberg::f_format_version); + + /// These arrays are optional per the Iceberg spec, so external metadata may omit them. + /// Seed an empty one (as Array::Ptr so getArray/extract see the right type tag) before use, + /// mirroring `generateNextMetadata`; otherwise appending to a missing log dereferences a null array. + for (const auto * field : {Iceberg::f_metadata_log, Iceberg::f_snapshot_log}) + if (!metadata_object->has(field)) + metadata_object->set(field, Poco::JSON::Array::Ptr(new Poco::JSON::Array)); + + /// A manifest-only rewrite always runs against a table with a current snapshot, so `snapshots` + /// must already be present. Guard the same way as `generateNextMetadata` instead of dereferencing + /// a null array below: with a live parent snapshot a missing `snapshots` list is corrupt metadata. + if (!metadata_object->has(Iceberg::f_snapshots)) + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Metadata has a current snapshot with id {} but no `snapshots` list", + parent_snapshot_id); + + Poco::JSON::Object::Ptr new_snapshot = new Poco::JSON::Object; + if (format_version > 1) + { + auto sequence_number = getMaxSequenceNumber() + 1; + new_snapshot->set(Iceberg::f_metadata_sequence_number, sequence_number); + metadata_object->set(Iceberg::f_last_sequence_number, sequence_number); + } + Int64 snapshot_id = static_cast(dis(gen)); + + auto manifest_list_path = generator.generateManifestListName(snapshot_id, format_version); + new_snapshot->set(Iceberg::f_metadata_snapshot_id, snapshot_id); + new_snapshot->set(Iceberg::f_parent_snapshot_id, parent_snapshot_id); + + auto now = std::chrono::system_clock::now(); + auto ms = duration_cast(now.time_since_epoch()); + Int64 timestamp = ms.count(); + new_snapshot->set(Iceberg::f_timestamp_ms, timestamp); + metadata_object->set(Iceberg::f_last_updated_ms, timestamp); + + auto parent_snapshot = getParentSnapshot(parent_snapshot_id); + + /// Manifest-only rewrite: all added-* deltas are zero so `total-*` counters are inherited unchanged from the parent. + Poco::JSON::Object::Ptr summary = new Poco::JSON::Object; + summary->set(Iceberg::f_operation, Iceberg::f_replace); + summary->set(Iceberg::f_added_data_files, "0"); + summary->set(Iceberg::f_added_records, "0"); + summary->set(Iceberg::f_added_files_size, "0"); + summary->set(Iceberg::f_changed_partition_count, "0"); + + setSnapshotTotals( + summary, + parent_snapshot, + /*added_records=*/0, + /*added_files_size=*/0, + /*added_data_files=*/0, + /*added_delete_files=*/0, + /*added_position_deletes=*/0, + /*added_equality_deletes=*/0); + new_snapshot->set(Iceberg::f_summary, summary); + + new_snapshot->set(Iceberg::f_schema_id, metadata_object->getValue(Iceberg::f_current_schema_id)); + new_snapshot->set(Iceberg::f_manifest_list, manifest_list_path.serialize()); + + metadata_object->getArray(Iceberg::f_snapshots)->add(new_snapshot); + metadata_object->set(Iceberg::f_current_snapshot_id, snapshot_id); + + if (!metadata_object->has(Iceberg::f_refs)) + metadata_object->set(Iceberg::f_refs, new Poco::JSON::Object); + + if (!metadata_object->getObject(Iceberg::f_refs)->has(Iceberg::f_main)) + { + Poco::JSON::Object::Ptr branch = new Poco::JSON::Object; + branch->set(Iceberg::f_metadata_snapshot_id, snapshot_id); + branch->set(Iceberg::f_type, Iceberg::f_branch); + metadata_object->getObject(Iceberg::f_refs)->set(Iceberg::f_main, branch); + } + else + { + metadata_object->getObject(Iceberg::f_refs)->getObject(Iceberg::f_main)->set(Iceberg::f_metadata_snapshot_id, snapshot_id); + } + + { + Poco::JSON::Object::Ptr new_metadata_item = new Poco::JSON::Object; + new_metadata_item->set(Iceberg::f_metadata_file, metadata_file_path.serialize()); + new_metadata_item->set(Iceberg::f_timestamp_ms, timestamp); + metadata_object->getArray(Iceberg::f_metadata_log)->add(new_metadata_item); + } + { + Poco::JSON::Object::Ptr new_snapshot_item = new Poco::JSON::Object; + new_snapshot_item->set(Iceberg::f_metadata_snapshot_id, snapshot_id); + new_snapshot_item->set(Iceberg::f_timestamp_ms, timestamp); + metadata_object->getArray(Iceberg::f_snapshot_log)->add(new_snapshot_item); + } + + return {new_snapshot, manifest_list_path}; +} + void MetadataGenerator::generateDropColumnMetadata(const String & column_name) { auto current_schema_id = metadata_object->getValue(Iceberg::f_current_schema_id); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.h index de7cbc86d99f..d7acc1b73d39 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.h @@ -22,9 +22,7 @@ class MetadataGenerator struct NextMetadataResult { Poco::JSON::Object::Ptr snapshot = nullptr; - /// Metadata path for the manifest list file (e.g. "wasb://container@account/table/metadata/snap-xxx.avro"). - /// Use IcebergPathResolver::resolve to get storage path for I/O. - /// Use .serialize() to get the path for writing into Iceberg metadata. + /// Metadata path for the manifest list file; resolve for I/O, serialize for writing into Iceberg metadata. Iceberg::IcebergPathFromMetadata manifest_list_path; }; @@ -41,6 +39,12 @@ class MetadataGenerator std::optional user_defined_snapshot_id = std::nullopt, std::optional user_defined_timestamp = std::nullopt); + /// Create a manifest-only rewrite snapshot (`replace` operation) carrying `total-*` counters forward so `OPTIMIZE ... MANIFEST` is idempotent. + NextMetadataResult generateManifestOnlySnapshot( + FileNamesGenerator & generator, + const Iceberg::IcebergPathFromMetadata & metadata_file_path, + Int64 parent_snapshot_id); + void generateAddColumnMetadata(const String & column_name, DataTypePtr type); void generateDropColumnMetadata(const String & column_name); void generateModifyColumnMetadata(const String & column_name, DataTypePtr type); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/StatelessMetadataFileGetter.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/StatelessMetadataFileGetter.cpp index 045470229dc9..d3daafa2c562 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/StatelessMetadataFileGetter.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/StatelessMetadataFileGetter.cpp @@ -208,8 +208,17 @@ ManifestFileCacheKeys getManifestList( content_type = Iceberg::ManifestFileContentType( manifest_list_deserializer.getValueFromRowByName(i, f_content, TypeIndex::Int32).safeGet()); } + if (!manifest_list_deserializer.hasPath(f_partition_spec_id)) + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Manifest list entry at index {} is missing required field '{}'", + i, + f_partition_spec_id); + Int32 partition_spec_id = static_cast( + manifest_list_deserializer.getValueFromRowByName(i, f_partition_spec_id, TypeIndex::Int32).safeGet()); manifest_file_cache_keys.emplace_back( - manifest_file_name, manifest_length, added_sequence_number, added_snapshot_id.safeGet(), content_type); + manifest_file_name, manifest_length, added_sequence_number, added_snapshot_id.safeGet(), content_type, + partition_spec_id); insertRowToLogTable( local_context, diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.h b/src/Storages/ObjectStorage/StorageObjectStorage.h index 6b8ac470f8ec..a2793a5654d4 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.h +++ b/src/Storages/ObjectStorage/StorageObjectStorage.h @@ -150,6 +150,8 @@ class StorageObjectStorage : public IStorage, public IBackgroundOperation IDataLakeMetadata * getExternalMetadata(ContextPtr query_context); + std::shared_ptr getCatalog() const { return catalog; } + std::optional totalRows(ContextPtr query_context) const override; std::optional totalBytes(ContextPtr query_context) const override; diff --git a/tests/integration/test_database_iceberg/test.py b/tests/integration/test_database_iceberg/test.py index 5d233f89d7ca..b1f70da2213c 100644 --- a/tests/integration/test_database_iceberg/test.py +++ b/tests/integration/test_database_iceberg/test.py @@ -745,6 +745,109 @@ def test_insert(started_cluster): assert node.query(f"SELECT * FROM {CATALOG_NAME}.`{root_namespace}.{table_name}` ORDER BY ALL") == "\\N\tAAPL\t193.24\t193.31\t('bot')\n\\N\tPavel Ivanov (pudge1000-7) pereezhai v amsterdam\t193.24\t193.31\t('bot')\n" +def test_optimize_manifest_with_catalog(started_cluster): + # OPTIMIZE TABLE ... MANIFEST on a catalog-managed table must consolidate the per-insert manifests + # and commit the new snapshot back through the catalog, without changing the data. + node = started_cluster.instances["node1"] + + test_ref = f"test_optimize_manifest_{uuid.uuid4()}" + table_name = f"{test_ref}_table" + root_namespace = f"{test_ref}_namespace" + + catalog = load_catalog_impl(started_cluster) + catalog.create_namespace(root_namespace) + # Unpartitioned table, so every per-insert data manifest can consolidate into a single one. + create_table(catalog, root_namespace, table_name, DEFAULT_SCHEMA, PartitionSpec(), DEFAULT_SORT_ORDER) + + create_clickhouse_iceberg_database(started_cluster, node, CATALOG_NAME) + + table_ref = f"{CATALOG_NAME}.`{root_namespace}.{table_name}`" + write_settings = {"allow_insert_into_iceberg": 1, "write_full_path_in_iceberg_metadata": 1} + + # Several separate inserts -> several snapshots, each adding its own data manifest. + num_inserts = 5 + for i in range(num_inserts): + node.query( + f"INSERT INTO {table_ref} VALUES (NULL, 'sym{i}', {100 + i}, {200 + i}, tuple('bot'));", + settings=write_settings, + ) + + def current_snapshot_id(): + # Read the current snapshot from the catalog's metadata.json (avoids parsing the manifest-list + # Avro, which pyiceberg rejects because ClickHouse omits field-ids there). + table = catalog.load_table(f"{root_namespace}.{table_name}") + assert table.current_snapshot() is not None, "expected a current snapshot after inserts" + return table.metadata.current_snapshot_id + + snapshot_id_before = current_snapshot_id() + rows_before = node.query(f"SELECT symbol, bid, ask FROM {table_ref} ORDER BY ALL") + + node.query( + f"OPTIMIZE TABLE {table_ref} MANIFEST", + settings={ + "allow_experimental_iceberg_compaction": 1, + "iceberg_manifest_min_count_to_compact": 2, + "allow_insert_into_iceberg": 1, + "write_full_path_in_iceberg_metadata": 1, + }, + ) + + # The compaction must commit a new (replace) snapshot back through the catalog. + assert current_snapshot_id() != snapshot_id_before, ( + "OPTIMIZE TABLE ... MANIFEST did not commit a new snapshot through the catalog" + ) + + # The metadata-only rewrite must not change the data. + rows_after = node.query(f"SELECT symbol, bid, ask FROM {table_ref} ORDER BY ALL") + assert rows_after == rows_before + + +@pytest.mark.parametrize( + "fields_to_remove", + [ + ["snapshots"], + ["metadata-log"], + ["snapshot-log"], + ["snapshots", "metadata-log", "snapshot-log"], + ], +) +def test_insert_into_table_without_optional_metadata_arrays(started_cluster, fields_to_remove): + # The Iceberg spec marks snapshots / metadata-log / snapshot-log as optional, so external + # engines may create empty-table metadata that omits any of them. Inserting into such a table + # must still succeed instead of aborting in the metadata write path. + node = started_cluster.instances["node1"] + + test_ref = f"test_insert_no_optional_arrays_{uuid.uuid4()}" + table_name = f"{test_ref}_table" + root_namespace = f"{test_ref}_namespace" + + catalog = load_catalog_impl(started_cluster) + catalog.create_namespace(root_namespace) + create_table(catalog, root_namespace, table_name, DEFAULT_SCHEMA, PartitionSpec(), DEFAULT_SORT_ORDER) + + iceberg_table = catalog.load_table(f"{root_namespace}.{table_name}") + assert iceberg_table.metadata_location.startswith("s3://") + metadata_bucket, metadata_key = iceberg_table.metadata_location[len("s3://"):].split("/", 1) + metadata = json.loads(get_file_contents(started_cluster.minio_client, metadata_bucket, metadata_key)) + for field in fields_to_remove: + metadata.pop(field, None) + metadata_bytes = json.dumps(metadata).encode() + started_cluster.minio_client.put_object( + metadata_bucket, + metadata_key, + io.BytesIO(metadata_bytes), + len(metadata_bytes), + content_type="application/json", + ) + + create_clickhouse_iceberg_database(started_cluster, node, CATALOG_NAME) + node.query( + f"INSERT INTO {CATALOG_NAME}.`{root_namespace}.{table_name}` VALUES (NULL, 'AAPL', 193.24, 193.31, tuple('bot'));", + settings={"allow_insert_into_iceberg": 1, "write_full_path_in_iceberg_metadata": 1}, + ) + assert node.query(f"SELECT * FROM {CATALOG_NAME}.`{root_namespace}.{table_name}`") == "\\N\tAAPL\t193.24\t193.31\t('bot')\n" + + def test_create(started_cluster): node = started_cluster.instances["node1"] diff --git a/tests/integration/test_storage_iceberg_with_spark/test_manifest_compaction.py b/tests/integration/test_storage_iceberg_with_spark/test_manifest_compaction.py new file mode 100644 index 000000000000..cb7c4b290080 --- /dev/null +++ b/tests/integration/test_storage_iceberg_with_spark/test_manifest_compaction.py @@ -0,0 +1,1761 @@ +import gzip +import json +import os +import re +import pytest +import threading +from datetime import datetime, timezone +import time + +from helpers.iceberg_utils import ( + create_iceberg_table, + default_upload_directory, + default_download_directory, + get_uuid_str, + get_last_snapshot +) + + +def _open_metadata_file(filepath): + """Open an Iceberg metadata file, transparently handling gzip compression. + + ClickHouse writes compressed metadata with the encoding name (e.g. `gzip`) + embedded in the middle of the file name, e.g. `v.gzip.metadata.json`, + so the filename still ends with `.json`. Detect gzip by the magic bytes + (0x1f 0x8b) to be agnostic to the exact naming convention. + """ + with open(filepath, "rb") as raw: + magic = raw.read(2) + if magic == b"\x1f\x8b": + return gzip.open(filepath, "rt") + return open(filepath, "r") + + +def _metadata_version_from_name(filename): + """Extract the leading metadata version from a metadata file name for tie-breaking. + + Iceberg metadata files are named like `v.metadata.json`, `v.gzip.metadata.json` + or `-.metadata.json`; return (0 if not parseable).""" + match = re.match(r"v?0*(\d+)", filename) + return int(match.group(1)) if match else 0 + + +def _load_latest_metadata(path_to_table): + """Return the parsed latest metadata file. When two files share last-updated-ms, the higher + metadata version wins, so the result is deterministic regardless of os.listdir order.""" + metadata_dir = f"{path_to_table}/metadata/" + best = None + best_key = None + for filename in os.listdir(metadata_dir): + if not filename.endswith(".json"): + continue + with _open_metadata_file(os.path.join(metadata_dir, filename)) as f: + data = json.load(f) + key = (data.get("last-updated-ms", 0), _metadata_version_from_name(filename)) + if best_key is None or key > best_key: + best_key = key + best = data + return best + + +def get_current_snapshot_summary(path_to_table): + """Return the summary dict of the current snapshot from the latest metadata file.""" + best = _load_latest_metadata(path_to_table) + if best is None: + return {} + current_id = best.get("current-snapshot-id") + for snap in best.get("snapshots", []): + if snap.get("snapshot-id") == current_id: + return snap.get("summary", {}) + return {} + + +def get_all_snapshot_ids(path_to_table): + """Return the set of all snapshot-ids recorded in the latest metadata file.""" + best = _load_latest_metadata(path_to_table) + return {snap.get("snapshot-id") for snap in (best or {}).get("snapshots", [])} + + +@pytest.mark.parametrize("format_version", ["2"]) +@pytest.mark.parametrize("storage_type", ["s3"]) +def test_optimize_manifest_files(started_cluster_iceberg_with_spark, storage_type, format_version): + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_optimize_manifests_v" + format_version + "_" + storage_type + "_" + get_uuid_str() + + # Merge-on-read modes are only valid for v2 tables; v1 has no row-level deletes. + if format_version == "2": + tbl_properties = ( + "'format-version' = '2', " + "'write.update.mode' = 'merge-on-read', " + "'write.delete.mode' = 'merge-on-read', " + "'write.merge.mode' = 'merge-on-read'" + ) + else: + tbl_properties = "'format-version' = '1'" + + spark.sql( + f""" + CREATE TABLE {TABLE_NAME} (id long, data string) USING iceberg TBLPROPERTIES ({tbl_properties}) + """ + ) + spark.sql(f"INSERT INTO {TABLE_NAME} select id, char(id + ascii('a')) from range(10, 100)") + + default_upload_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + + create_iceberg_table(storage_type, instance, TABLE_NAME, started_cluster_iceberg_with_spark) + snapshot_id = get_last_snapshot(f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}/") + + assert instance.query(f"SELECT id FROM {TABLE_NAME} ORDER BY id SETTINGS iceberg_snapshot_id = {snapshot_id}") == instance.query( + "SELECT number FROM numbers(10, 90)" + ) + + time.sleep(0.1) + assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == 90 + + spark.sql(f"INSERT INTO {TABLE_NAME} select id, char(id + ascii('a')) from range(100, 200)") + spark.sql(f"INSERT INTO {TABLE_NAME} select id, char(id + ascii('a')) from range(600, 700)") + default_upload_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + spark.sql(f"INSERT INTO {TABLE_NAME} select id, char(id + ascii('a')) from range(200, 300)") + + spark.sql(f"INSERT INTO {TABLE_NAME} select id, char(id + ascii('a')) from range(300, 400)") + spark.sql(f"INSERT INTO {TABLE_NAME} select id, char(id + ascii('a')) from range(400, 500)") + + default_upload_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + spark.sql(f"INSERT INTO {TABLE_NAME} select id, char(id + ascii('a')) from range(600, 700)") + default_upload_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + + instance.query(f"OPTIMIZE TABLE {TABLE_NAME} MANIFEST;", settings={"allow_experimental_iceberg_compaction" : 1}) + + # check that timetravel works with previous snapshot_ids and timestamps + assert instance.query(f"SELECT id FROM {TABLE_NAME} ORDER BY id SETTINGS iceberg_snapshot_id = {snapshot_id}") == instance.query( + "SELECT number FROM numbers(10, 90)" + ) + + instance.query(f"OPTIMIZE TABLE {TABLE_NAME} MANIFEST;", settings={"allow_experimental_iceberg_compaction" : 1}) + + # Verify Spark can still read the table correctly after manifest compaction. + # Total rows: range(10,100) + range(100,200) + range(600,700) + range(200,300) + # + range(300,400) + range(400,500) + range(600,700) = 690 + # (range(600,700) is inserted twice, so it contributes 200 rows.) + default_download_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}/", + f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}/", + ) + spark_rows = spark.read.format("iceberg").load( + f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}" + ).collect() + assert len(spark_rows) == 690 + + spark_ids = sorted(row["id"] for row in spark_rows) + clickhouse_ids = list(map(int, instance.query( + f"SELECT id FROM {TABLE_NAME} ORDER BY id" + ).split())) + assert spark_ids == clickhouse_ids + + +def test_optimize_manifest_files_preserves_stats(started_cluster_iceberg_with_spark): + """ + OPTIMIZE TABLE ... MANIFEST must preserve the per-column statistics carried by the source + manifest entries (column_sizes, value_counts, null_value_counts, lower_bounds, upper_bounds). + Dropping them would weaken predicate pushdown / file pruning after compaction. + """ + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + storage_type = "local" + TABLE_NAME = "test_optimize_manifest_stats_" + storage_type + "_" + get_uuid_str() + + spark.sql( + f"CREATE TABLE {TABLE_NAME} (id long, data string) USING iceberg " + f"TBLPROPERTIES ('format-version' = '2')" + ) + # Several separate inserts so the current snapshot has several data manifests (> threshold). + for lo in range(0, 50, 10): + spark.sql( + f"INSERT INTO {TABLE_NAME} SELECT id, char(id + ascii('a')) FROM range({lo}, {lo + 10})" + ) + + default_upload_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + create_iceberg_table(storage_type, instance, TABLE_NAME, started_cluster_iceberg_with_spark) + + metadata_dir = ( + f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}/metadata" + ) + + def list_data_manifests(): + return set( + instance.exec_in_container( + [ + "bash", + "-c", + f"find '{metadata_dir}' -maxdepth 1 -name '*.avro' " + f"-not -name 'snap-*.avro' -type f", + ] + ) + .strip() + .splitlines() + ) + + manifests_before = list_data_manifests() + + instance.query( + f"OPTIMIZE TABLE {TABLE_NAME} MANIFEST", + settings={ + "allow_experimental_iceberg_compaction": 1, + "iceberg_manifest_min_count_to_compact": 2, + }, + ) + + # The manifest-only rewrite does not delete old files, so the newly written consolidated + # manifest(s) are exactly the data manifests that appeared after the OPTIMIZE. + new_manifests = sorted(list_data_manifests() - manifests_before) + assert new_manifests, "OPTIMIZE TABLE ... MANIFEST did not produce a consolidated manifest" + + entries_checked = 0 + for manifest in new_manifests: + result = instance.query( + f""" + SELECT + tupleElement(data_file, 'content') AS content, + length(tupleElement(data_file, 'column_sizes')) AS n_column_sizes, + length(tupleElement(data_file, 'value_counts')) AS n_value_counts, + length(tupleElement(data_file, 'null_value_counts')) AS n_null_value_counts, + length(tupleElement(data_file, 'lower_bounds')) AS n_lower_bounds, + length(tupleElement(data_file, 'upper_bounds')) AS n_upper_bounds + FROM file('{manifest}', Avro) + FORMAT TSV + """ + ).strip() + if not result: + continue + for line in result.splitlines(): + content, n_col, n_val, n_null, n_lower, n_upper = map(int, line.split("\t")) + if content != 0: # data files only + continue + # The table has two columns (id, data); both carry stats in the source manifests. + assert n_col == 2, f"column_sizes dropped: {n_col}" + assert n_val == 2, f"value_counts dropped: {n_val}" + assert n_null == 2, f"null_value_counts dropped: {n_null}" + assert n_lower == 2, f"lower_bounds dropped: {n_lower}" + assert n_upper == 2, f"upper_bounds dropped: {n_upper}" + entries_checked += 1 + + assert entries_checked > 0, "no data-file entries found in consolidated manifest(s)" + + +def test_optimize_manifest_files_preserves_sort_order_id(started_cluster_iceberg_with_spark): + """ + OPTIMIZE TABLE ... MANIFEST must preserve each data file's sort_order_id. A manifest-only + rewrite does not touch the data files, so a table sorted before compaction must stay sorted + afterwards; dropping sort_order_id would make ClickHouse treat the table as unsorted. + """ + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + storage_type = "local" + TABLE_NAME = "test_optimize_manifest_sortorder_" + storage_type + "_" + get_uuid_str() + + spark.sql( + f"CREATE TABLE {TABLE_NAME} (id long, data string) USING iceberg " + f"TBLPROPERTIES ('format-version' = '2')" + ) + # Establish a sort order, so data files written afterwards carry a non-default sort_order_id. + spark.sql(f"ALTER TABLE {TABLE_NAME} WRITE ORDERED BY id") + for lo in range(0, 50, 10): + spark.sql( + f"INSERT INTO {TABLE_NAME} SELECT id, char(id + ascii('a')) FROM range({lo}, {lo + 10}) ORDER BY id" + ) + + default_upload_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + create_iceberg_table(storage_type, instance, TABLE_NAME, started_cluster_iceberg_with_spark) + + metadata_dir = ( + f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}/metadata" + ) + + def list_data_manifests(): + return set( + instance.exec_in_container( + [ + "bash", + "-c", + f"find '{metadata_dir}' -maxdepth 1 -name '*.avro' " + f"-not -name 'snap-*.avro' -type f", + ] + ) + .strip() + .splitlines() + ) + + def data_file_sort_order_ids(manifests): + ids = set() + for manifest in manifests: + result = instance.query( + f""" + SELECT + tupleElement(data_file, 'content') AS content, + tupleElement(data_file, 'sort_order_id') AS sort_order_id + FROM file('{manifest}', Avro) + FORMAT TSV + """ + ).strip() + for line in result.splitlines(): + if not line: + continue + content, sort_order_id = line.split("\t") + if int(content) != 0: # data files only + continue + ids.add(sort_order_id) + return ids + + manifests_before = list_data_manifests() + source_sort_order_ids = data_file_sort_order_ids(manifests_before) + # The source files must carry a concrete (non-null) sort_order_id for the test to be meaningful. + assert source_sort_order_ids and "\\N" not in source_sort_order_ids, ( + f"expected source data files to have a sort_order_id, got: {source_sort_order_ids}" + ) + + instance.query( + f"OPTIMIZE TABLE {TABLE_NAME} MANIFEST", + settings={ + "allow_experimental_iceberg_compaction": 1, + "iceberg_manifest_min_count_to_compact": 2, + }, + ) + + new_manifests = sorted(list_data_manifests() - manifests_before) + assert new_manifests, "OPTIMIZE TABLE ... MANIFEST did not produce a consolidated manifest" + + # The consolidated manifest must report the same sort_order_id(s) as the source files. + assert data_file_sort_order_ids(new_manifests) == source_sort_order_ids + + +@pytest.mark.parametrize("storage_type", ["s3"]) +def test_optimize_manifest_files_partition_evolution(started_cluster_iceberg_with_spark, storage_type): + """ + OPTIMIZE TABLE ... MANIFEST on a table whose partition spec evolved must rewrite each manifest + under the partition spec its source files were written with, not the default spec. Re-encoding + old partition tuples under the default spec would corrupt partition metadata, so the end-to-end + check is that the data is still correct and Spark (a reference reader) can read it back. + """ + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_optimize_manifest_partevo_" + storage_type + "_" + get_uuid_str() + + # spec 0: partitioned by bucket(4, id). + spark.sql( + f""" + CREATE TABLE {TABLE_NAME} (id long, data string) USING iceberg + PARTITIONED BY (bucket(4, id)) + TBLPROPERTIES ('format-version' = '2') + """ + ) + spark.sql(f"INSERT INTO {TABLE_NAME} SELECT id, char(id + ascii('a')) FROM range(0, 20)") + spark.sql(f"INSERT INTO {TABLE_NAME} SELECT id, char(id + ascii('a')) FROM range(20, 40)") + + # Evolve the partition spec → spec 1. + spark.sql(f"ALTER TABLE {TABLE_NAME} ADD PARTITION FIELD truncate(2, data)") + spark.sql(f"INSERT INTO {TABLE_NAME} SELECT id, char(id + ascii('a')) FROM range(40, 60)") + spark.sql(f"INSERT INTO {TABLE_NAME} SELECT id, char(id + ascii('a')) FROM range(60, 80)") + + default_upload_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + create_iceberg_table(storage_type, instance, TABLE_NAME, started_cluster_iceberg_with_spark) + + assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == 80 + + instance.query( + f"OPTIMIZE TABLE {TABLE_NAME} MANIFEST", + settings={ + "allow_experimental_iceberg_compaction": 1, + "iceberg_manifest_min_count_to_compact": 2, + }, + ) + + # Data is unchanged after the manifest-only rewrite of a partition-evolved table. + assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == 80 + assert instance.query(f"SELECT id FROM {TABLE_NAME} ORDER BY id") == instance.query( + "SELECT number FROM numbers(0, 80)" + ) + + # Spark must still read the table back correctly (partition metadata not corrupted). + default_download_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}/", + f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}/", + ) + spark_rows = spark.read.format("iceberg").load( + f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}" + ).collect() + assert len(spark_rows) == 80 + assert sorted(row["id"] for row in spark_rows) == list(range(0, 80)) + + +@pytest.mark.parametrize("storage_type", ["s3"]) +def test_optimize_manifest_files_dropped_partition_source_column( + started_cluster_iceberg_with_spark, storage_type +): + """ + OPTIMIZE TABLE ... MANIFEST must derive each preserved manifest's partition value types from a + schema that actually defines the spec's source columns, not unconditionally from the current + schema. After partition evolution drops a partition field and the source column itself is then + dropped, the current schema no longer contains that column, yet the current snapshot still + references manifests written under the old spec. Deriving the partition types from the current + schema would throw (the column is absent) or encode the preserved partition tuple under the + wrong type. The end-to-end check is that compaction succeeds and the data is still read back + correctly. + + Note: once the source column is dropped, Spark itself can no longer bind the orphaned spec 0 + (`SerializableTable.specs` eagerly binds every historical spec against the current schema and + throws `Cannot find source column for partition field`), so any Spark write or read of the + table fails. The mixed-spec data is therefore written before dropping the column, and the + post-drop state is verified through ClickHouse only. + """ + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_optimize_manifest_droppedcol_" + storage_type + "_" + get_uuid_str() + + # spec 0: partitioned by identity(region). 'region' is both a partition source and a column. + spark.sql( + f""" + CREATE TABLE {TABLE_NAME} (id long, data string, region string) USING iceberg + PARTITIONED BY (region) + TBLPROPERTIES ('format-version' = '2') + """ + ) + spark.sql( + f"INSERT INTO {TABLE_NAME} VALUES " + f"(0, 'a', 'us'), (1, 'b', 'us'), (2, 'c', 'eu'), (3, 'd', 'eu')" + ) + spark.sql(f"INSERT INTO {TABLE_NAME} VALUES (4, 'e', 'us'), (5, 'f', 'eu')") + + # Evolve: drop the partition field → new unpartitioned spec 1. + spark.sql(f"ALTER TABLE {TABLE_NAME} DROP PARTITION FIELD region") + + # Insert more rows under the new (unpartitioned) spec so specs are mixed. This must happen + # while 'region' still exists, because dropping it leaves spec 0 unbindable by Spark. + spark.sql(f"INSERT INTO {TABLE_NAME} VALUES (6, 'g', 'us'), (7, 'h', 'eu')") + + # Now drop the source column. The current snapshot still references the spec-0 manifests above, + # whose partition source column no longer exists in the current schema. + spark.sql(f"ALTER TABLE {TABLE_NAME} DROP COLUMN region") + + default_upload_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + create_iceberg_table(storage_type, instance, TABLE_NAME, started_cluster_iceberg_with_spark) + + assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == 8 + + # Without resolving the old spec's source-column type from a historical schema this throws. + instance.query( + f"OPTIMIZE TABLE {TABLE_NAME} MANIFEST", + settings={ + "allow_experimental_iceberg_compaction": 1, + "iceberg_manifest_min_count_to_compact": 2, + }, + ) + + # Re-read through ClickHouse after compaction to confirm partition metadata is not corrupted. + # (Spark cannot read this table: it fails to bind the orphaned spec 0 — see the docstring.) + assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == 8 + assert instance.query(f"SELECT id FROM {TABLE_NAME} ORDER BY id") == instance.query( + "SELECT number FROM numbers(0, 8)" + ) + + +@pytest.mark.parametrize("storage_type", ["s3"]) +def test_optimize_manifest_files_dropped_partition_source_column_schema_header( + started_cluster_iceberg_with_spark, storage_type +): + """ + A manifest-only rewrite (`OPTIMIZE TABLE ... MANIFEST`) must serialize into each compacted + manifest's Avro `schema` metadata a schema that still defines the manifest's partition-spec + `source-id`s — not unconditionally the current schema. After partition evolution drops a + partition field and the source column itself is then dropped, the current schema no longer + contains that column. If the rewritten manifest carried the current schema in its `schema` + header, the spec's `source-id`s would no longer resolve on read and `ManifestFileIterator` + would silently drop the partition field, so the manifest would stop faithfully describing the + files it carried forward. + + This is the schema-header regression for the partition-type scenario covered by + `test_optimize_manifest_files_dropped_partition_source_column`: here we additionally read the + Avro container metadata of the newly written manifest back and assert every partition-spec + `source-id` is still present in its `schema` header. + """ + from avro.datafile import DataFileReader + from avro.io import DatumReader + + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_optimize_manifest_droppedcol_schema_" + storage_type + "_" + get_uuid_str() + + # spec 0: partitioned by identity(region). 'region' is both a partition source and a column. + # `commit.manifest-merge.enabled = false` is essential: with Iceberg's default manifest merging + # every spec-0 append would be folded back into a single spec-0 manifest, so the current + # snapshot would carry only one manifest per spec (here 2 total). That is at or below the + # compaction threshold below, and even past it `writeConsolidatedManifestFile` would find the + # manifests already optimal (one per partition). Disabling the merge keeps each append as its + # own manifest, so spec 0 actually accumulates more manifests than partition groups. + spark.sql( + f""" + CREATE TABLE {TABLE_NAME} (id long, data string, region string) USING iceberg + PARTITIONED BY (region) + TBLPROPERTIES ('format-version' = '2', 'commit.manifest-merge.enabled' = 'false') + """ + ) + # Three separate inserts under spec 0 over the same two partitions (us, eu). With manifest + # merging disabled each Spark append writes one manifest, so spec 0 ends up with more manifests + # (3) than it has unique partition groups (2). This is what makes a manifest-only rewrite + # actually consolidate — otherwise `writeConsolidatedManifestFile` finds the manifests already + # optimal (one per partition) and writes nothing, leaving no compacted manifest to inspect the + # `schema` header of. + spark.sql( + f"INSERT INTO {TABLE_NAME} VALUES " + f"(0, 'a', 'us'), (1, 'b', 'us'), (2, 'c', 'eu'), (3, 'd', 'eu')" + ) + spark.sql(f"INSERT INTO {TABLE_NAME} VALUES (4, 'e', 'us'), (5, 'f', 'eu')") + spark.sql(f"INSERT INTO {TABLE_NAME} VALUES (8, 'i', 'us'), (9, 'j', 'eu')") + + # Evolve: drop the partition field → new unpartitioned spec 1. Insert more rows under the new + # spec while 'region' still exists (dropping it leaves spec 0 unbindable by Spark). + spark.sql(f"ALTER TABLE {TABLE_NAME} DROP PARTITION FIELD region") + spark.sql(f"INSERT INTO {TABLE_NAME} VALUES (6, 'g', 'us'), (7, 'h', 'eu')") + + # Now drop the source column. The current snapshot still references the spec-0 manifests above, + # whose partition source column no longer exists in the current schema. + spark.sql(f"ALTER TABLE {TABLE_NAME} DROP COLUMN region") + + default_upload_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + create_iceberg_table(storage_type, instance, TABLE_NAME, started_cluster_iceberg_with_spark) + + table_dir = f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}/" + local_metadata_dir = os.path.join(table_dir, "metadata") + + def download_and_list_manifests(): + # Mirror the table (including the manifests written to object storage) onto the test host so + # the Avro container metadata can be read directly. `snap-*.avro` are manifest lists, not + # manifests, so they are excluded. + default_download_directory( + started_cluster_iceberg_with_spark, storage_type, table_dir, table_dir + ) + return { + os.path.join(local_metadata_dir, name) + for name in os.listdir(local_metadata_dir) + if name.endswith(".avro") and not name.startswith("snap-") + } + + manifests_before = download_and_list_manifests() + + instance.query( + f"OPTIMIZE TABLE {TABLE_NAME} MANIFEST", + settings={ + "allow_experimental_iceberg_compaction": 1, + "iceberg_manifest_min_count_to_compact": 2, + }, + ) + + new_manifests = download_and_list_manifests() - manifests_before + assert new_manifests, "OPTIMIZE TABLE ... MANIFEST did not write any new manifest files" + + def read_avro_user_metadata(path): + with open(path, "rb") as f: + reader = DataFileReader(f, DatumReader()) + try: + decoded = {} + for key, value in reader.meta.items(): + key = key.decode("utf-8") if isinstance(key, bytes) else key + if isinstance(value, bytes): + value = value.decode("utf-8") + decoded[key] = value + return decoded + finally: + reader.close() + + # At least one newly written manifest must carry the old (partitioned) spec, and for every + # partition-spec source-id its `schema` header must still define a field with that id. + checked_partitioned_manifest = False + for manifest_path in sorted(new_manifests): + meta = read_avro_user_metadata(manifest_path) + if "partition-spec" not in meta or "schema" not in meta: + continue + partition_spec = json.loads(meta["partition-spec"]) + if not partition_spec: + # The unpartitioned spec-1 manifest — nothing to resolve. + continue + schema_field_ids = {field["id"] for field in json.loads(meta["schema"])["fields"]} + for spec_field in partition_spec: + source_id = spec_field["source-id"] + assert source_id in schema_field_ids, ( + f"Compacted manifest {os.path.basename(manifest_path)} has a schema header with " + f"field ids {sorted(schema_field_ids)} that does not define partition-spec " + f"source-id {source_id}; the rewrite serialized the current schema instead of one " + f"defining the spec's source columns" + ) + checked_partitioned_manifest = True + + assert checked_partitioned_manifest, ( + "No newly written manifest with a non-empty partition spec was found to verify" + ) + + +@pytest.mark.parametrize("storage_type", ["s3"]) +def test_optimize_manifest_files_bucket_partition(started_cluster_iceberg_with_spark, storage_type): + """ + OPTIMIZE TABLE ... MANIFEST on a bucket-partitioned table must recompute the manifest-list + partition summary for the bucket value. The `icebergBucket` transform resolves to ClickHouse + `UInt32`, which `getAvroType` maps to Avro `int`; the byte encoder must serialize that unsigned + type instead of throwing 'Can not dump such stats', otherwise a valid Iceberg bucket partition + cannot be compacted. + """ + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_optimize_manifest_bucket_" + storage_type + "_" + get_uuid_str() + + spark.sql( + f""" + CREATE TABLE {TABLE_NAME} (id long, data string) USING iceberg + PARTITIONED BY (bucket(4, id)) + TBLPROPERTIES ('format-version' = '2') + """ + ) + for lo in range(0, 80, 20): + spark.sql( + f"INSERT INTO {TABLE_NAME} SELECT id, char(id + ascii('a')) FROM range({lo}, {lo + 20})" + ) + + default_upload_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + create_iceberg_table(storage_type, instance, TABLE_NAME, started_cluster_iceberg_with_spark) + + assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == 80 + + instance.query( + f"OPTIMIZE TABLE {TABLE_NAME} MANIFEST", + settings={ + "allow_experimental_iceberg_compaction": 1, + "iceberg_manifest_min_count_to_compact": 2, + }, + ) + + assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == 80 + assert instance.query(f"SELECT id FROM {TABLE_NAME} ORDER BY id") == instance.query( + "SELECT number FROM numbers(0, 80)" + ) + + # Spark must still read the table back correctly after the bucket-partition manifest rewrite. + default_download_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}/", + f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}/", + ) + spark_rows = spark.read.format("iceberg").load( + f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}" + ).collect() + assert len(spark_rows) == 80 + assert sorted(row["id"] for row in spark_rows) == list(range(0, 80)) + + +def test_optimize_manifest_files_preserves_entry_lineage(started_cluster_iceberg_with_spark): + """ + OPTIMIZE TABLE ... MANIFEST is metadata-only, so each rewritten manifest entry must stay an + EXISTING entry that keeps the snapshot-id and data sequence number that originally added the + file, rather than being re-stamped as ADDED by the new (replace) snapshot. Otherwise the + snapshot is internally inconsistent (the manifest list reports the files as existing) and row + lineage / delete-file sequence-number matching would be corrupted. + """ + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + storage_type = "local" + TABLE_NAME = "test_optimize_manifest_lineage_" + storage_type + "_" + get_uuid_str() + + spark.sql( + f"CREATE TABLE {TABLE_NAME} (id long, data string) USING iceberg " + f"TBLPROPERTIES ('format-version' = '2')" + ) + # Several separate inserts → several snapshots, so files carry distinct original snapshot-ids. + for lo in range(0, 50, 10): + spark.sql( + f"INSERT INTO {TABLE_NAME} SELECT id, char(id + ascii('a')) FROM range({lo}, {lo + 10})" + ) + + default_upload_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + create_iceberg_table(storage_type, instance, TABLE_NAME, started_cluster_iceberg_with_spark) + + table_path = f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}/" + metadata_dir = f"{table_path}metadata" + + def list_data_manifests(): + return set( + instance.exec_in_container( + [ + "bash", + "-c", + f"find '{metadata_dir}' -maxdepth 1 -name '*.avro' " + f"-not -name 'snap-*.avro' -type f", + ] + ) + .strip() + .splitlines() + ) + + manifests_before = list_data_manifests() + # Snapshot-ids that exist before compaction; every preserved entry must reference one of these + # (its original adder), never the brand-new replace snapshot created by the compaction. + original_snapshot_ids = get_all_snapshot_ids(table_path) + assert original_snapshot_ids, "expected the pre-compaction metadata to record snapshots" + + def data_file_basename(path): + return path.rstrip("/").split("/")[-1] + + # Resolved (inheritance-applied) data sequence_number per data file, captured BEFORE compaction and + # keyed by the data file's basename. Reading the manifests raw would return null here: Spark writes + # ADDED entries without an explicit sequence number and Iceberg inherits it from the manifest list at + # read time, so system.iceberg_files (which resolves that inheritance) is the reliable source. + sequence_number_before = {} + rows_before = instance.query( + f""" + SELECT file_path, sequence_number + FROM system.iceberg_files + WHERE database = currentDatabase() AND table = '{TABLE_NAME}' AND content = 'DATA' + FORMAT TSV + """ + ).strip() + for line in rows_before.splitlines(): + file_path, sequence_number = line.split("\t") + sequence_number_before[data_file_basename(file_path)] = sequence_number + assert sequence_number_before, "expected to read pre-compaction data-file sequence numbers" + + instance.query( + f"OPTIMIZE TABLE {TABLE_NAME} MANIFEST", + settings={ + "allow_experimental_iceberg_compaction": 1, + "iceberg_manifest_min_count_to_compact": 2, + }, + ) + + new_manifests = sorted(list_data_manifests() - manifests_before) + assert new_manifests, "OPTIMIZE TABLE ... MANIFEST did not produce a consolidated manifest" + + entries_checked = 0 + for manifest in new_manifests: + result = instance.query( + f""" + SELECT + status, + snapshot_id, + sequence_number, + file_sequence_number, + tupleElement(data_file, 'file_path') AS file_path, + tupleElement(data_file, 'content') AS content + FROM file('{manifest}', Avro) + FORMAT TSV + """ + ).strip() + if not result: + continue + for line in result.splitlines(): + status, snapshot_id, sequence_number, file_sequence_number, file_path, content = line.split("\t") + if int(content) != 0: # data files only + continue + # A metadata-only rewrite carries files forward: entries are EXISTING (status 0), not + # ADDED, so the new snapshot is consistent with the manifest list (which reports them as + # existing) and incremental planning can tell them apart from additions. + assert int(status) == 0, f"expected EXISTING entry (status 0), got {status}" + # The original adding snapshot is preserved: the entry references one of the original + # snapshots, not the brand-new replace snapshot created by the compaction. + assert snapshot_id != "\\N", "snapshot_id must be preserved (non-null)" + assert int(snapshot_id) in original_snapshot_ids, ( + f"entry snapshot_id {snapshot_id} should be an original adder, " + f"not a snapshot created by the compaction" + ) + assert sequence_number != "\\N", "sequence_number must be preserved (non-null)" + assert file_sequence_number != "\\N", "file_sequence_number must be preserved (non-null)" + # The rewrite must carry each file's original sequence numbers forward, not re-stamp them with + # the new (replace) snapshot's sequence number. Compare against the resolved pre-compaction + # values captured above: both the data sequence_number and the file_sequence_number, which for + # these plain-inserted files equal the file's original data sequence number. + key = data_file_basename(file_path) + assert key in sequence_number_before, ( + f"carried-forward file {file_path} was not present before compaction" + ) + assert sequence_number == sequence_number_before[key], ( + f"data sequence_number for {file_path} changed from " + f"{sequence_number_before[key]} to {sequence_number}" + ) + assert file_sequence_number == sequence_number_before[key], ( + f"file_sequence_number for {file_path} changed from " + f"{sequence_number_before[key]} to {file_sequence_number}" + ) + entries_checked += 1 + + assert entries_checked > 0, "no data-file entries found in consolidated manifest(s)" + + +@pytest.mark.parametrize("format_version", ["2"]) +@pytest.mark.parametrize("storage_type", ["s3"]) +def test_optimize_manifest_files_with_deletes(started_cluster_iceberg_with_spark, storage_type, format_version): + """ + OPTIMIZE TABLE ... MANIFEST must preserve delete files. It consolidates only the data + manifests, while delete-file manifests are carried forward unchanged into the new manifest + list. If they were dropped, the previously deleted rows would reappear after compaction. + + Covers v2 (position delete files) row-level deletes. Format-version 3 is rejected by + manifest compaction for now (see test_optimize_manifest_files_v3_rejected), because the + writer does not yet round-trip the v3 row-lineage 'first_row_id' metadata. + """ + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_optimize_manifest_deletes_v" + format_version + "_" + storage_type + "_" + get_uuid_str() + + spark.sql( + f""" + CREATE TABLE {TABLE_NAME} (id long, data string) USING iceberg TBLPROPERTIES ( + 'format-version' = '{format_version}', + 'write.update.mode' = 'merge-on-read', + 'write.delete.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read' + ) + """ + ) + + # Separate inserts so the current snapshot accumulates several data manifests + # (above the compaction threshold set below). + for lo in range(10, 100, 10): + spark.sql( + f"INSERT INTO {TABLE_NAME} select id, char(id + ascii('a')) from range({lo}, {lo + 10})" + ) + + # Merge-on-read delete: produces a row-level delete (position-delete file in v2, deletion + # vector in v3) tracked by a delete-file manifest. + spark.sql(f"DELETE FROM {TABLE_NAME} WHERE id < 20") + + default_upload_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + + create_iceberg_table(storage_type, instance, TABLE_NAME, started_cluster_iceberg_with_spark) + + # 90 inserted, ids 10..19 deleted -> 80 live rows. + assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == 80 + assert instance.query(f"SELECT id FROM {TABLE_NAME} ORDER BY id") == instance.query( + "SELECT number FROM numbers(20, 80)" + ) + + optimize_settings = { + "allow_experimental_iceberg_compaction": 1, + "iceberg_manifest_min_count_to_compact": 2, + } + instance.query(f"OPTIMIZE TABLE {TABLE_NAME} MANIFEST", settings=optimize_settings) + + # The deletes must still be applied after manifest compaction (no resurrected rows). + # ClickHouse reads directly from storage, so no download is needed for these checks. + assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == 80 + assert instance.query(f"SELECT id FROM {TABLE_NAME} ORDER BY id") == instance.query( + "SELECT number FROM numbers(20, 80)" + ) + + # Download the ClickHouse-written metadata/manifests from storage so we can inspect the + # new snapshot summary locally and let Spark read the table back. + default_download_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}/", + f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}/", + ) + + # A manifest-only rewrite must have happened (replace operation). + summary = get_current_snapshot_summary( + f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}/" + ) + assert summary.get("operation") == "replace", ( + f"Expected operation='replace', got: {summary.get('operation')}" + ) + + # Spark must still read the same rows after ClickHouse rewrote the manifests. + spark_rows = spark.read.format("iceberg").load( + f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}" + ).collect() + assert len(spark_rows) == 80 + spark_ids = sorted(row["id"] for row in spark_rows) + assert spark_ids == list(range(20, 100)) + + +@pytest.mark.parametrize("storage_type", ["s3"]) +def test_optimize_manifest_files_v3_rejected(started_cluster_iceberg_with_spark, storage_type): + """ + Format-version 3 adds row lineage: each data file carries an inherited 'first_row_id' from + which readers assign '_row_id'. The manifest writer does not yet round-trip 'first_row_id' + (it uses the v2 Avro schema for v3), so a manifest-only rewrite would carry data files forward + while dropping their row ids, producing a v3 table with a valid-looking snapshot but broken row + lineage. Until the round-trip is implemented, OPTIMIZE TABLE ... MANIFEST must fail loudly on a + v3 table rather than silently corrupt lineage. + """ + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_optimize_manifest_v3_rejected_" + storage_type + "_" + get_uuid_str() + + spark.sql( + f"CREATE TABLE {TABLE_NAME} (id long, data string) USING iceberg " + f"TBLPROPERTIES ('format-version' = '3')" + ) + for lo in range(0, 40, 10): + spark.sql( + f"INSERT INTO {TABLE_NAME} select id, char(id + ascii('a')) from range({lo}, {lo + 10})" + ) + + default_upload_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + create_iceberg_table(storage_type, instance, TABLE_NAME, started_cluster_iceberg_with_spark) + + assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == 40 + + error_message = instance.query_and_get_error( + f"OPTIMIZE TABLE {TABLE_NAME} MANIFEST", + settings={ + "allow_experimental_iceberg_compaction": 1, + "iceberg_manifest_min_count_to_compact": 2, + }, + ) + assert "not yet supported for Iceberg format-version 3" in error_message + + # The rejected compaction must leave the table untouched and still readable. + assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == 40 + + +@pytest.mark.parametrize("storage_type", ["s3"]) +def test_optimize_manifest_files_v1_rejected(started_cluster_iceberg_with_spark, storage_type): + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_optimize_manifest_v1_rejected_" + storage_type + "_" + get_uuid_str() + + spark.sql( + f"CREATE TABLE {TABLE_NAME} (id long, data string) USING iceberg " + f"TBLPROPERTIES ('format-version' = '1')" + ) + for lo in range(0, 40, 10): + spark.sql( + f"INSERT INTO {TABLE_NAME} select id, char(id + ascii('a')) from range({lo}, {lo + 10})" + ) + + default_upload_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + create_iceberg_table(storage_type, instance, TABLE_NAME, started_cluster_iceberg_with_spark) + + assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == 40 + + error_message = instance.query_and_get_error( + f"OPTIMIZE TABLE {TABLE_NAME} MANIFEST", + settings={ + "allow_experimental_iceberg_compaction": 1, + "iceberg_manifest_min_count_to_compact": 2, + }, + ) + assert "supported only for Iceberg format_version 2" in error_message + + assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == 40 + + +@pytest.mark.parametrize("storage_type", ["s3"]) +def test_optimize_manifest_files_partitioned(started_cluster_iceberg_with_spark, storage_type): + """ + Test manifest-only compaction for a partitioned Iceberg table. + + The table is partitioned by 'region' (3 distinct values). We perform many + small inserts across all partitions so that the number of manifest files + grows well above the compaction threshold. After OPTIMIZE TABLE ... MANIFEST + the manifests should be consolidated to one per partition. + + Checks: + - Data correctness is preserved after compaction. + - Time-travel via snapshot_id still works after compaction. + - A second OPTIMIZE invocation is a no-op (already optimal). + - The compaction threshold setting is honoured: with the default threshold (30) + a table that already has <= 30 manifest files is left untouched, while with + a lower threshold (2) compaction is triggered sooner. + """ + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_optimize_manifests_partitioned_" + storage_type + "_" + get_uuid_str() + + # 3 distinct partition values + REGIONS = ["eu", "us", "ap"] + NUM_PARTITIONS = len(REGIONS) + + # ── Create partitioned table ────────────────────────────────────────────── + spark.sql( + f""" + CREATE TABLE {TABLE_NAME} (id long, data string, region string) + USING iceberg + PARTITIONED BY (region) + TBLPROPERTIES ( + 'format-version' = '2', + 'write.update.mode' = 'merge-on-read', + 'write.delete.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read' + ) + """ + ) + + # ── Initial insert – one batch per partition ────────────────────────────── + for region in REGIONS: + spark.sql( + f"INSERT INTO {TABLE_NAME} " + f"SELECT id, char(id + ascii('a')), '{region}' " + f"FROM range(0, 30)" + ) + + default_upload_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + + create_iceberg_table(storage_type, instance, TABLE_NAME, started_cluster_iceberg_with_spark) + first_snapshot_id = get_last_snapshot(f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}/") + snapshot_timestamp = datetime.now(timezone.utc) + + time.sleep(0.1) + # 30 rows × 3 regions = 90 rows + assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == 90 + + # Time-travel snapshot should also see 90 rows + assert int(instance.query( + f"SELECT count() FROM {TABLE_NAME} " + f"SETTINGS iceberg_snapshot_id = {first_snapshot_id}" + )) == 90 + + # ── Many more small inserts to create many manifest files ───────────────── + # 6 batches × 3 regions = 18 additional inserts → well above the lowered threshold (2) + for batch_start in range(30, 90, 10): + for region in REGIONS: + spark.sql( + f"INSERT INTO {TABLE_NAME} " + f"SELECT id, char(id + ascii('a')), '{region}' " + f"FROM range({batch_start}, {batch_start + 10})" + ) + default_upload_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + + snapshot_id = get_last_snapshot(f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}/") + + # 90 (initial) + 6 batches × 10 rows × 3 regions = 90 + 180 = 270 + total_rows = 90 + 6 * 10 * NUM_PARTITIONS + assert int(instance.query( + f"SELECT count() FROM {TABLE_NAME} " + f"SETTINGS iceberg_snapshot_id = {snapshot_id}" + )) == total_rows + + # ── Run manifest compaction ─────────────────────────────────────────────── + # Lower threshold to 2 so that compaction is definitely triggered + # (each partition will have at least 7 manifest files after the inserts above) + instance.query( + f"OPTIMIZE TABLE {TABLE_NAME} MANIFEST", + settings={ + "allow_experimental_iceberg_compaction": 1, + "iceberg_manifest_min_count_to_compact": 2, + }, + ) + + # ── Data correctness after compaction ──────────────────────────────────── + # Check the current (post-compaction) snapshot via the default read path. + assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == total_rows + + for region in REGIONS: + expected_count = 90 # 30 initial + 6 × 10 additional + actual_count = int(instance.query( + f"SELECT count() FROM {TABLE_NAME} WHERE region = '{region}'" + )) + assert actual_count == expected_count, \ + f"Region '{region}': expected {expected_count} rows after compaction, got {actual_count}" + + # Cross-check: the pre-compaction snapshot must also still be readable. + assert int(instance.query( + f"SELECT count() FROM {TABLE_NAME} " + f"SETTINGS iceberg_snapshot_id = {snapshot_id}" + )) == total_rows + + # ── Time-travel still works after compaction ────────────────────────────── + assert int(instance.query( + f"SELECT count() FROM {TABLE_NAME} " + f"SETTINGS iceberg_snapshot_id = {first_snapshot_id}" + )) == 90 + + assert int(instance.query( + f"SELECT count() FROM {TABLE_NAME} " + f"SETTINGS iceberg_timestamp_ms = {int(snapshot_timestamp.timestamp() * 1000)}" + )) == 90 + + # ── Second OPTIMIZE should be a no-op (already one manifest per partition) ─ + # This must not raise and must leave data intact. + instance.query( + f"OPTIMIZE TABLE {TABLE_NAME} MANIFEST;", + settings={ + "allow_experimental_iceberg_compaction": 1, + "iceberg_manifest_min_count_to_compact": 2, + }, + ) + # Verify the current snapshot is still intact after the no-op. + assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == total_rows + + # ── Third OPTIMIZE should throw exception + error_message = instance.query_and_get_error( + f"OPTIMIZE TABLE {TABLE_NAME} FINAL MANIFEST;", + settings={ + "allow_experimental_iceberg_compaction": 1, + "iceberg_manifest_min_count_to_compact": 2, + }, + ) + assert "OPTIMIZE MANIFEST is incompatible with FINAL, PARTITION, DEDUPLICATE, CLEANUP, and DRY RUN options" in error_message + + +@pytest.mark.parametrize("storage_type", ["s3"]) +def test_optimize_manifest_files_partitioned_concurrent(started_cluster_iceberg_with_spark, storage_type): + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_optimize_manifests_concurrent_" + storage_type + "_" + get_uuid_str() + + REGIONS = ["eu", "us", "ap"] + NUM_PARTITIONS = len(REGIONS) + + spark.sql( + f""" + CREATE TABLE {TABLE_NAME} (id long, data string, region string) + USING iceberg + PARTITIONED BY (region) + TBLPROPERTIES ( + 'format-version' = '2', + 'write.update.mode' = 'merge-on-read', + 'write.delete.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read' + ) + """ + ) + + # Initial insert – one batch per partition. + for region in REGIONS: + spark.sql( + f"INSERT INTO {TABLE_NAME} " + f"SELECT id, char(id + ascii('a')), '{region}' " + f"FROM range(0, 30)" + ) + + # Many more small inserts to create many manifest files (>> compaction threshold). + for batch_start in range(30, 90, 10): + for region in REGIONS: + spark.sql( + f"INSERT INTO {TABLE_NAME} " + f"SELECT id, char(id + ascii('a')), '{region}' " + f"FROM range({batch_start}, {batch_start + 10})" + ) + + default_upload_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + + create_iceberg_table(storage_type, instance, TABLE_NAME, started_cluster_iceberg_with_spark) + + # 30 initial + 6 × 10 additional rows per region. + expected_per_region = 90 + total_rows = expected_per_region * NUM_PARTITIONS + + # Sanity check before launching threads. + assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == total_rows + + NUM_READER_THREADS = 4 + NUM_OPTIMIZE_THREADS = 2 + # Iteration-bounded rather than wall-clock-bounded: a slow CI runner + # would otherwise truncate the test to a handful of iterations and miss + # the conflict windows we're trying to exercise. + OPTIMIZE_ITERATIONS_PER_THREAD = 5 + + errors = [] + errors_lock = threading.Lock() + optimize_attempts = [0] * NUM_OPTIMIZE_THREADS + reader_attempts = [0] * NUM_READER_THREADS + + optimizers_done_event = threading.Event() + finished_optimizers = [0] + finished_lock = threading.Lock() + + def report_error(label, exc): + with errors_lock: + errors.append(f"{label}: {type(exc).__name__}: {exc}") + + def reader_loop(idx): + # Readers run as long as any optimizer is still in flight, so the + # exposure to the conflict window scales with the optimize workload + # rather than with wall-clock time. + try: + while not optimizers_done_event.is_set(): + got_total = int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) + if got_total != total_rows: + raise AssertionError( + f"SELECT count() returned {got_total}, expected {total_rows}" + ) + region = REGIONS[idx % NUM_PARTITIONS] + got_part = int(instance.query( + f"SELECT count() FROM {TABLE_NAME} WHERE region = '{region}'" + )) + if got_part != expected_per_region: + raise AssertionError( + f"count(WHERE region={region}) returned {got_part}, " + f"expected {expected_per_region}" + ) + reader_attempts[idx] += 1 + except Exception as exc: + report_error(f"reader-{idx}", exc) + + def optimize_loop(idx): + try: + for _ in range(OPTIMIZE_ITERATIONS_PER_THREAD): + instance.query( + f"OPTIMIZE TABLE {TABLE_NAME} MANIFEST", + settings={ + "allow_experimental_iceberg_compaction": 1, + "iceberg_manifest_min_count_to_compact": 2, + }, + ) + optimize_attempts[idx] += 1 + except Exception as exc: + report_error(f"optimize-{idx}", exc) + finally: + # Wake the readers as soon as the last optimizer is done so the + # whole test ends in bounded time even if an optimizer raised. + with finished_lock: + finished_optimizers[0] += 1 + if finished_optimizers[0] == NUM_OPTIMIZE_THREADS: + optimizers_done_event.set() + + readers = [ + threading.Thread(target=reader_loop, args=(i,), daemon=True) + for i in range(NUM_READER_THREADS) + ] + optimizers = [ + threading.Thread(target=optimize_loop, args=(i,), daemon=True) + for i in range(NUM_OPTIMIZE_THREADS) + ] + + for t in optimizers: + t.start() + for t in readers: + t.start() + + # Generous per-thread join timeout so a slow CI runner does not flake; + # the test itself completes as soon as all threads finish their bounded work. + for t in optimizers + readers: + t.join(timeout=300) + assert not t.is_alive(), "Worker thread did not finish in time" + + assert not errors, "Concurrent run produced errors:\n" + "\n".join(errors) + assert sum(reader_attempts) > 0, "No reads were performed" + assert sum(optimize_attempts) == NUM_OPTIMIZE_THREADS * OPTIMIZE_ITERATIONS_PER_THREAD, ( + f"Expected {NUM_OPTIMIZE_THREADS * OPTIMIZE_ITERATIONS_PER_THREAD} OPTIMIZE iterations, " + f"got {sum(optimize_attempts)}" + ) + + # Final consistency check. + assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == total_rows + for region in REGIONS: + assert int(instance.query( + f"SELECT count() FROM {TABLE_NAME} WHERE region = '{region}'" + )) == expected_per_region + + +@pytest.mark.parametrize("storage_type", ["s3"]) +def test_optimize_manifest_totals_invariant(started_cluster_iceberg_with_spark, storage_type): + """ + Regression test: repeated OPTIMIZE TABLE ... MANIFEST must not inflate the + snapshot summary totals (total-data-files, total-records, total-files-size). + + Before the fix, each compaction call passed added_files = total_data_files and + added_records/added_files_size from the previous snapshot delta to + generateNextMetadata, which computes total_* = parent_total_* + added_*. + This caused totals to double (or more) with every OPTIMIZE run. + """ + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_optimize_totals_" + storage_type + "_" + get_uuid_str() + TABLE_PATH = f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}/" + + spark.sql( + f""" + CREATE TABLE {TABLE_NAME} (id long, data string) USING iceberg + TBLPROPERTIES ('format-version' = '2') + """ + ) + + # Several inserts to produce multiple manifest files. + for batch_start in range(0, 50, 10): + spark.sql( + f"INSERT INTO {TABLE_NAME} " + f"SELECT id, char(id + ascii('a')) FROM range({batch_start}, {batch_start + 10})" + ) + default_upload_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + + create_iceberg_table(storage_type, instance, TABLE_NAME, started_cluster_iceberg_with_spark) + assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == 50 + + # First compaction — consolidates manifests. + instance.query( + f"OPTIMIZE TABLE {TABLE_NAME} MANIFEST", + settings={ + "allow_experimental_iceberg_compaction": 1, + "iceberg_manifest_min_count_to_compact": 2, + }, + ) + default_download_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}/", + f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}/", + ) + summary_after_first = get_current_snapshot_summary(TABLE_PATH) + assert summary_after_first, "Could not read snapshot summary after first compaction" + + # Verify the operation type is correct for a manifest-only rewrite. + assert summary_after_first.get("operation") == "replace", ( + f"Expected operation='replace', got: {summary_after_first.get('operation')}" + ) + + total_files_1 = int(summary_after_first.get("total-data-files", -1)) + total_records_1 = int(summary_after_first.get("total-records", -1)) + total_size_1 = int(summary_after_first.get("total-files-size", -1)) + + assert total_files_1 >= 0 + assert total_records_1 == 50 + + # Second compaction — already optimal, should be a no-op that does NOT change totals. + instance.query( + f"OPTIMIZE TABLE {TABLE_NAME} MANIFEST", + settings={ + "allow_experimental_iceberg_compaction": 1, + "iceberg_manifest_min_count_to_compact": 2, + }, + ) + default_download_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}/", + f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}/", + ) + summary_after_second = get_current_snapshot_summary(TABLE_PATH) + assert summary_after_second, "Could not read snapshot summary after second compaction" + + # The second round is already optimal, so it must not change the totals. + assert int(summary_after_second.get("total-data-files", -1)) == total_files_1, ( + f"total-data-files changed by no-op compaction: {total_files_1} -> {summary_after_second.get('total-data-files')}" + ) + assert int(summary_after_second.get("total-records", -1)) == total_records_1, ( + f"total-records changed by no-op compaction: {total_records_1} -> {summary_after_second.get('total-records')}" + ) + assert int(summary_after_second.get("total-files-size", -1)) == total_size_1, ( + f"total-files-size changed by no-op compaction: {total_size_1} -> {summary_after_second.get('total-files-size')}" + ) + + # Third compaction — totals must remain identical. + instance.query( + f"OPTIMIZE TABLE {TABLE_NAME} MANIFEST", + settings={ + "allow_experimental_iceberg_compaction": 1, + "iceberg_manifest_min_count_to_compact": 1, + }, + ) + default_download_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}/", + f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}/", + ) + summary_after_third = get_current_snapshot_summary(TABLE_PATH) + assert summary_after_third, "Could not read snapshot summary after third compaction" + + total_files_3 = int(summary_after_third.get("total-data-files", -1)) + total_records_3 = int(summary_after_third.get("total-records", -1)) + total_size_3 = int(summary_after_third.get("total-files-size", -1)) + + assert total_files_3 == total_files_1, ( + f"total-data-files inflated: {total_files_1} -> {total_files_3}" + ) + assert total_records_3 == total_records_1, ( + f"total-records inflated: {total_records_1} -> {total_records_3}" + ) + assert total_size_3 == total_size_1, ( + f"total-files-size inflated: {total_size_1} -> {total_size_3}" + ) + + # Data must still be correct after all compaction rounds. + assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == 50 + + +@pytest.mark.parametrize("compression_method", ["", "gzip"]) +@pytest.mark.parametrize("storage_type", ["s3"]) +def test_optimize_manifest_totals_invariant_schema_evolution( + started_cluster_iceberg_with_spark, storage_type, compression_method +): + instance = started_cluster_iceberg_with_spark.instances["node1"] + suffix = compression_method or "none" + TABLE_NAME = f"test_optimize_totals_se_{suffix}_{storage_type}_{get_uuid_str()}" + TABLE_PATH = f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}/" + + base_settings = {"allow_insert_into_iceberg": 1} + if compression_method: + base_settings["iceberg_metadata_compression_method"] = compression_method + + create_iceberg_table( + storage_type, + instance, + TABLE_NAME, + started_cluster_iceberg_with_spark, + "(x Nullable(Int32))", + format_version=2, + compression_method=compression_method if compression_method else None, + ) + + # Schema evolution: widen, add, then drop a column to produce non-trivial metadata. + instance.query( + f"ALTER TABLE {TABLE_NAME} MODIFY COLUMN x Nullable(Int64);", + settings=base_settings, + ) + instance.query( + f"INSERT INTO {TABLE_NAME} SELECT number FROM numbers(0, 10);", + settings=base_settings, + ) + instance.query( + f"INSERT INTO {TABLE_NAME} SELECT number FROM numbers(10, 10);", + settings=base_settings, + ) + + instance.query( + f"ALTER TABLE {TABLE_NAME} ADD COLUMN y Nullable(Float64);", + settings=base_settings, + ) + instance.query( + f"INSERT INTO {TABLE_NAME} SELECT number, number + 0.5 FROM numbers(20, 10);", + settings=base_settings, + ) + instance.query( + f"INSERT INTO {TABLE_NAME} SELECT number, number + 0.5 FROM numbers(30, 10);", + settings=base_settings, + ) + + instance.query( + f"ALTER TABLE {TABLE_NAME} DROP COLUMN x;", + settings=base_settings, + ) + instance.query( + f"INSERT INTO {TABLE_NAME} SELECT number + 0.5 FROM numbers(40, 10);", + settings=base_settings, + ) + + total_rows = 50 + assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == total_rows + + optimize_settings = dict(base_settings) + optimize_settings.update({ + "allow_experimental_iceberg_compaction": 1, + "iceberg_manifest_min_count_to_compact": 2, + }) + + # First compaction — consolidates manifests. + instance.query( + f"OPTIMIZE TABLE {TABLE_NAME} MANIFEST", + settings=optimize_settings, + ) + default_download_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}/", + f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}/", + ) + summary_after_first = get_current_snapshot_summary(TABLE_PATH) + assert summary_after_first, ( + f"Could not read snapshot summary after first compaction " + f"(compression='{compression_method}')" + ) + assert summary_after_first.get("operation") == "replace", ( + f"Expected operation='replace', got: {summary_after_first.get('operation')}" + ) + + total_files_1 = int(summary_after_first.get("total-data-files", -1)) + total_records_1 = int(summary_after_first.get("total-records", -1)) + total_size_1 = int(summary_after_first.get("total-files-size", -1)) + + assert total_files_1 >= 0 + assert total_records_1 == total_rows + + # Second compaction — already optimal, totals must stay identical. + optimize_settings["iceberg_manifest_min_count_to_compact"] = 1 + instance.query( + f"OPTIMIZE TABLE {TABLE_NAME} MANIFEST", + settings=optimize_settings, + ) + default_download_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}/", + f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}/", + ) + summary_after_second = get_current_snapshot_summary(TABLE_PATH) + assert summary_after_second, ( + f"Could not read snapshot summary after second compaction " + f"(compression='{compression_method}')" + ) + + total_files_2 = int(summary_after_second.get("total-data-files", -1)) + total_records_2 = int(summary_after_second.get("total-records", -1)) + total_size_2 = int(summary_after_second.get("total-files-size", -1)) + + assert total_files_2 == total_files_1, ( + f"total-data-files inflated (compression='{compression_method}'): " + f"{total_files_1} -> {total_files_2}" + ) + assert total_records_2 == total_records_1, ( + f"total-records inflated (compression='{compression_method}'): " + f"{total_records_1} -> {total_records_2}" + ) + assert total_size_2 == total_size_1, ( + f"total-files-size inflated (compression='{compression_method}'): " + f"{total_size_1} -> {total_size_2}" + ) + + # Data must still be correct after compaction. + assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == total_rows + + +@pytest.mark.parametrize("storage_type", ["s3"]) +def test_optimize_manifest_parent_summary_missing_totals( + started_cluster_iceberg_with_spark, storage_type +): + """ + Regression test: OPTIMIZE TABLE ... MANIFEST must tolerate a parent snapshot + whose summary omits some of the carried `total-*` counters. + + Iceberg only requires totals on snapshots that change row-level state, so older + Spark-written tables and tables touched by tools like `removeOrphanFiles` + routinely drop fields like `total-position-deletes`, `total-equality-deletes`, + or `total-delete-files` from the summary. Before the fix, the carry-forward + helper would call `parse` on a missing field and throw + "Cannot parse Int64". + + This test simulates that situation by stripping those fields from the latest + metadata file before ClickHouse first sees the table. + """ + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_optimize_parent_missing_totals_" + storage_type + "_" + get_uuid_str() + TABLE_PATH = f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}/" + + spark.sql( + f""" + CREATE TABLE {TABLE_NAME} (id long, data string) USING iceberg + TBLPROPERTIES ('format-version' = '2') + """ + ) + for batch_start in range(0, 30, 10): + spark.sql( + f"INSERT INTO {TABLE_NAME} " + f"SELECT id, char(id + ascii('a')) FROM range({batch_start}, {batch_start + 10})" + ) + + default_upload_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + + + # Locate the latest metadata file and strip several total-* fields from the + # current snapshot's summary, mimicking older Spark / removeOrphanFiles output. + # Same deterministic tie-break as _load_latest_metadata: when two files share + # last-updated-ms, the higher metadata version wins (independent of os.listdir order). + metadata_dir = f"{TABLE_PATH}/metadata/" + latest_path = None + best_key = None + for filename in os.listdir(metadata_dir): + if not filename.endswith(".json"): + continue + fp = os.path.join(metadata_dir, filename) + with _open_metadata_file(fp) as f: + data = json.load(f) + key = (data.get("last-updated-ms", 0), _metadata_version_from_name(filename)) + if best_key is None or key > best_key: + best_key = key + latest_path = fp + assert latest_path is not None, "Could not locate latest metadata file" + + with _open_metadata_file(latest_path) as f: + data = json.load(f) + + stripped_fields = ( + "total-position-deletes", + "total-equality-deletes", + "total-delete-files", + ) + for snap in data.get("snapshots", []): + summary = snap.get("summary", {}) + for stripped in stripped_fields: + summary.pop(stripped, None) + + # Preserve the on-disk encoding (gzip vs plain) when writing back. + with open(latest_path, "rb") as raw: + magic = raw.read(2) + if magic == b"\x1f\x8b": + with gzip.open(latest_path, "wt") as f: + json.dump(data, f) + else: + with open(latest_path, "w") as f: + json.dump(data, f) + + # Re-upload the edited metadata so ClickHouse reads the stripped summary. + default_upload_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + + # Create the table only after the edit so no metadata cache is populated + # with the original (full) summary. + create_iceberg_table(storage_type, instance, TABLE_NAME, started_cluster_iceberg_with_spark) + assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == 30 + + # Must not throw despite the missing total-* fields in the parent summary. + instance.query( + f"OPTIMIZE TABLE {TABLE_NAME} MANIFEST", + settings={ + "allow_experimental_iceberg_compaction": 1, + "iceberg_manifest_min_count_to_compact": 1, + }, + ) + + assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == 30 + + # The newly-written manifest-only snapshot must carry the missing totals as "0". + default_download_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}/", + f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}/", + ) + summary = get_current_snapshot_summary(TABLE_PATH) + assert summary, "Could not read snapshot summary after compaction" + assert summary.get("operation") == "replace", ( + f"Expected operation='replace', got: {summary.get('operation')}" + ) + for stripped in stripped_fields: + assert summary.get(stripped) == "0", ( + f"Expected {stripped}='0' on the new manifest-only snapshot, " + f"got: {summary.get(stripped)!r}" + ) + + +@pytest.mark.parametrize("storage_type", ["s3"]) +def test_optimize_manifest_files_experimental_gate(started_cluster_iceberg_with_spark, storage_type): + """ + `OPTIMIZE TABLE ... MANIFEST` is gated behind the experimental + `allow_experimental_iceberg_compaction` setting. Running it without the setting must throw + rather than silently rewrite Iceberg metadata. + """ + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_optimize_manifest_gate_" + storage_type + "_" + get_uuid_str() + + spark.sql( + f""" + CREATE TABLE {TABLE_NAME} (id long, data string) USING iceberg TBLPROPERTIES ('format-version' = '2') + """ + ) + spark.sql(f"INSERT INTO {TABLE_NAME} SELECT id, char(id + ascii('a')) FROM range(0, 10)") + + default_upload_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + create_iceberg_table(storage_type, instance, TABLE_NAME, started_cluster_iceberg_with_spark) + + error_message = instance.query_and_get_error(f"OPTIMIZE TABLE {TABLE_NAME} MANIFEST") + assert "allow_experimental_iceberg_compaction" in error_message, ( + f"Expected the experimental-gate exception, got: {error_message}" + ) diff --git a/tests/integration/test_storage_iceberg_with_trino/test.py b/tests/integration/test_storage_iceberg_with_trino/test.py index 9f840d5e7968..a8a63c8a9a0e 100644 --- a/tests/integration/test_storage_iceberg_with_trino/test.py +++ b/tests/integration/test_storage_iceberg_with_trino/test.py @@ -471,3 +471,76 @@ def test_v3_iceberg_system_tables(iceberg_db): f'SELECT count(*) FROM "{NAMESPACE}"."{table_name}$history"', ) assert history.strip() == "2", f"$history count: {history!r}" + + +def test_optimize_manifest_trino_field_ids(iceberg_db): + # Field-id regression: renaming a column keeps its Iceberg field-id but changes the name, so a + # correct Trino read of the new name after OPTIMIZE ... MANIFEST proves the rewrite kept field-ids. + cluster = iceberg_db + node = cluster.instances["node1"] + + table_name = f"optimize_manifest_fieldid_{_get_uuid_str()}" + full = f"{CATALOG_DATABASE}.`{NAMESPACE}.{table_name}`" + + # Manifest compaction is supported only for Iceberg format-version 2 (v1 and v3 are rejected). + node.query( + f""" + CREATE TABLE {full} (id Int32, secret String) + {_engine_clause(table_name)} + SETTINGS iceberg_format_version = 2 + """, + settings=WRITE_SETTINGS, + ) + + # Several separate inserts -> several data manifests to consolidate. + for i in range(1, 6): + node.query( + f"INSERT INTO {full} VALUES ({i}, 'val{i}')", + settings=WRITE_SETTINGS, + ) + + # Rename the column: the Iceberg field-id is preserved, only the name changes. The data files + # keep the original field-id, so a correct read of the new name relies on field-id resolution. + node.query( + f"ALTER TABLE {full} RENAME COLUMN secret TO revealed", + settings=WRITE_SETTINGS, + ) + + read_sql = f'SELECT id, revealed FROM "{NAMESPACE}"."{table_name}" ORDER BY id' + expected = "1\tval1\n2\tval2\n3\tval3\n4\tval4\n5\tval5\n" + + before = _trino_exec(cluster, read_sql) + assert before == expected, ( + f"Trino read of renamed column before compaction: expected {expected!r}, got {before!r}" + ) + + snaps_before = _trino_exec( + cluster, f'SELECT count(*) FROM "{NAMESPACE}"."{table_name}$snapshots"' + ).strip() + + node.query( + f"OPTIMIZE TABLE {full} MANIFEST", + settings={ + "allow_experimental_iceberg_compaction": 1, + "iceberg_manifest_min_count_to_compact": 2, + "allow_insert_into_iceberg": 1, + "write_full_path_in_iceberg_metadata": 1, + }, + ) + + # Confirm the compaction actually ran (committed a new snapshot), so the read-back below is not a + # vacuous no-op pass. + snaps_after = _trino_exec( + cluster, f'SELECT count(*) FROM "{NAMESPACE}"."{table_name}$snapshots"' + ).strip() + assert int(snaps_after) > int(snaps_before), ( + f"OPTIMIZE TABLE ... MANIFEST did not commit a new snapshot " + f"({snaps_before} -> {snaps_after})" + ) + + # After the manifest rewrite, Trino must still resolve the renamed column by its preserved + # field-id and return the original data. + after = _trino_exec(cluster, read_sql) + assert after == expected, ( + f"Trino read of renamed column after OPTIMIZE MANIFEST: expected {expected!r}, got {after!r}" + ) From 9626fa1df68f8f5f962001947172730d0968404a Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Sun, 9 Aug 2026 03:38:56 +0200 Subject: [PATCH 2/2] Remove 26.6 changes from upstream --- src/Core/SettingsChangesHistory.cpp | 39 ----------------------------- 1 file changed, 39 deletions(-) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 7b298977e07d..ca41902b67f5 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -43,46 +43,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() { {"analyzer_compatibility_allow_non_aggregate_in_having", false, false, "New compatibility setting. When enabled, the new analyzer mimics the legacy `HAVING`-to-`WHERE` rewrite for non-aggregate AND-conjuncts instead of raising `NOT_AN_AGGREGATE`."}, {"reserve_memory", 0, 0, "New setting to reserve memory for specific workload before starting a query."}, - {"optimize_or_like_chain", false, true, "Enable by default: optimize OR chains of LIKE/ILIKE/match into multiSearchAny (pure-substring patterns) or multiMatchAny (other patterns, when Hyperscan/Vectorscan is permitted); when neither fast path applies the original OR chain is kept unchanged."}, - {"optimize_or_like_chain_min_patterns", 0, 10, "New setting controlling the minimum number of non-pure-substring LIKE/ILIKE/match branches (sharing the same LHS expression) required for optimize_or_like_chain to rewrite a chain into multiMatchAny. Shorter chains are kept as-is because the multiMatchAny (Hyperscan) rewrite only becomes faster than short-circuit OR evaluation from about nine branches."}, - {"optimize_or_like_chain_min_substrings", 0, 4, "New setting controlling the minimum number of pure-substring (%needle%) LIKE/ILIKE branches (sharing the same LHS expression) required for optimize_or_like_chain to rewrite a chain into multiSearchAny."}, - {"input_format_arrow_use_native_reader", false, true, "New setting to use the native ClickHouse reader for the Arrow and ArrowStream formats instead of the Apache Arrow library."}, - {"output_format_arrow_use_native_writer", false, true, "New setting to use the native ClickHouse writer for the Arrow and ArrowStream formats instead of the Apache Arrow library."}, - {"allow_minmax_index_for_json", true, false, "Forbid creating minmax skip index on JSON columns by default because the index serialization cannot handle heterogeneous Field values"}, - {"s3_allow_server_credentials_in_user_queries", true, false, "New setting to block S3 access from user SQL from resolving the server's own ambient credentials (environment/IMDS/IRSA/instance-profile/AWS-config-file/role_arn-STS/GCP-OAuth-metadata). The previous behavior (allowed) is restored with compatibility settings."}, - {"query_plan_merge_expression_into_join", false, true, "New setting. Allow to merge Expression step into JOIN step during join reordering optimization."}, - {"skip_unavailable_shards_mode", "unavailable_or_table_missing", "unavailable_or_table_missing", "New setting to control which exceptions from a remote shard are ignored when `skip_unavailable_shards` is enabled. The default matches the historical behavior: a shard whose table is missing is treated as unavailable."}, - {"use_text_index_tokens_cache", false, true, "Enabled the text index tokens cache globally."}, - {"use_text_index_header_cache", false, true, "Enabled the text index header cache globally."}, - {"optimize_aggregation_in_order_limit", false, true, "New setting to push the `LIMIT` into aggregation-in-order for early termination when the `ORDER BY` is a prefix of the `GROUP BY` sort description."}, - {"explain_query_plan_default", "legacy", "pretty", "From 26.7, `EXPLAIN PLAN` defaults to `actions=1, compact=1, pretty=1`. Set this to `legacy` to restore the pre-26.7 output."}, - {"format_geojson_validate_geometry", true, true, "New setting that controls whether the GeoJSON format enforces RFC 7946 geometry validity (minimum points per line and ring, ring closure, non-empty multi-geometries) when reading and writing"}, - {"use_partition_minmax_for_primary_key_pruning", false, true, "New setting to use the part's partition minmax to prune more granules during primary key analysis for `MergeTree` tables, when a primary key column is also an input column of the partition key."}, - {"allow_delta_lake_writes", false, false, "Added an alias for setting `allow_experimental_delta_lake_writes`, which was moved to Beta."}, - {"allow_experimental_delta_lake_writes", false, false, "Delta Lake writes were moved to Beta."}, - {"input_format_parquet_dictionary_filter_push_down", 0, 1024 * 1024, "New setting enabling Parquet row-group pruning based on dictionary page contents (reader v3). The value is the maximum dictionary page size in bytes for which the optimization applies; 0 (the previous behavior) disables it."}, - {"compile_regular_expressions", false, true, "New setting to enable JIT compilation of simple regular expressions in functions like `match` and `extract`."}, - {"min_count_to_compile_regular_expression", 3, 3, "New setting controlling how many times a regular expression must be used before it is JIT-compiled."}, - {"allow_aggregate_partitions_independently", false, true, "Enable independent per-partition aggregation by default when the partition key suits the GROUP BY key. The existing runtime heuristics in `ReadFromMergeTree::requestOutputEachPartitionThroughSeparatePortForAggregation` already skip the optimization when the partition layout is unfavorable (too few partitions, too many partitions, or significantly skewed partition sizes), so enabling the setting is safe in the cases where it would otherwise be a no-op."}, - {"text_index_lazy_intersection_density_threshold", 0.2, 0.2, "Renamed from `text_index_density_threshold` (kept as an alias); selects the posting list intersection algorithm in lazy posting list apply mode."}, - {"allow_experimental_text_index_lazy_apply", false, true, "Lazy posting list apply mode for the text index is no longer experimental; the setting is now obsolete and has no effect (lazy mode is selected via `text_index_posting_list_apply_mode = 'lazy'`)."}, - {"allow_experimental_url_wildcard_from_index_pages", false, false, "New setting to enable expanding wildcards in the `url` table function by listing HTTP index pages."}, - {"url_wildcard_max_directories_to_read", 100000, 100000, "New setting to limit the number of directories read when expanding wildcards in the `url` table function."}, - {"allow_experimental_eval_table_function", false, false, "New setting to enable the experimental table function `eval`."}, - {"output_format_csv_header_serialize_tuple_into_separate_columns", false, true, "New setting. When output_format_csv_serialize_tuple_into_separate_columns is enabled, the CSVWithNames/CSVWithNamesAndTypes header now flattens Tuple columns into their leaf fields so the header width matches the data. Set to false to restore the previous single-name header."}, - {"reader_executor_use_long_connections", false, false, "New experimental ReaderExecutor setting (off by default): reuse a held source connection across sequential windows."}, - {"reader_executor_min_bytes_for_seek", 2097152, 2097152, "New experimental ReaderExecutor setting: forward-gap bound for bridging on a held source connection."}, - {"reader_executor_max_tail_for_drain", 1048576, 1048576, "New experimental ReaderExecutor setting: drain bound for completing a dropped long connection."}, - {"precise_float_parsing", false, true, "Use the precise (closest-representable) float parsing algorithm by default, now that it is faster than the previous fast algorithm. Set to false to restore the pre-26.7 fast-but-less-accurate parsing in conversion functions."}, - {"optimize_and_compare_chain_max_hash_work", 0, 5'000'000, "New setting that bounds the work of the `optimize_and_compare_chain` optimization (measured in query-tree nodes hashed) so it cannot dominate analysis of queries with very many or very large `AND`-chains of comparisons. The previous value `0` (unlimited) reproduces the pre-26.7 behavior where the optimization was uncapped, so `compatibility` set to an earlier version keeps deriving transitive predicates without a budget. Set to `0` to disable the budget."}, {"iceberg_manifest_min_count_to_compact", 30, 30, "New setting to control manifest compaction for Iceberg tables."}, - {"show_remote_databases_in_system_tables", true, true, "New setting to control whether `MySQL` and `PostgreSQL` databases are shown in `system.tables`, `system.columns` and `system.completions`."}, - {"use_constant_folding_in_index_analysis", false, false, "New setting to fold partition-level constants into the filter predicate per part during MergeTree index analysis, improving pruning for filters whose branches depend on partition values."}, - {"join_runtime_filter_size_from_hash_table_stats", false, true, "Use hash table size statistics collected from previous executions to size the JOIN runtime filter. When disabled, fall back to the fixed `join_runtime_bloom_filter_bytes`."}, - }); - - addSettingsChanges(settings_changes_history, "26.6", - { {"output_format_image_width", 1024, 1024, "New setting controlling the width of the output image for image output formats such as PNG."}, {"output_format_image_height", 1024, 1024, "New setting controlling the height of the output image for image output formats such as PNG."}, {"output_format_image_terminal_mode", "", "", "New setting controlling whether image output formats such as PNG are rendered directly to the terminal using an inline image protocol."},