From f384ec465d063902254758fb6937a23a2074675f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Wed, 12 Aug 2026 22:35:48 +0800 Subject: [PATCH 1/3] [core] Speed up row-id manifest sorting --- docs/generated/core_configuration.html | 6 + .../java/org/apache/paimon/CoreOptions.java | 13 + .../org/apache/paimon/utils/ByteArrayKey.java | 3 +- .../paimon/utils/ByteArrayLookupKey.java | 48 +- .../apache/paimon/utils/ByteArrayKeyTest.java | 17 + .../org/apache/paimon/manifest/FileEntry.java | 13 + .../apache/paimon/manifest/ManifestFile.java | 9 + .../operation/ManifestEntryRunMerge.java | 550 ++++++++++++ .../operation/ManifestEntryRunMergeEntry.java | 308 +++++++ .../operation/ManifestEntryRunMergePlan.java | 797 ++++++++++++++++++ .../paimon/operation/ManifestFileSorter.java | 416 ++++++--- .../paimon/manifest/ManifestFileMetaTest.java | 538 +++++++++++- 12 files changed, 2603 insertions(+), 115 deletions(-) create mode 100644 paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMerge.java create mode 100644 paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergeEntry.java create mode 100644 paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergePlan.java diff --git a/docs/generated/core_configuration.html b/docs/generated/core_configuration.html index 0c576c393485..033e2194f53b 100644 --- a/docs/generated/core_configuration.html +++ b/docs/generated/core_configuration.html @@ -1023,6 +1023,12 @@ String Partition field name to sort manifest entries by. Validated by schema validation, if not configured, defaults to the first partition field. + +
manifest-sort.run-merge-optimize.enabled
+ true + Boolean + Whether to use streaming run merge for RowID-based manifest sorting. When disabled, the external sorter is used without changing the RowID sort semantics. +
manifest.compression
"zstd" diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index 12e02ee2a631..b4e34c574b24 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -589,6 +589,15 @@ public InlineElement getDescription() { + " skipped. Set to a larger value to allow more aggressive" + " sort rewriting. The cap only limits the sorted rewrite portion and full/minor cleanup may still happen beyond it."); + public static final ConfigOption MANIFEST_SORT_RUN_MERGE_OPTIMIZE_ENABLED = + key("manifest-sort.run-merge-optimize.enabled") + .booleanType() + .defaultValue(true) + .withDescription( + "Whether to use streaming run merge for RowID-based manifest sorting." + + " When disabled, the external sorter is used without changing" + + " the RowID sort semantics."); + public static final ConfigOption PARTITION_DEFAULT_NAME = key("partition.default-name") .stringType() @@ -3066,6 +3075,10 @@ public long manifestSortMaxRewriteSize() { return options.get(MANIFEST_SORT_MAX_REWRITE_SIZE).getBytes(); } + public boolean manifestSortRunMergeOptimizeEnabled() { + return options.get(MANIFEST_SORT_RUN_MERGE_OPTIMIZE_ENABLED); + } + public String partitionDefaultName() { return options.get(PARTITION_DEFAULT_NAME); } diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/ByteArrayKey.java b/paimon-common/src/main/java/org/apache/paimon/utils/ByteArrayKey.java index 09d9ded426a4..274e20abdc0a 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/ByteArrayKey.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/ByteArrayKey.java @@ -47,8 +47,7 @@ byte[] bytes() { public boolean equals(Object obj) { return obj == this || (obj instanceof ByteArrayKey && Arrays.equals(bytes, ((ByteArrayKey) obj).bytes)) - || (obj instanceof ByteArrayLookupKey - && Arrays.equals(bytes, ((ByteArrayLookupKey) obj).bytes())); + || (obj instanceof ByteArrayLookupKey && obj.equals(this)); } @Override diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/ByteArrayLookupKey.java b/paimon-common/src/main/java/org/apache/paimon/utils/ByteArrayLookupKey.java index aaa913ace7ee..023a6b686609 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/ByteArrayLookupKey.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/ByteArrayLookupKey.java @@ -20,8 +20,6 @@ import javax.annotation.Nullable; -import java.util.Arrays; - import static org.apache.paimon.utils.Preconditions.checkArgument; /** @@ -33,6 +31,8 @@ public final class ByteArrayLookupKey { private @Nullable byte[] bytes; + private int offset; + private int length; private int hash; public ByteArrayLookupKey() {} @@ -43,12 +43,26 @@ public ByteArrayLookupKey(byte[] bytes) { public void reset(byte[] bytes) { checkArgument(bytes != null, "Byte array cannot be null."); + reset(bytes, 0, bytes.length); + } + + public void reset(byte[] bytes, int offset, int length) { + checkArgument(bytes != null, "Byte array cannot be null."); + checkArgument(offset >= 0 && length >= 0 && offset <= bytes.length - length); this.bytes = bytes; - this.hash = Arrays.hashCode(bytes); + this.offset = offset; + this.length = length; + int hash = 1; + for (int i = offset; i < offset + length; i++) { + hash = 31 * hash + bytes[i]; + } + this.hash = hash; } public void clear() { bytes = null; + offset = 0; + length = 0; hash = 0; } @@ -62,14 +76,38 @@ public boolean equals(Object obj) { return obj == this || (bytes != null && obj instanceof ByteArrayKey - && Arrays.equals(bytes, ((ByteArrayKey) obj).bytes())) + && equals(((ByteArrayKey) obj).bytes())) || (bytes != null && obj instanceof ByteArrayLookupKey - && Arrays.equals(bytes, ((ByteArrayLookupKey) obj).bytes)); + && equals((ByteArrayLookupKey) obj)); } @Override public int hashCode() { return hash; } + + private boolean equals(byte[] other) { + if (length != other.length) { + return false; + } + for (int i = 0; i < length; i++) { + if (bytes[offset + i] != other[i]) { + return false; + } + } + return true; + } + + private boolean equals(ByteArrayLookupKey other) { + if (other.bytes == null || length != other.length) { + return false; + } + for (int i = 0; i < length; i++) { + if (bytes[offset + i] != other.bytes[other.offset + i]) { + return false; + } + } + return true; + } } diff --git a/paimon-common/src/test/java/org/apache/paimon/utils/ByteArrayKeyTest.java b/paimon-common/src/test/java/org/apache/paimon/utils/ByteArrayKeyTest.java index 89c2db09d426..8248cfcb36cc 100644 --- a/paimon-common/src/test/java/org/apache/paimon/utils/ByteArrayKeyTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/utils/ByteArrayKeyTest.java @@ -59,6 +59,23 @@ void testReusableMapLookup() { assertThat(lookup.hashCode()).isZero(); } + @Test + void testReusableSliceLookup() { + Map values = new HashMap<>(); + ByteArrayKey key = new ByteArrayKey(new byte[] {1, 2, 3}); + values.put(key, "value"); + ByteArrayLookupKey lookup = new ByteArrayLookupKey(); + + lookup.reset(new byte[] {9, 1, 2, 3, 8}, 1, 3); + assertThat(lookup).isEqualTo(key); + assertThat(key).isEqualTo(lookup); + assertThat(lookup.hashCode()).isEqualTo(key.hashCode()); + assertThat(values.get(lookup)).isEqualTo("value"); + + lookup.clear(); + assertThat(new ByteArrayLookupKey(new byte[] {1, 2, 3})).isNotEqualTo(lookup); + } + @Test void testLookupEqualityLifecycle() { ByteArrayLookupKey first = new ByteArrayLookupKey(new byte[] {1}); diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/FileEntry.java b/paimon-core/src/main/java/org/apache/paimon/manifest/FileEntry.java index 11f08cf6329e..8cc109b6a882 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/FileEntry.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/FileEntry.java @@ -213,6 +213,19 @@ public ReusableIdentifier replaceWithPartition(ProjectedManifestEntry entry) { return appendEntryFields(entry); } + /** Replaces this encoding with an already serialized identifier. */ + public ReusableIdentifier replace(byte[] value, int offset, int valueLength) { + checkArgument(value != null, "Serialized identifier cannot be null."); + checkArgument( + offset >= 0 && valueLength >= 0 && offset <= value.length - valueLength, + "Identifier byte range is invalid."); + length = 0; + ensureCapacity(valueLength); + System.arraycopy(value, offset, bytes, 0, valueLength); + length = valueLength; + return this; + } + private ReusableIdentifier appendEntryFields(ProjectedManifestEntry entry) { putInt(entry.bucket()); ProjectedDataFileMeta file = entry.file(); diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java index b9fa876f61de..0bac1bab0adc 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java @@ -203,6 +203,15 @@ private static CloseableIterator createManifestIterator( } } + /** Opens a low-allocation reader for the encoded manifest fields needed by run merge. */ + public ManifestAvroReader scanForRunMerge(String fileName, @Nullable Long fileSize) { + try { + return new ManifestAvroReader(fileIO.newInputStream(pathFactory.toPath(fileName))); + } catch (IOException e) { + throw new UncheckedIOException("Failed to read manifest file " + fileName, e); + } + } + @VisibleForTesting public long suggestedFileSize() { return suggestedFileSize; diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMerge.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMerge.java new file mode 100644 index 000000000000..b3d296e085a8 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMerge.java @@ -0,0 +1,550 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.operation; + +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.manifest.CompactFileIdentifierSet; +import org.apache.paimon.manifest.FileKind; +import org.apache.paimon.manifest.ManifestAvroReader; +import org.apache.paimon.manifest.ManifestAvroReader.RawBlock; +import org.apache.paimon.manifest.ManifestAvroReader.RowIterator; +import org.apache.paimon.manifest.ManifestAvroWriter.EncodedBlock; +import org.apache.paimon.manifest.ManifestEntry; +import org.apache.paimon.manifest.ManifestFile; +import org.apache.paimon.manifest.ManifestFileMeta; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.Pair; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.function.Function; + +import static org.apache.paimon.utils.ManifestReadThreadPool.sequentialBatchedExecute; + +/** Streaming merge of the naturally sorted runs in data-evolution manifest files. */ +final class ManifestEntryRunMerge { + + private static final int FRAGMENTED_RUN_THRESHOLD = 64; + private static final long MAX_IN_MEMORY_FRAGMENTED_ENTRIES = 25_000L; + private static final int MAX_STREAM_CURSORS = 128; + private static final int MAX_STREAM_READ_AMPLIFICATION = 8; + static final int KIND = 0; + static final int PARTITION = 1; + static final int BUCKET = 2; + static final int FILE = 3; + static final int FILE_NAME = 0; + static final int ROW_COUNT = 1; + static final int LEVEL = 2; + static final int SCHEMA_ID = 3; + static final int FIRST_ROW_ID = 4; + static final int MAX_SEQUENCE_NUMBER = 5; + static final int EXTRA_FILES = 6; + static final int EMBEDDED_FILE_INDEX = 7; + static final int EXTERNAL_PATH = 8; + static final int FILE_FIELD_COUNT = 9; + static final RowType ENTRY_LAYOUT = entryLayout(); + + private ManifestEntryRunMerge() {} + + private static RowType entryLayout() { + List fields = new ArrayList<>(); + fields.add(ManifestEntry.MANIFEST_ROW_TYPE.getField(ManifestEntry.KIND)); + fields.add(ManifestEntry.MANIFEST_ROW_TYPE.getField(ManifestEntry.PARTITION)); + fields.add(ManifestEntry.MANIFEST_ROW_TYPE.getField(ManifestEntry.BUCKET)); + fields.add( + ManifestEntry.MANIFEST_ROW_TYPE + .getField(ManifestEntry.FILE) + .newType( + DataFileMeta.SCHEMA.project( + DataFileMeta.FILE_NAME, + DataFileMeta.ROW_COUNT, + DataFileMeta.LEVEL, + DataFileMeta.SCHEMA_ID, + DataFileMeta.FIRST_ROW_ID, + DataFileMeta.MAX_SEQUENCE_NUMBER, + DataFileMeta.EXTRA_FILES, + DataFileMeta.EMBEDDED_FILE_INDEX, + DataFileMeta.EXTERNAL_PATH))); + return new RowType(false, fields); + } + + /** + * Returns null when the input is too fragmented for a bounded streaming merge. The caller must + * fall back to the spillable external sorter in that case. + */ + @Nullable + static List sortAndWriteFullEntries( + List section, + ManifestFileSorter.RowIdEntrySortKey sortKey, + ManifestFile manifestFile, + List newFilesForAbort, + CompactFileIdentifierSet deletedIdentifiers, + ManifestFileSorter.DeletedRowIdSet deletedRowIds, + @Nullable Integer manifestReadParallelism) + throws Exception { + ManifestEntryRunMergeEntry.Filter filter = + new ManifestEntryRunMergeEntry.Filter(deletedIdentifiers, deletedRowIds); + ManifestEntryRunMergePlan plan = + discoverRuns(section, sortKey, manifestFile, filter, manifestReadParallelism); + if (plan == null) { + return null; + } + return plan.mergeToManifest(sortKey, manifestFile, filter, newFilesForAbort); + } + + /** + * Returns null when the input is too fragmented for a bounded streaming merge or primitive + * manifest reading is unavailable. The caller must fall back to the spillable external sorter. + */ + @Nullable + static Pair, List> sortAndWriteMinorEntries( + List section, + ManifestFileSorter.RowIdEntrySortKey sortKey, + ManifestFile manifestFile, + List newFilesForAbort, + @Nullable Integer manifestReadParallelism) + throws Exception { + CompactFileIdentifierSet deletedIdentifiers = new CompactFileIdentifierSet(); + ManifestFileSorter.DeletedRowIdSet deletedRowIds = new ManifestFileSorter.DeletedRowIdSet(); + ManifestEntryRunMergeEntry.Filter.Minor filter = + new ManifestEntryRunMergeEntry.Filter.Minor(deletedIdentifiers, deletedRowIds); + try { + ManifestEntryRunMergePlan plan; + try { + plan = + discoverRuns( + section, sortKey, manifestFile, filter, manifestReadParallelism); + } finally { + deletedRowIds.releaseRangeIndex(); + } + if (plan == null) { + return null; + } + return plan.mergeMinorToManifest( + sortKey, + manifestFile, + filter, + deletedIdentifiers, + deletedRowIds, + newFilesForAbort); + } finally { + deletedIdentifiers.release(); + } + } + + @Nullable + private static ManifestEntryRunMergePlan discoverRuns( + List section, + ManifestFileSorter.RowIdEntrySortKey sortKey, + ManifestFile manifestFile, + ManifestEntryRunMergeEntry.Filter filter, + @Nullable Integer manifestReadParallelism) + throws Exception { + ManifestEntryRunMergeEntry.PartitionDictionary partitions = + new ManifestEntryRunMergeEntry.PartitionDictionary(sortKey); + List sources = new ArrayList<>(); + int streamCursorCount = 0; + long inMemoryEntries = 0; + List discovered = new ArrayList<>(section.size()); + if (section.size() <= 1 + || manifestReadParallelism == null + || manifestReadParallelism <= 1) { + for (ManifestFileMeta meta : section) { + Discovery.DiscoveredManifest manifest = + discoverManifestRuns(meta, manifestFile, partitions, filter); + if (manifest.requiresExternalSort) { + return null; + } + discovered.add(manifest); + } + } else { + Function> reader = + meta -> { + try { + return Collections.singletonList( + discoverManifestRuns(meta, manifestFile, partitions, filter)); + } catch (Exception e) { + throw new RuntimeException( + "Failed to discover sorted Avro runs in " + meta.fileName(), e); + } + }; + for (Discovery.DiscoveredManifest manifest : + sequentialBatchedExecute(reader, section, manifestReadParallelism)) { + discovered.add(manifest); + } + } + for (int manifestIndex = 0; manifestIndex < section.size(); manifestIndex++) { + ManifestFileMeta meta = section.get(manifestIndex); + Discovery.DiscoveredManifest manifest = discovered.get(manifestIndex); + if (manifest.requiresExternalSort) { + return null; + } + if (manifest.fragmented) { + long entryCount = meta.numAddedFiles() + meta.numDeletedFiles(); + inMemoryEntries += entryCount; + if (inMemoryEntries > MAX_IN_MEMORY_FRAGMENTED_ENTRIES) { + return null; + } + sources.add(new ManifestEntryRunMergePlan.Source.FragmentedManifestSpec(meta)); + streamCursorCount++; + } else { + sources.addAll(manifest.runs); + streamCursorCount += manifest.runs.size(); + } + if (streamCursorCount > MAX_STREAM_CURSORS) { + return null; + } + } + partitions.finish(); + for (Discovery.DiscoveredManifest manifest : discovered) { + manifest.finishFiltering(filter); + manifest.updatePartitionRanks(partitions); + } + return new ManifestEntryRunMergePlan(sources, partitions); + } + + private static Discovery.DiscoveredManifest discoverManifestRuns( + ManifestFileMeta meta, + ManifestFile manifestFile, + ManifestEntryRunMergeEntry.PartitionDictionary partitions, + ManifestEntryRunMergeEntry.Filter filter) + throws Exception { + try (ManifestAvroReader reader = + manifestFile.scanForRunMerge(meta.fileName(), meta.fileSize())) { + return discoverManifestRuns(meta, reader, partitions, filter); + } catch (UnsupportedOperationException unsupported) { + return Discovery.DiscoveredManifest.requiresExternalSort(); + } + } + + private static Discovery.DiscoveredManifest discoverManifestRuns( + ManifestFileMeta meta, + ManifestAvroReader reader, + ManifestEntryRunMergeEntry.PartitionDictionary partitions, + ManifestEntryRunMergeEntry.Filter filter) + throws Exception { + List runs = new ArrayList<>(); + List blocks = new ArrayList<>(); + ManifestEntryRunMergeEntry.Key previous = new ManifestEntryRunMergeEntry.Key(); + ManifestEntryRunMergeEntry.Key current = new ManifestEntryRunMergeEntry.Key(); + boolean hasPrevious = false; + long runStart = 0; + long position = 0; + long entryCount = meta.numAddedFiles() + meta.numDeletedFiles(); + boolean fragmented = false; + while (reader.hasNext()) { + RawBlock rawBlock = reader.next(); + RowIterator rows = rawBlock.toRows(ENTRY_LAYOUT); + while (rows.hasNext()) { + GenericRow row = rows.next(); + current.replace(row, partitions); + filter.observe(row, current); + if (fragmented) { + position++; + continue; + } + if (rows.recordIndex() == 0) { + blocks.add( + new Discovery.BlockInfo( + rawBlock.blockOrdinal(), + position, + rawBlock.rawBlockCopySupported(), + current.stableCopy())); + } + Discovery.BlockInfo block = blocks.get(blocks.size() - 1); + block.collect(row, current, partitions, filter); + boolean inversion = + hasPrevious && compareDiscoveryKeys(previous, current, partitions) > 0; + if (inversion) { + if (rows.recordIndex() > 0) { + block.sorted = false; + } + runs.add( + new ManifestEntryRunMergePlan.Source.ManifestRunSpec( + meta, runStart, position, blocks)); + runStart = position; + if (runs.size() >= FRAGMENTED_RUN_THRESHOLD) { + if (entryCount > MAX_IN_MEMORY_FRAGMENTED_ENTRIES) { + return Discovery.DiscoveredManifest.requiresExternalSort(); + } + fragmented = true; + runs.clear(); + blocks.clear(); + position++; + continue; + } + } + position++; + if (rows.recordIndex() + 1 == rawBlock.recordCount()) { + ManifestEntryRunMergeEntry.Key stableLastKey = current.stableCopy(); + block.finish(position, stableLastKey); + previous.copyFrom(stableLastKey); + } else { + previous.copyFrom(current); + } + hasPrevious = true; + } + } + if (fragmented) { + return Discovery.DiscoveredManifest.fragmented(); + } + if (position > runStart) { + runs.add( + new ManifestEntryRunMergePlan.Source.ManifestRunSpec( + meta, runStart, position, blocks)); + } + if (exceedsStreamingReadAmplification(runs, blocks.size())) { + return entryCount > MAX_IN_MEMORY_FRAGMENTED_ENTRIES + ? Discovery.DiscoveredManifest.requiresExternalSort() + : Discovery.DiscoveredManifest.fragmented(); + } + return Discovery.DiscoveredManifest.runs(runs, blocks); + } + + private static boolean exceedsStreamingReadAmplification( + List runs, int blockCount) { + if (runs.size() <= 1 || blockCount == 0) { + return false; + } + + long prefixBlocksRead = 0; + for (ManifestEntryRunMergePlan.Source.ManifestRunSpec run : runs) { + prefixBlocksRead += run.prefixBlockCount(); + } + return prefixBlocksRead > (long) blockCount * MAX_STREAM_READ_AMPLIFICATION; + } + + private static int compareDiscoveryKeys( + ManifestEntryRunMergeEntry.Key left, + ManifestEntryRunMergeEntry.Key right, + ManifestEntryRunMergeEntry.PartitionDictionary partitions) { + return compareRemainingKeys( + left, right, partitions.compareIds(left.partitionId, right.partitionId)); + } + + static int compareMergeKeys( + ManifestEntryRunMergeEntry.Key left, ManifestEntryRunMergeEntry.Key right) { + return compareRemainingKeys( + left, right, Integer.compare(left.partitionRank, right.partitionRank)); + } + + private static int compareRemainingKeys( + ManifestEntryRunMergeEntry.Key left, + ManifestEntryRunMergeEntry.Key right, + int comparison) { + if (comparison == 0) { + comparison = Byte.compare(left.kind, right.kind); + } + if (comparison == 0) { + comparison = Long.compare(left.firstRowId, right.firstRowId); + } + if (comparison == 0) { + comparison = Long.compare(left.rangeEnd, right.rangeEnd); + } + if (comparison == 0) { + comparison = Long.compare(left.reverseSequence, right.reverseSequence); + } + if (comparison == 0) { + comparison = compareBytes(left, right); + } + return comparison; + } + + private static int compareBytes( + ManifestEntryRunMergeEntry.Key left, ManifestEntryRunMergeEntry.Key right) { + int minLength = Math.min(left.fileNameLength, right.fileNameLength); + for (int i = 0; i < minLength; i++) { + int leftByte = left.fileNameBytes[left.fileNameOffset + i] & 0xFF; + int rightByte = right.fileNameBytes[right.fileNameOffset + i] & 0xFF; + if (leftByte != rightByte) { + return leftByte - rightByte; + } + } + return left.fileNameLength - right.fileNameLength; + } + + /** Results and Avro block metadata collected while discovering natural manifest runs. */ + static final class Discovery { + + private Discovery() {} + + static final class DiscoveredManifest { + + final List runs; + final List blocks; + final boolean fragmented; + final boolean requiresExternalSort; + + DiscoveredManifest( + List runs, + List blocks, + boolean fragmented, + boolean requiresExternalSort) { + this.runs = runs; + this.blocks = blocks; + this.fragmented = fragmented; + this.requiresExternalSort = requiresExternalSort; + } + + static DiscoveredManifest runs( + List runs, + List blocks) { + return new DiscoveredManifest(runs, blocks, false, false); + } + + static DiscoveredManifest fragmented() { + return new DiscoveredManifest( + Collections.emptyList(), Collections.emptyList(), true, false); + } + + static DiscoveredManifest requiresExternalSort() { + return new DiscoveredManifest( + Collections.emptyList(), Collections.emptyList(), false, true); + } + + void updatePartitionRanks(ManifestEntryRunMergeEntry.PartitionDictionary partitions) { + for (BlockInfo block : blocks) { + block.updatePartitionRanks(partitions); + } + } + + void finishFiltering(ManifestEntryRunMergeEntry.Filter filter) { + for (BlockInfo block : blocks) { + block.finishFiltering(filter); + } + } + } + + static final class BlockInfo { + + final long ordinal; + final long start; + final ManifestEntryRunMergeEntry.Key firstKey; + boolean eligible; + boolean sorted = true; + long end; + ManifestEntryRunMergeEntry.Key lastKey; + long addedFiles; + long deletedFiles; + long schemaId = Long.MIN_VALUE; + int minBucket = Integer.MAX_VALUE; + int maxBucket = Integer.MIN_VALUE; + int minLevel = Integer.MAX_VALUE; + int maxLevel = Integer.MIN_VALUE; + long minRowId = Long.MAX_VALUE; + long maxRowId = Long.MIN_VALUE; + BinaryRow nullPartition; + long nullPartitionCount; + BinaryRow minNonNullPartition; + BinaryRow maxNonNullPartition; + EncodedBlock metadata; + + BlockInfo( + long ordinal, + long start, + boolean eligible, + ManifestEntryRunMergeEntry.Key firstKey) { + this.ordinal = ordinal; + this.start = start; + this.eligible = eligible; + this.firstKey = firstKey; + } + + void collect( + GenericRow record, + ManifestEntryRunMergeEntry.Key key, + ManifestEntryRunMergeEntry.PartitionDictionary partitions, + ManifestEntryRunMergeEntry.Filter filter) { + BinaryRow partition = partitions.partition(key.partitionId); + eligible &= partition.getFieldCount() == 1 && filter.copyable(record, key); + if (!eligible) { + return; + } + InternalRow file = ManifestEntryRunMergeEntry.file(record); + if (key.kind == FileKind.ADD.toByteValue()) { + addedFiles++; + } else { + deletedFiles++; + } + schemaId = Math.max(schemaId, file.getLong(SCHEMA_ID)); + int bucket = record.getInt(BUCKET); + minBucket = Math.min(minBucket, bucket); + maxBucket = Math.max(maxBucket, bucket); + int level = file.getInt(LEVEL); + minLevel = Math.min(minLevel, level); + maxLevel = Math.max(maxLevel, level); + minRowId = Math.min(minRowId, key.firstRowId); + maxRowId = Math.max(maxRowId, key.rangeEnd); + if (partition.isNullAt(0)) { + nullPartition = partition; + nullPartitionCount++; + } else { + if (minNonNullPartition == null) { + minNonNullPartition = partition; + } + maxNonNullPartition = partition; + } + } + + void finish(long end, ManifestEntryRunMergeEntry.Key lastKey) { + this.end = end; + this.lastKey = lastKey; + if (eligible && sorted) { + metadata = + new EncodedBlock( + addedFiles, + deletedFiles, + schemaId, + minBucket, + maxBucket, + minLevel, + maxLevel, + minRowId, + maxRowId, + nullPartition, + nullPartitionCount, + minNonNullPartition, + maxNonNullPartition); + } + } + + boolean copyable(long runStart, long runEnd) { + return metadata != null && start >= runStart && end <= runEnd; + } + + void finishFiltering(ManifestEntryRunMergeEntry.Filter filter) { + if (metadata != null && !filter.copyableAfterDiscovery(minRowId, maxRowId)) { + metadata = null; + } + } + + void updatePartitionRanks(ManifestEntryRunMergeEntry.PartitionDictionary partitions) { + firstKey.partitionRank = partitions.rank(firstKey.partitionId); + lastKey.partitionRank = partitions.rank(lastKey.partitionId); + } + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergeEntry.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergeEntry.java new file mode 100644 index 000000000000..078184ffc9d5 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergeEntry.java @@ -0,0 +1,308 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.operation; + +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.manifest.CompactFileIdentifierSet; +import org.apache.paimon.manifest.FileEntry.ReusableIdentifier; +import org.apache.paimon.manifest.FileKind; +import org.apache.paimon.manifest.ProjectedManifestEntry; +import org.apache.paimon.memory.MemorySegmentUtils; +import org.apache.paimon.utils.ByteArrayKey; +import org.apache.paimon.utils.ByteArrayLookupKey; +import org.apache.paimon.utils.SerializationUtils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import static org.apache.paimon.utils.Preconditions.checkState; + +/** Entry-level state shared by manifest run discovery and merge execution. */ +final class ManifestEntryRunMergeEntry { + + private ManifestEntryRunMergeEntry() {} + + static final class Key { + + int partitionId; + int partitionRank; + byte kind; + long firstRowId; + long rangeEnd; + long reverseSequence; + byte[] fileNameBytes; + int fileNameOffset; + int fileNameLength; + + static Key viewOf(ProjectedManifestEntry entry, PartitionDictionary partitions) { + Key key = new Key(); + key.replace(entry, partitions); + return key; + } + + void replace(ProjectedManifestEntry entry, PartitionDictionary partitions) { + long firstRowId = entry.file().nonNullFirstRowId(); + this.partitionId = partitions.id(entry.partitionBytes()); + this.partitionRank = partitions.rank(partitionId); + this.kind = entry.kind().toByteValue(); + this.firstRowId = firstRowId; + this.rangeEnd = firstRowId + entry.file().rowCount() - 1L; + this.reverseSequence = Long.MAX_VALUE - entry.file().maxSequenceNumber(); + this.fileNameBytes = entry.file().fileNameBinary().toBytes(); + this.fileNameOffset = 0; + this.fileNameLength = fileNameBytes.length; + } + + void replace(GenericRow record, PartitionDictionary partitions) { + InternalRow file = file(record); + checkState( + !file.isNullAt(ManifestEntryRunMerge.FIRST_ROW_ID), + "First row id should not be null."); + this.partitionId = partitions.id(record.getBinary(ManifestEntryRunMerge.PARTITION)); + this.partitionRank = partitions.rank(partitionId); + this.kind = record.getByte(ManifestEntryRunMerge.KIND); + this.firstRowId = file.getLong(ManifestEntryRunMerge.FIRST_ROW_ID); + this.rangeEnd = firstRowId + file.getLong(ManifestEntryRunMerge.ROW_COUNT) - 1L; + this.reverseSequence = + Long.MAX_VALUE - file.getLong(ManifestEntryRunMerge.MAX_SEQUENCE_NUMBER); + BinaryString fileName = file.getString(ManifestEntryRunMerge.FILE_NAME); + this.fileNameBytes = + MemorySegmentUtils.copyToBytes( + fileName.getSegments(), + fileName.getOffset(), + fileName.getSizeInBytes()); + this.fileNameOffset = 0; + this.fileNameLength = fileNameBytes.length; + } + + void copyFrom(Key key) { + this.partitionId = key.partitionId; + this.partitionRank = key.partitionRank; + this.kind = key.kind; + this.firstRowId = key.firstRowId; + this.rangeEnd = key.rangeEnd; + this.reverseSequence = key.reverseSequence; + this.fileNameBytes = key.fileNameBytes; + this.fileNameOffset = key.fileNameOffset; + this.fileNameLength = key.fileNameLength; + } + + Key stableCopy() { + Key copy = new Key(); + copy.copyFrom(this); + copy.fileNameBytes = + Arrays.copyOfRange( + fileNameBytes, fileNameOffset, fileNameOffset + fileNameLength); + copy.fileNameOffset = 0; + return copy; + } + + void clear() { + fileNameBytes = null; + } + } + + /** Interns variable-width partition bytes once and assigns comparator-compatible ranks. */ + static final class PartitionDictionary { + + final ManifestFileSorter.RowIdEntrySortKey sortKey; + final Map ids = new ConcurrentHashMap<>(); + final ThreadLocal lookup = + ThreadLocal.withInitial(ByteArrayLookupKey::new); + volatile BinaryRow[] partitions = new BinaryRow[16]; + int partitionCount; + int[] ranks; + + PartitionDictionary(ManifestFileSorter.RowIdEntrySortKey sortKey) { + this.sortKey = sortKey; + } + + int id(byte[] bytes) { + return id(bytes, 0, bytes.length); + } + + int id(byte[] bytes, int offset, int length) { + ByteArrayLookupKey lookupKey = lookup.get(); + lookupKey.reset(bytes, offset, length); + try { + Integer existing = ids.get(lookupKey); + if (existing != null) { + return existing; + } + synchronized (this) { + existing = ids.get(lookupKey); + if (existing != null) { + return existing; + } + checkState(ranks == null, "Full manifest scan found an unknown partition."); + byte[] canonical = Arrays.copyOfRange(bytes, offset, offset + length); + int id = partitionCount; + if (id == partitions.length) { + partitions = Arrays.copyOf(partitions, partitions.length << 1); + } + partitions[id] = SerializationUtils.deserializeBinaryRow(canonical); + ids.put(new ByteArrayKey(canonical), id); + partitionCount = id + 1; + return id; + } + } finally { + lookupKey.clear(); + } + } + + int compareIds(int left, int right) { + return sortKey.comparePartitions(partitions[left], partitions[right]); + } + + void finish() { + List order = new ArrayList<>(partitionCount); + for (int id = 0; id < partitionCount; id++) { + order.add(id); + } + order.sort((left, right) -> compareIds(left, right)); + ranks = new int[partitionCount]; + int rank = 0; + for (int position = 0; position < order.size(); position++) { + if (position > 0 && compareIds(order.get(position - 1), order.get(position)) != 0) { + rank++; + } + ranks[order.get(position)] = rank; + } + } + + int rank(int id) { + return ranks == null ? 0 : ranks[id]; + } + + BinaryRow partition(int id) { + return partitions[id]; + } + } + + static class Filter { + + final CompactFileIdentifierSet deletedIdentifiers; + final ManifestFileSorter.DeletedRowIdSet deletedRowIds; + final ThreadLocal identifier = + ThreadLocal.withInitial(IdentifierEncoder::new); + + Filter( + CompactFileIdentifierSet deletedIdentifiers, + ManifestFileSorter.DeletedRowIdSet deletedRowIds) { + this.deletedIdentifiers = deletedIdentifiers; + this.deletedRowIds = deletedRowIds; + } + + boolean include(ProjectedManifestEntry entry) { + return entry.isAdd() && !deletedIdentifiers.contains(entry); + } + + boolean include(GenericRow record, Key key) { + if (key.kind != FileKind.ADD.toByteValue()) { + return false; + } + if (!deletedRowIds.contains(key.firstRowId)) { + return true; + } + + ReusableIdentifier reusable = identifier.get().replace(record); + return !deletedIdentifiers.contains(reusable); + } + + boolean copyable(GenericRow record, Key key) { + return include(record, key); + } + + void observe(GenericRow record, Key key) {} + + boolean copyableAfterDiscovery(long minRowId, long maxRowId) { + return true; + } + + ReusableIdentifier identifier(GenericRow record) { + return identifier.get().replace(record); + } + + static final class Minor extends Filter { + + Minor( + CompactFileIdentifierSet deletedIdentifiers, + ManifestFileSorter.DeletedRowIdSet deletedRowIds) { + super(deletedIdentifiers, deletedRowIds); + } + + @Override + boolean include(ProjectedManifestEntry entry) { + return true; + } + + @Override + boolean include(GenericRow record, Key key) { + return true; + } + + @Override + boolean copyable(GenericRow record, Key key) { + return key.kind == FileKind.ADD.toByteValue(); + } + + @Override + void observe(GenericRow record, Key key) { + if (key.kind != FileKind.DELETE.toByteValue()) { + return; + } + ReusableIdentifier reusable = identifier(record); + synchronized (this) { + deletedIdentifiers.add(reusable); + deletedRowIds.add(key.firstRowId); + } + } + + @Override + boolean copyableAfterDiscovery(long minRowId, long maxRowId) { + // A DELETE preserves the deleted ADD's globally unique first RowID. A range hit may + // be a false positive and only disables block copying; a miss proves the block has + // no deleted ADD. + return !deletedRowIds.intersects(minRowId, maxRowId); + } + } + + private static final class IdentifierEncoder { + + final ProjectedManifestEntry entry = + ProjectedManifestEntry.Projection.create(ManifestEntryRunMerge.ENTRY_LAYOUT) + .createEntry(); + final ReusableIdentifier identifier = new ReusableIdentifier(); + + ReusableIdentifier replace(GenericRow record) { + return identifier.replaceWithPartition(entry.replace(record)); + } + } + } + + static InternalRow file(GenericRow record) { + return record.getRow(ManifestEntryRunMerge.FILE, ManifestEntryRunMerge.FILE_FIELD_COUNT); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergePlan.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergePlan.java new file mode 100644 index 000000000000..2f88d97c1823 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergePlan.java @@ -0,0 +1,797 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.operation; + +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.data.serializer.InternalRowSerializer; +import org.apache.paimon.format.avro.AvroRawBlock; +import org.apache.paimon.manifest.CompactFileIdentifierSet; +import org.apache.paimon.manifest.FileEntry.ReusableIdentifier; +import org.apache.paimon.manifest.FileKind; +import org.apache.paimon.manifest.ManifestAvroReader; +import org.apache.paimon.manifest.ManifestAvroReader.RawBlock; +import org.apache.paimon.manifest.ManifestAvroReader.RowIterator; +import org.apache.paimon.manifest.ManifestAvroWriter; +import org.apache.paimon.manifest.ManifestAvroWriter.EncodedBlock; +import org.apache.paimon.manifest.ManifestAvroWriter.EncodedEntry; +import org.apache.paimon.manifest.ManifestEntry; +import org.apache.paimon.manifest.ManifestFile; +import org.apache.paimon.manifest.ManifestFileMeta; +import org.apache.paimon.manifest.ProjectedManifestEntry; +import org.apache.paimon.utils.CloseableIterator; +import org.apache.paimon.utils.Pair; + +import javax.annotation.Nullable; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.apache.paimon.utils.Preconditions.checkState; + +final class ManifestEntryRunMergePlan { + + final List sources; + final ManifestEntryRunMergeEntry.PartitionDictionary partitions; + + ManifestEntryRunMergePlan( + List sources, ManifestEntryRunMergeEntry.PartitionDictionary partitions) { + this.sources = sources; + this.partitions = partitions; + } + + List mergeToManifest( + ManifestFileSorter.RowIdEntrySortKey sortKey, + ManifestFile manifestFile, + ManifestEntryRunMergeEntry.Filter filter, + List newFilesForAbort) + throws Exception { + List cursors = new ArrayList<>(sources.size()); + Exception failure = null; + try { + for (Source.Spec source : sources) { + Cursor cursor = source.open(manifestFile, sortKey, filter, partitions); + cursors.add(cursor); + cursor.advance(); + } + SelectionTree selectionTree = new SelectionTree(cursors); + if (selectionTree.winner() < 0) { + return Collections.emptyList(); + } + List files = writeSelected(selectionTree, manifestFile); + newFilesForAbort.addAll(files); + return files; + } catch (Exception e) { + failure = e; + throw e; + } finally { + try { + closeCursors(cursors); + } catch (Exception closeFailure) { + if (failure == null) { + throw closeFailure; + } + failure.addSuppressed(closeFailure); + } + } + } + + Pair, List> mergeMinorToManifest( + ManifestFileSorter.RowIdEntrySortKey sortKey, + ManifestFile manifestFile, + ManifestEntryRunMergeEntry.Filter filter, + CompactFileIdentifierSet deletedIdentifiers, + ManifestFileSorter.DeletedRowIdSet deletedRowIds, + List newFilesForAbort) + throws Exception { + List cursors = new ArrayList<>(sources.size()); + Exception failure = null; + try { + for (Source.Spec source : sources) { + Cursor cursor = source.open(manifestFile, sortKey, filter, partitions); + cursors.add(cursor); + cursor.advance(); + } + SelectionTree selectionTree = new SelectionTree(cursors); + if (selectionTree.winner() < 0) { + return Pair.of(Collections.emptyList(), Collections.emptyList()); + } + Pair, List> files = + writeMinorSelected( + selectionTree, manifestFile, deletedIdentifiers, deletedRowIds); + newFilesForAbort.addAll(files.getLeft()); + newFilesForAbort.addAll(files.getRight()); + return files; + } catch (Exception e) { + failure = e; + throw e; + } finally { + try { + closeCursors(cursors); + } catch (Exception closeFailure) { + if (failure == null) { + throw closeFailure; + } + failure.addSuppressed(closeFailure); + } + } + } + + static List writeSelected( + SelectionTree selectionTree, ManifestFile manifestFile) throws Exception { + ManifestAvroWriter writer = manifestFile.createAvroWriter(); + Exception failure = null; + try { + int winner; + while ((winner = selectionTree.winner()) >= 0) { + Cursor cursor = selectionTree.cursor(winner); + if (cursor.hasCopyableBlock() + && selectionTree.blockPrecedesOthers(winner, cursor.blockLastKey())) { + writer.writeEncodedBlock(cursor.encodedBlock(), cursor.blockMetadata()); + selectionTree.update(winner, cursor.advanceAfterBlock()); + continue; + } + cursor.materializeCurrent(); + ByteBuffer encodedRecord = cursor.encodedRecord(); + if (encodedRecord == null) { + writer.write(cursor.current()); + } else { + writer.writeEncoded(encodedRecord, cursor.metadata()); + } + selectionTree.update(winner, cursor.advance()); + } + } catch (Exception e) { + failure = e; + } finally { + if (failure != null) { + writer.abort(); + throw failure; + } + writer.close(); + } + return writer.result(); + } + + private static Pair, List> writeMinorSelected( + SelectionTree selectionTree, + ManifestFile manifestFile, + CompactFileIdentifierSet deletedIdentifiers, + ManifestFileSorter.DeletedRowIdSet deletedRowIds) + throws Exception { + ManifestAvroWriter addWriter = manifestFile.createAvroWriter(); + ManifestAvroWriter deleteWriter = manifestFile.createAvroWriter(); + CompactFileIdentifierSet matchedEntries = new CompactFileIdentifierSet(); + CompactFileIdentifierSet emittedDeletes = new CompactFileIdentifierSet(); + Exception failure = null; + try { + int winner; + while ((winner = selectionTree.winner()) >= 0) { + Cursor cursor = selectionTree.cursor(winner); + if (cursor.hasCopyableBlock() + && selectionTree.blockPrecedesOthers(winner, cursor.blockLastKey())) { + addWriter.writeEncodedBlock(cursor.encodedBlock(), cursor.blockMetadata()); + selectionTree.update(winner, cursor.advanceAfterBlock()); + continue; + } + + cursor.materializeCurrent(); + if (cursor.key().kind == FileKind.ADD.toByteValue()) { + if (!deletedRowIds.contains(cursor.key().firstRowId)) { + writeCurrent(addWriter, cursor); + } else { + ReusableIdentifier identifier = cursor.identifier(); + if (deletedIdentifiers.contains(identifier)) { + matchedEntries.add(identifier); + } else { + writeCurrent(addWriter, cursor); + } + } + } else { + ReusableIdentifier identifier = cursor.identifier(); + if (!matchedEntries.contains(identifier) + && !emittedDeletes.contains(identifier)) { + emittedDeletes.add(identifier); + writeCurrent(deleteWriter, cursor); + } + } + selectionTree.update(winner, cursor.advance()); + } + addWriter.close(); + deleteWriter.close(); + } catch (Exception e) { + failure = e; + } finally { + matchedEntries.release(); + emittedDeletes.release(); + if (failure != null) { + addWriter.abort(); + deleteWriter.abort(); + throw failure; + } + } + return Pair.of(addWriter.result(), deleteWriter.result()); + } + + private static void writeCurrent(ManifestAvroWriter writer, Cursor cursor) throws Exception { + ByteBuffer encodedRecord = cursor.encodedRecord(); + if (encodedRecord == null) { + writer.write(cursor.current()); + } else { + writer.writeEncoded(encodedRecord, cursor.metadata()); + } + } + + static void closeCursors(List cursors) throws Exception { + Exception failure = null; + for (Cursor cursor : cursors) { + try { + cursor.close(); + } catch (Exception e) { + if (failure == null) { + failure = e; + } else { + failure.addSuppressed(e); + } + } + } + if (failure != null) { + throw failure; + } + } + + /** Describes the manifest inputs which become cursors when this plan starts executing. */ + static final class Source { + + private Source() {} + + interface Spec { + + Cursor open( + ManifestFile manifestFile, + ManifestFileSorter.RowIdEntrySortKey sortKey, + ManifestEntryRunMergeEntry.Filter filter, + ManifestEntryRunMergeEntry.PartitionDictionary partitions) + throws Exception; + } + + static final class ManifestRunSpec implements Spec { + + final ManifestFileMeta meta; + final long start; + final long end; + final List blocks; + + ManifestRunSpec( + ManifestFileMeta meta, + long start, + long end, + List blocks) { + this.meta = meta; + this.start = start; + this.end = end; + this.blocks = blocks; + } + + long prefixBlockCount() { + long lastBlockOrdinal = -1; + for (ManifestEntryRunMerge.Discovery.BlockInfo block : blocks) { + if (block.start >= end) { + break; + } + if (block.end > start) { + lastBlockOrdinal = block.ordinal; + } + } + checkState(lastBlockOrdinal >= 0, "Manifest run does not contain an Avro block."); + return lastBlockOrdinal + 1; + } + + @Override + public Cursor open( + ManifestFile manifestFile, + ManifestFileSorter.RowIdEntrySortKey sortKey, + ManifestEntryRunMergeEntry.Filter filter, + ManifestEntryRunMergeEntry.PartitionDictionary partitions) + throws Exception { + return new PrimitiveManifestRunCursor( + manifestFile, meta, start, end, blocks, filter, partitions); + } + } + + static final class FragmentedManifestSpec implements Spec { + + final ManifestFileMeta meta; + + FragmentedManifestSpec(ManifestFileMeta meta) { + this.meta = meta; + } + + @Override + public Cursor open( + ManifestFile manifestFile, + ManifestFileSorter.RowIdEntrySortKey sortKey, + ManifestEntryRunMergeEntry.Filter filter, + ManifestEntryRunMergeEntry.PartitionDictionary partitions) + throws Exception { + return new InMemoryManifestCursor(manifestFile, meta, sortKey, filter, partitions); + } + } + } + + interface Cursor extends AutoCloseable { + + boolean advance() throws Exception; + + boolean hasCurrent(); + + @Nullable + ProjectedManifestEntry current(); + + @Nullable + EncodedEntry metadata(); + + ManifestEntryRunMergeEntry.Key key(); + + @Nullable + ByteBuffer encodedRecord(); + + ReusableIdentifier identifier(); + + default boolean hasCopyableBlock() { + return false; + } + + default ManifestEntryRunMergeEntry.Key blockLastKey() { + throw new UnsupportedOperationException(); + } + + default AvroRawBlock encodedBlock() { + throw new UnsupportedOperationException(); + } + + default EncodedBlock blockMetadata() { + throw new UnsupportedOperationException(); + } + + default boolean advanceAfterBlock() throws Exception { + throw new UnsupportedOperationException(); + } + + default void materializeCurrent() throws Exception {} + + @Override + void close() throws Exception; + } + + static final class PrimitiveManifestRunCursor implements Cursor { + + final ManifestAvroReader reader; + final ManifestEntryRunMergeEntry.Filter filter; + final ManifestEntryRunMergeEntry.PartitionDictionary partitions; + final ManifestEntryRunMergeEntry.Key key = new ManifestEntryRunMergeEntry.Key(); + final EncodedEntry metadata = new EncodedEntry(); + final List blocks; + final long runStart; + final long runEnd; + int blockIndex; + long nextReaderBlockOrdinal; + long decodedRemaining; + boolean rawBlock; + boolean current; + @Nullable RawBlock currentRawBlock; + @Nullable RowIterator currentRows; + @Nullable GenericRow currentRow; + @Nullable ManifestEntryRunMerge.Discovery.BlockInfo currentBlock; + boolean closed; + + PrimitiveManifestRunCursor( + ManifestFile manifestFile, + ManifestFileMeta meta, + long start, + long end, + List blocks, + ManifestEntryRunMergeEntry.Filter filter, + ManifestEntryRunMergeEntry.PartitionDictionary partitions) + throws Exception { + this.reader = manifestFile.scanForRunMerge(meta.fileName(), meta.fileSize()); + this.filter = filter; + this.partitions = partitions; + this.blocks = blocks; + this.runStart = start; + this.runEnd = end; + try { + while (blockIndex < blocks.size() && blocks.get(blockIndex).end <= start) { + blockIndex++; + } + checkState( + blockIndex < blocks.size(), + "Manifest run starts after the end of the file."); + } catch (Exception e) { + try { + reader.close(); + } catch (Exception closeFailure) { + e.addSuppressed(closeFailure); + } + throw e; + } + } + + @Override + public boolean advance() throws Exception { + current = false; + while (true) { + if (decodedRemaining == 0) { + if (!prepareNextBlock()) { + key.clear(); + close(); + return false; + } + if (rawBlock) { + return true; + } + } + checkState( + currentRows != null && currentRows.hasNext(), + "Manifest block ends before its discovered boundary."); + currentRow = currentRows.next(); + decodedRemaining--; + key.replace(currentRow, partitions); + if (filter.include(currentRow, key)) { + current = true; + InternalRow file = ManifestEntryRunMergeEntry.file(currentRow); + metadata.replace( + key.kind, + partitions.partition(key.partitionId), + currentRow.getInt(ManifestEntryRunMerge.BUCKET), + file.getInt(ManifestEntryRunMerge.LEVEL), + file.getLong(ManifestEntryRunMerge.SCHEMA_ID), + key.firstRowId, + file.getLong(ManifestEntryRunMerge.ROW_COUNT)); + return true; + } + } + } + + boolean prepareNextBlock() throws Exception { + rawBlock = false; + current = false; + currentRows = null; + currentRow = null; + while (blockIndex < blocks.size()) { + ManifestEntryRunMerge.Discovery.BlockInfo info = blocks.get(blockIndex); + if (info.start >= runEnd) { + return false; + } + while (nextReaderBlockOrdinal < info.ordinal) { + checkState(reader.hasNext(), "Manifest block ordinal is missing."); + reader.next(); + nextReaderBlockOrdinal++; + } + checkState(reader.hasNext(), "Manifest run ends after the end of the file."); + currentRawBlock = reader.next(); + nextReaderBlockOrdinal++; + currentBlock = info; + if (info.copyable(runStart, runEnd)) { + rawBlock = true; + key.copyFrom(info.firstKey); + return true; + } + + long overlapStart = Math.max(runStart, info.start); + long overlapEnd = Math.min(runEnd, info.end); + long prefix = overlapStart - info.start; + currentRows = currentRawBlock.toRows(ManifestEntryRunMerge.ENTRY_LAYOUT); + for (long i = 0; i < prefix; i++) { + checkState( + currentRows.hasNext(), + "Manifest run starts after the end of its block."); + currentRows.next(); + } + decodedRemaining = overlapEnd - overlapStart; + blockIndex++; + if (decodedRemaining > 0) { + return true; + } + } + return false; + } + + @Override + public boolean hasCurrent() { + return current || rawBlock; + } + + @Override + public ProjectedManifestEntry current() { + return null; + } + + @Override + public EncodedEntry metadata() { + return metadata; + } + + @Override + public ManifestEntryRunMergeEntry.Key key() { + return key; + } + + @Override + public ByteBuffer encodedRecord() { + return current ? currentRows.encodedRecord() : null; + } + + @Override + public ReusableIdentifier identifier() { + checkState(current, "Manifest entry has not been materialized."); + return filter.identifier(currentRow); + } + + @Override + public boolean hasCopyableBlock() { + return rawBlock; + } + + @Override + public ManifestEntryRunMergeEntry.Key blockLastKey() { + return currentBlock.lastKey; + } + + @Override + public AvroRawBlock encodedBlock() { + return currentRawBlock.encodedBlock(); + } + + @Override + public EncodedBlock blockMetadata() { + return currentBlock.metadata; + } + + @Override + public boolean advanceAfterBlock() throws Exception { + checkState(rawBlock, "There is no raw block to advance."); + rawBlock = false; + currentRawBlock = null; + blockIndex++; + return advance(); + } + + @Override + public void materializeCurrent() throws Exception { + if (!rawBlock) { + return; + } + rawBlock = false; + decodedRemaining = currentBlock.end - currentBlock.start; + checkState(decodedRemaining > 0, "Raw Avro block is empty."); + currentRows = currentRawBlock.toRows(ManifestEntryRunMerge.ENTRY_LAYOUT); + checkState(currentRows.hasNext(), "Manifest block cannot be decompressed."); + currentRow = currentRows.next(); + decodedRemaining--; + key.replace(currentRow, partitions); + checkState( + filter.include(currentRow, key), + "Copyable manifest block contains a filtered entry."); + current = true; + InternalRow file = ManifestEntryRunMergeEntry.file(currentRow); + metadata.replace( + key.kind, + partitions.partition(key.partitionId), + currentRow.getInt(ManifestEntryRunMerge.BUCKET), + file.getInt(ManifestEntryRunMerge.LEVEL), + file.getLong(ManifestEntryRunMerge.SCHEMA_ID), + key.firstRowId, + file.getLong(ManifestEntryRunMerge.ROW_COUNT)); + blockIndex++; + } + + @Override + public void close() throws Exception { + if (closed) { + return; + } + closed = true; + current = false; + currentRawBlock = null; + currentRows = null; + currentRow = null; + currentBlock = null; + rawBlock = false; + key.clear(); + reader.close(); + } + } + + static final class InMemoryManifestCursor implements Cursor { + + final List entries; + final ProjectedManifestEntry current = + ProjectedManifestEntry.fullProjection().createEntry(); + final ReusableIdentifier identifier = new ReusableIdentifier(); + int position = -1; + + InMemoryManifestCursor( + ManifestFile manifestFile, + ManifestFileMeta meta, + ManifestFileSorter.RowIdEntrySortKey sortKey, + ManifestEntryRunMergeEntry.Filter filter, + ManifestEntryRunMergeEntry.PartitionDictionary partitions) + throws Exception { + long entryCount = meta.numAddedFiles() + meta.numDeletedFiles(); + this.entries = new ArrayList<>((int) entryCount); + InternalRowSerializer serializer = + new InternalRowSerializer(ManifestEntry.MANIFEST_ROW_TYPE); + ProjectedManifestEntry view = ProjectedManifestEntry.fullProjection().createEntry(); + try (CloseableIterator iterator = + manifestFile.scan(meta.fileName(), ProjectedManifestEntry.fullProjection())) { + while (iterator.hasNext()) { + ProjectedManifestEntry entry = iterator.next(); + if (!filter.include(entry)) { + continue; + } + BinaryRow row = serializer.toBinaryRow(entry.fullRow()).copy(); + entries.add( + new StoredEntry( + row, + ManifestEntryRunMergeEntry.Key.viewOf( + view.replace(row), partitions))); + } + } + entries.sort( + (left, right) -> ManifestEntryRunMerge.compareMergeKeys(left.key, right.key)); + view.clear(); + } + + @Override + public boolean advance() { + position++; + if (position >= entries.size()) { + current.clear(); + return false; + } + StoredEntry stored = entries.get(position); + current.replace(stored.row); + return true; + } + + @Override + public boolean hasCurrent() { + return position >= 0 && position < entries.size(); + } + + @Override + public ProjectedManifestEntry current() { + return current; + } + + @Override + public EncodedEntry metadata() { + return null; + } + + @Override + public ManifestEntryRunMergeEntry.Key key() { + return entries.get(position).key; + } + + @Override + public ByteBuffer encodedRecord() { + return null; + } + + @Override + public ReusableIdentifier identifier() { + return identifier.replaceWithPartition(current); + } + + @Override + public void close() { + current.clear(); + identifier.release(); + entries.clear(); + position = -1; + } + } + + private static final class StoredEntry { + + final BinaryRow row; + final ManifestEntryRunMergeEntry.Key key; + + StoredEntry(BinaryRow row, ManifestEntryRunMergeEntry.Key key) { + this.row = row; + this.key = key; + } + } + + /** Fixed-size tournament tree which selects a cursor with one comparison per tree level. */ + private static final class SelectionTree { + + final List cursors; + final int leafBase; + final int[] winners; + + SelectionTree(List cursors) { + this.cursors = cursors; + int base = 1; + while (base < cursors.size()) { + base <<= 1; + } + this.leafBase = base; + this.winners = new int[leafBase << 1]; + Arrays.fill(winners, -1); + for (int cursor = 0; cursor < cursors.size(); cursor++) { + if (cursors.get(cursor).hasCurrent()) { + winners[leafBase + cursor] = cursor; + } + } + for (int node = leafBase - 1; node > 0; node--) { + winners[node] = select(winners[node << 1], winners[(node << 1) + 1]); + } + } + + int winner() { + return winners[1]; + } + + Cursor cursor(int index) { + return cursors.get(index); + } + + void update(int cursor, boolean hasCurrent) { + int node = leafBase + cursor; + winners[node] = hasCurrent ? cursor : -1; + while ((node >>= 1) > 0) { + winners[node] = select(winners[node << 1], winners[(node << 1) + 1]); + } + } + + int select(int left, int right) { + if (left < 0) { + return right; + } + if (right < 0) { + return left; + } + int comparison = + ManifestEntryRunMerge.compareMergeKeys( + cursors.get(left).key(), cursors.get(right).key()); + return comparison < 0 || (comparison == 0 && left < right) ? left : right; + } + + boolean blockPrecedesOthers(int cursor, ManifestEntryRunMergeEntry.Key blockLastKey) { + for (int other = 0; other < cursors.size(); other++) { + if (other == cursor || !cursors.get(other).hasCurrent()) { + continue; + } + int comparison = + ManifestEntryRunMerge.compareMergeKeys( + blockLastKey, cursors.get(other).key()); + if (comparison > 0 || (comparison == 0 && cursor > other)) { + return false; + } + } + return true; + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java index 6ca30b644479..d109ebb76909 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java @@ -70,9 +70,11 @@ public class ManifestFileSorter { /** Context object that carries shared state across compaction methods. */ static class CompactionContext { final boolean fullCompaction; + final boolean runMergeOptimizeEnabled; final ManifestSortKey sortKey; final ManifestEntryExternalSort.ExternalSortConfig externalSortConfig; final CompactFileIdentifierSet deleteEntries; + final DeletedRowIdSet deletedRowIds; /** * Manifest files that need unsorted compaction. * @@ -81,38 +83,43 @@ static class CompactionContext { *

Value: true if fullCompaction is true and the file overlaps with delete partitions. It * means the file needs to eliminate delete entries file */ - final Map compactWithoutSort; + final Map defaultCompactFiles; final List levelRuns; final List pickedRuns; CompactionContext( boolean fullCompaction, + boolean runMergeOptimizeEnabled, ManifestSortKey sortKey, ManifestEntryExternalSort.ExternalSortConfig externalSortConfig, CompactFileIdentifierSet deleteEntries, - Map compactWithoutSort, + DeletedRowIdSet deletedRowIds, + Map defaultCompactFiles, List levelRuns, List pickedRuns) { this.fullCompaction = fullCompaction; + this.runMergeOptimizeEnabled = runMergeOptimizeEnabled; this.sortKey = sortKey; this.externalSortConfig = externalSortConfig; this.deleteEntries = deleteEntries; - this.compactWithoutSort = compactWithoutSort; + this.deletedRowIds = deletedRowIds; + this.defaultCompactFiles = defaultCompactFiles; this.levelRuns = levelRuns; this.pickedRuns = pickedRuns; } /** Check whether the given manifest file is marked for unsorted compaction. */ - boolean isMarkedForUnsortedCompaction(ManifestFileMeta file) { - return compactWithoutSort.containsKey(file); + boolean isMarkedForDefaultCompaction(ManifestFileMeta file) { + return defaultCompactFiles.containsKey(file); } } /** Result of classifying manifest files. */ - private static class ClassifyResult { + private static class ManifestClassification { final List lsmFiles; final CompactFileIdentifierSet deleteEntries; + final DeletedRowIdSet deletedRowIds; /** * Manifest files that need unsorted compaction. * @@ -121,29 +128,153 @@ private static class ClassifyResult { *

Value: true if fullCompaction is true and the file overlaps with delete partitions. It * means the file needs to eliminate delete entries file */ - final Map compactWithoutSort; + final Map defaultCompactFiles; - ClassifyResult( + ManifestClassification( List lsmFiles, CompactFileIdentifierSet deleteEntries, - Map compactWithoutSort) { + DeletedRowIdSet deletedRowIds, + Map defaultCompactFiles) { this.lsmFiles = lsmFiles; this.deleteEntries = deleteEntries; - this.compactWithoutSort = compactWithoutSort; + this.deletedRowIds = deletedRowIds; + this.defaultCompactFiles = defaultCompactFiles; } } /** Binary identifiers and partition values collected from DELETE entries. */ private static class DeletedEntryInfo { final CompactFileIdentifierSet identifiers; + final DeletedRowIdSet rowIds; final Set partitions; - private DeletedEntryInfo(CompactFileIdentifierSet identifiers, Set partitions) { + private DeletedEntryInfo( + CompactFileIdentifierSet identifiers, + DeletedRowIdSet rowIds, + Set partitions) { this.identifiers = identifiers; + this.rowIds = rowIds; this.partitions = partitions; } } + /** Primitive set used by RowID full compaction to avoid rebuilding file identifiers. */ + static final class DeletedRowIdSet { + + private static final long EMPTY = Long.MIN_VALUE; + private long[] table = emptyTable(16); + private int size; + private boolean containsMinValue; + private @Nullable long[] sortedRowIds; + + void add(long value) { + if (value == EMPTY) { + if (!containsMinValue) { + containsMinValue = true; + size++; + sortedRowIds = null; + } + return; + } + if ((size + 1) * 2 > table.length) { + grow(); + } + int slot = slot(value, table.length); + while (table[slot] != EMPTY) { + if (table[slot] == value) { + return; + } + slot = (slot + 1) & (table.length - 1); + } + table[slot] = value; + size++; + sortedRowIds = null; + } + + boolean contains(long value) { + if (value == EMPTY) { + return containsMinValue; + } + int slot = slot(value, table.length); + while (table[slot] != EMPTY) { + if (table[slot] == value) { + return true; + } + slot = (slot + 1) & (table.length - 1); + } + return false; + } + + boolean intersects(long minInclusive, long maxInclusive) { + if (minInclusive > maxInclusive) { + return true; + } + long[] values = sortedRowIds(); + int position = java.util.Arrays.binarySearch(values, minInclusive); + if (position < 0) { + position = -position - 1; + } + return position < values.length && values[position] <= maxInclusive; + } + + private long[] sortedRowIds() { + if (sortedRowIds != null) { + return sortedRowIds; + } + long[] values = new long[size]; + int position = 0; + if (containsMinValue) { + values[position++] = EMPTY; + } + for (long value : table) { + if (value != EMPTY) { + values[position++] = value; + } + } + if (position != size) { + throw new IllegalStateException("Failed to snapshot deleted RowID set."); + } + java.util.Arrays.sort(values); + sortedRowIds = values; + return values; + } + + void releaseRangeIndex() { + sortedRowIds = null; + } + + private void grow() { + long[] previous = table; + if (previous.length >= (1 << 30)) { + throw new IllegalStateException("Too many deleted RowIDs in one manifest group."); + } + table = emptyTable(previous.length << 1); + int previousSize = size; + size = containsMinValue ? 1 : 0; + for (long value : previous) { + if (value != EMPTY) { + add(value); + } + } + if (size != previousSize) { + throw new IllegalStateException("Failed to grow deleted RowID set."); + } + } + + private static int slot(long value, int length) { + value ^= value >>> 33; + value *= 0xff51afd7ed558ccdL; + value ^= value >>> 33; + return ((int) value) & (length - 1); + } + + private static long[] emptyTable(int length) { + long[] table = new long[length]; + java.util.Arrays.fill(table, EMPTY); + return table; + } + } + /** * Try to sort-rewrite the merged manifest list by a configured partition field. If the sort * field cannot be resolved, the input is returned as-is. @@ -160,6 +291,7 @@ static List trySortCompaction( @Nullable IOManager ioManager) throws Exception { String sortPartitionField = options.manifestSortPartitionField(); + boolean runMergeOptimizeEnabled = options.manifestSortRunMergeOptimizeEnabled(); long suggestedMetaSize = options.manifestTargetSize().getBytes(); int suggestedMinMetaCount = options.manifestMergeMinCount(); long fullCompactionThreshold = options.manifestFullCompactionThresholdSize().getBytes(); @@ -178,6 +310,7 @@ static List trySortCompaction( partitionType, sortPartitionField, options.dataEvolutionEnabled(), + runMergeOptimizeEnabled, suggestedMetaSize, suggestedMinMetaCount, fullCompactionThreshold, @@ -196,6 +329,7 @@ static List trySortCompaction( partitionType, sortPartitionField, options.dataEvolutionEnabled(), + runMergeOptimizeEnabled, suggestedMetaSize, suggestedMinMetaCount, maxRewriteSize, @@ -218,6 +352,7 @@ private static Optional> tryFullCompaction( RowType partitionType, String sortPartitionField, boolean dataEvolutionEnabled, + boolean runMergeOptimizeEnabled, long suggestedMetaSize, int suggestedMinMetaCount, long fullCompactionThreshold, @@ -246,6 +381,7 @@ private static Optional> tryFullCompaction( partitionType, sortPartitionField, dataEvolutionEnabled, + runMergeOptimizeEnabled, suggestedMetaSize, maxSizeAmplificationPercent, sortedRunSizeRatio, @@ -254,19 +390,19 @@ private static Optional> tryFullCompaction( List levelRuns = ctx.levelRuns; List pickedRuns = ctx.pickedRuns; - if (pickedRuns.isEmpty() && ctx.compactWithoutSort.isEmpty()) { + if (pickedRuns.isEmpty() && ctx.defaultCompactFiles.isEmpty()) { LOG.debug( - "Manifest sort full compact skipped: no runs picked and no compactWithoutSort files."); + "Manifest sort full compact skipped: no runs picked and no defaultCompactFiles."); return Optional.empty(); } LOG.info( "Manifest sort full compact: input={} files, lsm={} runs, picked={} runs, " - + "compactWithoutSort={} files.", + + "defaultCompactFiles={}.", input.size(), levelRuns.size(), pickedRuns.size(), - ctx.compactWithoutSort.size()); + ctx.defaultCompactFiles.size()); // Step 3: Collect reused files (not picked) and picked files Set pickedSet = new HashSet<>(pickedRuns); @@ -280,7 +416,7 @@ private static Optional> tryFullCompaction( for (ManifestAdjacentSortedRun run : pickedRuns) { pickedFiles.addAll(run.files()); } - pickedFiles.addAll(ctx.compactWithoutSort.keySet()); + pickedFiles.addAll(ctx.defaultCompactFiles.keySet()); // Step 4: Split into sections and merge small adjacent sections List

sections = splitIntoSections(pickedFiles, ctx); @@ -324,6 +460,7 @@ private static List tryMinorCompaction( RowType partitionType, String sortPartitionField, boolean dataEvolutionEnabled, + boolean runMergeOptimizeEnabled, long suggestedMetaSize, int suggestedMinMetaCount, long maxRewriteSize, @@ -341,6 +478,7 @@ private static List tryMinorCompaction( partitionType, sortPartitionField, dataEvolutionEnabled, + runMergeOptimizeEnabled, suggestedMetaSize, maxSizeAmplificationPercent, sortedRunSizeRatio, @@ -349,19 +487,19 @@ private static List tryMinorCompaction( List levelRuns = ctx.levelRuns; List pickedRuns = ctx.pickedRuns; - if (pickedRuns.isEmpty() && ctx.compactWithoutSort.isEmpty()) { + if (pickedRuns.isEmpty() && ctx.defaultCompactFiles.isEmpty()) { LOG.debug( - "Manifest sort minor compact skipped: no runs picked and no compactWithoutSort files."); + "Manifest sort minor compact skipped: no runs picked and no defaultCompactFiles."); return input; } LOG.info( "Manifest sort minor compact: input={} files, lsm={} runs, picked={} runs, " - + "compactWithoutSort={} files.", + + "defaultCompactFiles={}.", input.size(), levelRuns.size(), pickedRuns.size(), - ctx.compactWithoutSort.size()); + ctx.defaultCompactFiles.size()); // Step 2: Build fileName -> index mapping and initialize 2D result Map fileNameToIndex = new HashMap<>(); @@ -388,7 +526,7 @@ private static List tryMinorCompaction( for (ManifestAdjacentSortedRun run : pickedRuns) { pickedFiles.addAll(run.files()); } - pickedFiles.addAll(ctx.compactWithoutSort.keySet()); + pickedFiles.addAll(ctx.defaultCompactFiles.keySet()); // Step 4: Compute index range int minIdx = Integer.MAX_VALUE; @@ -450,26 +588,29 @@ private static CompactionContext prepareCompaction( RowType partitionType, String sortPartitionField, boolean dataEvolutionEnabled, + boolean runMergeOptimizeEnabled, long suggestedMetaSize, int maxSizeAmplificationPercent, int sortedRunSizeRatio, ManifestEntryExternalSort.ExternalSortConfig externalSortConfig, @Nullable Integer manifestReadParallelism) { + boolean rowIdSort = dataEvolutionEnabled && ManifestFileMeta.allContainsRowId(input); + boolean useRunMergeOptimize = rowIdSort && runMergeOptimizeEnabled; // Step 1: Resolve sort key. Data evolution tables prefer RowID ranges when available. - ManifestSortKey sortKey = - createSortKey(dataEvolutionEnabled, input, sortPartitionField, partitionType); + ManifestSortKey sortKey = createSortKey(rowIdSort, sortPartitionField, partitionType); // Step 2: Classify manifests into LSM files and collect delete entries. - ClassifyResult classifyResult = + ManifestClassification classification = classifyManifests( input, fullCompaction, manifestFile, partitionType, suggestedMetaSize, + useRunMergeOptimize, manifestReadParallelism); - List lsmFiles = classifyResult.lsmFiles; + List lsmFiles = classification.lsmFiles; // Step 3: Build level-sorted runs from LSM files based on partition order. List levelRuns = @@ -482,10 +623,12 @@ private static CompactionContext prepareCompaction( return new CompactionContext( fullCompaction, + useRunMergeOptimize, sortKey, externalSortConfig, - classifyResult.deleteEntries, - classifyResult.compactWithoutSort, + classification.deleteEntries, + classification.deletedRowIds, + classification.defaultCompactFiles, levelRuns, pickedRuns); } @@ -494,30 +637,34 @@ private static CompactionContext prepareCompaction( * Classify manifest files into default-compaction group and LSM group. * *

Full compaction: small files and files overlapping delete partitions go into - * compactWithoutSort; the rest are returned as lsmFiles. + * defaultCompactFiles; the rest are returned as lsmFiles. * - *

Non-full compaction: small files go to compactWithoutSort for minor-style merge; the rest + *

Non-full compaction: small files go to defaultCompactFiles for minor-style merge; the rest * are returned as lsmFiles. * - * @return ClassifyResult containing lsmFiles, deleteEntries, and compactWithoutSort + * @return classification containing lsmFiles, deleteEntries, and defaultCompactFiles */ - private static ClassifyResult classifyManifests( + private static ManifestClassification classifyManifests( List input, boolean fullCompaction, ManifestFile manifestFile, RowType partitionType, long suggestedMetaSize, + boolean runMergeOptimizeEnabled, @Nullable Integer manifestReadParallelism) { // Initialize classification containers and read delete entries - Map compactWithoutSort = new LinkedHashMap<>(); + Map defaultCompactFiles = new LinkedHashMap<>(); List lsmFiles = new LinkedList<>(input); CompactFileIdentifierSet classifiedDeleteEntries = new CompactFileIdentifierSet(); + DeletedRowIdSet deletedRowIds = new DeletedRowIdSet(); Set deletePartitions = Collections.emptySet(); PartitionPredicate predicate = null; if (fullCompaction) { DeletedEntryInfo deletedEntries = - readDeletedEntries(manifestFile, input, manifestReadParallelism); + readDeletedEntries( + manifestFile, input, runMergeOptimizeEnabled, manifestReadParallelism); classifiedDeleteEntries = deletedEntries.identifiers; + deletedRowIds = deletedEntries.rowIds; deletePartitions = deletedEntries.partitions; // Build partition predicate from delete entries for overlap detection. @@ -546,18 +693,21 @@ private static ClassifyResult classifyManifests( file.partitionStats().nullCounts()); if (small || inDeleteRange) { iterator.remove(); - compactWithoutSort.put(file, inDeleteRange); + defaultCompactFiles.put(file, inDeleteRange); } } - return new ClassifyResult(lsmFiles, classifiedDeleteEntries, compactWithoutSort); + return new ManifestClassification( + lsmFiles, classifiedDeleteEntries, deletedRowIds, defaultCompactFiles); } private static DeletedEntryInfo readDeletedEntries( ManifestFile manifestFile, List manifestFiles, + boolean runMergeOptimizeEnabled, @Nullable Integer manifestReadParallelism) { CompactFileIdentifierSet identifiers = new CompactFileIdentifierSet(); + DeletedRowIdSet rowIds = new DeletedRowIdSet(); Set partitions = new HashSet<>(); List filesWithDeletes = new ArrayList<>(); for (ManifestFileMeta meta : manifestFiles) { @@ -569,12 +719,26 @@ private static DeletedEntryInfo readDeletedEntries( if (filesWithDeletes.size() <= 1 || (manifestReadParallelism != null && manifestReadParallelism <= 1)) { for (ManifestFileMeta meta : filesWithDeletes) { - collectDeletedEntries(meta, manifestFile, identifiers, partitions, false); + collectDeletedEntries( + meta, + manifestFile, + identifiers, + rowIds, + partitions, + runMergeOptimizeEnabled, + false); } } else { Function> reader = meta -> { - collectDeletedEntries(meta, manifestFile, identifiers, partitions, true); + collectDeletedEntries( + meta, + manifestFile, + identifiers, + rowIds, + partitions, + runMergeOptimizeEnabled, + true); return Collections.singletonList(Boolean.TRUE); }; for (Boolean ignored : @@ -582,14 +746,16 @@ private static DeletedEntryInfo readDeletedEntries( // Iteration waits for each bounded batch of parallel reads. } } - return new DeletedEntryInfo(identifiers, partitions); + return new DeletedEntryInfo(identifiers, rowIds, partitions); } private static void collectDeletedEntries( ManifestFileMeta meta, ManifestFile manifestFile, CompactFileIdentifierSet identifiers, + DeletedRowIdSet rowIds, Set partitions, + boolean runMergeOptimizeEnabled, boolean synchronize) { try (CloseableIterator entries = manifestFile.scan( @@ -603,10 +769,16 @@ private static void collectDeletedEntries( if (synchronize) { synchronized (identifiers) { identifiers.add(entry); + if (runMergeOptimizeEnabled) { + rowIds.add(entry.file().nonNullFirstRowId()); + } partitions.add(partition); } } else { identifiers.add(entry); + if (runMergeOptimizeEnabled) { + rowIds.add(entry.file().nonNullFirstRowId()); + } partitions.add(partition); } } @@ -691,7 +863,7 @@ static List buildLevelSortedRuns( /** * Split picked files into sections. Files with overlapping sort-key intervals go into the same - * section. Each section is built with pre-computed totalSize and hasUnsortedCompactMeta. + * section. Each section is built with pre-computed totalSize and hasDefaultCompactFile. */ static List

splitIntoSections( List pickedFiles, CompactionContext ctx) { @@ -712,7 +884,7 @@ static List
splitIntoSections( currentSectionFiles.add(first); currentSectionTotalSize += first.fileSize(); - boolean currentSectionHasUnsortedCompactMeta = ctx.isMarkedForUnsortedCompaction(first); + boolean currentSectionHasDefaultCompactFile = ctx.isMarkedForDefaultCompaction(first); ManifestFileMeta sectionMaxFile = first; for (int i = 1; i < pickedFiles.size(); i++) { @@ -724,20 +896,20 @@ static List
splitIntoSections( new Section( currentSectionFiles, currentSectionTotalSize, - currentSectionHasUnsortedCompactMeta)); + currentSectionHasDefaultCompactFile)); // start a new section currentSectionFiles = new ArrayList<>(); currentSectionTotalSize = 0; currentSectionFiles.add(file); currentSectionTotalSize += file.fileSize(); - currentSectionHasUnsortedCompactMeta = ctx.isMarkedForUnsortedCompaction(file); + currentSectionHasDefaultCompactFile = ctx.isMarkedForDefaultCompaction(file); sectionMaxFile = file; } else { currentSectionFiles.add(file); currentSectionTotalSize += file.fileSize(); - if (!currentSectionHasUnsortedCompactMeta - && ctx.isMarkedForUnsortedCompaction(file)) { - currentSectionHasUnsortedCompactMeta = true; + if (!currentSectionHasDefaultCompactFile + && ctx.isMarkedForDefaultCompaction(file)) { + currentSectionHasDefaultCompactFile = true; } if (sortKey.compareMax(file, sectionMaxFile) > 0) { sectionMaxFile = file; @@ -748,7 +920,7 @@ static List
splitIntoSections( new Section( currentSectionFiles, currentSectionTotalSize, - currentSectionHasUnsortedCompactMeta)); + currentSectionHasDefaultCompactFile)); return sections; } @@ -791,7 +963,7 @@ private static List
mergeSmallAdjacentSections( *
  • First overflow: The current section is split. The rewritable part is sorted and * rewritten. The remaining part is appended back to the sections queue for later * processing. - *
  • Subsequent overflows: If the section has files in compactWithoutSort (needs unsorted + *
  • Subsequent overflows: If the section has files in defaultCompactFiles (needs default * compaction), unsortedCompactSection is called to process it in smaller chunks. * Otherwise, the section is skipped. * @@ -902,9 +1074,9 @@ private static Section splitSectionAndRewriteHead( List tailFiles = new ArrayList<>(); long headSize = 0; long tailSize = 0; - // Whether tail section has files in compactWithoutSort, if true, the section need to + // Whether the tail section has files in defaultCompactFiles. If so, the section needs to // be rewritten. - boolean tailHasUnsortedCompactMeta = false; + boolean tailHasDefaultCompactFile = false; for (ManifestFileMeta file : section.files) { // Rewrite budget is enforced at manifest-file granularity. Include the first file that @@ -916,8 +1088,8 @@ private static Section splitSectionAndRewriteHead( } else { tailFiles.add(file); tailSize += file.fileSize(); - if (ctx.isMarkedForUnsortedCompaction(file)) { - tailHasUnsortedCompactMeta = true; + if (ctx.isMarkedForDefaultCompaction(file)) { + tailHasDefaultCompactFile = true; } } } @@ -927,7 +1099,7 @@ private static Section splitSectionAndRewriteHead( if (tailFiles.isEmpty()) { return null; } - return new Section(tailFiles, tailSize, tailHasUnsortedCompactMeta); + return new Section(tailFiles, tailSize, tailHasDefaultCompactFile); } /** @@ -945,7 +1117,7 @@ private static void rewriteSectionBeyondBudget( int suggestedMinMetaCount, @Nullable Integer manifestReadParallelism) throws Exception { - if (section.hasUnsortedCompactMeta) { + if (section.hasDefaultCompactFile) { unsortedCompactSection( section.files, output, @@ -965,8 +1137,8 @@ private static void rewriteSectionBeyondBudget( * *

    Semantics difference from old minor merge: In the old ManifestFileMerger path, the * trailing candidates are kept unchanged when their count is below manifest.merge-min-count. In - * this sort path, unsortedCompactSection is triggered when compactWithoutSort is non-empty, - * regardless of the manifest count. This is because files in compactWithoutSort either: + * this sort path, unsortedCompactSection is triggered when defaultCompactFiles is non-empty, + * regardless of the manifest count. This is because files in defaultCompactFiles either: * *

      *
    • Are small files needing consolidation @@ -1036,7 +1208,7 @@ private static void rewriteSection( @Nullable Integer manifestReadParallelism) throws Exception { // Skip rewrite for single file not in delete-range. - if (section.size() == 1 && !ctx.compactWithoutSort.getOrDefault(section.get(0), false)) { + if (section.size() == 1 && !ctx.defaultCompactFiles.getOrDefault(section.get(0), false)) { output.addUnchanged(section.get(0)); return; } @@ -1062,24 +1234,38 @@ private static void rewriteFull( ManifestFile manifestFile, @Nullable Integer manifestReadParallelism) throws Exception { - List sorted = - ManifestEntryExternalSort.sortAndWriteFullEntries( - section, - ctx.sortKey, - ctx.externalSortConfig, - manifestFile, - sortNewFiles, - ctx.deleteEntries, - manifestReadParallelism); + List sorted = null; + if (ctx.runMergeOptimizeEnabled) { + sorted = + ManifestEntryRunMerge.sortAndWriteFullEntries( + section, + (RowIdEntrySortKey) ctx.sortKey, + manifestFile, + sortNewFiles, + ctx.deleteEntries, + ctx.deletedRowIds, + manifestReadParallelism); + } + if (sorted == null) { + sorted = + ManifestEntryExternalSort.sortAndWriteFullEntries( + section, + ctx.sortKey, + ctx.externalSortConfig, + manifestFile, + sortNewFiles, + ctx.deleteEntries, + manifestReadParallelism); + } if (!sorted.isEmpty()) { output.addSortedFiles(sorted); } } /** - * Minor compaction path: collect DELETE entries in memory while external-sorting all entries, - * then write surviving ADD entries from the sorted stream and remaining DELETE entries from - * memory. + * Minor compaction path: collect DELETE identities, merge the existing sorted runs, and write + * surviving ADD entries and unmatched DELETE entries separately. Falls back to external sort + * when the input is not suitable for run merge. */ private static void rewriteMinor( List section, @@ -1089,14 +1275,26 @@ private static void rewriteMinor( ManifestFile manifestFile, @Nullable Integer manifestReadParallelism) throws Exception { - Pair, List> sorted = - ManifestEntryExternalSort.sortAndWriteMinorEntries( - section, - ctx.sortKey, - ctx.externalSortConfig, - manifestFile, - sortNewFiles, - manifestReadParallelism); + Pair, List> sorted = null; + if (ctx.runMergeOptimizeEnabled) { + sorted = + ManifestEntryRunMerge.sortAndWriteMinorEntries( + section, + (RowIdEntrySortKey) ctx.sortKey, + manifestFile, + sortNewFiles, + manifestReadParallelism); + } + if (sorted == null) { + sorted = + ManifestEntryExternalSort.sortAndWriteMinorEntries( + section, + ctx.sortKey, + ctx.externalSortConfig, + manifestFile, + sortNewFiles, + manifestReadParallelism); + } if (!sorted.getLeft().isEmpty()) { output.addSortedFiles(sorted.getLeft()); @@ -1117,11 +1315,8 @@ private static boolean containsNoDeleteEntries(List section) { } private static ManifestSortKey createSortKey( - boolean dataEvolutionEnabled, - List input, - String sortPartitionField, - RowType partitionType) { - if (dataEvolutionEnabled && ManifestFileMeta.allContainsRowId(input)) { + boolean rowIdSort, String sortPartitionField, RowType partitionType) { + if (rowIdSort) { // RowID sorting uses the configured partition field as the primary key when specified, // otherwise it uses the full partition row to preserve partition locality. It then // orders files by RowID. @@ -1200,6 +1395,11 @@ void replaceExternalSortRow( InternalRow binaryManifestRow(BinaryRow row); } + interface RowIdEntrySortKey extends ManifestSortKey { + + int comparePartitions(BinaryRow left, BinaryRow right); + } + private static class PartitionSortKey implements ManifestSortKey { private final RecordComparator fieldComparator; @@ -1271,7 +1471,7 @@ public InternalRow binaryManifestRow(BinaryRow row) { } } - private static class RowIdSortKey implements ManifestSortKey { + private static class RowIdSortKey implements RowIdEntrySortKey { @Nullable private final RecordComparator partitionComparator; private final InternalRow.FieldGetter[] partitionFieldGetters; @@ -1286,21 +1486,8 @@ private RowIdSortKey( this.partitionComparator = partitionComparator; this.partitionFieldGetters = createPartitionFieldGetters(partitionType, partitionSortFields); - - List fieldTypes = new ArrayList<>(); - for (int partitionSortField : partitionSortFields) { - fieldTypes.add(partitionType.getTypeAt(partitionSortField)); - } - // ADD must precede DELETE for the same partition. Minor compaction streams the sorted - // rows once and uses this ordering to eliminate a matching pair without retaining all - // ADD identifiers. - fieldTypes.add(DataTypes.TINYINT()); - fieldTypes.add(DataTypes.BIGINT()); - fieldTypes.add(DataTypes.BIGINT()); - fieldTypes.add(DataTypes.BIGINT()); - fieldTypes.add(DataTypes.STRING()); - fieldTypes.add(ManifestEntry.MANIFEST_ROW_TYPE); - this.externalSortRowType = DataTypes.ROW(fieldTypes.toArray(new DataType[0])); + this.externalSortRowType = + createRowIdExternalSortRowType(partitionType, partitionSortFields); this.sortFieldNum = externalSortRowType.getFieldCount() - 1; this.externalSortKeyFields = createSequentialFields(sortFieldNum); } @@ -1334,7 +1521,7 @@ public boolean isAfterMax(ManifestFileMeta file, ManifestFileMeta maxFile) { return c > 0; } } - return Long.compare(nonNullMinRowId(file), nonNullMaxRowId(maxFile)) > 0; + return nonNullMinRowId(file) > nonNullMaxRowId(maxFile); } @Override @@ -1371,6 +1558,11 @@ public InternalRow binaryManifestRow(BinaryRow row) { return row.getRow(sortFieldNum, ManifestEntry.MANIFEST_ROW_TYPE.getFieldCount()); } + @Override + public int comparePartitions(BinaryRow left, BinaryRow right) { + return partitionComparator == null ? 0 : partitionComparator.compare(left, right); + } + private int comparePartitionMin(ManifestFileMeta a, ManifestFileMeta b) { if (partitionComparator == null) { return 0; @@ -1410,6 +1602,26 @@ private static long rowIdRangeEnd(ManifestEntry entry) { } } + private static RowType createRowIdExternalSortRowType( + RowType partitionType, int[] partitionSortFields) { + List fieldTypes = new ArrayList<>(partitionSortFields.length + 6); + for (int partitionSortField : partitionSortFields) { + fieldTypes.add(partitionType.getTypeAt(partitionSortField)); + } + // ADD must precede DELETE for the same partition. Minor compaction streams the sorted rows + // once and uses this ordering to eliminate a matching pair without retaining all ADD + // identifiers. + Collections.addAll( + fieldTypes, + DataTypes.TINYINT(), + DataTypes.BIGINT(), + DataTypes.BIGINT(), + DataTypes.BIGINT(), + DataTypes.STRING(), + ManifestEntry.MANIFEST_ROW_TYPE); + return DataTypes.ROW(fieldTypes.toArray(new DataType[0])); + } + private static int[] createSequentialFields(int fieldCount) { int[] fields = new int[fieldCount]; for (int i = 0; i < fieldCount; i++) { @@ -1528,12 +1740,12 @@ public void addDeleteFiles(List files) { static class Section { final List files; final long totalSize; - final boolean hasUnsortedCompactMeta; + final boolean hasDefaultCompactFile; - Section(List files, long totalSize, boolean hasUnsortedCompactMeta) { + Section(List files, long totalSize, boolean hasDefaultCompactFile) { this.files = files; this.totalSize = totalSize; - this.hasUnsortedCompactMeta = hasUnsortedCompactMeta; + this.hasDefaultCompactFile = hasDefaultCompactFile; } /** Create a merged section from two sections. */ @@ -1543,7 +1755,7 @@ static Section merge(Section a, Section b) { return new Section( merged, a.totalSize + b.totalSize, - a.hasUnsortedCompactMeta || b.hasUnsortedCompactMeta); + a.hasDefaultCompactFile || b.hasDefaultCompactFile); } } } diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java index 7dff697e8fc6..9fa1edb7615e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java @@ -59,10 +59,12 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.TreeSet; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; @@ -70,6 +72,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.IntStream; +import java.util.stream.LongStream; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -883,6 +886,28 @@ private void beforeFirstRead() throws IOException { } } + private static class CountingReadFileIO extends LocalFileIO { + + private final Map readCounts = new ConcurrentHashMap<>(); + + @Override + public SeekableInputStream newInputStream(Path path) throws IOException { + readCounts + .computeIfAbsent(path.getName(), ignored -> new AtomicInteger()) + .incrementAndGet(); + return super.newInputStream(path); + } + + private int readCount(String fileName) { + AtomicInteger count = readCounts.get(fileName); + return count == null ? 0 : count.get(); + } + + private void resetReadCounts() { + readCounts.clear(); + } + } + // ==================== Manifest Sort Tests ==================== /** @@ -1263,6 +1288,386 @@ public void testDataEvolutionManifestSortByPartitionAndRowId() { } } + @Test + public void testDisablingRunMergeOptimizePreservesDataEvolutionRowIdSort() { + assertThat( + CoreOptions.fromMap(Collections.emptyMap()) + .manifestSortRunMergeOptimizeEnabled()) + .isTrue(); + + List input = + Arrays.asList( + makeManifest( + makeRowIdEntry(true, "row-30", 0, 30, 5), + makeRowIdEntry(true, "row-10", 0, 10, 5)), + makeManifest( + makeRowIdEntry(true, "row-20", 0, 20, 5), + makeRowIdEntry(true, "row-0", 0, 0, 5))); + + Options testOptions = new Options(); + testOptions.set("manifest-sort.enabled", "true"); + testOptions.set("manifest-sort.run-merge-optimize.enabled", "false"); + testOptions.set("data-evolution.enabled", "true"); + testOptions.set("manifest.full-compaction-threshold-size", "1B"); + CoreOptions coreOptions = CoreOptions.fromMap(testOptions.toMap()); + + assertThat(coreOptions.manifestSortRunMergeOptimizeEnabled()).isFalse(); + + List merged = + ManifestFileMerger.merge(input, manifestFile, getPartitionType(), coreOptions); + + assertEquivalentEntries(input, merged); + assertThat(readEntries(merged).stream().map(entry -> entry.file().fileName())) + .containsExactly("row-0", "row-10", "row-20", "row-30"); + } + + @Test + public void testDataEvolutionManifestRunMergeSecondaryKeys() { + List firstManifest = new ArrayList<>(); + firstManifest.add(makeRowIdEntry(true, "range-short", 0, 100, 5, 1)); + firstManifest.add(makeRowIdEntry(true, "sequence-newer", 0, 100, 10, 9)); + for (int i = 19; i >= 10; i--) { + firstManifest.add(makeRowIdEntry(true, String.format("tie-%02d", i), 0, 100, 10, 5)); + } + + List secondManifest = new ArrayList<>(); + for (int i = 9; i >= 0; i--) { + secondManifest.add(makeRowIdEntry(true, String.format("tie-%02d", i), 0, 100, 10, 5)); + } + + List input = new ArrayList<>(); + input.add(makeManifest(firstManifest.toArray(new ManifestEntry[0]))); + input.add(makeManifest(secondManifest.toArray(new ManifestEntry[0]))); + + Options testOptions = new Options(); + testOptions.set("manifest-sort.enabled", "true"); + testOptions.set("data-evolution.enabled", "true"); + testOptions.set("manifest.full-compaction-threshold-size", "1B"); + testOptions.set("scan.manifest.parallelism", "2"); + + List merged = + ManifestFileMerger.merge( + input, + manifestFile, + getPartitionType(), + CoreOptions.fromMap(testOptions.toMap())); + + List expected = new ArrayList<>(); + expected.add("range-short"); + expected.add("sequence-newer"); + for (int i = 0; i < 20; i++) { + expected.add(String.format("tie-%02d", i)); + } + assertThat(readEntries(merged).stream().map(e -> e.file().fileName())) + .containsExactlyElementsOf(expected); + } + + @Test + public void testDataEvolutionManifestRunMergeUsesExactDeleteIdentifier() { + List input = + Arrays.asList( + makeManifest( + makeRowIdEntry(true, "deleted", 0, 100, 5), + makeRowIdEntry(true, "same-row-id-survivor", 0, 100, 5)), + makeManifest(makeRowIdEntry(false, "deleted", 0, 100, 5))); + + Options testOptions = new Options(); + testOptions.set("manifest-sort.enabled", "true"); + testOptions.set("data-evolution.enabled", "true"); + testOptions.set("manifest.full-compaction-threshold-size", "1B"); + testOptions.set("scan.manifest.parallelism", "2"); + + List merged = + ManifestFileMerger.merge( + input, + manifestFile, + getPartitionType(), + CoreOptions.fromMap(testOptions.toMap())); + + assertThat(readEntries(merged).stream().map(entry -> entry.file().fileName())) + .containsExactly("same-row-id-survivor"); + } + + @Test + public void testDataEvolutionManifestRunMergeUsesRawDeleteIdentityFields() { + ManifestEntry deleted = + makeRowIdEntry( + true, + "same-file-name", + 0, + 100, + 5, + 0, + Arrays.asList("extra-a", "extra-b"), + new byte[] {1, 2}, + "external-a"); + ManifestEntry survivor = + makeRowIdEntry( + true, + "same-file-name", + 0, + 100, + 5, + 0, + Collections.singletonList("other-extra"), + new byte[] {3, 4}, + "external-b"); + ManifestEntry delete = + makeRowIdEntry( + false, + "same-file-name", + 0, + 100, + 5, + 0, + Arrays.asList("extra-a", "extra-b"), + new byte[] {1, 2}, + "external-a"); + List input = + Arrays.asList(makeManifest(deleted, survivor), makeManifest(delete)); + + Options testOptions = new Options(); + testOptions.set("manifest-sort.enabled", "true"); + testOptions.set("data-evolution.enabled", "true"); + testOptions.set("manifest.full-compaction-threshold-size", "1B"); + + List merged = + ManifestFileMerger.merge( + input, + manifestFile, + getPartitionType(), + CoreOptions.fromMap(testOptions.toMap())); + + assertThat(readEntries(merged)).singleElement().isEqualTo(survivor); + } + + @Test + public void testDataEvolutionManifestRunMergeManyPartitions() { + List firstManifest = new ArrayList<>(); + List secondManifest = new ArrayList<>(); + for (int partition = 39; partition >= 0; partition--) { + ManifestEntry entry = + makeRowIdEntry( + true, + String.format("partition-%02d", partition), + partition, + partition * 10L, + 5); + (partition >= 20 ? firstManifest : secondManifest).add(entry); + } + + List input = + Arrays.asList( + makeManifest(firstManifest.toArray(new ManifestEntry[0])), + makeManifest(secondManifest.toArray(new ManifestEntry[0]))); + Options testOptions = new Options(); + testOptions.set("manifest-sort.enabled", "true"); + testOptions.set("data-evolution.enabled", "true"); + testOptions.set("manifest.full-compaction-threshold-size", "1B"); + + List merged = + ManifestFileMerger.merge( + input, + manifestFile, + getPartitionType(), + CoreOptions.fromMap(testOptions.toMap())); + + assertThat(readEntries(merged).stream().map(e -> e.partition().getInt(0))) + .containsExactlyElementsOf( + IntStream.range(0, 40).boxed().collect(Collectors.toList())); + } + + @Test + public void testDataEvolutionManifestRunMergeFragmentedSmallManifests() { + List firstManifest = new ArrayList<>(); + List secondManifest = new ArrayList<>(); + for (long firstRowId = 199; firstRowId >= 0; firstRowId--) { + ManifestEntry entry = + makeRowIdEntry(true, String.format("row-%03d", firstRowId), 0, firstRowId, 1); + (firstRowId >= 100 ? firstManifest : secondManifest).add(entry); + } + + List input = + Arrays.asList( + makeManifest(firstManifest.toArray(new ManifestEntry[0])), + makeManifest(secondManifest.toArray(new ManifestEntry[0]))); + Options testOptions = new Options(); + testOptions.set("manifest-sort.enabled", "true"); + testOptions.set("data-evolution.enabled", "true"); + testOptions.set("manifest.full-compaction-threshold-size", "1B"); + + List merged = + ManifestFileMerger.merge( + input, + manifestFile, + getPartitionType(), + CoreOptions.fromMap(testOptions.toMap())); + + assertThat(readEntries(merged).stream().map(entry -> entry.file().nonNullFirstRowId())) + .containsExactlyElementsOf( + LongStream.range(0, 200).boxed().collect(Collectors.toList())); + } + + @Test + public void testDataEvolutionManifestRunMergeFallsBackForLargeFragmentedManifest() { + List firstManifest = new ArrayList<>(); + List secondManifest = new ArrayList<>(); + for (long firstRowId = 25_000; firstRowId >= 12_500; firstRowId--) { + firstManifest.add( + makeRowIdEntry(true, String.format("row-%05d", firstRowId), 0, firstRowId, 1)); + } + for (long firstRowId = 12_499; firstRowId >= 0; firstRowId--) { + secondManifest.add( + makeRowIdEntry(true, String.format("row-%05d", firstRowId), 0, firstRowId, 1)); + } + + List input = + Arrays.asList( + makeManifest(firstManifest.toArray(new ManifestEntry[0])), + makeManifest(secondManifest.toArray(new ManifestEntry[0]))); + Options testOptions = new Options(); + testOptions.set("manifest-sort.enabled", "true"); + testOptions.set("data-evolution.enabled", "true"); + testOptions.set("manifest.full-compaction-threshold-size", "1B"); + + List merged = + ManifestFileMerger.merge( + input, + manifestFile, + getPartitionType(), + CoreOptions.fromMap(testOptions.toMap())); + + assertThat(readEntries(merged).stream().map(entry -> entry.file().nonNullFirstRowId())) + .containsExactlyElementsOf( + LongStream.rangeClosed(0, 25_000).boxed().collect(Collectors.toList())); + } + + @Test + public void testDataEvolutionMinorRunMergeFallsBackForLargeFragmentedManifest() { + List fragmentedEntries = new ArrayList<>(); + for (long firstRowId = 25_000; firstRowId >= 0; firstRowId--) { + fragmentedEntries.add( + makeRowIdEntry(true, String.format("row-%05d", firstRowId), 0, firstRowId, 1)); + } + + List input = + Arrays.asList( + makeManifest(fragmentedEntries.toArray(new ManifestEntry[0])), + makeManifest( + makeRowIdEntry(false, "row-12500", 0, 12_500, 1), + makeRowIdEntry(true, "row-30000", 0, 30_000, 1))); + + List expected = + LongStream.rangeClosed(0, 25_000).boxed().collect(Collectors.toList()); + expected.remove(Long.valueOf(12_500L)); + expected.add(30_000L); + + List externalResult = readEntries(mergeMinorManifestEntries(input, false)); + List runMergeResult = readEntries(mergeMinorManifestEntries(input, true)); + assertThat(runMergeResult).containsExactlyElementsOf(externalResult); + assertThat(runMergeResult.stream().map(entry -> entry.file().nonNullFirstRowId())) + .containsExactlyElementsOf(expected); + } + + @Test + public void testDataEvolutionManifestRunMergeLimitsReadAmplification() { + CountingReadFileIO fileIO = new CountingReadFileIO(); + manifestFile = createManifestFile(tempDir.toString(), fileIO); + + int runCount = 20; + int entriesPerRun = 600; + List fragmentedEntries = new ArrayList<>(); + for (int run = runCount - 1; run >= 0; run--) { + long runStart = (long) run * entriesPerRun; + for (int entry = 0; entry < entriesPerRun; entry++) { + long firstRowId = runStart + entry; + fragmentedEntries.add( + makeRowIdEntry( + true, + String.format("fragmented-%05d", firstRowId), + 0, + firstRowId, + 1)); + } + } + + ManifestFileMeta fragmented = makeManifest(fragmentedEntries.toArray(new ManifestEntry[0])); + ManifestFileMeta overlap = + makeManifest(makeRowIdEntry(true, "overlap", 0, entriesPerRun / 2L, 1)); + fileIO.resetReadCounts(); + + Options testOptions = new Options(); + testOptions.set("manifest-sort.enabled", "true"); + testOptions.set("data-evolution.enabled", "true"); + testOptions.set("manifest.full-compaction-threshold-size", "1B"); + + List merged = + ManifestFileMerger.merge( + Arrays.asList(fragmented, overlap), + manifestFile, + getPartitionType(), + CoreOptions.fromMap(testOptions.toMap())); + + assertThat(fileIO.readCount(fragmented.fileName())).isEqualTo(2); + List expectedRowIds = + LongStream.range(0, (long) runCount * entriesPerRun) + .boxed() + .collect(Collectors.toList()); + expectedRowIds.add(entriesPerRun / 2L); + Collections.sort(expectedRowIds); + assertThat(readEntries(merged).stream().map(entry -> entry.file().nonNullFirstRowId())) + .containsExactlyElementsOf(expectedRowIds); + } + + @Test + public void testDataEvolutionManifestRunMergePreservesBlockStats() { + List spanningManifest = new ArrayList<>(); + List middleManifest = new ArrayList<>(); + for (long firstRowId = 0; firstRowId < 5_000; firstRowId++) { + spanningManifest.add( + makeRowIdEntry( + true, String.format("row-%05d", firstRowId), null, firstRowId, 1)); + } + for (long firstRowId = 10_000; firstRowId < 15_000; firstRowId++) { + spanningManifest.add( + makeRowIdEntry(true, String.format("row-%05d", firstRowId), 7, firstRowId, 1)); + } + for (long firstRowId = 5_000; firstRowId < 10_000; firstRowId++) { + middleManifest.add( + makeRowIdEntry( + true, String.format("row-%05d", firstRowId), null, firstRowId, 1)); + } + + List input = + Arrays.asList( + makeManifest(spanningManifest.toArray(new ManifestEntry[0])), + makeManifest(middleManifest.toArray(new ManifestEntry[0]))); + Options testOptions = new Options(); + testOptions.set("manifest-sort.enabled", "true"); + testOptions.set("data-evolution.enabled", "true"); + testOptions.set("manifest.full-compaction-threshold-size", "1B"); + + List merged = + ManifestFileMerger.merge( + input, + manifestFile, + getPartitionType(), + CoreOptions.fromMap(testOptions.toMap())); + + assertThat(merged).hasSize(1); + ManifestFileMeta output = merged.get(0); + assertThat(output.numAddedFiles()).isEqualTo(15_000); + assertThat(output.numDeletedFiles()).isZero(); + assertThat(output.minRowId()).isZero(); + assertThat(output.maxRowId()).isEqualTo(14_999); + assertThat(output.partitionStats().minValues().getInt(0)).isEqualTo(7); + assertThat(output.partitionStats().maxValues().getInt(0)).isEqualTo(7); + assertThat(output.partitionStats().nullCounts().getLong(0)).isEqualTo(10_000); + assertThat(readEntries(merged).stream().map(entry -> entry.file().nonNullFirstRowId())) + .containsExactlyElementsOf( + LongStream.range(0, 15_000).boxed().collect(Collectors.toList())); + } + @Test public void testDataEvolutionManifestSortUsesConfiguredPartitionFieldBeforeRowId() { RowType multiPartitionType = RowType.of(new IntType(), new IntType(), new IntType()); @@ -1394,6 +1799,101 @@ public void testDataEvolutionMinorManifestSortPreservesUnmatchedDeleteEntries() .containsExactly("ADD-new-row20", "ADD-survivor-row30", "DELETE-old-row10"); } + @Test + public void testDataEvolutionMinorRunMergeMatchesExternalSort() { + ManifestEntry deleted = + makeRowIdEntry( + true, + "same-file-name", + 0, + 100, + 5, + 0, + Arrays.asList("extra-a", "extra-b"), + new byte[] {1, 2}, + "external-a"); + ManifestEntry sameRowIdSurvivor = + makeRowIdEntry( + true, + "same-file-name", + 0, + 100, + 5, + 0, + Collections.singletonList("other-extra"), + new byte[] {3, 4}, + "external-b"); + ManifestEntry delete = + makeRowIdEntry( + false, + "same-file-name", + 0, + 100, + 5, + 0, + Arrays.asList("extra-a", "extra-b"), + new byte[] {1, 2}, + "external-a"); + ManifestEntry unmatchedDelete = makeRowIdEntry(false, "old-row-200", 0, 200, 5); + List input = + Arrays.asList( + makeManifest( + deleted, + sameRowIdSurvivor, + makeRowIdEntry(true, "survivor-row-300", 0, 300, 5)), + makeManifest(delete, unmatchedDelete, unmatchedDelete)); + + List externalResult = readEntries(mergeMinorManifestEntries(input, false)); + List runMergeResult = readEntries(mergeMinorManifestEntries(input, true)); + + assertThat(runMergeResult).containsExactlyElementsOf(externalResult); + assertThat( + runMergeResult.stream() + .map(entry -> entry.kind() + "-" + entry.file().fileName()) + .collect(Collectors.toList())) + .containsExactly( + "ADD-same-file-name", "ADD-survivor-row-300", "DELETE-old-row-200"); + assertThat(runMergeResult.get(0)).isEqualTo(sameRowIdSurvivor); + } + + @Test + public void testDataEvolutionMinorRunMergeCollectsDeletesDuringDiscovery() { + CountingReadFileIO fileIO = new CountingReadFileIO(); + manifestFile = createManifestFile(tempDir.toString(), fileIO); + ManifestFileMeta base = + makeManifest( + makeRowIdEntry(true, "deleted-row-10", 0, 10, 5), + makeRowIdEntry(true, "survivor-row-20", 0, 20, 5)); + ManifestFileMeta delta = + makeManifest( + makeRowIdEntry(true, "survivor-row-30", 0, 30, 5), + makeRowIdEntry(false, "deleted-row-10", 0, 10, 5)); + fileIO.resetReadCounts(); + + List merged = mergeMinorManifestEntries(Arrays.asList(base, delta), true); + + assertThat(fileIO.readCount(base.fileName())).isEqualTo(2); + assertThat(fileIO.readCount(delta.fileName())).isEqualTo(2); + assertThat( + readEntries(merged).stream() + .map(entry -> entry.file().fileName()) + .collect(Collectors.toList())) + .containsExactly("survivor-row-20", "survivor-row-30"); + } + + private List mergeMinorManifestEntries( + List input, boolean runMergeOptimizeEnabled) { + Options options = new Options(); + options.set("manifest-sort.enabled", "true"); + options.set( + "manifest-sort.run-merge-optimize.enabled", + Boolean.toString(runMergeOptimizeEnabled)); + options.set("data-evolution.enabled", "true"); + options.set("manifest.full-compaction-threshold-size", Long.MAX_VALUE + "B"); + return ManifestFileMerger.merge( + input, manifestFile, getPartitionType(), CoreOptions.fromMap(options.toMap())); + } + /** * Test manifest sort with a multi-field partition type. * @@ -1909,20 +2409,46 @@ private List readFileNames( /** Create a ManifestEntry with row ID metadata for data evolution manifest sort tests. */ private ManifestEntry makeRowIdEntry( - boolean isAdd, String fileName, int partition, long firstRowId, long rowCount) { + boolean isAdd, String fileName, Integer partition, long firstRowId, long rowCount) { return makeRowIdEntry(isAdd, fileName, partition, firstRowId, rowCount, 0); } private ManifestEntry makeRowIdEntry( boolean isAdd, String fileName, - int partition, + Integer partition, long firstRowId, long rowCount, long sequenceNumber) { + return makeRowIdEntry( + isAdd, + fileName, + partition, + firstRowId, + rowCount, + sequenceNumber, + Collections.emptyList(), + null, + null); + } + + private ManifestEntry makeRowIdEntry( + boolean isAdd, + String fileName, + Integer partition, + long firstRowId, + long rowCount, + long sequenceNumber, + List extraFiles, + byte[] embeddedIndex, + String externalPath) { BinaryRow binaryRow = new BinaryRow(1); BinaryRowWriter writer = new BinaryRowWriter(binaryRow); - writer.writeInt(0, partition); + if (partition == null) { + writer.setNullAt(0); + } else { + writer.writeInt(0, partition); + } writer.complete(); return ManifestEntry.create( @@ -1942,13 +2468,13 @@ private ManifestEntry makeRowIdEntry( sequenceNumber, 0, 0, - Collections.emptyList(), + extraFiles, Timestamp.fromEpochMillis(200000), 0L, - null, + embeddedIndex, FileSource.APPEND, null, - null, + externalPath, firstRowId, Collections.singletonList("f0"))); } From cfc8358783b6baea17abefcac4fb89661c7ac5c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Thu, 13 Aug 2026 21:00:09 +0800 Subject: [PATCH 2/3] [core] Optimize manifest file merging --- docs/generated/core_configuration.html | 12 +- .../java/org/apache/paimon/CoreOptions.java | 15 +- .../paimon/manifest/ManifestAvroReader.java | 11 + .../paimon/manifest/ManifestAvroWriter.java | 208 ++- .../apache/paimon/manifest/ManifestFile.java | 4 +- .../operation/ManifestEntryRunMerge.java | 188 ++- .../operation/ManifestEntryRunMergeEntry.java | 58 +- .../operation/ManifestEntryRunMergePlan.java | 65 +- .../operation/ManifestFileBlockMerger.java | 1177 +++++++++++++++++ .../operation/ManifestFileLegacyMerger.java | 290 ++++ .../paimon/operation/ManifestFileMerger.java | 271 +--- .../paimon/operation/ManifestFileSorter.java | 13 +- .../paimon/manifest/ManifestFileMetaTest.java | 384 ++++-- .../paimon/manifest/ManifestFileTest.java | 89 +- .../ManifestFileMergerTestUtils.java | 53 + .../java/org/apache/avro/file/RawBlock.java | 28 +- .../paimon/format/avro/AvroBlockReader.java | 48 +- .../paimon/format/avro/AvroBlockWriter.java | 12 +- .../paimon/format/avro/AvroFileFormat.java | 2 +- .../paimon/format/avro/AvroRawBlock.java | 10 +- .../format/avro/AvroFileFormatTest.java | 41 + 21 files changed, 2441 insertions(+), 538 deletions(-) create mode 100644 paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileBlockMerger.java create mode 100644 paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileLegacyMerger.java create mode 100644 paimon-core/src/test/java/org/apache/paimon/operation/ManifestFileMergerTestUtils.java diff --git a/docs/generated/core_configuration.html b/docs/generated/core_configuration.html index 033e2194f53b..96ae7a1e1359 100644 --- a/docs/generated/core_configuration.html +++ b/docs/generated/core_configuration.html @@ -1023,12 +1023,6 @@ String Partition field name to sort manifest entries by. Validated by schema validation, if not configured, defaults to the first partition field. - -
      manifest-sort.run-merge-optimize.enabled
      - true - Boolean - Whether to use streaming run merge for RowID-based manifest sorting. When disabled, the external sorter is used without changing the RowID sort semantics. -
      manifest.compression
      "zstd" @@ -1053,6 +1047,12 @@ Integer To avoid frequent manifest merges, this parameter specifies the minimum number of ManifestFileMeta to merge.
      Note: when 'manifest-sort.enabled' is true, this minimum-count gate is only applied to the trailing sub-segment of a section that exceeds 'manifest-sort.max-rewrite-size'. Small under-budget sections are sorted and rewritten directly, so two small manifest files may be merged into one even when their count is below this threshold and full compaction is not triggered. + +
      manifest.merge-optimize.enabled
      + true + Boolean + Whether to enable optimized manifest compaction. When enabled, block-aware compaction and streaming run merge are used. When disabled, ordinary compaction uses the legacy merger and RowID-based sorting uses the external sorter. +
      manifest.target-file-size
      8 mb diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index b4e34c574b24..e4f720edd99d 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -589,14 +589,15 @@ public InlineElement getDescription() { + " skipped. Set to a larger value to allow more aggressive" + " sort rewriting. The cap only limits the sorted rewrite portion and full/minor cleanup may still happen beyond it."); - public static final ConfigOption MANIFEST_SORT_RUN_MERGE_OPTIMIZE_ENABLED = - key("manifest-sort.run-merge-optimize.enabled") + public static final ConfigOption MANIFEST_MERGE_OPTIMIZE_ENABLED = + key("manifest.merge-optimize.enabled") .booleanType() .defaultValue(true) .withDescription( - "Whether to use streaming run merge for RowID-based manifest sorting." - + " When disabled, the external sorter is used without changing" - + " the RowID sort semantics."); + "Whether to enable optimized manifest compaction. When enabled," + + " block-aware compaction and streaming run merge are used." + + " When disabled, ordinary compaction uses the legacy merger" + + " and RowID-based sorting uses the external sorter."); public static final ConfigOption PARTITION_DEFAULT_NAME = key("partition.default-name") @@ -3075,8 +3076,8 @@ public long manifestSortMaxRewriteSize() { return options.get(MANIFEST_SORT_MAX_REWRITE_SIZE).getBytes(); } - public boolean manifestSortRunMergeOptimizeEnabled() { - return options.get(MANIFEST_SORT_RUN_MERGE_OPTIMIZE_ENABLED); + public boolean manifestMergeOptimizeEnabled() { + return options.get(MANIFEST_MERGE_OPTIMIZE_ENABLED); } public String partitionDefaultName() { diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroReader.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroReader.java index 3fbb0008714e..54d52434f824 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroReader.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroReader.java @@ -74,6 +74,11 @@ public boolean hasNext() throws IOException { return blockReader.hasNextBlock(); } + /** Returns whether raw blocks can be copied into a writer for the current manifest schema. */ + public boolean rawBlockCopySupported() { + return rawBlockCopySupported; + } + /** Returns the next raw block without decompressing it. */ public RawBlock next() throws IOException { if (!hasNext()) { @@ -372,6 +377,12 @@ public boolean rawBlockCopySupported() { public AvroRawBlock encodedBlock() { return block; } + + /** Returns an independently owned block which remains valid after this reader advances. */ + public RawBlock stableCopy() { + return new RawBlock( + decoderContext, rawBlockCopySupported, block.stableCopy(), blockOrdinal); + } } /** Decoder state shared by the borrowed blocks produced by one reader. */ diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroWriter.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroWriter.java index 6244ed569c9e..39314c566690 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroWriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroWriter.java @@ -19,6 +19,7 @@ package org.apache.paimon.manifest; import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.InternalRow; import org.apache.paimon.format.SimpleColStats; import org.apache.paimon.format.SimpleStatsCollector; import org.apache.paimon.format.avro.AvroBlockWriter; @@ -29,6 +30,7 @@ import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.io.RollingFileWriter; import org.apache.paimon.schema.SchemaManager; +import org.apache.paimon.stats.SimpleStats; import org.apache.paimon.stats.SimpleStatsConverter; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.IOUtils; @@ -54,6 +56,8 @@ */ public final class ManifestAvroWriter implements AutoCloseable { + private static final int MAX_BUFFERED_ENCODED_PARTITIONS = 8_192; + private final FileIO fileIO; private final SchemaManager schemaManager; private final RowType partitionType; @@ -114,6 +118,17 @@ public void writeEncoded(ByteBuffer encodedRecord, EncodedEntry metadata) throws } } + /** Writes an already decoded manifest row while collecting its projected metadata. */ + public void writeRow(InternalRow row, EncodedEntry metadata) throws IOException { + try { + currentWriter().writeRow(row, metadata); + afterWrite(1, false); + } catch (IOException | RuntimeException | Error failure) { + abort(); + throw failure; + } + } + public void writeEncodedBlock(AvroRawBlock block, EncodedBlock metadata) throws IOException { if (metadata.addedFiles < 0 || metadata.deletedFiles < 0) { throw new IllegalArgumentException( @@ -137,6 +152,38 @@ public void writeEncodedBlock(AvroRawBlock block, EncodedBlock metadata) throws } } + /** Copies all raw blocks from one manifest and reuses its aggregate metadata. */ + public void writeEncodedManifest(ManifestAvroReader reader, ManifestFileMeta metadata) + throws IOException { + if (!reader.rawBlockCopySupported()) { + throw new IllegalArgumentException( + "Manifest schema is incompatible with raw block copying."); + } + try { + FileWriter fileWriter = currentWriter(); + long copiedRecords = 0; + while (reader.hasNext()) { + ManifestAvroReader.RawBlock block = reader.next(); + fileWriter.ensureOpen(); + fileWriter.writer.addEncodedBlock(block.encodedBlock()); + copiedRecords = Math.addExact(copiedRecords, block.recordCount()); + } + long metadataRecords = + Math.addExact(metadata.numAddedFiles(), metadata.numDeletedFiles()); + if (copiedRecords != metadataRecords) { + throw new IllegalArgumentException( + String.format( + "Manifest record count mismatch: metadata %s, blocks %s.", + metadataRecords, copiedRecords)); + } + fileWriter.collectStats(metadata); + afterWrite(copiedRecords, true); + } catch (IOException | RuntimeException | Error failure) { + abort(); + throw failure; + } + } + private FileWriter currentWriter() { if (closed) { throw new IllegalStateException("Manifest writer has already closed."); @@ -213,6 +260,7 @@ public static final class EncodedEntry { private int bucket; private int level; private long schemaId; + private boolean hasRowId; private long firstRowId; private long rowCount; @@ -229,10 +277,29 @@ public EncodedEntry replace( this.bucket = bucket; this.level = level; this.schemaId = schemaId; + this.hasRowId = true; this.firstRowId = firstRowId; this.rowCount = rowCount; return this; } + + public EncodedEntry replaceWithoutRowId( + byte kind, + BinaryRow partition, + int bucket, + int level, + long schemaId, + long rowCount) { + this.kind = kind; + this.partition = partition; + this.bucket = bucket; + this.level = level; + this.schemaId = schemaId; + this.hasRowId = false; + this.firstRowId = 0; + this.rowCount = rowCount; + return this; + } } /** Aggregate statistics for an encoded Avro block copied without decompression. */ @@ -247,10 +314,7 @@ public static final class EncodedBlock { private final int maxLevel; private final long minRowId; private final long maxRowId; - private final @Nullable BinaryRow nullPartition; - private final long nullPartitionCount; - private final @Nullable BinaryRow minNonNullPartition; - private final @Nullable BinaryRow maxNonNullPartition; + private final SimpleStats partitionStats; public EncodedBlock( long addedFiles, @@ -262,10 +326,7 @@ public EncodedBlock( int maxLevel, long minRowId, long maxRowId, - @Nullable BinaryRow nullPartition, - long nullPartitionCount, - @Nullable BinaryRow minNonNullPartition, - @Nullable BinaryRow maxNonNullPartition) { + SimpleStats partitionStats) { this.addedFiles = addedFiles; this.deletedFiles = deletedFiles; this.schemaId = schemaId; @@ -275,10 +336,7 @@ public EncodedBlock( this.maxLevel = maxLevel; this.minRowId = minRowId; this.maxRowId = maxRowId; - this.nullPartition = nullPartition; - this.nullPartitionCount = nullPartitionCount; - this.minNonNullPartition = minNonNullPartition; - this.maxNonNullPartition = maxNonNullPartition; + this.partitionStats = partitionStats; } } @@ -289,6 +347,9 @@ private final class FileWriter { private final SimpleStatsConverter partitionStatsSerializer; private final Map encodedPartitionCounts = new IdentityHashMap<>(); private final long[] repeatedNullCounts = new long[partitionType.getFieldCount()]; + private final long[] copiedManifestNullCounts = new long[partitionType.getFieldCount()]; + private final long[] copiedManifestRepresentativeNullCounts = + new long[partitionType.getFieldCount()]; private @Nullable PositionOutputStream out; private @Nullable AvroBlockWriter writer; private @Nullable Long outputBytes; @@ -299,6 +360,8 @@ private final class FileWriter { private int maxBucket = Integer.MIN_VALUE; private int minLevel = Integer.MAX_VALUE; private int maxLevel = Integer.MIN_VALUE; + private boolean bucketStatsKnown = true; + private boolean levelStatsKnown = true; private @Nullable RowIdStats rowIdStats = new RowIdStats(); private boolean closed; @@ -348,20 +411,19 @@ private void writeEncoded(ByteBuffer encodedRecord, EncodedEntry metadata) addEncodedPartition(metadata.partition, 1); } + private void writeRow(InternalRow row, EncodedEntry metadata) throws IOException { + ensureOpen(); + writer.addElement(row); + collectStats(metadata); + addEncodedPartition(metadata.partition, 1); + } + private void writeEncodedBlock(AvroRawBlock block, EncodedBlock metadata) throws IOException { ensureOpen(); writer.addEncodedBlock(block); collectStats(metadata); - if (metadata.nullPartitionCount > 0) { - addEncodedPartition(metadata.nullPartition, metadata.nullPartitionCount); - } - if (metadata.minNonNullPartition != null) { - addEncodedPartition(metadata.minNonNullPartition, 1); - if (metadata.maxNonNullPartition != metadata.minNonNullPartition) { - addEncodedPartition(metadata.maxNonNullPartition, 1); - } - } + collectCopiedPartitionStats(metadata.partitionStats); } private void collectStats(ManifestEntry entry) { @@ -408,7 +470,11 @@ private void collectStats(EncodedEntry entry) { minLevel = Math.min(minLevel, entry.level); maxLevel = Math.max(maxLevel, entry.level); if (rowIdStats != null) { - rowIdStats.collect(entry.firstRowId, entry.rowCount); + if (!entry.hasRowId) { + rowIdStats = null; + } else { + rowIdStats.collect(entry.firstRowId, entry.rowCount); + } } } @@ -425,6 +491,55 @@ private void collectStats(EncodedBlock block) { } } + private void collectStats(ManifestFileMeta manifest) { + numAddedFiles = Math.addExact(numAddedFiles, manifest.numAddedFiles()); + numDeletedFiles = Math.addExact(numDeletedFiles, manifest.numDeletedFiles()); + schemaId = Math.max(schemaId, manifest.schemaId()); + if (manifest.minBucket() == null || manifest.maxBucket() == null) { + bucketStatsKnown = false; + } else { + minBucket = Math.min(minBucket, manifest.minBucket()); + maxBucket = Math.max(maxBucket, manifest.maxBucket()); + } + if (manifest.minLevel() == null || manifest.maxLevel() == null) { + levelStatsKnown = false; + } else { + minLevel = Math.min(minLevel, manifest.minLevel()); + maxLevel = Math.max(maxLevel, manifest.maxLevel()); + } + if (rowIdStats != null) { + if (manifest.minRowId() == null || manifest.maxRowId() == null) { + rowIdStats = null; + } else { + rowIdStats.collectRange(manifest.minRowId(), manifest.maxRowId()); + } + } + + collectCopiedPartitionStats(manifest.partitionStats()); + } + + private void collectCopiedPartitionStats(SimpleStats partitionStats) { + collectCopiedPartitionRepresentative(partitionStats.minValues()); + if (!partitionStats.maxValues().equals(partitionStats.minValues())) { + collectCopiedPartitionRepresentative(partitionStats.maxValues()); + } + for (int field = 0; field < copiedManifestNullCounts.length; field++) { + copiedManifestNullCounts[field] = + Math.addExact( + copiedManifestNullCounts[field], + partitionStats.nullCounts().getLong(field)); + } + } + + private void collectCopiedPartitionRepresentative(BinaryRow partition) { + partitionStatsCollector.collect(partition); + for (int field = 0; field < partition.getFieldCount(); field++) { + if (partition.isNullAt(field)) { + copiedManifestRepresentativeNullCounts[field]++; + } + } + } + private void addEncodedPartition(@Nullable BinaryRow partition, long count) { if (partition == null || count <= 0) { return; @@ -432,9 +547,37 @@ private void addEncodedPartition(@Nullable BinaryRow partition, long count) { long[] value = encodedPartitionCounts.computeIfAbsent(partition, ignored -> new long[1]); value[0] = Math.addExact(value[0], count); + if (encodedPartitionCounts.size() >= MAX_BUFFERED_ENCODED_PARTITIONS) { + flushEncodedPartitions(); + } } private SimpleColStats[] partitionStats() { + flushEncodedPartitions(); + SimpleColStats[] stats = partitionStatsCollector.extract(); + for (int field = 0; field < stats.length; field++) { + // The collector sees only the min/max partition representatives of each copied + // block or manifest. Add the remaining nulls from its aggregate statistics. + long nullCountAdjustment = + Math.addExact( + repeatedNullCounts[field], + Math.subtractExact( + copiedManifestNullCounts[field], + copiedManifestRepresentativeNullCounts[field])); + if (nullCountAdjustment == 0) { + continue; + } + SimpleColStats current = stats[field]; + stats[field] = + new SimpleColStats( + current.min(), + current.max(), + Math.addExact(current.nullCount(), nullCountAdjustment)); + } + return stats; + } + + private void flushEncodedPartitions() { for (Map.Entry entry : encodedPartitionCounts.entrySet()) { BinaryRow partition = entry.getKey(); partitionStatsCollector.collect(partition); @@ -450,19 +593,6 @@ private SimpleColStats[] partitionStats() { } } encodedPartitionCounts.clear(); - SimpleColStats[] stats = partitionStatsCollector.extract(); - for (int field = 0; field < stats.length; field++) { - if (repeatedNullCounts[field] == 0) { - continue; - } - SimpleColStats current = stats[field]; - stats[field] = - new SimpleColStats( - current.min(), - current.max(), - Math.addExact(current.nullCount(), repeatedNullCounts[field])); - } - return stats; } private boolean reachTargetSize(boolean suggestedCheck, long targetSize) @@ -519,10 +649,10 @@ private ManifestFileMeta result() { numAddedFiles + numDeletedFiles > 0 ? schemaId : schemaManager.latest().get().id(), - minBucket, - maxBucket, - minLevel, - maxLevel, + bucketStatsKnown ? minBucket : null, + bucketStatsKnown ? maxBucket : null, + levelStatsKnown ? minLevel : null, + levelStatsKnown ? maxLevel : null, rowIdStats == null ? null : rowIdStats.minRowId, rowIdStats == null ? null : rowIdStats.maxRowId); } diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java index 0bac1bab0adc..e91f84a2e65c 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java @@ -203,8 +203,8 @@ private static CloseableIterator createManifestIterator( } } - /** Opens a low-allocation reader for the encoded manifest fields needed by run merge. */ - public ManifestAvroReader scanForRunMerge(String fileName, @Nullable Long fileSize) { + /** Opens a low-allocation reader over raw Avro manifest blocks. */ + public ManifestAvroReader scanAvroBlocks(String fileName, @Nullable Long fileSize) { try { return new ManifestAvroReader(fileIO.newInputStream(pathFactory.toPath(fileName))); } catch (IOException e) { diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMerge.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMerge.java index b3d296e085a8..121b1be89e7d 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMerge.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMerge.java @@ -18,9 +18,11 @@ package org.apache.paimon.operation; +import org.apache.paimon.data.BinaryArray; import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalRow; +import org.apache.paimon.format.SimpleStatsCollector; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.manifest.CompactFileIdentifierSet; import org.apache.paimon.manifest.FileKind; @@ -31,6 +33,8 @@ import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.manifest.ManifestFile; import org.apache.paimon.manifest.ManifestFileMeta; +import org.apache.paimon.stats.SimpleStats; +import org.apache.paimon.stats.SimpleStatsConverter; import org.apache.paimon.types.DataField; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.Pair; @@ -43,6 +47,7 @@ import java.util.function.Function; import static org.apache.paimon.utils.ManifestReadThreadPool.sequentialBatchedExecute; +import static org.apache.paimon.utils.Preconditions.checkState; /** Streaming merge of the naturally sorted runs in data-evolution manifest files. */ final class ManifestEntryRunMerge { @@ -65,10 +70,42 @@ final class ManifestEntryRunMerge { static final int EMBEDDED_FILE_INDEX = 7; static final int EXTERNAL_PATH = 8; static final int FILE_FIELD_COUNT = 9; + private static final String[] ENTRY_FILE_FIELD_NAMES = { + DataFileMeta.FILE_NAME, + DataFileMeta.ROW_COUNT, + DataFileMeta.LEVEL, + DataFileMeta.SCHEMA_ID, + DataFileMeta.FIRST_ROW_ID, + DataFileMeta.MAX_SEQUENCE_NUMBER, + DataFileMeta.EXTRA_FILES, + DataFileMeta.EMBEDDED_FILE_INDEX, + DataFileMeta.EXTERNAL_PATH + }; + private static final InternalRow.FieldGetter[] ENTRY_FILE_GETTERS = entryFileGetters(); static final RowType ENTRY_LAYOUT = entryLayout(); + private static final int FULL_KIND = + ManifestEntry.MANIFEST_ROW_TYPE.getFieldIndex(ManifestEntry.KIND); + private static final int FULL_PARTITION = + ManifestEntry.MANIFEST_ROW_TYPE.getFieldIndex(ManifestEntry.PARTITION); + private static final int FULL_BUCKET = + ManifestEntry.MANIFEST_ROW_TYPE.getFieldIndex(ManifestEntry.BUCKET); + private static final int FULL_FILE = + ManifestEntry.MANIFEST_ROW_TYPE.getFieldIndex(ManifestEntry.FILE); private ManifestEntryRunMerge() {} + private static InternalRow.FieldGetter[] entryFileGetters() { + InternalRow.FieldGetter[] getters = + new InternalRow.FieldGetter[ENTRY_FILE_FIELD_NAMES.length]; + for (int field = 0; field < getters.length; field++) { + int position = DataFileMeta.SCHEMA.getFieldIndex(ENTRY_FILE_FIELD_NAMES[field]); + getters[field] = + InternalRow.createFieldGetter( + DataFileMeta.SCHEMA.getTypeAt(position), position); + } + return getters; + } + private static RowType entryLayout() { List fields = new ArrayList<>(); fields.add(ManifestEntry.MANIFEST_ROW_TYPE.getField(ManifestEntry.KIND)); @@ -91,6 +128,19 @@ private static RowType entryLayout() { return new RowType(false, fields); } + static GenericRow projectEntryLayout( + GenericRow fullRow, GenericRow reuse, GenericRow reuseFile) { + reuse.setField(KIND, fullRow.getByte(FULL_KIND)); + reuse.setField(PARTITION, fullRow.getBinary(FULL_PARTITION)); + reuse.setField(BUCKET, fullRow.getInt(FULL_BUCKET)); + InternalRow fullFile = fullRow.getRow(FULL_FILE, DataFileMeta.SCHEMA.getFieldCount()); + for (int field = 0; field < ENTRY_FILE_GETTERS.length; field++) { + reuseFile.setField(field, ENTRY_FILE_GETTERS[field].getFieldOrNull(fullFile)); + } + reuse.setField(FILE, reuseFile); + return reuse; + } + /** * Returns null when the input is too fragmented for a bounded streaming merge. The caller must * fall back to the spillable external sorter in that case. @@ -99,6 +149,7 @@ private static RowType entryLayout() { static List sortAndWriteFullEntries( List section, ManifestFileSorter.RowIdEntrySortKey sortKey, + RowType partitionType, ManifestFile manifestFile, List newFilesForAbort, CompactFileIdentifierSet deletedIdentifiers, @@ -106,9 +157,15 @@ static List sortAndWriteFullEntries( @Nullable Integer manifestReadParallelism) throws Exception { ManifestEntryRunMergeEntry.Filter filter = - new ManifestEntryRunMergeEntry.Filter(deletedIdentifiers, deletedRowIds); + new ManifestEntryRunMergeEntry.Filter(deletedIdentifiers, deletedRowIds, true); ManifestEntryRunMergePlan plan = - discoverRuns(section, sortKey, manifestFile, filter, manifestReadParallelism); + discoverRuns( + section, + sortKey, + partitionType, + manifestFile, + filter, + manifestReadParallelism); if (plan == null) { return null; } @@ -123,6 +180,7 @@ static List sortAndWriteFullEntries( static Pair, List> sortAndWriteMinorEntries( List section, ManifestFileSorter.RowIdEntrySortKey sortKey, + RowType partitionType, ManifestFile manifestFile, List newFilesForAbort, @Nullable Integer manifestReadParallelism) @@ -130,13 +188,19 @@ static Pair, List> sortAndWriteMinorEnt CompactFileIdentifierSet deletedIdentifiers = new CompactFileIdentifierSet(); ManifestFileSorter.DeletedRowIdSet deletedRowIds = new ManifestFileSorter.DeletedRowIdSet(); ManifestEntryRunMergeEntry.Filter.Minor filter = - new ManifestEntryRunMergeEntry.Filter.Minor(deletedIdentifiers, deletedRowIds); + new ManifestEntryRunMergeEntry.Filter.Minor( + deletedIdentifiers, deletedRowIds, true); try { ManifestEntryRunMergePlan plan; try { plan = discoverRuns( - section, sortKey, manifestFile, filter, manifestReadParallelism); + section, + sortKey, + partitionType, + manifestFile, + filter, + manifestReadParallelism); } finally { deletedRowIds.releaseRangeIndex(); } @@ -159,6 +223,7 @@ static Pair, List> sortAndWriteMinorEnt private static ManifestEntryRunMergePlan discoverRuns( List section, ManifestFileSorter.RowIdEntrySortKey sortKey, + RowType partitionType, ManifestFile manifestFile, ManifestEntryRunMergeEntry.Filter filter, @Nullable Integer manifestReadParallelism) @@ -174,7 +239,7 @@ private static ManifestEntryRunMergePlan discoverRuns( || manifestReadParallelism <= 1) { for (ManifestFileMeta meta : section) { Discovery.DiscoveredManifest manifest = - discoverManifestRuns(meta, manifestFile, partitions, filter); + discoverManifestRuns(meta, manifestFile, partitionType, partitions, filter); if (manifest.requiresExternalSort) { return null; } @@ -185,7 +250,8 @@ private static ManifestEntryRunMergePlan discoverRuns( meta -> { try { return Collections.singletonList( - discoverManifestRuns(meta, manifestFile, partitions, filter)); + discoverManifestRuns( + meta, manifestFile, partitionType, partitions, filter)); } catch (Exception e) { throw new RuntimeException( "Failed to discover sorted Avro runs in " + meta.fileName(), e); @@ -229,12 +295,13 @@ private static ManifestEntryRunMergePlan discoverRuns( private static Discovery.DiscoveredManifest discoverManifestRuns( ManifestFileMeta meta, ManifestFile manifestFile, + RowType partitionType, ManifestEntryRunMergeEntry.PartitionDictionary partitions, ManifestEntryRunMergeEntry.Filter filter) throws Exception { try (ManifestAvroReader reader = - manifestFile.scanForRunMerge(meta.fileName(), meta.fileSize())) { - return discoverManifestRuns(meta, reader, partitions, filter); + manifestFile.scanAvroBlocks(meta.fileName(), meta.fileSize())) { + return discoverManifestRuns(meta, reader, partitionType, partitions, filter); } catch (UnsupportedOperationException unsupported) { return Discovery.DiscoveredManifest.requiresExternalSort(); } @@ -243,9 +310,11 @@ private static Discovery.DiscoveredManifest discoverManifestRuns( private static Discovery.DiscoveredManifest discoverManifestRuns( ManifestFileMeta meta, ManifestAvroReader reader, + RowType partitionType, ManifestEntryRunMergeEntry.PartitionDictionary partitions, ManifestEntryRunMergeEntry.Filter filter) throws Exception { + SimpleStatsConverter partitionStatsConverter = new SimpleStatsConverter(partitionType); List runs = new ArrayList<>(); List blocks = new ArrayList<>(); ManifestEntryRunMergeEntry.Key previous = new ManifestEntryRunMergeEntry.Key(); @@ -272,10 +341,11 @@ private static Discovery.DiscoveredManifest discoverManifestRuns( rawBlock.blockOrdinal(), position, rawBlock.rawBlockCopySupported(), - current.stableCopy())); + current.stableCopy(), + partitionType)); } Discovery.BlockInfo block = blocks.get(blocks.size() - 1); - block.collect(row, current, partitions, filter); + block.collectForSort(row, current, partitions, filter); boolean inversion = hasPrevious && compareDiscoveryKeys(previous, current, partitions) > 0; if (inversion) { @@ -300,7 +370,7 @@ private static Discovery.DiscoveredManifest discoverManifestRuns( position++; if (rows.recordIndex() + 1 == rawBlock.recordCount()) { ManifestEntryRunMergeEntry.Key stableLastKey = current.stableCopy(); - block.finish(position, stableLastKey); + block.finishSort(position, stableLastKey, partitionStatsConverter); previous.copyFrom(stableLastKey); } else { previous.copyFrom(current); @@ -442,11 +512,11 @@ static final class BlockInfo { final long ordinal; final long start; - final ManifestEntryRunMergeEntry.Key firstKey; + final @Nullable ManifestEntryRunMergeEntry.Key firstKey; boolean eligible; boolean sorted = true; long end; - ManifestEntryRunMergeEntry.Key lastKey; + @Nullable ManifestEntryRunMergeEntry.Key lastKey; long addedFiles; long deletedFiles; long schemaId = Long.MIN_VALUE; @@ -456,33 +526,65 @@ static final class BlockInfo { int maxLevel = Integer.MIN_VALUE; long minRowId = Long.MAX_VALUE; long maxRowId = Long.MIN_VALUE; - BinaryRow nullPartition; + final boolean singleFieldSortedPartitionStats; + @Nullable SimpleStatsCollector partitionStats; + final RowType partitionType; + @Nullable BinaryRow nullPartition; long nullPartitionCount; - BinaryRow minNonNullPartition; - BinaryRow maxNonNullPartition; + @Nullable BinaryRow minNonNullPartition; + @Nullable BinaryRow maxNonNullPartition; EncodedBlock metadata; BlockInfo( long ordinal, long start, boolean eligible, - ManifestEntryRunMergeEntry.Key firstKey) { + ManifestEntryRunMergeEntry.Key firstKey, + RowType partitionType) { this.ordinal = ordinal; this.start = start; this.eligible = eligible; this.firstKey = firstKey; + this.partitionType = partitionType; + this.singleFieldSortedPartitionStats = + eligible && firstKey != null && partitionType.getFieldCount() == 1; + this.partitionStats = + eligible && firstKey != null && !singleFieldSortedPartitionStats + ? new SimpleStatsCollector(partitionType) + : null; } - void collect( + void collectForSort( GenericRow record, ManifestEntryRunMergeEntry.Key key, ManifestEntryRunMergeEntry.PartitionDictionary partitions, ManifestEntryRunMergeEntry.Filter filter) { - BinaryRow partition = partitions.partition(key.partitionId); - eligible &= partition.getFieldCount() == 1 && filter.copyable(record, key); if (!eligible) { return; } + if (!filter.copyable(record, key)) { + eligible = false; + releasePartitionStats(); + return; + } + collectEntryStats(record, key); + BinaryRow partition = partitions.partition(key.partitionId); + if (singleFieldSortedPartitionStats) { + if (partition.isNullAt(0)) { + nullPartition = partition; + nullPartitionCount++; + } else { + if (minNonNullPartition == null) { + minNonNullPartition = partition; + } + maxNonNullPartition = partition; + } + } else { + partitionStats.collect(partition); + } + } + + private void collectEntryStats(GenericRow record, ManifestEntryRunMergeEntry.Key key) { InternalRow file = ManifestEntryRunMergeEntry.file(record); if (key.kind == FileKind.ADD.toByteValue()) { addedFiles++; @@ -498,21 +600,33 @@ void collect( maxLevel = Math.max(maxLevel, level); minRowId = Math.min(minRowId, key.firstRowId); maxRowId = Math.max(maxRowId, key.rangeEnd); - if (partition.isNullAt(0)) { - nullPartition = partition; - nullPartitionCount++; - } else { - if (minNonNullPartition == null) { - minNonNullPartition = partition; - } - maxNonNullPartition = partition; - } } - void finish(long end, ManifestEntryRunMergeEntry.Key lastKey) { + void finishSort( + long end, + ManifestEntryRunMergeEntry.Key lastKey, + SimpleStatsConverter partitionStatsConverter) { this.end = end; this.lastKey = lastKey; if (eligible && sorted) { + SimpleStats encodedPartitionStats; + if (singleFieldSortedPartitionStats) { + BinaryRow min = + minNonNullPartition == null ? nullPartition : minNonNullPartition; + BinaryRow max = + maxNonNullPartition == null ? nullPartition : maxNonNullPartition; + checkState(min != null && max != null, "Manifest block has no partition."); + encodedPartitionStats = + new SimpleStats( + min, + max, + BinaryArray.fromLongArray(new Long[] {nullPartitionCount})); + } else { + checkState( + partitionStats != null, "Manifest block has no partition stats."); + encodedPartitionStats = + partitionStatsConverter.toBinaryAllMode(partitionStats.extract()); + } metadata = new EncodedBlock( addedFiles, @@ -524,11 +638,16 @@ void finish(long end, ManifestEntryRunMergeEntry.Key lastKey) { maxLevel, minRowId, maxRowId, - nullPartition, - nullPartitionCount, - minNonNullPartition, - maxNonNullPartition); + encodedPartitionStats); } + releasePartitionStats(); + } + + private void releasePartitionStats() { + partitionStats = null; + nullPartition = null; + minNonNullPartition = null; + maxNonNullPartition = null; } boolean copyable(long runStart, long runEnd) { @@ -542,6 +661,7 @@ void finishFiltering(ManifestEntryRunMergeEntry.Filter filter) { } void updatePartitionRanks(ManifestEntryRunMergeEntry.PartitionDictionary partitions) { + checkState(firstKey != null && lastKey != null, "Manifest block has no sort keys."); firstKey.partitionRank = partitions.rank(firstKey.partitionId); lastKey.partitionRank = partitions.rank(lastKey.partitionId); } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergeEntry.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergeEntry.java index 078184ffc9d5..2dc8b3d86509 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergeEntry.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergeEntry.java @@ -49,6 +49,7 @@ static final class Key { int partitionId; int partitionRank; byte kind; + boolean hasRowId; long firstRowId; long rangeEnd; long reverseSequence; @@ -67,6 +68,7 @@ void replace(ProjectedManifestEntry entry, PartitionDictionary partitions) { this.partitionId = partitions.id(entry.partitionBytes()); this.partitionRank = partitions.rank(partitionId); this.kind = entry.kind().toByteValue(); + this.hasRowId = true; this.firstRowId = firstRowId; this.rangeEnd = firstRowId + entry.file().rowCount() - 1L; this.reverseSequence = Long.MAX_VALUE - entry.file().maxSequenceNumber(); @@ -83,6 +85,7 @@ void replace(GenericRow record, PartitionDictionary partitions) { this.partitionId = partitions.id(record.getBinary(ManifestEntryRunMerge.PARTITION)); this.partitionRank = partitions.rank(partitionId); this.kind = record.getByte(ManifestEntryRunMerge.KIND); + this.hasRowId = true; this.firstRowId = file.getLong(ManifestEntryRunMerge.FIRST_ROW_ID); this.rangeEnd = firstRowId + file.getLong(ManifestEntryRunMerge.ROW_COUNT) - 1L; this.reverseSequence = @@ -97,10 +100,21 @@ void replace(GenericRow record, PartitionDictionary partitions) { this.fileNameLength = fileNameBytes.length; } + void replaceForCompaction(GenericRow record) { + InternalRow file = file(record); + this.kind = record.getByte(ManifestEntryRunMerge.KIND); + this.hasRowId = !file.isNullAt(ManifestEntryRunMerge.FIRST_ROW_ID); + if (hasRowId) { + this.firstRowId = file.getLong(ManifestEntryRunMerge.FIRST_ROW_ID); + this.rangeEnd = firstRowId + file.getLong(ManifestEntryRunMerge.ROW_COUNT) - 1L; + } + } + void copyFrom(Key key) { this.partitionId = key.partitionId; this.partitionRank = key.partitionRank; this.kind = key.kind; + this.hasRowId = key.hasRowId; this.firstRowId = key.firstRowId; this.rangeEnd = key.rangeEnd; this.reverseSequence = key.reverseSequence; @@ -139,6 +153,10 @@ static final class PartitionDictionary { this.sortKey = sortKey; } + PartitionDictionary() { + this.sortKey = null; + } + int id(byte[] bytes) { return id(bytes, 0, bytes.length); } @@ -173,6 +191,7 @@ int id(byte[] bytes, int offset, int length) { } int compareIds(int left, int right) { + checkState(sortKey != null, "Partition dictionary has no sort key."); return sortKey.comparePartitions(partitions[left], partitions[right]); } @@ -205,14 +224,17 @@ static class Filter { final CompactFileIdentifierSet deletedIdentifiers; final ManifestFileSorter.DeletedRowIdSet deletedRowIds; + final boolean useRowIdFilter; final ThreadLocal identifier = ThreadLocal.withInitial(IdentifierEncoder::new); Filter( CompactFileIdentifierSet deletedIdentifiers, - ManifestFileSorter.DeletedRowIdSet deletedRowIds) { + ManifestFileSorter.DeletedRowIdSet deletedRowIds, + boolean useRowIdFilter) { this.deletedIdentifiers = deletedIdentifiers; this.deletedRowIds = deletedRowIds; + this.useRowIdFilter = useRowIdFilter; } boolean include(ProjectedManifestEntry entry) { @@ -220,15 +242,7 @@ boolean include(ProjectedManifestEntry entry) { } boolean include(GenericRow record, Key key) { - if (key.kind != FileKind.ADD.toByteValue()) { - return false; - } - if (!deletedRowIds.contains(key.firstRowId)) { - return true; - } - - ReusableIdentifier reusable = identifier.get().replace(record); - return !deletedIdentifiers.contains(reusable); + return key.kind == FileKind.ADD.toByteValue() && !isDeleted(record, key); } boolean copyable(GenericRow record, Key key) { @@ -245,12 +259,25 @@ ReusableIdentifier identifier(GenericRow record) { return identifier.get().replace(record); } + boolean isDeleted(GenericRow record, Key key) { + // RowID is only a cheap negative filter. The complete identifier remains the + // authoritative match, and is also sufficient for manifests which predate RowID. + if (useRowIdFilter) { + checkState(key.hasRowId, "First row id should not be null."); + if (!deletedRowIds.contains(key.firstRowId)) { + return false; + } + } + return deletedIdentifiers.contains(identifier(record)); + } + static final class Minor extends Filter { Minor( CompactFileIdentifierSet deletedIdentifiers, - ManifestFileSorter.DeletedRowIdSet deletedRowIds) { - super(deletedIdentifiers, deletedRowIds); + ManifestFileSorter.DeletedRowIdSet deletedRowIds, + boolean useRowIdFilter) { + super(deletedIdentifiers, deletedRowIds, useRowIdFilter); } @Override @@ -276,7 +303,10 @@ void observe(GenericRow record, Key key) { ReusableIdentifier reusable = identifier(record); synchronized (this) { deletedIdentifiers.add(reusable); - deletedRowIds.add(key.firstRowId); + if (useRowIdFilter) { + checkState(key.hasRowId, "First row id should not be null."); + deletedRowIds.add(key.firstRowId); + } } } @@ -285,7 +315,7 @@ boolean copyableAfterDiscovery(long minRowId, long maxRowId) { // A DELETE preserves the deleted ADD's globally unique first RowID. A range hit may // be a false positive and only disables block copying; a miss proves the block has // no deleted ADD. - return !deletedRowIds.intersects(minRowId, maxRowId); + return useRowIdFilter && !deletedRowIds.intersects(minRowId, maxRowId); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergePlan.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergePlan.java index 2f88d97c1823..8b2129e10b32 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergePlan.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergePlan.java @@ -152,12 +152,7 @@ static List writeSelected( continue; } cursor.materializeCurrent(); - ByteBuffer encodedRecord = cursor.encodedRecord(); - if (encodedRecord == null) { - writer.write(cursor.current()); - } else { - writer.writeEncoded(encodedRecord, cursor.metadata()); - } + writeCurrent(writer, cursor); selectionTree.update(winner, cursor.advance()); } } catch (Exception e) { @@ -233,6 +228,11 @@ private static Pair, List> writeMinorSe } private static void writeCurrent(ManifestAvroWriter writer, Cursor cursor) throws Exception { + InternalRow decodedRow = cursor.decodedRow(); + if (decodedRow != null) { + writer.writeRow(decodedRow, cursor.metadata()); + return; + } ByteBuffer encodedRecord = cursor.encodedRecord(); if (encodedRecord == null) { writer.write(cursor.current()); @@ -355,6 +355,10 @@ interface Cursor extends AutoCloseable { @Nullable ByteBuffer encodedRecord(); + default @Nullable InternalRow decodedRow() { + return null; + } + ReusableIdentifier identifier(); default boolean hasCopyableBlock() { @@ -386,6 +390,7 @@ default void materializeCurrent() throws Exception {} static final class PrimitiveManifestRunCursor implements Cursor { final ManifestAvroReader reader; + final boolean encodedRecordsCompatible; final ManifestEntryRunMergeEntry.Filter filter; final ManifestEntryRunMergeEntry.PartitionDictionary partitions; final ManifestEntryRunMergeEntry.Key key = new ManifestEntryRunMergeEntry.Key(); @@ -401,6 +406,9 @@ static final class PrimitiveManifestRunCursor implements Cursor { @Nullable RawBlock currentRawBlock; @Nullable RowIterator currentRows; @Nullable GenericRow currentRow; + @Nullable GenericRow currentSourceRow; + @Nullable GenericRow compactRow; + @Nullable GenericRow compactFile; @Nullable ManifestEntryRunMerge.Discovery.BlockInfo currentBlock; boolean closed; @@ -413,7 +421,13 @@ static final class PrimitiveManifestRunCursor implements Cursor { ManifestEntryRunMergeEntry.Filter filter, ManifestEntryRunMergeEntry.PartitionDictionary partitions) throws Exception { - this.reader = manifestFile.scanForRunMerge(meta.fileName(), meta.fileSize()); + this.reader = manifestFile.scanAvroBlocks(meta.fileName(), meta.fileSize()); + this.encodedRecordsCompatible = reader.rawBlockCopySupported(); + if (!encodedRecordsCompatible) { + this.compactRow = + new GenericRow(ManifestEntryRunMerge.ENTRY_LAYOUT.getFieldCount()); + this.compactFile = new GenericRow(ManifestEntryRunMerge.FILE_FIELD_COUNT); + } this.filter = filter; this.partitions = partitions; this.blocks = blocks; @@ -453,7 +467,12 @@ public boolean advance() throws Exception { checkState( currentRows != null && currentRows.hasNext(), "Manifest block ends before its discovered boundary."); - currentRow = currentRows.next(); + currentSourceRow = currentRows.next(); + currentRow = + encodedRecordsCompatible + ? currentSourceRow + : ManifestEntryRunMerge.projectEntryLayout( + currentSourceRow, compactRow, compactFile); decodedRemaining--; key.replace(currentRow, partitions); if (filter.include(currentRow, key)) { @@ -477,6 +496,7 @@ boolean prepareNextBlock() throws Exception { current = false; currentRows = null; currentRow = null; + currentSourceRow = null; while (blockIndex < blocks.size()) { ManifestEntryRunMerge.Discovery.BlockInfo info = blocks.get(blockIndex); if (info.start >= runEnd) { @@ -500,7 +520,11 @@ boolean prepareNextBlock() throws Exception { long overlapStart = Math.max(runStart, info.start); long overlapEnd = Math.min(runEnd, info.end); long prefix = overlapStart - info.start; - currentRows = currentRawBlock.toRows(ManifestEntryRunMerge.ENTRY_LAYOUT); + currentRows = + currentRawBlock.toRows( + encodedRecordsCompatible + ? ManifestEntryRunMerge.ENTRY_LAYOUT + : ManifestEntry.MANIFEST_ROW_TYPE); for (long i = 0; i < prefix; i++) { checkState( currentRows.hasNext(), @@ -538,7 +562,12 @@ public ManifestEntryRunMergeEntry.Key key() { @Override public ByteBuffer encodedRecord() { - return current ? currentRows.encodedRecord() : null; + return current && encodedRecordsCompatible ? currentRows.encodedRecord() : null; + } + + @Override + public InternalRow decodedRow() { + return current && !encodedRecordsCompatible ? currentSourceRow : null; } @Override @@ -584,9 +613,18 @@ public void materializeCurrent() throws Exception { rawBlock = false; decodedRemaining = currentBlock.end - currentBlock.start; checkState(decodedRemaining > 0, "Raw Avro block is empty."); - currentRows = currentRawBlock.toRows(ManifestEntryRunMerge.ENTRY_LAYOUT); + currentRows = + currentRawBlock.toRows( + encodedRecordsCompatible + ? ManifestEntryRunMerge.ENTRY_LAYOUT + : ManifestEntry.MANIFEST_ROW_TYPE); checkState(currentRows.hasNext(), "Manifest block cannot be decompressed."); - currentRow = currentRows.next(); + currentSourceRow = currentRows.next(); + currentRow = + encodedRecordsCompatible + ? currentSourceRow + : ManifestEntryRunMerge.projectEntryLayout( + currentSourceRow, compactRow, compactFile); decodedRemaining--; key.replace(currentRow, partitions); checkState( @@ -615,6 +653,9 @@ public void close() throws Exception { currentRawBlock = null; currentRows = null; currentRow = null; + currentSourceRow = null; + compactRow = null; + compactFile = null; currentBlock = null; rawBlock = false; key.clear(); diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileBlockMerger.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileBlockMerger.java new file mode 100644 index 000000000000..0d4b88509a90 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileBlockMerger.java @@ -0,0 +1,1177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.operation; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.format.SimpleColStats; +import org.apache.paimon.format.SimpleStatsCollector; +import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.manifest.CompactFileIdentifierSet; +import org.apache.paimon.manifest.FileEntry.ReusableIdentifier; +import org.apache.paimon.manifest.FileKind; +import org.apache.paimon.manifest.ManifestAvroReader; +import org.apache.paimon.manifest.ManifestAvroReader.RawBlock; +import org.apache.paimon.manifest.ManifestAvroReader.RowIterator; +import org.apache.paimon.manifest.ManifestAvroWriter; +import org.apache.paimon.manifest.ManifestAvroWriter.EncodedBlock; +import org.apache.paimon.manifest.ManifestAvroWriter.EncodedEntry; +import org.apache.paimon.manifest.ManifestEntry; +import org.apache.paimon.manifest.ManifestFile; +import org.apache.paimon.manifest.ManifestFileMeta; +import org.apache.paimon.manifest.ProjectedManifestEntry; +import org.apache.paimon.partition.PartitionPredicate; +import org.apache.paimon.stats.SimpleStatsConverter; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.ByteArrayKey; +import org.apache.paimon.utils.ByteArrayLookupKey; +import org.apache.paimon.utils.CloseableIterator; +import org.apache.paimon.utils.Filter; +import org.apache.paimon.utils.SerializationUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; + +import static org.apache.paimon.manifest.ManifestFileMeta.allContainsRowId; +import static org.apache.paimon.utils.ManifestReadThreadPool.sequentialBatchedExecute; +import static org.apache.paimon.utils.Preconditions.checkArgument; +import static org.apache.paimon.utils.Preconditions.checkState; + +/** Block-aware manifest compaction which never delegates to the legacy full-entry merger. */ +final class ManifestFileBlockMerger { + + private static final Logger LOG = LoggerFactory.getLogger(ManifestFileBlockMerger.class); + + private static final int KIND = 0; + private static final int PARTITION = 1; + private static final int BUCKET = 2; + private static final int FILE = 3; + private static final int FILE_NAME = 0; + private static final int ROW_COUNT = 1; + private static final int LEVEL = 2; + private static final int SCHEMA_ID = 3; + private static final int FIRST_ROW_ID = 4; + private static final int MAX_SEQUENCE_NUMBER = 5; + private static final int EXTRA_FILES = 6; + private static final int EMBEDDED_FILE_INDEX = 7; + private static final int EXTERNAL_PATH = 8; + private static final int FILE_FIELD_COUNT = 9; + private static final String[] ENTRY_FILE_FIELD_NAMES = { + DataFileMeta.FILE_NAME, + DataFileMeta.ROW_COUNT, + DataFileMeta.LEVEL, + DataFileMeta.SCHEMA_ID, + DataFileMeta.FIRST_ROW_ID, + DataFileMeta.MAX_SEQUENCE_NUMBER, + DataFileMeta.EXTRA_FILES, + DataFileMeta.EMBEDDED_FILE_INDEX, + DataFileMeta.EXTERNAL_PATH + }; + private static final InternalRow.FieldGetter[] ENTRY_FILE_GETTERS = entryFileGetters(); + private static final RowType ENTRY_LAYOUT = entryLayout(); + private static final int FULL_KIND = + ManifestEntry.MANIFEST_ROW_TYPE.getFieldIndex(ManifestEntry.KIND); + private static final int FULL_PARTITION = + ManifestEntry.MANIFEST_ROW_TYPE.getFieldIndex(ManifestEntry.PARTITION); + private static final int FULL_BUCKET = + ManifestEntry.MANIFEST_ROW_TYPE.getFieldIndex(ManifestEntry.BUCKET); + private static final int FULL_FILE = + ManifestEntry.MANIFEST_ROW_TYPE.getFieldIndex(ManifestEntry.FILE); + + private ManifestFileBlockMerger() {} + + private static InternalRow.FieldGetter[] entryFileGetters() { + InternalRow.FieldGetter[] getters = + new InternalRow.FieldGetter[ENTRY_FILE_FIELD_NAMES.length]; + for (int field = 0; field < getters.length; field++) { + int position = DataFileMeta.SCHEMA.getFieldIndex(ENTRY_FILE_FIELD_NAMES[field]); + getters[field] = + InternalRow.createFieldGetter( + DataFileMeta.SCHEMA.getTypeAt(position), position); + } + return getters; + } + + private static RowType entryLayout() { + List fields = new ArrayList<>(); + fields.add(ManifestEntry.MANIFEST_ROW_TYPE.getField(ManifestEntry.KIND)); + fields.add(ManifestEntry.MANIFEST_ROW_TYPE.getField(ManifestEntry.PARTITION)); + fields.add(ManifestEntry.MANIFEST_ROW_TYPE.getField(ManifestEntry.BUCKET)); + fields.add( + ManifestEntry.MANIFEST_ROW_TYPE + .getField(ManifestEntry.FILE) + .newType( + DataFileMeta.SCHEMA.project( + DataFileMeta.FILE_NAME, + DataFileMeta.ROW_COUNT, + DataFileMeta.LEVEL, + DataFileMeta.SCHEMA_ID, + DataFileMeta.FIRST_ROW_ID, + DataFileMeta.MAX_SEQUENCE_NUMBER, + DataFileMeta.EXTRA_FILES, + DataFileMeta.EMBEDDED_FILE_INDEX, + DataFileMeta.EXTERNAL_PATH))); + return new RowType(false, fields); + } + + private static GenericRow projectEntryLayout( + GenericRow fullRow, GenericRow reuse, GenericRow reuseFile) { + reuse.setField(KIND, fullRow.getByte(FULL_KIND)); + reuse.setField(PARTITION, fullRow.getBinary(FULL_PARTITION)); + reuse.setField(BUCKET, fullRow.getInt(FULL_BUCKET)); + InternalRow fullFile = fullRow.getRow(FULL_FILE, DataFileMeta.SCHEMA.getFieldCount()); + for (int field = 0; field < ENTRY_FILE_GETTERS.length; field++) { + reuseFile.setField(field, ENTRY_FILE_GETTERS[field].getFieldOrNull(fullFile)); + } + reuse.setField(FILE, reuseFile); + return reuse; + } + + static List merge( + List input, + List newFilesForAbort, + ManifestFile manifestFile, + RowType partitionType, + CoreOptions options) + throws Exception { + long suggestedMetaSize = options.manifestTargetSize().getBytes(); + Integer manifestReadParallelism = options.scanManifestParallelism(); + Optional> fullCompacted = + tryFullCompaction( + input, + newFilesForAbort, + manifestFile, + suggestedMetaSize, + options.manifestFullCompactionThresholdSize().getBytes(), + partitionType, + manifestReadParallelism); + if (fullCompacted.isPresent()) { + return fullCompacted.get(); + } + return compactMinor( + input, + newFilesForAbort, + manifestFile, + partitionType, + suggestedMetaSize, + options.manifestMergeMinCount(), + manifestReadParallelism); + } + + static Optional> tryFullCompaction( + List inputs, + List newFilesForAbort, + ManifestFile manifestFile, + long suggestedMetaSize, + long sizeTrigger, + RowType partitionType, + @Nullable Integer manifestReadParallelism) + throws Exception { + checkArgument(sizeTrigger > 0, "Manifest full compaction size trigger cannot be zero."); + + Filter mustChange = + file -> file.numDeletedFiles() > 0 || file.fileSize() < suggestedMetaSize; + long totalManifestSize = 0; + long deltaDeleteFileNum = 0; + long totalDeltaFileSize = 0; + List deltaManifests = new ArrayList<>(); + for (ManifestFileMeta file : inputs) { + totalManifestSize += file.fileSize(); + if (mustChange.test(file)) { + totalDeltaFileSize += file.fileSize(); + deltaDeleteFileNum += file.numDeletedFiles(); + deltaManifests.add(file); + } + } + + if (totalDeltaFileSize < sizeTrigger) { + return Optional.empty(); + } + + LOG.info( + "Start Block-aware Manifest File Full Compaction: totalManifestSize: {}, deltaDeleteFileNum {}, totalDeltaFileSize {}", + totalManifestSize, + deltaDeleteFileNum, + totalDeltaFileSize); + + boolean useRowIdFilter = allContainsRowId(inputs); + CollectedDeletes deletes = + collectDeletes( + deltaManifests, + manifestFile, + useRowIdFilter, + true, + manifestReadParallelism); + try { + PartitionPredicate predicate; + if (deletes.identifiers.isEmpty()) { + predicate = PartitionPredicate.ALWAYS_FALSE; + } else if (partitionType.getFieldCount() > 0) { + predicate = PartitionPredicate.fromMultiple(partitionType, deletes.partitions); + } else { + predicate = PartitionPredicate.ALWAYS_TRUE; + } + + List result = new ArrayList<>(); + List toCompact = new LinkedList<>(inputs); + if (predicate != null) { + Iterator iterator = toCompact.iterator(); + while (iterator.hasNext()) { + ManifestFileMeta file = iterator.next(); + if (mustChange.test(file)) { + continue; + } + if (!predicate.test( + file.numAddedFiles() + file.numDeletedFiles(), + file.partitionStats().minValues(), + file.partitionStats().maxValues(), + file.partitionStats().nullCounts())) { + iterator.remove(); + result.add(file); + } + } + } + + if (toCompact.size() <= 1) { + return Optional.empty(); + } + + CompactionFilter filter = + new CompactionFilter(deletes.identifiers, deletes.rowIds, useRowIdFilter); + List rewritten = + rewriteManifests( + toCompact, + manifestFile, + partitionType, + filter, + true, + mustChange, + result, + manifestReadParallelism); + result.addAll(rewritten); + newFilesForAbort.addAll(rewritten); + return Optional.of(result); + } finally { + deletes.release(); + } + } + + private static List compactMinor( + List input, + List newFilesForAbort, + ManifestFile manifestFile, + RowType partitionType, + long suggestedMetaSize, + int suggestedMinMetaCount, + @Nullable Integer manifestReadParallelism) + throws Exception { + List result = new ArrayList<>(); + List candidates = new ArrayList<>(); + long totalSize = 0; + for (ManifestFileMeta manifest : input) { + totalSize += manifest.fileSize(); + candidates.add(manifest); + if (totalSize >= suggestedMetaSize) { + compactMinorBatch( + candidates, + result, + newFilesForAbort, + manifestFile, + partitionType, + manifestReadParallelism); + candidates.clear(); + totalSize = 0; + } + } + + if (candidates.size() >= suggestedMinMetaCount) { + compactMinorBatch( + candidates, + result, + newFilesForAbort, + manifestFile, + partitionType, + manifestReadParallelism); + } else { + result.addAll(candidates); + } + return result; + } + + private static void compactMinorBatch( + List candidates, + List result, + List newFilesForAbort, + ManifestFile manifestFile, + RowType partitionType, + @Nullable Integer manifestReadParallelism) + throws Exception { + if (candidates.size() == 1) { + result.add(candidates.get(0)); + return; + } + + List compacted = + mergeMinorManifests( + candidates, manifestFile, partitionType, manifestReadParallelism); + result.addAll(compacted); + newFilesForAbort.addAll(compacted); + } + + private static CollectedDeletes collectDeletes( + List manifests, + ManifestFile manifestFile, + boolean collectRowIds, + boolean collectPartitions, + @Nullable Integer manifestReadParallelism) { + List manifestsWithDeletes = new ArrayList<>(); + for (ManifestFileMeta manifest : manifests) { + if (manifest.numDeletedFiles() > 0) { + manifestsWithDeletes.add(manifest); + } + } + + CollectedDeletes deletes = new CollectedDeletes(); + if (manifestReadParallelism == null + || manifestReadParallelism <= 1 + || manifestsWithDeletes.size() <= 1) { + for (ManifestFileMeta manifest : manifestsWithDeletes) { + collectDeletedEntries( + manifest, manifestFile, collectRowIds, collectPartitions, deletes, false); + } + return deletes; + } + + Function> scan = + manifest -> { + collectDeletedEntries( + manifest, + manifestFile, + collectRowIds, + collectPartitions, + deletes, + true); + return Collections.singletonList(Boolean.TRUE); + }; + for (Boolean ignored : + sequentialBatchedExecute(scan, manifestsWithDeletes, manifestReadParallelism)) { + // Iteration waits for every bounded batch of manifest scans. + } + return deletes; + } + + private static void collectDeletedEntries( + ManifestFileMeta manifest, + ManifestFile manifestFile, + boolean collectRowIds, + boolean collectPartitions, + CollectedDeletes deletes, + boolean synchronize) { + try (CloseableIterator entries = + manifestFile.scan( + manifest.fileName(), ProjectedManifestEntry.DELETE_ENTRY_PROJECTION)) { + while (entries.hasNext()) { + ProjectedManifestEntry entry = entries.next(); + if (!entry.isDelete()) { + continue; + } + if (synchronize) { + synchronized (deletes) { + deletes.add(entry, collectRowIds, collectPartitions); + } + } else { + deletes.add(entry, collectRowIds, collectPartitions); + } + } + } catch (Exception e) { + throw new RuntimeException( + "Failed to collect DELETE entries from manifest " + manifest.fileName(), e); + } + } + + private static InternalRow entryFile(GenericRow record) { + return record.getRow(FILE, FILE_FIELD_COUNT); + } + + private static final class CompactionKey { + + private byte kind; + private boolean hasRowId; + private long firstRowId; + private long rangeEnd; + + private void replace(GenericRow record) { + InternalRow file = entryFile(record); + kind = record.getByte(KIND); + hasRowId = !file.isNullAt(FIRST_ROW_ID); + if (hasRowId) { + firstRowId = file.getLong(FIRST_ROW_ID); + rangeEnd = firstRowId + file.getLong(ROW_COUNT) - 1L; + } else { + firstRowId = 0; + rangeEnd = 0; + } + } + } + + private static class CompactionFilter { + + final CompactFileIdentifierSet deletedIdentifiers; + final ManifestFileSorter.DeletedRowIdSet deletedRowIds; + final boolean useRowIdFilter; + private final ProjectedManifestEntry identifierEntry = + ProjectedManifestEntry.Projection.create(ENTRY_LAYOUT).createEntry(); + private final ReusableIdentifier identifier = new ReusableIdentifier(); + + private CompactionFilter( + CompactFileIdentifierSet deletedIdentifiers, + ManifestFileSorter.DeletedRowIdSet deletedRowIds, + boolean useRowIdFilter) { + this.deletedIdentifiers = deletedIdentifiers; + this.deletedRowIds = deletedRowIds; + this.useRowIdFilter = useRowIdFilter; + } + + boolean copyable(GenericRow record, CompactionKey key, boolean deferDeletedAddCheck) { + return key.kind == FileKind.ADD.toByteValue() + && (deferDeletedAddCheck || !isDeleted(record, key)); + } + + boolean canCopyRange(long minRowId, long maxRowId) { + return !deletedRowIds.intersects(minRowId, maxRowId); + } + + private ReusableIdentifier identifier(GenericRow record) { + return identifier.replaceWithPartition(identifierEntry.replace(record)); + } + + private boolean isDeleted(GenericRow record, CompactionKey key) { + if (useRowIdFilter) { + checkState(key.hasRowId, "First row id should not be null."); + if (!deletedRowIds.contains(key.firstRowId)) { + return false; + } + } + return deletedIdentifiers.contains(identifier(record)); + } + } + + private static final class PartitionDictionary { + + private final Map ids = new HashMap<>(); + private final ByteArrayLookupKey lookup = new ByteArrayLookupKey(); + private BinaryRow[] partitions = new BinaryRow[16]; + private int partitionCount; + + private int id(byte[] bytes) { + lookup.reset(bytes); + try { + Integer existing = ids.get(lookup); + if (existing != null) { + return existing; + } + byte[] canonical = Arrays.copyOf(bytes, bytes.length); + int id = partitionCount; + if (id == partitions.length) { + partitions = Arrays.copyOf(partitions, partitions.length << 1); + } + partitions[id] = SerializationUtils.deserializeBinaryRow(canonical); + ids.put(new ByteArrayKey(canonical), id); + partitionCount = id + 1; + return id; + } finally { + lookup.clear(); + } + } + + private BinaryRow partition(int id) { + return partitions[id]; + } + } + + private static final class CollectedDeletes { + + private final CompactFileIdentifierSet identifiers = new CompactFileIdentifierSet(); + private final ManifestFileSorter.DeletedRowIdSet rowIds = + new ManifestFileSorter.DeletedRowIdSet(); + private final Set partitions = new HashSet<>(); + + private void add( + ProjectedManifestEntry entry, boolean collectRowIds, boolean collectPartitions) { + identifiers.add(entry); + if (collectPartitions) { + partitions.add(entry.partition().copy()); + } + if (collectRowIds) { + rowIds.add(entry.file().nonNullFirstRowId()); + } + } + + private void release() { + identifiers.release(); + rowIds.releaseRangeIndex(); + } + } + + /** + * Compacts manifests in input order. RowID manifests can copy unaffected ADD-only Avro blocks + * verbatim; manifests without RowID use identifiers to filter decoded entries. + */ + private static List mergeMinorManifests( + List manifests, + ManifestFile manifestFile, + RowType partitionType, + @Nullable Integer manifestReadParallelism) + throws Exception { + boolean useRowIdFilter = allContainsRowId(manifests); + CollectedDeletes deletes = + collectDeletes( + manifests, manifestFile, useRowIdFilter, false, manifestReadParallelism); + try { + CompactionFilter filter = + new CompactionFilter(deletes.identifiers, deletes.rowIds, useRowIdFilter); + return rewriteManifests( + manifests, + manifestFile, + partitionType, + filter, + false, + null, + null, + manifestReadParallelism); + } finally { + deletes.release(); + } + } + + private static List rewriteManifests( + List manifests, + ManifestFile manifestFile, + RowType partitionType, + CompactionFilter filter, + boolean fullCompaction, + @Nullable Filter mustChange, + @Nullable List unchangedManifests, + @Nullable Integer manifestReadParallelism) + throws Exception { + ManifestAvroWriter writer = manifestFile.createAvroWriter(); + CompactFileIdentifierSet matchedEntries = new CompactFileIdentifierSet(); + CompactFileIdentifierSet emittedDeletes = new CompactFileIdentifierSet(); + PartitionDictionary partitions = new PartitionDictionary(); + SimpleStatsConverter partitionStatsConverter = new SimpleStatsConverter(partitionType); + EncodedEntry metadata = new EncodedEntry(); + boolean hasDeletes = !filter.deletedIdentifiers.isEmpty(); + try { + if (hasDeletes + && filter.useRowIdFilter + && manifestReadParallelism != null + && manifestReadParallelism > 1 + && manifests.size() > 1) { + // Keep decompression and primitive entry inspection parallel. The batched executor + // bounds retained raw blocks to at most one manifest per planning thread, while the + // single writer still emits manifests in input order. + filter.deletedRowIds.prepareRangeIndex(); + Function> planner = + manifest -> { + try { + return Collections.singletonList( + planManifestRewrite( + manifest, manifestFile, partitionType, filter)); + } catch (Exception e) { + throw new RuntimeException( + "Failed to plan manifest rewrite for " + + manifest.fileName(), + e); + } + }; + for (ManifestRewritePlan plan : + sequentialBatchedExecute(planner, manifests, manifestReadParallelism)) { + if (fullCompaction + && mustChange != null + && !mustChange.test(plan.manifest) + && plan.unchanged()) { + checkState( + unchangedManifests != null, + "Full compaction requires an unchanged manifest result."); + unchangedManifests.add(plan.manifest); + continue; + } + for (PlannedBlock block : plan.blocks) { + if (block.compaction.metadata != null) { + writer.writeEncodedBlock( + block.raw.encodedBlock(), block.compaction.metadata); + } else { + writeBlockEntries( + block.raw, + writer, + filter, + fullCompaction, + matchedEntries, + emittedDeletes, + metadata, + plan.encodedRecordsCompatible); + } + } + } + writer.close(); + return writer.result(); + } + + for (ManifestFileMeta manifest : manifests) { + try (ManifestAvroReader reader = + manifestFile.scanAvroBlocks(manifest.fileName(), manifest.fileSize())) { + boolean encodedRecordsCompatible = reader.rawBlockCopySupported(); + if (fullCompaction && mustChange != null && !mustChange.test(manifest)) { + boolean rewritten = + rewriteOptionalManifest( + reader, + writer, + partitionType, + partitionStatsConverter, + partitions, + filter, + matchedEntries, + emittedDeletes, + metadata, + encodedRecordsCompatible); + if (!rewritten) { + checkState( + unchangedManifests != null, + "Full compaction requires an unchanged manifest result."); + unchangedManifests.add(manifest); + } + continue; + } + if (!hasDeletes + && manifest.numDeletedFiles() == 0 + && encodedRecordsCompatible) { + writer.writeEncodedManifest(reader, manifest); + continue; + } + writeRemainingBlocks( + reader, + writer, + partitionType, + partitionStatsConverter, + partitions, + filter, + fullCompaction, + matchedEntries, + emittedDeletes, + metadata, + encodedRecordsCompatible); + } + } + writer.close(); + return writer.result(); + } catch (Exception | Error failure) { + writer.abort(); + throw failure; + } finally { + matchedEntries.release(); + emittedDeletes.release(); + } + } + + private static ManifestRewritePlan planManifestRewrite( + ManifestFileMeta manifest, + ManifestFile manifestFile, + RowType partitionType, + CompactionFilter filter) + throws Exception { + CompactionFilter taskFilter = + new CompactionFilter( + filter.deletedIdentifiers, filter.deletedRowIds, filter.useRowIdFilter); + PartitionDictionary partitions = new PartitionDictionary(); + SimpleStatsConverter partitionStatsConverter = new SimpleStatsConverter(partitionType); + try (ManifestAvroReader reader = + manifestFile.scanAvroBlocks(manifest.fileName(), manifest.fileSize())) { + boolean encodedRecordsCompatible = reader.rawBlockCopySupported(); + List blocks = new ArrayList<>(); + while (reader.hasNext()) { + RawBlock raw = reader.next(); + blocks.add( + new PlannedBlock( + raw.stableCopy(), + inspectBlock( + raw, + partitionType, + partitionStatsConverter, + partitions, + taskFilter, + encodedRecordsCompatible))); + } + return new ManifestRewritePlan(manifest, encodedRecordsCompatible, blocks); + } + } + + private static boolean rewriteOptionalManifest( + ManifestAvroReader reader, + ManifestAvroWriter writer, + RowType partitionType, + SimpleStatsConverter partitionStatsConverter, + PartitionDictionary partitions, + CompactionFilter filter, + CompactFileIdentifierSet matchedEntries, + CompactFileIdentifierSet emittedDeletes, + EncodedEntry metadata, + boolean encodedRecordsCompatible) + throws Exception { + List pendingBlocks = new ArrayList<>(); + List pendingMetadata = new ArrayList<>(); + while (reader.hasNext()) { + RawBlock rawBlock = reader.next(); + CompactionBlock block = + inspectBlock( + rawBlock, + partitionType, + partitionStatsConverter, + partitions, + filter, + encodedRecordsCompatible); + if (block.unchanged) { + pendingBlocks.add(rawBlock.stableCopy()); + pendingMetadata.add(block.metadata); + continue; + } + + for (int i = 0; i < pendingBlocks.size(); i++) { + RawBlock pending = pendingBlocks.get(i); + EncodedBlock encodedBlock = pendingMetadata.get(i); + if (encodedBlock == null) { + writeBlockEntries( + pending, + writer, + filter, + true, + matchedEntries, + emittedDeletes, + metadata, + encodedRecordsCompatible); + } else { + writer.writeEncodedBlock(pending.encodedBlock(), encodedBlock); + } + } + writeBlockEntries( + rawBlock, + writer, + filter, + true, + matchedEntries, + emittedDeletes, + metadata, + encodedRecordsCompatible); + writeRemainingBlocks( + reader, + writer, + partitionType, + partitionStatsConverter, + partitions, + filter, + true, + matchedEntries, + emittedDeletes, + metadata, + encodedRecordsCompatible); + return true; + } + return false; + } + + private static void writeRemainingBlocks( + ManifestAvroReader reader, + ManifestAvroWriter writer, + RowType partitionType, + SimpleStatsConverter partitionStatsConverter, + PartitionDictionary partitions, + CompactionFilter filter, + boolean fullCompaction, + CompactFileIdentifierSet matchedEntries, + CompactFileIdentifierSet emittedDeletes, + EncodedEntry metadata, + boolean encodedRecordsCompatible) + throws Exception { + while (reader.hasNext()) { + RawBlock rawBlock = reader.next(); + if (encodedRecordsCompatible && filter.useRowIdFilter) { + CompactionBlock block = + inspectBlock( + rawBlock, + partitionType, + partitionStatsConverter, + partitions, + filter, + true); + if (block.metadata != null) { + writer.writeEncodedBlock(rawBlock.encodedBlock(), block.metadata); + continue; + } + } + + writeBlockEntries( + rawBlock, + writer, + filter, + fullCompaction, + matchedEntries, + emittedDeletes, + metadata, + encodedRecordsCompatible); + } + } + + private static CompactionBlock inspectBlock( + RawBlock rawBlock, + RowType partitionType, + SimpleStatsConverter partitionStatsConverter, + PartitionDictionary partitions, + CompactionFilter filter, + boolean encodedRecordsCompatible) + throws Exception { + boolean deferDeletedAddCheck = encodedRecordsCompatible && filter.useRowIdFilter; + CompactionBlock block = new CompactionBlock(deferDeletedAddCheck, partitionType); + RowIterator rows = rawBlock.toRows(ENTRY_LAYOUT); + CompactionKey key = new CompactionKey(); + while (rows.hasNext()) { + GenericRow row = rows.next(); + key.replace(row); + block.collect(row, key, filter, partitions, deferDeletedAddCheck); + } + block.finish(partitionStatsConverter, partitions); + block.finishFiltering(filter, deferDeletedAddCheck); + return block; + } + + private static void writeBlockEntries( + RawBlock rawBlock, + ManifestAvroWriter writer, + CompactionFilter filter, + boolean fullCompaction, + CompactFileIdentifierSet matchedEntries, + CompactFileIdentifierSet emittedDeletes, + EncodedEntry metadata, + boolean encodedRecordsCompatible) + throws Exception { + RowIterator rows = + rawBlock.toRows( + encodedRecordsCompatible ? ENTRY_LAYOUT : ManifestEntry.MANIFEST_ROW_TYPE); + CompactionKey key = new CompactionKey(); + GenericRow compactRow = + encodedRecordsCompatible ? null : new GenericRow(ENTRY_LAYOUT.getFieldCount()); + GenericRow compactFile = encodedRecordsCompatible ? null : new GenericRow(FILE_FIELD_COUNT); + while (rows.hasNext()) { + GenericRow sourceRow = rows.next(); + GenericRow row = + encodedRecordsCompatible + ? sourceRow + : projectEntryLayout(sourceRow, compactRow, compactFile); + key.replace(row); + if (fullCompaction) { + if (key.kind == FileKind.ADD.toByteValue() && !filter.isDeleted(row, key)) { + writeCompactedEntry( + writer, rows, sourceRow, row, key, metadata, encodedRecordsCompatible); + } + } else if (key.kind == FileKind.ADD.toByteValue()) { + if (filter.isDeleted(row, key)) { + matchedEntries.add(filter.identifier(row)); + } else { + writeCompactedEntry( + writer, rows, sourceRow, row, key, metadata, encodedRecordsCompatible); + } + } else { + ReusableIdentifier identifier = filter.identifier(row); + if (!matchedEntries.contains(identifier) && !emittedDeletes.contains(identifier)) { + emittedDeletes.add(identifier); + writeCompactedEntry( + writer, rows, sourceRow, row, key, metadata, encodedRecordsCompatible); + } + } + } + } + + private static void writeCompactedEntry( + ManifestAvroWriter writer, + RowIterator rows, + GenericRow sourceRow, + GenericRow row, + CompactionKey key, + EncodedEntry metadata, + boolean encodedRecordsCompatible) + throws Exception { + InternalRow file = entryFile(row); + BinaryRow partition = SerializationUtils.deserializeBinaryRow(row.getBinary(PARTITION)); + if (key.hasRowId) { + metadata.replace( + key.kind, + partition, + row.getInt(BUCKET), + file.getInt(LEVEL), + file.getLong(SCHEMA_ID), + key.firstRowId, + file.getLong(ROW_COUNT)); + } else { + metadata.replaceWithoutRowId( + key.kind, + partition, + row.getInt(BUCKET), + file.getInt(LEVEL), + file.getLong(SCHEMA_ID), + file.getLong(ROW_COUNT)); + } + if (encodedRecordsCompatible) { + writer.writeEncoded(rows.encodedRecord(), metadata); + } else { + writer.writeRow(sourceRow, metadata); + } + } + + /** Aggregate metadata for one raw Avro block considered by ordinary manifest compaction. */ + private static final class CompactionBlock { + + private boolean unchanged; + private long addedFiles; + private long deletedFiles; + private long schemaId = Long.MIN_VALUE; + private int minBucket = Integer.MAX_VALUE; + private int maxBucket = Integer.MIN_VALUE; + private int minLevel = Integer.MAX_VALUE; + private int maxLevel = Integer.MIN_VALUE; + private long minRowId = Long.MAX_VALUE; + private long maxRowId = Long.MIN_VALUE; + private final RowType partitionType; + private @Nullable PartitionCounts partitionCounts; + private @Nullable EncodedBlock metadata; + + private CompactionBlock(boolean collectMetadata, RowType partitionType) { + this.unchanged = true; + this.partitionType = partitionType; + this.partitionCounts = collectMetadata ? new PartitionCounts() : null; + } + + private void collect( + GenericRow record, + CompactionKey key, + CompactionFilter filter, + PartitionDictionary partitions, + boolean deferDeletedAddCheck) { + if (!unchanged) { + return; + } + if (!filter.copyable(record, key, deferDeletedAddCheck)) { + unchanged = false; + partitionCounts = null; + return; + } + if (partitionCounts == null) { + return; + } + + InternalRow file = entryFile(record); + if (key.kind == FileKind.ADD.toByteValue()) { + addedFiles++; + } else { + deletedFiles++; + } + schemaId = Math.max(schemaId, file.getLong(SCHEMA_ID)); + int bucket = record.getInt(BUCKET); + minBucket = Math.min(minBucket, bucket); + maxBucket = Math.max(maxBucket, bucket); + int level = file.getInt(LEVEL); + minLevel = Math.min(minLevel, level); + maxLevel = Math.max(maxLevel, level); + minRowId = Math.min(minRowId, key.firstRowId); + maxRowId = Math.max(maxRowId, key.rangeEnd); + partitionCounts.add(partitions.id(record.getBinary(PARTITION))); + } + + private void finish( + SimpleStatsConverter partitionStatsConverter, PartitionDictionary partitions) { + if (!unchanged || partitionCounts == null) { + return; + } + + SimpleStatsCollector collector = new SimpleStatsCollector(partitionType); + long[] nullCounts = new long[partitionType.getFieldCount()]; + partitionCounts.collect(partitions, collector, nullCounts); + SimpleColStats[] stats = collector.extract(); + for (int field = 0; field < stats.length; field++) { + stats[field] = + new SimpleColStats( + stats[field].min(), stats[field].max(), nullCounts[field]); + } + metadata = + new EncodedBlock( + addedFiles, + deletedFiles, + schemaId, + minBucket, + maxBucket, + minLevel, + maxLevel, + minRowId, + maxRowId, + partitionStatsConverter.toBinaryAllMode(stats)); + partitionCounts = null; + } + + private void finishFiltering(CompactionFilter filter, boolean deferDeletedAddCheck) { + if (deferDeletedAddCheck + && metadata != null + && !filter.canCopyRange(minRowId, maxRowId)) { + metadata = null; + unchanged = false; + } + } + } + + private static final class PlannedBlock { + + private final RawBlock raw; + private final CompactionBlock compaction; + + private PlannedBlock(RawBlock raw, CompactionBlock compaction) { + this.raw = raw; + this.compaction = compaction; + } + } + + private static final class ManifestRewritePlan { + + private final ManifestFileMeta manifest; + private final boolean encodedRecordsCompatible; + private final List blocks; + + private ManifestRewritePlan( + ManifestFileMeta manifest, + boolean encodedRecordsCompatible, + List blocks) { + this.manifest = manifest; + this.encodedRecordsCompatible = encodedRecordsCompatible; + this.blocks = blocks; + } + + private boolean unchanged() { + for (PlannedBlock block : blocks) { + if (!block.compaction.unchanged) { + return false; + } + } + return true; + } + } + + /** Primitive partition-id counts retained only while one Avro block is inspected. */ + private static final class PartitionCounts { + + private static final float LOAD_FACTOR = 0.75f; + + private int[] keys = new int[16]; + private long[] counts = new long[16]; + private int size; + + private PartitionCounts() { + Arrays.fill(keys, -1); + } + + private void add(int partitionId) { + if (size + 1 > keys.length * LOAD_FACTOR) { + resize(); + } + int mask = keys.length - 1; + int slot = mix(partitionId) & mask; + while (true) { + int existing = keys[slot]; + if (existing == -1) { + keys[slot] = partitionId; + counts[slot] = 1; + size++; + return; + } + if (existing == partitionId) { + counts[slot]++; + return; + } + slot = (slot + 1) & mask; + } + } + + private void collect( + PartitionDictionary partitions, SimpleStatsCollector collector, long[] nullCounts) { + for (int slot = 0; slot < keys.length; slot++) { + if (keys[slot] < 0) { + continue; + } + BinaryRow partition = partitions.partition(keys[slot]); + collector.collect(partition); + for (int field = 0; field < nullCounts.length; field++) { + if (partition.isNullAt(field)) { + nullCounts[field] = Math.addExact(nullCounts[field], counts[slot]); + } + } + } + } + + private void resize() { + int[] previousKeys = keys; + long[] previousCounts = counts; + keys = new int[previousKeys.length << 1]; + counts = new long[keys.length]; + Arrays.fill(keys, -1); + int mask = keys.length - 1; + for (int slot = 0; slot < previousKeys.length; slot++) { + int key = previousKeys[slot]; + if (key < 0) { + continue; + } + int target = mix(key) & mask; + while (keys[target] != -1) { + target = (target + 1) & mask; + } + keys[target] = key; + counts[target] = previousCounts[slot]; + } + } + + private static int mix(int value) { + value ^= value >>> 16; + value *= 0x7feb352d; + value ^= value >>> 15; + value *= 0x846ca68b; + return value ^ (value >>> 16); + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileLegacyMerger.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileLegacyMerger.java new file mode 100644 index 000000000000..12c633252634 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileLegacyMerger.java @@ -0,0 +1,290 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.operation; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.manifest.FileEntry; +import org.apache.paimon.manifest.ManifestAvroWriter; +import org.apache.paimon.manifest.ManifestEntry; +import org.apache.paimon.manifest.ManifestFile; +import org.apache.paimon.manifest.ManifestFileMeta; +import org.apache.paimon.partition.PartitionPredicate; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.Filter; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; + +import static java.util.Collections.singletonList; +import static org.apache.paimon.utils.ManifestReadThreadPool.sequentialBatchedExecute; +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Legacy full-entry manifest merger used when optimized manifest merging is disabled. */ +final class ManifestFileLegacyMerger { + + private static final Logger LOG = LoggerFactory.getLogger(ManifestFileLegacyMerger.class); + + private ManifestFileLegacyMerger() {} + + static List merge( + List input, + List newFilesForAbort, + ManifestFile manifestFile, + RowType partitionType, + CoreOptions options) + throws Exception { + long suggestedMetaSize = options.manifestTargetSize().getBytes(); + Integer manifestReadParallelism = options.scanManifestParallelism(); + Optional> fullCompacted = + tryFullCompaction( + input, + newFilesForAbort, + manifestFile, + suggestedMetaSize, + options.manifestFullCompactionThresholdSize().getBytes(), + partitionType, + manifestReadParallelism); + if (fullCompacted.isPresent()) { + return fullCompacted.get(); + } + return compactMinor( + input, + newFilesForAbort, + manifestFile, + suggestedMetaSize, + options.manifestMergeMinCount(), + manifestReadParallelism); + } + + private static List compactMinor( + List input, + List newFilesForAbort, + ManifestFile manifestFile, + long suggestedMetaSize, + int suggestedMinMetaCount, + @Nullable Integer manifestReadParallelism) { + List result = new ArrayList<>(); + List candidates = new ArrayList<>(); + long totalSize = 0; + for (ManifestFileMeta manifest : input) { + totalSize += manifest.fileSize(); + candidates.add(manifest); + if (totalSize >= suggestedMetaSize) { + mergeCandidates( + candidates, + manifestFile, + result, + newFilesForAbort, + manifestReadParallelism); + candidates.clear(); + totalSize = 0; + } + } + + if (candidates.size() >= suggestedMinMetaCount) { + mergeCandidates( + candidates, manifestFile, result, newFilesForAbort, manifestReadParallelism); + } else { + result.addAll(candidates); + } + return result; + } + + private static void mergeCandidates( + List candidates, + ManifestFile manifestFile, + List result, + List newMetas, + @Nullable Integer manifestReadParallelism) { + if (candidates.size() == 1) { + result.add(candidates.get(0)); + return; + } + + Map map = new LinkedHashMap<>(); + FileEntry.mergeEntries(manifestFile, candidates, map, manifestReadParallelism); + if (!map.isEmpty()) { + List merged = manifestFile.write(new ArrayList<>(map.values())); + result.addAll(merged); + newMetas.addAll(merged); + } + } + + static Optional> tryFullCompaction( + List inputs, + List newFilesForAbort, + ManifestFile manifestFile, + long suggestedMetaSize, + long sizeTrigger, + RowType partitionType, + @Nullable Integer manifestReadParallelism) + throws Exception { + checkArgument(sizeTrigger > 0, "Manifest full compaction size trigger cannot be zero."); + + Filter mustChange = + file -> file.numDeletedFiles() > 0 || file.fileSize() < suggestedMetaSize; + long totalManifestSize = 0; + long deltaDeleteFileNum = 0; + long totalDeltaFileSize = 0; + for (ManifestFileMeta file : inputs) { + totalManifestSize += file.fileSize(); + if (mustChange.test(file)) { + totalDeltaFileSize += file.fileSize(); + deltaDeleteFileNum += file.numDeletedFiles(); + } + } + + if (totalDeltaFileSize < sizeTrigger) { + return Optional.empty(); + } + + LOG.info( + "Start Legacy Manifest File Full Compaction: totalManifestSize: {}, deltaDeleteFileNum {}, totalDeltaFileSize {}", + totalManifestSize, + deltaDeleteFileNum, + totalDeltaFileSize); + + Set deleteEntries = + FileEntry.readDeletedEntries(manifestFile, inputs, manifestReadParallelism); + + PartitionPredicate predicate; + if (deleteEntries.isEmpty()) { + predicate = PartitionPredicate.ALWAYS_FALSE; + } else if (partitionType.getFieldCount() > 0) { + predicate = + PartitionPredicate.fromMultiple( + partitionType, computeDeletePartitions(deleteEntries)); + } else { + predicate = PartitionPredicate.ALWAYS_TRUE; + } + + List result = new ArrayList<>(); + List toBeMerged = new LinkedList<>(inputs); + if (predicate != null) { + Iterator iterator = toBeMerged.iterator(); + while (iterator.hasNext()) { + ManifestFileMeta file = iterator.next(); + if (mustChange.test(file)) { + continue; + } + if (!predicate.test( + file.numAddedFiles() + file.numDeletedFiles(), + file.partitionStats().minValues(), + file.partitionStats().maxValues(), + file.partitionStats().nullCounts())) { + iterator.remove(); + result.add(file); + } + } + } + + if (toBeMerged.size() <= 1) { + return Optional.empty(); + } + + ManifestAvroWriter writer = manifestFile.createAvroWriter(); + Function> reader = + file -> + singletonList( + readForFullCompaction( + file, manifestFile, mustChange, deleteEntries)); + Exception exception = null; + try { + for (FullCompactionReadResult readResult : + sequentialBatchedExecute(reader, toBeMerged, manifestReadParallelism)) { + if (readResult.requireChange) { + writer.write(readResult.entries); + } else { + result.add(readResult.file); + } + } + } catch (Exception e) { + exception = e; + } finally { + if (exception != null) { + writer.abort(); + throw exception; + } + writer.close(); + } + + List merged = writer.result(); + result.addAll(merged); + newFilesForAbort.addAll(merged); + return Optional.of(result); + } + + private static FullCompactionReadResult readForFullCompaction( + ManifestFileMeta file, + ManifestFile manifestFile, + Filter mustChange, + Set deleteEntries) { + List entries = new ArrayList<>(); + boolean requireChange = mustChange.test(file); + for (ManifestEntry entry : + manifestFile.read( + file.fileName(), + file.fileSize(), + FileEntry.addFilter(), + Filter.alwaysTrue())) { + if (deleteEntries.contains(entry.identifier())) { + requireChange = true; + } else { + entries.add(entry); + } + } + return new FullCompactionReadResult(file, requireChange, entries); + } + + private static Set computeDeletePartitions(Set deleteEntries) { + Set partitions = new HashSet<>(); + for (FileEntry.Identifier identifier : deleteEntries) { + partitions.add(identifier.partition); + } + return partitions; + } + + private static final class FullCompactionReadResult { + + private final ManifestFileMeta file; + private final boolean requireChange; + private final List entries; + + private FullCompactionReadResult( + ManifestFileMeta file, boolean requireChange, List entries) { + this.file = file; + this.requireChange = requireChange; + this.entries = entries; + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileMerger.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileMerger.java index 7c5019f1e33c..7c2dee8b4672 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileMerger.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileMerger.java @@ -19,43 +19,22 @@ package org.apache.paimon.operation; import org.apache.paimon.CoreOptions; -import org.apache.paimon.data.BinaryRow; import org.apache.paimon.disk.IOManager; -import org.apache.paimon.manifest.FileEntry; -import org.apache.paimon.manifest.ManifestAvroWriter; import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.manifest.ManifestFile; import org.apache.paimon.manifest.ManifestFileMeta; -import org.apache.paimon.partition.PartitionPredicate; import org.apache.paimon.types.RowType; -import org.apache.paimon.utils.Filter; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import javax.annotation.Nullable; import java.util.ArrayList; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.LinkedList; import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.function.Function; -import static java.util.Collections.singletonList; import static org.apache.paimon.manifest.ManifestFileMeta.allContainsRowId; -import static org.apache.paimon.utils.ManifestReadThreadPool.sequentialBatchedExecute; -import static org.apache.paimon.utils.Preconditions.checkArgument; /** Manifest file merger with standard merge logic and optional sort rewrite. */ public class ManifestFileMerger { - private static final Logger LOG = LoggerFactory.getLogger(ManifestFileMerger.class); - /** * Merge several {@link ManifestFileMeta}s. {@link ManifestEntry}s representing first adding and * then deleting the same data file will cancel each other. @@ -76,12 +55,6 @@ public static List merge( RowType partitionType, CoreOptions options, @Nullable IOManager ioManager) { - // Extract configuration from options - long suggestedMetaSize = options.manifestTargetSize().getBytes(); - int suggestedMinMetaCount = options.manifestMergeMinCount(); - long manifestFullCompactionSize = options.manifestFullCompactionThresholdSize().getBytes(); - Integer manifestReadParallelism = options.scanManifestParallelism(); - // these are the newly created manifest files, clean them up if exception occurs List newFilesForAbort = new ArrayList<>(); @@ -94,27 +67,14 @@ public static List merge( || (options.dataEvolutionEnabled() && allContainsRowId(input)))) { return ManifestFileSorter.trySortCompaction( input, newFilesForAbort, manifestFile, partitionType, options, ioManager); - } else { - // Otherwise try full compaction first, then minor compaction if needed - Optional> fullCompacted = - tryFullCompaction( - input, - newFilesForAbort, - manifestFile, - suggestedMetaSize, - manifestFullCompactionSize, - partitionType, - manifestReadParallelism); - return fullCompacted.orElseGet( - () -> - tryMinorCompaction( - input, - newFilesForAbort, - manifestFile, - suggestedMetaSize, - suggestedMinMetaCount, - manifestReadParallelism)); } + + if (options.manifestMergeOptimizeEnabled()) { + return ManifestFileBlockMerger.merge( + input, newFilesForAbort, manifestFile, partitionType, options); + } + return ManifestFileLegacyMerger.merge( + input, newFilesForAbort, manifestFile, partitionType, options); } catch (Throwable e) { // exception occurs, clean up and rethrow for (ManifestFileMeta manifest : newFilesForAbort) { @@ -123,221 +83,4 @@ public static List merge( throw new RuntimeException(e); } } - - private static List tryMinorCompaction( - List input, - List newFilesForAbort, - ManifestFile manifestFile, - long suggestedMetaSize, - int suggestedMinMetaCount, - @Nullable Integer manifestReadParallelism) { - List result = new ArrayList<>(); - List candidates = new ArrayList<>(); - long totalSize = 0; - // merge existing small manifest files - for (ManifestFileMeta manifest : input) { - totalSize += manifest.fileSize(); - candidates.add(manifest); - if (totalSize >= suggestedMetaSize) { - // reach suggested file size, perform merging and produce new file - mergeCandidates( - candidates, - manifestFile, - result, - newFilesForAbort, - manifestReadParallelism); - candidates.clear(); - totalSize = 0; - } - } - - // merge the last bit of manifests if there are too many - if (candidates.size() >= suggestedMinMetaCount) { - mergeCandidates( - candidates, manifestFile, result, newFilesForAbort, manifestReadParallelism); - } else { - result.addAll(candidates); - } - return result; - } - - private static void mergeCandidates( - List candidates, - ManifestFile manifestFile, - List result, - List newMetas, - @Nullable Integer manifestReadParallelism) { - if (candidates.size() == 1) { - result.add(candidates.get(0)); - return; - } - - Map map = new LinkedHashMap<>(); - FileEntry.mergeEntries(manifestFile, candidates, map, manifestReadParallelism); - if (!map.isEmpty()) { - List merged = manifestFile.write(new ArrayList<>(map.values())); - result.addAll(merged); - newMetas.addAll(merged); - } - } - - public static Optional> tryFullCompaction( - List inputs, - List newFilesForAbort, - ManifestFile manifestFile, - long suggestedMetaSize, - long sizeTrigger, - RowType partitionType, - @Nullable Integer manifestReadParallelism) - throws Exception { - checkArgument(sizeTrigger > 0, "Manifest full compaction size trigger cannot be zero."); - - // 1. should trigger full compaction - - Filter mustChange = - file -> file.numDeletedFiles() > 0 || file.fileSize() < suggestedMetaSize; - long totalManifestSize = 0; - long deltaDeleteFileNum = 0; - long totalDeltaFileSize = 0; - for (ManifestFileMeta file : inputs) { - totalManifestSize += file.fileSize(); - if (mustChange.test(file)) { - totalDeltaFileSize += file.fileSize(); - deltaDeleteFileNum += file.numDeletedFiles(); - } - } - - if (totalDeltaFileSize < sizeTrigger) { - return Optional.empty(); - } - - // 2. do full compaction - - LOG.info( - "Start Manifest File Full Compaction: totalManifestSize: {}, deltaDeleteFileNum {}, totalDeltaFileSize {}", - totalManifestSize, - deltaDeleteFileNum, - totalDeltaFileSize); - - // 2.1. read all delete entries - - Set deleteEntries = - FileEntry.readDeletedEntries(manifestFile, inputs, manifestReadParallelism); - - // 2.2. try to skip base files by partition filter - - PartitionPredicate predicate; - if (deleteEntries.isEmpty()) { - predicate = PartitionPredicate.ALWAYS_FALSE; - } else { - if (partitionType.getFieldCount() > 0) { - Set deletePartitions = computeDeletePartitions(deleteEntries); - predicate = PartitionPredicate.fromMultiple(partitionType, deletePartitions); - } else { - predicate = PartitionPredicate.ALWAYS_TRUE; - } - } - - List result = new ArrayList<>(); - List toBeMerged = new LinkedList<>(inputs); - - if (predicate != null) { - Iterator iterator = toBeMerged.iterator(); - while (iterator.hasNext()) { - ManifestFileMeta file = iterator.next(); - if (mustChange.test(file)) { - continue; - } - if (!predicate.test( - file.numAddedFiles() + file.numDeletedFiles(), - file.partitionStats().minValues(), - file.partitionStats().maxValues(), - file.partitionStats().nullCounts())) { - iterator.remove(); - result.add(file); - } - } - } - - // 2.2. merge - if (toBeMerged.size() <= 1) { - return Optional.empty(); - } - - ManifestAvroWriter writer = manifestFile.createAvroWriter(); - Function> reader = - file -> - singletonList( - readForFullCompaction( - file, manifestFile, mustChange, deleteEntries)); - Exception exception = null; - try { - for (FullCompactionReadResult readResult : - sequentialBatchedExecute(reader, toBeMerged, manifestReadParallelism)) { - if (readResult.requireChange) { - writer.write(readResult.entries); - } else { - result.add(readResult.file); - } - } - } catch (Exception e) { - exception = e; - } finally { - if (exception != null) { - writer.abort(); - throw exception; - } - writer.close(); - } - - List merged = writer.result(); - result.addAll(merged); - newFilesForAbort.addAll(merged); - return Optional.of(result); - } - - private static FullCompactionReadResult readForFullCompaction( - ManifestFileMeta file, - ManifestFile manifestFile, - Filter mustChange, - Set deleteEntries) { - List entries = new ArrayList<>(); - boolean requireChange = mustChange.test(file); - for (ManifestEntry entry : - manifestFile.read( - file.fileName(), - file.fileSize(), - FileEntry.addFilter(), - Filter.alwaysTrue())) { - if (deleteEntries.contains(entry.identifier())) { - requireChange = true; - } else { - entries.add(entry); - } - } - - return new FullCompactionReadResult(file, requireChange, entries); - } - - static Set computeDeletePartitions(Set deleteEntries) { - Set partitions = new HashSet<>(); - for (FileEntry.Identifier identifier : deleteEntries) { - partitions.add(identifier.partition); - } - return partitions; - } - - static class FullCompactionReadResult { - - final ManifestFileMeta file; - final boolean requireChange; - final List entries; - - FullCompactionReadResult( - ManifestFileMeta file, boolean requireChange, List entries) { - this.file = file; - this.requireChange = requireChange; - this.entries = entries; - } - } } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java index d109ebb76909..574286771fb8 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java @@ -72,6 +72,7 @@ static class CompactionContext { final boolean fullCompaction; final boolean runMergeOptimizeEnabled; final ManifestSortKey sortKey; + final RowType partitionType; final ManifestEntryExternalSort.ExternalSortConfig externalSortConfig; final CompactFileIdentifierSet deleteEntries; final DeletedRowIdSet deletedRowIds; @@ -92,6 +93,7 @@ static class CompactionContext { boolean fullCompaction, boolean runMergeOptimizeEnabled, ManifestSortKey sortKey, + RowType partitionType, ManifestEntryExternalSort.ExternalSortConfig externalSortConfig, CompactFileIdentifierSet deleteEntries, DeletedRowIdSet deletedRowIds, @@ -101,6 +103,7 @@ static class CompactionContext { this.fullCompaction = fullCompaction; this.runMergeOptimizeEnabled = runMergeOptimizeEnabled; this.sortKey = sortKey; + this.partitionType = partitionType; this.externalSortConfig = externalSortConfig; this.deleteEntries = deleteEntries; this.deletedRowIds = deletedRowIds; @@ -239,6 +242,11 @@ private long[] sortedRowIds() { return values; } + void prepareRangeIndex() { + // Publish the immutable sorted snapshot before concurrent manifest planning starts. + sortedRowIds(); + } + void releaseRangeIndex() { sortedRowIds = null; } @@ -291,7 +299,7 @@ static List trySortCompaction( @Nullable IOManager ioManager) throws Exception { String sortPartitionField = options.manifestSortPartitionField(); - boolean runMergeOptimizeEnabled = options.manifestSortRunMergeOptimizeEnabled(); + boolean runMergeOptimizeEnabled = options.manifestMergeOptimizeEnabled(); long suggestedMetaSize = options.manifestTargetSize().getBytes(); int suggestedMinMetaCount = options.manifestMergeMinCount(); long fullCompactionThreshold = options.manifestFullCompactionThresholdSize().getBytes(); @@ -625,6 +633,7 @@ private static CompactionContext prepareCompaction( fullCompaction, useRunMergeOptimize, sortKey, + partitionType, externalSortConfig, classification.deleteEntries, classification.deletedRowIds, @@ -1240,6 +1249,7 @@ private static void rewriteFull( ManifestEntryRunMerge.sortAndWriteFullEntries( section, (RowIdEntrySortKey) ctx.sortKey, + ctx.partitionType, manifestFile, sortNewFiles, ctx.deleteEntries, @@ -1281,6 +1291,7 @@ private static void rewriteMinor( ManifestEntryRunMerge.sortAndWriteMinorEntries( section, (RowIdEntrySortKey) ctx.sortKey, + ctx.partitionType, manifestFile, sortNewFiles, manifestReadParallelism); diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java index 9fa1edb7615e..02bdde74ca26 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java @@ -27,7 +27,6 @@ import org.apache.paimon.fs.FileIOFinder; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.SeekableInputStream; -import org.apache.paimon.fs.SeekableInputStreamWrapper; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.operation.ManifestFileMerger; @@ -65,15 +64,13 @@ import java.util.TreeSet; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.LongStream; +import static org.apache.paimon.operation.ManifestFileMergerTestUtils.tryFullCompaction; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -239,6 +236,151 @@ public void testMergeWithoutDelta() { assertEquivalentEntries(input1, merged1); } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testAddOnlyCompactionCopiesRawBlocks(boolean fullCompaction) throws Exception { + List input = + Arrays.asList( + makeManifest( + makeRowIdEntry(true, "a", null, 0, 1), makeEntry(true, "b", 1)), + makeManifest(makeEntry(true, "c", 7)), + makeManifest(makeEntry(true, "d", 5), makeEntry(true, "e", 9))); + int inputBlocks = 0; + for (ManifestFileMeta meta : input) { + inputBlocks += rawBlockCount(manifestFile, meta); + } + + Options testOptions = new Options(); + testOptions.set("manifest.target-file-size", "1MB"); + testOptions.set("manifest.merge-min-count", "2"); + testOptions.set( + "manifest.full-compaction-threshold-size", + fullCompaction ? "1B" : Long.MAX_VALUE + "B"); + List merged = + ManifestFileMerger.merge( + input, + manifestFile, + getPartitionType(), + CoreOptions.fromMap(testOptions.toMap())); + + assertThat(merged).hasSize(1); + assertEquivalentEntries(input, merged); + ManifestFileMeta output = merged.get(0); + assertThat(rawBlockCount(manifestFile, output)).isEqualTo(inputBlocks); + assertThat(output.numAddedFiles()).isEqualTo(5); + assertThat(output.numDeletedFiles()).isZero(); + assertThat(output.partitionStats().minValues().getInt(0)).isEqualTo(1); + assertThat(output.partitionStats().maxValues().getInt(0)).isEqualTo(9); + assertThat(output.partitionStats().nullCounts().getLong(0)).isEqualTo(1); + assertThat(output.minRowId()).isNull(); + assertThat(output.maxRowId()).isNull(); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testCompactionCopiesUnaffectedBlocksAroundDeletes(boolean fullCompaction) + throws Exception { + List input = + Arrays.asList( + makeManifest( + makeRowIdEntry(true, "deleted", 0, 0, 5), + makeRowIdEntry(true, "survivor-10", 0, 10, 5)), + makeManifest( + makeRowIdEntry(true, "survivor-100", 9, 100, 5), + makeRowIdEntry(true, "survivor-120", 1, 120, 5)), + makeManifest(makeRowIdEntry(false, "deleted", 0, 0, 5)), + makeManifest(makeRowIdEntry(true, "survivor-300", 0, 300, 5))); + assertThat(input.get(2).numDeletedFiles()).isEqualTo(1); + assertThat(ManifestFileMeta.allContainsRowId(input)).isTrue(); + + Options testOptions = new Options(); + testOptions.set("manifest.target-file-size", "1MB"); + testOptions.set("manifest.merge-min-count", "2"); + testOptions.set("scan.manifest.parallelism", "2"); + testOptions.set( + "manifest.full-compaction-threshold-size", + fullCompaction ? "1B" : Long.MAX_VALUE + "B"); + List merged = + ManifestFileMerger.merge( + input, + manifestFile, + getPartitionType(), + CoreOptions.fromMap(testOptions.toMap())); + + assertThat(merged).hasSize(1); + assertEquivalentEntries(input, merged); + ManifestFileMeta output = merged.get(0); + assertThat(rawBlockCount(manifestFile, output)).isEqualTo(3); + assertThat(output.numAddedFiles()).isEqualTo(4); + assertThat(output.numDeletedFiles()).isZero(); + assertThat(output.partitionStats().minValues().getInt(0)).isZero(); + assertThat(output.partitionStats().maxValues().getInt(0)).isEqualTo(9); + assertThat(output.minRowId()).isEqualTo(10); + assertThat(output.maxRowId()).isEqualTo(304); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testCompactionWithoutRowIdFiltersDeletes(boolean fullCompaction) { + List input = + Arrays.asList( + makeManifest(makeEntry(true, "deleted", 0), makeEntry(true, "survivor", 0)), + makeManifest(makeEntry(true, "other", 1)), + makeManifest(makeEntry(false, "deleted", 0))); + assertThat(ManifestFileMeta.allContainsRowId(input)).isFalse(); + + Options testOptions = new Options(); + testOptions.set("manifest.target-file-size", "1MB"); + testOptions.set("manifest.merge-min-count", "2"); + testOptions.set("scan.manifest.parallelism", "2"); + testOptions.set( + "manifest.full-compaction-threshold-size", + fullCompaction ? "1B" : Long.MAX_VALUE + "B"); + List merged = + ManifestFileMerger.merge( + input, + manifestFile, + getPartitionType(), + CoreOptions.fromMap(testOptions.toMap())); + + assertThat(merged).hasSize(1); + assertEquivalentEntries(input, merged); + ManifestFileMeta output = merged.get(0); + assertThat(output.numAddedFiles()).isEqualTo(2); + assertThat(output.numDeletedFiles()).isZero(); + assertThat(output.minRowId()).isNull(); + assertThat(output.maxRowId()).isNull(); + } + + @Test + public void testDisablingManifestMergeOptimizeUsesLegacyMerger() throws Exception { + List input = + Arrays.asList( + makeManifest(makeEntry(true, "a", 0)), + makeManifest(makeEntry(true, "b", 1)), + makeManifest(makeEntry(true, "c", 2))); + int inputBlocks = 0; + for (ManifestFileMeta manifest : input) { + inputBlocks += rawBlockCount(manifestFile, manifest); + } + + Options testOptions = new Options(); + testOptions.set("manifest.merge-optimize.enabled", "false"); + testOptions.set("manifest.target-file-size", "1MB"); + testOptions.set("manifest.merge-min-count", "2"); + testOptions.set("manifest.full-compaction-threshold-size", Long.MAX_VALUE + "B"); + List merged = + ManifestFileMerger.merge( + input, + manifestFile, + getPartitionType(), + CoreOptions.fromMap(testOptions.toMap())); + + assertThat(merged).hasSize(1); + assertEquivalentEntries(input, merged); + assertThat(rawBlockCount(manifestFile, merged.get(0))).isLessThan(inputBlocks); + } + @Test public void testMergeWithoutBase() { List input = new ArrayList<>(); @@ -324,7 +466,7 @@ public void testTriggerFullCompaction() throws Exception { Arrays.asList(manifest1, manifest2, manifest3, manifest4, manifest5, manifest6)); List newMetas1 = new ArrayList<>(); Optional> fullCompacted1 = - ManifestFileMerger.tryFullCompaction( + tryFullCompaction( input, newMetas1, manifestFile, @@ -340,7 +482,7 @@ public void testTriggerFullCompaction() throws Exception { input.add(manifest7); List newMetas2 = new ArrayList<>(); Optional> fullCompacted2 = - ManifestFileMerger.tryFullCompaction( + tryFullCompaction( input, newMetas1, manifestFile, 500, 100, getPartitionType(), null); assertThat(fullCompacted2).isEmpty(); assertThat(newMetas2).isEmpty(); @@ -351,7 +493,7 @@ public void testTriggerFullCompaction() throws Exception { input.addAll(Arrays.asList(manifest1, manifest2, manifest3)); List newMetas3 = new ArrayList<>(); Optional> fullCompacted3 = - ManifestFileMerger.tryFullCompaction( + tryFullCompaction( input, newMetas3, manifestFile, 500, 100, getPartitionType(), null); assertThat(fullCompacted3).isEmpty(); assertThat(newMetas3).isEmpty(); @@ -362,7 +504,7 @@ public void testTriggerFullCompaction() throws Exception { input.addAll(Arrays.asList(manifest1, manifest2, manifest3)); List newMetas4 = new ArrayList<>(); List fullCompacted4 = - ManifestFileMerger.tryFullCompaction( + tryFullCompaction( input, newMetas4, manifestFile, 5000, 100, getPartitionType(), null) .get(); assertThat(fullCompacted4.size()).isEqualTo(1); @@ -374,7 +516,7 @@ public void testTriggerFullCompaction() throws Exception { input.addAll(Arrays.asList(manifest1, manifest2, manifest3, manifest4)); List newMetas5 = new ArrayList<>(); List fullCompacted5 = - ManifestFileMerger.tryFullCompaction( + tryFullCompaction( input, newMetas5, manifestFile, 1800, 100, getPartitionType(), null) .get(); assertThat(fullCompacted5.size()).isEqualTo(3); @@ -390,7 +532,7 @@ public void testTriggerFullCompaction() throws Exception { manifest7)); List newMetas6 = new ArrayList<>(); List fullCompacted6 = - ManifestFileMerger.tryFullCompaction( + tryFullCompaction( input, newMetas6, manifestFile, 500, 100, getPartitionType(), null) .get(); @@ -405,7 +547,7 @@ public void testTriggerFullCompaction() throws Exception { assertThrows( IllegalArgumentException.class, () -> { - ManifestFileMerger.tryFullCompaction( + tryFullCompaction( input, newMetas7, manifestFile, 500, 0, getPartitionType(), null); }); @@ -415,7 +557,7 @@ public void testTriggerFullCompaction() throws Exception { assertThrows( Exception.class, () -> { - ManifestFileMerger.tryFullCompaction( + tryFullCompaction( input, newMetas8, manifestFile, 500, 100, getPartitionType(), null); }); assertThat(newMetas8).isEmpty(); @@ -431,8 +573,7 @@ public void testMultiPartitionsFullCompaction() throws Exception { List newMetas = new ArrayList<>(); List mergedManifest = - ManifestFileMerger.tryFullCompaction( - input, newMetas, manifestFile, 500, 100, getPartitionType(), null) + tryFullCompaction(input, newMetas, manifestFile, 500, 100, getPartitionType(), null) .get(); List expected = Lists.newArrayList("ADD-C2", "ADD-D2", "ADD-G"); @@ -505,8 +646,7 @@ public void testIdentifierAfterFullCompaction() throws Exception { new ArrayList<>(Arrays.asList(manifest1, manifest2, manifest3, manifest4)); List newMetas = new ArrayList<>(); List fullCompacted = - ManifestFileMerger.tryFullCompaction( - input, newMetas, manifestFile, 500, 100, getPartitionType(), null) + tryFullCompaction(input, newMetas, manifestFile, 500, 100, getPartitionType(), null) .get(); assertThat(fullCompacted.size()).isEqualTo(1); assertThat(newMetas.size()).isEqualTo(1); @@ -568,8 +708,8 @@ public void testMergeFullCompactionWithoutDeleteFile() { } @Test - public void testFullCompactionReadManifestsInParallel() throws Exception { - BlockingReadFileIO fileIO = new BlockingReadFileIO(); + public void testFullCompactionReadsSelectedManifestsOnce() throws Exception { + CountingReadFileIO fileIO = new CountingReadFileIO(); manifestFile = createManifestFile(tempDir.toString(), fileIO); List input = new ArrayList<>(); @@ -577,28 +717,74 @@ public void testFullCompactionReadManifestsInParallel() throws Exception { input.add(makeManifest(makeEntry(true, "parallel-" + i))); } - List newMetas = new ArrayList<>(); - Optional> fullCompacted; - fileIO.blockManifestReads(); - try { - fullCompacted = - ManifestFileMerger.tryFullCompaction( - input, - newMetas, - manifestFile, - Long.MAX_VALUE, - 1, - getPartitionType(), - 2); - } finally { - fileIO.stopBlockingManifestReads(); - } + fileIO.resetReadCounts(); + Optional> fullCompacted = + tryFullCompaction( + input, + new ArrayList<>(), + manifestFile, + Long.MAX_VALUE, + 1, + getPartitionType(), + 2); - assertThat(fileIO.maxConcurrentManifestReads()).isGreaterThanOrEqualTo(2); assertThat(fullCompacted).isPresent(); + for (ManifestFileMeta manifest : input) { + assertThat(fileIO.readCount(manifest.fileName())).isEqualTo(1); + } assertEquivalentEntries(input, fullCompacted.get()); } + @Test + public void testFullCompactionReadsDeleteCandidateOnce() throws Exception { + CountingReadFileIO fileIO = new CountingReadFileIO(); + manifestFile = createManifestFile(tempDir.toString(), fileIO); + ManifestFileMeta base = + makeManifest(makeEntry(true, "deleted", 0), makeEntry(true, "survivor", 0)); + ManifestFileMeta delta = makeManifest(makeEntry(false, "deleted", 0)); + fileIO.resetReadCounts(); + + Optional> fullCompacted = + tryFullCompaction( + Arrays.asList(base, delta), + new ArrayList<>(), + manifestFile, + 1, + 1, + getPartitionType(), + null); + + assertThat(fullCompacted).isPresent(); + assertThat(fileIO.readCount(base.fileName())).isEqualTo(1); + assertThat(fileIO.readCount(delta.fileName())).isEqualTo(2); + assertThat(readEntries(fullCompacted.get()).stream().map(entry -> entry.file().fileName())) + .containsExactly("survivor"); + } + + @Test + public void testFullCompactionRewritesOptionalManifestWhenDeleteHitsBlockRange() + throws Exception { + ManifestFileMeta base = + makeManifest( + makeRowIdEntry(true, "deleted", 0, 0, 5), + makeRowIdEntry(true, "survivor", 0, 10, 5)); + ManifestFileMeta delta = makeManifest(makeRowIdEntry(false, "deleted", 0, 0, 5)); + + List fullCompacted = + tryFullCompaction( + Arrays.asList(base, delta), + new ArrayList<>(), + manifestFile, + 1, + 1, + getPartitionType(), + null) + .get(); + + assertThat(readEntries(fullCompacted).stream().map(entry -> entry.file().fileName())) + .containsExactly("survivor"); + } + @RepeatedTest(10) public void testRandomFullCompaction() throws Exception { List input = new ArrayList<>(); @@ -617,7 +803,7 @@ public void testRandomFullCompaction() throws Exception { int sizeTrigger = ThreadLocalRandom.current().nextInt(40000) + 1; List newMetas = new ArrayList<>(); Optional> fullCompacted = - ManifestFileMerger.tryFullCompaction( + tryFullCompaction( input, newMetas, manifestFile, @@ -798,94 +984,6 @@ ManifestFile getManifestFile() { return manifestFile; } - private static class BlockingReadFileIO extends LocalFileIO { - - private final AtomicBoolean blockManifestReads = new AtomicBoolean(false); - private final AtomicInteger activeManifestReads = new AtomicInteger(0); - private final AtomicInteger maxConcurrentManifestReads = new AtomicInteger(0); - private final CountDownLatch readersReady = new CountDownLatch(2); - private final CountDownLatch releaseReaders = new CountDownLatch(1); - - private void blockManifestReads() { - blockManifestReads.set(true); - } - - private void stopBlockingManifestReads() { - blockManifestReads.set(false); - releaseReaders.countDown(); - } - - private int maxConcurrentManifestReads() { - return maxConcurrentManifestReads.get(); - } - - @Override - public SeekableInputStream newInputStream(Path path) throws IOException { - SeekableInputStream inputStream = super.newInputStream(path); - if (!blockManifestReads.get() || !path.toString().contains("/manifest/")) { - return inputStream; - } - return new BlockingSeekableInputStream(inputStream); - } - - private class BlockingSeekableInputStream extends SeekableInputStreamWrapper { - - private boolean entered; - private boolean closed; - - private BlockingSeekableInputStream(SeekableInputStream inputStream) { - super(inputStream); - } - - @Override - public int read() throws IOException { - beforeFirstRead(); - return super.read(); - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - beforeFirstRead(); - return super.read(b, off, len); - } - - @Override - public void close() throws IOException { - try { - super.close(); - } finally { - if (entered && !closed) { - activeManifestReads.decrementAndGet(); - } - closed = true; - } - } - - private void beforeFirstRead() throws IOException { - if (entered) { - return; - } - - entered = true; - int activeReads = activeManifestReads.incrementAndGet(); - maxConcurrentManifestReads.accumulateAndGet(activeReads, Math::max); - readersReady.countDown(); - if (readersReady.getCount() == 0) { - releaseReaders.countDown(); - } - - try { - if (!releaseReaders.await(3, TimeUnit.SECONDS)) { - throw new IOException("Manifest reads were not parallelized."); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new IOException(e); - } - } - } - } - private static class CountingReadFileIO extends LocalFileIO { private final Map readCounts = new ConcurrentHashMap<>(); @@ -1290,9 +1388,7 @@ public void testDataEvolutionManifestSortByPartitionAndRowId() { @Test public void testDisablingRunMergeOptimizePreservesDataEvolutionRowIdSort() { - assertThat( - CoreOptions.fromMap(Collections.emptyMap()) - .manifestSortRunMergeOptimizeEnabled()) + assertThat(CoreOptions.fromMap(Collections.emptyMap()).manifestMergeOptimizeEnabled()) .isTrue(); List input = @@ -1306,12 +1402,12 @@ public void testDisablingRunMergeOptimizePreservesDataEvolutionRowIdSort() { Options testOptions = new Options(); testOptions.set("manifest-sort.enabled", "true"); - testOptions.set("manifest-sort.run-merge-optimize.enabled", "false"); + testOptions.set("manifest.merge-optimize.enabled", "false"); testOptions.set("data-evolution.enabled", "true"); testOptions.set("manifest.full-compaction-threshold-size", "1B"); CoreOptions coreOptions = CoreOptions.fromMap(testOptions.toMap()); - assertThat(coreOptions.manifestSortRunMergeOptimizeEnabled()).isFalse(); + assertThat(coreOptions.manifestMergeOptimizeEnabled()).isFalse(); List merged = ManifestFileMerger.merge(input, manifestFile, getPartitionType(), coreOptions); @@ -1669,7 +1765,8 @@ public void testDataEvolutionManifestRunMergePreservesBlockStats() { } @Test - public void testDataEvolutionManifestSortUsesConfiguredPartitionFieldBeforeRowId() { + public void testDataEvolutionManifestSortUsesConfiguredPartitionFieldBeforeRowId() + throws Exception { RowType multiPartitionType = RowType.of(new IntType(), new IntType(), new IntType()); ManifestFile multiPartManifestFile = createManifestFileForPartitionType(multiPartitionType); @@ -1703,12 +1800,23 @@ public void testDataEvolutionManifestSortUsesConfiguredPartitionFieldBeforeRowId multiPartManifestFile, multiPartitionType, CoreOptions.fromMap(defaultOptions.toMap())); + assertThat(sortedByFullPartition).hasSize(1); assertThat(readFileNames(multiPartManifestFile, sortedByFullPartition)) .containsExactly( "region0-dt1-row20", "region5-dt2-row10", "region10-dt2-row30", "region20-dt1-row5"); + ManifestFileMeta fullPartitionOutput = sortedByFullPartition.get(0); + assertThat(rawBlockCount(multiPartManifestFile, fullPartitionOutput)).isEqualTo(2); + assertThat(fullPartitionOutput.partitionStats().minValues().getInt(0)).isZero(); + assertThat(fullPartitionOutput.partitionStats().maxValues().getInt(0)).isEqualTo(20); + assertThat(fullPartitionOutput.partitionStats().minValues().getInt(1)).isEqualTo(1); + assertThat(fullPartitionOutput.partitionStats().maxValues().getInt(1)).isEqualTo(2); + assertThat(fullPartitionOutput.partitionStats().minValues().getInt(2)).isZero(); + assertThat(fullPartitionOutput.partitionStats().maxValues().getInt(2)).isZero(); + assertThat(fullPartitionOutput.partitionStats().nullCounts().toLongArray()) + .containsExactly(0L, 0L, 0L); Options configuredFieldOptions = new Options(); configuredFieldOptions.set("manifest-sort.enabled", "true"); @@ -1885,9 +1993,7 @@ private List mergeMinorManifestEntries( List input, boolean runMergeOptimizeEnabled) { Options options = new Options(); options.set("manifest-sort.enabled", "true"); - options.set( - "manifest-sort.run-merge-optimize.enabled", - Boolean.toString(runMergeOptimizeEnabled)); + options.set("manifest.merge-optimize.enabled", Boolean.toString(runMergeOptimizeEnabled)); options.set("data-evolution.enabled", "true"); options.set("manifest.full-compaction-threshold-size", Long.MAX_VALUE + "B"); return ManifestFileMerger.merge( @@ -2486,4 +2592,16 @@ private List readEntries(List manifestMetas) { } return entries; } + + private int rawBlockCount(ManifestFile manifestFile, ManifestFileMeta meta) throws Exception { + int blocks = 0; + try (ManifestAvroReader reader = + manifestFile.scanAvroBlocks(meta.fileName(), meta.fileSize())) { + while (reader.hasNext()) { + reader.next(); + blocks++; + } + } + return blocks; + } } diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java index 3bd052ab7ad3..fd80db719291 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java @@ -184,6 +184,84 @@ void testWriteEncodedRecords() throws Exception { assertThat(manifestFile.read(result.fileName())).containsExactlyElementsOf(entries); } + @Test + void testWriteEncodedRecordsFlushesPartitionStatsBuffer() throws Exception { + ManifestEntry generated = gen.next(); + ManifestEntry source = + ManifestEntry.create( + generated.kind(), + generated.partition(), + generated.bucket(), + generated.totalBuckets(), + generated.file().newFirstRowId(0L)); + ManifestFile manifestFile = createManifestFile(tempDir.toString(), Long.MAX_VALUE); + ManifestFileMeta sourceMeta = + writeSingleManifest(manifestFile, Collections.singletonList(source)); + ManifestAvroWriter writer = manifestFile.createAvroWriter(); + ManifestAvroWriter.EncodedEntry metadata = new ManifestAvroWriter.EncodedEntry(); + int recordCount = 8_200; + + try (ManifestAvroReader reader = openManifestReader(sourceMeta)) { + ManifestAvroReader.RowIterator rows = + reader.next().toRows(ManifestEntry.MANIFEST_ROW_TYPE); + rows.next(); + ByteBuffer encodedRecord = rows.encodedRecord(); + for (int i = 0; i < recordCount; i++) { + writer.writeEncoded( + encodedRecord.duplicate(), + metadata.replace( + source.kind().toByteValue(), + source.partition().copy(), + source.bucket(), + source.level(), + source.file().schemaId(), + source.file().firstRowId(), + source.file().rowCount())); + } + } + writer.close(); + + ManifestFileMeta result = writer.result().get(0); + assertThat(result.numAddedFiles()).isEqualTo(recordCount); + assertThat(result.numDeletedFiles()).isZero(); + assertThat(result.partitionStats()).isEqualTo(sourceMeta.partitionStats()); + } + + @Test + void testWriteEncodedManifestPreservesUnknownAggregateStats() throws Exception { + List entries = Arrays.asList(gen.next(), gen.next()); + ManifestFile manifestFile = createManifestFile(tempDir.toString(), Long.MAX_VALUE); + ManifestFileMeta source = writeSingleManifest(manifestFile, entries); + ManifestFileMeta unknownStats = + new ManifestFileMeta( + source.fileName(), + source.fileSize(), + source.numAddedFiles(), + source.numDeletedFiles(), + source.partitionStats(), + source.schemaId(), + null, + null, + null, + null, + source.minRowId(), + source.maxRowId()); + + ManifestAvroWriter writer = manifestFile.createAvroWriter(); + try (ManifestAvroReader reader = openManifestReader(source)) { + writer.writeEncodedManifest(reader, unknownStats); + } + writer.close(); + + ManifestFileMeta result = writer.result().get(0); + assertThat(result.minBucket()).isNull(); + assertThat(result.maxBucket()).isNull(); + assertThat(result.minLevel()).isNull(); + assertThat(result.maxLevel()).isNull(); + assertThat(result.partitionStats()).isEqualTo(source.partitionStats()); + assertThat(manifestFile.read(result.fileName())).containsExactlyElementsOf(entries); + } + @Test void testReadMissingManifestFile() { ManifestFile manifestFile = createManifestFile(tempDir.toString()); @@ -889,7 +967,7 @@ private void assertEncodedBlockCounts(FileKind... kinds) throws Exception { assertThat(reader.hasNext()).isTrue(); ManifestAvroReader.RawBlock block = reader.next(); assertThat(block.recordCount()).isEqualTo(entries.size()); - writer.writeEncodedBlock(block.encodedBlock(), encodedBlock(sourceMeta, entries)); + writer.writeEncodedBlock(block.encodedBlock(), encodedBlock(sourceMeta)); assertThat(reader.hasNext()).isFalse(); } writer.close(); @@ -903,9 +981,7 @@ private void assertEncodedBlockCounts(FileKind... kinds) throws Exception { assertThat(manifestFile.read(result.fileName())).containsExactlyElementsOf(entries); } - private ManifestAvroWriter.EncodedBlock encodedBlock( - ManifestFileMeta meta, List entries) { - boolean nullPartition = entries.get(0).partition().isNullAt(0); + private ManifestAvroWriter.EncodedBlock encodedBlock(ManifestFileMeta meta) { return new ManifestAvroWriter.EncodedBlock( meta.numAddedFiles(), meta.numDeletedFiles(), @@ -916,10 +992,7 @@ private ManifestAvroWriter.EncodedBlock encodedBlock( meta.maxLevel(), meta.minRowId(), meta.maxRowId(), - nullPartition ? entries.get(0).partition() : null, - nullPartition ? entries.size() : 0, - nullPartition ? null : entries.get(0).partition(), - nullPartition ? null : entries.get(0).partition()); + meta.partitionStats()); } private ManifestAvroReader openManifestReader(ManifestFileMeta manifest) throws IOException { diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/ManifestFileMergerTestUtils.java b/paimon-core/src/test/java/org/apache/paimon/operation/ManifestFileMergerTestUtils.java new file mode 100644 index 000000000000..b94bd7ff9a23 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/operation/ManifestFileMergerTestUtils.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.operation; + +import org.apache.paimon.manifest.ManifestFile; +import org.apache.paimon.manifest.ManifestFileMeta; +import org.apache.paimon.types.RowType; + +import javax.annotation.Nullable; + +import java.util.List; +import java.util.Optional; + +/** Test access to full compaction without also triggering minor compaction. */ +public final class ManifestFileMergerTestUtils { + + private ManifestFileMergerTestUtils() {} + + public static Optional> tryFullCompaction( + List inputs, + List newFilesForAbort, + ManifestFile manifestFile, + long suggestedMetaSize, + long sizeTrigger, + RowType partitionType, + @Nullable Integer manifestReadParallelism) + throws Exception { + return ManifestFileBlockMerger.tryFullCompaction( + inputs, + newFilesForAbort, + manifestFile, + suggestedMetaSize, + sizeTrigger, + partitionType, + manifestReadParallelism); + } +} diff --git a/paimon-format/src/main/java/org/apache/avro/file/RawBlock.java b/paimon-format/src/main/java/org/apache/avro/file/RawBlock.java index 3386074b9df4..8600279380d0 100644 --- a/paimon-format/src/main/java/org/apache/avro/file/RawBlock.java +++ b/paimon-format/src/main/java/org/apache/avro/file/RawBlock.java @@ -57,6 +57,22 @@ public long recordCount() { return block.getNumEntries(); } + public Schema schema() { + return schema; + } + + /** Returns an independently owned copy which remains valid after the reader advances. */ + public RawBlock stableCopy() { + ByteBuffer source = block.getAsByteBuffer().duplicate(); + ByteBuffer copy = ByteBuffer.allocate(source.remaining()); + copy.put(source); + copy.flip(); + DataFileStream.DataBlock copiedBlock = + new DataFileStream.DataBlock(copy, block.getNumEntries()); + copiedBlock.setFlushOnWrite(block.isFlushOnWrite()); + return new RawBlock(copiedBlock, codec, schema); + } + public ByteBuffer decompress(ByteBuffer reuse) throws IOException { if (!decompressed) { if (codec instanceof ZstandardCodec) { @@ -100,20 +116,16 @@ public ByteBuffer decompress(ByteBuffer reuse) throws IOException { target.limit(size); decompressedBuffer = target.duplicate(); } else { - block.decompressUsing(codec); - decompressedBuffer = block.getAsByteBuffer().duplicate(); + decompressedBuffer = codec.decompress(block.getAsByteBuffer().duplicate()); } decompressed = true; } return decompressedBuffer.duplicate(); } - /** Returns a single-block stream for appending this compressed block to an Avro writer. */ - public DataFileStream asStream() throws IOException { - if (decompressed) { - throw new IllegalStateException("A decompressed Avro block cannot be copied raw."); - } - return new SingleBlockStream(schema, codec, block); + /** Returns a single-block stream using a binary-compatible target schema. */ + public DataFileStream asStream(Schema targetSchema) throws IOException { + return new SingleBlockStream(targetSchema, codec, block); } private static final class SingleBlockStream extends DataFileStream { diff --git a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockReader.java b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockReader.java index c79b0e198573..c4359afcda43 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockReader.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockReader.java @@ -22,6 +22,7 @@ import org.apache.paimon.utils.IOUtils; import org.apache.avro.AvroRuntimeException; +import org.apache.avro.Schema; import org.apache.avro.file.RawBlock; import org.apache.avro.file.RawBlockReader; @@ -60,8 +61,51 @@ public AvroRecordDecoder createRecordDecoder() { /** Returns whether blocks use the default Avro schema for the given row type. */ public boolean supportsRawBlockCopy(RowType rowType) { - return AvroSchemaConverter.convertToSchema(rowType, Collections.emptyMap()) - .equals(reader.getSchema()); + return hasSameBinaryLayout( + AvroSchemaConverter.convertToSchema(rowType, Collections.emptyMap()), + reader.getSchema()); + } + + static boolean hasSameBinaryLayout(Schema expected, Schema actual) { + if (expected.getType() != actual.getType()) { + return false; + } + switch (expected.getType()) { + case RECORD: + if (expected.getFields().size() != actual.getFields().size()) { + return false; + } + for (int i = 0; i < expected.getFields().size(); i++) { + if (!expected.getFields().get(i).name().equals(actual.getFields().get(i).name()) + || !hasSameBinaryLayout( + expected.getFields().get(i).schema(), + actual.getFields().get(i).schema())) { + return false; + } + } + return true; + case ARRAY: + return hasSameBinaryLayout(expected.getElementType(), actual.getElementType()); + case MAP: + return hasSameBinaryLayout(expected.getValueType(), actual.getValueType()); + case UNION: + if (expected.getTypes().size() != actual.getTypes().size()) { + return false; + } + for (int i = 0; i < expected.getTypes().size(); i++) { + if (!hasSameBinaryLayout( + expected.getTypes().get(i), actual.getTypes().get(i))) { + return false; + } + } + return true; + case FIXED: + return expected.getFixedSize() == actual.getFixedSize(); + case ENUM: + return expected.getEnumSymbols().equals(actual.getEnumSymbols()); + default: + return true; + } } /** Returns whether another block is available. */ diff --git a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockWriter.java b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockWriter.java index 377a7862d099..5cf4a803e03a 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockWriter.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockWriter.java @@ -22,6 +22,7 @@ import org.apache.paimon.format.FormatWriter; import org.apache.paimon.fs.PositionOutputStream; +import org.apache.avro.Schema; import org.apache.avro.file.DataFileWriter; import java.io.IOException; @@ -32,10 +33,13 @@ public final class AvroBlockWriter implements FormatWriter { private final DataFileWriter writer; private final PositionOutputStream out; + private final Schema schema; - public AvroBlockWriter(DataFileWriter writer, PositionOutputStream out) { + public AvroBlockWriter( + DataFileWriter writer, PositionOutputStream out, Schema schema) { this.writer = writer; this.out = out; + this.schema = schema; } @Override @@ -48,7 +52,11 @@ public void addEncoded(ByteBuffer record) throws IOException { } public void addEncodedBlock(AvroRawBlock block) throws IOException { - writer.appendAllFrom(block.asStream(), false); + if (!AvroBlockReader.hasSameBinaryLayout(schema, block.rawBlock().schema())) { + throw new IllegalArgumentException( + "Avro block schema is not binary-compatible with the writer schema."); + } + writer.appendAllFrom(block.rawBlock().asStream(schema), false); } @Override diff --git a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java index 73b08eec474f..69512d5ee7cf 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java @@ -95,7 +95,7 @@ public AvroBlockWriter createBlockWriter( writer.setCodec(createCodecFactory(compression)); writer.setFlushOnEveryBlock(false); writer.create(schema, new CloseShieldOutputStream(out)); - return new AvroBlockWriter(writer, out); + return new AvroBlockWriter(writer, out, schema); } @Override diff --git a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroRawBlock.java b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroRawBlock.java index c7d21d4cc8d9..152d5dc30087 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroRawBlock.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroRawBlock.java @@ -18,7 +18,6 @@ package org.apache.paimon.format.avro; -import org.apache.avro.file.DataFileStream; import org.apache.avro.file.RawBlock; import javax.annotation.Nullable; @@ -48,6 +47,11 @@ public long recordCount() { return block.recordCount(); } + /** Returns an independently owned copy which is not reused by the reader. */ + public AvroRawBlock stableCopy() { + return new AvroRawBlock(block.stableCopy()); + } + /** * Lazily decompresses this block, reusing the supplied heap buffer when possible. * @@ -57,8 +61,4 @@ public long recordCount() { public ByteBuffer decompress(@Nullable ByteBuffer reuse) throws IOException { return block.decompress(reuse); } - - DataFileStream asStream() throws IOException { - return block.asStream(); - } } diff --git a/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java b/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java index 8c05c5b4bf05..ab6031531b35 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java @@ -197,6 +197,47 @@ void testReadBorrowedRawBlocks() throws IOException { assertThat(records).isEqualTo(numRecords); } + @Test + void testRawBlockCompatibilityUsesBinaryLayoutAndFieldIdentity() { + Schema expected = + SchemaBuilder.record("Expected") + .fields() + .requiredInt("id") + .name("nested") + .type( + SchemaBuilder.record("ExpectedNested") + .fields() + .requiredLong("value") + .endRecord()) + .noDefault() + .endRecord(); + Schema renamedRecords = + SchemaBuilder.record("Actual") + .fields() + .requiredInt("id") + .name("nested") + .type( + SchemaBuilder.record("ActualNested") + .fields() + .requiredLong("value") + .endRecord()) + .noDefault() + .endRecord(); + Schema renamedField = + SchemaBuilder.record("Actual") + .fields() + .requiredInt("other_id") + .name("nested") + .type(renamedRecords.getField("nested").schema()) + .noDefault() + .endRecord(); + Schema missingField = SchemaBuilder.record("Actual").fields().requiredInt("id").endRecord(); + + assertThat(AvroBlockReader.hasSameBinaryLayout(expected, renamedRecords)).isTrue(); + assertThat(AvroBlockReader.hasSameBinaryLayout(expected, renamedField)).isFalse(); + assertThat(AvroBlockReader.hasSameBinaryLayout(expected, missingField)).isFalse(); + } + @Test void testRowReaderProjectsIntoReusedRow() throws IOException { Schema writerSchema = From 8ccbc6f73cd6589f02bbde44cfd2e59c7a67efb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Thu, 13 Aug 2026 23:54:43 +0800 Subject: [PATCH 3/3] [core] Refine block-aware manifest merging --- .../paimon/io/ProjectedDataFileMeta.java | 8 + .../paimon/manifest/CollectedDeletes.java | 114 ++++ .../manifest/CompactFileIdentifierSet.java | 51 +- .../paimon/manifest/DeletedRowIdSet.java | 157 +++++ .../paimon/manifest/PartitionDictionary.java | 62 ++ .../manifest/ProjectedManifestEntry.java | 26 +- .../operation/ManifestEntryRunMerge.java | 5 +- .../operation/ManifestEntryRunMergeEntry.java | 7 +- .../operation/ManifestEntryRunMergePlan.java | 5 +- .../operation/ManifestFileBlockMerger.java | 593 +++++------------- .../paimon/operation/ManifestFileSorter.java | 123 +--- .../CompactFileIdentifierSetTest.java | 19 + 12 files changed, 588 insertions(+), 582 deletions(-) create mode 100644 paimon-core/src/main/java/org/apache/paimon/manifest/CollectedDeletes.java create mode 100644 paimon-core/src/main/java/org/apache/paimon/manifest/DeletedRowIdSet.java create mode 100644 paimon-core/src/main/java/org/apache/paimon/manifest/PartitionDictionary.java diff --git a/paimon-core/src/main/java/org/apache/paimon/io/ProjectedDataFileMeta.java b/paimon-core/src/main/java/org/apache/paimon/io/ProjectedDataFileMeta.java index b707065eb19f..d774e9fc1720 100644 --- a/paimon-core/src/main/java/org/apache/paimon/io/ProjectedDataFileMeta.java +++ b/paimon-core/src/main/java/org/apache/paimon/io/ProjectedDataFileMeta.java @@ -230,6 +230,14 @@ public boolean hasFirstRowId() { return !currentRow().isNullAt(requiredPosition(Fields.FIRST_ROW_ID)); } + @Override + public long nonNullFirstRowId() { + int position = requiredPosition(Fields.FIRST_ROW_ID); + InternalRow row = currentRow(); + checkState(!row.isNullAt(position), "First row id cannot be null."); + return row.getLong(position); + } + @Nullable @Override public Long firstRowId() { diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/CollectedDeletes.java b/paimon-core/src/main/java/org/apache/paimon/manifest/CollectedDeletes.java new file mode 100644 index 000000000000..49465c303d20 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/CollectedDeletes.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.manifest; + +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.io.ProjectedDataFileMeta; +import org.apache.paimon.manifest.FileEntry.ReusableIdentifier; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +import static org.apache.paimon.utils.Preconditions.checkState; + +/** DELETE identifiers and optional RowID and partition indexes collected for manifest merging. */ +public final class CollectedDeletes { + + private final CompactFileIdentifierSet identifiers = new CompactFileIdentifierSet(); + private final DeletedRowIdSet rowIds = new DeletedRowIdSet(); + private Set partitions = new HashSet<>(); + private final boolean useRowIdFilter; + private boolean immutable; + + public CollectedDeletes(boolean useRowIdFilter) { + this.useRowIdFilter = useRowIdFilter; + } + + public void add( + ProjectedManifestEntry entry, boolean collectRowIds, boolean collectPartitions) { + checkState(!immutable, "Cannot modify an immutable DELETE collection."); + identifiers.add(entry); + if (collectPartitions) { + partitions.add(entry.partition().copy()); + } + if (collectRowIds) { + rowIds.add(entry.file().nonNullFirstRowId()); + } + } + + public void combine(CollectedDeletes other) { + checkState(!immutable, "Cannot modify an immutable DELETE collection."); + checkState( + useRowIdFilter == other.useRowIdFilter, + "Cannot combine DELETE collections with different RowID modes."); + identifiers.addAll(other.identifiers); + rowIds.addAll(other.rowIds); + partitions.addAll(other.partitions); + } + + public CollectedDeletes toImmutable() { + checkState(!immutable, "Cannot modify an immutable DELETE collection."); + if (useRowIdFilter) { + rowIds.prepareRangeIndex(); + } + partitions = Collections.unmodifiableSet(partitions); + immutable = true; + return this; + } + + public boolean isEmpty() { + return identifiers.isEmpty(); + } + + public Set partitions() { + return partitions; + } + + public boolean useRowIdFilter() { + return useRowIdFilter; + } + + public boolean isDeleted(ProjectedManifestEntry entry, ReusableIdentifier reusableIdentifier) { + if (useRowIdFilter) { + ProjectedDataFileMeta file = entry.file(); + checkState(file.hasFirstRowId(), "First row id should not be null."); + if (!rowIds.contains(file.nonNullFirstRowId())) { + return false; + } + } + return identifiers.contains(reusableIdentifier.replaceWithPartition(entry)); + } + + public boolean copyable( + ProjectedManifestEntry entry, + ReusableIdentifier reusableIdentifier, + boolean deferDeletedAddCheck) { + return entry.isAdd() && (deferDeletedAddCheck || !isDeleted(entry, reusableIdentifier)); + } + + public boolean intersectsRowIds(long minRowId, long maxRowId) { + return rowIds.intersects(minRowId, maxRowId); + } + + public void release() { + identifiers.release(); + rowIds.releaseRangeIndex(); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/CompactFileIdentifierSet.java b/paimon-core/src/main/java/org/apache/paimon/manifest/CompactFileIdentifierSet.java index b11b21345889..eca97f75ddbc 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/CompactFileIdentifierSet.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/CompactFileIdentifierSet.java @@ -74,6 +74,13 @@ public void add(int partitionId, ReusableIdentifier identifier) { add(partitionId, identifier.bytes(), identifier.length()); } + public void addAll(CompactFileIdentifierSet other) { + checkArgument(other != null, "Identifier set cannot be null."); + for (int entry = 0; entry < other.size; entry++) { + add(other.partitionIds[entry], other.arena, other.offsets[entry], other.lengths[entry]); + } + } + public boolean contains(ProjectedManifestEntry entry) { return contains(reusableIdentifier().replaceWithPartition(entry)); } @@ -104,9 +111,13 @@ public void release() { } void add(int partitionId, byte[] identifier, int length) { - checkIdentifier(identifier, length); - long hash = hash(partitionId, identifier, length); - if (contains(partitionId, identifier, length, hash)) { + add(partitionId, identifier, 0, length); + } + + private void add(int partitionId, byte[] identifier, int offset, int length) { + checkIdentifier(identifier, offset, length); + long hash = hash(partitionId, identifier, offset, length); + if (contains(partitionId, identifier, offset, length, hash)) { return; } if (size + 1 > (int) (buckets.length * LOAD_FACTOR)) { @@ -114,14 +125,14 @@ void add(int partitionId, byte[] identifier, int length) { } ensureEntryCapacity(size + 1); ensureArenaCapacity(length); - int offset = arenaSize; - System.arraycopy(identifier, 0, arena, offset, length); + int arenaOffset = arenaSize; + System.arraycopy(identifier, offset, arena, arenaOffset, length); arenaSize = Math.addExact(arenaSize, length); int bucket = bucket(hash); hashes[size] = hash; partitionIds[size] = partitionId; - offsets[size] = offset; + offsets[size] = arenaOffset; lengths[size] = length; next[size] = buckets[bucket]; buckets[bucket] = size; @@ -129,16 +140,18 @@ void add(int partitionId, byte[] identifier, int length) { } boolean contains(int partitionId, byte[] identifier, int length) { - checkIdentifier(identifier, length); - return contains(partitionId, identifier, length, hash(partitionId, identifier, length)); + checkIdentifier(identifier, 0, length); + return contains( + partitionId, identifier, 0, length, hash(partitionId, identifier, 0, length)); } - private boolean contains(int partitionId, byte[] identifier, int length, long hash) { + private boolean contains( + int partitionId, byte[] identifier, int offset, int length, long hash) { for (int entry = buckets[bucket(hash)]; entry >= 0; entry = next[entry]) { if (hashes[entry] == hash && partitionIds[entry] == partitionId && lengths[entry] == length - && bytesEqual(arena, offsets[entry], identifier, length)) { + && bytesEqual(arena, offsets[entry], identifier, offset, length)) { return true; } } @@ -194,20 +207,21 @@ private static int bucket(long hash, int bucketCount) { return ((int) (hash ^ (hash >>> 32))) & (bucketCount - 1); } - private static long hash(int partitionId, byte[] bytes, int length) { + private static long hash(int partitionId, byte[] bytes, int offset, int length) { long hash = 0xcbf29ce484222325L; hash ^= Integer.toUnsignedLong(partitionId); hash *= 0x100000001b3L; for (int i = 0; i < length; i++) { - hash ^= bytes[i] & 0xFFL; + hash ^= bytes[offset + i] & 0xFFL; hash *= 0x100000001b3L; } return hash; } - private static boolean bytesEqual(byte[] left, int leftOffset, byte[] right, int length) { + private static boolean bytesEqual( + byte[] left, int leftOffset, byte[] right, int rightOffset, int length) { for (int i = 0; i < length; i++) { - if (left[leftOffset + i] != right[i]) { + if (left[leftOffset + i] != right[rightOffset + i]) { return false; } } @@ -225,12 +239,13 @@ private ReusableIdentifier reusableIdentifier() { return reusableIdentifier; } - private static void checkIdentifier(byte[] identifier, int length) { + private static void checkIdentifier(byte[] identifier, int offset, int length) { checkArgument(identifier != null, "Identifier bytes cannot be null."); checkArgument( - length >= 0 && length <= identifier.length, - "Invalid identifier length %s.", - length); + offset >= 0 && length >= 0 && offset <= identifier.length - length, + "Invalid identifier range [%s, %s).", + offset, + offset + length); } private static int[] filledWithMinusOne(int length) { diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/DeletedRowIdSet.java b/paimon-core/src/main/java/org/apache/paimon/manifest/DeletedRowIdSet.java new file mode 100644 index 000000000000..2e5e87883d51 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/DeletedRowIdSet.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.manifest; + +import javax.annotation.Nullable; + +import java.util.Arrays; + +/** Primitive set used by RowID compaction to avoid rebuilding file identifiers. */ +public final class DeletedRowIdSet { + + private static final long EMPTY = Long.MIN_VALUE; + + private long[] table = emptyTable(16); + private int size; + private boolean containsMinValue; + private @Nullable long[] sortedRowIds; + + public void add(long value) { + if (value == EMPTY) { + if (!containsMinValue) { + containsMinValue = true; + size++; + sortedRowIds = null; + } + return; + } + if ((size + 1) * 2 > table.length) { + grow(); + } + int slot = slot(value, table.length); + while (table[slot] != EMPTY) { + if (table[slot] == value) { + return; + } + slot = (slot + 1) & (table.length - 1); + } + table[slot] = value; + size++; + sortedRowIds = null; + } + + public void addAll(DeletedRowIdSet other) { + if (other.containsMinValue) { + add(EMPTY); + } + for (long value : other.table) { + if (value != EMPTY) { + add(value); + } + } + } + + public boolean contains(long value) { + if (value == EMPTY) { + return containsMinValue; + } + int slot = slot(value, table.length); + while (table[slot] != EMPTY) { + if (table[slot] == value) { + return true; + } + slot = (slot + 1) & (table.length - 1); + } + return false; + } + + public boolean intersects(long minInclusive, long maxInclusive) { + if (minInclusive > maxInclusive) { + return true; + } + long[] values = sortedRowIds(); + int position = Arrays.binarySearch(values, minInclusive); + if (position < 0) { + position = -position - 1; + } + return position < values.length && values[position] <= maxInclusive; + } + + public void prepareRangeIndex() { + // Publish the immutable sorted snapshot before concurrent manifest planning starts. + sortedRowIds(); + } + + public void releaseRangeIndex() { + sortedRowIds = null; + } + + private long[] sortedRowIds() { + if (sortedRowIds != null) { + return sortedRowIds; + } + long[] values = new long[size]; + int position = 0; + if (containsMinValue) { + values[position++] = EMPTY; + } + for (long value : table) { + if (value != EMPTY) { + values[position++] = value; + } + } + if (position != size) { + throw new IllegalStateException("Failed to snapshot deleted RowID set."); + } + Arrays.sort(values); + sortedRowIds = values; + return values; + } + + private void grow() { + long[] previous = table; + if (previous.length >= (1 << 30)) { + throw new IllegalStateException("Too many deleted RowIDs in one manifest group."); + } + table = emptyTable(previous.length << 1); + int previousSize = size; + size = containsMinValue ? 1 : 0; + for (long value : previous) { + if (value != EMPTY) { + add(value); + } + } + if (size != previousSize) { + throw new IllegalStateException("Failed to grow deleted RowID set."); + } + } + + private static int slot(long value, int length) { + value ^= value >>> 33; + value *= 0xff51afd7ed558ccdL; + value ^= value >>> 33; + return ((int) value) & (length - 1); + } + + private static long[] emptyTable(int length) { + long[] table = new long[length]; + Arrays.fill(table, EMPTY); + return table; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/PartitionDictionary.java b/paimon-core/src/main/java/org/apache/paimon/manifest/PartitionDictionary.java new file mode 100644 index 000000000000..b925457e68dd --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/PartitionDictionary.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.manifest; + +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.utils.ByteArrayKey; +import org.apache.paimon.utils.ByteArrayLookupKey; +import org.apache.paimon.utils.SerializationUtils; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +/** Deduplicates serialized partitions and assigns compact integer identifiers. */ +public final class PartitionDictionary { + + private final Map ids = new HashMap<>(); + private final ByteArrayLookupKey lookup = new ByteArrayLookupKey(); + private BinaryRow[] partitions = new BinaryRow[16]; + private int partitionCount; + + public int id(byte[] bytes) { + lookup.reset(bytes); + try { + Integer existing = ids.get(lookup); + if (existing != null) { + return existing; + } + byte[] canonical = Arrays.copyOf(bytes, bytes.length); + int id = partitionCount; + if (id == partitions.length) { + partitions = Arrays.copyOf(partitions, partitions.length << 1); + } + partitions[id] = SerializationUtils.deserializeBinaryRow(canonical); + ids.put(new ByteArrayKey(canonical), id); + partitionCount = id + 1; + return id; + } finally { + lookup.clear(); + } + } + + public BinaryRow partition(int id) { + return partitions[id]; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ProjectedManifestEntry.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ProjectedManifestEntry.java index e7a279f0f0cd..591d86759e6e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ProjectedManifestEntry.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ProjectedManifestEntry.java @@ -49,6 +49,7 @@ public final class ProjectedManifestEntry implements ManifestEntry { private static final Projection FULL_PROJECTION = Projection.create(MANIFEST_ROW_TYPE); public static final Projection DELETE_ENTRY_PROJECTION = createDeleteEntryProjection(); public static final Projection ROW_RANGE_PROJECTION = createRowRangeProjection(); + public static final Projection ENTRY_LAYOUT_PROJECTION = createEntryLayoutProjection(); private final Projection projection; private final @Nullable ProjectedDataFileMeta file; @@ -132,6 +133,29 @@ private static Projection createRowRangeProjection() { DataFileMeta.FIRST_ROW_ID))))); } + private static Projection createEntryLayoutProjection() { + RowType manifestType = MANIFEST_ROW_TYPE; + return Projection.create( + new RowType( + false, + Arrays.asList( + manifestType.getField(ManifestEntry.KIND), + manifestType.getField(ManifestEntry.PARTITION), + manifestType.getField(ManifestEntry.BUCKET), + manifestType + .getField(ManifestEntry.FILE) + .newType( + DataFileMeta.SCHEMA.project( + DataFileMeta.FILE_NAME, + DataFileMeta.ROW_COUNT, + DataFileMeta.LEVEL, + DataFileMeta.SCHEMA_ID, + DataFileMeta.FIRST_ROW_ID, + DataFileMeta.EXTRA_FILES, + DataFileMeta.EMBEDDED_FILE_INDEX, + DataFileMeta.EXTERNAL_PATH))))); + } + /** Drops references to the current row before its reader batch is released. */ public void clear() { row = null; @@ -366,7 +390,7 @@ private static void validateProjection(RowType projectedType) { } } - RowType projectedType() { + public RowType projectedType() { return projectedType; } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMerge.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMerge.java index 121b1be89e7d..0e808e798fd0 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMerge.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMerge.java @@ -25,6 +25,7 @@ import org.apache.paimon.format.SimpleStatsCollector; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.manifest.CompactFileIdentifierSet; +import org.apache.paimon.manifest.DeletedRowIdSet; import org.apache.paimon.manifest.FileKind; import org.apache.paimon.manifest.ManifestAvroReader; import org.apache.paimon.manifest.ManifestAvroReader.RawBlock; @@ -153,7 +154,7 @@ static List sortAndWriteFullEntries( ManifestFile manifestFile, List newFilesForAbort, CompactFileIdentifierSet deletedIdentifiers, - ManifestFileSorter.DeletedRowIdSet deletedRowIds, + DeletedRowIdSet deletedRowIds, @Nullable Integer manifestReadParallelism) throws Exception { ManifestEntryRunMergeEntry.Filter filter = @@ -186,7 +187,7 @@ static Pair, List> sortAndWriteMinorEnt @Nullable Integer manifestReadParallelism) throws Exception { CompactFileIdentifierSet deletedIdentifiers = new CompactFileIdentifierSet(); - ManifestFileSorter.DeletedRowIdSet deletedRowIds = new ManifestFileSorter.DeletedRowIdSet(); + DeletedRowIdSet deletedRowIds = new DeletedRowIdSet(); ManifestEntryRunMergeEntry.Filter.Minor filter = new ManifestEntryRunMergeEntry.Filter.Minor( deletedIdentifiers, deletedRowIds, true); diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergeEntry.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergeEntry.java index 2dc8b3d86509..5e8c318e9fda 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergeEntry.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergeEntry.java @@ -23,6 +23,7 @@ import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.manifest.CompactFileIdentifierSet; +import org.apache.paimon.manifest.DeletedRowIdSet; import org.apache.paimon.manifest.FileEntry.ReusableIdentifier; import org.apache.paimon.manifest.FileKind; import org.apache.paimon.manifest.ProjectedManifestEntry; @@ -223,14 +224,14 @@ BinaryRow partition(int id) { static class Filter { final CompactFileIdentifierSet deletedIdentifiers; - final ManifestFileSorter.DeletedRowIdSet deletedRowIds; + final DeletedRowIdSet deletedRowIds; final boolean useRowIdFilter; final ThreadLocal identifier = ThreadLocal.withInitial(IdentifierEncoder::new); Filter( CompactFileIdentifierSet deletedIdentifiers, - ManifestFileSorter.DeletedRowIdSet deletedRowIds, + DeletedRowIdSet deletedRowIds, boolean useRowIdFilter) { this.deletedIdentifiers = deletedIdentifiers; this.deletedRowIds = deletedRowIds; @@ -275,7 +276,7 @@ static final class Minor extends Filter { Minor( CompactFileIdentifierSet deletedIdentifiers, - ManifestFileSorter.DeletedRowIdSet deletedRowIds, + DeletedRowIdSet deletedRowIds, boolean useRowIdFilter) { super(deletedIdentifiers, deletedRowIds, useRowIdFilter); } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergePlan.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergePlan.java index 8b2129e10b32..18785455e942 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergePlan.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergePlan.java @@ -24,6 +24,7 @@ import org.apache.paimon.data.serializer.InternalRowSerializer; import org.apache.paimon.format.avro.AvroRawBlock; import org.apache.paimon.manifest.CompactFileIdentifierSet; +import org.apache.paimon.manifest.DeletedRowIdSet; import org.apache.paimon.manifest.FileEntry.ReusableIdentifier; import org.apache.paimon.manifest.FileKind; import org.apache.paimon.manifest.ManifestAvroReader; @@ -101,7 +102,7 @@ Pair, List> mergeMinorToManifest( ManifestFile manifestFile, ManifestEntryRunMergeEntry.Filter filter, CompactFileIdentifierSet deletedIdentifiers, - ManifestFileSorter.DeletedRowIdSet deletedRowIds, + DeletedRowIdSet deletedRowIds, List newFilesForAbort) throws Exception { List cursors = new ArrayList<>(sources.size()); @@ -171,7 +172,7 @@ private static Pair, List> writeMinorSe SelectionTree selectionTree, ManifestFile manifestFile, CompactFileIdentifierSet deletedIdentifiers, - ManifestFileSorter.DeletedRowIdSet deletedRowIds) + DeletedRowIdSet deletedRowIds) throws Exception { ManifestAvroWriter addWriter = manifestFile.createAvroWriter(); ManifestAvroWriter deleteWriter = manifestFile.createAvroWriter(); diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileBlockMerger.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileBlockMerger.java index 0d4b88509a90..a7ea3f224174 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileBlockMerger.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileBlockMerger.java @@ -21,32 +21,27 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.GenericRow; -import org.apache.paimon.data.InternalRow; import org.apache.paimon.format.SimpleColStats; import org.apache.paimon.format.SimpleStatsCollector; -import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.io.ProjectedDataFileMeta; +import org.apache.paimon.manifest.CollectedDeletes; import org.apache.paimon.manifest.CompactFileIdentifierSet; import org.apache.paimon.manifest.FileEntry.ReusableIdentifier; -import org.apache.paimon.manifest.FileKind; import org.apache.paimon.manifest.ManifestAvroReader; import org.apache.paimon.manifest.ManifestAvroReader.RawBlock; import org.apache.paimon.manifest.ManifestAvroReader.RowIterator; import org.apache.paimon.manifest.ManifestAvroWriter; import org.apache.paimon.manifest.ManifestAvroWriter.EncodedBlock; import org.apache.paimon.manifest.ManifestAvroWriter.EncodedEntry; -import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.manifest.ManifestFile; import org.apache.paimon.manifest.ManifestFileMeta; +import org.apache.paimon.manifest.PartitionDictionary; import org.apache.paimon.manifest.ProjectedManifestEntry; import org.apache.paimon.partition.PartitionPredicate; import org.apache.paimon.stats.SimpleStatsConverter; -import org.apache.paimon.types.DataField; import org.apache.paimon.types.RowType; -import org.apache.paimon.utils.ByteArrayKey; -import org.apache.paimon.utils.ByteArrayLookupKey; import org.apache.paimon.utils.CloseableIterator; import org.apache.paimon.utils.Filter; -import org.apache.paimon.utils.SerializationUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,16 +49,13 @@ import javax.annotation.Nullable; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.function.Function; import static org.apache.paimon.manifest.ManifestFileMeta.allContainsRowId; @@ -76,91 +68,8 @@ final class ManifestFileBlockMerger { private static final Logger LOG = LoggerFactory.getLogger(ManifestFileBlockMerger.class); - private static final int KIND = 0; - private static final int PARTITION = 1; - private static final int BUCKET = 2; - private static final int FILE = 3; - private static final int FILE_NAME = 0; - private static final int ROW_COUNT = 1; - private static final int LEVEL = 2; - private static final int SCHEMA_ID = 3; - private static final int FIRST_ROW_ID = 4; - private static final int MAX_SEQUENCE_NUMBER = 5; - private static final int EXTRA_FILES = 6; - private static final int EMBEDDED_FILE_INDEX = 7; - private static final int EXTERNAL_PATH = 8; - private static final int FILE_FIELD_COUNT = 9; - private static final String[] ENTRY_FILE_FIELD_NAMES = { - DataFileMeta.FILE_NAME, - DataFileMeta.ROW_COUNT, - DataFileMeta.LEVEL, - DataFileMeta.SCHEMA_ID, - DataFileMeta.FIRST_ROW_ID, - DataFileMeta.MAX_SEQUENCE_NUMBER, - DataFileMeta.EXTRA_FILES, - DataFileMeta.EMBEDDED_FILE_INDEX, - DataFileMeta.EXTERNAL_PATH - }; - private static final InternalRow.FieldGetter[] ENTRY_FILE_GETTERS = entryFileGetters(); - private static final RowType ENTRY_LAYOUT = entryLayout(); - private static final int FULL_KIND = - ManifestEntry.MANIFEST_ROW_TYPE.getFieldIndex(ManifestEntry.KIND); - private static final int FULL_PARTITION = - ManifestEntry.MANIFEST_ROW_TYPE.getFieldIndex(ManifestEntry.PARTITION); - private static final int FULL_BUCKET = - ManifestEntry.MANIFEST_ROW_TYPE.getFieldIndex(ManifestEntry.BUCKET); - private static final int FULL_FILE = - ManifestEntry.MANIFEST_ROW_TYPE.getFieldIndex(ManifestEntry.FILE); - private ManifestFileBlockMerger() {} - private static InternalRow.FieldGetter[] entryFileGetters() { - InternalRow.FieldGetter[] getters = - new InternalRow.FieldGetter[ENTRY_FILE_FIELD_NAMES.length]; - for (int field = 0; field < getters.length; field++) { - int position = DataFileMeta.SCHEMA.getFieldIndex(ENTRY_FILE_FIELD_NAMES[field]); - getters[field] = - InternalRow.createFieldGetter( - DataFileMeta.SCHEMA.getTypeAt(position), position); - } - return getters; - } - - private static RowType entryLayout() { - List fields = new ArrayList<>(); - fields.add(ManifestEntry.MANIFEST_ROW_TYPE.getField(ManifestEntry.KIND)); - fields.add(ManifestEntry.MANIFEST_ROW_TYPE.getField(ManifestEntry.PARTITION)); - fields.add(ManifestEntry.MANIFEST_ROW_TYPE.getField(ManifestEntry.BUCKET)); - fields.add( - ManifestEntry.MANIFEST_ROW_TYPE - .getField(ManifestEntry.FILE) - .newType( - DataFileMeta.SCHEMA.project( - DataFileMeta.FILE_NAME, - DataFileMeta.ROW_COUNT, - DataFileMeta.LEVEL, - DataFileMeta.SCHEMA_ID, - DataFileMeta.FIRST_ROW_ID, - DataFileMeta.MAX_SEQUENCE_NUMBER, - DataFileMeta.EXTRA_FILES, - DataFileMeta.EMBEDDED_FILE_INDEX, - DataFileMeta.EXTERNAL_PATH))); - return new RowType(false, fields); - } - - private static GenericRow projectEntryLayout( - GenericRow fullRow, GenericRow reuse, GenericRow reuseFile) { - reuse.setField(KIND, fullRow.getByte(FULL_KIND)); - reuse.setField(PARTITION, fullRow.getBinary(FULL_PARTITION)); - reuse.setField(BUCKET, fullRow.getInt(FULL_BUCKET)); - InternalRow fullFile = fullRow.getRow(FULL_FILE, DataFileMeta.SCHEMA.getFieldCount()); - for (int field = 0; field < ENTRY_FILE_GETTERS.length; field++) { - reuseFile.setField(field, ENTRY_FILE_GETTERS[field].getFieldOrNull(fullFile)); - } - reuse.setField(FILE, reuseFile); - return reuse; - } - static List merge( List input, List newFilesForAbort, @@ -229,19 +138,20 @@ static Optional> tryFullCompaction( totalDeltaFileSize); boolean useRowIdFilter = allContainsRowId(inputs); - CollectedDeletes deletes = + final CollectedDeletes deletes = collectDeletes( - deltaManifests, - manifestFile, - useRowIdFilter, - true, - manifestReadParallelism); + deltaManifests, + manifestFile, + useRowIdFilter, + true, + manifestReadParallelism) + .toImmutable(); try { PartitionPredicate predicate; - if (deletes.identifiers.isEmpty()) { + if (deletes.isEmpty()) { predicate = PartitionPredicate.ALWAYS_FALSE; } else if (partitionType.getFieldCount() > 0) { - predicate = PartitionPredicate.fromMultiple(partitionType, deletes.partitions); + predicate = PartitionPredicate.fromMultiple(partitionType, deletes.partitions()); } else { predicate = PartitionPredicate.ALWAYS_TRUE; } @@ -270,14 +180,12 @@ static Optional> tryFullCompaction( return Optional.empty(); } - CompactionFilter filter = - new CompactionFilter(deletes.identifiers, deletes.rowIds, useRowIdFilter); List rewritten = rewriteManifests( toCompact, manifestFile, partitionType, - filter, + deletes, true, mustChange, result, @@ -365,42 +273,39 @@ private static CollectedDeletes collectDeletes( } } - CollectedDeletes deletes = new CollectedDeletes(); + CollectedDeletes result = new CollectedDeletes(collectRowIds); if (manifestReadParallelism == null || manifestReadParallelism <= 1 || manifestsWithDeletes.size() <= 1) { for (ManifestFileMeta manifest : manifestsWithDeletes) { - collectDeletedEntries( - manifest, manifestFile, collectRowIds, collectPartitions, deletes, false); + CollectedDeletes deletes = + collectDeletedEntries( + manifest, manifestFile, collectRowIds, collectPartitions); + result.combine(deletes); + deletes.release(); } - return deletes; + return result; } - Function> scan = - manifest -> { - collectDeletedEntries( - manifest, - manifestFile, - collectRowIds, - collectPartitions, - deletes, - true); - return Collections.singletonList(Boolean.TRUE); - }; - for (Boolean ignored : + Function> scan = + manifest -> + Collections.singletonList( + collectDeletedEntries( + manifest, manifestFile, collectRowIds, collectPartitions)); + for (CollectedDeletes deletes : sequentialBatchedExecute(scan, manifestsWithDeletes, manifestReadParallelism)) { - // Iteration waits for every bounded batch of manifest scans. + result.combine(deletes); + deletes.release(); } - return deletes; + return result; } - private static void collectDeletedEntries( + private static CollectedDeletes collectDeletedEntries( ManifestFileMeta manifest, ManifestFile manifestFile, boolean collectRowIds, - boolean collectPartitions, - CollectedDeletes deletes, - boolean synchronize) { + boolean collectPartitions) { + CollectedDeletes deletes = new CollectedDeletes(collectRowIds); try (CloseableIterator entries = manifestFile.scan( manifest.fileName(), ProjectedManifestEntry.DELETE_ENTRY_PROJECTION)) { @@ -409,144 +314,16 @@ private static void collectDeletedEntries( if (!entry.isDelete()) { continue; } - if (synchronize) { - synchronized (deletes) { - deletes.add(entry, collectRowIds, collectPartitions); - } - } else { - deletes.add(entry, collectRowIds, collectPartitions); - } + deletes.add(entry, collectRowIds, collectPartitions); } + return deletes; } catch (Exception e) { + deletes.release(); throw new RuntimeException( "Failed to collect DELETE entries from manifest " + manifest.fileName(), e); } } - private static InternalRow entryFile(GenericRow record) { - return record.getRow(FILE, FILE_FIELD_COUNT); - } - - private static final class CompactionKey { - - private byte kind; - private boolean hasRowId; - private long firstRowId; - private long rangeEnd; - - private void replace(GenericRow record) { - InternalRow file = entryFile(record); - kind = record.getByte(KIND); - hasRowId = !file.isNullAt(FIRST_ROW_ID); - if (hasRowId) { - firstRowId = file.getLong(FIRST_ROW_ID); - rangeEnd = firstRowId + file.getLong(ROW_COUNT) - 1L; - } else { - firstRowId = 0; - rangeEnd = 0; - } - } - } - - private static class CompactionFilter { - - final CompactFileIdentifierSet deletedIdentifiers; - final ManifestFileSorter.DeletedRowIdSet deletedRowIds; - final boolean useRowIdFilter; - private final ProjectedManifestEntry identifierEntry = - ProjectedManifestEntry.Projection.create(ENTRY_LAYOUT).createEntry(); - private final ReusableIdentifier identifier = new ReusableIdentifier(); - - private CompactionFilter( - CompactFileIdentifierSet deletedIdentifiers, - ManifestFileSorter.DeletedRowIdSet deletedRowIds, - boolean useRowIdFilter) { - this.deletedIdentifiers = deletedIdentifiers; - this.deletedRowIds = deletedRowIds; - this.useRowIdFilter = useRowIdFilter; - } - - boolean copyable(GenericRow record, CompactionKey key, boolean deferDeletedAddCheck) { - return key.kind == FileKind.ADD.toByteValue() - && (deferDeletedAddCheck || !isDeleted(record, key)); - } - - boolean canCopyRange(long minRowId, long maxRowId) { - return !deletedRowIds.intersects(minRowId, maxRowId); - } - - private ReusableIdentifier identifier(GenericRow record) { - return identifier.replaceWithPartition(identifierEntry.replace(record)); - } - - private boolean isDeleted(GenericRow record, CompactionKey key) { - if (useRowIdFilter) { - checkState(key.hasRowId, "First row id should not be null."); - if (!deletedRowIds.contains(key.firstRowId)) { - return false; - } - } - return deletedIdentifiers.contains(identifier(record)); - } - } - - private static final class PartitionDictionary { - - private final Map ids = new HashMap<>(); - private final ByteArrayLookupKey lookup = new ByteArrayLookupKey(); - private BinaryRow[] partitions = new BinaryRow[16]; - private int partitionCount; - - private int id(byte[] bytes) { - lookup.reset(bytes); - try { - Integer existing = ids.get(lookup); - if (existing != null) { - return existing; - } - byte[] canonical = Arrays.copyOf(bytes, bytes.length); - int id = partitionCount; - if (id == partitions.length) { - partitions = Arrays.copyOf(partitions, partitions.length << 1); - } - partitions[id] = SerializationUtils.deserializeBinaryRow(canonical); - ids.put(new ByteArrayKey(canonical), id); - partitionCount = id + 1; - return id; - } finally { - lookup.clear(); - } - } - - private BinaryRow partition(int id) { - return partitions[id]; - } - } - - private static final class CollectedDeletes { - - private final CompactFileIdentifierSet identifiers = new CompactFileIdentifierSet(); - private final ManifestFileSorter.DeletedRowIdSet rowIds = - new ManifestFileSorter.DeletedRowIdSet(); - private final Set partitions = new HashSet<>(); - - private void add( - ProjectedManifestEntry entry, boolean collectRowIds, boolean collectPartitions) { - identifiers.add(entry); - if (collectPartitions) { - partitions.add(entry.partition().copy()); - } - if (collectRowIds) { - rowIds.add(entry.file().nonNullFirstRowId()); - } - } - - private void release() { - identifiers.release(); - rowIds.releaseRangeIndex(); - } - } - /** * Compacts manifests in input order. RowID manifests can copy unaffected ADD-only Avro blocks * verbatim; manifests without RowID use identifiers to filter decoded entries. @@ -558,17 +335,20 @@ private static List mergeMinorManifests( @Nullable Integer manifestReadParallelism) throws Exception { boolean useRowIdFilter = allContainsRowId(manifests); - CollectedDeletes deletes = + final CollectedDeletes deletes = collectDeletes( - manifests, manifestFile, useRowIdFilter, false, manifestReadParallelism); + manifests, + manifestFile, + useRowIdFilter, + false, + manifestReadParallelism) + .toImmutable(); try { - CompactionFilter filter = - new CompactionFilter(deletes.identifiers, deletes.rowIds, useRowIdFilter); return rewriteManifests( manifests, manifestFile, partitionType, - filter, + deletes, false, null, null, @@ -582,7 +362,7 @@ private static List rewriteManifests( List manifests, ManifestFile manifestFile, RowType partitionType, - CompactionFilter filter, + CollectedDeletes deletes, boolean fullCompaction, @Nullable Filter mustChange, @Nullable List unchangedManifests, @@ -594,23 +374,23 @@ private static List rewriteManifests( PartitionDictionary partitions = new PartitionDictionary(); SimpleStatsConverter partitionStatsConverter = new SimpleStatsConverter(partitionType); EncodedEntry metadata = new EncodedEntry(); - boolean hasDeletes = !filter.deletedIdentifiers.isEmpty(); + ReusableIdentifier reusableIdentifier = new ReusableIdentifier(); + boolean hasDeletes = !deletes.isEmpty(); try { if (hasDeletes - && filter.useRowIdFilter + && deletes.useRowIdFilter() && manifestReadParallelism != null && manifestReadParallelism > 1 && manifests.size() > 1) { // Keep decompression and primitive entry inspection parallel. The batched executor // bounds retained raw blocks to at most one manifest per planning thread, while the // single writer still emits manifests in input order. - filter.deletedRowIds.prepareRangeIndex(); Function> planner = manifest -> { try { return Collections.singletonList( planManifestRewrite( - manifest, manifestFile, partitionType, filter)); + manifest, manifestFile, partitionType, deletes)); } catch (Exception e) { throw new RuntimeException( "Failed to plan manifest rewrite for " @@ -638,7 +418,8 @@ private static List rewriteManifests( writeBlockEntries( block.raw, writer, - filter, + deletes, + reusableIdentifier, fullCompaction, matchedEntries, emittedDeletes, @@ -663,7 +444,8 @@ private static List rewriteManifests( partitionType, partitionStatsConverter, partitions, - filter, + deletes, + reusableIdentifier, matchedEntries, emittedDeletes, metadata, @@ -688,7 +470,8 @@ private static List rewriteManifests( partitionType, partitionStatsConverter, partitions, - filter, + deletes, + reusableIdentifier, fullCompaction, matchedEntries, emittedDeletes, @@ -702,6 +485,7 @@ private static List rewriteManifests( writer.abort(); throw failure; } finally { + reusableIdentifier.release(); matchedEntries.release(); emittedDeletes.release(); } @@ -711,11 +495,9 @@ private static ManifestRewritePlan planManifestRewrite( ManifestFileMeta manifest, ManifestFile manifestFile, RowType partitionType, - CompactionFilter filter) + CollectedDeletes deletes) throws Exception { - CompactionFilter taskFilter = - new CompactionFilter( - filter.deletedIdentifiers, filter.deletedRowIds, filter.useRowIdFilter); + ReusableIdentifier reusableIdentifier = new ReusableIdentifier(); PartitionDictionary partitions = new PartitionDictionary(); SimpleStatsConverter partitionStatsConverter = new SimpleStatsConverter(partitionType); try (ManifestAvroReader reader = @@ -732,10 +514,13 @@ private static ManifestRewritePlan planManifestRewrite( partitionType, partitionStatsConverter, partitions, - taskFilter, + deletes, + reusableIdentifier, encodedRecordsCompatible))); } return new ManifestRewritePlan(manifest, encodedRecordsCompatible, blocks); + } finally { + reusableIdentifier.release(); } } @@ -745,7 +530,8 @@ private static boolean rewriteOptionalManifest( RowType partitionType, SimpleStatsConverter partitionStatsConverter, PartitionDictionary partitions, - CompactionFilter filter, + CollectedDeletes deletes, + ReusableIdentifier reusableIdentifier, CompactFileIdentifierSet matchedEntries, CompactFileIdentifierSet emittedDeletes, EncodedEntry metadata, @@ -761,7 +547,8 @@ private static boolean rewriteOptionalManifest( partitionType, partitionStatsConverter, partitions, - filter, + deletes, + reusableIdentifier, encodedRecordsCompatible); if (block.unchanged) { pendingBlocks.add(rawBlock.stableCopy()); @@ -776,7 +563,8 @@ private static boolean rewriteOptionalManifest( writeBlockEntries( pending, writer, - filter, + deletes, + reusableIdentifier, true, matchedEntries, emittedDeletes, @@ -789,7 +577,8 @@ private static boolean rewriteOptionalManifest( writeBlockEntries( rawBlock, writer, - filter, + deletes, + reusableIdentifier, true, matchedEntries, emittedDeletes, @@ -801,7 +590,8 @@ private static boolean rewriteOptionalManifest( partitionType, partitionStatsConverter, partitions, - filter, + deletes, + reusableIdentifier, true, matchedEntries, emittedDeletes, @@ -818,7 +608,8 @@ private static void writeRemainingBlocks( RowType partitionType, SimpleStatsConverter partitionStatsConverter, PartitionDictionary partitions, - CompactionFilter filter, + CollectedDeletes deletes, + ReusableIdentifier reusableIdentifier, boolean fullCompaction, CompactFileIdentifierSet matchedEntries, CompactFileIdentifierSet emittedDeletes, @@ -827,14 +618,15 @@ private static void writeRemainingBlocks( throws Exception { while (reader.hasNext()) { RawBlock rawBlock = reader.next(); - if (encodedRecordsCompatible && filter.useRowIdFilter) { + if (encodedRecordsCompatible && deletes.useRowIdFilter()) { CompactionBlock block = inspectBlock( rawBlock, partitionType, partitionStatsConverter, partitions, - filter, + deletes, + reusableIdentifier, true); if (block.metadata != null) { writer.writeEncodedBlock(rawBlock.encodedBlock(), block.metadata); @@ -845,7 +637,8 @@ private static void writeRemainingBlocks( writeBlockEntries( rawBlock, writer, - filter, + deletes, + reusableIdentifier, fullCompaction, matchedEntries, emittedDeletes, @@ -859,65 +652,65 @@ private static CompactionBlock inspectBlock( RowType partitionType, SimpleStatsConverter partitionStatsConverter, PartitionDictionary partitions, - CompactionFilter filter, + CollectedDeletes deletes, + ReusableIdentifier reusableIdentifier, boolean encodedRecordsCompatible) throws Exception { - boolean deferDeletedAddCheck = encodedRecordsCompatible && filter.useRowIdFilter; + boolean deferDeletedAddCheck = encodedRecordsCompatible && deletes.useRowIdFilter(); CompactionBlock block = new CompactionBlock(deferDeletedAddCheck, partitionType); - RowIterator rows = rawBlock.toRows(ENTRY_LAYOUT); - CompactionKey key = new CompactionKey(); + RowIterator rows = + rawBlock.toRows(ProjectedManifestEntry.ENTRY_LAYOUT_PROJECTION.projectedType()); + ProjectedManifestEntry entry = ProjectedManifestEntry.ENTRY_LAYOUT_PROJECTION.createEntry(); while (rows.hasNext()) { - GenericRow row = rows.next(); - key.replace(row); - block.collect(row, key, filter, partitions, deferDeletedAddCheck); + entry.replace(rows.next()); + if (!block.collect( + entry, deletes, reusableIdentifier, partitions, deferDeletedAddCheck)) { + break; + } } block.finish(partitionStatsConverter, partitions); - block.finishFiltering(filter, deferDeletedAddCheck); + block.finishFiltering(deletes, deferDeletedAddCheck); return block; } private static void writeBlockEntries( RawBlock rawBlock, ManifestAvroWriter writer, - CompactionFilter filter, + CollectedDeletes deletes, + ReusableIdentifier reusableIdentifier, boolean fullCompaction, CompactFileIdentifierSet matchedEntries, CompactFileIdentifierSet emittedDeletes, EncodedEntry metadata, boolean encodedRecordsCompatible) throws Exception { - RowIterator rows = - rawBlock.toRows( - encodedRecordsCompatible ? ENTRY_LAYOUT : ManifestEntry.MANIFEST_ROW_TYPE); - CompactionKey key = new CompactionKey(); - GenericRow compactRow = - encodedRecordsCompatible ? null : new GenericRow(ENTRY_LAYOUT.getFieldCount()); - GenericRow compactFile = encodedRecordsCompatible ? null : new GenericRow(FILE_FIELD_COUNT); + ProjectedManifestEntry.Projection projection = + (encodedRecordsCompatible + ? ProjectedManifestEntry.ENTRY_LAYOUT_PROJECTION + : ProjectedManifestEntry.fullProjection()); + RowIterator rows = rawBlock.toRows(projection.projectedType()); + ProjectedManifestEntry entry = projection.createEntry(); while (rows.hasNext()) { GenericRow sourceRow = rows.next(); - GenericRow row = - encodedRecordsCompatible - ? sourceRow - : projectEntryLayout(sourceRow, compactRow, compactFile); - key.replace(row); + entry.replace(sourceRow); if (fullCompaction) { - if (key.kind == FileKind.ADD.toByteValue() && !filter.isDeleted(row, key)) { + if (entry.isAdd() && !deletes.isDeleted(entry, reusableIdentifier)) { writeCompactedEntry( - writer, rows, sourceRow, row, key, metadata, encodedRecordsCompatible); + writer, rows, sourceRow, entry, metadata, encodedRecordsCompatible); } - } else if (key.kind == FileKind.ADD.toByteValue()) { - if (filter.isDeleted(row, key)) { - matchedEntries.add(filter.identifier(row)); + } else if (entry.isAdd()) { + if (deletes.isDeleted(entry, reusableIdentifier)) { + matchedEntries.add(reusableIdentifier.replaceWithPartition(entry)); } else { writeCompactedEntry( - writer, rows, sourceRow, row, key, metadata, encodedRecordsCompatible); + writer, rows, sourceRow, entry, metadata, encodedRecordsCompatible); } } else { - ReusableIdentifier identifier = filter.identifier(row); + ReusableIdentifier identifier = reusableIdentifier.replaceWithPartition(entry); if (!matchedEntries.contains(identifier) && !emittedDeletes.contains(identifier)) { emittedDeletes.add(identifier); writeCompactedEntry( - writer, rows, sourceRow, row, key, metadata, encodedRecordsCompatible); + writer, rows, sourceRow, entry, metadata, encodedRecordsCompatible); } } } @@ -927,30 +720,29 @@ private static void writeCompactedEntry( ManifestAvroWriter writer, RowIterator rows, GenericRow sourceRow, - GenericRow row, - CompactionKey key, + ProjectedManifestEntry entry, EncodedEntry metadata, boolean encodedRecordsCompatible) throws Exception { - InternalRow file = entryFile(row); - BinaryRow partition = SerializationUtils.deserializeBinaryRow(row.getBinary(PARTITION)); - if (key.hasRowId) { + ProjectedDataFileMeta file = entry.file(); + BinaryRow partition = entry.partition(); + if (file.hasFirstRowId()) { metadata.replace( - key.kind, + entry.kind().toByteValue(), partition, - row.getInt(BUCKET), - file.getInt(LEVEL), - file.getLong(SCHEMA_ID), - key.firstRowId, - file.getLong(ROW_COUNT)); + entry.bucket(), + file.level(), + file.schemaId(), + file.nonNullFirstRowId(), + file.rowCount()); } else { metadata.replaceWithoutRowId( - key.kind, + entry.kind().toByteValue(), partition, - row.getInt(BUCKET), - file.getInt(LEVEL), - file.getLong(SCHEMA_ID), - file.getLong(ROW_COUNT)); + entry.bucket(), + file.level(), + file.schemaId(), + file.rowCount()); } if (encodedRecordsCompatible) { writer.writeEncoded(rows.encodedRecord(), metadata); @@ -973,60 +765,74 @@ private static final class CompactionBlock { private long minRowId = Long.MAX_VALUE; private long maxRowId = Long.MIN_VALUE; private final RowType partitionType; - private @Nullable PartitionCounts partitionCounts; + private final boolean collectMetadata; + private @Nullable Map partitionCounts; private @Nullable EncodedBlock metadata; private CompactionBlock(boolean collectMetadata, RowType partitionType) { this.unchanged = true; this.partitionType = partitionType; - this.partitionCounts = collectMetadata ? new PartitionCounts() : null; + this.collectMetadata = collectMetadata; + this.partitionCounts = collectMetadata ? new HashMap<>() : null; } - private void collect( - GenericRow record, - CompactionKey key, - CompactionFilter filter, + private boolean collect( + ProjectedManifestEntry entry, + CollectedDeletes deletes, + ReusableIdentifier reusableIdentifier, PartitionDictionary partitions, boolean deferDeletedAddCheck) { if (!unchanged) { - return; + return false; } - if (!filter.copyable(record, key, deferDeletedAddCheck)) { + if (!deletes.copyable(entry, reusableIdentifier, deferDeletedAddCheck)) { unchanged = false; partitionCounts = null; - return; + return false; } - if (partitionCounts == null) { - return; + if (!collectMetadata) { + return true; } - InternalRow file = entryFile(record); - if (key.kind == FileKind.ADD.toByteValue()) { + checkState(partitionCounts != null, "Partition counts have already been released."); + ProjectedDataFileMeta file = entry.file(); + if (entry.isAdd()) { addedFiles++; } else { deletedFiles++; } - schemaId = Math.max(schemaId, file.getLong(SCHEMA_ID)); - int bucket = record.getInt(BUCKET); + schemaId = Math.max(schemaId, file.schemaId()); + int bucket = entry.bucket(); minBucket = Math.min(minBucket, bucket); maxBucket = Math.max(maxBucket, bucket); - int level = file.getInt(LEVEL); + int level = file.level(); minLevel = Math.min(minLevel, level); maxLevel = Math.max(maxLevel, level); - minRowId = Math.min(minRowId, key.firstRowId); - maxRowId = Math.max(maxRowId, key.rangeEnd); - partitionCounts.add(partitions.id(record.getBinary(PARTITION))); + long firstRowId = file.nonNullFirstRowId(); + minRowId = Math.min(minRowId, firstRowId); + maxRowId = Math.max(maxRowId, firstRowId + file.rowCount() - 1L); + partitionCounts.merge(partitions.id(entry.partitionBytes()), 1, Integer::sum); + return true; } private void finish( SimpleStatsConverter partitionStatsConverter, PartitionDictionary partitions) { - if (!unchanged || partitionCounts == null) { + if (!unchanged || !collectMetadata) { return; } + checkState(partitionCounts != null, "Partition counts have already been released."); SimpleStatsCollector collector = new SimpleStatsCollector(partitionType); long[] nullCounts = new long[partitionType.getFieldCount()]; - partitionCounts.collect(partitions, collector, nullCounts); + for (Map.Entry entry : partitionCounts.entrySet()) { + BinaryRow partition = partitions.partition(entry.getKey()); + collector.collect(partition); + for (int field = 0; field < nullCounts.length; field++) { + if (partition.isNullAt(field)) { + nullCounts[field] = Math.addExact(nullCounts[field], entry.getValue()); + } + } + } SimpleColStats[] stats = collector.extract(); for (int field = 0; field < stats.length; field++) { stats[field] = @@ -1048,10 +854,10 @@ private void finish( partitionCounts = null; } - private void finishFiltering(CompactionFilter filter, boolean deferDeletedAddCheck) { + private void finishFiltering(CollectedDeletes deletes, boolean deferDeletedAddCheck) { if (deferDeletedAddCheck && metadata != null - && !filter.canCopyRange(minRowId, maxRowId)) { + && deletes.intersectsRowIds(minRowId, maxRowId)) { metadata = null; unchanged = false; } @@ -1093,85 +899,4 @@ private boolean unchanged() { return true; } } - - /** Primitive partition-id counts retained only while one Avro block is inspected. */ - private static final class PartitionCounts { - - private static final float LOAD_FACTOR = 0.75f; - - private int[] keys = new int[16]; - private long[] counts = new long[16]; - private int size; - - private PartitionCounts() { - Arrays.fill(keys, -1); - } - - private void add(int partitionId) { - if (size + 1 > keys.length * LOAD_FACTOR) { - resize(); - } - int mask = keys.length - 1; - int slot = mix(partitionId) & mask; - while (true) { - int existing = keys[slot]; - if (existing == -1) { - keys[slot] = partitionId; - counts[slot] = 1; - size++; - return; - } - if (existing == partitionId) { - counts[slot]++; - return; - } - slot = (slot + 1) & mask; - } - } - - private void collect( - PartitionDictionary partitions, SimpleStatsCollector collector, long[] nullCounts) { - for (int slot = 0; slot < keys.length; slot++) { - if (keys[slot] < 0) { - continue; - } - BinaryRow partition = partitions.partition(keys[slot]); - collector.collect(partition); - for (int field = 0; field < nullCounts.length; field++) { - if (partition.isNullAt(field)) { - nullCounts[field] = Math.addExact(nullCounts[field], counts[slot]); - } - } - } - } - - private void resize() { - int[] previousKeys = keys; - long[] previousCounts = counts; - keys = new int[previousKeys.length << 1]; - counts = new long[keys.length]; - Arrays.fill(keys, -1); - int mask = keys.length - 1; - for (int slot = 0; slot < previousKeys.length; slot++) { - int key = previousKeys[slot]; - if (key < 0) { - continue; - } - int target = mix(key) & mask; - while (keys[target] != -1) { - target = (target + 1) & mask; - } - keys[target] = key; - counts[target] = previousCounts[slot]; - } - } - - private static int mix(int value) { - value ^= value >>> 16; - value *= 0x7feb352d; - value ^= value >>> 15; - value *= 0x846ca68b; - return value ^ (value >>> 16); - } - } } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java index 574286771fb8..eb9010461fc9 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java @@ -27,6 +27,7 @@ import org.apache.paimon.data.InternalRow; import org.apache.paimon.disk.IOManager; import org.apache.paimon.manifest.CompactFileIdentifierSet; +import org.apache.paimon.manifest.DeletedRowIdSet; import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.manifest.ManifestFile; import org.apache.paimon.manifest.ManifestFileMeta; @@ -161,128 +162,6 @@ private DeletedEntryInfo( } } - /** Primitive set used by RowID full compaction to avoid rebuilding file identifiers. */ - static final class DeletedRowIdSet { - - private static final long EMPTY = Long.MIN_VALUE; - private long[] table = emptyTable(16); - private int size; - private boolean containsMinValue; - private @Nullable long[] sortedRowIds; - - void add(long value) { - if (value == EMPTY) { - if (!containsMinValue) { - containsMinValue = true; - size++; - sortedRowIds = null; - } - return; - } - if ((size + 1) * 2 > table.length) { - grow(); - } - int slot = slot(value, table.length); - while (table[slot] != EMPTY) { - if (table[slot] == value) { - return; - } - slot = (slot + 1) & (table.length - 1); - } - table[slot] = value; - size++; - sortedRowIds = null; - } - - boolean contains(long value) { - if (value == EMPTY) { - return containsMinValue; - } - int slot = slot(value, table.length); - while (table[slot] != EMPTY) { - if (table[slot] == value) { - return true; - } - slot = (slot + 1) & (table.length - 1); - } - return false; - } - - boolean intersects(long minInclusive, long maxInclusive) { - if (minInclusive > maxInclusive) { - return true; - } - long[] values = sortedRowIds(); - int position = java.util.Arrays.binarySearch(values, minInclusive); - if (position < 0) { - position = -position - 1; - } - return position < values.length && values[position] <= maxInclusive; - } - - private long[] sortedRowIds() { - if (sortedRowIds != null) { - return sortedRowIds; - } - long[] values = new long[size]; - int position = 0; - if (containsMinValue) { - values[position++] = EMPTY; - } - for (long value : table) { - if (value != EMPTY) { - values[position++] = value; - } - } - if (position != size) { - throw new IllegalStateException("Failed to snapshot deleted RowID set."); - } - java.util.Arrays.sort(values); - sortedRowIds = values; - return values; - } - - void prepareRangeIndex() { - // Publish the immutable sorted snapshot before concurrent manifest planning starts. - sortedRowIds(); - } - - void releaseRangeIndex() { - sortedRowIds = null; - } - - private void grow() { - long[] previous = table; - if (previous.length >= (1 << 30)) { - throw new IllegalStateException("Too many deleted RowIDs in one manifest group."); - } - table = emptyTable(previous.length << 1); - int previousSize = size; - size = containsMinValue ? 1 : 0; - for (long value : previous) { - if (value != EMPTY) { - add(value); - } - } - if (size != previousSize) { - throw new IllegalStateException("Failed to grow deleted RowID set."); - } - } - - private static int slot(long value, int length) { - value ^= value >>> 33; - value *= 0xff51afd7ed558ccdL; - value ^= value >>> 33; - return ((int) value) & (length - 1); - } - - private static long[] emptyTable(int length) { - long[] table = new long[length]; - java.util.Arrays.fill(table, EMPTY); - return table; - } - } - /** * Try to sort-rewrite the merged manifest list by a configured partition field. If the sort * field cannot be resolved, the input is returned as-is. diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/CompactFileIdentifierSetTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/CompactFileIdentifierSetTest.java index 95d071eabee3..7eeab708411d 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/CompactFileIdentifierSetTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/CompactFileIdentifierSetTest.java @@ -79,6 +79,25 @@ void testGrowsAndReleases() { assertThat(identifiers.contains(1, new byte[] {1}, 1)).isTrue(); } + @Test + void testAddAll() { + CompactFileIdentifierSet first = new CompactFileIdentifierSet(); + first.add(1, new byte[] {1, 2}, 2); + first.add(2, new byte[] {3, 4, 5}, 3); + + CompactFileIdentifierSet second = new CompactFileIdentifierSet(); + second.add(2, new byte[] {3, 4, 5}, 3); + second.add(3, new byte[] {6, 7, 8, 9}, 4); + + first.addAll(second); + + assertThat(first.size()).isEqualTo(3); + assertThat(first.retainedIdentifierBytes()).isEqualTo(9); + assertThat(first.contains(1, new byte[] {1, 2}, 2)).isTrue(); + assertThat(first.contains(2, new byte[] {3, 4, 5}, 3)).isTrue(); + assertThat(first.contains(3, new byte[] {6, 7, 8, 9}, 4)).isTrue(); + } + @Test void testRejectsInvalidIdentifier() { CompactFileIdentifierSet identifiers = new CompactFileIdentifierSet();