[core] Speed up row-id manifest sorting

docs/generated/core_configuration.html

@@ -1047,6 +1047,12 @@
             <td>Integer</td>
             <td>To avoid frequent manifest merges, this parameter specifies the minimum number of ManifestFileMeta to merge.<br />Note: when 'manifest-sort.enabled' is true, this minimum-count gate is only applied to the trailing sub-segment of a section that exceeds 'manifest-sort.max-rewrite-size'. Small under-budget sections are sorted and rewritten directly, so two small manifest files may be merged into one even when their count is below this threshold and full compaction is not triggered.</td>
         </tr>
+        <tr>
+            <td><h5>manifest.merge-optimize.enabled</h5></td>
+            <td style="word-wrap: break-word;">true</td>
+            <td>Boolean</td>
+            <td>Whether to enable optimized manifest compaction. When enabled, block-aware compaction and streaming run merge are used. When disabled, ordinary compaction uses the legacy merger and RowID-based sorting uses the external sorter.</td>
+        </tr>
         <tr>
             <td><h5>manifest.target-file-size</h5></td>
             <td style="word-wrap: break-word;">8 mb</td>

paimon-api/src/main/java/org/apache/paimon/CoreOptions.java

@@ -589,6 +589,16 @@ public InlineElement getDescription() {
                                     + " skipped. Set to a larger value to allow more aggressive"
                                     + " sort rewriting. The cap only limits the sorted rewrite portion and full/minor cleanup may still happen beyond it.");
 
+    public static final ConfigOption<Boolean> MANIFEST_MERGE_OPTIMIZE_ENABLED =
+            key("manifest.merge-optimize.enabled")
+                    .booleanType()
+                    .defaultValue(true)
+                    .withDescription(
+                            "Whether to enable optimized manifest compaction. When enabled,"
+                                    + " block-aware compaction and streaming run merge are used."
+                                    + " When disabled, ordinary compaction uses the legacy merger"
+                                    + " and RowID-based sorting uses the external sorter.");
+
     public static final ConfigOption<String> PARTITION_DEFAULT_NAME =
             key("partition.default-name")
                     .stringType()
@@ -3066,6 +3076,10 @@ public long manifestSortMaxRewriteSize() {
         return options.get(MANIFEST_SORT_MAX_REWRITE_SIZE).getBytes();
     }
 
+    public boolean manifestMergeOptimizeEnabled() {
+        return options.get(MANIFEST_MERGE_OPTIMIZE_ENABLED);
+    }
+
     public String partitionDefaultName() {
         return options.get(PARTITION_DEFAULT_NAME);
     }

paimon-common/src/main/java/org/apache/paimon/utils/ByteArrayKey.java

@@ -47,8 +47,7 @@ byte[] bytes() {
     public boolean equals(Object obj) {
         return obj == this
                 || (obj instanceof ByteArrayKey && Arrays.equals(bytes, ((ByteArrayKey) obj).bytes))
-                || (obj instanceof ByteArrayLookupKey
-                        && Arrays.equals(bytes, ((ByteArrayLookupKey) obj).bytes()));
+                || (obj instanceof ByteArrayLookupKey && obj.equals(this));
     }
 
     @Override

paimon-common/src/main/java/org/apache/paimon/utils/ByteArrayLookupKey.java

@@ -20,8 +20,6 @@
 
 import javax.annotation.Nullable;
 
-import java.util.Arrays;
-
 import static org.apache.paimon.utils.Preconditions.checkArgument;
 
 /**
@@ -33,6 +31,8 @@
 public final class ByteArrayLookupKey {
 
     private @Nullable byte[] bytes;
+    private int offset;
+    private int length;
     private int hash;
 
     public ByteArrayLookupKey() {}
@@ -43,12 +43,26 @@ public ByteArrayLookupKey(byte[] bytes) {
 
     public void reset(byte[] bytes) {
         checkArgument(bytes != null, "Byte array cannot be null.");
+        reset(bytes, 0, bytes.length);
+    }
+
+    public void reset(byte[] bytes, int offset, int length) {
+        checkArgument(bytes != null, "Byte array cannot be null.");
+        checkArgument(offset >= 0 && length >= 0 && offset <= bytes.length - length);
         this.bytes = bytes;
-        this.hash = Arrays.hashCode(bytes);
+        this.offset = offset;
+        this.length = length;
+        int hash = 1;
+        for (int i = offset; i < offset + length; i++) {
+            hash = 31 * hash + bytes[i];
+        }
+        this.hash = hash;
     }
 
     public void clear() {
         bytes = null;
+        offset = 0;
+        length = 0;
         hash = 0;
     }
 
@@ -62,14 +76,38 @@ public boolean equals(Object obj) {
         return obj == this
                 || (bytes != null
                         && obj instanceof ByteArrayKey
-                        && Arrays.equals(bytes, ((ByteArrayKey) obj).bytes()))
+                        && equals(((ByteArrayKey) obj).bytes()))
                 || (bytes != null
                         && obj instanceof ByteArrayLookupKey
-                        && Arrays.equals(bytes, ((ByteArrayLookupKey) obj).bytes));
+                        && equals((ByteArrayLookupKey) obj));
     }
 
     @Override
     public int hashCode() {
         return hash;
     }
+
+    private boolean equals(byte[] other) {
+        if (length != other.length) {
+            return false;
+        }
+        for (int i = 0; i < length; i++) {
+            if (bytes[offset + i] != other[i]) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    private boolean equals(ByteArrayLookupKey other) {
+        if (other.bytes == null || length != other.length) {
+            return false;
+        }
+        for (int i = 0; i < length; i++) {
+            if (bytes[offset + i] != other.bytes[other.offset + i]) {
+                return false;
+            }
+        }
+        return true;
+    }
 }

paimon-common/src/test/java/org/apache/paimon/utils/ByteArrayKeyTest.java

@@ -59,6 +59,23 @@ void testReusableMapLookup() {
         assertThat(lookup.hashCode()).isZero();
     }
 
+    @Test
+    void testReusableSliceLookup() {
+        Map<ByteArrayKey, String> values = new HashMap<>();
+        ByteArrayKey key = new ByteArrayKey(new byte[] {1, 2, 3});
+        values.put(key, "value");
+        ByteArrayLookupKey lookup = new ByteArrayLookupKey();
+
+        lookup.reset(new byte[] {9, 1, 2, 3, 8}, 1, 3);
+        assertThat(lookup).isEqualTo(key);
+        assertThat(key).isEqualTo(lookup);
+        assertThat(lookup.hashCode()).isEqualTo(key.hashCode());
+        assertThat(values.get(lookup)).isEqualTo("value");
+
+        lookup.clear();
+        assertThat(new ByteArrayLookupKey(new byte[] {1, 2, 3})).isNotEqualTo(lookup);
+    }
+
     @Test
     void testLookupEqualityLifecycle() {
         ByteArrayLookupKey first = new ByteArrayLookupKey(new byte[] {1});

paimon-core/src/main/java/org/apache/paimon/io/ProjectedDataFileMeta.java

@@ -230,6 +230,14 @@ public boolean hasFirstRowId() {
         return !currentRow().isNullAt(requiredPosition(Fields.FIRST_ROW_ID));
     }
 
+    @Override
+    public long nonNullFirstRowId() {
+        int position = requiredPosition(Fields.FIRST_ROW_ID);
+        InternalRow row = currentRow();
+        checkState(!row.isNullAt(position), "First row id cannot be null.");
+        return row.getLong(position);
+    }
+
     @Nullable
     @Override
     public Long firstRowId() {

paimon-core/src/main/java/org/apache/paimon/manifest/CollectedDeletes.java

@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.manifest;
+
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.io.ProjectedDataFileMeta;
+import org.apache.paimon.manifest.FileEntry.ReusableIdentifier;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.apache.paimon.utils.Preconditions.checkState;
+
+/** DELETE identifiers and optional RowID and partition indexes collected for manifest merging. */
+public final class CollectedDeletes {
+
+    private final CompactFileIdentifierSet identifiers = new CompactFileIdentifierSet();
+    private final DeletedRowIdSet rowIds = new DeletedRowIdSet();
+    private Set<BinaryRow> partitions = new HashSet<>();
+    private final boolean useRowIdFilter;
+    private boolean immutable;
+
+    public CollectedDeletes(boolean useRowIdFilter) {
+        this.useRowIdFilter = useRowIdFilter;
+    }
+
+    public void add(
+            ProjectedManifestEntry entry, boolean collectRowIds, boolean collectPartitions) {
+        checkState(!immutable, "Cannot modify an immutable DELETE collection.");
+        identifiers.add(entry);
+        if (collectPartitions) {
+            partitions.add(entry.partition().copy());
+        }
+        if (collectRowIds) {
+            rowIds.add(entry.file().nonNullFirstRowId());
+        }
+    }
+
+    public void combine(CollectedDeletes other) {
+        checkState(!immutable, "Cannot modify an immutable DELETE collection.");
+        checkState(
+                useRowIdFilter == other.useRowIdFilter,
+                "Cannot combine DELETE collections with different RowID modes.");
+        identifiers.addAll(other.identifiers);
+        rowIds.addAll(other.rowIds);
+        partitions.addAll(other.partitions);
+    }
+
+    public CollectedDeletes toImmutable() {
+        checkState(!immutable, "Cannot modify an immutable DELETE collection.");
+        if (useRowIdFilter) {
+            rowIds.prepareRangeIndex();
+        }
+        partitions = Collections.unmodifiableSet(partitions);
+        immutable = true;
+        return this;
+    }
+
+    public boolean isEmpty() {
+        return identifiers.isEmpty();
+    }
+
+    public Set<BinaryRow> partitions() {
+        return partitions;
+    }
+
+    public boolean useRowIdFilter() {
+        return useRowIdFilter;
+    }
+
+    public boolean isDeleted(ProjectedManifestEntry entry, ReusableIdentifier reusableIdentifier) {
+        if (useRowIdFilter) {
+            ProjectedDataFileMeta file = entry.file();
+            checkState(file.hasFirstRowId(), "First row id should not be null.");
+            if (!rowIds.contains(file.nonNullFirstRowId())) {
+                return false;
+            }
+        }
+        return identifiers.contains(reusableIdentifier.replaceWithPartition(entry));
+    }
+
+    public boolean copyable(
+            ProjectedManifestEntry entry,
+            ReusableIdentifier reusableIdentifier,
+            boolean deferDeletedAddCheck) {
+        return entry.isAdd() && (deferDeletedAddCheck || !isDeleted(entry, reusableIdentifier));
+    }
+
+    public boolean intersectsRowIds(long minRowId, long maxRowId) {
+        return rowIds.intersects(minRowId, maxRowId);
+    }
+
+    public void release() {
+        identifiers.release();
+        rowIds.releaseRangeIndex();
+    }
+}