[feature](fe) Add byte-weighted external metadata caches

fe/fe-benchmark/pom.xml

@@ -0,0 +1,92 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.apache.doris</groupId>
+        <artifactId>fe</artifactId>
+        <version>${revision}</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+
+    <artifactId>fe-benchmark</artifactId>
+    <name>Doris FE Benchmarks</name>
+
+    <properties>
+        <jmh.version>1.37</jmh.version>
+        <jol.version>0.17</jol.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>fe-connector-hive</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>fe-connector-iceberg</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>fe-connector-paimon</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.openjdk.jmh</groupId>
+            <artifactId>jmh-core</artifactId>
+            <version>${jmh.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.openjdk.jmh</groupId>
+            <artifactId>jmh-generator-annprocess</artifactId>
+            <version>${jmh.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.openjdk.jol</groupId>
+            <artifactId>jol-core</artifactId>
+            <version>${jol.version}</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <release>17</release>
+                    <annotationProcessorPaths>
+                        <path>
+                            <groupId>org.openjdk.jmh</groupId>
+                            <artifactId>jmh-generator-annprocess</artifactId>
+                            <version>${jmh.version}</version>
+                        </path>
+                    </annotationProcessorPaths>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+</project>

fe/fe-benchmark/run-metacache-estimator-benchmark.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -euo pipefail
+
+BENCHMARK_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
+FE_DIR=$(cd -- "${BENCHMARK_DIR}/.." && pwd)
+BENCHMARK_PATTERN=${1:-'(HiveFileListingSizeBenchmark|IcebergPartitionSizeBenchmark|IcebergManifestSizeBenchmark|PaimonPartitionViewSizeBenchmark)'}
+if [[ $# -gt 0 ]]; then
+    shift
+fi
+
+CLASSPATH_FILE=$(mktemp)
+trap 'rm -f "${CLASSPATH_FILE}"' EXIT
+
+(
+    cd "${FE_DIR}"
+    mvn -Pbenchmark -pl fe-benchmark -am test-compile -DskipTests
+    mvn -Pbenchmark -pl fe-benchmark -am dependency:build-classpath \
+        -DincludeScope=test \
+        -Dmdep.outputFile="${CLASSPATH_FILE}"
+)
+
+REACTOR_CLASSES=$(find "${FE_DIR}" -type d -path '*/target/classes' -printf '%p:')
+DEPENDENCY_CLASSES=$(tr -d '\n' < "${CLASSPATH_FILE}")
+
+java \
+    -Djol.magicFieldOffset=true \
+    --add-opens=java.base/java.lang=ALL-UNNAMED \
+    --add-opens=java.base/java.util=ALL-UNNAMED \
+    --add-opens=java.base/java.util.concurrent=ALL-UNNAMED \
+    -classpath "${REACTOR_CLASSES}${DEPENDENCY_CLASSES}" \
+    org.openjdk.jmh.Main "${BENCHMARK_PATTERN}" "$@"

fe/fe-benchmark/src/main/java/org/apache/doris/connector/hive/HiveFileListingSizeBenchmark.java

@@ -0,0 +1,90 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.connector.hive;
+
+import org.apache.doris.connector.hive.HiveFileListingCache.FileListingKey;
+import org.apache.doris.connector.hive.HiveFileListingCache.FileListingValue;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jol.info.GraphLayout;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+/** Compares the O(1) production file-listing weight with construction-time accounting and JOL traversal. */
+@BenchmarkMode(Mode.AverageTime)
+@Warmup(iterations = 1, time = 500, timeUnit = TimeUnit.MILLISECONDS)
+@Measurement(iterations = 3, time = 500, timeUnit = TimeUnit.MILLISECONDS)
+@Fork(value = 1, jvmArgsAppend = {"-Xms1g", "-Xmx4g", "-Djol.magicFieldOffset=true"})
+public class HiveFileListingSizeBenchmark {
+
+    @Benchmark
+    @OutputTimeUnit(TimeUnit.NANOSECONDS)
+    public long cachedWeight(BenchmarkState state) {
+        return HiveFileListingSizeEstimator.estimateEntry(state.key, state.value);
+    }
+
+    @Benchmark
+    @OutputTimeUnit(TimeUnit.MILLISECONDS)
+    public long constructAndEstimate(BenchmarkState state) {
+        return new FileListingValue(state.files).estimatedBytes;
+    }
+
+    @Benchmark
+    @OutputTimeUnit(TimeUnit.MILLISECONDS)
+    public long jolRetainedGraph(BenchmarkState state) {
+        return GraphLayout.parseInstance(state.key, state.value).totalSize();
+    }
+
+    @State(Scope.Thread)
+    public static class BenchmarkState {
+        @Param({"10000", "100000"})
+        public int size;
+
+        private FileListingKey key;
+        private FileListingValue value;
+        private List<HiveFileStatus> files;
+
+        @Setup(Level.Trial)
+        public void setup() {
+            files = new ArrayList<>(size);
+            for (int i = 0; i < size; i++) {
+                files.add(new HiveFileStatus(
+                        "s3://warehouse/db/table/dt=2026-08-07/part-" + i + ".parquet",
+                        128L * 1024L * 1024L + i,
+                        1_786_048_000_000L + i));
+            }
+            key = new FileListingKey("db", "table", "s3://warehouse/db/table/dt=2026-08-07",
+                    Collections.singletonList("2026-08-07"));
+            value = new FileListingValue(files);
+        }
+    }
+}

fe/fe-benchmark/src/main/java/org/apache/doris/connector/iceberg/IcebergManifestSizeBenchmark.java

@@ -0,0 +1,92 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.connector.iceberg;
+
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DataFiles;
+import org.apache.iceberg.ManifestContent;
+import org.apache.iceberg.PartitionSpec;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jol.info.GraphLayout;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+/** Compares the O(1) production manifest weight with construction-time accounting and JOL traversal. */
+@BenchmarkMode(Mode.AverageTime)
+@Warmup(iterations = 1, time = 500, timeUnit = TimeUnit.MILLISECONDS)
+@Measurement(iterations = 3, time = 500, timeUnit = TimeUnit.MILLISECONDS)
+@Fork(value = 1, jvmArgsAppend = {"-Xms1g", "-Xmx4g", "-Djol.magicFieldOffset=true"})
+public class IcebergManifestSizeBenchmark {
+
+    @Benchmark
+    @OutputTimeUnit(TimeUnit.NANOSECONDS)
+    public long cachedWeight(BenchmarkState state) {
+        return IcebergCacheSizeEstimator.estimateManifestEntry(state.key, state.value);
+    }
+
+    @Benchmark
+    @OutputTimeUnit(TimeUnit.MILLISECONDS)
+    public long constructAndEstimate(BenchmarkState state) {
+        return ManifestCacheValue.forDataFiles(state.files).getEstimatedBytes();
+    }
+
+    @Benchmark
+    @OutputTimeUnit(TimeUnit.MILLISECONDS)
+    public long jolRetainedGraph(BenchmarkState state) {
+        return GraphLayout.parseInstance(state.key, state.value).totalSize();
+    }
+
+    @State(Scope.Thread)
+    public static class BenchmarkState {
+        @Param({"10000", "100000"})
+        public int size;
+
+        private IcebergManifestEntryKey key;
+        private ManifestCacheValue value;
+        private List<DataFile> files;
+
+        @Setup(Level.Trial)
+        public void setup() {
+            PartitionSpec spec = PartitionSpec.unpartitioned();
+            files = new ArrayList<>(size);
+            for (int i = 0; i < size; i++) {
+                files.add(DataFiles.builder(spec)
+                        .withPath("s3://warehouse/db/table/data/part-" + i + ".parquet")
+                        .withFileSizeInBytes(128L * 1024L * 1024L + i)
+                        .withRecordCount(1_000_000L + i)
+                        .build());
+            }
+            key = new IcebergManifestEntryKey(
+                    "s3://benchmark-bucket/warehouse/db/table/metadata/manifest.avro", ManifestContent.DATA);
+            value = ManifestCacheValue.forDataFiles(files);
+        }
+    }
+}