From 37040b9a77060d187022429b6f9fe42e9a59b4fd Mon Sep 17 00:00:00 2001 From: guoqiang Date: Thu, 6 Aug 2026 10:41:15 +0800 Subject: [PATCH 1/5] [feature](fe) Add byte-weighted metadata cache framework ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: External metadata caches are currently bounded only by entry count. Add framework support for an optional catalog-level max-weight, an entry-specific size estimator contract, mutually exclusive Caffeine size/weight construction, saturated integer weight conversion, and weighted cache statistics. Complete the framework by supporting weighted caches with synchronous removal listeners, accepting convenient binary size suffixes such as MB while retaining bare-byte compatibility, strictly rejecting malformed, negative, and overflowing max-weight values, and keeping statistics reads lightweight and side-effect free. Existing entries continue to use maximumSize unless they explicitly register an estimator and configure max-weight. Catalog-specific estimators and information_schema exposure are intentionally not included. ### Release note None ### Check List (For Author) - Test: Unit Test - ./run-fe-ut.sh --run org.apache.doris.common.util.ParseUtilTest,org.apache.doris.common.CacheFactoryTest,org.apache.doris.datasource.metacache.CacheSpecTest,org.apache.doris.datasource.metacache.MetaCacheEntryTest,org.apache.doris.datasource.metacache.AbstractExternalMetaCacheTest (80 tests passed) - DISABLE_BUILD_UI=ON ./build.sh --fe (passed, including Checkstyle) - Behavior changed: Yes. Entries that register an estimator and configure max-weight use weighted eviction, invalid max-weight values are rejected, and statistics reads no longer trigger Caffeine maintenance. - Does this need documentation: No --- .../org/apache/doris/common/CacheFactory.java | 43 +++++- .../apache/doris/common/util/ParseUtil.java | 30 +++++ .../metacache/AbstractExternalMetaCache.java | 2 +- .../doris/datasource/metacache/CacheSpec.java | 85 +++++++++++- .../datasource/metacache/MetaCacheEntry.java | 86 +++++++++++- .../metacache/MetaCacheEntryDef.java | 24 +++- .../metacache/MetaCacheEntryStats.java | 38 +++++- .../metacache/MetaCacheSizeEstimator.java | 30 +++++ .../doris/common/util/ParseUtilTest.java | 40 ++++++ .../AbstractExternalMetaCacheTest.java | 54 ++++++++ .../datasource/metacache/CacheSpecTest.java | 49 +++++++ .../metacache/MetaCacheEntryTest.java | 123 ++++++++++++++++++ 12 files changed, 581 insertions(+), 23 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimator.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/common/util/ParseUtilTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/CacheFactory.java b/fe/fe-core/src/main/java/org/apache/doris/common/CacheFactory.java index 2b3abfce4e9a76..eab8103d575daf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/CacheFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/CacheFactory.java @@ -24,6 +24,7 @@ import com.github.benmanes.caffeine.cache.LoadingCache; import com.github.benmanes.caffeine.cache.RemovalListener; import com.github.benmanes.caffeine.cache.Ticker; +import com.github.benmanes.caffeine.cache.Weigher; import org.jetbrains.annotations.NotNull; import java.time.Duration; @@ -70,22 +71,49 @@ public CacheFactory( // Build a loading cache, without executor, it will use fork-join pool for refresh public LoadingCache buildCache(CacheLoader cacheLoader) { - Caffeine builder = buildWithParams(); + Caffeine builder = buildSizeBoundedWithParams(); return builder.build(cacheLoader); } // Build a loading cache, with executor, it will use given executor for refresh public LoadingCache buildCache(CacheLoader cacheLoader, ExecutorService executor) { - Caffeine builder = buildWithParams(); + Caffeine builder = buildSizeBoundedWithParams(); builder.executor(executor); return builder.build(cacheLoader); } + /** + * Build a loading cache bounded by weight instead of entry count. + */ + public LoadingCache buildCacheWithWeight(CacheLoader cacheLoader, + ExecutorService executor, long maxWeight, Weigher weigher) { + Caffeine builder = buildWithParams() + .maximumWeight(maxWeight) + .weigher(weigher); + builder.executor(executor); + return builder.build(cacheLoader); + } + + /** + * Build a loading cache bounded by weight with a synchronous removal listener. + */ + public LoadingCache buildCacheWithWeightAndSyncRemovalListener(CacheLoader cacheLoader, + long maxWeight, Weigher weigher, RemovalListener removalListener) { + Caffeine builder = buildWithParams() + .maximumWeight(maxWeight) + .weigher(weigher); + if (removalListener != null) { + builder.removalListener(removalListener); + } + builder.executor(Runnable::run); // Sync execution to avoid thread pool deadlock + return builder.build(cacheLoader); + } + // Build cache with sync removal listener to prevent deadlock when listener calls invalidateAll() public LoadingCache buildCacheWithSyncRemovalListener(CacheLoader cacheLoader, RemovalListener removalListener) { - Caffeine builder = buildWithParams(); + Caffeine builder = buildSizeBoundedWithParams(); if (removalListener != null) { builder.removalListener(removalListener); } @@ -96,7 +124,7 @@ public LoadingCache buildCacheWithSyncRemovalListener(CacheLoader AsyncLoadingCache buildAsyncCache(AsyncCacheLoader cacheLoader, ExecutorService executor) { - Caffeine builder = buildWithParams(); + Caffeine builder = buildSizeBoundedWithParams(); builder.executor(executor); return builder.buildAsync(cacheLoader); } @@ -104,8 +132,6 @@ public AsyncLoadingCache buildAsyncCache(AsyncCacheLoader cac @NotNull private Caffeine buildWithParams() { Caffeine builder = Caffeine.newBuilder(); - builder.maximumSize(maxSize); - if (expireAfterAccessSec.isPresent()) { builder.expireAfterAccess(Duration.ofSeconds(expireAfterAccessSec.getAsLong())); } @@ -122,4 +148,9 @@ private Caffeine buildWithParams() { } return builder; } + + @NotNull + private Caffeine buildSizeBoundedWithParams() { + return buildWithParams().maximumSize(maxSize); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/ParseUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/ParseUtil.java index 135a8eeac0a2df..fa3494f099a4c9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/ParseUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/ParseUtil.java @@ -70,6 +70,36 @@ public static long analyzeDataVolume(String dataVolumnStr) throws AnalysisExcept return dataVolumn; } + /** + * Parse a data volume while allowing zero and rejecting arithmetic overflow. + */ + public static long analyzeDataVolumeAllowZero(String dataVolumnStr) throws AnalysisException { + long dataVolumn; + Matcher m = dataVolumnPattern.matcher(dataVolumnStr); + if (!m.matches()) { + throw new AnalysisException("invalid data volume expression:" + dataVolumnStr); + } + try { + dataVolumn = Long.parseLong(m.group(1)); + } catch (NumberFormatException nfe) { + throw new AnalysisException("invalid data volume:" + m.group(1)); + } + + String unit = "B"; + String tmpUnit = m.group(2); + if (!Strings.isNullOrEmpty(tmpUnit)) { + unit = tmpUnit.toUpperCase(); + } + if (!validDataVolumnUnitMultiplier.containsKey(unit)) { + throw new AnalysisException("invalid unit:" + tmpUnit); + } + try { + return Math.multiplyExact(dataVolumn, validDataVolumnUnitMultiplier.get(unit)); + } catch (ArithmeticException e) { + throw new AnalysisException("data volume is too large:" + dataVolumnStr); + } + } + public static long analyzeReplicaNumber(String replicaNumberStr) throws AnalysisException { long replicaNumber = 0; try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java index eca9816fd5f982..25b758f55f7a46 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java @@ -300,7 +300,7 @@ private MetaCacheEntry newMetaCacheEntry( wrapSchemaValidator(entryDef.getLoader(), entryDef.getValueType()), cacheSpec, refreshExecutor, entryDef.isAutoRefresh(), entryDef.isContextualOnly(), - MetaCacheEntry.defaultObjectStripeCount()); + MetaCacheEntry.defaultObjectStripeCount(), entryDef.getSizeEstimator()); } private Function wrapSchemaValidator(Function loader, Class valueType) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CacheSpec.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CacheSpec.java index 0bb640ad0d753c..cbab3458ef9ed1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CacheSpec.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CacheSpec.java @@ -17,7 +17,9 @@ package org.apache.doris.datasource.metacache; +import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; +import org.apache.doris.common.util.ParseUtil; import org.apache.commons.lang3.math.NumberUtils; @@ -33,7 +35,9 @@ *
    *
  • enable=false disables cache
    • *
  • ttlSecond=0 disables cache, ttlSecond=-1 means no expiration
    • - *
  • capacity=0 disables cache; capacity is count-based
    • + *
  • capacity=0 disables a count-bounded cache
    • + *
  • when maxWeight is present, it replaces capacity as the effective bound
    • + *
  • maxWeight accepts an optional binary unit such as KB, MB, or GB; a bare number means bytes
    • *
*/ public final class CacheSpec { @@ -43,19 +47,26 @@ public final class CacheSpec { private static final String KEY_ENABLE = ".enable"; private static final String KEY_TTL_SECOND = ".ttl-second"; private static final String KEY_CAPACITY = ".capacity"; + private static final String KEY_MAX_WEIGHT = ".max-weight"; private final boolean enable; private final long ttlSecond; private final long capacity; + private final OptionalLong maxWeight; - private CacheSpec(boolean enable, long ttlSecond, long capacity) { + private CacheSpec(boolean enable, long ttlSecond, long capacity, OptionalLong maxWeight) { this.enable = enable; this.ttlSecond = ttlSecond; this.capacity = capacity; + this.maxWeight = Objects.requireNonNull(maxWeight, "maxWeight is required"); } public static CacheSpec of(boolean enable, long ttlSecond, long capacity) { - return new CacheSpec(enable, ttlSecond, capacity); + return new CacheSpec(enable, ttlSecond, capacity, OptionalLong.empty()); + } + + public static CacheSpec ofWeight(boolean enable, long ttlSecond, long capacity, long maxWeight) { + return new CacheSpec(enable, ttlSecond, capacity, OptionalLong.of(maxWeight)); } public static PropertySpec.Builder propertySpecBuilder() { @@ -77,12 +88,14 @@ public static CacheSpec fromProperties(Map properties, PropertyS boolean enable = getBooleanProperty(properties, propertySpec.getEnableKey(), propertySpec.isDefaultEnable()); long ttlSecond = getLongProperty(properties, propertySpec.getTtlKey(), propertySpec.getDefaultTtlSecond()); long capacity = getLongProperty(properties, propertySpec.getCapacityKey(), propertySpec.getDefaultCapacity()); - return of(enable, ttlSecond, capacity); + OptionalLong maxWeight = getOptionalDataSizeProperty( + properties, propertySpec.getMaxWeightKey(), propertySpec.getDefaultMaxWeight()); + return new CacheSpec(enable, ttlSecond, capacity, maxWeight); } /** * Build a cache spec from catalog properties by standard external meta cache key pattern: - * meta.cache.<engine>.<entry>.(enable|ttl-second|capacity) + * meta.cache.<engine>.<entry>.(enable|ttl-second|capacity|max-weight) */ public static CacheSpec fromProperties(Map properties, String engine, String entryName, CacheSpec defaultSpec) { @@ -95,6 +108,7 @@ public static PropertySpec metaCachePropertySpec(String engine, String entryName .enable(cacheKeyPrefix + KEY_ENABLE, defaultSpec.isEnable()) .ttl(cacheKeyPrefix + KEY_TTL_SECOND, defaultSpec.getTtlSecond()) .capacity(cacheKeyPrefix + KEY_CAPACITY, defaultSpec.getCapacity()) + .maxWeight(cacheKeyPrefix + KEY_MAX_WEIGHT, defaultSpec.getMaxWeight()) .build(); } @@ -193,6 +207,26 @@ private static long getLongProperty(Map properties, String key, return NumberUtils.toLong(value, defaultValue); } + private static OptionalLong getOptionalDataSizeProperty( + Map properties, String key, OptionalLong defaultValue) { + if (key == null) { + return defaultValue; + } + String value = properties.get(key); + if (value == null) { + return defaultValue; + } + try { + return OptionalLong.of(ParseUtil.analyzeDataVolumeAllowZero(value)); + } catch (AnalysisException e) { + throw invalidDataSizeProperty(key, value); + } + } + + private static IllegalArgumentException invalidDataSizeProperty(String key, String value) { + return new IllegalArgumentException("The parameter " + key + " is wrong, value is " + value); + } + public boolean isEnable() { return enable; } @@ -205,6 +239,20 @@ public long getCapacity() { return capacity; } + public OptionalLong getMaxWeight() { + return maxWeight; + } + + public boolean isWeightBounded() { + return maxWeight.isPresent(); + } + + public boolean isCacheEnabled() { + return enable + && ttlSecond != CACHE_TTL_DISABLE_CACHE + && maxWeight.orElse(capacity) != 0L; + } + public static final class PropertySpec { private final String enableKey; private final boolean defaultEnable; @@ -212,15 +260,20 @@ public static final class PropertySpec { private final long defaultTtlSecond; private final String capacityKey; private final long defaultCapacity; + private final String maxWeightKey; + private final OptionalLong defaultMaxWeight; private PropertySpec(String enableKey, boolean defaultEnable, String ttlKey, - long defaultTtlSecond, String capacityKey, long defaultCapacity) { + long defaultTtlSecond, String capacityKey, long defaultCapacity, + String maxWeightKey, OptionalLong defaultMaxWeight) { this.enableKey = enableKey; this.defaultEnable = defaultEnable; this.ttlKey = ttlKey; this.defaultTtlSecond = defaultTtlSecond; this.capacityKey = capacityKey; this.defaultCapacity = defaultCapacity; + this.maxWeightKey = maxWeightKey; + this.defaultMaxWeight = defaultMaxWeight; } public String getEnableKey() { @@ -247,6 +300,14 @@ public long getDefaultCapacity() { return defaultCapacity; } + public String getMaxWeightKey() { + return maxWeightKey; + } + + public OptionalLong getDefaultMaxWeight() { + return defaultMaxWeight; + } + public static final class Builder { private String enableKey; private boolean defaultEnable; @@ -254,6 +315,8 @@ public static final class Builder { private long defaultTtlSecond; private String capacityKey; private long defaultCapacity; + private String maxWeightKey; + private OptionalLong defaultMaxWeight = OptionalLong.empty(); public Builder enable(String key, boolean defaultValue) { this.enableKey = key; @@ -273,6 +336,12 @@ public Builder capacity(String key, long defaultValue) { return this; } + public Builder maxWeight(String key, OptionalLong defaultValue) { + this.maxWeightKey = Objects.requireNonNull(key, "maxWeightKey is required"); + this.defaultMaxWeight = Objects.requireNonNull(defaultValue, "defaultMaxWeight is required"); + return this; + } + public PropertySpec build() { return new PropertySpec( Objects.requireNonNull(enableKey, "enableKey is required"), @@ -280,7 +349,9 @@ public PropertySpec build() { Objects.requireNonNull(ttlKey, "ttlKey is required"), defaultTtlSecond, Objects.requireNonNull(capacityKey, "capacityKey is required"), - defaultCapacity); + defaultCapacity, + maxWeightKey, + defaultMaxWeight); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java index d3590790afed87..15b97f5185281f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java @@ -23,6 +23,7 @@ import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.CacheLoader; import com.github.benmanes.caffeine.cache.LoadingCache; +import com.github.benmanes.caffeine.cache.Policy; import com.github.benmanes.caffeine.cache.RemovalListener; import com.github.benmanes.caffeine.cache.stats.CacheStats; import com.google.common.base.Preconditions; @@ -116,6 +117,13 @@ public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec ca defaultObjectStripeCount(), null, false); } + public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec cacheSpec, + ExecutorService refreshExecutor, boolean autoRefresh, boolean contextualOnly, + @Nullable MetaCacheSizeEstimator sizeEstimator) { + this(name, loader, cacheSpec, refreshExecutor, autoRefresh, contextualOnly, + defaultObjectStripeCount(), sizeEstimator, null, false); + } + public MetaCacheEntry(String name, Function loader, CacheSpec cacheSpec, ExecutorService refreshExecutor, boolean autoRefresh, int stripeCount) { this(name, loader, cacheSpec, refreshExecutor, autoRefresh, false, stripeCount, null, false); @@ -126,6 +134,13 @@ public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec ca this(name, loader, cacheSpec, refreshExecutor, autoRefresh, contextualOnly, stripeCount, null, false); } + public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec cacheSpec, + ExecutorService refreshExecutor, boolean autoRefresh, boolean contextualOnly, int stripeCount, + @Nullable MetaCacheSizeEstimator sizeEstimator) { + this(name, loader, cacheSpec, refreshExecutor, autoRefresh, contextualOnly, + stripeCount, sizeEstimator, null, false); + } + public static MetaCacheEntry withSyncRemovalListener(String name, Function loader, CacheSpec cacheSpec, ExecutorService refreshExecutor, RemovalListener removalListener) { return withSyncRemovalListener(name, loader, cacheSpec, refreshExecutor, @@ -147,9 +162,40 @@ public static MetaCacheEntry withSyncRemovalListener(String name, F true); } + public static MetaCacheEntry withSyncRemovalListener(String name, Function loader, + CacheSpec cacheSpec, ExecutorService refreshExecutor, MetaCacheSizeEstimator sizeEstimator, + RemovalListener removalListener) { + return withSyncRemovalListener(name, loader, cacheSpec, refreshExecutor, + defaultObjectStripeCount(), sizeEstimator, removalListener); + } + + public static MetaCacheEntry withSyncRemovalListener(String name, Function loader, + CacheSpec cacheSpec, ExecutorService refreshExecutor, int stripeCount, + MetaCacheSizeEstimator sizeEstimator, RemovalListener removalListener) { + return new MetaCacheEntry<>( + name, + loader, + cacheSpec, + refreshExecutor, + false, + false, + stripeCount, + Objects.requireNonNull(sizeEstimator, "sizeEstimator can not be null"), + Objects.requireNonNull(removalListener, "removalListener can not be null"), + true); + } + private MetaCacheEntry(String name, @Nullable Function loader, CacheSpec cacheSpec, ExecutorService refreshExecutor, boolean autoRefresh, boolean contextualOnly, int stripeCount, @Nullable RemovalListener removalListener, boolean syncRemovalListener) { + this(name, loader, cacheSpec, refreshExecutor, autoRefresh, contextualOnly, + stripeCount, null, removalListener, syncRemovalListener); + } + + private MetaCacheEntry(String name, @Nullable Function loader, CacheSpec cacheSpec, + ExecutorService refreshExecutor, boolean autoRefresh, boolean contextualOnly, + int stripeCount, @Nullable MetaCacheSizeEstimator sizeEstimator, + @Nullable RemovalListener removalListener, boolean syncRemovalListener) { this.name = Objects.requireNonNull(name, "name can not be null"); if (contextualOnly) { if (loader != null) { @@ -170,6 +216,9 @@ private MetaCacheEntry(String name, @Nullable Function loader, CacheSpec c this.loader = loader; this.cacheSpec = Objects.requireNonNull(cacheSpec, "cacheSpec can not be null"); this.autoRefresh = autoRefresh; + if (cacheSpec.isWeightBounded() && sizeEstimator == null) { + throw new IllegalArgumentException("max-weight requires an entry size estimator: " + name); + } if (stripeCount < 1) { throw new IllegalArgumentException("stripeCount must be positive"); } @@ -180,15 +229,14 @@ private MetaCacheEntry(String name, @Nullable Function loader, CacheSpec c stripeStates.set(0, new StripeState<>()); } Objects.requireNonNull(refreshExecutor, "refreshExecutor can not be null"); - this.effectiveEnabled = CacheSpec.isCacheEnabled( - this.cacheSpec.isEnable(), this.cacheSpec.getTtlSecond(), this.cacheSpec.getCapacity()); + this.effectiveEnabled = this.cacheSpec.isCacheEnabled(); OptionalLong expireAfterAccessSec = effectiveEnabled ? CacheSpec.toExpireAfterAccess(this.cacheSpec.getTtlSecond()) : OptionalLong.empty(); OptionalLong refreshAfterWriteSec = effectiveEnabled && autoRefresh ? OptionalLong.of(Config.external_cache_refresh_time_minutes * 60) : OptionalLong.empty(); - long maxSize = effectiveEnabled ? this.cacheSpec.getCapacity() : 0L; + long maxSize = effectiveEnabled && !cacheSpec.isWeightBounded() ? this.cacheSpec.getCapacity() : 0L; CacheFactory cacheFactory = new CacheFactory( expireAfterAccessSec, refreshAfterWriteSec, @@ -197,7 +245,22 @@ private MetaCacheEntry(String name, @Nullable Function loader, CacheSpec c null); // Build through a dedicated loader so refresh results admitted under an older generation are rejected. CacheLoader cacheLoader = newCacheLoader(); - if (syncRemovalListener) { + if (cacheSpec.isWeightBounded()) { + long maxWeight = effectiveEnabled ? cacheSpec.getMaxWeight().getAsLong() : 0L; + if (syncRemovalListener) { + this.loadingData = cacheFactory.buildCacheWithWeightAndSyncRemovalListener( + cacheLoader, + maxWeight, + (key, value) -> toCaffeineWeight(sizeEstimator.estimateBytes(key, value)), + removalListener); + } else { + this.loadingData = cacheFactory.buildCacheWithWeight( + cacheLoader, + refreshExecutor, + maxWeight, + (key, value) -> toCaffeineWeight(sizeEstimator.estimateBytes(key, value))); + } + } else if (syncRemovalListener) { this.loadingData = cacheFactory.buildCacheWithSyncRemovalListener(cacheLoader, removalListener); } else { this.loadingData = cacheFactory.buildCache(cacheLoader, refreshExecutor); @@ -209,6 +272,13 @@ public String name() { return name; } + private int toCaffeineWeight(long estimatedBytes) { + if (estimatedBytes < 0L) { + throw new IllegalStateException("entry size estimator returned a negative weight: " + name); + } + return estimatedBytes >= Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) estimatedBytes; + } + public V get(K key) { return getWithManualLoad(key, this::applyDefaultLoader, null, null); } @@ -373,7 +443,11 @@ public void forEach(BiConsumer consumer) { } public MetaCacheEntryStats stats() { + // Keep statistics reads lightweight and side-effect free. Caffeine policy values may be briefly stale while + // asynchronous maintenance is pending; querying stats must not trigger expiration or removal listeners. CacheStats cacheStats = loadingData.stats(); + Policy.Eviction evictionPolicy = loadingData.policy().eviction() + .orElseThrow(() -> new IllegalStateException("cache has no eviction policy: " + name)); long successCount = loadSuccessCount.get(); long failureCount = loadFailureCount.get(); long totalLoadTime = totalLoadTimeNanos.get(); @@ -384,7 +458,10 @@ public MetaCacheEntryStats stats() { autoRefresh, cacheSpec.getTtlSecond(), cacheSpec.getCapacity(), + cacheSpec.isWeightBounded(), + cacheSpec.getMaxWeight().orElse(-1L), data.estimatedSize(), + evictionPolicy.weightedSize().orElse(-1L), cacheStats.requestCount(), cacheStats.hitCount(), cacheStats.missCount(), @@ -394,6 +471,7 @@ public MetaCacheEntryStats stats() { totalLoadTime, totalLoadCount == 0 ? 0D : (double) totalLoadTime / totalLoadCount, cacheStats.evictionCount(), + cacheStats.evictionWeight(), invalidateCount.get(), lastLoadSuccessTimeMs.get(), lastLoadFailureTimeMs.get(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java index 1f48057a44fc40..fa475ab2331317 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java @@ -101,10 +101,12 @@ public final class MetaCacheEntryDef { private final boolean autoRefresh; private final boolean contextualOnly; private final MetaCacheEntryInvalidation invalidation; + @Nullable + private final MetaCacheSizeEstimator sizeEstimator; private MetaCacheEntryDef(String name, Class keyType, Class valueType, @Nullable Function loader, CacheSpec defaultCacheSpec, boolean autoRefresh, boolean contextualOnly, - MetaCacheEntryInvalidation invalidation) { + MetaCacheEntryInvalidation invalidation, @Nullable MetaCacheSizeEstimator sizeEstimator) { this.name = Objects.requireNonNull(name, "entry name is required"); this.keyType = Objects.requireNonNull(keyType, "entry key type is required"); this.valueType = Objects.requireNonNull(valueType, "entry value type is required"); @@ -123,6 +125,7 @@ private MetaCacheEntryDef(String name, Class keyType, Class valueType, this.autoRefresh = autoRefresh; this.contextualOnly = contextualOnly; this.invalidation = Objects.requireNonNull(invalidation, "entry invalidation is required"); + this.sizeEstimator = sizeEstimator; } /** @@ -142,7 +145,7 @@ public static MetaCacheEntryDef of(String name, Class keyType, C public static MetaCacheEntryDef of(String name, Class keyType, Class valueType, Function loader, CacheSpec defaultCacheSpec, MetaCacheEntryInvalidation invalidation) { return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, true, false, - invalidation); + invalidation, null); } /** @@ -164,7 +167,7 @@ public static MetaCacheEntryDef of(String name, Class keyType, C Function loader, CacheSpec defaultCacheSpec, boolean autoRefresh, MetaCacheEntryInvalidation invalidation) { return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, autoRefresh, false, - invalidation); + invalidation, null); } /** @@ -179,7 +182,15 @@ public static MetaCacheEntryDef contextualOnly( String name, Class keyType, Class valueType, CacheSpec defaultCacheSpec, MetaCacheEntryInvalidation invalidation) { return new MetaCacheEntryDef<>(name, keyType, valueType, null, defaultCacheSpec, false, true, - invalidation); + invalidation, null); + } + + /** + * Return a copy of this definition with the estimator used by a maximum-weight cache. + */ + public MetaCacheEntryDef withSizeEstimator(MetaCacheSizeEstimator estimator) { + return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, autoRefresh, + contextualOnly, invalidation, Objects.requireNonNull(estimator, "entry size estimator is required")); } /** @@ -232,4 +243,9 @@ public boolean isContextualOnly() { public MetaCacheEntryInvalidation getInvalidation() { return invalidation; } + + @Nullable + public MetaCacheSizeEstimator getSizeEstimator() { + return sizeEstimator; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryStats.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryStats.java index 0c8b875e73038a..fb5dee675c0bf2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryStats.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryStats.java @@ -30,6 +30,8 @@ * *

For last-load timestamps, {@code -1} means no corresponding event happened yet. * {@code lastError} keeps the latest load failure message; empty string means no failure recorded. + * Snapshots do not trigger Caffeine maintenance, so size, weight, and eviction values may be briefly stale while + * asynchronous maintenance is pending. */ public final class MetaCacheEntryStats { private final boolean configEnabled; @@ -37,7 +39,10 @@ public final class MetaCacheEntryStats { private final boolean autoRefresh; private final long ttlSecond; private final long capacity; + private final boolean weightBounded; + private final long maxWeight; private final long estimatedSize; + private final long estimatedWeight; private final long requestCount; private final long hitCount; private final long missCount; @@ -47,6 +52,7 @@ public final class MetaCacheEntryStats { private final long totalLoadTimeNanos; private final double averageLoadPenaltyNanos; private final long evictionCount; + private final long evictionWeight; private final long invalidateCount; private final long lastLoadSuccessTimeMs; private final long lastLoadFailureTimeMs; @@ -61,7 +67,10 @@ public MetaCacheEntryStats( boolean autoRefresh, long ttlSecond, long capacity, + boolean weightBounded, + long maxWeight, long estimatedSize, + long estimatedWeight, long requestCount, long hitCount, long missCount, @@ -71,6 +80,7 @@ public MetaCacheEntryStats( long totalLoadTimeNanos, double averageLoadPenaltyNanos, long evictionCount, + long evictionWeight, long invalidateCount, long lastLoadSuccessTimeMs, long lastLoadFailureTimeMs, @@ -80,7 +90,10 @@ public MetaCacheEntryStats( this.autoRefresh = autoRefresh; this.ttlSecond = ttlSecond; this.capacity = capacity; + this.weightBounded = weightBounded; + this.maxWeight = maxWeight; this.estimatedSize = estimatedSize; + this.estimatedWeight = estimatedWeight; this.requestCount = requestCount; this.hitCount = hitCount; this.missCount = missCount; @@ -90,6 +103,7 @@ public MetaCacheEntryStats( this.totalLoadTimeNanos = totalLoadTimeNanos; this.averageLoadPenaltyNanos = averageLoadPenaltyNanos; this.evictionCount = evictionCount; + this.evictionWeight = evictionWeight; this.invalidateCount = invalidateCount; this.lastLoadSuccessTimeMs = lastLoadSuccessTimeMs; this.lastLoadFailureTimeMs = lastLoadFailureTimeMs; @@ -101,7 +115,7 @@ public boolean isConfigEnabled() { } /** - * Effective cache enable state evaluated by {@link CacheSpec#isCacheEnabled(boolean, long, long)}. + * Effective cache enable state evaluated by {@link CacheSpec#isCacheEnabled()}. */ public boolean isEffectiveEnabled() { return effectiveEnabled; @@ -119,10 +133,28 @@ public long getCapacity() { return capacity; } + public boolean isWeightBounded() { + return weightBounded; + } + + /** + * Returns the configured maximum weight in bytes, or -1 for a count-bounded cache. + */ + public long getMaxWeight() { + return maxWeight; + } + public long getEstimatedSize() { return estimatedSize; } + /** + * Returns Caffeine's current weighted size in bytes, or -1 for a count-bounded cache. + */ + public long getEstimatedWeight() { + return estimatedWeight; + } + public long getRequestCount() { return requestCount; } @@ -162,6 +194,10 @@ public long getEvictionCount() { return evictionCount; } + public long getEvictionWeight() { + return evictionWeight; + } + public double getEvictionRate() { if (requestCount == 0) { return 0D; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimator.java new file mode 100644 index 00000000000000..78e7a506cb264d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimator.java @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +/** + * Estimates the retained heap bytes owned by one cache key/value entry. + * + *

The estimator runs when Caffeine admits or replaces an entry, so implementations should be deterministic and + * inexpensive. Results must be non-negative. Values larger than {@link Integer#MAX_VALUE} are saturated because + * Caffeine's weigher API uses an integer weight. + */ +@FunctionalInterface +public interface MetaCacheSizeEstimator { + long estimateBytes(K key, V value); +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/util/ParseUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/util/ParseUtilTest.java new file mode 100644 index 00000000000000..e7673983182396 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/ParseUtilTest.java @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.util; + +import org.apache.doris.common.AnalysisException; + +import org.junit.Assert; +import org.junit.Test; + +public class ParseUtilTest { + + @Test + public void testLegacyDataVolumeStillRejectsZero() throws Exception { + Assert.assertEquals(1024L, ParseUtil.analyzeDataVolume("1KB")); + Assert.assertThrows(AnalysisException.class, () -> ParseUtil.analyzeDataVolume("0GB")); + } + + @Test + public void testStrictDataVolumeAllowsZeroAndRejectsOverflow() throws Exception { + Assert.assertEquals(0L, ParseUtil.analyzeDataVolumeAllowZero("0GB")); + Assert.assertEquals(512L * 1024 * 1024, ParseUtil.analyzeDataVolumeAllowZero("512MB")); + Assert.assertThrows(AnalysisException.class, + () -> ParseUtil.analyzeDataVolumeAllowZero("9223372036854775807KB")); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCacheTest.java index fee5f074edcc09..11c786220c6e26 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCacheTest.java @@ -31,6 +31,7 @@ import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; public class AbstractExternalMetaCacheTest { @@ -72,6 +73,46 @@ public void testEngineEntriesDoNotInitializeMultiKeyStripeStatesEagerly() { } } + @Test + public void testMaximumWeightRequiresRegisteredEntryEstimator() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + try { + TestExternalMetaCache cache = new TestExternalMetaCache(refreshExecutor); + Map properties = Maps.newHashMap(); + properties.put("meta.cache.test_engine.schema.max-weight", "5"); + + IllegalArgumentException exception = Assert.assertThrows( + IllegalArgumentException.class, () -> cache.initCatalog(1L, properties)); + Assert.assertTrue(exception.getMessage().contains("size estimator")); + } finally { + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testRegisteredEntryEstimatorEnablesMaximumWeight() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + try { + WeightedExternalMetaCache cache = new WeightedExternalMetaCache(refreshExecutor); + Map properties = Maps.newHashMap(); + properties.put("meta.cache.weighted_engine.value.max-weight", "5"); + cache.initCatalog(1L, properties); + + MetaCacheEntry entry = cache.entry(1L, "value", String.class, Integer.class); + entry.put("first", 4); + entry.put("second", 4); + // Wait for Caffeine's queued maintenance without making the statistics read trigger it. + refreshExecutor.submit(() -> null).get(3L, TimeUnit.SECONDS); + + MetaCacheEntryStats stats = entry.stats(); + Assert.assertTrue(stats.isWeightBounded()); + Assert.assertEquals(5L, stats.getMaxWeight()); + Assert.assertTrue(stats.getEstimatedWeight() <= 5L); + } finally { + refreshExecutor.shutdownNow(); + } + } + @Test public void testCheckCatalogInitializedRequiresExplicitInit() { ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); @@ -159,4 +200,17 @@ private TestExternalMetaCache(ExecutorService refreshExecutor) { MetaCacheEntryInvalidation.forNameMapping(SchemaCacheKey::getNameMapping))); } } + + private static final class WeightedExternalMetaCache extends AbstractExternalMetaCache { + private WeightedExternalMetaCache(ExecutorService refreshExecutor) { + super("weighted_engine", refreshExecutor); + registerEntry(MetaCacheEntryDef.of( + "value", + String.class, + Integer.class, + String::length, + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 100L)) + .withSizeEstimator((key, value) -> value)); + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/CacheSpecTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/CacheSpecTest.java index 05acbb539a26d6..373bd03704504e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/CacheSpecTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/CacheSpecTest.java @@ -68,6 +68,7 @@ public void testFromPropertiesWithPropertySpecBuilder() { public void testFromPropertiesWithEngineEntryKeys() { Map properties = Maps.newHashMap(); properties.put("meta.cache.hive.schema.ttl-second", "0"); + properties.put("meta.cache.hive.schema.max-weight", "4KB"); CacheSpec defaultSpec = CacheSpec.fromProperties( Maps.newHashMap(), @@ -79,6 +80,47 @@ public void testFromPropertiesWithEngineEntryKeys() { Assert.assertTrue(spec.isEnable()); Assert.assertEquals(0, spec.getTtlSecond()); Assert.assertEquals(100, spec.getCapacity()); + Assert.assertTrue(spec.isWeightBounded()); + Assert.assertEquals(4096L, spec.getMaxWeight().getAsLong()); + } + + @Test + public void testFromPropertiesRejectsInvalidMaximumWeight() { + String key = "meta.cache.test.schema.max-weight"; + CacheSpec defaultSpec = CacheSpec.of(true, 60L, 100L); + Map properties = Maps.newHashMap(); + + for (String invalidValue : new String[] { + "abc", "-1MB", "1.5GB", "1XB", "9223372036854775808", "9223372036854775807KB"}) { + properties.put(key, invalidValue); + IllegalArgumentException exception = Assert.assertThrows( + IllegalArgumentException.class, + () -> CacheSpec.fromProperties(properties, "test", "schema", defaultSpec)); + Assert.assertEquals( + "The parameter " + key + " is wrong, value is " + invalidValue, + exception.getMessage()); + } + } + + @Test + public void testFromPropertiesParsesMaximumWeightUnitsAndZero() { + String key = "meta.cache.test.schema.max-weight"; + CacheSpec defaultSpec = CacheSpec.of(true, 60L, 100L); + Map properties = Maps.newHashMap(); + + properties.put(key, "4096"); + CacheSpec byteSpec = CacheSpec.fromProperties(properties, "test", "schema", defaultSpec); + Assert.assertEquals(4096L, byteSpec.getMaxWeight().getAsLong()); + + properties.put(key, "512MB"); + CacheSpec megabyteSpec = CacheSpec.fromProperties(properties, "test", "schema", defaultSpec); + Assert.assertEquals(512L * 1024 * 1024, megabyteSpec.getMaxWeight().getAsLong()); + Assert.assertTrue(megabyteSpec.isCacheEnabled()); + + properties.put(key, "0GB"); + CacheSpec disabledSpec = CacheSpec.fromProperties(properties, "test", "schema", defaultSpec); + Assert.assertEquals(0L, disabledSpec.getMaxWeight().getAsLong()); + Assert.assertFalse(disabledSpec.isCacheEnabled()); } @Test @@ -118,6 +160,11 @@ public void testOfSemantics() { Assert.assertFalse(disabled.isEnable()); Assert.assertEquals(60, disabled.getTtlSecond()); Assert.assertEquals(100, disabled.getCapacity()); + + CacheSpec weighted = CacheSpec.ofWeight(true, 60, 100, 1024); + Assert.assertTrue(weighted.isWeightBounded()); + Assert.assertEquals(1024L, weighted.getMaxWeight().getAsLong()); + Assert.assertTrue(weighted.isCacheEnabled()); } @Test @@ -147,6 +194,8 @@ public void testIsCacheEnabled() { Assert.assertFalse(CacheSpec.isCacheEnabled(false, CacheSpec.CACHE_NO_TTL, 1)); Assert.assertFalse(CacheSpec.isCacheEnabled(true, 0, 1)); Assert.assertFalse(CacheSpec.isCacheEnabled(true, CacheSpec.CACHE_NO_TTL, 0)); + Assert.assertTrue(CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 0, 1).isCacheEnabled()); + Assert.assertFalse(CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 1, 0).isCacheEnabled()); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java index 340c6ae7f3b50f..efff73e3206c86 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java @@ -20,6 +20,7 @@ import org.apache.doris.common.Config; import com.github.benmanes.caffeine.cache.LoadingCache; +import com.github.benmanes.caffeine.cache.RemovalCause; import com.google.common.collect.Maps; import com.google.common.util.concurrent.MoreExecutors; import org.junit.Assert; @@ -33,6 +34,7 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.BooleanSupplier; import java.util.function.Consumer; import java.util.function.Supplier; @@ -225,6 +227,97 @@ public void testGetWithMissLoaderAndDisableAutoRefresh() throws Exception { } } + @Test + public void testMaximumWeightUsesEstimatorAndExposesWeightStats() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + try { + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", + String::length, + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 100L, 5L), + refreshExecutor, + false, + false, + (key, value) -> value); + + entry.put("first", 4); + waitUntil(() -> entry.stats().getEstimatedWeight() == 4L); + entry.put("second", 4); + waitUntil(() -> entry.stats().getEvictionCount() == 1L); + + MetaCacheEntryStats stats = entry.stats(); + Assert.assertTrue(stats.isWeightBounded()); + Assert.assertEquals(5L, stats.getMaxWeight()); + Assert.assertTrue(stats.getEstimatedWeight() <= 5L); + Assert.assertEquals(1L, stats.getEvictionCount()); + Assert.assertEquals(4L, stats.getEvictionWeight()); + } finally { + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testMaximumWeightRequiresEstimator() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + try { + IllegalArgumentException exception = Assert.assertThrows( + IllegalArgumentException.class, + () -> new MetaCacheEntry<>( + "weighted", + String::length, + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 100L, 5L), + refreshExecutor, + false)); + Assert.assertTrue(exception.getMessage().contains("size estimator")); + } finally { + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testMaximumWeightRejectsNegativeEstimatorResult() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + try { + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", + String::length, + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 100L, 5L), + refreshExecutor, + false, + false, + (key, value) -> -1L); + + IllegalStateException exception = Assert.assertThrows( + IllegalStateException.class, () -> entry.put("key", 1)); + Assert.assertTrue(exception.getMessage().contains("negative weight")); + } finally { + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testMaximumWeightSaturatesEstimatorResultToCaffeineLimit() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + try { + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", + String::length, + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 100L, Integer.MAX_VALUE), + refreshExecutor, + false, + false, + (key, value) -> Long.MAX_VALUE); + + entry.put("key", 1); + waitUntil(() -> entry.stats().getEstimatedWeight() == Integer.MAX_VALUE); + MetaCacheEntryStats stats = entry.stats(); + Assert.assertEquals(1L, stats.getEstimatedSize()); + Assert.assertEquals(Integer.MAX_VALUE, stats.getEstimatedWeight()); + } finally { + refreshExecutor.shutdownNow(); + } + } + @Test public void testStatsSnapshotTracksLoadAndLastError() { ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); @@ -1079,6 +1172,36 @@ public void testSyncRemovalListenerDisablesRefreshAndRunsSynchronously() throws } } + @Test + public void testMaximumWeightEvictionRunsSyncRemovalListener() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + try { + CacheSpec cacheSpec = CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 100L, 5L); + AtomicInteger removalCounter = new AtomicInteger(); + AtomicReference removalCause = new AtomicReference<>(); + MetaCacheEntry entry = MetaCacheEntry.withSyncRemovalListener( + "weighted-sync-listener", + String::length, + cacheSpec, + refreshExecutor, + (key, value) -> value, + (key, value, cause) -> { + removalCounter.incrementAndGet(); + removalCause.set(cause); + }); + + entry.put("first", 4); + entry.put("second", 4); + + Assert.assertEquals(1, removalCounter.get()); + Assert.assertEquals(RemovalCause.SIZE, removalCause.get()); + Assert.assertEquals(1L, entry.stats().getEstimatedSize()); + Assert.assertEquals(4L, entry.stats().getEstimatedWeight()); + } finally { + refreshExecutor.shutdownNow(); + } + } + @Test public void testInvalidateAllDoesNotPutAfterInFlightManualMissLoad() throws Exception { ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); From c63fc746d9f04b8d9fbc16348e90cf3092bdd364 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Fri, 7 Aug 2026 16:34:58 +0800 Subject: [PATCH 2/5] [feature](fe) Add weighted connector metadata caches ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: External connector metadata caches are bounded only by entry count, so a small number of large file, partition, table, or manifest values can consume excessive FE heap. Add optional byte-weighted cache construction and typed, precomputed estimators for Hive file listings and Iceberg table, partition, and manifest entries while preserving count-based defaults. ### Release note External metadata caches can opt into byte-based limits with per-entry meta.cache...max-weight properties. ### Check List (For Author) - Test: Unit Test - Connector cache CacheSpecTest and MetaCacheEntryTest (28 tests passed) - Hive and Iceberg production sources compiled in the FE unit-test reactor - Behavior changed: Yes, configured connector caches can use maximumWeight; existing maximumSize behavior remains the default. - Does this need documentation: No --- .../doris/connector/cache/CacheFactory.java | 42 +- .../doris/connector/cache/CacheSpec.java | 154 ++++- .../doris/connector/cache/JvmSizeUtils.java | 181 +++++ .../doris/connector/cache/MetaCacheEntry.java | 43 +- .../connector/cache/MetaCacheEntryStats.java | 38 +- .../cache/MetaCacheSizeEstimator.java | 30 + .../doris/connector/cache/CacheSpecTest.java | 38 ++ .../connector/cache/MetaCacheEntryTest.java | 60 ++ .../connector/hive/HiveCatalogProperties.java | 3 + .../connector/hive/HiveFileListingCache.java | 33 +- .../hive/HiveFileListingSizeEstimator.java | 73 +++ .../hive/HiveFileListingCacheTest.java | 23 + .../iceberg/IcebergCacheSizeEstimator.java | 616 ++++++++++++++++++ .../iceberg/IcebergCatalogProperties.java | 4 + .../connector/iceberg/IcebergConnector.java | 31 +- .../iceberg/IcebergManifestCache.java | 23 +- .../iceberg/IcebergManifestEntryKey.java | 6 + .../iceberg/IcebergPartitionCache.java | 31 +- .../iceberg/IcebergPartitionUtils.java | 12 +- .../connector/iceberg/IcebergTableCache.java | 27 +- .../connector/iceberg/ManifestCacheValue.java | 6 + 21 files changed, 1428 insertions(+), 46 deletions(-) create mode 100644 fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/JvmSizeUtils.java create mode 100644 fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheSizeEstimator.java create mode 100644 fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveFileListingSizeEstimator.java create mode 100644 fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCacheSizeEstimator.java diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheFactory.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheFactory.java index 03e4126e9911be..47e7ae5b0309f7 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheFactory.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheFactory.java @@ -24,6 +24,7 @@ import com.github.benmanes.caffeine.cache.LoadingCache; import com.github.benmanes.caffeine.cache.RemovalListener; import com.github.benmanes.caffeine.cache.Ticker; +import com.github.benmanes.caffeine.cache.Weigher; import java.time.Duration; import java.util.OptionalLong; @@ -73,22 +74,49 @@ public CacheFactory( // Build a loading cache, without executor, it will use fork-join pool for refresh public LoadingCache buildCache(CacheLoader cacheLoader) { - Caffeine builder = buildWithParams(); + Caffeine builder = buildSizeBoundedWithParams(); return builder.build(cacheLoader); } // Build a loading cache, with executor, it will use given executor for refresh public LoadingCache buildCache(CacheLoader cacheLoader, ExecutorService executor) { - Caffeine builder = buildWithParams(); + Caffeine builder = buildSizeBoundedWithParams(); builder.executor(executor); return builder.build(cacheLoader); } + /** + * Build a loading cache bounded by weight instead of entry count. + */ + public LoadingCache buildCacheWithWeight(CacheLoader cacheLoader, + ExecutorService executor, long maxWeight, Weigher weigher) { + Caffeine builder = buildWithParams() + .maximumWeight(maxWeight) + .weigher(weigher); + builder.executor(executor); + return builder.build(cacheLoader); + } + + /** + * Build a loading cache bounded by weight with a synchronous removal listener. + */ + public LoadingCache buildCacheWithWeightAndSyncRemovalListener(CacheLoader cacheLoader, + long maxWeight, Weigher weigher, RemovalListener removalListener) { + Caffeine builder = buildWithParams() + .maximumWeight(maxWeight) + .weigher(weigher); + if (removalListener != null) { + builder.removalListener(removalListener); + } + builder.executor(Runnable::run); // Sync execution to avoid thread pool deadlock + return builder.build(cacheLoader); + } + // Build cache with sync removal listener to prevent deadlock when listener calls invalidateAll() public LoadingCache buildCacheWithSyncRemovalListener(CacheLoader cacheLoader, RemovalListener removalListener) { - Caffeine builder = buildWithParams(); + Caffeine builder = buildSizeBoundedWithParams(); if (removalListener != null) { builder.removalListener(removalListener); } @@ -99,15 +127,13 @@ public LoadingCache buildCacheWithSyncRemovalListener(CacheLoader AsyncLoadingCache buildAsyncCache(AsyncCacheLoader cacheLoader, ExecutorService executor) { - Caffeine builder = buildWithParams(); + Caffeine builder = buildSizeBoundedWithParams(); builder.executor(executor); return builder.buildAsync(cacheLoader); } private Caffeine buildWithParams() { Caffeine builder = Caffeine.newBuilder(); - builder.maximumSize(maxSize); - if (expireAfterAccessSec.isPresent()) { builder.expireAfterAccess(Duration.ofSeconds(expireAfterAccessSec.getAsLong())); } @@ -124,4 +150,8 @@ private Caffeine buildWithParams() { } return builder; } + + private Caffeine buildSizeBoundedWithParams() { + return buildWithParams().maximumSize(maxSize); + } } diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheSpec.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheSpec.java index 0524b31d402f48..6b1850437cd1cc 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheSpec.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheSpec.java @@ -18,9 +18,12 @@ package org.apache.doris.connector.cache; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.OptionalLong; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Common cache specification for external metadata caches. @@ -42,7 +45,9 @@ *

    *
  • enable=false disables cache
    • *
  • ttlSecond=0 disables cache, ttlSecond=-1 means no expiration
    • - *
  • capacity=0 disables cache; capacity is count-based
    • + *
  • capacity=0 disables a count-bounded cache
    • + *
  • when maxWeight is present, it replaces capacity as the effective bound
    • + *
  • maxWeight accepts an optional binary unit such as KB, MB, or GB; a bare number means bytes
    • *
*/ public final class CacheSpec { @@ -52,19 +57,27 @@ public final class CacheSpec { private static final String KEY_ENABLE = ".enable"; private static final String KEY_TTL_SECOND = ".ttl-second"; private static final String KEY_CAPACITY = ".capacity"; + private static final String KEY_MAX_WEIGHT = ".max-weight"; + private static final Pattern DATA_SIZE_PATTERN = Pattern.compile("(\\d+)([a-zA-Z]*)"); private final boolean enable; private final long ttlSecond; private final long capacity; + private final OptionalLong maxWeight; - private CacheSpec(boolean enable, long ttlSecond, long capacity) { + private CacheSpec(boolean enable, long ttlSecond, long capacity, OptionalLong maxWeight) { this.enable = enable; this.ttlSecond = ttlSecond; this.capacity = capacity; + this.maxWeight = Objects.requireNonNull(maxWeight, "maxWeight is required"); } public static CacheSpec of(boolean enable, long ttlSecond, long capacity) { - return new CacheSpec(enable, ttlSecond, capacity); + return new CacheSpec(enable, ttlSecond, capacity, OptionalLong.empty()); + } + + public static CacheSpec ofWeight(boolean enable, long ttlSecond, long capacity, long maxWeight) { + return new CacheSpec(enable, ttlSecond, capacity, OptionalLong.of(maxWeight)); } /** @@ -103,12 +116,14 @@ public static CacheSpec fromProperties(Map properties, PropertyS boolean enable = getBooleanProperty(properties, propertySpec.getEnableKey(), propertySpec.isDefaultEnable()); long ttlSecond = getLongProperty(properties, propertySpec.getTtlKey(), propertySpec.getDefaultTtlSecond()); long capacity = getLongProperty(properties, propertySpec.getCapacityKey(), propertySpec.getDefaultCapacity()); - return of(enable, ttlSecond, capacity); + OptionalLong maxWeight = getOptionalDataSizeProperty( + properties, propertySpec.getMaxWeightKey(), propertySpec.getDefaultMaxWeight()); + return new CacheSpec(enable, ttlSecond, capacity, maxWeight); } /** * Build a cache spec from catalog properties by standard external meta cache key pattern: - * meta.cache.<engine>.<entry>.(enable|ttl-second|capacity) + * meta.cache.<engine>.<entry>.(enable|ttl-second|capacity|max-weight) */ public static CacheSpec fromProperties(Map properties, String engine, String entryName, CacheSpec defaultSpec) { @@ -121,6 +136,7 @@ public static PropertySpec metaCachePropertySpec(String engine, String entryName .enable(cacheKeyPrefix + KEY_ENABLE, defaultSpec.isEnable()) .ttl(cacheKeyPrefix + KEY_TTL_SECOND, defaultSpec.getTtlSecond()) .capacity(cacheKeyPrefix + KEY_CAPACITY, defaultSpec.getCapacity()) + .maxWeight(cacheKeyPrefix + KEY_MAX_WEIGHT, defaultSpec.getMaxWeight()) .build(); } @@ -173,6 +189,17 @@ public static void checkLongProperty(String value, long minValue, String key) { } } + public static void checkDataSizeProperty(String value, String key) { + if (value == null) { + return; + } + try { + parseDataSizeAllowZero(value); + } catch (IllegalArgumentException e) { + throw invalidDataSizeProperty(key, value); + } + } + public static boolean isCacheEnabled(boolean enable, long ttlSecond, long capacity) { return enable && ttlSecond != 0 && capacity != 0; } @@ -196,6 +223,13 @@ public static String metaCacheTtlKey(String engine, String entryName) { return META_CACHE_PREFIX + engine + "." + entryName + KEY_TTL_SECOND; } + /** + * Build the standard external meta cache maximum-weight key for one engine+entry. + */ + public static String metaCacheMaxWeightKey(String engine, String entryName) { + return META_CACHE_PREFIX + engine + "." + entryName + KEY_MAX_WEIGHT; + } + /** * Returns true when the given property key belongs to one engine's meta cache namespace. */ @@ -234,6 +268,75 @@ private static long getLongProperty(Map properties, String key, } } + private static OptionalLong getOptionalDataSizeProperty( + Map properties, String key, OptionalLong defaultValue) { + if (key == null) { + return defaultValue; + } + String value = properties.get(key); + if (value == null) { + return defaultValue; + } + try { + return OptionalLong.of(parseDataSizeAllowZero(value)); + } catch (IllegalArgumentException e) { + throw invalidDataSizeProperty(key, value); + } + } + + private static long parseDataSizeAllowZero(String value) { + Matcher matcher = DATA_SIZE_PATTERN.matcher(value); + if (!matcher.matches()) { + throw new IllegalArgumentException("invalid data size"); + } + + long number; + try { + number = Long.parseLong(matcher.group(1)); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("invalid data size", e); + } + + long multiplier; + switch (matcher.group(2).toUpperCase(Locale.ROOT)) { + case "": + case "B": + multiplier = 1L; + break; + case "K": + case "KB": + multiplier = 1L << 10; + break; + case "M": + case "MB": + multiplier = 1L << 20; + break; + case "G": + case "GB": + multiplier = 1L << 30; + break; + case "T": + case "TB": + multiplier = 1L << 40; + break; + case "P": + case "PB": + multiplier = 1L << 50; + break; + default: + throw new IllegalArgumentException("invalid data size unit"); + } + try { + return Math.multiplyExact(number, multiplier); + } catch (ArithmeticException e) { + throw new IllegalArgumentException("data size is too large", e); + } + } + + private static IllegalArgumentException invalidDataSizeProperty(String key, String value) { + return new IllegalArgumentException("The parameter " + key + " is wrong, value is " + value); + } + public boolean isEnable() { return enable; } @@ -246,6 +349,20 @@ public long getCapacity() { return capacity; } + public OptionalLong getMaxWeight() { + return maxWeight; + } + + public boolean isWeightBounded() { + return maxWeight.isPresent(); + } + + public boolean isCacheEnabled() { + return enable + && ttlSecond != CACHE_TTL_DISABLE_CACHE + && maxWeight.orElse(capacity) != 0L; + } + public static final class PropertySpec { private final String enableKey; private final boolean defaultEnable; @@ -253,15 +370,20 @@ public static final class PropertySpec { private final long defaultTtlSecond; private final String capacityKey; private final long defaultCapacity; + private final String maxWeightKey; + private final OptionalLong defaultMaxWeight; private PropertySpec(String enableKey, boolean defaultEnable, String ttlKey, - long defaultTtlSecond, String capacityKey, long defaultCapacity) { + long defaultTtlSecond, String capacityKey, long defaultCapacity, + String maxWeightKey, OptionalLong defaultMaxWeight) { this.enableKey = enableKey; this.defaultEnable = defaultEnable; this.ttlKey = ttlKey; this.defaultTtlSecond = defaultTtlSecond; this.capacityKey = capacityKey; this.defaultCapacity = defaultCapacity; + this.maxWeightKey = maxWeightKey; + this.defaultMaxWeight = defaultMaxWeight; } public String getEnableKey() { @@ -288,6 +410,14 @@ public long getDefaultCapacity() { return defaultCapacity; } + public String getMaxWeightKey() { + return maxWeightKey; + } + + public OptionalLong getDefaultMaxWeight() { + return defaultMaxWeight; + } + public static final class Builder { private String enableKey; private boolean defaultEnable; @@ -295,6 +425,8 @@ public static final class Builder { private long defaultTtlSecond; private String capacityKey; private long defaultCapacity; + private String maxWeightKey; + private OptionalLong defaultMaxWeight = OptionalLong.empty(); public Builder enable(String key, boolean defaultValue) { this.enableKey = key; @@ -314,6 +446,12 @@ public Builder capacity(String key, long defaultValue) { return this; } + public Builder maxWeight(String key, OptionalLong defaultValue) { + this.maxWeightKey = Objects.requireNonNull(key, "maxWeightKey is required"); + this.defaultMaxWeight = Objects.requireNonNull(defaultValue, "defaultMaxWeight is required"); + return this; + } + public PropertySpec build() { return new PropertySpec( Objects.requireNonNull(enableKey, "enableKey is required"), @@ -321,7 +459,9 @@ public PropertySpec build() { Objects.requireNonNull(ttlKey, "ttlKey is required"), defaultTtlSecond, Objects.requireNonNull(capacityKey, "capacityKey is required"), - defaultCapacity); + defaultCapacity, + maxWeightKey, + defaultMaxWeight); } } } diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/JvmSizeUtils.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/JvmSizeUtils.java new file mode 100644 index 00000000000000..6be78915365b1e --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/JvmSizeUtils.java @@ -0,0 +1,181 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import com.sun.management.HotSpotDiagnosticMXBean; + +import java.lang.management.ManagementFactory; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; + +/** + * Low-cost JVM heap layout formulas for type-specific connector cache estimators. + * + *

This class reflects class declarations once to calculate shallow sizes. It never reads fields from a runtime + * object and never walks an object graph. + */ +public final class JvmSizeUtils { + private static final VmLayout VM_LAYOUT = VmLayout.detect(); + private static final boolean COMPACT_STRINGS = vmBoolean("CompactStrings"); + + private static final ClassValue SHALLOW_SIZES = new ClassValue<>() { + @Override + protected Long computeValue(Class type) { + if (type.isArray()) { + throw new IllegalArgumentException("Array size depends on its length: " + type); + } + long size = VM_LAYOUT.objectHeaderBytes(); + for (Class current = type; current != null; current = current.getSuperclass()) { + for (Field field : current.getDeclaredFields()) { + if (!Modifier.isStatic(field.getModifiers())) { + size = saturatedAdd(size, fieldSize(field.getType())); + } + } + } + return align(size); + } + }; + private static final long STRING_SHALLOW_BYTES = instanceSize(String.class); + private static final long ARRAY_LIST_SHALLOW_BYTES = instanceSize(java.util.ArrayList.class); + + private JvmSizeUtils() { + } + + public static long instanceSize(Class type) { + return SHALLOW_SIZES.get(type); + } + + public static long objectArraySize(int length) { + return arraySize(length, VM_LAYOUT.referenceBytes()); + } + + public static long byteArraySize(int length) { + return arraySize(length, Byte.BYTES); + } + + public static long intArraySize(int length) { + return arraySize(length, Integer.BYTES); + } + + public static long longArraySize(int length) { + return arraySize(length, Long.BYTES); + } + + public static long arrayListSize(int backingArrayCapacity) { + return saturatedAdd(ARRAY_LIST_SHALLOW_BYTES, objectArraySize(backingArrayCapacity)); + } + + /** Estimate the heap retained by a Java 17 String and its compact-string byte array. */ + public static long stringSize(String value) { + if (value == null) { + return 0L; + } + int bytesPerCharacter = COMPACT_STRINGS && isLatin1(value) ? Byte.BYTES : Character.BYTES; + long valueBytes = saturatedMultiply(value.length(), bytesPerCharacter); + int arrayLength = valueBytes >= Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) valueBytes; + return saturatedAdd(STRING_SHALLOW_BYTES, byteArraySize(arrayLength)); + } + + public static long saturatedAdd(long left, long right) { + if (right > 0L && left > Long.MAX_VALUE - right) { + return Long.MAX_VALUE; + } + return left + right; + } + + public static long saturatedMultiply(long left, long right) { + if (left == 0L || right == 0L) { + return 0L; + } + if (left > Long.MAX_VALUE / right) { + return Long.MAX_VALUE; + } + return left * right; + } + + private static long arraySize(int length, int elementBytes) { + long elements = saturatedMultiply(length, elementBytes); + return align(saturatedAdd(VM_LAYOUT.arrayHeaderBytes(), elements)); + } + + private static long fieldSize(Class type) { + if (!type.isPrimitive()) { + return VM_LAYOUT.referenceBytes(); + } + if (type == long.class || type == double.class) { + return Long.BYTES; + } + if (type == int.class || type == float.class) { + return Integer.BYTES; + } + if (type == short.class || type == char.class) { + return Short.BYTES; + } + return Byte.BYTES; + } + + private static boolean isLatin1(String value) { + for (int i = 0; i < value.length(); i++) { + if (value.charAt(i) > 0xff) { + return false; + } + } + return true; + } + + private static long align(long value) { + long remainder = value % VM_LAYOUT.objectAlignmentBytes(); + return remainder == 0L + ? value + : saturatedAdd(value, VM_LAYOUT.objectAlignmentBytes() - remainder); + } + + private static boolean vmBoolean(String option) { + return Boolean.parseBoolean(hotSpotDiagnostic().getVMOption(option).getValue()); + } + + private static int vmInt(String option) { + return Integer.parseInt(hotSpotDiagnostic().getVMOption(option).getValue()); + } + + private static HotSpotDiagnosticMXBean hotSpotDiagnostic() { + return ManagementFactory.getPlatformMXBean(HotSpotDiagnosticMXBean.class); + } + + private record VmLayout( + int referenceBytes, + int objectHeaderBytes, + int arrayHeaderBytes, + int objectAlignmentBytes) { + + private static VmLayout detect() { + int referenceBytes = vmBoolean("UseCompressedOops") ? Integer.BYTES : Long.BYTES; + int classPointerBytes = vmBoolean("UseCompressedClassPointers") ? Integer.BYTES : Long.BYTES; + int alignment = vmInt("ObjectAlignmentInBytes"); + int objectHeaderBytes = Long.BYTES + classPointerBytes; + int arrayHeaderBytes = Math.toIntExact( + alignWithoutLayout(objectHeaderBytes + Integer.BYTES, alignment)); + return new VmLayout(referenceBytes, objectHeaderBytes, arrayHeaderBytes, alignment); + } + + private static long alignWithoutLayout(long value, int alignment) { + long remainder = value % alignment; + return remainder == 0L ? value : value + alignment - remainder; + } + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheEntry.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheEntry.java index 425697ef9b1098..d0dc2657cc9f4c 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheEntry.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheEntry.java @@ -19,6 +19,7 @@ import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.LoadingCache; +import com.github.benmanes.caffeine.cache.Policy; import com.github.benmanes.caffeine.cache.stats.CacheStats; import java.util.Objects; @@ -84,6 +85,14 @@ public MetaCacheEntry(String name, Function loader, CacheSpec cacheSpec, E public MetaCacheEntry(String name, Function loader, CacheSpec cacheSpec, ExecutorService refreshExecutor, boolean autoRefresh, boolean contextualOnly, long refreshAfterWriteSeconds, boolean manualMissLoadEnabled) { + this(name, loader, cacheSpec, refreshExecutor, autoRefresh, contextualOnly, + refreshAfterWriteSeconds, manualMissLoadEnabled, null); + } + + public MetaCacheEntry(String name, Function loader, CacheSpec cacheSpec, + ExecutorService refreshExecutor, boolean autoRefresh, boolean contextualOnly, + long refreshAfterWriteSeconds, boolean manualMissLoadEnabled, + MetaCacheSizeEstimator sizeEstimator) { this.name = name; if (contextualOnly) { if (loader != null) { @@ -101,22 +110,35 @@ public MetaCacheEntry(String name, Function loader, CacheSpec cacheSpec, this.refreshAfterWriteSeconds = refreshAfterWriteSeconds; this.manualMissLoadEnabled = manualMissLoadEnabled; Objects.requireNonNull(refreshExecutor, "refreshExecutor can not be null"); - this.effectiveEnabled = CacheSpec.isCacheEnabled( - this.cacheSpec.isEnable(), this.cacheSpec.getTtlSecond(), this.cacheSpec.getCapacity()); + if (this.cacheSpec.isWeightBounded()) { + Objects.requireNonNull(sizeEstimator, "sizeEstimator is required when max-weight is configured"); + } + this.effectiveEnabled = this.cacheSpec.isCacheEnabled(); OptionalLong expireAfterAccessSec = effectiveEnabled ? CacheSpec.toExpireAfterAccess(this.cacheSpec.getTtlSecond()) : OptionalLong.empty(); OptionalLong refreshAfterWriteSec = effectiveEnabled && autoRefresh ? OptionalLong.of(refreshAfterWriteSeconds) : OptionalLong.empty(); - long maxSize = effectiveEnabled ? this.cacheSpec.getCapacity() : 0L; + long maxSize = effectiveEnabled && !this.cacheSpec.isWeightBounded() + ? this.cacheSpec.getCapacity() + : 0L; CacheFactory cacheFactory = new CacheFactory( expireAfterAccessSec, refreshAfterWriteSec, maxSize, true, null); - this.loadingData = cacheFactory.buildCache(this::loadFromDefaultLoader, refreshExecutor); + if (this.cacheSpec.isWeightBounded()) { + long maxWeight = effectiveEnabled ? this.cacheSpec.getMaxWeight().getAsLong() : 0L; + this.loadingData = cacheFactory.buildCacheWithWeight( + this::loadFromDefaultLoader, + refreshExecutor, + maxWeight, + (key, value) -> toCaffeineWeight(sizeEstimator.estimateBytes(key, value))); + } else { + this.loadingData = cacheFactory.buildCache(this::loadFromDefaultLoader, refreshExecutor); + } this.data = loadingData; // Initialize striped locks eagerly to keep the hot path allocation-free. for (int i = 0; i < loadLocks.length; i++) { @@ -227,6 +249,8 @@ public void forEach(BiConsumer consumer) { public MetaCacheEntryStats stats() { CacheStats cacheStats = loadingData.stats(); + Policy.Eviction evictionPolicy = loadingData.policy().eviction() + .orElseThrow(() -> new IllegalStateException("cache has no eviction policy: " + name)); long successCount = loadSuccessCount.get(); long failureCount = loadFailureCount.get(); long totalLoadTime = totalLoadTimeNanos.get(); @@ -237,7 +261,10 @@ public MetaCacheEntryStats stats() { autoRefresh, cacheSpec.getTtlSecond(), cacheSpec.getCapacity(), + cacheSpec.isWeightBounded(), + cacheSpec.getMaxWeight().orElse(-1L), data.estimatedSize(), + evictionPolicy.weightedSize().orElse(-1L), cacheStats.requestCount(), cacheStats.hitCount(), cacheStats.missCount(), @@ -247,12 +274,20 @@ public MetaCacheEntryStats stats() { totalLoadTime, totalLoadCount == 0 ? 0D : (double) totalLoadTime / totalLoadCount, cacheStats.evictionCount(), + cacheStats.evictionWeight(), invalidateCount.get(), lastLoadSuccessTimeMs.get(), lastLoadFailureTimeMs.get(), lastError.get()); } + private static int toCaffeineWeight(long estimatedBytes) { + if (estimatedBytes < 0L) { + throw new IllegalArgumentException("estimated cache entry bytes can not be negative: " + estimatedBytes); + } + return estimatedBytes >= Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) estimatedBytes; + } + // Injected at construction (fe-core reads Config.enable_external_meta_cache_manual_miss_load dynamically). private boolean isManualMissLoadEnabled() { return manualMissLoadEnabled; diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheEntryStats.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheEntryStats.java index 41c8b89192cd1b..1c8ef49cc59e35 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheEntryStats.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheEntryStats.java @@ -35,6 +35,8 @@ * *

For last-load timestamps, {@code -1} means no corresponding event happened yet. * {@code lastError} keeps the latest load failure message; empty string means no failure recorded. + * Snapshots do not trigger Caffeine maintenance, so size, weight, and eviction values may be briefly stale while + * asynchronous maintenance is pending. */ public final class MetaCacheEntryStats { private final boolean configEnabled; @@ -42,7 +44,10 @@ public final class MetaCacheEntryStats { private final boolean autoRefresh; private final long ttlSecond; private final long capacity; + private final boolean weightBounded; + private final long maxWeight; private final long estimatedSize; + private final long estimatedWeight; private final long requestCount; private final long hitCount; private final long missCount; @@ -52,6 +57,7 @@ public final class MetaCacheEntryStats { private final long totalLoadTimeNanos; private final double averageLoadPenaltyNanos; private final long evictionCount; + private final long evictionWeight; private final long invalidateCount; private final long lastLoadSuccessTimeMs; private final long lastLoadFailureTimeMs; @@ -66,7 +72,10 @@ public MetaCacheEntryStats( boolean autoRefresh, long ttlSecond, long capacity, + boolean weightBounded, + long maxWeight, long estimatedSize, + long estimatedWeight, long requestCount, long hitCount, long missCount, @@ -76,6 +85,7 @@ public MetaCacheEntryStats( long totalLoadTimeNanos, double averageLoadPenaltyNanos, long evictionCount, + long evictionWeight, long invalidateCount, long lastLoadSuccessTimeMs, long lastLoadFailureTimeMs, @@ -85,7 +95,10 @@ public MetaCacheEntryStats( this.autoRefresh = autoRefresh; this.ttlSecond = ttlSecond; this.capacity = capacity; + this.weightBounded = weightBounded; + this.maxWeight = maxWeight; this.estimatedSize = estimatedSize; + this.estimatedWeight = estimatedWeight; this.requestCount = requestCount; this.hitCount = hitCount; this.missCount = missCount; @@ -95,6 +108,7 @@ public MetaCacheEntryStats( this.totalLoadTimeNanos = totalLoadTimeNanos; this.averageLoadPenaltyNanos = averageLoadPenaltyNanos; this.evictionCount = evictionCount; + this.evictionWeight = evictionWeight; this.invalidateCount = invalidateCount; this.lastLoadSuccessTimeMs = lastLoadSuccessTimeMs; this.lastLoadFailureTimeMs = lastLoadFailureTimeMs; @@ -106,7 +120,7 @@ public boolean isConfigEnabled() { } /** - * Effective cache enable state evaluated by {@link CacheSpec#isCacheEnabled(boolean, long, long)}. + * Effective cache enable state evaluated by {@link CacheSpec#isCacheEnabled()}. */ public boolean isEffectiveEnabled() { return effectiveEnabled; @@ -124,10 +138,28 @@ public long getCapacity() { return capacity; } + public boolean isWeightBounded() { + return weightBounded; + } + + /** + * Returns the configured maximum weight in bytes, or -1 for a count-bounded cache. + */ + public long getMaxWeight() { + return maxWeight; + } + public long getEstimatedSize() { return estimatedSize; } + /** + * Returns Caffeine's current weighted size in bytes, or -1 for a count-bounded cache. + */ + public long getEstimatedWeight() { + return estimatedWeight; + } + public long getRequestCount() { return requestCount; } @@ -167,6 +199,10 @@ public long getEvictionCount() { return evictionCount; } + public long getEvictionWeight() { + return evictionWeight; + } + public double getEvictionRate() { if (requestCount == 0) { return 0D; diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheSizeEstimator.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheSizeEstimator.java new file mode 100644 index 00000000000000..bf824ad7c45e0e --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheSizeEstimator.java @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +/** + * Estimates the retained bytes owned by one connector metadata cache entry. + * + *

Implementations should be type-specific, deterministic, non-blocking, and normally O(1) at cache admission. + * Large object graphs should precompute their owned size when the value is built instead of being traversed again + * whenever Caffeine inserts, replaces, refreshes, or computes an entry. + */ +@FunctionalInterface +public interface MetaCacheSizeEstimator { + long estimateBytes(K key, V value); +} diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CacheSpecTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CacheSpecTest.java index 276735b40c2f9b..27ced4beb40377 100644 --- a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CacheSpecTest.java +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CacheSpecTest.java @@ -247,4 +247,42 @@ public void isMetaCacheKeyForEngine() { CacheSpec.isMetaCacheKeyForEngine("meta.cache.paimon.table.ttl-second", "iceberg")); Assertions.assertFalse(CacheSpec.isMetaCacheKeyForEngine(null, "iceberg")); } + + @Test + public void fromPropertiesParsesMaximumWeight() { + Map properties = new HashMap<>(); + properties.put("meta.cache.hive.file.max-weight", "512MB"); + + CacheSpec spec = CacheSpec.fromProperties( + properties, "hive", "file", CacheSpec.of(true, 60L, 100L)); + + Assertions.assertTrue(spec.isWeightBounded()); + Assertions.assertEquals(512L * 1024 * 1024, spec.getMaxWeight().getAsLong()); + Assertions.assertTrue(spec.isCacheEnabled()); + } + + @Test + public void maximumWeightOverridesCountCapacity() { + CacheSpec spec = CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 0L, 1024L); + Assertions.assertTrue(spec.isCacheEnabled()); + + CacheSpec disabled = CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 100L, 0L); + Assertions.assertFalse(disabled.isCacheEnabled()); + } + + @Test + public void maximumWeightRejectsInvalidValues() { + String key = "meta.cache.iceberg.table.max-weight"; + for (String value : new String[] {"abc", "-1", "9223372036854775807KB"}) { + Map properties = new HashMap<>(); + properties.put(key, value); + IllegalArgumentException error = Assertions.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.fromProperties( + properties, "iceberg", "table", CacheSpec.of(true, 60L, 100L))); + Assertions.assertEquals("The parameter " + key + " is wrong, value is " + value, + error.getMessage()); + Assertions.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.checkDataSizeProperty(value, key)); + } + } } diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheEntryTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheEntryTest.java index 8284f4ae1ed672..287f98098f53f8 100644 --- a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheEntryTest.java +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheEntryTest.java @@ -243,6 +243,66 @@ public void putIfNotInvalidatedSinceHonorsGenerationGuard() { } } + @Test + public void weightedEntryEvictsAndReportsWeight() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + try { + CacheSpec cacheSpec = CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 100L, 4L); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", key -> key, cacheSpec, refreshExecutor, + false, false, 0L, false, (key, value) -> value.length()); + + entry.get("aaa"); + entry.get("bbb"); + extractLoadingCache(entry).cleanUp(); + + MetaCacheEntryStats stats = entry.stats(); + Assertions.assertTrue(stats.isWeightBounded()); + Assertions.assertEquals(4L, stats.getMaxWeight()); + Assertions.assertTrue(stats.getEstimatedWeight() <= 4L); + Assertions.assertEquals(1L, stats.getEvictionCount()); + Assertions.assertEquals(3L, stats.getEvictionWeight()); + } finally { + refreshExecutor.shutdownNow(); + } + } + + @Test + public void weightedEntryRequiresEstimator() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + try { + CacheSpec cacheSpec = CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 100L, 4L); + NullPointerException error = Assertions.assertThrows(NullPointerException.class, + () -> new MetaCacheEntry<>("weighted", String::length, cacheSpec, refreshExecutor)); + Assertions.assertEquals("sizeEstimator is required when max-weight is configured", error.getMessage()); + } finally { + refreshExecutor.shutdownNow(); + } + } + + @Test + public void weightedEntryRejectsNegativeAndSaturatesLargeEstimate() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + try { + CacheSpec smallSpec = CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 100L, 4L); + MetaCacheEntry negative = new MetaCacheEntry<>( + "negative", key -> key, smallSpec, refreshExecutor, + false, false, 0L, false, (key, value) -> -1L); + Assertions.assertThrows(IllegalArgumentException.class, () -> negative.get("a")); + + CacheSpec largeSpec = CacheSpec.ofWeight( + true, CacheSpec.CACHE_NO_TTL, 100L, Integer.MAX_VALUE); + MetaCacheEntry saturated = new MetaCacheEntry<>( + "saturated", key -> key, largeSpec, refreshExecutor, + false, false, 0L, false, (key, value) -> Long.MAX_VALUE); + saturated.get("a"); + extractLoadingCache(saturated).cleanUp(); + Assertions.assertEquals(Integer.MAX_VALUE, saturated.stats().getEstimatedWeight()); + } finally { + refreshExecutor.shutdownNow(); + } + } + @SuppressWarnings("unchecked") private static LoadingCache extractLoadingCache(MetaCacheEntry entry) throws Exception { Field field = MetaCacheEntry.class.getDeclaredField("loadingData"); diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveCatalogProperties.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveCatalogProperties.java index d38571a11f2503..c16c7dfb14507e 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveCatalogProperties.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveCatalogProperties.java @@ -187,6 +187,9 @@ public HiveCatalogProperties checkCreateTimeOnlyRules() { // "The parameter ... is wrong, value is ..." message. CacheSpec.checkLongProperty(raw.get("file.meta.cache.ttl-second"), 0L, "file.meta.cache.ttl-second"); CacheSpec.checkLongProperty(raw.get("partition.cache.ttl-second"), 0L, "partition.cache.ttl-second"); + String fileMaxWeightKey = CacheSpec.metaCacheMaxWeightKey( + HiveFileListingCache.ENGINE, HiveFileListingCache.ENTRY_FILE); + CacheSpec.checkDataSizeProperty(raw.get(fileMaxWeightKey), fileMaxWeightKey); return this; } diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveFileListingCache.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveFileListingCache.java index af7d2ba558acc8..951fed609ce810 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveFileListingCache.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveFileListingCache.java @@ -19,6 +19,7 @@ import org.apache.doris.connector.cache.CacheSpec; import org.apache.doris.connector.cache.MetaCacheEntry; +import org.apache.doris.connector.cache.MetaCacheEntryStats; import org.apache.doris.connector.spi.DorisConnectorException; import org.apache.doris.filesystem.FileEntry; import org.apache.doris.filesystem.FileIterator; @@ -104,7 +105,7 @@ interface DirectoryLister { List list(String location, FileSystem fs); } - private final MetaCacheEntry> cache; + private final MetaCacheEntry cache; private final DirectoryLister lister; public HiveFileListingCache(HiveCatalogProperties properties) { @@ -138,7 +139,9 @@ private static DirectoryLister defaultLister(HiveCatalogProperties properties) { CacheSpec.of(true, DEFAULT_TTL_SECOND, DEFAULT_FILE_CAPACITY)); // Contextual-only + manual-miss so the slow listStatus runs on the caller (TCCL-pinned) thread outside // Caffeine's sync compute lock, deduplicated by a striped lock — mirrors CachingHmsClient's entries. - this.cache = new MetaCacheEntry<>("hive.file", null, spec, ForkJoinPool.commonPool(), false, true, 0L, true); + this.cache = new MetaCacheEntry<>( + "hive.file", null, spec, ForkJoinPool.commonPool(), false, true, 0L, true, + HiveFileListingSizeEstimator::estimateEntry); this.lister = Objects.requireNonNull(lister, "lister can not be null"); } @@ -165,7 +168,7 @@ public List listDataFiles(String dbName, String tableName, Strin public List listDataFiles(String dbName, String tableName, String location, List partitionValues, FileSystem fs) { return cache.get(new FileListingKey(dbName, tableName, location, partitionValues), - key -> lister.list(key.location, fs)); + key -> new FileListingValue(lister.list(key.location, fs))).files; } /** Drops every cached listing for one table. Backs {@code REFRESH TABLE}. */ @@ -206,6 +209,10 @@ long size() { return count[0]; } + MetaCacheEntryStats stats() { + return cache.stats(); + } + /** * The production {@link DirectoryLister}: a LITERAL listing through the engine-injected Doris * {@link FileSystem} (a per-catalog {@code SpiSwitchingFileSystem}), filtering out directories and @@ -334,10 +341,11 @@ private static boolean isSystemicResolutionFailure(Throwable t) { * size-estimate paths sharing the same entry while making per-partition invalidation possible. */ static final class FileListingKey { - private final String dbName; - private final String tableName; - private final String location; - private final List partitionValues; + final String dbName; + final String tableName; + final String location; + final List partitionValues; + final long estimatedBytes; FileListingKey(String dbName, String tableName, String location, List partitionValues) { this.dbName = dbName; @@ -346,6 +354,7 @@ static final class FileListingKey { this.partitionValues = partitionValues == null ? Collections.emptyList() : Collections.unmodifiableList(new ArrayList<>(partitionValues)); + this.estimatedBytes = HiveFileListingSizeEstimator.estimateKey(this); } boolean matches(String db, String table) { @@ -376,4 +385,14 @@ public int hashCode() { return Objects.hash(dbName, tableName, location, partitionValues); } } + + static final class FileListingValue { + final List files; + final long estimatedBytes; + + private FileListingValue(List files) { + this.files = Collections.unmodifiableList(new ArrayList<>(files)); + this.estimatedBytes = HiveFileListingSizeEstimator.estimateValue(this); + } + } } diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveFileListingSizeEstimator.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveFileListingSizeEstimator.java new file mode 100644 index 00000000000000..641b3e1957e5d5 --- /dev/null +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveFileListingSizeEstimator.java @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.hive; + +import org.apache.doris.connector.cache.JvmSizeUtils; +import org.apache.doris.connector.hive.HiveFileListingCache.FileListingKey; +import org.apache.doris.connector.hive.HiveFileListingCache.FileListingValue; + +/** Type-specific retained-heap estimator for one Hive directory-listing cache entry. */ +final class HiveFileListingSizeEstimator { + private static final long KEY_SHALLOW_BYTES = JvmSizeUtils.instanceSize(FileListingKey.class); + private static final long VALUE_SHALLOW_BYTES = JvmSizeUtils.instanceSize(FileListingValue.class); + private static final long FILE_STATUS_SHALLOW_BYTES = JvmSizeUtils.instanceSize(HiveFileStatus.class); + + private HiveFileListingSizeEstimator() { + } + + /** Caffeine callback: both key and value sizes were computed once during construction. */ + static long estimateEntry(FileListingKey key, FileListingValue value) { + return add(key.estimatedBytes, value.estimatedBytes); + } + + static long estimateKey(FileListingKey key) { + long bytes = KEY_SHALLOW_BYTES; + bytes = add(bytes, JvmSizeUtils.stringSize(key.dbName)); + bytes = add(bytes, JvmSizeUtils.stringSize(key.tableName)); + bytes = add(bytes, JvmSizeUtils.stringSize(key.location)); + bytes = add(bytes, estimateOwnedStringList(key.partitionValues)); + return bytes; + } + + static long estimateValue(FileListingValue value) { + long bytes = VALUE_SHALLOW_BYTES; + bytes = add(bytes, estimateArrayBackedList(value.files)); + for (HiveFileStatus file : value.files) { + bytes = add(bytes, FILE_STATUS_SHALLOW_BYTES); + bytes = add(bytes, JvmSizeUtils.stringSize(file.getPath())); + } + return bytes; + } + + private static long estimateOwnedStringList(java.util.List values) { + long bytes = estimateArrayBackedList(values); + for (String value : values) { + bytes = add(bytes, JvmSizeUtils.stringSize(value)); + } + return bytes; + } + + private static long estimateArrayBackedList(java.util.List values) { + long bytes = JvmSizeUtils.instanceSize(values.getClass()); + return add(bytes, JvmSizeUtils.arrayListSize(values.size())); + } + + private static long add(long left, long right) { + return JvmSizeUtils.saturatedAdd(left, right); + } +} diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveFileListingCacheTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveFileListingCacheTest.java index 08f567a9d718a8..5206f18b279fc1 100644 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveFileListingCacheTest.java +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveFileListingCacheTest.java @@ -192,6 +192,29 @@ public void legacyFileMetaCacheTtlZeroBypassesTheCache() { Assertions.assertEquals(2, lister.totalCalls); } + @Test + public void maximumWeightUsesTypedFileListingEstimator() { + CountingLister lister = new CountingLister(); + HiveFileListingCache cache = new HiveFileListingCache( + HiveCatalogProperties.of(props("meta.cache.hive.file.max-weight", "1MB")), lister); + + cache.listDataFiles("db", "table", "s3://bucket/table/p=1", Collections.singletonList("1"), FS); + + Assertions.assertTrue(cache.stats().isWeightBounded()); + Assertions.assertEquals(1024L * 1024, cache.stats().getMaxWeight()); + Assertions.assertTrue(cache.stats().getEstimatedWeight() > 0L); + } + + @Test + public void invalidMaximumWeightIsRejectedAtCreateTime() { + HiveCatalogProperties properties = HiveCatalogProperties.of( + props("meta.cache.hive.file.max-weight", "broken")); + IllegalArgumentException error = Assertions.assertThrows( + IllegalArgumentException.class, properties::checkCreateTimeOnlyRules); + Assertions.assertEquals( + "The parameter meta.cache.hive.file.max-weight is wrong, value is broken", error.getMessage()); + } + // ==================== the production lister: filters dirs + hidden files, lists literally ==================== @Test diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCacheSizeEstimator.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCacheSizeEstimator.java new file mode 100644 index 00000000000000..5557abfe8d7091 --- /dev/null +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCacheSizeEstimator.java @@ -0,0 +1,616 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.iceberg; + +import org.apache.doris.connector.cache.JvmSizeUtils; +import org.apache.doris.connector.iceberg.IcebergPartitionCache.CachedPartitions; +import org.apache.doris.connector.iceberg.IcebergPartitionCache.Key; +import org.apache.doris.connector.iceberg.IcebergPartitionUtils.IcebergRawPartition; +import org.apache.doris.connector.iceberg.IcebergTableCache.CachedTable; + +import org.apache.iceberg.BlobMetadata; +import org.apache.iceberg.ContentFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.HistoryEntry; +import org.apache.iceberg.MetadataUpdate; +import org.apache.iceberg.PartitionField; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.PartitionStatisticsFile; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.SortField; +import org.apache.iceberg.SortOrder; +import org.apache.iceberg.StatisticsFile; +import org.apache.iceberg.StructLike; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.UnboundPartitionSpec; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.IdentityHashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** Type-specific retained-heap estimators for the large Iceberg connector cache values. */ +final class IcebergCacheSizeEstimator { + private static final String[] CONTENT_FILE_FIELD_NAMES = { + "content", "file_path", "file_format", "partition", "record_count", "file_size_in_bytes", + "column_sizes", "value_counts", "null_value_counts", "nan_value_counts", "lower_bounds", + "upper_bounds", "key_metadata", "split_offsets", "equality_ids", "sort_order_id", "first_row_id", + "referenced_data_file", "content_offset", "content_size_in_bytes" + }; + private static final long TABLE_IDENTIFIER_SHALLOW_BYTES = JvmSizeUtils.instanceSize(TableIdentifier.class); + private static final long CACHED_TABLE_SHALLOW_BYTES = JvmSizeUtils.instanceSize(CachedTable.class); + private static final long TABLE_METADATA_SHALLOW_BYTES = JvmSizeUtils.instanceSize(TableMetadata.class); + private static final long PARTITION_KEY_SHALLOW_BYTES = JvmSizeUtils.instanceSize(Key.class); + private static final long CACHED_PARTITIONS_SHALLOW_BYTES = JvmSizeUtils.instanceSize(CachedPartitions.class); + private static final long RAW_PARTITION_SHALLOW_BYTES = JvmSizeUtils.instanceSize(IcebergRawPartition.class); + private static final long MANIFEST_KEY_SHALLOW_BYTES = JvmSizeUtils.instanceSize(IcebergManifestEntryKey.class); + private static final long MANIFEST_VALUE_SHALLOW_BYTES = JvmSizeUtils.instanceSize(ManifestCacheValue.class); + private static final long INTEGER_SHALLOW_BYTES = JvmSizeUtils.instanceSize(Integer.class); + private static final long LONG_SHALLOW_BYTES = JvmSizeUtils.instanceSize(Long.class); + private static final long HASH_MAP_NODE_SHALLOW_BYTES = classSize("java.util.HashMap$Node"); + private static final long LINKED_HASH_MAP_ENTRY_SHALLOW_BYTES = classSize("java.util.LinkedHashMap$Entry"); + private static final long CONTENT_FILE_SCHEMA_BYTES = estimateContentFileSchema(); + + private IcebergCacheSizeEstimator() { + } + + /** Caffeine callback: the expensive table graph was sized once when {@link CachedTable} was constructed. */ + static long estimateTableEntry(TableIdentifier key, CachedTable value) { + return add(estimateTableIdentifier(key), value.estimatedBytes); + } + + static long estimateTable(Table table) { + long bytes = add(CACHED_TABLE_SHALLOW_BYTES, JvmSizeUtils.instanceSize(table.getClass())); + bytes = add(bytes, JvmSizeUtils.stringSize(table.name())); + if (!(table instanceof HasTableOperations)) { + bytes = add(bytes, JvmSizeUtils.stringSize(table.location())); + return add(bytes, estimateStringMap(table.properties())); + } + + TableOperations operations = ((HasTableOperations) table).operations(); + bytes = add(bytes, JvmSizeUtils.instanceSize(operations.getClass())); + return add(bytes, estimateTableMetadata(operations.current())); + } + + static long estimatePartitionKey(Key key) { + return add(PARTITION_KEY_SHALLOW_BYTES, estimateTableIdentifier(key.id)); + } + + static long estimatePartitions(List partitions) { + long bytes = CACHED_PARTITIONS_SHALLOW_BYTES; + // CachedPartitions owns an unmodifiable wrapper around an exact-size ArrayList copy. + bytes = add(bytes, JvmSizeUtils.instanceSize(partitions.getClass())); + bytes = add(bytes, JvmSizeUtils.arrayListSize(partitions.size())); + for (IcebergRawPartition partition : partitions) { + bytes = add(bytes, RAW_PARTITION_SHALLOW_BYTES); + bytes = add(bytes, JvmSizeUtils.stringSize(partition.name)); + bytes = add(bytes, estimateStringList(partition.columnNames)); + bytes = add(bytes, estimateStringList(partition.values)); + bytes = add(bytes, estimateStringList(partition.transforms)); + } + return bytes; + } + + /** Caffeine callback: the partition list was sized once when {@link CachedPartitions} was constructed. */ + static long estimatePartitionEntry(Key key, CachedPartitions value) { + return add(key.estimatedBytes, value.estimatedBytes); + } + + static long estimateManifestKey(IcebergManifestEntryKey key) { + return add(MANIFEST_KEY_SHALLOW_BYTES, JvmSizeUtils.stringSize(key.getManifestPath())); + } + + static long estimateManifestValue(ManifestCacheValue value) { + long bytes = MANIFEST_VALUE_SHALLOW_BYTES; + bytes = add(bytes, estimateContentFileList(value.getDataFiles())); + return add(bytes, estimateContentFileList(value.getDeleteFiles())); + } + + /** Caffeine callback: key and manifest payload sizes are precomputed during construction. */ + static long estimateManifestEntry(IcebergManifestEntryKey key, ManifestCacheValue value) { + return add(key.getEstimatedBytes(), value.getEstimatedBytes()); + } + + private static long estimateTableIdentifier(TableIdentifier identifier) { + long bytes = TABLE_IDENTIFIER_SHALLOW_BYTES; + Namespace namespace = identifier.namespace(); + bytes = add(bytes, JvmSizeUtils.instanceSize(namespace.getClass())); + String[] levels = namespace.levels(); + bytes = add(bytes, JvmSizeUtils.objectArraySize(levels.length)); + for (String level : levels) { + bytes = add(bytes, JvmSizeUtils.stringSize(level)); + } + return add(bytes, JvmSizeUtils.stringSize(identifier.name())); + } + + private static long estimateTableMetadata(TableMetadata metadata) { + long bytes = TABLE_METADATA_SHALLOW_BYTES; + bytes = add(bytes, JvmSizeUtils.stringSize(metadata.metadataFileLocation())); + bytes = add(bytes, JvmSizeUtils.stringSize(metadata.uuid())); + bytes = add(bytes, JvmSizeUtils.stringSize(metadata.location())); + bytes = add(bytes, estimateStringMap(metadata.properties())); + + List schemas = metadata.schemas(); + bytes = add(bytes, estimateListStructure(schemas)); + bytes = add(bytes, estimateIntegerIndexMap(metadata.schemasById())); + for (Schema schema : schemas) { + bytes = add(bytes, estimateSchema(schema)); + } + + List specs = metadata.specs(); + bytes = add(bytes, estimateListStructure(specs)); + bytes = add(bytes, estimateIntegerIndexMap(metadata.specsById())); + for (PartitionSpec spec : specs) { + bytes = add(bytes, estimatePartitionSpec(spec)); + } + + List sortOrders = metadata.sortOrders(); + bytes = add(bytes, estimateListStructure(sortOrders)); + bytes = add(bytes, estimateIntegerIndexMap(metadata.sortOrdersById())); + for (SortOrder sortOrder : sortOrders) { + bytes = add(bytes, estimateSortOrder(sortOrder)); + } + + List snapshots = metadata.snapshots(); + bytes = add(bytes, estimateListStructure(snapshots)); + bytes = add(bytes, estimateLongIndexMap(snapshots)); + for (Snapshot snapshot : snapshots) { + bytes = add(bytes, estimateSnapshot(snapshot)); + } + + bytes = add(bytes, estimateHistory(metadata.snapshotLog())); + bytes = add(bytes, estimateMetadataLog(metadata.previousFiles())); + bytes = add(bytes, estimateSnapshotRefs(metadata.refs())); + bytes = add(bytes, estimateStatisticsFiles(metadata.statisticsFiles())); + bytes = add(bytes, estimatePartitionStatisticsFiles(metadata.partitionStatisticsFiles())); + bytes = add(bytes, estimateMetadataUpdates(metadata.changes())); + bytes = add(bytes, estimateShallowList(metadata.encryptionKeys())); + // TableMetadata retains a serializable snapshot supplier after the immutable snapshot list is loaded. + return add(bytes, JvmSizeUtils.objectArraySize(1)); + } + + private static long estimateSchema(Schema schema) { + List columns = schema.columns(); + long bytes = JvmSizeUtils.instanceSize(schema.getClass()); + bytes = add(bytes, JvmSizeUtils.instanceSize(schema.asStruct().getClass())); + bytes = add(bytes, estimateListStructure(columns)); + for (Types.NestedField field : columns) { + bytes = add(bytes, estimateNestedField(field)); + } + bytes = add(bytes, JvmSizeUtils.objectArraySize(schema.identifierFieldIds().size())); + bytes = add(bytes, estimateMapStructure(schema.getAliases())); + return add(bytes, estimateSchemaIndexes(columns.size())); + } + + private static long estimateNestedField(Types.NestedField field) { + long bytes = JvmSizeUtils.instanceSize(field.getClass()); + bytes = add(bytes, JvmSizeUtils.stringSize(field.name())); + bytes = add(bytes, JvmSizeUtils.stringSize(field.doc())); + return add(bytes, estimateIcebergType(field.type())); + } + + private static long estimateIcebergType(Type type) { + long bytes = JvmSizeUtils.instanceSize(type.getClass()); + if (type.isStructType()) { + List fields = type.asStructType().fields(); + bytes = add(bytes, estimateListStructure(fields)); + for (Types.NestedField field : fields) { + bytes = add(bytes, estimateNestedField(field)); + } + } else if (type.isListType()) { + bytes = add(bytes, estimateNestedField(type.asListType().fields().get(0))); + } else if (type.isMapType()) { + for (Types.NestedField field : type.asMapType().fields()) { + bytes = add(bytes, estimateNestedField(field)); + } + } + return bytes; + } + + private static long estimatePartitionSpec(PartitionSpec spec) { + List fields = spec.fields(); + long bytes = add(JvmSizeUtils.instanceSize(spec.getClass()), JvmSizeUtils.objectArraySize(fields.size())); + for (PartitionField field : fields) { + bytes = add(bytes, JvmSizeUtils.instanceSize(field.getClass())); + bytes = add(bytes, JvmSizeUtils.stringSize(field.name())); + bytes = add(bytes, JvmSizeUtils.instanceSize(field.transform().getClass())); + } + return bytes; + } + + private static long estimateSortOrder(SortOrder sortOrder) { + List fields = sortOrder.fields(); + long bytes = add(JvmSizeUtils.instanceSize(sortOrder.getClass()), JvmSizeUtils.objectArraySize(fields.size())); + for (SortField field : fields) { + bytes = add(bytes, JvmSizeUtils.instanceSize(field.getClass())); + bytes = add(bytes, JvmSizeUtils.instanceSize(field.transform().getClass())); + } + return bytes; + } + + private static long estimateSnapshot(Snapshot snapshot) { + long bytes = JvmSizeUtils.instanceSize(snapshot.getClass()); + bytes = add(bytes, estimateBoxed(snapshot.parentId(), LONG_SHALLOW_BYTES)); + bytes = add(bytes, estimateBoxed(snapshot.schemaId(), INTEGER_SHALLOW_BYTES)); + bytes = add(bytes, estimateBoxed(snapshot.firstRowId(), LONG_SHALLOW_BYTES)); + bytes = add(bytes, estimateBoxed(snapshot.addedRows(), LONG_SHALLOW_BYTES)); + bytes = add(bytes, JvmSizeUtils.stringSize(snapshot.operation())); + bytes = add(bytes, JvmSizeUtils.stringSize(snapshot.manifestListLocation())); + bytes = add(bytes, JvmSizeUtils.stringSize(snapshot.keyId())); + return add(bytes, estimateStringMap(snapshot.summary())); + } + + private static long estimateHistory(List history) { + long bytes = estimateListStructure(history); + for (HistoryEntry entry : history) { + bytes = add(bytes, JvmSizeUtils.instanceSize(entry.getClass())); + } + return bytes; + } + + private static long estimateMetadataLog(List entries) { + long bytes = estimateListStructure(entries); + for (TableMetadata.MetadataLogEntry entry : entries) { + bytes = add(bytes, JvmSizeUtils.instanceSize(entry.getClass())); + bytes = add(bytes, JvmSizeUtils.stringSize(entry.file())); + } + return bytes; + } + + private static long estimateSnapshotRefs(Map refs) { + long bytes = estimateMapStructure(refs); + for (Map.Entry entry : refs.entrySet()) { + bytes = add(bytes, JvmSizeUtils.stringSize(entry.getKey())); + SnapshotRef ref = entry.getValue(); + bytes = add(bytes, JvmSizeUtils.instanceSize(ref.getClass())); + bytes = add(bytes, estimateBoxed(ref.minSnapshotsToKeep(), INTEGER_SHALLOW_BYTES)); + bytes = add(bytes, estimateBoxed(ref.maxSnapshotAgeMs(), LONG_SHALLOW_BYTES)); + bytes = add(bytes, estimateBoxed(ref.maxRefAgeMs(), LONG_SHALLOW_BYTES)); + } + return bytes; + } + + private static long estimateStatisticsFiles(List files) { + long bytes = estimateListStructure(files); + for (StatisticsFile file : files) { + bytes = add(bytes, JvmSizeUtils.instanceSize(file.getClass())); + bytes = add(bytes, JvmSizeUtils.stringSize(file.path())); + List blobs = file.blobMetadata(); + bytes = add(bytes, estimateListStructure(blobs)); + for (BlobMetadata blob : blobs) { + bytes = add(bytes, JvmSizeUtils.instanceSize(blob.getClass())); + bytes = add(bytes, JvmSizeUtils.stringSize(blob.type())); + bytes = add(bytes, estimateBoxedList(blob.fields(), INTEGER_SHALLOW_BYTES)); + bytes = add(bytes, estimateStringMap(blob.properties())); + } + } + return bytes; + } + + private static long estimatePartitionStatisticsFiles(List files) { + long bytes = estimateListStructure(files); + for (PartitionStatisticsFile file : files) { + bytes = add(bytes, JvmSizeUtils.instanceSize(file.getClass())); + bytes = add(bytes, JvmSizeUtils.stringSize(file.path())); + } + return bytes; + } + + private static long estimateShallowList(List values) { + long bytes = estimateListStructure(values); + for (Object value : values) { + bytes = add(bytes, JvmSizeUtils.instanceSize(value.getClass())); + } + return bytes; + } + + private static long estimateMetadataUpdates(List updates) { + long bytes = estimateListStructure(updates); + for (MetadataUpdate update : updates) { + bytes = add(bytes, JvmSizeUtils.instanceSize(update.getClass())); + if (update instanceof MetadataUpdate.SetProperties) { + bytes = add(bytes, estimateMapStructure(((MetadataUpdate.SetProperties) update).updated())); + } else if (update instanceof MetadataUpdate.AddPartitionSpec) { + UnboundPartitionSpec spec = ((MetadataUpdate.AddPartitionSpec) update).spec(); + bytes = add(bytes, JvmSizeUtils.instanceSize(spec.getClass())); + bytes = add(bytes, estimateListStructure(spec.fields())); + } else if (update instanceof MetadataUpdate.AddSortOrder) { + bytes = add(bytes, JvmSizeUtils.instanceSize( + ((MetadataUpdate.AddSortOrder) update).sortOrder().getClass())); + } + } + return bytes; + } + + private static long estimateContentFileList(List> files) { + if (files.isEmpty()) { + return 0L; + } + long bytes = add(estimateListStructure(files), CONTENT_FILE_SCHEMA_BYTES); + Set ownedObjects = java.util.Collections.newSetFromMap(new IdentityHashMap<>()); + for (ContentFile file : files) { + bytes = add(bytes, estimateContentFile(file, ownedObjects)); + } + return bytes; + } + + private static long estimateContentFileSchema() { + long bytes = JvmSizeUtils.instanceSize(Types.StructType.class); + bytes = add(bytes, JvmSizeUtils.arrayListSize(CONTENT_FILE_FIELD_NAMES.length)); + for (String name : CONTENT_FILE_FIELD_NAMES) { + bytes = add(bytes, JvmSizeUtils.instanceSize(Types.NestedField.class)); + bytes = add(bytes, JvmSizeUtils.stringSize(name)); + } + return bytes; + } + + private static long estimateContentFile(ContentFile file, Set ownedObjects) { + long bytes = JvmSizeUtils.instanceSize(file.getClass()); + if (file instanceof StructLike) { + bytes = add(bytes, JvmSizeUtils.intArraySize(((StructLike) file).size())); + bytes = add(bytes, LONG_SHALLOW_BYTES); + } + bytes = add(bytes, estimateOwnedCharSequence(file.path(), ownedObjects)); + bytes = add(bytes, estimateOwnedString(file.manifestLocation(), ownedObjects)); + bytes = add(bytes, estimatePartition(file.partition(), ownedObjects)); + bytes = add(bytes, estimateLongMap(file.columnSizes())); + bytes = add(bytes, estimateLongMap(file.valueCounts())); + bytes = add(bytes, estimateLongMap(file.nullValueCounts())); + bytes = add(bytes, estimateLongMap(file.nanValueCounts())); + bytes = add(bytes, estimateByteBufferMap(file.lowerBounds())); + bytes = add(bytes, estimateByteBufferMap(file.upperBounds())); + ByteBuffer keyMetadata = file.keyMetadata(); + if (keyMetadata != null) { + bytes = add(bytes, JvmSizeUtils.byteArraySize(keyMetadata.remaining())); + } + List splitOffsets = file.splitOffsets(); + if (splitOffsets != null) { + bytes = add(bytes, JvmSizeUtils.longArraySize(splitOffsets.size())); + } + List equalityFieldIds = file.equalityFieldIds(); + if (equalityFieldIds != null) { + bytes = add(bytes, JvmSizeUtils.intArraySize(equalityFieldIds.size())); + } + bytes = add(bytes, estimateBoxed(file.pos(), LONG_SHALLOW_BYTES)); + bytes = add(bytes, estimateBoxed(file.sortOrderId(), INTEGER_SHALLOW_BYTES)); + bytes = add(bytes, estimateBoxed(file.dataSequenceNumber(), LONG_SHALLOW_BYTES)); + bytes = add(bytes, estimateBoxed(file.fileSequenceNumber(), LONG_SHALLOW_BYTES)); + bytes = add(bytes, estimateBoxed(file.firstRowId(), LONG_SHALLOW_BYTES)); + if (file instanceof DeleteFile) { + DeleteFile deleteFile = (DeleteFile) file; + bytes = add(bytes, estimateOwnedString(deleteFile.referencedDataFile(), ownedObjects)); + bytes = add(bytes, estimateBoxed(deleteFile.contentOffset(), LONG_SHALLOW_BYTES)); + bytes = add(bytes, estimateBoxed(deleteFile.contentSizeInBytes(), LONG_SHALLOW_BYTES)); + } + return bytes; + } + + private static long estimatePartition(StructLike partition, Set ownedObjects) { + if (partition == null || !ownedObjects.add(partition)) { + return 0L; + } + long bytes = JvmSizeUtils.instanceSize(partition.getClass()); + bytes = add(bytes, JvmSizeUtils.objectArraySize(partition.size())); + for (int i = 0; i < partition.size(); i++) { + bytes = add(bytes, estimateOwnedScalar(partition.get(i, Object.class), ownedObjects)); + } + return bytes; + } + + private static long estimateOwnedScalar(Object value, Set ownedObjects) { + if (value == null || !ownedObjects.add(value)) { + return 0L; + } + if (value instanceof CharSequence) { + return estimateCharSequence((CharSequence) value); + } + if (value instanceof ByteBuffer) { + return estimateByteBuffer((ByteBuffer) value); + } + return JvmSizeUtils.instanceSize(value.getClass()); + } + + private static long estimateOwnedCharSequence(CharSequence value, Set ownedObjects) { + return value == null || !ownedObjects.add(value) ? 0L : estimateCharSequence(value); + } + + private static long estimateOwnedString(String value, Set ownedStrings) { + return value == null || !ownedStrings.add(value) ? 0L : JvmSizeUtils.stringSize(value); + } + + private static long estimateCharSequence(CharSequence value) { + if (value instanceof String) { + return JvmSizeUtils.stringSize((String) value); + } + return add(JvmSizeUtils.instanceSize(value.getClass()), JvmSizeUtils.stringSize(value.toString())); + } + + private static long estimateLongMap(Map values) { + if (values == null || values.isEmpty()) { + return 0L; + } + return add(estimateMapStructure(values), multiply(values.size(), INTEGER_SHALLOW_BYTES + LONG_SHALLOW_BYTES)); + } + + private static long estimateByteBufferMap(Map values) { + if (values == null || values.isEmpty()) { + return 0L; + } + long bytes = add(estimateMapStructure(values), multiply(values.size(), INTEGER_SHALLOW_BYTES)); + for (ByteBuffer value : values.values()) { + bytes = add(bytes, estimateByteBuffer(value)); + } + return bytes; + } + + private static long estimateByteBuffer(ByteBuffer value) { + if (value == null) { + return 0L; + } + long bytes = JvmSizeUtils.instanceSize(value.getClass()); + return value.hasArray() ? add(bytes, JvmSizeUtils.byteArraySize(value.capacity())) : bytes; + } + + private static long estimateSchemaIndexes(int fieldCount) { + if (fieldCount == 0) { + return 0L; + } + long oneIndex = JvmSizeUtils.instanceSize(HashMap.class); + oneIndex = add(oneIndex, JvmSizeUtils.objectArraySize(hashCapacity(fieldCount))); + oneIndex = add(oneIndex, multiply(fieldCount, HASH_MAP_NODE_SHALLOW_BYTES)); + long bytes = multiply(3L, oneIndex); + return add(bytes, multiply(2L * fieldCount, INTEGER_SHALLOW_BYTES)); + } + + private static long estimateIntegerIndexMap(Map values) { + return add(estimateMapStructure(values), multiply(values.size(), INTEGER_SHALLOW_BYTES)); + } + + private static long estimateLongIndexMap(List values) { + if (values.isEmpty()) { + return JvmSizeUtils.instanceSize(HashMap.class); + } + long bytes = JvmSizeUtils.instanceSize(HashMap.class); + bytes = add(bytes, JvmSizeUtils.objectArraySize(hashCapacity(values.size()))); + bytes = add(bytes, multiply(values.size(), HASH_MAP_NODE_SHALLOW_BYTES)); + return add(bytes, multiply(values.size(), LONG_SHALLOW_BYTES)); + } + + private static long estimateStringMap(Map values) { + if (values == null) { + return 0L; + } + long bytes = estimateMapStructure(values); + for (Map.Entry entry : values.entrySet()) { + bytes = add(bytes, JvmSizeUtils.stringSize(entry.getKey())); + bytes = add(bytes, JvmSizeUtils.stringSize(entry.getValue())); + } + return bytes; + } + + private static long estimateStringList(List values) { + if (values == null || values.isEmpty()) { + return 0L; + } + long bytes = estimateListStructure(values); + for (String value : values) { + bytes = add(bytes, JvmSizeUtils.stringSize(value)); + } + return bytes; + } + + private static long estimateBoxedList(List values, long elementBytes) { + if (values == null || values.isEmpty()) { + return 0L; + } + return add(estimateListStructure(values), multiply(values.size(), elementBytes)); + } + + private static long estimateBoxed(Object value, long bytes) { + return value == null ? 0L : bytes; + } + + private static long estimateListStructure(List values) { + int capacity = values instanceof ArrayList ? arrayListCapacity(values.size()) : values.size(); + long bytes = JvmSizeUtils.instanceSize(values.getClass()); + if (values.getClass().getName().equals("java.util.ImmutableCollections$List12")) { + return bytes; + } + return add(bytes, JvmSizeUtils.objectArraySize(capacity)); + } + + private static long estimateMapStructure(Map values) { + if (values == null) { + return 0L; + } + long bytes = JvmSizeUtils.instanceSize(values.getClass()); + if (values.isEmpty()) { + return bytes; + } + if (values instanceof HashMap) { + int capacity = hashCapacity(values.size()); + long nodeBytes = values instanceof LinkedHashMap + ? LINKED_HASH_MAP_ENTRY_SHALLOW_BYTES + : HASH_MAP_NODE_SHALLOW_BYTES; + bytes = add(bytes, JvmSizeUtils.objectArraySize(capacity)); + return add(bytes, multiply(values.size(), nodeBytes)); + } + bytes = add(bytes, JvmSizeUtils.objectArraySize(saturatedDouble(values.size()))); + return add(bytes, multiply(values.size(), HASH_MAP_NODE_SHALLOW_BYTES)); + } + + private static int arrayListCapacity(int size) { + if (size == 0) { + return 0; + } + int capacity = 10; + while (capacity < size) { + int grown = capacity + (capacity >> 1); + if (grown < 0) { + return Integer.MAX_VALUE; + } + capacity = grown; + } + return capacity; + } + + private static int hashCapacity(int size) { + if (size == 0) { + return 0; + } + long needed = (size * 4L + 2L) / 3L; + int capacity = 16; + while (capacity < needed && capacity < 1 << 30) { + capacity <<= 1; + } + return capacity; + } + + private static int saturatedDouble(int value) { + return value > Integer.MAX_VALUE / 2 ? Integer.MAX_VALUE : value * 2; + } + + private static long multiply(long left, long right) { + return JvmSizeUtils.saturatedMultiply(left, right); + } + + private static long classSize(String className) { + try { + return JvmSizeUtils.instanceSize(Class.forName(className)); + } catch (ClassNotFoundException e) { + throw new IllegalStateException("Required JVM collection class is missing: " + className, e); + } + } + + private static long add(long left, long right) { + return JvmSizeUtils.saturatedAdd(left, right); + } +} diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogProperties.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogProperties.java index 8cf4db88386e8e..987ba22e3a169a 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogProperties.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogProperties.java @@ -158,6 +158,8 @@ private static void checkMetaCacheProperties(Map properties) { -1L, IcebergConnector.TABLE_CACHE_TTL_SECOND); CacheSpec.checkLongProperty(properties.get(IcebergConnector.TABLE_CACHE_CAPACITY), 0L, IcebergConnector.TABLE_CACHE_CAPACITY); + CacheSpec.checkDataSizeProperty(properties.get(IcebergConnector.TABLE_CACHE_MAX_WEIGHT), + IcebergConnector.TABLE_CACHE_MAX_WEIGHT); CacheSpec.checkBooleanProperty(properties.get(IcebergConnector.MANIFEST_CACHE_ENABLE), IcebergConnector.MANIFEST_CACHE_ENABLE); @@ -165,6 +167,8 @@ private static void checkMetaCacheProperties(Map properties) { -1L, IcebergConnector.MANIFEST_CACHE_TTL_SECOND); CacheSpec.checkLongProperty(properties.get(IcebergConnector.MANIFEST_CACHE_CAPACITY), 0L, IcebergConnector.MANIFEST_CACHE_CAPACITY); + CacheSpec.checkDataSizeProperty(properties.get(IcebergConnector.MANIFEST_CACHE_MAX_WEIGHT), + IcebergConnector.MANIFEST_CACHE_MAX_WEIGHT); } /** The metastore backend, lower-cased; {@code null} when the catalog does not name one. */ diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java index 47df2d3f9c6f06..79c580705ec858 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java @@ -17,6 +17,7 @@ package org.apache.doris.connector.iceberg; +import org.apache.doris.connector.cache.CacheSpec; import org.apache.doris.connector.cache.ConnectorMetadataCache; import org.apache.doris.connector.metastore.HmsMetaStoreProperties; import org.apache.doris.connector.metastore.iceberg.jdbc.IcebergJdbcMetaStoreProperties; @@ -141,9 +142,11 @@ public class IcebergConnector implements Connector { static final String TABLE_CACHE_ENABLE = "meta.cache.iceberg.table.enable"; static final String TABLE_CACHE_TTL_SECOND = "meta.cache.iceberg.table.ttl-second"; static final String TABLE_CACHE_CAPACITY = "meta.cache.iceberg.table.capacity"; + static final String TABLE_CACHE_MAX_WEIGHT = "meta.cache.iceberg.table.max-weight"; static final String MANIFEST_CACHE_ENABLE = "meta.cache.iceberg.manifest.enable"; static final String MANIFEST_CACHE_TTL_SECOND = "meta.cache.iceberg.manifest.ttl-second"; static final String MANIFEST_CACHE_CAPACITY = "meta.cache.iceberg.manifest.capacity"; + static final String MANIFEST_CACHE_MAX_WEIGHT = "meta.cache.iceberg.manifest.max-weight"; static final long DEFAULT_TABLE_CACHE_TTL_SECOND = 86400L; static final int DEFAULT_TABLE_CACHE_CAPACITY = 1000; @@ -224,7 +227,7 @@ public class IcebergConnector implements Connector { listPartitionsViewCache; // Manifest content cache — pure metadata, default-off (meta.cache.iceberg.manifest.enable), and consumed // ONLY after a per-user resolveTable(ForRead) -- exempt: no read path without a per-user load. - private final IcebergManifestCache manifestCache = new IcebergManifestCache(); + private final IcebergManifestCache manifestCache; // Lazily-built plugin-side Kerberos authenticator (single-owner auth; see TcclPinningConnectorContext). // null for a non-Kerberos catalog. Its doAs acts on the PLUGIN's UserGroupInformation copy — the one the @@ -248,6 +251,8 @@ public IcebergConnector(Map properties, ConnectorContext context // authenticator never logs in — so without this the DDL/read hits secured HDFS as SIMPLE auth. this.context = new TcclPinningConnectorContext(context, getClass().getClassLoader(), this::pluginAuthenticator); + CacheSpec tableCacheSpec = resolveTableCacheSpec(this.properties); + this.manifestCache = new IcebergManifestCache(this.properties); // Authorization-sensitive projection (snapshotId/schemaId). Under iceberg.rest.session=user the value is // per-user AUTHORIZED metadata that a "can-list-cannot-load" principal must not see. beginQuerySnapshot // reads this cache WITHOUT a preceding per-user loadTable, so a shared (table-keyed, no user dimension) @@ -270,8 +275,7 @@ public IcebergConnector(Map properties, ConnectorContext context this.tableCache = (isUserSessionEnabled() || IcebergScanPlanProvider.restVendedCredentialsEnabled(this.properties)) ? null - : new IcebergTableCache( - resolveTableCacheTtlSecond(this.properties), DEFAULT_TABLE_CACHE_CAPACITY); + : new IcebergTableCache(tableCacheSpec); // PERF-02: partition-view cache. Authorization-sensitive projection: a shared (table+snapshot-keyed, no // user dimension) hit would disclose one user's partition list. Its readers are all downstream of a // per-user resolveTableForRead today (so a hit cannot precede authz), but that safety rests entirely on @@ -280,8 +284,7 @@ public IcebergConnector(Map properties, ConnectorContext context // otherwise (single static identity). Readers already tolerate a null cache (loadRawPartitions). this.partitionCache = isUserSessionEnabled() ? null - : new IcebergPartitionCache( - resolveTableCacheTtlSecond(this.properties), DEFAULT_TABLE_CACHE_CAPACITY); + : new IcebergPartitionCache(tableCacheSpec); // PERF-03: inferred-file-format cache. Same authorization-sensitive treatment as partitionCache (disabled // under session=user, kept otherwise); readers already tolerate a null cache (resolveFileFormatName). this.formatCache = isUserSessionEnabled() @@ -330,6 +333,24 @@ static long resolveTableCacheTtlSecond(Map properties) { } } + private static CacheSpec resolveTableCacheSpec(Map properties) { + CacheSpec parsed = CacheSpec.fromProperties( + properties, + "iceberg", + "table", + CacheSpec.of(true, DEFAULT_TABLE_CACHE_TTL_SECOND, DEFAULT_TABLE_CACHE_CAPACITY)); + if (parsed.getTtlSecond() > 0L) { + return parsed; + } + if (parsed.isWeightBounded()) { + return CacheSpec.ofWeight( + parsed.isEnable(), CacheSpec.CACHE_TTL_DISABLE_CACHE, + parsed.getCapacity(), parsed.getMaxWeight().getAsLong()); + } + return CacheSpec.of( + parsed.isEnable(), CacheSpec.CACHE_TTL_DISABLE_CACHE, parsed.getCapacity()); + } + @Override public ConnectorMetadata getMetadata(ConnectorSession session) { return new IcebergConnectorMetadata(newCatalogBackedOps(session), catalogProps, context, diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergManifestCache.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergManifestCache.java index f3c46732fb2fc1..afe6bedf1ab166 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergManifestCache.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergManifestCache.java @@ -19,6 +19,7 @@ import org.apache.doris.connector.cache.CacheSpec; import org.apache.doris.connector.cache.MetaCacheEntry; +import org.apache.doris.connector.cache.MetaCacheEntryStats; import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; @@ -100,12 +101,26 @@ private static final class ScanStats { this(maxSize, DEFAULT_STATS_TTL_SECONDS, System::nanoTime); } + IcebergManifestCache(Map properties) { + this(CacheSpec.fromProperties( + properties, + "iceberg", + "manifest", + CacheSpec.of(false, CacheSpec.CACHE_NO_TTL, DEFAULT_MANIFEST_CACHE_CAPACITY)), + DEFAULT_STATS_TTL_SECONDS, + System::nanoTime); + } + /** Visible for testing: injectable stats TTL + clock so the leak sweep is deterministic without sleeping. */ IcebergManifestCache(int maxSize, long statsTtlSeconds, LongSupplier nanoClock) { // Always enabled, no expiry, capacity-bounded (CACHE_NO_TTL == -1 means "no expiration", enabled). - CacheSpec spec = CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, Math.max(1, maxSize)); + this(CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, Math.max(1, maxSize)), statsTtlSeconds, nanoClock); + } + + private IcebergManifestCache(CacheSpec spec, long statsTtlSeconds, LongSupplier nanoClock) { this.entry = new MetaCacheEntry<>("iceberg-manifest", null, spec, - ForkJoinPool.commonPool(), false, true, 0L, true); + ForkJoinPool.commonPool(), false, true, 0L, true, + IcebergCacheSizeEstimator::estimateManifestEntry); this.statsTtlNanos = TimeUnit.SECONDS.toNanos(Math.max(1L, statsTtlSeconds)); this.nanoClock = nanoClock; } @@ -232,4 +247,8 @@ int size() { entry.forEach((key, value) -> count[0]++); return count[0]; } + + MetaCacheEntryStats stats() { + return entry.stats(); + } } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergManifestEntryKey.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergManifestEntryKey.java index 90552bef6d5d3a..c9901d387d0403 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergManifestEntryKey.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergManifestEntryKey.java @@ -35,10 +35,12 @@ public class IcebergManifestEntryKey { private final String manifestPath; private final ManifestContent content; + private final long estimatedBytes; public IcebergManifestEntryKey(String manifestPath, ManifestContent content) { this.manifestPath = Objects.requireNonNull(manifestPath, "manifestPath can not be null"); this.content = Objects.requireNonNull(content, "content can not be null"); + this.estimatedBytes = IcebergCacheSizeEstimator.estimateManifestKey(this); } public static IcebergManifestEntryKey of(ManifestFile manifest) { @@ -53,6 +55,10 @@ public ManifestContent getContent() { return content; } + long getEstimatedBytes() { + return estimatedBytes; + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionCache.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionCache.java index a85e2187043e7c..30a3834f8721b1 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionCache.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionCache.java @@ -19,11 +19,14 @@ import org.apache.doris.connector.cache.CacheSpec; import org.apache.doris.connector.cache.MetaCacheEntry; +import org.apache.doris.connector.cache.MetaCacheEntryStats; import org.apache.doris.connector.iceberg.IcebergPartitionUtils.IcebergRawPartition; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.concurrent.ForkJoinPool; @@ -61,10 +64,12 @@ final class IcebergPartitionCache { static final class Key { final TableIdentifier id; final long snapshotId; + final long estimatedBytes; Key(TableIdentifier id, long snapshotId) { this.id = id; this.snapshotId = snapshotId; + this.estimatedBytes = IcebergCacheSizeEstimator.estimatePartitionKey(this); } @Override @@ -85,13 +90,27 @@ public int hashCode() { } } - private final MetaCacheEntry> entry; + static final class CachedPartitions { + final List partitions; + final long estimatedBytes; + + CachedPartitions(List partitions) { + this.partitions = Collections.unmodifiableList(new ArrayList<>(partitions)); + this.estimatedBytes = IcebergCacheSizeEstimator.estimatePartitions(this.partitions); + } + } + + private final MetaCacheEntry entry; IcebergPartitionCache(long ttlSeconds, int maxSize) { // "<= 0 disables" connector TTL contract, folded to CacheSpec's disable sentinel (CacheSpec.ofConnectorTtl). - CacheSpec spec = CacheSpec.ofConnectorTtl(ttlSeconds, maxSize); + this(CacheSpec.ofConnectorTtl(ttlSeconds, maxSize)); + } + + IcebergPartitionCache(CacheSpec spec) { this.entry = new MetaCacheEntry<>("iceberg-partition", null, spec, - ForkJoinPool.commonPool(), false, true, 0L, true); + ForkJoinPool.commonPool(), false, true, 0L, true, + IcebergCacheSizeEstimator::estimatePartitionEntry); } /** Caching is on only when the TTL is positive; ttl-second <= 0 means "always scan live". */ @@ -105,7 +124,7 @@ boolean isEnabled() { * loader runs OUTSIDE Caffeine's compute lock (single-flight per key) and its exception propagates unwrapped. */ List getOrLoad(Key key, Supplier> loader) { - return entry.get(key, ignored -> loader.get()); + return entry.get(key, ignored -> new CachedPartitions(loader.get())).partitions; } /** Drops every cached snapshot entry for one table so the next read scans live (REFRESH TABLE). */ @@ -135,4 +154,8 @@ int size() { long loadCountForTest() { return entry.stats().getLoadSuccessCount(); } + + MetaCacheEntryStats stats() { + return entry.stats(); + } } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java index a5ff4562e915cd..2307cf7de7d422 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java @@ -971,15 +971,15 @@ static long latestSnapshotId(String name, Map> mergeMap, /** One PARTITIONS-metadata-table row reduced to what the MTMV partition view needs (port of IcebergPartition). */ static final class IcebergRawPartition { - private final String name; + final String name; // Partition-field SOURCE column names (lowercased), parallel to {@link #values}, so listPartitions can // build a value map keyed by the generic partition-column remote name (see IcebergConnectorMetadata // buildTableSchema's "partition_columns" derivation). - private final List columnNames; - private final List values; - private final List transforms; - private final long lastUpdateTime; - private final long lastSnapshotId; + final List columnNames; + final List values; + final List transforms; + final long lastUpdateTime; + final long lastSnapshotId; IcebergRawPartition(String name, List columnNames, List values, List transforms, long lastUpdateTime, long lastSnapshotId) { diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableCache.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableCache.java index 426b706cf11f90..e7bac2d6f46fcf 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableCache.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableCache.java @@ -19,6 +19,7 @@ import org.apache.doris.connector.cache.CacheSpec; import org.apache.doris.connector.cache.MetaCacheEntry; +import org.apache.doris.connector.cache.MetaCacheEntryStats; import org.apache.iceberg.Table; import org.apache.iceberg.catalog.Namespace; @@ -58,13 +59,27 @@ */ final class IcebergTableCache { - private final MetaCacheEntry entry; + static final class CachedTable { + final Table table; + final long estimatedBytes; + + CachedTable(Table table) { + this.table = table; + this.estimatedBytes = IcebergCacheSizeEstimator.estimateTable(table); + } + } + + private final MetaCacheEntry entry; IcebergTableCache(long ttlSeconds, int maxSize) { // "<= 0 disables" connector TTL contract, folded to CacheSpec's disable sentinel (CacheSpec.ofConnectorTtl). - CacheSpec spec = CacheSpec.ofConnectorTtl(ttlSeconds, maxSize); + this(CacheSpec.ofConnectorTtl(ttlSeconds, maxSize)); + } + + IcebergTableCache(CacheSpec spec) { this.entry = new MetaCacheEntry<>("iceberg-table", null, spec, - ForkJoinPool.commonPool(), false, true, 0L, true); + ForkJoinPool.commonPool(), false, true, 0L, true, + IcebergCacheSizeEstimator::estimateTableEntry); } /** Caching is on only when the TTL is positive; ttl-second <= 0 means "always read live". */ @@ -80,7 +95,7 @@ boolean isEnabled() { * propagates unwrapped. */ Table getOrLoad(TableIdentifier identifier, Supplier loader) { - return entry.get(identifier, ignored -> loader.get()); + return entry.get(identifier, ignored -> new CachedTable(loader.get())).table; } /** Drops the cached entry for one table so the next read goes live (REFRESH TABLE). */ @@ -110,4 +125,8 @@ int size() { entry.forEach((key, value) -> count[0]++); return count[0]; } + + MetaCacheEntryStats stats() { + return entry.stats(); + } } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/ManifestCacheValue.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/ManifestCacheValue.java index b9b9da68dbd3c5..06e68ad769878e 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/ManifestCacheValue.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/ManifestCacheValue.java @@ -33,10 +33,12 @@ public class ManifestCacheValue { private final List dataFiles; private final List deleteFiles; + private final long estimatedBytes; private ManifestCacheValue(List dataFiles, List deleteFiles) { this.dataFiles = dataFiles == null ? Collections.emptyList() : dataFiles; this.deleteFiles = deleteFiles == null ? Collections.emptyList() : deleteFiles; + this.estimatedBytes = IcebergCacheSizeEstimator.estimateManifestValue(this); } public static ManifestCacheValue forDataFiles(List dataFiles) { @@ -54,4 +56,8 @@ public List getDataFiles() { public List getDeleteFiles() { return deleteFiles; } + + long getEstimatedBytes() { + return estimatedBytes; + } } From 41bcc116bf4efda13c1efb16dde81ef9e82cf491 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Fri, 7 Aug 2026 16:51:53 +0800 Subject: [PATCH 3/5] [test](fe) Add metadata cache estimator benchmarks ### What problem does this PR solve? Issue Number: None Related PR: #66533 Problem Summary: Add opt-in JMH benchmarks for the production Hive file-listing and Iceberg partition and manifest cache estimators. The benchmarks compare the constant-time cached weight lookup, the one-time value construction and estimation cost, and JOL retained-graph traversal without adding benchmark dependencies to the default FE reactor. ### Release note None ### Check List (For Author) - Test: Manual test - `mvn -Pbenchmark -pl fe-benchmark -am test-compile -DskipTests` - Behavior changed: No - Does this need documentation: No --- fe/fe-benchmark/pom.xml | 87 ++++++++++++++++ .../run-metacache-estimator-benchmark.sh | 49 +++++++++ .../hive/HiveFileListingSizeBenchmark.java | 90 +++++++++++++++++ .../iceberg/IcebergManifestSizeBenchmark.java | 92 +++++++++++++++++ .../IcebergPartitionSizeBenchmark.java | 99 +++++++++++++++++++ .../connector/hive/HiveFileListingCache.java | 2 +- fe/pom.xml | 6 ++ 7 files changed, 424 insertions(+), 1 deletion(-) create mode 100644 fe/fe-benchmark/pom.xml create mode 100755 fe/fe-benchmark/run-metacache-estimator-benchmark.sh create mode 100644 fe/fe-benchmark/src/main/java/org/apache/doris/connector/hive/HiveFileListingSizeBenchmark.java create mode 100644 fe/fe-benchmark/src/main/java/org/apache/doris/connector/iceberg/IcebergManifestSizeBenchmark.java create mode 100644 fe/fe-benchmark/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionSizeBenchmark.java diff --git a/fe/fe-benchmark/pom.xml b/fe/fe-benchmark/pom.xml new file mode 100644 index 00000000000000..5a4412a9081afd --- /dev/null +++ b/fe/fe-benchmark/pom.xml @@ -0,0 +1,87 @@ + + + + 4.0.0 + + + org.apache.doris + fe + ${revision} + ../pom.xml + + + fe-benchmark + Doris FE Benchmarks + + + 1.37 + 0.17 + + + + + ${project.groupId} + fe-connector-hive + ${project.version} + + + ${project.groupId} + fe-connector-iceberg + ${project.version} + + + org.openjdk.jmh + jmh-core + ${jmh.version} + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + provided + + + org.openjdk.jol + jol-core + ${jol.version} + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 17 + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + + + + + + + diff --git a/fe/fe-benchmark/run-metacache-estimator-benchmark.sh b/fe/fe-benchmark/run-metacache-estimator-benchmark.sh new file mode 100755 index 00000000000000..4936885e30b592 --- /dev/null +++ b/fe/fe-benchmark/run-metacache-estimator-benchmark.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +BENCHMARK_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +FE_DIR=$(cd -- "${BENCHMARK_DIR}/.." && pwd) +BENCHMARK_PATTERN=${1:-'(HiveFileListingSizeBenchmark|IcebergPartitionSizeBenchmark|IcebergManifestSizeBenchmark)'} +if [[ $# -gt 0 ]]; then + shift +fi + +CLASSPATH_FILE=$(mktemp) +trap 'rm -f "${CLASSPATH_FILE}"' EXIT + +( + cd "${FE_DIR}" + mvn -Pbenchmark -pl fe-benchmark -am test-compile -DskipTests + mvn -Pbenchmark -pl fe-benchmark -am dependency:build-classpath \ + -DincludeScope=test \ + -Dmdep.outputFile="${CLASSPATH_FILE}" +) + +REACTOR_CLASSES=$(find "${FE_DIR}" -type d -path '*/target/classes' -printf '%p:') +DEPENDENCY_CLASSES=$(tr -d '\n' < "${CLASSPATH_FILE}") + +java \ + -Djol.magicFieldOffset=true \ + --add-opens=java.base/java.lang=ALL-UNNAMED \ + --add-opens=java.base/java.util=ALL-UNNAMED \ + --add-opens=java.base/java.util.concurrent=ALL-UNNAMED \ + -classpath "${REACTOR_CLASSES}${DEPENDENCY_CLASSES}" \ + org.openjdk.jmh.Main "${BENCHMARK_PATTERN}" "$@" diff --git a/fe/fe-benchmark/src/main/java/org/apache/doris/connector/hive/HiveFileListingSizeBenchmark.java b/fe/fe-benchmark/src/main/java/org/apache/doris/connector/hive/HiveFileListingSizeBenchmark.java new file mode 100644 index 00000000000000..ac89947272eb3e --- /dev/null +++ b/fe/fe-benchmark/src/main/java/org/apache/doris/connector/hive/HiveFileListingSizeBenchmark.java @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.hive; + +import org.apache.doris.connector.hive.HiveFileListingCache.FileListingKey; +import org.apache.doris.connector.hive.HiveFileListingCache.FileListingValue; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jol.info.GraphLayout; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** Compares the O(1) production file-listing weight with construction-time accounting and JOL traversal. */ +@BenchmarkMode(Mode.AverageTime) +@Warmup(iterations = 1, time = 500, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 3, time = 500, timeUnit = TimeUnit.MILLISECONDS) +@Fork(value = 1, jvmArgsAppend = {"-Xms1g", "-Xmx4g", "-Djol.magicFieldOffset=true"}) +public class HiveFileListingSizeBenchmark { + + @Benchmark + @OutputTimeUnit(TimeUnit.NANOSECONDS) + public long cachedWeight(BenchmarkState state) { + return HiveFileListingSizeEstimator.estimateEntry(state.key, state.value); + } + + @Benchmark + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public long constructAndEstimate(BenchmarkState state) { + return new FileListingValue(state.files).estimatedBytes; + } + + @Benchmark + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public long jolRetainedGraph(BenchmarkState state) { + return GraphLayout.parseInstance(state.key, state.value).totalSize(); + } + + @State(Scope.Thread) + public static class BenchmarkState { + @Param({"10000", "100000"}) + public int size; + + private FileListingKey key; + private FileListingValue value; + private List files; + + @Setup(Level.Trial) + public void setup() { + files = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + files.add(new HiveFileStatus( + "s3://warehouse/db/table/dt=2026-08-07/part-" + i + ".parquet", + 128L * 1024L * 1024L + i, + 1_786_048_000_000L + i)); + } + key = new FileListingKey("db", "table", "s3://warehouse/db/table/dt=2026-08-07", + Collections.singletonList("2026-08-07")); + value = new FileListingValue(files); + } + } +} diff --git a/fe/fe-benchmark/src/main/java/org/apache/doris/connector/iceberg/IcebergManifestSizeBenchmark.java b/fe/fe-benchmark/src/main/java/org/apache/doris/connector/iceberg/IcebergManifestSizeBenchmark.java new file mode 100644 index 00000000000000..70782a04211879 --- /dev/null +++ b/fe/fe-benchmark/src/main/java/org/apache/doris/connector/iceberg/IcebergManifestSizeBenchmark.java @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.iceberg; + +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DataFiles; +import org.apache.iceberg.ManifestContent; +import org.apache.iceberg.PartitionSpec; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jol.info.GraphLayout; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** Compares the O(1) production manifest weight with construction-time accounting and JOL traversal. */ +@BenchmarkMode(Mode.AverageTime) +@Warmup(iterations = 1, time = 500, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 3, time = 500, timeUnit = TimeUnit.MILLISECONDS) +@Fork(value = 1, jvmArgsAppend = {"-Xms1g", "-Xmx4g", "-Djol.magicFieldOffset=true"}) +public class IcebergManifestSizeBenchmark { + + @Benchmark + @OutputTimeUnit(TimeUnit.NANOSECONDS) + public long cachedWeight(BenchmarkState state) { + return IcebergCacheSizeEstimator.estimateManifestEntry(state.key, state.value); + } + + @Benchmark + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public long constructAndEstimate(BenchmarkState state) { + return ManifestCacheValue.forDataFiles(state.files).getEstimatedBytes(); + } + + @Benchmark + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public long jolRetainedGraph(BenchmarkState state) { + return GraphLayout.parseInstance(state.key, state.value).totalSize(); + } + + @State(Scope.Thread) + public static class BenchmarkState { + @Param({"10000", "100000"}) + public int size; + + private IcebergManifestEntryKey key; + private ManifestCacheValue value; + private List files; + + @Setup(Level.Trial) + public void setup() { + PartitionSpec spec = PartitionSpec.unpartitioned(); + files = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + files.add(DataFiles.builder(spec) + .withPath("s3://warehouse/db/table/data/part-" + i + ".parquet") + .withFileSizeInBytes(128L * 1024L * 1024L + i) + .withRecordCount(1_000_000L + i) + .build()); + } + key = new IcebergManifestEntryKey( + "s3://warehouse/db/table/metadata/manifest.avro", ManifestContent.DATA); + value = ManifestCacheValue.forDataFiles(files); + } + } +} diff --git a/fe/fe-benchmark/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionSizeBenchmark.java b/fe/fe-benchmark/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionSizeBenchmark.java new file mode 100644 index 00000000000000..e3d6ad39f4faf6 --- /dev/null +++ b/fe/fe-benchmark/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionSizeBenchmark.java @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.iceberg; + +import org.apache.doris.connector.iceberg.IcebergPartitionCache.CachedPartitions; +import org.apache.doris.connector.iceberg.IcebergPartitionCache.Key; +import org.apache.doris.connector.iceberg.IcebergPartitionUtils.IcebergRawPartition; + +import org.apache.iceberg.catalog.TableIdentifier; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jol.info.GraphLayout; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** Compares the O(1) production partition weight with construction-time accounting and JOL traversal. */ +@BenchmarkMode(Mode.AverageTime) +@Warmup(iterations = 1, time = 500, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 3, time = 500, timeUnit = TimeUnit.MILLISECONDS) +@Fork(value = 1, jvmArgsAppend = {"-Xms1g", "-Xmx4g", "-Djol.magicFieldOffset=true"}) +public class IcebergPartitionSizeBenchmark { + + @Benchmark + @OutputTimeUnit(TimeUnit.NANOSECONDS) + public long cachedWeight(BenchmarkState state) { + return IcebergCacheSizeEstimator.estimatePartitionEntry(state.key, state.value); + } + + @Benchmark + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public long constructAndEstimate(BenchmarkState state) { + return new CachedPartitions(state.partitions).estimatedBytes; + } + + @Benchmark + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public long jolRetainedGraph(BenchmarkState state) { + return GraphLayout.parseInstance(state.key, state.value).totalSize(); + } + + @State(Scope.Thread) + public static class BenchmarkState { + @Param({"10000", "100000"}) + public int size; + + private Key key; + private CachedPartitions value; + private List partitions; + + @Setup(Level.Trial) + public void setup() { + partitions = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + List columns = new ArrayList<>(1); + columns.add("dt"); + List values = new ArrayList<>(1); + values.add("2026-08-" + (i % 28 + 1)); + List transforms = new ArrayList<>(1); + transforms.add("identity"); + partitions.add(new IcebergRawPartition( + "dt=" + values.get(0) + "/bucket=" + i, + columns, + values, + transforms, + 1_786_048_000_000L + i, + 10_000L + i)); + } + key = new Key(TableIdentifier.of("db", "table"), 10_000L); + value = new CachedPartitions(partitions); + } + } +} diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveFileListingCache.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveFileListingCache.java index 951fed609ce810..f7814e35e964a0 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveFileListingCache.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveFileListingCache.java @@ -390,7 +390,7 @@ static final class FileListingValue { final List files; final long estimatedBytes; - private FileListingValue(List files) { + FileListingValue(List files) { this.files = Collections.unmodifiableList(new ArrayList<>(files)); this.estimatedBytes = HiveFileListingSizeEstimator.estimateValue(this); } diff --git a/fe/pom.xml b/fe/pom.xml index 1d124c69c26640..a13a8bd7534865 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -508,6 +508,12 @@ under the License. + + benchmark + + fe-benchmark + + From d4eb16b3a5dfd3a1e2d560d4f217ceeea9e4c267 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Fri, 7 Aug 2026 17:15:43 +0800 Subject: [PATCH 4/5] [feature](fe) Add weighted Paimon partition cache ### What problem does this PR solve? Issue Number: None Related PR: #66533 Problem Summary: Paimon's derived partition-view cache was limited only by entry count, so a catalog containing large partition views could retain substantially more FE heap than its configured entry capacity implied. Extend the generic connector metadata cache to accept a type-specific estimator, add a Paimon partition-view estimator that computes the complete immutable entry weight once when max-weight is enabled, and use the stored value for O(1) Caffeine weighing. Keep the legacy count-bounded path unchanged. Add JMH/JOL benchmarks for 10,000 and 100,000 Paimon partitions. ### Release note Paimon partition-view caches support the catalog property `meta.cache.paimon.partition_view.max-weight`. ### Check List (For Author) - Test: Unit Test - `./run-fe-ut.sh --run org.apache.doris.connector.cache.ConnectorMetadataCacheTest,org.apache.doris.connector.paimon.PaimonPartitionViewSizeEstimatorTest,org.apache.doris.connector.paimon.PaimonConnectorValidatePropertiesTest` - `mvn -Pbenchmark -pl fe-benchmark -am test-compile -DskipTests` - Behavior changed: Yes. Paimon partition-view caches can opt into byte-weighted eviction; existing capacity behavior remains the default. - Does this need documentation: No --- fe/fe-benchmark/pom.xml | 5 + .../run-metacache-estimator-benchmark.sh | 2 +- .../iceberg/IcebergManifestSizeBenchmark.java | 2 +- .../PaimonPartitionViewSizeBenchmark.java | 101 ++++++++++++++++ .../cache/ConnectorMetadataCache.java | 34 ++++-- .../cache/ConnectorMetadataCacheTest.java | 27 +++++ .../paimon/PaimonCatalogProperties.java | 2 + .../connector/paimon/PaimonConnector.java | 10 +- .../paimon/PaimonConnectorMetadata.java | 7 +- .../connector/paimon/PaimonPartitionView.java | 52 ++++++++ .../PaimonPartitionViewSizeEstimator.java | 113 ++++++++++++++++++ ...PaimonConnectorValidatePropertiesTest.java | 10 ++ .../PaimonPartitionViewSizeEstimatorTest.java | 65 ++++++++++ 13 files changed, 417 insertions(+), 13 deletions(-) create mode 100644 fe/fe-benchmark/src/main/java/org/apache/doris/connector/paimon/PaimonPartitionViewSizeBenchmark.java create mode 100644 fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonPartitionView.java create mode 100644 fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonPartitionViewSizeEstimator.java create mode 100644 fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonPartitionViewSizeEstimatorTest.java diff --git a/fe/fe-benchmark/pom.xml b/fe/fe-benchmark/pom.xml index 5a4412a9081afd..8034a2d4fb44bf 100644 --- a/fe/fe-benchmark/pom.xml +++ b/fe/fe-benchmark/pom.xml @@ -48,6 +48,11 @@ under the License. fe-connector-iceberg ${project.version} + + ${project.groupId} + fe-connector-paimon + ${project.version} + org.openjdk.jmh jmh-core diff --git a/fe/fe-benchmark/run-metacache-estimator-benchmark.sh b/fe/fe-benchmark/run-metacache-estimator-benchmark.sh index 4936885e30b592..4a1854b8a87c80 100755 --- a/fe/fe-benchmark/run-metacache-estimator-benchmark.sh +++ b/fe/fe-benchmark/run-metacache-estimator-benchmark.sh @@ -21,7 +21,7 @@ set -euo pipefail BENCHMARK_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) FE_DIR=$(cd -- "${BENCHMARK_DIR}/.." && pwd) -BENCHMARK_PATTERN=${1:-'(HiveFileListingSizeBenchmark|IcebergPartitionSizeBenchmark|IcebergManifestSizeBenchmark)'} +BENCHMARK_PATTERN=${1:-'(HiveFileListingSizeBenchmark|IcebergPartitionSizeBenchmark|IcebergManifestSizeBenchmark|PaimonPartitionViewSizeBenchmark)'} if [[ $# -gt 0 ]]; then shift fi diff --git a/fe/fe-benchmark/src/main/java/org/apache/doris/connector/iceberg/IcebergManifestSizeBenchmark.java b/fe/fe-benchmark/src/main/java/org/apache/doris/connector/iceberg/IcebergManifestSizeBenchmark.java index 70782a04211879..8b62801ac53c47 100644 --- a/fe/fe-benchmark/src/main/java/org/apache/doris/connector/iceberg/IcebergManifestSizeBenchmark.java +++ b/fe/fe-benchmark/src/main/java/org/apache/doris/connector/iceberg/IcebergManifestSizeBenchmark.java @@ -85,7 +85,7 @@ public void setup() { .build()); } key = new IcebergManifestEntryKey( - "s3://warehouse/db/table/metadata/manifest.avro", ManifestContent.DATA); + "s3://benchmark-bucket/warehouse/db/table/metadata/manifest.avro", ManifestContent.DATA); value = ManifestCacheValue.forDataFiles(files); } } diff --git a/fe/fe-benchmark/src/main/java/org/apache/doris/connector/paimon/PaimonPartitionViewSizeBenchmark.java b/fe/fe-benchmark/src/main/java/org/apache/doris/connector/paimon/PaimonPartitionViewSizeBenchmark.java new file mode 100644 index 00000000000000..b5635b5abec2bb --- /dev/null +++ b/fe/fe-benchmark/src/main/java/org/apache/doris/connector/paimon/PaimonPartitionViewSizeBenchmark.java @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.paimon; + +import org.apache.doris.connector.cache.ConnectorTableKey; +import org.apache.doris.connector.spi.ConnectorPartitionInfo; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jol.info.GraphLayout; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** Compares Paimon's O(1) cached partition-view weight with construction-time accounting and JOL traversal. */ +@BenchmarkMode(Mode.AverageTime) +@Warmup(iterations = 1, time = 500, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 3, time = 500, timeUnit = TimeUnit.MILLISECONDS) +@Fork(value = 1, jvmArgsAppend = {"-Xms1g", "-Xmx4g", "-Djol.magicFieldOffset=true"}) +public class PaimonPartitionViewSizeBenchmark { + + @Benchmark + @OutputTimeUnit(TimeUnit.NANOSECONDS) + public long cachedWeight(BenchmarkState state) { + return PaimonPartitionViewSizeEstimator.estimateEntry(state.key, state.value); + } + + @Benchmark + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public long constructAndEstimate(BenchmarkState state) { + return new PaimonPartitionView(state.key, state.partitions).getEstimatedBytes(); + } + + @Benchmark + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public long jolRetainedGraph(BenchmarkState state) { + return GraphLayout.parseInstance(state.key, state.value).totalSize(); + } + + @State(Scope.Thread) + public static class BenchmarkState { + @Param({"10000", "100000"}) + public int size; + + private ConnectorTableKey key; + private PaimonPartitionView value; + private List partitions; + + @Setup(Level.Trial) + public void setup() { + partitions = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + Map values = new LinkedHashMap<>(); + values.put("dt", "2026-08-" + (i % 28 + 1)); + values.put("bucket", Integer.toString(i)); + partitions.add(new ConnectorPartitionInfo( + "dt=" + values.get("dt") + "/bucket=" + i, + values, + Collections.emptyMap(), + 10_000L + i, + 128L * 1024L * 1024L + i, + 1_786_048_000_000L + i, + 4L, + new ArrayList<>(values.values()), + Arrays.asList(false, false))); + } + key = new ConnectorTableKey("db", "table", 10_000L, -1L); + value = new PaimonPartitionView(key, partitions); + } + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ConnectorMetadataCache.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ConnectorMetadataCache.java index fde68573e73c50..1bbbb6be93d38f 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ConnectorMetadataCache.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ConnectorMetadataCache.java @@ -33,10 +33,12 @@ * the hive/iceberg/paimon derived partition-view caches (entry {@code "partition_view"}); a connector may hold * several instances under distinct entry names. * - *

Config: {@code meta.cache...(enable|ttl-second|capacity)}, default ON / 86400s / 1000 - * entries (matching {@code IcebergPartitionCache}'s {@code DEFAULT_TABLE_CACHE_CAPACITY}). {@code enable=false} / - * {@code ttl-second=0} / {@code capacity=0} each disable the cache (see {@link CacheSpec#isCacheEnabled}): {@link - * #get} then calls the loader on every call, matching {@code IcebergPartitionCache}'s disabled-cache bypass. + *

Config: {@code meta.cache...(enable|ttl-second|capacity|max-weight)}, default ON / + * 86400s / 1000 entries (matching {@code IcebergPartitionCache}'s {@code DEFAULT_TABLE_CACHE_CAPACITY}). + * {@code enable=false} / {@code ttl-second=0} / an effective bound of zero each disable the cache (see + * {@link CacheSpec#isCacheEnabled}): {@link #get} then calls the loader on every call, matching + * {@code IcebergPartitionCache}'s disabled-cache bypass. A caller that supports {@code max-weight} must use the + * estimator constructor; count-bounded callers can keep using the original constructor. * *

Concurrency: mirrors {@code IcebergPartitionCache} / {@code MaxComputePartitionCache} exactly — the * entry is contextual-only (no built-in loader; the caller supplies one per {@link #get} call) with manual miss @@ -51,25 +53,38 @@ public final class ConnectorMetadataCache { static final long DEFAULT_CAPACITY = 1000L; private final MetaCacheEntry entry; + private final boolean weightBounded; /** * @param engine engine token for the {@code meta.cache...*} property namespace, e.g. * {@code "iceberg"}/{@code "paimon"}/{@code "hive"}/{@code "max_compute"}. * @param entryName the entry name within that namespace (e.g. {@code "partition_view"}); a connector may hold * several {@code ConnectorMetadataCache}s under distinct entry names. - * @param props the catalog properties; drives the {@link CacheSpec} (enable/ttl-second/capacity). May be - * {@code null}, treated as empty (defaults apply). + * @param props the catalog properties; drives the {@link CacheSpec} + * (enable/ttl-second/capacity/max-weight). May be {@code null}, treated as empty. */ public ConnectorMetadataCache(String engine, String entryName, Map props) { + this(engine, entryName, props, null); + } + + /** + * Builds a generic connector metadata cache with an optional type-specific byte estimator. + * + * @param sizeEstimator required when {@code meta.cache...max-weight} is configured; ignored by + * the count-bounded branch + */ + public ConnectorMetadataCache(String engine, String entryName, Map props, + MetaCacheSizeEstimator sizeEstimator) { Objects.requireNonNull(engine, "engine can not be null"); Objects.requireNonNull(entryName, "entryName can not be null"); Map properties = props == null ? Collections.emptyMap() : props; CacheSpec spec = CacheSpec.fromProperties(properties, engine, entryName, CacheSpec.of(true, DEFAULT_TTL_SECOND, DEFAULT_CAPACITY)); + this.weightBounded = spec.isWeightBounded(); // contextual-only (loader == null, supplied per-call by get()) + manual-miss-load, no auto-refresh -- // identical shape to IcebergPartitionCache / MaxComputePartitionCache's entry construction. this.entry = new MetaCacheEntry<>(engine + "." + entryName, null, spec, - ForkJoinPool.commonPool(), false, true, 0L, true); + ForkJoinPool.commonPool(), false, true, 0L, true, sizeEstimator); } /** Caching is on only when the resolved {@link CacheSpec} is effectively enabled (see {@link #entry}'s spec). */ @@ -77,6 +92,11 @@ public boolean isEnabled() { return entry.stats().isEffectiveEnabled(); } + /** Whether this cache is configured with {@code max-weight} instead of the legacy entry-count capacity. */ + public boolean isWeightBounded() { + return weightBounded; + } + /** * Returns the cached value for {@code key} if present, else runs {@code loader}, caches and returns it. * Disabled cache -> {@code loader} runs on every call. The loader runs OUTSIDE Caffeine's compute lock diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ConnectorMetadataCacheTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ConnectorMetadataCacheTest.java index 8a6c1596fccad4..224b29e8da8c95 100644 --- a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ConnectorMetadataCacheTest.java +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ConnectorMetadataCacheTest.java @@ -263,6 +263,33 @@ public void defaultsAreEnabledWithNoProperties() { Assertions.assertTrue(cache.isEnabled(), "the cache must be ON by default with no override properties"); } + @Test + public void maxWeightRequiresEstimator() { + Map props = new HashMap<>(); + props.put("meta.cache." + ENGINE + ".partition_view.max-weight", "1KB"); + + Assertions.assertThrows(NullPointerException.class, + () -> new ConnectorMetadataCache<>(ENGINE, "partition_view", props)); + } + + @Test + public void maxWeightUsesProvidedEstimator() { + Map props = new HashMap<>(); + props.put("meta.cache." + ENGINE + ".partition_view.max-weight", "1B"); + AtomicInteger estimates = new AtomicInteger(); + ConnectorMetadataCache cache = new ConnectorMetadataCache<>( + ENGINE, "partition_view", props, (key, value) -> { + estimates.incrementAndGet(); + return value.length(); + }); + + String value = cache.get(key("db", "t", 5L, 1L), () -> "weighted"); + + Assertions.assertEquals("weighted", value); + Assertions.assertEquals(1, estimates.get(), "the max-weight branch must call the provided estimator"); + Assertions.assertTrue(cache.isWeightBounded()); + } + @Test public void loaderExceptionPropagatesUnwrappedAndIsNotCached() { ConnectorMetadataCache cache = newCache(); diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonCatalogProperties.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonCatalogProperties.java index 2a16f1e81f67d0..6909ab7a6a3dca 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonCatalogProperties.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonCatalogProperties.java @@ -165,6 +165,8 @@ private static void checkMetaCacheProperties(Map properties) { -1L, PaimonConnector.TABLE_CACHE_TTL_SECOND); CacheSpec.checkLongProperty(properties.get(PaimonConnector.TABLE_CACHE_CAPACITY), 0L, PaimonConnector.TABLE_CACHE_CAPACITY); + CacheSpec.checkDataSizeProperty(properties.get(PaimonConnector.PARTITION_VIEW_CACHE_MAX_WEIGHT), + PaimonConnector.PARTITION_VIEW_CACHE_MAX_WEIGHT); } // R2: warn (do not reject, do not strip) when a CREATE/ALTER CATALOG carries the now-dead paimon diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java index 5a620f1a222448..05a30ed9baae03 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java @@ -17,6 +17,7 @@ package org.apache.doris.connector.paimon; +import org.apache.doris.connector.cache.CacheSpec; import org.apache.doris.connector.cache.ConnectorMetadataCache; import org.apache.doris.connector.metastore.HmsMetaStoreProperties; import org.apache.doris.connector.metastore.paimon.jdbc.PaimonJdbcMetaStoreProperties; @@ -101,6 +102,8 @@ public class PaimonConnector implements Connector { // still validated at CREATE/ALTER for legacy parity (reject non-boolean / out-of-range garbage). static final String TABLE_CACHE_ENABLE = "meta.cache.paimon.table.enable"; static final String TABLE_CACHE_CAPACITY = "meta.cache.paimon.table.capacity"; + static final String PARTITION_VIEW_CACHE_MAX_WEIGHT = + CacheSpec.metaCacheMaxWeightKey("paimon", "partition_view"); // Legacy default = Config.external_cache_expire_time_seconds_after_access (24h); the connector is isolated // from fe-core Config, so the legacy default is mirrored here (an explicit ttl-second always overrides it). static final long DEFAULT_TABLE_CACHE_TTL_SECOND = 86400L; @@ -165,9 +168,10 @@ public PaimonConnector(Map properties, ConnectorContext context) this::pluginAuthenticator); this.latestSnapshotCache = new PaimonLatestSnapshotCache(resolveTableCacheTtlSecond(properties), DEFAULT_TABLE_CACHE_CAPACITY); - // Reads its own meta.cache.paimon.partition_view.(enable|ttl-second|capacity) from the catalog - // properties via the framework's CacheSpec (default ON / 24h / 1000). - this.partitionViewCache = new ConnectorMetadataCache<>("paimon", "partition_view", properties); + // Reads its own meta.cache.paimon.partition_view.(enable|ttl-second|capacity|max-weight) from the + // catalog properties via the framework's CacheSpec (default ON / 24h / 1000). + this.partitionViewCache = new ConnectorMetadataCache<>("paimon", "partition_view", properties, + PaimonPartitionViewSizeEstimator::estimateEntry); } /** diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java index 7f124db18c55fb..2e712dd61b7db2 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java @@ -1285,7 +1285,12 @@ private List cachedPartitions(PaimonTableHandle paimonHa return collectPartitions(paimonHandle); } ConnectorTableKey key = partitionViewCacheKey(paimonHandle); - return partitionViewCache.get(key, () -> collectPartitions(paimonHandle)); + return partitionViewCache.get(key, () -> { + List partitions = collectPartitions(paimonHandle); + return partitionViewCache.isWeightBounded() + ? new PaimonPartitionView(key, partitions) + : partitions; + }); } /** diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonPartitionView.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonPartitionView.java new file mode 100644 index 00000000000000..3ae03c6b88d77c --- /dev/null +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonPartitionView.java @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.paimon; + +import org.apache.doris.connector.cache.ConnectorTableKey; +import org.apache.doris.connector.spi.ConnectorPartitionInfo; + +import java.util.AbstractList; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.RandomAccess; + +/** Immutable Paimon partition-view cache value with its entry weight computed once at construction. */ +final class PaimonPartitionView extends AbstractList implements RandomAccess { + private final List partitions; + private final long estimatedBytes; + + PaimonPartitionView(ConnectorTableKey key, List partitions) { + this.partitions = Collections.unmodifiableList(new ArrayList<>(partitions)); + this.estimatedBytes = PaimonPartitionViewSizeEstimator.estimateEntryOnConstruction(key, this); + } + + @Override + public ConnectorPartitionInfo get(int index) { + return partitions.get(index); + } + + @Override + public int size() { + return partitions.size(); + } + + long getEstimatedBytes() { + return estimatedBytes; + } +} diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonPartitionViewSizeEstimator.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonPartitionViewSizeEstimator.java new file mode 100644 index 00000000000000..bc83996fd54199 --- /dev/null +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonPartitionViewSizeEstimator.java @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.paimon; + +import org.apache.doris.connector.cache.ConnectorTableKey; +import org.apache.doris.connector.cache.JvmSizeUtils; +import org.apache.doris.connector.spi.ConnectorPartitionInfo; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; + +/** Type-specific retained-heap estimator for the Paimon derived partition-view cache. */ +final class PaimonPartitionViewSizeEstimator { + private static final long KEY_SHALLOW_BYTES = JvmSizeUtils.instanceSize(ConnectorTableKey.class); + private static final long VIEW_SHALLOW_BYTES = JvmSizeUtils.instanceSize(PaimonPartitionView.class); + private static final long PARTITION_SHALLOW_BYTES = JvmSizeUtils.instanceSize(ConnectorPartitionInfo.class); + private static final long UNMODIFIABLE_LIST_SHALLOW_BYTES = JvmSizeUtils.instanceSize( + Collections.unmodifiableList(Collections.emptyList()).getClass()); + private static final long UNMODIFIABLE_MAP_SHALLOW_BYTES = JvmSizeUtils.instanceSize( + Collections.unmodifiableMap(Collections.emptyMap()).getClass()); + private static final long LINKED_HASH_MAP_SHALLOW_BYTES = JvmSizeUtils.instanceSize(LinkedHashMap.class); + private static final long LINKED_HASH_MAP_ENTRY_SHALLOW_BYTES = classSize("java.util.LinkedHashMap$Entry"); + + private PaimonPartitionViewSizeEstimator() { + } + + /** Caffeine callback: the complete key/value weight was computed when the immutable view was built. */ + static long estimateEntry(ConnectorTableKey key, List value) { + return ((PaimonPartitionView) value).getEstimatedBytes(); + } + + static long estimateEntryOnConstruction(ConnectorTableKey key, PaimonPartitionView value) { + long bytes = KEY_SHALLOW_BYTES; + bytes = add(bytes, JvmSizeUtils.stringSize(key.getDb())); + bytes = add(bytes, JvmSizeUtils.stringSize(key.getTable())); + bytes = add(bytes, VIEW_SHALLOW_BYTES); + bytes = add(bytes, UNMODIFIABLE_LIST_SHALLOW_BYTES); + bytes = add(bytes, JvmSizeUtils.arrayListSize(value.size())); + for (ConnectorPartitionInfo partition : value) { + bytes = add(bytes, estimatePartition(partition)); + } + return bytes; + } + + private static long estimatePartition(ConnectorPartitionInfo partition) { + long bytes = PARTITION_SHALLOW_BYTES; + bytes = add(bytes, JvmSizeUtils.stringSize(partition.getPartitionName())); + + int valueCount = partition.getPartitionValues().size(); + if (valueCount > 0) { + bytes = add(bytes, UNMODIFIABLE_MAP_SHALLOW_BYTES); + bytes = add(bytes, LINKED_HASH_MAP_SHALLOW_BYTES); + bytes = add(bytes, JvmSizeUtils.objectArraySize(hashCapacity(valueCount))); + bytes = add(bytes, multiply(valueCount, LINKED_HASH_MAP_ENTRY_SHALLOW_BYTES)); + } + + List orderedValues = partition.getOrderedPartitionValues(); + bytes = add(bytes, estimateCopiedList(orderedValues)); + for (String value : orderedValues) { + // The same rendered String is retained by both partitionValues and orderedPartitionValues. + bytes = add(bytes, JvmSizeUtils.stringSize(value)); + } + return add(bytes, estimateCopiedList(partition.getPartitionValueNullFlags())); + } + + private static long estimateCopiedList(List values) { + if (values.isEmpty()) { + return 0L; + } + return add(JvmSizeUtils.instanceSize(values.getClass()), JvmSizeUtils.arrayListSize(values.size())); + } + + private static int hashCapacity(int size) { + long needed = (size * 4L + 2L) / 3L; + int capacity = 16; + while (capacity < needed && capacity < 1 << 30) { + capacity <<= 1; + } + return capacity; + } + + private static long classSize(String className) { + try { + return JvmSizeUtils.instanceSize(Class.forName(className)); + } catch (ClassNotFoundException e) { + throw new IllegalStateException("Required JVM collection class is missing: " + className, e); + } + } + + private static long multiply(long left, long right) { + return JvmSizeUtils.saturatedMultiply(left, right); + } + + private static long add(long left, long right) { + return JvmSizeUtils.saturatedAdd(left, right); + } +} diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorValidatePropertiesTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorValidatePropertiesTest.java index 516a027870b883..1c167c47b5f4ef 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorValidatePropertiesTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorValidatePropertiesTest.java @@ -114,6 +114,16 @@ public void acceptsValidMetaCacheKnobs() { "meta.cache.paimon.table.ttl-second", "0"))); } + @Test + public void validatesPartitionViewMaxWeight() { + Assertions.assertDoesNotThrow(() -> validate(props( + "paimon.catalog.type", "filesystem", "warehouse", "/wh", + PaimonConnector.PARTITION_VIEW_CACHE_MAX_WEIGHT, "256MB"))); + Assertions.assertThrows(IllegalArgumentException.class, () -> validate(props( + "paimon.catalog.type", "filesystem", "warehouse", "/wh", + PaimonConnector.PARTITION_VIEW_CACHE_MAX_WEIGHT, "not-a-size"))); + } + @Test public void requiresWarehouseForRest() { // Legacy parity: AbstractPaimonProperties requires warehouse and PaimonRestMetaStoreProperties diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonPartitionViewSizeEstimatorTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonPartitionViewSizeEstimatorTest.java new file mode 100644 index 00000000000000..75ddcecc1bd575 --- /dev/null +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonPartitionViewSizeEstimatorTest.java @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.paimon; + +import org.apache.doris.connector.cache.ConnectorTableKey; +import org.apache.doris.connector.spi.ConnectorPartitionInfo; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +public class PaimonPartitionViewSizeEstimatorTest { + + @Test + public void weightIsPrecomputedAndGrowsWithOwnedPartitions() { + ConnectorTableKey key = new ConnectorTableKey("db", "table", 10L, -1L); + PaimonPartitionView one = new PaimonPartitionView(key, + Collections.singletonList(partition(1))); + PaimonPartitionView two = new PaimonPartitionView(key, + Arrays.asList(partition(1), partition(2))); + + Assertions.assertEquals(one.getEstimatedBytes(), + PaimonPartitionViewSizeEstimator.estimateEntry(key, one)); + Assertions.assertTrue(two.getEstimatedBytes() > one.getEstimatedBytes()); + Assertions.assertThrows(UnsupportedOperationException.class, () -> one.add(partition(3))); + } + + private static ConnectorPartitionInfo partition(int bucket) { + Map values = new LinkedHashMap<>(); + values.put("dt", "2026-08-07"); + values.put("bucket", Integer.toString(bucket)); + List orderedValues = new ArrayList<>(values.values()); + return new ConnectorPartitionInfo( + "dt=2026-08-07/bucket=" + bucket, + values, + Collections.emptyMap(), + 10_000L + bucket, + 128L * 1024L * 1024L, + 1_786_048_000_000L + bucket, + 4L, + orderedValues, + Arrays.asList(false, false)); + } +} From 65313edc54fe2032be10e7444944382ac8916703 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Mon, 10 Aug 2026 18:05:59 +0800 Subject: [PATCH 5/5] [improvement](fe) Add reflective cache estimator fallback ### What problem does this PR solve? Issue Number: None Related PR: #66533 Problem Summary: Type-specific Iceberg cache estimators are fast, but dependency upgrades can add owned reference fields that a manual estimator does not yet cover. Cache each runtime class layout and accessible reference-field plan with ClassValue, inspect a bounded sample of each new object graph once during cache-value construction, and retain the larger of the type-specific and reflective estimates for Iceberg table metadata and manifest payloads. The stored estimate keeps the Caffeine callback O(1). ### Release note None ### Check List (For Author) - Test: Unit Test - ./run-fe-ut.sh --run org.apache.doris.connector.cache.ReflectiveObjectSizeEstimatorTest,org.apache.doris.connector.iceberg.IcebergTableCacheTest,org.apache.doris.connector.iceberg.IcebergManifestCacheTest - mvn -Pbenchmark -pl fe-benchmark -am test-compile -DskipTests - Behavior changed: Yes (Iceberg cache weights may conservatively increase when the reflective fallback finds retained fields missed by the type-specific estimator) - Does this need documentation: No --- .../doris/connector/cache/JvmSizeUtils.java | 8 + .../cache/ReflectiveObjectSizeEstimator.java | 359 ++++++++++++++++++ .../ReflectiveObjectSizeEstimatorTest.java | 123 ++++++ .../iceberg/IcebergCacheSizeEstimator.java | 7 +- 4 files changed, 495 insertions(+), 2 deletions(-) create mode 100644 fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ReflectiveObjectSizeEstimator.java create mode 100644 fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ReflectiveObjectSizeEstimatorTest.java diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/JvmSizeUtils.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/JvmSizeUtils.java index 6be78915365b1e..67e5fc7f1e81e3 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/JvmSizeUtils.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/JvmSizeUtils.java @@ -76,6 +76,14 @@ public static long longArraySize(int length) { return arraySize(length, Long.BYTES); } + /** Estimate an array whose component is a primitive type. */ + public static long primitiveArraySize(Class componentType, int length) { + if (!componentType.isPrimitive() || componentType == void.class) { + throw new IllegalArgumentException("Not an array component primitive: " + componentType); + } + return arraySize(length, Math.toIntExact(fieldSize(componentType))); + } + public static long arrayListSize(int backingArrayCapacity) { return saturatedAdd(ARRAY_LIST_SHALLOW_BYTES, objectArraySize(backingArrayCapacity)); } diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ReflectiveObjectSizeEstimator.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ReflectiveObjectSizeEstimator.java new file mode 100644 index 00000000000000..4d55d15e3b32da --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ReflectiveObjectSizeEstimator.java @@ -0,0 +1,359 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import java.lang.reflect.Array; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.RandomAccess; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Bounded reflective object-graph estimator for validating type-specific cache estimators. + * + *

The expensive class discovery is cached in a {@link ClassValue}: each class hierarchy is reflected once to + * retain its shallow size and accessible reference fields. Runtime estimation still reads each new root's actual + * fields because two instances of the same class can retain very different lists, maps, arrays, and strings. + * Collections and object arrays are sampled to keep this fallback bounded on large metadata values. + * + *

This is intentionally a construction-time safety net, not a Caffeine hit-path weigher. Cache values should + * combine it with a type-specific estimate once, store the larger result, and expose that stored number to + * Caffeine in O(1). + */ +public final class ReflectiveObjectSizeEstimator { + private static final int DEFAULT_SAMPLE_SIZE = 5; + private static final int DEFAULT_MAX_DEPTH = 20; + + private static final long HASH_MAP_NODE_BYTES = classSize("java.util.HashMap$Node"); + private static final long LINKED_HASH_MAP_ENTRY_BYTES = classSize("java.util.LinkedHashMap$Entry"); + private static final long TREE_MAP_ENTRY_BYTES = classSize("java.util.TreeMap$Entry"); + private static final long LINKED_LIST_NODE_BYTES = classSize("java.util.LinkedList$Node"); + private static final long CONCURRENT_HASH_MAP_NODE_BYTES = classSize("java.util.concurrent.ConcurrentHashMap$Node"); + + private static final Set> SHALLOW_LEAF_TYPES = Set.of( + Boolean.class, + Byte.class, + Character.class, + Short.class, + Integer.class, + Float.class, + Long.class, + Double.class); + + private static final ClassValue CLASS_PLANS = new ClassValue<>() { + @Override + protected ClassPlan computeValue(Class type) { + List referenceFields = new ArrayList<>(); + for (Class current = type; current != null; current = current.getSuperclass()) { + for (Field field : current.getDeclaredFields()) { + if (Modifier.isStatic(field.getModifiers()) + || field.getType().isPrimitive() + || field.getType().isEnum()) { + continue; + } + try { + if (field.trySetAccessible()) { + referenceFields.add(field); + } + } catch (RuntimeException ignored) { + // Strongly encapsulated JDK fields are expected; container storage is modeled separately. + } + } + } + return new ClassPlan( + JvmSizeUtils.instanceSize(type), + referenceFields.toArray(new Field[0])); + } + }; + + private ReflectiveObjectSizeEstimator() { + } + + public static long estimate(Object root) { + return estimate(root, DEFAULT_SAMPLE_SIZE, DEFAULT_MAX_DEPTH); + } + + public static long estimate(Object root, int sampleSize, int maxDepth) { + if (sampleSize <= 0) { + throw new IllegalArgumentException("sampleSize must be positive: " + sampleSize); + } + if (maxDepth < 0) { + throw new IllegalArgumentException("maxDepth can not be negative: " + maxDepth); + } + return new Walker(sampleSize).estimate(root, maxDepth); + } + + private static final class Walker { + private final int sampleSize; + private final Map visited = new IdentityHashMap<>(); + + private Walker(int sampleSize) { + this.sampleSize = sampleSize; + } + + private long estimate(Object value, int depth) { + if (value == null || visited.put(value, Boolean.TRUE) != null) { + return 0L; + } + + Class type = value.getClass(); + if (type.isEnum() || value instanceof Class) { + return 0L; + } + if (value instanceof String) { + return JvmSizeUtils.stringSize((String) value); + } + if (SHALLOW_LEAF_TYPES.contains(type) || type.isHidden()) { + return JvmSizeUtils.instanceSize(type); + } + if (type.isArray()) { + return estimateArray(value, depth); + } + if (value instanceof ByteBuffer) { + return estimateByteBuffer((ByteBuffer) value, depth); + } + if (value instanceof Optional) { + return estimateOptional((Optional) value, depth); + } + if (value instanceof Map) { + return estimateMap((Map) value, depth); + } + if (value instanceof Collection) { + return estimateCollection((Collection) value, depth); + } + + ClassPlan plan = CLASS_PLANS.get(type); + long bytes = plan.shallowBytes; + if (depth == 0) { + return bytes; + } + for (Field field : plan.referenceFields) { + bytes = add(bytes, estimateField(value, field, depth - 1)); + } + return bytes; + } + + private long estimateArray(Object value, int depth) { + int length = Array.getLength(value); + Class componentType = value.getClass().getComponentType(); + if (componentType.isPrimitive()) { + return JvmSizeUtils.primitiveArraySize(componentType, length); + } + + long bytes = JvmSizeUtils.objectArraySize(length); + if (depth == 0 || length == 0) { + return bytes; + } + int samples = Math.min(length, sampleSize); + long sampledBytes = 0L; + for (int i = 0; i < samples; i++) { + int index = sampleIndex(i, samples, length); + sampledBytes = add(sampledBytes, estimate(Array.get(value, index), depth - 1)); + } + return add(bytes, scale(sampledBytes, samples, length)); + } + + private long estimateByteBuffer(ByteBuffer value, int depth) { + long bytes = JvmSizeUtils.instanceSize(value.getClass()); + if (depth > 0 && value.hasArray()) { + bytes = add(bytes, estimate(value.array(), depth - 1)); + } + return bytes; + } + + private long estimateOptional(Optional value, int depth) { + long bytes = JvmSizeUtils.instanceSize(value.getClass()); + return depth == 0 || value.isEmpty() + ? bytes + : add(bytes, estimate(value.get(), depth - 1)); + } + + private long estimateCollection(Collection values, int depth) { + int size = values.size(); + long bytes = add( + JvmSizeUtils.instanceSize(values.getClass()), + collectionStorageBytes(values, size)); + if (depth == 0 || size == 0) { + return bytes; + } + + int samples = Math.min(size, sampleSize); + long sampledBytes = 0L; + if (values instanceof List && values instanceof RandomAccess) { + List list = (List) values; + for (int i = 0; i < samples; i++) { + sampledBytes = add(sampledBytes, + estimate(list.get(sampleIndex(i, samples, size)), depth - 1)); + } + } else { + sampledBytes = estimateIterableSamples(values, samples, depth - 1); + } + return add(bytes, scale(sampledBytes, samples, size)); + } + + private long estimateMap(Map values, int depth) { + int size = values.size(); + long bytes = add( + JvmSizeUtils.instanceSize(values.getClass()), + mapStorageBytes(values, size)); + if (depth == 0 || size == 0) { + return bytes; + } + + int samples = Math.min(size, sampleSize); + long sampledBytes = 0L; + Iterator> iterator = values.entrySet().iterator(); + for (int i = 0; i < samples; i++) { + Map.Entry entry = iterator.next(); + sampledBytes = add(sampledBytes, estimate(entry.getKey(), depth - 1)); + sampledBytes = add(sampledBytes, estimate(entry.getValue(), depth - 1)); + } + return add(bytes, scale(sampledBytes, samples, size)); + } + + private long estimateIterableSamples(Collection values, int samples, int depth) { + Iterator iterator = values.iterator(); + long sampledBytes = 0L; + for (int i = 0; i < samples; i++) { + sampledBytes = add(sampledBytes, estimate(iterator.next(), depth)); + } + return sampledBytes; + } + + private long estimateField(Object owner, Field field, int depth) { + try { + return estimate(field.get(owner), depth); + } catch (IllegalAccessException e) { + throw new IllegalStateException("Cached reference field is no longer accessible: " + field, e); + } + } + } + + private static long collectionStorageBytes(Collection values, int size) { + if (size == 0) { + return 0L; + } + if (values instanceof LinkedHashSet) { + return hashStorageBytes(size, LINKED_HASH_MAP_ENTRY_BYTES); + } + if (values instanceof HashSet) { + return hashStorageBytes(size, HASH_MAP_NODE_BYTES); + } + if (values instanceof TreeSet) { + return multiply(size, TREE_MAP_ENTRY_BYTES); + } + if (values instanceof LinkedList) { + return multiply(size, LINKED_LIST_NODE_BYTES); + } + return JvmSizeUtils.objectArraySize(size); + } + + private static long mapStorageBytes(Map values, int size) { + if (size == 0) { + return 0L; + } + if (values instanceof LinkedHashMap) { + return hashStorageBytes(size, LINKED_HASH_MAP_ENTRY_BYTES); + } + if (values instanceof HashMap) { + return hashStorageBytes(size, HASH_MAP_NODE_BYTES); + } + if (values instanceof ConcurrentHashMap) { + return hashStorageBytes(size, CONCURRENT_HASH_MAP_NODE_BYTES); + } + if (values instanceof TreeMap) { + return multiply(size, TREE_MAP_ENTRY_BYTES); + } + return JvmSizeUtils.objectArraySize(saturatedDouble(size)); + } + + private static long hashStorageBytes(int size, long nodeBytes) { + return add( + JvmSizeUtils.objectArraySize(hashCapacity(size)), + multiply(size, nodeBytes)); + } + + private static int hashCapacity(int size) { + long needed = (size * 4L + 2L) / 3L; + int capacity = 16; + while (capacity < needed && capacity < 1 << 30) { + capacity <<= 1; + } + return capacity; + } + + private static int sampleIndex(int sample, int sampleCount, int totalCount) { + return sampleCount == 1 + ? 0 + : (int) ((long) sample * (totalCount - 1) / (sampleCount - 1)); + } + + private static long scale(long sampledBytes, int sampleCount, int totalCount) { + if (sampleCount == totalCount) { + return sampledBytes; + } + double scaled = (double) sampledBytes * totalCount / sampleCount; + return scaled >= Long.MAX_VALUE ? Long.MAX_VALUE : (long) scaled; + } + + private static int saturatedDouble(int value) { + return value > Integer.MAX_VALUE / 2 ? Integer.MAX_VALUE : value * 2; + } + + private static long classSize(String className) { + try { + return JvmSizeUtils.instanceSize(Class.forName(className)); + } catch (ClassNotFoundException e) { + throw new IllegalStateException("Required JVM collection class is missing: " + className, e); + } + } + + private static long multiply(long left, long right) { + return JvmSizeUtils.saturatedMultiply(left, right); + } + + private static long add(long left, long right) { + return JvmSizeUtils.saturatedAdd(left, right); + } + + private static final class ClassPlan { + private final long shallowBytes; + private final Field[] referenceFields; + + private ClassPlan(long shallowBytes, Field[] referenceFields) { + this.shallowBytes = shallowBytes; + this.referenceFields = referenceFields; + } + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ReflectiveObjectSizeEstimatorTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ReflectiveObjectSizeEstimatorTest.java new file mode 100644 index 00000000000000..2ec71bf27611cf --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ReflectiveObjectSizeEstimatorTest.java @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class ReflectiveObjectSizeEstimatorTest { + + @Test + public void usesActualRuntimeCollectionSize() { + Holder small = new Holder(strings(1)); + Holder large = new Holder(strings(100)); + + long smallBytes = ReflectiveObjectSizeEstimator.estimate(small); + long largeBytes = ReflectiveObjectSizeEstimator.estimate(large); + + Assertions.assertTrue(largeBytes > smallBytes); + } + + @Test + public void usesActualRuntimeMapSizeWithCachedClassPlan() { + Map values = new HashMap<>(); + values.put("key-0", "value-0"); + MapHolder holder = new MapHolder(values); + long smallBytes = ReflectiveObjectSizeEstimator.estimate(holder); + + for (int i = 1; i < 100; i++) { + values.put("key-" + i, "value-" + i); + } + + Assertions.assertTrue(ReflectiveObjectSizeEstimator.estimate(holder) > smallBytes); + } + + @Test + public void countsSharedReferenceOnceAndStopsCycles() { + String shared = new String("shared-value"); + SharedHolder holder = new SharedHolder(shared, shared); + long expected = JvmSizeUtils.instanceSize(SharedHolder.class) + JvmSizeUtils.stringSize(shared); + + Assertions.assertEquals(expected, ReflectiveObjectSizeEstimator.estimate(holder)); + + Cycle cycle = new Cycle(); + cycle.next = cycle; + Assertions.assertEquals( + JvmSizeUtils.instanceSize(Cycle.class), + ReflectiveObjectSizeEstimator.estimate(cycle)); + } + + @Test + public void usesExactPrimitiveArrayLayout() { + int[] values = new int[37]; + Assertions.assertEquals( + JvmSizeUtils.intArraySize(values.length), + ReflectiveObjectSizeEstimator.estimate(values)); + } + + @Test + public void validatesBounds() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> ReflectiveObjectSizeEstimator.estimate(new Object(), 0, 1)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> ReflectiveObjectSizeEstimator.estimate(new Object(), 1, -1)); + } + + private static List strings(int size) { + List values = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + values.add(new String("value-" + i)); + } + return values; + } + + private static final class Holder { + private final List values; + + private Holder(List values) { + this.values = values; + } + } + + private static final class SharedHolder { + private final String left; + private final String right; + + private SharedHolder(String left, String right) { + this.left = left; + this.right = right; + } + } + + private static final class MapHolder { + private final Map values; + + private MapHolder(Map values) { + this.values = values; + } + } + + private static final class Cycle { + private Cycle next; + } +} diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCacheSizeEstimator.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCacheSizeEstimator.java index 5557abfe8d7091..0ec5079097e6bd 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCacheSizeEstimator.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCacheSizeEstimator.java @@ -18,6 +18,7 @@ package org.apache.doris.connector.iceberg; import org.apache.doris.connector.cache.JvmSizeUtils; +import org.apache.doris.connector.cache.ReflectiveObjectSizeEstimator; import org.apache.doris.connector.iceberg.IcebergPartitionCache.CachedPartitions; import org.apache.doris.connector.iceberg.IcebergPartitionCache.Key; import org.apache.doris.connector.iceberg.IcebergPartitionUtils.IcebergRawPartition; @@ -131,7 +132,8 @@ static long estimateManifestKey(IcebergManifestEntryKey key) { static long estimateManifestValue(ManifestCacheValue value) { long bytes = MANIFEST_VALUE_SHALLOW_BYTES; bytes = add(bytes, estimateContentFileList(value.getDataFiles())); - return add(bytes, estimateContentFileList(value.getDeleteFiles())); + bytes = add(bytes, estimateContentFileList(value.getDeleteFiles())); + return Math.max(bytes, ReflectiveObjectSizeEstimator.estimate(value)); } /** Caffeine callback: key and manifest payload sizes are precomputed during construction. */ @@ -194,7 +196,8 @@ private static long estimateTableMetadata(TableMetadata metadata) { bytes = add(bytes, estimateMetadataUpdates(metadata.changes())); bytes = add(bytes, estimateShallowList(metadata.encryptionKeys())); // TableMetadata retains a serializable snapshot supplier after the immutable snapshot list is loaded. - return add(bytes, JvmSizeUtils.objectArraySize(1)); + bytes = add(bytes, JvmSizeUtils.objectArraySize(1)); + return Math.max(bytes, ReflectiveObjectSizeEstimator.estimate(metadata)); } private static long estimateSchema(Schema schema) {