From bd18dda3c758882397198110db7da3558e1b2eca Mon Sep 17 00:00:00 2001 From: Jay DeLuca Date: Thu, 4 Jun 2026 15:29:55 -0400 Subject: [PATCH 1/2] add exmplar supplier Signed-off-by: Jay DeLuca --- .../prometheus-metrics-core.txt | 19 +- docs/content/otel/tracing.md | 40 ++++ .../exemplars/ExemplarLabelsSupplier.java | 55 +++++ .../core/exemplars/ExemplarSampler.java | 94 +++++++- .../metrics/core/metrics/Counter.java | 6 +- .../metrics/core/metrics/Gauge.java | 6 +- .../metrics/core/metrics/Histogram.java | 5 +- .../metrics/core/metrics/StatefulMetric.java | 21 ++ .../metrics/core/metrics/Summary.java | 5 +- .../metrics/core/metrics/CounterTest.java | 212 ++++++++++++++++++ 10 files changed, 455 insertions(+), 8 deletions(-) create mode 100644 prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/exemplars/ExemplarLabelsSupplier.java diff --git a/docs/apidiffs/current_vs_latest/prometheus-metrics-core.txt b/docs/apidiffs/current_vs_latest/prometheus-metrics-core.txt index 85d68d53a..fa164a17e 100644 --- a/docs/apidiffs/current_vs_latest/prometheus-metrics-core.txt +++ b/docs/apidiffs/current_vs_latest/prometheus-metrics-core.txt @@ -1,2 +1,19 @@ Comparing source compatibility of prometheus-metrics-core-1.7.1-SNAPSHOT.jar against prometheus-metrics-core-1.7.0.jar -No changes. ++++ NEW CLASS: PUBLIC(+) io.prometheus.metrics.core.exemplars.ExemplarLabelsSupplier (not serializable) + +++ CLASS FILE FORMAT VERSION: 52.0 <- n.a. + +++ NEW SUPERCLASS: java.lang.Object + +++ NEW METHOD: PUBLIC(+) STATIC(+) java.util.function.Supplier getExemplarLabelsSupplier() + +++ NEW ANNOTATION: javax.annotation.Nullable + +++ NEW METHOD: PUBLIC(+) STATIC(+) void setExemplarLabelsSupplier(java.util.function.Supplier) +*** MODIFIED CLASS: PUBLIC io.prometheus.metrics.core.exemplars.ExemplarSampler (not serializable) + === CLASS FILE FORMAT VERSION: 52.0 <- 52.0 + +++ NEW CONSTRUCTOR: PUBLIC(+) ExemplarSampler(io.prometheus.metrics.core.exemplars.ExemplarSamplerConfig, io.prometheus.metrics.tracer.common.SpanContext, java.util.function.Supplier) +*** MODIFIED CLASS: PUBLIC io.prometheus.metrics.core.metrics.Counter (not serializable) + === CLASS FILE FORMAT VERSION: 52.0 <- 52.0 +*** MODIFIED CLASS: PUBLIC io.prometheus.metrics.core.metrics.Gauge (not serializable) + === CLASS FILE FORMAT VERSION: 52.0 <- 52.0 +*** MODIFIED CLASS: PUBLIC io.prometheus.metrics.core.metrics.Histogram (not serializable) + === CLASS FILE FORMAT VERSION: 52.0 <- 52.0 +*** MODIFIED CLASS: PUBLIC io.prometheus.metrics.core.metrics.Summary (not serializable) + === CLASS FILE FORMAT VERSION: 52.0 <- 52.0 + diff --git a/docs/content/otel/tracing.md b/docs/content/otel/tracing.md index f00af9afe..9d598c6f4 100644 --- a/docs/content/otel/tracing.md +++ b/docs/content/otel/tracing.md @@ -75,3 +75,43 @@ The [examples/example-exemplar-tail-sampling/](https://github.com/prometheus/cli directory has a complete end-to-end example, with a distributed Java application with two services, an OpenTelemetry collector, Prometheus, Tempo as a trace database, and Grafana dashboards. Use docker-compose as described in the example's readme to run the example and explore the results. + +## Adding custom labels to exemplars + +Automatically-sampled exemplars carry the `trace_id` and `span_id` labels. You can attach +additional, custom labels (for example an internal identifier) to every automatically-sampled +exemplar. There are two options. + +### Global (all metrics) + +Register a global supplier to add custom labels to the exemplars of _all_ metrics, including +metrics registered by third-party libraries that you do not control. This is the right option when +you cannot modify the code that creates the metric: + +```java +ExemplarLabelsSupplier.setExemplarLabelsSupplier( + () -> Labels.of("management_id", currentManagementId())); +``` + +### Per metric + +If you only want the extra labels on a specific metric you define yourself, use the builder: + +```java +Counter counter = + Counter.builder() + .name("requests_total") + .exemplarLabelsSupplier(() -> Labels.of("management_id", currentManagementId())) + .build(); +``` + +### Notes + +- The supplier is invoked on the (rate-limited) hot path each time an exemplar is sampled, so it + should be cheap. It may return dynamic, request-scoped values (e.g. read from a thread-local). +- Custom labels are only added when a valid, sampled span context is present; the supplier never + causes an exemplar to be created on its own. +- Precedence on a label-name collision: the reserved `trace_id`/`span_id` labels always win, then + the per-metric supplier, then the global supplier. Colliding labels are silently dropped. +- If the supplier throws, the exception is swallowed and the exemplar is created without the + additional labels, so a misbehaving supplier never breaks metric collection. diff --git a/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/exemplars/ExemplarLabelsSupplier.java b/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/exemplars/ExemplarLabelsSupplier.java new file mode 100644 index 000000000..672df100c --- /dev/null +++ b/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/exemplars/ExemplarLabelsSupplier.java @@ -0,0 +1,55 @@ +package io.prometheus.metrics.core.exemplars; + +import io.prometheus.metrics.annotations.StableApi; +import io.prometheus.metrics.model.snapshots.Labels; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; +import javax.annotation.Nullable; + +/** + * Global holder for a {@link Supplier} of additional {@link Labels} that are merged into every + * automatically-sampled Exemplar across the entire application. + * + *

This is the global counterpart to the per-metric {@code exemplarLabelsSupplier(...)} builder + * method. Registering a supplier here affects all metrics, including metrics registered by + * third-party libraries that the application does not control. This makes it the right tool when + * you cannot modify the code that creates the metrics. + * + *

The supplier is invoked on the metric hot path (rate-limited by the exemplar sampler), each + * time an Exemplar is sampled from a valid, sampled span context. It should therefore be cheap and + * non-blocking. It may return dynamic, request-scoped values, for example an identifier read from a + * thread-local: + * + * 

{@code
+ * ExemplarLabelsSupplier.setExemplarLabelsSupplier(
+ *     () -> Labels.of("management_id", currentManagementId()));
+ * }
+ * + *

Labels returned by the supplier that collide with {@code trace_id}/{@code span_id} (or, when a + * per-metric supplier is also configured, with that supplier's labels) are silently dropped rather + * than causing an error: the per-metric supplier takes precedence over the global one, and the + * reserved {@code trace_id}/{@code span_id} labels always win. If the supplier throws, the + * exception is swallowed and the Exemplar is created without the additional labels, so a + * misbehaving supplier never breaks metric collection. + */ +@StableApi +public class ExemplarLabelsSupplier { + + private static final AtomicReference> supplierRef = new AtomicReference<>(); + + private ExemplarLabelsSupplier() {} + + /** + * Register a global supplier of additional exemplar labels. Pass {@code null} to remove a + * previously registered supplier. The most recently registered supplier wins. + */ + public static void setExemplarLabelsSupplier(@Nullable Supplier supplier) { + supplierRef.set(supplier); + } + + /** Returns the registered global supplier, or {@code null} if none has been set. */ + @Nullable + public static Supplier getExemplarLabelsSupplier() { + return supplierRef.get(); + } +} diff --git a/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/exemplars/ExemplarSampler.java b/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/exemplars/ExemplarSampler.java index ad1591c21..9d44e97da 100644 --- a/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/exemplars/ExemplarSampler.java +++ b/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/exemplars/ExemplarSampler.java @@ -13,6 +13,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.LongSupplier; +import java.util.function.Supplier; import javax.annotation.Nullable; /** @@ -47,8 +48,10 @@ public class ExemplarSampler { private final SpanContext spanContext; // may be null, in that case SpanContextSupplier.getSpanContext() is used. + @Nullable private final Supplier additionalLabelsSupplier; + public ExemplarSampler(ExemplarSamplerConfig config) { - this(config, null); + this(config, null, null); } /** @@ -60,10 +63,24 @@ public ExemplarSampler(ExemplarSamplerConfig config) { * SpanContextSupplier.getSpanContext()} is called to find a span context. */ public ExemplarSampler(ExemplarSamplerConfig config, @Nullable SpanContext spanContext) { + this(config, spanContext, null); + } + + /** + * Constructor that additionally accepts a supplier of labels to be merged into every + * automatically-sampled exemplar. The supplier is called each time an exemplar is sampled from a + * span context, so it can return dynamic values (e.g. a request-scoped identifier). The supplier + * is only called when a valid, sampled span context is present. + */ + public ExemplarSampler( + ExemplarSamplerConfig config, + @Nullable SpanContext spanContext, + @Nullable Supplier additionalLabelsSupplier) { this.config = config; this.exemplars = new Exemplar[config.getNumberOfExemplars()]; this.customExemplars = new Exemplar[exemplars.length]; this.spanContext = spanContext; + this.additionalLabelsSupplier = additionalLabelsSupplier; } public Exemplars collect() { @@ -322,7 +339,7 @@ private long durationUntilNextExemplarExpires(long now) { private long updateCustomExemplar(int index, double value, Labels labels, long now) { if (!labels.contains(Exemplar.TRACE_ID) && !labels.contains(Exemplar.SPAN_ID)) { - labels = labels.merge(doSampleExemplar()); + labels = mergeLabels(labels, doSampleExemplar()); } customExemplars[index] = Exemplar.builder().value(value).labels(labels).timestampMillis(now).build(); @@ -357,7 +374,14 @@ private Labels doSampleExemplar() { String traceId = spanContext.getCurrentTraceId(); if (spanId != null && traceId != null) { spanContext.markCurrentSpanAsExemplar(); - return Labels.of(Exemplar.TRACE_ID, traceId, Exemplar.SPAN_ID, spanId); + Labels labels = Labels.of(Exemplar.TRACE_ID, traceId, Exemplar.SPAN_ID, spanId); + // Per-metric supplier first (more specific), then the global supplier. On a name + // collision the earlier (more specific) value is kept; the reserved trace_id/span_id + // labels always win over both. + labels = mergeAdditionalLabels(labels, additionalLabelsSupplier); + labels = + mergeAdditionalLabels(labels, ExemplarLabelsSupplier.getExemplarLabelsSupplier()); + return labels; } } } @@ -366,4 +390,68 @@ private Labels doSampleExemplar() { } return Labels.EMPTY; } + + /** + * Merge labels from {@code supplier} into {@code base}, dropping any label whose name already + * exists in {@code base}. Never throws: a {@code null} supplier, a {@code null}/empty result, a + * colliding label name, or an exception thrown by the supplier all result in {@code base} being + * returned unchanged (minus the offending labels). A misbehaving supplier must never break metric + * collection. + */ + private static Labels mergeAdditionalLabels(Labels base, @Nullable Supplier supplier) { + if (supplier == null) { + return base; + } + Labels extra; + try { + extra = supplier.get(); + } catch (Throwable ignored) { + // A misbehaving supplier (any RuntimeException or Error) must never break metric collection. + return base; + } + if (extra == null || extra.isEmpty()) { + return base; + } + return mergeLabels(base, extra); + } + + /** + * Merge {@code extra} into {@code base}, dropping any label whose name already exists in {@code + * base}. + */ + private static Labels mergeLabels(Labels base, Labels extra) { + if (extra.isEmpty()) { + return base; + } + // Count name collisions with base in a single pass so we can merge exactly once below: base + // (trace_id/span_id and any more-specific supplier) always wins, so colliding labels are + // dropped. extra is itself a valid Labels (no internal duplicates), so the surviving labels + // never collide with each other and merge() cannot throw on a duplicate name. + int size = extra.size(); + int collisions = 0; + for (int i = 0; i < size; i++) { + if (base.contains(extra.getName(i))) { + collisions++; + } + } + if (collisions == 0) { + return base.merge(extra); + } + if (collisions == size) { + return base; + } + int kept = size - collisions; + String[] names = new String[kept]; + String[] values = new String[kept]; + int j = 0; + for (int i = 0; i < size; i++) { + String name = extra.getName(i); + if (!base.contains(name)) { + names[j] = name; + values[j] = extra.getValue(i); + j++; + } + } + return base.merge(names, values); + } } diff --git a/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/metrics/Counter.java b/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/metrics/Counter.java index 742043449..1c614bdf3 100644 --- a/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/metrics/Counter.java +++ b/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/metrics/Counter.java @@ -15,6 +15,7 @@ import java.util.List; import java.util.concurrent.atomic.DoubleAdder; import java.util.concurrent.atomic.LongAdder; +import java.util.function.Supplier; import javax.annotation.Nullable; /** @@ -37,6 +38,7 @@ public class Counter extends StatefulMetric implements CounterDataPoint { @Nullable private final ExemplarSamplerConfig exemplarSamplerConfig; + @Nullable private final Supplier exemplarLabelsSupplier; private Counter(Builder builder, PrometheusProperties prometheusProperties) { super(builder); @@ -49,6 +51,7 @@ private Counter(Builder builder, PrometheusProperties prometheusProperties) { } else { exemplarSamplerConfig = null; } + exemplarLabelsSupplier = builder.exemplarLabelsSupplier; } @Override @@ -108,7 +111,8 @@ public MetricType getMetricType() { @Override protected DataPoint newDataPoint() { if (exemplarSamplerConfig != null) { - return new DataPoint(new ExemplarSampler(exemplarSamplerConfig)); + return new DataPoint( + new ExemplarSampler(exemplarSamplerConfig, null, exemplarLabelsSupplier)); } else { return new DataPoint(null); } diff --git a/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/metrics/Gauge.java b/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/metrics/Gauge.java index 44b61a976..ca743c859 100644 --- a/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/metrics/Gauge.java +++ b/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/metrics/Gauge.java @@ -14,6 +14,7 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; import javax.annotation.Nullable; /** @@ -44,6 +45,7 @@ public class Gauge extends StatefulMetric implements GaugeDataPoint { @Nullable private final ExemplarSamplerConfig exemplarSamplerConfig; + @Nullable private final Supplier exemplarLabelsSupplier; private Gauge(Builder builder, PrometheusProperties prometheusProperties) { super(builder); @@ -56,6 +58,7 @@ private Gauge(Builder builder, PrometheusProperties prometheusProperties) { } else { exemplarSamplerConfig = null; } + exemplarLabelsSupplier = builder.exemplarLabelsSupplier; } @Override @@ -110,7 +113,8 @@ public MetricType getMetricType() { @Override protected DataPoint newDataPoint() { if (exemplarSamplerConfig != null) { - return new DataPoint(new ExemplarSampler(exemplarSamplerConfig)); + return new DataPoint( + new ExemplarSampler(exemplarSamplerConfig, null, exemplarLabelsSupplier)); } else { return new DataPoint(null); } diff --git a/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/metrics/Histogram.java b/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/metrics/Histogram.java index c10d583b1..9a7f9b7c9 100644 --- a/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/metrics/Histogram.java +++ b/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/metrics/Histogram.java @@ -26,6 +26,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.DoubleAdder; import java.util.concurrent.atomic.LongAdder; +import java.util.function.Supplier; import javax.annotation.Nullable; /** @@ -73,6 +74,7 @@ public class Histogram extends StatefulMetric exemplarLabelsSupplier; // Upper bounds for the classic histogram buckets. Contains at least +Inf. // An empty array indicates that this is a native histogram only. @@ -171,6 +173,7 @@ private Histogram(Histogram.Builder builder, PrometheusProperties prometheusProp } else { exemplarSamplerConfig = null; } + exemplarLabelsSupplier = builder.exemplarLabelsSupplier; } @Override @@ -212,7 +215,7 @@ public class DataPoint implements DistributionDataPoint { private DataPoint() { if (exemplarSamplerConfig != null) { - exemplarSampler = new ExemplarSampler(exemplarSamplerConfig); + exemplarSampler = new ExemplarSampler(exemplarSamplerConfig, null, exemplarLabelsSupplier); } else { exemplarSampler = null; } diff --git a/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/metrics/StatefulMetric.java b/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/metrics/StatefulMetric.java index fabad19d0..6ad26fda3 100644 --- a/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/metrics/StatefulMetric.java +++ b/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/metrics/StatefulMetric.java @@ -14,6 +14,7 @@ import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; +import java.util.function.Supplier; import javax.annotation.Nullable; /** @@ -200,11 +201,31 @@ abstract static class Builder, M extends StatefulMetric< extends MetricWithFixedMetadata.Builder { @Nullable protected Boolean exemplarsEnabled; + @Nullable protected Supplier exemplarLabelsSupplier; protected Builder(List illegalLabelNames, PrometheusProperties config) { super(illegalLabelNames, config); } + /** + * Provide additional labels to be merged into every automatically-sampled exemplar of this + * metric. The supplier is called each time an exemplar is sampled, so it can return + * dynamic values (e.g. a request-scoped identifier from a thread-local). The supplier is only + * invoked when a valid, sampled span context is present; it has no effect when tracing is not + * active. + * + *

For a global supplier that applies to all metrics (including metrics registered by + * third-party libraries you do not control), see {@link + * io.prometheus.metrics.core.exemplars.ExemplarLabelsSupplier}. When both are configured, this + * per-metric supplier takes precedence over the global one on a label-name collision, and the + * reserved {@code trace_id}/{@code span_id} labels always win over both. Labels that collide + * are silently dropped. + */ + public B exemplarLabelsSupplier(Supplier supplier) { + this.exemplarLabelsSupplier = supplier; + return self(); + } + /** Allow Exemplars for this metric. */ public B withExemplars() { this.exemplarsEnabled = true; diff --git a/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/metrics/Summary.java b/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/metrics/Summary.java index 1e380cc77..043f31129 100644 --- a/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/metrics/Summary.java +++ b/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/metrics/Summary.java @@ -20,6 +20,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.DoubleAdder; import java.util.concurrent.atomic.LongAdder; +import java.util.function.Supplier; import javax.annotation.Nullable; /** @@ -52,6 +53,7 @@ public class Summary extends StatefulMetric exemplarLabelsSupplier; private Summary(Builder builder, PrometheusProperties prometheusProperties) { super(builder); @@ -67,6 +69,7 @@ private Summary(Builder builder, PrometheusProperties prometheusProperties) { } else { exemplarSamplerConfig = null; } + exemplarLabelsSupplier = builder.exemplarLabelsSupplier; } private List makeQuantiles(MetricsProperties[] properties) { @@ -160,7 +163,7 @@ private DataPoint() { ageBuckets); } if (exemplarSamplerConfig != null) { - exemplarSampler = new ExemplarSampler(exemplarSamplerConfig); + exemplarSampler = new ExemplarSampler(exemplarSamplerConfig, null, exemplarLabelsSupplier); } else { exemplarSampler = null; } diff --git a/prometheus-metrics-core/src/test/java/io/prometheus/metrics/core/metrics/CounterTest.java b/prometheus-metrics-core/src/test/java/io/prometheus/metrics/core/metrics/CounterTest.java index b13e50f8d..69e472eb9 100644 --- a/prometheus-metrics-core/src/test/java/io/prometheus/metrics/core/metrics/CounterTest.java +++ b/prometheus-metrics-core/src/test/java/io/prometheus/metrics/core/metrics/CounterTest.java @@ -8,7 +8,9 @@ import io.prometheus.metrics.config.EscapingScheme; import io.prometheus.metrics.config.MetricsProperties; import io.prometheus.metrics.config.PrometheusProperties; +import io.prometheus.metrics.core.exemplars.ExemplarLabelsSupplier; import io.prometheus.metrics.core.exemplars.ExemplarSamplerConfigTestUtil; +import io.prometheus.metrics.expositionformats.OpenMetricsTextFormatWriter; import io.prometheus.metrics.expositionformats.generated.Metrics; import io.prometheus.metrics.expositionformats.internal.PrometheusProtobufWriterImpl; import io.prometheus.metrics.expositionformats.internal.ProtobufUtil; @@ -17,9 +19,11 @@ import io.prometheus.metrics.model.snapshots.Exemplar; import io.prometheus.metrics.model.snapshots.Label; import io.prometheus.metrics.model.snapshots.Labels; +import io.prometheus.metrics.model.snapshots.MetricSnapshots; import io.prometheus.metrics.model.snapshots.Unit; import io.prometheus.metrics.tracer.common.SpanContext; import io.prometheus.metrics.tracer.initializer.SpanContextSupplier; +import java.io.ByteArrayOutputStream; import java.util.Arrays; import java.util.Iterator; import org.junit.jupiter.api.AfterEach; @@ -51,6 +55,37 @@ void setUp() throws NoSuchFieldException, IllegalAccessException { @AfterEach void tearDown() { SpanContextSupplier.setSpanContext(origSpanContext); + ExemplarLabelsSupplier.setExemplarLabelsSupplier(null); + } + + /** A {@link SpanContext} that always returns a fixed, sampled span. */ + private static SpanContext sampledSpanContext(String traceId, String spanId) { + return new SpanContext() { + @Override + public String getCurrentTraceId() { + return traceId; + } + + @Override + public String getCurrentSpanId() { + return spanId; + } + + @Override + public boolean isCurrentSpanSampled() { + return true; + } + + @Override + public void markCurrentSpanAsExemplar() {} + }; + } + + private static String openMetrics(Counter counter) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new OpenMetricsTextFormatWriter(false, true) + .write(out, MetricSnapshots.of(counter.collect()), EscapingScheme.ALLOW_UTF8); + return out.toString(); } private CounterSnapshot.CounterDataPointSnapshot getData(Counter counter, String... labels) { @@ -321,6 +356,183 @@ void incWithExemplar2() { getData(counter).getExemplar()); } + @Test + void incWithExemplarCustomMetadataInExposition() throws Exception { + Counter counter = Counter.builder().name("requests_total").build(); + counter.incWithExemplar( + Labels.of( + Exemplar.TRACE_ID, "abc123", Exemplar.SPAN_ID, "def456", "management_id", "mgmt-42")); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new OpenMetricsTextFormatWriter(false, true) + .write(out, MetricSnapshots.of(counter.collect()), EscapingScheme.ALLOW_UTF8); + + assertThat(out.toString()) + .contains("management_id=\"mgmt-42\"") + .contains("trace_id=\"abc123\"") + .contains("span_id=\"def456\""); + } + + @Test + void exemplarLabelsSupplierAppearsInAutomaticallySampledExemplar() throws Exception { + SpanContextSupplier.setSpanContext( + new SpanContext() { + @Override + public String getCurrentTraceId() { + return "trace-abc"; + } + + @Override + public String getCurrentSpanId() { + return "span-def"; + } + + @Override + public boolean isCurrentSpanSampled() { + return true; + } + + @Override + public void markCurrentSpanAsExemplar() {} + }); + + Counter counter = + Counter.builder() + .name("requests_total") + .exemplarLabelsSupplier(() -> Labels.of("management_id", "mgmt-42")) + .build(); + counter.inc(); // automatic sampling path + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new OpenMetricsTextFormatWriter(false, true) + .write(out, MetricSnapshots.of(counter.collect()), EscapingScheme.ALLOW_UTF8); + + assertThat(out.toString()) + .contains("management_id=\"mgmt-42\"") + .contains("trace_id=\"trace-abc\"") + .contains("span_id=\"span-def\""); + } + + @Test + void globalExemplarLabelsSupplierAppearsInAutomaticallySampledExemplar() throws Exception { + SpanContextSupplier.setSpanContext(sampledSpanContext("trace-abc", "span-def")); + ExemplarLabelsSupplier.setExemplarLabelsSupplier(() -> Labels.of("management_id", "mgmt-42")); + + Counter counter = Counter.builder().name("requests_total").build(); + counter.inc(); // automatic sampling path + + assertThat(openMetrics(counter)) + .contains("management_id=\"mgmt-42\"") + .contains("trace_id=\"trace-abc\"") + .contains("span_id=\"span-def\""); + } + + @Test + void globalAndPerMetricSuppliersBothApply() throws Exception { + SpanContextSupplier.setSpanContext(sampledSpanContext("trace-abc", "span-def")); + ExemplarLabelsSupplier.setExemplarLabelsSupplier(() -> Labels.of("global_k", "g")); + + Counter counter = + Counter.builder() + .name("requests_total") + .exemplarLabelsSupplier(() -> Labels.of("metric_k", "m")) + .build(); + counter.inc(); + + assertThat(openMetrics(counter)) + .contains("global_k=\"g\"") + .contains("metric_k=\"m\"") + .contains("trace_id=\"trace-abc\"") + .contains("span_id=\"span-def\""); + } + + @Test + void perMetricSupplierWinsOverGlobalOnCollision() throws Exception { + SpanContextSupplier.setSpanContext(sampledSpanContext("trace-abc", "span-def")); + ExemplarLabelsSupplier.setExemplarLabelsSupplier(() -> Labels.of("k", "global")); + + Counter counter = + Counter.builder() + .name("requests_total") + .exemplarLabelsSupplier(() -> Labels.of("k", "metric")) + .build(); + counter.inc(); + + assertThat(openMetrics(counter)).contains("k=\"metric\"").doesNotContain("k=\"global\""); + } + + @Test + void globalSupplierCollidingWithTraceIdIsDropped() throws Exception { + SpanContextSupplier.setSpanContext(sampledSpanContext("trace-abc", "span-def")); + ExemplarLabelsSupplier.setExemplarLabelsSupplier(() -> Labels.of(Exemplar.TRACE_ID, "evil")); + + Counter counter = Counter.builder().name("requests_total").build(); + counter.inc(); + + assertThat(openMetrics(counter)) + .contains("trace_id=\"trace-abc\"") + .doesNotContain("trace_id=\"evil\""); + } + + @Test + void globalSupplierThrowingDoesNotBreakCollection() throws Exception { + SpanContextSupplier.setSpanContext(sampledSpanContext("trace-abc", "span-def")); + ExemplarLabelsSupplier.setExemplarLabelsSupplier( + () -> { + throw new RuntimeException("boom"); + }); + + Counter counter = Counter.builder().name("requests_total").build(); + counter.inc(); + + assertThat(openMetrics(counter)) + .contains("trace_id=\"trace-abc\"") + .contains("span_id=\"span-def\""); + } + + @Test + void globalSupplierWithoutSpanContextProducesNoExemplar() { + SpanContextSupplier.setSpanContext(null); + ExemplarLabelsSupplier.setExemplarLabelsSupplier(() -> Labels.of("management_id", "mgmt-42")); + + Counter counter = Counter.builder().name("requests_total").build(); + counter.inc(); + + assertThat(getData(counter).getExemplar()).isNull(); + } + + @Test + void globalSupplierMergedIntoCustomExemplar() throws Exception { + SpanContextSupplier.setSpanContext(sampledSpanContext("trace-abc", "span-def")); + ExemplarLabelsSupplier.setExemplarLabelsSupplier(() -> Labels.of("management_id", "mgmt-42")); + + Counter counter = Counter.builder().name("requests_total").build(); + counter.incWithExemplar(Labels.of("k", "v")); + + assertThat(openMetrics(counter)) + .contains("management_id=\"mgmt-42\"") + .contains("k=\"v\"") + .contains("trace_id=\"trace-abc\"") + .contains("span_id=\"span-def\""); + } + + @Test + void callerLabelsWinOverGlobalSupplierInCustomExemplar() throws Exception { + SpanContextSupplier.setSpanContext(sampledSpanContext("trace-abc", "span-def")); + ExemplarLabelsSupplier.setExemplarLabelsSupplier( + () -> Labels.of("k", "global", "management_id", "mgmt-42")); + + Counter counter = Counter.builder().name("requests_total").build(); + counter.incWithExemplar(Labels.of("k", "caller")); + + assertThat(openMetrics(counter)) + .contains("k=\"caller\"") + .contains("management_id=\"mgmt-42\"") + .contains("trace_id=\"trace-abc\"") + .contains("span_id=\"span-def\"") + .doesNotContain("k=\"global\""); + } + @Test void testExemplarSamplerDisabled() { Counter counter = Counter.builder().name("count_total").withoutExemplars().build(); From 0e8dfb66a76634a7cf9ebf661e2ee6923705e8bf Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 5 Jun 2026 15:57:29 +0200 Subject: [PATCH 2/2] fix: keep custom exemplar labels caller-controlled (#2194) Signed-off-by: Jay DeLuca --- .../core/exemplars/ExemplarSampler.java | 24 ++++++++++++------- .../metrics/core/metrics/CounterTest.java | 12 +++++----- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/exemplars/ExemplarSampler.java b/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/exemplars/ExemplarSampler.java index 9d44e97da..3a6320955 100644 --- a/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/exemplars/ExemplarSampler.java +++ b/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/exemplars/ExemplarSampler.java @@ -339,7 +339,7 @@ private long durationUntilNextExemplarExpires(long now) { private long updateCustomExemplar(int index, double value, Labels labels, long now) { if (!labels.contains(Exemplar.TRACE_ID) && !labels.contains(Exemplar.SPAN_ID)) { - labels = mergeLabels(labels, doSampleExemplar()); + labels = mergeLabels(labels, sampleTraceContextLabels()); } customExemplars[index] = Exemplar.builder().value(value).labels(labels).timestampMillis(now).build(); @@ -358,6 +358,19 @@ private long updateExemplar(int index, double value, long now) { } private Labels doSampleExemplar() { + Labels labels = sampleTraceContextLabels(); + if (labels.isEmpty()) { + return labels; + } + // Per-metric supplier first (more specific), then the global supplier. On a name + // collision the earlier (more specific) value is kept; the reserved trace_id/span_id + // labels always win over both. + labels = mergeAdditionalLabels(labels, additionalLabelsSupplier); + labels = mergeAdditionalLabels(labels, ExemplarLabelsSupplier.getExemplarLabelsSupplier()); + return labels; + } + + private Labels sampleTraceContextLabels() { // Using the qualified name so that Micrometer can exclude the dependency on // prometheus-metrics-tracer-initializer // as they provide their own implementation of SpanContextSupplier. @@ -374,14 +387,7 @@ private Labels doSampleExemplar() { String traceId = spanContext.getCurrentTraceId(); if (spanId != null && traceId != null) { spanContext.markCurrentSpanAsExemplar(); - Labels labels = Labels.of(Exemplar.TRACE_ID, traceId, Exemplar.SPAN_ID, spanId); - // Per-metric supplier first (more specific), then the global supplier. On a name - // collision the earlier (more specific) value is kept; the reserved trace_id/span_id - // labels always win over both. - labels = mergeAdditionalLabels(labels, additionalLabelsSupplier); - labels = - mergeAdditionalLabels(labels, ExemplarLabelsSupplier.getExemplarLabelsSupplier()); - return labels; + return Labels.of(Exemplar.TRACE_ID, traceId, Exemplar.SPAN_ID, spanId); } } } diff --git a/prometheus-metrics-core/src/test/java/io/prometheus/metrics/core/metrics/CounterTest.java b/prometheus-metrics-core/src/test/java/io/prometheus/metrics/core/metrics/CounterTest.java index 69e472eb9..e86842190 100644 --- a/prometheus-metrics-core/src/test/java/io/prometheus/metrics/core/metrics/CounterTest.java +++ b/prometheus-metrics-core/src/test/java/io/prometheus/metrics/core/metrics/CounterTest.java @@ -502,7 +502,7 @@ void globalSupplierWithoutSpanContextProducesNoExemplar() { } @Test - void globalSupplierMergedIntoCustomExemplar() throws Exception { + void globalSupplierDoesNotApplyToCustomExemplar() throws Exception { SpanContextSupplier.setSpanContext(sampledSpanContext("trace-abc", "span-def")); ExemplarLabelsSupplier.setExemplarLabelsSupplier(() -> Labels.of("management_id", "mgmt-42")); @@ -510,14 +510,14 @@ void globalSupplierMergedIntoCustomExemplar() throws Exception { counter.incWithExemplar(Labels.of("k", "v")); assertThat(openMetrics(counter)) - .contains("management_id=\"mgmt-42\"") .contains("k=\"v\"") .contains("trace_id=\"trace-abc\"") - .contains("span_id=\"span-def\""); + .contains("span_id=\"span-def\"") + .doesNotContain("management_id=\"mgmt-42\""); } @Test - void callerLabelsWinOverGlobalSupplierInCustomExemplar() throws Exception { + void callerControlsCustomExemplarLabels() throws Exception { SpanContextSupplier.setSpanContext(sampledSpanContext("trace-abc", "span-def")); ExemplarLabelsSupplier.setExemplarLabelsSupplier( () -> Labels.of("k", "global", "management_id", "mgmt-42")); @@ -527,10 +527,10 @@ void callerLabelsWinOverGlobalSupplierInCustomExemplar() throws Exception { assertThat(openMetrics(counter)) .contains("k=\"caller\"") - .contains("management_id=\"mgmt-42\"") .contains("trace_id=\"trace-abc\"") .contains("span_id=\"span-def\"") - .doesNotContain("k=\"global\""); + .doesNotContain("k=\"global\"") + .doesNotContain("management_id=\"mgmt-42\""); } @Test