Skip to content

Limit exemplar label characters to conform to Prometheus limits #8362

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
anuq wants to merge 2 commits into
base: main
Choose a base branch
from
+180 −8
Open

Limit exemplar label characters to conform to Prometheus limits #8362

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a0627db
Limit exemplar label characters to conform to Prometheus limits
anuq May 2, 2026
4291d57
Add coverage for invalid span context exemplar branches
anuq May 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 43 additions & 8 deletions ...ometheus/src/main/java/io/opentelemetry/exporter/prometheus/Otel2PrometheusConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,9 @@ final class Otel2PrometheusConverter {

private static final Logger LOGGER = Logger.getLogger(Otel2PrometheusConverter.class.getName());
private static final ThrottlingLogger THROTTLING_LOGGER = new ThrottlingLogger(LOGGER);
// Prometheus limits the total UTF-8 character count across all exemplar label names and values
// to 128. See https://github.com/open-telemetry/opentelemetry-java/issues/6770
static final int EXEMPLAR_MAX_LABEL_SET_LENGTH = 128;
private static final String OTEL_SCOPE_NAME = "otel_scope_name";
private static final String OTEL_SCOPE_VERSION = "otel_scope_version";
private static final String OTEL_SCOPE_SCHEMA_URL = "otel_scope_schema_url";
Expand Down Expand Up @@ -418,26 +421,58 @@ private Exemplars convertDoubleExemplars(List<DoubleExemplarData> exemplars) {
private Exemplar convertExemplar(double value, ExemplarData exemplar) {
SpanContext spanContext = exemplar.getSpanContext();
if (spanContext.isValid()) {
return new Exemplar(
value,
Labels labels =
convertAttributes(
null, // resource attributes are only copied for point's attributes
null, // scope attributes are only needed for point's attributes
exemplar.getFilteredAttributes(),
"trace_id",
spanContext.getTraceId(),
"span_id",
spanContext.getSpanId()),
exemplar.getEpochNanos() / NANOS_PER_MILLISECOND);
spanContext.getSpanId());
if (labelSetLength(labels) > EXEMPLAR_MAX_LABEL_SET_LENGTH) {
// Drop filtered attributes to stay within Prometheus 128-char exemplar label limit,
// keeping trace_id and span_id which are the most valuable for correlation.
THROTTLING_LOGGER.log(
Level.WARNING,
"Exemplar attributes exceeded Prometheus limit of "
+ EXEMPLAR_MAX_LABEL_SET_LENGTH
+ " UTF-8 characters; dropping filtered attributes.");
labels =
convertAttributes(
null, // resource attributes are only copied for point's attributes
null, // scope attributes are only needed for point's attributes
Attributes.empty(),
"trace_id",
spanContext.getTraceId(),
"span_id",
spanContext.getSpanId());
}
return new Exemplar(value, labels, exemplar.getEpochNanos() / NANOS_PER_MILLISECOND);
} else {
return new Exemplar(
value,
Labels labels =
convertAttributes(
null, // resource attributes are only copied for point's attributes
null, // scope attributes are only needed for point's attributes
exemplar.getFilteredAttributes()),
exemplar.getEpochNanos() / NANOS_PER_MILLISECOND);
exemplar.getFilteredAttributes());
if (labelSetLength(labels) > EXEMPLAR_MAX_LABEL_SET_LENGTH) {
THROTTLING_LOGGER.log(
Level.WARNING,
"Exemplar attributes exceeded Prometheus limit of "
+ EXEMPLAR_MAX_LABEL_SET_LENGTH
+ " UTF-8 characters; dropping filtered attributes.");
labels = Labels.EMPTY;
}
return new Exemplar(value, labels, exemplar.getEpochNanos() / NANOS_PER_MILLISECOND);
}
}

private static int labelSetLength(Labels labels) {
int length = 0;
for (int i = 0; i < labels.size(); i++) {
length += labels.getName(i).length() + labels.getValue(i).length();
Copy link
Copy Markdown
Contributor

@psx95 psx95 May 9, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looking at the OpenMetrics Exemplar spec, I think this function should be using codePointCount instead of length. This was also used in a previous PR addressing the same issue.

That PR already had reviews. Not sure if you've looked through that PR (it seems it has been inactive for a while now).

}
return length;
}

private InfoSnapshot makeTargetInfo(Resource resource) {
Expand Down
137 changes: 137 additions & 0 deletions ...heus/src/test/java/io/opentelemetry/exporter/prometheus/Otel2PrometheusConverterTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,14 @@
import io.opentelemetry.api.common.Attributes;
import io.opentelemetry.api.common.KeyValue;
import io.opentelemetry.api.common.Value;
import io.opentelemetry.api.trace.SpanContext;
import io.opentelemetry.api.trace.TraceFlags;
import io.opentelemetry.api.trace.TraceState;
import io.opentelemetry.sdk.common.InstrumentationScopeInfo;
import io.opentelemetry.sdk.metrics.data.AggregationTemporality;
import io.opentelemetry.sdk.metrics.data.MetricData;
import io.opentelemetry.sdk.metrics.data.MetricDataType;
import io.opentelemetry.sdk.metrics.internal.data.ImmutableDoubleExemplarData;
import io.opentelemetry.sdk.metrics.internal.data.ImmutableDoublePointData;
import io.opentelemetry.sdk.metrics.internal.data.ImmutableExponentialHistogramBuckets;
import io.opentelemetry.sdk.metrics.internal.data.ImmutableExponentialHistogramData;
Expand All @@ -39,6 +43,7 @@
import io.opentelemetry.sdk.resources.Resource;
import io.prometheus.metrics.expositionformats.ExpositionFormats;
import io.prometheus.metrics.model.snapshots.CounterSnapshot;
import io.prometheus.metrics.model.snapshots.GaugeSnapshot.GaugeDataPointSnapshot;
import io.prometheus.metrics.model.snapshots.Labels;
import io.prometheus.metrics.model.snapshots.MetricSnapshot;
import io.prometheus.metrics.model.snapshots.MetricSnapshots;
Expand Down Expand Up @@ -555,4 +560,136 @@ void validateCacheIsBounded() {
// it never saw those resources before.
assertThat(predicateCalledCount.get()).isEqualTo(2);
}

@Test
Copy link
Copy Markdown
Contributor

@psx95 psx95 May 9, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Quick Question - would it be possible to convert these added unit tests into parameterized tests?

void exemplarLabelsWithinLimit() {
SpanContext spanContext =
SpanContext.create(
"00000000000000000000000000000001",
"0000000000000001",
TraceFlags.getSampled(),
TraceState.getDefault());
ImmutableDoubleExemplarData exemplar =
(ImmutableDoubleExemplarData)
ImmutableDoubleExemplarData.create(
Attributes.of(stringKey("short"), "val"), 1000L, spanContext, 1.0);

MetricData metricData =
ImmutableMetricData.createDoubleGauge(
Resource.getDefault(),
InstrumentationScopeInfo.create("test"),
"my.gauge",
"desc",
"unit",
ImmutableGaugeData.create(
Collections.singletonList(
ImmutableDoublePointData.create(
0, 1000, Attributes.empty(), 1.0, Collections.singletonList(exemplar)))));

MetricSnapshots snapshots = converter.convert(Collections.singletonList(metricData));
assertThat(snapshots).isNotNull();
// Labels within limit — both trace/span and filtered attribute should be present
GaugeDataPointSnapshot point = (GaugeDataPointSnapshot) snapshots.get(0).getDataPoints().get(0);
Labels exemplarLabels = point.getExemplar().getLabels();
assertThat(exemplarLabels.get("trace_id")).isEqualTo(spanContext.getTraceId());
assertThat(exemplarLabels.get("span_id")).isEqualTo(spanContext.getSpanId());
assertThat(exemplarLabels.get("short")).isEqualTo("val");
}

@Test
void exemplarLabelsExceedingLimitDropsFilteredAttributes() {
SpanContext spanContext =
SpanContext.create(
"00000000000000000000000000000001",
"0000000000000001",
TraceFlags.getSampled(),
TraceState.getDefault());
// Build a filtered attribute whose name+value alone would push total over 128
char[] chars = new char[100];
Arrays.fill(chars, 'x');
String longValue = new String(chars);
ImmutableDoubleExemplarData exemplar =
(ImmutableDoubleExemplarData)
ImmutableDoubleExemplarData.create(
Attributes.of(stringKey("long_attr"), longValue), 1000L, spanContext, 1.0);

MetricData metricData =
ImmutableMetricData.createDoubleGauge(
Resource.getDefault(),
InstrumentationScopeInfo.create("test"),
"my.gauge",
"desc",
"unit",
ImmutableGaugeData.create(
Collections.singletonList(
ImmutableDoublePointData.create(
0, 1000, Attributes.empty(), 1.0, Collections.singletonList(exemplar)))));

MetricSnapshots snapshots = converter.convert(Collections.singletonList(metricData));
assertThat(snapshots).isNotNull();
GaugeDataPointSnapshot point = (GaugeDataPointSnapshot) snapshots.get(0).getDataPoints().get(0);
Labels exemplarLabels = point.getExemplar().getLabels();
// trace_id and span_id are preserved
assertThat(exemplarLabels.get("trace_id")).isEqualTo(spanContext.getTraceId());
assertThat(exemplarLabels.get("span_id")).isEqualTo(spanContext.getSpanId());
// filtered attribute is dropped to stay within limit
assertThat(exemplarLabels.get("long_attr")).isNull();
}

@Test
void exemplarWithoutSpanContextExceedingLimitDropsFilteredAttributes() {
char[] chars = new char[150];
Arrays.fill(chars, 'x');
String longValue = new String(chars);
ImmutableDoubleExemplarData exemplar =
(ImmutableDoubleExemplarData)
ImmutableDoubleExemplarData.create(
Attributes.of(stringKey("long_attr"), longValue),
1000L,
SpanContext.getInvalid(),
1.0);

MetricData metricData =
ImmutableMetricData.createDoubleGauge(
Resource.getDefault(),
InstrumentationScopeInfo.create("test"),
"my.gauge",
"desc",
"unit",
ImmutableGaugeData.create(
Collections.singletonList(
ImmutableDoublePointData.create(
0, 1000, Attributes.empty(), 1.0, Collections.singletonList(exemplar)))));

MetricSnapshots snapshots = converter.convert(Collections.singletonList(metricData));
GaugeDataPointSnapshot point = (GaugeDataPointSnapshot) snapshots.get(0).getDataPoints().get(0);
Labels exemplarLabels = point.getExemplar().getLabels();
// No span context means no trace_id/span_id, and oversized filtered attr is dropped
assertThat(exemplarLabels.size()).isZero();
}

@Test
void exemplarWithoutSpanContextWithinLimit() {
ImmutableDoubleExemplarData exemplar =
(ImmutableDoubleExemplarData)
ImmutableDoubleExemplarData.create(
Attributes.of(stringKey("short"), "val"), 1000L, SpanContext.getInvalid(), 1.0);

MetricData metricData =
ImmutableMetricData.createDoubleGauge(
Resource.getDefault(),
InstrumentationScopeInfo.create("test"),
"my.gauge",
"desc",
"unit",
ImmutableGaugeData.create(
Collections.singletonList(
ImmutableDoublePointData.create(
0, 1000, Attributes.empty(), 1.0, Collections.singletonList(exemplar)))));

MetricSnapshots snapshots = converter.convert(Collections.singletonList(metricData));
GaugeDataPointSnapshot point = (GaugeDataPointSnapshot) snapshots.get(0).getDataPoints().get(0);
Labels exemplarLabels = point.getExemplar().getLabels();
assertThat(exemplarLabels.get("short")).isEqualTo("val");
}
}
Loading