From 307e31a4c30416f5901b094e4a6d1ed1e1e8c18e Mon Sep 17 00:00:00 2001 From: Rahul Yadav Date: Tue, 14 Jul 2026 11:36:27 +0530 Subject: [PATCH] feat(spanner): add optional endpoint attribute to client metrics for location-aware routing Adds the ENDPOINT_KEY metric attribute: KeyAwareChannel labels each attempt with the endpoint the location-aware fastpath routed it to, and the custom-export attempt views allow the label through their attribute filter. Cloud Monitoring views never include it. Stacked follow-up to the client metrics export change (#13679); until that merges this commit also carries the client metrics export work. Merge after #13679 lands. --- java-spanner/.readme-partials.yaml | 89 +++-- java-spanner/README.md | 47 ++- .../cloud/spanner/BuiltInMetricsConstant.java | 120 +++++-- .../cloud/spanner/BuiltInMetricsProvider.java | 27 +- .../cloud/spanner/BuiltInMetricsView.java | 2 +- .../CustomOpenTelemetryMetricsProvider.java | 93 ++++++ .../cloud/spanner/DefaultMetricsProvider.java | 30 ++ .../google/cloud/spanner/MetricsProvider.java | 39 +++ .../cloud/spanner/NoopMetricsProvider.java | 29 ++ .../google/cloud/spanner/SpannerMetrics.java | 48 +++ .../google/cloud/spanner/SpannerOptions.java | 220 +++++++++++-- .../cloud/spanner/spi/v1/GapicSpannerRpc.java | 24 +- .../cloud/spanner/spi/v1/KeyAwareChannel.java | 12 + .../spanner/BuiltInMetricsBothSinksTest.java | 145 +++++++++ ...BuiltInMetricsCustomOpenTelemetryTest.java | 174 ++++++++++ .../BuiltInMetricsCustomViewsTest.java | 255 +++++++++++++++ .../spanner/GrpcMetricsDualWireTest.java | 122 +++++++ .../LocationAwareEndpointMetricsTest.java | 149 +++++++++ .../cloud/spanner/MetricsProviderTest.java | 307 ++++++++++++++++++ .../cloud/spanner/SpannerOptionsTest.java | 34 ++ .../spanner/spi/v1/GapicSpannerRpcTest.java | 52 ++- .../spanner/spi/v1/KeyAwareChannelTest.java | 127 ++++++++ 22 files changed, 2041 insertions(+), 104 deletions(-) create mode 100644 java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/CustomOpenTelemetryMetricsProvider.java create mode 100644 java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/DefaultMetricsProvider.java create mode 100644 java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MetricsProvider.java create mode 100644 java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/NoopMetricsProvider.java create mode 100644 java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerMetrics.java create mode 100644 java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/BuiltInMetricsBothSinksTest.java create mode 100644 java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/BuiltInMetricsCustomOpenTelemetryTest.java create mode 100644 java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/BuiltInMetricsCustomViewsTest.java create mode 100644 java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/GrpcMetricsDualWireTest.java create mode 100644 java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/LocationAwareEndpointMetricsTest.java create mode 100644 java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MetricsProviderTest.java diff --git a/java-spanner/.readme-partials.yaml b/java-spanner/.readme-partials.yaml index 392f727e622a..b48f204eeb78 100644 --- a/java-spanner/.readme-partials.yaml +++ b/java-spanner/.readme-partials.yaml @@ -53,29 +53,70 @@ custom_content: | ## Metrics Cloud Spanner client supports [client-side metrics](https://cloud.google.com/spanner/docs/view-manage-client-side-metrics) that you can use along with server-side metrics to optimize performance and troubleshoot performance issues if they occur. - - Client-side metrics are measured from the time a request leaves your application to the time your application receives the response. + + Client-side metrics are measured from the time a request leaves your application to the time your application receives the response. In contrast, server-side metrics are measured from the time Spanner receives a request until the last byte of data is sent to the client. - - These metrics are enabled by default. You can opt out of using client-side metrics with the following code: - + + The default Cloud Monitoring export for these metrics is enabled by default. You can opt out of the default Cloud Monitoring export with the following code: + ``` SpannerOptions options = SpannerOptions.newBuilder() .setBuiltInMetricsEnabled(false) .build(); ``` - - You can also disable these metrics by setting `SPANNER_DISABLE_BUILTIN_METRICS` to `true`. - - > Note: Client-side metrics needs `monitoring.timeSeries.create` IAM permission to export metrics data. Ask your administrator to grant your service account the [Monitoring Metric Writer](https://cloud.google.com/iam/docs/roles-permissions/monitoring#monitoring.metricWriter) (roles/monitoring.metricWriter) IAM role on the project. + + You can also disable the default Cloud Monitoring export by setting `SPANNER_DISABLE_BUILTIN_METRICS` to `true`. These controls affect only the Cloud Monitoring export. They do not affect a caller-owned client-metrics export configured with `CustomOpenTelemetryMetricsProvider`. + + > Note: Client-side metrics needs `monitoring.timeSeries.create` IAM permission to export metrics data to Cloud Monitoring. Ask your administrator to grant your service account the [Monitoring Metric Writer](https://cloud.google.com/iam/docs/roles-permissions/monitoring#monitoring.metricWriter) (roles/monitoring.metricWriter) IAM role on the project. + + #### Exporting client metrics to OpenTelemetry + + Client metrics export to a caller-owned OpenTelemetry destination is controlled by a `MetricsProvider`, + set with `SpannerOptions.Builder.setClientMetricsProvider(MetricsProvider)`. The available + providers are: + + * `DefaultMetricsProvider` (the default): no caller-owned client-metrics destination is configured. + The built-in Cloud Monitoring export follows `setBuiltInMetricsEnabled` and the + `SPANNER_DISABLE_BUILTIN_METRICS` environment variable. On Spanner Omni, where the Cloud Monitoring + export is not available, the default provider results in no client-metrics export. + * `NoopMetricsProvider`: caller-owned client metrics are explicitly disabled. The Cloud Monitoring + export is controlled separately. + * `CustomOpenTelemetryMetricsProvider`: the same Spanner client instruments are additionally recorded + on an `OpenTelemetry` instance that you provide. You own the metrics pipeline (readers, exporters + and resource). This custom destination is independent of the built-in Cloud Monitoring export on all + instance types, including Spanner Omni (`InstanceType.OMNI`), for which Cloud Monitoring export is + not available. + + Client metrics are not recorded when the client runs against the Spanner emulator, regardless of the + configured `MetricsProvider`. gRPC-layer metrics are recorded on a custom destination only when the + provided `OpenTelemetry` instance is an `OpenTelemetrySdk`. + + When using `CustomOpenTelemetryMetricsProvider`, it is recommended to register the Spanner + client-metrics views on a dedicated `SdkMeterProviderBuilder` with + `SpannerMetrics.configureMeterProviderBuilder(SdkMeterProviderBuilder)` before creating the + `OpenTelemetry` instance. The views rename the raw instruments, apply the Spanner latency + histogram buckets and restrict the recorded attributes to the supported client-metric labels. Attempt + metrics include the routed `endpoint` label when Spanner Omni location-aware routing supplies it. + + ```java + SdkMeterProviderBuilder meterProviderBuilder = + SdkMeterProvider.builder().registerMetricReader(PeriodicMetricReader.create(myExporter)); + SpannerMetrics.configureMeterProviderBuilder(meterProviderBuilder); + OpenTelemetry openTelemetry = + OpenTelemetrySdk.builder().setMeterProvider(meterProviderBuilder.build()).build(); + SpannerOptions options = + SpannerOptions.newBuilder() + .setClientMetricsProvider(CustomOpenTelemetryMetricsProvider.create(openTelemetry)) + .build(); + ``` ## Traces - Cloud Spanner client supports OpenTelemetry Traces, which gives insight into the client internals and aids in debugging/troubleshooting production issues. + Cloud Spanner client supports OpenTelemetry Traces, which gives insight into the client internals and aids in debugging/troubleshooting production issues. By default, the functionality is disabled. You need to add OpenTelemetry dependencies, enable OpenTelemetry traces and must configure the OpenTelemetry with appropriate exporters at the startup of your application. See [Configure client-side tracing](https://cloud.google.com/spanner/docs/set-up-tracing#configure-client-side-tracing) for more details on configuring traces. - + #### OpenTelemetry Dependencies If you are using Maven, add this to your pom.xml file @@ -129,7 +170,7 @@ custom_content: | Spanner spanner = options.getService(); ``` - + #### OpenTelemetry SQL Statement Tracing The OpenTelemetry traces that are generated by the Java client include any request and transaction tags that have been set. The traces can also include the SQL statements that are executed and the @@ -149,42 +190,42 @@ custom_content: | #### OpenTelemetry API Tracing You can enable tracing of each API call that the Spanner client executes with the `enableApiTracing` option. These traces also include any retry attempts for an API call: - + ``` SpannerOptions options = SpannerOptions.newBuilder() .setOpenTelemetry(openTelemetry) .setEnableApiTracing(true) .build(); ``` - + This option can also be enabled by setting the environment variable `SPANNER_ENABLE_API_TRACING=true`. > Note: The attribute keys that are used for additional information about retry attempts and the number of requests might change in a future release. - - #### End-to-end Tracing - + + #### End-to-end Tracing + In addition to client-side tracing, you can opt in for [end-to-end tracing](https://cloud.google.com/spanner/docs/tracing-overview#end-to-end-side-tracing). End-to-end tracing helps you understand and debug latency issues that are specific to Spanner such as the following: * Identify whether the latency is due to network latency between your application and Spanner, or if the latency is occurring within Spanner. * Identify the Google Cloud regions that your application requests are being routed through and if there is a cross-region request. A cross-region request usually means higher latencies between your application and Spanner. - + ``` SpannerOptions options = SpannerOptions.newBuilder() .setOpenTelemetry(openTelemetry) .setEnableEndToEndTracing(true) .build(); ``` - + Refer to [Configure end-to-end tracing](https://cloud.google.com/spanner/docs/set-up-tracing#configure-end-to-end-tracing) to configure end-to-end tracing and to understand its attributes. - + > Note: End-to-end traces can only be exported to [Cloud Trace](https://cloud.google.com/trace/docs). - - + + ## Instrument with OpenCensus > Note: OpenCensus project is deprecated. See [Sunsetting OpenCensus](https://opentelemetry.io/blog/2023/sunsetting-opencensus/). We recommend migrating to OpenTelemetry, the successor project. - + ## Migrate from OpenCensus to OpenTelemetry > Using the [OpenTelemetry OpenCensus Bridge](https://mvnrepository.com/artifact/io.opentelemetry/opentelemetry-opencensus-shim), you can immediately begin exporting your metrics and traces with OpenTelemetry. @@ -214,4 +255,4 @@ custom_content: | Update your dashboards and alerts to reflect below changes * **Metrics name** : `cloud.google.com/java` prefix has been removed from OpenTelemery metrics and instead has been added as Instrumenation Scope. - * **Metrics namespace** : OpenTelmetry exporters uses `workload.googleapis.com` namespace opposed to `custom.googleapis.com` with OpenCensus. \ No newline at end of file + * **Metrics namespace** : OpenTelmetry exporters uses `workload.googleapis.com` namespace opposed to `custom.googleapis.com` with OpenCensus. diff --git a/java-spanner/README.md b/java-spanner/README.md index 8eef414741bb..b4e631c1a3a1 100644 --- a/java-spanner/README.md +++ b/java-spanner/README.md @@ -155,7 +155,7 @@ Cloud Spanner client supports [client-side metrics](https://cloud.google.com/spa Client-side metrics are measured from the time a request leaves your application to the time your application receives the response. In contrast, server-side metrics are measured from the time Spanner receives a request until the last byte of data is sent to the client. -These metrics are enabled by default. You can opt out of using client-side metrics with the following code: +The default Cloud Monitoring export for these metrics is enabled by default. You can opt out of the default Cloud Monitoring export with the following code: ``` SpannerOptions options = SpannerOptions.newBuilder() @@ -163,9 +163,50 @@ SpannerOptions options = SpannerOptions.newBuilder() .build(); ``` -You can also disable these metrics by setting `SPANNER_DISABLE_BUILTIN_METRICS` to `true`. +You can also disable the default Cloud Monitoring export by setting `SPANNER_DISABLE_BUILTIN_METRICS` to `true`. These controls affect only the Cloud Monitoring export. They do not affect a caller-owned client-metrics export configured with `CustomOpenTelemetryMetricsProvider`. -> Note: Client-side metrics needs `monitoring.timeSeries.create` IAM permission to export metrics data. Ask your administrator to grant your service account the [Monitoring Metric Writer](https://cloud.google.com/iam/docs/roles-permissions/monitoring#monitoring.metricWriter) (roles/monitoring.metricWriter) IAM role on the project. +> Note: Client-side metrics needs `monitoring.timeSeries.create` IAM permission to export metrics data to Cloud Monitoring. Ask your administrator to grant your service account the [Monitoring Metric Writer](https://cloud.google.com/iam/docs/roles-permissions/monitoring#monitoring.metricWriter) (roles/monitoring.metricWriter) IAM role on the project. + +#### Exporting client metrics to OpenTelemetry + +Client metrics export to a caller-owned OpenTelemetry destination is controlled by a `MetricsProvider`, +set with `SpannerOptions.Builder.setClientMetricsProvider(MetricsProvider)`. The available +providers are: + +* `DefaultMetricsProvider` (the default): no caller-owned client-metrics destination is configured. + The built-in Cloud Monitoring export follows `setBuiltInMetricsEnabled` and the + `SPANNER_DISABLE_BUILTIN_METRICS` environment variable. On Spanner Omni, where the Cloud Monitoring + export is not available, the default provider results in no client-metrics export. +* `NoopMetricsProvider`: caller-owned client metrics are explicitly disabled. The Cloud Monitoring + export is controlled separately. +* `CustomOpenTelemetryMetricsProvider`: the same Spanner client instruments are additionally recorded + on an `OpenTelemetry` instance that you provide. You own the metrics pipeline (readers, exporters + and resource). This custom destination is independent of the built-in Cloud Monitoring export on all + instance types, including Spanner Omni (`InstanceType.OMNI`), for which Cloud Monitoring export is + not available. + +Client metrics are not recorded when the client runs against the Spanner emulator, regardless of the +configured `MetricsProvider`. gRPC-layer metrics are recorded on a custom destination only when the +provided `OpenTelemetry` instance is an `OpenTelemetrySdk`. + +When using `CustomOpenTelemetryMetricsProvider`, it is recommended to register the Spanner +client-metrics views on a dedicated `SdkMeterProviderBuilder` with +`SpannerMetrics.configureMeterProviderBuilder(SdkMeterProviderBuilder)` before creating the +`OpenTelemetry` instance. The views rename the raw instruments, apply the Spanner latency +histogram buckets and restrict the recorded attributes to the supported client-metric labels. Attempt +metrics include the routed `endpoint` label when Spanner Omni location-aware routing supplies it. + +```java +SdkMeterProviderBuilder meterProviderBuilder = + SdkMeterProvider.builder().registerMetricReader(PeriodicMetricReader.create(myExporter)); +SpannerMetrics.configureMeterProviderBuilder(meterProviderBuilder); +OpenTelemetry openTelemetry = + OpenTelemetrySdk.builder().setMeterProvider(meterProviderBuilder.build()).build(); +SpannerOptions options = + SpannerOptions.newBuilder() + .setClientMetricsProvider(CustomOpenTelemetryMetricsProvider.create(openTelemetry)) + .build(); +``` ## Traces Cloud Spanner client supports OpenTelemetry Traces, which gives insight into the client internals and aids in debugging/troubleshooting production issues. diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java index 76291a53458c..39c5e1e8f798 100644 --- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java @@ -25,6 +25,7 @@ import com.google.common.collect.ImmutableSet; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.sdk.metrics.Aggregation; +import io.opentelemetry.sdk.metrics.ExplicitBucketHistogramOptions; import io.opentelemetry.sdk.metrics.InstrumentSelector; import io.opentelemetry.sdk.metrics.InstrumentType; import io.opentelemetry.sdk.metrics.View; @@ -40,6 +41,16 @@ public class BuiltInMetricsConstant { public static final String METER_NAME = "spanner.googleapis.com/internal/client"; + + /** + * Instrument namespace used when client metrics are recorded on a caller-provided {@link + * io.opentelemetry.api.OpenTelemetry} via {@link CustomOpenTelemetryMetricsProvider}. This is + * deliberately distinct from the reserved {@code spanner.googleapis.com} namespace, so that + * re-exporting a caller-owned pipeline to Cloud Monitoring cannot conflict with the curated + * Spanner metric descriptors. + */ + public static final String CUSTOM_EXPORT_METER_NAME = "spanner/client"; + public static final String GAX_METER_NAME = OpenTelemetryMetricsRecorder.GAX_METER_NAME; public static final String GRPC_GCP_METER_NAME = "grpc-gcp"; static final String SPANNER_METER_NAME = "spanner-java"; @@ -168,6 +179,14 @@ public class BuiltInMetricsConstant { AttributeKey.stringKey("directpath_used"); public static final AttributeKey REQUEST_ID_KEY = AttributeKey.stringKey(REQUEST_ID_HEADER_NAME); + + /** + * The endpoint an attempt was routed to by the location-aware fastpath. Only produced when the + * experimental location API is enabled (Spanner Omni), and only allowed through the attribute + * filter of the custom-export views; the Cloud Monitoring views never include it. + */ + public static final AttributeKey ENDPOINT_KEY = AttributeKey.stringKey("endpoint"); + public static Set ALLOWED_EXEMPLARS_ATTRIBUTES = new HashSet<>(Arrays.asList(REQUEST_ID_HEADER_NAME)); @@ -197,7 +216,8 @@ public class BuiltInMetricsConstant { 200.0, 250.0, 300.0, 400.0, 500.0, 650.0, 800.0, 1000.0, 2000.0, 5000.0, 10000.0, 20000.0, 50000.0, 100000.0, 200000.0, 400000.0, 800000.0, 1600000.0, 3200000.0); static Aggregation AGGREGATION_WITH_MILLIS_HISTOGRAM = - Aggregation.explicitBucketHistogram(BUCKET_BOUNDARIES); + Aggregation.explicitBucketHistogram( + ExplicitBucketHistogramOptions.builder().setBucketBoundaries(BUCKET_BOUNDARIES).build()); static final Collection GRPC_METRICS_ENABLED_BY_DEFAULT = ImmutableList.of( @@ -208,102 +228,138 @@ public class BuiltInMetricsConstant { static Map getAllViews() { ImmutableMap.Builder views = ImmutableMap.builder(); + addViews(views, METER_NAME, false); + defineGrpcGcpView(views); + return views.build(); + } + + /** + * Views for client metrics recorded on a caller-provided OpenTelemetry via {@link + * CustomOpenTelemetryMetricsProvider}. The instruments live in the {@link + * #CUSTOM_EXPORT_METER_NAME} namespace, and the attempt metric attribute filters additionally + * allow the {@link #ENDPOINT_KEY} label. The grpc-gcp DirectPath-fallback views are deliberately + * excluded: those metrics are only recorded on the Cloud Monitoring path, so registering their + * views on a custom sink would be dead configuration. + */ + static Map getCustomExportViews() { + ImmutableMap.Builder views = ImmutableMap.builder(); + addViews(views, CUSTOM_EXPORT_METER_NAME, true); + return views.build(); + } + + private static void addViews( + ImmutableMap.Builder views, + String metricPrefix, + boolean includeEndpointAttribute) { defineView( views, + metricPrefix, BuiltInMetricsConstant.GAX_METER_NAME, BuiltInMetricsConstant.OPERATION_LATENCY_NAME, BuiltInMetricsConstant.OPERATION_LATENCIES_NAME, BuiltInMetricsConstant.AGGREGATION_WITH_MILLIS_HISTOGRAM, InstrumentType.HISTOGRAM, - "ms"); + "ms", + false); defineView( views, + metricPrefix, BuiltInMetricsConstant.GAX_METER_NAME, BuiltInMetricsConstant.ATTEMPT_LATENCY_NAME, BuiltInMetricsConstant.ATTEMPT_LATENCIES_NAME, BuiltInMetricsConstant.AGGREGATION_WITH_MILLIS_HISTOGRAM, InstrumentType.HISTOGRAM, - "ms"); + "ms", + includeEndpointAttribute); defineView( views, + metricPrefix, BuiltInMetricsConstant.GAX_METER_NAME, BuiltInMetricsConstant.OPERATION_COUNT_NAME, BuiltInMetricsConstant.OPERATION_COUNT_NAME, Aggregation.sum(), InstrumentType.COUNTER, - "1"); + "1", + false); defineView( views, + metricPrefix, BuiltInMetricsConstant.GAX_METER_NAME, BuiltInMetricsConstant.ATTEMPT_COUNT_NAME, BuiltInMetricsConstant.ATTEMPT_COUNT_NAME, Aggregation.sum(), InstrumentType.COUNTER, - "1"); - defineSpannerView(views); - defineGRPCView(views); - defineGrpcGcpView(views); - return views.build(); + "1", + includeEndpointAttribute); + defineSpannerView(views, false); + defineGRPCView(views, metricPrefix, false); + } + + private static Set baseAttributesFilter(boolean includeEndpointAttribute) { + Set attributesFilter = + BuiltInMetricsConstant.COMMON_ATTRIBUTES.stream() + .map(AttributeKey::getKey) + .collect(Collectors.toCollection(HashSet::new)); + if (includeEndpointAttribute) { + attributesFilter.add(ENDPOINT_KEY.getKey()); + } + return attributesFilter; } private static void defineView( ImmutableMap.Builder viewMap, + String metricPrefix, String meterName, String metricName, String metricViewName, Aggregation aggregation, InstrumentType type, - String unit) { + String unit, + boolean includeEndpointAttribute) { InstrumentSelector selector = InstrumentSelector.builder() - .setName(BuiltInMetricsConstant.METER_NAME + '/' + metricName) + .setName(metricPrefix + '/' + metricName) .setMeterName(meterName) .setType(type) .setUnit(unit) .build(); - Set attributesFilter = - BuiltInMetricsConstant.COMMON_ATTRIBUTES.stream() - .map(AttributeKey::getKey) - .collect(Collectors.toSet()); View view = View.builder() - .setName(BuiltInMetricsConstant.METER_NAME + '/' + metricViewName) + .setName(metricPrefix + '/' + metricViewName) .setAggregation(aggregation) - .setAttributeFilter(attributesFilter) + .setAttributeFilter(baseAttributesFilter(includeEndpointAttribute)) .build(); viewMap.put(selector, view); } - private static void defineSpannerView(ImmutableMap.Builder viewMap) { + private static void defineSpannerView( + ImmutableMap.Builder viewMap, boolean includeEndpointAttribute) { InstrumentSelector selector = InstrumentSelector.builder() .setMeterName(BuiltInMetricsConstant.SPANNER_METER_NAME) .build(); - Set attributesFilter = - BuiltInMetricsConstant.COMMON_ATTRIBUTES.stream() - .map(AttributeKey::getKey) - .collect(Collectors.toSet()); - View view = View.builder().setAttributeFilter(attributesFilter).build(); + View view = + View.builder().setAttributeFilter(baseAttributesFilter(includeEndpointAttribute)).build(); viewMap.put(selector, view); } - private static void defineGRPCView(ImmutableMap.Builder viewMap) { + private static void defineGRPCView( + ImmutableMap.Builder viewMap, + String metricPrefix, + boolean includeEndpointAttribute) { for (String metric : BuiltInMetricsConstant.GRPC_METRICS_TO_ENABLE) { InstrumentSelector selector = InstrumentSelector.builder() .setName(metric) .setMeterName(BuiltInMetricsConstant.GRPC_METER_NAME) .build(); - Set attributesFilter = - BuiltInMetricsConstant.COMMON_ATTRIBUTES.stream() - .map(AttributeKey::getKey) - .collect(Collectors.toSet()); + Set attributesFilter = baseAttributesFilter(includeEndpointAttribute); attributesFilter.addAll( GRPC_METRIC_ADDITIONAL_ATTRIBUTES.getOrDefault(metric, ImmutableSet.of())); View view = View.builder() - .setName(BuiltInMetricsConstant.METER_NAME + '/' + metric.replace(".", "/")) + .setName(metricPrefix + '/' + metric.replace(".", "/")) .setAttributeFilter(attributesFilter) .build(); viewMap.put(selector, view); @@ -318,11 +374,7 @@ private static void defineGrpcGcpView(ImmutableMap.Builder attributesFilter = - BuiltInMetricsConstant.COMMON_ATTRIBUTES.stream() - .map(AttributeKey::getKey) - .collect(Collectors.toSet()); - + Set attributesFilter = baseAttributesFilter(false); attributesFilter.addAll( GRPC_GCP_METRIC_ADDITIONAL_ATTRIBUTES.getOrDefault(metric, ImmutableSet.of())); diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsProvider.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsProvider.java index 5ba34207ea08..9c551dce70de 100644 --- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsProvider.java +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsProvider.java @@ -130,6 +130,16 @@ synchronized OpenTelemetry getOpenTelemetry() { return this.openTelemetry; } + /** + * Injects the Cloud Monitoring OpenTelemetry so that {@link #getOrCreateOpenTelemetry} returns it + * instead of building one backed by the live Cloud Monitoring exporter. Used to observe the + * reserved-namespace sink in-memory in tests. + */ + @VisibleForTesting + synchronized void setOpenTelemetry(OpenTelemetry openTelemetry) { + this.openTelemetry = openTelemetry; + } + synchronized String getProjectId() { return this.projectId; } @@ -188,11 +198,22 @@ void enableGrpcMetrics( @Nullable Credentials credentials, @Nullable String monitoringHost, String universeDomain) { + enableGrpcMetrics( + channelProviderBuilder, + this.getOrCreateOpenTelemetry(projectId, credentials, monitoringHost, universeDomain)); + } + + /** + * Wires gRPC-layer metrics into the channel builder using the given OpenTelemetry instance + * instead of the process-wide Cloud Monitoring one. Used for {@link + * CustomOpenTelemetryMetricsProvider}; does not touch any state of this singleton. + */ + void enableGrpcMetrics( + InstantiatingGrpcChannelProvider.Builder channelProviderBuilder, + OpenTelemetry openTelemetry) { GrpcOpenTelemetry grpcOpenTelemetry = GrpcOpenTelemetry.newBuilder() - .sdk( - this.getOrCreateOpenTelemetry( - projectId, credentials, monitoringHost, universeDomain)) + .sdk(openTelemetry) .enableMetrics(BuiltInMetricsConstant.GRPC_METRICS_TO_ENABLE) // Disable gRPCs default metrics as they are not needed for Spanner. .disableMetrics(BuiltInMetricsConstant.GRPC_METRICS_ENABLED_BY_DEFAULT) diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsView.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsView.java index e72eeb9425a4..73a3738e8dcc 100644 --- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsView.java +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsView.java @@ -24,7 +24,7 @@ class BuiltInMetricsView { private BuiltInMetricsView() {} - /** Register built-in metrics on the {@link SdkMeterProviderBuilder} with credentials. */ + /** Registers Cloud Monitoring built-in metrics views and exporter on the builder. */ static void registerBuiltinMetrics( MetricExporter metricExporter, SdkMeterProviderBuilder builder) { BuiltInMetricsConstant.getAllViews().forEach(builder::registerView); diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/CustomOpenTelemetryMetricsProvider.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/CustomOpenTelemetryMetricsProvider.java new file mode 100644 index 000000000000..3585b63262b5 --- /dev/null +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/CustomOpenTelemetryMetricsProvider.java @@ -0,0 +1,93 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.spanner; + +import com.google.common.base.Preconditions; +import io.opentelemetry.api.OpenTelemetry; + +/** + * A {@link MetricsProvider} that records Spanner client metrics on a caller-provided {@link + * OpenTelemetry} instance. The caller owns the metrics pipeline (readers, exporters and resource); + * Spanner only records the client metric instruments on it. The caller-owned sink is never exported + * to Google Cloud Monitoring by the Spanner client and is independent of the built-in Cloud + * Monitoring export. + * + *

Metrics recorded through this provider use the {@code spanner/client/} instrument namespace + * instead of the reserved {@code spanner.googleapis.com/internal/client/} namespace that is used + * for the Cloud Monitoring export, so that re-exporting the caller-owned pipeline to Cloud + * Monitoring does not conflict with the curated Spanner metric descriptors. + * + *

It is recommended to register the Spanner client-metrics views on the {@link + * io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder} with {@link + * SpannerMetrics#configureMeterProviderBuilder(io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder)} + * before creating the {@link OpenTelemetry} instance. Without the views, metrics are still + * recorded, but with the raw instrument names, default histogram buckets and without attribute + * filtering. When registering the views, use a meter provider that is dedicated to Spanner client + * metrics; see {@link + * SpannerMetrics#configureMeterProviderBuilder(io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder)} + * for details. + * + *

gRPC-layer metrics are only recorded if the given {@link OpenTelemetry} instance is an + * instance of {@link io.opentelemetry.sdk.OpenTelemetrySdk}; for any other implementation they are + * skipped. All other client metrics work with any {@link OpenTelemetry} implementation. + * + *

Client metrics are not recorded when the client runs against the Spanner emulator, regardless + * of the configured {@link MetricsProvider}; the caller-owned sink is inactive in that mode. + * + *

This provider is independent of the default Cloud Monitoring export: the caller-owned + * destination recorded here keeps recording even when the Cloud Monitoring (GCM) sink is disabled + * via {@link SpannerOptions.Builder#setBuiltInMetricsEnabled(boolean)} or the {@code + * SPANNER_DISABLE_BUILTIN_METRICS} environment variable. On {@link + * SpannerOptions.InstanceType#OMNI} clients, where the GCM sink is never active, this provider is + * the way to export client metrics. + * + *

Example usage: + * + * 

{@code
+ * SdkMeterProviderBuilder meterProviderBuilder =
+ *     SdkMeterProvider.builder().registerMetricReader(PeriodicMetricReader.create(myExporter));
+ * SpannerMetrics.configureMeterProviderBuilder(meterProviderBuilder);
+ * OpenTelemetry openTelemetry =
+ *     OpenTelemetrySdk.builder().setMeterProvider(meterProviderBuilder.build()).build();
+ * SpannerOptions options =
+ *     SpannerOptions.newBuilder()
+ *         .setClientMetricsProvider(CustomOpenTelemetryMetricsProvider.create(openTelemetry))
+ *         .build();
+ * }
+ */ +public final class CustomOpenTelemetryMetricsProvider implements MetricsProvider { + + private final OpenTelemetry openTelemetry; + + private CustomOpenTelemetryMetricsProvider(OpenTelemetry openTelemetry) { + this.openTelemetry = openTelemetry; + } + + /** + * Creates a provider that records Spanner client metrics on the given caller-owned {@link + * OpenTelemetry} instance. + */ + public static CustomOpenTelemetryMetricsProvider create(OpenTelemetry openTelemetry) { + Preconditions.checkNotNull(openTelemetry, "openTelemetry cannot be null"); + return new CustomOpenTelemetryMetricsProvider(openTelemetry); + } + + /** Returns the caller-owned {@link OpenTelemetry} instance used for client metrics. */ + public OpenTelemetry getOpenTelemetry() { + return openTelemetry; + } +} diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/DefaultMetricsProvider.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/DefaultMetricsProvider.java new file mode 100644 index 000000000000..f1b3c9d5bcb5 --- /dev/null +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/DefaultMetricsProvider.java @@ -0,0 +1,30 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.spanner; + +/** + * The default {@link MetricsProvider}: no caller-owned client-metrics destination is configured. + * Built-in Cloud Monitoring export is controlled separately by {@link + * SpannerOptions.Builder#setBuiltInMetricsEnabled(boolean)} and the {@code + * SPANNER_DISABLE_BUILTIN_METRICS} environment variable. + */ +public final class DefaultMetricsProvider implements MetricsProvider { + + public static final DefaultMetricsProvider INSTANCE = new DefaultMetricsProvider(); + + private DefaultMetricsProvider() {} +} diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MetricsProvider.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MetricsProvider.java new file mode 100644 index 000000000000..c3cafd0881b2 --- /dev/null +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MetricsProvider.java @@ -0,0 +1,39 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.spanner; + +import com.google.api.core.InternalExtensionOnly; + +/** + * Provider for Spanner client metrics exported to a caller-owned OpenTelemetry destination. This is + * independent of the built-in Cloud Monitoring export controlled by {@link + * SpannerOptions.Builder#setBuiltInMetricsEnabled(boolean)}. + * + *

The available providers are: + * + *

    + *
  • {@link DefaultMetricsProvider}: No caller-owned client-metrics destination is configured. + * This is the default. On {@link SpannerOptions.InstanceType#OMNI}, where the Cloud + * Monitoring export is not available, this means no client metrics are exported. + *
  • {@link NoopMetricsProvider}: Caller-owned client metrics are explicitly disabled. The Cloud + * Monitoring export is controlled separately. + *
  • {@link CustomOpenTelemetryMetricsProvider}: The same Spanner client instruments are + * recorded on a caller-provided {@link io.opentelemetry.api.OpenTelemetry} instance. + *
+ */ +@InternalExtensionOnly +public interface MetricsProvider {} diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/NoopMetricsProvider.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/NoopMetricsProvider.java new file mode 100644 index 000000000000..d12c1caa48f5 --- /dev/null +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/NoopMetricsProvider.java @@ -0,0 +1,29 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.spanner; + +/** + * A {@link MetricsProvider} that disables caller-owned client-metrics export. This does not affect + * the built-in Cloud Monitoring export controlled by {@link + * SpannerOptions.Builder#setBuiltInMetricsEnabled(boolean)}. + */ +public final class NoopMetricsProvider implements MetricsProvider { + + public static final NoopMetricsProvider INSTANCE = new NoopMetricsProvider(); + + private NoopMetricsProvider() {} +} diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerMetrics.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerMetrics.java new file mode 100644 index 000000000000..f3f0cad4f4e9 --- /dev/null +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerMetrics.java @@ -0,0 +1,48 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.spanner; + +import com.google.common.base.Preconditions; +import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; + +/** Utility methods for configuring Spanner client metrics. */ +public final class SpannerMetrics { + private SpannerMetrics() {} + + /** + * Applies the Spanner client-metrics views to a caller-owned {@link SdkMeterProviderBuilder}. + * Recommended (but not required) when using {@link CustomOpenTelemetryMetricsProvider}: the views + * rename the raw instruments to the exported {@code spanner/client/...} metric names, apply the + * Spanner latency histogram bucket boundaries, and restrict the recorded attributes to the + * supported client-metric labels. Attempt metrics include the routed {@code endpoint} label when + * Spanner Omni location-aware routing supplies it. Without the views, metrics are still recorded + * with the raw instrument names, default buckets and unfiltered attributes. + * + *

Use a dedicated {@link SdkMeterProviderBuilder} (and resulting meter provider) for the + * Spanner client metrics, not one that is shared with other instrumentation. Some of the + * registered views select generic gRPC instruments (for example {@code + * grpc.client.attempt.duration} on the {@code grpc-java} meter), so on a shared meter provider + * they would also rename metrics recorded by unrelated gRPC clients into the {@code + * spanner/client} namespace and filter their attributes down to the Spanner client-metric labels. + * + *

Register the views at most once per {@link SdkMeterProviderBuilder}. + */ + public static void configureMeterProviderBuilder(SdkMeterProviderBuilder meterProviderBuilder) { + Preconditions.checkNotNull(meterProviderBuilder, "meterProviderBuilder cannot be null"); + BuiltInMetricsConstant.getCustomExportViews().forEach(meterProviderBuilder::registerView); + } +} diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java index 0a9c619b763d..4986cce7e0c2 100644 --- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java @@ -89,6 +89,7 @@ import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.sdk.OpenTelemetrySdk; import java.io.File; import java.io.IOException; import java.net.MalformedURLException; @@ -110,12 +111,15 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Level; +import java.util.logging.Logger; import javax.annotation.Nonnull; import javax.annotation.Nullable; /** Options for the Cloud Spanner service. */ public class SpannerOptions extends ServiceOptions { private static final long serialVersionUID = 2789571558532701170L; + private static final Logger logger = Logger.getLogger(SpannerOptions.class.getName()); private static SpannerEnvironment environment = SpannerEnvironmentImpl.INSTANCE; private static boolean enableOpenCensusMetrics = true; private static boolean enableOpenTelemetryMetrics = false; @@ -299,6 +303,7 @@ static GcpChannelPoolOptions mergeWithDefaultChannelPoolOptions( private final CallCredentialsProvider callCredentialsProvider; private final CloseableExecutorProvider asyncExecutorProvider; private final String compressorName; + private final String emulatorHost; private final boolean leaderAwareRoutingEnabled; private final boolean enableDirectAccess; private final boolean enableGcpFallback; @@ -306,7 +311,10 @@ static GcpChannelPoolOptions mergeWithDefaultChannelPoolOptions( private final boolean useVirtualThreads; private final OpenTelemetry openTelemetry; private final boolean enableApiTracing; + private final boolean rawEnableBuiltInMetrics; private final boolean enableBuiltInMetrics; + private final MetricsProvider metricsProvider; + private final MetricsProvider resolvedMetricsProvider; private final boolean enableLocationApi; private final boolean enableExtendedTracing; private final boolean enableEndToEndTracing; @@ -993,6 +1001,7 @@ protected SpannerOptions(Builder builder) { callCredentialsProvider = builder.callCredentialsProvider; asyncExecutorProvider = builder.asyncExecutorProvider; compressorName = builder.compressorName; + emulatorHost = builder.emulatorHost; leaderAwareRoutingEnabled = builder.leaderAwareRoutingEnabled; enableDirectAccess = builder.enableDirectAccess; enableGcpFallback = builder.enableGcpFallback; @@ -1001,11 +1010,26 @@ protected SpannerOptions(Builder builder) { openTelemetry = builder.openTelemetry; enableApiTracing = builder.enableApiTracing; enableExtendedTracing = builder.enableExtendedTracing; + // Resolve the client-metrics provider and the Cloud Monitoring (GCM) built-in metrics flag + // independently. The built-in metrics flag/env controls only the GCM sink; the client-metrics + // provider controls only the caller-owned OpenTelemetry sink. + // - Non-OMNI: GCM is gated by enableBuiltInMetrics/env. A custom provider records the same + // client instruments to the caller-owned spanner/client namespace regardless of that flag. + // - OMNI: GCM is always off (the backend rejects reserved-namespace writes). The Default + // provider is coerced to Noop, while a custom provider records client metrics regardless of + // enableBuiltInMetrics/env. + rawEnableBuiltInMetrics = builder.enableBuiltInMetrics; + metricsProvider = builder.metricsProvider; + MetricsProvider resolved = builder.metricsProvider; if (builder.instanceType == InstanceType.OMNI) { + if (resolved instanceof DefaultMetricsProvider) { + resolved = NoopMetricsProvider.INSTANCE; + } enableBuiltInMetrics = false; } else { enableBuiltInMetrics = builder.enableBuiltInMetrics; } + resolvedMetricsProvider = resolved; // Enable location API when InstanceType is OMNI, unless explicitly disabled // via GOOGLE_SPANNER_EXPERIMENTAL_LOCATION_API=false. if (builder.instanceType == InstanceType.OMNI) { @@ -1337,6 +1361,7 @@ private static Builder prepareBuilder(Builder builder) { private boolean enableExtendedTracing = SpannerOptions.environment.isEnableExtendedTracing(); private boolean enableEndToEndTracing = SpannerOptions.environment.isEnableEndToEndTracing(); private boolean enableBuiltInMetrics = SpannerOptions.environment.isEnableBuiltInMetrics(); + private MetricsProvider metricsProvider = DefaultMetricsProvider.INSTANCE; private boolean enableLocationApi = SpannerOptions.environment.isEnableLocationApi(); private String monitoringHost = SpannerOptions.environment.getMonitoringHost(); private SslContext mTLSContext = null; @@ -1409,11 +1434,7 @@ protected Builder() { Builder(SpannerOptions options) { super(options); - if (options.getHost() != null - && this.emulatorHost != null - && !options.getHost().equals(this.emulatorHost)) { - this.emulatorHost = null; - } + this.emulatorHost = options.emulatorHost; this.numChannels = options.numChannels; this.transportChannelExecutorThreadNameFormat = options.transportChannelExecutorThreadNameFormat; @@ -1450,7 +1471,8 @@ protected Builder() { this.useVirtualThreads = options.useVirtualThreads; this.enableApiTracing = options.enableApiTracing; this.enableExtendedTracing = options.enableExtendedTracing; - this.enableBuiltInMetrics = options.enableBuiltInMetrics; + this.enableBuiltInMetrics = options.rawEnableBuiltInMetrics; + this.metricsProvider = options.metricsProvider; this.enableLocationApi = options.enableLocationApi; this.enableEndToEndTracing = options.enableEndToEndTracing; this.monitoringHost = options.monitoringHost; @@ -2180,14 +2202,46 @@ public Builder setEnableApiTracing(boolean enableApiTracing) { } /** - * Sets whether to enable or disable built in metrics for Data client operations. Built in - * metrics are enabled by default. + * Sets whether to enable or disable the Cloud Monitoring export destination for Spanner + * built-in client metrics. Built-in metrics are enabled by default. + * + *

This flag controls only the default Cloud Monitoring (GCM) export destination on non-Omni + * clients. It has no effect on a client-metrics provider configured with {@link + * #setClientMetricsProvider(MetricsProvider)}; that caller-owned OpenTelemetry destination is + * controlled separately by the configured provider. + * + *

On {@link InstanceType#OMNI} clients the GCM sink is unavailable, so this flag does not + * enable Cloud Monitoring export there. */ public Builder setBuiltInMetricsEnabled(boolean enableBuiltInMetrics) { this.enableBuiltInMetrics = enableBuiltInMetrics; return this; } + /** + * Sets the {@link MetricsProvider} that controls client-metrics export to a caller-owned + * OpenTelemetry destination. Defaults to {@link DefaultMetricsProvider#INSTANCE}, which does + * not configure a caller-owned destination. Use {@link NoopMetricsProvider#INSTANCE} to disable + * caller-owned client metrics explicitly, or {@link CustomOpenTelemetryMetricsProvider} to + * record the same Spanner client instruments on a caller-provided {@link OpenTelemetry} + * instance. + * + *

Client metrics are independent of the built-in Cloud Monitoring (GCM) export controlled by + * {@link #setBuiltInMetricsEnabled(boolean)} and the {@code SPANNER_DISABLE_BUILTIN_METRICS} + * environment variable. A custom client-metrics provider keeps recording when the GCM sink is + * disabled. On non-Omni clients, the GCM and client-metrics destinations can both be active. On + * {@link InstanceType#OMNI} clients, the GCM sink is unavailable and a custom provider is the + * way to export client metrics. + * + *

Client metrics are not recorded when the client runs against the Spanner emulator, + * regardless of the configured {@link MetricsProvider}. + */ + public Builder setClientMetricsProvider(MetricsProvider metricsProvider) { + this.metricsProvider = + Preconditions.checkNotNull(metricsProvider, "metricsProvider cannot be null"); + return this; + } + /** Sets the monitoring host to be used for Built-in client side metrics */ @Deprecated @ObsoleteApi( @@ -2323,7 +2377,8 @@ public SpannerOptions build() { if (!emulatorHost.startsWith("http")) { emulatorHost = "http://" + emulatorHost; } - this.setHost(emulatorHost); + this.host = emulatorHost; + super.setHost(this.host); // Channels are secure by default (via SSL/TLS). For the example we disable TLS to avoid // needing certificates. this.setChannelConfigurator(ManagedChannelBuilder::usePlaintext); @@ -2643,8 +2698,38 @@ public OpenTelemetry getBuiltInOpenTelemetry() { this.getProjectId(), getCredentials(), this.monitoringHost, getUniverseDomain()); } + /** + * Wires gRPC-layer built-in metrics into the given channel provider builder using the + * pre-emulator-aware behavior. + * + * @deprecated for internal callers only. Use {@link #enablegRPCMetrics( + * InstantiatingGrpcChannelProvider.Builder, boolean)}. + */ + @Deprecated + @InternalApi public void enablegRPCMetrics(InstantiatingGrpcChannelProvider.Builder channelProviderBuilder) { - if (isEnableBuiltInMetrics() && SpannerOptions.environment.isEnableGRPCBuiltInMetrics()) { + enablegRPCMetrics(channelProviderBuilder, /* isEmulatorEnabled= */ false); + } + + /** + * Wires gRPC-layer built-in metrics into the given channel provider builder for each active + * metrics sink: the Cloud Monitoring (GCM) sink when {@link #isEnableBuiltInMetrics()} is true, + * and the caller-owned OpenTelemetry sink when a {@link CustomOpenTelemetryMetricsProvider} is + * configured. Does nothing when {@code isEmulatorEnabled} is true or gRPC built-in metrics are + * disabled via the environment. + */ + @InternalApi + public void enablegRPCMetrics( + InstantiatingGrpcChannelProvider.Builder channelProviderBuilder, boolean isEmulatorEnabled) { + if (isEmulatorEnabled || !SpannerOptions.environment.isEnableGRPCBuiltInMetrics()) { + return; + } + // The GCM and client-metrics sinks are independent; wire each active sink onto the channel. + // grpc-java + // records each instrument once per GrpcOpenTelemetry instance, so wiring two instances onto one + // channel does not double-count attempt/operation metrics (verified by + // GrpcMetricsDualWireTest). + if (isEnableBuiltInMetrics()) { this.builtInMetricsProvider.enableGrpcMetrics( channelProviderBuilder, this.getProjectId(), @@ -2652,6 +2737,19 @@ public void enablegRPCMetrics(InstantiatingGrpcChannelProvider.Builder channelPr this.monitoringHost, getUniverseDomain()); } + if (usesCustomMetricsProvider()) { + OpenTelemetry customOpenTelemetry = + ((CustomOpenTelemetryMetricsProvider) resolvedMetricsProvider).getOpenTelemetry(); + // GrpcOpenTelemetry can only record metrics on an SDK instance. + if (customOpenTelemetry instanceof OpenTelemetrySdk) { + this.builtInMetricsProvider.enableGrpcMetrics(channelProviderBuilder, customOpenTelemetry); + } else { + logger.log( + Level.FINE, + "Skipping gRPC-layer built-in metrics: the OpenTelemetry instance of the" + + " CustomOpenTelemetryMetricsProvider is not an OpenTelemetrySdk."); + } + } } void initializeBuiltInMetrics(DatabaseId databaseId) { @@ -2671,12 +2769,22 @@ private ApiTracerFactory createApiTracerFactory( apiTracerFactories.add( MoreObjects.firstNonNull(super.getApiTracerFactory(), getDefaultApiTracerFactory())); - // Add Metrics Tracer factory if built in metrics are enabled and if the client is data client - // and if emulator is not enabled. - if (isEnableBuiltInMetrics() && !isAdminClient && !isEmulatorEnabled && !usesNoCredentials()) { - ApiTracerFactory metricsTracerFactory = createMetricsApiTracerFactory(); - if (metricsTracerFactory != null) { - apiTracerFactories.add(metricsTracerFactory); + // Add built-in metrics tracer factories for a data client (never admin/emulator). The Cloud + // Monitoring (GCM) sink and a caller-provided client-metrics sink are independent and may both + // be + // active, so each active sink gets its own tracer factory recording into its own OpenTelemetry + // and namespace. The credentials guard only applies to the GCM sink: a client-metrics sink + // needs no + // Google credentials to export (e.g. Spanner Omni clients). + if (!isAdminClient && !isEmulatorEnabled) { + if (isEnableBuiltInMetrics() && !usesNoCredentials()) { + ApiTracerFactory cloudMonitoringTracerFactory = createCloudMonitoringMetricsTracerFactory(); + if (cloudMonitoringTracerFactory != null) { + apiTracerFactories.add(cloudMonitoringTracerFactory); + } + } + if (usesCustomMetricsProvider()) { + apiTracerFactories.add(createCustomMetricsTracerFactory()); } } @@ -2699,7 +2807,31 @@ private ApiTracerFactory getDefaultApiTracerFactory() { return BaseApiTracerFactory.getInstance(); } - private ApiTracerFactory createMetricsApiTracerFactory() { + /** + * Records client metrics on the caller-provided OpenTelemetry with the custom-export namespace. + * This deliberately does not touch the process-wide BuiltInMetricsProvider singleton, so a client + * with a client-metrics sink can coexist with clients that export to Cloud Monitoring in the same + * JVM. + */ + private ApiTracerFactory createCustomMetricsTracerFactory() { + OpenTelemetry customOpenTelemetry = + ((CustomOpenTelemetryMetricsProvider) resolvedMetricsProvider).getOpenTelemetry(); + // The Cloud Monitoring exporter adds client_name/client_uid at export time; on the custom + // path they must be recorded as metric attributes instead. + Map clientAttributes = this.builtInMetricsProvider.createClientAttributes(); + return new BuiltInMetricsTracerFactory( + new BuiltInMetricsRecorder( + customOpenTelemetry, BuiltInMetricsConstant.CUSTOM_EXPORT_METER_NAME), + clientAttributes, + createTraceWrapper()); + } + + /** + * Records built-in metrics on the process-wide Cloud Monitoring OpenTelemetry with the reserved + * {@code spanner.googleapis.com} namespace. Returns {@code null} when the Cloud Monitoring + * OpenTelemetry could not be created. + */ + private ApiTracerFactory createCloudMonitoringMetricsTracerFactory() { OpenTelemetry openTelemetry = this.builtInMetricsProvider.getOrCreateOpenTelemetry( this.getProjectId(), getCredentials(), this.monitoringHost, getUniverseDomain()); @@ -2708,17 +2840,21 @@ private ApiTracerFactory createMetricsApiTracerFactory() { ? new BuiltInMetricsTracerFactory( new BuiltInMetricsRecorder(openTelemetry, BuiltInMetricsConstant.METER_NAME), new HashMap<>(), - new TraceWrapper( - Tracing.getTracer(), - // Using the OpenTelemetry object set in Spanner Options, will be NoOp if not set - this.getOpenTelemetry() - .getTracer( - MetricRegistryConstants.INSTRUMENTATION_SCOPE, - GaxProperties.getLibraryVersion(getClass())), - true)) + createTraceWrapper()) : null; } + private TraceWrapper createTraceWrapper() { + return new TraceWrapper( + Tracing.getTracer(), + // Using the OpenTelemetry object set in Spanner Options, will be NoOp if not set + this.getOpenTelemetry() + .getTracer( + MetricRegistryConstants.INSTRUMENTATION_SCOPE, + GaxProperties.getLibraryVersion(getClass())), + true); + } + /** * Returns true if an {@link com.google.api.gax.tracing.ApiTracer} should be created and set on * the Spanner client. Enabling this only has effect if an OpenTelemetry or OpenCensus trace @@ -2729,13 +2865,35 @@ public boolean isEnableApiTracing() { } /** - * Returns true if an {@link com.google.api.gax.tracing.MetricsTracer} should be created and set - * on the Spanner client. + * Returns true if built-in metrics are exported to the default Cloud Monitoring (GCM) + * destination. Always false for {@link InstanceType#OMNI} clients, where the GCM sink is + * unavailable. This flag does not affect a caller-owned client-metrics destination configured + * with {@link Builder#setClientMetricsProvider(MetricsProvider)}. */ public boolean isEnableBuiltInMetrics() { return enableBuiltInMetrics; } + /** + * Returns the effective {@link MetricsProvider} used for client metrics. This is the provider set + * with {@link Builder#setClientMetricsProvider(MetricsProvider)} after applying instance type + * rules: {@link NoopMetricsProvider} is returned when the {@link DefaultMetricsProvider} is used + * with an {@link InstanceType#OMNI} client. A {@link CustomOpenTelemetryMetricsProvider} records + * the same client instruments to an independent caller-owned OpenTelemetry destination on all + * instance types, including Omni, and is not affected by {@link + * Builder#setBuiltInMetricsEnabled(boolean)} or the {@code SPANNER_DISABLE_BUILTIN_METRICS} + * environment variable. {@link #toBuilder()} preserves the provider as originally set, so + * rebuilding the options re-applies these rules against the new configuration. + */ + public MetricsProvider getClientMetricsProvider() { + return resolvedMetricsProvider; + } + + /** Returns true if client metrics are recorded on a caller-provided OpenTelemetry. */ + private boolean usesCustomMetricsProvider() { + return resolvedMetricsProvider instanceof CustomOpenTelemetryMetricsProvider; + } + @InternalApi public boolean isEnableLocationApi() { return enableLocationApi; @@ -2889,4 +3047,12 @@ public String getEndpoint() { return String.format( "%s:%s", url.getHost(), url.getPort() < 0 ? url.getDefaultPort() : url.getPort()); } + + @InternalApi + public boolean isEmulatorEnabled() { + return getChannelProvider() == null + && emulatorHost != null + && getHost() != null + && getHost().equals(emulatorHost); + } } diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java index 831e5bf69da0..b5013b289457 100644 --- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java @@ -768,7 +768,9 @@ private InstantiatingGrpcChannelProvider.Builder createBaseChannelProviderBuilde defaultChannelProviderBuilder.setAttemptDirectPathXds(); } - options.enablegRPCMetrics(defaultChannelProviderBuilder); + options.enablegRPCMetrics( + defaultChannelProviderBuilder, + isEmulatorEnabled(options, System.getenv("SPANNER_EMULATOR_HOST"))); if (options.isUseVirtualThreads()) { ExecutorService executor = @@ -917,8 +919,8 @@ private static void checkEmulatorConnection( CredentialsProvider credentialsProvider, String emulatorHost) throws IOException { - // Only do the check if the emulator environment variable has been set to localhost. if (isEmulatorEnabled(options, emulatorHost)) { + String resolvedEmulatorHost = emulatorHost != null ? emulatorHost : options.getEndpoint(); // Do a quick check to see if the emulator is actually running. try { InstanceAdminStubSettings.Builder testEmulatorSettings = @@ -940,22 +942,22 @@ private static void checkEmulatorConnection( throw SpannerExceptionFactory.newSpannerException( ErrorCode.UNAVAILABLE, String.format( - "The environment variable SPANNER_EMULATOR_HOST has been set to %s, but no running" + "The Spanner emulator host has been set to %s, but no running" + " emulator could be found at that address.\n" + "Did you forget to start the emulator, or to unset the environment" - + " variable?", - emulatorHost)); + + " configuration?", + resolvedEmulatorHost)); } } } private static boolean isEmulatorEnabled(SpannerOptions options, String emulatorHost) { - // Only do the check if the emulator environment variable has been set to localhost. - return options.getChannelProvider() == null - && emulatorHost != null - && options.getHost() != null - && options.getHost().startsWith("http://localhost") - && options.getHost().endsWith(emulatorHost); + return options.isEmulatorEnabled() + || (options.getChannelProvider() == null + && emulatorHost != null + && options.getHost() != null + && options.getHost().startsWith("http://localhost") + && options.getHost().endsWith(emulatorHost)); } public static boolean isEnableAFEServerTiming() { diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/KeyAwareChannel.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/KeyAwareChannel.java index c0dfdb8210fe..432a793b8c50 100644 --- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/KeyAwareChannel.java +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/KeyAwareChannel.java @@ -19,7 +19,11 @@ import static com.google.cloud.spanner.XGoogSpannerRequestId.REQUEST_ID_CALL_OPTIONS_KEY; import com.google.api.core.InternalApi; +import com.google.api.gax.grpc.GrpcCallContext; import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; +import com.google.api.gax.tracing.ApiTracer; +import com.google.cloud.spanner.BuiltInMetricsConstant; +import com.google.cloud.spanner.CompositeTracer; import com.google.cloud.spanner.XGoogSpannerRequestId; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Ticker; @@ -626,6 +630,14 @@ public void sendMessage(RequestT message) { } selectedEndpoint = endpoint; selectedTargetEndpoint = endpoint.getAddress(); + // Label built-in attempt metrics with the endpoint this attempt was routed to. Retries + // re-enter sendMessage with the same operation-scoped tracer, so every attempt overwrites + // the attribute with the endpoint it actually used. + ApiTracer tracer = callOptions.getOption(GrpcCallContext.TRACER_KEY); + if (tracer instanceof CompositeTracer) { + ((CompositeTracer) tracer) + .addAttributes(BuiltInMetricsConstant.ENDPOINT_KEY.getKey(), selectedTargetEndpoint); + } selectedDatabaseScope = databaseScope != null ? databaseScope : routingScope(finder); selectedOperationUid = operationUid; selectedPreferLeader = preferLeader; diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/BuiltInMetricsBothSinksTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/BuiltInMetricsBothSinksTest.java new file mode 100644 index 000000000000..efce731caf3d --- /dev/null +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/BuiltInMetricsBothSinksTest.java @@ -0,0 +1,145 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.spanner; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.NoCredentials; +import com.google.cloud.spanner.connection.AbstractMockServerTest; +import io.grpc.ManagedChannelBuilder; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; +import io.opentelemetry.sdk.metrics.data.MetricData; +import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader; +import java.util.Collection; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * End-to-end proof of the additive dual-export semantics: a non-OMNI client with a custom sink AND + * the Cloud Monitoring (GCM) sink enabled records the app-layer built-in metrics into BOTH + * OpenTelemetry instances at once, each in its own namespace, with no cross-contamination. + * + *

The reserved-namespace GCM OpenTelemetry is injected in-memory (it is normally backed by the + * live Cloud Monitoring exporter), and {@code jmh.enabled} lifts the {@code NoCredentials} guard so + * the GCM app-layer tracer is created just as it would be with real credentials. + */ +@RunWith(JUnit4.class) +public class BuiltInMetricsBothSinksTest extends AbstractMockServerTest { + private static final Statement SELECT1 = Statement.of("SELECT 1"); + + private String previousJmhEnabled; + + @Before + public void enableCloudMonitoringGuard() { + // usesNoCredentials() returns false when jmh.enabled=true, so the GCM app-layer tracer is + // created even though the client uses NoCredentials against the mock server. + previousJmhEnabled = System.getProperty("jmh.enabled"); + System.setProperty("jmh.enabled", "true"); + BuiltInMetricsProvider.INSTANCE.reset(); + } + + @After + public void restore() { + BuiltInMetricsProvider.INSTANCE.reset(); + if (previousJmhEnabled == null) { + System.clearProperty("jmh.enabled"); + } else { + System.setProperty("jmh.enabled", previousJmhEnabled); + } + } + + private static OpenTelemetrySdk gcmOpenTelemetry(InMemoryMetricReader reader) { + SdkMeterProviderBuilder meterProviderBuilder = + SdkMeterProvider.builder().registerMetricReader(reader); + // Reserved-namespace views: rename raw instruments to spanner.googleapis.com/internal/client/*. + BuiltInMetricsConstant.getAllViews().forEach(meterProviderBuilder::registerView); + return OpenTelemetrySdk.builder().setMeterProvider(meterProviderBuilder.build()).build(); + } + + private static OpenTelemetrySdk customOpenTelemetry(InMemoryMetricReader reader) { + SdkMeterProviderBuilder meterProviderBuilder = + SdkMeterProvider.builder().registerMetricReader(reader); + // Custom-namespace views: rename raw instruments to spanner/client/*. + SpannerMetrics.configureMeterProviderBuilder(meterProviderBuilder); + return OpenTelemetrySdk.builder().setMeterProvider(meterProviderBuilder.build()).build(); + } + + private Spanner createSpanner(OpenTelemetry customSink) { + return SpannerOptions.newBuilder() + .setProjectId("my-project") + .setHost(String.format("http://localhost:%d", getPort())) + .setChannelConfigurator(ManagedChannelBuilder::usePlaintext) + .setCredentials(NoCredentials.getInstance()) + .setSessionPoolOption(SessionPoolOptions.newBuilder().setFailOnSessionLeak().build()) + .setClientMetricsProvider(CustomOpenTelemetryMetricsProvider.create(customSink)) + .build() + .getService(); + } + + private static void executeSelect1(Spanner spanner) { + DatabaseClient client = + spanner.getDatabaseClient(DatabaseId.of("my-project", "my-instance", "my-database")); + try (ResultSet resultSet = client.singleUse().executeQuery(SELECT1)) { + assertThat(resultSet.next()).isTrue(); + assertThat(resultSet.getLong(0)).isEqualTo(1L); + } + } + + private static MetricData getMetric(InMemoryMetricReader reader, String name) { + Collection metrics = reader.collectAllMetrics(); + return metrics.stream().filter(m -> m.getName().equals(name)).findFirst().orElse(null); + } + + private static long operationCount(InMemoryMetricReader reader, String meterName) { + MetricData metric = + getMetric(reader, meterName + "/" + BuiltInMetricsConstant.OPERATION_COUNT_NAME); + assertThat(metric).isNotNull(); + return metric.getLongSumData().getPoints().stream().mapToLong(point -> point.getValue()).sum(); + } + + @Test + public void bothSinksRecordAppLayerMetricsInTheirOwnNamespaces() { + InMemoryMetricReader gcmReader = InMemoryMetricReader.create(); + InMemoryMetricReader customReader = InMemoryMetricReader.create(); + BuiltInMetricsProvider.INSTANCE.setOpenTelemetry(gcmOpenTelemetry(gcmReader)); + + try (Spanner spanner = createSpanner(customOpenTelemetry(customReader))) { + executeSelect1(spanner); + } + + // Both sinks recorded the operation, each in its own namespace. + assertThat(operationCount(gcmReader, BuiltInMetricsConstant.METER_NAME)).isGreaterThan(0L); + assertThat(operationCount(customReader, BuiltInMetricsConstant.CUSTOM_EXPORT_METER_NAME)) + .isGreaterThan(0L); + + // No cross-contamination: the reserved names never land on the custom sink, and the custom + // names never land on the GCM sink. + for (MetricData metric : customReader.collectAllMetrics()) { + assertThat(metric.getName()).doesNotContain("spanner.googleapis.com"); + } + for (MetricData metric : gcmReader.collectAllMetrics()) { + assertThat(metric.getName()) + .doesNotContain(BuiltInMetricsConstant.CUSTOM_EXPORT_METER_NAME + "/"); + } + } +} diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/BuiltInMetricsCustomOpenTelemetryTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/BuiltInMetricsCustomOpenTelemetryTest.java new file mode 100644 index 000000000000..8d46ba81579d --- /dev/null +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/BuiltInMetricsCustomOpenTelemetryTest.java @@ -0,0 +1,174 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.spanner; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import com.google.cloud.NoCredentials; +import com.google.cloud.spanner.connection.AbstractMockServerTest; +import io.grpc.ManagedChannelBuilder; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; +import io.opentelemetry.sdk.metrics.data.MetricData; +import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader; +import java.util.Collection; +import org.junit.After; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Integration test (mock server) for {@link CustomOpenTelemetryMetricsProvider}: built-in metrics + * are recorded on the caller-provided OpenTelemetry with the {@code spanner/client/} names, and + * nothing is recorded for {@link NoopMetricsProvider} or the coerced OMNI default. + */ +@RunWith(JUnit4.class) +public class BuiltInMetricsCustomOpenTelemetryTest extends AbstractMockServerTest { + private static final Statement SELECT1 = Statement.of("SELECT 1"); + + @After + public void resetGcmSingleton() { + // A cloud custom+enabled client now also activates the Cloud Monitoring sink (additive + // semantics), which initializes the process-wide singleton via the gRPC layer. Reset it so the + // state does not leak into other tests sharing the JVM fork. + BuiltInMetricsProvider.INSTANCE.reset(); + } + + private Spanner createSpanner(MetricsProvider metricsProvider) { + return spannerBuilder(metricsProvider).build().getService(); + } + + private SpannerOptions.Builder spannerBuilder(MetricsProvider metricsProvider) { + return SpannerOptions.newBuilder() + .setProjectId("my-project") + .setHost(String.format("http://localhost:%d", getPort())) + .setChannelConfigurator(ManagedChannelBuilder::usePlaintext) + .setCredentials(NoCredentials.getInstance()) + .setSessionPoolOption(SessionPoolOptions.newBuilder().setFailOnSessionLeak().build()) + .setClientMetricsProvider(metricsProvider); + } + + private static OpenTelemetrySdk createOpenTelemetry(InMemoryMetricReader metricReader) { + SdkMeterProviderBuilder meterProviderBuilder = + SdkMeterProvider.builder().registerMetricReader(metricReader); + SpannerMetrics.configureMeterProviderBuilder(meterProviderBuilder); + return OpenTelemetrySdk.builder().setMeterProvider(meterProviderBuilder.build()).build(); + } + + private static void executeSelect1(Spanner spanner) { + DatabaseClient client = + spanner.getDatabaseClient(DatabaseId.of("my-project", "my-instance", "my-database")); + try (ResultSet resultSet = client.singleUse().executeQuery(SELECT1)) { + assertTrue(resultSet.next()); + assertThat(resultSet.getLong(0)).isEqualTo(1L); + } + } + + private static MetricData getMetric(InMemoryMetricReader reader, String name) { + Collection metrics = reader.collectAllMetrics(); + return metrics.stream().filter(m -> m.getName().equals(name)).findFirst().orElse(null); + } + + @Test + public void builtInMetricsAreRecordedOnCustomOpenTelemetry() { + InMemoryMetricReader metricReader = InMemoryMetricReader.create(); + try (Spanner spanner = + createSpanner( + CustomOpenTelemetryMetricsProvider.create(createOpenTelemetry(metricReader)))) { + executeSelect1(spanner); + } + + MetricData operationCount = + getMetric( + metricReader, + BuiltInMetricsConstant.CUSTOM_EXPORT_METER_NAME + + "/" + + BuiltInMetricsConstant.OPERATION_COUNT_NAME); + assertNotNull(operationCount); + assertThat( + operationCount.getLongSumData().getPoints().stream() + .anyMatch( + point -> + point.getValue() > 0 + && "Spanner.ExecuteStreamingSql" + .equals( + point.getAttributes().get(BuiltInMetricsConstant.METHOD_KEY)) + && "OK" + .equals( + point.getAttributes().get(BuiltInMetricsConstant.STATUS_KEY)) + && "my-database" + .equals( + point.getAttributes().get(BuiltInMetricsConstant.DATABASE_KEY)) + && point.getAttributes().get(BuiltInMetricsConstant.CLIENT_NAME_KEY) + != null)) + .isTrue(); + + MetricData attemptLatencies = + getMetric( + metricReader, + BuiltInMetricsConstant.CUSTOM_EXPORT_METER_NAME + + "/" + + BuiltInMetricsConstant.ATTEMPT_LATENCIES_NAME); + assertNotNull(attemptLatencies); + + // A default cloud client never produces the endpoint label, even on the custom sink. + assertThat( + attemptLatencies.getHistogramData().getPoints().stream() + .allMatch( + point -> + point.getAttributes().get(BuiltInMetricsConstant.ENDPOINT_KEY) == null)) + .isTrue(); + + // The reserved Cloud Monitoring names never appear on the custom sink. + for (MetricData metric : metricReader.collectAllMetrics()) { + assertThat(metric.getName()).doesNotContain("spanner.googleapis.com"); + } + } + + @Test + public void noMetricsAreRecordedForNoopProvider() { + InMemoryMetricReader noopReader = InMemoryMetricReader.create(); + // The OpenTelemetry instance exists, but the provider is Noop: nothing may be recorded. + OpenTelemetrySdk unusedOpenTelemetry = createOpenTelemetry(noopReader); + + try (Spanner noopSpanner = createSpanner(NoopMetricsProvider.INSTANCE)) { + executeSelect1(noopSpanner); + } + + assertThat(noopReader.collectAllMetrics()).isEmpty(); + } + + @Test + public void gcmSingletonIsNotInitializedByCustomOnlyClient() { + // With the Cloud Monitoring sink disabled, a custom-sink client must not create or modify the + // process-wide Cloud Monitoring OpenTelemetry singleton (custom sink only, no GCM). + BuiltInMetricsProvider.INSTANCE.reset(); + InMemoryMetricReader metricReader = InMemoryMetricReader.create(); + try (Spanner spanner = + spannerBuilder(CustomOpenTelemetryMetricsProvider.create(createOpenTelemetry(metricReader))) + .setBuiltInMetricsEnabled(false) + .build() + .getService()) { + executeSelect1(spanner); + } + assertNull(BuiltInMetricsProvider.INSTANCE.getOpenTelemetry()); + } +} diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/BuiltInMetricsCustomViewsTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/BuiltInMetricsCustomViewsTest.java new file mode 100644 index 000000000000..3ff2589f84f7 --- /dev/null +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/BuiltInMetricsCustomViewsTest.java @@ -0,0 +1,255 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.spanner; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertNotNull; + +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; +import io.opentelemetry.sdk.metrics.data.HistogramPointData; +import io.opentelemetry.sdk.metrics.data.MetricData; +import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Tests for the custom-export metric views registered with {@link + * SpannerMetrics#configureMeterProviderBuilder}, and for the schema gate that keeps the {@code + * endpoint} label out of the Cloud Monitoring export. + */ +@RunWith(JUnit4.class) +public class BuiltInMetricsCustomViewsTest { + + private static Map recordedAttributes(boolean includeEndpoint) { + Map attributes = new HashMap<>(); + attributes.put(BuiltInMetricsConstant.METHOD_KEY.getKey(), "Spanner.ExecuteSql"); + attributes.put(BuiltInMetricsConstant.STATUS_KEY.getKey(), "OK"); + attributes.put(BuiltInMetricsConstant.DATABASE_KEY.getKey(), "test-database"); + attributes.put(BuiltInMetricsConstant.CLIENT_NAME_KEY.getKey(), "spanner-java/test"); + attributes.put("some_attribute_that_is_not_allowed", "should-be-filtered"); + if (includeEndpoint) { + attributes.put(BuiltInMetricsConstant.ENDPOINT_KEY.getKey(), "routed-endpoint:1234"); + } + return attributes; + } + + private static MetricData getMetric(InMemoryMetricReader reader, String name) { + Collection metrics = reader.collectAllMetrics(); + return metrics.stream().filter(m -> m.getName().equals(name)).findFirst().orElse(null); + } + + @Test + public void customViewsRenameInstrumentsIntoCustomNamespace() { + InMemoryMetricReader reader = InMemoryMetricReader.create(); + SdkMeterProviderBuilder meterProviderBuilder = + SdkMeterProvider.builder().registerMetricReader(reader); + SpannerMetrics.configureMeterProviderBuilder(meterProviderBuilder); + OpenTelemetrySdk openTelemetry = + OpenTelemetrySdk.builder().setMeterProvider(meterProviderBuilder.build()).build(); + BuiltInMetricsRecorder recorder = + new BuiltInMetricsRecorder(openTelemetry, BuiltInMetricsConstant.CUSTOM_EXPORT_METER_NAME); + + recorder.recordAttemptLatency(5.0, recordedAttributes(true)); + recorder.recordAttemptCount(1, recordedAttributes(true)); + recorder.recordOperationLatency(6.0, recordedAttributes(true)); + recorder.recordOperationCount(1, recordedAttributes(true)); + + // Renamed metric names in the custom namespace. + assertNotNull( + getMetric( + reader, + BuiltInMetricsConstant.CUSTOM_EXPORT_METER_NAME + + "/" + + BuiltInMetricsConstant.ATTEMPT_LATENCIES_NAME)); + assertNotNull( + getMetric( + reader, + BuiltInMetricsConstant.CUSTOM_EXPORT_METER_NAME + + "/" + + BuiltInMetricsConstant.OPERATION_LATENCIES_NAME)); + assertNotNull( + getMetric( + reader, + BuiltInMetricsConstant.CUSTOM_EXPORT_METER_NAME + + "/" + + BuiltInMetricsConstant.ATTEMPT_COUNT_NAME)); + assertNotNull( + getMetric( + reader, + BuiltInMetricsConstant.CUSTOM_EXPORT_METER_NAME + + "/" + + BuiltInMetricsConstant.OPERATION_COUNT_NAME)); + + // The reserved Cloud Monitoring namespace never appears on the custom path. + for (MetricData metric : reader.collectAllMetrics()) { + assertThat(metric.getName()).doesNotContain("spanner.googleapis.com"); + } + } + + @Test + public void customViewsApplySpannerHistogramBuckets() { + InMemoryMetricReader reader = InMemoryMetricReader.create(); + SdkMeterProviderBuilder meterProviderBuilder = + SdkMeterProvider.builder().registerMetricReader(reader); + SpannerMetrics.configureMeterProviderBuilder(meterProviderBuilder); + OpenTelemetrySdk openTelemetry = + OpenTelemetrySdk.builder().setMeterProvider(meterProviderBuilder.build()).build(); + BuiltInMetricsRecorder recorder = + new BuiltInMetricsRecorder(openTelemetry, BuiltInMetricsConstant.CUSTOM_EXPORT_METER_NAME); + + recorder.recordAttemptLatency(5.0, recordedAttributes(true)); + + MetricData attemptLatencies = + getMetric( + reader, + BuiltInMetricsConstant.CUSTOM_EXPORT_METER_NAME + + "/" + + BuiltInMetricsConstant.ATTEMPT_LATENCIES_NAME); + assertNotNull(attemptLatencies); + HistogramPointData point = attemptLatencies.getHistogramData().getPoints().iterator().next(); + assertThat(point.getBoundaries()).isEqualTo(BuiltInMetricsConstant.BUCKET_BOUNDARIES); + } + + @Test + public void customViewsAllowEndpointAttributeAndFilterUnknownAttributes() { + InMemoryMetricReader reader = InMemoryMetricReader.create(); + SdkMeterProviderBuilder meterProviderBuilder = + SdkMeterProvider.builder().registerMetricReader(reader); + SpannerMetrics.configureMeterProviderBuilder(meterProviderBuilder); + OpenTelemetrySdk openTelemetry = + OpenTelemetrySdk.builder().setMeterProvider(meterProviderBuilder.build()).build(); + BuiltInMetricsRecorder recorder = + new BuiltInMetricsRecorder(openTelemetry, BuiltInMetricsConstant.CUSTOM_EXPORT_METER_NAME); + + recorder.recordAttemptLatency(5.0, recordedAttributes(true)); + + MetricData attemptLatencies = + getMetric( + reader, + BuiltInMetricsConstant.CUSTOM_EXPORT_METER_NAME + + "/" + + BuiltInMetricsConstant.ATTEMPT_LATENCIES_NAME); + assertNotNull(attemptLatencies); + HistogramPointData point = attemptLatencies.getHistogramData().getPoints().iterator().next(); + assertThat(point.getAttributes().get(BuiltInMetricsConstant.ENDPOINT_KEY)) + .isEqualTo("routed-endpoint:1234"); + assertThat(point.getAttributes().get(BuiltInMetricsConstant.METHOD_KEY)) + .isEqualTo("Spanner.ExecuteSql"); + assertThat( + point + .getAttributes() + .get( + io.opentelemetry.api.common.AttributeKey.stringKey( + "some_attribute_that_is_not_allowed"))) + .isNull(); + } + + @Test + public void customOperationViewsDropEndpointAttribute() { + InMemoryMetricReader reader = InMemoryMetricReader.create(); + SdkMeterProviderBuilder meterProviderBuilder = + SdkMeterProvider.builder().registerMetricReader(reader); + SpannerMetrics.configureMeterProviderBuilder(meterProviderBuilder); + OpenTelemetrySdk openTelemetry = + OpenTelemetrySdk.builder().setMeterProvider(meterProviderBuilder.build()).build(); + BuiltInMetricsRecorder recorder = + new BuiltInMetricsRecorder(openTelemetry, BuiltInMetricsConstant.CUSTOM_EXPORT_METER_NAME); + + recorder.recordOperationLatency(5.0, recordedAttributes(true)); + recorder.recordOperationCount(1, recordedAttributes(true)); + + MetricData operationLatencies = + getMetric( + reader, + BuiltInMetricsConstant.CUSTOM_EXPORT_METER_NAME + + "/" + + BuiltInMetricsConstant.OPERATION_LATENCIES_NAME); + assertNotNull(operationLatencies); + HistogramPointData latencyPoint = + operationLatencies.getHistogramData().getPoints().iterator().next(); + assertThat(latencyPoint.getAttributes().get(BuiltInMetricsConstant.ENDPOINT_KEY)).isNull(); + + MetricData operationCount = + getMetric( + reader, + BuiltInMetricsConstant.CUSTOM_EXPORT_METER_NAME + + "/" + + BuiltInMetricsConstant.OPERATION_COUNT_NAME); + assertNotNull(operationCount); + assertThat( + operationCount + .getLongSumData() + .getPoints() + .iterator() + .next() + .getAttributes() + .get(BuiltInMetricsConstant.ENDPOINT_KEY)) + .isNull(); + } + + @Test + public void customViewsExcludeGrpcGcpFallbackViews() { + // grpc-gcp DirectPath-fallback metrics are only recorded on the Cloud Monitoring path, so the + // custom-export views must not include them. + assertThat( + BuiltInMetricsConstant.getCustomExportViews().keySet().stream() + .anyMatch( + selector -> + BuiltInMetricsConstant.GRPC_GCP_METER_NAME.equals(selector.getMeterName()))) + .isFalse(); + assertThat( + BuiltInMetricsConstant.getAllViews().keySet().stream() + .anyMatch( + selector -> + BuiltInMetricsConstant.GRPC_GCP_METER_NAME.equals(selector.getMeterName()))) + .isTrue(); + } + + @Test + public void cloudMonitoringViewsDropEndpointAttribute() { + // Schema gate: even if an endpoint attribute is recorded, the Cloud Monitoring views filter + // it out, so the label can never reach the curated spanner.googleapis.com descriptors. + InMemoryMetricReader reader = InMemoryMetricReader.create(); + SdkMeterProviderBuilder meterProviderBuilder = + SdkMeterProvider.builder().registerMetricReader(reader); + BuiltInMetricsConstant.getAllViews().forEach(meterProviderBuilder::registerView); + OpenTelemetrySdk openTelemetry = + OpenTelemetrySdk.builder().setMeterProvider(meterProviderBuilder.build()).build(); + BuiltInMetricsRecorder recorder = + new BuiltInMetricsRecorder(openTelemetry, BuiltInMetricsConstant.METER_NAME); + + recorder.recordAttemptLatency(5.0, recordedAttributes(true)); + + MetricData attemptLatencies = + getMetric( + reader, + BuiltInMetricsConstant.METER_NAME + + "/" + + BuiltInMetricsConstant.ATTEMPT_LATENCIES_NAME); + assertNotNull(attemptLatencies); + HistogramPointData point = attemptLatencies.getHistogramData().getPoints().iterator().next(); + assertThat(point.getAttributes().get(BuiltInMetricsConstant.ENDPOINT_KEY)).isNull(); + assertThat(point.getAttributes().get(BuiltInMetricsConstant.METHOD_KEY)) + .isEqualTo("Spanner.ExecuteSql"); + } +} diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/GrpcMetricsDualWireTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/GrpcMetricsDualWireTest.java new file mode 100644 index 000000000000..08191cfc566a --- /dev/null +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/GrpcMetricsDualWireTest.java @@ -0,0 +1,122 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.spanner; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; +import com.google.cloud.spanner.connection.AbstractMockServerTest; +import com.google.spanner.v1.CreateSessionRequest; +import com.google.spanner.v1.SpannerGrpc; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Verifies that wiring two {@link io.grpc.opentelemetry.GrpcOpenTelemetry} instances onto a single + * channel builder (as done when a Cloud Monitoring sink and a caller-provided sink are both active) + * records gRPC-layer metrics on each sink exactly once, with no cross-sink inflation. + * + *

This is the empirical gate for the gRPC-layer fan-out: the additive semantics wire both + * OpenTelemetry instances into {@code enablegRPCMetrics}, and this test proves each instrument is + * counted once per sink rather than twice. + */ +@RunWith(JUnit4.class) +public class GrpcMetricsDualWireTest extends AbstractMockServerTest { + + private static final String GRPC_ATTEMPT_STARTED = "grpc.client.attempt.started"; + + private static OpenTelemetrySdk newOpenTelemetry(InMemoryMetricReader reader) { + return OpenTelemetrySdk.builder() + .setMeterProvider(SdkMeterProvider.builder().registerMetricReader(reader).build()) + .build(); + } + + /** Wires each supplied OpenTelemetry onto the builder using the exact production gRPC config. */ + private static ManagedChannel buildWiredChannel(OpenTelemetry... openTelemetries) { + InstantiatingGrpcChannelProvider.Builder providerBuilder = + InstantiatingGrpcChannelProvider.newBuilder(); + for (OpenTelemetry openTelemetry : openTelemetries) { + BuiltInMetricsProvider.INSTANCE.enableGrpcMetrics(providerBuilder, openTelemetry); + } + ManagedChannelBuilder channelBuilder = + NettyChannelBuilder.forAddress("localhost", getPort()).usePlaintext(); + // Apply the chained channel configurator installed by enableGrpcMetrics. + ManagedChannelBuilder configured = + providerBuilder.getChannelConfigurator().apply(channelBuilder); + return configured.build(); + } + + private static void issueUnaryRpc(ManagedChannel channel) { + SpannerGrpc.SpannerBlockingStub stub = SpannerGrpc.newBlockingStub(channel); + stub.createSession( + CreateSessionRequest.newBuilder() + .setDatabase("projects/p/instances/i/databases/d") + .build()); + } + + private static long attemptStartedCount(InMemoryMetricReader reader) { + return reader.collectAllMetrics().stream() + .filter(m -> m.getName().equals(GRPC_ATTEMPT_STARTED)) + .flatMap(m -> m.getLongSumData().getPoints().stream()) + .mapToLong(point -> point.getValue()) + .sum(); + } + + @Test + public void twoGrpcOpenTelemetryInstancesDoNotDoubleCount() { + // Baseline: a single wired sink sees exactly the real attempt count for one unary RPC. + InMemoryMetricReader baselineReader = InMemoryMetricReader.create(); + OpenTelemetrySdk baselineOtel = newOpenTelemetry(baselineReader); + ManagedChannel baselineChannel = buildWiredChannel(baselineOtel); + long baseline; + try { + issueUnaryRpc(baselineChannel); + baseline = attemptStartedCount(baselineReader); + } finally { + baselineChannel.shutdownNow(); + baselineOtel.close(); + } + assertThat(baseline).isGreaterThan(0L); + + // Dual-wire: two independent sinks on one channel. Each must see the SAME count as the + // single-sink baseline -- neither inflated (double-counting) nor starved. + InMemoryMetricReader readerA = InMemoryMetricReader.create(); + InMemoryMetricReader readerB = InMemoryMetricReader.create(); + OpenTelemetrySdk otelA = newOpenTelemetry(readerA); + OpenTelemetrySdk otelB = newOpenTelemetry(readerB); + ManagedChannel dualChannel = buildWiredChannel(otelA, otelB); + try { + issueUnaryRpc(dualChannel); + long countA = attemptStartedCount(readerA); + long countB = attemptStartedCount(readerB); + assertThat(countA).isEqualTo(baseline); + assertThat(countB).isEqualTo(baseline); + } finally { + dualChannel.shutdownNow(); + otelA.close(); + otelB.close(); + } + } +} diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/LocationAwareEndpointMetricsTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/LocationAwareEndpointMetricsTest.java new file mode 100644 index 000000000000..a3b80be2a9bc --- /dev/null +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/LocationAwareEndpointMetricsTest.java @@ -0,0 +1,149 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.spanner; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import com.google.cloud.NoCredentials; +import com.google.cloud.spanner.connection.AbstractMockServerTest; +import io.grpc.ManagedChannelBuilder; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; +import io.opentelemetry.sdk.metrics.data.HistogramPointData; +import io.opentelemetry.sdk.metrics.data.MetricData; +import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader; +import java.util.Collection; +import org.junit.After; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Integration test (mock server) for the {@code endpoint} label on the location-aware fastpath: + * with the experimental location API enabled and a {@link CustomOpenTelemetryMetricsProvider}, + * attempt metrics carry the endpoint the attempt was routed to; a regular cloud client never + * produces the label. + */ +@RunWith(JUnit4.class) +public class LocationAwareEndpointMetricsTest extends AbstractMockServerTest { + private static final Statement SELECT1 = Statement.of("SELECT 1"); + + @After + public void resetEnvironment() { + SpannerOptions.useDefaultEnvironment(); + } + + private Spanner createSpanner(boolean enableLocationApi, InMemoryMetricReader metricReader) { + if (enableLocationApi) { + SpannerOptions.useEnvironment( + new SpannerOptions.SpannerEnvironment() { + @Override + public boolean isEnableLocationApi() { + return true; + } + }); + } else { + SpannerOptions.useDefaultEnvironment(); + } + try { + SdkMeterProviderBuilder meterProviderBuilder = + SdkMeterProvider.builder().registerMetricReader(metricReader); + SpannerMetrics.configureMeterProviderBuilder(meterProviderBuilder); + OpenTelemetrySdk openTelemetry = + OpenTelemetrySdk.builder().setMeterProvider(meterProviderBuilder.build()).build(); + return SpannerOptions.newBuilder() + .setProjectId("my-project") + .setHost(String.format("http://localhost:%d", getPort())) + .setChannelConfigurator(ManagedChannelBuilder::usePlaintext) + .setCredentials(NoCredentials.getInstance()) + .setSessionPoolOption(SessionPoolOptions.newBuilder().setFailOnSessionLeak().build()) + .setClientMetricsProvider(CustomOpenTelemetryMetricsProvider.create(openTelemetry)) + .build() + .getService(); + } finally { + SpannerOptions.useDefaultEnvironment(); + } + } + + private static void executeSelect1(Spanner spanner) { + DatabaseClient client = + spanner.getDatabaseClient(DatabaseId.of("my-project", "my-instance", "my-database")); + try (ResultSet resultSet = client.singleUse().executeQuery(SELECT1)) { + assertTrue(resultSet.next()); + } + } + + private static MetricData getMetric(InMemoryMetricReader reader, String name) { + Collection metrics = reader.collectAllMetrics(); + return metrics.stream().filter(m -> m.getName().equals(name)).findFirst().orElse(null); + } + + @Test + public void attemptMetricsCarryEndpointLabelOnFastpath() { + InMemoryMetricReader metricReader = InMemoryMetricReader.create(); + try (Spanner spanner = createSpanner(/* enableLocationApi= */ true, metricReader)) { + executeSelect1(spanner); + } + + MetricData attemptLatencies = + getMetric( + metricReader, + BuiltInMetricsConstant.CUSTOM_EXPORT_METER_NAME + + "/" + + BuiltInMetricsConstant.ATTEMPT_LATENCIES_NAME); + assertNotNull(attemptLatencies); + HistogramPointData queryPoint = + attemptLatencies.getHistogramData().getPoints().stream() + .filter( + point -> + "Spanner.ExecuteStreamingSql" + .equals(point.getAttributes().get(BuiltInMetricsConstant.METHOD_KEY))) + .findFirst() + .orElse(null); + assertNotNull(queryPoint); + String endpoint = queryPoint.getAttributes().get(BuiltInMetricsConstant.ENDPOINT_KEY); + assertNotNull(endpoint); + // Without any routing information from the server, the attempt is routed to the default + // endpoint of the mock server. + assertThat(endpoint).contains(String.valueOf(getPort())); + } + + @Test + public void cloudClientNeverProducesEndpointLabel() { + InMemoryMetricReader metricReader = InMemoryMetricReader.create(); + try (Spanner spanner = createSpanner(/* enableLocationApi= */ false, metricReader)) { + executeSelect1(spanner); + } + + MetricData attemptLatencies = + getMetric( + metricReader, + BuiltInMetricsConstant.CUSTOM_EXPORT_METER_NAME + + "/" + + BuiltInMetricsConstant.ATTEMPT_LATENCIES_NAME); + assertNotNull(attemptLatencies); + assertThat( + attemptLatencies.getHistogramData().getPoints().stream() + .allMatch( + point -> + point.getAttributes().get(BuiltInMetricsConstant.ENDPOINT_KEY) == null)) + .isTrue(); + } +} diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MetricsProviderTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MetricsProviderTest.java new file mode 100644 index 000000000000..36e7ee96412d --- /dev/null +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MetricsProviderTest.java @@ -0,0 +1,307 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.spanner; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import com.google.cloud.NoCredentials; +import io.opentelemetry.api.OpenTelemetry; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Tests for the {@link MetricsProvider} semantics matrix on {@link SpannerOptions}. The Cloud + * Monitoring (GCM) sink and a caller-provided client-metrics sink are independent booleans that can + * both be active at once: + * + * 

+ * instanceType | provider | built-in disabled?      | GCM sink | client-metrics sink
+ * CLOUD        | Default  | no                       | on       | -
+ * CLOUD        | Default  | setBuiltInMetricsEnabled | off      | -
+ * CLOUD        | Noop     | -                        | on       | off
+ * CLOUD        | Custom   | no                       | on       | on
+ * CLOUD        | Custom   | setBuiltInMetricsEnabled | off      | on
+ * CLOUD        | Custom   | env kill-switch          | off      | on
+ * OMNI         | Default  | -                        | off (Noop) | -
+ * OMNI         | Noop     | -                        | off      | -
+ * OMNI         | Custom   | no                       | off      | on
+ * OMNI         | Custom   | setBuiltInMetricsEnabled | off      | on
+ * OMNI         | Custom   | env kill-switch          | off      | on
+ *
+ */ +@RunWith(JUnit4.class) +public class MetricsProviderTest { + + @Before + public void setUp() { + SpannerOptions.useEnvironment(new SpannerOptions.SpannerEnvironment() {}); + } + + @After + public void reset() { + SpannerOptions.useDefaultEnvironment(); + BuiltInMetricsProvider.INSTANCE.reset(); + } + + private static SpannerOptions.Builder cloudBuilder() { + return SpannerOptions.newBuilder() + .setProjectId("test-project") + .setCredentials(NoCredentials.getInstance()); + } + + private static SpannerOptions.Builder omniBuilder() { + return SpannerOptions.newBuilder() + .setType(SpannerOptions.InstanceType.OMNI) + .setHost("http://localhost:8080") + .setCredentials(NoCredentials.getInstance()); + } + + @Test + public void cloudDefaultProviderKeepsCloudMonitoringSink() { + SpannerOptions options = cloudBuilder().build(); + assertSame(DefaultMetricsProvider.INSTANCE, options.getClientMetricsProvider()); + assertTrue(options.isEnableBuiltInMetrics()); + } + + @Test + public void cloudNoopProviderDisablesOnlyClientMetrics() { + SpannerOptions options = + cloudBuilder().setClientMetricsProvider(NoopMetricsProvider.INSTANCE).build(); + assertSame(NoopMetricsProvider.INSTANCE, options.getClientMetricsProvider()); + assertTrue(options.isEnableBuiltInMetrics()); + } + + @Test + public void cloudCustomProviderKeepsBothSinks() { + // Additive semantics: adding a client-metrics sink does not silence the Cloud Monitoring sink. + CustomOpenTelemetryMetricsProvider provider = + CustomOpenTelemetryMetricsProvider.create(OpenTelemetry.noop()); + SpannerOptions options = cloudBuilder().setClientMetricsProvider(provider).build(); + assertSame(provider, options.getClientMetricsProvider()); + assertTrue(options.isEnableBuiltInMetrics()); + } + + @Test + public void cloudCustomProviderWithBuiltInMetricsDisabledIsCustomOnly() { + CustomOpenTelemetryMetricsProvider provider = + CustomOpenTelemetryMetricsProvider.create(OpenTelemetry.noop()); + SpannerOptions options = + cloudBuilder().setClientMetricsProvider(provider).setBuiltInMetricsEnabled(false).build(); + // The client-metrics sink survives; only the Cloud Monitoring sink is turned off. + assertSame(provider, options.getClientMetricsProvider()); + assertFalse(options.isEnableBuiltInMetrics()); + } + + @Test + public void explicitProviderCombinesWithBuiltInMetricsEnabledFlag() { + CustomOpenTelemetryMetricsProvider provider = + CustomOpenTelemetryMetricsProvider.create(OpenTelemetry.noop()); + SpannerOptions options = + cloudBuilder().setBuiltInMetricsEnabled(true).setClientMetricsProvider(provider).build(); + // Custom provider plus the built-in flag on: both sinks active. + assertSame(provider, options.getClientMetricsProvider()); + assertTrue(options.isEnableBuiltInMetrics()); + + options = + cloudBuilder() + .setBuiltInMetricsEnabled(true) + .setClientMetricsProvider(NoopMetricsProvider.INSTANCE) + .build(); + // A Noop provider means no caller-owned client metrics; Cloud Monitoring is separate. + assertSame(NoopMetricsProvider.INSTANCE, options.getClientMetricsProvider()); + assertTrue(options.isEnableBuiltInMetrics()); + } + + @Test + public void builtInMetricsEnabledFlagStillControlsDefaultProvider() { + SpannerOptions options = cloudBuilder().setBuiltInMetricsEnabled(false).build(); + assertSame(DefaultMetricsProvider.INSTANCE, options.getClientMetricsProvider()); + assertFalse(options.isEnableBuiltInMetrics()); + } + + @Test + public void omniDefaultProviderIsCoercedToNoop() { + SpannerOptions options = omniBuilder().build(); + assertSame(NoopMetricsProvider.INSTANCE, options.getClientMetricsProvider()); + assertFalse(options.isEnableBuiltInMetrics()); + } + + @Test + public void omniDefaultProviderIsCoercedToNoopEvenWithBuiltInMetricsEnabled() { + SpannerOptions options = omniBuilder().setBuiltInMetricsEnabled(true).build(); + assertSame(NoopMetricsProvider.INSTANCE, options.getClientMetricsProvider()); + assertFalse(options.isEnableBuiltInMetrics()); + } + + @Test + public void omniCustomProviderIsHonored() { + CustomOpenTelemetryMetricsProvider provider = + CustomOpenTelemetryMetricsProvider.create(OpenTelemetry.noop()); + SpannerOptions options = omniBuilder().setClientMetricsProvider(provider).build(); + assertSame(provider, options.getClientMetricsProvider()); + // The Cloud Monitoring sink stays off on OMNI. + assertFalse(options.isEnableBuiltInMetrics()); + + options = + omniBuilder().setClientMetricsProvider(provider).setBuiltInMetricsEnabled(true).build(); + assertSame(provider, options.getClientMetricsProvider()); + assertFalse(options.isEnableBuiltInMetrics()); + } + + @Test + public void omniCustomProviderWithBuiltInMetricsDisabledStaysActive() { + CustomOpenTelemetryMetricsProvider provider = + CustomOpenTelemetryMetricsProvider.create(OpenTelemetry.noop()); + SpannerOptions options = + omniBuilder().setClientMetricsProvider(provider).setBuiltInMetricsEnabled(false).build(); + assertSame(provider, options.getClientMetricsProvider()); + assertFalse(options.isEnableBuiltInMetrics()); + } + + @Test + public void environmentKillSwitchTurnsOffGcmButKeepsCustomSink() { + SpannerOptions.useEnvironment( + new SpannerOptions.SpannerEnvironment() { + @Override + public boolean isEnableBuiltInMetrics() { + // Simulates SPANNER_DISABLE_BUILTIN_METRICS=true. + return false; + } + }); + CustomOpenTelemetryMetricsProvider provider = + CustomOpenTelemetryMetricsProvider.create(OpenTelemetry.noop()); + + // Non-OMNI: the kill-switch only turns off the Cloud Monitoring sink; the client-metrics sink + // stays. + SpannerOptions cloudOptions = cloudBuilder().setClientMetricsProvider(provider).build(); + assertSame(provider, cloudOptions.getClientMetricsProvider()); + assertFalse(cloudOptions.isEnableBuiltInMetrics()); + + // OMNI: the kill-switch only affects Cloud Monitoring, which is impossible there anyway. + SpannerOptions omniOptions = omniBuilder().setClientMetricsProvider(provider).build(); + assertSame(provider, omniOptions.getClientMetricsProvider()); + assertFalse(omniOptions.isEnableBuiltInMetrics()); + } + + @Test + public void omniCustomProviderEnvironmentKillSwitchStaysActive() { + SpannerOptions.useEnvironment( + new SpannerOptions.SpannerEnvironment() { + @Override + public boolean isEnableBuiltInMetrics() { + return false; + } + }); + CustomOpenTelemetryMetricsProvider provider = + CustomOpenTelemetryMetricsProvider.create(OpenTelemetry.noop()); + + SpannerOptions options = + omniBuilder().setClientMetricsProvider(provider).setBuiltInMetricsEnabled(true).build(); + assertSame(provider, options.getClientMetricsProvider()); + assertFalse(options.isEnableBuiltInMetrics()); + } + + @Test + public void nullMetricsProviderIsRejected() { + assertThrows(NullPointerException.class, () -> cloudBuilder().setClientMetricsProvider(null)); + } + + @Test + public void toBuilderKeepsMetricsProvider() { + CustomOpenTelemetryMetricsProvider provider = + CustomOpenTelemetryMetricsProvider.create(OpenTelemetry.noop()); + SpannerOptions options = cloudBuilder().setClientMetricsProvider(provider).build(); + SpannerOptions copy = options.toBuilder().build(); + assertSame(provider, copy.getClientMetricsProvider()); + // Additive: a non-OMNI custom provider keeps the Cloud Monitoring sink on across toBuilder(). + assertTrue(copy.isEnableBuiltInMetrics()); + } + + @Test + public void toBuilderPreservesRawProviderAcrossInstanceTypeChange() { + SpannerOptions omniOptions = omniBuilder().build(); + assertSame(NoopMetricsProvider.INSTANCE, omniOptions.getClientMetricsProvider()); + + SpannerOptions cloudCopy = + omniOptions.toBuilder() + .setType(SpannerOptions.InstanceType.CLOUD) + .setProjectId("test-project") + .build(); + assertSame(DefaultMetricsProvider.INSTANCE, cloudCopy.getClientMetricsProvider()); + } + + @Test + public void toBuilderReResolvesBuiltInMetricsFlagAgainstNewProvider() { + CustomOpenTelemetryMetricsProvider provider = + CustomOpenTelemetryMetricsProvider.create(OpenTelemetry.noop()); + SpannerOptions options = + cloudBuilder().setClientMetricsProvider(provider).setBuiltInMetricsEnabled(false).build(); + assertFalse(options.isEnableBuiltInMetrics()); + + // Switching back to the default provider re-enables the Cloud Monitoring sink, and the raw + // built-in flag is re-evaluated rather than pinned to the previously resolved value. + SpannerOptions copy = + options.toBuilder() + .setClientMetricsProvider(DefaultMetricsProvider.INSTANCE) + .setBuiltInMetricsEnabled(true) + .build(); + assertSame(DefaultMetricsProvider.INSTANCE, copy.getClientMetricsProvider()); + assertTrue(copy.isEnableBuiltInMetrics()); + } + + @Test + public void toBuilderPreservesRawProviderAcrossKillSwitchOnOmni() { + SpannerOptions.useEnvironment( + new SpannerOptions.SpannerEnvironment() { + @Override + public boolean isEnableBuiltInMetrics() { + return false; + } + }); + CustomOpenTelemetryMetricsProvider provider = + CustomOpenTelemetryMetricsProvider.create(OpenTelemetry.noop()); + // On OMNI the kill-switch does not affect the custom provider. + SpannerOptions options = omniBuilder().setClientMetricsProvider(provider).build(); + assertSame(provider, options.getClientMetricsProvider()); + + SpannerOptions.useDefaultEnvironment(); + SpannerOptions copy = options.toBuilder().build(); + assertSame(provider, copy.getClientMetricsProvider()); + } + + @Test + public void customProviderDoesNotTouchProcessWideCloudMonitoringSingleton() { + BuiltInMetricsProvider.INSTANCE.reset(); + CustomOpenTelemetryMetricsProvider provider = + CustomOpenTelemetryMetricsProvider.create(OpenTelemetry.noop()); + + SpannerOptions cloudOptions = cloudBuilder().setClientMetricsProvider(provider).build(); + cloudOptions.getApiTracerFactory(/* isAdminClient= */ false, /* isEmulatorEnabled= */ false); + assertNull(BuiltInMetricsProvider.INSTANCE.getOpenTelemetry()); + + SpannerOptions omniOptions = omniBuilder().setClientMetricsProvider(provider).build(); + omniOptions.getApiTracerFactory(/* isAdminClient= */ false, /* isEmulatorEnabled= */ false); + assertNull(BuiltInMetricsProvider.INSTANCE.getOpenTelemetry()); + } +} diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/SpannerOptionsTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/SpannerOptionsTest.java index 2ca142aa9544..bcff79ad3e0d 100644 --- a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/SpannerOptionsTest.java +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/SpannerOptionsTest.java @@ -31,6 +31,7 @@ import static org.mockito.Mockito.mock; import com.google.api.gax.grpc.GrpcCallContext; +import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; import com.google.api.gax.retrying.RetrySettings; import com.google.api.gax.rpc.ApiCallContext; import com.google.api.gax.rpc.ServerStreamingCallSettings; @@ -1451,6 +1452,39 @@ public void testOmniEnforcedInConstructor() { Duration.ofSeconds(42), options.getSessionPoolOptions().getAcquireSessionTimeout()); } + @Test + public void enableGrpcMetricsKeepsOneArgOverloadForCompatibility() throws Exception { + assertNotNull( + SpannerOptions.class.getMethod( + "enablegRPCMetrics", InstantiatingGrpcChannelProvider.Builder.class)); + } + + @Test + public void testCustomProviderKeepsCloudMonitoringSinkOnNonOmni() { + SpannerOptions.useEnvironment(new SpannerOptions.SpannerEnvironment() {}); + try { + CustomOpenTelemetryMetricsProvider provider = + CustomOpenTelemetryMetricsProvider.create(OpenTelemetry.noop()); + // Additive semantics: a custom sink on a non-OMNI client leaves the Cloud Monitoring sink on. + SpannerOptions options = + SpannerOptions.newBuilder() + .setProjectId("some-project") + .setCredentials(NoCredentials.getInstance()) + .setClientMetricsProvider(provider) + .build(); + assertSame(provider, options.getClientMetricsProvider()); + assertTrue(options.isEnableBuiltInMetrics()); + + // Disabling the built-in flag turns off only the Cloud Monitoring sink; the custom sink + // stays. + SpannerOptions customOnly = options.toBuilder().setBuiltInMetricsEnabled(false).build(); + assertSame(provider, customOnly.getClientMetricsProvider()); + assertFalse(customOnly.isEnableBuiltInMetrics()); + } finally { + SpannerOptions.useDefaultEnvironment(); + } + } + @Test public void testLogin() { SpannerOptions.Builder builder = diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpcTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpcTest.java index e39d6be2a33c..3b2b3f6d854b 100644 --- a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpcTest.java +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpcTest.java @@ -1300,7 +1300,7 @@ public boolean isEnableGRPCBuiltInMetrics() { InstantiatingGrpcChannelProvider.Builder channelProviderBuilder = InstantiatingGrpcChannelProvider.newBuilder(); - options.enablegRPCMetrics(channelProviderBuilder); + options.enablegRPCMetrics(channelProviderBuilder, /* isEmulatorEnabled= */ false); assertNull(channelProviderBuilder.getChannelConfigurator()); } finally { @@ -1308,6 +1308,56 @@ public boolean isEnableGRPCBuiltInMetrics() { } } + @Test + public void testEmulatorSkipsGrpcBuiltInMetricsConfigurator() { + try { + SpannerOptions.useEnvironment( + new SpannerOptions.SpannerEnvironment() { + @Override + public boolean isEnableGRPCBuiltInMetrics() { + return true; + } + }); + + SpannerOptions options = + SpannerOptions.newBuilder() + .setProjectId("[PROJECT]") + .setCredentials(STATIC_CREDENTIALS) + .setBuiltInMetricsEnabled(true) + .build(); + InstantiatingGrpcChannelProvider.Builder channelProviderBuilder = + InstantiatingGrpcChannelProvider.newBuilder(); + + options.enablegRPCMetrics(channelProviderBuilder, /* isEmulatorEnabled= */ true); + + assertNull(channelProviderBuilder.getChannelConfigurator()); + } finally { + SpannerOptions.useDefaultEnvironment(); + } + } + + @Test + public void testSetEmulatorHostIsDetectedWithoutEnvironmentVariable() throws Exception { + SpannerOptions emulatorOptions = + SpannerOptions.newBuilder() + .setProjectId("[PROJECT]") + .setEmulatorHost("localhost:1234") + .build(); + SpannerOptions localhostOptions = + SpannerOptions.newBuilder() + .setProjectId("[PROJECT]") + .setHost("http://localhost:1234") + .setCredentials(NoCredentials.getInstance()) + .build(); + java.lang.reflect.Method isEmulatorEnabled = + GapicSpannerRpc.class.getDeclaredMethod( + "isEmulatorEnabled", SpannerOptions.class, String.class); + isEmulatorEnabled.setAccessible(true); + + assertTrue((boolean) isEmulatorEnabled.invoke(null, emulatorOptions, null)); + assertFalse((boolean) isEmulatorEnabled.invoke(null, localhostOptions, null)); + } + private static final class RecordingTransportChannelProvider implements TransportChannelProvider { private final String host; private final int port; diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/KeyAwareChannelTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/KeyAwareChannelTest.java index af9366020da9..48f0c641f6f1 100644 --- a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/KeyAwareChannelTest.java +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/KeyAwareChannelTest.java @@ -22,9 +22,17 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThrows; +import com.google.api.gax.grpc.GrpcCallContext; import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; +import com.google.api.gax.tracing.BaseApiTracer; +import com.google.api.gax.tracing.MethodName; +import com.google.api.gax.tracing.MetricsRecorder; +import com.google.api.gax.tracing.MetricsTracer; +import com.google.cloud.spanner.BuiltInMetricsConstant; +import com.google.cloud.spanner.CompositeTracer; import com.google.cloud.spanner.XGoogSpannerRequestId; import com.google.common.base.Ticker; +import com.google.common.collect.ImmutableList; import com.google.common.testing.FakeTicker; import com.google.protobuf.ByteString; import com.google.protobuf.Empty; @@ -1678,4 +1686,123 @@ private static void assertRoutedEndpointErrorPenaltyRecorded(Status status, long databaseScope, routedOperationUid, true, "server-b:1234")) .isFalse(); } + + /** A gax {@link MetricsTracer} that records the attributes it receives. */ + private static final class CapturingMetricsTracer extends MetricsTracer { + private final Map capturedAttributes = new HashMap<>(); + + CapturingMetricsTracer() { + super(MethodName.of("Spanner", "ExecuteSql"), new MetricsRecorder() {}); + } + + @Override + public void addAttributes(String key, String value) { + super.addAttributes(key, value); + capturedAttributes.put(key, value); + } + } + + private static CallOptions callOptionsWithCompositeTracer(CapturingMetricsTracer metricsTracer) { + CompositeTracer compositeTracer = new CompositeTracer(ImmutableList.of(metricsTracer)); + return CallOptions.DEFAULT.withOption(GrpcCallContext.TRACER_KEY, compositeTracer); + } + + @Test + public void sendMessageInjectsEndpointAttributeIntoCompositeTracer() throws Exception { + TestHarness harness = createHarness(); + CapturingMetricsTracer metricsTracer = new CapturingMetricsTracer(); + + ClientCall call = + harness.channel.newCall( + SpannerGrpc.getExecuteSqlMethod(), callOptionsWithCompositeTracer(metricsTracer)); + call.start(new CapturingListener(), new Metadata()); + call.sendMessage(ExecuteSqlRequest.newBuilder().setSession(SESSION).build()); + + assertEquals( + DEFAULT_ADDRESS, + metricsTracer.capturedAttributes.get(BuiltInMetricsConstant.ENDPOINT_KEY.getKey())); + } + + @Test + public void endpointAttributeIsOverwrittenWhenRetryRoutesToDifferentEndpoint() throws Exception { + TestHarness harness = createHarness(); + CapturingMetricsTracer metricsTracer = new CapturingMetricsTracer(); + CallOptions callOptions = callOptionsWithCompositeTracer(metricsTracer); + ExecuteSqlRequest request = + ExecuteSqlRequest.newBuilder() + .setSession(SESSION) + .setRoutingHint(RoutingHint.newBuilder().setKey(bytes("a")).build()) + .build(); + + // First attempt: no routing information yet, routed to the default endpoint. + ClientCall firstCall = + harness.channel.newCall(SpannerGrpc.getExecuteSqlMethod(), callOptions); + firstCall.start(new CapturingListener(), new Metadata()); + firstCall.sendMessage(request); + assertEquals( + DEFAULT_ADDRESS, + metricsTracer.capturedAttributes.get(BuiltInMetricsConstant.ENDPOINT_KEY.getKey())); + + @SuppressWarnings("unchecked") + RecordingClientCall firstDelegate = + (RecordingClientCall) + harness.defaultManagedChannel.latestCall(); + firstDelegate.emitOnMessage( + ResultSet.newBuilder() + .setCacheUpdate( + CacheUpdate.newBuilder() + .setDatabaseId(7L) + .addRange( + Range.newBuilder() + .setStartKey(bytes("a")) + .setLimitKey(bytes("z")) + .setGroupUid(9L) + .setSplitId(1L) + .setGeneration(bytes("1"))) + .addGroup( + Group.newBuilder() + .setGroupUid(9L) + .setGeneration(bytes("1")) + .addTablets( + Tablet.newBuilder() + .setTabletUid(3L) + .setServerAddress("routed:1234") + .setIncarnation(bytes("1")) + .setDistance(0))) + .build()) + .build()); + harness.channel.awaitPendingCacheUpdates(); + + // Retry with the same operation-scoped tracer: now routed to the resolved endpoint, and the + // endpoint attribute is overwritten with the endpoint this attempt actually used. + ClientCall secondCall = + harness.channel.newCall(SpannerGrpc.getExecuteSqlMethod(), callOptions); + secondCall.start(new CapturingListener(), new Metadata()); + secondCall.sendMessage(request); + + assertEquals( + "routed:1234", + metricsTracer.capturedAttributes.get(BuiltInMetricsConstant.ENDPOINT_KEY.getKey())); + } + + @Test + public void sendMessageWithoutCompositeTracerDoesNotFail() throws Exception { + TestHarness harness = createHarness(); + + // No tracer at all. + ClientCall call = + harness.channel.newCall(SpannerGrpc.getExecuteSqlMethod(), CallOptions.DEFAULT); + call.start(new CapturingListener(), new Metadata()); + call.sendMessage(ExecuteSqlRequest.newBuilder().setSession(SESSION).build()); + assertThat(harness.defaultManagedChannel.callCount()).isEqualTo(1); + + // A tracer that is not a CompositeTracer. + ClientCall callWithPlainTracer = + harness.channel.newCall( + SpannerGrpc.getExecuteSqlMethod(), + CallOptions.DEFAULT.withOption(GrpcCallContext.TRACER_KEY, new BaseApiTracer() {})); + callWithPlainTracer.start(new CapturingListener(), new Metadata()); + callWithPlainTracer.sendMessage(ExecuteSqlRequest.newBuilder().setSession(SESSION).build()); + assertThat(harness.defaultManagedChannel.callCount()).isEqualTo(2); + } }