feat(bqjdbc): implement OTel SDK and exporting traces and logs to gcp (#13039)

b/491238299
b/511147053

This PR completes the implementation of the OpenTelemetry SDK lifecycle
and cross-project authentication for the BigQuery JDBC driver. It
introduces thread-safe caching of heavy OTel SDK instances to support
multi-project tracing without global side effects.

### Changes

#### `BigQueryJdbcOpenTelemetry.java`
- A `ConcurrentHashMap` caches `OpenTelemetrySdk` instances, keyed by a
concatenated string of `ProjectId` and `Credentials`
- The `getOpenTelemetry()` method lazily loads and initializes the SDK
only when requested and not present in the cache.
- A JVM shutdown hook closes each cached SDK to ensure pending traces
are flushed on application exit.

#### `BigQueryJdbcUrlUtility.java`
- `gcpTelemetryCredentials` and `gcpTelemetryProjectId` are added to
connection properties.

#### `BigQueryJdbcOAuthUtility.java`
- The `isJson()` helper method is changed from private to
package-private to allow reuse in `BigQueryJdbcOpenTelemetry.`

#### `BigQueryConnection.java`
- Added new connection properties to be used
- Uses `Boolean.TRUE.equals()` when checking `enableGcpLogExporter` and
`enableGcpTraceExporter` to safely handle cases where these properties
are not specified `null` and avoid `NullPointerException` during
auto-unboxing.

#### `BigQueryJdbcOpenTelemetryTest.java`
- New unit tests verify that SDK instances are correctly cached for
identical keys and isolated for different keys.

#### `BigQueryArrowStructTest.java`
- A fix is applied to avoid varargs ambiguity in array creation,
resolving a specific coercion failure in the structOfArrays test.

Author: keshavdandeva

java-bigquery/google-cloud-bigquery-jdbc/pom.xml

@@ -185,7 +185,7 @@
     <dependency>
       <groupId>com.google.cloud</groupId>
       <artifactId>google-cloud-logging</artifactId>
-      <version>3.33.0-SNAPSHOT</version><!-- {x-version-update:google-cloud-logging:current} -->
+      <version>3.32.0</version><!-- {x-version-update:google-cloud-logging:current} -->
     </dependency>
     <dependency>
       <groupId>com.google.http-client</groupId>
@@ -301,6 +301,24 @@
       <groupId>io.opentelemetry</groupId>
       <artifactId>opentelemetry-context</artifactId>
     </dependency>
+    <!-- OpenTelemetry SDK and Exporters (will be shaded via existing 'io' relocation) -->
+    <dependency>
+      <groupId>io.opentelemetry</groupId>
+      <artifactId>opentelemetry-sdk</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.opentelemetry</groupId>
+      <artifactId>opentelemetry-exporter-otlp</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.opentelemetry</groupId>
+      <artifactId>opentelemetry-sdk-extension-autoconfigure</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.opentelemetry.contrib</groupId>
+      <artifactId>opentelemetry-gcp-auth-extension</artifactId>
+      <version>1.56.0-alpha</version>
+    </dependency>
 
     <!--  Test Dependencies  -->
     <dependency>

java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java

@@ -41,6 +41,7 @@
 import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient;
 import com.google.cloud.bigquery.storage.v1.BigQueryWriteSettings;
 import com.google.cloud.http.HttpTransportOptions;
+import com.google.cloud.logging.Logging;
 import io.opentelemetry.api.OpenTelemetry;
 import io.opentelemetry.api.trace.Tracer;
 import java.io.IOException;
@@ -88,6 +89,8 @@ public class BigQueryConnection extends BigQueryNoOpsConnection {
   int transactionIsolation;
   List<SQLWarning> sqlWarnings;
   String catalog;
+  String gcpTelemetryCredentials;
+  String gcpTelemetryProjectId;
   int holdability;
   long retryTimeoutInSeconds;
   Duration retryTimeoutDuration;
@@ -142,9 +145,10 @@ public class BigQueryConnection extends BigQueryNoOpsConnection {
   Long connectionPoolSize;
   Long listenerPoolSize;
   String partnerToken;
-  Boolean enableGcpTraceExporter;
-  Boolean enableGcpLogExporter;
+  boolean enableGcpTraceExporter;
+  boolean enableGcpLogExporter;
   OpenTelemetry customOpenTelemetry;
+  private OpenTelemetry openTelemetry;
   Tracer tracer =
       OpenTelemetry.noop().getTracer(BigQueryJdbcOpenTelemetry.INSTRUMENTATION_SCOPE_NAME);
   DatabaseMetaData databaseMetaData;
@@ -169,6 +173,8 @@ public class BigQueryConnection extends BigQueryNoOpsConnection {
 
       this.labels = ds.getLabels() != null ? ds.getLabels() : new java.util.HashMap<>();
       this.maxBytesBilled = ds.getMaximumBytesBilled();
+      this.gcpTelemetryCredentials = ds.getGcpTelemetryCredentials();
+      this.gcpTelemetryProjectId = ds.getGcpTelemetryProjectId();
       this.retryTimeoutInSeconds = ds.getTimeout();
       this.retryTimeoutDuration = Duration.ofMillis(retryTimeoutInSeconds * 1000L);
       this.retryInitialDelayInSeconds = ds.getRetryInitialDelay();
@@ -277,6 +283,7 @@ public class BigQueryConnection extends BigQueryNoOpsConnection {
       this.enableGcpTraceExporter = ds.getEnableGcpTraceExporter();
       this.enableGcpLogExporter = ds.getEnableGcpLogExporter();
       this.customOpenTelemetry = ds.getCustomOpenTelemetry();
+      this.openTelemetry = getOpenTelemetryInstance();
       this.bigQuery = getBigQueryConnection();
     }
   }
@@ -961,6 +968,61 @@ void removeStatement(Statement statement) {
     this.openStatements.remove(statement);
   }
 
+  private OpenTelemetry getOpenTelemetryInstance() {
+    boolean hasCustomOtel = this.customOpenTelemetry != null;
+
+    String effectiveProjectId =
+        (this.gcpTelemetryProjectId != null) ? this.gcpTelemetryProjectId : this.catalog;
+    String effectiveCredentials = resolveEffectiveCredentials();
+
+    validateTraceConfiguration(this.enableGcpTraceExporter, effectiveCredentials);
+
+    OpenTelemetry openTelemetry =
+        BigQueryJdbcOpenTelemetry.getOpenTelemetry(
+            this.enableGcpTraceExporter,
+            this.enableGcpLogExporter,
+            this.customOpenTelemetry,
+            effectiveCredentials,
+            effectiveProjectId);
+
+    Logging localLoggingClient = null;
+    if (this.enableGcpLogExporter && !hasCustomOtel) {
+      localLoggingClient =
+          BigQueryJdbcOpenTelemetry.createLoggingClient(
+              true, null, effectiveCredentials, effectiveProjectId, this.credentials);
+    }
+
+    if (this.enableGcpLogExporter || hasCustomOtel) {
+      BigQueryJdbcOpenTelemetry.registerConnection(
+          this.connectionId, openTelemetry, localLoggingClient, this.enableGcpLogExporter);
+    }
+
+    return openTelemetry;
+  }
+
+  private String resolveEffectiveCredentials() {
+    String creds = this.gcpTelemetryCredentials;
+    String authTypeStr = this.authProperties.get(BigQueryJdbcUrlUtility.OAUTH_TYPE_PROPERTY_NAME);
+    if (creds == null
+        && BigQueryJdbcOAuthUtility.AuthType.GOOGLE_SERVICE_ACCOUNT.name().equals(authTypeStr)) {
+      return this.authProperties.get(BigQueryJdbcUrlUtility.OAUTH_PVT_KEY_PROPERTY_NAME);
+    }
+    return creds;
+  }
+
+  private void validateTraceConfiguration(boolean isTraceEnabled, String effectiveCredentials) {
+    if (isTraceEnabled && effectiveCredentials == null) {
+      String authTypeStr = this.authProperties.get(BigQueryJdbcUrlUtility.OAUTH_TYPE_PROPERTY_NAME);
+      if (!BigQueryJdbcOAuthUtility.AuthType.GOOGLE_SERVICE_ACCOUNT.name().equals(authTypeStr)
+          && !BigQueryJdbcOAuthUtility.AuthType.APPLICATION_DEFAULT_CREDENTIALS
+              .name()
+              .equals(authTypeStr)) {
+        throw new BigQueryJdbcRuntimeException(
+            "Exporting traces to Google Cloud is only supported when using Application Default Credentials (ADC) or Service Account authentication.");
+      }
+    }
+  }
+
   private BigQuery getBigQueryConnection() {
     BigQueryOptions.Builder bigQueryOptions = BigQueryOptions.newBuilder();
     if (this.retryTimeoutInSeconds > 0L
@@ -997,18 +1059,8 @@ private BigQuery getBigQueryConnection() {
     if (this.httpTransportOptions != null) {
       bigQueryOptions.setTransportOptions(this.httpTransportOptions);
     }
-
-    OpenTelemetry openTelemetry =
-        BigQueryJdbcOpenTelemetry.getOpenTelemetry(
-            this.enableGcpTraceExporter, this.enableGcpLogExporter, this.customOpenTelemetry);
-
-    if (this.enableGcpLogExporter || this.customOpenTelemetry != null) {
-      BigQueryJdbcOpenTelemetry.registerConnection(
-          this.connectionId, openTelemetry, null, this.enableGcpLogExporter);
-    }
-
-    if (this.enableGcpTraceExporter || this.customOpenTelemetry != null) {
-      this.tracer = BigQueryJdbcOpenTelemetry.getTracer(openTelemetry);
+    if (Boolean.TRUE.equals(this.enableGcpTraceExporter) || this.customOpenTelemetry != null) {
+      this.tracer = BigQueryJdbcOpenTelemetry.getTracer(this.openTelemetry);
       bigQueryOptions.setOpenTelemetryTracer(this.tracer);
     }
 
@@ -1060,11 +1112,8 @@ private BigQueryReadClient getBigQueryReadClientConnection() throws IOException
 
     bigQueryReadSettings.setTransportChannelProvider(activeProvider);
 
-    OpenTelemetry openTelemetry =
-        BigQueryJdbcOpenTelemetry.getOpenTelemetry(
-            this.enableGcpTraceExporter, this.enableGcpLogExporter, this.customOpenTelemetry);
-    if (this.enableGcpTraceExporter || this.customOpenTelemetry != null) {
-      bigQueryReadSettings.setOpenTelemetryTracerProvider(openTelemetry.getTracerProvider());
+    if (Boolean.TRUE.equals(this.enableGcpTraceExporter) || this.customOpenTelemetry != null) {
+      bigQueryReadSettings.setOpenTelemetryTracerProvider(this.openTelemetry.getTracerProvider());
     }
 
     return BigQueryReadClient.create(bigQueryReadSettings.build());

java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOAuthUtility.java

@@ -307,7 +307,7 @@ private static boolean isFileExists(String filename) {
     }
   }
 
-  private static boolean isJson(byte[] value) {
+  static boolean isJson(byte[] value) {
     try {
       // This is done this way to ensure strict Json parsing
       // https://github.com/google/gson/issues/1208#issuecomment-2120764686