[codex] Trace first-party relay clients

.github/workflows/release.yml

@@ -179,13 +179,36 @@ jobs:
       clerk_jwt_template: ${{ steps.public_config.outputs.clerk_jwt_template }}
       clerk_cli_oauth_client_id: ${{ steps.public_config.outputs.clerk_cli_oauth_client_id }}
       relay_url: ${{ steps.public_config.outputs.relay_url }}
+      client_tracing_url: ${{ steps.relay_state.outputs.client_tracing_url }}
+      client_tracing_dataset: ${{ steps.relay_state.outputs.client_tracing_dataset }}
+      client_tracing_token: ${{ steps.relay_state.outputs.client_tracing_token }}
     env:
+      CLOUDFLARE_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }}
+      CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
       RELAY_DOMAIN: ${{ vars.RELAY_DOMAIN }}
       RELAY_API_ZONE_NAME: ${{ vars.RELAY_API_ZONE_NAME }}
       CLERK_PUBLISHABLE_KEY: ${{ vars.CLERK_PUBLISHABLE_KEY }}
       CLERK_JWT_TEMPLATE: ${{ vars.CLERK_JWT_TEMPLATE }}
       CLERK_CLI_OAUTH_CLIENT_ID: ${{ vars.CLERK_CLI_OAUTH_CLIENT_ID }}
     steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ needs.preflight.outputs.ref }}
+
+      - name: Setup Vite+
+        uses: voidzero-dev/setup-vp@v1
+        with:
+          node-version-file: package.json
+          cache: true
+          run-install: |
+            args:
+              - --filter=t3code-relay...
+
+      - id: relay_state
+        name: Read production relay tracing config
+        run: vp run --filter t3code-relay deploy --stage prod --read-state --github-output
+
       - id: public_config
         name: Resolve production relay public config
         shell: bash
@@ -229,6 +252,9 @@ jobs:
       T3CODE_CLERK_JWT_TEMPLATE: ${{ needs.relay_public_config.outputs.clerk_jwt_template }}
       T3CODE_CLERK_CLI_OAUTH_CLIENT_ID: ${{ needs.relay_public_config.outputs.clerk_cli_oauth_client_id }}
       T3CODE_RELAY_URL: ${{ needs.relay_public_config.outputs.relay_url }}
+      T3CODE_RELAY_CLIENT_OTLP_TRACES_URL: ${{ needs.relay_public_config.outputs.client_tracing_url }}
+      T3CODE_RELAY_CLIENT_OTLP_TRACES_DATASET: ${{ needs.relay_public_config.outputs.client_tracing_dataset }}
+      T3CODE_RELAY_CLIENT_OTLP_TRACES_TOKEN: ${{ needs.relay_public_config.outputs.client_tracing_token }}
     strategy:
       fail-fast: false
       matrix:
@@ -491,6 +517,9 @@ jobs:
       T3CODE_CLERK_JWT_TEMPLATE: ${{ needs.relay_public_config.outputs.clerk_jwt_template }}
       T3CODE_CLERK_CLI_OAUTH_CLIENT_ID: ${{ needs.relay_public_config.outputs.clerk_cli_oauth_client_id }}
       T3CODE_RELAY_URL: ${{ needs.relay_public_config.outputs.relay_url }}
+      T3CODE_RELAY_CLIENT_OTLP_TRACES_URL: ${{ needs.relay_public_config.outputs.client_tracing_url }}
+      T3CODE_RELAY_CLIENT_OTLP_TRACES_DATASET: ${{ needs.relay_public_config.outputs.client_tracing_dataset }}
+      T3CODE_RELAY_CLIENT_OTLP_TRACES_TOKEN: ${{ needs.relay_public_config.outputs.client_tracing_token }}
     steps:
       - name: Checkout
         uses: actions/checkout@v6
@@ -647,6 +676,9 @@ jobs:
       T3CODE_CLERK_PUBLISHABLE_KEY: ${{ needs.relay_public_config.outputs.clerk_publishable_key }}
       T3CODE_CLERK_JWT_TEMPLATE: ${{ needs.relay_public_config.outputs.clerk_jwt_template }}
       T3CODE_RELAY_URL: ${{ needs.relay_public_config.outputs.relay_url }}
+      T3CODE_RELAY_CLIENT_OTLP_TRACES_URL: ${{ needs.relay_public_config.outputs.client_tracing_url }}
+      T3CODE_RELAY_CLIENT_OTLP_TRACES_DATASET: ${{ needs.relay_public_config.outputs.client_tracing_dataset }}
+      T3CODE_RELAY_CLIENT_OTLP_TRACES_TOKEN: ${{ needs.relay_public_config.outputs.client_tracing_token }}
       VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }}
       VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }}
       VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }}
@@ -716,6 +748,9 @@ jobs:
               --build-env "T3CODE_CLERK_PUBLISHABLE_KEY=${T3CODE_CLERK_PUBLISHABLE_KEY:-}" \
               --build-env "T3CODE_CLERK_JWT_TEMPLATE=${T3CODE_CLERK_JWT_TEMPLATE:-}" \
               --build-env "T3CODE_RELAY_URL=${T3CODE_RELAY_URL:-}" \
+              --build-env "T3CODE_RELAY_CLIENT_OTLP_TRACES_URL=${T3CODE_RELAY_CLIENT_OTLP_TRACES_URL:-}" \
+              --build-env "T3CODE_RELAY_CLIENT_OTLP_TRACES_DATASET=${T3CODE_RELAY_CLIENT_OTLP_TRACES_DATASET:-}" \
+              --build-env "T3CODE_RELAY_CLIENT_OTLP_TRACES_TOKEN=${T3CODE_RELAY_CLIENT_OTLP_TRACES_TOKEN:-}" \
               --build-env "VITE_HOSTED_APP_URL=$router_url" \
               --build-env "VITE_HOSTED_APP_CHANNEL=$channel_name"
           )"

apps/mobile/src/features/observability/mobileTracing.test.ts

@@ -4,6 +4,7 @@ import * as Layer from "effect/Layer";
 import { vi } from "vite-plus/test";
 
 import { remoteHttpClientLayer } from "@t3tools/client-runtime";
+import { withRelayClientTracing } from "@t3tools/shared/relayTracing";
 
 import { makeMobileTracingLayer } from "./mobileTracing";
 
@@ -29,7 +30,7 @@ it.effect("exports spans through the scoped mobile OTLP layer", () => {
     },
   ).pipe(Layer.provide(remoteHttpClientLayer(fetchFn)));
   const tracedApplication = Layer.effectDiscard(
-    Effect.void.pipe(Effect.withSpan("mobile.test.span")),
+    Effect.void.pipe(Effect.withSpan("mobile.test.span"), withRelayClientTracing),
   ).pipe(Layer.provide(tracingLayer));
 
   return Effect.gen(function* () {

apps/mobile/src/features/observability/mobileTracing.ts

@@ -1,7 +1,5 @@
 import Constants from "expo-constants";
-import * as Layer from "effect/Layer";
-import type { HttpClient } from "effect/unstable/http";
-import { OtlpSerialization, OtlpTracer } from "effect/unstable/observability";
+import { makeRelayClientTracingLayer } from "@t3tools/shared/relayTracing";
 
 import { hasMobileTracingPublicConfig, resolveCloudPublicConfig } from "../cloud/publicConfig";
 
@@ -28,27 +26,13 @@ export function resolveMobileTracingConfig(): MobileTracingConfig | null {
 export function makeMobileTracingLayer(
   config: MobileTracingConfig | null,
   resource: MobileTracingResource,
-): Layer.Layer<never, never, HttpClient.HttpClient> {
-  if (config === null) {
-    return Layer.empty;
-  }
-
-  return OtlpTracer.layer({
-    url: config.tracesUrl,
-    headers: {
-      Authorization: `Bearer ${config.tracesToken}`,
-      "X-Axiom-Dataset": config.tracesDataset,
-    },
-    resource: {
-      serviceName: "t3-mobile",
-      serviceVersion: resource.serviceVersion,
-      attributes: {
-        "service.runtime": "react-native",
-        "service.component": "mobile",
-        "deployment.environment.name": resource.appVariant,
-      },
-    },
-  }).pipe(Layer.provide(OtlpSerialization.layerJson));
+) {
+  return makeRelayClientTracingLayer(config, {
+    serviceName: "t3-mobile-relay-client",
+    serviceVersion: resource.serviceVersion,
+    runtime: "react-native",
+    client: `mobile-${resource.appVariant}`,
+  });
 }
 
 export const mobileTracingLayer = makeMobileTracingLayer(resolveMobileTracingConfig(), {

apps/server/src/cli/cloud.ts

@@ -6,6 +6,7 @@ import {
 } from "@t3tools/contracts";
 import { RelayOkResponse } from "@t3tools/contracts/relay";
 import * as RelayClient from "@t3tools/shared/relayClient";
+import { withRelayClientTracing } from "@t3tools/shared/relayTracing";
 import * as Console from "effect/Console";
 import * as Duration from "effect/Duration";
 import * as Effect from "effect/Effect";
@@ -29,6 +30,7 @@ import * as CliState from "../cloud/CliState.ts";
 import * as CliTokenManager from "../cloud/CliTokenManager.ts";
 import { CLOUD_LINKED_USER_ID, RELAY_URL_SECRET } from "../cloud/config.ts";
 import { relayUrlConfig } from "../cloud/publicConfig.ts";
+import { headlessRelayClientTracingLayer } from "../cloud/relayTracing.ts";
 import { ServerConfig } from "../config.ts";
 import { ServerEnvironmentLive } from "../environment/Layers/ServerEnvironment.ts";
 import { ServerEnvironment } from "../environment/Services/ServerEnvironment.ts";
@@ -228,6 +230,7 @@ const unlinkRelayEnvironment = Effect.fn("cloud.cli.unlink_relay_environment")(f
     httpClient.execute,
     Effect.flatMap(HttpClientResponse.filterStatusOk),
     Effect.flatMap(HttpClientResponse.schemaBodyJson(RelayOkResponse)),
+    withRelayClientTracing,
   );
   return response.ok
     ? ({ status: "revoked" } satisfies RelayUnlinkResult)
@@ -299,6 +302,7 @@ const runCloudCommand = <A, E>(
       RelayClient.layerCloudflared({ baseDir: config.baseDir }),
       EnvironmentAuth.runtimeLayer,
       ServerEnvironmentLive,
+      headlessRelayClientTracingLayer,
     ).pipe(
       Layer.provideMerge(FetchHttpClient.layer),
       Layer.provideMerge(Layer.succeed(ServerConfig, config)),

apps/server/src/cloud/http.test.ts

@@ -3,13 +3,19 @@ import { describe, expect, it } from "@effect/vitest";
 import * as Effect from "effect/Effect";
 import * as Option from "effect/Option";
 import * as PlatformError from "effect/PlatformError";
-import { HttpClient } from "effect/unstable/http";
+import * as Tracer from "effect/Tracer";
+import { HttpClient, HttpServerRequest } from "effect/unstable/http";
 
+import { RelayClientTracer } from "@t3tools/shared/relayTracing";
 import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts";
 import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
 import { ServerEnvironment } from "../environment/Services/ServerEnvironment.ts";
 import * as CliTokenManager from "./CliTokenManager.ts";
-import { consumeCloudReplayGuards, reconcileDesiredCloudLink } from "./http.ts";
+import {
+  consumeCloudReplayGuards,
+  reconcileDesiredCloudLink,
+  traceRelayBrokerHandler,
+} from "./http.ts";
 import {
   CloudManagedEndpointRuntime,
   type CloudManagedEndpointRuntimeShape,
@@ -69,6 +75,38 @@ describe("consumeCloudReplayGuards", () => {
   );
 });
 
+describe("traceRelayBrokerHandler", () => {
+  it.effect("continues the incoming relay trace with the product tracer", () =>
+    Effect.gen(function* () {
+      const spans: Array<Tracer.Span> = [];
+      const productTracer = Tracer.make({
+        span: (options) => {
+          const span = new Tracer.NativeSpan(options);
+          spans.push(span);
+          return span;
+        },
+      });
+      const request = HttpServerRequest.fromWeb(
+        new Request("https://environment.example.test/api/t3-cloud/mint-credential", {
+          headers: {
+            traceparent: "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01",
+          },
+        }),
+      );
+
+      yield* traceRelayBrokerHandler(Effect.void.pipe(Effect.withSpan("relay.mint.handler"))).pipe(
+        Effect.provideService(HttpServerRequest.HttpServerRequest, request),
+        Effect.provideService(RelayClientTracer, Option.some(productTracer)),
+      );
+
+      expect(spans).toHaveLength(1);
+      const span = spans[0]!;
+      expect(span.traceId).toBe("0123456789abcdef0123456789abcdef");
+      expect(Option.getOrUndefined(span.parent)?.spanId).toBe("0123456789abcdef");
+    }),
+  );
+});
+
 describe("reconcileDesiredCloudLink", () => {
   it.effect("requires stored CLI authorization without exposing an HTTP endpoint", () =>
     Effect.gen(function* () {

apps/server/src/cloud/http.ts

@@ -29,6 +29,7 @@ import {
   RelayLinkProofRequest,
   RelayManagedEndpointOrigin,
 } from "@t3tools/contracts/relay";
+import { withRelayClientTracing } from "@t3tools/shared/relayTracing";
 import {
   normalizeRelayIssuer,
   RELAY_HEALTH_REQUEST_TYP,
@@ -47,7 +48,7 @@ import * as Effect from "effect/Effect";
 import * as Option from "effect/Option";
 import * as Schema from "effect/Schema";
 import * as HttpEffect from "effect/unstable/http/HttpEffect";
-import { HttpServerRequest, HttpServerResponse } from "effect/unstable/http";
+import { HttpServerRequest, HttpServerResponse, HttpTraceContext } from "effect/unstable/http";
 import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http";
 import * as HttpApiBuilder from "effect/unstable/httpapi/HttpApiBuilder";
 
@@ -110,6 +111,19 @@ const requireRelayUrl = relayUrlConfig.pipe(
   ),
 );
 
+export const traceRelayBrokerHandler = <A, E, R>(
+  effect: Effect.Effect<A, E, R>,
+): Effect.Effect<A, E, R | HttpServerRequest.HttpServerRequest> =>
+  HttpServerRequest.HttpServerRequest.pipe(
+    Effect.flatMap((request) =>
+      Option.match(HttpTraceContext.fromHeaders(request.headers), {
+        onNone: () => effect,
+        onSome: (parent) => effect.pipe(Effect.withParentSpan(parent)),
+      }),
+    ),
+    withRelayClientTracing,
+  );
+
 function bytesToString(bytes: Uint8Array): string {
   return new TextDecoder().decode(bytes);
 }
@@ -512,6 +526,7 @@ const relayClientRequest = <A>(
           message: `T3 Cloud relay request failed: ${String(cause)}`,
         }),
     ),
+    withRelayClientTracing,
   );
 
 const reconcileDesiredCloudLinkWith = Effect.fn("environment.cloud.reconcileDesiredLinkWith")(
@@ -938,7 +953,7 @@ export const cloudHttpApiLayer = HttpApiBuilder.group(
       .handle("health", ({ payload }) => cloudEnvironmentHealthHandler(dependencies, payload))
       .handle("mintCredential", ({ payload }) => cloudMintCredentialHandler(dependencies, payload))
       .handle("t3MintCredential", ({ payload }) =>
-        cloudMintCredentialHandler(dependencies, payload),
+        traceRelayBrokerHandler(cloudMintCredentialHandler(dependencies, payload)),
       );
   }),
 );

apps/server/src/cloud/publicConfig.test.ts

@@ -2,7 +2,11 @@ import { assert, it } from "@effect/vitest";
 import * as ConfigProvider from "effect/ConfigProvider";
 import * as Effect from "effect/Effect";
 
-import { makeCloudCliOAuthConfig, makeRelayUrlConfig } from "./publicConfig.ts";
+import {
+  makeCloudCliOAuthConfig,
+  makeRelayUrlConfig,
+  resolveRelayClientTracingConfig,
+} from "./publicConfig.ts";
 
 const provideEnv = (env: Readonly<Record<string, string>>) =>
   Effect.provide(ConfigProvider.layer(ConfigProvider.fromEnv({ env })));
@@ -83,3 +87,39 @@ it.effect("requires Clerk OAuth config when the server bundle has no injected va
     clerkCliOAuthClientIdFallback: "",
   }).pipe(provideEnv({}), Effect.flip),
 );
+
+it("resolves relay client tracing from runtime config with build-time fallback", () => {
+  const fallback = {
+    tracesUrl: "https://embedded.example.test/v1/traces",
+    tracesDataset: "embedded-dataset",
+    tracesToken: "embedded-token",
+  };
+
+  assert.deepEqual(resolveRelayClientTracingConfig({}, fallback), fallback);
+  assert.deepEqual(
+    resolveRelayClientTracingConfig(
+      {
+        T3CODE_RELAY_CLIENT_OTLP_TRACES_URL: "https://runtime.example.test/v1/traces",
+        T3CODE_RELAY_CLIENT_OTLP_TRACES_DATASET: "runtime-dataset",
+        T3CODE_RELAY_CLIENT_OTLP_TRACES_TOKEN: "runtime-token",
+      },
+      fallback,
+    ),
+    {
+      tracesUrl: "https://runtime.example.test/v1/traces",
+      tracesDataset: "runtime-dataset",
+      tracesToken: "runtime-token",
+    },
+  );
+  assert.equal(
+    resolveRelayClientTracingConfig(
+      {
+        T3CODE_RELAY_CLIENT_OTLP_TRACES_URL: "http://insecure.example.test/v1/traces",
+        T3CODE_RELAY_CLIENT_OTLP_TRACES_DATASET: "runtime-dataset",
+        T3CODE_RELAY_CLIENT_OTLP_TRACES_TOKEN: "runtime-token",
+      },
+      fallback,
+    ),
+    null,
+  );
+});