feat(dev): collect local OTEL traces in project dev

package.json

@@ -58,6 +58,7 @@
     "@aws-sdk/client-iam": "^3.1080.0",
     "@opentelemetry/api": "^1.9.1",
     "@opentelemetry/exporter-metrics-otlp-http": "^0.221.0",
+    "@opentelemetry/otlp-transformer": "0.213.0",
     "@opentelemetry/resources": "^2.10.0",
     "@opentelemetry/sdk-metrics": "^2.10.0",
     "@smithy/core": "3.29.3",
@@ -75,5 +76,8 @@
     "winston": "^3.19.0",
     "winston-daily-rotate-file": "^5.0.0",
     "zod": "^4.4.3"
+  },
+  "overrides": {
+    "@opentelemetry/core": "^2.10.0"
   }
 }

src/assets/templates/hello-world-python-container/Dockerfile

@@ -34,4 +34,6 @@ USER bedrock_agentcore
 # 9000: A2A Mode
 EXPOSE 8080 8000 9000
 
-CMD ["python", "-m", "main"]
+# opentelemetry-instrument (from aws-opentelemetry-distro) starts a real
+# TracerProvider; plain `python -m main` would export nothing.
+CMD ["opentelemetry-instrument", "python", "-m", "main"]

src/assets/templates/shared/env.local.template

@@ -1,7 +1,11 @@
 # Environment variables for local development.
 # `agentcore project dev` loads this file into your agent's process. Values here
 # override injected values except PORT, FASTMCP_PORT, and LOCAL_DEV, which the
-# CLI owns. This file is gitignored — keep secrets out of version control.
+# CLI owns. While trace collection is on (the default), the CLI also owns the
+# OTEL_* and AGENT_OBSERVABILITY_ENABLED variables so traces reach its local
+# collector — pass --no-traces (or set instrumentation.enableOtel to false in
+# agentcore.json) to disable collection and set your own.
+# This file is gitignored — keep secrets out of version control.
 #
 # Example:
 # MY_API_KEY=...

src/core/dev/codezip.test.ts

@@ -1,11 +1,11 @@
 import { afterEach, describe, expect, test } from "bun:test";
 import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
-import { join, relative } from "node:path";
+import { delimiter, join, relative } from "node:path";
 import { InputValidationError } from "../../errors";
 import type { ProjectRuntime } from "../../projectSchemas/runtime";
 import type { DevEvent, DevServerInput } from "../../handlers/project/dev/types";
-import type { ProcessEvent, ProcessStreamer, StreamProcessOptions } from "../../io";
+import type { ProcessEvent, ProcessRunner, ProcessStreamer, StreamProcessOptions } from "../../io";
 import { CodeZipDevRunner } from "./codezip";
 
 type ProcessCall = {
@@ -54,15 +54,26 @@ async function projectRoot(withNodeModules = false): Promise<string> {
   return root;
 }
 
-function harness(output: ProcessEvent[] = []) {
+function harness(
+  output: ProcessEvent[] = [],
+  site: { dir?: string; fail?: boolean; noise?: string[] } = {},
+) {
   const calls: ProcessCall[] = [];
+  const discoverCalls: string[][] = [];
   const fakeStreamProcess: ProcessStreamer = async function* (command, options) {
     calls.push({ command, options });
     yield* output;
   };
+  const fakeRunProcess: ProcessRunner = async (command, options) => {
+    discoverCalls.push(command);
+    if (site.fail) throw new Error("discovery failed");
+    for (const line of site.noise ?? []) options.onOutput?.(`${line}\n`);
+    if (site.dir !== undefined) options.onOutput?.(`AGENTCORE_OTEL_SITECUSTOMIZE=${site.dir}\n`);
+  };
   return {
     calls,
-    runner: new CodeZipDevRunner({ streamProcess: fakeStreamProcess }),
+    discoverCalls,
+    runner: new CodeZipDevRunner({ streamProcess: fakeStreamProcess, runProcess: fakeRunProcess }),
   };
 }
 
@@ -193,3 +204,82 @@ describe("CodeZipDevRunner", () => {
     ]);
   });
 });
+
+describe("CodeZipDevRunner OTEL instrumentation", () => {
+  async function sitecustomizeDir(): Promise<string> {
+    const directory = await mkdtemp(join(tmpdir(), "otel-site-"));
+    tempDirectories.push(directory);
+    await writeFile(join(directory, "sitecustomize.py"), "");
+    return directory;
+  }
+
+  function otelInput(root: string, extraEnv: Record<string, string> = {}): DevServerInput {
+    const base = input(root, runtime());
+    return {
+      ...base,
+      env: { ...base.env, OTEL_EXPORTER_OTLP_ENDPOINT: "http://127.0.0.1:4318", ...extraEnv },
+    };
+  }
+
+  test("prepends the sitecustomize directory to PYTHONPATH when instrumentation is installed", async () => {
+    const root = await projectRoot();
+    const directory = await sitecustomizeDir();
+    const { calls, discoverCalls, runner } = harness([], { dir: directory });
+
+    await collect(runner.run(otelInput(root)));
+
+    expect(discoverCalls[0]?.slice(0, 4)).toEqual(["uv", "run", "python", "-c"]);
+    expect(calls[0]?.options.env?.PYTHONPATH).toBe(directory);
+  });
+
+  test("reads the marked path even when uv writes progress to the merged output", async () => {
+    const root = await projectRoot();
+    const directory = await sitecustomizeDir();
+    const { calls, runner } = harness([], {
+      dir: directory,
+      noise: ["Resolved 12 packages", "Installed 12 packages"],
+    });
+
+    await collect(runner.run(otelInput(root)));
+
+    expect(calls[0]?.options.env?.PYTHONPATH).toBe(directory);
+  });
+
+  test("preserves an existing PYTHONPATH", async () => {
+    const root = await projectRoot();
+    const directory = await sitecustomizeDir();
+    const { calls, runner } = harness([], { dir: directory });
+
+    await collect(runner.run(otelInput(root, { PYTHONPATH: "/existing" })));
+
+    expect(calls[0]?.options.env?.PYTHONPATH).toBe(`${directory}${delimiter}/existing`);
+  });
+
+  test("does not run discovery without an OTEL endpoint or for Node entrypoints", async () => {
+    const root = await projectRoot(true);
+    const { discoverCalls, runner } = harness();
+
+    await collect(runner.run(input(root, runtime())));
+    await collect(runner.run({ ...otelInput(root), runtime: runtime({ entrypoint: "index.js" }) }));
+
+    expect(discoverCalls).toEqual([]);
+  });
+
+  test.each([
+    ["discovery failure", { fail: true }],
+    ["missing sitecustomize.py", { dir: "/nonexistent" }],
+  ] as const)("warns and starts untraced on %s", async (_case, site) => {
+    const root = await projectRoot();
+    const { calls, discoverCalls, runner } = harness([], site);
+
+    const events = await collect(runner.run(otelInput(root)));
+
+    expect(discoverCalls).toHaveLength(1);
+    expect(calls).toHaveLength(1);
+    expect(calls[0]?.options.env?.PYTHONPATH).toBeUndefined();
+    expect(events).toContainEqual({
+      type: "status",
+      message: expect.stringContaining("traces will not be collected"),
+    });
+  });
+});

src/core/dev/codezip.ts

@@ -1,19 +1,30 @@
 import { existsSync } from "node:fs";
-import { join, resolve } from "node:path";
+import { delimiter, join, resolve } from "node:path";
 import { InputValidationError } from "../../errors";
 import type { DevEvent, DevRunner, DevServerInput } from "../../handlers/project/dev/types";
-import { streamProcess, type ProcessStreamer, type StreamProcessOptions } from "../../io";
+import {
+  runProcess,
+  streamProcess,
+  type ProcessRunner,
+  type ProcessStreamer,
+  type StreamProcessOptions,
+} from "../../io";
 import { isDirectory, isFile, resolvePathWithinProject } from "./path";
 
 type CodeZipDevRunnerConfig = {
   streamProcess?: ProcessStreamer;
+  runProcess?: ProcessRunner;
 };
 
+const SITECUSTOMIZE_MARKER = "AGENTCORE_OTEL_SITECUSTOMIZE=";
+
 export class CodeZipDevRunner implements DevRunner {
   private readonly streamProcess: ProcessStreamer;
+  private readonly runProcess: ProcessRunner;
 
   constructor(config: CodeZipDevRunnerConfig = {}) {
     this.streamProcess = config.streamProcess ?? streamProcess;
+    this.runProcess = config.runProcess ?? runProcess;
   }
 
   public async *run(input: DevServerInput): AsyncGenerator<DevEvent, void> {
@@ -41,8 +52,58 @@ export class CodeZipDevRunner implements DevRunner {
 
     yield { type: "status", message: "Starting development server" };
     const serverProcess = commandForRuntime(entrypoint!, directory, input);
+    if (entrypoint!.endsWith(".py") && input.env?.OTEL_EXPORTER_OTLP_ENDPOINT) {
+      const sitecustomizeDir = await this.findOtelSitecustomizeDir(directory, input.signal);
+      if (sitecustomizeDir) {
+        const existing = serverProcess.options.env?.PYTHONPATH;
+        serverProcess.options.env = {
+          ...serverProcess.options.env,
+          PYTHONPATH: existing ? `${sitecustomizeDir}${delimiter}${existing}` : sitecustomizeDir,
+        };
+      } else {
+        yield {
+          type: "status",
+          message:
+            "OTEL auto-instrumentation is not installed in the agent environment; traces will not be collected. Add aws-opentelemetry-distro to the agent's dependencies to enable them.",
+        };
+      }
+    }
     yield* this.streamProcess(serverProcess.command, serverProcess.options);
   }
+
+  /**
+   * Locate the auto-instrumentation sitecustomize.py directory inside the agent's
+   * uv environment. Prepending it to PYTHONPATH instruments every Python process —
+   * an `opentelemetry-instrument` wrapper would only instrument uvicorn's reloader
+   * parent, leaving the re-spawned worker processes untraced.
+   */
+  private async findOtelSitecustomizeDir(
+    directory: string,
+    signal: AbortSignal,
+  ): Promise<string | undefined> {
+    const output: string[] = [];
+    // uv writes sync progress to stderr, which merges into onOutput, so the path
+    // is printed behind a marker and read from that line rather than the last one.
+    const script = `import opentelemetry.instrumentation.auto_instrumentation as m, os; print("${SITECUSTOMIZE_MARKER}" + os.path.dirname(m.__file__))`;
+    try {
+      await this.runProcess(["uv", "run", "python", "-c", script], {
+        cwd: directory,
+        onOutput: (chunk) => output.push(chunk),
+        signal,
+      });
+    } catch {
+      return undefined;
+    }
+    const marked = output
+      .join("")
+      .split("\n")
+      .map((line) => line.trim())
+      .find((line) => line.startsWith(SITECUSTOMIZE_MARKER));
+    const sitecustomizeDir = marked?.slice(SITECUSTOMIZE_MARKER.length);
+    if (!sitecustomizeDir || !existsSync(join(sitecustomizeDir, "sitecustomize.py")))
+      return undefined;
+    return sitecustomizeDir;
+  }
 }
 
 function commandForRuntime(

src/core/dev/container.test.ts

@@ -207,6 +207,8 @@ describe("ContainerDevRunner", () => {
       containerName(root),
       "-p",
       `127.0.0.1:3000:${containerPort}`,
+      "--add-host",
+      "host.docker.internal:host-gateway",
       "--env-file",
       run.envFile!.path,
       imageTag(root),

src/core/dev/container.ts

@@ -183,6 +183,11 @@ export class ContainerDevRunner implements DevRunner {
       containerName,
       "-p",
       `127.0.0.1:${input.port}:${containerPort}`,
+      // Docker Engine on Linux does not define host.docker.internal (Desktop,
+      // Podman, and Finch do); the mapping makes the OTLP endpoint rewrite
+      // resolve everywhere and is harmless where the name already exists.
+      "--add-host",
+      "host.docker.internal:host-gateway",
       ...awsMount,
       "--env-file",
       envFile,