diff --git a/.claude/agents/connect-rpc-go.md b/.claude/agents/connect-rpc-go.md index f72c51f..b4575cb 100644 --- a/.claude/agents/connect-rpc-go.md +++ b/.claude/agents/connect-rpc-go.md @@ -983,6 +983,7 @@ package main import ( "connectrpc.com/connect" + "connectrpc.com/grpcreflect" contentv1connect "/gen/proto/content/v1/contentv1connect" contentapi "/internal/api/content" @@ -1002,6 +1003,13 @@ func setupGateway(cfg *config.Config, domains *Domains) connectapp.App { ) application.Handle(path, h) + // gRPC server reflection — enables k6 reflect:true and tools like grpcurl + reflector := grpcreflect.NewStaticReflector( + contentv1connect.ContentServiceName, + ) + application.Handle(grpcreflect.NewHandlerV1(reflector)) + application.Handle(grpcreflect.NewHandlerV1Alpha(reflector)) + return application } ``` @@ -1139,6 +1147,7 @@ services: |---|---| | Go | 1.25.6 | | connectrpc.com/connect | v1.19.1 | +| connectrpc.com/grpcreflect | latest stable | | connectrpc.com/validate | latest stable | | google.golang.org/protobuf | v1.36.11 | | github.com/jackc/pgx/v5 | latest stable | diff --git a/.claude/commands/scaffold-implementation.md b/.claude/commands/scaffold-implementation.md index 010a29f..72f75a8 100644 --- a/.claude/commands/scaffold-implementation.md +++ b/.claude/commands/scaffold-implementation.md @@ -123,7 +123,7 @@ Add a target entry to `benchmark.config.json`: For protobuf targets, adjust: - `protocol`: `"grpc"` -- `k6.env`: use `GRPC_HOST` (set to `localhost:8080`) and `PROTO_DIR` (e.g., `./projects//_shared/protobuf`) instead of `BASE_URL` +- `k6.env`: use `GRPC_HOST` (set to `localhost:8080`) instead of `BASE_URL` ## Step 7 — Verify diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..5aadb66 --- /dev/null +++ b/.env.example @@ -0,0 +1,9 @@ +# Grafana Cloud OTLP credentials (standard OpenTelemetry env vars) +# Find these at: Grafana Cloud portal > Your Stack > Connections > OpenTelemetry (OTLP) +# Generate an API token and copy the two environment variables shown +# +# These env vars serve two purposes: +# 1. Publishing benchmark metrics (build/deploy/loadtest) to Grafana Cloud +# 2. BuildKit automatically picks them up and sends per-stage build traces to Tempo +OTEL_EXPORTER_OTLP_ENDPOINT= +OTEL_EXPORTER_OTLP_HEADERS= diff --git a/README.md b/README.md index 19d1939..0e0f82d 100644 --- a/README.md +++ b/README.md @@ -6,16 +6,36 @@ API benchmark platform for comparing backend service implementations. Define you - [Node.js](https://nodejs.org/) >= 22 - [Docker](https://docs.docker.com/get-docker/) with Compose v2 -- [k6](https://grafana.com/docs/k6/latest/set-up/install-k6/) for load testing - [Claude Code](https://docs.anthropic.com/en/docs/claude-code) for scaffolding implementations - A [Grafana Cloud](https://grafana.com/products/cloud/) account (optional, for publishing results) +k6 load tests run inside Docker (`grafana/k6:1.6.1`) — no local k6 installation required. + ## Setup ```bash npm install ``` +### Grafana Cloud (optional) + +To publish benchmark results to Grafana Cloud, create a `.env` file at the repository root: + +```bash +cp .env.example .env +``` + +Then fill in the standard OpenTelemetry env vars: + +```env +OTEL_EXPORTER_OTLP_ENDPOINT=https://otlp-gateway-prod-us-central-0.grafana.net/otlp +OTEL_EXPORTER_OTLP_HEADERS=Authorization=Basic +``` + +To find these values, sign in to [Grafana Cloud](https://grafana.com), open your stack, and go to **Connections** > **OpenTelemetry (OTLP)**. Generate an API token and copy the two environment variables shown on the page. + +The toolkit uses the official [OpenTelemetry JS SDK](https://opentelemetry.io/docs/languages/js/) to export metrics, so it reads these env vars natively. + ## Quick start ### 1. Define an API @@ -81,7 +101,7 @@ npm run benchmark -- run content-api/spring-boot npm run benchmark -- compare results/content-api-connect-rpc-*.json results/content-api-spring-boot-*.json # publish results to Grafana Cloud -npm run benchmark -- run content-api/connect-rpc --publish +npm run benchmark -- publish results/content-api-connect-rpc-*.json ``` ## Configuration @@ -129,11 +149,6 @@ All targets are defined in `benchmark.config.json` at the repo root. Each target "timeoutMs": 120000 } }, - "grafana": { - "endpoint": "https://otlp-gateway-prod-us-central-0.grafana.net/otlp", - "instanceId": "${GRAFANA_INSTANCE_ID}", - "apiKey": "${GRAFANA_API_KEY}" - }, "output": { "dir": "./results" } } ``` @@ -153,14 +168,12 @@ All targets are defined in `benchmark.config.json` at the repo root. Each target ### Environment variables -Grafana Cloud credentials are resolved from environment variables. Create a `.env` file or export them in your shell: - -```bash -export GRAFANA_INSTANCE_ID=your-instance-id -export GRAFANA_API_KEY=your-api-key -``` +Grafana Cloud credentials are read from the standard `OTEL_EXPORTER_OTLP_*` environment variables when publishing results. The CLI automatically loads a `.env` file from the repository root (see [Setup](#grafana-cloud-optional)). -Values in the config using `${VAR_NAME}` syntax are interpolated from the environment at load time. +| Variable | Description | +| ------------------------------ | -------------------------------------------- | +| `OTEL_EXPORTER_OTLP_ENDPOINT` | Grafana Cloud OTLP gateway URL | +| `OTEL_EXPORTER_OTLP_HEADERS` | Auth header (`Authorization=Basic `) | ## Commands @@ -212,12 +225,16 @@ npm run benchmark -- loadtest my-api npm run benchmark -- loadtest my-api --k6-vus 100 --k6-duration 1m ``` -### `benchmark publish ` +### `benchmark publish ` -Publishes a results JSON file to Grafana Cloud. +Publishes one or more results JSON files to Grafana Cloud. Accepts multiple files so you can batch-publish results from previous runs. ```bash +# publish a single result npm run benchmark -- publish results/my-api-2026-03-04T12-00-00-000Z.json + +# publish multiple results at once +npm run benchmark -- publish results/connect-rpc-*.json results/spring-boot-*.json ``` ### `benchmark compare ` @@ -265,33 +282,52 @@ Time from `docker compose up -d` until the service health check passes. Health i ### Load test (k6) -Parsed from k6's `--summary-export` JSON output: +Parsed from k6's `--summary-export` JSON output. Supports both HTTP (`http_req_duration`) and gRPC (`grpc_req_duration`) protocols: -| Metric | Description | -| --------------------- | ---------------------------------- | -| `httpReqs` | Total HTTP requests | -| `httpReqsPerSec` | Throughput (requests/second) | -| `httpReqDuration.avg` | Average response time (ms) | -| `httpReqDuration.med` | Median / p50 response time (ms) | -| `httpReqDuration.p90` | 90th percentile response time (ms) | -| `httpReqDuration.p95` | 95th percentile response time (ms) | -| `httpReqDuration.p99` | 99th percentile response time (ms) | -| `httpReqFailed` | Error rate (0.0 - 1.0) | -| `checksPassRate` | k6 check pass rate (0.0 - 1.0) | +| Metric | Description | +| ------------------ | ---------------------------------- | +| `reqs` | Total requests (iterations) | +| `reqsPerSec` | Throughput (requests/second) | +| `reqDuration.avg` | Average response time (ms) | +| `reqDuration.med` | Median / p50 response time (ms) | +| `reqDuration.p90` | 90th percentile response time (ms) | +| `reqDuration.p95` | 95th percentile response time (ms) | +| `reqDuration.p99` | 99th percentile response time (ms) | +| `reqFailedRate` | Error rate (0.0 - 1.0) | +| `checksPassRate` | k6 check pass rate (0.0 - 1.0) | ## Grafana Cloud publishing -Results are pushed to Grafana Cloud using the [OTLP/HTTP JSON](https://opentelemetry.io/docs/specs/otlp/) protocol. Each metric is sent as a gauge data point labeled with the target name, tag, and environment info. +Results are pushed to Grafana Cloud using the [OpenTelemetry JS SDK](https://opentelemetry.io/docs/languages/js/) via OTLP/HTTP. Each metric is sent as a gauge data point with the target name, tag, and environment info as resource attributes. The SDK reads `OTEL_EXPORTER_OTLP_ENDPOINT` and `OTEL_EXPORTER_OTLP_HEADERS` from the environment (loaded via `.env`). Metrics appear in Grafana with the `benchmark.*` prefix: - `benchmark.build.duration` - `benchmark.deploy.duration` -- `benchmark.http_reqs_per_sec` -- `benchmark.http_req.duration.{avg,med,p90,p95,p99}` -- `benchmark.http_req_failed_rate` +- `benchmark.reqs_per_sec` +- `benchmark.req.duration.{avg,med,p90,p95,p99}` +- `benchmark.req_failed_rate` - `benchmark.checks_pass_rate` +### Grafana dashboard + +A pre-built dashboard is included at `grafana/benchmark-dashboard.json`. To import it: + +1. In Grafana, go to **Dashboards** > **New** > **Import** +2. Upload `grafana/benchmark-dashboard.json` +3. Select your Prometheus data source +4. Click **Import** + +The dashboard includes: + +- **Overview** — stat panels for throughput, error rate, checks pass rate, build and deploy times +- **Latency Comparison** — bar chart of avg/med/p90/p95/p99 grouped by target +- **Throughput & Volume** — bar gauges for requests/sec and iterations/sec +- **Build & Deploy Comparison** — bar gauges comparing build and deploy times +- **Summary Table** — all metrics in a sortable table + +Use the **Target** and **Tag** dropdowns at the top to filter by implementation and run tag. + ## Custom k6 scripts The toolkit ships with default k6 scripts for HTTP and gRPC APIs in `toolkit/k6/scripts/`. To use a custom script, set the `k6.script` path in your target config: @@ -372,7 +408,7 @@ toolkit/ # the benchmark CLI toolkit config/ # zod schema, loader, defaults core/ # docker, k6, timer, health check metrics/ # result collector, k6 parser - publish/ # OTLP formatter, Grafana Cloud client + publish/ # OpenTelemetry OTLP metrics export report/ # comparison logic, terminal table, JSON writer k6/scripts/ # bundled k6 test scripts projects/ # benchmark target projects @@ -382,6 +418,7 @@ projects/ # benchmark target projects .claude/ commands/ # Claude Code slash commands agents/ # architecture agents for code generation +grafana/ # Grafana dashboard JSON (importable) results/ # benchmark output (gitignored) ``` diff --git a/benchmark.config.json b/benchmark.config.json index 99a6820..e10938c 100644 --- a/benchmark.config.json +++ b/benchmark.config.json @@ -18,8 +18,7 @@ "vus": 50, "duration": "30s", "env": { - "GRPC_HOST": "localhost:8080", - "PROTO_DIR": "./projects/content-api/_shared/protobuf" + "GRPC_HOST": "localhost:8080" } }, "tags": { @@ -41,11 +40,6 @@ "timeoutMs": 120000 } }, - "grafana": { - "endpoint": "https://otlp-gateway-prod-us-central-0.grafana.net/otlp", - "instanceId": "${GRAFANA_INSTANCE_ID}", - "apiKey": "${GRAFANA_API_KEY}" - }, "output": { "dir": "./results" } diff --git a/grafana/benchmark-dashboard.json b/grafana/benchmark-dashboard.json new file mode 100644 index 0000000..abd8d04 --- /dev/null +++ b/grafana/benchmark-dashboard.json @@ -0,0 +1,269 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "Prometheus data source for benchmark metrics", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "11.0.0" }, + { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }, + { "type": "panel", "id": "barchart", "name": "Bar chart", "version": "" }, + { "type": "panel", "id": "bargauge", "name": "Bar gauge", "version": "" } + ], + "id": null, + "uid": null, + "title": "API Benchmark Results", + "description": "Compare benchmark results across API implementations", + "tags": ["benchmark"], + "timezone": "browser", + "editable": true, + "graphTooltip": 1, + "time": { "from": "now-7d", "to": "now" }, + "refresh": "", + "schemaVersion": 39, + "version": 1, + "templating": { + "list": [ + { + "name": "datasource", + "type": "datasource", + "query": "prometheus", + "current": {}, + "hide": 0, + "includeAll": false, + "multi": false, + "label": "Data Source" + }, + { + "name": "target", + "type": "query", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "query": "label_values(benchmark_reqs_per_sec, benchmark_target)", + "current": {}, + "hide": 0, + "includeAll": true, + "multi": true, + "allValue": ".*", + "label": "Target", + "refresh": 2, + "sort": 1 + }, + { + "name": "tag", + "type": "query", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "query": "label_values(benchmark_reqs_per_sec{benchmark_target=~\"$target\"}, benchmark_tag)", + "current": {}, + "hide": 0, + "includeAll": true, + "multi": true, + "allValue": ".*", + "label": "Tag", + "refresh": 2, + "sort": 1 + } + ] + }, + "panels": [ + { + "type": "bargauge", + "title": "Throughput (req/s)", + "description": "Higher is better", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "color": { "mode": "palette-classic-by-name" }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "orientation": "horizontal", + "displayMode": "gradient", + "showUnfilled": true, + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "valueMode": "color", + "namePlacement": "left", + "sizing": "auto" + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "benchmark_reqs_per_sec{benchmark_target=~\"$target\", benchmark_tag=~\"$tag\"}", + "legendFormat": "{{benchmark_target}}", + "refId": "A", + "instant": true + } + ] + }, + { + "type": "bargauge", + "title": "Error Rate", + "description": "Lower is better", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "value": null, "color": "green" }, + { "value": 0.01, "color": "yellow" }, + { "value": 0.05, "color": "red" } + ] + }, + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "options": { + "orientation": "horizontal", + "displayMode": "gradient", + "showUnfilled": true, + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "valueMode": "color", + "namePlacement": "left", + "sizing": "auto" + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "benchmark_req_failed_rate{benchmark_target=~\"$target\", benchmark_tag=~\"$tag\"}", + "legendFormat": "{{benchmark_target}}", + "refId": "A", + "instant": true + } + ] + }, + { + "type": "barchart", + "title": "Response Time by Percentile (ms)", + "description": "Lower is better", + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 8 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "unit": "ms", + "color": { "mode": "palette-classic" } + }, + "overrides": [] + }, + "options": { + "orientation": "vertical", + "showValue": "always", + "groupWidth": 0.7, + "barWidth": 0.8, + "stacking": "none", + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi" }, + "xTickLabelRotation": 0 + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "benchmark_req_duration_avg{benchmark_target=~\"$target\", benchmark_tag=~\"$tag\"}", + "legendFormat": "{{benchmark_target}} avg", + "refId": "A", + "instant": true + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "benchmark_req_duration_p90{benchmark_target=~\"$target\", benchmark_tag=~\"$tag\"}", + "legendFormat": "{{benchmark_target}} p90", + "refId": "B", + "instant": true + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "benchmark_req_duration_p95{benchmark_target=~\"$target\", benchmark_tag=~\"$tag\"}", + "legendFormat": "{{benchmark_target}} p95", + "refId": "C", + "instant": true + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "benchmark_req_duration_p99{benchmark_target=~\"$target\", benchmark_tag=~\"$tag\"}", + "legendFormat": "{{benchmark_target}} p99", + "refId": "D", + "instant": true + } + ] + }, + { + "type": "bargauge", + "title": "Build Time", + "description": "Total Docker build duration (lower is better)", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 18 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "unit": "ms", + "color": { "mode": "palette-classic-by-name" }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "orientation": "horizontal", + "displayMode": "gradient", + "showUnfilled": true, + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "valueMode": "color", + "namePlacement": "left", + "sizing": "auto" + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "benchmark_build_duration{benchmark_target=~\"$target\", benchmark_tag=~\"$tag\"}", + "legendFormat": "{{benchmark_target}}", + "refId": "A", + "instant": true + } + ] + }, + { + "type": "bargauge", + "title": "Deploy Time", + "description": "Time to healthy (lower is better)", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 18 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "unit": "ms", + "color": { "mode": "palette-classic-by-name" }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "orientation": "horizontal", + "displayMode": "gradient", + "showUnfilled": true, + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "valueMode": "color", + "namePlacement": "left", + "sizing": "auto" + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "benchmark_deploy_duration{benchmark_target=~\"$target\", benchmark_tag=~\"$tag\"}", + "legendFormat": "{{benchmark_target}}", + "refId": "A", + "instant": true + } + ] + } + ] +} diff --git a/package-lock.json b/package-lock.json index a92b1ec..07383f4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -553,12 +553,244 @@ "resolved": "toolkit", "link": true }, + "node_modules/@opentelemetry/api": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", + "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/api-logs": { + "version": "0.213.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.213.0.tgz", + "integrity": "sha512-zRM5/Qj6G84Ej3F1yt33xBVY/3tnMxtL1fiDIxYbDWYaZ/eudVw3/PBiZ8G7JwUxXxjW8gU4g6LnOyfGKYHYgw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.3.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/core": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.6.0.tgz", + "integrity": "sha512-HLM1v2cbZ4TgYN6KEOj+Bbj8rAKriOdkF9Ed3tG25FoprSiQl7kYc+RRT6fUZGOvx0oMi5U67GoFdT+XUn8zEg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-metrics-otlp-http": { + "version": "0.213.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-metrics-otlp-http/-/exporter-metrics-otlp-http-0.213.0.tgz", + "integrity": "sha512-yw3fTIw4KQIRXC/ZyYQq5gtA3Ogfdfz/g5HVgleobQAcjUUE8Nj3spGMx8iQPp+S+u6/js7BixufRkXhzLmpJA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.6.0", + "@opentelemetry/otlp-exporter-base": "0.213.0", + "@opentelemetry/otlp-transformer": "0.213.0", + "@opentelemetry/resources": "2.6.0", + "@opentelemetry/sdk-metrics": "2.6.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-exporter-base": { + "version": "0.213.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-exporter-base/-/otlp-exporter-base-0.213.0.tgz", + "integrity": "sha512-MegxAP1/n09Ob2dQvY5NBDVjAFkZRuKtWKxYev1R2M8hrsgXzQGkaMgoEKeUOyQ0FUyYcO29UOnYdQWmWa0PXg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.6.0", + "@opentelemetry/otlp-transformer": "0.213.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-transformer": { + "version": "0.213.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-transformer/-/otlp-transformer-0.213.0.tgz", + "integrity": "sha512-RSuAlxFFPjeK4d5Y6ps8L2WhaQI6CXWllIjvo5nkAlBpmq2XdYWEBGiAbOF4nDs8CX4QblJDv5BbMUft3sEfDw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.213.0", + "@opentelemetry/core": "2.6.0", + "@opentelemetry/resources": "2.6.0", + "@opentelemetry/sdk-logs": "0.213.0", + "@opentelemetry/sdk-metrics": "2.6.0", + "@opentelemetry/sdk-trace-base": "2.6.0", + "protobufjs": "^7.0.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/resources": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.6.0.tgz", + "integrity": "sha512-D4y/+OGe3JSuYUCBxtH5T9DSAWNcvCb/nQWIga8HNtXTVPQn59j0nTBAgaAXxUVBDl40mG3Tc76b46wPlZaiJQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.6.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-logs": { + "version": "0.213.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-logs/-/sdk-logs-0.213.0.tgz", + "integrity": "sha512-00xlU3GZXo3kXKve4DLdrAL0NAFUaZ9appU/mn00S/5kSUdAvyYsORaDUfR04Mp2CLagAOhrzfUvYozY/EZX2g==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.213.0", + "@opentelemetry/core": "2.6.0", + "@opentelemetry/resources": "2.6.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.4.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-metrics": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.6.0.tgz", + "integrity": "sha512-CicxWZxX6z35HR83jl+PLgtFgUrKRQ9LCXyxgenMnz5A1lgYWfAog7VtdOvGkJYyQgMNPhXQwkYrDLujk7z1Iw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.6.0", + "@opentelemetry/resources": "2.6.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.9.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-base": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.6.0.tgz", + "integrity": "sha512-g/OZVkqlxllgFM7qMKqbPV9c1DUPhQ7d4n3pgZFcrnrNft9eJXZM2TNHTPYREJBrtNdRytYyvwjgL5geDKl3EQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.6.0", + "@opentelemetry/resources": "2.6.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/semantic-conventions": { + "version": "1.40.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.40.0.tgz", + "integrity": "sha512-cifvXDhcqMwwTlTK04GBNeIe7yyo28Mfby85QXFe1Yk8nmi36Ab/5UQwptOx84SsoGNRg+EVSjwzfSZMy6pmlw==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, "node_modules/@pinojs/redact": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz", "integrity": "sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==", "license": "MIT" }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", + "license": "BSD-3-Clause" + }, "node_modules/@sec-ant/readable-stream": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/@sec-ant/readable-stream/-/readable-stream-0.4.1.tgz", @@ -602,7 +834,6 @@ "version": "25.3.5", "resolved": "https://registry.npmjs.org/@types/node/-/node-25.3.5.tgz", "integrity": "sha512-oX8xrhvpiyRCQkG1MFchB09f+cXftgIXb3a7UUa4Y3wpmZPw5tyZGTLWhlESOLq1Rq6oDlc8npVU2/9xiCuXMA==", - "dev": true, "license": "MIT", "dependencies": { "undici-types": "~7.18.0" @@ -770,18 +1001,6 @@ "dev": true, "license": "MIT" }, - "node_modules/dotenv": { - "version": "16.6.1", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.6.1.tgz", - "integrity": "sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==", - "license": "BSD-2-Clause", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://dotenvx.com" - } - }, "node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", @@ -1324,6 +1543,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, "node_modules/minimatch": { "version": "10.2.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.4.tgz", @@ -1617,6 +1842,30 @@ ], "license": "MIT" }, + "node_modules/protobufjs": { + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.4.tgz", + "integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/pump": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", @@ -1813,7 +2062,6 @@ "version": "7.18.2", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", - "dev": true, "license": "MIT" }, "node_modules/unicorn-magic": { @@ -1908,6 +2156,9 @@ "version": "1.0.0", "license": "Apache-2.0", "dependencies": { + "@opentelemetry/exporter-metrics-otlp-http": "^0.213.0", + "@opentelemetry/resources": "^2.6.0", + "@opentelemetry/sdk-metrics": "^2.6.0", "chalk": "^5.4.1", "cli-table3": "^0.6.5", "commander": "^13.1.0", @@ -1931,6 +2182,18 @@ "engines": { "node": ">=22.0.0" } + }, + "toolkit/node_modules/dotenv": { + "version": "16.6.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.6.1.tgz", + "integrity": "sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } } } } diff --git a/package.json b/package.json index ceb20fe..3b58914 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,8 @@ "lint:fix": "npm run lint:fix --workspaces --if-present", "format": "npm run format --workspaces --if-present", "format:check": "npm run format:check --workspaces --if-present", - "build": "npm run build --workspaces --if-present" + "build": "npm run build --workspaces --if-present", + "fix": "npm run fix --workspaces --if-present" }, "devDependencies": { "@anthropic-ai/claude-code": "^2.1.63" diff --git a/projects/content-api/_shared/protobuf/k6/content-api.js b/projects/content-api/_shared/protobuf/k6/content-api.js index 9ff1a9d..7a8be0d 100644 --- a/projects/content-api/_shared/protobuf/k6/content-api.js +++ b/projects/content-api/_shared/protobuf/k6/content-api.js @@ -2,18 +2,9 @@ import grpc from 'k6/net/grpc'; import { check, group, sleep } from 'k6'; const GRPC_HOST = __ENV.GRPC_HOST || 'localhost:8080'; -const PROTO_DIR = __ENV.PROTO_DIR || ''; const client = new grpc.Client(); -if (PROTO_DIR) { - client.load( - [PROTO_DIR], - 'content/v1/content_service.proto', - 'content/v1/content_model.proto', - ); -} - export const options = { thresholds: { grpc_req_duration: ['p(95)<500'], @@ -21,7 +12,7 @@ export const options = { }; export default function () { - client.connect(GRPC_HOST, { plaintext: true }); + client.connect(GRPC_HOST, { plaintext: true, reflect: true }); let contentId; @@ -76,7 +67,7 @@ export default function () { title: `Updated ${crypto.randomUUID()}`, status: 'CONTENT_STATUS_PUBLISHED', }, - updateMask: { paths: ['title', 'status'] }, + updateMask: 'title,status', }); check(res, { diff --git a/projects/content-api/connect-rpc/cmd/server/setup_gateway.go b/projects/content-api/connect-rpc/cmd/server/setup_gateway.go index 960d227..86ad610 100644 --- a/projects/content-api/connect-rpc/cmd/server/setup_gateway.go +++ b/projects/content-api/connect-rpc/cmd/server/setup_gateway.go @@ -2,6 +2,7 @@ package main import ( "connectrpc.com/connect" + "connectrpc.com/grpcreflect" contentv1connect "content-api-connect-rpc/gen/proto/content/v1/contentv1connect" contentapi "content-api-connect-rpc/internal/api/content" @@ -21,5 +22,11 @@ func setupGateway(cfg *config.Config, domains *Domains) connectapp.App { ) application.Handle(path, h) + reflector := grpcreflect.NewStaticReflector( + contentv1connect.ContentServiceName, + ) + application.Handle(grpcreflect.NewHandlerV1(reflector)) + application.Handle(grpcreflect.NewHandlerV1Alpha(reflector)) + return application } diff --git a/projects/content-api/connect-rpc/go.mod b/projects/content-api/connect-rpc/go.mod index f903c8c..68bcc57 100644 --- a/projects/content-api/connect-rpc/go.mod +++ b/projects/content-api/connect-rpc/go.mod @@ -3,7 +3,9 @@ module content-api-connect-rpc go 1.25.0 require ( + buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.9-20250912141014-52f32327d4b0.1 connectrpc.com/connect v1.19.1 + connectrpc.com/grpcreflect v1.3.0 connectrpc.com/validate v0.6.0 github.com/gofrs/uuid/v5 v5.4.0 github.com/jackc/pgx/v5 v5.8.0 @@ -17,7 +19,6 @@ require ( ) require ( - buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.9-20250912141014-52f32327d4b0.1 // indirect buf.build/go/protovalidate v1.0.0 // indirect cel.dev/expr v0.24.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect diff --git a/projects/content-api/connect-rpc/go.sum b/projects/content-api/connect-rpc/go.sum index ab1322c..fe5ca72 100644 --- a/projects/content-api/connect-rpc/go.sum +++ b/projects/content-api/connect-rpc/go.sum @@ -6,6 +6,8 @@ cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= connectrpc.com/connect v1.19.1 h1:R5M57z05+90EfEvCY1b7hBxDVOUl45PrtXtAV2fOC14= connectrpc.com/connect v1.19.1/go.mod h1:tN20fjdGlewnSFeZxLKb0xwIZ6ozc3OQs2hTXy4du9w= +connectrpc.com/grpcreflect v1.3.0 h1:Y4V+ACf8/vOb1XOc251Qun7jMB75gCUNw6llvB9csXc= +connectrpc.com/grpcreflect v1.3.0/go.mod h1:nfloOtCS8VUQOQ1+GTdFzVg2CJo4ZGaat8JIovCtDYs= connectrpc.com/validate v0.6.0 h1:DcrgDKt2ZScrUs/d/mh9itD2yeEa0UbBBa+i0mwzx+4= connectrpc.com/validate v0.6.0/go.mod h1:ihrpI+8gVbLH1fvVWJL1I3j0CfWnF8P/90LsmluRiZs= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= diff --git a/toolkit/bin/benchmark.js b/toolkit/bin/benchmark.js index fd023ef..028cb9e 100755 --- a/toolkit/bin/benchmark.js +++ b/toolkit/bin/benchmark.js @@ -1,5 +1,6 @@ #!/usr/bin/env node +import 'dotenv/config'; import process from 'node:process'; import { createProgram } from '../src/cli/index.js'; diff --git a/toolkit/package.json b/toolkit/package.json index 6b756b6..d2d825a 100644 --- a/toolkit/package.json +++ b/toolkit/package.json @@ -15,9 +15,14 @@ "lint": "eslint .", "lint:fix": "eslint . --fix", "format": "prettier --write .", - "format:check": "prettier --check ." + "format:check": "prettier --check .", + "build": "npm run format:check && npm run lint", + "fix": "npm run format && npm run lint:fix" }, "dependencies": { + "@opentelemetry/exporter-metrics-otlp-http": "^0.213.0", + "@opentelemetry/resources": "^2.6.0", + "@opentelemetry/sdk-metrics": "^2.6.0", "chalk": "^5.4.1", "cli-table3": "^0.6.5", "commander": "^13.1.0", diff --git a/toolkit/src/cli/commands/build.js b/toolkit/src/cli/commands/build.js index 884d643..bba779e 100644 --- a/toolkit/src/cli/commands/build.js +++ b/toolkit/src/cli/commands/build.js @@ -25,6 +25,7 @@ export function buildCommand() { }); const { durationMs, durationSec } = timer.stop(); + log.info({ target: targetName, durationMs, durationSec, msg: 'build completed' }); return { durationMs, cached: options.cache }; diff --git a/toolkit/src/cli/commands/loadtest.js b/toolkit/src/cli/commands/loadtest.js index aa29eb3..7a389b8 100644 --- a/toolkit/src/cli/commands/loadtest.js +++ b/toolkit/src/cli/commands/loadtest.js @@ -1,6 +1,7 @@ import { Command } from 'commander'; import { resolve } from 'node:path'; import { loadConfig, resolveTarget } from '../../config/loader.js'; +import { waitForHealthy } from '../../core/health.js'; import { runK6 } from '../../core/k6.js'; import { parseK6Summary } from '../../metrics/k6-parser.js'; import { getLogger } from '../../util/logger.js'; @@ -17,6 +18,9 @@ export function loadtestCommand() { const config = await loadConfig(globalOpts.config); const target = resolveTarget(config, targetName); + // Verify the service is healthy before starting the load test + await waitForHealthy(target); + const scriptPath = resolve(target.k6.script); const vus = options.k6Vus ? parseInt(options.k6Vus, 10) : target.k6.vus; const duration = options.k6Duration ?? target.k6.duration; @@ -31,9 +35,9 @@ export function loadtestCommand() { log.info({ target: targetName, - reqsPerSec: summary.httpReqsPerSec.toFixed(1), - p95: summary.httpReqDuration.p95.toFixed(1), - errorRate: summary.httpReqFailed.toFixed(4), + reqsPerSec: summary.reqsPerSec.toFixed(1), + p95: summary.reqDuration.p95.toFixed(1), + errorRate: summary.reqFailedRate.toFixed(4), msg: 'load test completed', }); diff --git a/toolkit/src/cli/commands/publish.js b/toolkit/src/cli/commands/publish.js index 4126ff8..ef21448 100644 --- a/toolkit/src/cli/commands/publish.js +++ b/toolkit/src/cli/commands/publish.js @@ -1,25 +1,25 @@ import { Command } from 'commander'; import { readFile } from 'node:fs/promises'; import { resolve } from 'node:path'; -import { loadConfig } from '../../config/loader.js'; -import { publishToGrafanaCloud } from '../../publish/grafana-cloud.js'; +import { publishResults } from '../../publish/otlp.js'; import { getLogger } from '../../util/logger.js'; export function publishCommand() { return new Command('publish') .description('publish benchmark results to Grafana Cloud') - .argument('', 'path to results JSON file') - .action(async (resultsPath, options, command) => { + .argument('', 'path(s) to results JSON file(s)') + .action(async (resultsPaths) => { const log = getLogger(); - const globalOpts = command.parent.opts(); - const config = await loadConfig(globalOpts.config); - const absolutePath = resolve(resultsPath); - const content = await readFile(absolutePath, 'utf-8'); - const results = JSON.parse(content); + for (const resultsPath of resultsPaths) { + const absolutePath = resolve(resultsPath); + const content = await readFile(absolutePath, 'utf-8'); + const results = JSON.parse(content); - log.info({ file: absolutePath, target: results.target, msg: 'publishing results' }); - await publishToGrafanaCloud(results, config); - log.info({ msg: 'publish completed' }); + log.info({ file: absolutePath, target: results.target, msg: 'publishing results' }); + await publishResults(results); + } + + log.info({ count: resultsPaths.length, msg: 'publish completed' }); }); } diff --git a/toolkit/src/cli/commands/run.js b/toolkit/src/cli/commands/run.js index 983edac..e4b0150 100644 --- a/toolkit/src/cli/commands/run.js +++ b/toolkit/src/cli/commands/run.js @@ -81,8 +81,8 @@ export function runCommand() { summary, }; log.info({ - reqsPerSec: summary.httpReqsPerSec.toFixed(1), - p95: summary.httpReqDuration.p95.toFixed(1), + reqsPerSec: summary.reqsPerSec.toFixed(1), + p95: summary.reqDuration.p95.toFixed(1), msg: 'loadtest completed', }); } @@ -101,9 +101,8 @@ export function runCommand() { // --- Publish (if requested) --- if (options.publish) { - const { publishToGrafanaCloud } = await import('../../publish/grafana-cloud.js'); - await publishToGrafanaCloud(results, config); - log.info({ msg: 'results published to Grafana Cloud' }); + const { publishResults } = await import('../../publish/otlp.js'); + await publishResults(results); } } finally { // --- Cleanup (only if deploy was attempted) --- diff --git a/toolkit/src/config/schema.js b/toolkit/src/config/schema.js index 391ed41..dad1942 100644 --- a/toolkit/src/config/schema.js +++ b/toolkit/src/config/schema.js @@ -31,12 +31,6 @@ const targetSchema = z.object({ tags: z.record(z.string()).default({}), }); -const grafanaSchema = z.object({ - endpoint: z.string().url(), - instanceId: z.string(), - apiKey: z.string(), -}); - export const configSchema = z.object({ targets: z.record(targetSchema), defaults: z @@ -45,7 +39,6 @@ export const configSchema = z.object({ readinessProbe: readinessProbeSchema.partial().default({}), }) .default({}), - grafana: grafanaSchema.optional(), output: z .object({ dir: z.string().default('./results'), diff --git a/toolkit/src/core/docker.js b/toolkit/src/core/docker.js index 41bd3d4..7deb9e8 100644 --- a/toolkit/src/core/docker.js +++ b/toolkit/src/core/docker.js @@ -9,13 +9,12 @@ export async function composeBuild(projectDir, options = {}) { const log = getLogger(); const { composeFile = 'docker-compose.yml', noCache = true } = options; - const args = composeArgs(composeFile, ['build']); + const args = composeArgs(composeFile, ['build', '--progress=plain']); if (noCache) args.push('--no-cache'); log.info({ projectDir, noCache, msg: 'building docker compose project' }); return exec('docker', ['compose', ...args], { cwd: projectDir, - stdio: 'inherit', }); } diff --git a/toolkit/src/core/k6.js b/toolkit/src/core/k6.js index 67110bf..14612ab 100644 --- a/toolkit/src/core/k6.js +++ b/toolkit/src/core/k6.js @@ -1,33 +1,65 @@ -import { join } from 'node:path'; +import { join, resolve, relative } from 'node:path'; import { tmpdir } from 'node:os'; import { randomUUID } from 'node:crypto'; import { readFile, unlink } from 'node:fs/promises'; import { exec } from '../util/exec.js'; import { getLogger } from '../util/logger.js'; +const K6_IMAGE = 'grafana/k6:1.6.1'; + export async function runK6(scriptPath, options = {}) { const log = getLogger(); const { vus = 50, duration = '30s', env = {} } = options; - const summaryFile = join(tmpdir(), `benchmark-k6-${randomUUID()}.json`); + const workDir = resolve('.'); + const summaryName = `benchmark-k6-${randomUUID()}.json`; + const summaryDir = tmpdir(); + const summaryFile = join(summaryDir, summaryName); + + const toContainerPath = (hostPath) => { + const abs = resolve(hostPath); + return `/workspace/${relative(workDir, abs)}`; + }; + + const isLinux = process.platform === 'linux'; - const args = [ + const dockerArgs = [ + 'run', + '--rm', + '--network', + 'host', + '-v', + `${workDir}:/workspace:ro`, + '-v', + `${summaryDir}:/results`, + ]; + + for (const [key, value] of Object.entries(env)) { + let mapped = value.startsWith('./') ? toContainerPath(value) : value; + // Docker Desktop (macOS/Windows): --network host still works but localhost + // inside the container resolves to the VM, not the macOS host. + // Rewrite localhost to host.docker.internal so k6 can reach published ports. + if (!isLinux) { + mapped = mapped.replace(/localhost|127\.0\.0\.1/g, 'host.docker.internal'); + } + dockerArgs.push('-e', `${key}=${mapped}`); + } + + dockerArgs.push( + K6_IMAGE, 'run', '--summary-export', - summaryFile, + `/results/${summaryName}`, '--vus', String(vus), '--duration', duration, - scriptPath, - ]; + toContainerPath(scriptPath) + ); log.info({ scriptPath, vus, duration, msg: 'running k6 load test' }); - await exec('k6', args, { - stdio: 'inherit', - env: { ...process.env, ...env }, - }); + await exec('docker', dockerArgs, { stdio: 'inherit' }); try { const raw = await readFile(summaryFile, 'utf-8'); diff --git a/toolkit/src/metrics/collector.js b/toolkit/src/metrics/collector.js index 0608878..48e1ee9 100644 --- a/toolkit/src/metrics/collector.js +++ b/toolkit/src/metrics/collector.js @@ -11,11 +11,11 @@ async function getVersion(command, args) { } async function collectEnvironment() { - const [dockerVersion, nodeVersion, k6Version] = await Promise.all([ + const [dockerVersion, nodeVersion] = await Promise.all([ getVersion('docker', ['--version']), getVersion('node', ['--version']), - getVersion('k6', ['version']), ]); + const k6Version = 'grafana/k6:1.6.1'; return { os: platform(), diff --git a/toolkit/src/metrics/k6-parser.js b/toolkit/src/metrics/k6-parser.js index 055a7ff..4504a4c 100644 --- a/toolkit/src/metrics/k6-parser.js +++ b/toolkit/src/metrics/k6-parser.js @@ -1,31 +1,32 @@ export function parseK6Summary(raw) { const metrics = raw.metrics || {}; - const httpReqDuration = metrics.http_req_duration?.values || {}; - const httpReqs = metrics.http_reqs?.values || {}; - const httpReqFailed = metrics.http_req_failed?.values || {}; - const checks = metrics.checks?.values || {}; - const iterations = metrics.iterations?.values || {}; - const dataReceived = metrics.data_received?.values || {}; - const dataSent = metrics.data_sent?.values || {}; + // Support both HTTP and gRPC protocols. + const reqDuration = metrics.http_req_duration || metrics.grpc_req_duration || {}; + const reqs = metrics.http_reqs || {}; + const reqFailed = metrics.http_req_failed || {}; + const checks = metrics.checks || {}; + const iterations = metrics.iterations || {}; + const dataReceived = metrics.data_received || {}; + const dataSent = metrics.data_sent || {}; return { - httpReqs: httpReqs.count ?? 0, - httpReqsPerSec: httpReqs.rate ?? 0, - httpReqDuration: { - avg: httpReqDuration.avg ?? 0, - min: httpReqDuration.min ?? 0, - med: httpReqDuration.med ?? 0, - max: httpReqDuration.max ?? 0, - p90: httpReqDuration['p(90)'] ?? 0, - p95: httpReqDuration['p(95)'] ?? 0, - p99: httpReqDuration['p(99)'] ?? 0, + reqs: reqs.count ?? iterations.count ?? 0, + reqsPerSec: reqs.rate ?? iterations.rate ?? 0, + reqDuration: { + avg: reqDuration.avg ?? 0, + min: reqDuration.min ?? 0, + med: reqDuration.med ?? 0, + max: reqDuration.max ?? 0, + p90: reqDuration['p(90)'] ?? 0, + p95: reqDuration['p(95)'] ?? 0, + p99: reqDuration['p(99)'] ?? 0, }, - httpReqFailed: httpReqFailed.rate ?? 0, + reqFailedRate: reqFailed.rate ?? 0, iterations: iterations.count ?? 0, iterationsPerSec: iterations.rate ?? 0, dataReceived: dataReceived.count ?? 0, dataSent: dataSent.count ?? 0, - checksPassRate: checks.rate ?? 0, + checksPassRate: checks.value ?? 0, }; } diff --git a/toolkit/src/publish/formatter.js b/toolkit/src/publish/formatter.js deleted file mode 100644 index f22e4cf..0000000 --- a/toolkit/src/publish/formatter.js +++ /dev/null @@ -1,179 +0,0 @@ -function makeAttribute(key, value) { - if (typeof value === 'number') { - return { key, value: { intValue: String(Math.round(value)) } }; - } - return { key, value: { stringValue: String(value) } }; -} - -function makeGauge(name, unit, value, timeUnixNano, attributes) { - const dataPoint = { - timeUnixNano: String(timeUnixNano), - attributes, - }; - - if (Number.isInteger(value)) { - dataPoint.asInt = String(value); - } else { - dataPoint.asDouble = value; - } - - return { - name, - unit, - gauge: { - dataPoints: [dataPoint], - }, - }; -} - -export function formatAsOtlpMetrics(results) { - const timeUnixNano = BigInt(new Date(results.timestamp).getTime()) * 1_000_000n; - - const resourceAttributes = [ - makeAttribute('benchmark.target', results.target), - makeAttribute('benchmark.tag', results.tag), - makeAttribute('host.os', results.environment.os), - makeAttribute('host.arch', results.environment.arch), - ]; - - // Add target tags as resource attributes - if (results.metrics.loadtest?.config) { - const { vus, duration } = results.metrics.loadtest.config; - resourceAttributes.push(makeAttribute('benchmark.k6.vus', vus)); - resourceAttributes.push(makeAttribute('benchmark.k6.duration', duration)); - } - - const metricAttributes = [ - makeAttribute('benchmark.target', results.target), - makeAttribute('benchmark.tag', results.tag), - ]; - - const metrics = []; - - // Build metrics - if (results.metrics.build) { - metrics.push( - makeGauge( - 'benchmark.build.duration', - 'ms', - results.metrics.build.durationMs, - timeUnixNano, - metricAttributes - ) - ); - } - - // Deploy metrics - if (results.metrics.deploy) { - metrics.push( - makeGauge( - 'benchmark.deploy.duration', - 'ms', - results.metrics.deploy.durationMs, - timeUnixNano, - metricAttributes - ) - ); - } - - // Load test metrics - if (results.metrics.loadtest) { - const { summary } = results.metrics.loadtest; - - metrics.push( - makeGauge('benchmark.http_reqs', '1', summary.httpReqs, timeUnixNano, metricAttributes), - makeGauge( - 'benchmark.http_reqs_per_sec', - '1/s', - summary.httpReqsPerSec, - timeUnixNano, - metricAttributes - ), - makeGauge( - 'benchmark.http_req_failed_rate', - '1', - summary.httpReqFailed, - timeUnixNano, - metricAttributes - ), - makeGauge( - 'benchmark.http_req.duration.avg', - 'ms', - summary.httpReqDuration.avg, - timeUnixNano, - metricAttributes - ), - makeGauge( - 'benchmark.http_req.duration.min', - 'ms', - summary.httpReqDuration.min, - timeUnixNano, - metricAttributes - ), - makeGauge( - 'benchmark.http_req.duration.med', - 'ms', - summary.httpReqDuration.med, - timeUnixNano, - metricAttributes - ), - makeGauge( - 'benchmark.http_req.duration.max', - 'ms', - summary.httpReqDuration.max, - timeUnixNano, - metricAttributes - ), - makeGauge( - 'benchmark.http_req.duration.p90', - 'ms', - summary.httpReqDuration.p90, - timeUnixNano, - metricAttributes - ), - makeGauge( - 'benchmark.http_req.duration.p95', - 'ms', - summary.httpReqDuration.p95, - timeUnixNano, - metricAttributes - ), - makeGauge( - 'benchmark.http_req.duration.p99', - 'ms', - summary.httpReqDuration.p99, - timeUnixNano, - metricAttributes - ), - makeGauge( - 'benchmark.checks_pass_rate', - '1', - summary.checksPassRate, - timeUnixNano, - metricAttributes - ), - makeGauge('benchmark.iterations', '1', summary.iterations, timeUnixNano, metricAttributes), - makeGauge( - 'benchmark.iterations_per_sec', - '1/s', - summary.iterationsPerSec, - timeUnixNano, - metricAttributes - ) - ); - } - - return { - resourceMetrics: [ - { - resource: { attributes: resourceAttributes }, - scopeMetrics: [ - { - scope: { name: '@labset/benchmark-toolkit', version: '1.0.0' }, - metrics, - }, - ], - }, - ], - }; -} diff --git a/toolkit/src/publish/grafana-cloud.js b/toolkit/src/publish/grafana-cloud.js deleted file mode 100644 index 7c314eb..0000000 --- a/toolkit/src/publish/grafana-cloud.js +++ /dev/null @@ -1,71 +0,0 @@ -import { formatAsOtlpMetrics } from './formatter.js'; -import { getLogger } from '../util/logger.js'; - -export async function publishToGrafanaCloud(results, config) { - const log = getLogger(); - - if (!config.grafana) { - throw new Error('grafana config is required for publishing'); - } - - const { endpoint, instanceId, apiKey } = config.grafana; - - if (!instanceId || !apiKey) { - throw new Error( - 'grafana instanceId and apiKey are required. Set GRAFANA_INSTANCE_ID and GRAFANA_API_KEY environment variables.' - ); - } - - const body = formatAsOtlpMetrics(results); - const url = `${endpoint}/v1/metrics`; - const auth = Buffer.from(`${instanceId}:${apiKey}`).toString('base64'); - - log.debug({ - url, - metricsCount: body.resourceMetrics[0].scopeMetrics[0].metrics.length, - msg: 'publishing to Grafana Cloud', - }); - - const maxRetries = 3; - let lastError; - - for (let attempt = 1; attempt <= maxRetries; attempt++) { - try { - const response = await fetch(url, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Basic ${auth}`, - }, - body: JSON.stringify(body), - }); - - if (response.ok) { - log.info({ - status: response.status, - target: results.target, - msg: 'published to Grafana Cloud', - }); - return; - } - - if (response.status === 429 && attempt < maxRetries) { - const retryAfter = parseInt(response.headers.get('retry-after') || '5', 10); - log.warn({ retryAfter, attempt, msg: 'rate limited, retrying' }); - await new Promise((r) => setTimeout(r, retryAfter * 1000)); - continue; - } - - const responseText = await response.text(); - lastError = new Error(`Grafana Cloud returned ${response.status}: ${responseText}`); - } catch (err) { - lastError = err; - if (attempt < maxRetries) { - log.warn({ error: err.message, attempt, msg: 'publish failed, retrying' }); - await new Promise((r) => setTimeout(r, 2000 * attempt)); - } - } - } - - throw lastError; -} diff --git a/toolkit/src/publish/otlp.js b/toolkit/src/publish/otlp.js new file mode 100644 index 0000000..e8eba6c --- /dev/null +++ b/toolkit/src/publish/otlp.js @@ -0,0 +1,81 @@ +import { MeterProvider, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'; +import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http'; +import { resourceFromAttributes } from '@opentelemetry/resources'; +import { getLogger } from '../util/logger.js'; + +export async function publishResults(results) { + const log = getLogger(); + + if (!process.env.OTEL_EXPORTER_OTLP_ENDPOINT) { + throw new Error('OTEL_EXPORTER_OTLP_ENDPOINT is not set. Configure it in your .env file.'); + } + + const resource = resourceFromAttributes({ + 'service.name': '@labset/benchmark-toolkit', + 'service.version': '1.0.0', + }); + + // Data point attributes become Prometheus labels directly. + // Resource attributes only appear as labels if promoted by Grafana Cloud, + // and custom attributes like benchmark.target are not in the default promoted list. + const attributes = { + 'benchmark.target': results.target, + 'benchmark.tag': results.tag, + 'host.os': results.environment.os, + 'host.arch': results.environment.arch, + }; + + const exporter = new OTLPMetricExporter(); + const meterProvider = new MeterProvider({ + resource, + readers: [ + new PeriodicExportingMetricReader({ + exporter, + exportIntervalMillis: 60_000, + }), + ], + }); + + const meter = meterProvider.getMeter('benchmark'); + + // Helper to record a gauge with shared attributes. + // Units are omitted to avoid Grafana Cloud appending suffixes + // (e.g. _milliseconds, _ratio) which vary by configuration. + const gauge = (name, value) => { + meter.createGauge(name).record(value, attributes); + }; + + // Build metrics (per-stage durations are reported via BuildKit OTLP traces) + if (results.metrics.build) { + gauge('benchmark.build.duration', results.metrics.build.durationMs); + } + + // Deploy metrics + if (results.metrics.deploy) { + gauge('benchmark.deploy.duration', results.metrics.deploy.durationMs); + } + + // Load test metrics + if (results.metrics.loadtest) { + const s = results.metrics.loadtest.summary; + + gauge('benchmark.reqs', s.reqs); + gauge('benchmark.reqs_per_sec', s.reqsPerSec); + gauge('benchmark.req_failed_rate', s.reqFailedRate); + gauge('benchmark.req.duration.avg', s.reqDuration.avg); + gauge('benchmark.req.duration.min', s.reqDuration.min); + gauge('benchmark.req.duration.med', s.reqDuration.med); + gauge('benchmark.req.duration.max', s.reqDuration.max); + gauge('benchmark.req.duration.p90', s.reqDuration.p90); + gauge('benchmark.req.duration.p95', s.reqDuration.p95); + gauge('benchmark.req.duration.p99', s.reqDuration.p99); + gauge('benchmark.checks_pass_rate', s.checksPassRate); + gauge('benchmark.iterations', s.iterations); + gauge('benchmark.iterations_per_sec', s.iterationsPerSec); + } + + await meterProvider.forceFlush(); + await meterProvider.shutdown(); + + log.info({ target: results.target, msg: 'published to Grafana Cloud' }); +} diff --git a/toolkit/src/report/compare.js b/toolkit/src/report/compare.js index eb9b01d..d3a4343 100644 --- a/toolkit/src/report/compare.js +++ b/toolkit/src/report/compare.js @@ -28,12 +28,12 @@ export function compareResults(results) { if (r.metrics.loadtest) { const s = r.metrics.loadtest.summary; - row.reqsPerSec = s.httpReqsPerSec; - row.avgMs = s.httpReqDuration.avg; - row.p90Ms = s.httpReqDuration.p90; - row.p95Ms = s.httpReqDuration.p95; - row.p99Ms = s.httpReqDuration.p99; - row.errorRate = s.httpReqFailed; + row.reqsPerSec = s.reqsPerSec; + row.avgMs = s.reqDuration.avg; + row.p90Ms = s.reqDuration.p90; + row.p95Ms = s.reqDuration.p95; + row.p99Ms = s.reqDuration.p99; + row.errorRate = s.reqFailedRate; } return row; diff --git a/toolkit/src/report/table.js b/toolkit/src/report/table.js index 3795c35..bb113f3 100644 --- a/toolkit/src/report/table.js +++ b/toolkit/src/report/table.js @@ -13,19 +13,21 @@ function highlight(value, bestValue) { } export function renderComparisonTable({ rows, best }) { + const head = [ + chalk.bold('Target'), + chalk.bold('Tag'), + chalk.bold('Build (ms)'), + chalk.bold('Deploy (ms)'), + chalk.bold('Reqs/s'), + chalk.bold('Avg (ms)'), + chalk.bold('p90 (ms)'), + chalk.bold('p95 (ms)'), + chalk.bold('p99 (ms)'), + chalk.bold('Error %'), + ]; + const table = new Table({ - head: [ - chalk.bold('Target'), - chalk.bold('Tag'), - chalk.bold('Build (ms)'), - chalk.bold('Deploy (ms)'), - chalk.bold('Reqs/s'), - chalk.bold('Avg (ms)'), - chalk.bold('p90 (ms)'), - chalk.bold('p95 (ms)'), - chalk.bold('p99 (ms)'), - chalk.bold('Error %'), - ], + head, style: { head: [], border: [] }, }); diff --git a/toolkit/src/util/exec.js b/toolkit/src/util/exec.js index 06f982c..751e698 100644 --- a/toolkit/src/util/exec.js +++ b/toolkit/src/util/exec.js @@ -5,7 +5,7 @@ export async function exec(command, args = [], options = {}) { const log = getLogger(); log.debug({ command, args, cwd: options.cwd, msg: 'executing command' }); - await execa(command, args, { + return await execa(command, args, { stdio: options.stdio ?? 'pipe', cwd: options.cwd, env: options.env,