From 4400d2b807561728ad34d896a21f5dc6dc07d934 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Wed, 8 Jul 2026 07:32:05 +0800 Subject: [PATCH 1/9] Coverage --- .github/workflows/coverage.yaml | 78 +++++++++++++++++++++++++++++++++ .github/workflows/test.yaml | 8 ++-- .mise/config.coverage.toml | 21 +++++++++ 3 files changed, 103 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/coverage.yaml create mode 100644 .mise/config.coverage.toml diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml new file mode 100644 index 0000000..589ed36 --- /dev/null +++ b/.github/workflows/coverage.yaml @@ -0,0 +1,78 @@ +--- +name: coverage + +"on": + pull_request: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + # Codecov's tokenless upload uses an OIDC id-token instead of a repository upload token. + id-token: write + +concurrency: + group: "${{ github.workflow }}-${{ github.ref }}" + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} + +defaults: + run: + shell: bash --noprofile --norc -euo pipefail {0} + +env: + # Exact full guest-language set -- gha.rego pins this literal value at the workflow level. + # (It's what lets `build-modules` build every ws-module so the Rust tests run.) The coverage job appends the + # `coverage` env at job level below, mirroring test.yaml's override job's `,${{ matrix.compiler }}`. + MISE_ENV: dart,dotnet,java,js,python,rust,zig + +jobs: + coverage: + runs-on: ubuntu-latest + timeout-minutes: 150 + env: + MISE_ENV: dart,dotnet,java,js,python,rust,zig,coverage + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + persist-credentials: false + + - name: Free disk space on Linux + uses: jlumbroso/free-disk-space@v1.3.1 + + - name: Install Mesa Vulkan drivers (lavapipe) + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends mesa-vulkan-drivers + + - name: Install mise + coverage tools + uses: ./.github/actions/install-mise-tools + timeout-minutes: 20 + with: + github-token: ${{ github.token }} + install-action-tools: cargo-llvm-cov + + - name: Prefetch dependencies + timeout-minutes: 15 + env: + GITHUB_TOKEN: ${{ github.token }} + run: mise run prefetch-ci + + - name: Build WASM modules + timeout-minutes: 25 + env: + GITHUB_TOKEN: ${{ github.token }} + run: mise run build-modules + + - name: Collect Rust coverage + timeout-minutes: 75 + run: mise run cargo-llvm-cov + + - name: Upload coverage to Codecov (OIDC) + uses: codecov/codecov-action@v5 + with: + use_oidc: true + files: lcov.info + fail_ci_if_error: true diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 5771af9..1d2c25a 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -53,8 +53,9 @@ jobs: # Compute the OS list here in the strategy block, where `github.*` IS in scope. # `matrix` context isn't available in job-level `if:`, so we compute the # OS list right here: - # pull_request -> ubuntu-latest + ubuntu-24.04-arm + macos-latest - # + windows-latest (skip slow Intel Mac; + # pull_request -> ubuntu-24.04-arm + macos-latest + windows-latest + # (ubuntu-latest x64 is exercised by coverage.yaml, so + # it's dispatch-only here; skip slow Intel Mac; # ubuntu-26.04 + windows-11-arm are dispatch-only) # workflow_dispatch -> the OS picked in the dropdown, or all of them # The combos are emitted as a fully-formed list of `{os, timeout}` @@ -81,8 +82,7 @@ jobs: || github.event.inputs.os == 'windows-latest' && '[{"os":"windows-latest","timeout":120}]' || github.event.inputs.os == 'windows-11-arm' && '[{"os":"windows-11-arm","timeout":60}]' ) - || format('[{0},{1},{2},{3}]', - '{"os":"ubuntu-latest","timeout":40}', + || format('[{0},{1},{2}]', '{"os":"ubuntu-24.04-arm","timeout":40}', '{"os":"macos-latest","timeout":45}', '{"os":"windows-latest","timeout":120}') diff --git a/.mise/config.coverage.toml b/.mise/config.coverage.toml new file mode 100644 index 0000000..15953b1 --- /dev/null +++ b/.mise/config.coverage.toml @@ -0,0 +1,21 @@ +# Coverage env (MISE_ENV includes `coverage`). +# A special stackable env like mingw/msvc, not a language -- so it is deliberately NOT in config.toml's ALL_LANGS +# and the *-all tasks never touch it. Selected only by the coverage workflow, stacked on the language envs it also +# loads (rust + the guest langs whose ws-modules the coverage build exercises, so the module-loading tests run +# rather than self-skip). Houses the cargo-llvm-cov task so it stays out of the always-loaded config.toml. +# cargo-llvm-cov is not a mise [tool] -- the workflow co-installs it via taiki-e/install-action -- so there is no +# [tools] table here. + +[tasks.cargo-llvm-cov] +description = "Rust coverage via cargo-llvm-cov; emits lcov.info for Codecov" +# cargo-llvm-cov is deliberately NOT a mise [tool] (like cargo-unmaintained). +# The coverage workflow installs it via taiki-e/install-action; this task just runs the binary. It needs the +# `llvm-tools-preview` rustup component, which cargo-llvm-cov adds on demand against the mise-managed toolchain. +# Mirrors cargo-test's split -- workspace minus et-ws-web-runner, then et-ws-web-runner serialized (same +# RUST_TEST_THREADS=1 as test-ws-web-runner) -- with --no-report so both runs merge into one lcov report. +run = """ +cargo llvm-cov --no-report --workspace --exclude et-ws-web-runner +RUST_TEST_THREADS=1 cargo llvm-cov --no-report -p et-ws-web-runner +cargo llvm-cov report --lcov --output-path lcov.info +""" +shell = "bash -euo pipefail -c" From c95ef6baa90c2db1c657b271193fdf35f0c1dd6e Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Wed, 8 Jul 2026 08:33:47 +0800 Subject: [PATCH 2/9] fixes --- .../actions/install-mise-tools/action.yaml | 2 +- .github/workflows/docker-linux.yaml | 4 + .github/workflows/docker-windows.yaml | 8 +- .github/workflows/upstream-cache.yaml | 20 ++- .mise/config.toml | 19 +- CLAUDE.md | 18 ++ Cargo.lock | 1 + config/conftest/policy/gha_uses/gha_uses.rego | 48 ++++++ config/deny.toml | 6 + config/taplo/no-banned-deps.schema.json | 3 + generated/zig-rest/src/et_rest_client.zig | 20 ++- utilities/int-gen/Cargo.toml | 2 + .../int-gen/src/zig.in/request_raw_body.zig | 5 + utilities/int-gen/src/zig.rs | 163 ++++++++++++------ 14 files changed, 249 insertions(+), 70 deletions(-) create mode 100644 config/conftest/policy/gha_uses/gha_uses.rego diff --git a/.github/actions/install-mise-tools/action.yaml b/.github/actions/install-mise-tools/action.yaml index 017b1c9..f18d88a 100644 --- a/.github/actions/install-mise-tools/action.yaml +++ b/.github/actions/install-mise-tools/action.yaml @@ -29,7 +29,7 @@ runs: - name: Install mise binary uses: ./.github/actions/install-mise with: - install-action: ${{ inputs.install-action-tools }} + install-action-tools: ${{ inputs.install-action-tools }} # Optional npm backend, installed before the main `mise install`. # Only useful when js env is loaded (it's the backend for npm:* tools, all of which live in config.js.toml); skip diff --git a/.github/workflows/docker-linux.yaml b/.github/workflows/docker-linux.yaml index 1ea771a..de1dd30 100644 --- a/.github/workflows/docker-linux.yaml +++ b/.github/workflows/docker-linux.yaml @@ -89,6 +89,8 @@ jobs: run: | args="--target test --build-arg BASE_IMAGE --build-arg MISE_ENV" args="$args --secret id=gh_token,env=GITHUB_TOKEN -t et-test" + # $args is a word-split flag list by design; do not quote it. + # shellcheck disable=SC2086 docker build $args . # The check stage needs `.git/` + every tracked Dockerfile. @@ -110,6 +112,8 @@ jobs: git ls-files '*Dockerfile' '*Dockerfile.*' | xargs -I{} cp --parents {} "$CTX/" args="--target check --build-arg BASE_IMAGE --build-arg MISE_ENV" args="$args --build-context extras=$CTX --secret id=gh_token -t $TAG" + # $args is a word-split flag list by design; do not quote it. + # shellcheck disable=SC2086 docker build $args . # Capture image + intermediate stage sizes after the builds. diff --git a/.github/workflows/docker-windows.yaml b/.github/workflows/docker-windows.yaml index a88ca25..6d36c8c 100644 --- a/.github/workflows/docker-windows.yaml +++ b/.github/workflows/docker-windows.yaml @@ -108,6 +108,8 @@ jobs: args="$args --build-arg WINDOWS_VERSION=${{ matrix.windows_version }}" fi args="$args --target build-minimal -t $IMAGE_TAG" + # $args is a word-split flag list by design; do not quote it. + # shellcheck disable=SC2086 docker build $args . # Diagnose whether the et-rp (rustpython) binary actually runs inside this base image. @@ -139,10 +141,10 @@ jobs: DOCKERFILE: ${{ steps.dockerfile.outputs.dockerfile }} run: | if [ "${{ matrix.base }}" = "servercore" ]; then - target=test + target="test" tag="-t et-windows-test" else - target=precompile + target="precompile" tag="" fi args="-f $DOCKERFILE --build-arg MISE_ENV" @@ -150,6 +152,8 @@ jobs: args="$args --build-arg WINDOWS_VERSION=${{ matrix.windows_version }}" fi args="$args --target $target $tag" + # $args is a word-split flag list by design; do not quote it. + # shellcheck disable=SC2086 docker build $args . # Run the test suite split the same way docker-linux does. diff --git a/.github/workflows/upstream-cache.yaml b/.github/workflows/upstream-cache.yaml index 69bd5c6..d733166 100644 --- a/.github/workflows/upstream-cache.yaml +++ b/.github/workflows/upstream-cache.yaml @@ -282,6 +282,8 @@ jobs: run: | sudo apt-get update pkgs="autoconf automake bison flex libtool-bin libxml2-dev libreadline-dev pkg-config" + # $pkgs is a word-split package list by design; do not quote it. + # shellcheck disable=SC2086 sudo apt-get install -y --no-install-recommends $pkgs # On macOS, brew the build deps, mirroring the fork's build.yml `macos` job. @@ -334,6 +336,8 @@ jobs: installdir="$RUNNER_TEMP/install" mkdir -p "$installdir" clone_args="--depth 1 --branch $AUG_REF --recurse-submodules" + # $clone_args is a word-split flag list by design; do not quote it. + # shellcheck disable=SC2086 git clone $clone_args "$AUG_FORK_REPO" "$srcdir" cd "$srcdir" if [ "$RUNNER_OS" = "macOS" ]; then @@ -347,6 +351,8 @@ jobs: fi autogen_args="--prefix=$installdir --disable-static --disable-gnulib-tests" autogen_args="$autogen_args --disable-dependency-tracking" + # $autogen_args is a word-split flag list by design; do not quote it. + # shellcheck disable=SC2086 ./autogen.sh $autogen_args make -j"$(nproc 2>/dev/null || sysctl -n hw.ncpu)" make install @@ -361,6 +367,8 @@ jobs: sudo apt-get install -y --no-install-recommends patchelf for f in bin/* lib/lib*.so*; do [ -f "$f" ] && [ ! -L "$f" ] || continue + # '$ORIGIN' is a literal rpath token for the dynamic linker; single quotes are intentional. + # shellcheck disable=SC2016 patchelf --set-rpath '$ORIGIN/../lib' "$f" 2>/dev/null || true done else @@ -397,6 +405,8 @@ jobs: # otool -L on augtool tells us # which absolute brew paths got baked in; copy each, rewrite # its install_name + nested refs, and rewrite consumers. + # $1 is awk's field reference inside a single-quoted awk program, not a shell expansion. + # shellcheck disable=SC2016 brew_filter='NR>1 && $1 ~ /^\/(usr\/local|opt\/homebrew)\// {print $1}' brew_deps_otool=$(otool -L bin/augtool lib/libaugeas*.dylib lib/libfa*.dylib 2>/dev/null) queue=$(echo "$brew_deps_otool" | awk "$brew_filter" | sort -u) @@ -542,9 +552,11 @@ jobs: echo "::notice::$asset already in $DTG_RELEASE_TAG; skipping" echo "work=" >> "$GITHUB_OUTPUT" else - echo "work=yes" >> "$GITHUB_OUTPUT" - echo "triple=$triple" >> "$GITHUB_OUTPUT" - echo "asset=$asset" >> "$GITHUB_OUTPUT" + { + echo "work=yes" + echo "triple=$triple" + echo "asset=$asset" + } >> "$GITHUB_OUTPUT" fi # No explicit setup-rust step is needed for this build. @@ -564,6 +576,8 @@ jobs: # otherwise pick gnullvm and trip the same build failure we're working around. cargo_args="--version $DTG_VERSION --root $installdir" cargo_args="$cargo_args --target x86_64-pc-windows-msvc --locked" + # $cargo_args is a word-split flag list by design; do not quote it. + # shellcheck disable=SC2086 cargo install dart-typegen $cargo_args ls "$installdir/bin" # `cd` into the dir and tar a relative path to avoid the colon. diff --git a/.mise/config.toml b/.mise/config.toml index 998e7f8..b9f720d 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -57,6 +57,7 @@ action-validator = { version = "latest", os = ["linux", "macos"] } "aqua:EmbarkStudios/cargo-deny" = "latest" "aqua:StyraInc/regal" = "latest" "aqua:koalaman/shellcheck" = "latest" +"aqua:rhysd/actionlint" = "latest" "aqua:rustwasm/wasm-pack" = "latest" "aqua:vectordotdev/vector" = "0.56.0" ast-grep = "latest" @@ -454,6 +455,7 @@ description = "Run all formatters: fmt:rust + any loaded guest fmt:" # loaded MISE_ENV configs). depends = [ "action-validator-check", + "actionlint-check", "ast-grep-check", "cargo-check", "cargo-clippy-check", @@ -507,6 +509,16 @@ description = "Apply all lint-fix passes: fix:rust + any loaded guest fix: description = "Validate GitHub Actions workflow + composite-action YAML" run = "action-validator .github/workflows/*.yaml .github/actions/*/action.yaml" +[tasks.actionlint-check] +description = "Lint GitHub Actions workflows (actionlint): local-action input validation, expression syntax, shellcheck" +# Lint every workflow with actionlint's native checks plus its shellcheck integration over each `run:` script. +# shellcheck is a mise [tool], so it is on PATH here and actionlint auto-detects and invokes it. The most +# load-bearing native rule is `[action]`, which fails when a workflow step passes a `with:` key the referenced +# local action does not declare (GHA itself only warns). actionlint lints workflows only, so it does NOT see +# composite-action -> action calls; the gha_uses conftest rule covers that direction. Intentional word-splitting +# sites carry an inline `# shellcheck disable=SC2086` directive in the run script, as the mise-task bodies do. +run = "actionlint .github/workflows/*.yaml" + [tasks.zizmor-check] description = "Audit GitHub Actions workflows + composite actions for security issues (zizmor)" # --offline so it needs no GH token and stays deterministic. @@ -612,9 +624,11 @@ description = "Run conftest OPA/Rego policies over the GitHub Actions workflow + # workflow YAML for cross-cutting invariants where the rule has to compare each workflow's parsed body # against its own filename (the `paths:` self-reference check; conftest's per-file mode has no `input.path`); # gha_action over the composite actions under .github/actions/*/action.yaml (different schema, so its -# own namespace); and gha_mise over the workflow YAML combined with .mise (auto-detected parsers) so a rule can +# own namespace); gha_mise over the workflow YAML combined with .mise (auto-detected parsers) so a rule can # cross-check a hard-coded action-input version against the matching mise [tools] pin (setup-dotnet's -# dotnet-version vs the `dotnet` pin). +# dotnet-version vs the `dotnet` pin); and gha_uses over the workflow YAML combined with the composite actions so a +# rule can resolve each `uses: ./.github/actions/` against that action's declared `inputs:` (GHA only warns on +# an undeclared input key, silently dropping the value). run = """ ns="--namespace gha --namespace no_trailing_backslash" # $ns expands to repeated `--namespace ` pairs; word-splitting is intentional. @@ -623,6 +637,7 @@ conftest test --parser yaml $ns -p config/conftest/policy .github/workflows conftest test --parser yaml --combine --namespace gha_combined -p config/conftest/policy .github/workflows conftest test --parser yaml --namespace gha_action -p config/conftest/policy .github/actions/*/action.yaml conftest test --combine --namespace gha_mise -p config/conftest/policy .github/workflows .mise +conftest test --parser yaml --combine --namespace gha_uses -p config/conftest/policy .github/workflows .github/actions """ shell = "bash -euo pipefail -c" diff --git a/CLAUDE.md b/CLAUDE.md index 82df03d..5035374 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -25,6 +25,24 @@ Reason: agents are bad at zsh-specific quoting and expansion rules, and the syst bash everyone else uses. Pinning to homebrew bash makes ad-hoc shell behavior predictable and matches what mise tasks already use. +## Ad-hoc agent commands: use mise-managed tools, or recommend adding one + +The "Don't depend on host tools" rule further down is written for mise task bodies, but the same discipline applies to +the ad-hoc commands the agent runs while working -- investigations, scratch-area probes, one-off transforms. Prefer the +repo's mise-managed, version-pinned, cross-platform tools -- `coreutils` (uutils multicall), `rg` (ripgrep), +`find`/`xargs` (uutils findutils), `goawk`, and whatever else the `[tools]` tables pin -- over whatever binary happens +to be on the host (`perl`, BSD `sed`, a random CLI). A host binary may be absent, a different version, or missing on +another OS, and its shell-quoting is the exact fragility the homebrew-bash rule exists to avoid (a mis-escaped +`perl -pi -e` once silently no-op'd its edit and produced a bogus "passing" test result here before it was caught). + +For file edits specifically, reach for the Edit/Write tools rather than a stream editor (`perl -pi`, `sed -i`): they +never touch the shell, so there is no quoting to get wrong. + +If you genuinely need a tool that is not installed (or is not mise-managed) to do the job well, do NOT quietly fall +back to the host binary -- surface it. Recommend to the user that the tool be added as a mise `[tools]` entry +(prebuilt-binary backend, cross-platform, per the "Tools must work on every OS" rule), and let them decide. A tool +worth using in this repo is worth pinning. + ## Polling cadence when monitoring a PR's CI runs When an agent is watching a PR (e.g. via `/loop`), the next-poll delay follows three tiers -- tight at first (catch diff --git a/Cargo.lock b/Cargo.lock index 05e54ec..a9a24b4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4122,6 +4122,7 @@ dependencies = [ "prettyplease", "progenitor", "quote", + "regex", "reqwest 0.13.4", "schemars 1.2.1", "semver", diff --git a/config/conftest/policy/gha_uses/gha_uses.rego b/config/conftest/policy/gha_uses/gha_uses.rego new file mode 100644 index 0000000..7af5df8 --- /dev/null +++ b/config/conftest/policy/gha_uses/gha_uses.rego @@ -0,0 +1,48 @@ +# Cross-file check: every input passed to a LOCAL composite action must be one that the action declares. +# Run with `--namespace gha_uses` over the workflow YAML plus every .github/actions/*/action.yaml, combined +# (--combine) so a single evaluation resolves each `uses: ./.github/actions/` against that action's `inputs:`. +# GitHub Actions only emits a *warning* -- never an error -- when a caller passes a `with:` key the target action +# does not declare, so a typo'd input name silently defaults to "" and the intended value is dropped. (Concrete +# regression: install-mise-tools passed `install-action:` to install-mise, whose input is `install-action-tools`, +# so co-installed tools like cargo-llvm-cov silently never installed and the coverage job failed with +# `no such command: llvm-cov`.) This promotes that class of typo to a hard failure. +package gha_uses + +# action dir name -> set of declared input keys, one entry per parsed .github/actions//action.yaml. +declared_inputs[name] := names if { + some file in input + parts := split(file.path, "/") + n := count(parts) + parts[n - 1] == "action.yaml" + name := parts[n - 2] + names := object.keys(object.get(file.contents, "inputs", {})) +} + +# {path, step} for every step in a workflow job (jobs..steps[]) ... +caller_steps contains entry if { + some file in input + some job in file.contents.jobs + some step in job.steps + entry := {"path": file.path, "step": step} +} + +# ... and every step in a composite action's runs block (runs.steps[]). +caller_steps contains entry if { + some file in input + some step in file.contents.runs.steps + entry := {"path": file.path, "step": step} +} + +deny contains msg if { + some entry in caller_steps + step := entry.step + startswith(step.uses, "./.github/actions/") + name := trim_prefix(step.uses, "./.github/actions/") + declared := declared_inputs[name] + some key, _ in step.with + not declared[key] + msg := sprintf( + "%s: step passes input %q to local action %q, which declares no such input (GHA only warns -- treat as error)", + [entry.path, key, name], + ) +} diff --git a/config/deny.toml b/config/deny.toml index b1b7463..a8363cb 100644 --- a/config/deny.toml +++ b/config/deny.toml @@ -152,6 +152,12 @@ deny = [ # parent (`ort-sys` via the `download-binaries` feature) is gone. The workspace `ort` dep now stages ONNX # Runtime through mise instead. { crate = "ureq" }, + # `regress` (a JS-flavoured regex engine) is forbidden except via the wrapper below. + # It arrives only transitively through progenitor -> typify -> typify-impl, which uses it to validate the + # JSON-Schema `pattern` regexes in our generated REST client. We standardise on the `regex` crate everywhere + # else; any new parent that drags `regress` back in should migrate to `regex`. Direct workspace declaration is + # separately blocked by config/taplo/no-banned-deps.schema.json. + { crate = "regress", use-instead = "regex", wrappers = ["typify-impl"] }, ] [sources] diff --git a/config/taplo/no-banned-deps.schema.json b/config/taplo/no-banned-deps.schema.json index b469855..747d54c 100644 --- a/config/taplo/no-banned-deps.schema.json +++ b/config/taplo/no-banned-deps.schema.json @@ -15,6 +15,9 @@ "openssl-sys": { "const": "forbidden; use rustls + aws-lc-rs. One TLS/crypto stack only." }, + "regress": { + "const": "forbidden; use the regex crate. Transitive via typify-impl only (gated in config/deny.toml)." + }, "ring": { "const": "forbidden; use aws-lc-rs. Transitive via rcgen only (gated in config/deny.toml's bans.deny)." }, diff --git a/generated/zig-rest/src/et_rest_client.zig b/generated/zig-rest/src/et_rest_client.zig index d867df3..6c48711 100644 --- a/generated/zig-rest/src/et_rest_client.zig +++ b/generated/zig-rest/src/et_rest_client.zig @@ -129,10 +129,19 @@ fn appendQueryParam(writer: *std.Io.Writer, first_query: *bool, name: []const u8 } pub fn requestRaw(client: *Client, method: std.http.Method, url: []const u8, payload: ?[]const u8) !RawResponse { + return requestRawWithContentType(client, method, url, payload, "application/json"); +} + +pub fn requestRawWithContentType(client: *Client, method: std.http.Method, url: []const u8, payload: ?[]const u8, content_type_value: []const u8) !RawResponse { // Replaced by et-int-gen: dispatch via the host JS shim instead of // `std.http.Client.fetch`, which can't reach the network from // browser wasm. The JS side proxies to `fetch()` via // SharedArrayBuffer + Atomics so this stays synchronous in Zig. + // This is the single shared request function; `requestRaw` and every + // per-operation wrapper (including binary bodies rerouted by et-int-gen) + // funnel through here. `content_type_value` is carried in the signature + // for source compatibility but the JS shim derives it from the payload. + _ = content_type_value; const allocator = client.allocator; const method_str = @tagName(method); const body_slice = payload orelse ""; @@ -316,7 +325,7 @@ fn streamJson(client: *Client, path: []const u8, requestBody: anytype, callback: var headers = std.ArrayList(std.http.Header).empty; defer headers.deinit(allocator); - const auth_header = try appendClientHeaders(allocator, &headers, client, true, "text/event-stream"); + const auth_header = try appendClientHeaders(allocator, &headers, client, "application/json", "text/event-stream"); defer if (auth_header) |value| allocator.free(value); const url = try std.fmt.allocPrint(allocator, "{s}{s}", .{ client.base_url, path }); @@ -347,9 +356,9 @@ fn streamJson(client: *Client, path: []const u8, requestBody: anytype, callback: }; } -fn appendClientHeaders(allocator: std.mem.Allocator, headers: *std.ArrayList(std.http.Header), client: *Client, include_content_type: bool, accept: []const u8) !?[]u8 { - if (include_content_type) { - try headers.append(allocator, .{ .name = "Content-Type", .value = "application/json" }); +fn appendClientHeaders(allocator: std.mem.Allocator, headers: *std.ArrayList(std.http.Header), client: *Client, content_type: ?[]const u8, accept: []const u8) !?[]u8 { + if (content_type) |ct| { + try headers.append(allocator, .{ .name = "Content-Type", .value = ct }); } try headers.append(allocator, .{ .name = "Accept", .value = accept }); @@ -480,10 +489,9 @@ pub fn put_fileRaw(client: *Client, agent_id: []const u8, filename: []const u8, var uri_buf: std.Io.Writer.Allocating = .init(allocator); defer uri_buf.deinit(); try uri_buf.writer.print("{s}/storage/{s}/{s}", .{ client.base_url, agent_id, filename }); - const payload: ?[]const u8 = requestBody; - return requestRaw(client, std.http.Method.PUT, uri_buf.written(), payload); + return requestRawWithContentType(client, std.http.Method.PUT, uri_buf.written(), payload, "application/octet-stream"); } ///////////////// diff --git a/utilities/int-gen/Cargo.toml b/utilities/int-gen/Cargo.toml index 828d731..ae94c05 100644 --- a/utilities/int-gen/Cargo.toml +++ b/utilities/int-gen/Cargo.toml @@ -29,6 +29,8 @@ pretty_yaml.workspace = true prettyplease.workspace = true progenitor.workspace = true quote.workspace = true +# Workspace regex has default-features off; `std` is what provides the `Regex` type used by the zig post-processor. +regex = { workspace = true, features = ["std"] } reqwest = { workspace = true, features = ["blocking", "rustls"] } schemars.workspace = true semver.workspace = true diff --git a/utilities/int-gen/src/zig.in/request_raw_body.zig b/utilities/int-gen/src/zig.in/request_raw_body.zig index 045ef87..b75833c 100644 --- a/utilities/int-gen/src/zig.in/request_raw_body.zig +++ b/utilities/int-gen/src/zig.in/request_raw_body.zig @@ -3,6 +3,11 @@ // `std.http.Client.fetch`, which can't reach the network from // browser wasm. The JS side proxies to `fetch()` via // SharedArrayBuffer + Atomics so this stays synchronous in Zig. + // This is the single shared request function; `requestRaw` and every + // per-operation wrapper (including binary bodies rerouted by et-int-gen) + // funnel through here. `content_type_value` is carried in the signature + // for source compatibility but the JS shim derives it from the payload. + _ = content_type_value; const allocator = client.allocator; const method_str = @tagName(method); const body_slice = payload orelse ""; diff --git a/utilities/int-gen/src/zig.rs b/utilities/int-gen/src/zig.rs index 9a6c830..a3ae011 100644 --- a/utilities/int-gen/src/zig.rs +++ b/utilities/int-gen/src/zig.rs @@ -1,31 +1,51 @@ //! Post-process `openapi2zig`'s output into a `wasm32-unknown-unknown`- //! compatible Zig REST client. //! -//! `openapi2zig` generates a fully typed client whose `requestRaw` calls -//! `std.http.Client.fetch` -- which compiles for `wasm32-freestanding` but -//! can't actually reach the network from a browser sandbox. We swap the -//! body of `requestRaw` for one that delegates to a single -//! `extern fn js_rest_request(...)` import (host-implemented via `fetch()` -//! and `SharedArrayBuffer` in the JS shim), and append the extern -//! declaration. Everything else -- schemas, `RawResponse`, `ApiResult`, -//! per-operation wrappers, SSE helpers -- is left untouched; Zig's lazy -//! evaluation + dead-code elimination shake out the now-unused +//! `openapi2zig` generates a fully typed client that reaches the network +//! through `std.http.Client.fetch` -- which compiles for +//! `wasm32-freestanding` but can't actually reach the network from a +//! browser sandbox. As of openapi2zig 0.3 the emission has two shapes: +//! +//! * JSON / form / empty bodies route through a single shared +//! `requestRawWithContentType` (with `requestRaw` a thin default-`content-type` +//! wrapper over it), which holds the one `client.http.fetch` call. +//! * Binary / text bodies (e.g. `application/octet-stream`) *inline* their +//! own `client.http.fetch` in the per-operation `*Raw` function rather +//! than delegating. +//! +//! We funnel everything through one host import. First we swap the body of +//! the shared `requestRawWithContentType` for one that delegates to a single +//! `extern fn js_rest_request(...)` (host-implemented via `fetch()` and +//! `SharedArrayBuffer` in the JS shim). Then we rewrite each inlined binary +//! operation to delegate to that same shared function instead of calling +//! `client.http.fetch` directly. Finally we assert no reachable +//! `client.http.fetch` survived and append the extern declaration. +//! +//! Everything else -- schemas, `RawResponse`, `ApiResult`, per-operation +//! wrappers, the unreachable SSE `streamJson` helper -- is left untouched; +//! Zig's lazy evaluation + dead-code elimination shake out the now-unused //! `std.http.Client`/`std.Io` machinery (verified: the resulting wasm has //! a single `env.js_rest_request` import and is ~6 KB at `-O ReleaseSmall`). //! -//! We use `tree-sitter-zig` to find `requestRaw` by name rather than -//! string-matching its body -- that way `openapi2zig` version bumps that -//! reshuffle the implementation don't break us. +//! We use `tree-sitter-zig` to find `requestRawWithContentType` by name +//! rather than string-matching its body -- that way `openapi2zig` version +//! bumps that reshuffle the implementation don't break us. use std::path::Path; use std::process::Command; use fs_err as fs; +use regex::Regex; use tree_sitter::{Node, Parser}; use crate::Error; -/// The replacement body spliced into `requestRaw` by [`rewrite`]. +/// Name of the single shared request function whose body we replace with the +/// host-import dispatch; `requestRaw` and every per-operation wrapper funnel +/// through it. +const SHARED_REQUEST_FN: &str = "requestRawWithContentType"; + +/// The replacement body spliced into [`SHARED_REQUEST_FN`] by [`rewrite`]. const REQUEST_RAW_BODY: &str = include_str!("zig.in/request_raw_body.zig"); /// The `extern fn js_rest_request(...)` declaration appended to the generated client by [`rewrite`]. @@ -79,11 +99,11 @@ fn run_openapi2zig(rest_json: &Path, raw_out: &Path) -> Result<(), Error> { Ok(()) } -/// Replace `requestRaw`'s body with our extern-backed implementation, fix -/// the JSON-encoding of binary request bodies (openapi2zig blindly applies -/// `std.json.Stringify.value` even when the `OpenAPI` content-type is -/// `application/octet-stream`), and append the `extern fn js_rest_request` -/// declaration. Everything else passes through verbatim. +/// Replace the shared request function's body with our extern-backed +/// implementation, reroute the inlined binary operations through it, assert +/// no reachable `client.http.fetch` survived, and append the +/// `extern fn js_rest_request` declaration. Everything else passes through +/// verbatim. #[expect( clippy::single_call_fn, reason = "named helper; pairs with run_openapi2zig() as the two halves of render()" @@ -95,11 +115,11 @@ fn rewrite(source: &str) -> Result { .parse(source, None) .ok_or_else(|| Error::ZigCodegen("tree-sitter parse returned None".into()))?; - let request_raw = find_fn(tree.root_node(), source, "requestRaw") - .ok_or_else(|| Error::ZigCodegen("requestRaw function not found in openapi2zig output".into()))?; - let body = request_raw + let shared_fn = find_fn(tree.root_node(), source, SHARED_REQUEST_FN) + .ok_or_else(|| Error::ZigCodegen(format!("{SHARED_REQUEST_FN} function not found in openapi2zig output")))?; + let body = shared_fn .child_by_field_name("body") - .ok_or_else(|| Error::ZigCodegen("requestRaw has no body field".into()))?; + .ok_or_else(|| Error::ZigCodegen(format!("{SHARED_REQUEST_FN} has no body field")))?; let body_start = body.start_byte(); let body_end = body.end_byte(); @@ -120,49 +140,80 @@ fn rewrite(source: &str) -> Result { out.push_str(REQUEST_RAW_BODY.trim_end()); out.push_str(&source[body_end..]); } - out = fix_binary_request_body(&out)?; + out = reroute_inline_binary_ops(&out)?; + if out.contains("client.http.fetch") { + return Err(Error::ZigCodegen( + concat!( + "openapi2zig output still contains a reachable `client.http.fetch` after rewriting ", + "- an operation shape changed; it would attempt real HTTP from wasm. Verify the codegen.", + ) + .into(), + )); + } out.push_str(JS_REST_REQUEST_EXTERN); Ok(out) } -/// Replace openapi2zig's `std.json.Stringify.value(requestBody, ...)` block -/// with a direct `requestBody` pass-through. +/// Reroute openapi2zig's inlined binary/text-body operations through the +/// shared, extern-backed request function. +/// +/// Since openapi2zig 0.3, an operation whose request body is not JSON/form +/// (e.g. `application/octet-stream`) does not delegate to +/// `requestRawWithContentType`; it inlines the whole +/// header-build + `std.Uri.parse` + `client.http.fetch` + `RawResponse` +/// dance directly (`src/generators/unified/api_generator.zig`'s `.binary`/ +/// `.text` arm). That inlined `client.http.fetch` would try to reach the +/// network from wasm. We rewrite the inlined tail back into a single +/// `return requestRawWithContentType(...)` call so it funnels through the +/// host import like every other operation; the captured method and +/// content-type are preserved. The `rewrite` caller then asserts no +/// `client.http.fetch` survived, so an emission shape we don't recognise +/// fails loudly rather than silently emitting a real-HTTP path. #[expect( clippy::single_call_fn, reason = "named helper called once by rewrite(); kept separate for the long comment + clear scope" )] -fn fix_binary_request_body(source: &str) -> Result { - // openapi2zig generates the `bad` block for every operation that has a - // `requestBody`, regardless of content-type. - // - // For `application/octet-stream` endpoints (every body in our spec) this - // corrupts the wire bytes -- `"Hello"` ships as `"\"Hello\""`. Replace - // the block with a direct `requestBody` pass-through. If openapi2zig - // changes the pattern we fail loudly rather than silently emitting - // JSON-encoded bodies. - // - // Upstream bug: - // (emission sites: `src/generators/unified/api_generator.zig:611-617` and - // `:1431-1436`, both unconditional on content-type). Drop this workaround - // once the issue is fixed and we bump the pinned openapi2zig version. - let bad = concat!( - " var str: std.Io.Writer.Allocating = .init(allocator);\n", - " defer str.deinit();\n", - " try std.json.Stringify.value(requestBody, .{ .emit_null_optional_fields = false }, &str.writer);\n", - " const payload: ?[]const u8 = str.written();", - ); - let good = " const payload: ?[]const u8 = requestBody;"; - let count = source.matches(bad).count(); - if count == 0 { - return Err(Error::ZigCodegen( - concat!( - "openapi2zig output no longer contains the std.json.Stringify(requestBody) pattern ", - "- its body-encoding may have changed; verify before relying on the binary fix-up", +fn reroute_inline_binary_ops(source: &str) -> Result { + // The inlined block openapi2zig emits after `const payload = requestBody;`: + // build headers (the 4th `appendClientHeaders` arg is the request + // content-type), parse the pre-built `uri_buf`, allocate a response + // writer, `client.http.fetch`, and assemble a `RawResponse`. The two + // capture groups pull out the content-type and the `std.http.Method.*` + // literal so the delegating call reproduces them. `[A-Z]+` (not `\w`) + // keeps the pattern ASCII so no unicode regex feature is required. + let inline_block = Regex::new(concat!( + r#"\n var headers = std\.ArrayList\(std\.http\.Header\)\.empty;\n"#, + r#" defer headers\.deinit\(allocator\);\n"#, + r#" const auth_header = try appendClientHeaders\(allocator, &headers, client, "#, + r#""([^"]*)", "application/json"\);\n"#, + r#" defer if \(auth_header\) \|value\| allocator\.free\(value\);\n\n"#, + r#" const uri = try std\.Uri\.parse\(uri_buf\.written\(\)\);\n"#, + r#" var response_body: std\.Io\.Writer\.Allocating = \.init\(allocator\);\n"#, + r#" defer response_body\.deinit\(\);\n\n"#, + r#" const result = try client\.http\.fetch\(\.\{\n"#, + r#" \.location = \.\{ \.uri = uri \},\n"#, + r#" \.method = (std\.http\.Method\.[A-Z]+),\n"#, + r#" \.extra_headers = headers\.items,\n"#, + r#" \.payload = payload,\n"#, + r#" \.response_writer = &response_body\.writer,\n"#, + r#" \}\);\n\n"#, + r#" return \.\{\n"#, + r#" \.allocator = allocator,\n"#, + r#" \.status = result\.status,\n"#, + r#" \.body = try response_body\.toOwnedSlice\(\),\n"#, + r#" \};"#, + )) + .map_err(|err| Error::ZigCodegen(format!("inline-binary-op regex failed to compile: {err}")))?; + + Ok(inline_block + .replace_all(source, |caps: ®ex::Captures<'_>| { + let content_type = &caps[1]; + let method = &caps[2]; + format!( + "\n return {SHARED_REQUEST_FN}(client, {method}, uri_buf.written(), payload, \"{content_type}\");" ) - .into(), - )); - } - Ok(source.replace(bad, good)) + }) + .into_owned()) } /// Recursive walk: return the first `function_declaration` whose `name` From c50da17b7483cfbf493fd87686a7d08573366844 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Wed, 8 Jul 2026 09:29:41 +0800 Subject: [PATCH 3/9] fixes --- .codacy.yaml | 7 +++ .github/workflows/coverage.yaml | 16 +++++- .mise/config.coverage.toml | 24 +++++---- CLAUDE.md | 12 ++++- .../ast-grep/rules/no-concat-raw-strings.yaml | 17 +++++++ config/nextest.toml | 17 +++++++ utilities/int-gen/Cargo.toml | 1 - utilities/int-gen/src/lib.rs | 2 + utilities/int-gen/src/zig.rs | 49 ++++++------------- 9 files changed, 96 insertions(+), 49 deletions(-) create mode 100644 config/ast-grep/rules/no-concat-raw-strings.yaml create mode 100644 config/nextest.toml diff --git a/.codacy.yaml b/.codacy.yaml index 64bea9d..5c23482 100644 --- a/.codacy.yaml +++ b/.codacy.yaml @@ -5,7 +5,14 @@ # deliberately ignores and the repo's own hadolint-check + DeepSource Docker already cover. # - dotnet-data1/Program.cs: Codacy's CA1054 wants URI-typed params, but [JSImport] partial methods can only # marshal `string` across the JS boundary; DeepSource covers the C# there and reports no such issue. +# - .github/workflows/ + .github/actions/: Codacy flags GitHub Actions `uses:` not pinned to a commit SHA, but +# this repo deliberately pins actions by tag (config/zizmor.yaml turns off zizmor's matching hash-pin policy for +# the same reason). Workflows + composite actions are already linted by actionlint, zizmor, action-validator, the +# ast-grep gha rules, and the gha* conftest policies; since Codacy cannot disable an individual pattern in-repo, +# excluding these trees is the only way to silence its pin rule globally. exclude_paths: + - ".github/actions/**" + - ".github/workflows/**" - "Dockerfile" - "config/semgrep/**" - "services/ws-modules/dotnet-data1/Program.cs" diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index 589ed36..65447c8 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -52,7 +52,7 @@ jobs: timeout-minutes: 20 with: github-token: ${{ github.token }} - install-action-tools: cargo-llvm-cov + install-action-tools: cargo-llvm-cov,cargo-nextest - name: Prefetch dependencies timeout-minutes: 15 @@ -66,7 +66,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: mise run build-modules - - name: Collect Rust coverage + - name: Collect Rust coverage + test results timeout-minutes: 75 run: mise run cargo-llvm-cov @@ -76,3 +76,15 @@ jobs: use_oidc: true files: lcov.info fail_ci_if_error: true + + # Upload test results even when tests failed. + # nextest still wrote the JUnit report, and a failing run is exactly when the per-test results on Codecov + # matter most; `report_type: test_results` selects the JUnit upload path. + - name: Upload test results to Codecov (OIDC) + if: ${{ !cancelled() }} + uses: codecov/codecov-action@v5 + with: + use_oidc: true + report_type: test_results + files: target/nextest/ci/junit.xml + fail_ci_if_error: true diff --git a/.mise/config.coverage.toml b/.mise/config.coverage.toml index 15953b1..11e3555 100644 --- a/.mise/config.coverage.toml +++ b/.mise/config.coverage.toml @@ -3,19 +3,23 @@ # and the *-all tasks never touch it. Selected only by the coverage workflow, stacked on the language envs it also # loads (rust + the guest langs whose ws-modules the coverage build exercises, so the module-loading tests run # rather than self-skip). Houses the cargo-llvm-cov task so it stays out of the always-loaded config.toml. -# cargo-llvm-cov is not a mise [tool] -- the workflow co-installs it via taiki-e/install-action -- so there is no -# [tools] table here. +# cargo-llvm-cov and cargo-nextest are not mise [tools] -- the workflow co-installs them via +# taiki-e/install-action -- so there is no [tools] table here. [tasks.cargo-llvm-cov] -description = "Rust coverage via cargo-llvm-cov; emits lcov.info for Codecov" -# cargo-llvm-cov is deliberately NOT a mise [tool] (like cargo-unmaintained). -# The coverage workflow installs it via taiki-e/install-action; this task just runs the binary. It needs the -# `llvm-tools-preview` rustup component, which cargo-llvm-cov adds on demand against the mise-managed toolchain. -# Mirrors cargo-test's split -- workspace minus et-ws-web-runner, then et-ws-web-runner serialized (same -# RUST_TEST_THREADS=1 as test-ws-web-runner) -- with --no-report so both runs merge into one lcov report. +description = "Rust coverage (cargo-llvm-cov) + JUnit test results (nextest); emits lcov.info + junit.xml" +# cargo-llvm-cov and cargo-nextest are deliberately NOT mise [tools] (like cargo-unmaintained). +# The coverage workflow installs both via taiki-e/install-action; this task just runs them. cargo-llvm-cov needs +# the `llvm-tools-preview` rustup component, which it adds on demand against the mise-managed toolchain. +# Running the suite through nextest (NEXTEST_PROFILE=ci) emits target/nextest/ci/junit.xml for the Codecov +# test-results upload; et-ws-web-runner is serialized by the `web-runner` test-group in config/nextest.toml +# rather than RUST_TEST_THREADS, which nextest ignores. `--config-file` points at that config since it lives under +# config/ rather than the default .config/. nextest does not run doctests, so a separate --doc pass keeps their +# coverage; --no-report on both passes so they merge into one lcov report. +env = { NEXTEST_PROFILE = "ci" } run = """ -cargo llvm-cov --no-report --workspace --exclude et-ws-web-runner -RUST_TEST_THREADS=1 cargo llvm-cov --no-report -p et-ws-web-runner +cargo llvm-cov --no-report nextest --config-file config/nextest.toml +cargo llvm-cov --no-report --doc cargo llvm-cov report --lcov --output-path lcov.info """ shell = "bash -euo pipefail -c" diff --git a/CLAUDE.md b/CLAUDE.md index 5035374..9067fda 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -32,7 +32,7 @@ the ad-hoc commands the agent runs while working -- investigations, scratch-area repo's mise-managed, version-pinned, cross-platform tools -- `coreutils` (uutils multicall), `rg` (ripgrep), `find`/`xargs` (uutils findutils), `goawk`, and whatever else the `[tools]` tables pin -- over whatever binary happens to be on the host (`perl`, BSD `sed`, a random CLI). A host binary may be absent, a different version, or missing on -another OS, and its shell-quoting is the exact fragility the homebrew-bash rule exists to avoid (a mis-escaped +another OS, and its shell-quoting is the exact fragility the homebrew-bash rule exists to avoid (a mangled `perl -pi -e` once silently no-op'd its edit and produced a bogus "passing" test result here before it was caught). For file edits specifically, reach for the Edit/Write tools rather than a stream editor (`perl -pi`, `sed -i`): they @@ -45,6 +45,16 @@ worth using in this repo is worth pinning. ## Polling cadence when monitoring a PR's CI runs +**Watch at job/check granularity, never run granularity.** `gh run list` reports a _workflow run_ as `in_progress` +for as long as any job in its matrix is still going -- even when other jobs in that same run have **already failed**. +Polling only `gh run list` (or filtering to `status == "completed"`) therefore hides live failures and reads as +"still healthy" when the PR is already red. Use `gh pr checks ` as the primary signal: it lists every individual +check (`check (ubuntu-latest, 25)`, `Codacy ...`, `coverage`, external DeepSource/Codacy/Codecov statuses, ...) with +its own pass/fail/pending, so a failed matrix leg or external check shows immediately. Once a job shows `fail`, pull +its failing step with `gh api repos///actions/jobs//logs` -- that returns the completed job's log +even while the parent run is still `in_progress` (whereas `gh run view --log-failed` refuses until the whole run +finishes). Only after `gh pr checks` is clean is the PR actually green. + When an agent is watching a PR (e.g. via `/loop`), the next-poll delay follows three tiers -- tight at first (catch fail-fast errors), loose in the middle (long compile/test phases run on a 30-90 min scale), then loose again once everything's settled (waiting for the user to push): diff --git a/config/ast-grep/rules/no-concat-raw-strings.yaml b/config/ast-grep/rules/no-concat-raw-strings.yaml new file mode 100644 index 0000000..c5f8215 --- /dev/null +++ b/config/ast-grep/rules/no-concat-raw-strings.yaml @@ -0,0 +1,17 @@ +id: no-concat-raw-strings +language: Rust +severity: error +message: | + `concat!(...)` of raw-string (`r#"..."#`) fragments is forbidden -- it is an unreadable way to build one string + (a regex, an embedded template) out of a stack of `r#"..."#,` lines. Use a single multi-line raw string literal + instead, or a named `const`, or -- for a regex -- `(?x)` verbose mode so the pattern can span lines without + concatenation. Plain-string `concat!` (wrapping a long message, `concat!(env!(...), "...")`) is fine; only + raw-string fragments are banned. +rule: + all: + - pattern: concat!($$$ARGS) + - has: + kind: raw_string_literal + stopBy: end +ignores: + - generated/** diff --git a/config/nextest.toml b/config/nextest.toml new file mode 100644 index 0000000..ea93caf --- /dev/null +++ b/config/nextest.toml @@ -0,0 +1,17 @@ +# nextest profile used by the coverage workflow via `cargo llvm-cov nextest` (NEXTEST_PROFILE=ci). +# It writes a JUnit report to target/nextest/ci/junit.xml, which the coverage job uploads to Codecov as test +# results (report_type: test_results). Passed explicitly with `--config-file` since it lives here, not the +# default .config/nextest.toml. +[profile.ci.junit] +path = "junit.xml" + +# Serialize et-ws-web-runner's tests. +# Each boots a heavy deno_runtime MainWorker, and several at once exhaust fd/port budgets. `max-threads = 1` +# runs them one at a time while the rest of the workspace stays parallel -- nextest's equivalent of that +# suite's single-threaded run (nextest ignores RUST_TEST_THREADS). +[test-groups] +web-runner = { max-threads = 1 } + +[[profile.ci.overrides]] +filter = 'package(et-ws-web-runner)' +test-group = 'web-runner' diff --git a/utilities/int-gen/Cargo.toml b/utilities/int-gen/Cargo.toml index ae94c05..e2e8853 100644 --- a/utilities/int-gen/Cargo.toml +++ b/utilities/int-gen/Cargo.toml @@ -29,7 +29,6 @@ pretty_yaml.workspace = true prettyplease.workspace = true progenitor.workspace = true quote.workspace = true -# Workspace regex has default-features off; `std` is what provides the `Regex` type used by the zig post-processor. regex = { workspace = true, features = ["std"] } reqwest = { workspace = true, features = ["blocking", "rustls"] } schemars.workspace = true diff --git a/utilities/int-gen/src/lib.rs b/utilities/int-gen/src/lib.rs index 2965ccd..f91d4f8 100644 --- a/utilities/int-gen/src/lib.rs +++ b/utilities/int-gen/src/lib.rs @@ -74,6 +74,8 @@ pub enum Error { Semver(#[from] semver::Error), #[error(transparent)] Fmt(#[from] std::fmt::Error), + #[error(transparent)] + Regex(#[from] regex::Error), #[error("AsyncAPI spec missing required node: {0}")] SpecNodeMissing(&'static str), diff --git a/utilities/int-gen/src/zig.rs b/utilities/int-gen/src/zig.rs index a3ae011..f71aa3b 100644 --- a/utilities/int-gen/src/zig.rs +++ b/utilities/int-gen/src/zig.rs @@ -51,6 +51,14 @@ const REQUEST_RAW_BODY: &str = include_str!("zig.in/request_raw_body.zig"); /// The `extern fn js_rest_request(...)` declaration appended to the generated client by [`rewrite`]. const JS_REST_REQUEST_EXTERN: &str = include_str!("zig.in/js_rest_request_extern.zig"); +/// Regex spanning the inlined request tail openapi2zig 0.3 emits for a binary/text body (its `.binary`/`.text` +/// arm, which -- unlike JSON bodies -- does not delegate to [`SHARED_REQUEST_FN`]). It anchors on the +/// `= requestBody;` payload assignment, which is unique to those ops (JSON ops assign `str.written()`, empty +/// bodies `null`), then skips to the two captures: 1 = content-type, 2 = HTTP method. `[A-Z]+` (not `\w`) keeps +/// it ASCII so no unicode regex feature is needed. [`reroute_inline_binary_ops`] splices both into a delegation. +const INLINE_FETCH_BLOCK_PATTERN: &str = + r#"(?s)requestBody;.*?"([^"]+)".*?Method\.([A-Z]+).*?toOwnedSlice\(\),\n \};"#; + /// Return `true` if the `openapi2zig` binary is on `PATH`. /// /// Upstream doesn't publish a `linux/arm64` release (see `.mise/config.zig.toml`). @@ -174,43 +182,14 @@ fn rewrite(source: &str) -> Result { reason = "named helper called once by rewrite(); kept separate for the long comment + clear scope" )] fn reroute_inline_binary_ops(source: &str) -> Result { - // The inlined block openapi2zig emits after `const payload = requestBody;`: - // build headers (the 4th `appendClientHeaders` arg is the request - // content-type), parse the pre-built `uri_buf`, allocate a response - // writer, `client.http.fetch`, and assemble a `RawResponse`. The two - // capture groups pull out the content-type and the `std.http.Method.*` - // literal so the delegating call reproduces them. `[A-Z]+` (not `\w`) - // keeps the pattern ASCII so no unicode regex feature is required. - let inline_block = Regex::new(concat!( - r#"\n var headers = std\.ArrayList\(std\.http\.Header\)\.empty;\n"#, - r#" defer headers\.deinit\(allocator\);\n"#, - r#" const auth_header = try appendClientHeaders\(allocator, &headers, client, "#, - r#""([^"]*)", "application/json"\);\n"#, - r#" defer if \(auth_header\) \|value\| allocator\.free\(value\);\n\n"#, - r#" const uri = try std\.Uri\.parse\(uri_buf\.written\(\)\);\n"#, - r#" var response_body: std\.Io\.Writer\.Allocating = \.init\(allocator\);\n"#, - r#" defer response_body\.deinit\(\);\n\n"#, - r#" const result = try client\.http\.fetch\(\.\{\n"#, - r#" \.location = \.\{ \.uri = uri \},\n"#, - r#" \.method = (std\.http\.Method\.[A-Z]+),\n"#, - r#" \.extra_headers = headers\.items,\n"#, - r#" \.payload = payload,\n"#, - r#" \.response_writer = &response_body\.writer,\n"#, - r#" \}\);\n\n"#, - r#" return \.\{\n"#, - r#" \.allocator = allocator,\n"#, - r#" \.status = result\.status,\n"#, - r#" \.body = try response_body\.toOwnedSlice\(\),\n"#, - r#" \};"#, - )) - .map_err(|err| Error::ZigCodegen(format!("inline-binary-op regex failed to compile: {err}")))?; - - Ok(inline_block + let block = Regex::new(INLINE_FETCH_BLOCK_PATTERN)?; + Ok(block .replace_all(source, |caps: ®ex::Captures<'_>| { - let content_type = &caps[1]; - let method = &caps[2]; format!( - "\n return {SHARED_REQUEST_FN}(client, {method}, uri_buf.written(), payload, \"{content_type}\");" + r#"requestBody; + + return {SHARED_REQUEST_FN}(client, std.http.Method.{}, uri_buf.written(), payload, "{}");"#, + &caps[2], &caps[1], ) }) .into_owned()) From 6085e94409f811d41d7af1c01c3e540b3089510e Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Wed, 8 Jul 2026 06:43:00 +0800 Subject: [PATCH 4/9] Enable local C# roslynator --- .deepsource.toml | 7 +++++ .mise/config.dotnet.toml | 39 ++++++++++++++++++++--- .mise/config.toml | 15 +++++---- .mise/config.windows.toml | 45 +++++++++++++++------------ config/conftest/policy/mise/mise.rego | 30 +++++++++++++++--- 5 files changed, 101 insertions(+), 35 deletions(-) diff --git a/.deepsource.toml b/.deepsource.toml index e97af7a..0784103 100644 --- a/.deepsource.toml +++ b/.deepsource.toml @@ -10,6 +10,13 @@ exclude_patterns = ["**/pkg/**", "generated/**", "services/ws-web-runner/mingw-s # Repo convention: tests live in a `tests/` directory or in source files prefixed `test_`. test_patterns = ["**/test_*.py", "**/tests/**"] +# Dart is explicitly deactivated; its coverage comes from the mise pipeline's check:dart instead. +# DeepSource's Dart support is an AI-only analyzer (no classic static analysis), so with AI review off it +# auto-surfaces as a perpetually-skipped "DeepSource: Dart" check on every PR. +[[analyzers]] +enabled = false +name = "dart" + [[analyzers]] enabled = true name = "python" diff --git a/.mise/config.dotnet.toml b/.mise/config.dotnet.toml index d9f445d..bff0c21 100644 --- a/.mise/config.dotnet.toml +++ b/.mise/config.dotnet.toml @@ -3,9 +3,14 @@ [tools] # Pinned: SDK churn re-keys the wasm-tools workload manifest, forcing a ~1.5GB pack re-download on every bump. dotnet = "10.0.300" -dotnet-core = "10.0.300" "dotnet:roslynator.dotnet.cli" = "latest" +[vars] +# MSBuild dir inside the mise dotnet SDK, for the roslynator `-m` flags below. +# `$DOTNET_ROOT` stays literal here; the task shell expands it. The version segment is the [tools] dotnet +# pin above -- bump together. +dotnet_msbuild = "$DOTNET_ROOT/sdk/10.0.300" + [tasks.dotnet-fmt] description = "Format dotnet C# code" run = "dotnet format" @@ -14,15 +19,41 @@ run = "dotnet format" description = "Check dotnet C# formatting" run = "dotnet format --verify-no-changes" -# Namespaced aggregators picked up by the default config's globbed `check`/`fmt`. +[tasks.roslynator-check] +description = "Analyze C# sources (Roslynator)" +# Roslynator analyzes with its embedded analyzer assemblies, against the mise dotnet SDK's own MSBuild. +# The dotnet:roslynator.dotnet.cli tool ships the analyzers, so the csproj needs no analyzer +# PackageReference. Diagnostics at the default info-and-up severity fail the task; loading the project +# performs an implicit restore, so no `dotnet restore` dependency is needed. +# +# `-m` pins MSBuild to the mise dotnet SDK instead of MSBuildLocator's automatic discovery. On Windows the +# inline-task shell is busybox ash (windows_default_inline_shell_args), and busybox-w32 rewrites `\` to `/` +# in every env var it passes to children -- DOTNET_ROOT arrives as `C:/Users/...`, a form hostfxr accepts +# (dotnet-check works) but MSBuildLocator's SDK discovery does not, failing with: +# Cannot choose MSBuild location automatically. Use option '-m, --msbuild-path' to specify MSBuild location +# Observed on commit ef55945932f5d417b8d205530f0868cff6d73ffa (locally; reproduced under `mise exec` by +# setting a forward-slashed DOTNET_ROOT). The env var itself is fine as the `-m` argument -- mise's core +# dotnet plugin exports DOTNET_ROOT on every OS, and the forward-slashed form ash delivers on Windows is +# accepted there; only MSBuildLocator's own probing of it breaks. +run = 'roslynator analyze -m "{{ vars.dotnet_msbuild }}" services/ws-modules/dotnet-data1/dotnet-data1.csproj' + +[tasks.roslynator-fix] +description = "Apply Roslynator diagnostics' machine-applicable C# fixes in place" +run = 'roslynator fix -m "{{ vars.dotnet_msbuild }}" services/ws-modules/dotnet-data1/dotnet-data1.csproj' + +# Namespaced aggregators picked up by the default config's globbed `check`/`fmt`/`fix`. [tasks."check:dotnet"] -depends = ["dotnet-check"] -description = "Run .NET checks (format verification)" +depends = ["dotnet-check", "roslynator-check"] +description = "Run .NET checks (format verification + Roslynator analysis)" [tasks."fmt:dotnet"] depends = ["dotnet-fmt"] description = "Format .NET sources" +[tasks."fix:dotnet"] +depends = ["roslynator-fix"] +description = "Apply .NET lint-fix passes (roslynator fix)" + [tasks.build-ws-dotnet-data1-module] description = "Build the dotnet-data1 C# WASM workflow module" dir = "services/ws-modules/dotnet-data1" diff --git a/.mise/config.toml b/.mise/config.toml index b9f720d..781f59e 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -359,6 +359,10 @@ a_rp_wasm_short = "08a1d6f" # ("C:/Users/runneradmin/AppData/Local"), and a mixed-slash path trips uutils-coreutils' Windows path # normalizer (`cp: The system cannot find the path specified. (os error 3)`). gh_http = "--http-url https://github.com --progress --ignore-existing" +# OpenObserve docker image for the openobserve/o2 dev-observability task. +o2_image = "openobserve/openobserve:v0.70.3" +# pywasm1's own module version, templated into the pkg/package.json its build task emits. +pywasm1_pkg_version = "0.1.0" rp_wasm_dir = '''{% if os() == "windows" -%} {%- set base = (env?.LOCALAPPDATA or "") | replace(from="\\", to="/") -%} {{- base -}}/mise/installs/http-rp-wasm/{{- vars.a_rp_wasm_short -}} @@ -649,16 +653,15 @@ description = "Cross-check Dockerfiles against mise [tools] pins + heredoc body- # 2. `ignore` parser, which preserves every line of the file as a separate entry -- conftest's dockerfile parser # flattens heredoc bodies out of the AST, so the body-first-line `set -euo pipefail` check lives in its # own namespace + parser. -# Each git-tracked Dockerfile* gets both passes. +# Each git-tracked Dockerfile* gets both passes, except that the `ignore` pass only visits Dockerfiles that +# actually contain heredocs: conftest 0.68.2's ignore parser panics with `runtime error: slice bounds out of +# range` on plain Dockerfiles (no heredocs), and the body-first-line rule has nothing to check there anyway. run = """ files=$(git ls-files '*Dockerfile' '*Dockerfile.*') # $files is a newline-separated path list; word-splitting is intentional (no spaces in paths). # shellcheck disable=SC2086 conftest test --combine --namespace dockerfile -p config/conftest/policy $files .mise # Filter to Dockerfiles that actually use heredocs before the `ignore` parser. -# conftest 0.68.2's ignore parser panics with -# `runtime error: slice bounds out of range` on plain Dockerfiles (no -# heredocs). The body-first-line rule has nothing to check there anyway. for f in $files; do if rg -q '^RUN[^\n]*<<' "$f"; then conftest test --parser ignore --namespace dockerfile_heredoc -p config/conftest/policy "$f" @@ -1125,7 +1128,7 @@ description = "Run both the ws-server and ws-wasm-agent using Chrome" [tasks.openobserve] alias = "o2" -run = "docker run --rm -it --name openobserve -p 5080:5080 --env-file config/o2.env openobserve/openobserve:v0.70.3" +run = "docker run --rm -it --name openobserve -p 5080:5080 --env-file config/o2.env {{ vars.o2_image }}" [tasks.demo] depends = ["openobserve", "ws-server"] @@ -1252,7 +1255,7 @@ coreutils cat > pkg/package.json <<'PKG_JSON' { "name": "et-ws-pywasm1", "type": "module", - "version": "0.1.0", + "version": "{{ vars.pywasm1_pkg_version }}", "main": "index.js", "files": [ "index.js", diff --git a/.mise/config.windows.toml b/.mise/config.windows.toml index a8384e3..7283398 100644 --- a/.mise/config.windows.toml +++ b/.mise/config.windows.toml @@ -212,18 +212,31 @@ _.path = ["{{ vars.m2_gnupg_bin }}", "{{ vars.maven_bin }}", "{{ vars.win_libcla # flips rustup's default host to gnullvm so both host and target link via llvm-mingw rather than the absent # MSVC link.exe (no Visual Studio Build Tools needed). The prereqs mise CAN'T supply (a recent mise + the # VC++ runtime) are in the README. +# +# The body opens with a sha256sum fail-fast diagnostic for the Nano CI lane. mise's conda backend (mise +# 2026.6.5, src/hash.rs:file_hash_sha256, files > 50 MB only) shells out to whichever `sha256sum` PATH finds +# to verify each conda tarball download. On the Nano docker build that lookup historically hit a +# sha256sum.exe that fails to load with `STATUS_ENTRYPOINT_NOT_FOUND` (0xC0000139 == decimal -1073741511, +# the msys2-runtime autoload-trap signature), surfacing as: +# mise ERROR Failed to install conda:m2-git@latest: command ["sha256sum", +# ""] exited with code -1073741511 +# Dockerfile.nanoserver stages a busybox-w32-backed `sha256sum.exe` ahead of the broken shim on PATH so the +# check should now pass; failing fast tells us which binary PATH picked when the next CI break hits. +# +# The pipx bootstrap's rustpython arm carries pinned-version baggage: RustPython's bundled ensurepip is pip +# 26.1.1, which crashes on import via `sys.addaudithook` (rustpython 0.5.0 doesn't implement -- +# RustPython#7956), so that arm fetches pip 26.0.1 (the last release before that audit hook landed, commit +# fb01b1830 / 2026-04-14), unzips the wheel, and puts it on RUSTPYTHONPATH so `rustpython -m pip` resolves +# to that copy. config.maint.toml's `rustpython-investigation` task documents the workaround at length, and +# also explains the PIPX_SHARED_LIBS pre-creation: pipx's first install creates a shared-libs venv via +# `python -m venv --clear` (no --without-pip), which triggers RustPython's bundled ensurepip audit-hook +# crash again. Pre-creating that venv with --without-pip and dropping the unpacked pip site-packages in lets +# pipx's `is_valid()` short-circuit past `create()`. The Nano Dockerfile then exports PATH + PIPX_* env vars +# so `mise install pipx:` can find this pipx and reuse the pre-created shared venv. depends = ["_setup_all"] description = "Preinstall: toolchain + pipx + flip rustup to the gnullvm host" run = """ -# sha256sum diagnostic for the Nano CI lane. -# mise's conda backend in 2026.6.5 shells out to a `sha256sum` it finds on PATH (via src/hash.rs:file_hash_sha256, -# only for files > 50 MB) to verify each conda tarball download. On the Nano docker build that lookup -# historically hit a sha256sum.exe that fails to load with `STATUS_ENTRYPOINT_NOT_FOUND` (0xC0000139 == -# decimal -1073741511, the msys2-runtime autoload-trap signature), surfacing as: -# mise ERROR Failed to install conda:m2-git@latest: command ["sha256sum", -# ""] exited with code -1073741511 -# Dockerfile.nanoserver stages a busybox-w32-backed `sha256sum.exe` ahead of the broken shim on PATH so this -# should now pass; the diagnostic is fail-fast so the next CI break here tells us which binary PATH picked. +# sha256sum fail-fast diagnostic for the Nano CI lane. sha256sum_path="$(command -v sha256sum 2>/dev/null || true)" echo "[sha256sum-check] command -v sha256sum: ${sha256sum_path:-}" if [ -z "$sha256sum_path" ]; then @@ -256,17 +269,9 @@ mise exec -- rustup set default-host x86_64-pc-windows-gnullvm # pipx backs the pipx:* tools but isn't a mise tool on Windows. # On a workstation, the system python's Scripts dir already has pipx on PATH; use that python. On the Nano # container there's no CPython (Dockerfile.nanoserver drops the `mise install python` step now that we can -# pipx-bootstrap via the `http:et-rp` rustpython mise installs as part of the global tool set). RustPython's -# bundled ensurepip is pip 26.1.1, which crashes on import via `sys.addaudithook` (rustpython 0.5.0 doesn't -# implement -- RustPython#7956), so the rustpython arm fetches pip 26.0.1 (the last release before that audit -# hook landed, see commit fb01b1830 / 2026-04-14), unzips the wheel, and puts it on RUSTPYTHONPATH so -# `rustpython -m pip` resolves to that copy. The matching workaround is documented at length in config.maint.toml's -# `rustpython-investigation` task. The same task also explains the PIPX_SHARED_LIBS pre-creation below: -# pipx's first install creates a shared-libs venv via `python -m venv --clear` (no --without-pip), which -# triggers RustPython's bundled ensurepip + pip 26.1.1 audit-hook crash again. Pre-creating that venv with -# --without-pip and dropping the unpacked pip-26.0.1 site-packages in lets pipx's `is_valid()` short-circuit -# past `create()`. The Nano Dockerfile then exports PATH + PIPX_* env vars so `mise install pipx:` can -# find this pipx and reuse the pre-created shared venv. +# pipx-bootstrap via the `http:et-rp` rustpython mise installs as part of the global tool set), so the +# rustpython arm fetches a pinned pre-audit-hook pip wheel, unzips it, and puts it on RUSTPYTHONPATH so +# `rustpython -m pip` resolves to that copy. if command -v python >/dev/null 2>&1; then python -m pip install pipx elif [ "${ENABLE_RUSTPYTHON_PIPX_BOOTSTRAP:-}" = "1" ]; then diff --git a/config/conftest/policy/mise/mise.rego b/config/conftest/policy/mise/mise.rego index 45bd919..eeecc8d 100644 --- a/config/conftest/policy/mise/mise.rego +++ b/config/conftest/policy/mise/mise.rego @@ -50,24 +50,44 @@ deny contains msg if { # `compgen` is a bash builtin that busybox-w32 ash (Nano Server's shell) does not provide. # A task using it fails on Nano with the literal error `: not found` (verified in the build-rp-native task). -# Skip lines whose first non-whitespace char is `#` so explanatory comments (like this one's siblings) don't -# false-positive. POSIX-portable alternatives include `[ -e "$prefix/bin/foo" ] || [ -e "$prefix/bin/foo.exe" ]` for an +# The whole run body is checked, comments included -- compgen-narrating prose belongs in a TOML comment on +# the task, not inside the body. config.maint.toml is exempt: maintainer-only, never runs on Nano. +# POSIX-portable alternatives include `[ -e "$prefix/bin/foo" ] || [ -e "$prefix/bin/foo.exe" ]` for an # extension-agnostic existence check, and `for f in "$prefix/bin/foo"*` to walk a literal glob (no-match leaves the # literal as the loop var). deny contains msg if { some file in input is_mise(file) + replace(file.path, "\\", "/") != ".mise/config.maint.toml" some name, task in file.contents.tasks is_string(task.run) - some line in split(task.run, "\n") - not startswith(trim_space(line), "#") - regex.match(`\bcompgen\b`, line) + regex.match(`\bcompgen\b`, task.run) msg := sprintf( "%s: task %q uses `compgen`, which is bash-only -- busybox ash (Nano) errors `compgen: not found`", [file.path, name], ) } +# A task `run` must not embed version-like text (an `X.Y.Z` literal), comments included. +# A version baked into a run body silently goes stale when the pin it mirrors is bumped -- version_drift +# only cross-checks [vars]/[env] values, never run strings. Keep the versioned fragment in a [vars] entry +# beside the [tools] pin it tracks and template it into the run with `{{ vars. }}`; version-narrating +# prose belongs in a TOML comment on the task, not inside the body. +# config.maint.toml is exempt: its maintainer-only investigation/publish tasks narrate and probe specific +# upstream versions by design. +deny contains msg if { + some file in input + is_mise(file) + replace(file.path, "\\", "/") != ".mise/config.maint.toml" + some name, task in file.contents.tasks + is_string(task.run) + some m in regex.find_all_string_submatch_n(`[0-9]+\.[0-9]+\.[0-9]+`, task.run, -1) + msg := sprintf( + "%s: task %q run embeds version-like text %q -- move it into a [vars] entry beside the pin it tracks", + [file.path, name, m[0]], + ) +} + # `cargo:` tools may build from source; prefer a prebuilt backend. # Allowed only when either (a) allowlisted by name below -- cargo-binstall fetches a prebuilt there (e.g. a # cargo-quickinstall release, routed via an `install_env` CARGO_BUILD_TARGET override), or the tool has no prebuilt From 87c6a120c48c5648b2941b564ad7bdcd8720d550 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Wed, 8 Jul 2026 05:17:19 +0800 Subject: [PATCH 5/9] claude note --- CLAUDE.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 9067fda..08ecd2b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -441,8 +441,8 @@ If you believe a skip is genuinely warranted, stop and ask the user; do not add ### Known intermittent CI failure: pyo3-runner torch registration timeout -`et-ws-pyo3-runner`'s `module_behaves::case_5_torch` intermittently fails on test.yaml's Windows lane with the -literal failure line +`et-ws-pyo3-runner`'s `module_behaves::case_5_torch` intermittently fails on test.yaml's Windows and macOS lanes +with the literal failure line Error: "runner never registered" @@ -454,7 +454,11 @@ torch's import when the test's registration timeout expired. The ~117 MB `pipx:t cold runner is the slow step; a rerun passes because the import caches warm. Observed on commit `6479913bdc288dd680fbe0520f63054e8c71fe6c` at `https://github.com/edge-toolkit/core/actions/runs/28686533955/job/85080173283` (PR #70; the rerun passed and the PR -merged). If this signature recurs, stop rerunning and fix the root cause: raise (or make torch-case-specific) the +merged), and on the `default (macos-latest, 45)` job -- same signature, unix-form +`torch/lib/python3.13/site-packages/torch/_subclasses/functional_tensor.py:362` warning path -- on commit +`f2a85307a2415f4a50625627795e02c84f1c8e5d` at +`https://github.com/edge-toolkit/core/actions/runs/28865327221/job/85613919558` (PR #73), 20.68s into the test +run. If this signature recurs, stop rerunning and fix the root cause: raise (or make torch-case-specific) the runner-registration timeout in the pyo3-runner module tests, or warm the torch import before the registration clock starts. From 77cbbb7c3ecf0432492c340e8da5adeaada24b7a Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Wed, 8 Jul 2026 10:43:59 +0800 Subject: [PATCH 6/9] small fixes --- .mise/config.coverage.toml | 8 ++++---- CLAUDE.md | 4 ++-- config/nextest.toml | 3 +-- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.mise/config.coverage.toml b/.mise/config.coverage.toml index 11e3555..d7dae8c 100644 --- a/.mise/config.coverage.toml +++ b/.mise/config.coverage.toml @@ -13,13 +13,13 @@ description = "Rust coverage (cargo-llvm-cov) + JUnit test results (nextest); em # the `llvm-tools-preview` rustup component, which it adds on demand against the mise-managed toolchain. # Running the suite through nextest (NEXTEST_PROFILE=ci) emits target/nextest/ci/junit.xml for the Codecov # test-results upload; et-ws-web-runner is serialized by the `web-runner` test-group in config/nextest.toml -# rather than RUST_TEST_THREADS, which nextest ignores. `--config-file` points at that config since it lives under -# config/ rather than the default .config/. nextest does not run doctests, so a separate --doc pass keeps their -# coverage; --no-report on both passes so they merge into one lcov report. +# rather than RUST_TEST_THREADS, which nextest ignores; `--config-file config/nextest.toml` selects that config. +# Doctests are not covered here -- nextest does not run them, and `cargo llvm-cov --doc` needs a nightly toolchain +# this stable job does not have (the pre-nextest task skipped them too). `--no-report` defers reporting so `report` +# writes lcov.info from the run's profiles. env = { NEXTEST_PROFILE = "ci" } run = """ cargo llvm-cov --no-report nextest --config-file config/nextest.toml -cargo llvm-cov --no-report --doc cargo llvm-cov report --lcov --output-path lcov.info """ shell = "bash -euo pipefail -c" diff --git a/CLAUDE.md b/CLAUDE.md index 08ecd2b..d3fb270 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -31,8 +31,8 @@ The "Don't depend on host tools" rule further down is written for mise task bodi the ad-hoc commands the agent runs while working -- investigations, scratch-area probes, one-off transforms. Prefer the repo's mise-managed, version-pinned, cross-platform tools -- `coreutils` (uutils multicall), `rg` (ripgrep), `find`/`xargs` (uutils findutils), `goawk`, and whatever else the `[tools]` tables pin -- over whatever binary happens -to be on the host (`perl`, BSD `sed`, a random CLI). A host binary may be absent, a different version, or missing on -another OS, and its shell-quoting is the exact fragility the homebrew-bash rule exists to avoid (a mangled +to be on the host (`perl`, the host's own `sed`, a random CLI). A host binary may be absent, a different version, or +missing on another OS, and its shell-quoting is the exact fragility the homebrew-bash rule exists to avoid (a mangled `perl -pi -e` once silently no-op'd its edit and produced a bogus "passing" test result here before it was caught). For file edits specifically, reach for the Edit/Write tools rather than a stream editor (`perl -pi`, `sed -i`): they diff --git a/config/nextest.toml b/config/nextest.toml index ea93caf..90c926e 100644 --- a/config/nextest.toml +++ b/config/nextest.toml @@ -1,7 +1,6 @@ # nextest profile used by the coverage workflow via `cargo llvm-cov nextest` (NEXTEST_PROFILE=ci). # It writes a JUnit report to target/nextest/ci/junit.xml, which the coverage job uploads to Codecov as test -# results (report_type: test_results). Passed explicitly with `--config-file` since it lives here, not the -# default .config/nextest.toml. +# results (report_type: test_results). The coverage task loads it via `--config-file config/nextest.toml`. [profile.ci.junit] path = "junit.xml" From b2a7abb627632da4ce09e92c09a16e5356690071 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Wed, 8 Jul 2026 11:05:41 +0800 Subject: [PATCH 7/9] two more --- .codacy.yaml | 4 ++++ .mise/config.coverage.toml | 12 +++++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.codacy.yaml b/.codacy.yaml index 5c23482..4fc7f6a 100644 --- a/.codacy.yaml +++ b/.codacy.yaml @@ -10,9 +10,13 @@ # the same reason). Workflows + composite actions are already linted by actionlint, zizmor, action-validator, the # ast-grep gha rules, and the gha* conftest policies; since Codacy cannot disable an individual pattern in-repo, # excluding these trees is the only way to silence its pin rule globally. +# - CLAUDE.md: Codacy's prose rules (undefined-acronym, absolute-rule-without-escape-hatch) fight this +# agent-instructions doc, which is imperative and full of deliberate absolutes ("NEVER ...", "ALWAYS ...") by +# design; dprint + typos + editorconfig already lint it, so Codacy adds only false positives here. exclude_paths: - ".github/actions/**" - ".github/workflows/**" + - "CLAUDE.md" - "Dockerfile" - "config/semgrep/**" - "services/ws-modules/dotnet-data1/Program.cs" diff --git a/.mise/config.coverage.toml b/.mise/config.coverage.toml index d7dae8c..9847eef 100644 --- a/.mise/config.coverage.toml +++ b/.mise/config.coverage.toml @@ -14,12 +14,14 @@ description = "Rust coverage (cargo-llvm-cov) + JUnit test results (nextest); em # Running the suite through nextest (NEXTEST_PROFILE=ci) emits target/nextest/ci/junit.xml for the Codecov # test-results upload; et-ws-web-runner is serialized by the `web-runner` test-group in config/nextest.toml # rather than RUST_TEST_THREADS, which nextest ignores; `--config-file config/nextest.toml` selects that config. -# Doctests are not covered here -- nextest does not run them, and `cargo llvm-cov --doc` needs a nightly toolchain -# this stable job does not have (the pre-nextest task skipped them too). `--no-report` defers reporting so `report` -# writes lcov.info from the run's profiles. +# Every pass runs on `+nightly` because doctest coverage (`--doc`, via `-Z persist-doctests`) is nightly-only; +# keeping all passes on one toolchain lets their profiles merge, and cargo-llvm-cov adds the `llvm-tools-preview` +# rustup component to that toolchain on demand. `--no-report` on the nextest + doc passes, then `report` writes the +# merged lcov.info. env = { NEXTEST_PROFILE = "ci" } run = """ -cargo llvm-cov --no-report nextest --config-file config/nextest.toml -cargo llvm-cov report --lcov --output-path lcov.info +cargo +nightly llvm-cov --no-report nextest --config-file config/nextest.toml +cargo +nightly llvm-cov --no-report --doc +cargo +nightly llvm-cov report --lcov --output-path lcov.info """ shell = "bash -euo pipefail -c" From 2468a2e19ac7ad53d07e956c810e7f683c5d8e57 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Wed, 8 Jul 2026 11:50:37 +0800 Subject: [PATCH 8/9] disable doctest coverage --- .mise/config.coverage.toml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.mise/config.coverage.toml b/.mise/config.coverage.toml index 9847eef..54286c0 100644 --- a/.mise/config.coverage.toml +++ b/.mise/config.coverage.toml @@ -14,14 +14,13 @@ description = "Rust coverage (cargo-llvm-cov) + JUnit test results (nextest); em # Running the suite through nextest (NEXTEST_PROFILE=ci) emits target/nextest/ci/junit.xml for the Codecov # test-results upload; et-ws-web-runner is serialized by the `web-runner` test-group in config/nextest.toml # rather than RUST_TEST_THREADS, which nextest ignores; `--config-file config/nextest.toml` selects that config. -# Every pass runs on `+nightly` because doctest coverage (`--doc`, via `-Z persist-doctests`) is nightly-only; -# keeping all passes on one toolchain lets their profiles merge, and cargo-llvm-cov adds the `llvm-tools-preview` -# rustup component to that toolchain on demand. `--no-report` on the nextest + doc passes, then `report` writes the -# merged lcov.info. +# No `--doc` pass: the workspace sets `[lib] doctest = false` on essentially every crate (doc comments are prose, +# not runnable examples), and `cargo llvm-cov --doc` ignores that setting -- it drives rustdoc directly and chokes +# on the generated et-rest-client's backtick prose. So coverage stays on stable, tests-only. `--no-report` defers +# the report so `report` writes lcov.info from the nextest run's profiles. env = { NEXTEST_PROFILE = "ci" } run = """ -cargo +nightly llvm-cov --no-report nextest --config-file config/nextest.toml -cargo +nightly llvm-cov --no-report --doc -cargo +nightly llvm-cov report --lcov --output-path lcov.info +cargo llvm-cov --no-report nextest --config-file config/nextest.toml +cargo llvm-cov report --lcov --output-path lcov.info """ shell = "bash -euo pipefail -c" From 71e70e56946abefcea6daf47a7745e5f8f186dad Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Wed, 8 Jul 2026 14:46:56 +0800 Subject: [PATCH 9/9] fix actionlint on windows --- .mise/config.toml | 15 +++++++++------ .mise/config.windows.toml | 11 +++++++++++ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/.mise/config.toml b/.mise/config.toml index 781f59e..c35cd2d 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -358,6 +358,8 @@ a_rp_wasm_short = "08a1d6f" # Forward slashes throughout the Windows suffix: GHA's LOCALAPPDATA arrives already-forward-slashed # ("C:/Users/runneradmin/AppData/Local"), and a mixed-slash path trips uutils-coreutils' Windows path # normalizer (`cp: The system cannot find the path specified. (os error 3)`). +# Extra actionlint flags for actionlint-check; empty everywhere except the config.windows.toml override. +actionlint_args = "" gh_http = "--http-url https://github.com --progress --ignore-existing" # OpenObserve docker image for the openobserve/o2 dev-observability task. o2_image = "openobserve/openobserve:v0.70.3" @@ -516,12 +518,13 @@ run = "action-validator .github/workflows/*.yaml .github/actions/*/action.yaml" [tasks.actionlint-check] description = "Lint GitHub Actions workflows (actionlint): local-action input validation, expression syntax, shellcheck" # Lint every workflow with actionlint's native checks plus its shellcheck integration over each `run:` script. -# shellcheck is a mise [tool], so it is on PATH here and actionlint auto-detects and invokes it. The most -# load-bearing native rule is `[action]`, which fails when a workflow step passes a `with:` key the referenced -# local action does not declare (GHA itself only warns). actionlint lints workflows only, so it does NOT see -# composite-action -> action calls; the gha_uses conftest rule covers that direction. Intentional word-splitting -# sites carry an inline `# shellcheck disable=SC2086` directive in the run script, as the mise-task bodies do. -run = "actionlint .github/workflows/*.yaml" +# shellcheck is a mise [tool], so it is on PATH here and actionlint auto-detects and invokes it (except under +# the Windows vars.actionlint_args override). The most load-bearing native rule is `[action]`, which fails +# when a workflow step passes a `with:` key the referenced local action does not declare (GHA itself only +# warns). actionlint lints workflows only, so it does NOT see composite-action -> action calls; the gha_uses +# conftest rule covers that direction. Intentional word-splitting sites carry an inline +# `# shellcheck disable=SC2086` directive in the run script, as the mise-task bodies do. +run = "actionlint {{ vars.actionlint_args }} .github/workflows/*.yaml" [tasks.zizmor-check] description = "Audit GitHub Actions workflows + composite actions for security issues (zizmor)" diff --git a/.mise/config.windows.toml b/.mise/config.windows.toml index 7283398..974db92 100644 --- a/.mise/config.windows.toml +++ b/.mise/config.windows.toml @@ -153,6 +153,17 @@ no_opt = "--no-opt" # The build-based cargo checks (check/clippy/doc) skip it here, matching cargo test's --exclude and the # skipped test-ws-web-runner task. cargo_ws_excludes = "--exclude et-ws-web-runner" +# Disable actionlint's shellcheck integration on Windows (user-approved 2026-07-08). +# actionlint deadlocks here whenever a workflow `run:` script exceeds the ~4KB anonymous-pipe buffer. Its +# process.go `cmdExecution.run()` writes the whole script to the child's stdin pipe BEFORE starting it, so a +# big write blocks forever with no reader -- `mise run actionlint-check` then hangs with no output and an +# `actionlint.exe` process pinned in tasklist. Present in every actionlint since the shellcheck rule landed +# (v1.4.x; verified hanging on 1.7.7/1.7.11/1.7.12), masked on Linux/macOS by their 64KB pipe buffers, so +# those lanes keep full shellcheck coverage. Reproduced on commit ef55945932f5d417b8d205530f0868cff6d73ffa +# with a synthetic 29KB run body (hangs) vs a 178B one (passes); a fake instantly-exiting shellcheck still +# hangs it, proving shellcheck itself is uninvolved. Upstream fix is one line +# (`cmd.Stdin = strings.NewReader(e.stdin)`); drop this override once a fixed actionlint releases. +actionlint_args = "-shellcheck=" [env] # Run `shell = "bash ..."` tasks on busybox ash rather than the Windows WSL launcher.