From cea703d49fc94d4987a2566bf2b4f514defcac64 Mon Sep 17 00:00:00 2001 From: Bryant Date: Thu, 13 Aug 2026 18:38:42 +0800 Subject: [PATCH 1/9] =?UTF-8?q?=E2=9C=85=20(test):=20Bind=20each=20documen?= =?UTF-8?q?ted=20quick-start=20claim=20to=20the=20control=20that=20proves?= =?UTF-8?q?=20it?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit docs/quick-start.md §"What just happened" carries the Python quick-start's enforcement claims, and nothing connected them to the negative controls. A claim could be added, reworded, or left standing after the behaviour beneath it changed, and no gate would notice. Each claim is now bound to the controls that prove it. The claim list is parsed from the document and the control names are extracted from the negative-control module's AST, so neither side is a transcribed constant checking another transcribed constant: adding a claim, rewording one, renaming a control, or renaming the exception the document names each fail here. Claim 1 is registered as deliberately unproven, naming AAASM-5661. Every control installs a fake native core that the documented configuration does not have, so binding it to one would launder that gap into evidence. The SDK symbols a claim names are resolved lazily rather than imported at module scope. Imported at module scope, a rename becomes a collection error that aborts before the assertion meant to catch it can run — the same inverted-order defect the round-1 review of this ticket found in all three SDKs. Refs AAASM-5529 --- test/unit/test_quickstart_claim_bindings.py | 335 ++++++++++++++++++++ 1 file changed, 335 insertions(+) create mode 100644 test/unit/test_quickstart_claim_bindings.py diff --git a/test/unit/test_quickstart_claim_bindings.py b/test/unit/test_quickstart_claim_bindings.py new file mode 100644 index 00000000..bb3c9f4e --- /dev/null +++ b/test/unit/test_quickstart_claim_bindings.py @@ -0,0 +1,335 @@ +"""Drift gate binding the quick-start's enforcement claims to the controls that prove them. + +AAASM-5529, Epic AAASM-5526. + +``docs/quick-start.md`` §"What just happened" is where the Python quick-start +tells a reader what governance did for them. Those sentences are the product's +load-bearing enforcement claims, and until now nothing connected them to the +negative controls in :mod:`test.unit.test_quickstart_negative_control`. A claim +could be added, reworded or left standing after the behaviour beneath it changed, +and no gate would notice. + +What this gate proves +--------------------- + +#. **Every enforcement claim in the section is bound to a named control.** The + claim list is parsed out of the document, so adding a fifth numbered claim + without registering a binding for it fails here rather than shipping an + unbacked sentence. +#. **Every binding still describes the document.** Each binding quotes the + load-bearing fragment of its claim; rewording the sentence in the document + breaks the quote and fails. +#. **Every control a binding names still exists.** The control names are + extracted from the negative-control module's AST, not transcribed, so + renaming or deleting a control fails here. +#. **Every SDK symbol the section names is real.** The symbol is imported and + its ``__name__`` compared, so renaming ``ToolExecutionBlockedError`` in the + SDK fails here instead of leaving the documentation pointing at a class that + no longer exists. + +What this gate does **not** prove +--------------------------------- + +It does not execute the quick-start, and it cannot: ``quickstart_snippets/`` is +a vendored, verbatim copy of regions from the ``examples`` repository +(``ruff.toml`` excludes it for this reason), and the snippets are partial +governance slices that reference names they never define — ``gateway_url``, +``api_key``, ``src.policy`` — so they are not importable modules. The existing +``quickstart-tabs-check`` workflow round-trips them as *text*, asserting only +that the generated document matches the vendored copy. Neither that gate nor +this one type-checks, imports or runs a snippet. + +Nor does binding a claim make the claim *true*. A binding records which control +stands behind a sentence; where no control does, the binding must say so and +name the ticket, which is the state claim 1 is in today (AAASM-5661). +""" + +from __future__ import annotations + +import ast +import importlib +import re +from dataclasses import dataclass, field +from pathlib import Path + +import pytest + +# NOTE: the SDK symbols a claim names are resolved lazily, by module path and +# attribute name, rather than imported here. Importing them at module scope +# makes a rename a *collection* error, which aborts before +# test_named_sdk_symbols_resolve_to_that_name can run — leaving the assertion +# that is supposed to catch the rename permanently unexercised. That is the +# same inverted-order defect the round-1 review of this ticket found in all +# three SDKs, and it is invisible unless you mutate and watch which line fails. + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_QUICK_START = _REPO_ROOT / "docs" / "quick-start.md" +_NEGATIVE_CONTROL = Path(__file__).with_name("test_quickstart_negative_control.py") + +_SECTION_HEADING = "## What just happened" + +#: Claims that assert governance acted on a tool call. These are the ones the +#: Epic exists for, and the ones a binding must back with a control. +ENFORCEMENT = "enforcement" +#: Claims about setup or teardown. They are still bound, so the parser's +#: completeness check cannot be satisfied by silently dropping one, but they do +#: not require an enforcement control. +LIFECYCLE = "lifecycle" + + +@dataclass(frozen=True) +class ClaimBinding: + """One documented claim and the controls that stand behind it.""" + + claim_id: str + kind: str + #: A verbatim fragment of the claim as it appears in the document, with + #: newlines collapsed. Rewording the document breaks this. + quote: str + #: ``ClassName::test_name`` node ids in the negative-control module. + controls: tuple[str, ...] = () + #: Set when no control proves the claim. Must name the ticket that tracks it. + unproven_reason: str = "" + #: Backticked SDK identifiers the claim names, mapped to the module they + #: must be importable from. Resolved lazily — see the note at the top. + symbols: dict[str, str] = field(default_factory=dict) + + +BINDINGS: tuple[ClaimBinding, ...] = ( + ClaimBinding( + claim_id="init-routes-every-tool-call", + kind=ENFORCEMENT, + quote="every tool call from this point on is routed", + # Deliberately unbacked. AAASM-5661 measured the documented + # configuration and found this sentence overstates it: the controls in + # the negative-control module all call install_fake_core(), supplying an + # authoritative runtime the documented configuration does not have, so + # they are structurally incapable of proving a claim about the + # documented path. Binding it to one of them would launder that gap into + # evidence. The honest state is a named, ticketed absence. + unproven_reason=( + "AAASM-5661: the documented configuration was measured and no control " + "covers it. Every control in test_quickstart_negative_control.py " + "installs a fake native core, which the documented path does not have." + ), + ), + ClaimBinding( + claim_id="sdk-only-enforces-on-tool-calls", + kind=ENFORCEMENT, + quote="The in-process adapter enforces on tool calls with no", + controls=( + "TestFilesystemSideEffect::test_negative_control_denied_write_leaves_no_file", + "TestNetworkSideEffect::test_negative_control_denied_egress_never_reaches_the_listener", + ), + ), + ClaimBinding( + claim_id="verdict-precedes-execution", + kind=ENFORCEMENT, + quote="asks the policy engine for an allow/deny verdict before the tool actually runs", + # The two negative controls prove the *before* by absence of the side + # effect; the two positive controls prove the probe would have seen the + # effect had it happened. Both halves are named, because either alone is + # the vacuous evidence this Epic exists to remove. + controls=( + "TestFilesystemSideEffect::test_positive_control_allowed_write_creates_the_file", + "TestFilesystemSideEffect::test_negative_control_denied_write_leaves_no_file", + "TestNetworkSideEffect::test_positive_control_allowed_egress_reaches_the_listener", + "TestNetworkSideEffect::test_negative_control_denied_egress_never_reaches_the_listener", + ), + ), + ClaimBinding( + claim_id="with-block-unwinds", + kind=LIFECYCLE, + quote="tore everything down on exit", + unproven_reason=( + "Teardown is covered by the context-manager tests, not by the enforcement controls this gate binds." + ), + ), + ClaimBinding( + claim_id="deny-surfaces-as-tool-execution-blocked", + kind=ENFORCEMENT, + quote="that is not a bug — the policy denied the", + controls=( + "TestFilesystemSideEffect::test_negative_control_denied_write_leaves_no_file", + "TestDegradedRuntimeCannotLookProtected" + "::test_an_unavailable_native_runtime_denies_rather_than_silently_allowing", + ), + symbols={"ToolExecutionBlockedError": "agent_assembly.exceptions"}, + ), +) + + +def _section_text() -> str: + """Return the "What just happened" section, up to the next ``##`` heading.""" + document = _QUICK_START.read_text(encoding="utf-8") + start = document.find(_SECTION_HEADING) + assert start != -1, ( + f"{_QUICK_START} no longer contains a '{_SECTION_HEADING}' section. " + "If the quick-start was restructured, re-point this gate at the section " + "that now carries the enforcement claims — do not delete it." + ) + body = document[start + len(_SECTION_HEADING) :] + end = body.find("\n## ") + return body if end == -1 else body[:end] + + +def _flatten(text: str) -> str: + """Collapse Markdown's soft wrapping so a quote can span wrapped lines.""" + return re.sub(r"\s+", " ", text).strip() + + +def _documented_claims() -> dict[str, str]: + """Parse the section into ``claim_key -> flattened text``. + + Numbered list items become ``item-N``; the trailing prose paragraph becomes + ``prose-N``. Both are derived from the document, so a newly added claim + appears here without anyone updating this module — which is the point. + """ + section = _section_text() + claims: dict[str, str] = {} + + # Numbered items: "1. ..." through the line before the next "N. " or a blank + # line followed by unindented prose. + item_pattern = re.compile(r"^(\d+)\.\s+(.*(?:\n(?![ ]*\d+\.\s|\n).*)*)", re.MULTILINE) + for match in item_pattern.finditer(section): + claims[f"item-{match.group(1)}"] = _flatten(match.group(2)) + + # Prose paragraphs that make a claim, i.e. mention denial or blocking. Link + # lists and prose that merely points elsewhere are not claims. + consumed = {match.group(0) for match in item_pattern.finditer(section)} + remainder = section + for chunk in consumed: + remainder = remainder.replace(chunk, "\n") + for index, paragraph in enumerate(p for p in remainder.split("\n\n") if p.strip()): + flat = _flatten(paragraph) + if re.search(r"\bdenied\b|\bblocked\b|\bdeny\b", flat, re.IGNORECASE): + claims[f"prose-{index}"] = flat + + return claims + + +def _control_node_ids() -> set[str]: + """Extract ``ClassName::test_name`` ids from the negative-control module's AST. + + Derived from the source rather than transcribed, so this set changes when a + control is renamed or removed and the bindings above then fail. + """ + tree = ast.parse(_NEGATIVE_CONTROL.read_text(encoding="utf-8")) + node_ids: set[str] = set() + for node in tree.body: + if isinstance(node, ast.ClassDef): + for child in node.body: + if isinstance(child, ast.FunctionDef | ast.AsyncFunctionDef) and child.name.startswith("test_"): + node_ids.add(f"{node.name}::{child.name}") + elif isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef) and node.name.startswith("test_"): + node_ids.add(node.name) + return node_ids + + +class TestTheGateCanSeeWhatItGates: + """Positive controls. Every check below reads a real artifact; prove it arrived.""" + + def test_the_quick_start_section_is_found_and_non_empty(self) -> None: + section = _section_text() + assert len(section.strip()) > 200, "the parsed section is too short to contain the claim list" + + def test_the_parser_finds_the_numbered_claims(self) -> None: + claims = _documented_claims() + numbered = [key for key in claims if key.startswith("item-")] + # A count is asserted, not a list, because the list is the thing under + # test. If the document grows a claim this fails, which is the gate. + assert len(numbered) >= 4, f"expected the four documented claims, parsed {sorted(claims)}" + + def test_the_ast_extraction_finds_the_negative_controls(self) -> None: + node_ids = _control_node_ids() + assert len(node_ids) >= 8, f"AST extraction found only {len(node_ids)} controls: {sorted(node_ids)}" + # A named one, so an extraction that silently returned an unrelated set + # cannot satisfy the count above. + assert "TestFilesystemSideEffect::test_negative_control_denied_write_leaves_no_file" in node_ids + + +class TestEveryDocumentedClaimIsBound: + def test_no_claim_in_the_section_is_unbound(self) -> None: + """Adding a claim to the quick-start without a binding fails here. + + This is the check that makes the gate load-bearing rather than + decorative: a new enforcement sentence cannot reach the published + quick-start without someone naming the control that stands behind it, + or recording in the binding that none does. + """ + documented = _documented_claims() + unmatched = { + key: text for key, text in documented.items() if not any(binding.quote in text for binding in BINDINGS) + } + assert not unmatched, ( + "These quick-start claims have no ClaimBinding in BINDINGS:\n" + + "\n".join(f" {key}: {text}" for key, text in unmatched.items()) + + "\n\nAdd a ClaimBinding naming the control that proves each one. If no " + "control does, set unproven_reason and name the ticket — do not delete " + "the claim from this gate to make it pass." + ) + + @pytest.mark.parametrize("binding", BINDINGS, ids=lambda b: b.claim_id) + def test_each_binding_still_quotes_the_document(self, binding: ClaimBinding) -> None: + """Rewording a claim in the document fails here.""" + documented = _documented_claims() + assert any(binding.quote in text for text in documented.values()), ( + f"ClaimBinding {binding.claim_id!r} quotes:\n {binding.quote!r}\n" + f"which no longer appears in {_SECTION_HEADING!r} of {_QUICK_START.name}. " + "The claim was reworded or removed. Update the quote and re-check that " + "the named controls still prove the new wording." + ) + + +class TestEveryBindingNamesSomethingReal: + @pytest.mark.parametrize("binding", BINDINGS, ids=lambda b: b.claim_id) + def test_named_controls_exist(self, binding: ClaimBinding) -> None: + """Renaming or deleting a control fails here.""" + available = _control_node_ids() + missing = [control for control in binding.controls if control not in available] + assert not missing, ( + f"ClaimBinding {binding.claim_id!r} names controls that do not exist in " + f"{_NEGATIVE_CONTROL.name}: {missing}\n" + "The control was renamed or removed. Re-point the binding at the control " + "that now proves the claim, or mark the claim unproven and name the ticket." + ) + + @pytest.mark.parametrize("binding", BINDINGS, ids=lambda b: b.claim_id) + def test_an_enforcement_claim_is_either_proven_or_openly_unproven(self, binding: ClaimBinding) -> None: + """An enforcement claim may not be silently unbacked.""" + if binding.kind != ENFORCEMENT: + return + assert binding.controls or binding.unproven_reason, ( + f"Enforcement claim {binding.claim_id!r} names no control and gives no " + "unproven_reason. One or the other is required: a documented enforcement " + "claim with neither is exactly the unbacked assertion AAASM-5526 exists " + "to eliminate." + ) + if not binding.controls: + assert re.search(r"AAASM-\d+", binding.unproven_reason), ( + f"Claim {binding.claim_id!r} is unproven but its reason names no " + "ticket. An unproven claim must be traceable to the work that " + f"resolves it. Reason given: {binding.unproven_reason!r}" + ) + + @pytest.mark.parametrize("binding", BINDINGS, ids=lambda b: b.claim_id) + def test_named_sdk_symbols_resolve_to_that_name(self, binding: ClaimBinding) -> None: + """Renaming an SDK class the document names fails here.""" + for documented_name, module_path in binding.symbols.items(): + module = importlib.import_module(module_path) + resolved = getattr(module, documented_name, None) + assert resolved is not None, ( + f"{_QUICK_START.name} names `{documented_name}` but that symbol no " + f"longer exists in {module_path}. The class was renamed or moved, so " + "the documented quick-start now points at something a reader cannot " + "import. Update the documentation and this binding together." + ) + assert resolved.__name__ == documented_name, ( + f"{_QUICK_START.name} names {documented_name!r} but the resolved " + f"symbol reports __name__ == {resolved.__name__!r} — the documented " + "name is an alias for a class that has been renamed underneath it." + ) + assert f"`{documented_name}`" in _section_text(), ( + f"ClaimBinding {binding.claim_id!r} declares the symbol " + f"{documented_name!r} but the section no longer mentions it." + ) From 91a8d5014245b2d1705d4d8bd59c1b6e3b6f7129 Mon Sep 17 00:00:00 2001 From: Bryant Date: Thu, 13 Aug 2026 18:38:53 +0800 Subject: [PATCH 2/9] =?UTF-8?q?=F0=9F=94=A7=20(ci):=20Run=20the=20claim-bi?= =?UTF-8?q?nding=20gate=20where=20a=20docs-only=20PR=20can=20reach=20it?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ci.yaml's paths allow-list excludes docs/**, so a PR that only rewords a quick-start claim gets no CI at all — which is exactly the change the claim-binding gate exists to catch. quickstart-tabs-check.yml already triggers on docs/quick-start.md, so the gate runs there. The negative-control module and the exceptions package are added to the trigger paths as well. ci.yaml already covers both under test/**/*.py and agent_assembly/**/*.py; the entries here are what make a renamed control or a renamed exception re-run this workflow too. What the two jobs prove is deliberately different, and neither covers the other: drift-check round-trips the §3 tabs as text and never parses, imports or executes a snippet; claim-bindings gates the prose claims in §"What just happened" against named controls. Refs AAASM-5529 --- .github/workflows/quickstart-tabs-check.yml | 34 +++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/.github/workflows/quickstart-tabs-check.yml b/.github/workflows/quickstart-tabs-check.yml index a3f18b54..6c680457 100644 --- a/.github/workflows/quickstart-tabs-check.yml +++ b/.github/workflows/quickstart-tabs-check.yml @@ -13,6 +13,13 @@ on: - "scripts/generate_quickstart_tabs.py" - "docs/quick-start.md" - ".github/workflows/quickstart-tabs-check.yml" +# AAASM-5529: the claim-binding gate below reads the negative controls and the +# exception the quick-start names, so an edit to either must re-run it here. +# ci.yaml already covers them under agent_assembly/**/*.py and test/**/*.py; +# these entries matter for the docs-only PR, which ci.yaml deliberately skips. + - "test/unit/test_quickstart_claim_bindings.py" + - "test/unit/test_quickstart_negative_control.py" + - "agent_assembly/exceptions/**" push: branches: - main @@ -47,3 +54,30 @@ jobs: echo "::error:: python scripts/generate_quickstart_tabs.py" exit 1 fi + + claim-bindings: + # AAASM-5529. The drift-check job above round-trips the §3 tabs as *text*: it + # proves the generated document matches the vendored snippets and nothing + # more — the snippets are never parsed, imported or executed (ruff.toml + # excludes them; they reference names they never define). + # + # This job gates a different surface: §"What just happened", where the + # quick-start states what governance did. It binds each claim to the control + # that proves it, so a claim cannot be added or reworded without someone + # naming the evidence. + # + # It runs here rather than only in ci.yaml because ci.yaml's paths allow-list + # excludes docs/**, so a docs-only PR — precisely the change that rewords a + # claim — gets no CI at all. + name: quick-start claim bindings + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v7 + + - name: Install the SDK and its dev dependencies + run: uv sync + + - name: Every documented enforcement claim names the control that proves it + run: uv run pytest test/unit/test_quickstart_claim_bindings.py -q --no-cov From a3c5d6ef598909bedf4ee4aa5050304df135dc19 Mon Sep 17 00:00:00 2001 From: Bryant Date: Thu, 13 Aug 2026 19:25:54 +0800 Subject: [PATCH 3/9] =?UTF-8?q?=F0=9F=90=9B=20(test):=20Gate=20whole=20sen?= =?UTF-8?q?tences=20across=20the=20whole=20document,=20not=20fragments=20i?= =?UTF-8?q?n=20a=20region?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review found the central promise breaks in under a minute. Three defects, all in this file: F1 — the gate asked whether the region CONTAINED a binding's quote, so a sentence could carry unlimited extra unbound claims as long as one fragment survived. Python bound at list-item granularity, so an appended sentence rode free. Quotes are now whole sentences compared with ==, and exactly one binding may match a sentence. F6 — only two regions were scanned, so claims elsewhere were invisible. Notably "the other modes add network/kernel interception" at the foot of the page, which is a named acceptance check of this very ticket. The scan now covers the whole document minus two named allow-lists, each entry carrying a reason. An allow-listed sentence must still be present verbatim, so an entry cannot silently cover a reworded claim. F4 — kind=LIFECYCLE was an unchecked one-word bypass: relabel a claim, drop its controls, no ticket needed, green. It was also already live in-tree, on a binding whose unproven_reason named no ticket. The field is removed rather than fixed; every claim now needs controls or a ticketed reason. F5 — the vocabulary was three alternatives, so a claim phrased around them was not treated as a claim. Widened to match the Go gate's, plus throws, routed, intercepts, governed, verified, protection and bypass. Refs AAASM-5529 --- test/unit/test_quickstart_claim_bindings.py | 429 ++++++++++++-------- 1 file changed, 264 insertions(+), 165 deletions(-) diff --git a/test/unit/test_quickstart_claim_bindings.py b/test/unit/test_quickstart_claim_bindings.py index bb3c9f4e..a3e56827 100644 --- a/test/unit/test_quickstart_claim_bindings.py +++ b/test/unit/test_quickstart_claim_bindings.py @@ -2,30 +2,37 @@ AAASM-5529, Epic AAASM-5526. -``docs/quick-start.md`` §"What just happened" is where the Python quick-start -tells a reader what governance did for them. Those sentences are the product's -load-bearing enforcement claims, and until now nothing connected them to the -negative controls in :mod:`test.unit.test_quickstart_negative_control`. A claim -could be added, reworded or left standing after the behaviour beneath it changed, -and no gate would notice. +``docs/quick-start.md`` tells a reader what governance did for them. Those +sentences are the product's load-bearing enforcement claims, and until now +nothing connected them to the negative controls in +:mod:`test.unit.test_quickstart_negative_control`. A claim could be added, +reworded, or left standing after the behaviour beneath it changed, and no gate +would notice. What this gate proves --------------------- -#. **Every enforcement claim in the section is bound to a named control.** The - claim list is parsed out of the document, so adding a fifth numbered claim - without registering a binding for it fails here rather than shipping an - unbacked sentence. -#. **Every binding still describes the document.** Each binding quotes the - load-bearing fragment of its claim; rewording the sentence in the document - breaks the quote and fails. -#. **Every control a binding names still exists.** The control names are - extracted from the negative-control module's AST, not transcribed, so - renaming or deleting a control fails here. -#. **Every SDK symbol the section names is real.** The symbol is imported and - its ``__name__`` compared, so renaming ``ToolExecutionBlockedError`` in the - SDK fails here instead of leaving the documentation pointing at a class that - no longer exists. +#. **The whole document is scanned, not an opted-in section.** Every sentence + that uses enforcement vocabulary anywhere in the quick-start must be bound. + Regions and sentences may be excluded only through the two named allow-lists + below, each entry carrying a reason and an exact sentence — so an allow-list + entry cannot cover a reworded or newly added claim. +#. **A binding must match a whole sentence, exactly.** ``quote`` is compared + with ``==`` against the flattened sentence, never with ``in``. Substring + containment let a sentence carry unlimited extra unbound claims — including + its own negation — as long as one bound fragment survived, which is the + defect this revision exists to close. +#. **Exactly one binding may match a sentence,** so two bindings cannot quietly + split responsibility for one claim and leave neither owning it. +#. **Every control a binding names still exists.** Control names are extracted + from the negative-control module's AST, not transcribed, so renaming or + deleting one fails here. +#. **Every claim is proven or openly unproven.** There is no claim category + exempt from that: a binding names controls, or names a ticket. The former + ``kind`` field was removed because it was a one-word bypass — relabelling a + claim as lifecycle disabled the requirement entirely. +#. **Every SDK symbol the document names is real,** resolved lazily so a rename + fails on the assertion rather than aborting collection. What this gate does **not** prove --------------------------------- @@ -35,13 +42,13 @@ (``ruff.toml`` excludes it for this reason), and the snippets are partial governance slices that reference names they never define — ``gateway_url``, ``api_key``, ``src.policy`` — so they are not importable modules. The existing -``quickstart-tabs-check`` workflow round-trips them as *text*, asserting only -that the generated document matches the vendored copy. Neither that gate nor +``quickstart-tabs-check`` drift job round-trips them as *text*, asserting only +that the generated document matches the vendored copy. Neither that job nor this one type-checks, imports or runs a snippet. Nor does binding a claim make the claim *true*. A binding records which control -stands behind a sentence; where no control does, the binding must say so and -name the ticket, which is the state claim 1 is in today (AAASM-5661). +stands behind a sentence; where none does, the binding must say so and name the +ticket. """ from __future__ import annotations @@ -58,23 +65,43 @@ # attribute name, rather than imported here. Importing them at module scope # makes a rename a *collection* error, which aborts before # test_named_sdk_symbols_resolve_to_that_name can run — leaving the assertion -# that is supposed to catch the rename permanently unexercised. That is the -# same inverted-order defect the round-1 review of this ticket found in all -# three SDKs, and it is invisible unless you mutate and watch which line fails. +# that is supposed to catch the rename permanently unexercised. _REPO_ROOT = Path(__file__).resolve().parents[2] _QUICK_START = _REPO_ROOT / "docs" / "quick-start.md" _NEGATIVE_CONTROL = Path(__file__).with_name("test_quickstart_negative_control.py") -_SECTION_HEADING = "## What just happened" +#: Sentences using any of these make a claim about what governance does. Kept +#: deliberately wide: a narrow vocabulary is itself a bypass, because a new +#: enforcement paragraph phrased around it is not treated as a claim at all. +_ENFORCEMENT_VOCABULARY = re.compile( + r"(?i)\bdenie[sd]\b|\bdeny\b|\bblocked\b|\bblocking\b|\bnever runs?\b" + r"|\bbefore execution\b|\bchecked against\b|\benforces?\b|\benforced\b" + r"|\bpassthrough\b|\bdiscards?\b|\bdiscarded\b|\bthrows?\b|\brejects?\b" + r"|\brouted\b|\bintercepts?\b|\binterception\b|\bgoverned\b|\bverified\b" + r"|\bprotection\b|\bunprotected\b|\bbypass(ed|es)?\b" +) -#: Claims that assert governance acted on a tool call. These are the ones the -#: Epic exists for, and the ones a binding must back with a control. -ENFORCEMENT = "enforcement" -#: Claims about setup or teardown. They are still bound, so the parser's -#: completeness check cannot be satisfied by silently dropping one, but they do -#: not require an enforcement control. -LIFECYCLE = "lifecycle" +#: Whole sections excluded from the scan, each with the reason. Keyed by the +#: exact heading line. +_EXCLUDED_SECTIONS: dict[str, str] = { + "## Next steps": ( + "A link list. Every line is a cross-reference to another page; the " + "claims themselves live on the pages linked to and are gated there." + ), +} + +#: Individual sentences excluded from the scan, each with the reason. These are +#: exact flattened sentences, never patterns, so an entry cannot silently cover +#: a reworded or newly added claim — changing the sentence makes the entry stale +#: and test_every_excluded_sentence_is_still_present_verbatim fails. +_EXCLUDED_SENTENCES: dict[str, str] = { + "See [Handling allow/deny decisions](guides/handling-decisions.md) for how to catch and respond to " + "those, and [Troubleshooting](troubleshooting.md) if `init_assembly()` itself raised.": ( + "Navigational cross-reference. It makes no capability claim of its " + "own; it matches the vocabulary only through the linked page's title." + ), +} @dataclass(frozen=True) @@ -82,130 +109,157 @@ class ClaimBinding: """One documented claim and the controls that stand behind it.""" claim_id: str - kind: str - #: A verbatim fragment of the claim as it appears in the document, with - #: newlines collapsed. Rewording the document breaks this. + #: The claim as a WHOLE sentence, flattened. Compared with ==, not `in`. quote: str #: ``ClassName::test_name`` node ids in the negative-control module. controls: tuple[str, ...] = () #: Set when no control proves the claim. Must name the ticket that tracks it. unproven_reason: str = "" #: Backticked SDK identifiers the claim names, mapped to the module they - #: must be importable from. Resolved lazily — see the note at the top. + #: must be importable from. Resolved lazily. symbols: dict[str, str] = field(default_factory=dict) +_DENY_CONTROLS = ( + "TestFilesystemSideEffect::test_negative_control_denied_write_leaves_no_file", + "TestNetworkSideEffect::test_negative_control_denied_egress_never_reaches_the_listener", +) +_ALLOW_AND_DENY_CONTROLS = ( + "TestFilesystemSideEffect::test_positive_control_allowed_write_creates_the_file", + "TestFilesystemSideEffect::test_negative_control_denied_write_leaves_no_file", + "TestNetworkSideEffect::test_positive_control_allowed_egress_reaches_the_listener", + "TestNetworkSideEffect::test_negative_control_denied_egress_never_reaches_the_listener", +) + BINDINGS: tuple[ClaimBinding, ...] = ( ClaimBinding( - claim_id="init-routes-every-tool-call", - kind=ENFORCEMENT, - quote="every tool call from this point on is routed", - # Deliberately unbacked. AAASM-5661 measured the documented - # configuration and found this sentence overstates it: the controls in - # the negative-control module all call install_fake_core(), supplying an - # authoritative runtime the documented configuration does not have, so - # they are structurally incapable of proving a claim about the - # documented path. Binding it to one of them would launder that gap into - # evidence. The honest state is a named, ticketed absence. + claim_id="gateway-returns-allow-deny-decisions", + quote=("`init_assembly()` needs to reach a **gateway** — the policy brain that returns allow/deny decisions."), + # AAASM-5661 measured the documented configuration: it reaches no + # gateway and installs a deny-all fail-closed interceptor instead. No + # control covers the documented path, because every control here + # supplies a fake native core the documented path does not have. unproven_reason=( - "AAASM-5661: the documented configuration was measured and no control " - "covers it. Every control in test_quickstart_negative_control.py " - "installs a fake native core, which the documented path does not have." + "AAASM-5661: the documented configuration was measured and reaches no gateway. " + "Every control in test_quickstart_negative_control.py installs a fake native " + "core, so none of them exercises the path this sentence describes." ), ), ClaimBinding( - claim_id="sdk-only-enforces-on-tool-calls", - kind=ENFORCEMENT, - quote="The in-process adapter enforces on tool calls with no", - controls=( - "TestFilesystemSideEffect::test_negative_control_denied_write_leaves_no_file", - "TestNetworkSideEffect::test_negative_control_denied_egress_never_reaches_the_listener", + claim_id="init-routes-every-tool-call", + quote=( + "**`init_assembly()` wired in governance.** It registered the agent with the gateway " + "and auto-loaded the adapter for your framework — every tool call from this point on " + "is routed through the policy gate." + ), + unproven_reason=( + "AAASM-5661: measured false for the documented configuration. Binding this to a " + "control that installs a fake native core would launder that gap into evidence." ), ), ClaimBinding( - claim_id="verdict-precedes-execution", - kind=ENFORCEMENT, - quote="asks the policy engine for an allow/deny verdict before the tool actually runs", - # The two negative controls prove the *before* by absence of the side - # effect; the two positive controls prove the probe would have seen the - # effect had it happened. Both halves are named, because either alone is - # the vacuous evidence this Epic exists to remove. - controls=( - "TestFilesystemSideEffect::test_positive_control_allowed_write_creates_the_file", - "TestFilesystemSideEffect::test_negative_control_denied_write_leaves_no_file", - "TestNetworkSideEffect::test_positive_control_allowed_egress_reaches_the_listener", - "TestNetworkSideEffect::test_negative_control_denied_egress_never_reaches_the_listener", + claim_id="sdk-only-enforces-on-tool-calls", + quote=( + '**`mode="sdk-only"` kept it offline.** The in-process adapter enforces on tool calls ' + "with no network sidecar, so the example runs deterministically with no real LLM or " + "gateway round-trip." ), + controls=_DENY_CONTROLS, ), ClaimBinding( - claim_id="with-block-unwinds", - kind=LIFECYCLE, - quote="tore everything down on exit", - unproven_reason=( - "Teardown is covered by the context-manager tests, not by the enforcement controls this gate binds." + claim_id="verdict-precedes-execution", + quote=( + "**Tool calls were governed.** The adapter intercepts the framework's tool-invocation " + "path and asks the policy engine for an allow/deny verdict before the tool actually " + "runs." ), + # Both halves are named. The negative controls prove the "before" by + # absence of the side effect; the positive controls prove the probe + # would have seen that effect had it happened. Either alone is the + # vacuous evidence this Epic exists to remove. + controls=_ALLOW_AND_DENY_CONTROLS, ), ClaimBinding( claim_id="deny-surfaces-as-tool-execution-blocked", - kind=ENFORCEMENT, - quote="that is not a bug — the policy denied the", + quote=("If a tool call raises a `ToolExecutionBlockedError`, that is not a bug — the policy denied the call."), controls=( - "TestFilesystemSideEffect::test_negative_control_denied_write_leaves_no_file", + *_DENY_CONTROLS, "TestDegradedRuntimeCannotLookProtected" "::test_an_unavailable_native_runtime_denies_rather_than_silently_allowing", ), symbols={"ToolExecutionBlockedError": "agent_assembly.exceptions"}, ), + ClaimBinding( + claim_id="sdk-only-is-the-in-process-interception-layer", + quote=( + '`mode="sdk-only"` is the in-process-only interception layer: the framework adapter ' + "enforces on tool calls, with no network sidecar to start." + ), + controls=_DENY_CONTROLS, + ), + ClaimBinding( + claim_id="other-modes-add-network-kernel-interception", + quote=( + "The other modes (`auto`, `proxy`, `ebpf`) add network/kernel interception — see " + "[Core Concepts → Modes](concepts/index.md#runtime-modes)." + ), + # This is the sentence AAASM-5529's own SDK-specific check names: + # "mode=auto/proxy/ebpf does not report verified network protection + # unless the corresponding layer is actually running and probed." No + # control in this repo probes a proxy or eBPF layer, so the claim rests + # on the Core Concepts page's authority, not on evidence here. + unproven_reason=( + "AAASM-5529: this ticket's own mode-probing acceptance check is not delivered. " + "No control in the Python SDK starts or probes a proxy or eBPF layer, so nothing " + "here can distinguish 'the mode adds interception' from 'the mode is selected'." + ), + ), ) -def _section_text() -> str: - """Return the "What just happened" section, up to the next ``##`` heading.""" - document = _QUICK_START.read_text(encoding="utf-8") - start = document.find(_SECTION_HEADING) - assert start != -1, ( - f"{_QUICK_START} no longer contains a '{_SECTION_HEADING}' section. " - "If the quick-start was restructured, re-point this gate at the section " - "that now carries the enforcement claims — do not delete it." - ) - body = document[start + len(_SECTION_HEADING) :] - end = body.find("\n## ") - return body if end == -1 else body[:end] +def _document() -> str: + return _QUICK_START.read_text(encoding="utf-8") def _flatten(text: str) -> str: - """Collapse Markdown's soft wrapping so a quote can span wrapped lines.""" + """Collapse Markdown's soft wrapping so a sentence is one line.""" return re.sub(r"\s+", " ", text).strip() -def _documented_claims() -> dict[str, str]: - """Parse the section into ``claim_key -> flattened text``. +def _scanned_sentences() -> dict[str, str]: + """Return ``flattened sentence -> section heading`` for the whole document. - Numbered list items become ``item-N``; the trailing prose paragraph becomes - ``prose-N``. Both are derived from the document, so a newly added claim - appears here without anyone updating this module — which is the point. + Fenced code is dropped, and sections named in :data:`_EXCLUDED_SECTIONS` are + skipped. Everything else is in scope — the gate opts sections *out* by name + rather than opting them in, so a claim added to a section nobody thought + about is still caught. """ - section = _section_text() - claims: dict[str, str] = {} - - # Numbered items: "1. ..." through the line before the next "N. " or a blank - # line followed by unindented prose. - item_pattern = re.compile(r"^(\d+)\.\s+(.*(?:\n(?![ ]*\d+\.\s|\n).*)*)", re.MULTILINE) - for match in item_pattern.finditer(section): - claims[f"item-{match.group(1)}"] = _flatten(match.group(2)) - - # Prose paragraphs that make a claim, i.e. mention denial or blocking. Link - # lists and prose that merely points elsewhere are not claims. - consumed = {match.group(0) for match in item_pattern.finditer(section)} - remainder = section - for chunk in consumed: - remainder = remainder.replace(chunk, "\n") - for index, paragraph in enumerate(p for p in remainder.split("\n\n") if p.strip()): - flat = _flatten(paragraph) - if re.search(r"\bdenied\b|\bblocked\b|\bdeny\b", flat, re.IGNORECASE): - claims[f"prose-{index}"] = flat - - return claims + body = re.sub(r"```.*?```", " ", _document(), flags=re.DOTALL) + + sentences: dict[str, str] = {} + section = "(preamble)" + for chunk in re.split(r"(?m)^(#{2,6} .*)$", body): + if chunk is None: + continue + if re.match(r"^#{2,6} ", chunk): + section = chunk.strip() + continue + if section in _EXCLUDED_SECTIONS: + continue + for raw in re.split(r"(?<=\.)\s+", chunk): + flat = _flatten(raw) + if flat: + sentences[flat] = section + return sentences + + +def _claim_sentences() -> dict[str, str]: + """The scanned sentences that make an enforcement claim, minus the allow-list.""" + return { + sentence: section + for sentence, section in _scanned_sentences().items() + if _ENFORCEMENT_VOCABULARY.search(sentence) and sentence not in _EXCLUDED_SENTENCES + } def _control_node_ids() -> set[str]: @@ -227,57 +281,104 @@ def _control_node_ids() -> set[str]: class TestTheGateCanSeeWhatItGates: - """Positive controls. Every check below reads a real artifact; prove it arrived.""" + """Positive controls. Every check below reads a real artifact; prove it arrived. + + An empty parse and a clean result are otherwise indistinguishable, which is + the failure mode that makes a drift gate worthless without ever going red. + """ - def test_the_quick_start_section_is_found_and_non_empty(self) -> None: - section = _section_text() - assert len(section.strip()) > 200, "the parsed section is too short to contain the claim list" + def test_the_document_is_read_and_split_into_sentences(self) -> None: + sentences = _scanned_sentences() + assert len(sentences) > 40, f"only {len(sentences)} sentences parsed from the whole quick-start" - def test_the_parser_finds_the_numbered_claims(self) -> None: - claims = _documented_claims() - numbered = [key for key in claims if key.startswith("item-")] - # A count is asserted, not a list, because the list is the thing under - # test. If the document grows a claim this fails, which is the gate. - assert len(numbered) >= 4, f"expected the four documented claims, parsed {sorted(claims)}" + def test_the_scan_finds_enforcement_claims(self) -> None: + claims = _claim_sentences() + assert len(claims) >= 7, f"only {len(claims)} claim sentences found: {sorted(claims)}" + + def test_the_scan_reaches_beyond_the_what_just_happened_section(self) -> None: + """The whole document is in scope, not one opted-in region. + + Without this, narrowing the scan back to a single section would look + identical to a clean pass. + """ + sections = set(_claim_sentences().values()) + assert len(sections) >= 3, f"claims were found in only these sections: {sections}" def test_the_ast_extraction_finds_the_negative_controls(self) -> None: node_ids = _control_node_ids() assert len(node_ids) >= 8, f"AST extraction found only {len(node_ids)} controls: {sorted(node_ids)}" - # A named one, so an extraction that silently returned an unrelated set - # cannot satisfy the count above. assert "TestFilesystemSideEffect::test_negative_control_denied_write_leaves_no_file" in node_ids +class TestTheAllowListCannotBecomeABypass: + def test_every_excluded_section_is_still_a_real_heading(self) -> None: + document = _document() + for heading, reason in _EXCLUDED_SECTIONS.items(): + assert heading in document, ( + f"_EXCLUDED_SECTIONS names {heading!r}, which is no longer a heading in " + f"{_QUICK_START.name}. A stale exclusion silently widens over time — remove it." + ) + assert reason.strip(), f"exclusion {heading!r} carries no reason" + + def test_every_excluded_sentence_is_still_present_verbatim(self) -> None: + """An allow-listed sentence must still exist, exactly. + + This is what stops the allow-list becoming the new bypass: an entry is a + whole sentence, so rewording the claim makes the entry stale and fails + here rather than silently exempting the new wording. + """ + scanned = _scanned_sentences() + for sentence, reason in _EXCLUDED_SENTENCES.items(): + assert sentence in scanned, ( + f"_EXCLUDED_SENTENCES contains a sentence that no longer appears in " + f"{_QUICK_START.name}:\n {sentence!r}\n" + "It was reworded or removed. Delete the stale entry, and if the replacement " + "makes an enforcement claim, bind it." + ) + assert reason.strip(), f"exclusion of {sentence!r} carries no reason" + + class TestEveryDocumentedClaimIsBound: - def test_no_claim_in_the_section_is_unbound(self) -> None: - """Adding a claim to the quick-start without a binding fails here. + def test_no_enforcement_sentence_is_unbound(self) -> None: + """Adding an enforcement claim anywhere in the quick-start fails here. This is the check that makes the gate load-bearing rather than decorative: a new enforcement sentence cannot reach the published - quick-start without someone naming the control that stands behind it, - or recording in the binding that none does. + quick-start without someone naming the control that stands behind it. """ - documented = _documented_claims() - unmatched = { - key: text for key, text in documented.items() if not any(binding.quote in text for binding in BINDINGS) - } + quotes = {binding.quote for binding in BINDINGS} + unmatched = {sentence: section for sentence, section in _claim_sentences().items() if sentence not in quotes} assert not unmatched, ( - "These quick-start claims have no ClaimBinding in BINDINGS:\n" - + "\n".join(f" {key}: {text}" for key, text in unmatched.items()) - + "\n\nAdd a ClaimBinding naming the control that proves each one. If no " - "control does, set unproven_reason and name the ticket — do not delete " - "the claim from this gate to make it pass." + "These quick-start sentences make an enforcement claim and have no ClaimBinding:\n" + + "\n".join(f" [{section}] {sentence}" for sentence, section in unmatched.items()) + + "\n\nAdd a ClaimBinding whose quote is the WHOLE sentence, naming the control that " + "proves it. If no control does, set unproven_reason and name the ticket. If the " + "sentence genuinely makes no capability claim, add it to _EXCLUDED_SENTENCES with a " + "reason — do not delete the claim from this gate to make it pass." ) @pytest.mark.parametrize("binding", BINDINGS, ids=lambda b: b.claim_id) - def test_each_binding_still_quotes_the_document(self, binding: ClaimBinding) -> None: - """Rewording a claim in the document fails here.""" - documented = _documented_claims() - assert any(binding.quote in text for text in documented.values()), ( - f"ClaimBinding {binding.claim_id!r} quotes:\n {binding.quote!r}\n" - f"which no longer appears in {_SECTION_HEADING!r} of {_QUICK_START.name}. " - "The claim was reworded or removed. Update the quote and re-check that " - "the named controls still prove the new wording." + def test_each_binding_matches_exactly_one_whole_sentence(self, binding: ClaimBinding) -> None: + """Rewording any part of a bound claim fails here. + + Whole-sentence equality, not containment. Containment allowed a sentence + to carry extra unbound claims — up to and including its own negation — + while one bound fragment kept the gate green. + """ + matches = [sentence for sentence in _scanned_sentences() if sentence == binding.quote] + assert len(matches) == 1, ( + f"ClaimBinding {binding.claim_id!r} must match exactly one whole sentence in " + f"{_QUICK_START.name}; it matched {len(matches)}.\nIts quote is:\n {binding.quote!r}\n" + "The claim was reworded, split, or merged. Update the quote to the new whole " + "sentence and re-check that the named controls still prove it." + ) + + def test_no_two_bindings_claim_the_same_sentence(self) -> None: + quotes = [binding.quote for binding in BINDINGS] + duplicates = {quote for quote in quotes if quotes.count(quote) > 1} + assert not duplicates, ( + f"More than one ClaimBinding quotes the same sentence: {duplicates}. " + "Split responsibility like that and neither binding owns the claim." ) @@ -295,21 +396,23 @@ def test_named_controls_exist(self, binding: ClaimBinding) -> None: ) @pytest.mark.parametrize("binding", BINDINGS, ids=lambda b: b.claim_id) - def test_an_enforcement_claim_is_either_proven_or_openly_unproven(self, binding: ClaimBinding) -> None: - """An enforcement claim may not be silently unbacked.""" - if binding.kind != ENFORCEMENT: - return + def test_a_claim_is_either_proven_or_openly_unproven(self, binding: ClaimBinding) -> None: + """Every claim, with no exempt category. + + There used to be a ``kind`` field here, and setting it to "lifecycle" + skipped this check entirely — a one-word bypass that needed no ticket + and no control. It was removed rather than fixed. + """ assert binding.controls or binding.unproven_reason, ( - f"Enforcement claim {binding.claim_id!r} names no control and gives no " - "unproven_reason. One or the other is required: a documented enforcement " - "claim with neither is exactly the unbacked assertion AAASM-5526 exists " - "to eliminate." + f"Claim {binding.claim_id!r} names no control and gives no unproven_reason. One or " + "the other is required: a documented claim with neither is exactly the unbacked " + "assertion AAASM-5526 exists to eliminate." ) if not binding.controls: assert re.search(r"AAASM-\d+", binding.unproven_reason), ( - f"Claim {binding.claim_id!r} is unproven but its reason names no " - "ticket. An unproven claim must be traceable to the work that " - f"resolves it. Reason given: {binding.unproven_reason!r}" + f"Claim {binding.claim_id!r} is unproven but its reason names no ticket. An " + "unproven claim must be traceable to the work that resolves it. Reason given: " + f"{binding.unproven_reason!r}" ) @pytest.mark.parametrize("binding", BINDINGS, ids=lambda b: b.claim_id) @@ -329,7 +432,3 @@ def test_named_sdk_symbols_resolve_to_that_name(self, binding: ClaimBinding) -> f"symbol reports __name__ == {resolved.__name__!r} — the documented " "name is an alias for a class that has been renamed underneath it." ) - assert f"`{documented_name}`" in _section_text(), ( - f"ClaimBinding {binding.claim_id!r} declares the symbol " - f"{documented_name!r} but the section no longer mentions it." - ) From d73c14c2ee7b2145d8373c3b9e799afd6b90aa3a Mon Sep 17 00:00:00 2001 From: Bryant Date: Thu, 13 Aug 2026 19:37:06 +0800 Subject: [PATCH 4/9] =?UTF-8?q?=F0=9F=90=9B=20(test):=20Treat=20a=20fenced?= =?UTF-8?q?=20block=20as=20a=20paragraph=20break,=20not=20a=20space?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replacing a fenced block with a space glued the sentence before a code sample to the sentence after it. Measured on this document: 18 such pairs. A glued pair is fragment containment one level up — a binding quoting it would cover two claims at once, and the second could then be reworded or negated without the gate noticing. Found while fixing the same bug in the Go and Node gates, then checked for here rather than assumed absent. No binding changed: none of the 18 pairs carries enforcement vocabulary today. The hole was latent, not live. Refs AAASM-5529 --- test/unit/test_quickstart_claim_bindings.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/test/unit/test_quickstart_claim_bindings.py b/test/unit/test_quickstart_claim_bindings.py index a3e56827..7dc0b04a 100644 --- a/test/unit/test_quickstart_claim_bindings.py +++ b/test/unit/test_quickstart_claim_bindings.py @@ -234,7 +234,12 @@ def _scanned_sentences() -> dict[str, str]: rather than opting them in, so a claim added to a section nobody thought about is still caught. """ - body = re.sub(r"```.*?```", " ", _document(), flags=re.DOTALL) + # A fenced block becomes a PARAGRAPH break, not a space. Replacing it with a + # space glued the sentence before a code sample to the sentence after it — + # 18 such pairs in this document — and a binding quoting the glued pair + # would then cover two claims at once, which is fragment containment one + # level up. + body = re.sub(r"```.*?```", "\n\n", _document(), flags=re.DOTALL) sentences: dict[str, str] = {} section = "(preamble)" @@ -246,10 +251,11 @@ def _scanned_sentences() -> dict[str, str]: continue if section in _EXCLUDED_SECTIONS: continue - for raw in re.split(r"(?<=\.)\s+", chunk): - flat = _flatten(raw) - if flat: - sentences[flat] = section + for paragraph in chunk.split("\n\n"): + for raw in re.split(r"(?<=\.)\s+", paragraph): + flat = _flatten(raw) + if flat: + sentences[flat] = section return sentences From fd850be2c6eb126abf21e7b1a0a9c4878b6c7cb1 Mon Sep 17 00:00:00 2001 From: Bryant Date: Thu, 13 Aug 2026 19:44:34 +0800 Subject: [PATCH 5/9] =?UTF-8?q?=F0=9F=90=9B=20(test):=20Normalise=20line?= =?UTF-8?q?=20endings=20before=20splitting=20the=20document?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Node's four Windows CI legs failed the equivalent gate: git checks the doc out with CRLF, so the paragraph split never fires and the section collapses into one "sentence" that matches no binding. This repo's CI is Linux-only, so the bug is latent here rather than live. Normalised anyway — a gate whose result depends on the checkout's line endings is not a gate. Verified against a CRLF copy of the document. Refs AAASM-5529 --- test/unit/test_quickstart_claim_bindings.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/test/unit/test_quickstart_claim_bindings.py b/test/unit/test_quickstart_claim_bindings.py index 7dc0b04a..11f37033 100644 --- a/test/unit/test_quickstart_claim_bindings.py +++ b/test/unit/test_quickstart_claim_bindings.py @@ -218,7 +218,16 @@ class ClaimBinding: def _document() -> str: - return _QUICK_START.read_text(encoding="utf-8") + """Read the quick-start with line endings normalised to LF. + + Without this the paragraph split never fires on a CRLF checkout: the whole + section collapses into one "sentence" that matches no binding. Node's four + Windows CI legs caught exactly that while Linux and macOS stayed green. + This repo's CI is Linux-only, so the bug is latent here — normalised anyway, + because a gate whose result depends on the checkout's line endings is not a + gate. + """ + return _QUICK_START.read_text(encoding="utf-8").replace("\r\n", "\n") def _flatten(text: str) -> str: From 57654354d68c7a06f0bcb2dbb084574b266f56e7 Mon Sep 17 00:00:00 2001 From: Bryant Date: Thu, 13 Aug 2026 20:13:39 +0800 Subject: [PATCH 6/9] =?UTF-8?q?=F0=9F=90=9B=20(test):=20Invert=20the=20def?= =?UTF-8?q?ault=20=E2=80=94=20every=20sentence=20bound=20or=20allow-listed?= =?UTF-8?q?,=20no=20keyword=20filter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review appended three plain sentences to the quick-start, the last of them "Tool bodies always execute; the policy result is recorded alongside them" — the negation of the product — and this gate stayed green. None of the three used any of the 21 vocabulary terms. Widening 3 -> 21 closed the instance, not the class: a keyword allow-list cannot be completed, because whoever adds the claim picks the words after reading the list. R1 — the vocabulary no longer gates anything. Every one of the document's 88 sentences must be bound to a control or allow-listed by exact text. The vocabulary survives as a severity hint in the failure message, and as the trigger for a stricter allow-list rule: waving through a sentence that reads like a claim costs a written justification, not a category. R3 — section exclusions are gone. An excluded section was a black hole: the guard checked the heading still existed and said nothing about its contents, so a claim inserted into "## Next steps" was never scanned. A positive control now asserts that section is in the scan. R4 — HTML and MDX comments are stripped, as fences already were, and for the same reason: a reader cannot see them. Leaving them in let a bound claim be commented out of the rendered page while the gate still counted it. R5 — an unproven_reason may no longer name AAASM-5529, the ticket this module implements. The mode=auto registration did, and would have resolved to a closed issue the moment this merged, with nothing noticing. Repointed at AAASM-5536, and the reason now states plainly that no ticket owns proving the claim and the expected resolution is qualification. R6 — the splitter no longer emits bare list markers as sentences, treats "!" as text so mkdocs "!!! note" survives intact, handles "?" as a terminator, splits table rows and list items as units, and strips front matter, which Python lacked where Go and Node had it. Inverting the default immediately found five claims every keyword-gated revision was blind to, including the page's own central promise ("whose tool calls pass through the Agent Assembly policy gate") and an unbounded breadth claim ("governs whichever agent framework you already use"). Four are now registered unproven against AAASM-5661 and AAASM-5536; the teardown claim, previously exempt under the removed `kind` field, is bound to a real control. Refs AAASM-5529 --- test/unit/test_quickstart_claim_bindings.py | 684 ++++++++++++-------- 1 file changed, 430 insertions(+), 254 deletions(-) diff --git a/test/unit/test_quickstart_claim_bindings.py b/test/unit/test_quickstart_claim_bindings.py index 11f37033..418949ad 100644 --- a/test/unit/test_quickstart_claim_bindings.py +++ b/test/unit/test_quickstart_claim_bindings.py @@ -1,54 +1,61 @@ -"""Drift gate binding the quick-start's enforcement claims to the controls that prove them. +"""Drift gate binding the quick-start's claims to the controls that prove them. AAASM-5529, Epic AAASM-5526. -``docs/quick-start.md`` tells a reader what governance did for them. Those -sentences are the product's load-bearing enforcement claims, and until now -nothing connected them to the negative controls in -:mod:`test.unit.test_quickstart_negative_control`. A claim could be added, -reworded, or left standing after the behaviour beneath it changed, and no gate -would notice. +Every sentence in ``docs/quick-start.md`` must be either **bound** to a control +that proves it, or **explicitly allow-listed** as making no capability claim. +There is no third state and no keyword filter. + +Why the default is inverted +--------------------------- + +Earlier revisions only scanned sentences matching an enforcement vocabulary. +Review appended three plain sentences that used none of the 21 terms — the last +of them, *"Tool bodies always execute; the policy result is recorded alongside +them"*, is the negation of the product — and all three gates stayed green. +Widening 3 → 21 terms closed the instance and not the class: **a keyword +allow-list cannot be completed, because whoever adds the claim picks the words +after reading the list.** + +So the vocabulary no longer gates anything. It survives only as a *severity +hint* in the failure message, and as the trigger for a stricter allow-list rule +(:data:`_ALLOWED` entries whose sentence matches it need a bespoke written +justification, not a category). + +There are no section-level exclusions either. An excluded section was a black +hole: the guard checked the heading still existed and said nothing about its +contents, so a claim inserted into ``## Next steps`` was never scanned at all. What this gate proves --------------------- -#. **The whole document is scanned, not an opted-in section.** Every sentence - that uses enforcement vocabulary anywhere in the quick-start must be bound. - Regions and sentences may be excluded only through the two named allow-lists - below, each entry carrying a reason and an exact sentence — so an allow-list - entry cannot cover a reworded or newly added claim. -#. **A binding must match a whole sentence, exactly.** ``quote`` is compared - with ``==`` against the flattened sentence, never with ``in``. Substring - containment let a sentence carry unlimited extra unbound claims — including - its own negation — as long as one bound fragment survived, which is the - defect this revision exists to close. -#. **Exactly one binding may match a sentence,** so two bindings cannot quietly - split responsibility for one claim and leave neither owning it. -#. **Every control a binding names still exists.** Control names are extracted - from the negative-control module's AST, not transcribed, so renaming or - deleting one fails here. -#. **Every claim is proven or openly unproven.** There is no claim category - exempt from that: a binding names controls, or names a ticket. The former - ``kind`` field was removed because it was a one-word bypass — relabelling a - claim as lifecycle disabled the requirement entirely. -#. **Every SDK symbol the document names is real,** resolved lazily so a rename - fails on the assertion rather than aborting collection. +#. **Every sentence in the document is accounted for.** Add a sentence anywhere + — any section, any wording — and it fails until someone binds it or + allow-lists it by exact text. +#. **A binding matches a whole sentence, exactly** (``==``, never ``in``), and + exactly one binding may match a sentence. Substring containment let a + sentence carry extra unbound claims, including its own negation. +#. **Every control a binding names still exists**, extracted from the control + modules' ASTs rather than transcribed. +#. **Every claim is proven or openly unproven**, with no exempt category, and an + unproven claim must name a ticket that is *not* the ticket this module + implements — a pointer at one's own ticket resolves to a closed issue the + moment that ticket merges. +#. **Comments are stripped before scanning**, because a reader cannot see them. + Leaving them in let a bound claim be commented out of the rendered page while + the gate still counted it. What this gate does **not** prove --------------------------------- -It does not execute the quick-start, and it cannot: ``quickstart_snippets/`` is -a vendored, verbatim copy of regions from the ``examples`` repository -(``ruff.toml`` excludes it for this reason), and the snippets are partial -governance slices that reference names they never define — ``gateway_url``, -``api_key``, ``src.policy`` — so they are not importable modules. The existing -``quickstart-tabs-check`` drift job round-trips them as *text*, asserting only -that the generated document matches the vendored copy. Neither that job nor -this one type-checks, imports or runs a snippet. - -Nor does binding a claim make the claim *true*. A binding records which control -stands behind a sentence; where none does, the binding must say so and name the -ticket. +It does not execute, import or type-check a quick-start snippet. +``quickstart_snippets/`` is a vendored verbatim copy of regions from the +``examples`` repository (``ruff.toml`` excludes it), and the snippets reference +names they never define. The ``quickstart-tabs-check`` drift job round-trips +them as *text* only. Neither job runs a snippet. + +Nor does binding a claim make it true. A binding records which control stands +behind a sentence; where none does, it says so and names the ticket. """ from __future__ import annotations @@ -61,47 +68,39 @@ import pytest -# NOTE: the SDK symbols a claim names are resolved lazily, by module path and +# NOTE: SDK symbols a claim names are resolved lazily, by module path and # attribute name, rather than imported here. Importing them at module scope -# makes a rename a *collection* error, which aborts before -# test_named_sdk_symbols_resolve_to_that_name can run — leaving the assertion -# that is supposed to catch the rename permanently unexercised. +# makes a rename a *collection* error, aborting before the assertion meant to +# catch it can run. + +#: The ticket this module implements. An unproven claim may not name it — see +#: test_an_unproven_reason_does_not_name_the_implementing_ticket. +IMPLEMENTING_TICKET = "AAASM-5529" _REPO_ROOT = Path(__file__).resolve().parents[2] _QUICK_START = _REPO_ROOT / "docs" / "quick-start.md" -_NEGATIVE_CONTROL = Path(__file__).with_name("test_quickstart_negative_control.py") -#: Sentences using any of these make a claim about what governance does. Kept -#: deliberately wide: a narrow vocabulary is itself a bypass, because a new -#: enforcement paragraph phrased around it is not treated as a claim at all. +#: Modules a binding may name a control from. +_CONTROL_MODULES = ( + Path(__file__).with_name("test_quickstart_negative_control.py"), + Path(__file__).with_name("test_assembly.py"), +) + +#: NOT a gate. A severity hint in the failure message, and the trigger for the +#: stricter allow-list rule below. See the module docstring for why gating on a +#: keyword list is unsound. _ENFORCEMENT_VOCABULARY = re.compile( r"(?i)\bdenie[sd]\b|\bdeny\b|\bblocked\b|\bblocking\b|\bnever runs?\b" r"|\bbefore execution\b|\bchecked against\b|\benforces?\b|\benforced\b" r"|\bpassthrough\b|\bdiscards?\b|\bdiscarded\b|\bthrows?\b|\brejects?\b" - r"|\brouted\b|\bintercepts?\b|\binterception\b|\bgoverned\b|\bverified\b" - r"|\bprotection\b|\bunprotected\b|\bbypass(ed|es)?\b" + r"|\brouted\b|\bintercepts?\b|\binterception\b|\bgovern(s|ed|ance)?\b" + r"|\bverified\b|\bprotection\b|\bunprotected\b|\bbypass(ed|es)?\b" ) -#: Whole sections excluded from the scan, each with the reason. Keyed by the -#: exact heading line. -_EXCLUDED_SECTIONS: dict[str, str] = { - "## Next steps": ( - "A link list. Every line is a cross-reference to another page; the " - "claims themselves live on the pages linked to and are gated there." - ), -} - -#: Individual sentences excluded from the scan, each with the reason. These are -#: exact flattened sentences, never patterns, so an entry cannot silently cover -#: a reworded or newly added claim — changing the sentence makes the entry stale -#: and test_every_excluded_sentence_is_still_present_verbatim fails. -_EXCLUDED_SENTENCES: dict[str, str] = { - "See [Handling allow/deny decisions](guides/handling-decisions.md) for how to catch and respond to " - "those, and [Troubleshooting](troubleshooting.md) if `init_assembly()` itself raised.": ( - "Navigational cross-reference. It makes no capability claim of its " - "own; it matches the vocabulary only through the linked page's title." - ), -} +#: Allow-list categories. A category is only permitted for a sentence that does +#: NOT match the vocabulary above; anything that does needs a written reason. +_NOT_A_CAPABILITY_CLAIM = "Descriptive or instructional prose. Says nothing about what governance does to a tool call." +_NAVIGATION = "A cross-reference. The claim, if any, lives on the page linked to and is gated there." @dataclass(frozen=True) @@ -111,12 +110,12 @@ class ClaimBinding: claim_id: str #: The claim as a WHOLE sentence, flattened. Compared with ==, not `in`. quote: str - #: ``ClassName::test_name`` node ids in the negative-control module. + #: ``ClassName::test_name`` or ``test_name`` ids from _CONTROL_MODULES. controls: tuple[str, ...] = () - #: Set when no control proves the claim. Must name the ticket that tracks it. + #: Set when no control proves the claim. Must name a ticket, and must not + #: name IMPLEMENTING_TICKET. unproven_reason: str = "" - #: Backticked SDK identifiers the claim names, mapped to the module they - #: must be importable from. Resolved lazily. + #: Backticked SDK identifiers the claim names -> the module they live in. symbols: dict[str, str] = field(default_factory=dict) @@ -126,25 +125,81 @@ class ClaimBinding: ) _ALLOW_AND_DENY_CONTROLS = ( "TestFilesystemSideEffect::test_positive_control_allowed_write_creates_the_file", - "TestFilesystemSideEffect::test_negative_control_denied_write_leaves_no_file", + *_DENY_CONTROLS, "TestNetworkSideEffect::test_positive_control_allowed_egress_reaches_the_listener", - "TestNetworkSideEffect::test_negative_control_denied_egress_never_reaches_the_listener", +) + +#: AAASM-5661 measured the documented configuration: it reaches no gateway and +#: installs a deny-all fail-closed interceptor. Every control here calls +#: install_fake_core(), supplying an authoritative runtime the documented path +#: does not have, so none of them exercises what these sentences describe. +#: Binding one would launder that gap into evidence. +_DOCUMENTED_PATH_UNMEASURED = ( + "AAASM-5661: the documented configuration was measured and does not behave as this " + "sentence says. No control covers it — every control in " + "test_quickstart_negative_control.py installs a fake native core the documented " + "path does not have." ) BINDINGS: tuple[ClaimBinding, ...] = ( ClaimBinding( - claim_id="gateway-returns-allow-deny-decisions", - quote=("`init_assembly()` needs to reach a **gateway** — the policy brain that returns allow/deny decisions."), - # AAASM-5661 measured the documented configuration: it reaches no - # gateway and installs a deny-all fail-closed interceptor instead. No - # control covers the documented path, because every control here - # supplies a fake native core the documented path does not have. + claim_id="tool-calls-pass-through-the-policy-gate", + quote=( + "By the end you'll have an agent — in whichever framework you already use — whose " + "tool calls pass through the Agent Assembly policy gate, and it runs **offline** " + "against a local policy, so you need no API keys and no network access to the " + "outside world." + ), + # Found by inverting the default. It states the page's central promise + # and matches no enforcement keyword, so every earlier revision of this + # gate was blind to it. + unproven_reason=_DOCUMENTED_PATH_UNMEASURED, + ), + ClaimBinding( + claim_id="governs-whichever-framework-you-use", + quote="Agent Assembly governs whichever agent framework you already use.", + # A breadth claim with no boundary. The controls prove the shared + # governed-tool chain, not "whichever framework"; per ADR 0033 §6 a + # claim like this needs a named boundary or a qualification. unproven_reason=( - "AAASM-5661: the documented configuration was measured and reaches no gateway. " - "Every control in test_quickstart_negative_control.py installs a fake native " - "core, so none of them exercises the path this sentence describes." + "AAASM-5536: an unbounded breadth claim. The controls prove the shared " + "governed-tool chain behind two adapter tabs, not every framework the page " + "offers, and no control enumerates them. AAASM-5536 is the documentation " + "claim gate that forces such a sentence to name its boundary or be qualified." ), ), + ClaimBinding( + claim_id="auto-start-probes-and-starts-a-gateway", + quote=( + "**Let the SDK auto-start one.** Call `init_assembly()` with no `gateway_url`; the " + "SDK probes `http://localhost:7391` and, if nothing answers, runs `aasm start " + "--mode local --foreground` for you." + ), + # Measured during this ticket's scoping pass and reported: the + # [project.scripts] aasm console script shadows the bundled Rust binary + # on PATH, so find_aasm_binary() resolves the Python one, which has no + # `start` subcommand. The documented auto-start therefore cannot work + # from a clean install. + unproven_reason=( + "AAASM-5661: no control covers the documented auto-start path, and it was " + "measured not to work from a clean install — the [project.scripts] aasm " + "console script shadows the bundled binary, and the shadowing one has no " + "'start' subcommand." + ), + ), + ClaimBinding( + claim_id="no-arg-init-connects-and-appears-in-dashboard", + quote=( + "You don't configure `:50051` yourself — registration dials it automatically — so a " + "no-argument `init_assembly()` both connects and shows the agent in the dashboard." + ), + unproven_reason=_DOCUMENTED_PATH_UNMEASURED, + ), + ClaimBinding( + claim_id="gateway-returns-allow-deny-decisions", + quote=("`init_assembly()` needs to reach a **gateway** — the policy brain that returns allow/deny decisions."), + unproven_reason=_DOCUMENTED_PATH_UNMEASURED, + ), ClaimBinding( claim_id="init-routes-every-tool-call", quote=( @@ -152,10 +207,7 @@ class ClaimBinding: "and auto-loaded the adapter for your framework — every tool call from this point on " "is routed through the policy gate." ), - unproven_reason=( - "AAASM-5661: measured false for the documented configuration. Binding this to a " - "control that installs a fake native core would launder that gap into evidence." - ), + unproven_reason=_DOCUMENTED_PATH_UNMEASURED, ), ClaimBinding( claim_id="sdk-only-enforces-on-tool-calls", @@ -173,12 +225,21 @@ class ClaimBinding: "path and asks the policy engine for an allow/deny verdict before the tool actually " "runs." ), - # Both halves are named. The negative controls prove the "before" by - # absence of the side effect; the positive controls prove the probe - # would have seen that effect had it happened. Either alone is the - # vacuous evidence this Epic exists to remove. + # Both halves. The negative controls prove the "before" by absence of + # the side effect; the positive controls prove the probe would have seen + # that effect had it happened. Either alone is vacuous. controls=_ALLOW_AND_DENY_CONTROLS, ), + ClaimBinding( + claim_id="with-block-tears-everything-down", + quote=( + "**The `with` block tore everything down on exit** — adapter hooks were unwound and " + "the gateway connection closed, leaving the process exactly as it was before." + ), + # Previously exempt under the removed `kind` field. Under the inverted + # default it is a claim like any other and needs a control. + controls=("test_context_manager_shutdown_calls_adapter_unregister_hooks",), + ), ClaimBinding( claim_id="deny-surfaces-as-tool-execution-blocked", quote=("If a tool call raises a `ToolExecutionBlockedError`, that is not a bug — the policy denied the call."), @@ -203,247 +264,362 @@ class ClaimBinding: "The other modes (`auto`, `proxy`, `ebpf`) add network/kernel interception — see " "[Core Concepts → Modes](concepts/index.md#runtime-modes)." ), - # This is the sentence AAASM-5529's own SDK-specific check names: - # "mode=auto/proxy/ebpf does not report verified network protection - # unless the corresponding layer is actually running and probed." No - # control in this repo probes a proxy or eBPF layer, so the claim rests - # on the Core Concepts page's authority, not on evidence here. + # This previously named AAASM-5529 — the ticket this module implements — + # which would have resolved to a closed issue the moment this merged. + # No ticket currently owns *proving* it: nothing in this SDK starts or + # probes a proxy or eBPF layer, so the honest resolution is + # qualification rather than proof, which is AAASM-5536's job. unproven_reason=( - "AAASM-5529: this ticket's own mode-probing acceptance check is not delivered. " - "No control in the Python SDK starts or probes a proxy or eBPF layer, so nothing " - "here can distinguish 'the mode adds interception' from 'the mode is selected'." + "AAASM-5536: no control in the Python SDK starts or probes a proxy or eBPF " + "layer, so nothing here distinguishes 'the mode adds interception' from 'the " + "mode is selected'. No ticket owns proving this; the expected resolution is " + "that AAASM-5536's claim gate forces the sentence to name its boundary." ), ), ) +#: Every sentence in the quick-start that makes no capability claim, keyed +#: exactly. A category is only permitted where the sentence does not match +#: _ENFORCEMENT_VOCABULARY; anything that does needs a written justification. +_ALLOWED: dict[str, str] = { + "Govern your first agent in about five minutes.": ( + "The page's title line. It matches the vocabulary on the imperative verb " + "'Govern', which names what the reader is about to do, not what the product " + "guarantees. The substantive promise is the next sentence, which IS bound." + ), + "The package is published on PyPI as [`{{ aa.python_sdk.package_name }}`]({{ aa.urls.pypi }}) (current version: `{{ aa.python_sdk.version }}`).": _NOT_A_CAPABILITY_CLAIM, + '=== "pip"': _NOT_A_CAPABILITY_CLAIM, + '=== "uv"': _NOT_A_CAPABILITY_CLAIM, + '=== "poetry"': _NOT_A_CAPABILITY_CLAIM, + '=== "conda"': _NOT_A_CAPABILITY_CLAIM, + "`{{ aa.python_sdk.package_name }}` is not published on conda-forge or the Anaconda default channel — create a conda environment, then install from PyPI with `pip` inside it:": _NOT_A_CAPABILITY_CLAIM, + '!!! note "`--pre` is required for now" Agent Assembly is currently published only as a pre-release on PyPI, and `pip` skips pre-releases unless you pass `--pre` (already included above).': _NOT_A_CAPABILITY_CLAIM, + "Drop the flag once a stable (non-pre-release) version is published.": _NOT_A_CAPABILITY_CLAIM, + "`{{ aa.python_sdk.package_name }}` is the pure-Python client.": _NOT_A_CAPABILITY_CLAIM, + "`{{ aa.python_sdk.package_name }}[runtime]` additionally pulls a platform wheel (`manylinux`, `macosx`) that bundles the `{{ aa.python_sdk.cli_name }}` gateway/runtime binary, so a local gateway is available without a separate install.": _NOT_A_CAPABILITY_CLAIM, + "You have three options:": _NOT_A_CAPABILITY_CLAIM, + "This needs the `aasm` binary on your `PATH` (the `agent-assembly[runtime]` extra provides it).": _NOT_A_CAPABILITY_CLAIM, + "**Run one yourself** with `aasm start --mode local --foreground` in a separate terminal.": _NOT_A_CAPABILITY_CLAIM, + "For a full gateway walkthrough, see the core [Run the gateway](https://docs.agent-assembly.com/core/latest/quick-start/first-run.html) guide.": _NAVIGATION, + "**Pass an explicit URL**, as the example below does.": _NOT_A_CAPABILITY_CLAIM, + "See [Configuration](configuration.md) for the full URL/key resolution chain (`7391` is the local default port).": _NAVIGATION, + '!!! note "Local-mode transports: `:7391` REST + `:50051` gRPC" Starting local mode binds **two** loopback surfaces in one process:': _NOT_A_CAPABILITY_CLAIM, + "This runs the REST/dashboard API on `http://localhost:7391` (what `gateway_url` points to, and what the SDK probes and auto-starts) **and** the gRPC `AgentLifecycleService` on `127.0.0.1:50051`, which is the endpoint the native SDK uses to **register** your agent.": _NOT_A_CAPABILITY_CLAIM, + "`:8080` is **not** the local gateway port; ignore older docs or examples that point registration there.": _NOT_A_CAPABILITY_CLAIM, + "To confirm both surfaces are actually up rather than guessing from the SDK's behavior, check them directly:": _NOT_A_CAPABILITY_CLAIM, + "Pick your framework below — each tab is the **governance-wiring slice** (`init_assembly()` plus " + "that framework's adapter hookup) taken verbatim from that framework's runnable example in the " + "[examples repo](https://github.com/ai-agent-assembly/examples/tree/master/python).": ( + "Describes where the tab content comes from. 'governance-wiring slice' names the " + "excerpt's provenance, not an enforcement outcome; it asserts nothing about " + "whether a denied call is stopped." + ), + "Copy the full, runnable script — imports, tools, and the agent run — from the linked example; " + "the slice below is the part that wires in governance.": ( + "An instruction to the reader about which lines to copy. 'wires in governance' " + "identifies the excerpt, and makes no claim about what that wiring then does to " + "a tool call." + ), + 'Every example runs **offline** in `mode="sdk-only"` against a local policy, so you can try it with no API keys and no outbound network.': _NOT_A_CAPABILITY_CLAIM, + '=== "Agno"': _NOT_A_CAPABILITY_CLAIM, + '!!! note "Version compatibility" Agno was previously published as **Phidata**; the rename replaced every `phi.*` import with `agno.*`.': _NOT_A_CAPABILITY_CLAIM, + "Before (Phidata): `from phi.agent import Agent`": _NOT_A_CAPABILITY_CLAIM, + "After (Agno): `from agno.agent import Agent`": _NOT_A_CAPABILITY_CLAIM, + "Source: [Agno's official Phidata → Agno migration guide](https://docs.agno.com/how-to/phidata-to-agno).": _NAVIGATION, + '=== "AutoGen"': _NOT_A_CAPABILITY_CLAIM, + "!!! note \"Version compatibility\" AutoGen's `v0.4` rewrite (2024) replaced the single `pyautogen` package's `autogen.agentchat` namespace with separate `autogen-agentchat` / `autogen-core` / `autogen-ext` packages, and `llm_config` with an explicit `model_client`.": _NOT_A_CAPABILITY_CLAIM, + "Before (v0.2, `pyautogen`): `from autogen.agentchat import AssistantAgent`": _NOT_A_CAPABILITY_CLAIM, + "After (v0.4+): `from autogen_agentchat.agents import AssistantAgent`": _NOT_A_CAPABILITY_CLAIM, + "Source: [AutoGen's official v0.2 → v0.4 migration guide](https://microsoft.github.io/autogen/stable/user-guide/agentchat-user-guide/migration-guide.html).": _NAVIGATION, + '=== "CrewAI"': _NOT_A_CAPABILITY_CLAIM, + '=== "Custom (no framework)"': _NOT_A_CAPABILITY_CLAIM, + '=== "Google ADK"': _NOT_A_CAPABILITY_CLAIM, + '=== "Haystack"': _NOT_A_CAPABILITY_CLAIM, + '!!! note "Version compatibility" Haystack 2.0 replaced the `farm-haystack` package with `haystack-ai` and flattened node imports into `haystack.components.*`; the two package versions cannot coexist in one environment.': _NOT_A_CAPABILITY_CLAIM, + "Before (Haystack 1.x, `farm-haystack`): `from haystack.nodes import BM25Retriever`": _NOT_A_CAPABILITY_CLAIM, + "After (Haystack 2.x, `haystack-ai`): `from haystack.components.retrievers.in_memory import InMemoryBM25Retriever`": _NOT_A_CAPABILITY_CLAIM, + "Source: [Haystack's official migration guide](https://docs.haystack.deepset.ai/docs/migration).": _NAVIGATION, + '=== "LangChain"': _NOT_A_CAPABILITY_CLAIM, + '!!! note "Version compatibility" LangChain\'s import surface moved twice: `langchain-core` split out of `langchain` across the `0.1` → `0.3` series (2024), and the `1.0` rewrite (2025) moved legacy chains/agents/tools out of `langchain` entirely into `langchain-classic`.': _NOT_A_CAPABILITY_CLAIM, + "Before (`<1.0`): `from langchain.agents import AgentExecutor, create_react_agent`": _NOT_A_CAPABILITY_CLAIM, + "After (`>=1.0`): `from langchain_classic.agents import AgentExecutor, create_react_agent` (requires the separate `langchain-classic` package)": _NOT_A_CAPABILITY_CLAIM, + "This SDK's own quick-start sample hit exactly this break — see AAASM-4451.": _NOT_A_CAPABILITY_CLAIM, + "Sources: [LangChain's official v1 migration guide](https://docs.langchain.com/oss/python/migrate/langchain-v1) and the [LangChain v0.3 announcement](https://www.langchain.com/blog/announcing-langchain-v0-3).": _NAVIGATION, + '=== "LangChain (Research Agent)"': _NOT_A_CAPABILITY_CLAIM, + '=== "LangGraph"': _NOT_A_CAPABILITY_CLAIM, + '!!! note "Version compatibility" LangGraph `1.0` deprecated `langgraph.prebuilt.create_react_agent` in favor of LangChain\'s own agent constructor.': _NOT_A_CAPABILITY_CLAIM, + "Before (`<1.0`): `from langgraph.prebuilt import create_react_agent`": _NOT_A_CAPABILITY_CLAIM, + "After (`>=1.0`): `from langchain.agents import create_agent`": _NOT_A_CAPABILITY_CLAIM, + "Source: [LangGraph's official v1 migration guide](https://docs.langchain.com/oss/python/migrate/langgraph-v1).": _NAVIGATION, + '=== "LlamaIndex"': _NOT_A_CAPABILITY_CLAIM, + '!!! note "Version compatibility" LlamaIndex `v0.10.0` (February 2024) split the monolithic `llama_index` package into a slim `llama-index-core` plus versioned per-provider packages (`llama-index-llms-openai`, etc.).': _NOT_A_CAPABILITY_CLAIM, + "An automated `llamaindex-cli upgrade` tool is provided for the migration.": _NOT_A_CAPABILITY_CLAIM, + "Before (`<0.10`): `from llama_index.llms import OpenAI`": _NOT_A_CAPABILITY_CLAIM, + "After (`>=0.10`): `from llama_index.llms.openai import OpenAI` (from the separate `llama-index-llms-openai` package)": _NOT_A_CAPABILITY_CLAIM, + "Source: [LlamaIndex's official v0.10 migration guide](https://www.llamaindex.ai/blog/llamaindex-v0-10-838e735948f8).": _NAVIGATION, + '=== "Microsoft Agent Framework"': _NOT_A_CAPABILITY_CLAIM, + '=== "OpenAI Agents SDK"': _NOT_A_CAPABILITY_CLAIM, + '=== "Pydantic AI"': _NOT_A_CAPABILITY_CLAIM, + '=== "Semantic Kernel"': _NOT_A_CAPABILITY_CLAIM, + '=== "smolagents"': _NOT_A_CAPABILITY_CLAIM, + '!!! note "Version compatibility" smolagents `v1.14.0` (April 2025) renamed `HfApiModel` to `InferenceClientModel` to reflect that it wraps any Hugging Face Inference Provider, not just the HF Hub; backward-compatible re-export was restored in `v1.24.0`.': _NOT_A_CAPABILITY_CLAIM, + "Before (`<1.14`): `from smolagents import HfApiModel`": _NOT_A_CAPABILITY_CLAIM, + "After (`>=1.14`): `from smolagents import InferenceClientModel`": _NOT_A_CAPABILITY_CLAIM, + "Source: [smolagents releases](https://github.com/huggingface/smolagents/releases).": _NAVIGATION, + '=== "Strands Agents"': _NOT_A_CAPABILITY_CLAIM, + "That's the product working.": _NOT_A_CAPABILITY_CLAIM, + "It's the most portable mode and the best choice for deterministic, offline examples and tests.": _NOT_A_CAPABILITY_CLAIM, + "**[Core Concepts](concepts/index.md)** — the adapter pattern, the `init_assembly()` lifecycle, and the modes/enforcement model.": _NAVIGATION, + "**[Examples](examples/index.md)** — wire the SDK into the framework you actually use.": _NAVIGATION, + "**[Configuration](configuration.md)** — drop the hard-coded URL and key; let the resolver chain find them.": _NAVIGATION, + "See [Handling allow/deny decisions](guides/handling-decisions.md) for how to catch and respond " + "to those, and [Troubleshooting](troubleshooting.md) if `init_assembly()` itself raised.": ( + "Navigational cross-reference. It matches the vocabulary only through the linked " + "page's title ('allow/deny decisions'); it asserts nothing about what governance " + "does. The claim it points at is gated on that page." + ), +} + + +_FRONT_MATTER = re.compile(r"\A---\n.*?\n---\n", re.DOTALL) +_FENCE = re.compile(r"```.*?```", re.DOTALL) +#: Comments are invisible to a reader, so a bound claim commented out of the +#: rendered page must not still satisfy this gate. Stripped for the same reason +#: fences are. Go's CommonMark HTML blocks and MDX's {/* */} form are covered +#: too, so the three gates strip the same things. +_HTML_COMMENT = re.compile(r"", re.DOTALL) +_MDX_COMMENT = re.compile(r"\{/\*.*?\*/\}", re.DOTALL) +_LIST_MARKER = re.compile(r"(?m)^\s*(?:[-*+]|\d+\.)\s+") +_UNIT_SPLIT = re.compile(r"(?m)^(?=\s*(?:[-*+]|\d+\.)\s|\|)") +#: '.' and '?' only. '!' is not a terminator here because mkdocs admonitions +#: open with '!!! note', which would otherwise split into a bare '!!!' unit. +_SENTENCE_END = re.compile(r"(?<=[.?])\s+") + def _document() -> str: """Read the quick-start with line endings normalised to LF. - Without this the paragraph split never fires on a CRLF checkout: the whole - section collapses into one "sentence" that matches no binding. Node's four - Windows CI legs caught exactly that while Linux and macOS stayed green. - This repo's CI is Linux-only, so the bug is latent here — normalised anyway, - because a gate whose result depends on the checkout's line endings is not a - gate. + Without this the paragraph split never fires on a CRLF checkout and the + whole section collapses into one sentence. Node's Windows CI legs caught + exactly that; this repo's CI is Linux-only, so it is latent here. """ return _QUICK_START.read_text(encoding="utf-8").replace("\r\n", "\n") -def _flatten(text: str) -> str: - """Collapse Markdown's soft wrapping so a sentence is one line.""" - return re.sub(r"\s+", " ", text).strip() - - def _scanned_sentences() -> dict[str, str]: - """Return ``flattened sentence -> section heading`` for the whole document. + """Return ``flattened sentence -> section heading`` for the WHOLE document. - Fenced code is dropped, and sections named in :data:`_EXCLUDED_SECTIONS` are - skipped. Everything else is in scope — the gate opts sections *out* by name - rather than opting them in, so a claim added to a section nobody thought - about is still caught. + No section is skipped. A section-level exclusion was a black hole: the guard + checked the heading still existed and said nothing about its contents, so a + claim inserted into an excluded section was never scanned. """ - # A fenced block becomes a PARAGRAPH break, not a space. Replacing it with a - # space glued the sentence before a code sample to the sentence after it — - # 18 such pairs in this document — and a binding quoting the glued pair - # would then cover two claims at once, which is fragment containment one - # level up. - body = re.sub(r"```.*?```", "\n\n", _document(), flags=re.DOTALL) + body = _document() + body = _FRONT_MATTER.sub("", body) + for pattern in (_FENCE, _HTML_COMMENT, _MDX_COMMENT): + body = pattern.sub("\n\n", body) sentences: dict[str, str] = {} section = "(preamble)" - for chunk in re.split(r"(?m)^(#{2,6} .*)$", body): + for chunk in re.split(r"(?m)^(#{1,6} .*)$", body): if chunk is None: continue - if re.match(r"^#{2,6} ", chunk): + if re.match(r"^#{1,6} ", chunk): section = chunk.strip() continue - if section in _EXCLUDED_SECTIONS: - continue for paragraph in chunk.split("\n\n"): - for raw in re.split(r"(?<=\.)\s+", paragraph): - flat = _flatten(raw) - if flat: - sentences[flat] = section + for unit in _UNIT_SPLIT.split(paragraph): + for raw in _SENTENCE_END.split(_LIST_MARKER.sub("", unit)): + flat = re.sub(r"\s+", " ", raw).strip() + if flat: + sentences[flat] = section return sentences -def _claim_sentences() -> dict[str, str]: - """The scanned sentences that make an enforcement claim, minus the allow-list.""" - return { - sentence: section - for sentence, section in _scanned_sentences().items() - if _ENFORCEMENT_VOCABULARY.search(sentence) and sentence not in _EXCLUDED_SENTENCES - } - - def _control_node_ids() -> set[str]: - """Extract ``ClassName::test_name`` ids from the negative-control module's AST. - - Derived from the source rather than transcribed, so this set changes when a - control is renamed or removed and the bindings above then fail. - """ - tree = ast.parse(_NEGATIVE_CONTROL.read_text(encoding="utf-8")) + """Extract control ids from the control modules' ASTs, not transcribed.""" node_ids: set[str] = set() - for node in tree.body: - if isinstance(node, ast.ClassDef): - for child in node.body: - if isinstance(child, ast.FunctionDef | ast.AsyncFunctionDef) and child.name.startswith("test_"): - node_ids.add(f"{node.name}::{child.name}") - elif isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef) and node.name.startswith("test_"): - node_ids.add(node.name) + for module in _CONTROL_MODULES: + tree = ast.parse(module.read_text(encoding="utf-8")) + for node in tree.body: + if isinstance(node, ast.ClassDef): + for child in node.body: + if isinstance(child, ast.FunctionDef | ast.AsyncFunctionDef) and child.name.startswith("test_"): + node_ids.add(f"{node.name}::{child.name}") + elif isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef) and node.name.startswith("test_"): + node_ids.add(node.name) return node_ids class TestTheGateCanSeeWhatItGates: - """Positive controls. Every check below reads a real artifact; prove it arrived. + """Positive controls. An empty parse and a clean result look identical.""" - An empty parse and a clean result are otherwise indistinguishable, which is - the failure mode that makes a drift gate worthless without ever going red. - """ - - def test_the_document_is_read_and_split_into_sentences(self) -> None: - sentences = _scanned_sentences() - assert len(sentences) > 40, f"only {len(sentences)} sentences parsed from the whole quick-start" + def test_the_whole_document_is_read_and_split(self) -> None: + assert len(_scanned_sentences()) > 60, "too few sentences parsed from the whole quick-start" - def test_the_scan_finds_enforcement_claims(self) -> None: - claims = _claim_sentences() - assert len(claims) >= 7, f"only {len(claims)} claim sentences found: {sorted(claims)}" + def test_the_scan_covers_every_section_including_the_last(self) -> None: + sections = set(_scanned_sentences().values()) + assert len(sections) >= 6, f"the scan reached only {len(sections)} sections: {sections}" + assert "## Next steps" in sections, ( + "'## Next steps' is not in the scan. It used to be excluded by name, which made it " + "a black hole: a claim inserted there was never seen. It must be scanned." + ) - def test_the_scan_reaches_beyond_the_what_just_happened_section(self) -> None: - """The whole document is in scope, not one opted-in region. + def test_comments_are_stripped_before_scanning(self) -> None: + """A commented-out sentence must not satisfy a binding. - Without this, narrowing the scan back to a single section would look - identical to a clean pass. + Positive control for the strip: the document contains HTML comments, and + none of their content may appear in the scan. """ - sections = set(_claim_sentences().values()) - assert len(sections) >= 3, f"claims were found in only these sections: {sections}" + assert "