From 31c64e7f942939f1af85f39f6241e87ed3515c48 Mon Sep 17 00:00:00 2001 From: Soumava2023 Date: Fri, 21 Aug 2026 13:08:21 -0400 Subject: [PATCH 01/13] docs: design markdown code references and gap triage --- ...wn-code-reference-and-gap-triage-design.md | 412 ++++++++++++++++++ 1 file changed, 412 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-21-markdown-code-reference-and-gap-triage-design.md diff --git a/docs/superpowers/specs/2026-08-21-markdown-code-reference-and-gap-triage-design.md b/docs/superpowers/specs/2026-08-21-markdown-code-reference-and-gap-triage-design.md new file mode 100644 index 0000000000..df01966887 --- /dev/null +++ b/docs/superpowers/specs/2026-08-21-markdown-code-reference-and-gap-triage-design.md @@ -0,0 +1,412 @@ +# Design: Markdown-to-Code References and Actionable Gap Triage + +**Date:** 2026-08-21 +**Branch:** v8 +**Consumer validation corpus:** DebtGPS + +--- + +## Problem + +Graphify currently builds useful structural graphs for code and useful semantic graphs for documents, but three deterministic boundaries leave avoidable gaps in a mixed code-and-document corpus: + +1. The Markdown extractor follows links only when the target is another document. A design or domain document can link to `server/routes_planning.py` or `server/routes_planning.py#L83`, but Graphify discards that reference instead of connecting the document to the implementation it describes. +2. Python import-guided resolution canonicalizes callable uses, but imported type aliases such as `OrderFn` can remain as sourceless stubs beside their source-backed definition. +3. Knowledge-gap analysis treats several benign leaves as actionable gaps. Framework symbols, test-library symbols, rationale leaves, and metadata/configuration keys can dominate isolated-node and thin-community counts even when no project work is missing. + +DebtGPS exposes all three limitations. Its domain-model documentation contains explicit code references that should become graph edges; `OrderFn` is split across imported and defined representations; and symbols such as `route`, `Flask`, `parametrize`, `given`, and `composite` are dependencies rather than missing local implementations. + +The enhancement must improve graph fidelity without introducing LLM inference into code resolution, deleting useful semantic nodes, or changing existing document-to-document link behavior. + +--- + +## Goals + +- Convert local Markdown links to code files into deterministic `references` edges. +- Use a `#L` fragment to connect a document to the exact or nearest source-backed symbol in that file. +- Canonicalize Python imported type-alias references when import evidence uniquely identifies a source-backed alias. +- Mark provable external symbols consistently and remove them from actionable gap counts while retaining them in the graph. +- Separate actionable local gaps from benign external, rationale, and metadata/configuration leaves in analysis and reporting. +- Preserve semantic extraction cache behavior and merge the four currently uncached DebtGPS documents with the 134 cached semantic results. +- Add the missing rationale for the DebtGPS refinance route and prove the documented implementation-to-verification links are represented in the rebuilt graph. + +## Non-goals + +- Resolving arbitrary prose mentions that are not links. +- Resolving Markdown heading fragments to document-heading nodes; existing document-link behavior remains unchanged. +- Inferring symbol ranges with a language server or compiler. +- Binding a sourceless name to a same-named local symbol without unique path/import evidence. +- Removing external, rationale, or metadata nodes from `graph.json`. +- Running an LLM over code; code remains AST-extracted and deterministically resolved. +- Reclassifying low-cohesion communities as bugs. The change only makes their reported composition explicit. + +--- + +## Design Principles + +1. **Evidence before name matching.** A local path plus line anchor, or a Python import plus module/name pair, is sufficient evidence. A bare matching label is not. +2. **Lossless graph, selective reporting.** Benign nodes remain queryable; only their gap classification changes. +3. **Backward-compatible extraction.** Existing Markdown document links, wikilinks, reference definitions, cache stamps, and incremental remapping continue to work. +4. **Deterministic fallback.** Ambiguous or malformed references fall back to a file node or remain unresolved; they never pick a symbol nondeterministically. +5. **One classification vocabulary.** Analysis and report rendering consume the same node-classification helper so their counts cannot drift. + +--- + +## Architecture + +```text +Markdown extractor + local link + optional #L line + | + v + references edge stamped with target_file / target_line + | + v +whole-corpus deterministic resolution pass + target file index + source-line symbol index + | + +--> exact symbol at line + +--> nearest preceding symbol in file + +--> target file node fallback + +Python extraction fragments + imports + type-alias definitions/references + | + v +import-guided symbol resolution + unique (module, alias) source-backed match + | + v +canonical alias node; duplicate stub rewired/removed + +final graph + | + v +shared gap classifier + actionable_local | external | rationale | metadata | structural + | + +--> suggest_questions() + +--> GRAPH_REPORT.md gap and community breakdowns +``` + +The Markdown extractor records resolution evidence but does not select a code symbol itself. At per-file extraction time, it cannot see all source nodes and would produce order-dependent results. Symbol selection therefore happens in the existing whole-corpus normalization/resolution stage after all file fragments are available and before edge stamps are removed. + +--- + +## Component 1: Structured Markdown Link Targets + +### Files + +- `graphify/extractors/markdown.py` +- `graphify/extractors/base.py` or a new small shared path-classification helper if importing the central extractor registry would create a cycle +- `tests/test_languages.py` +- `tests/test_incremental.py` +- `tests/test_cache.py` + +### Target representation + +Introduce an immutable internal value object, `ResolvedMarkdownTarget`, with: + +```python +path: Path +line: int | None +``` + +A new parser/resolver returns this structured target. `_resolve_markdown_link()` remains as a compatibility wrapper that returns only `Path | None` for existing callers and tests. + +### Accepted targets + +- Existing document extensions remain accepted exactly as today. +- Local files with extensions already recognized as code by Graphify are accepted as code targets. +- `#L83` and case-insensitive `#l83` are accepted as one-based line anchors for code targets. +- A query string may precede the fragment; both are removed from the filesystem path. +- External URLs, protocol-relative URLs, `mailto:`, `tel:`, `data:`, images, and pure in-page anchors continue to be skipped. +- An absent, zero, negative, non-numeric, or overflow line anchor is treated as no usable line evidence; the local file link can still resolve to its file node. +- Extensionless wikilinks remain document links. The feature does not guess that an extensionless name is code. + +### Extracted edge + +The source remains the Markdown page node. The edge remains: + +```json +{ + "relation": "references", + "confidence": "EXTRACTED", + "target_file": "", + "target_line": 83 +} +``` + +`target_line` is omitted when no valid code-line fragment exists. The provisional target ID is the normal file-node ID so the edge is still valid if later symbol resolution cannot improve it. The existing `target_file` stamp remains the authoritative incremental/canonical-path evidence. + +### Existing behavior preserved + +- Document targets still end at the target document page node, even if their URL has a heading fragment. +- Obsidian vault fallback is applied only to document wikilinks. +- Inline, reference-style, and wikilink extraction share the same structured parser. +- Cached paths continue to be remapped when a corpus root or file location changes. + +--- + +## Component 2: Line-Aware Code Target Resolution + +### Files + +- `graphify/extract.py` +- `graphify/symbol_resolution.py` +- `tests/test_symbol_resolution.py` +- `tests/test_node_id_canonical.py` +- `tests/test_incremental.py` + +### Index + +Build a per-source-file index from final source-backed code nodes. Each candidate must have: + +- `file_type == "code"` +- a non-empty `source_file` +- a parseable one-based `source_location` beginning with `L` +- a non-file node ID + +Candidates are sorted by `(start_line, node_id)` for deterministic lookup. File nodes are indexed separately through the existing canonical file-node mapping. + +### Resolution rule + +For every `references` edge carrying `target_file`: + +1. Canonicalize `target_file` with the same root/path logic used by other cross-file edges. +2. If `target_line` is absent, target the canonical file node. +3. If one or more non-file symbols start exactly at `target_line`, choose the most specific candidate deterministically. Specificity is based on node kind, preferring method/function/class/type-alias definitions over generic container nodes; ties use the stable node ID. +4. Otherwise choose the nearest preceding candidate in the same file. +5. If no preceding source-backed symbol exists, target the canonical file node. +6. Remove transient `target_file` and `target_line` stamps only after canonicalization, matching the current cleanup contract. + +Nearest-preceding resolution is intentionally conservative. The current graph stores start lines reliably but not uniform end lines across all languages. A following symbol must never be selected for a line that appears before it. + +### Failure behavior + +- A missing target file does not create a ghost code symbol. +- An ambiguous path preserves the existing deterministic path handling and falls back to a file node only when a unique canonical target exists. +- Malformed edge metadata is ignored without aborting extraction. +- Direct calls to a single-file extractor still return a valid file-target edge; symbol refinement is a corpus-level feature. + +--- + +## Component 3: Canonical Python Type Aliases + +### Files + +- `graphify/symbol_resolution.py` +- the Python extractor module that emits assignment/type-reference facts +- `tests/test_symbol_resolution.py` +- the focused Python extraction test module selected during implementation + +### Definition discovery + +Recognize source-backed module-level aliases from: + +- `OrderFn = Callable[...]` +- `OrderFn: TypeAlias = Callable[...]` +- Python 3.12 `type OrderFn = ...` when the running parser exposes `ast.TypeAlias` + +The alias node is a code node with its source file, source location, stable ID, label, and `node_kind: "type_alias"`. Only module-level definitions participate in file-wide import resolution. + +### Import-guided canonicalization + +Extend the existing `ImportedSymbol`-based flow with a symbol-reference resolution path that is independent of callable resolution: + +1. Parse top-level `from module import OrderFn` and aliased forms. +2. Index source-backed type aliases by normalized `(module stem, exported name)`. +3. Resolve only when the import evidence maps to exactly one candidate. +4. Rewire annotation/reference edges from the imported stub to the canonical source-backed alias. +5. Remove the sourceless stub only when all of its incident evidence has been transferred and it is not shared by an unresolved external reference. + +The callable resolver remains unchanged for function calls. Type aliases are not added to the callable label index merely to make this feature work. + +### Safety constraints + +- Plain `import module` member annotations are not resolved until receiver/module facts are retained by extraction. +- Star imports do not justify resolution. +- Function-local imports do not become file-wide evidence. +- Two matching source aliases remain unresolved and visible rather than being guessed. +- An external package exporting the same alias name never collapses onto a local alias without a uniquely matching local module path. + +--- + +## Component 4: External Symbol Classification + +### Files + +- `graphify/analyze.py` +- a shared classification module if report code cannot safely import the analysis helper +- `tests/test_analyze.py` + +Add a single node classifier used by both analysis and report generation. A node is `external` only when Graphify has affirmative evidence, including one of: + +- `external: true` +- `node_kind == "external_symbol"` +- sanitized metadata with `scip_kind == "external"` +- a recognized external-reference/stub ID namespace emitted by an extractor +- an extractor-originated external import/reference marker on the node + +Absence of `source_file` alone is not enough because semantic concepts also lack source paths. During graph normalization, provable external stubs should receive the common `external: true` and `node_kind: "external_symbol"` fields while retaining extractor-specific metadata. + +This classification covers dependency/framework nodes such as `route`, `Flask`, `parametrize`, `given`, and `composite` when their extraction evidence identifies them as external. It does not maintain a hard-coded library-name denylist. + +--- + +## Component 5: Actionable Gap and Thin-Community Reporting + +### Files + +- `graphify/analyze.py` +- `graphify/report.py` +- `tests/test_analyze.py` +- the focused report test module selected during implementation + +### Categories + +Every weakly connected node considered for gap reporting receives one category: + +| Category | Meaning | Actionable gap? | +|---|---|---:| +| `actionable_local` | Source-backed local implementation or project documentation with insufficient graph connections | Yes | +| `external` | Dependency/framework/test-library symbol | No | +| `rationale` | Decision rationale retained for semantic queries | No | +| `metadata` | Known manifest/configuration key or structural serialization leaf | No | +| `structural` | File/page/heading or intentional extractor scaffolding | No | + +The main “Knowledge Gaps” number and the generated isolated-node question use only `actionable_local`. The report also prints the suppressed-category counts so users can audit why the total changed. + +### Thin communities + +A thin community is no longer presented as one undifferentiated gap. For each low-cohesion community, the report records: + +- total nodes +- actionable local nodes +- external nodes +- rationale nodes +- metadata/structural nodes + +Communities with zero actionable local nodes are labeled benign and omitted from the top actionable-gap list, but remain in the community table. Communities with actionable nodes retain the current structural question and include representative local labels. + +This closes the DebtGPS triage gap without hiding the approximately 150 isolated and 115 thin-community nodes that motivated the enhancement: each is retained and placed into an auditable category. + +--- + +## Component 6: DebtGPS Rationale and Semantic Refresh + +### DebtGPS code change + +Add a concise rationale-bearing docstring to `server/routes_planning.py::refinance`. It must explain that the route validates request/ownership state and delegates financial calculations to the canonical refinance service; it must not duplicate calculation formulas or introduce runtime behavior. + +The docstring completes route-level rationale coverage alongside the existing domain model, budget, planning-state, refinance-tool, and verification documentation. + +### Semantic extraction + +After the patched Graphify package passes its tests and is installed into the active local tool environment: + +1. Re-run corpus detection and semantic-cache comparison. +2. Use the host-agent semantic backend because no Gemini/Google API key is configured. +3. Extract only the four uncached DebtGPS documents as one document chunk, following Graphify's extraction schema exactly. +4. Save the fresh semantic results in the normal content-hash cache. +5. Merge them with the 134 cached semantic files and the AST extraction results. +6. Build, cluster, analyze, and write the final graph/report/wiki outputs. + +No image extraction is needed unless the re-check discovers a changed or uncached image. Any newly uncached images must be processed one per extraction task. + +--- + +## Data Flow and Ordering + +The full validation build uses this order: + +```text +detect corpus and cache state + -> AST extraction for code + -> semantic extraction for uncached documents only + -> merge all per-file fragments + -> canonicalize file paths and imported symbols + -> refine Markdown code-link targets by line + -> normalize external-symbol metadata + -> build graph and prune replaced/deleted source data + -> cluster and score + -> classify actionable and benign gaps + -> write graph.json, analysis JSON, GRAPH_REPORT.md, and wiki +``` + +Line-aware Markdown resolution must happen after code node IDs are canonical and before transient edge stamps are removed. Gap classification must happen after external normalization and graph construction, because it depends on both node metadata and final degree. + +--- + +## Testing Strategy + +Implementation follows test-driven development: add one focused failing test, observe the intended failure, implement the smallest behavior, and then run the relevant regression group. + +### Markdown links + +- A Markdown link to `module.py` produces a `references` edge to the canonical file node. +- `module.py#L10` resolves to a symbol starting on line 10. +- A line inside a function resolves to the nearest preceding function/method/class definition. +- A line before the first symbol falls back to the file node. +- Invalid line anchors fall back to the file node. +- Document heading fragments retain document-page behavior. +- Images and external URLs remain absent from references. +- Cached and incremental extraction remap `target_file` and preserve `target_line` until final resolution. +- Full and incremental builds produce the same canonical target IDs. + +### Type aliases + +- Each supported alias syntax creates one source-backed `type_alias` node. +- Direct and `as` imports resolve annotation/reference edges to that node. +- Function-local, star, external, and ambiguous imports do not collapse. +- A duplicate sourceless `OrderFn` stub is removed only after safe rewiring. +- Callable resolution regressions remain green. + +### Gap classification + +- Explicit and SCIP external nodes are categorized as external. +- A sourceless semantic concept is not misclassified as external. +- Rationale, JSON noise, files/pages/headings, and external nodes do not increase actionable isolated counts. +- Actionable source-backed leaves still generate a question. +- Thin-community output exposes category totals and suppresses only all-benign communities from the actionable list. +- Report totals equal the sum of category counts. + +### DebtGPS acceptance checks + +- The refinance route contract test fails before and passes after the rationale docstring is added. +- All local links in `docs/debt-model.md` resolve. +- Every critical implementation node named by the DebtGPS documentation map gains at least one incoming document `references` edge; the current acceptance set contains nine nodes. +- `OrderFn` has exactly one source-backed canonical node and no unresolved local duplicate. +- The known framework/test symbols are retained but excluded from actionable gaps. +- Every previously isolated or thin-community node is either actionable or assigned a benign category. +- Existing focused DebtGPS verification suites remain green. +- The rebuilt report surfaces graph-health warnings, if any, rather than treating successful file generation as proof of graph health. + +--- + +## Installation and Rollback + +After the Graphify test suite passes, reinstall the active `graphify` tool from this local checkout using the same isolated `uv tool` environment currently providing the command. Verify the executable path and version/source before rebuilding DebtGPS. + +Rollback is reinstalling the published `graphifyy` package into that isolated tool environment and rebuilding DebtGPS with the previously saved graph outputs. The source patch remains committed in this checkout, so it can be reviewed or submitted upstream independently of the installed tool state. + +--- + +## Verification Gates + +The work is complete only when all gates pass: + +1. Focused Graphify tests for Markdown extraction, canonical IDs, incremental caching, Python symbol resolution, analysis, and reporting pass. +2. The broader Graphify test suite passes, or any unrelated pre-existing failures are identified with evidence. +3. The local Graphify build is installed and is the executable used for DebtGPS. +4. Semantic extraction processes only the cache misses discovered at execution time. +5. `graphify update .` or the full semantic build completes and refreshes DebtGPS graph artifacts. +6. The nine documentation targets, canonical `OrderFn`, external-node classification, isolated-node categories, and thin-community categories satisfy the acceptance checks above. +7. DebtGPS refinance and focused domain/planning/refinance/route verification tests pass. +8. The final handoff reports exact graph node/edge/community counts, semantic cache hits/misses, test results, remaining actionable gaps, and any graph-health warning. + +--- + +## Expected Outcome + +DebtGPS documentation becomes a first-class structural bridge into the implementation instead of a parallel semantic island. Local type aliases resolve to one canonical source definition. External framework/test symbols remain visible for queries but stop masquerading as missing work. The knowledge-gap report becomes an actionable backlog with an auditable benign remainder, and semantic extraction refreshes only the content that is actually uncached. From 945c35d211f288c01fe2a2527073fa30bf066e84 Mon Sep 17 00:00:00 2001 From: Soumava2023 Date: Fri, 21 Aug 2026 14:35:15 -0400 Subject: [PATCH 02/13] docs: plan markdown code references and gap triage --- ...-markdown-code-reference-and-gap-triage.md | 1260 +++++++++++++++++ 1 file changed, 1260 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-21-markdown-code-reference-and-gap-triage.md diff --git a/docs/superpowers/plans/2026-08-21-markdown-code-reference-and-gap-triage.md b/docs/superpowers/plans/2026-08-21-markdown-code-reference-and-gap-triage.md new file mode 100644 index 0000000000..2c7ce86112 --- /dev/null +++ b/docs/superpowers/plans/2026-08-21-markdown-code-reference-and-gap-triage.md @@ -0,0 +1,1260 @@ +# Markdown-to-Code References and Gap Triage Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make Graphify connect Markdown documentation to exact code symbols, canonicalize imported Python type aliases, and report only actionable local graph gaps, then use the enhanced extractor to refresh and verify DebtGPS. + +**Architecture:** Markdown extraction records a normalized target path and optional line anchor; a whole-corpus resolver maps that evidence to canonical code nodes after ID normalization. A separate deterministic Python pass materializes module-level type aliases and rewires uniquely import-supported references. A shared gap-classification module powers both analysis questions and report breakdowns without removing benign nodes from the graph. + +**Tech Stack:** Python 3.10+, tree-sitter-backed Graphify AST extraction, `ast`, NetworkX, pytest, uv tool environments, Markdown semantic extraction cache, DebtGPS Flask/pytest suite. + +--- + +## File Structure + +### Graphify checkout + +- Modify `graphify/extractors/markdown.py` — parse local document/code links into structured targets and retain line evidence on reference edges. +- Modify `graphify/symbol_resolution.py` — resolve line-targeted Markdown edges and canonicalize Python module-level type aliases. +- Modify `graphify/extract.py` — call the two deterministic corpus-level resolvers at the correct normalization points. +- Create `graphify/gaps.py` — single classification vocabulary and gap/community breakdown helpers. +- Modify `graphify/analyze.py` — use actionable-local filtering for isolated-node and low-cohesion questions. +- Modify `graphify/report.py` — render actionable counts plus auditable benign-category counts. +- Modify `tests/test_languages.py` — unit coverage for structured Markdown target parsing/extraction. +- Modify `tests/test_incremental.py` — full/incremental parity for line-targeted Markdown edges. +- Modify `tests/test_symbol_resolution.py` — code-line and Python type-alias resolution coverage. +- Modify `tests/test_analyze.py` — classification and suggested-question coverage. +- Create `tests/test_gap_reporting.py` — focused report breakdown tests. + +### DebtGPS consumer + +- Modify `server/routes_planning.py` — add the refinance route rationale docstring without changing runtime behavior. +- Modify `tests/test_coverage_gaps.py` — enforce that the route documents validation/delegation authority. +- Refresh `graphify-out/` — merge cached and newly extracted semantics with the patched AST graph. + +--- + +### Task 1: Parse and Extract Markdown Code Targets + +**Files:** +- Modify: `graphify/extractors/markdown.py:16-230` +- Modify: `graphify/extractors/markdown.py:295-350` +- Test: `tests/test_languages.py` + +- [ ] **Step 1: Write failing parser tests** + +Add imports for `ResolvedMarkdownTarget` and `_resolve_markdown_target`, then add: + +```python +def test_markdown_target_keeps_code_line_anchor(tmp_path): + source_dir = tmp_path / "docs" + source_dir.mkdir() + + target = _resolve_markdown_target( + "../src/service.py#L17", source_dir, wikilink=False + ) + + assert target == ResolvedMarkdownTarget( + path=tmp_path / "src" / "service.py", + line=17, + ) + + +@pytest.mark.parametrize("fragment", ["#L0", "#L-2", "#Labc", "#section"]) +def test_markdown_target_invalid_code_line_falls_back_to_file(tmp_path, fragment): + target = _resolve_markdown_target( + f"../src/service.py{fragment}", tmp_path / "docs", wikilink=False + ) + + assert target == ResolvedMarkdownTarget( + path=tmp_path / "src" / "service.py", + line=None, + ) + + +def test_markdown_target_keeps_document_fragment_behavior(tmp_path): + target = _resolve_markdown_target( + "./architecture.md#decisions", tmp_path, wikilink=False + ) + + assert target == ResolvedMarkdownTarget( + path=tmp_path / "architecture.md", + line=None, + ) +``` + +- [ ] **Step 2: Run parser tests and verify RED** + +Run: + +```powershell +uv run pytest tests/test_languages.py -k "markdown_target" -v +``` + +Expected: collection/import failure because `ResolvedMarkdownTarget` and `_resolve_markdown_target` do not exist. + +- [ ] **Step 3: Implement the structured target parser** + +In `graphify/extractors/markdown.py`, add: + +```python +from dataclasses import dataclass +from graphify.detect import CODE_EXTENSIONS + +_MD_LINE_FRAGMENT_RE = re.compile(r"^L([1-9][0-9]*)$", re.IGNORECASE) + + +@dataclass(frozen=True) +class ResolvedMarkdownTarget: + path: Path + line: int | None = None + + +def _resolve_markdown_target( + raw: str, + source_dir: Path, + wikilink: bool = False, +) -> ResolvedMarkdownTarget | None: + target = raw.strip() + if not target: + return None + path_and_query, separator, fragment = target.partition("#") + path_text = path_and_query.split("?", 1)[0].strip() + if not path_text: + return None + low = path_text.lower() + if "://" in path_text or low.startswith(("mailto:", "tel:", "//", "data:")): + return None + suffix = Path(path_text).suffix.lower() + if not suffix: + path_text += ".md" + suffix = ".md" + if suffix not in _MD_LINKABLE_EXTS and suffix not in CODE_EXTENSIONS: + return None + candidate = Path(path_text) + if not candidate.is_absolute(): + candidate = source_dir / candidate + resolved = Path(os.path.normpath(str(candidate))) + if wikilink and suffix in _MD_LINKABLE_EXTS and not Path(path_text).is_absolute(): + try: + missing = not resolved.is_file() + except OSError: + missing = False + if missing: + root = _active_scan_root() + hit = _vault_lookup(path_text, root) if root is not None else None + if hit is not None: + resolved = Path(os.path.normpath(str(hit))) + match = _MD_LINE_FRAGMENT_RE.fullmatch(fragment.strip()) if separator else None + line = int(match.group(1)) if match and suffix in CODE_EXTENSIONS else None + return ResolvedMarkdownTarget(path=resolved, line=line) + + +def _resolve_markdown_link( + raw: str, + source_dir: Path, + wikilink: bool = False, +) -> Path | None: + target = _resolve_markdown_target(raw, source_dir, wikilink=wikilink) + return target.path if target is not None else None +``` + +Retain the existing vault helper and remove only the superseded body of `_resolve_markdown_link`. + +- [ ] **Step 4: Run parser tests and existing Markdown tests** + +Run: + +```powershell +uv run pytest tests/test_languages.py -k "markdown" -v +``` + +Expected: all selected tests pass. + +- [ ] **Step 5: Write a failing extraction-edge test** + +Add: + +```python +def test_markdown_code_link_emits_target_file_and_line(tmp_path): + docs = tmp_path / "docs" + src = tmp_path / "src" + docs.mkdir() + src.mkdir() + target = src / "service.py" + target.write_text("def run():\n return 1\n", encoding="utf-8") + page = docs / "architecture.md" + page.write_text("[implementation](../src/service.py#L2)\n", encoding="utf-8") + + result = extract_markdown(page) + edge = next(edge for edge in result["edges"] if edge["relation"] == "references") + + assert edge["target_file"] == str(target) + assert edge["target_line"] == 2 +``` + +- [ ] **Step 6: Run extraction test and verify RED** + +Run: + +```powershell +uv run pytest tests/test_languages.py::test_markdown_code_link_emits_target_file_and_line -v +``` + +Expected: FAIL because the extracted edge has no `target_line`. + +- [ ] **Step 7: Retain target-line evidence on extracted edges** + +Change `add_edge` and `add_link` to: + +```python +def add_edge( + src: str, + tgt: str, + relation: str, + line: int, + confidence: str = "EXTRACTED", + weight: float = 1.0, + target_file: str | None = None, + target_line: int | None = None, +) -> None: + edge = { + "source": src, + "target": tgt, + "relation": relation, + "confidence": confidence, + "source_file": str_path, + "source_location": f"L{line}", + "weight": weight, + } + if target_file is not None: + edge["target_file"] = target_file + if target_line is not None: + edge["target_line"] = target_line + edges.append(edge) + + +def add_link(raw: str, line: int, wikilink: bool = False) -> None: + target = _resolve_markdown_target(raw, source_dir, wikilink=wikilink) + if target is None: + return + tgt_nid = _make_id(str(target.path)) + dedupe_key = f"{tgt_nid}:L{target.line or 0}" + if tgt_nid == file_nid or dedupe_key in linked_targets: + return + linked_targets.add(dedupe_key) + target_file = None + try: + if target.path.is_file(): + target_file = str(target.path) + except OSError: + pass + add_edge( + file_nid, + tgt_nid, + "references", + line, + target_file=target_file, + target_line=target.line, + ) +``` + +- [ ] **Step 8: Run Markdown regression tests** + +Run: + +```powershell +uv run pytest tests/test_languages.py -k "markdown" -v +``` + +Expected: all selected tests pass. + +- [ ] **Step 9: Commit Task 1** + +```powershell +git add graphify/extractors/markdown.py tests/test_languages.py +git commit -m "feat: retain markdown code link targets" +``` + +--- + +### Task 2: Resolve Markdown Line Anchors to Canonical Code Symbols + +**Files:** +- Modify: `graphify/symbol_resolution.py` +- Modify: `graphify/extract.py:6760-6850` +- Modify: `tests/test_symbol_resolution.py` +- Modify: `tests/test_incremental.py` + +- [ ] **Step 1: Write failing resolver tests** + +Add: + +```python +from graphify.symbol_resolution import resolve_markdown_code_references + + +def test_markdown_code_reference_prefers_exact_symbol(tmp_path): + source = tmp_path / "service.py" + source.write_text("def run():\n return 1\n", encoding="utf-8") + nodes = [ + {"id": "service", "label": "service.py", "file_type": "code", "source_file": str(source), "source_location": "L1"}, + {"id": "service_run", "label": "run", "file_type": "code", "node_kind": "function", "source_file": str(source), "source_location": "L1"}, + ] + edges = [{"source": "architecture", "target": "service", "relation": "references", "target_file": str(source), "target_line": 1}] + + resolve_markdown_code_references(nodes, edges) + + assert edges[0]["target"] == "service_run" + assert "target_line" not in edges[0] + + +def test_markdown_code_reference_uses_nearest_preceding_symbol(tmp_path): + source = tmp_path / "service.py" + source.write_text("def first():\n pass\n\ndef second():\n pass\n", encoding="utf-8") + nodes = [ + {"id": "service", "label": "service.py", "file_type": "code", "source_file": str(source), "source_location": "L1"}, + {"id": "first", "label": "first", "file_type": "code", "node_kind": "function", "source_file": str(source), "source_location": "L1"}, + {"id": "second", "label": "second", "file_type": "code", "node_kind": "function", "source_file": str(source), "source_location": "L4"}, + ] + edges = [{"source": "architecture", "target": "service", "relation": "references", "target_file": str(source), "target_line": 5}] + + resolve_markdown_code_references(nodes, edges) + + assert edges[0]["target"] == "second" +``` + +- [ ] **Step 2: Run resolver tests and verify RED** + +Run: + +```powershell +uv run pytest tests/test_symbol_resolution.py -k "markdown_code_reference" -v +``` + +Expected: import/collection failure because `resolve_markdown_code_references` does not exist. + +- [ ] **Step 3: Implement deterministic line resolution** + +Add to `graphify/symbol_resolution.py`: + +```python +_SOURCE_LINE_RE = re.compile(r"^L([1-9][0-9]*)") +_SYMBOL_KIND_PRIORITY = { + "method": 0, + "function": 1, + "class": 2, + "type_alias": 3, +} + + +def _source_line(node: dict[str, Any]) -> int | None: + match = _SOURCE_LINE_RE.match(str(node.get("source_location", ""))) + return int(match.group(1)) if match else None + + +def resolve_markdown_code_references( + nodes: list[dict[str, Any]], + edges: list[dict[str, Any]], +) -> None: + by_path: dict[Path, list[tuple[int, int, str]]] = {} + file_ids: dict[Path, str] = {} + for node in nodes: + source_file = node.get("source_file") + if not source_file or node.get("file_type") != "code" or not node.get("id"): + continue + try: + path = Path(str(source_file)).resolve() + except (OSError, RuntimeError): + continue + label = str(node.get("label", "")) + if label == Path(str(source_file)).name: + file_ids[path] = str(node["id"]) + continue + line = _source_line(node) + if line is None: + continue + priority = _SYMBOL_KIND_PRIORITY.get(str(node.get("node_kind", "")), 10) + by_path.setdefault(path, []).append((line, priority, str(node["id"]))) + for candidates in by_path.values(): + candidates.sort(key=lambda item: (item[0], item[1], item[2])) + + for edge in edges: + if edge.get("relation") != "references" or not edge.get("target_file"): + continue + raw_line = edge.pop("target_line", None) + try: + path = Path(str(edge["target_file"])).resolve() + except (OSError, RuntimeError): + continue + fallback = file_ids.get(path) + if fallback is not None: + edge["target"] = fallback + if not isinstance(raw_line, int) or raw_line < 1: + continue + preceding = [item for item in by_path.get(path, []) if item[0] <= raw_line] + if not preceding: + continue + best_line = preceding[-1][0] + same_line = [item for item in preceding if item[0] == best_line] + edge["target"] = min(same_line, key=lambda item: (item[1], item[2]))[2] +``` + +- [ ] **Step 4: Run resolver tests and verify GREEN** + +Run: + +```powershell +uv run pytest tests/test_symbol_resolution.py -k "markdown_code_reference" -v +``` + +Expected: all selected tests pass. + +- [ ] **Step 5: Write failing full/incremental parity test** + +Add a test to `tests/test_incremental.py` that creates `docs/map.md` linking to `src/service.py#L2`, performs a full extraction, changes only the Markdown document, performs an incremental extraction, and asserts both results target the same `service_run` node and ship neither `target_file` nor `target_line`. + +Use this complete assertion helper: + +```python +def _reference_target(result): + edge = next( + edge for edge in result["edges"] + if edge.get("relation") == "references" + and edge.get("source_file", "").endswith("map.md") + ) + assert "target_file" not in edge + assert "target_line" not in edge + return edge["target"] + + +assert _reference_target(full) == "src_service_run" +assert _reference_target(incremental) == _reference_target(full) +``` + +- [ ] **Step 6: Run parity test and verify RED** + +Run the exact new node ID reported by pytest: + +```powershell +uv run pytest tests/test_incremental.py -k "markdown_code_line" -v +``` + +Expected: FAIL because extraction still targets the file node. + +- [ ] **Step 7: Integrate the resolver after final ID/path normalization** + +Import the resolver in `graphify/extract.py` and call it immediately before transient edge metadata is removed and before the AST provenance loop: + +```python +from graphify.symbol_resolution import resolve_markdown_code_references + +# All node IDs and source paths are canonical at this point. Refine Markdown +# code references before transient target evidence is removed. +resolve_markdown_code_references(all_nodes, all_edges) +for edge in all_edges: + edge.pop("target_file", None) + edge.pop("target_line", None) +``` + +If an earlier pass already removes `target_file`, move that removal to this single cleanup point without changing other resolver ordering. + +- [ ] **Step 8: Run parity and canonical-ID regressions** + +Run: + +```powershell +uv run pytest tests/test_incremental.py -k "markdown or target_file" -v +uv run pytest tests/test_node_id_canonical.py -v +``` + +Expected: all selected tests pass. + +- [ ] **Step 9: Commit Task 2** + +```powershell +git add graphify/extract.py graphify/symbol_resolution.py tests/test_symbol_resolution.py tests/test_incremental.py +git commit -m "feat: resolve markdown links to code symbols" +``` + +--- + +### Task 3: Materialize and Canonicalize Python Type Aliases + +**Files:** +- Modify: `graphify/symbol_resolution.py` +- Modify: `graphify/extract.py` +- Modify: `tests/test_symbol_resolution.py` + +- [ ] **Step 1: Write failing alias-discovery tests** + +Add: + +```python +from graphify.symbol_resolution import parse_python_type_aliases + + +def test_parse_python_type_aliases_supports_assignment_and_typealias(tmp_path): + module = tmp_path / "ordering.py" + module.write_text( + "from typing import Callable, TypeAlias\n" + "OrderFn = Callable[[int], str]\n" + "ExplicitOrderFn: TypeAlias = Callable[[int], str]\n", + encoding="utf-8", + ) + + aliases = parse_python_type_aliases(module) + + assert [(alias.name, alias.source_location) for alias in aliases] == [ + ("OrderFn", "L2"), + ("ExplicitOrderFn", "L3"), + ] + + +def test_parse_python_type_aliases_ignores_function_local_assignments(tmp_path): + module = tmp_path / "ordering.py" + module.write_text( + "from typing import Callable\n" + "def build():\n" + " LocalOrderFn = Callable[[int], str]\n", + encoding="utf-8", + ) + + assert parse_python_type_aliases(module) == [] +``` + +- [ ] **Step 2: Run discovery tests and verify RED** + +Run: + +```powershell +uv run pytest tests/test_symbol_resolution.py -k "parse_python_type_aliases" -v +``` + +Expected: import/collection failure because the parser does not exist. + +- [ ] **Step 3: Implement conservative alias discovery** + +Add: + +```python +@dataclass(frozen=True) +class PythonTypeAlias: + name: str + module_stem: str + source_file: str + source_location: str + + +def _annotation_names(node: ast.AST | None) -> set[str]: + return { + child.id + for child in ast.walk(node) + if isinstance(child, ast.Name) + } if node is not None else set() + + +def parse_python_type_aliases(path: Path) -> list[PythonTypeAlias]: + try: + tree = ast.parse(path.read_text(encoding="utf-8", errors="replace")) + except (OSError, SyntaxError): + return [] + result: list[PythonTypeAlias] = [] + for node in tree.body: + name: str | None = None + value: ast.AST | None = None + if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name): + name = node.targets[0].id + value = node.value + elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): + if "TypeAlias" in _annotation_names(node.annotation): + name = node.target.id + value = node.value + elif hasattr(ast, "TypeAlias") and isinstance(node, ast.TypeAlias): + name_node = getattr(node, "name", None) + name = getattr(name_node, "id", None) + value = getattr(node, "value", None) + if not name or value is None: + continue + value_names = _annotation_names(value) + if not value_names.intersection({"Callable", "Union", "Literal", "Protocol", "Type", "Annotated"}): + continue + result.append(PythonTypeAlias( + name=name, + module_stem=path.stem, + source_file=str(path), + source_location=f"L{getattr(node, 'lineno', 1)}", + )) + return result +``` + +- [ ] **Step 4: Run discovery tests and verify GREEN** + +Run: + +```powershell +uv run pytest tests/test_symbol_resolution.py -k "parse_python_type_aliases" -v +``` + +Expected: all selected tests pass. + +- [ ] **Step 5: Write a failing import-canonicalization test** + +Add: + +```python +from graphify.symbol_resolution import canonicalize_python_type_aliases + + +def test_canonicalize_python_type_aliases_rewires_imported_stub(tmp_path): + definitions = tmp_path / "ordering.py" + consumer = tmp_path / "planner.py" + definitions.write_text( + "from typing import Callable\nOrderFn = Callable[[int], str]\n", + encoding="utf-8", + ) + consumer.write_text( + "from ordering import OrderFn\ndef plan(order: OrderFn):\n return order(1)\n", + encoding="utf-8", + ) + nodes = [ + {"id": "ordering", "label": "ordering.py", "file_type": "code", "source_file": str(definitions), "source_location": "L1"}, + {"id": "planner", "label": "planner.py", "file_type": "code", "source_file": str(consumer), "source_location": "L1"}, + {"id": "planner_plan", "label": "plan", "file_type": "code", "node_kind": "function", "source_file": str(consumer), "source_location": "L2"}, + {"id": "stub_orderfn", "label": "OrderFn", "file_type": "code"}, + ] + edges = [{"source": "planner_plan", "target": "stub_orderfn", "relation": "references", "source_file": str(consumer)}] + + canonicalize_python_type_aliases([definitions, consumer], nodes, edges) + + aliases = [node for node in nodes if node.get("node_kind") == "type_alias"] + assert len(aliases) == 1 + assert aliases[0]["label"] == "OrderFn" + assert aliases[0]["source_file"] == str(definitions) + assert edges[0]["target"] == aliases[0]["id"] + assert all(node["id"] != "stub_orderfn" for node in nodes) +``` + +- [ ] **Step 6: Run canonicalization test and verify RED** + +Run: + +```powershell +uv run pytest tests/test_symbol_resolution.py -k "canonicalize_python_type_aliases" -v +``` + +Expected: import/collection failure because `canonicalize_python_type_aliases` does not exist. + +- [ ] **Step 7: Implement source-backed alias nodes and safe rewiring** + +Add a helper that derives the containing file node by resolved `source_file`, creates stable alias IDs with `_shared_make_id(file_node_id, alias.name)`, appends a `contains` edge, indexes aliases by `(module_stem, lower_name)`, and rewires only edges whose `source_file` has matching top-level import evidence. + +Use this public shape: + +```python +def canonicalize_python_type_aliases( + paths: Sequence[Path], + nodes: list[dict[str, Any]], + edges: list[dict[str, Any]], +) -> None: + file_ids = _file_node_ids_by_resolved_path(nodes) + alias_index = _materialize_python_type_alias_nodes(paths, nodes, edges, file_ids) + node_by_id = {str(node.get("id")): node for node in nodes if node.get("id")} + rewired_stub_ids: set[str] = set() + for path in paths: + if path.suffix.lower() not in {".py", ".pyw"}: + continue + imports = parse_python_import_aliases(path) + for edge in edges: + if _resolved_source(edge.get("source_file")) != _resolved_source(str(path)): + continue + stub = node_by_id.get(str(edge.get("target"))) + if not stub or stub.get("source_file"): + continue + imported = imports.get(str(stub.get("label", ""))) + if imported is None: + continue + candidates = alias_index.get((imported.module_stem, imported.imported_name.lower()), []) + if len(candidates) != 1: + continue + edge["target"] = candidates[0] + rewired_stub_ids.add(str(stub["id"])) + referenced = {str(edge.get(key)) for edge in edges for key in ("source", "target")} + nodes[:] = [ + node for node in nodes + if str(node.get("id")) not in rewired_stub_ids + or str(node.get("id")) in referenced + ] +``` + +Implement `_file_node_ids_by_resolved_path`, `_materialize_python_type_alias_nodes`, and `_resolved_source` as private helpers in the same module. `_materialize_python_type_alias_nodes` must deduplicate existing source-backed alias nodes before appending and sanitize metadata consistently with the other resolvers. + +- [ ] **Step 8: Integrate alias canonicalization before final Markdown refinement** + +In `graphify/extract.py`, after all node IDs and source paths are canonical but before `resolve_markdown_code_references`, call: + +```python +canonicalize_python_type_aliases(paths, all_nodes, all_edges) +resolve_markdown_code_references(all_nodes, all_edges) +``` + +- [ ] **Step 9: Add and run ambiguity regressions** + +Add tests proving star imports, function-local imports, external modules, and two same-module candidates leave the stub unresolved. Run: + +```powershell +uv run pytest tests/test_symbol_resolution.py -k "type_alias or import_guided" -v +``` + +Expected: all selected tests pass. + +- [ ] **Step 10: Commit Task 3** + +```powershell +git add graphify/symbol_resolution.py graphify/extract.py tests/test_symbol_resolution.py +git commit -m "feat: canonicalize imported python type aliases" +``` + +--- + +### Task 4: Classify External and Benign Gap Nodes + +**Files:** +- Create: `graphify/gaps.py` +- Modify: `graphify/analyze.py:428-540` +- Modify: `tests/test_analyze.py` + +- [ ] **Step 1: Write failing classifier tests** + +Add: + +```python +from graphify.gaps import GapCategory, classify_gap_node, gap_breakdown + + +@pytest.mark.parametrize("attrs", [ + {"external": True, "file_type": "code"}, + {"node_kind": "external_symbol", "file_type": "code"}, + {"metadata": {"scip_kind": "external"}, "file_type": "code"}, +]) +def test_external_evidence_is_benign(attrs): + graph = nx.Graph() + graph.add_node("external", label="Flask", **attrs) + + assert classify_gap_node(graph, "external") is GapCategory.EXTERNAL + + +def test_sourceless_semantic_concept_is_not_external(): + graph = nx.Graph() + graph.add_node("concept", label="Debt Strategy", file_type="concept") + + assert classify_gap_node(graph, "concept") is GapCategory.STRUCTURAL + + +def test_source_backed_leaf_is_actionable(): + graph = nx.Graph() + graph.add_node("service", label="Service", file_type="code", source_file="src/service.py") + + assert classify_gap_node(graph, "service") is GapCategory.ACTIONABLE_LOCAL +``` + +- [ ] **Step 2: Run classifier tests and verify RED** + +Run: + +```powershell +uv run pytest tests/test_analyze.py -k "external_evidence or sourceless_semantic or source_backed_leaf" -v +``` + +Expected: import/collection failure because `graphify.gaps` does not exist. + +- [ ] **Step 3: Implement the shared classifier** + +Create `graphify/gaps.py`: + +```python +from __future__ import annotations + +from collections import Counter +from enum import StrEnum +from typing import Iterable + +import networkx as nx + + +class GapCategory(StrEnum): + ACTIONABLE_LOCAL = "actionable_local" + EXTERNAL = "external" + RATIONALE = "rationale" + METADATA = "metadata" + STRUCTURAL = "structural" + + +def _is_external(attrs: dict) -> bool: + metadata = attrs.get("metadata") if isinstance(attrs.get("metadata"), dict) else {} + return bool( + attrs.get("external") is True + or attrs.get("node_kind") == "external_symbol" + or metadata.get("scip_kind") == "external" + or str(attrs.get("id", "")).startswith("ref_") + ) + + +def classify_gap_node(graph: nx.Graph, node_id: str) -> GapCategory: + from graphify.analyze import _is_concept_node, _is_file_node, _is_json_key_node + + attrs = dict(graph.nodes[node_id]) + attrs.setdefault("id", node_id) + if _is_external(attrs): + return GapCategory.EXTERNAL + if attrs.get("file_type") == "rationale": + return GapCategory.RATIONALE + if _is_json_key_node(graph, node_id): + return GapCategory.METADATA + if _is_file_node(graph, node_id) or _is_concept_node(graph, node_id): + return GapCategory.STRUCTURAL + if attrs.get("node_kind") in {"page", "heading"}: + return GapCategory.STRUCTURAL + if attrs.get("source_file"): + return GapCategory.ACTIONABLE_LOCAL + return GapCategory.STRUCTURAL + + +def gap_breakdown(graph: nx.Graph, node_ids: Iterable[str]) -> dict[str, int]: + counts = Counter(classify_gap_node(graph, node_id).value for node_id in node_ids) + return {category.value: counts.get(category.value, 0) for category in GapCategory} +``` + +If the supported Python floor does not provide `StrEnum`, use `class GapCategory(str, Enum)` with the same values. + +- [ ] **Step 4: Run classifier tests and verify GREEN** + +Run: + +```powershell +uv run pytest tests/test_analyze.py -k "external_evidence or sourceless_semantic or source_backed_leaf" -v +``` + +Expected: all selected tests pass. + +- [ ] **Step 5: Write failing suggested-question tests** + +Add a graph with one local code leaf, one explicit external leaf, one rationale leaf, and one JSON noise leaf. Assert the generated `isolated_nodes` question reports exactly one weakly connected node and names only the local code node. + +```python +def test_suggest_questions_counts_only_actionable_local_leaves(): + graph = nx.Graph() + graph.add_node("local", label="LocalService", file_type="code", source_file="src/local.py") + graph.add_node("external", label="Flask", file_type="code", external=True) + graph.add_node("reason", label="Why", file_type="rationale", source_file="src/local.py") + graph.add_node("json", label="name", file_type="code", source_file="fixtures/data.json") + + questions = suggest_questions(graph, {}, {}, top_n=10) + isolated = next(item for item in questions if item["type"] == "isolated_nodes") + + assert isolated["why"].startswith("1 weakly-connected node") + assert "LocalService" in isolated["question"] + assert "Flask" not in isolated["question"] +``` + +- [ ] **Step 6: Run question test and verify RED** + +Run: + +```powershell +uv run pytest tests/test_analyze.py::test_suggest_questions_counts_only_actionable_local_leaves -v +``` + +Expected: FAIL because current analysis counts the external leaf. + +- [ ] **Step 7: Filter analysis through the shared category** + +Replace the isolated-node filter in `suggest_questions` with: + +```python +from graphify.gaps import GapCategory, classify_gap_node + +isolated = [ + node_id + for node_id in G.nodes() + if G.degree(node_id) <= 1 + and classify_gap_node(G, node_id) is GapCategory.ACTIONABLE_LOCAL +] +``` + +For low-cohesion community questions, compute a breakdown and skip the question when `actionable_local == 0`; retain the community itself in clustering output. + +- [ ] **Step 8: Run analysis regressions** + +Run: + +```powershell +uv run pytest tests/test_analyze.py -v +``` + +Expected: all tests pass. + +- [ ] **Step 9: Commit Task 4** + +```powershell +git add graphify/gaps.py graphify/analyze.py tests/test_analyze.py +git commit -m "feat: classify actionable graph gaps" +``` + +--- + +### Task 5: Render Auditable Gap and Thin-Community Breakdowns + +**Files:** +- Modify: `graphify/report.py:270-315` +- Create: `tests/test_gap_reporting.py` + +- [ ] **Step 1: Write failing report tests** + +Create `tests/test_gap_reporting.py` with a small helper that calls `report.generate` using empty scores/surprises/detection/token inputs, then add: + +```python +def test_report_separates_actionable_and_benign_isolated_nodes(): + graph = nx.Graph() + graph.add_node("local", label="LocalService", file_type="code", source_file="src/local.py") + graph.add_node("external", label="Flask", file_type="code", external=True) + graph.add_node("reason", label="Decision", file_type="rationale", source_file="src/local.py") + + report = _generate(graph, communities={0: ["local"], 1: ["external"], 2: ["reason"]}) + + assert "1 actionable isolated node(s)" in report + assert "external: 1" in report + assert "rationale: 1" in report + assert "`LocalService`" in report + + +def test_report_marks_all_benign_thin_community_non_actionable(): + graph = nx.Graph() + graph.add_node("external", label="parametrize", file_type="code", external=True) + + report = _generate(graph, communities={0: ["external"]}) + + assert "benign thin communities: 1" in report + assert "actionable thin communities: 0" in report +``` + +- [ ] **Step 2: Run report tests and verify RED** + +Run: + +```powershell +uv run pytest tests/test_gap_reporting.py -v +``` + +Expected: FAIL because the report has no categorized breakdown. + +- [ ] **Step 3: Replace the undifferentiated gaps section** + +Use `classify_gap_node` and `gap_breakdown` in `graphify/report.py`: + +```python +from .gaps import GapCategory, classify_gap_node, gap_breakdown + +weak_nodes = [node_id for node_id in G.nodes() if G.degree(node_id) <= 1] +weak_breakdown = gap_breakdown(G, weak_nodes) +actionable_isolated = [ + node_id for node_id in weak_nodes + if classify_gap_node(G, node_id) is GapCategory.ACTIONABLE_LOCAL +] +thin_communities = { + cid: nodes for cid, nodes in communities.items() + if 0 < sum(1 for node_id in nodes if not _is_file_node(G, node_id)) < min_community_size +} +thin_breakdowns = { + cid: gap_breakdown(G, nodes) for cid, nodes in thin_communities.items() +} +actionable_thin = { + cid: counts for cid, counts in thin_breakdowns.items() + if counts[GapCategory.ACTIONABLE_LOCAL.value] > 0 +} +benign_thin_count = len(thin_breakdowns) - len(actionable_thin) +``` + +Render the section with these exact labels: + +```python +lines.append(f"- **{len(actionable_isolated)} actionable isolated node(s):** {labels}{suffix}") +lines.append( + "- **Benign weak-node breakdown:** " + f"external: {weak_breakdown['external']}, " + f"rationale: {weak_breakdown['rationale']}, " + f"metadata: {weak_breakdown['metadata']}, " + f"structural: {weak_breakdown['structural']}" +) +lines.append(f"- **actionable thin communities: {len(actionable_thin)}**") +lines.append(f"- **benign thin communities: {benign_thin_count}**") +``` + +Keep the ambiguity warning unchanged. + +- [ ] **Step 4: Run report and analysis tests** + +Run: + +```powershell +uv run pytest tests/test_gap_reporting.py tests/test_analyze.py -v +``` + +Expected: all tests pass. + +- [ ] **Step 5: Commit Task 5** + +```powershell +git add graphify/report.py tests/test_gap_reporting.py +git commit -m "feat: report actionable and benign graph gaps" +``` + +--- + +### Task 6: Document the DebtGPS Refinance Route Rationale + +**Files:** +- Modify: `../../server/routes_planning.py:384-406` +- Modify: `../../tests/test_coverage_gaps.py` + +- [ ] **Step 1: Write a failing route-contract test** + +Add to `../../tests/test_coverage_gaps.py`: + +```python +def test_refinance_route_documents_validation_and_engine_authority(): + from server.routes_planning import refinance + + rationale = refinance.__doc__ or "" + + assert "validat" in rationale.lower() + assert "delegat" in rationale.lower() + assert "analyze_refinance" in rationale +``` + +- [ ] **Step 2: Run the contract test and verify RED** + +From the DebtGPS root, run: + +```powershell +pytest tests/test_coverage_gaps.py::test_refinance_route_documents_validation_and_engine_authority -v +``` + +Expected: FAIL because `refinance.__doc__` is empty. + +- [ ] **Step 3: Add the minimal rationale docstring** + +Add immediately inside `refinance()`: + +```python +def refinance(): + """Validate request and ownership state, then delegate pricing to analyze_refinance. + + The route is an HTTP boundary only; canonical payoff and refinance math stays + in the engine so route behavior cannot diverge from planning simulations. + """ +``` + +Do not change the route body. + +- [ ] **Step 4: Run the focused route tests** + +From the DebtGPS root, run: + +```powershell +pytest tests/test_coverage_gaps.py::test_refinance_route_documents_validation_and_engine_authority tests/test_coverage_gaps.py::test_refinance_requires_a_term_or_a_payment tests/test_routes.py::test_refinance_route_consolidates_and_validates -v +``` + +Expected: 3 passed. + +- [ ] **Step 5: Commit Task 6 in the DebtGPS repository only if the user-owned worktree policy permits a focused commit** + +If existing unrelated changes make a focused commit unsafe, leave the verified modification uncommitted and report it. Otherwise: + +```powershell +git add server/routes_planning.py tests/test_coverage_gaps.py +git commit -m "docs: explain refinance route authority" +``` + +--- + +### Task 7: Run Graphify Regression Gates and Install the Local Build + +**Files:** +- Verify all Graphify changes +- Update the active uv tool environment + +- [ ] **Step 1: Run formatting/static checks configured by the repository** + +From the Graphify checkout: + +```powershell +uv run ruff check graphify tests/test_languages.py tests/test_incremental.py tests/test_symbol_resolution.py tests/test_analyze.py tests/test_gap_reporting.py +``` + +Expected: exit 0. If `ruff` is not configured in the environment, record that fact and run the repository's configured pre-commit checks instead. + +- [ ] **Step 2: Run focused Graphify regression suites** + +```powershell +uv run pytest tests/test_languages.py tests/test_incremental.py tests/test_symbol_resolution.py tests/test_analyze.py tests/test_gap_reporting.py tests/test_node_id_canonical.py -v +``` + +Expected: all selected tests pass with zero failures. + +- [ ] **Step 3: Run the full Graphify test suite** + +```powershell +uv run pytest -q +``` + +Expected: exit 0. If unrelated pre-existing failures occur, rerun each failure in isolation, capture evidence, and do not classify the patch as fully green until the distinction is proven. + +- [ ] **Step 4: Install Graphify from the verified checkout** + +```powershell +uv tool install --force --from . graphifyy +``` + +Expected: the `graphifyy` tool environment is replaced from the local checkout. + +- [ ] **Step 5: Verify the active executable** + +```powershell +Get-Command graphify | Format-List Source +graphify --version +``` + +Expected: the known local executable path is used and its version matches the checkout metadata. + +--- + +### Task 8: Perform the Cached Semantic Extraction and Rebuild DebtGPS + +**Files:** +- Read: `../../.codex/skills/graphify/references/extraction-spec.md` +- Refresh: `../../graphify-out/` + +- [ ] **Step 1: Re-read the semantic extraction contract completely** + +Read the full DebtGPS Graphify extraction specification before launching semantic work. Confirm the allowed node schema, relation directions, file-type enum, rationale policy, cache output requirements, and token-usage reporting. + +- [ ] **Step 2: Re-detect the corpus and cache state** + +From the DebtGPS root, run the Graphify detection/cache procedure and save the exact uncached list. Expected baseline from design time: + +```text +448 files +641,073 words +310 code files +113 documents +25 images +138 semantic files +134 cached +4 uncached documents +0 uncached images +``` + +The execution-time result is authoritative; do not force these baseline numbers if files changed. + +- [ ] **Step 3: Extract only current cache misses** + +Because no `GEMINI_API_KEY` or `GOOGLE_API_KEY` is configured, use one host semantic extraction task for up to 25 uncached documents. Each image cache miss, if any, gets its own task. Require schema-valid JSON, exact `source_file`, cache save, and reported input/output token usage. + +- [ ] **Step 4: Merge cached semantics and rebuild** + +Run the Graphify merge/build/cluster/report sequence prescribed by the installed skill so cached semantic output, fresh semantic output, and AST output all contribute to the final graph. Then run: + +```powershell +graphify update . +``` + +Expected: AST state is current after all DebtGPS code changes, with no API cost for the update step. + +- [ ] **Step 5: Capture build metrics and warnings** + +Record final node, edge, and community counts; cached/fresh semantic counts; token usage; and every graph-health warning. A generated report with a health warning is not considered a clean build. + +--- + +### Task 9: Verify DebtGPS Graph Acceptance and Close Remaining Gaps + +**Files:** +- Verify: `../../docs/debt-model.md` +- Verify: `../../graphify-out/graph.json` +- Verify: `../../graphify-out/GRAPH_REPORT.md` +- Verify: DebtGPS focused test suites + +- [ ] **Step 1: Verify all documented local links** + +Run a local-link checker over `docs/debt-model.md` that strips Markdown fragments, resolves paths relative to the document, and asserts every referenced local file exists. Expected: 0 broken local links. + +- [ ] **Step 2: Verify the nine documentation-to-code targets** + +Load `graphify-out/graph.json`; for each critical implementation link in the DebtGPS canonical map, resolve the target file and optional line with the same deterministic rule and assert at least one incoming `references` edge from a document node. Print the nine source-document/target-node pairs as evidence. + +- [ ] **Step 3: Verify canonical `OrderFn`** + +Query graph nodes whose normalized label is `orderfn`. Assert: + +```python +assert len(orderfn_nodes) == 1 +assert orderfn_nodes[0].get("source_file") +assert orderfn_nodes[0].get("node_kind") == "type_alias" +``` + +- [ ] **Step 4: Verify known external symbols are retained but benign** + +For `route`, `Flask`, `parametrize`, `given`, and `composite`, print matching nodes and their classification. Assert no matching node appears in the actionable-isolated set while externally evidenced matches remain in `graph.json`. + +- [ ] **Step 5: Verify every weak/thin node is categorized** + +Recompute weak nodes and thin communities using the report rules. Assert the category counts sum to their respective totals and print: + +```text +actionable_local +external +rationale +metadata +structural +``` + +Any remaining actionable local node is reported with label, source file, degree, and community rather than silently declared closed. + +- [ ] **Step 6: Run focused DebtGPS verification suites** + +From the DebtGPS root: + +```powershell +pytest tests/test_planning.py tests/test_math_oracle.py tests/test_event_integrity.py tests/test_audit_batch1.py tests/test_audit_batch6.py tests/test_audit_batch7.py tests/test_routes.py tests/test_coverage_gaps.py tests/test_scenario_authority.py tests/test_scenario_provenance.py -q +``` + +Expected: zero failures. + +- [ ] **Step 7: Review diffs and repository state** + +In both repositories, run: + +```powershell +git status --short +git diff --check +git log --oneline -8 +``` + +Confirm no unrelated user change was overwritten and every Graphify production change has a previously observed RED test and a current GREEN test. + +- [ ] **Step 8: Final evidence summary** + +Report: + +- Graphify commits and changed components +- DebtGPS changed files +- focused and full test counts +- final graph node/edge/community counts +- semantic cache hits/misses and token usage +- nine document-to-code reference results +- canonical `OrderFn` result +- external/benign/actionable gap breakdown +- every unresolved graph-health warning or actionable local node + +Do not state that all gaps are closed unless the fresh acceptance checks show zero remaining actionable local gaps. From 5b9427dbfb4dd4ac719f83b558e7d3e3220ad0cb Mon Sep 17 00:00:00 2001 From: Soumava2023 Date: Fri, 21 Aug 2026 14:45:52 -0400 Subject: [PATCH 03/13] feat: retain markdown code link targets --- graphify/extractors/markdown.py | 94 ++++++++++++++++++++++++--------- tests/test_languages.py | 64 ++++++++++++++++++++++ 2 files changed, 134 insertions(+), 24 deletions(-) diff --git a/graphify/extractors/markdown.py b/graphify/extractors/markdown.py index 50c1754a78..6cb1ebb74d 100644 --- a/graphify/extractors/markdown.py +++ b/graphify/extractors/markdown.py @@ -5,7 +5,9 @@ import os import unicodedata +from dataclasses import dataclass from pathlib import Path +from graphify.detect import CODE_EXTENSIONS from graphify.extractors.base import _file_stem, _make_id from graphify.security import sanitize_metadata @@ -18,6 +20,16 @@ _MD_LINKABLE_EXTS = {".md", ".mdx", ".qmd", ".markdown", ".rst", ".txt"} +_MD_LINE_FRAGMENT_RE = re.compile(r"^L([1-9][0-9]*)$", re.IGNORECASE) + + +@dataclass(frozen=True) +class ResolvedMarkdownTarget: + """A local Markdown target plus optional one-based code line evidence.""" + + path: Path + line: int | None = None + # A YAML frontmatter block is only frontmatter when the opening `---` is the # very first line of the file. A `---` further down is a horizontal rule and # must not be mistaken for one. Bounded so a file that opens a fence and never @@ -182,18 +194,22 @@ def _vault_lookup(target: str, root: Path) -> "Path | None": return min(matches)[2] -def _resolve_markdown_link(raw: str, source_dir: Path, - wikilink: bool = False) -> "Path | None": - """Resolve a markdown link target to the absolute path of a sibling document. +def _resolve_markdown_target( + raw: str, + source_dir: Path, + wikilink: bool = False, +) -> "ResolvedMarkdownTarget | None": + """Resolve a local Markdown link and retain valid code-line evidence. Returns the resolved (normalized, not necessarily existing) path when the - target is a *local* relative/absolute file-path link to a document, or None + target is a *local* relative/absolute file-path link to a document or code + file, or None when it should be skipped: external URLs (http/https/mailto/protocol- - relative/data), pure in-page anchors (``#section``), and links to non-doc - file types (code/assets are handled by their own extractors). + relative/data), pure in-page anchors (``#section``), and links to other + file types. - The anchor fragment (``#section``) and query (``?x=1``) are stripped before - resolution so ``./repo.md#setup`` resolves to the same node as ``./repo.md``. + A code fragment of the form ``#L83`` becomes one-based line evidence. + Other fragments are stripped, preserving existing document-link behavior. Extension-less targets (typical of wikilinks) are treated as sibling ``.md``. With ``wikilink=True``, a target whose lexically resolved path does not @@ -202,11 +218,11 @@ def _resolve_markdown_link(raw: str, source_dir: Path, reference-style links keep pure relative semantics: for them a missing relative target is an authoring error, not an alternate link convention. """ - target = raw.strip() - if not target: + raw_target = raw.strip() + if not raw_target: return None - # Drop anchor / query so #section links still resolve to the target doc. - target = target.split("#", 1)[0].split("?", 1)[0].strip() + path_and_query, separator, fragment = raw_target.partition("#") + target = path_and_query.split("?", 1)[0].strip() if not target: return None low = target.lower() @@ -216,7 +232,7 @@ def _resolve_markdown_link(raw: str, source_dir: Path, if suffix == "": target = target + ".md" suffix = ".md" - if suffix not in _MD_LINKABLE_EXTS: + if suffix not in _MD_LINKABLE_EXTS and suffix not in CODE_EXTENSIONS: return None candidate = Path(target) if not candidate.is_absolute(): @@ -232,8 +248,27 @@ def _resolve_markdown_link(raw: str, source_dir: Path, if scan_root is not None: hit = _vault_lookup(target, scan_root) if hit is not None: - return Path(os.path.normpath(str(hit))) - return resolved + resolved = Path(os.path.normpath(str(hit))) + line_match = ( + _MD_LINE_FRAGMENT_RE.fullmatch(fragment.strip()) if separator else None + ) + line = ( + int(line_match.group(1)) + if line_match is not None and suffix in CODE_EXTENSIONS + else None + ) + return ResolvedMarkdownTarget(path=resolved, line=line) + + +def _resolve_markdown_link( + raw: str, + source_dir: Path, + wikilink: bool = False, +) -> "Path | None": + """Compatibility wrapper returning only the resolved target path.""" + + target = _resolve_markdown_target(raw, source_dir, wikilink=wikilink) + return target.path if target is not None else None def extract_markdown(path: Path) -> dict: """Extract structural nodes and edges from a Markdown file. @@ -298,12 +333,15 @@ def add_node(nid: str, label: str, line: int, file_type: str = "document", def add_edge(src: str, tgt: str, relation: str, line: int, confidence: str = "EXTRACTED", weight: float = 1.0, - target_file: "str | None" = None) -> None: + target_file: "str | None" = None, + target_line: "int | None" = None) -> None: edge = {"source": src, "target": tgt, "relation": relation, "confidence": confidence, "source_file": str_path, "source_location": f"L{line}", "weight": weight} if target_file is not None: edge["target_file"] = target_file + if target_line is not None: + edge["target_line"] = target_line edges.append(edge) lines = source.splitlines() @@ -320,18 +358,19 @@ def add_edge(src: str, tgt: str, relation: str, line: int, linked_targets: set[str] = set() def add_link(raw: str, line: int, wikilink: bool = False) -> None: - resolved = _resolve_markdown_link(raw, source_dir, wikilink=wikilink) - if resolved is None: + target = _resolve_markdown_target(raw, source_dir, wikilink=wikilink) + if target is None: return # Build the target ID with the SAME recipe as the target file's own # node (_make_id(str(path)) at extract time, canonicalized to # _file_node_id(rel) by the extract() post-pass). Using the absolute # resolved path means both endpoints get remapped identically, so the # edge merges into the existing doc node instead of spawning a ghost. - tgt_nid = _make_id(str(resolved)) - if tgt_nid == file_nid or tgt_nid in linked_targets: + tgt_nid = _make_id(str(target.path)) + dedupe_key = f"{tgt_nid}:L{target.line or 0}" + if tgt_nid == file_nid or dedupe_key in linked_targets: return - linked_targets.add(tgt_nid) + linked_targets.add(dedupe_key) # Stamp the resolved target file (mirroring the JS/Python import # stamps, #1814/#2213) so the #2169 remap pass can canonicalize this # edge's target on an incremental run where the linked doc is not in @@ -342,11 +381,18 @@ def add_link(raw: str, line: int, wikilink: bool = False) -> None: # and popped before graph.json ships. target_file = None try: - if resolved.is_file(): - target_file = str(resolved) + if target.path.is_file(): + target_file = str(target.path) except OSError: pass - add_edge(file_nid, tgt_nid, "references", line, target_file=target_file) + add_edge( + file_nid, + tgt_nid, + "references", + line, + target_file=target_file, + target_line=target.line, + ) # Track heading stack for nesting: [(level, nid), ...] heading_stack: list[tuple[int, str]] = [] diff --git a/tests/test_languages.py b/tests/test_languages.py index 2c0ba62072..e22f7dfa07 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -2484,6 +2484,70 @@ def test_ts_injected_field_ambiguous_type_emits_no_edge(tmp_path): # ── Markdown ───────────────────────────────────────────────────────────────── from graphify.extract import extract_markdown +from graphify.extractors.markdown import ( + ResolvedMarkdownTarget, + _resolve_markdown_target, +) + + +def test_markdown_target_keeps_code_line_anchor(tmp_path): + source_dir = tmp_path / "docs" + source_dir.mkdir() + + target = _resolve_markdown_target( + "../src/service.py#L17", source_dir, wikilink=False + ) + + assert target == ResolvedMarkdownTarget( + path=tmp_path / "src" / "service.py", + line=17, + ) + + +@pytest.mark.parametrize("fragment", ["#L0", "#L-2", "#Labc", "#section"]) +def test_markdown_target_invalid_code_line_falls_back_to_file( + tmp_path, fragment +): + target = _resolve_markdown_target( + f"../src/service.py{fragment}", tmp_path / "docs", wikilink=False + ) + + assert target == ResolvedMarkdownTarget( + path=tmp_path / "src" / "service.py", + line=None, + ) + + +def test_markdown_target_keeps_document_fragment_behavior(tmp_path): + target = _resolve_markdown_target( + "./architecture.md#decisions", tmp_path, wikilink=False + ) + + assert target == ResolvedMarkdownTarget( + path=tmp_path / "architecture.md", + line=None, + ) + + +def test_markdown_code_link_emits_target_file_and_line(tmp_path): + docs = tmp_path / "docs" + src = tmp_path / "src" + docs.mkdir() + src.mkdir() + target = src / "service.py" + target.write_text("def run():\n return 1\n", encoding="utf-8") + page = docs / "architecture.md" + page.write_text( + "[implementation](../src/service.py#L2)\n", encoding="utf-8" + ) + + result = extract_markdown(page) + edge = next( + edge for edge in result["edges"] if edge["relation"] == "references" + ) + + assert edge["target_file"] == str(target) + assert edge["target_line"] == 2 def test_markdown_no_error(): r = extract_markdown(FIXTURES / "deploy_guide.md") From cf8f508d628844d93acaa3ae8056739515f770ad Mon Sep 17 00:00:00 2001 From: Soumava2023 Date: Fri, 21 Aug 2026 14:52:14 -0400 Subject: [PATCH 04/13] feat: resolve markdown links to code symbols --- graphify/extract.py | 18 ++++++- graphify/symbol_resolution.py | 83 +++++++++++++++++++++++++++++++++ tests/test_incremental.py | 51 ++++++++++++++++++++ tests/test_symbol_resolution.py | 82 ++++++++++++++++++++++++++++++++ 4 files changed, 233 insertions(+), 1 deletion(-) diff --git a/graphify/extract.py b/graphify/extract.py index ffc6153f82..f6281dddbc 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -141,7 +141,10 @@ _workspace_globs, ) -from graphify.symbol_resolution import resolve_bash_source_edges # noqa: E402 +from graphify.symbol_resolution import ( # noqa: E402 + resolve_bash_source_edges, + resolve_markdown_code_references, +) from graphify.extractors.engine import REFERENCE_CONTEXTS, _CSHARP_TYPE_PARAMETER_SCOPE_DECLARATIONS, _C_PRIMITIVE_TYPE_NODES, _JAVA_BUILTIN_TYPES, _JAVA_TYPE_PARAMETER_SCOPE_DECLARATIONS, _JS_FUNCTION_VALUE_TYPES, _JS_SCOPE_BOUNDARY, _PYTHON_ANNOTATION_NOISE, _PYTHON_TYPE_CONTAINERS, _RUBY_CLASS_FACTORIES, _c_collect_type_refs, _cpp_collect_type_refs, _cpp_declarator_name, _cpp_local_var_types, _csharp_attribute_names, _csharp_classify_base, _csharp_collect_type_refs, _csharp_extra_walk, _csharp_namespace_id, _csharp_namespace_name, _csharp_pre_scan_interfaces, _csharp_type_parameters_in_scope, _dynamic_import_js, _extract_generic, _find_body, _find_require_call, _get_cpp_func_name, _java_annotation_names, _java_collect_type_refs, _java_extra_walk, _java_type_parameters_in_scope, _js_collect_pattern_idents, _js_dispatch_value_idents, _js_extra_walk, _js_local_bound_names, _js_member_assignment_target, _js_module_bound_names, _kotlin_collect_type_refs, _kotlin_function_return_type_node, _kotlin_property_type_node, _kotlin_user_type_name, _php_collect_type_refs, _php_method_return_type_node, _php_name_text, _python_collect_assignment_targets, _python_collect_param_refs, _python_collect_type_refs, _python_local_bound_names, _python_module_bound_names, _python_param_names, _read_csharp_type_name, _require_imports_js, _ruby_const_last_name, _ruby_extra_walk, _ruby_local_class_bindings, _ruby_new_class_name, _scala_collect_type_refs, _semantic_reference_edge, _source_location, _swift_classify_base, _swift_collect_type_refs, _swift_constructor_type, _swift_declaration_keyword, _swift_extra_walk, _swift_local_var_types, _swift_pre_scan, _swift_property_name, _swift_property_type_node, _swift_receiver_name, _swift_user_type_name, _ts_decorator_name, _ts_descendant_decorators, _ts_emit_decorator_edges, _ts_extra_walk, _ts_method_name, _ts_receiver_type_table # noqa: E402,F401 @@ -6199,6 +6202,19 @@ def _learn(e: dict) -> None: _repoint_python_package_imports(paths, all_nodes, all_edges, root) _merge_swift_extensions(per_file, all_nodes, all_edges) _merge_csharp_partial_class_nodes(per_file, all_nodes, all_edges, paths, root) + _markdown_resolution_nodes = all_nodes + if resolution_context_nodes: + _fresh_markdown_ids = {node.get("id") for node in all_nodes} + _markdown_resolution_nodes = all_nodes + [ + node + for node in resolution_context_nodes + if node.get("id") and node.get("id") not in _fresh_markdown_ids + ] + resolve_markdown_code_references( + _markdown_resolution_nodes, + all_edges, + root=root, + ) _disambiguate_colliding_node_ids(all_nodes, all_edges, all_raw_calls, root) _canonicalize_csharp_namespace_nodes(all_nodes, all_edges) # PHP namespace/use disambiguation must run BEFORE the unique-stub rewire: diff --git a/graphify/symbol_resolution.py b/graphify/symbol_resolution.py index 6adb4a2aca..99902a5e8e 100644 --- a/graphify/symbol_resolution.py +++ b/graphify/symbol_resolution.py @@ -27,6 +27,89 @@ class ImportedSymbol: source_location: str +_SOURCE_LINE_RE = re.compile(r"^L([1-9][0-9]*)") +_SYMBOL_KIND_PRIORITY = { + "method": 0, + "function": 1, + "class": 2, + "type_alias": 3, +} + + +def _source_line(node: dict[str, Any]) -> int | None: + """Return a node's one-based starting line when present.""" + + match = _SOURCE_LINE_RE.match(str(node.get("source_location", ""))) + return int(match.group(1)) if match else None + + +def resolve_markdown_code_references( + nodes: list[dict[str, Any]], + edges: list[dict[str, Any]], + root: Path | None = None, +) -> None: + """Refine stamped Markdown code links to exact or preceding symbols. + + The target path and optional line are deterministic extraction evidence. + When the line cannot identify a symbol, the canonical file node remains the + target. No node is fabricated for a missing file. + """ + + by_path: dict[Path, list[tuple[int, int, str]]] = {} + file_ids: dict[Path, str] = {} + + def resolved_graph_path(value: object) -> Path: + path = Path(str(value)) + if root is not None and not path.is_absolute(): + path = root / path + return path.resolve() + + for node in nodes: + source_file = node.get("source_file") + node_id = node.get("id") + if ( + not source_file + or not node_id + or node.get("file_type") != "code" + ): + continue + try: + path = resolved_graph_path(source_file) + except (OSError, RuntimeError): + continue + label = str(node.get("label", "")) + if label == Path(str(source_file)).name: + file_ids[path] = str(node_id) + continue + line = _source_line(node) + if line is None: + continue + priority = _SYMBOL_KIND_PRIORITY.get(str(node.get("node_kind", "")), 10) + by_path.setdefault(path, []).append((line, priority, str(node_id))) + for candidates in by_path.values(): + candidates.sort(key=lambda item: (item[0], item[1], item[2])) + + for edge in edges: + if edge.get("relation") != "references" or not edge.get("target_file"): + continue + raw_line = edge.pop("target_line", None) + try: + path = resolved_graph_path(edge["target_file"]) + except (OSError, RuntimeError): + continue + fallback = file_ids.get(path) + if fallback is not None: + edge["target"] = fallback + if not isinstance(raw_line, int) or raw_line < 1: + continue + preceding = [item for item in by_path.get(path, []) if item[0] <= raw_line] + if not preceding: + continue + best_line = preceding[-1][0] + same_line = [item for item in preceding if item[0] == best_line] + edge["target"] = min(same_line, key=lambda item: (item[1], item[2]))[2] + + def normalise_callable_label(label: str) -> str: """Normalize a node label into the key used for call resolution.""" diff --git a/tests/test_incremental.py b/tests/test_incremental.py index 62c1203257..460db23ba1 100644 --- a/tests/test_incremental.py +++ b/tests/test_incremental.py @@ -308,6 +308,57 @@ def test_incremental_md_reference_target_canonicalizes(tmp_path): assert "target_file" not in e, e +def test_incremental_markdown_code_line_resolves_like_full_build(tmp_path): + """A changed doc keeps targeting an unchanged code symbol by line.""" + from graphify.extract import extract + + root = Path(os.path.realpath(tmp_path)) + docs = root / "docs" + src = root / "src" + docs.mkdir() + src.mkdir() + service = src / "service.py" + service.write_text( + "def run():\n return 1\n", encoding="utf-8" + ) + page = docs / "map.md" + page.write_text( + "[implementation](../src/service.py#L2)\n", encoding="utf-8" + ) + + full = extract( + [page, service], cache_root=root, root=root, parallel=False + ) + page.write_text( + "# Updated\n[implementation](../src/service.py#L2)\n", + encoding="utf-8", + ) + incremental = extract( + [page], + cache_root=root, + root=root, + parallel=False, + resolution_context_nodes=full["nodes"], + resolution_context_edges=full["edges"], + ) + + def reference_target(result): + edge = next( + edge + for edge in result["edges"] + if edge.get("relation") == "references" + and str(edge.get("source_file", "")).endswith("map.md") + ) + assert "target_file" not in edge + assert "target_line" not in edge + return edge["target"] + + target = reference_target(full) + by_id = {node["id"]: node for node in full["nodes"]} + assert by_id[target]["label"].strip("().") == "run" + assert reference_target(incremental) == target + + def test_update_prunes_a_removed_imports_edge(tmp_path): """#1521: when an import is deleted from a file, `graphify update` must prune the edge it produced — preserving it (keyed only on endpoint membership) left a diff --git a/tests/test_symbol_resolution.py b/tests/test_symbol_resolution.py index d8737d2367..46de25d660 100644 --- a/tests/test_symbol_resolution.py +++ b/tests/test_symbol_resolution.py @@ -14,10 +14,92 @@ parse_python_import_aliases, resolve_bash_source_edges, resolve_cross_file_raw_calls, + resolve_markdown_code_references, resolve_python_import_guided_calls, ) +def test_markdown_code_reference_prefers_exact_symbol(tmp_path): + source = tmp_path / "service.py" + source.write_text("def run():\n return 1\n", encoding="utf-8") + nodes = [ + { + "id": "service", + "label": "service.py", + "file_type": "code", + "source_file": str(source), + "source_location": "L1", + }, + { + "id": "service_run", + "label": "run", + "file_type": "code", + "node_kind": "function", + "source_file": str(source), + "source_location": "L1", + }, + ] + edges = [ + { + "source": "architecture", + "target": "service", + "relation": "references", + "target_file": str(source), + "target_line": 1, + } + ] + + resolve_markdown_code_references(nodes, edges) + + assert edges[0]["target"] == "service_run" + assert "target_line" not in edges[0] + + +def test_markdown_code_reference_uses_nearest_preceding_symbol(tmp_path): + source = tmp_path / "service.py" + source.write_text( + "def first():\n pass\n\ndef second():\n pass\n", encoding="utf-8" + ) + nodes = [ + { + "id": "service", + "label": "service.py", + "file_type": "code", + "source_file": str(source), + "source_location": "L1", + }, + { + "id": "first", + "label": "first", + "file_type": "code", + "node_kind": "function", + "source_file": str(source), + "source_location": "L1", + }, + { + "id": "second", + "label": "second", + "file_type": "code", + "node_kind": "function", + "source_file": str(source), + "source_location": "L4", + }, + ] + edges = [ + { + "source": "architecture", + "target": "service", + "relation": "references", + "target_file": str(source), + "target_line": 5, + } + ] + + resolve_markdown_code_references(nodes, edges) + + assert edges[0]["target"] == "second" + + def test_normalise_callable_label_strips_function_punctuation() -> None: assert normalise_callable_label("run()") == "run" assert normalise_callable_label(".process()") == "process" From 5917a7f6073517ff281ec824ab31ab33b7536606 Mon Sep 17 00:00:00 2001 From: Soumava2023 Date: Fri, 21 Aug 2026 14:59:37 -0400 Subject: [PATCH 05/13] feat: canonicalize imported python type aliases --- graphify/extract.py | 2 + graphify/symbol_resolution.py | 216 +++++++++++++++++++++++++++++ tests/test_symbol_resolution.py | 237 ++++++++++++++++++++++++++++++++ 3 files changed, 455 insertions(+) diff --git a/graphify/extract.py b/graphify/extract.py index f6281dddbc..1596970501 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -142,6 +142,7 @@ ) from graphify.symbol_resolution import ( # noqa: E402 + canonicalize_python_type_aliases, resolve_bash_source_edges, resolve_markdown_code_references, ) @@ -6202,6 +6203,7 @@ def _learn(e: dict) -> None: _repoint_python_package_imports(paths, all_nodes, all_edges, root) _merge_swift_extensions(per_file, all_nodes, all_edges) _merge_csharp_partial_class_nodes(per_file, all_nodes, all_edges, paths, root) + canonicalize_python_type_aliases(paths, all_nodes, all_edges) _markdown_resolution_nodes = all_nodes if resolution_context_nodes: _fresh_markdown_ids = {node.get("id") for node in all_nodes} diff --git a/graphify/symbol_resolution.py b/graphify/symbol_resolution.py index 99902a5e8e..b1e0b6c4c7 100644 --- a/graphify/symbol_resolution.py +++ b/graphify/symbol_resolution.py @@ -27,6 +27,222 @@ class ImportedSymbol: source_location: str +@dataclass(frozen=True) +class PythonTypeAlias: + """A source-backed, module-level Python type alias definition.""" + + name: str + module_stem: str + source_file: str + source_location: str + + +def _annotation_names(node: ast.AST | None) -> set[str]: + """Return unqualified names occurring in a type expression.""" + + if node is None: + return set() + return { + child.id + for child in ast.walk(node) + if isinstance(child, ast.Name) + } + + +def parse_python_type_aliases(path: Path) -> list[PythonTypeAlias]: + """Discover conservative module-level aliases backed by typing forms.""" + + try: + tree = ast.parse(path.read_text(encoding="utf-8", errors="replace")) + except (OSError, SyntaxError): + return [] + + result: list[PythonTypeAlias] = [] + alias_value_names = { + "Callable", + "Union", + "Literal", + "Protocol", + "Type", + "Annotated", + } + for node in tree.body: + name: str | None = None + value: ast.AST | None = None + if ( + isinstance(node, ast.Assign) + and len(node.targets) == 1 + and isinstance(node.targets[0], ast.Name) + ): + name = node.targets[0].id + value = node.value + elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): + if "TypeAlias" in _annotation_names(node.annotation): + name = node.target.id + value = node.value + elif hasattr(ast, "TypeAlias") and isinstance(node, ast.TypeAlias): + name_node = getattr(node, "name", None) + name = getattr(name_node, "id", None) + value = getattr(node, "value", None) + if not name or value is None: + continue + if not _annotation_names(value).intersection(alias_value_names): + continue + result.append( + PythonTypeAlias( + name=name, + module_stem=path.stem, + source_file=str(path), + source_location=f"L{getattr(node, 'lineno', 1)}", + ) + ) + return result + + +def _resolved_source(value: object) -> Path | None: + """Resolve a graph source path defensively for equality/indexing.""" + + if not value: + return None + try: + return Path(str(value)).resolve() + except (OSError, RuntimeError): + return None + + +def _file_node_ids_by_resolved_path( + nodes: Sequence[dict[str, Any]], +) -> dict[Path, str]: + """Map source paths to their source-backed file-node IDs.""" + + result: dict[Path, str] = {} + for node in nodes: + source = _resolved_source(node.get("source_file")) + node_id = node.get("id") + if source is None or not node_id: + continue + if str(node.get("label", "")) == Path(str(node["source_file"])).name: + result[source] = str(node_id) + return result + + +def _materialize_python_type_alias_nodes( + paths: Sequence[Path], + nodes: list[dict[str, Any]], + edges: list[dict[str, Any]], + file_ids: dict[Path, str], +) -> dict[tuple[str, str], list[str]]: + """Create source-backed alias nodes and return their strict lookup index.""" + + existing = { + ( + _resolved_source(node.get("source_file")), + str(node.get("label", "")).lower(), + ): str(node["id"]) + for node in nodes + if node.get("id") and node.get("node_kind") == "type_alias" + } + edge_keys = { + (str(edge.get("source")), str(edge.get("target")), edge.get("relation")) + for edge in edges + } + index: dict[tuple[str, str], list[str]] = {} + for path in paths: + if path.suffix.lower() not in {".py", ".pyw"}: + continue + source = _resolved_source(path) + if source is None: + continue + file_id = file_ids.get(source) + if file_id is None: + continue + for alias in parse_python_type_aliases(path): + key = (source, alias.name.lower()) + alias_id = existing.get(key) + if alias_id is None: + alias_id = _shared_make_id(file_id, alias.name) + nodes.append( + { + "id": alias_id, + "label": alias.name, + "file_type": "code", + "node_kind": "type_alias", + "source_file": alias.source_file, + "source_location": alias.source_location, + } + ) + existing[key] = alias_id + contains_key = (file_id, alias_id, "contains") + if contains_key not in edge_keys: + edges.append( + { + "source": file_id, + "target": alias_id, + "relation": "contains", + "confidence": "EXTRACTED", + "source_file": alias.source_file, + "source_location": alias.source_location, + "weight": 1.0, + } + ) + edge_keys.add(contains_key) + index.setdefault( + (alias.module_stem, alias.name.lower()), [] + ).append(alias_id) + return index + + +def canonicalize_python_type_aliases( + paths: Sequence[Path], + nodes: list[dict[str, Any]], + edges: list[dict[str, Any]], +) -> None: + """Materialize aliases and rewire uniquely import-supported local stubs.""" + + file_ids = _file_node_ids_by_resolved_path(nodes) + alias_index = _materialize_python_type_alias_nodes( + paths, nodes, edges, file_ids + ) + node_by_id = { + str(node.get("id")): node for node in nodes if node.get("id") + } + rewired_stub_ids: set[str] = set() + for path in paths: + if path.suffix.lower() not in {".py", ".pyw"}: + continue + imports = parse_python_import_aliases(path) + source = _resolved_source(path) + for edge in edges: + if _resolved_source(edge.get("source_file")) != source: + continue + stub = node_by_id.get(str(edge.get("target"))) + if not stub or stub.get("source_file"): + continue + imported = imports.get(str(stub.get("label", ""))) + if imported is None: + continue + candidates = alias_index.get( + (imported.module_stem, imported.imported_name.lower()), [] + ) + if len(candidates) != 1: + continue + edge["target"] = candidates[0] + rewired_stub_ids.add(str(stub["id"])) + + referenced = { + str(edge.get(key)) + for edge in edges + for key in ("source", "target") + if edge.get(key) + } + nodes[:] = [ + node + for node in nodes + if str(node.get("id")) not in rewired_stub_ids + or str(node.get("id")) in referenced + ] + + _SOURCE_LINE_RE = re.compile(r"^L([1-9][0-9]*)") _SYMBOL_KIND_PRIORITY = { "method": 0, diff --git a/tests/test_symbol_resolution.py b/tests/test_symbol_resolution.py index 46de25d660..be49e2db90 100644 --- a/tests/test_symbol_resolution.py +++ b/tests/test_symbol_resolution.py @@ -4,14 +4,18 @@ from pathlib import Path +import pytest + from graphify.symbol_resolution import ( _bash_make_id, build_label_index, build_python_symbol_index, + canonicalize_python_type_aliases, find_unique_python_symbol, node_is_resolvable_symbol, normalise_callable_label, parse_python_import_aliases, + parse_python_type_aliases, resolve_bash_source_edges, resolve_cross_file_raw_calls, resolve_markdown_code_references, @@ -19,6 +23,239 @@ ) +def test_parse_python_type_aliases_supports_assignment_and_typealias(tmp_path): + module = tmp_path / "ordering.py" + module.write_text( + "from typing import Callable, TypeAlias\n" + "OrderFn = Callable[[int], str]\n" + "ExplicitOrderFn: TypeAlias = Callable[[int], str]\n", + encoding="utf-8", + ) + + aliases = parse_python_type_aliases(module) + + assert [(alias.name, alias.source_location) for alias in aliases] == [ + ("OrderFn", "L2"), + ("ExplicitOrderFn", "L3"), + ] + + +def test_parse_python_type_aliases_ignores_function_local_assignments(tmp_path): + module = tmp_path / "ordering.py" + module.write_text( + "from typing import Callable\n" + "def build():\n" + " LocalOrderFn = Callable[[int], str]\n", + encoding="utf-8", + ) + + assert parse_python_type_aliases(module) == [] + + +def test_canonicalize_python_type_aliases_rewires_imported_stub(tmp_path): + definitions = tmp_path / "ordering.py" + consumer = tmp_path / "planner.py" + definitions.write_text( + "from typing import Callable\nOrderFn = Callable[[int], str]\n", + encoding="utf-8", + ) + consumer.write_text( + "from ordering import OrderFn\n" + "def plan(order: OrderFn):\n" + " return order(1)\n", + encoding="utf-8", + ) + nodes = [ + { + "id": "ordering", + "label": "ordering.py", + "file_type": "code", + "source_file": str(definitions), + "source_location": "L1", + }, + { + "id": "planner", + "label": "planner.py", + "file_type": "code", + "source_file": str(consumer), + "source_location": "L1", + }, + { + "id": "planner_plan", + "label": "plan", + "file_type": "code", + "node_kind": "function", + "source_file": str(consumer), + "source_location": "L2", + }, + { + "id": "stub_orderfn", + "label": "OrderFn", + "file_type": "code", + }, + ] + edges = [ + { + "source": "planner_plan", + "target": "stub_orderfn", + "relation": "references", + "source_file": str(consumer), + } + ] + + canonicalize_python_type_aliases([definitions, consumer], nodes, edges) + + aliases = [ + node for node in nodes if node.get("node_kind") == "type_alias" + ] + assert len(aliases) == 1 + assert aliases[0]["label"] == "OrderFn" + assert aliases[0]["source_file"] == str(definitions) + assert edges[0]["target"] == aliases[0]["id"] + assert all(node["id"] != "stub_orderfn" for node in nodes) + + +def test_extract_canonicalizes_imported_python_type_alias(tmp_path): + from graphify.extract import extract + + definitions = tmp_path / "ordering.py" + consumer = tmp_path / "planner.py" + definitions.write_text( + "from typing import Callable\nOrderFn = Callable[[int], str]\n", + encoding="utf-8", + ) + consumer.write_text( + "from ordering import OrderFn\n" + "def plan(order: OrderFn):\n" + " return order(1)\n", + encoding="utf-8", + ) + + result = extract( + [definitions, consumer], + cache_root=tmp_path, + root=tmp_path, + parallel=False, + ) + + aliases = [ + node + for node in result["nodes"] + if node.get("label") == "OrderFn" + ] + assert len(aliases) == 1 + assert aliases[0].get("node_kind") == "type_alias" + assert aliases[0].get("source_file") == "ordering.py" + assert any( + edge.get("target") == aliases[0]["id"] + and edge.get("relation") == "references" + for edge in result["edges"] + ) + + +@pytest.mark.parametrize( + "consumer_source", + [ + "from ordering import *\ndef plan(order: OrderFn):\n return order(1)\n", + "def plan(order: OrderFn):\n from ordering import OrderFn\n return order(1)\n", + "from external_lib import OrderFn\ndef plan(order: OrderFn):\n return order(1)\n", + ], +) +def test_python_type_alias_resolution_requires_top_level_exact_import( + tmp_path, consumer_source +): + definitions = tmp_path / "ordering.py" + consumer = tmp_path / "planner.py" + definitions.write_text( + "from typing import Callable\nOrderFn = Callable[[int], str]\n", + encoding="utf-8", + ) + consumer.write_text(consumer_source, encoding="utf-8") + nodes = [ + { + "id": "ordering", + "label": "ordering.py", + "file_type": "code", + "source_file": str(definitions), + }, + { + "id": "planner", + "label": "planner.py", + "file_type": "code", + "source_file": str(consumer), + }, + {"id": "stub", "label": "OrderFn", "file_type": "code"}, + ] + edges = [ + { + "source": "planner", + "target": "stub", + "relation": "references", + "source_file": str(consumer), + } + ] + + canonicalize_python_type_aliases([definitions, consumer], nodes, edges) + + assert edges[0]["target"] == "stub" + assert any(node["id"] == "stub" for node in nodes) + + +def test_python_type_alias_resolution_leaves_ambiguous_modules_unresolved( + tmp_path, +): + first = tmp_path / "one" / "ordering.py" + second = tmp_path / "two" / "ordering.py" + consumer = tmp_path / "planner.py" + first.parent.mkdir() + second.parent.mkdir() + alias_source = ( + "from typing import Callable\nOrderFn = Callable[[int], str]\n" + ) + first.write_text(alias_source, encoding="utf-8") + second.write_text(alias_source, encoding="utf-8") + consumer.write_text( + "from ordering import OrderFn\n" + "def plan(order: OrderFn):\n" + " return order(1)\n", + encoding="utf-8", + ) + nodes = [ + { + "id": "one_ordering", + "label": "ordering.py", + "file_type": "code", + "source_file": str(first), + }, + { + "id": "two_ordering", + "label": "ordering.py", + "file_type": "code", + "source_file": str(second), + }, + { + "id": "planner", + "label": "planner.py", + "file_type": "code", + "source_file": str(consumer), + }, + {"id": "stub", "label": "OrderFn", "file_type": "code"}, + ] + edges = [ + { + "source": "planner", + "target": "stub", + "relation": "references", + "source_file": str(consumer), + } + ] + + canonicalize_python_type_aliases([first, second, consumer], nodes, edges) + + assert edges[0]["target"] == "stub" + assert any(node["id"] == "stub" for node in nodes) + + def test_markdown_code_reference_prefers_exact_symbol(tmp_path): source = tmp_path / "service.py" source.write_text("def run():\n return 1\n", encoding="utf-8") From 099763c1f47fa3a7ea3b8305eacf991f3c7c2904 Mon Sep 17 00:00:00 2001 From: Soumava2023 Date: Fri, 21 Aug 2026 15:02:49 -0400 Subject: [PATCH 06/13] feat: classify actionable graph gaps --- graphify/analyze.py | 18 ++++++---- graphify/gaps.py | 78 +++++++++++++++++++++++++++++++++++++++++++ tests/test_analyze.py | 67 +++++++++++++++++++++++++++++++++++++ 3 files changed, 157 insertions(+), 6 deletions(-) create mode 100644 graphify/gaps.py diff --git a/graphify/analyze.py b/graphify/analyze.py index 0707e2be78..17aa892da0 100644 --- a/graphify/analyze.py +++ b/graphify/analyze.py @@ -511,12 +511,13 @@ def suggest_questions( }) # 4. Isolated or weakly-connected nodes → exploration questions + from .gaps import GapCategory, classify_gap_node, gap_breakdown + isolated = [ - n for n in G.nodes() - if G.degree(n) <= 1 - and not _is_file_node(G, n) - and not _is_concept_node(G, n) - and G.nodes[n].get("file_type") != "rationale" + node_id + for node_id in G.nodes() + if G.degree(node_id) <= 1 + and classify_gap_node(G, node_id) is GapCategory.ACTIONABLE_LOCAL ] if isolated: labels = [G.nodes[n].get("label", n) for n in isolated[:3]] @@ -530,7 +531,12 @@ def suggest_questions( from .cluster import cohesion_score for cid, nodes in communities.items(): score = cohesion_score(G, nodes) - if score < 0.15 and len(nodes) >= 5: + breakdown = gap_breakdown(G, nodes) + if ( + score < 0.15 + and len(nodes) >= 5 + and breakdown[GapCategory.ACTIONABLE_LOCAL.value] > 0 + ): label = community_labels.get(cid, f"Community {cid}") questions.append({ "type": "low_cohesion", diff --git a/graphify/gaps.py b/graphify/gaps.py new file mode 100644 index 0000000000..177c62482a --- /dev/null +++ b/graphify/gaps.py @@ -0,0 +1,78 @@ +"""Shared classification for actionable and benign graph gaps.""" + +from __future__ import annotations + +from collections import Counter +from enum import Enum +from typing import Iterable + +import networkx as nx + + +class GapCategory(str, Enum): + """Mutually exclusive reasons a weak graph node is or is not actionable.""" + + ACTIONABLE_LOCAL = "actionable_local" + EXTERNAL = "external" + RATIONALE = "rationale" + METADATA = "metadata" + STRUCTURAL = "structural" + + +def _is_external(attrs: dict) -> bool: + """Return True only for affirmative extractor-originated evidence.""" + + metadata = ( + attrs.get("metadata") + if isinstance(attrs.get("metadata"), dict) + else {} + ) + return bool( + attrs.get("external") is True + or attrs.get("node_kind") == "external_symbol" + or metadata.get("scip_kind") == "external" + or str(attrs.get("id", "")).startswith("ref_") + ) + + +def classify_gap_node(graph: nx.Graph, node_id: str) -> GapCategory: + """Classify one graph node without deleting or hiding it from queries.""" + + from graphify.analyze import ( + _is_concept_node, + _is_file_node, + _is_json_key_node, + ) + + attrs = dict(graph.nodes[node_id]) + attrs.setdefault("id", node_id) + if _is_external(attrs): + return GapCategory.EXTERNAL + if attrs.get("file_type") == "rationale": + return GapCategory.RATIONALE + if _is_json_key_node(graph, node_id): + return GapCategory.METADATA + if ( + _is_file_node(graph, node_id) + or _is_concept_node(graph, node_id) + or attrs.get("node_kind") in {"page", "heading"} + ): + return GapCategory.STRUCTURAL + if attrs.get("source_file"): + return GapCategory.ACTIONABLE_LOCAL + return GapCategory.STRUCTURAL + + +def gap_breakdown( + graph: nx.Graph, + node_ids: Iterable[str], +) -> dict[str, int]: + """Count every supplied node in the shared classification vocabulary.""" + + counts = Counter( + classify_gap_node(graph, node_id).value for node_id in node_ids + ) + return { + category.value: counts.get(category.value, 0) + for category in GapCategory + } diff --git a/tests/test_analyze.py b/tests/test_analyze.py index 7bff432cf7..09028f4ab1 100644 --- a/tests/test_analyze.py +++ b/tests/test_analyze.py @@ -7,10 +7,77 @@ from graphify.cluster import cluster from graphify.analyze import god_nodes, surprising_connections, _is_concept_node, graph_diff, _surprise_score, _file_category, _is_json_key_node, find_import_cycles, suggest_questions from graphify.extract import _make_id +from graphify.gaps import GapCategory, classify_gap_node FIXTURES = Path(__file__).parent / "fixtures" +@pytest.mark.parametrize( + "attrs", + [ + {"external": True, "file_type": "code"}, + {"node_kind": "external_symbol", "file_type": "code"}, + {"metadata": {"scip_kind": "external"}, "file_type": "code"}, + ], +) +def test_external_evidence_is_benign(attrs): + graph = nx.Graph() + graph.add_node("external", label="Flask", **attrs) + + assert classify_gap_node(graph, "external") is GapCategory.EXTERNAL + + +def test_sourceless_semantic_concept_is_not_external(): + graph = nx.Graph() + graph.add_node("concept", label="Debt Strategy", file_type="concept") + + assert classify_gap_node(graph, "concept") is GapCategory.STRUCTURAL + + +def test_source_backed_leaf_is_actionable(): + graph = nx.Graph() + graph.add_node( + "service", + label="Service", + file_type="code", + source_file="src/service.py", + ) + + assert classify_gap_node(graph, "service") is GapCategory.ACTIONABLE_LOCAL + + +def test_suggest_questions_counts_only_actionable_local_leaves(): + graph = nx.Graph() + graph.add_node( + "local", + label="LocalService", + file_type="code", + source_file="src/local.py", + ) + graph.add_node("external", label="Flask", file_type="code", external=True) + graph.add_node( + "reason", + label="Why", + file_type="rationale", + source_file="src/local.py", + ) + graph.add_node( + "json", + label="name", + file_type="code", + source_file="fixtures/data.json", + ) + + questions = suggest_questions(graph, {}, {}, top_n=10) + isolated = next( + item for item in questions if item["type"] == "isolated_nodes" + ) + + assert isolated["why"].startswith("1 weakly-connected node") + assert "LocalService" in isolated["question"] + assert "Flask" not in isolated["question"] + + def make_graph(): return build_from_json(json.loads((FIXTURES / "extraction.json").read_text())) From f9295fdff5d1e0af833193630016809cca549147 Mon Sep 17 00:00:00 2001 From: Soumava2023 Date: Fri, 21 Aug 2026 15:05:16 -0400 Subject: [PATCH 07/13] feat: report actionable and benign graph gaps --- graphify/report.py | 63 +++++++++++++++++++++++++---------- tests/test_gap_reporting.py | 66 +++++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 17 deletions(-) create mode 100644 tests/test_gap_reporting.py diff --git a/graphify/report.py b/graphify/report.py index 69657f0d9d..d967b4909b 100644 --- a/graphify/report.py +++ b/graphify/report.py @@ -275,30 +275,59 @@ def generate( ] # --- Gaps section --- - from .analyze import _is_file_node, _is_concept_node - - isolated = [ - n for n in G.nodes() - if G.degree(n) <= 1 - and not _is_file_node(G, n) - and not _is_concept_node(G, n) - and G.nodes[n].get("file_type") != "rationale" + from .analyze import _is_file_node + from .gaps import GapCategory, classify_gap_node, gap_breakdown + + weak_nodes = [node_id for node_id in G.nodes() if G.degree(node_id) <= 1] + weak_breakdown = gap_breakdown(G, weak_nodes) + actionable_isolated = [ + node_id + for node_id in weak_nodes + if classify_gap_node(G, node_id) is GapCategory.ACTIONABLE_LOCAL ] thin_communities = { cid: nodes for cid, nodes in communities.items() - if 0 < sum(1 for n in nodes if not _is_file_node(G, n)) < 3 + if 0 < sum(1 for n in nodes if not _is_file_node(G, n)) < min_community_size } - gap_count = len(isolated) + len(thin_communities) + thin_breakdowns = { + cid: gap_breakdown(G, nodes) + for cid, nodes in thin_communities.items() + } + actionable_thin = { + cid: counts + for cid, counts in thin_breakdowns.items() + if counts[GapCategory.ACTIONABLE_LOCAL.value] > 0 + } + benign_thin_count = len(thin_breakdowns) - len(actionable_thin) - if gap_count > 0 or amb_pct > 20: + if weak_nodes or thin_breakdowns or amb_pct > 20: lines += ["", "## Knowledge Gaps"] - if isolated: - isolated_labels = [G.nodes[n].get("label", n) for n in isolated[:5]] - suffix = f" (+{len(isolated)-5} more)" if len(isolated) > 5 else "" - lines.append(f"- **{len(isolated)} isolated node(s):** {', '.join(f'`{l}`' for l in isolated_labels)}{suffix}") + if actionable_isolated: + isolated_labels = [ + G.nodes[node_id].get("label", node_id) + for node_id in actionable_isolated[:5] + ] + suffix = ( + f" (+{len(actionable_isolated) - 5} more)" + if len(actionable_isolated) > 5 + else "" + ) + lines.append( + f"- **{len(actionable_isolated)} actionable isolated node(s):** " + f"{', '.join(f'`{label}`' for label in isolated_labels)}{suffix}" + ) lines.append(" These have ≤1 connection - possible missing edges or undocumented components.") - if thin_communities: - lines.append(f"- **{len(thin_communities)} thin communities (<{min_community_size} nodes) omitted from report** — run `graphify query` to explore isolated nodes.") + lines.append( + "- **Benign weak-node breakdown:** " + f"external: {weak_breakdown['external']}, " + f"rationale: {weak_breakdown['rationale']}, " + f"metadata: {weak_breakdown['metadata']}, " + f"structural: {weak_breakdown['structural']}" + ) + lines.append( + f"- **actionable thin communities: {len(actionable_thin)}**" + ) + lines.append(f"- **benign thin communities: {benign_thin_count}**") if amb_pct > 20: lines.append(f"- **High ambiguity: {amb_pct}% of edges are AMBIGUOUS.** Review the Ambiguous Edges section above.") diff --git a/tests/test_gap_reporting.py b/tests/test_gap_reporting.py new file mode 100644 index 0000000000..2891138bdd --- /dev/null +++ b/tests/test_gap_reporting.py @@ -0,0 +1,66 @@ +"""Focused report tests for actionable versus benign graph gaps.""" + +import networkx as nx + +from graphify.report import generate + + +def _generate(graph: nx.Graph, communities: dict[int, list[str]]) -> str: + return generate( + graph, + communities, + {community_id: 0.0 for community_id in communities}, + {community_id: f"Community {community_id}" for community_id in communities}, + [], + [], + { + "total_files": 3, + "total_words": 30, + "needs_graph": True, + "warning": None, + }, + {"input": 0, "output": 0}, + "./project", + ) + + +def test_report_separates_actionable_and_benign_isolated_nodes(): + graph = nx.Graph() + graph.add_node( + "local", + label="LocalService", + file_type="code", + source_file="src/local.py", + ) + graph.add_node("external", label="Flask", file_type="code", external=True) + graph.add_node( + "reason", + label="Decision", + file_type="rationale", + source_file="src/local.py", + ) + + report = _generate( + graph, + communities={0: ["local"], 1: ["external"], 2: ["reason"]}, + ) + + assert "1 actionable isolated node(s)" in report + assert "external: 1" in report + assert "rationale: 1" in report + assert "`LocalService`" in report + + +def test_report_marks_all_benign_thin_community_non_actionable(): + graph = nx.Graph() + graph.add_node( + "external", + label="parametrize", + file_type="code", + external=True, + ) + + report = _generate(graph, communities={0: ["external"]}) + + assert "benign thin communities: 1" in report + assert "actionable thin communities: 0" in report From 5b59eb823a87e05dc617a71fcfbeab9fb75d4f89 Mon Sep 17 00:00:00 2001 From: Soumava2023 Date: Fri, 21 Aug 2026 16:08:09 -0400 Subject: [PATCH 08/13] perf: index type alias references by source --- graphify/symbol_resolution.py | 11 +++++--- tests/test_symbol_resolution.py | 46 +++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/graphify/symbol_resolution.py b/graphify/symbol_resolution.py index b1e0b6c4c7..e88ce0da69 100644 --- a/graphify/symbol_resolution.py +++ b/graphify/symbol_resolution.py @@ -206,15 +206,20 @@ def canonicalize_python_type_aliases( node_by_id = { str(node.get("id")): node for node in nodes if node.get("id") } + edges_by_source: dict[Path, list[dict[str, Any]]] = {} + for edge in edges: + edge_source = _resolved_source(edge.get("source_file")) + if edge_source is not None: + edges_by_source.setdefault(edge_source, []).append(edge) rewired_stub_ids: set[str] = set() for path in paths: if path.suffix.lower() not in {".py", ".pyw"}: continue imports = parse_python_import_aliases(path) source = _resolved_source(path) - for edge in edges: - if _resolved_source(edge.get("source_file")) != source: - continue + if source is None: + continue + for edge in edges_by_source.get(source, ()): stub = node_by_id.get(str(edge.get("target"))) if not stub or stub.get("source_file"): continue diff --git a/tests/test_symbol_resolution.py b/tests/test_symbol_resolution.py index be49e2db90..10df10b813 100644 --- a/tests/test_symbol_resolution.py +++ b/tests/test_symbol_resolution.py @@ -115,6 +115,52 @@ def test_canonicalize_python_type_aliases_rewires_imported_stub(tmp_path): assert all(node["id"] != "stub_orderfn" for node in nodes) +def test_canonicalize_python_type_aliases_indexes_edges_by_source( + monkeypatch, tmp_path +): + """A repository pass must resolve each edge source a bounded number of times.""" + import graphify.symbol_resolution as sr + + paths = [] + nodes = [] + edges = [] + for index in range(12): + path = tmp_path / f"module_{index}.py" + path.write_text("VALUE = 1\n", encoding="utf-8") + paths.append(path) + nodes.append( + { + "id": f"module_{index}", + "label": path.name, + "file_type": "code", + "source_file": str(path), + } + ) + for edge_index in range(10): + edges.append( + { + "source": f"module_{index}", + "target": f"external_{edge_index}", + "relation": "references", + "source_file": str(path), + } + ) + + original = sr._resolved_source + calls = 0 + + def counted(value): + nonlocal calls + calls += 1 + return original(value) + + monkeypatch.setattr(sr, "_resolved_source", counted) + + canonicalize_python_type_aliases(paths, nodes, edges) + + assert calls < 400 + + def test_extract_canonicalizes_imported_python_type_alias(tmp_path): from graphify.extract import extract From e88c8e98c647cd14f775e8fc716ebc8a47ec6b68 Mon Sep 17 00:00:00 2001 From: Soumava2023 Date: Fri, 21 Aug 2026 20:17:40 -0400 Subject: [PATCH 09/13] docs: design extracted type-use relationships --- ...extracted-type-use-relationships-design.md | 167 ++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-21-extracted-type-use-relationships-design.md diff --git a/docs/superpowers/specs/2026-08-21-extracted-type-use-relationships-design.md b/docs/superpowers/specs/2026-08-21-extracted-type-use-relationships-design.md new file mode 100644 index 0000000000..f901be9e13 --- /dev/null +++ b/docs/superpowers/specs/2026-08-21-extracted-type-use-relationships-design.md @@ -0,0 +1,167 @@ +# Extracted Type-Use Relationships — Design + +**Date:** 2026-08-21 +**Status:** Proposed for user review + +## Problem + +Graphify's Python cross-file resolver currently turns every reference to an +imported symbol inside a class or function into the same edge: + +```json +{ + "relation": "uses", + "confidence": "INFERRED", + "confidence_score": 0.95 +} +``` + +That is appropriate for an imported name whose role is only known from a body +reference. It is inaccurate for a parameter, return, or field annotation. Those +annotations are direct syntax-tree evidence, so they should not be presented as +model-inferred relationships. The generic edge also hides whether the type is +accepted, returned, or stored. + +DebtGPS exposed the practical impact: 38 relationships involving its canonical +`Debt` type were reported as inferred even though every relationship was backed +by a concrete Python annotation or constructor reference. + +## Constraint: One Edge Per Endpoint Pair + +Graphify builds an `nx.Graph` or `nx.DiGraph`, not a multigraph. A source and +target therefore retain one edge in the built graph even if raw extraction +emits several relations between them. + +Emitting separate `accepts_type`, `returns_type`, and `field_type` edges would +lose information when one symbol uses the same type in multiple roles. It could +also overwrite a more specific runtime `calls` edge unless every downstream +edge-selection rule were updated. + +The design must preserve all annotation roles in one edge and must not weaken a +runtime relationship. + +## Chosen Model + +An exact annotation produces one `uses_type` edge per source-target pair: + +```json +{ + "source": "api_transform", + "target": "models_payload", + "relation": "uses_type", + "context": "type_annotation", + "type_roles": ["parameter", "return"], + "confidence": "EXTRACTED", + "confidence_score": 1.0, + "source_file": "api.py", + "source_location": "L12", + "weight": 1.0 +} +``` + +`type_roles` is a sorted, unique list. Supported roles are: + +- `parameter`: a top-level function or method parameter annotation; +- `return`: a top-level function or method return annotation; +- `field`: an annotated class attribute; +- `nested_parameter`: a parameter annotation on a function nested inside the + source symbol; +- `nested_return`: a return annotation on a function nested inside the source + symbol; +- `nested_field`: an annotated assignment inside a nested class owned by the + source symbol. + +Nested roles prevent a factory such as `order_custom()` from falsely claiming +that its own signature accepts `Debt` when the annotation actually belongs to +the closure it returns. + +## Extraction Flow + +The Python cross-file import resolver will continue to resolve imported names +against source-backed in-corpus definitions. While walking each source symbol, +it will classify identifier occurrences by syntax context: + +1. Detect whether an imported identifier occurs inside a parameter annotation, + return annotation, or annotated class field. +2. Record the role and first evidence line per source-target pair. +3. Aggregate all roles for that pair into one `uses_type` edge. +4. Continue collecting non-annotation body references as generic `uses` facts. +5. Preserve the existing extracted `calls` relationship for runtime constructor + and function calls. + +Annotations nested inside generic containers retain the surrounding role. For +example, both `list[Debt]` and `Debt | None` record the role of the full +annotation rather than treating `Debt` as an unclassified body reference. + +Forward-reference string annotations are included when their imported symbol +can be resolved exactly. Arbitrary strings are not evaluated. + +## Edge Precedence and Compatibility + +`uses_type` is a generic relationship for graph-construction precedence. A +specific runtime edge such as `calls`, `inherits`, or `implements` wins when the +same endpoints carry both facts. This keeps `analyze_refinance() -> Debt` as a +runtime construction relationship while annotation-only ordering functions use +`uses_type`. + +Raw extraction may contain both facts. The built graph keeps the specific fact; +diagnostics and future multigraph output can still inspect the raw evidence. + +Existing `uses` consumers remain supported: + +- runtime and otherwise-unclassified imported-name references still emit + `uses/INFERRED`; +- query and display code accepts the new relation without special handling; +- call-flow and affected traversals do not treat type-only relationships as + runtime calls; +- gap and centrality analysis may traverse `uses_type` as ordinary structural + connectivity, but must not apply the cross-language inferred-edge penalty to + an extracted edge. + +No semantic-cache migration is required because these are deterministic AST +relationships. A code graph rebuild replaces the old inferred edges. + +## Error Handling and Conservatism + +- An unresolved, ambiguous, external, or star-imported annotation does not bind + to an arbitrary local type. +- Built-in annotations such as `str`, `int`, and `list` do not create local type + nodes. +- Malformed syntax continues to fail open under the extractor's existing error + handling; it must not fabricate a type relationship. +- Duplicate annotation occurrences merge their roles and keep the earliest + evidence line. +- A runtime reference and an annotation reference may coexist in raw output, + but graph construction must retain the specific runtime relation. + +## Verification Contract + +Tests will establish the behavior before implementation: + +1. Parameter-only annotation emits `uses_type`, role `parameter`, and + `EXTRACTED 1.0`. +2. Return-only annotation emits role `return`. +3. A parameter-and-return use of the same type produces one edge with both + sorted roles. +4. A class attribute annotation emits role `field`. +5. A nested closure records nested roles on its owning source symbol. +6. A runtime constructor call remains `calls/EXTRACTED`, not `uses_type`, in the + built graph. +7. A non-annotation body reference retains `uses/INFERRED 0.95` when no more + specific runtime relationship is available. +8. Import aliases, generics, unions, forward references, ambiguity, and built-in + exclusions retain their existing safety behavior. +9. Incremental extraction produces the same type edge as a full extraction. +10. Rebuilding DebtGPS changes the 34 production annotation-only `Debt` edges + from inferred `uses` to extracted `uses_type`. Its four test constructor + sites must not be mislabeled as annotation-backed `uses_type`; when the + existing call resolver resolves them, they remain extracted runtime facts. + The refinance constructor relationship remains an extracted runtime fact. + +## Scope + +This change applies to Python cross-file annotation resolution. It does not +redesign Graphify as a multigraph, change semantic-LLM extraction, or rewrite +the established `references` contexts produced by other language extractors. +Cross-language unification can be designed separately after this Python model +has proven stable. From aca6c4fe2bada86913bfd8221dff446d1546980f Mon Sep 17 00:00:00 2001 From: Soumava2023 Date: Fri, 21 Aug 2026 20:40:28 -0400 Subject: [PATCH 10/13] docs: plan extracted type-use relationships --- ...-08-21-extracted-type-use-relationships.md | 733 ++++++++++++++++++ ...extracted-type-use-relationships-design.md | 2 +- 2 files changed, 734 insertions(+), 1 deletion(-) create mode 100644 docs/superpowers/plans/2026-08-21-extracted-type-use-relationships.md diff --git a/docs/superpowers/plans/2026-08-21-extracted-type-use-relationships.md b/docs/superpowers/plans/2026-08-21-extracted-type-use-relationships.md new file mode 100644 index 0000000000..f86545616e --- /dev/null +++ b/docs/superpowers/plans/2026-08-21-extracted-type-use-relationships.md @@ -0,0 +1,733 @@ +# Extracted Type-Use Relationships Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace annotation-backed Python `uses/INFERRED` edges with deterministic `uses_type/EXTRACTED` edges that preserve every type role without weakening runtime relationships. + +**Architecture:** Extend the existing two-pass Python import resolver instead of adding a second parser. The resolver will classify imported identifiers by their Tree-sitter ancestor context, aggregate type roles per source-target pair, and use unchanged-corpus nodes as read-only targets during incremental extraction. `uses_type` joins the graph builder's generic-relation set so `calls`, `inherits`, and other runtime facts remain the single surviving edge in Graphify's simple graph. + +**Tech Stack:** Python 3.10+, Tree-sitter Python, NetworkX, pytest + +--- + +## File Structure + +- `graphify/extractors/resolution.py` owns Python import-target indexing, annotation-context classification, role aggregation, and raw `uses_type` edge emission. +- `graphify/extract.py` supplies unchanged-corpus nodes and the scan root to the resolver during incremental extraction. +- `graphify/build.py` classifies `uses_type` as generic for same-endpoint collapse precedence. +- `tests/test_extract.py` verifies raw full-extraction semantics, nesting, aliases, forward references, and conservative fallback behavior. +- `tests/test_incremental.py` proves a changed importer resolves the same type target and metadata when its definition file is unchanged. +- `tests/test_relation_collapse_precedence.py` proves runtime edges beat `uses_type` in both input orders and retain their own metadata. +- `docs/superpowers/specs/2026-08-21-extracted-type-use-relationships-design.md` is the approved design contract; implementation must not broaden beyond its Python-only scope. + +### Task 1: Emit extracted type-use edges from Python annotations + +**Files:** +- Modify: `tests/test_extract.py:3873` +- Modify: `graphify/extractors/resolution.py:1880-2076` + +- [ ] **Step 1: Add focused failing extraction tests** + +Add this helper and these tests immediately after `_inferred_uses` in `tests/test_extract.py`: + +```python +def _type_uses(result): + """Every deterministic cross-file Python type-use edge.""" + return [e for e in result["edges"] if e.get("relation") == "uses_type"] + + +def test_cross_file_annotations_emit_extracted_roles(tmp_path): + (tmp_path / "models.py").write_text( + "class Payload:\n pass\n", encoding="utf-8" + ) + (tmp_path / "api.py").write_text( + "from models import Payload as P\n\n\n" + "class Envelope:\n" + " value: P\n\n\n" + "def convert(values: list[P | None]) -> \"P\":\n" + " return values[0]\n\n\n" + "def build(value: P) -> P:\n" + " return P()\n", + encoding="utf-8", + ) + + result = extract( + [tmp_path / "api.py", tmp_path / "models.py"], + cache_root=tmp_path, + ) + by_source = {edge["source"]: edge for edge in _type_uses(result)} + + assert by_source["api_envelope"] == { + "source": "api_envelope", + "target": "models_payload", + "relation": "uses_type", + "context": "type_annotation", + "type_roles": ["field"], + "confidence": "EXTRACTED", + "confidence_score": 1.0, + "source_file": str(tmp_path / "api.py"), + "source_location": "L5", + "weight": 1.0, + } + assert by_source["api_convert"]["target"] == "models_payload" + assert by_source["api_convert"]["type_roles"] == ["parameter", "return"] + assert by_source["api_convert"]["confidence"] == "EXTRACTED" + assert by_source["api_convert"]["confidence_score"] == 1.0 + assert by_source["api_build"]["type_roles"] == ["parameter", "return"] + inferred_pairs = { + (edge["source"], edge["target"]) + for edge in result["edges"] + if edge.get("relation") == "uses" and edge.get("confidence") == "INFERRED" + } + assert not inferred_pairs & { + ("api_envelope", "models_payload"), + ("api_convert", "models_payload"), + } + # Raw extraction retains both facts. The graph builder chooses `calls` for + # this endpoint pair in Task 3. + assert ("api_build", "models_payload") in inferred_pairs + + +def test_nested_annotations_record_nested_roles_on_the_owner(tmp_path): + (tmp_path / "models.py").write_text("class Debt:\n pass\n", encoding="utf-8") + (tmp_path / "order.py").write_text( + "from models import Debt\n\n\n" + "def order_custom():\n" + " def key(debt: Debt) -> Debt:\n" + " return debt\n" + " return key\n\n\n" + "class Holder:\n" + " class Inner:\n" + " debt: Debt\n", + encoding="utf-8", + ) + + result = extract( + [tmp_path / "order.py", tmp_path / "models.py"], + cache_root=tmp_path, + ) + by_source = {edge["source"]: edge for edge in _type_uses(result)} + + assert by_source["order_order_custom"]["type_roles"] == [ + "nested_parameter", + "nested_return", + ] + assert by_source["order_holder"]["type_roles"] == ["nested_field"] + + +def test_local_annotation_and_body_reference_keep_conservative_uses(tmp_path): + (tmp_path / "models.py").write_text("class Helper:\n pass\n", encoding="utf-8") + (tmp_path / "api.py").write_text( + "from models import Helper\n\n\n" + "def handler():\n" + " local: Helper = Helper()\n" + " return local\n", + encoding="utf-8", + ) + + result = extract( + [tmp_path / "api.py", tmp_path / "models.py"], + cache_root=tmp_path, + ) + + assert ("api_handler", "models_helper") in _inferred_uses(result) + assert not any(edge["source"] == "api_handler" for edge in _type_uses(result)) + + +def test_type_use_skips_ambiguous_and_external_targets(tmp_path): + for package in ("one", "two"): + folder = tmp_path / package + folder.mkdir() + (folder / "models.py").write_text( + "class Payload:\n pass\n", encoding="utf-8" + ) + api = tmp_path / "api.py" + api.write_text( + "from models import Payload\n" + "from pathlib import Path\n\n\n" + "def load(value: Payload, path: Path) -> Payload:\n" + " return value\n", + encoding="utf-8", + ) + star = tmp_path / "star.py" + star.write_text( + "from one.models import *\n\n\n" + "def load(value: Payload) -> Payload:\n" + " return value\n", + encoding="utf-8", + ) + + result = extract( + [ + api, + star, + tmp_path / "one" / "models.py", + tmp_path / "two" / "models.py", + ], + cache_root=tmp_path, + ) + + assert not any(edge["source"] == "api_load" for edge in _type_uses(result)) + assert not any(edge["source"] == "star_load" for edge in _type_uses(result)) +``` + +- [ ] **Step 2: Run the new tests and verify the red state** + +Run: + +```powershell +.\.venv\Scripts\python.exe -m pytest tests/test_extract.py -k "cross_file_annotations_emit_extracted_roles or nested_annotations_record_nested_roles or local_annotation_and_body_reference" -q +``` + +Expected: the annotation and ambiguity tests fail because `uses_type` classification and ambiguous-target rejection are not implemented; the conservative local-annotation test may already pass. + +- [ ] **Step 3: Add syntax-context classification and role aggregation** + +Replace `_resolve_cross_file_imports`' docstring with: + +```python + """Resolve source-backed Python imports at the symbol level. + + Pass one indexes project definitions by directory-qualified module stem. + Pass two attributes each imported-name occurrence to its top-level owning + class or function. Supported annotations emit deterministic ``uses_type`` + edges with aggregated roles; other body references retain the conservative + ``uses/INFERRED`` relationship. + """ +``` + +In `_resolve_cross_file_imports`, replace the single `ref_sources` declaration and the existing `visit` implementation with the following complete block. Keep `resolve_import` unchanged above it. + +```python + # referenced name -> {source symbol nid: first body-reference line} + ref_sources: dict[str, dict[str, int]] = {} + # referenced name -> source symbol nid -> roles + first annotation line + type_ref_sources: dict[str, dict[str, dict[str, Any]]] = {} + + def _same_node(left, right) -> bool: + return ( + left is not None + and right is not None + and left.type == right.type + and left.start_byte == right.start_byte + and left.end_byte == right.end_byte + ) + + def _contains(outer, inner) -> bool: + return ( + outer is not None + and outer.start_byte <= inner.start_byte + and inner.end_byte <= outer.end_byte + ) + + def _annotation_role(ref_node, owner_node) -> str | None: + """Classify an identifier/string only when it is inside a supported annotation.""" + cursor = ref_node + kind: str | None = None + anchor = None + while cursor.parent is not None and not _same_node(cursor, owner_node): + parent = cursor.parent + if parent.type in ("typed_parameter", "typed_default_parameter"): + type_node = parent.child_by_field_name("type") + if _contains(type_node, ref_node): + kind, anchor = "parameter", parent + break + if parent.type == "function_definition": + return_node = parent.child_by_field_name("return_type") + if _contains(return_node, ref_node): + kind, anchor = "return", parent + break + if parent.type == "assignment": + type_node = parent.child_by_field_name("type") + if _contains(type_node, ref_node): + kind, anchor = "field", parent + break + cursor = parent + + if kind is None or anchor is None: + return None + + if kind == "field": + scope = anchor.parent + while scope is not None and not _same_node(scope, owner_node): + if scope.type in ("class_definition", "function_definition"): + break + scope = scope.parent + if scope is None or scope.type != "class_definition": + return None + return "field" if _same_node(scope, owner_node) else "nested_field" + + annotation_function = anchor + while ( + annotation_function is not None + and annotation_function.type != "function_definition" + ): + annotation_function = annotation_function.parent + if annotation_function is None: + return None + if owner_node.type == "function_definition": + nested = not _same_node(annotation_function, owner_node) + else: + nested = False + scope = annotation_function.parent + while scope is not None and not _same_node(scope, owner_node): + if scope.type in ("class_definition", "function_definition"): + nested = True + scope = scope.parent + if scope is None: + return None + return f"nested_{kind}" if nested else kind + + def _record_type_ref(name: str, source_nid: str, role: str, line: int) -> None: + by_source = type_ref_sources.setdefault(name, {}) + evidence = by_source.setdefault(source_nid, {"roles": set(), "line": line}) + evidence["roles"].add(role) + evidence["line"] = min(evidence["line"], line) + + def visit(node, current_nid: str | None, owner_node=None) -> None: + # Identifiers inside an import statement are the import itself, not a + # real use — resolve the import here and don't descend into it. + if node.type == "import_from_statement": + resolve_import(node) + return + # Attribute references to the top-level symbol that contains them. + if current_nid is None and node.type in ("class_definition", "function_definition"): + name_node = node.child_by_field_name("name") + if name_node is not None: + mapped = name_to_nid.get(_text(name_node)) + if mapped is not None: + current_nid = mapped + owner_node = node + if current_nid is not None and owner_node is not None: + if node.type == "identifier": + name = _text(node) + role = _annotation_role(node, owner_node) + if role is None: + slot = ref_sources.setdefault(name, {}) + slot.setdefault(current_nid, node.start_point[0] + 1) + else: + _record_type_ref( + name, current_nid, role, node.start_point[0] + 1 + ) + elif node.type == "string": + role = _annotation_role(node, owner_node) + if role is not None: + # Tokenize only a string already proven to be an annotation; + # do not evaluate it as Python. + for name in re.findall(r"\b[A-Za-z_]\w*\b", _text(node)): + _record_type_ref( + name, current_nid, role, node.start_point[0] + 1 + ) + for child in node.children: + visit(child, current_nid, owner_node) +``` + +Then replace the emission loop after `visit(tree.root_node, None)` with this complete loop: + +```python + for name, tgt_nid in import_targets.items(): + for src_nid, line in ref_sources.get(name, {}).items(): + if src_nid == tgt_nid: + continue + new_edges.append({ + "source": src_nid, + "target": tgt_nid, + "relation": "uses", + "confidence": "INFERRED", + "confidence_score": 0.95, + "source_file": str_path, + "source_location": f"L{line}", + "weight": 0.8, + }) + for src_nid, evidence in type_ref_sources.get(name, {}).items(): + if src_nid == tgt_nid: + continue + new_edges.append({ + "source": src_nid, + "target": tgt_nid, + "relation": "uses_type", + "context": "type_annotation", + "type_roles": sorted(evidence["roles"]), + "confidence": "EXTRACTED", + "confidence_score": 1.0, + "source_file": str_path, + "source_location": f"L{evidence['line']}", + "weight": 1.0, + }) +``` + +Also make the existing bare-module fallback reject collisions instead of retaining its first match. Change its declaration and assignment to: + +```python + bare_to_qualified: dict[str, str | None] = {} + + # Inside the pass-one node loop, after fq_stem is known: + bare = src_path.stem + if bare not in bare_to_qualified: + bare_to_qualified[bare] = fq_stem + elif bare_to_qualified[bare] != fq_stem: + bare_to_qualified[bare] = None +``` + +`resolve_import` already returns when `target_fq` is falsey, so ambiguous absolute imports now produce no local type binding. Exact relative imports still use their directory-qualified stem and remain unaffected. + +- [ ] **Step 4: Run the focused extraction tests and existing resolver regressions** + +Run: + +```powershell +.\.venv\Scripts\python.exe -m pytest tests/test_extract.py -k "cross_file_annotations_emit_extracted_roles or nested_annotations_record_nested_roles or local_annotation_and_body_reference or type_use_skips_ambiguous or inferred_uses_edge or cross_file_type_annotation_refs" -q +``` + +Expected: all selected tests pass. Existing body-only references remain `uses/INFERRED 0.95`; supported annotations become `uses_type/EXTRACTED 1.0`. + +- [ ] **Step 5: Commit the annotation extractor behavior** + +```powershell +git add graphify/extractors/resolution.py tests/test_extract.py +git commit -m "feat: extract Python type-use relationships" +``` + +### Task 2: Make type-use resolution identical in full and incremental extraction + +**Files:** +- Modify: `tests/test_incremental.py:214` +- Modify: `graphify/extractors/resolution.py:1880-1936` +- Modify: `graphify/extract.py:5484-5522` +- Modify: `graphify/extract.py:6270-6279` + +- [ ] **Step 1: Add a failing changed-importer regression test** + +Add this test before `test_incremental_python_relative_import_target_canonicalizes` in `tests/test_incremental.py`: + +```python +def test_incremental_python_type_use_matches_full_extraction(tmp_path): + from graphify.extract import extract + + pkg = tmp_path / "pkg" + pkg.mkdir() + model = pkg / "models.py" + model.write_text("class Debt:\n pass\n", encoding="utf-8") + planner = pkg / "planner.py" + planner.write_text( + "from .models import Debt\n\n\n" + "def prioritize(debts: list[Debt]) -> Debt:\n" + " return debts[0]\n", + encoding="utf-8", + ) + + full = extract( + [planner, model], cache_root=tmp_path, root=tmp_path, parallel=False + ) + incremental = extract( + [planner], + cache_root=tmp_path, + root=tmp_path, + parallel=False, + resolution_context_nodes=full["nodes"], + resolution_context_edges=full["edges"], + ) + + def type_edge(result): + return next( + edge + for edge in result["edges"] + if edge.get("relation") == "uses_type" + and edge.get("source") == "pkg_planner_prioritize" + ) + + full_edge = type_edge(full) + incremental_edge = type_edge(incremental) + assert incremental_edge["target"] == full_edge["target"] == "pkg_models_debt" + assert incremental_edge["type_roles"] == ["parameter", "return"] + assert incremental_edge["confidence"] == "EXTRACTED" + assert incremental_edge["confidence_score"] == 1.0 +``` + +- [ ] **Step 2: Run the incremental test and verify the red state** + +Run: + +```powershell +.\.venv\Scripts\python.exe -m pytest tests/test_incremental.py::test_incremental_python_type_use_matches_full_extraction -q +``` + +Expected: FAIL because the unchanged `models.py` definition is absent from `_resolve_cross_file_imports`' target index. + +- [ ] **Step 3: Extend only the resolver's target index with context nodes** + +Change the resolver signature to: + +```python +def _resolve_cross_file_imports( + per_file: list[dict], + paths: list[Path], + *, + resolution_context_nodes: list[dict] | None = None, + root: Path | None = None, +) -> list[dict]: +``` + +Replace the pass-one indexing loop with this implementation. Fresh definitions are indexed first and win; context nodes are read-only fallbacks. + +```python + stem_to_entities: dict[str, dict[str, str]] = {} + bare_to_qualified: dict[str, str | None] = {} + + def index_definition(node: dict, *, overwrite: bool) -> None: + src = node.get("source_file", "") + if not src: + return + src_path = Path(src) + if root is not None and not src_path.is_absolute(): + src_path = Path(root) / src_path + if src_path.suffix not in (".py", ".pyi"): + return + fq_stem = _file_stem(src_path) + label = node.get("label", "") + nid = node.get("id", "") + if ( + not label + or label.endswith((")", ".py", ".pyi")) + or "_" in label[:1] + or node.get("file_type") == "rationale" + ): + return + entities = stem_to_entities.setdefault(fq_stem, {}) + if overwrite: + entities[label] = nid + else: + entities.setdefault(label, nid) + bare = src_path.stem + if bare not in bare_to_qualified: + bare_to_qualified[bare] = fq_stem + elif bare_to_qualified[bare] != fq_stem: + bare_to_qualified[bare] = None + + for file_result in per_file: + for node in file_result.get("nodes", []): + index_definition(node, overwrite=True) + for node in resolution_context_nodes or []: + index_definition(node, overwrite=False) +``` + +In `graphify/extract.py`, update the invocation to: + +```python + cross_file_edges = _resolve_cross_file_imports( + py_results, + py_paths, + resolution_context_nodes=resolution_context_nodes, + root=root, + ) +``` + +In `extract()`'s docstring, replace the second process step with: + +```python + 2. Cross-file import resolution: emits source-backed runtime relationships + and deterministic Python ``uses_type`` edges. +``` + +Replace the `resolution_context_nodes` argument paragraph with: + +```python + resolution_context_nodes: read-only AST nodes from files that are NOT + being extracted this run (an incremental rebuild's unchanged + corpus, #2406). They extend the Python import/type-use target index, + the shared direct-call label/file indexes, the indirect_call + callable guard (via persisted `_callable` / `_callable_class` + markers, #2438), and member-call resolvers run by + `run_language_resolvers` (#2437). They are never parsed, mutated, + or returned; only edges sourced by re-extracted files are emitted. +``` + +- [ ] **Step 4: Run incremental and full-extraction tests** + +Run: + +```powershell +.\.venv\Scripts\python.exe -m pytest tests/test_incremental.py::test_incremental_python_type_use_matches_full_extraction tests/test_incremental.py::test_incremental_python_relative_import_target_canonicalizes tests/test_extract.py -k "type_use or annotations_emit or nested_annotations or inferred_uses_edge" -q +``` + +Expected: all selected tests pass, and the full and incremental type edges have identical target, roles, and confidence. + +- [ ] **Step 5: Commit incremental parity** + +```powershell +git add graphify/extract.py graphify/extractors/resolution.py tests/test_incremental.py +git commit -m "fix: resolve type uses against incremental context" +``` + +### Task 3: Preserve runtime relationships during simple-graph collapse + +**Files:** +- Modify: `tests/test_relation_collapse_precedence.py:25-29` +- Modify: `graphify/build.py:55-63` + +- [ ] **Step 1: Expand the precedence test matrix and pin metadata survival** + +Change the test module's generic list and add the focused regression: + +```python +GENERIC = ["references", "uses", "uses_type", "mentions"] + + +def test_calls_beats_uses_type_and_keeps_runtime_metadata(): + for edges in ( + [ + _edge("calls", source_location="L9", weight=1.0, confidence_score=1.0), + _edge( + "uses_type", + source_location="L4", + context="type_annotation", + type_roles=["return"], + weight=1.0, + confidence_score=1.0, + ), + ], + [ + _edge( + "uses_type", + source_location="L4", + context="type_annotation", + type_roles=["return"], + weight=1.0, + confidence_score=1.0, + ), + _edge("calls", source_location="L9", weight=1.0, confidence_score=1.0), + ], + ): + graph = build_from_json(_extraction(edges)) + data = edge_data(graph, "a", "b") + assert data["relation"] == "calls" + assert data["source_location"] == "L9" + assert "type_roles" not in data +``` + +- [ ] **Step 2: Run the precedence tests and verify the red state** + +Run: + +```powershell +.\.venv\Scripts\python.exe -m pytest tests/test_relation_collapse_precedence.py -q +``` + +Expected: cases containing `uses_type` fail because an unknown relation is currently treated as specific. + +- [ ] **Step 3: Mark `uses_type` as a generic structural relationship** + +Change the constant in `graphify/build.py` to: + +```python +_GENERIC_RELATIONS: frozenset[str] = frozenset( + {"references", "uses", "uses_type", "mentions"} +) +``` + +Do not add a total relation ranking. The existing specific-versus-generic guard is sufficient and preserves prior behavior between two specific or two generic relations. + +- [ ] **Step 4: Run the precedence and build suites** + +Run: + +```powershell +.\.venv\Scripts\python.exe -m pytest tests/test_relation_collapse_precedence.py tests/test_build.py -q +``` + +Expected: all tests pass; `calls` survives over `uses_type` in either input order. + +- [ ] **Step 5: Commit graph precedence support** + +```powershell +git add graphify/build.py tests/test_relation_collapse_precedence.py +git commit -m "fix: preserve runtime edges over type uses" +``` + +### Task 4: Verify Graphify and rebuild the DebtGPS knowledge graph + +**Files:** +- Verify: `graphify/extractors/resolution.py` +- Verify: `graphify/extract.py` +- Verify: `graphify/build.py` +- Update generated output: `C:/Users/souma/OneDrive/Desktop/Debt Application/Application/DebtGPS/graphify-out/` + +- [ ] **Step 1: Run formatting guards and the focused feature suite** + +Run from `tmp/graphify-src`: + +```powershell +git diff --check +.\.venv\Scripts\python.exe -m pytest tests/test_extract.py tests/test_incremental.py tests/test_relation_collapse_precedence.py tests/test_build.py -q +``` + +Expected: `git diff --check` prints nothing; all selected tests pass. + +- [ ] **Step 2: Run the complete Graphify test suite** + +```powershell +.\.venv\Scripts\python.exe -m pytest -q +``` + +Expected: the suite completes with zero failures. Pre-existing permission warnings from ignored `.pytest-*` directories do not count as failures. + +- [ ] **Step 3: Install the working tree into the configured Graphify tool environment** + +The DebtGPS graph records its interpreter in `graphify-out/.graphify_python`. Ensure that environment has pip, then install this checkout without downloading dependencies: + +```powershell +& 'C:\Users\souma\AppData\Roaming\uv\tools\graphifyy\Scripts\python.exe' -m ensurepip --upgrade +& 'C:\Users\souma\AppData\Roaming\uv\tools\graphifyy\Scripts\python.exe' -m pip install --no-build-isolation --no-deps --editable 'C:\Users\souma\OneDrive\Desktop\Debt Application\Application\DebtGPS\tmp\graphify-src' +graphify --version +``` + +Expected: the editable install succeeds and `graphify --version` reports `0.9.48` from the enhanced checkout. This step writes to the user-managed tool environment and therefore may require the normal workspace approval prompt. + +- [ ] **Step 4: Rebuild DebtGPS's AST tier while preserving semantic and document tiers** + +Run from the DebtGPS root: + +```powershell +graphify update . --force +``` + +Expected: Graphify performs a full code-corpus rebuild because `update` calls `_rebuild_code` with no changed-path subset. Existing semantic/document nodes are retained by tier-aware merge behavior. + +- [ ] **Step 5: Assert the corrected Debt relationships** + +Run from the DebtGPS root: + +```powershell +.\tmp\graphify-src\.venv\Scripts\python.exe -c "import json; p='graphify-out/graph.json'; d=json.load(open(p,encoding='utf-8')); es=d.get('links',d.get('edges',[])); debt='engine_models_debt'; rel=[e for e in es if debt in (e.get('source'),e.get('target'),e.get('_src'),e.get('_tgt'))]; typed=[e for e in rel if e.get('relation')=='uses_type']; inferred=[e for e in rel if e.get('confidence')=='INFERRED']; assert len(typed)==34,(len(typed),typed); assert all(e.get('confidence')=='EXTRACTED' and e.get('confidence_score')==1.0 and e.get('type_roles') for e in typed); assert len(inferred)<=4,(len(inferred),inferred); ps=next(e for e in typed if 'engine_action_plan_planstate' in (e.get('source'),e.get('target'),e.get('_src'),e.get('_tgt'))); assert 'field' in ps['type_roles']; ref=next(e for e in rel if 'engine_refinance_analyze_refinance' in (e.get('source'),e.get('target'),e.get('_src'),e.get('_tgt'))); assert ref['relation']=='calls' and ref['confidence']=='EXTRACTED',ref; print({'debt_type_edges':len(typed),'remaining_inferred':len(inferred),'plan_state_roles':ps['type_roles'],'refinance_relation':ref['relation']})" +``` + +Expected output contains: + +```text +'debt_type_edges': 34 +'plan_state_roles': ['field'] +'refinance_relation': 'calls' +``` + +The remaining inferred count is at most four and is limited to body-only test constructor sites rather than annotations. + +- [ ] **Step 6: Run DebtGPS's focused domain verification suite** + +Use the project's existing test environment and run the same focused coverage used for the original audit: + +```powershell +pytest tests/test_models.py tests/test_budget.py tests/test_action_plan.py tests/test_planning.py tests/test_routes.py tests/test_scenario_authority.py tests/test_audit_batch1.py tests/test_audit_batch6.py tests/test_audit_batch7.py tests/test_coverage_gaps.py -q +``` + +Expected: all tests pass; the previous baseline was 161 passing tests. + +- [ ] **Step 7: Commit any final source-only cleanup** + +If verification required a source or test correction, commit only those Graphify files: + +```powershell +git add graphify tests +git commit -m "test: verify extracted type-use relationships" +``` + +If no correction was required, do not create an empty commit. Leave DebtGPS's generated `graphify-out/` changes uncommitted unless the user separately asks to commit generated graph artifacts. diff --git a/docs/superpowers/specs/2026-08-21-extracted-type-use-relationships-design.md b/docs/superpowers/specs/2026-08-21-extracted-type-use-relationships-design.md index f901be9e13..5f210b5cee 100644 --- a/docs/superpowers/specs/2026-08-21-extracted-type-use-relationships-design.md +++ b/docs/superpowers/specs/2026-08-21-extracted-type-use-relationships-design.md @@ -1,7 +1,7 @@ # Extracted Type-Use Relationships — Design **Date:** 2026-08-21 -**Status:** Proposed for user review +**Status:** Approved ## Problem From 0af22a3a005b20eb5f2a4426a1bfc980115a709e Mon Sep 17 00:00:00 2001 From: Soumava2023 Date: Fri, 21 Aug 2026 21:30:57 -0400 Subject: [PATCH 11/13] feat: extract Python type-use relationships --- graphify/extractors/resolution.py | 156 +++++++++++++++++++++++++----- tests/test_extract.py | 136 ++++++++++++++++++++++++++ 2 files changed, 270 insertions(+), 22 deletions(-) diff --git a/graphify/extractors/resolution.py b/graphify/extractors/resolution.py index a75b450f8e..8ff75db198 100644 --- a/graphify/extractors/resolution.py +++ b/graphify/extractors/resolution.py @@ -1881,19 +1881,13 @@ def _resolve_cross_file_imports( per_file: list[dict], paths: list[Path], ) -> list[dict]: - """ - Two-pass import resolution: turn file-level imports into class-level edges. - - Pass 1 - build a global map: class/function name → node_id, per stem. - Pass 2 - for each `from .module import Name`, look up Name in the global - map and add a direct INFERRED edge from each class in the - importing file to the imported entity. - - This turns: - auth.py --imports_from--> models.py (obvious, filtered out) - Into: - DigestAuth --uses--> Response [INFERRED] (cross-file, interesting!) - BasicAuth --uses--> Request [INFERRED] + """Resolve source-backed Python imports at the symbol level. + + Pass one indexes project definitions by directory-qualified module stem. + Pass two attributes each imported-name occurrence to its top-level owning + class or function. Supported annotations emit deterministic ``uses_type`` + edges with aggregated roles; other body references retain the conservative + ``uses/INFERRED`` relationship. """ try: import tree_sitter_python as tspython @@ -1910,7 +1904,7 @@ def _resolve_cross_file_imports( # A secondary bare-stem index handles absolute imports where only the module # name is known — first writer wins when names collide (inherently ambiguous). stem_to_entities: dict[str, dict[str, str]] = {} - bare_to_qualified: dict[str, str] = {} + bare_to_qualified: dict[str, str | None] = {} for file_result in per_file: for node in file_result.get("nodes", []): src = node.get("source_file", "") @@ -1932,8 +1926,11 @@ def _resolve_cross_file_imports( and node.get("file_type") != "rationale" ): stem_to_entities.setdefault(fq_stem, {})[label] = nid - if src_path.stem not in bare_to_qualified: - bare_to_qualified[src_path.stem] = fq_stem + bare = src_path.stem + if bare not in bare_to_qualified: + bare_to_qualified[bare] = fq_stem + elif bare_to_qualified[bare] != fq_stem: + bare_to_qualified[bare] = None # Pass 2: for each file, find `from .X import A, B, C`, then attribute the # `uses` edge to the specific local symbol (class OR function) whose body @@ -1972,8 +1969,10 @@ def _resolve_cross_file_imports( # local_name -> target node id (local_name honours `import X as Y`, so a # reference to the alias in the body still attributes correctly). import_targets: dict[str, str] = {} - # referenced name -> {source symbol nid: first reference line} + # referenced name -> {source symbol nid: first body-reference line} ref_sources: dict[str, dict[str, int]] = {} + # referenced name -> source symbol nid -> roles + first annotation line + type_ref_sources: dict[str, dict[str, dict[str, Any]]] = {} def _text(n) -> str: return source[n.start_byte:n.end_byte].decode("utf-8", errors="replace") @@ -2028,7 +2027,87 @@ def resolve_import(node) -> None: if tgt_nid: import_targets[local_name] = tgt_nid - def visit(node, current_nid: str | None) -> None: + def _same_node(left, right) -> bool: + return ( + left is not None + and right is not None + and left.type == right.type + and left.start_byte == right.start_byte + and left.end_byte == right.end_byte + ) + + def _contains(outer, inner) -> bool: + return ( + outer is not None + and outer.start_byte <= inner.start_byte + and inner.end_byte <= outer.end_byte + ) + + def _annotation_role(ref_node, owner_node) -> str | None: + """Classify a reference only when it is inside a supported annotation.""" + cursor = ref_node + kind: str | None = None + anchor = None + while cursor.parent is not None and not _same_node(cursor, owner_node): + parent = cursor.parent + if parent.type in ("typed_parameter", "typed_default_parameter"): + type_node = parent.child_by_field_name("type") + if _contains(type_node, ref_node): + kind, anchor = "parameter", parent + break + if parent.type == "function_definition": + return_node = parent.child_by_field_name("return_type") + if _contains(return_node, ref_node): + kind, anchor = "return", parent + break + if parent.type == "assignment": + type_node = parent.child_by_field_name("type") + if _contains(type_node, ref_node): + kind, anchor = "field", parent + break + cursor = parent + + if kind is None or anchor is None: + return None + + if kind == "field": + scope = anchor.parent + while scope is not None and not _same_node(scope, owner_node): + if scope.type in ("class_definition", "function_definition"): + break + scope = scope.parent + if scope is None or scope.type != "class_definition": + return None + return "field" if _same_node(scope, owner_node) else "nested_field" + + annotation_function = anchor + while ( + annotation_function is not None + and annotation_function.type != "function_definition" + ): + annotation_function = annotation_function.parent + if annotation_function is None: + return None + if owner_node.type == "function_definition": + nested = not _same_node(annotation_function, owner_node) + else: + nested = False + scope = annotation_function.parent + while scope is not None and not _same_node(scope, owner_node): + if scope.type in ("class_definition", "function_definition"): + nested = True + scope = scope.parent + if scope is None: + return None + return f"nested_{kind}" if nested else kind + + def _record_type_ref(name: str, source_nid: str, role: str, line: int) -> None: + by_source = type_ref_sources.setdefault(name, {}) + evidence = by_source.setdefault(source_nid, {"roles": set(), "line": line}) + evidence["roles"].add(role) + evidence["line"] = min(evidence["line"], line) + + def visit(node, current_nid: str | None, owner_node=None) -> None: # Identifiers inside an import statement are the import itself, not a # real use — resolve the import here and don't descend into it. if node.type == "import_from_statement": @@ -2045,11 +2124,29 @@ def visit(node, current_nid: str | None) -> None: mapped = name_to_nid.get(_text(name_node)) if mapped is not None: current_nid = mapped - if node.type == "identifier" and current_nid is not None: - slot = ref_sources.setdefault(_text(node), {}) - slot.setdefault(current_nid, node.start_point[0] + 1) + owner_node = node + if current_nid is not None and owner_node is not None: + if node.type == "identifier": + name = _text(node) + role = _annotation_role(node, owner_node) + if role is None: + slot = ref_sources.setdefault(name, {}) + slot.setdefault(current_nid, node.start_point[0] + 1) + else: + _record_type_ref( + name, current_nid, role, node.start_point[0] + 1 + ) + elif node.type == "string": + role = _annotation_role(node, owner_node) + if role is not None: + # Tokenize only a string already proven to be an annotation; + # do not evaluate it as Python. + for name in re.findall(r"\b[A-Za-z_]\w*\b", _text(node)): + _record_type_ref( + name, current_nid, role, node.start_point[0] + 1 + ) for child in node.children: - visit(child, current_nid) + visit(child, current_nid, owner_node) visit(tree.root_node, None) @@ -2072,6 +2169,21 @@ def visit(node, current_nid: str | None) -> None: "source_location": f"L{line}", "weight": 0.8, }) + for src_nid, evidence in type_ref_sources.get(name, {}).items(): + if src_nid == tgt_nid: + continue + new_edges.append({ + "source": src_nid, + "target": tgt_nid, + "relation": "uses_type", + "context": "type_annotation", + "type_roles": sorted(evidence["roles"]), + "confidence": "EXTRACTED", + "confidence_score": 1.0, + "source_file": str_path, + "source_location": f"L{evidence['line']}", + "weight": 1.0, + }) return new_edges diff --git a/tests/test_extract.py b/tests/test_extract.py index c9790e4ab5..c440e7dd75 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -3879,6 +3879,142 @@ def _inferred_uses(result): } +def _type_uses(result): + """Every deterministic cross-file Python type-use edge.""" + return [e for e in result["edges"] if e.get("relation") == "uses_type"] + + +def test_cross_file_annotations_emit_extracted_roles(tmp_path): + (tmp_path / "models.py").write_text( + "class Payload:\n pass\n", encoding="utf-8" + ) + (tmp_path / "api.py").write_text( + "from models import Payload as P\n\n\n" + "class Envelope:\n" + " value: P\n\n\n" + "def convert(values: list[P | None]) -> \"P\":\n" + " return values[0]\n\n\n" + "def build(value: P) -> P:\n" + " return P()\n", + encoding="utf-8", + ) + + result = extract( + [tmp_path / "api.py", tmp_path / "models.py"], + cache_root=tmp_path, + ) + by_source = {edge["source"]: edge for edge in _type_uses(result)} + + envelope = by_source["api_envelope"] + assert envelope["target"] == "models_payload" + assert envelope["context"] == "type_annotation" + assert envelope["type_roles"] == ["field"] + assert envelope["confidence"] == "EXTRACTED" + assert envelope["confidence_score"] == 1.0 + assert envelope["source_file"] == "api.py" + assert envelope["source_location"] == "L5" + assert envelope["weight"] == 1.0 + assert envelope["_origin"] == "ast" + assert by_source["api_convert"]["target"] == "models_payload" + assert by_source["api_convert"]["type_roles"] == ["parameter", "return"] + assert by_source["api_convert"]["confidence"] == "EXTRACTED" + assert by_source["api_convert"]["confidence_score"] == 1.0 + assert by_source["api_build"]["type_roles"] == ["parameter", "return"] + inferred_pairs = { + (edge["source"], edge["target"]) + for edge in result["edges"] + if edge.get("relation") == "uses" and edge.get("confidence") == "INFERRED" + } + assert not inferred_pairs & { + ("api_envelope", "models_payload"), + ("api_convert", "models_payload"), + } + assert ("api_build", "models_payload") in inferred_pairs + + +def test_nested_annotations_record_nested_roles_on_the_owner(tmp_path): + (tmp_path / "models.py").write_text("class Debt:\n pass\n", encoding="utf-8") + (tmp_path / "order.py").write_text( + "from models import Debt\n\n\n" + "def order_custom():\n" + " def key(debt: Debt) -> Debt:\n" + " return debt\n" + " return key\n\n\n" + "class Holder:\n" + " class Inner:\n" + " debt: Debt\n", + encoding="utf-8", + ) + + result = extract( + [tmp_path / "order.py", tmp_path / "models.py"], + cache_root=tmp_path, + ) + by_source = {edge["source"]: edge for edge in _type_uses(result)} + + assert by_source["order_order_custom"]["type_roles"] == [ + "nested_parameter", + "nested_return", + ] + assert by_source["order_holder"]["type_roles"] == ["nested_field"] + + +def test_local_annotation_and_body_reference_keep_conservative_uses(tmp_path): + (tmp_path / "models.py").write_text("class Helper:\n pass\n", encoding="utf-8") + (tmp_path / "api.py").write_text( + "from models import Helper\n\n\n" + "def handler():\n" + " local: Helper = Helper()\n" + " return local\n", + encoding="utf-8", + ) + + result = extract( + [tmp_path / "api.py", tmp_path / "models.py"], + cache_root=tmp_path, + ) + + assert ("api_handler", "models_helper") in _inferred_uses(result) + assert not any(edge["source"] == "api_handler" for edge in _type_uses(result)) + + +def test_type_use_skips_ambiguous_and_external_targets(tmp_path): + for package in ("one", "two"): + folder = tmp_path / package + folder.mkdir() + (folder / "models.py").write_text( + "class Payload:\n pass\n", encoding="utf-8" + ) + api = tmp_path / "api.py" + api.write_text( + "from models import Payload\n" + "from pathlib import Path\n\n\n" + "def load(value: Payload, path: Path) -> Payload:\n" + " return value\n", + encoding="utf-8", + ) + star = tmp_path / "star.py" + star.write_text( + "from one.models import *\n\n\n" + "def load(value: Payload) -> Payload:\n" + " return value\n", + encoding="utf-8", + ) + + result = extract( + [ + api, + star, + tmp_path / "one" / "models.py", + tmp_path / "two" / "models.py", + ], + cache_root=tmp_path, + ) + + assert not any(edge["source"] == "api_load" for edge in _type_uses(result)) + assert not any(edge["source"] == "star_load" for edge in _type_uses(result)) + + def test_inferred_uses_edge_attributes_to_the_referencing_symbol(tmp_path): """A cross-file INFERRED `uses` edge binds to the symbol that actually references the import — a function is a valid source and a co-located class From 063b0d205c31a8c263a7bb23ce63d6af863bc625 Mon Sep 17 00:00:00 2001 From: Soumava2023 Date: Fri, 21 Aug 2026 21:33:48 -0400 Subject: [PATCH 12/13] fix: resolve type uses against incremental context --- graphify/extract.py | 26 +++++++------ graphify/extractors/resolution.py | 63 +++++++++++++++++++------------ tests/test_incremental.py | 43 +++++++++++++++++++++ 3 files changed, 95 insertions(+), 37 deletions(-) diff --git a/graphify/extract.py b/graphify/extract.py index 1596970501..9bf81790fa 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -5485,8 +5485,8 @@ def extract( Two-pass process: 1. Per-file structural extraction (classes, functions, imports) - 2. Cross-file import resolution: turns file-level imports into - class-level INFERRED edges (DigestAuth --uses--> Response) + 2. Cross-file import resolution: emits source-backed runtime relationships + and deterministic Python ``uses_type`` edges. Args: paths: files to extract from @@ -5504,15 +5504,12 @@ def extract( value of GRAPHIFY_MAX_WORKERS if set), bounded by len(uncached_work). resolution_context_nodes: read-only AST nodes from files that are NOT being extracted this run (an incremental rebuild's unchanged - corpus, #2406). They extend the cross-file resolution indexes — - the shared direct-call pass's label/file indexes, the - indirect_call callable guard (via the persisted `_callable` / - `_callable_class` markers, #2438), and the member-call resolvers - run by `run_language_resolvers` (#2437) — so a changed caller can - still bind `foo()`, `obj.method()`, or `submit(handler)` to an - unchanged callee. They are never parsed, mutated, or returned; - raw_calls come only from `paths`, so only edges sourced by the - re-extracted files are emitted. + corpus, #2406). They extend the Python import/type-use target index, + the shared direct-call label/file indexes, the indirect_call + callable guard (via persisted `_callable` / `_callable_class` + markers, #2438), and member-call resolvers run by + `run_language_resolvers` (#2437). They are never parsed, mutated, + or returned; only edges sourced by re-extracted files are emitted. resolution_context_edges: the `contains`/`method` edges of the same unchanged corpus (#2437). The member-call resolvers walk these to map a receiver type to the single class owning the called method; @@ -6272,7 +6269,12 @@ def _learn(e: dict) -> None: if py_paths: py_results = [r for r, p in zip(per_file, paths) if p.suffix == ".py"] try: - cross_file_edges = _resolve_cross_file_imports(py_results, py_paths) + cross_file_edges = _resolve_cross_file_imports( + py_results, + py_paths, + resolution_context_nodes=resolution_context_nodes, + root=root, + ) all_edges.extend(cross_file_edges) except Exception as exc: import logging diff --git a/graphify/extractors/resolution.py b/graphify/extractors/resolution.py index 8ff75db198..96a0d6abd9 100644 --- a/graphify/extractors/resolution.py +++ b/graphify/extractors/resolution.py @@ -1880,6 +1880,9 @@ def _augment_symbol_resolution_edges( def _resolve_cross_file_imports( per_file: list[dict], paths: list[Path], + *, + resolution_context_nodes: list[dict] | None = None, + root: Path | None = None, ) -> list[dict]: """Resolve source-backed Python imports at the symbol level. @@ -1902,35 +1905,45 @@ class or function. Supported annotations emit deterministic ``uses_type`` # Keyed by directory-qualified stem (e.g. "auth_models") to avoid collisions # when multiple files share the same filename in different directories. # A secondary bare-stem index handles absolute imports where only the module - # name is known — first writer wins when names collide (inherently ambiguous). + # name is known. Collisions are marked ambiguous rather than bound arbitrarily. stem_to_entities: dict[str, dict[str, str]] = {} bare_to_qualified: dict[str, str | None] = {} + + def index_definition(node: dict, *, overwrite: bool) -> None: + src = node.get("source_file", "") + if not src: + return + src_path = Path(src) + if root is not None and not src_path.is_absolute(): + src_path = Path(root) / src_path + if src_path.suffix not in (".py", ".pyi"): + return + fq_stem = _file_stem(src_path) + label = node.get("label", "") + nid = node.get("id", "") + if ( + not label + or label.endswith((")", ".py", ".pyi")) + or "_" in label[:1] + or node.get("file_type") == "rationale" + ): + return + entities = stem_to_entities.setdefault(fq_stem, {}) + if overwrite: + entities[label] = nid + else: + entities.setdefault(label, nid) + bare = src_path.stem + if bare not in bare_to_qualified: + bare_to_qualified[bare] = fq_stem + elif bare_to_qualified[bare] != fq_stem: + bare_to_qualified[bare] = None + for file_result in per_file: for node in file_result.get("nodes", []): - src = node.get("source_file", "") - if not src: - continue - src_path = Path(src) - fq_stem = _file_stem(src_path) - label = node.get("label", "") - nid = node.get("id", "") - # Index class-level entities only. Function/method labels end in "()" - # so are excluded by the `endswith(")")` filter; file nodes end in ".py"; - # private/internal labels start with "_"; rationale nodes carry - # file_type=="rationale" and must never participate in cross-file - # import resolution (#563). - if ( - label - and not label.endswith((")", ".py")) - and "_" not in label[:1] - and node.get("file_type") != "rationale" - ): - stem_to_entities.setdefault(fq_stem, {})[label] = nid - bare = src_path.stem - if bare not in bare_to_qualified: - bare_to_qualified[bare] = fq_stem - elif bare_to_qualified[bare] != fq_stem: - bare_to_qualified[bare] = None + index_definition(node, overwrite=True) + for node in resolution_context_nodes or []: + index_definition(node, overwrite=False) # Pass 2: for each file, find `from .X import A, B, C`, then attribute the # `uses` edge to the specific local symbol (class OR function) whose body diff --git a/tests/test_incremental.py b/tests/test_incremental.py index 460db23ba1..378e68d38f 100644 --- a/tests/test_incremental.py +++ b/tests/test_incremental.py @@ -211,6 +211,49 @@ def test_extract_no_cluster_incremental_code_only_preserves_doc_nodes(tmp_path): assert any("beta" in i for i in after_by_id), sorted(after_by_id) +def test_incremental_python_type_use_matches_full_extraction(tmp_path): + from graphify.extract import extract + + pkg = tmp_path / "pkg" + pkg.mkdir() + model = pkg / "models.py" + model.write_text("class Debt:\n pass\n", encoding="utf-8") + planner = pkg / "planner.py" + planner.write_text( + "from .models import Debt\n\n\n" + "def prioritize(debts: list[Debt]) -> Debt:\n" + " return debts[0]\n", + encoding="utf-8", + ) + + full = extract( + [planner, model], cache_root=tmp_path, root=tmp_path, parallel=False + ) + incremental = extract( + [planner], + cache_root=tmp_path, + root=tmp_path, + parallel=False, + resolution_context_nodes=full["nodes"], + resolution_context_edges=full["edges"], + ) + + def type_edge(result): + return next( + edge + for edge in result["edges"] + if edge.get("relation") == "uses_type" + and edge.get("source") == "pkg_planner_prioritize" + ) + + full_edge = type_edge(full) + incremental_edge = type_edge(incremental) + assert incremental_edge["target"] == full_edge["target"] == "pkg_models_debt" + assert incremental_edge["type_roles"] == ["parameter", "return"] + assert incremental_edge["confidence"] == "EXTRACTED" + assert incremental_edge["confidence_score"] == 1.0 + + def test_incremental_python_relative_import_target_canonicalizes(tmp_path): """#2213 (defect 1, shared root with #2211): a Python relative import's imports_from edge must stamp target_file so the #2169 remap canonicalizes From 3fe81a03c493b21ee092b849f6363f2756bb44eb Mon Sep 17 00:00:00 2001 From: Soumava2023 Date: Fri, 21 Aug 2026 21:35:18 -0400 Subject: [PATCH 13/13] fix: preserve runtime edges over type uses --- graphify/build.py | 4 ++- tests/test_relation_collapse_precedence.py | 34 +++++++++++++++++++++- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/graphify/build.py b/graphify/build.py index 8efdcbd6e2..4573298f4a 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -60,7 +60,9 @@ def _is_ast_tier(item: dict) -> bool: # every relation: ranking `contains` against `calls` would be inventing a # cross-axis judgement, whereas "specific beats generic" is the only comparison # this collapse actually needs. -_GENERIC_RELATIONS: frozenset[str] = frozenset({"references", "uses", "mentions"}) +_GENERIC_RELATIONS: frozenset[str] = frozenset( + {"references", "uses", "uses_type", "mentions"} +) # Language interop families, keyed by extension, for the cross-language phantom-edge # guard in the edge loop below. Families group by REAL interop (JS/TS share a module diff --git a/tests/test_relation_collapse_precedence.py b/tests/test_relation_collapse_precedence.py index 2ab15104aa..eeb2b15373 100644 --- a/tests/test_relation_collapse_precedence.py +++ b/tests/test_relation_collapse_precedence.py @@ -25,7 +25,7 @@ SPECIFIC = ["calls", "imports", "imports_from", "inherits", "implements", "method", "indirect_call", "re_exports", "contains"] -GENERIC = ["references", "uses", "mentions"] +GENERIC = ["references", "uses", "uses_type", "mentions"] def _extraction(edges): @@ -48,6 +48,38 @@ def _relation(G): return edge_data(G, "a", "b").get("relation") +def test_calls_beats_uses_type_and_keeps_runtime_metadata(): + for edges in ( + [ + _edge("calls", source_location="L9", weight=1.0, confidence_score=1.0), + _edge( + "uses_type", + source_location="L4", + context="type_annotation", + type_roles=["return"], + weight=1.0, + confidence_score=1.0, + ), + ], + [ + _edge( + "uses_type", + source_location="L4", + context="type_annotation", + type_roles=["return"], + weight=1.0, + confidence_score=1.0, + ), + _edge("calls", source_location="L9", weight=1.0, confidence_score=1.0), + ], + ): + graph = build_from_json(_extraction(edges)) + data = edge_data(graph, "a", "b") + assert data["relation"] == "calls" + assert data["source_location"] == "L9" + assert "type_roles" not in data + + # --------------------------------------------------------------------------- # The bug # ---------------------------------------------------------------------------