diff --git a/graphify/cli.py b/graphify/cli.py index 02e55b944..1ebf56879 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -3129,6 +3129,16 @@ def _parse_float(name: str, raw: str) -> float: # --force: full scan, not the manifest-gated incremental diff — a warm # unchanged tree would otherwise dispatch zero files (#1894). incremental_mode = incremental_mode and not force + # #2923: --force --code-only must NOT drop the existing semantic layer. + # It still performs a full AST re-scan (so --force bypasses the + # manifest cache), but the semantic pass is itself skipped entirely, so + # the existing graph is merged instead of replaced. + preserve_semantic = force and code_only and existing_graph_path.exists() + if preserve_semantic: + print( + "[graphify extract] --force --code-only: full AST re-scan, " + "existing semantic layer preserved (no semantic pass this run)" + ) if force: print("[graphify extract] --force: full re-scan, semantic cache reads skipped") elif incremental_mode and not manifest_path.exists(): @@ -3213,6 +3223,16 @@ def _parse_float(name: str, raw: str) -> float: excluded_files = [] graph_stale_sources = [] unchanged_total = 0 + if preserve_semantic: + # A full scan re-extracts every code file, but doc/paper/image + # nodes are not re-dispatched, so the existing graph must still + # be merged. Compute stale sources (deleted/excluded files) so + # the merge can prune them while carrying the rest forward. + _seen_files = {f for _fl in files_by_type.values() for f in _fl} + _seen_files.update(detection.get("unclassified", [])) + graph_stale_sources = _stale_graph_sources( + existing_graph_path, target, _seen_files, detection=detection + ) semantic_files = doc_files + paper_files + image_files # --code-only: index code (pure local AST, no key) and skip the semantic @@ -3794,7 +3814,7 @@ def _invalidate_file_manifest_for_db_graph() -> None: existing_graph_node_count as _existing_graph_node_count, ) if ( - incremental_mode + (incremental_mode or preserve_semantic) and not code_files and not semantic_files and not deleted_files @@ -3828,7 +3848,7 @@ def _invalidate_file_manifest_for_db_graph() -> None: stages.total() sys.exit(0) - if incremental_mode: + if incremental_mode or preserve_semantic: # #2169: this raw path used to write ONLY this run's extraction # over graph.json — on an incremental run that is just the # changed files, silently dropping every node/edge owned by an @@ -3956,7 +3976,7 @@ def _invalidate_file_manifest_for_db_graph() -> None: from graphify.export import to_json as _to_json from graphify.analyze import god_nodes as _god_nodes, surprising_connections as _surprising dedup_backend = backend if dedup_llm else None - if incremental_mode: + if incremental_mode or preserve_semantic: # Prune everything the current scan no longer covers: genuinely # deleted manifest rows, excluded-but-alive manifest rows (#1908), # and the graph's own stale sources — which catches files that diff --git a/tests/test_extract_code_only_cli.py b/tests/test_extract_code_only_cli.py index 93fec48d3..1de97b2c7 100644 --- a/tests/test_extract_code_only_cli.py +++ b/tests/test_extract_code_only_cli.py @@ -258,3 +258,169 @@ def test_extract_names_skipped_sensitive_files(tmp_path): out = r.stdout + r.stderr assert "skipped as potentially sensitive" in out assert "github_token.txt" in out, "the skipped filename must be surfaced (#2106)" + + +def test_code_only_force_preserves_existing_semantic_layer(tmp_path): + """#2923 regression: --code-only --force must not drop the existing semantic + layer. The AST pass is fully replaced (full re-scan, semantic cache reads + skipped) but the semantic pass is itself skipped, so doc/paper/image nodes + from graph.json must be carried forward. Before the fix this combination + silently rewrote graph.json with only the AST tier, losing every semantic + node and every hyperedge connected to one. + """ + repo = _mixed_repo(tmp_path) + out = repo / "graphify-out" + out.mkdir() + graph = out / "graph.json" + # Seed a graph.json as if a prior full extract with an LLM backend had run: + # 2 AST nodes from app.py + 4 SEMANTIC nodes from README.md/NOTES.txt. + graph.write_text(json.dumps({ + "nodes": [ + {"id": "app_py", "label": "app.py", "type": "file", + "source_file": "app.py", "origin": "AST"}, + {"id": "app_hello", "label": "hello()", "type": "function", + "source_file": "app.py", "origin": "AST"}, + {"id": "readme_md", "label": "readme.md", "type": "file", + "source_file": "README.md", "origin": "SEMANTIC"}, + {"id": "readme_design", "label": "Design", "type": "concept", + "source_file": "README.md", "origin": "SEMANTIC"}, + {"id": "notes_txt", "label": "NOTES.txt", "type": "file", + "source_file": "NOTES.txt", "origin": "SEMANTIC"}, + {"id": "notes_architecture", "label": "Architecture", "type": "concept", + "source_file": "NOTES.txt", "origin": "SEMANTIC"}, + ], + "edges": [ + {"id": "e1", "source": "app_py", "target": "app_hello", + "relation": "contains", "source_file": "app.py"}, + {"id": "e2", "source": "readme_md", "target": "readme_design", + "relation": "concept_about", "source_file": "README.md"}, + {"id": "e3", "source": "notes_txt", "target": "notes_architecture", + "relation": "concept_about", "source_file": "NOTES.txt"}, + ], + "hyperedges": [], + "input_tokens": 0, + "output_tokens": 0, + })) + + r = _run(repo, "--code-only", "--force", "--no-cluster") + assert r.returncode == 0, r.stderr + + out_graph = json.loads(graph.read_text()) + semantic_labels = {n["label"] for n in out_graph["nodes"] + if n.get("origin") == "SEMANTIC"} + semantic_source_files = { + Path(str(n["source_file"])).name.lower() + for n in out_graph["nodes"] + if n.get("origin") == "SEMANTIC" + } + # Every seeded semantic node must survive. The AST pass may add new nodes + # (or relabel existing ones — e.g. hello vs hello()) but it must not + # silently drop the semantic tier. + assert {"readme.md", "notes.txt"}.issubset(semantic_source_files), ( + "code-only --force erased the existing semantic layer (#2923); " + f"semantic nodes remaining: {semantic_labels}" + ) + # Hyperedges are also semantic tier; the seeded graph had none but the + # AST re-extract must not have invented any non-semantic work, and the + # surviving edges list must not have been wholesale replaced. + assert "edges" in out_graph, "graph.json must still have an edges key" + # And the user-visible console line must explain why a semantic-layer- + # preserving branch fired. + assert "existing semantic layer preserved" in r.stdout + r.stderr, ( + "the --force --code-only print must announce the semantic-preserving branch" + ) + + +def test_code_only_force_prunes_removed_semantic_files(tmp_path): + """#2923 follow-up: --code-only --force preserves surviving semantic nodes + but must still prune semantic nodes for files that have been removed from + disk (the doc/paper/image tier cannot outlive the corpus it indexes). + """ + repo = _mixed_repo(tmp_path) + out = repo / "graphify-out" + out.mkdir() + graph = out / "graph.json" + graph.write_text(json.dumps({ + "nodes": [ + {"id": "app_py", "label": "app.py", "type": "file", + "source_file": "app.py", "origin": "AST"}, + {"id": "app_hello", "label": "hello()", "type": "function", + "source_file": "app.py", "origin": "AST"}, + {"id": "notes_txt", "label": "NOTES.txt", "type": "file", + "source_file": "NOTES.txt", "origin": "SEMANTIC"}, + {"id": "notes_architecture", "label": "Architecture", "type": "concept", + "source_file": "NOTES.txt", "origin": "SEMANTIC"}, + ], + "edges": [], + "hyperedges": [], + "input_tokens": 0, + "output_tokens": 0, + })) + + # Delete NOTES.txt between seed and re-run. The merge's graph_stale_sources + # path must drop its semantic nodes because the file no longer exists. + (repo / "NOTES.txt").unlink() + + r = _run(repo, "--code-only", "--force", "--no-cluster") + assert r.returncode == 0, r.stderr + out_graph = json.loads(graph.read_text()) + remaining_sources = { + Path(n["source_file"]).name + for n in out_graph["nodes"] + if n.get("origin") == "SEMANTIC" + } + assert "NOTES.txt" not in remaining_sources, ( + "NOTES.txt was deleted from disk; its semantic nodes must be pruned " + "(#2923 follow-up)" + ) + + +def test_code_only_force_rescans_unchanged_code_with_manifest_and_preserves_seeded_semantic_nodes(tmp_path): + """#2923: --force --code-only must perform a full AST re-scan even when the + manifest reports no code changes. Without this, a warm unchanged tree causes + the old fix to dispatch zero code files, so the AST tier is not rebuilt and + the existing graph is only merged unchanged. + """ + repo = _mixed_repo(tmp_path) + out = repo / "graphify-out" + out.mkdir() + + # Initial code-only extract writes a manifest and AST-only graph. + r1 = _run(repo, "--code-only", "--no-cluster") + assert r1.returncode == 0, r1.stderr + graph_path = out / "graph.json" + manifest_path = out / "manifest.json" + assert manifest_path.exists(), "code-only run must write a manifest" + g = json.loads(graph_path.read_text()) + assert any(n.get("label") == "hello()" for n in g["nodes"]) + + # Seed a semantic layer as if a prior full extract had produced it, then + # remove an unchanged AST node to verify the full re-scan restores it. + g["nodes"].extend([ + {"id": "readme_md", "label": "README.md", "type": "file", + "source_file": "README.md", "origin": "SEMANTIC"}, + {"id": "readme_design", "label": "Design", "type": "concept", + "source_file": "README.md", "origin": "SEMANTIC"}, + {"id": "notes_txt", "label": "NOTES.txt", "type": "file", + "source_file": "NOTES.txt", "origin": "SEMANTIC"}, + {"id": "notes_architecture", "label": "Architecture", "type": "concept", + "source_file": "NOTES.txt", "origin": "SEMANTIC"}, + ]) + g["nodes"] = [n for n in g["nodes"] if n.get("label") != "hello()"] + graph_path.write_text(json.dumps(g)) + + r2 = _run(repo, "--code-only", "--force", "--no-cluster") + assert r2.returncode == 0, r2.stderr + out_graph = json.loads(graph_path.read_text()) + assert any(n.get("label") == "hello()" for n in out_graph["nodes"]), ( + "--force --code-only must re-extract unchanged code and restore the AST node" + ) + semantic_labels = {n["label"] for n in out_graph["nodes"] + if n.get("origin") == "SEMANTIC"} + assert semantic_labels >= {"README.md", "Design", "NOTES.txt", "Architecture"}, ( + "seeded semantic layer must survive the full AST re-scan: " + f"{semantic_labels}" + ) + assert "existing semantic layer preserved" in r2.stdout + r2.stderr, ( + "the --force --code-only print must announce the semantic-preserving branch" + )