diff --git a/scripts/incremental-benchmark.ts b/scripts/incremental-benchmark.ts index a37f132c..f3d19a7f 100644 --- a/scripts/incremental-benchmark.ts +++ b/scripts/incremental-benchmark.ts @@ -181,6 +181,15 @@ const PROBE_FILE = path.join(root, 'src', 'domain', 'queries.ts'); // CI-amplified false regressions on sub-30ms metrics like No-op rebuild. const WARMUP_RUNS = 2; +// Resolution-benchmark fixtures live under the repo root and get pulled into +// every self-build sweep. They are hand-annotated test corpora — not real +// codegraph code — and a single heavy grammar (e.g. tree-sitter-verilog) can +// add hundreds of milliseconds to fullBuildMs purely from fixture parsing +// (#1112). Exclude them so adding native support for a new language doesn't +// silently inflate the incremental-benchmark numbers. +const BENCH_EXCLUDE = ['tests/benchmarks/resolution/fixtures/**']; +const BUILD_OPTS = { engine, exclude: BENCH_EXCLUDE }; + function median(arr) { const sorted = [...arr].sort((a, b) => a - b); const mid = Math.floor(sorted.length / 2); @@ -194,7 +203,7 @@ const fullTimings = []; for (let i = 0; i < RUNS; i++) { if (fs.existsSync(dbPath)) fs.unlinkSync(dbPath); const start = performance.now(); - await buildGraph(root, { engine, incremental: false }); + await buildGraph(root, { ...BUILD_OPTS, incremental: false }); fullTimings.push(performance.now() - start); } const fullBuildMs = Math.round(median(fullTimings)); @@ -203,12 +212,12 @@ const fullBuildMs = Math.round(median(fullTimings)); let noopRebuildMs = null; try { for (let i = 0; i < WARMUP_RUNS; i++) { - await buildGraph(root, { engine, incremental: true }); + await buildGraph(root, { ...BUILD_OPTS, incremental: true }); } const noopTimings = []; for (let i = 0; i < RUNS; i++) { const start = performance.now(); - await buildGraph(root, { engine, incremental: true }); + await buildGraph(root, { ...BUILD_OPTS, incremental: true }); noopTimings.push(performance.now() - start); } noopRebuildMs = Math.round(median(noopTimings)); @@ -223,13 +232,13 @@ let oneFilePhases = null; try { for (let i = 0; i < WARMUP_RUNS; i++) { fs.writeFileSync(PROBE_FILE, original + `\n// warmup-${i}\n`); - await buildGraph(root, { engine, incremental: true }); + await buildGraph(root, { ...BUILD_OPTS, incremental: true }); } const oneFileRuns = []; for (let i = 0; i < RUNS; i++) { fs.writeFileSync(PROBE_FILE, original + `\n// probe-${i}\n`); const start = performance.now(); - const res = await buildGraph(root, { engine, incremental: true }); + const res = await buildGraph(root, { ...BUILD_OPTS, incremental: true }); oneFileRuns.push({ ms: performance.now() - start, phases: res?.phases || null }); } oneFileRuns.sort((a, b) => a.ms - b.ms); @@ -241,7 +250,7 @@ try { } finally { fs.writeFileSync(PROBE_FILE, original); try { - await buildGraph(root, { engine, incremental: true }); + await buildGraph(root, { ...BUILD_OPTS, incremental: true }); } catch { // Cleanup rebuild failed — probe file is already restored, move on } diff --git a/src/domain/graph/builder/pipeline.ts b/src/domain/graph/builder/pipeline.ts index 72a8ca20..c83f72d9 100644 --- a/src/domain/graph/builder/pipeline.ts +++ b/src/domain/graph/builder/pipeline.ts @@ -185,6 +185,9 @@ function setupPipeline(ctx: PipelineContext): void { initSchema(ctx.db); ctx.config = loadConfig(ctx.rootDir); + if (ctx.opts.exclude && ctx.opts.exclude.length > 0) { + ctx.config = { ...ctx.config, exclude: [...ctx.config.exclude, ...ctx.opts.exclude] }; + } ctx.incremental = ctx.opts.incremental !== false && ctx.config.build && ctx.config.build.incremental !== false; diff --git a/src/types.ts b/src/types.ts index 08eb9431..6d8f31b7 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1064,6 +1064,12 @@ export interface BuildGraphOpts { cfg?: boolean; scope?: string[]; skipRegistry?: boolean; + /** + * Extra exclude globs appended to `config.exclude`. Lets benchmark scripts + * skip fixture directories that bloat self-build timings without permanently + * affecting `.codegraphrc.json` for normal users. + */ + exclude?: string[]; } /** Build timing result from buildGraph. */ diff --git a/tests/benchmarks/regression-guard.test.ts b/tests/benchmarks/regression-guard.test.ts index 4125325e..1694c0d5 100644 --- a/tests/benchmarks/regression-guard.test.ts +++ b/tests/benchmarks/regression-guard.test.ts @@ -176,18 +176,6 @@ const SKIP_VERSIONS = new Set(['3.8.0']); * absolute delta 10.4ms exactly at the MIN_ABSOLUTE_DELTA floor. Exempt * this release; remove once 3.11.0+ data confirms stabilization. * - * - 3.10.0:Full build — adding native Verilog support (#1107) pulled the - * 4 `.v` resolution-benchmark fixtures into the corpus the incremental - * benchmark sweeps (it runs against the repo root). tree-sitter-verilog - * is a large grammar (SystemVerilog is one of the heaviest in the - * tree-sitter ecosystem) so each file costs noticeably more than the - * other fixture languages. Local measurement: 1959 → 2809 (+43%, run - * 25716010487). The cost is real and structural — not a regression in - * shared code paths. Resolution: either exclude `tests/benchmarks/ - * resolution/fixtures/verilog/**` from the benchmark sweep or accept the - * one-time bump as the cost of supporting Verilog. Tracked separately; - * exempt this release. - * * - 3.10.0:Query time — cumulative effect of adding two native extractors * (Solidity #1100 + R #1102) in quick succession. Neither tripped the * threshold individually (Solidity PR's Query time stayed at 49ms, R PR @@ -230,7 +218,6 @@ const KNOWN_REGRESSIONS = new Set([ '3.10.0:fnDeps depth 1', '3.10.0:fnDeps depth 3', '3.10.0:fnDeps depth 5', - '3.10.0:Full build', '3.10.0:Query time', ]);