Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 15 additions & 6 deletions scripts/incremental-benchmark.ts
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,15 @@ const PROBE_FILE = path.join(root, 'src', 'domain', 'queries.ts');
// CI-amplified false regressions on sub-30ms metrics like No-op rebuild.
const WARMUP_RUNS = 2;

// Resolution-benchmark fixtures live under the repo root and get pulled into
// every self-build sweep. They are hand-annotated test corpora — not real
// codegraph code — and a single heavy grammar (e.g. tree-sitter-verilog) can
// add hundreds of milliseconds to fullBuildMs purely from fixture parsing
// (#1112). Exclude them so adding native support for a new language doesn't
// silently inflate the incremental-benchmark numbers.
const BENCH_EXCLUDE = ['tests/benchmarks/resolution/fixtures/**'];
const BUILD_OPTS = { engine, exclude: BENCH_EXCLUDE };

function median(arr) {
const sorted = [...arr].sort((a, b) => a - b);
const mid = Math.floor(sorted.length / 2);
Expand All @@ -194,7 +203,7 @@ const fullTimings = [];
for (let i = 0; i < RUNS; i++) {
if (fs.existsSync(dbPath)) fs.unlinkSync(dbPath);
const start = performance.now();
await buildGraph(root, { engine, incremental: false });
await buildGraph(root, { ...BUILD_OPTS, incremental: false });
fullTimings.push(performance.now() - start);
}
const fullBuildMs = Math.round(median(fullTimings));
Expand All @@ -203,12 +212,12 @@ const fullBuildMs = Math.round(median(fullTimings));
let noopRebuildMs = null;
try {
for (let i = 0; i < WARMUP_RUNS; i++) {
await buildGraph(root, { engine, incremental: true });
await buildGraph(root, { ...BUILD_OPTS, incremental: true });
}
const noopTimings = [];
for (let i = 0; i < RUNS; i++) {
const start = performance.now();
await buildGraph(root, { engine, incremental: true });
await buildGraph(root, { ...BUILD_OPTS, incremental: true });
noopTimings.push(performance.now() - start);
}
noopRebuildMs = Math.round(median(noopTimings));
Expand All @@ -223,13 +232,13 @@ let oneFilePhases = null;
try {
for (let i = 0; i < WARMUP_RUNS; i++) {
fs.writeFileSync(PROBE_FILE, original + `\n// warmup-${i}\n`);
await buildGraph(root, { engine, incremental: true });
await buildGraph(root, { ...BUILD_OPTS, incremental: true });
}
const oneFileRuns = [];
for (let i = 0; i < RUNS; i++) {
fs.writeFileSync(PROBE_FILE, original + `\n// probe-${i}\n`);
const start = performance.now();
const res = await buildGraph(root, { engine, incremental: true });
const res = await buildGraph(root, { ...BUILD_OPTS, incremental: true });
oneFileRuns.push({ ms: performance.now() - start, phases: res?.phases || null });
}
oneFileRuns.sort((a, b) => a.ms - b.ms);
Expand All @@ -241,7 +250,7 @@ try {
} finally {
fs.writeFileSync(PROBE_FILE, original);
try {
await buildGraph(root, { engine, incremental: true });
await buildGraph(root, { ...BUILD_OPTS, incremental: true });
} catch {
// Cleanup rebuild failed — probe file is already restored, move on
}
Expand Down
3 changes: 3 additions & 0 deletions src/domain/graph/builder/pipeline.ts
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,9 @@ function setupPipeline(ctx: PipelineContext): void {
initSchema(ctx.db);

ctx.config = loadConfig(ctx.rootDir);
if (ctx.opts.exclude && ctx.opts.exclude.length > 0) {
ctx.config = { ...ctx.config, exclude: [...ctx.config.exclude, ...ctx.opts.exclude] };
}
ctx.incremental =
ctx.opts.incremental !== false && ctx.config.build && ctx.config.build.incremental !== false;

Expand Down
6 changes: 6 additions & 0 deletions src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1064,6 +1064,12 @@ export interface BuildGraphOpts {
cfg?: boolean;
scope?: string[];
skipRegistry?: boolean;
/**
* Extra exclude globs appended to `config.exclude`. Lets benchmark scripts
* skip fixture directories that bloat self-build timings without permanently
* affecting `.codegraphrc.json` for normal users.
*/
exclude?: string[];
}

/** Build timing result from buildGraph. */
Expand Down
13 changes: 0 additions & 13 deletions tests/benchmarks/regression-guard.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -176,18 +176,6 @@ const SKIP_VERSIONS = new Set(['3.8.0']);
* absolute delta 10.4ms exactly at the MIN_ABSOLUTE_DELTA floor. Exempt
* this release; remove once 3.11.0+ data confirms stabilization.
*
* - 3.10.0:Full build — adding native Verilog support (#1107) pulled the
* 4 `.v` resolution-benchmark fixtures into the corpus the incremental
* benchmark sweeps (it runs against the repo root). tree-sitter-verilog
* is a large grammar (SystemVerilog is one of the heaviest in the
* tree-sitter ecosystem) so each file costs noticeably more than the
* other fixture languages. Local measurement: 1959 → 2809 (+43%, run
* 25716010487). The cost is real and structural — not a regression in
* shared code paths. Resolution: either exclude `tests/benchmarks/
* resolution/fixtures/verilog/**` from the benchmark sweep or accept the
* one-time bump as the cost of supporting Verilog. Tracked separately;
* exempt this release.
*
* - 3.10.0:Query time — cumulative effect of adding two native extractors
* (Solidity #1100 + R #1102) in quick succession. Neither tripped the
* threshold individually (Solidity PR's Query time stayed at 49ms, R PR
Expand Down Expand Up @@ -230,7 +218,6 @@ const KNOWN_REGRESSIONS = new Set([
'3.10.0:fnDeps depth 1',
'3.10.0:fnDeps depth 3',
'3.10.0:fnDeps depth 5',
'3.10.0:Full build',
'3.10.0:Query time',
]);

Expand Down
Loading