Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,20 @@ rules below are the parts that were worth keeping.
enforce a path convention, so update the explicit mapping when a suite is added or its doc moves.
Groups without a matching explainer render no docs link. Do not guess a path or link to an
unrelated general guide.
- **Every group kind has a summary, by default.** [`web/lib/summary.ts`](web/lib/summary.ts)
has no allowlist gate: a suite that lands in one of the five fact tables gets a rollup card
from its first ingest. The three timing families (query, random access, vector search) rank
through one `rankSeries` model, so a new suite in any of them needs no summary code at all.
`collectGroupSummary`'s exhaustive switch makes a missing arm for a sixth fact table a compile
error rather than a silently blank card. Do NOT reintroduce a per-dataset allowlist; the v2-era
one is what left `spatialbench`, `fineweb`, `gharchive`, `appian`, `public-bi`,
`clickbench-sorted`, and every vector-search group with no card.
- **A summary ranks the whole group, never one chart.** Random access is the cautionary case:
the producer emits `dataset` as `{dataset}/{pattern}`, so the group holds ~nine charts, and the
old summary published the alphabetically first chart's raw times under the group-wide title
"Random Access Performance" — reporting `lance` at 352us when it is over 1ms on most of the
other charts. Rank across every bucket in the group, impute the missing-series penalty where a
series skipped one, and report `measured`/`total` so a partially covered series is legible.
- **Don't write a server-side classifier for live ingest.** The emitter produces structured
records directly. Classifying loose name strings at read time was the v2-era weakness every
later generation existed to escape; it belongs nowhere in the live pipeline.
Expand Down
2 changes: 1 addition & 1 deletion web/components/GroupSection.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ const RANDOM_ACCESS: Group = {
summary: {
type: 'randomAccess',
title: 'Random Access Performance',
rankings: [{ name: 'vortex', time: 1_500_000, ratio: 1 }],
rankings: [{ name: 'vortex', score: 1, totalRuntime: 1_500_000, measured: 1, total: 1 }],
explanation: 'lower is better',
},
description: 'Tests selecting arbitrary row indices on NVMe',
Expand Down
52 changes: 46 additions & 6 deletions web/components/SummaryCard.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,15 @@ describe('SummaryCard', () => {
expect(render(undefined)).toBe('');
});

it('renders a randomAccess card with ranks, ns times, and ratios', () => {
it('renders a randomAccess card with ranks, scores, and total runtimes', () => {
const html = render({
type: 'randomAccess',
title: 'Random Access Performance',
rankings: [
{ name: 'vortex', time: 1_500_000, ratio: 1 },
{ name: 'parquet', time: 3_000_000, ratio: 2 },
{ name: 'vortex', score: 1, totalRuntime: 1_500_000, measured: 2, total: 2 },
{ name: 'parquet', score: 2, totalRuntime: 3_000_000, measured: 2, total: 2 },
],
explanation: 'Random access time | Ratio to fastest (lower is better)',
explanation: 'Geomean of take time ratio to fastest across every chart (lower is better)',
});
expect(html).toContain('class="benchmark-scores-summary"');
expect(html).toContain('<h3 class="scores-title">Random Access Performance</h3>');
Expand All @@ -36,7 +36,25 @@ describe('SummaryCard', () => {
expect(html).toContain('parquet');
expect(html).toContain('3.00 ms');
expect(html).toContain('2.00x');
expect(html).toContain('Random access time | Ratio to fastest (lower is better)');
expect(html).toContain(
'Geomean of take time ratio to fastest across every chart (lower is better)',
);
});

it('flags a partially measured series in its hover text', () => {
const html = render({
type: 'randomAccess',
title: 'Random Access Performance',
rankings: [
{ name: 'vortex', score: 1, totalRuntime: 1_500_000, measured: 9, total: 9 },
{ name: 'lance', score: 3, totalRuntime: 3_000_000, measured: 4, total: 9 },
],
explanation: 'e',
});
// The full-coverage series keeps a bare label; the partial one says so, so
// a penalty-inflated score is never presented as a like-for-like number.
expect(html).toContain('title="vortex"');
expect(html).toContain('measured in 4 of 9 charts');
});

it('renders nothing for a randomAccess card with no rankings', () => {
Expand Down Expand Up @@ -96,12 +114,34 @@ describe('SummaryCard', () => {
const html = render({
type: 'queryBenchmark',
title: 'Performance Summary',
rankings: [{ name: 'vortex:vortex-file', score: 1.0, totalRuntime: 5_000_000_000 }],
rankings: [
{
name: 'vortex:vortex-file',
score: 1.0,
totalRuntime: 5_000_000_000,
measured: 1,
total: 1,
},
],
explanation: 'lower is better',
});
expect(html).toContain('#1');
expect(html).toContain('vortex:vortex-file');
expect(html).toContain('1.00x');
expect(html).toContain('5.00 s');
});

it('renders a vectorSearch card through the shared timing arm', () => {
const html = render({
type: 'vectorSearch',
title: 'Vector Search Performance',
rankings: [
{ name: 'vortex-turboquant', score: 1.0, totalRuntime: 7_000, measured: 2, total: 2 },
],
explanation: 'lower is better',
});
expect(html).toContain('<h3 class="scores-title">Vector Search Performance</h3>');
expect(html).toContain('vortex-turboquant');
expect(html).toContain('1.00x');
});
});
46 changes: 19 additions & 27 deletions web/components/SummaryCard.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,29 @@

import { displayFormat, displaySeriesLabel } from '@/lib/chart-format';
import { formatTimeNs } from '@/lib/format';
import type { Summary } from '@/lib/summary';
import type { SeriesRanking, Summary } from '@/lib/summary';

/**
* Hover text for a ranked series: its full label plus, when the series was not
* measured everywhere, how much of its score came from the missing-bucket
* penalty. A partially measured series is ranked, not hidden, so the coverage
* has to be legible somewhere.
*/
function seriesTitle(item: SeriesRanking): string {
const label = displaySeriesLabel(item.name);
return item.measured >= item.total
? label
: `${label} - measured in ${item.measured} of ${item.total} charts; the rest scored by the missing-series penalty`;
}

/**
* The per-group summary card.
*
* Every [`Summary`] variant renders the same `.benchmark-scores-summary` shape
* (a `.scores-title`, a `.scores-list` of `.score-item` rows, and a
* `.scores-explanation` footer); only the rank label, value, and optional
* runtime change. The card stays visible whether or not the enclosing group is
* runtime change. The three timing families (query, random access, vector
* search) share one arm because they share one [`SeriesRanking`] shape. The card stays visible whether or not the enclosing group is
* expanded (the CSS only hides `.chart-grid` when the disclosure is closed), so
* the at-a-glance rankings show without expanding the group.
*
Expand All @@ -22,30 +36,6 @@ export function SummaryCard({ summary }: { summary?: Summary }) {
return null;
}
switch (summary.type) {
case 'randomAccess':
if (summary.rankings.length === 0) {
return null;
}
return (
<section className="benchmark-scores-summary" aria-label={summary.title}>
<h3 className="scores-title">{summary.title}</h3>
<div className="scores-list">
{summary.rankings.map((item, idx) => (
<div className="score-item" key={item.name}>
<span className="score-rank">#{idx + 1}</span>
<span className="score-series" title={displaySeriesLabel(item.name)}>
{displaySeriesLabel(item.name)}
</span>
<span className="score-metrics">
<span className="score-value">{formatTimeNs(item.time)}</span>
<span className="score-runtime">{item.ratio.toFixed(2)}x</span>
</span>
</div>
))}
</div>
<div className="scores-explanation">{summary.explanation}</div>
</section>
);
case 'compression': {
if (summary.rankings.length === 0) {
return null;
Expand Down Expand Up @@ -118,6 +108,8 @@ export function SummaryCard({ summary }: { summary?: Summary }) {
</section>
);
case 'queryBenchmark':
case 'randomAccess':
case 'vectorSearch':
if (summary.rankings.length === 0) {
return null;
}
Expand All @@ -128,7 +120,7 @@ export function SummaryCard({ summary }: { summary?: Summary }) {
{summary.rankings.map((item, idx) => (
<div className="score-item" key={item.name}>
<span className="score-rank">#{idx + 1}</span>
<span className="score-series" title={displaySeriesLabel(item.name)}>
<span className="score-series" title={seriesTitle(item)}>
{displaySeriesLabel(item.name)}
</span>
<span className="score-metrics">
Expand Down
Loading
Loading