From 3cce5e36dccadb88b1c7763c47261b4923b65070 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 14 Jul 2026 08:03:02 +0000 Subject: [PATCH 1/5] refactor: harden core architecture per review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - merge: resolve cross-source license conflicts conservatively (stricter license wins; incomparable claims collapse to unknown), surfaced in meta.warnings and via merge.onRightsConflict - client: unified load-more cursor (SearchInput.cursor + meta.nextCursor) with cross-page dedup; cache keys embed the full normalized query (no more 32-bit hash collisions); new cacheRaw option; runProvider extracted to provider-run.ts - query: single-track capability routing — queryFeatures/filters deprecated, capabilities.controls is the only routing source, and the legacy NormalizedQuery.filters channel is derived from routed controls - rerank: CJK-aware tokenize (character bigrams) so CJK queries score - mcp: BYOK providers become lazily-loaded optionalDependencies; search_references gains rerank + cursor - providers: wire controls.page in openverse, pixabay, internet-archive, wikimedia-commons (gsroffset), artic Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_018hoN6wKWxcs8WbUNCM9rtt --- .changeset/architecture-review-core.md | 13 ++ .changeset/architecture-review-mcp.md | 6 + .changeset/architecture-review-providers.md | 9 + README.md | 24 +-- packages/core/src/__tests__/client.test.ts | 50 ++++++ packages/core/src/__tests__/merge.test.ts | 57 ++++++- .../core/src/__tests__/provider-run.test.ts | 66 ++++++++ packages/core/src/__tests__/query.test.ts | 20 ++- packages/core/src/__tests__/rerank.test.ts | 16 ++ packages/core/src/client.ts | 158 +++++++----------- packages/core/src/cursor.ts | 48 ++++++ packages/core/src/index.ts | 6 +- packages/core/src/merge.ts | 97 ++++++++++- packages/core/src/provider-run.ts | 123 ++++++++++++++ packages/core/src/provider.ts | 14 +- packages/core/src/query.ts | 13 +- packages/core/src/reference.ts | 5 +- packages/core/src/rerank.ts | 19 ++- packages/mcp/package.json | 20 ++- packages/mcp/src/__tests__/mcp.test.ts | 44 ++--- packages/mcp/src/cli.ts | 118 +++++++++---- packages/mcp/src/index.ts | 11 +- packages/provider-artic/src/index.ts | 4 +- packages/provider-brave/src/index.ts | 1 - packages/provider-europeana/src/index.ts | 1 - packages/provider-flickr/src/index.ts | 1 - packages/provider-freesound/src/index.ts | 1 - packages/provider-gutendex/src/index.ts | 1 - .../provider-internet-archive/src/index.ts | 5 +- packages/provider-jamendo/src/index.ts | 1 - packages/provider-met/src/index.ts | 1 - packages/provider-openverse/src/index.ts | 8 +- packages/provider-pexels/src/index.ts | 2 - packages/provider-pixabay/src/index.ts | 8 +- packages/provider-poetrydb/src/index.ts | 1 - packages/provider-polyhaven/src/index.ts | 2 - packages/provider-rijksmuseum/src/index.ts | 1 - packages/provider-smithsonian/src/index.ts | 1 - packages/provider-unsplash/src/index.ts | 1 - .../provider-wikimedia-commons/src/index.ts | 5 +- 40 files changed, 753 insertions(+), 229 deletions(-) create mode 100644 .changeset/architecture-review-core.md create mode 100644 .changeset/architecture-review-mcp.md create mode 100644 .changeset/architecture-review-providers.md create mode 100644 packages/core/src/__tests__/provider-run.test.ts create mode 100644 packages/core/src/cursor.ts create mode 100644 packages/core/src/provider-run.ts diff --git a/.changeset/architecture-review-core.md b/.changeset/architecture-review-core.md new file mode 100644 index 0000000..9bef09a --- /dev/null +++ b/.changeset/architecture-review-core.md @@ -0,0 +1,13 @@ +--- +"@refkit/core": minor +--- + +Architecture-review hardening: + +- **Conservative cross-source rights merge** — when two sources disagree about the license of the same canonical URL, the stricter license wins (incomparable claims collapse to `unknown` → needs-review); conflicts surface in `meta.warnings` and via the new `merge.onRightsConflict` observer. New export: `stricterLicense`, `RightsConflict`. +- **Unified pagination cursor** — `SearchInput.cursor` + `meta.nextCursor`: opaque load-more cursor that advances the provider-local page and dedupes against previously returned results. +- **CJK-aware `tokenize`** — CJK runs tokenize into character bigrams, so `lexicalReranker` scores Chinese/Japanese/Korean queries instead of dropping them. +- **Collision-proof cache keys** — per-provider cache keys embed the full normalized query instead of a 32-bit hash (two distinct queries can no longer silently share a cache entry). Existing cache entries are invalidated by the key-format change. +- New `cacheRaw: false` option strips `raw` provider payloads from cache entries. +- **Deprecations (single-track capability routing)** — `SearchFilters`, `SearchInput.filters`, `NormalizedQuery.filters`, `ReferenceProvider.queryFeatures` (now optional), and `QueryFeature` are deprecated. Routing is driven solely by `capabilities.controls`; legacy `filters` are merged into `controls` and the deprecated `NormalizedQuery.filters` channel is derived from the routed controls, so both channels always agree. Providers that declared filter support only via `queryFeatures` must declare `capabilities.controls` to keep receiving those values. +- `runProviderSearch` / `providerCacheKey` / `stableStringify` extracted and exported (`provider-run`), shrinking the search orchestrator. diff --git a/.changeset/architecture-review-mcp.md b/.changeset/architecture-review-mcp.md new file mode 100644 index 0000000..fcbf6fa --- /dev/null +++ b/.changeset/architecture-review-mcp.md @@ -0,0 +1,6 @@ +--- +"@refkit/mcp": minor +--- + +- `search_references` gains `rerank: true` (query-aware re-ranking via `lexicalReranker`, CJK-aware) and `cursor` (load-more pagination with cross-page dedup; the next cursor rides on `meta.nextCursor` with `explain: true`). +- BYOK provider packages moved from `dependencies` to `optionalDependencies` and are now loaded lazily, only when their key is present. Default installs (incl. `npx -y @refkit/mcp`) still get everything; installs with `--omit=optional` skip BYOK sources, and a key whose package is missing logs a stderr warning instead of crashing. `defaultProviders()` is now async. diff --git a/.changeset/architecture-review-providers.md b/.changeset/architecture-review-providers.md new file mode 100644 index 0000000..6b86f19 --- /dev/null +++ b/.changeset/architecture-review-providers.md @@ -0,0 +1,9 @@ +--- +"@refkit/provider-openverse": patch +"@refkit/provider-pixabay": patch +"@refkit/provider-internet-archive": patch +"@refkit/provider-wikimedia-commons": patch +"@refkit/provider-artic": patch +--- + +Declare and honor the `page` search control (`capabilities.controls: ['page']`), wiring `controls.page` to each source's native pagination (Wikimedia Commons translates it to `gsroffset`). Enables core's unified load-more cursor across these sources. diff --git a/README.md b/README.md index 08cb8fd..e20a817 100644 --- a/README.md +++ b/README.md @@ -108,17 +108,17 @@ console.log(meta.providers) console.log(meta.warnings) ``` -`controls.page` is a **provider-local** cursor: each provider paginates its own result stream independently, and refkit does not track a unified offset across sources. Because pages are fused with Reciprocal Rank Fusion per request, page N+1 is not guaranteed to be disjoint from page N — results can overlap or shift relative to the previous page. For a "load more" UI, dedupe across pages by `canonicalUrl` rather than assuming stable, non-overlapping windows: +### Pagination ("load more") -```ts -import { canonicalizeUrl } from '@refkit/core' +Use the built-in cursor: every `searchWithMeta` result carries an opaque `meta.nextCursor`; pass it back as `cursor` and refkit advances the provider-local page **and dedupes against everything already returned** — no caller-side bookkeeping: -const seen = new Set(prev.map((r) => canonicalizeUrl(r.canonicalUrl))) -const nextPage = await refkit.search({ query, modalities: ['image'], controls: { page: 2 } }) -const newOnly = nextPage.filter((r) => !seen.has(canonicalizeUrl(r.canonicalUrl))) +```ts +const page1 = await refkit.searchWithMeta({ query: 'forest path', modalities: ['image'] }) +const page2 = await refkit.searchWithMeta({ query: 'forest path', modalities: ['image'], cursor: page1.meta.nextCursor }) +// page2.references never repeats page1's; an empty page (no meta.nextCursor) means exhausted. ``` -(core's own merge/dedup normalizes URLs the same way, so this recipe stays consistent with what refkit dedupes internally.) +Under the hood `controls.page` is still a **provider-local** cursor (each source paginates its own stream; RRF-fused pages can overlap or shift), which is exactly why the cursor tracks seen results for you. Raw `controls.page` remains available if you want to manage pages yourself — then dedupe across pages by `canonicalizeUrl(r.canonicalUrl)`. ## Ranking & rerank @@ -148,6 +148,10 @@ rerank: async ({ query, refs }) => myEmbeddingRerank(query, refs) Rerank is **opt-in** — omit it for the default RRF order. It runs post-merge, before the `gateFor` license filter and the limit. +`lexicalReranker`'s term matching understands CJK text (character bigrams), so Chinese/Japanese/Korean queries score against titles instead of tokenizing to nothing. Note that most bundled sources index English metadata — for best recall, query in English (or have your agent translate) even though ranking handles CJK. + +When two sources disagree about the license of the **same canonical URL**, the merge resolves conservatively: the stricter license wins, and incomparable claims collapse to `unknown` (→ needs-review). Each conflict is reported in `meta.warnings` (and to an optional `merge.onRightsConflict` observer) — results never silently inherit the more permissive claim. + URL dedupe is built in, and perceptual hashes are supported when providers or hosts supply them. For host-computed fingerprints or embeddings, add a duplicate hook without making core fetch or decode media: ```ts @@ -170,10 +174,10 @@ createRefkit({ providers, resilience: { timeoutMs: 4000, retries: 2 } }) createRefkit({ providers, resilience: false }) // raw fan-out, no timeout/retry ``` -Pass a `cache` to memoize **per-provider** results (keyed by provider + normalized query, TTL `cacheTtlMs`, default 5 min). Merging, reranking, and the license gate always run fresh; cache hits are flagged `cached: true` in `meta.providers`, and every provider status carries `latencyMs`: +Pass a `cache` to memoize **per-provider** results (keyed by provider + the full normalized query — keys can be a few hundred bytes; TTL `cacheTtlMs`, default 5 min). Merging, reranking, and the license gate always run fresh; cache hits are flagged `cached: true` in `meta.providers`, and every provider status carries `latencyMs`. Pass `cacheRaw: false` to strip each result's `raw` provider payload from cache entries (smaller entries; raw-reading `isDuplicate` hooks then won't see `raw` on hits): ```ts -createRefkit({ providers, cache: myKvCache, cacheTtlMs: 60_000 }) +createRefkit({ providers, cache: myKvCache, cacheTtlMs: 60_000, cacheRaw: false }) ``` ## Providers @@ -241,7 +245,7 @@ Agents can use refkit in two ways: npx -y @refkit/mcp ``` -It boots with the keyless sources (Met, Art Institute, Wikimedia, Openverse, Project Gutenberg, PoetryDB, Rijksmuseum, Poly Haven, ambientCG, Internet Archive) and auto-adds any BYOK source whose key is in the environment (`REFKIT_UNSPLASH_KEY`, `REFKIT_PEXELS_KEY`, `REFKIT_BRAVE_KEY`, … — legacy names like `UNSPLASH_KEY`, `PEXELS_KEY`, `BRAVE_TOKEN` still work as fallbacks). Pass `intent` to annotate each result with a use-verdict (may I use this, is attribution required); `gateFor` to return only allowed results. Beyond search, `evaluate_use` and `build_attribution` expose the same license-verdict and attribution logic as standalone stateless tools, for when an agent already has a license id and just needs a verdict or a credit line. Or wire your own providers/keys via `serveStdio(createRefkit({ … }))` — see [`@refkit/mcp`](https://www.npmjs.com/package/@refkit/mcp). +It boots with the keyless sources (Met, Art Institute, Wikimedia, Openverse, Project Gutenberg, PoetryDB, Rijksmuseum, Poly Haven, ambientCG, Internet Archive) and auto-adds any BYOK source whose key is in the environment (`REFKIT_UNSPLASH_KEY`, `REFKIT_PEXELS_KEY`, `REFKIT_BRAVE_KEY`, … — legacy names like `UNSPLASH_KEY`, `PEXELS_KEY`, `BRAVE_TOKEN` still work as fallbacks). Pass `intent` to annotate each result with a use-verdict (may I use this, is attribution required); `gateFor` to return only allowed results; `rerank: true` for query-aware re-ranking (term coverage incl. CJK, resolution, source diversity); `cursor` (from `meta.nextCursor`, with `explain: true`) to page through results without repeats. BYOK provider packages are `optionalDependencies` of `@refkit/mcp` — installed by default (zero-config `npx` keeps working), but an install with `--omit=optional` skips them, and a key whose package is missing just logs a stderr warning instead of crashing the server. Beyond search, `evaluate_use` and `build_attribution` expose the same license-verdict and attribution logic as standalone stateless tools, for when an agent already has a license id and just needs a verdict or a credit line. Or wire your own providers/keys via `serveStdio(createRefkit({ … }))` — see [`@refkit/mcp`](https://www.npmjs.com/package/@refkit/mcp). ## Not legal advice diff --git a/packages/core/src/__tests__/client.test.ts b/packages/core/src/__tests__/client.test.ts index c5e642b..93e0486 100644 --- a/packages/core/src/__tests__/client.test.ts +++ b/packages/core/src/__tests__/client.test.ts @@ -77,6 +77,56 @@ describe('createRefkit', () => { expect(rk.buildAttribution(r).required).toBe(true) }) + it('cursor: pages via controls.page, dedupes across pages, and chains nextCursor', async () => { + // A paging provider whose page 2 overlaps page 1 — the RRF reality. + const pages: Record = { + 1: [ref('a-1', 'https://a/1'), ref('a-2', 'https://a/2')], + 2: [ref('a-2', 'https://a/2'), ref('a-3', 'https://a/3')], + } + const seenPages: Array = [] + const paging = defineProvider({ + id: 'a', + modalities: ['image'], + capabilities: { controls: ['page'] }, + search: async (q) => { + seenPages.push(q.controls?.page) + return pages[q.controls?.page ?? 1] ?? [] + }, + }) + const rk = createRefkit({ providers: [paging] }) + + const page1 = await rk.searchWithMeta({ query: 'x', modalities: ['image'], limit: 2 }) + expect(page1.references.map(r => r.canonicalUrl)).toEqual(['https://a/1', 'https://a/2']) + expect(page1.meta.nextCursor).toBeDefined() + + const page2 = await rk.searchWithMeta({ query: 'x', modalities: ['image'], limit: 2, cursor: page1.meta.nextCursor }) + expect(seenPages).toEqual([undefined, 2]) // cursor routed page 2 to the provider + expect(page2.references.map(r => r.canonicalUrl)).toEqual(['https://a/3']) // overlap deduped + + const page3 = await rk.searchWithMeta({ query: 'x', modalities: ['image'], limit: 2, cursor: page2.meta.nextCursor }) + expect(page3.references).toEqual([]) // exhausted + expect(page3.meta.nextCursor).toBeUndefined() // empty page ends the chain + }) + + it('cursor: rejects strings that did not come from meta.nextCursor', async () => { + const rk = createRefkit({ providers: [provider('a', [ref('a-1', 'https://a/1')])] }) + await expect(rk.search({ query: 'x', modalities: ['image'], cursor: 'not-a-cursor' })).rejects.toThrow(/invalid cursor/) + await expect(rk.search({ query: 'x', modalities: ['image'], cursor: '{"v":9}' })).rejects.toThrow(/invalid cursor/) + }) + + it('surfaces cross-source license conflicts as meta.warnings with conservative rights', async () => { + const rk = createRefkit({ + providers: [ + provider('a', [ref('a-1', 'https://shared/1', 'CC-BY')]), + provider('b', [ref('b-1', 'https://shared/1', 'CC-BY-NC')]), + ], + }) + const { references, meta } = await rk.searchWithMeta({ query: 'x', modalities: ['image'] }) + expect(references).toHaveLength(1) + expect(references[0].rights.license).toBe('CC-BY-NC') + expect(meta.warnings.some(w => w.includes('cross-source license conflict'))).toBe(true) + }) + it('queries only providers matching the modality', async () => { const textOnly = defineProvider({ id: 't', modalities: ['text'], queryFeatures: [], diff --git a/packages/core/src/__tests__/merge.test.ts b/packages/core/src/__tests__/merge.test.ts index 822b930..fede5b8 100644 --- a/packages/core/src/__tests__/merge.test.ts +++ b/packages/core/src/__tests__/merge.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest' -import { mergeReferences } from '../merge' +import { mergeReferences, stricterLicense, type RightsConflict } from '../merge' import type { Reference } from '../reference' +import type { LicenseId } from '../license' const make = (id: string, url: string, hash?: string): Reference => ({ id, @@ -13,6 +14,11 @@ const make = (id: string, url: string, hash?: string): Reference => ({ perceptualHash: hash, }) +const withLicense = (ref: Reference, license: LicenseId, licenseVersion?: string): Reference => ({ + ...ref, + rights: { ...ref.rights, license, licenseVersion }, +}) + describe('mergeReferences (RRF)', () => { it('returns a single ranked list with relevance in 0..1, top item = 1', () => { const out = mergeReferences([ @@ -46,6 +52,55 @@ describe('mergeReferences (RRF)', () => { expect(mergeReferences([[], []])).toEqual([]) }) + it('resolves a cross-source license conflict to the stricter license, regardless of source order', () => { + const a = withLicense(make('a-1', 'https://shared/1'), 'CC-BY') + const b = withLicense(make('b-1', 'https://shared/1'), 'CC-BY-NC') + for (const perSource of [[[a], [b]], [[b], [a]]]) { + const out = mergeReferences(perSource) + expect(out).toHaveLength(1) + expect(out[0].rights.license).toBe('CC-BY-NC') + } + }) + + it('collapses an incomparable license conflict to unknown (strict-deny) and drops licenseVersion', () => { + // unsplash grants derivatives but not redistribution; CC-BY-ND the reverse — + // neither dominates, so no single honest license id exists. + const a = withLicense(make('a-1', 'https://shared/1'), 'CC-BY-ND', '4.0') + const b = withLicense(make('b-1', 'https://shared/1'), 'unsplash') + const out = mergeReferences([[a], [b]]) + expect(out).toHaveLength(1) + expect(out[0].rights.license).toBe('unknown') + expect(out[0].rights.licenseVersion).toBeUndefined() + }) + + it('reports conflicts via onRightsConflict; same-license duplicates never conflict', () => { + const seen: RightsConflict[] = [] + const a = withLicense(make('a-1', 'https://shared/1'), 'CC0-1.0') + const b = withLicense(make('b-1', 'https://shared/1'), 'proprietary') + const c1 = withLicense(make('a-2', 'https://same/2'), 'CC-BY', '4.0') + const c2 = withLicense(make('b-2', 'https://same/2'), 'CC-BY', '2.0') + const out = mergeReferences([[a, c1], [b, c2]], { onRightsConflict: (c) => seen.push(c) }) + expect(seen).toHaveLength(1) + expect(seen[0]).toEqual({ + canonicalUrl: 'https://shared/1', + licenses: ['CC0-1.0', 'proprietary'], + resolvedLicense: 'proprietary', + }) + const shared = out.find(r => r.canonicalUrl === 'https://shared/1')! + expect(shared.rights.license).toBe('proprietary') + // same license id, different version: not a conflict, representative's record kept + const same = out.find(r => r.canonicalUrl === 'https://same/2')! + expect(same.rights.license).toBe('CC-BY') + }) + + it('stricterLicense: dominance picks the stricter; incomparable pairs return undefined', () => { + expect(stricterLicense('CC-BY', 'CC-BY-NC')).toBe('CC-BY-NC') + expect(stricterLicense('CC0-1.0', 'proprietary')).toBe('proprietary') + expect(stricterLicense('CC0-1.0', 'PD')).toBeDefined() // equal permissiveness — either + expect(stricterLicense('unsplash', 'CC-BY-ND')).toBeUndefined() + expect(stricterLicense('CC-BY-SA', 'unknown')).toBe('unknown') // unknown grants nothing determinable + }) + it('handles a large pool without a Math.max(...spread) stack overflow', () => { // The fused-score max must not be computed via `Math.max(...scores)`: spreading // ~10^5 args overflows the call stack (RangeError). Pool size here is well past diff --git a/packages/core/src/__tests__/provider-run.test.ts b/packages/core/src/__tests__/provider-run.test.ts new file mode 100644 index 0000000..fce8b37 --- /dev/null +++ b/packages/core/src/__tests__/provider-run.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it } from 'vitest' +import { providerCacheKey, runProviderSearch } from '../provider-run' +import type { KeyValueCache, ReferenceProvider } from '../provider' +import type { Reference } from '../reference' + +const ref = (url: string): Reference => ({ + id: `p:${url}`, + modality: 'image', + source: { providerId: 'p', sourceUrl: url }, + canonicalUrl: url, + rights: { license: 'CC0-1.0', rehostPolicy: 'cache-allowed', raw: { sourceTerms: 't', sourceUrl: url } }, + verifiedAt: '2026-06-22T00:00:00.000Z', + relevance: 0, + raw: { upstream: 'payload' }, +}) + +const provider = (results: Reference[]): ReferenceProvider => ({ + id: 'p', + modalities: ['image'], + search: async () => results, +}) + +function memoryCache(): KeyValueCache & { store: Map } { + const store = new Map() + return { + store, + get: async (k) => store.get(k), + set: async (k, v) => { store.set(k, v) }, + } +} + +describe('providerCacheKey', () => { + it('embeds the full normalized query — distinct queries can never collide', () => { + const a = providerCacheKey('p', { text: 'lion', modalities: ['image'] }) + const b = providerCacheKey('p', { text: 'tiger', modalities: ['image'] }) + expect(a).not.toBe(b) + expect(a).toContain('lion') + }) + + it('is insensitive to object key order', () => { + const a = providerCacheKey('p', { text: 'x', modalities: ['image'], providerOptions: { a: 1, b: 2 } }) + const b = providerCacheKey('p', { modalities: ['image'], providerOptions: { b: 2, a: 1 }, text: 'x' }) + expect(a).toBe(b) + }) +}) + +describe('runProviderSearch cacheRaw', () => { + const deps = { fetch: (() => { throw new Error('unused') }) as unknown as typeof fetch, cacheTtlMs: 60_000 } + + it('cacheRaw: true (default behavior) keeps raw in the cached payload', async () => { + const cache = memoryCache() + await runProviderSearch(provider([ref('https://a/1')]), { text: 'q', modalities: ['image'] }, { ...deps, cache, cacheRaw: true }) + await new Promise(r => setTimeout(r)) // cache write is fire-and-forget + const [payload] = [...cache.store.values()] + expect(JSON.parse(payload)[0].raw).toEqual({ upstream: 'payload' }) + }) + + it('cacheRaw: false strips raw from the cached payload but not from live results', async () => { + const cache = memoryCache() + const run = await runProviderSearch(provider([ref('https://a/1')]), { text: 'q', modalities: ['image'] }, { ...deps, cache, cacheRaw: false }) + expect(run.ok && run.valid[0].raw).toEqual({ upstream: 'payload' }) + await new Promise(r => setTimeout(r)) + const [payload] = [...cache.store.values()] + expect(JSON.parse(payload)[0].raw).toBeUndefined() + }) +}) diff --git a/packages/core/src/__tests__/query.test.ts b/packages/core/src/__tests__/query.test.ts index b3e3b91..055edf9 100644 --- a/packages/core/src/__tests__/query.test.ts +++ b/packages/core/src/__tests__/query.test.ts @@ -3,34 +3,36 @@ import { normalizeQuery } from '../query' import type { ReferenceProvider } from '../provider' const provider = ( - qf: ReferenceProvider['queryFeatures'], + controls: NonNullable['controls'] = [], modalities: ReferenceProvider['modalities'] = ['image'], -): ReferenceProvider => ({ id: 'p', modalities, queryFeatures: qf, search: async () => [] }) +): ReferenceProvider => ({ id: 'p', modalities, capabilities: { controls }, search: async () => [] }) describe('normalizeQuery', () => { - it('drops filters the provider does not support', () => { + it('routes legacy filters by capabilities.controls and mirrors them on both channels', () => { const nq = normalizeQuery( { query: 'cat', modalities: ['image'], filters: { color: 'red', orientation: 'landscape' } }, - provider(['keyword', 'color']), + provider(['color']), ) - expect(nq.filters).toEqual({ color: 'red' }) // orientation dropped + expect(nq.filters).toEqual({ color: 'red' }) // orientation dropped (not in capabilities) + expect(nq.controls).toEqual({ color: 'red' }) // derived channel stays consistent }) it('omits filters entirely when none survive', () => { const nq = normalizeQuery( { query: 'cat', modalities: ['image'], filters: { color: 'red' } }, - provider(['keyword']), + provider([]), ) expect(nq.filters).toBeUndefined() + expect(nq.controls).toBeUndefined() }) it('intersects modalities with the provider', () => { - const nq = normalizeQuery({ query: 'x', modalities: ['image', 'text'] }, provider(['keyword'], ['image'])) + const nq = normalizeQuery({ query: 'x', modalities: ['image', 'text'] }, provider([], ['image'])) expect(nq.modalities).toEqual(['image']) }) it('passes through query text and limit', () => { - const nq = normalizeQuery({ query: 'cat', modalities: ['image'], limit: 10 }, provider(['keyword'])) + const nq = normalizeQuery({ query: 'cat', modalities: ['image'], limit: 10 }, provider()) expect(nq.text).toBe('cat') expect(nq.limit).toBe(10) }) @@ -45,7 +47,7 @@ describe('normalizeQuery', () => { other: { orderBy: 'relevant' }, }, }, - provider(['keyword']), + provider(), ) expect(nq.providerOptions).toEqual({ orderBy: 'latest' }) }) diff --git a/packages/core/src/__tests__/rerank.test.ts b/packages/core/src/__tests__/rerank.test.ts index 3bb87d5..918b779 100644 --- a/packages/core/src/__tests__/rerank.test.ts +++ b/packages/core/src/__tests__/rerank.test.ts @@ -107,6 +107,22 @@ describe('tokenize', () => { expect(tokenize(' the of a ')).toEqual([]) expect(tokenize('')).toEqual([]) }) + + it('tokenizes CJK runs into character bigrams (lone char stays a unigram)', () => { + expect(tokenize('青花瓷')).toEqual(['青花', '花瓷']) + expect(tokenize('瓷')).toEqual(['瓷']) + expect(tokenize('Ming 青花瓷 vase')).toEqual(['ming', 'vase', '青花', '花瓷']) + }) + + it('lexicalReranker scores CJK queries against CJK titles', async () => { + const rerank = lexicalReranker({ qualityWeight: 0, sourceDiversity: 0 }) + const out = await rerank({ + query: '青花瓷', + refs: [ref('miss', 'Roman marble bust'), ref('match', '明代青花瓷盘')], + }) + expect(out.map((r) => r.id)).toEqual(['match', 'miss']) + expect(out[0].relevance).toBeGreaterThan(out[1].relevance) + }) }) describe('public surface', () => { diff --git a/packages/core/src/client.ts b/packages/core/src/client.ts index 8dfa756..6bb1a33 100644 --- a/packages/core/src/client.ts +++ b/packages/core/src/client.ts @@ -1,5 +1,4 @@ import type { Reference } from './reference' -import { parseReference } from './reference' import type { Reranker } from './rerank' import type { Modality } from './modality' import type { Intent, Verdict } from './evaluate-use' @@ -8,17 +7,17 @@ import type { Attribution } from './attribution' import { buildAttribution } from './attribution' import type { ReferenceProvider, - ProviderContext, KeyValueCache, SearchFilters, SearchControls, SearchControlKey, ProviderOptionsById, } from './provider' -import { mergeReferences, type MergeOptions } from './merge' +import { mergeReferences, type MergeOptions, type RightsConflict } from './merge' import { mergeSearchControls, normalizeQuery, requestedControlKeys, supportedControlKeys, unsupportedControlKeys } from './query' -import { retryingFetch, withTimeout } from './resilience' -import { fnv1a } from './hash' +import { retryingFetch } from './resilience' +import { runProviderSearch, type ProviderRun } from './provider-run' +import { cursorSeenKey, decodeCursor, encodeCursor } from './cursor' export interface ResilienceOptions { /** Soft deadline per provider search. Default 10_000. */ @@ -37,6 +36,10 @@ export interface RefkitOptions { resilience?: ResilienceOptions | false /** TTL for per-provider cached results; used only when `cache` is set. Default 300_000. */ cacheTtlMs?: number + /** Include each result's `raw` provider payload in cached entries. Default true. + * Pass false to shrink cache entries — cache-hit refs then carry no `raw`, so a + * `merge.isDuplicate` hook reading `raw` won't see it on hits. */ + cacheRaw?: boolean } export interface ProviderError { @@ -80,6 +83,10 @@ export interface SearchMeta { providerOptions?: string[] providers: ProviderSearchStatus[] gate?: SearchGateMeta + /** Opaque "load more" cursor: pass as `SearchInput.cursor` to fetch the next + * page with cross-page dedup handled internally. Present when this page + * returned at least one result. */ + nextCursor?: string warnings: string[] } @@ -91,12 +98,19 @@ export interface SearchResult { export interface SearchInput { query: string modalities: Modality[] + /** @deprecated Compatibility alias for `controls.color` / `controls.orientation` + * / `controls.language` (controls win on conflict). Use `controls`. */ filters?: SearchFilters controls?: SearchControls /** Provider-specific search controls keyed by provider id. Core routes only the * matching entry to each provider; providers whitelist what they translate. */ providerOptions?: ProviderOptionsById limit?: number + /** Opaque cursor from a previous search's `meta.nextCursor`. Sets the + * provider-local page (overriding `controls.page`) and filters out results + * already returned on earlier pages, so "load more" needs no caller-side + * dedup. Throws on a string that did not come from `meta.nextCursor`. */ + cursor?: string /** Overfetch this many × `limit` candidates per provider before merge/rerank/gate, * then narrow to `limit` — a wider pool means better dedup + ranking. Default 4 * (capped so a source is never asked for more than {@link MAX_POOL_LIMIT}); min 1. @@ -130,19 +144,6 @@ function errorSummary(error: unknown): string { return 'unknown error' } -// Deterministic JSON for cache keys: object keys sorted recursively, so a -// caller's providerOptions key order can't split otherwise-identical keys. -// Keys whose value is `undefined` are skipped, matching JSON.stringify's own -// object semantics — `{ a: 1, b: undefined }` and `{ a: 1 }` must key alike. -function stableStringify(value: unknown): string { - if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]` - if (value && typeof value === 'object') { - const rec = value as Record - return `{${Object.keys(rec).filter(k => rec[k] !== undefined).sort().map(k => `${JSON.stringify(k)}:${stableStringify(rec[k])}`).join(',')}}` - } - return JSON.stringify(value) ?? 'null' -} - export function createRefkit(options: RefkitOptions): RefkitClient { if (!options.providers || options.providers.length === 0) { throw new Error('createRefkit: at least one provider is required') @@ -162,7 +163,11 @@ export function createRefkit(options: RefkitOptions): RefkitClient { // Overfetch a wider candidate pool per provider, then narrow to `limit` after // merge/rerank/gate — you can't rank or dedup candidates you never fetched. const fetchLimit = Math.max(limit, Math.min(Math.ceil(limit * poolFactor), MAX_POOL_LIMIT)) - const requestedControlsSource = mergeSearchControls(input.controls, input.filters) + // A cursor overrides controls.page and carries the already-seen set; the + // effective controls are what routing, providers, and meta all see. + const cursorState = input.cursor !== undefined ? decodeCursor(input.cursor) : undefined + const controls = cursorState ? { ...input.controls, page: cursorState.page } : input.controls + const requestedControlsSource = mergeSearchControls(controls, input.filters) const requestedControls = requestedControlKeys(requestedControlsSource) const controlsMeta = requestedControls.length > 0 ? { requested: requestedControls, @@ -182,88 +187,24 @@ export function createRefkit(options: RefkitOptions): RefkitClient { // a fresh wrapper per provider. const sharedFetch = resilience && resilience.retries > 0 ? retryingFetch(doFetch, { retries: resilience.retries }) : doFetch - type ProviderRun = - | { ok: true; valid: Reference[]; returned: number; latencyMs: number; cached?: boolean } - | { ok: false; error: unknown; latencyMs: number } - - const runProvider = async (p: ReferenceProvider): Promise => { - const started = Date.now() - const timeout = resilience ? withTimeout(input.signal ?? options.signal, resilience.timeoutMs) : undefined - const ctx: ProviderContext = { - fetch: sharedFetch, - cache: options.cache, - signal: timeout?.signal ?? input.signal ?? options.signal, - } + const runProvider = (p: ReferenceProvider): Promise => { const query = normalizeQuery({ query: input.query, modalities: input.modalities, filters: input.filters, - controls: input.controls, + controls, providerOptions: input.providerOptions, limit: fetchLimit, }, p) - const cacheKey = options.cache - ? `refkit:v1:${p.id}:${fnv1a(stableStringify(query))}` - : undefined - // Race a promise against the deadline without leaking an unhandled rejection - // for whichever side loses the race. - const raceDeadline = (p: Promise): Promise => { - p.catch(() => {}) - return timeout ? Promise.race([p, timeout.expired]) : p - } - // Parse raw provider items one at a time — a single bad item must not - // discard the rest (shared by the cache-hit and live-search paths). - const parseItems = (raw: unknown[]): Reference[] => { - const valid: Reference[] = [] - for (const item of raw) { - try { - valid.push(parseReference(item)) - } catch (error) { - input.onProviderError?.({ providerId: p.id, error }) - } - } - return valid - } - try { - if (options.cache && cacheKey) { - // best-effort: a broken/corrupt/stale cache degrades to a live search - const pending = options.cache.get(cacheKey) - // A slow cache read must not outlive the provider deadline: at expiry it - // degrades to a miss, and the live search below then fails fast on the - // same (already-expired) deadline. - const hit = await (timeout - ? Promise.race([pending, timeout.expired.catch(() => undefined)]) - : pending - ).catch(() => undefined) - pending.catch(() => {}) // raced-past rejection must not go unhandled - if (hit !== undefined) { - try { - // cached refs keep their original verifiedAt — staleness is bounded - // by the TTL when the cache honors ttlMs. Only a whole-payload - // failure (not JSON, not an array) falls through to live; a single - // bad item within an otherwise-valid array is reported and dropped, - // same as the live path. - const raw = JSON.parse(hit) as unknown[] - if (!Array.isArray(raw)) throw new Error('cached payload is not an array') - const valid = parseItems(raw) - return { ok: true, valid, returned: raw.length, latencyMs: Date.now() - started, cached: true } - } catch { /* fall through to live */ } - } - } - const searching = p.search(query, ctx) - const raw = await raceDeadline(searching) - const valid = parseItems(raw) - if (options.cache && cacheKey) { - // fire-and-forget: cache write failure must never fail the search - void options.cache.set(cacheKey, JSON.stringify(valid), options.cacheTtlMs ?? DEFAULT_CACHE_TTL_MS).catch(() => {}) - } - return { ok: true, valid, returned: raw.length, latencyMs: Date.now() - started } - } catch (error) { - input.onProviderError?.({ providerId: p.id, error }) - return { ok: false, error, latencyMs: Date.now() - started } - } finally { - timeout?.cancel() - } + return runProviderSearch(p, query, { + fetch: sharedFetch, + cache: options.cache, + cacheTtlMs: options.cacheTtlMs ?? DEFAULT_CACHE_TTL_MS, + cacheRaw: options.cacheRaw ?? true, + timeoutMs: resilience?.timeoutMs, + signal: input.signal ?? options.signal, + onError: (error) => input.onProviderError?.({ providerId: p.id, error }), + }) } const runs = await Promise.all(chosen.map(runProvider)) @@ -293,7 +234,16 @@ export function createRefkit(options: RefkitOptions): RefkitClient { throw new AggregateError(runs.filter(r => !r.ok).map(r => (r as { error: unknown }).error), 'refkit.search: all providers failed') } - let refs = mergeReferences(perSource, options.merge) + // Collect cross-source license conflicts for meta.warnings while still + // forwarding them to a host-supplied observer. + const rightsConflicts: RightsConflict[] = [] + let refs = mergeReferences(perSource, { + ...options.merge, + onRightsConflict: (c) => { + rightsConflicts.push(c) + options.merge?.onRightsConflict?.(c) + }, + }) // Rerank runs over the FULL merged pool, before the license gate — ordering // (and a reranker's batch-relative scoring, e.g. quality normalised across // the pool) is computed against every candidate, then the gate drops denied @@ -309,10 +259,27 @@ export function createRefkit(options: RefkitOptions): RefkitClient { refs = refs.filter(r => evaluateUse(r.rights, intent).decision.startsWith('allowed')) gate = { intent, before: beforeGate, after: refs.length, dropped: beforeGate - refs.length } } + // Cursor pagination: drop results already returned on earlier pages (RRF + // pages overlap by design), AFTER rank/gate so ordering is batch-consistent + // but BEFORE the limit so repeats don't consume the page budget. + if (cursorState) { + const seen = new Set(cursorState.seen) + refs = refs.filter(r => !seen.has(cursorSeenKey(r.canonicalUrl))) + } const references = refs.slice(0, limit) + const nextCursor = references.length > 0 + ? encodeCursor({ + v: 1, + page: (controls?.page ?? 1) + 1, + seen: [...(cursorState?.seen ?? []), ...references.map(r => cursorSeenKey(r.canonicalUrl))], + }) + : undefined const warnings: string[] = [] const failedCount = [...statusByProvider.values()].filter(s => s.status === 'failed').length if (failedCount > 0) warnings.push(`${failedCount} provider(s) failed; returning partial results.`) + for (const c of rightsConflicts) { + warnings.push(`cross-source license conflict for ${c.canonicalUrl}: ${c.licenses.join(' vs ')} → resolved to ${c.resolvedLicense}.`) + } if (gate && gate.dropped > 0) warnings.push(`${gate.dropped} result(s) dropped by ${gate.intent} gate.`) return { references, @@ -327,6 +294,7 @@ export function createRefkit(options: RefkitOptions): RefkitClient { ...(input.providerOptions ? { providerOptions: Object.keys(input.providerOptions) } : {}), providers: options.providers.map(p => statusByProvider.get(p.id) ?? { providerId: p.id, status: 'skipped', reason: 'unsupported-modality' }), ...(gate ? { gate } : {}), + ...(nextCursor ? { nextCursor } : {}), warnings, }, } diff --git a/packages/core/src/cursor.ts b/packages/core/src/cursor.ts new file mode 100644 index 0000000..34d1e02 --- /dev/null +++ b/packages/core/src/cursor.ts @@ -0,0 +1,48 @@ +// Unified "load more" cursor (v1). RRF fusion means provider page N+1 overlaps +// page N, so raw `controls.page` pushes cross-page dedup onto every caller. The +// cursor internalizes that: it carries the next provider-local page plus compact +// hashes of every already-returned result, and the client filters repeats out +// before applying `limit`. The string is an implementation detail — treat it as +// opaque; only `meta.nextCursor` from a previous search is a valid input. +import { fnv1a } from './hash' +import { canonicalizeUrl } from './dedup-key' + +export interface SearchCursorState { + v: 1 + /** Provider-local page to request next (routed as controls.page; 1-based). */ + page: number + /** {@link cursorSeenKey} hashes of results returned on previous pages. Grows + * linearly with pages consumed — a 32-bit hash keeps entries compact; the + * worst case of a collision is one new result suppressed as already-seen. */ + seen: string[] +} + +/** Compact already-seen key for a result — same URL canonicalization as merge/dedup. */ +export function cursorSeenKey(canonicalUrl: string): string { + return fnv1a(canonicalizeUrl(canonicalUrl)) +} + +export function encodeCursor(state: SearchCursorState): string { + return JSON.stringify(state) +} + +/** Parse and validate a cursor string. Throws on anything that is not a cursor + * this library produced — a corrupted cursor must fail loudly, not quietly + * restart from page 1. */ +export function decodeCursor(cursor: string): SearchCursorState { + let parsed: unknown + try { + parsed = JSON.parse(cursor) + } catch { + throw new Error('refkit.search: invalid cursor (not produced by meta.nextCursor)') + } + const s = parsed as Partial | null + if ( + !s || typeof s !== 'object' || s.v !== 1 + || typeof s.page !== 'number' || !Number.isInteger(s.page) || s.page < 1 + || !Array.isArray(s.seen) || !s.seen.every(k => typeof k === 'string') + ) { + throw new Error('refkit.search: invalid cursor (not produced by meta.nextCursor)') + } + return { v: 1, page: s.page, seen: s.seen } +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 354e78f..e902967 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -18,8 +18,8 @@ export { fnv1a } from './hash' export { canonicalizeUrl, referenceId } from './dedup-key' export { hammingDistance, dedupeReferences } from './dedup' export type { DedupeOptions } from './dedup' -export { mergeReferences } from './merge' -export type { MergeOptions } from './merge' +export { mergeReferences, stricterLicense } from './merge' +export type { MergeOptions, RightsConflict } from './merge' export { evaluateUse, evaluatePermissions, NOT_LEGAL_ADVICE, INTENTS } from './evaluate-use' export type { Intent, Decision, Verdict, PermissionKey, EvaluateOptions } from './evaluate-use' export { defineProvider } from './provider' @@ -50,6 +50,8 @@ export { isLikelyImageUrl, imageMediaType, IMAGE_EXT, } from './provider-helpers' export { normalizeQuery } from './query' +export { runProviderSearch, providerCacheKey, stableStringify } from './provider-run' +export type { ProviderRun, ProviderRunDeps } from './provider-run' export { createRefkit } from './client' export type { RefkitClient, diff --git a/packages/core/src/merge.ts b/packages/core/src/merge.ts index 5a14e58..2d8202e 100644 --- a/packages/core/src/merge.ts +++ b/packages/core/src/merge.ts @@ -1,10 +1,74 @@ import type { Reference } from './reference' +import type { RightsRecord } from './rights' +import { factsFor, type LicenseId, type Tri } from './license' import { canonicalizeUrl } from './dedup-key' import { dedupeReferences, type DedupeOptions } from './dedup' +/** A cross-source disagreement about the license of the same canonical URL, + * reported once per conflicting pair as the merge encounters it. */ +export interface RightsConflict { + canonicalUrl: string + /** The two license ids that disagreed (already-resolved vs newly seen). */ + licenses: [LicenseId, LicenseId] + /** What the merge resolved to: the stricter of the two, or 'unknown' when + * neither is stricter on every axis (strict-deny → needs-review). */ + resolvedLicense: LicenseId +} + export interface MergeOptions extends DedupeOptions { /** RRF dampening constant. Standard default 60. */ k?: number + /** Observe cross-source license conflicts (the client surfaces them as + * meta.warnings). Resolution itself is built in and always conservative. */ + onRightsConflict?: (conflict: RightsConflict) => void +} + +// — conservative rights resolution for cross-source URL conflicts — +// Two sources describing the SAME canonical URL are making claims about the same +// work; when their license ids disagree, believing the more permissive claim +// would be fail-open. Rank every axis so smaller = stricter, then keep a license +// only if it is no more permissive on EVERY axis; incomparable pairs collapse to +// 'unknown' (→ needs-review), matching the strict-deny invariant. +const triRank = (t: Tri): number => (t === true ? 2 : t === 'unknown' ? 1 : 0) + +function permissivenessVector(license: LicenseId): number[] { + const f = factsFor(license) + return [ + triRank(f.commercialUse), + triRank(f.derivatives), + triRank(f.redistribution), + f.attributionRequired ? 0 : 1, // carrying the obligation is stricter + f.shareAlike ? 0 : 1, + ] +} + +/** The stricter of two license ids when one dominates on every axis; undefined + * when they are incomparable (each grants something the other doesn't). */ +export function stricterLicense(a: LicenseId, b: LicenseId): LicenseId | undefined { + // 'unknown' grants nothing determinable — a conflict involving it can only + // resolve to it (its obligation axes are meaningless, not "no obligations"). + if (a === 'unknown' || b === 'unknown') return 'unknown' + const va = permissivenessVector(a) + const vb = permissivenessVector(b) + let aNoMorePermissive = true + let bNoMorePermissive = true + for (let i = 0; i < va.length; i++) { + if (va[i] > vb[i]) aNoMorePermissive = false + if (vb[i] > va[i]) bNoMorePermissive = false + } + if (aNoMorePermissive) return a + if (bNoMorePermissive) return b + return undefined +} + +function resolveRightsConflict(current: RightsRecord, incoming: RightsRecord): RightsRecord { + const winner = stricterLicense(current.license, incoming.license) + if (winner === current.license) return current + if (winner === incoming.license) return incoming + // Incomparable claims about the same work: no honest single license id exists, + // so strict-deny to 'unknown' (needs-review). Keep the current record's per-item + // data as the audit anchor; drop licenseVersion (meaningless off a CC family). + return { ...current, license: 'unknown', licenseVersion: undefined } } // Reciprocal Rank Fusion across per-source ranked lists. Each list is assumed already @@ -15,6 +79,8 @@ export function mergeReferences(perSource: Reference[][], opts: MergeOptions = { const k = opts.k ?? 60 const score = new Map() // dedup key -> accumulated RRF score const rep = new Map() // dedup key -> best representative + const rights = new Map() // dedup key -> conservatively-resolved rights + const conflicted = new Set() // keys whose rights were resolved across sources for (const list of perSource) { list.forEach((ref, rank) => { @@ -22,6 +88,24 @@ export function mergeReferences(perSource: Reference[][], opts: MergeOptions = { score.set(key, (score.get(key) ?? 0) + 1 / (k + rank)) const cur = rep.get(key) if (!cur || ref.relevance > cur.relevance) rep.set(key, ref) + // Cross-source license conflict: same canonical URL, different license id. + // Resolve conservatively (see resolveRightsConflict); the resolved record + // replaces the representative's rights below. Same-id records never + // conflict — differing versions/authors are per-source metadata, and the + // representative's own record stays authoritative for them. + const known = rights.get(key) + if (known === undefined) { + rights.set(key, ref.rights) + } else if (known.license !== ref.rights.license) { + const resolved = resolveRightsConflict(known, ref.rights) + rights.set(key, resolved) + conflicted.add(key) + opts.onRightsConflict?.({ + canonicalUrl: ref.canonicalUrl, + licenses: [known.license, ref.rights.license], + resolvedLicense: resolved.license, + }) + } }) } @@ -34,9 +118,18 @@ export function mergeReferences(perSource: Reference[][], opts: MergeOptions = { let maxScore = -Infinity for (const s of score.values()) if (s > maxScore) maxScore = s const fused: Reference[] = [...score.entries()] - .map(([key, s]) => ({ ...rep.get(key)!, relevance: s / maxScore })) + .map(([key, s]) => ({ + ...rep.get(key)!, + // A conflicted key carries the conservatively-resolved rights instead of + // whichever source happened to supply the representative. + ...(conflicted.has(key) ? { rights: rights.get(key)! } : {}), + relevance: s / maxScore, + })) .sort((a, b) => b.relevance - a.relevance) - // Perceptual-hash dedup as a second pass (URL dedup already happened via the key map). + // Perceptual-hash dedup as a second pass (URL dedup already happened via the key + // map). No rights resolution here: perceptual duplicates from different sources + // are DIFFERENT postings — each is a genuine offer under its own license, so + // keeping the representative's license is correct, unlike the same-URL case. return dedupeReferences(fused, opts) } diff --git a/packages/core/src/provider-run.ts b/packages/core/src/provider-run.ts new file mode 100644 index 0000000..c28f61e --- /dev/null +++ b/packages/core/src/provider-run.ts @@ -0,0 +1,123 @@ +// One provider's slice of a search fan-out: cache read → live search → per-item +// parse → cache write, under the orchestrator's deadline. Extracted from the +// client so the pipeline stage is testable on its own; the client owns fan-out, +// merge/rerank/gate and meta assembly. +import type { Reference } from './reference' +import { parseReference } from './reference' +import type { KeyValueCache, NormalizedQuery, ProviderContext, ReferenceProvider } from './provider' +import { withTimeout } from './resilience' + +// Deterministic JSON for cache keys: object keys sorted recursively, so a +// caller's providerOptions key order can't split otherwise-identical keys. +// Keys whose value is `undefined` are skipped, matching JSON.stringify's own +// object semantics — `{ a: 1, b: undefined }` and `{ a: 1 }` must key alike. +export function stableStringify(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]` + if (value && typeof value === 'object') { + const rec = value as Record + return `{${Object.keys(rec).filter(k => rec[k] !== undefined).sort().map(k => `${JSON.stringify(k)}:${stableStringify(rec[k])}`).join(',')}}` + } + return JSON.stringify(value) ?? 'null' +} + +/** Cache key for one provider's slice of a search. Embeds the FULL normalized + * query (not a truncated hash): a 32-bit digest would let two different queries + * silently collide and serve each other's cached results. Keys can therefore be + * a few hundred bytes — KeyValueCache implementations must tolerate that. */ +export function providerCacheKey(providerId: string, query: NormalizedQuery): string { + return `refkit:v1:${providerId}:${stableStringify(query)}` +} + +export interface ProviderRunDeps { + /** Already retry-wrapped by the orchestrator and shared across the fan-out. */ + fetch: typeof fetch + cache?: KeyValueCache + cacheTtlMs: number + /** false → strip `raw` from cached payloads (smaller cache entries; cache-hit + * refs then have no `raw`, so raw-reading isDuplicate hooks won't see it). */ + cacheRaw: boolean + /** Per-provider soft deadline; undefined → no deadline (resilience disabled). */ + timeoutMs?: number + signal?: AbortSignal + /** Reports BOTH per-item parse failures and a failed search. */ + onError?: (error: unknown) => void +} + +export type ProviderRun = + | { ok: true; valid: Reference[]; returned: number; latencyMs: number; cached?: boolean } + | { ok: false; error: unknown; latencyMs: number } + +export async function runProviderSearch( + provider: ReferenceProvider, + query: NormalizedQuery, + deps: ProviderRunDeps, +): Promise { + const started = Date.now() + const timeout = deps.timeoutMs !== undefined ? withTimeout(deps.signal, deps.timeoutMs) : undefined + const ctx: ProviderContext = { + fetch: deps.fetch, + cache: deps.cache, + signal: timeout?.signal ?? deps.signal, + } + const cacheKey = deps.cache ? providerCacheKey(provider.id, query) : undefined + // Race a promise against the deadline without leaking an unhandled rejection + // for whichever side loses the race. + const raceDeadline = (p: Promise): Promise => { + p.catch(() => {}) + return timeout ? Promise.race([p, timeout.expired]) : p + } + // Parse raw provider items one at a time — a single bad item must not + // discard the rest (shared by the cache-hit and live-search paths). + const parseItems = (raw: unknown[]): Reference[] => { + const valid: Reference[] = [] + for (const item of raw) { + try { + valid.push(parseReference(item)) + } catch (error) { + deps.onError?.(error) + } + } + return valid + } + try { + if (deps.cache && cacheKey) { + // best-effort: a broken/corrupt/stale cache degrades to a live search + const pending = deps.cache.get(cacheKey) + // A slow cache read must not outlive the provider deadline: at expiry it + // degrades to a miss, and the live search below then fails fast on the + // same (already-expired) deadline. + const hit = await (timeout + ? Promise.race([pending, timeout.expired.catch(() => undefined)]) + : pending + ).catch(() => undefined) + pending.catch(() => {}) // raced-past rejection must not go unhandled + if (hit !== undefined) { + try { + // cached refs keep their original verifiedAt — staleness is bounded + // by the TTL when the cache honors ttlMs. Only a whole-payload + // failure (not JSON, not an array) falls through to live; a single + // bad item within an otherwise-valid array is reported and dropped, + // same as the live path. + const raw = JSON.parse(hit) as unknown[] + if (!Array.isArray(raw)) throw new Error('cached payload is not an array') + const valid = parseItems(raw) + return { ok: true, valid, returned: raw.length, latencyMs: Date.now() - started, cached: true } + } catch { /* fall through to live */ } + } + } + const searching = provider.search(query, ctx) + const raw = await raceDeadline(searching) + const valid = parseItems(raw) + if (deps.cache && cacheKey) { + const payload = deps.cacheRaw ? valid : valid.map(({ raw: _raw, ...rest }) => rest) + // fire-and-forget: cache write failure must never fail the search + void deps.cache.set(cacheKey, JSON.stringify(payload), deps.cacheTtlMs).catch(() => {}) + } + return { ok: true, valid, returned: raw.length, latencyMs: Date.now() - started } + } catch (error) { + deps.onError?.(error) + return { ok: false, error, latencyMs: Date.now() - started } + } finally { + timeout?.cancel() + } +} diff --git a/packages/core/src/provider.ts b/packages/core/src/provider.ts index 2be5721..0b77867 100644 --- a/packages/core/src/provider.ts +++ b/packages/core/src/provider.ts @@ -1,6 +1,9 @@ import type { Modality } from './modality' import type { Reference } from './reference' +/** @deprecated Superseded by {@link ProviderCapabilities}`.controls` — declare + * supported {@link SearchControlKey}s instead. No longer read by core routing; + * will be removed in a future minor. */ export type QueryFeature = | 'keyword' | 'color' @@ -74,6 +77,9 @@ export interface ProviderCapabilities { controls: readonly SearchControlKey[] } +/** @deprecated Compatibility alias for {@link SearchControls} `color` / + * `orientation` / `language`. Values are merged into `controls` (controls win on + * conflict) and routed by `capabilities.controls`; use `controls` directly. */ export interface SearchFilters { color?: string orientation?: 'landscape' | 'portrait' | 'square' @@ -87,6 +93,9 @@ export type ProviderOptionsById = Record export interface NormalizedQuery { text: string modalities: Modality[] + /** @deprecated Mirror of the routed `controls` color/orientation/language, kept + * for providers still reading the legacy channel — always consistent with + * `controls`. Read `controls` instead. */ filters?: SearchFilters controls?: SearchControls providerOptions?: ProviderOptions @@ -111,13 +120,14 @@ export interface ProviderContext { signal?: AbortSignal } -// 4 load-bearing fields. Keys are held by the provider's factory closure +// 3 load-bearing fields. Keys are held by the provider's factory closure // (e.g. `unsplash({ accessKey })`), not declared here; rate-limit metadata is added // in P1 when the orchestrator implements throttling. export interface ReferenceProvider { id: string modalities: Modality[] - queryFeatures: QueryFeature[] + /** @deprecated Not read by core anymore — declare `capabilities.controls`. */ + queryFeatures?: QueryFeature[] capabilities?: ProviderCapabilities search(query: NormalizedQuery, ctx: ProviderContext): Promise } diff --git a/packages/core/src/query.ts b/packages/core/src/query.ts index 5d90700..3a0f33c 100644 --- a/packages/core/src/query.ts +++ b/packages/core/src/query.ts @@ -102,13 +102,16 @@ export function normalizeQuery( input: { query: string; modalities: Modality[]; filters?: SearchFilters; controls?: SearchControls; providerOptions?: ProviderOptionsById; limit?: number }, provider: ReferenceProvider, ): NormalizedQuery { - const feats = new Set(provider.queryFeatures) + // Single-track routing: legacy `filters` are merged into `controls` (controls + // win on conflict) and routed by `capabilities.controls` alone. The deprecated + // NormalizedQuery.filters channel is then DERIVED from the routed controls, so + // a provider reading either channel sees the same values — no double semantics. + const controls = normalizeControlsForProvider(input, provider) const filters: SearchFilters = {} - if (input.filters?.color && feats.has('color')) filters.color = input.filters.color - if (input.filters?.orientation && feats.has('orientation')) filters.orientation = input.filters.orientation - if (input.filters?.language && feats.has('language')) filters.language = input.filters.language + if (controls?.color) filters.color = controls.color + if (controls?.orientation) filters.orientation = controls.orientation + if (controls?.language) filters.language = controls.language const hasFilters = Object.keys(filters).length > 0 - const controls = normalizeControlsForProvider(input, provider) return { text: input.query, modalities: input.modalities.filter(m => provider.modalities.includes(m)), diff --git a/packages/core/src/reference.ts b/packages/core/src/reference.ts index 27d0269..bbe9f88 100644 --- a/packages/core/src/reference.ts +++ b/packages/core/src/reference.ts @@ -30,7 +30,10 @@ export interface Reference { visual?: VisualMeta text?: TextMeta // — retrieval — - relevance: number // 0..1 orderable score; RRF-fused at merge, may be rewritten by a reranker + // 0..1. Providers emit a placeholder 0 (per-source order carries the signal); + // mergeReferences rewrites it to the normalized RRF score, and a reranker may + // rewrite it again. Only post-merge values are meaningful for comparison. + relevance: number raw?: unknown } diff --git a/packages/core/src/rerank.ts b/packages/core/src/rerank.ts index b9a6a03..f40b77c 100644 --- a/packages/core/src/rerank.ts +++ b/packages/core/src/rerank.ts @@ -28,12 +28,25 @@ const STOPWORDS = new Set([ 'by', 'from', 'as', 'is', 'are', 'it', 'this', 'that', ]) -/** Lowercase, split on runs of non-alphanumerics, drop stopwords and 1-char tokens. */ +// CJK scripts have no word boundaries to split on, so character bigrams are the +// standard zero-dependency indexing unit (Han incl. compatibility ideographs, +// kana, hangul — BMP ranges). +const CJK_RUNS = /[぀-ヿ㐀-䶿一-鿿豈-﫿가-힯]+/g + +/** Latin/digit runs: lowercase, split on non-alphanumerics, drop stopwords and + * 1-char tokens. CJK runs: character bigrams (a lone char stays a unigram), so + * CJK queries score instead of tokenizing to nothing. */ export function tokenize(text: string): string[] { - return text - .toLowerCase() + const lower = text.toLowerCase() + const out = lower .split(/[^a-z0-9]+/) .filter((t) => t.length > 1 && !STOPWORDS.has(t)) + for (const run of lower.match(CJK_RUNS) ?? []) { + const chars = [...run] + if (chars.length === 1) out.push(chars[0]) + else for (let i = 0; i < chars.length - 1; i++) out.push(chars[i] + chars[i + 1]) + } + return out } /** Tuning weights for {@link lexicalReranker}. All weights are clamped to ≥ 0. */ diff --git a/packages/mcp/package.json b/packages/mcp/package.json index bb03ad7..7c16b76 100644 --- a/packages/mcp/package.json +++ b/packages/mcp/package.json @@ -38,22 +38,13 @@ "dependencies": { "@refkit/core": "workspace:*", "@refkit/provider-artic": "workspace:*", - "@refkit/provider-brave": "workspace:*", - "@refkit/provider-europeana": "workspace:*", - "@refkit/provider-flickr": "workspace:*", - "@refkit/provider-freesound": "workspace:*", "@refkit/provider-gutendex": "workspace:*", "@refkit/provider-internet-archive": "workspace:*", - "@refkit/provider-jamendo": "workspace:*", "@refkit/provider-met": "workspace:*", "@refkit/provider-openverse": "workspace:*", - "@refkit/provider-pexels": "workspace:*", - "@refkit/provider-pixabay": "workspace:*", "@refkit/provider-poetrydb": "workspace:*", "@refkit/provider-polyhaven": "workspace:*", "@refkit/provider-rijksmuseum": "workspace:*", - "@refkit/provider-smithsonian": "workspace:*", - "@refkit/provider-unsplash": "workspace:*", "@refkit/provider-wikimedia-commons": "workspace:*", "@modelcontextprotocol/sdk": "^1.29.0", "zod": "^4.3.6" @@ -71,5 +62,16 @@ "import": "./dist/index.js" } } + }, + "optionalDependencies": { + "@refkit/provider-brave": "workspace:*", + "@refkit/provider-europeana": "workspace:*", + "@refkit/provider-flickr": "workspace:*", + "@refkit/provider-freesound": "workspace:*", + "@refkit/provider-jamendo": "workspace:*", + "@refkit/provider-pexels": "workspace:*", + "@refkit/provider-pixabay": "workspace:*", + "@refkit/provider-smithsonian": "workspace:*", + "@refkit/provider-unsplash": "workspace:*" } } diff --git a/packages/mcp/src/__tests__/mcp.test.ts b/packages/mcp/src/__tests__/mcp.test.ts index 99f6f9f..20980d3 100644 --- a/packages/mcp/src/__tests__/mcp.test.ts +++ b/packages/mcp/src/__tests__/mcp.test.ts @@ -89,7 +89,7 @@ describe('@refkit/mcp', () => { const fakeProvider = defineProvider({ id: 'fake', modalities: ['image'], - queryFeatures: ['keyword', 'orientation'], + capabilities: { controls: ['orientation'] }, search: async (q) => { seen = { filters: q.filters, providerOptions: q.providerOptions } return [] @@ -357,43 +357,43 @@ describe('build_attribution tool', () => { }) describe('defaultProviders (zero-config CLI wiring)', () => { - it('includes every keyless provider by default', () => { - const ids = defaultProviders({}).map(p => p.id) + it('includes every keyless provider by default', async () => { + const ids = (await defaultProviders({})).map(p => p.id) for (const id of ['openverse', 'wikimedia-commons', 'met', 'artic', 'gutendex', 'poetrydb', 'rijksmuseum', 'polyhaven', 'ambientcg', 'internet-archive']) { expect(ids).toContain(id) } }) - it('adds a BYOK provider only when its env key is present', () => { - expect(defaultProviders({}).map(p => p.id)).not.toContain('unsplash') - expect(defaultProviders({ UNSPLASH_KEY: 'k' }).map(p => p.id)).toContain('unsplash') + it('adds a BYOK provider only when its env key is present', async () => { + expect((await defaultProviders({})).map(p => p.id)).not.toContain('unsplash') + expect((await defaultProviders({ UNSPLASH_KEY: 'k' })).map(p => p.id)).toContain('unsplash') }) - it('adds a BYOK provider when only the unified REFKIT_ env key is present', () => { - expect(defaultProviders({ REFKIT_UNSPLASH_KEY: 'k' }).map(p => p.id)).toContain('unsplash') + it('adds a BYOK provider when only the unified REFKIT_ env key is present', async () => { + expect((await defaultProviders({ REFKIT_UNSPLASH_KEY: 'k' })).map(p => p.id)).toContain('unsplash') }) - it('adds freesound only when FREESOUND_TOKEN is present', () => { - expect(defaultProviders({}).map(p => p.id)).not.toContain('freesound') - expect(defaultProviders({ FREESOUND_TOKEN: 'k' }).map(p => p.id)).toContain('freesound') + it('adds freesound only when FREESOUND_TOKEN is present', async () => { + expect((await defaultProviders({})).map(p => p.id)).not.toContain('freesound') + expect((await defaultProviders({ FREESOUND_TOKEN: 'k' })).map(p => p.id)).toContain('freesound') }) - it('adds jamendo only when JAMENDO_CLIENT_ID is present', () => { - expect(defaultProviders({}).map(p => p.id)).not.toContain('jamendo') - expect(defaultProviders({ JAMENDO_CLIENT_ID: 'k' }).map(p => p.id)).toContain('jamendo') + it('adds jamendo only when JAMENDO_CLIENT_ID is present', async () => { + expect((await defaultProviders({})).map(p => p.id)).not.toContain('jamendo') + expect((await defaultProviders({ JAMENDO_CLIENT_ID: 'k' })).map(p => p.id)).toContain('jamendo') }) - it('adds europeana only when EUROPEANA_KEY is present', () => { - expect(defaultProviders({}).map(p => p.id)).not.toContain('europeana') - expect(defaultProviders({ EUROPEANA_KEY: 'k' }).map(p => p.id)).toContain('europeana') + it('adds europeana only when EUROPEANA_KEY is present', async () => { + expect((await defaultProviders({})).map(p => p.id)).not.toContain('europeana') + expect((await defaultProviders({ EUROPEANA_KEY: 'k' })).map(p => p.id)).toContain('europeana') }) - it('adds europeana when only the unified REFKIT_EUROPEANA_KEY is present', () => { - expect(defaultProviders({ REFKIT_EUROPEANA_KEY: 'k' }).map(p => p.id)).toContain('europeana') + it('adds europeana when only the unified REFKIT_EUROPEANA_KEY is present', async () => { + expect((await defaultProviders({ REFKIT_EUROPEANA_KEY: 'k' })).map(p => p.id)).toContain('europeana') }) - it('adds smithsonian via the legacy SI_KEY name (no unified alias renames the id)', () => { - expect(defaultProviders({}).map(p => p.id)).not.toContain('smithsonian') - expect(defaultProviders({ SI_KEY: 'k' }).map(p => p.id)).toContain('smithsonian') + it('adds smithsonian via the legacy SI_KEY name (no unified alias renames the id)', async () => { + expect((await defaultProviders({})).map(p => p.id)).not.toContain('smithsonian') + expect((await defaultProviders({ SI_KEY: 'k' })).map(p => p.id)).toContain('smithsonian') }) }) diff --git a/packages/mcp/src/cli.ts b/packages/mcp/src/cli.ts index 4fa5ba0..3f10bca 100644 --- a/packages/mcp/src/cli.ts +++ b/packages/mcp/src/cli.ts @@ -10,49 +10,97 @@ import { met } from '@refkit/provider-met' import { artic } from '@refkit/provider-artic' import { gutendex } from '@refkit/provider-gutendex' import { poetrydb } from '@refkit/provider-poetrydb' -import { unsplash } from '@refkit/provider-unsplash' -import { pexels, pexelsVideo } from '@refkit/provider-pexels' -import { pixabay, pixabayVideo } from '@refkit/provider-pixabay' -import { flickr } from '@refkit/provider-flickr' -import { smithsonian } from '@refkit/provider-smithsonian' -import { brave } from '@refkit/provider-brave' import { rijksmuseum } from '@refkit/provider-rijksmuseum' import { polyhaven, ambientcg } from '@refkit/provider-polyhaven' -import { freesound } from '@refkit/provider-freesound' -import { jamendo } from '@refkit/provider-jamendo' -import { europeana } from '@refkit/provider-europeana' import { internetArchive } from '@refkit/provider-internet-archive' import { serveStdio } from './index' -/** Providers a zero-config server boots with: all keyless sources, plus any BYOK - * source whose key is present in `env`. Exported so the wiring is unit-testable. - * - * Env var convention: each BYOK key is read as `REFKIT__KEY` first (the - * unified name), falling back to the provider's legacy env var name — both are - * honored indefinitely, the unified name is just preferred going forward. */ -export function defaultProviders(env: NodeJS.ProcessEnv = process.env): ReferenceProvider[] { +/** One BYOK source: its unified + legacy env var names, and a lazy loader for its + * (optional) provider package. Loaded only when a key is present, so installs + * that omitted optionalDependencies still boot — the missing source is skipped + * with a stderr warning instead of crashing the server. */ +interface ByokSource { + pkg: string + key: (env: NodeJS.ProcessEnv) => string | undefined + load: (key: string) => Promise +} + +// Env var convention: each BYOK key is read as `REFKIT__KEY` first (the +// unified name), falling back to the provider's legacy env var name — both are +// honored indefinitely, the unified name is just preferred going forward. +const BYOK_SOURCES: ByokSource[] = [ + { + pkg: '@refkit/provider-unsplash', + key: (env) => env.REFKIT_UNSPLASH_KEY ?? env.UNSPLASH_KEY, + load: async (accessKey) => [(await import('@refkit/provider-unsplash')).unsplash({ accessKey })], + }, + { + pkg: '@refkit/provider-pexels', + key: (env) => env.REFKIT_PEXELS_KEY ?? env.PEXELS_KEY, + load: async (apiKey) => { + const m = await import('@refkit/provider-pexels') + return [m.pexels({ apiKey }), m.pexelsVideo({ apiKey })] + }, + }, + { + pkg: '@refkit/provider-pixabay', + key: (env) => env.REFKIT_PIXABAY_KEY ?? env.PIXABAY_KEY, + load: async (key) => { + const m = await import('@refkit/provider-pixabay') + return [m.pixabay({ key }), m.pixabayVideo({ key })] + }, + }, + { + pkg: '@refkit/provider-flickr', + key: (env) => env.REFKIT_FLICKR_KEY ?? env.FLICKR_KEY, + load: async (apiKey) => [(await import('@refkit/provider-flickr')).flickr({ apiKey })], + }, + { + pkg: '@refkit/provider-smithsonian', + key: (env) => env.REFKIT_SMITHSONIAN_KEY ?? env.SI_KEY, + load: async (apiKey) => [(await import('@refkit/provider-smithsonian')).smithsonian({ apiKey })], + }, + { + pkg: '@refkit/provider-brave', + key: (env) => env.REFKIT_BRAVE_KEY ?? env.BRAVE_TOKEN, + load: async (token) => [(await import('@refkit/provider-brave')).brave({ token })], + }, + { + pkg: '@refkit/provider-freesound', + key: (env) => env.REFKIT_FREESOUND_KEY ?? env.FREESOUND_TOKEN, + load: async (apiKey) => [(await import('@refkit/provider-freesound')).freesound({ apiKey })], + }, + { + pkg: '@refkit/provider-jamendo', + key: (env) => env.REFKIT_JAMENDO_CLIENT_ID ?? env.JAMENDO_CLIENT_ID, + load: async (clientId) => [(await import('@refkit/provider-jamendo')).jamendo({ clientId })], + }, + { + pkg: '@refkit/provider-europeana', + key: (env) => env.REFKIT_EUROPEANA_KEY ?? env.EUROPEANA_KEY, + load: async (apiKey) => [(await import('@refkit/provider-europeana')).europeana({ apiKey })], + }, +] + +/** Providers a zero-config server boots with: all keyless sources (hard deps, + * statically imported), plus any BYOK source whose key is present in `env` + * (optionalDependencies, dynamically imported on demand). Exported so the + * wiring is unit-testable. */ +export async function defaultProviders(env: NodeJS.ProcessEnv = process.env): Promise { const providers: ReferenceProvider[] = [ openverse(), openverseAudio(), wikimediaCommons(), met(), artic(), gutendex(), poetrydb(), rijksmuseum(), polyhaven(), ambientcg(), internetArchive(), ] - const unsplashKey = env.REFKIT_UNSPLASH_KEY ?? env.UNSPLASH_KEY - const pexelsKey = env.REFKIT_PEXELS_KEY ?? env.PEXELS_KEY - const pixabayKey = env.REFKIT_PIXABAY_KEY ?? env.PIXABAY_KEY - const flickrKey = env.REFKIT_FLICKR_KEY ?? env.FLICKR_KEY - const smithsonianKey = env.REFKIT_SMITHSONIAN_KEY ?? env.SI_KEY - const braveKey = env.REFKIT_BRAVE_KEY ?? env.BRAVE_TOKEN - const freesoundKey = env.REFKIT_FREESOUND_KEY ?? env.FREESOUND_TOKEN - const jamendoClientId = env.REFKIT_JAMENDO_CLIENT_ID ?? env.JAMENDO_CLIENT_ID - const europeanaKey = env.REFKIT_EUROPEANA_KEY ?? env.EUROPEANA_KEY - if (unsplashKey) providers.push(unsplash({ accessKey: unsplashKey })) - if (pexelsKey) providers.push(pexels({ apiKey: pexelsKey }), pexelsVideo({ apiKey: pexelsKey })) - if (pixabayKey) providers.push(pixabay({ key: pixabayKey }), pixabayVideo({ key: pixabayKey })) - if (flickrKey) providers.push(flickr({ apiKey: flickrKey })) - if (smithsonianKey) providers.push(smithsonian({ apiKey: smithsonianKey })) - if (braveKey) providers.push(brave({ token: braveKey })) - if (freesoundKey) providers.push(freesound({ apiKey: freesoundKey })) - if (jamendoClientId) providers.push(jamendo({ clientId: jamendoClientId })) - if (europeanaKey) providers.push(europeana({ apiKey: europeanaKey })) + for (const source of BYOK_SOURCES) { + const key = source.key(env) + if (!key) continue + try { + providers.push(...await source.load(key)) + } catch { + // stderr only — stdout is the MCP transport. + console.error(`[refkit-mcp] key for ${source.pkg} is set but the package is not installed — skipping this source. Reinstall @refkit/mcp with optional dependencies (or add ${source.pkg}) to enable it.`) + } + } return providers } @@ -68,5 +116,5 @@ const isEntry = (() => { })() if (isEntry) { - await serveStdio(createRefkit({ providers: defaultProviders() })) + await serveStdio(createRefkit({ providers: await defaultProviders() })) } diff --git a/packages/mcp/src/index.ts b/packages/mcp/src/index.ts index 9115052..33c7ca0 100644 --- a/packages/mcp/src/index.ts +++ b/packages/mcp/src/index.ts @@ -2,7 +2,7 @@ import { readFileSync } from 'node:fs' import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js' import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js' import { z } from 'zod' -import { LICENSE_IDS, INTENTS, evaluateUse, buildAttribution, ccVersionFor } from '@refkit/core' +import { LICENSE_IDS, INTENTS, evaluateUse, buildAttribution, ccVersionFor, lexicalReranker } from '@refkit/core' import type { RefkitClient, Reference, Verdict, Attribution, SearchFilters, SearchControls, SearchControlKey, ProviderOptionsById, SearchMeta, RightsRecord } from '@refkit/core' const MODALITIES = ['image', 'video', 'audio', 'text'] as const @@ -148,6 +148,7 @@ const searchMetaSchema: z.ZodType = z.object({ after: z.number(), dropped: z.number(), }).optional(), + nextCursor: z.string().optional().describe('opaque load-more cursor; pass back as `cursor` to fetch the next page with cross-page dedup'), warnings: z.array(z.string()), }) @@ -170,14 +171,16 @@ export function createRefkitMcpServer(refkit: RefkitClient): McpServer { filters: filtersSchema.optional().describe('compatibility alias for controls.orientation, controls.color, and controls.language'), controls: searchControlsSchema.optional().describe('provider-neutral search controls; providers translate supported controls and report ignored controls in explain metadata'), providerOptions: providerOptionsSchema.optional().describe('provider-specific search controls keyed by provider id; each provider whitelists supported keys'), - explain: z.boolean().optional().describe('include provider status, applied and ignored controls, warnings, and gate/drop metadata'), + explain: z.boolean().optional().describe('include provider status, applied and ignored controls, warnings, gate/drop metadata, and the load-more cursor'), limit: z.number().int().positive().optional(), + cursor: z.string().optional().describe('opaque cursor from a previous result\'s meta.nextCursor — fetches the next page, deduped against earlier pages (requires explain to read the next cursor)'), + rerank: z.boolean().optional().describe('re-rank results by query relevance (term coverage incl. CJK, resolution, source diversity) instead of raw cross-source rank fusion'), intent: z.enum(INTENTS).optional().describe('annotate each result with a use-verdict for this intended use (no filtering)'), gateFor: z.enum(INTENTS).optional().describe('only return results whose license allows this intended use'), }, outputSchema: { references: z.array(agentRefSchema), meta: searchMetaSchema.optional() }, }, - async ({ query, modalities, filters, controls, providerOptions, explain, limit, intent, gateFor }) => { + async ({ query, modalities, filters, controls, providerOptions, explain, limit, cursor, rerank, intent, gateFor }) => { const searchInput = { query, modalities: modalities ?? ['image'], @@ -185,6 +188,8 @@ export function createRefkitMcpServer(refkit: RefkitClient): McpServer { controls: controls as SearchControls | undefined, providerOptions: providerOptions as ProviderOptionsById | undefined, limit, + cursor, + ...(rerank ? { rerank: lexicalReranker() } : {}), gateFor, } const result = explain ? await refkit.searchWithMeta(searchInput) : { references: await refkit.search(searchInput), meta: undefined } diff --git a/packages/provider-artic/src/index.ts b/packages/provider-artic/src/index.ts index 54f86cc..70845bb 100644 --- a/packages/provider-artic/src/index.ts +++ b/packages/provider-artic/src/index.ts @@ -70,8 +70,7 @@ export function artic() { return defineProvider({ id: 'artic', modalities: ['image'], - queryFeatures: ['keyword'], - capabilities: { controls: [] }, + capabilities: { controls: ['page'] }, async search(q: NormalizedQuery, ctx: ProviderContext): Promise { const url = new URL('https://api.artic.edu/api/v1/artworks/search') url.searchParams.set('q', q.text) @@ -80,6 +79,7 @@ export function artic() { url.searchParams.set('query[term][is_public_domain]', 'true') url.searchParams.set('fields', articFields(opts?.fields)) url.searchParams.set('limit', String(q.limit ?? 20)) + if (q.controls?.page) url.searchParams.set('page', String(q.controls.page)) setIfString(url, 'sort', opts?.sort) setIfNonNegativeInt(url, 'from', opts?.from) setIfNonNegativeInt(url, 'size', opts?.size) diff --git a/packages/provider-brave/src/index.ts b/packages/provider-brave/src/index.ts index 5c639b4..5bb09a9 100644 --- a/packages/provider-brave/src/index.ts +++ b/packages/provider-brave/src/index.ts @@ -59,7 +59,6 @@ export function brave(config: BraveConfig) { return defineProvider({ id: 'brave', modalities: ['image'], - queryFeatures: ['keyword'], capabilities: { controls: ['safety'] }, async search(q: NormalizedQuery, ctx: ProviderContext): Promise { const url = new URL('https://api.search.brave.com/res/v1/images/search') diff --git a/packages/provider-europeana/src/index.ts b/packages/provider-europeana/src/index.ts index b9fe0b5..8ad4f3d 100644 --- a/packages/provider-europeana/src/index.ts +++ b/packages/provider-europeana/src/index.ts @@ -96,7 +96,6 @@ export function europeana(config: EuropeanaConfig) { return defineProvider({ id: 'europeana', modalities: ['image'], - queryFeatures: ['keyword'], capabilities: { controls: [] }, async search(q: NormalizedQuery, ctx: ProviderContext): Promise { const url = new URL(BASE) diff --git a/packages/provider-flickr/src/index.ts b/packages/provider-flickr/src/index.ts index d1ffd89..60161cd 100644 --- a/packages/provider-flickr/src/index.ts +++ b/packages/provider-flickr/src/index.ts @@ -179,7 +179,6 @@ export function flickr(config: FlickrConfig) { return defineProvider({ id: 'flickr', modalities: ['image'], - queryFeatures: ['keyword'], capabilities: { controls: ['sort', 'safety', 'license.commercial', 'license.modification', 'license.allowUnknown', 'creator.id'] }, async search(q: NormalizedQuery, ctx: ProviderContext): Promise { const opts = q.providerOptions as FlickrSearchOptions | undefined diff --git a/packages/provider-freesound/src/index.ts b/packages/provider-freesound/src/index.ts index 5228587..ce0c512 100644 --- a/packages/provider-freesound/src/index.ts +++ b/packages/provider-freesound/src/index.ts @@ -94,7 +94,6 @@ export function freesound(config: FreesoundConfig) { return defineProvider({ id: 'freesound', modalities: ['audio'], - queryFeatures: ['keyword'], capabilities: { controls: [] }, async search(q: NormalizedQuery, ctx: ProviderContext): Promise { const opts = q.providerOptions as FreesoundSearchOptions | undefined diff --git a/packages/provider-gutendex/src/index.ts b/packages/provider-gutendex/src/index.ts index 3a53c26..97da611 100644 --- a/packages/provider-gutendex/src/index.ts +++ b/packages/provider-gutendex/src/index.ts @@ -73,7 +73,6 @@ export function gutendex(config: GutendexConfig = {}) { return defineProvider({ id: 'gutendex', modalities: ['text'], - queryFeatures: ['keyword'], capabilities: { controls: ['language', 'text.copyright', 'page'] }, async search(q: NormalizedQuery, ctx: ProviderContext): Promise { const url = new URL('https://gutendex.com/books/') diff --git a/packages/provider-internet-archive/src/index.ts b/packages/provider-internet-archive/src/index.ts index af10186..6818e4c 100644 --- a/packages/provider-internet-archive/src/index.ts +++ b/packages/provider-internet-archive/src/index.ts @@ -84,8 +84,7 @@ export function internetArchive(config: InternetArchiveConfig = {}) { return defineProvider({ id: 'internet-archive', modalities: ['video', 'text'], - queryFeatures: ['keyword'], - capabilities: { controls: [] }, + capabilities: { controls: ['page'] }, async search(q: NormalizedQuery, ctx: ProviderContext): Promise { const url = new URL(BASE) url.searchParams.set('q', `(${q.text}) AND mediatype:(movies OR texts)`) @@ -93,7 +92,7 @@ export function internetArchive(config: InternetArchiveConfig = {}) { url.searchParams.append('fl[]', f) } url.searchParams.set('output', 'json') - url.searchParams.set('page', '1') + url.searchParams.set('page', String(q.controls?.page ?? 1)) const rows = Math.min(config.maxRows ?? q.limit ?? 20, 100) url.searchParams.set('rows', String(rows)) const res = await ctx.fetch(url.toString(), { signal: ctx.signal }) diff --git a/packages/provider-jamendo/src/index.ts b/packages/provider-jamendo/src/index.ts index 4febf75..866bba9 100644 --- a/packages/provider-jamendo/src/index.ts +++ b/packages/provider-jamendo/src/index.ts @@ -91,7 +91,6 @@ export function jamendo(config: JamendoConfig) { return defineProvider({ id: 'jamendo', modalities: ['audio'], - queryFeatures: ['keyword'], capabilities: { controls: [] }, async search(q: NormalizedQuery, ctx: ProviderContext): Promise { const url = new URL(BASE) diff --git a/packages/provider-met/src/index.ts b/packages/provider-met/src/index.ts index 8da2e66..c3f4b0d 100644 --- a/packages/provider-met/src/index.ts +++ b/packages/provider-met/src/index.ts @@ -69,7 +69,6 @@ export function met(config: MetConfig = {}) { return defineProvider({ id: 'met', modalities: ['image'], - queryFeatures: ['keyword'], capabilities: { controls: [] }, async search(q: NormalizedQuery, ctx: ProviderContext): Promise { const searchUrl = new URL(`${BASE}/search`) diff --git a/packages/provider-openverse/src/index.ts b/packages/provider-openverse/src/index.ts index a5e06e6..e05de15 100644 --- a/packages/provider-openverse/src/index.ts +++ b/packages/provider-openverse/src/index.ts @@ -152,13 +152,13 @@ export function openverse(config: OpenverseConfig = {}) { return defineProvider({ id: 'openverse', modalities: ['image'], - queryFeatures: ['keyword'], - capabilities: { controls: ['license.commercial', 'license.modification', 'license.allowUnknown'] }, + capabilities: { controls: ['license.commercial', 'license.modification', 'license.allowUnknown', 'page'] }, async search(q: NormalizedQuery, ctx: ProviderContext): Promise { const url = new URL('https://api.openverse.org/v1/images/') url.searchParams.set('q', q.text) url.searchParams.set('license_type', openverseLicenseType(q.controls?.license)) // performance/relevance hint only — the AUTHORITATIVE rights gate is mapOpenverseLicense below, not this filter url.searchParams.set('page_size', String(q.limit ?? 20)) + if (q.controls?.page) url.searchParams.set('page', String(q.controls.page)) const opts = q.providerOptions as OpenverseImageSearchOptions | undefined applyOpenverseSearchOptions(url, opts) setIfStringList(url, 'aspect_ratio', opts?.aspectRatio) @@ -223,13 +223,13 @@ export function openverseAudio(config: OpenverseConfig = {}) { return defineProvider({ id: 'openverse-audio', modalities: ['audio'], - queryFeatures: ['keyword'], - capabilities: { controls: ['license.commercial', 'license.modification', 'license.allowUnknown'] }, + capabilities: { controls: ['license.commercial', 'license.modification', 'license.allowUnknown', 'page'] }, async search(q: NormalizedQuery, ctx: ProviderContext): Promise { const url = new URL('https://api.openverse.org/v1/audio/') url.searchParams.set('q', q.text) url.searchParams.set('license_type', openverseLicenseType(q.controls?.license)) // relevance hint; mapOpenverseLicense authoritative url.searchParams.set('page_size', String(q.limit ?? 20)) + if (q.controls?.page) url.searchParams.set('page', String(q.controls.page)) const opts = q.providerOptions as OpenverseAudioSearchOptions | undefined applyOpenverseSearchOptions(url, opts) setIfStringList(url, 'length', opts?.length) diff --git a/packages/provider-pexels/src/index.ts b/packages/provider-pexels/src/index.ts index 3eb5e92..63a1445 100644 --- a/packages/provider-pexels/src/index.ts +++ b/packages/provider-pexels/src/index.ts @@ -80,7 +80,6 @@ export function pexels(config: PexelsConfig) { return defineProvider({ id: 'pexels', modalities: ['image'], - queryFeatures: ['keyword', 'color', 'orientation', 'language'], capabilities: { controls: ['orientation', 'color', 'language', 'media.size', 'page'] }, async search(q: NormalizedQuery, ctx: ProviderContext): Promise { const url = new URL('https://api.pexels.com/v1/search') @@ -141,7 +140,6 @@ export function pexelsVideo(config: PexelsConfig) { return defineProvider({ id: 'pexels-video', modalities: ['video'], - queryFeatures: ['keyword', 'orientation', 'language'], capabilities: { controls: ['orientation', 'language', 'media.size', 'page'] }, async search(q: NormalizedQuery, ctx: ProviderContext): Promise { const url = new URL('https://api.pexels.com/videos/search') diff --git a/packages/provider-pixabay/src/index.ts b/packages/provider-pixabay/src/index.ts index ea50f4c..39a54fb 100644 --- a/packages/provider-pixabay/src/index.ts +++ b/packages/provider-pixabay/src/index.ts @@ -102,14 +102,14 @@ export function pixabay(config: PixabayConfig) { return defineProvider({ id: 'pixabay', modalities: ['image'], - queryFeatures: ['keyword', 'color', 'orientation', 'language'], - capabilities: { controls: ['orientation', 'color', 'language', 'sort', 'safety', 'media.kind', 'media.minWidth', 'media.minHeight'] }, + capabilities: { controls: ['orientation', 'color', 'language', 'sort', 'safety', 'media.kind', 'media.minWidth', 'media.minHeight', 'page'] }, async search(q: NormalizedQuery, ctx: ProviderContext): Promise { const url = new URL('https://pixabay.com/api/') url.searchParams.set('key', config.key) url.searchParams.set('q', q.text) url.searchParams.set('image_type', 'photo') url.searchParams.set('per_page', String(Math.min(Math.max(q.limit ?? 20, 3), 200))) + if (q.controls?.page) url.searchParams.set('page', String(q.controls.page)) if (q.controls?.language) url.searchParams.set('lang', q.controls.language) if (q.controls?.color) url.searchParams.set('colors', q.controls.color) const controlsOrientation = pixabayOrientation(q.controls?.orientation) @@ -191,13 +191,13 @@ export function pixabayVideo(config: PixabayConfig) { return defineProvider({ id: 'pixabay-video', modalities: ['video'], - queryFeatures: ['keyword', 'language'], - capabilities: { controls: ['language', 'sort', 'safety', 'media.kind', 'media.minWidth', 'media.minHeight'] }, + capabilities: { controls: ['language', 'sort', 'safety', 'media.kind', 'media.minWidth', 'media.minHeight', 'page'] }, async search(q: NormalizedQuery, ctx: ProviderContext): Promise { const url = new URL('https://pixabay.com/api/videos/') url.searchParams.set('key', config.key) url.searchParams.set('q', q.text) url.searchParams.set('per_page', String(Math.min(Math.max(q.limit ?? 20, 3), 200))) + if (q.controls?.page) url.searchParams.set('page', String(q.controls.page)) if (q.controls?.language) url.searchParams.set('lang', q.controls.language) if (q.controls?.sort === 'latest' || q.controls?.sort === 'popular') url.searchParams.set('order', q.controls.sort) if (q.controls?.safety === 'strict') url.searchParams.set('safesearch', 'true') diff --git a/packages/provider-poetrydb/src/index.ts b/packages/provider-poetrydb/src/index.ts index 3770407..d346bbf 100644 --- a/packages/provider-poetrydb/src/index.ts +++ b/packages/provider-poetrydb/src/index.ts @@ -107,7 +107,6 @@ export function poetrydb() { return defineProvider({ id: 'poetrydb', modalities: ['text'], - queryFeatures: ['keyword'], capabilities: { controls: [] }, async search(q: NormalizedQuery, ctx: ProviderContext): Promise { // /lines/ finds poems whose line content contains the term (closest to keyword search) diff --git a/packages/provider-polyhaven/src/index.ts b/packages/provider-polyhaven/src/index.ts index 55406c4..cd407dc 100644 --- a/packages/provider-polyhaven/src/index.ts +++ b/packages/provider-polyhaven/src/index.ts @@ -83,7 +83,6 @@ export function polyhaven(config: PolyHavenConfig = {}) { return defineProvider({ id: 'polyhaven', modalities: ['image'], - queryFeatures: ['keyword'], capabilities: { controls: [] }, async search(q: NormalizedQuery, ctx: ProviderContext): Promise { const listUrl = new URL(`${PH_BASE}/assets`) @@ -171,7 +170,6 @@ export function ambientcg(config: AmbientCgConfig = {}) { return defineProvider({ id: 'ambientcg', modalities: ['image'], - queryFeatures: ['keyword'], capabilities: { controls: [] }, async search(q: NormalizedQuery, ctx: ProviderContext): Promise { const url = new URL(ACG_BASE) diff --git a/packages/provider-rijksmuseum/src/index.ts b/packages/provider-rijksmuseum/src/index.ts index 82e83dd..5dc13c1 100644 --- a/packages/provider-rijksmuseum/src/index.ts +++ b/packages/provider-rijksmuseum/src/index.ts @@ -138,7 +138,6 @@ export function rijksmuseum(config: RijksmuseumConfig = {}) { return defineProvider({ id: 'rijksmuseum', modalities: ['image'], - queryFeatures: ['keyword'], capabilities: { controls: [] }, async search(q: NormalizedQuery, ctx: ProviderContext): Promise { const opts = q.providerOptions as RijksmuseumSearchOptions | undefined diff --git a/packages/provider-smithsonian/src/index.ts b/packages/provider-smithsonian/src/index.ts index 5db4d02..2ec62b4 100644 --- a/packages/provider-smithsonian/src/index.ts +++ b/packages/provider-smithsonian/src/index.ts @@ -67,7 +67,6 @@ export function smithsonian(config: SmithsonianConfig) { return defineProvider({ id: 'smithsonian', modalities: ['image'], - queryFeatures: ['keyword'], capabilities: { controls: [] }, async search(q: NormalizedQuery, ctx: ProviderContext): Promise { const url = new URL('https://api.si.edu/openaccess/api/v1.0/search') diff --git a/packages/provider-unsplash/src/index.ts b/packages/provider-unsplash/src/index.ts index f51a8b7..fde114b 100644 --- a/packages/provider-unsplash/src/index.ts +++ b/packages/provider-unsplash/src/index.ts @@ -63,7 +63,6 @@ export function unsplash(config: UnsplashConfig) { return defineProvider({ id: 'unsplash', modalities: ['image'], - queryFeatures: ['keyword', 'color', 'orientation', 'language'], capabilities: { controls: ['orientation', 'color', 'language', 'sort', 'safety'] }, async search(q: NormalizedQuery, ctx: ProviderContext): Promise { const url = new URL('https://api.unsplash.com/search/photos') diff --git a/packages/provider-wikimedia-commons/src/index.ts b/packages/provider-wikimedia-commons/src/index.ts index c86807b..b55e9af 100644 --- a/packages/provider-wikimedia-commons/src/index.ts +++ b/packages/provider-wikimedia-commons/src/index.ts @@ -142,8 +142,7 @@ export function wikimediaCommons(config: WikimediaCommonsConfig = {}) { return defineProvider({ id: 'wikimedia-commons', modalities: ['image'], - queryFeatures: ['keyword'], - capabilities: { controls: [] }, + capabilities: { controls: ['page'] }, async search(q: NormalizedQuery, ctx: ProviderContext): Promise { const url = new URL('https://commons.wikimedia.org/w/api.php') url.searchParams.set('action', 'query') @@ -152,6 +151,8 @@ export function wikimediaCommons(config: WikimediaCommonsConfig = {}) { url.searchParams.set('gsrsearch', `${q.text} filetype:bitmap`) // raster images only url.searchParams.set('gsrnamespace', '6') // File: url.searchParams.set('gsrlimit', String(q.limit ?? 20)) + // offset-based API: translate the 1-based page control (providerOptions.gsroffset overrides below) + if (q.controls?.page && q.controls.page > 1) url.searchParams.set('gsroffset', String((q.controls.page - 1) * (q.limit ?? 20))) url.searchParams.set('prop', 'imageinfo') url.searchParams.set('iiprop', 'url|mime|size|extmetadata') url.searchParams.set('iiurlwidth', String(config.thumbWidth ?? 1024)) From a88f1056b2ed802b57cde69d4151514380e33edc Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 14 Jul 2026 09:07:06 +0000 Subject: [PATCH 2/5] =?UTF-8?q?feat:=20follow-ups=20=E2=80=94=20full=20pag?= =?UTF-8?q?e=20coverage,=20bounded=20fan-out,=20lint=20gate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - providers: wire controls.page across every source with usable pagination (unsplash/flickr/freesound native page; smithsonian/ europeana start; jamendo/ambientcg offset; met/polyhaven windowed slice) — the unified cursor now advances every paginatable source - core: RefkitOptions.concurrency bounds in-flight provider searches (default unlimited; queued providers keep their full timeout) - tooling: eslint (typescript-eslint syntactic recommended) + pnpm lint + CI lint step; fix the four existing violations Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_018hoN6wKWxcs8WbUNCM9rtt --- .changeset/architecture-review-core.md | 1 + .changeset/architecture-review-providers.md | 10 +- .github/workflows/ci.yml | 1 + README.md | 7 + eslint.config.mjs | 31 + package.json | 17 +- packages/core/src/__tests__/client.test.ts | 21 + packages/core/src/__tests__/reference.test.ts | 4 +- packages/core/src/__tests__/rights.test.ts | 2 +- packages/core/src/client.ts | 26 +- packages/core/src/resilience.ts | 13 +- packages/provider-europeana/src/index.ts | 4 +- packages/provider-flickr/src/index.ts | 3 +- packages/provider-freesound/src/index.ts | 3 +- packages/provider-jamendo/src/index.ts | 4 +- packages/provider-met/src/index.ts | 6 +- packages/provider-polyhaven/src/index.ts | 10 +- packages/provider-smithsonian/src/index.ts | 4 +- packages/provider-unsplash/src/index.ts | 3 +- pnpm-lock.yaml | 676 +++++++++++++++++- 20 files changed, 789 insertions(+), 57 deletions(-) create mode 100644 eslint.config.mjs diff --git a/.changeset/architecture-review-core.md b/.changeset/architecture-review-core.md index 9bef09a..4c2708c 100644 --- a/.changeset/architecture-review-core.md +++ b/.changeset/architecture-review-core.md @@ -9,5 +9,6 @@ Architecture-review hardening: - **CJK-aware `tokenize`** — CJK runs tokenize into character bigrams, so `lexicalReranker` scores Chinese/Japanese/Korean queries instead of dropping them. - **Collision-proof cache keys** — per-provider cache keys embed the full normalized query instead of a 32-bit hash (two distinct queries can no longer silently share a cache entry). Existing cache entries are invalidated by the key-format change. - New `cacheRaw: false` option strips `raw` provider payloads from cache entries. +- New `concurrency` option bounds how many provider searches run at once per search call (default unlimited, matching previous behavior); a queued provider's timeout starts only when it actually runs. - **Deprecations (single-track capability routing)** — `SearchFilters`, `SearchInput.filters`, `NormalizedQuery.filters`, `ReferenceProvider.queryFeatures` (now optional), and `QueryFeature` are deprecated. Routing is driven solely by `capabilities.controls`; legacy `filters` are merged into `controls` and the deprecated `NormalizedQuery.filters` channel is derived from the routed controls, so both channels always agree. Providers that declared filter support only via `queryFeatures` must declare `capabilities.controls` to keep receiving those values. - `runProviderSearch` / `providerCacheKey` / `stableStringify` extracted and exported (`provider-run`), shrinking the search orchestrator. diff --git a/.changeset/architecture-review-providers.md b/.changeset/architecture-review-providers.md index 6b86f19..d371dbc 100644 --- a/.changeset/architecture-review-providers.md +++ b/.changeset/architecture-review-providers.md @@ -4,6 +4,14 @@ "@refkit/provider-internet-archive": patch "@refkit/provider-wikimedia-commons": patch "@refkit/provider-artic": patch +"@refkit/provider-unsplash": patch +"@refkit/provider-flickr": patch +"@refkit/provider-smithsonian": patch +"@refkit/provider-freesound": patch +"@refkit/provider-jamendo": patch +"@refkit/provider-europeana": patch +"@refkit/provider-met": patch +"@refkit/provider-polyhaven": patch --- -Declare and honor the `page` search control (`capabilities.controls: ['page']`), wiring `controls.page` to each source's native pagination (Wikimedia Commons translates it to `gsroffset`). Enables core's unified load-more cursor across these sources. +Declare and honor the `page` search control (`capabilities.controls: ['page']`), wiring `controls.page` to each source's native pagination — native `page` params where they exist, offset translation for offset-based APIs (Wikimedia `gsroffset`, Smithsonian/Europeana `start`, Jamendo/ambientCG `offset`), and a window over the full result list for Met/Poly Haven. Enables core's unified load-more cursor across these sources. (Brave, PoetryDB, and Rijksmuseum expose no usable offset pagination and keep `page` undeclared.) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 79dff1e..548777c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,5 +17,6 @@ jobs: cache: pnpm - run: pnpm install --frozen-lockfile - run: pnpm typecheck + - run: pnpm lint - run: pnpm build - run: pnpm test:run diff --git a/README.md b/README.md index e20a817..10c1c7c 100644 --- a/README.md +++ b/README.md @@ -174,6 +174,12 @@ createRefkit({ providers, resilience: { timeoutMs: 4000, retries: 2 } }) createRefkit({ providers, resilience: false }) // raw fan-out, no timeout/retry ``` +With many sources registered, bound the fan-out with `concurrency` — at most N provider searches run at once (a queued provider's timeout only starts when its slot starts): + +```ts +createRefkit({ providers, concurrency: 6 }) // default: unlimited +``` + Pass a `cache` to memoize **per-provider** results (keyed by provider + the full normalized query — keys can be a few hundred bytes; TTL `cacheTtlMs`, default 5 min). Merging, reranking, and the license gate always run fresh; cache hits are flagged `cached: true` in `meta.providers`, and every provider status carries `latencyMs`. Pass `cacheRaw: false` to strip each result's `raw` provider payload from cache entries (smaller entries; raw-reading `isDuplicate` hooks then won't see `raw` on hits): ```ts @@ -256,6 +262,7 @@ It boots with the keyless sources (Met, Art Institute, Wikimedia, Openverse, Pro ```bash pnpm install pnpm typecheck # all packages +pnpm lint # eslint (typescript-eslint, syntactic rules; tsc stays the type gate) pnpm test:run # all packages pnpm build # tsup → dist for every package ``` diff --git a/eslint.config.mjs b/eslint.config.mjs new file mode 100644 index 0000000..1ff9986 --- /dev/null +++ b/eslint.config.mjs @@ -0,0 +1,31 @@ +// Deliberately lean: typescript-eslint's syntactic recommended set (no +// type-checked rules — they'd re-do tsc's job slowly in CI). tsc --noEmit +// remains the type gate; lint catches the bug-shaped patterns tsc allows. +import tseslint from 'typescript-eslint' + +export default tseslint.config( + { ignores: ['**/dist/**', '**/node_modules/**', 'docs/**'] }, + ...tseslint.configs.recommended, + { + rules: { + // `catch { /* fall through */ }` and `_`-prefixed placeholders are idioms here. + '@typescript-eslint/no-unused-vars': ['error', { + argsIgnorePattern: '^_', + varsIgnorePattern: '^_', + caughtErrors: 'none', + }], + // Providers cast `q.providerOptions as XSearchOptions` by design (typed whitelists). + '@typescript-eslint/consistent-type-assertions': 'off', + // zod schemas + Reference contracts legitimately use interface & type interchangeably. + '@typescript-eslint/no-empty-object-type': 'off', + }, + }, + { + files: ['**/__tests__/**', '**/*.test.ts'], + rules: { + // tests stub partial shapes and poke internals + '@typescript-eslint/no-explicit-any': 'off', + '@typescript-eslint/no-non-null-assertion': 'off', + }, + }, +) diff --git a/package.json b/package.json index 6b293db..96ad51f 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "packageManager": "pnpm@10.32.1", "scripts": { "typecheck": "pnpm -r --parallel typecheck", + "lint": "eslint .", "build": "pnpm -r --if-present build", "test": "vitest", "test:run": "vitest run", @@ -18,17 +19,19 @@ "release": "pnpm -r --if-present build && changeset publish" }, "devDependencies": { - "@types/node": "^22.10.0", - "@resvg/resvg-js": "^2.6.2", + "@changesets/cli": "^2.27.0", "@refkit/core": "workspace:*", + "@refkit/provider-artic": "workspace:*", + "@refkit/provider-met": "workspace:*", "@refkit/provider-openverse": "workspace:*", "@refkit/provider-wikimedia-commons": "workspace:*", - "@refkit/provider-met": "workspace:*", - "@refkit/provider-artic": "workspace:*", - "tsx": "^4.22.4", + "@resvg/resvg-js": "^2.6.2", + "@types/node": "^22.10.0", + "eslint": "^10.7.0", "tsup": "^8.3.5", + "tsx": "^4.22.4", "typescript": "^5.7.2", - "vitest": "^3.2.6", - "@changesets/cli": "^2.27.0" + "typescript-eslint": "^8.64.0", + "vitest": "^3.2.6" } } diff --git a/packages/core/src/__tests__/client.test.ts b/packages/core/src/__tests__/client.test.ts index 93e0486..a934699 100644 --- a/packages/core/src/__tests__/client.test.ts +++ b/packages/core/src/__tests__/client.test.ts @@ -127,6 +127,27 @@ describe('createRefkit', () => { expect(meta.warnings.some(w => w.includes('cross-source license conflict'))).toBe(true) }) + it('concurrency bounds in-flight provider searches without changing results', async () => { + let active = 0 + let maxActive = 0 + const slow = (id: string) => defineProvider({ + id, + modalities: ['image'], + search: async () => { + active++ + maxActive = Math.max(maxActive, active) + await new Promise(r => setTimeout(r, 5)) + active-- + return [ref(`${id}-1`, `https://${id}/1`)] + }, + }) + const providers = [slow('a'), slow('b'), slow('c'), slow('d')] + const rk = createRefkit({ providers, concurrency: 2 }) + const out = await rk.search({ query: 'x', modalities: ['image'] }) + expect(out).toHaveLength(4) + expect(maxActive).toBeLessThanOrEqual(2) + }) + it('queries only providers matching the modality', async () => { const textOnly = defineProvider({ id: 't', modalities: ['text'], queryFeatures: [], diff --git a/packages/core/src/__tests__/reference.test.ts b/packages/core/src/__tests__/reference.test.ts index 2a15299..8cd4eb2 100644 --- a/packages/core/src/__tests__/reference.test.ts +++ b/packages/core/src/__tests__/reference.test.ts @@ -23,12 +23,12 @@ describe('referenceSchema / parseReference', () => { }) it('rejects a reference missing canonicalUrl (provenance required)', () => { - const { canonicalUrl, ...bad } = ref + const { canonicalUrl: _canonicalUrl, ...bad } = ref expect(() => parseReference(bad)).toThrow() }) it('rejects a reference missing rights (provenance required)', () => { - const { rights, ...bad } = ref + const { rights: _rights, ...bad } = ref expect(() => parseReference(bad)).toThrow() }) diff --git a/packages/core/src/__tests__/rights.test.ts b/packages/core/src/__tests__/rights.test.ts index f4e6891..95fa529 100644 --- a/packages/core/src/__tests__/rights.test.ts +++ b/packages/core/src/__tests__/rights.test.ts @@ -19,7 +19,7 @@ describe('rightsRecordSchema', () => { }) it('requires the auditable raw anchor', () => { - const { raw, ...withoutRaw } = valid + const { raw: _raw, ...withoutRaw } = valid expect(() => rightsRecordSchema.parse(withoutRaw)).toThrow() }) diff --git a/packages/core/src/client.ts b/packages/core/src/client.ts index 6bb1a33..b3a749b 100644 --- a/packages/core/src/client.ts +++ b/packages/core/src/client.ts @@ -40,6 +40,11 @@ export interface RefkitOptions { * Pass false to shrink cache entries — cache-hit refs then carry no `raw`, so a * `merge.isDuplicate` hook reading `raw` won't see it on hits. */ cacheRaw?: boolean + /** Max provider searches in flight at once per search call. Default: unlimited + * (every matching provider fires simultaneously). Set when querying many + * sources at once — a provider's timeout only starts when its slot starts, so + * queueing never burns a queued provider's deadline. */ + concurrency?: number } export interface ProviderError { @@ -144,6 +149,20 @@ function errorSummary(error: unknown): string { return 'unknown error' } +// Bounded-parallel map: at most `limit` fn calls in flight, results in input +// order. fn never rejects here (runProvider returns failures as values). +async function mapBounded(items: readonly T[], limit: number, fn: (item: T) => Promise): Promise { + const results = new Array(items.length) + let next = 0 + const worker = async () => { + for (let i = next++; i < items.length; i = next++) { + results[i] = await fn(items[i]) + } + } + await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker)) + return results +} + export function createRefkit(options: RefkitOptions): RefkitClient { if (!options.providers || options.providers.length === 0) { throw new Error('createRefkit: at least one provider is required') @@ -207,7 +226,12 @@ export function createRefkit(options: RefkitOptions): RefkitClient { }) } - const runs = await Promise.all(chosen.map(runProvider)) + const concurrency = options.concurrency !== undefined && options.concurrency >= 1 + ? Math.floor(options.concurrency) + : undefined + const runs = concurrency + ? await mapBounded(chosen, concurrency, runProvider) + : await Promise.all(chosen.map(runProvider)) const perSource: Reference[][] = [] let anyOk = false diff --git a/packages/core/src/resilience.ts b/packages/core/src/resilience.ts index 3e8e712..5947727 100644 --- a/packages/core/src/resilience.ts +++ b/packages/core/src/resilience.ts @@ -16,12 +16,17 @@ export interface TimeoutHandle { * (H9: keeps core runtime-agnostic). */ export function withTimeout(parent: AbortSignal | undefined, timeoutMs: number): TimeoutHandle { const ctrl = new AbortController() - let timer: ReturnType | undefined let rejectExpired: (e: Error) => void const expired = new Promise((_, reject) => { rejectExpired = reject }) expired.catch(() => {}) // the race may settle first; never let this become an unhandled rejection + const timer = setTimeout(() => { + const err = new Error(`timeout after ${timeoutMs}ms`) + ctrl.abort(err) + rejectExpired(err) + }, timeoutMs) + const onParentAbort = () => { clearTimeout(timer) // self-clean: a parent abort makes the deadline moot ctrl.abort(parent?.reason) @@ -30,12 +35,6 @@ export function withTimeout(parent: AbortSignal | undefined, timeoutMs: number): if (parent?.aborted) onParentAbort() else parent?.addEventListener('abort', onParentAbort, { once: true }) - timer = setTimeout(() => { - const err = new Error(`timeout after ${timeoutMs}ms`) - ctrl.abort(err) - rejectExpired(err) - }, timeoutMs) - return { signal: ctrl.signal, expired, diff --git a/packages/provider-europeana/src/index.ts b/packages/provider-europeana/src/index.ts index 8ad4f3d..4df0f35 100644 --- a/packages/provider-europeana/src/index.ts +++ b/packages/provider-europeana/src/index.ts @@ -96,12 +96,14 @@ export function europeana(config: EuropeanaConfig) { return defineProvider({ id: 'europeana', modalities: ['image'], - capabilities: { controls: [] }, + capabilities: { controls: ['page'] }, async search(q: NormalizedQuery, ctx: ProviderContext): Promise { const url = new URL(BASE) url.searchParams.set('wskey', config.apiKey) url.searchParams.set('query', q.text) url.searchParams.set('rows', String(q.limit ?? 20)) + // 1-based `start` offset: item index of the first result, not a page number + if (q.controls?.page && q.controls.page > 1) url.searchParams.set('start', String((q.controls.page - 1) * (q.limit ?? 20) + 1)) url.searchParams.set('media', 'true') // only items that actually carry media url.searchParams.set('qf', 'TYPE:IMAGE') // v1 image-only scope (D1) const res = await ctx.fetch(url.toString(), { signal: ctx.signal }) diff --git a/packages/provider-flickr/src/index.ts b/packages/provider-flickr/src/index.ts index 60161cd..57ae268 100644 --- a/packages/provider-flickr/src/index.ts +++ b/packages/provider-flickr/src/index.ts @@ -179,7 +179,7 @@ export function flickr(config: FlickrConfig) { return defineProvider({ id: 'flickr', modalities: ['image'], - capabilities: { controls: ['sort', 'safety', 'license.commercial', 'license.modification', 'license.allowUnknown', 'creator.id'] }, + capabilities: { controls: ['sort', 'safety', 'license.commercial', 'license.modification', 'license.allowUnknown', 'creator.id', 'page'] }, async search(q: NormalizedQuery, ctx: ProviderContext): Promise { const opts = q.providerOptions as FlickrSearchOptions | undefined const url = new URL('https://api.flickr.com/services/rest/') @@ -216,6 +216,7 @@ export function flickr(config: FlickrConfig) { setBooleanFlag(url, 'in_gallery', opts?.inGallery) setBooleanFlag(url, 'is_getty', opts?.isGetty) url.searchParams.set('extras', flickrExtras(opts?.extras)) + if (q.controls?.page) url.searchParams.set('page', String(q.controls.page)) setIfInt(url, 'page', opts?.page, { min: 1 }) url.searchParams.set('per_page', String(q.limit ?? 20)) setIfInt(url, 'per_page', opts?.perPage, { min: 1, max: 500 }) diff --git a/packages/provider-freesound/src/index.ts b/packages/provider-freesound/src/index.ts index ce0c512..7be4f55 100644 --- a/packages/provider-freesound/src/index.ts +++ b/packages/provider-freesound/src/index.ts @@ -94,7 +94,7 @@ export function freesound(config: FreesoundConfig) { return defineProvider({ id: 'freesound', modalities: ['audio'], - capabilities: { controls: [] }, + capabilities: { controls: ['page'] }, async search(q: NormalizedQuery, ctx: ProviderContext): Promise { const opts = q.providerOptions as FreesoundSearchOptions | undefined const url = new URL(BASE) @@ -104,6 +104,7 @@ export function freesound(config: FreesoundConfig) { url.searchParams.set('page_size', String(opts?.pageSize ?? q.limit ?? 20)) setIfString(url, 'sort', opts?.sort) setIfString(url, 'filter', opts?.filter) + if (q.controls?.page) url.searchParams.set('page', String(q.controls.page)) setIfPositiveInt(url, 'page', opts?.page) const res = await ctx.fetch(url.toString(), { signal: ctx.signal }) if (!res.ok) throw new Error(`freesound search failed: ${res.status}`) diff --git a/packages/provider-jamendo/src/index.ts b/packages/provider-jamendo/src/index.ts index 866bba9..0df5566 100644 --- a/packages/provider-jamendo/src/index.ts +++ b/packages/provider-jamendo/src/index.ts @@ -91,7 +91,7 @@ export function jamendo(config: JamendoConfig) { return defineProvider({ id: 'jamendo', modalities: ['audio'], - capabilities: { controls: [] }, + capabilities: { controls: ['page'] }, async search(q: NormalizedQuery, ctx: ProviderContext): Promise { const url = new URL(BASE) url.searchParams.set('client_id', config.clientId) @@ -107,6 +107,8 @@ export function jamendo(config: JamendoConfig) { // jamendo joins tags with a SPACE (not the core default comma). setIfStringList(url, 'tags', opts?.tags, { separator: ' ' }) setIfString(url, 'artist_name', opts?.artist_name) + // offset-based API: translate the 1-based page control (providerOptions.offset overrides below) + if (q.controls?.page && q.controls.page > 1) url.searchParams.set('offset', String((q.controls.page - 1) * Math.min(q.limit ?? 20, 200))) // jamendo's offset is non-negative (0 is valid) → setIfNonNegativeInt, not PositiveInt. setIfNonNegativeInt(url, 'offset', opts?.offset) const res = await ctx.fetch(url.toString(), { signal: ctx.signal }) diff --git a/packages/provider-met/src/index.ts b/packages/provider-met/src/index.ts index c3f4b0d..2d7daf7 100644 --- a/packages/provider-met/src/index.ts +++ b/packages/provider-met/src/index.ts @@ -69,7 +69,7 @@ export function met(config: MetConfig = {}) { return defineProvider({ id: 'met', modalities: ['image'], - capabilities: { controls: [] }, + capabilities: { controls: ['page'] }, async search(q: NormalizedQuery, ctx: ProviderContext): Promise { const searchUrl = new URL(`${BASE}/search`) searchUrl.searchParams.set('q', q.text) @@ -90,7 +90,9 @@ export function met(config: MetConfig = {}) { const { objectIDs } = (await res.json()) as MetSearchResponse if (!objectIDs || objectIDs.length === 0) return [] const n = Math.min(config.maxObjects ?? q.limit ?? 12, 30) - const objects = await Promise.all(objectIDs.slice(0, n).map(async (id) => { + // the search endpoint returns every matching id — page = a window over that list + const from = ((q.controls?.page ?? 1) - 1) * n + const objects = await Promise.all(objectIDs.slice(from, from + n).map(async (id) => { try { const r = await ctx.fetch(`${BASE}/objects/${id}`, { signal: ctx.signal }) if (!r.ok) return null diff --git a/packages/provider-polyhaven/src/index.ts b/packages/provider-polyhaven/src/index.ts index cd407dc..8f006e6 100644 --- a/packages/provider-polyhaven/src/index.ts +++ b/packages/provider-polyhaven/src/index.ts @@ -83,7 +83,7 @@ export function polyhaven(config: PolyHavenConfig = {}) { return defineProvider({ id: 'polyhaven', modalities: ['image'], - capabilities: { controls: [] }, + capabilities: { controls: ['page'] }, async search(q: NormalizedQuery, ctx: ProviderContext): Promise { const listUrl = new URL(`${PH_BASE}/assets`) listUrl.searchParams.set('t', assetType) @@ -101,7 +101,9 @@ export function polyhaven(config: PolyHavenConfig = {}) { a.tags?.some((t) => t.toLowerCase().includes(text))) } const n = Math.min(config.maxAssets ?? q.limit ?? 12, 30) - const picked = entries.slice(0, n) + // the list endpoint returns everything — page = a window over the filtered list + const from = ((q.controls?.page ?? 1) - 1) * n + const picked = entries.slice(from, from + n) const refs = await Promise.all(picked.map(async ([id, asset]) => { try { const fr = await ctx.fetch(`${PH_BASE}/files/${id}`, { signal: ctx.signal }) @@ -170,12 +172,14 @@ export function ambientcg(config: AmbientCgConfig = {}) { return defineProvider({ id: 'ambientcg', modalities: ['image'], - capabilities: { controls: [] }, + capabilities: { controls: ['page'] }, async search(q: NormalizedQuery, ctx: ProviderContext): Promise { const url = new URL(ACG_BASE) url.searchParams.set('type', 'Material') // image-based PBR materials only (D1) url.searchParams.set('include', 'displayData,imageData') url.searchParams.set('limit', String(Math.min(config.limit ?? q.limit ?? 12, 30))) + // offset-based API: translate the 1-based page control + if (q.controls?.page && q.controls.page > 1) url.searchParams.set('offset', String((q.controls.page - 1) * Math.min(config.limit ?? q.limit ?? 12, 30))) if (q.text?.trim()) url.searchParams.set('q', q.text.trim()) const res = await ctx.fetch(url.toString(), { signal: ctx.signal }) if (!res.ok) throw new Error(`ambientcg search failed: ${res.status}`) diff --git a/packages/provider-smithsonian/src/index.ts b/packages/provider-smithsonian/src/index.ts index 2ec62b4..b8c83e3 100644 --- a/packages/provider-smithsonian/src/index.ts +++ b/packages/provider-smithsonian/src/index.ts @@ -67,12 +67,14 @@ export function smithsonian(config: SmithsonianConfig) { return defineProvider({ id: 'smithsonian', modalities: ['image'], - capabilities: { controls: [] }, + capabilities: { controls: ['page'] }, async search(q: NormalizedQuery, ctx: ProviderContext): Promise { const url = new URL('https://api.si.edu/openaccess/api/v1.0/search') url.searchParams.set('api_key', config.apiKey) url.searchParams.set('q', q.text) url.searchParams.set('rows', String(q.limit ?? 20)) + // offset-based API: translate the 1-based page control (providerOptions.start overrides below) + if (q.controls?.page && q.controls.page > 1) url.searchParams.set('start', String((q.controls.page - 1) * (q.limit ?? 20))) // bias toward CC0 image records; toReference stays authoritative per media url.searchParams.set('fq', 'online_media_type:"Images" AND media_usage:"CC0"') const opts = q.providerOptions as SmithsonianSearchOptions | undefined diff --git a/packages/provider-unsplash/src/index.ts b/packages/provider-unsplash/src/index.ts index fde114b..2c8c28c 100644 --- a/packages/provider-unsplash/src/index.ts +++ b/packages/provider-unsplash/src/index.ts @@ -63,12 +63,13 @@ export function unsplash(config: UnsplashConfig) { return defineProvider({ id: 'unsplash', modalities: ['image'], - capabilities: { controls: ['orientation', 'color', 'language', 'sort', 'safety'] }, + capabilities: { controls: ['orientation', 'color', 'language', 'sort', 'safety', 'page'] }, async search(q: NormalizedQuery, ctx: ProviderContext): Promise { const url = new URL('https://api.unsplash.com/search/photos') url.searchParams.set('query', q.text) url.searchParams.set('per_page', String(Math.min(q.limit ?? 10, 30))) // Unsplash hard-caps per_page at 30; default kept low for free-tier rate limits const controls = q.controls + if (controls?.page) url.searchParams.set('page', String(controls.page)) if (controls?.color) url.searchParams.set('color', controls.color) if (controls?.orientation) url.searchParams.set('orientation', controls.orientation === 'square' ? 'squarish' : controls.orientation) if (controls?.language) url.searchParams.set('lang', controls.language) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 541173f..2124a0b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -32,6 +32,9 @@ importers: '@types/node': specifier: ^22.10.0 version: 22.20.0 + eslint: + specifier: ^10.7.0 + version: 10.7.0 tsup: specifier: ^8.3.5 version: 8.5.1(postcss@8.5.15)(tsx@4.22.4)(typescript@5.9.3) @@ -41,6 +44,9 @@ importers: typescript: specifier: ^5.7.2 version: 5.9.3 + typescript-eslint: + specifier: ^8.64.0 + version: 8.64.0(eslint@10.7.0)(typescript@5.9.3) vitest: specifier: ^3.2.6 version: 3.2.6(@types/node@22.20.0)(tsx@4.22.4) @@ -62,39 +68,18 @@ importers: '@refkit/provider-artic': specifier: workspace:* version: link:../provider-artic - '@refkit/provider-brave': - specifier: workspace:* - version: link:../provider-brave - '@refkit/provider-europeana': - specifier: workspace:* - version: link:../provider-europeana - '@refkit/provider-flickr': - specifier: workspace:* - version: link:../provider-flickr - '@refkit/provider-freesound': - specifier: workspace:* - version: link:../provider-freesound '@refkit/provider-gutendex': specifier: workspace:* version: link:../provider-gutendex '@refkit/provider-internet-archive': specifier: workspace:* version: link:../provider-internet-archive - '@refkit/provider-jamendo': - specifier: workspace:* - version: link:../provider-jamendo '@refkit/provider-met': specifier: workspace:* version: link:../provider-met '@refkit/provider-openverse': specifier: workspace:* version: link:../provider-openverse - '@refkit/provider-pexels': - specifier: workspace:* - version: link:../provider-pexels - '@refkit/provider-pixabay': - specifier: workspace:* - version: link:../provider-pixabay '@refkit/provider-poetrydb': specifier: workspace:* version: link:../provider-poetrydb @@ -104,18 +89,40 @@ importers: '@refkit/provider-rijksmuseum': specifier: workspace:* version: link:../provider-rijksmuseum - '@refkit/provider-smithsonian': - specifier: workspace:* - version: link:../provider-smithsonian - '@refkit/provider-unsplash': - specifier: workspace:* - version: link:../provider-unsplash '@refkit/provider-wikimedia-commons': specifier: workspace:* version: link:../provider-wikimedia-commons zod: specifier: ^4.3.6 version: 4.4.3 + optionalDependencies: + '@refkit/provider-brave': + specifier: workspace:* + version: link:../provider-brave + '@refkit/provider-europeana': + specifier: workspace:* + version: link:../provider-europeana + '@refkit/provider-flickr': + specifier: workspace:* + version: link:../provider-flickr + '@refkit/provider-freesound': + specifier: workspace:* + version: link:../provider-freesound + '@refkit/provider-jamendo': + specifier: workspace:* + version: link:../provider-jamendo + '@refkit/provider-pexels': + specifier: workspace:* + version: link:../provider-pexels + '@refkit/provider-pixabay': + specifier: workspace:* + version: link:../provider-pixabay + '@refkit/provider-smithsonian': + specifier: workspace:* + version: link:../provider-smithsonian + '@refkit/provider-unsplash': + specifier: workspace:* + version: link:../provider-unsplash packages/provider-artic: dependencies: @@ -688,12 +695,62 @@ packages: cpu: [x64] os: [win32] + '@eslint-community/eslint-utils@4.9.1': + resolution: {integrity: sha512-phrYmNiYppR7znFEdqgfWHXR6NCkZEK7hwWDHZUjit/2/U0r6XvkDl0SYnoM51Hq7FhCGdLDT6zxCCOY1hexsQ==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + peerDependencies: + eslint: ^6.0.0 || ^7.0.0 || >=8.0.0 + + '@eslint-community/regexpp@4.12.2': + resolution: {integrity: sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==} + engines: {node: ^12.0.0 || ^14.0.0 || >=16.0.0} + + '@eslint/config-array@0.23.5': + resolution: {integrity: sha512-Y3kKLvC1dvTOT+oGlqNQ1XLqK6D1HU2YXPc52NmAlJZbMMWDzGYXMiPRJ8TYD39muD/OTjlZmNJ4ib7dvSrMBA==} + engines: {node: ^20.19.0 || ^22.13.0 || >=24} + + '@eslint/config-helpers@0.6.0': + resolution: {integrity: sha512-ii6Bw9jJ2zi2cWA2Z+9/QZ/+3DX6kwaV5Q986D/CdP3Lap3w/pgQZ373FV7byY/i7L4IRH/G43I5dz1ClsCbpA==} + engines: {node: ^20.19.0 || ^22.13.0 || >=24} + + '@eslint/core@1.2.1': + resolution: {integrity: sha512-MwcE1P+AZ4C6DWlpin/OmOA54mmIZ/+xZuJiQd4SyB29oAJjN30UW9wkKNptW2ctp4cEsvhlLY/CsQ1uoHDloQ==} + engines: {node: ^20.19.0 || ^22.13.0 || >=24} + + '@eslint/object-schema@3.0.5': + resolution: {integrity: sha512-vqTaUEgxzm+YDSdElad6PiRoX4t8VGDjCtt05zn4nU810UIx/uNEV7/lZJ6KwFThKZOzOxzXy48da+No7HZaMw==} + engines: {node: ^20.19.0 || ^22.13.0 || >=24} + + '@eslint/plugin-kit@0.7.2': + resolution: {integrity: sha512-+CNAzxglkrpNf/kKywqQfk74QjtceuOE7Qm+AF8miRvPF/wmmK5+OJOgVh3AVTT3RP2mH3+FOaxlE5v72owk0A==} + engines: {node: ^20.19.0 || ^22.13.0 || >=24} + '@hono/node-server@1.19.14': resolution: {integrity: sha512-GwtvgtXxnWsucXvbQXkRgqksiH2Qed37H9xHZocE5sA3N8O8O8/8FA3uclQXxXVzc9XBZuEOMK7+r02FmSpHtw==} engines: {node: '>=18.14.1'} peerDependencies: hono: ^4 + '@humanfs/core@0.19.2': + resolution: {integrity: sha512-UhXNm+CFMWcbChXywFwkmhqjs3PRCmcSa/hfBgLIb7oQ5HNb1wS0icWsGtSAUNgefHeI+eBrA8I1fxmbHsGdvA==} + engines: {node: '>=18.18.0'} + + '@humanfs/node@0.16.8': + resolution: {integrity: sha512-gE1eQNZ3R++kTzFUpdGlpmy8kDZD/MLyHqDwqjkVQI0JMdI1D51sy1H958PNXYkM2rAac7e5/CnIKZrHtPh3BQ==} + engines: {node: '>=18.18.0'} + + '@humanfs/types@0.15.0': + resolution: {integrity: sha512-ZZ1w0aoQkwuUuC7Yf+7sdeaNfqQiiLcSRbfI08oAxqLtpXQr9AIVX7Ay7HLDuiLYAaFPu8oBYNq/QIi9URHJ3Q==} + engines: {node: '>=18.18.0'} + + '@humanwhocodes/module-importer@1.0.1': + resolution: {integrity: sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==} + engines: {node: '>=12.22'} + + '@humanwhocodes/retry@0.4.3': + resolution: {integrity: sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==} + engines: {node: '>=18.18'} + '@inquirer/external-editor@1.0.3': resolution: {integrity: sha512-RWbSrDiYmO4LbejWY7ttpxczuwQyZLBUyygsA9Nsv95hpzUWwnNTVQmAq3xuh7vNwCp07UTmE5i11XAEExx4RA==} engines: {node: '>=18'} @@ -968,15 +1025,80 @@ packages: '@types/deep-eql@4.0.2': resolution: {integrity: sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==} + '@types/esrecurse@4.3.1': + resolution: {integrity: sha512-xJBAbDifo5hpffDBuHl0Y8ywswbiAp/Wi7Y/GtAgSlZyIABppyurxVueOPE8LUQOxdlgi6Zqce7uoEpqNTeiUw==} + '@types/estree@1.0.9': resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==} + '@types/json-schema@7.0.15': + resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==} + '@types/node@12.20.55': resolution: {integrity: sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ==} '@types/node@22.20.0': resolution: {integrity: sha512-QWlFW2wf3nTjC13/DqRnBpR4ZO36VJH/JVBkA/vcnmbTBNQIlnObqyqZE1tUR7+Ni23Lda8R1BxMfbXRpCUx5g==} + '@typescript-eslint/eslint-plugin@8.64.0': + resolution: {integrity: sha512-CGvQPBxN3wZLu6Rz2kFUpZeoCm78xUic92ck39KPePkO1NPOwjCqdQnm5Q87tpWw9vcBvW8XLrDXjH9PWYtJ3Q==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + '@typescript-eslint/parser': ^8.64.0 + eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 + typescript: '>=4.8.4 <6.1.0' + + '@typescript-eslint/parser@8.64.0': + resolution: {integrity: sha512-KA0OshtlcCCXmbfqyZkM5pV3/WNraJf7DkJRLpyrmwPtud57H5BDX7C3k0LPSPxpprfRL+cJDGabF10mvNCoCw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 + typescript: '>=4.8.4 <6.1.0' + + '@typescript-eslint/project-service@8.64.0': + resolution: {integrity: sha512-tk4WpOJ6IEbGrVHaNmM0YRrwAD3exZlIK3iadQNAxh4YKk6jvUQ4ecq18n+v7+meh+cJ3j+D8nbk8sRKhlwLQg==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + typescript: '>=4.8.4 <6.1.0' + + '@typescript-eslint/scope-manager@8.64.0': + resolution: {integrity: sha512-CXEaFdYXjSTgKhisNkwCcJwTP8Pl+fmRrEQrri4nm3vU743bALrxzLmq7fHG/7e6a5xO0lDYeURpZmBuhHk54w==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@typescript-eslint/tsconfig-utils@8.64.0': + resolution: {integrity: sha512-2yo8rRNKuzbVWQp5kslhANqZ2uDAeROQHBRZNPu8JDsHmeFNj/XJJhX/FhNUWmkHHvoNsKa6+tHJiig87EzsQw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + typescript: '>=4.8.4 <6.1.0' + + '@typescript-eslint/type-utils@8.64.0': + resolution: {integrity: sha512-XWG4Fmmv/6SvyS9nH8jWrKs6terwJvE8cyRt1CzYYqzp9OrPhCT4cMc/f7C6RZCwG+qMmiffJS1/qJP8G1URtg==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 + typescript: '>=4.8.4 <6.1.0' + + '@typescript-eslint/types@8.64.0': + resolution: {integrity: sha512-qjhfuTfLXjA4IOzXvz0rTjT01BqEiIgPoUeMwiEjnaHKJMTNo8rH5pYW1a2L/0Dnux2fPC85AeyJoWaGa8WxTA==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@typescript-eslint/typescript-estree@8.64.0': + resolution: {integrity: sha512-Pztpsn1aCE1oWDvDEfUk31nngvvF7vUB5SwHFEaZIFpvw7WJtqUHHL4plBZDA9HfWJJjL13BdG0YrJInTUvoVA==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + typescript: '>=4.8.4 <6.1.0' + + '@typescript-eslint/utils@8.64.0': + resolution: {integrity: sha512-aJUGVB3+U0htrrCjoA8qukw8cm8fNCGAxK/tVoS70k8aeb7DETKeFozRiVFIwEeN9WJLsjaP3ph8I60tY2XZoQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 + typescript: '>=4.8.4 <6.1.0' + + '@typescript-eslint/visitor-keys@8.64.0': + resolution: {integrity: sha512-mrtuL8Nsn6gi2H4mo5KMTp823M+3Q19Ew/i+Zlikq20tIMm99C3Ez0dCmkWWnxut20esQvTg8aUSEhMcAOXhEw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + '@vitest/expect@3.2.6': resolution: {integrity: sha512-1+7q9BtaKzEmO+fmNT3kYvoNn5Y71XWAx2Q5HRim4tTVRQVRv4uJFAQ5FbK0OPUeNP/WmVCpxYxoJdvuHVjzBQ==} @@ -1010,6 +1132,11 @@ packages: resolution: {integrity: sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==} engines: {node: '>= 0.6'} + acorn-jsx@5.3.2: + resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==} + peerDependencies: + acorn: ^6.0.0 || ^7.0.0 || ^8.0.0 + acorn@8.17.0: resolution: {integrity: sha512-xRQbDb9BnwDafYNn6Vwl839DYVjqXYb1XVGtWAZ1kcDc6iwAL4hg3B1dZlRiuENFeO2H53gFG3in621AdERVAg==} engines: {node: '>=0.4.0'} @@ -1023,6 +1150,9 @@ packages: ajv: optional: true + ajv@6.15.0: + resolution: {integrity: sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==} + ajv@8.20.0: resolution: {integrity: sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==} @@ -1051,6 +1181,10 @@ packages: resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} engines: {node: '>=12'} + balanced-match@4.0.4: + resolution: {integrity: sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==} + engines: {node: 18 || 20 || >=22} + better-path-resolve@1.0.0: resolution: {integrity: sha512-pbnl5XzGBdrFU/wT4jqmJVPn2B6UHPBOhzMQkY/SPUPB6QtUXtmBHBIwCbXJol93mOpGMnQyP/+BB19q04xj7g==} engines: {node: '>=4'} @@ -1059,6 +1193,10 @@ packages: resolution: {integrity: sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==} engines: {node: '>=18'} + brace-expansion@5.0.7: + resolution: {integrity: sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==} + engines: {node: 18 || 20 || >=22} + braces@3.0.3: resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} engines: {node: '>=8'} @@ -1152,6 +1290,9 @@ packages: resolution: {integrity: sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==} engines: {node: '>=6'} + deep-is@0.1.4: + resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} + depd@2.0.0: resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} engines: {node: '>= 0.8'} @@ -1207,14 +1348,60 @@ packages: escape-html@1.0.3: resolution: {integrity: sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==} + escape-string-regexp@4.0.0: + resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} + engines: {node: '>=10'} + + eslint-scope@9.1.2: + resolution: {integrity: sha512-xS90H51cKw0jltxmvmHy2Iai1LIqrfbw57b79w/J7MfvDfkIkFZ+kj6zC3BjtUwh150HsSSdxXZcsuv72miDFQ==} + engines: {node: ^20.19.0 || ^22.13.0 || >=24} + + eslint-visitor-keys@3.4.3: + resolution: {integrity: sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + + eslint-visitor-keys@5.0.1: + resolution: {integrity: sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==} + engines: {node: ^20.19.0 || ^22.13.0 || >=24} + + eslint@10.7.0: + resolution: {integrity: sha512-GVTD7s1vdIl6UYvAfriOPeY1Df8LIZjfofLvHwde+erDHGGuHyuM6xoxRxmHiebhYuD2p1vN4wWh0XzPARSGDQ==} + engines: {node: ^20.19.0 || ^22.13.0 || >=24} + hasBin: true + peerDependencies: + jiti: '*' + peerDependenciesMeta: + jiti: + optional: true + + espree@11.2.0: + resolution: {integrity: sha512-7p3DrVEIopW1B1avAGLuCSh1jubc01H2JHc8B4qqGblmg5gI9yumBgACjWo4JlIc04ufug4xJ3SQI8HkS/Rgzw==} + engines: {node: ^20.19.0 || ^22.13.0 || >=24} + esprima@4.0.1: resolution: {integrity: sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==} engines: {node: '>=4'} hasBin: true + esquery@1.7.0: + resolution: {integrity: sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==} + engines: {node: '>=0.10'} + + esrecurse@4.3.0: + resolution: {integrity: sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==} + engines: {node: '>=4.0'} + + estraverse@5.3.0: + resolution: {integrity: sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==} + engines: {node: '>=4.0'} + estree-walker@3.0.3: resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} + esutils@2.0.3: + resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==} + engines: {node: '>=0.10.0'} + etag@1.8.1: resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} engines: {node: '>= 0.6'} @@ -1251,6 +1438,12 @@ packages: resolution: {integrity: sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==} engines: {node: '>=8.6.0'} + fast-json-stable-stringify@2.1.0: + resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==} + + fast-levenshtein@2.0.6: + resolution: {integrity: sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==} + fast-uri@3.1.2: resolution: {integrity: sha512-rVjf7ArG3LTk+FS6Yw81V1DLuZl1bRbNrev6Tmd/9RaroeeRRJhAt7jg/6YFxbvAQXUCavSoZhPPj6oOx+5KjQ==} @@ -1266,6 +1459,10 @@ packages: picomatch: optional: true + file-entry-cache@8.0.0: + resolution: {integrity: sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==} + engines: {node: '>=16.0.0'} + fill-range@7.1.1: resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} engines: {node: '>=8'} @@ -1278,9 +1475,20 @@ packages: resolution: {integrity: sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==} engines: {node: '>=8'} + find-up@5.0.0: + resolution: {integrity: sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==} + engines: {node: '>=10'} + fix-dts-default-cjs-exports@1.0.1: resolution: {integrity: sha512-pVIECanWFC61Hzl2+oOCtoJ3F17kglZC/6N94eRWycFgBH35hHx0Li604ZIzhseh97mf2p0cv7vVrOZGoqhlEg==} + flat-cache@4.0.1: + resolution: {integrity: sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==} + engines: {node: '>=16'} + + flatted@3.4.2: + resolution: {integrity: sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==} + forwarded@0.2.0: resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} engines: {node: '>= 0.6'} @@ -1317,6 +1525,10 @@ packages: resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} engines: {node: '>= 6'} + glob-parent@6.0.2: + resolution: {integrity: sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==} + engines: {node: '>=10.13.0'} + globby@11.1.0: resolution: {integrity: sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==} engines: {node: '>=10'} @@ -1356,6 +1568,14 @@ packages: resolution: {integrity: sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==} engines: {node: '>= 4'} + ignore@7.0.6: + resolution: {integrity: sha512-BAg6QkE8W+TuQLrrw0Ugr7HegXduRuuj8/ti2kSOc+jz1dmx8/WNcjr6XGnq5YpDWxFwwaavqD0+jIUOKelTsw==} + engines: {node: '>= 4'} + + imurmurhash@0.1.4: + resolution: {integrity: sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==} + engines: {node: '>=0.8.19'} + inherits@2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} @@ -1411,15 +1631,31 @@ packages: resolution: {integrity: sha512-ePWsvanv0DWuDRsW8dnt+R4jQ31SCRCQ7hhNcPXZPsoBZiemuZNYGf7adZdqX2D86j6rvKp3RpCxVTSb8WQlOw==} hasBin: true + json-buffer@3.0.1: + resolution: {integrity: sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==} + + json-schema-traverse@0.4.1: + resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} + json-schema-traverse@1.0.0: resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} json-schema-typed@8.0.2: resolution: {integrity: sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==} + json-stable-stringify-without-jsonify@1.0.1: + resolution: {integrity: sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==} + jsonfile@4.0.0: resolution: {integrity: sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==} + keyv@4.5.4: + resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} + + levn@0.4.1: + resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==} + engines: {node: '>= 0.8.0'} + lilconfig@3.1.3: resolution: {integrity: sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==} engines: {node: '>=14'} @@ -1435,6 +1671,10 @@ packages: resolution: {integrity: sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==} engines: {node: '>=8'} + locate-path@6.0.0: + resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} + engines: {node: '>=10'} + lodash.startcase@4.4.0: resolution: {integrity: sha512-+WKqsK294HMSc2jEbNgpHpd0JfIBhp7rEV4aqXWqFr6AlXov+SlcgB1Fv01y2kGe3Gc8nMW7VA0SrGuSkRfIEg==} @@ -1472,6 +1712,10 @@ packages: resolution: {integrity: sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==} engines: {node: '>=18'} + minimatch@10.2.5: + resolution: {integrity: sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==} + engines: {node: 18 || 20 || >=22} + mlly@1.8.2: resolution: {integrity: sha512-d+ObxMQFmbt10sretNDytwt85VrbkhhUA/JBGm1MPaWJ65Cl4wOgLaB1NYvJSZ0Ef03MMEU/0xpPMXUIQ29UfA==} @@ -1490,6 +1734,9 @@ packages: engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true + natural-compare@1.4.0: + resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} + negotiator@1.0.0: resolution: {integrity: sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==} engines: {node: '>= 0.6'} @@ -1509,6 +1756,10 @@ packages: once@1.4.0: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + optionator@0.9.4: + resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} + engines: {node: '>= 0.8.0'} + outdent@0.5.0: resolution: {integrity: sha512-/jHxFIzoMXdqPzTaCpFzAAWhpkSjZPF4Vsn6jAfNpmbH/ymsmd7Qc6VE9BGn0L6YMj6uwpQLxCECpus4ukKS9Q==} @@ -1520,10 +1771,18 @@ packages: resolution: {integrity: sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==} engines: {node: '>=6'} + p-limit@3.1.0: + resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==} + engines: {node: '>=10'} + p-locate@4.1.0: resolution: {integrity: sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==} engines: {node: '>=8'} + p-locate@5.0.0: + resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==} + engines: {node: '>=10'} + p-map@2.1.0: resolution: {integrity: sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==} engines: {node: '>=6'} @@ -1609,6 +1868,10 @@ packages: resolution: {integrity: sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==} engines: {node: ^10 || ^12 || >=14} + prelude-ls@1.2.1: + resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} + engines: {node: '>= 0.8.0'} + prettier@2.8.8: resolution: {integrity: sha512-tdN8qQGvNjw4CHbY+XXk0JgCXn9QiF21a55rBe5LJAU+kDyC4WQn4+awm2Xfk2lQMk5fKup9XgzTZtGkjBdP9Q==} engines: {node: '>=10.13.0'} @@ -1618,6 +1881,10 @@ packages: resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} engines: {node: '>= 0.10'} + punycode@2.3.1: + resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} + engines: {node: '>=6'} + qs@6.15.2: resolution: {integrity: sha512-Rzq0KEyX/w/tEybncDgdkZrJgVUsUMk3xjh3t5bv3S1HTAtg+uOYt72+ZfwiQwKdysThkTBdL/rTi6HDmX9Ddw==} engines: {node: '>=0.6'} @@ -1807,6 +2074,12 @@ packages: resolution: {integrity: sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==} hasBin: true + ts-api-utils@2.5.0: + resolution: {integrity: sha512-OJ/ibxhPlqrMM0UiNHJ/0CKQkoKF243/AEmplt3qpRgkW8VG7IfOS41h7V8TjITqdByHzrjcS/2si+y4lIh8NA==} + engines: {node: '>=18.12'} + peerDependencies: + typescript: '>=4.8.4' + ts-interface-checker@0.1.13: resolution: {integrity: sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==} @@ -1834,10 +2107,21 @@ packages: engines: {node: '>=18.0.0'} hasBin: true + type-check@0.4.0: + resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} + engines: {node: '>= 0.8.0'} + type-is@2.1.0: resolution: {integrity: sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==} engines: {node: '>= 18'} + typescript-eslint@8.64.0: + resolution: {integrity: sha512-0qg+pDNMnqYzqH9AnNK+39tejHvsShUOUUoRUgtnTGE7QuMZhiFDnozq8nHJVq+Wae6NMLKNWLg5WmkcC/ndyQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 + typescript: '>=4.8.4 <6.1.0' + typescript@5.9.3: resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} engines: {node: '>=14.17'} @@ -1857,6 +2141,9 @@ packages: resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} engines: {node: '>= 0.8'} + uri-js@4.4.1: + resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} + vary@1.1.2: resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} engines: {node: '>= 0.8'} @@ -1944,9 +2231,17 @@ packages: engines: {node: '>=8'} hasBin: true + word-wrap@1.2.5: + resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==} + engines: {node: '>=0.10.0'} + wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + yocto-queue@0.1.0: + resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} + engines: {node: '>=10'} + zod-to-json-schema@3.25.2: resolution: {integrity: sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==} peerDependencies: @@ -2258,10 +2553,56 @@ snapshots: '@esbuild/win32-x64@0.28.1': optional: true + '@eslint-community/eslint-utils@4.9.1(eslint@10.7.0)': + dependencies: + eslint: 10.7.0 + eslint-visitor-keys: 3.4.3 + + '@eslint-community/regexpp@4.12.2': {} + + '@eslint/config-array@0.23.5': + dependencies: + '@eslint/object-schema': 3.0.5 + debug: 4.4.3 + minimatch: 10.2.5 + transitivePeerDependencies: + - supports-color + + '@eslint/config-helpers@0.6.0': + dependencies: + '@eslint/core': 1.2.1 + + '@eslint/core@1.2.1': + dependencies: + '@types/json-schema': 7.0.15 + + '@eslint/object-schema@3.0.5': {} + + '@eslint/plugin-kit@0.7.2': + dependencies: + '@eslint/core': 1.2.1 + levn: 0.4.1 + '@hono/node-server@1.19.14(hono@4.12.26)': dependencies: hono: 4.12.26 + '@humanfs/core@0.19.2': + dependencies: + '@humanfs/types': 0.15.0 + + '@humanfs/node@0.16.8': + dependencies: + '@humanfs/core': 0.19.2 + '@humanfs/types': 0.15.0 + '@humanwhocodes/retry': 0.4.3 + + '@humanfs/types@0.15.0': {} + + '@humanwhocodes/module-importer@1.0.1': {} + + '@humanwhocodes/retry@0.4.3': {} + '@inquirer/external-editor@1.0.3(@types/node@22.20.0)': dependencies: chardet: 2.2.0 @@ -2466,14 +2807,109 @@ snapshots: '@types/deep-eql@4.0.2': {} + '@types/esrecurse@4.3.1': {} + '@types/estree@1.0.9': {} + '@types/json-schema@7.0.15': {} + '@types/node@12.20.55': {} '@types/node@22.20.0': dependencies: undici-types: 6.21.0 + '@typescript-eslint/eslint-plugin@8.64.0(@typescript-eslint/parser@8.64.0(eslint@10.7.0)(typescript@5.9.3))(eslint@10.7.0)(typescript@5.9.3)': + dependencies: + '@eslint-community/regexpp': 4.12.2 + '@typescript-eslint/parser': 8.64.0(eslint@10.7.0)(typescript@5.9.3) + '@typescript-eslint/scope-manager': 8.64.0 + '@typescript-eslint/type-utils': 8.64.0(eslint@10.7.0)(typescript@5.9.3) + '@typescript-eslint/utils': 8.64.0(eslint@10.7.0)(typescript@5.9.3) + '@typescript-eslint/visitor-keys': 8.64.0 + eslint: 10.7.0 + ignore: 7.0.6 + natural-compare: 1.4.0 + ts-api-utils: 2.5.0(typescript@5.9.3) + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/parser@8.64.0(eslint@10.7.0)(typescript@5.9.3)': + dependencies: + '@typescript-eslint/scope-manager': 8.64.0 + '@typescript-eslint/types': 8.64.0 + '@typescript-eslint/typescript-estree': 8.64.0(typescript@5.9.3) + '@typescript-eslint/visitor-keys': 8.64.0 + debug: 4.4.3 + eslint: 10.7.0 + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/project-service@8.64.0(typescript@5.9.3)': + dependencies: + '@typescript-eslint/tsconfig-utils': 8.64.0(typescript@5.9.3) + '@typescript-eslint/types': 8.64.0 + debug: 4.4.3 + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/scope-manager@8.64.0': + dependencies: + '@typescript-eslint/types': 8.64.0 + '@typescript-eslint/visitor-keys': 8.64.0 + + '@typescript-eslint/tsconfig-utils@8.64.0(typescript@5.9.3)': + dependencies: + typescript: 5.9.3 + + '@typescript-eslint/type-utils@8.64.0(eslint@10.7.0)(typescript@5.9.3)': + dependencies: + '@typescript-eslint/types': 8.64.0 + '@typescript-eslint/typescript-estree': 8.64.0(typescript@5.9.3) + '@typescript-eslint/utils': 8.64.0(eslint@10.7.0)(typescript@5.9.3) + debug: 4.4.3 + eslint: 10.7.0 + ts-api-utils: 2.5.0(typescript@5.9.3) + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/types@8.64.0': {} + + '@typescript-eslint/typescript-estree@8.64.0(typescript@5.9.3)': + dependencies: + '@typescript-eslint/project-service': 8.64.0(typescript@5.9.3) + '@typescript-eslint/tsconfig-utils': 8.64.0(typescript@5.9.3) + '@typescript-eslint/types': 8.64.0 + '@typescript-eslint/visitor-keys': 8.64.0 + debug: 4.4.3 + minimatch: 10.2.5 + semver: 7.8.5 + tinyglobby: 0.2.17 + ts-api-utils: 2.5.0(typescript@5.9.3) + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/utils@8.64.0(eslint@10.7.0)(typescript@5.9.3)': + dependencies: + '@eslint-community/eslint-utils': 4.9.1(eslint@10.7.0) + '@typescript-eslint/scope-manager': 8.64.0 + '@typescript-eslint/types': 8.64.0 + '@typescript-eslint/typescript-estree': 8.64.0(typescript@5.9.3) + eslint: 10.7.0 + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/visitor-keys@8.64.0': + dependencies: + '@typescript-eslint/types': 8.64.0 + eslint-visitor-keys: 5.0.1 + '@vitest/expect@3.2.6': dependencies: '@types/chai': 5.2.3 @@ -2521,12 +2957,23 @@ snapshots: mime-types: 3.0.2 negotiator: 1.0.0 + acorn-jsx@5.3.2(acorn@8.17.0): + dependencies: + acorn: 8.17.0 + acorn@8.17.0: {} ajv-formats@3.0.1(ajv@8.20.0): optionalDependencies: ajv: 8.20.0 + ajv@6.15.0: + dependencies: + fast-deep-equal: 3.1.3 + fast-json-stable-stringify: 2.1.0 + json-schema-traverse: 0.4.1 + uri-js: 4.4.1 + ajv@8.20.0: dependencies: fast-deep-equal: 3.1.3 @@ -2550,6 +2997,8 @@ snapshots: assertion-error@2.0.1: {} + balanced-match@4.0.4: {} + better-path-resolve@1.0.0: dependencies: is-windows: 1.0.2 @@ -2568,6 +3017,10 @@ snapshots: transitivePeerDependencies: - supports-color + brace-expansion@5.0.7: + dependencies: + balanced-match: 4.0.4 + braces@3.0.3: dependencies: fill-range: 7.1.1 @@ -2640,6 +3093,8 @@ snapshots: deep-eql@5.0.2: {} + deep-is@0.1.4: {} + depd@2.0.0: {} detect-indent@6.1.0: {} @@ -2733,12 +3188,78 @@ snapshots: escape-html@1.0.3: {} + escape-string-regexp@4.0.0: {} + + eslint-scope@9.1.2: + dependencies: + '@types/esrecurse': 4.3.1 + '@types/estree': 1.0.9 + esrecurse: 4.3.0 + estraverse: 5.3.0 + + eslint-visitor-keys@3.4.3: {} + + eslint-visitor-keys@5.0.1: {} + + eslint@10.7.0: + dependencies: + '@eslint-community/eslint-utils': 4.9.1(eslint@10.7.0) + '@eslint-community/regexpp': 4.12.2 + '@eslint/config-array': 0.23.5 + '@eslint/config-helpers': 0.6.0 + '@eslint/core': 1.2.1 + '@eslint/plugin-kit': 0.7.2 + '@humanfs/node': 0.16.8 + '@humanwhocodes/module-importer': 1.0.1 + '@humanwhocodes/retry': 0.4.3 + '@types/estree': 1.0.9 + ajv: 6.15.0 + cross-spawn: 7.0.6 + debug: 4.4.3 + escape-string-regexp: 4.0.0 + eslint-scope: 9.1.2 + eslint-visitor-keys: 5.0.1 + espree: 11.2.0 + esquery: 1.7.0 + esutils: 2.0.3 + fast-deep-equal: 3.1.3 + file-entry-cache: 8.0.0 + find-up: 5.0.0 + glob-parent: 6.0.2 + ignore: 5.3.2 + imurmurhash: 0.1.4 + is-glob: 4.0.3 + json-stable-stringify-without-jsonify: 1.0.1 + minimatch: 10.2.5 + natural-compare: 1.4.0 + optionator: 0.9.4 + transitivePeerDependencies: + - supports-color + + espree@11.2.0: + dependencies: + acorn: 8.17.0 + acorn-jsx: 5.3.2(acorn@8.17.0) + eslint-visitor-keys: 5.0.1 + esprima@4.0.1: {} + esquery@1.7.0: + dependencies: + estraverse: 5.3.0 + + esrecurse@4.3.0: + dependencies: + estraverse: 5.3.0 + + estraverse@5.3.0: {} + estree-walker@3.0.3: dependencies: '@types/estree': 1.0.9 + esutils@2.0.3: {} + etag@1.8.1: {} eventsource-parser@3.1.0: {} @@ -2799,6 +3320,10 @@ snapshots: merge2: 1.4.1 micromatch: 4.0.8 + fast-json-stable-stringify@2.1.0: {} + + fast-levenshtein@2.0.6: {} + fast-uri@3.1.2: {} fastq@1.20.1: @@ -2809,6 +3334,10 @@ snapshots: optionalDependencies: picomatch: 4.0.4 + file-entry-cache@8.0.0: + dependencies: + flat-cache: 4.0.1 + fill-range@7.1.1: dependencies: to-regex-range: 5.0.1 @@ -2829,12 +3358,24 @@ snapshots: locate-path: 5.0.0 path-exists: 4.0.0 + find-up@5.0.0: + dependencies: + locate-path: 6.0.0 + path-exists: 4.0.0 + fix-dts-default-cjs-exports@1.0.1: dependencies: magic-string: 0.30.21 mlly: 1.8.2 rollup: 4.62.2 + flat-cache@4.0.1: + dependencies: + flatted: 3.4.2 + keyv: 4.5.4 + + flatted@3.4.2: {} + forwarded@0.2.0: {} fresh@2.0.0: {} @@ -2878,6 +3419,10 @@ snapshots: dependencies: is-glob: 4.0.3 + glob-parent@6.0.2: + dependencies: + is-glob: 4.0.3 + globby@11.1.0: dependencies: array-union: 2.1.0 @@ -2915,6 +3460,10 @@ snapshots: ignore@5.3.2: {} + ignore@7.0.6: {} + + imurmurhash@0.1.4: {} + inherits@2.0.4: {} ip-address@10.2.0: {} @@ -2954,14 +3503,29 @@ snapshots: dependencies: argparse: 2.0.1 + json-buffer@3.0.1: {} + + json-schema-traverse@0.4.1: {} + json-schema-traverse@1.0.0: {} json-schema-typed@8.0.2: {} + json-stable-stringify-without-jsonify@1.0.1: {} + jsonfile@4.0.0: optionalDependencies: graceful-fs: 4.2.11 + keyv@4.5.4: + dependencies: + json-buffer: 3.0.1 + + levn@0.4.1: + dependencies: + prelude-ls: 1.2.1 + type-check: 0.4.0 + lilconfig@3.1.3: {} lines-and-columns@1.2.4: {} @@ -2972,6 +3536,10 @@ snapshots: dependencies: p-locate: 4.1.0 + locate-path@6.0.0: + dependencies: + p-locate: 5.0.0 + lodash.startcase@4.4.0: {} loupe@3.2.1: {} @@ -2999,6 +3567,10 @@ snapshots: dependencies: mime-db: 1.54.0 + minimatch@10.2.5: + dependencies: + brace-expansion: 5.0.7 + mlly@1.8.2: dependencies: acorn: 8.17.0 @@ -3018,6 +3590,8 @@ snapshots: nanoid@3.3.15: {} + natural-compare@1.4.0: {} + negotiator@1.0.0: {} object-assign@4.1.1: {} @@ -3032,6 +3606,15 @@ snapshots: dependencies: wrappy: 1.0.2 + optionator@0.9.4: + dependencies: + deep-is: 0.1.4 + fast-levenshtein: 2.0.6 + levn: 0.4.1 + prelude-ls: 1.2.1 + type-check: 0.4.0 + word-wrap: 1.2.5 + outdent@0.5.0: {} p-filter@2.1.0: @@ -3042,10 +3625,18 @@ snapshots: dependencies: p-try: 2.2.0 + p-limit@3.1.0: + dependencies: + yocto-queue: 0.1.0 + p-locate@4.1.0: dependencies: p-limit: 2.3.0 + p-locate@5.0.0: + dependencies: + p-limit: 3.1.0 + p-map@2.1.0: {} p-try@2.2.0: {} @@ -3099,6 +3690,8 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 + prelude-ls@1.2.1: {} + prettier@2.8.8: {} proxy-addr@2.0.7: @@ -3106,6 +3699,8 @@ snapshots: forwarded: 0.2.0 ipaddr.js: 1.9.1 + punycode@2.3.1: {} + qs@6.15.2: dependencies: side-channel: 1.1.1 @@ -3324,6 +3919,10 @@ snapshots: tree-kill@1.2.2: {} + ts-api-utils@2.5.0(typescript@5.9.3): + dependencies: + typescript: 5.9.3 + ts-interface-checker@0.1.13: {} tsup@8.5.1(postcss@8.5.15)(tsx@4.22.4)(typescript@5.9.3): @@ -3360,12 +3959,27 @@ snapshots: optionalDependencies: fsevents: 2.3.3 + type-check@0.4.0: + dependencies: + prelude-ls: 1.2.1 + type-is@2.1.0: dependencies: content-type: 2.0.0 media-typer: 1.1.0 mime-types: 3.0.2 + typescript-eslint@8.64.0(eslint@10.7.0)(typescript@5.9.3): + dependencies: + '@typescript-eslint/eslint-plugin': 8.64.0(@typescript-eslint/parser@8.64.0(eslint@10.7.0)(typescript@5.9.3))(eslint@10.7.0)(typescript@5.9.3) + '@typescript-eslint/parser': 8.64.0(eslint@10.7.0)(typescript@5.9.3) + '@typescript-eslint/typescript-estree': 8.64.0(typescript@5.9.3) + '@typescript-eslint/utils': 8.64.0(eslint@10.7.0)(typescript@5.9.3) + eslint: 10.7.0 + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + typescript@5.9.3: {} ufo@1.6.4: {} @@ -3376,6 +3990,10 @@ snapshots: unpipe@1.0.0: {} + uri-js@4.4.1: + dependencies: + punycode: 2.3.1 + vary@1.1.2: {} vite-node@3.2.4(@types/node@22.20.0)(tsx@4.22.4): @@ -3462,8 +4080,12 @@ snapshots: siginfo: 2.0.0 stackback: 0.0.2 + word-wrap@1.2.5: {} + wrappy@1.0.2: {} + yocto-queue@0.1.0: {} + zod-to-json-schema@3.25.2(zod@4.4.3): dependencies: zod: 4.4.3 From 913c4e6af259b5f3c1c17222d0b1410ce34f574c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 14 Jul 2026 13:49:46 +0000 Subject: [PATCH 3/5] fix: address code-review findings on the architecture branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - mcp/tsup: externalize @refkit/* so optionalDependencies providers are imported from node_modules at runtime instead of being bundled into dist (--omit=optional now works; provider patches reach users) - cursor: drain the overfetched pool of the current provider page before advancing (no more skipped ranked results); advance internally up to 3 pages when a page is fully seen/gated; cap the seen set; validate the cursor with zod; bind fixes with regression tests - mcp: return nextCursor unconditionally at the top level of search_references output — pagination no longer requires explain - mcp/cli: classify BYOK load failures (ERR_MODULE_NOT_FOUND vs real errors, real error surfaced) and load sources concurrently - merge: report each cross-source rights conflict once per URL with only source-declared licenses (no phantom 'unknown' claims, no duplicates); RightsConflict.licenses is now LicenseId[] - cache: fixed-shape hashed keys (strict-KV-safe) + query fingerprint verified on read, so a key collision degrades to a miss - createRefkit: reject non-array providers (un-awaited async factory) with a clear error at construction time - query: capabilities-less providers declaring legacy queryFeatures keep receiving their filters (loud deprecation, not silent degradation) - internet-archive: escape Lucene syntax in the user query - provider-helpers: shared offsetForPage; providers use validated setIfPositiveInt/offsetForPage for all page wiring; drop the dead useLegacyFilter fallbacks Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_018hoN6wKWxcs8WbUNCM9rtt --- .changeset/architecture-review-core.md | 4 +- .changeset/architecture-review-mcp.md | 3 +- README.md | 12 +- packages/core/src/__tests__/client.test.ts | 50 ++-- packages/core/src/__tests__/merge.test.ts | 14 + .../core/src/__tests__/provider-run.test.ts | 11 +- packages/core/src/__tests__/query.test.ts | 31 +++ packages/core/src/client.ts | 259 +++++++++++------- packages/core/src/cursor.ts | 40 +-- packages/core/src/index.ts | 2 +- packages/core/src/merge.ts | 48 +++- packages/core/src/provider-helpers.ts | 9 + packages/core/src/provider-run.ts | 46 +++- packages/core/src/query.ts | 24 +- packages/mcp/src/cli.ts | 26 +- packages/mcp/src/index.ts | 18 +- packages/mcp/tsup.config.ts | 6 + packages/provider-artic/src/index.ts | 3 +- packages/provider-europeana/src/index.ts | 3 +- packages/provider-flickr/src/index.ts | 3 +- packages/provider-freesound/src/index.ts | 2 +- packages/provider-gutendex/src/index.ts | 2 +- .../src/__tests__/internet-archive.test.ts | 15 +- .../provider-internet-archive/src/index.ts | 13 +- packages/provider-jamendo/src/index.ts | 6 +- packages/provider-met/src/index.ts | 3 +- packages/provider-openverse/src/index.ts | 4 +- packages/provider-pexels/src/index.ts | 12 +- packages/provider-pixabay/src/index.ts | 16 +- packages/provider-polyhaven/src/index.ts | 8 +- packages/provider-smithsonian/src/index.ts | 3 +- .../src/__tests__/unsplash.test.ts | 4 +- packages/provider-unsplash/src/index.ts | 12 +- .../provider-wikimedia-commons/src/index.ts | 3 +- 34 files changed, 461 insertions(+), 254 deletions(-) diff --git a/.changeset/architecture-review-core.md b/.changeset/architecture-review-core.md index 4c2708c..30c3771 100644 --- a/.changeset/architecture-review-core.md +++ b/.changeset/architecture-review-core.md @@ -5,9 +5,9 @@ Architecture-review hardening: - **Conservative cross-source rights merge** — when two sources disagree about the license of the same canonical URL, the stricter license wins (incomparable claims collapse to `unknown` → needs-review); conflicts surface in `meta.warnings` and via the new `merge.onRightsConflict` observer. New export: `stricterLicense`, `RightsConflict`. -- **Unified pagination cursor** — `SearchInput.cursor` + `meta.nextCursor`: opaque load-more cursor that advances the provider-local page and dedupes against previously returned results. +- **Unified pagination cursor** — `SearchInput.cursor` + `meta.nextCursor`: opaque load-more cursor that first drains the current provider page's overfetched pool, then advances the provider-local page internally, deduping against previously returned results (seen-set capped so cursors stay small). - **CJK-aware `tokenize`** — CJK runs tokenize into character bigrams, so `lexicalReranker` scores Chinese/Japanese/Korean queries instead of dropping them. -- **Collision-proof cache keys** — per-provider cache keys embed the full normalized query instead of a 32-bit hash (two distinct queries can no longer silently share a cache entry). Existing cache entries are invalidated by the key-format change. +- **Collision-proof cache keys** — per-provider cache keys stay short and fixed-shape (safe for strict KV backends), while the cached value embeds the full normalized-query fingerprint and is verified on read: a key collision degrades to a cache miss, never to another query's results. Existing cache entries are invalidated by the format change (`refkit:v2:`). - New `cacheRaw: false` option strips `raw` provider payloads from cache entries. - New `concurrency` option bounds how many provider searches run at once per search call (default unlimited, matching previous behavior); a queued provider's timeout starts only when it actually runs. - **Deprecations (single-track capability routing)** — `SearchFilters`, `SearchInput.filters`, `NormalizedQuery.filters`, `ReferenceProvider.queryFeatures` (now optional), and `QueryFeature` are deprecated. Routing is driven solely by `capabilities.controls`; legacy `filters` are merged into `controls` and the deprecated `NormalizedQuery.filters` channel is derived from the routed controls, so both channels always agree. Providers that declared filter support only via `queryFeatures` must declare `capabilities.controls` to keep receiving those values. diff --git a/.changeset/architecture-review-mcp.md b/.changeset/architecture-review-mcp.md index fcbf6fa..74d44f4 100644 --- a/.changeset/architecture-review-mcp.md +++ b/.changeset/architecture-review-mcp.md @@ -2,5 +2,6 @@ "@refkit/mcp": minor --- -- `search_references` gains `rerank: true` (query-aware re-ranking via `lexicalReranker`, CJK-aware) and `cursor` (load-more pagination with cross-page dedup; the next cursor rides on `meta.nextCursor` with `explain: true`). +- `search_references` gains `rerank: true` (query-aware re-ranking via `lexicalReranker`, CJK-aware) and `cursor` (load-more pagination with cross-page dedup; the continuation token is returned as top-level `nextCursor` on every call, independent of `explain`). +- BYOK provider packages are explicitly externalized from the tsup bundle so dynamic imports resolve from node_modules at runtime — `--omit=optional` actually omits them, and provider patch releases reach `@refkit/mcp` users without a republish. - BYOK provider packages moved from `dependencies` to `optionalDependencies` and are now loaded lazily, only when their key is present. Default installs (incl. `npx -y @refkit/mcp`) still get everything; installs with `--omit=optional` skip BYOK sources, and a key whose package is missing logs a stderr warning instead of crashing. `defaultProviders()` is now async. diff --git a/README.md b/README.md index 10c1c7c..c33dee2 100644 --- a/README.md +++ b/README.md @@ -113,12 +113,12 @@ console.log(meta.warnings) Use the built-in cursor: every `searchWithMeta` result carries an opaque `meta.nextCursor`; pass it back as `cursor` and refkit advances the provider-local page **and dedupes against everything already returned** — no caller-side bookkeeping: ```ts -const page1 = await refkit.searchWithMeta({ query: 'forest path', modalities: ['image'] }) -const page2 = await refkit.searchWithMeta({ query: 'forest path', modalities: ['image'], cursor: page1.meta.nextCursor }) -// page2.references never repeats page1's; an empty page (no meta.nextCursor) means exhausted. +const batch1 = await refkit.searchWithMeta({ query: 'forest path', modalities: ['image'] }) +const batch2 = await refkit.searchWithMeta({ query: 'forest path', modalities: ['image'], cursor: batch1.meta.nextCursor }) +// batch2 never repeats batch1; an empty batch (no meta.nextCursor) means exhausted. ``` -Under the hood `controls.page` is still a **provider-local** cursor (each source paginates its own stream; RRF-fused pages can overlap or shift), which is exactly why the cursor tracks seen results for you. Raw `controls.page` remains available if you want to manage pages yourself — then dedupe across pages by `canonicalizeUrl(r.canonicalUrl)`. +The cursor first **drains the overfetched pool** of the current provider page (each search fetches `limit × poolFactor` candidates but returns `limit`), and only then advances the provider-local page — so ranked results are never skipped. Under the hood `controls.page` is still provider-local (each source paginates its own stream; RRF-fused pages can overlap), which is exactly why the cursor tracks seen results for you. The seen set is capped (oldest evicted) so cursors stay small. Raw `controls.page` remains available if you want to manage pages yourself — then dedupe across pages by `canonicalizeUrl(r.canonicalUrl)`. ## Ranking & rerank @@ -180,7 +180,7 @@ With many sources registered, bound the fan-out with `concurrency` — at most N createRefkit({ providers, concurrency: 6 }) // default: unlimited ``` -Pass a `cache` to memoize **per-provider** results (keyed by provider + the full normalized query — keys can be a few hundred bytes; TTL `cacheTtlMs`, default 5 min). Merging, reranking, and the license gate always run fresh; cache hits are flagged `cached: true` in `meta.providers`, and every provider status carries `latencyMs`. Pass `cacheRaw: false` to strip each result's `raw` provider payload from cache entries (smaller entries; raw-reading `isDuplicate` hooks then won't see `raw` on hits): +Pass a `cache` to memoize **per-provider** results (short fixed-shape hashed keys, safe for strict KV backends; the cached value embeds the full normalized query and is verified on read, so a key collision degrades to a miss instead of serving another query's results; TTL `cacheTtlMs`, default 5 min). Merging, reranking, and the license gate always run fresh; cache hits are flagged `cached: true` in `meta.providers`, and every provider status carries `latencyMs`. Pass `cacheRaw: false` to strip each result's `raw` provider payload from cache entries (smaller entries; raw-reading `isDuplicate` hooks then won't see `raw` on hits): ```ts createRefkit({ providers, cache: myKvCache, cacheTtlMs: 60_000, cacheRaw: false }) @@ -251,7 +251,7 @@ Agents can use refkit in two ways: npx -y @refkit/mcp ``` -It boots with the keyless sources (Met, Art Institute, Wikimedia, Openverse, Project Gutenberg, PoetryDB, Rijksmuseum, Poly Haven, ambientCG, Internet Archive) and auto-adds any BYOK source whose key is in the environment (`REFKIT_UNSPLASH_KEY`, `REFKIT_PEXELS_KEY`, `REFKIT_BRAVE_KEY`, … — legacy names like `UNSPLASH_KEY`, `PEXELS_KEY`, `BRAVE_TOKEN` still work as fallbacks). Pass `intent` to annotate each result with a use-verdict (may I use this, is attribution required); `gateFor` to return only allowed results; `rerank: true` for query-aware re-ranking (term coverage incl. CJK, resolution, source diversity); `cursor` (from `meta.nextCursor`, with `explain: true`) to page through results without repeats. BYOK provider packages are `optionalDependencies` of `@refkit/mcp` — installed by default (zero-config `npx` keeps working), but an install with `--omit=optional` skips them, and a key whose package is missing just logs a stderr warning instead of crashing the server. Beyond search, `evaluate_use` and `build_attribution` expose the same license-verdict and attribution logic as standalone stateless tools, for when an agent already has a license id and just needs a verdict or a credit line. Or wire your own providers/keys via `serveStdio(createRefkit({ … }))` — see [`@refkit/mcp`](https://www.npmjs.com/package/@refkit/mcp). +It boots with the keyless sources (Met, Art Institute, Wikimedia, Openverse, Project Gutenberg, PoetryDB, Rijksmuseum, Poly Haven, ambientCG, Internet Archive) and auto-adds any BYOK source whose key is in the environment (`REFKIT_UNSPLASH_KEY`, `REFKIT_PEXELS_KEY`, `REFKIT_BRAVE_KEY`, … — legacy names like `UNSPLASH_KEY`, `PEXELS_KEY`, `BRAVE_TOKEN` still work as fallbacks). Pass `intent` to annotate each result with a use-verdict (may I use this, is attribution required); `gateFor` to return only allowed results; `rerank: true` for query-aware re-ranking (term coverage incl. CJK, resolution, source diversity); `cursor` (from the previous result's top-level `nextCursor` — always returned, no `explain` needed) to page through results without repeats. BYOK provider packages are `optionalDependencies` of `@refkit/mcp` — installed by default (zero-config `npx` keeps working), but an install with `--omit=optional` skips them, and a key whose package is missing just logs a stderr warning instead of crashing the server. Beyond search, `evaluate_use` and `build_attribution` expose the same license-verdict and attribution logic as standalone stateless tools, for when an agent already has a license id and just needs a verdict or a credit line. Or wire your own providers/keys via `serveStdio(createRefkit({ … }))` — see [`@refkit/mcp`](https://www.npmjs.com/package/@refkit/mcp). ## Not legal advice diff --git a/packages/core/src/__tests__/client.test.ts b/packages/core/src/__tests__/client.test.ts index a934699..5ee4b0d 100644 --- a/packages/core/src/__tests__/client.test.ts +++ b/packages/core/src/__tests__/client.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from 'vitest' import { createRefkit } from '../client' -import { defineProvider } from '../provider' +import { defineProvider, type ReferenceProvider } from '../provider' import { lexicalReranker } from '../rerank' import type { Reference } from '../reference' import type { LicenseId } from '../license' @@ -77,11 +77,13 @@ describe('createRefkit', () => { expect(rk.buildAttribution(r).required).toBe(true) }) - it('cursor: pages via controls.page, dedupes across pages, and chains nextCursor', async () => { - // A paging provider whose page 2 overlaps page 1 — the RRF reality. + it('cursor: drains the overfetched pool before advancing the provider page', async () => { + // fetchLimit > limit: the page-1 pool holds MORE than one batch. The cursor + // must keep returning from the same provider page until it is exhausted — + // advancing per batch would skip ranked results forever. const pages: Record = { - 1: [ref('a-1', 'https://a/1'), ref('a-2', 'https://a/2')], - 2: [ref('a-2', 'https://a/2'), ref('a-3', 'https://a/3')], + 1: [ref('a-1', 'https://a/1'), ref('a-2', 'https://a/2'), ref('a-3', 'https://a/3'), ref('a-4', 'https://a/4')], + 2: [ref('a-5', 'https://a/5')], } const seenPages: Array = [] const paging = defineProvider({ @@ -95,17 +97,25 @@ describe('createRefkit', () => { }) const rk = createRefkit({ providers: [paging] }) - const page1 = await rk.searchWithMeta({ query: 'x', modalities: ['image'], limit: 2 }) - expect(page1.references.map(r => r.canonicalUrl)).toEqual(['https://a/1', 'https://a/2']) - expect(page1.meta.nextCursor).toBeDefined() + const batch1 = await rk.searchWithMeta({ query: 'x', modalities: ['image'], limit: 2 }) + expect(batch1.references.map(r => r.canonicalUrl)).toEqual(['https://a/1', 'https://a/2']) + expect(batch1.meta.nextCursor).toBeDefined() - const page2 = await rk.searchWithMeta({ query: 'x', modalities: ['image'], limit: 2, cursor: page1.meta.nextCursor }) - expect(seenPages).toEqual([undefined, 2]) // cursor routed page 2 to the provider - expect(page2.references.map(r => r.canonicalUrl)).toEqual(['https://a/3']) // overlap deduped + // Batch 2 comes from the REMAINDER of page 1 — no page advance. + const batch2 = await rk.searchWithMeta({ query: 'x', modalities: ['image'], limit: 2, cursor: batch1.meta.nextCursor }) + expect(batch2.references.map(r => r.canonicalUrl)).toEqual(['https://a/3', 'https://a/4']) + expect(seenPages).toEqual([undefined, 1]) - const page3 = await rk.searchWithMeta({ query: 'x', modalities: ['image'], limit: 2, cursor: page2.meta.nextCursor }) - expect(page3.references).toEqual([]) // exhausted - expect(page3.meta.nextCursor).toBeUndefined() // empty page ends the chain + // Page 1 exhausted → the next call advances to page 2 internally. + const batch3 = await rk.searchWithMeta({ query: 'x', modalities: ['image'], limit: 2, cursor: batch2.meta.nextCursor }) + expect(batch3.references.map(r => r.canonicalUrl)).toEqual(['https://a/5']) + expect(seenPages).toEqual([undefined, 1, 1, 2]) + + // Page 2 exhausted and page 3 empty → chain ends. + const batch4 = await rk.searchWithMeta({ query: 'x', modalities: ['image'], limit: 2, cursor: batch3.meta.nextCursor }) + expect(batch4.references).toEqual([]) + expect(batch4.meta.nextCursor).toBeUndefined() + expect(seenPages).toEqual([undefined, 1, 1, 2, 2, 3]) }) it('cursor: rejects strings that did not come from meta.nextCursor', async () => { @@ -114,6 +124,12 @@ describe('createRefkit', () => { await expect(rk.search({ query: 'x', modalities: ['image'], cursor: '{"v":9}' })).rejects.toThrow(/invalid cursor/) }) + it('rejects a Promise passed as providers (un-awaited async factory)', () => { + const promised = Promise.resolve([provider('a', [])]) + expect(() => createRefkit({ providers: promised as unknown as ReferenceProvider[] })).toThrow(/non-empty array/) + promised.catch(() => {}) + }) + it('surfaces cross-source license conflicts as meta.warnings with conservative rights', async () => { const rk = createRefkit({ providers: [ @@ -615,10 +631,10 @@ describe('createRefkit', () => { }) const rk = createRefkit({ providers: [counted], cache }) await rk.search({ query: 'x', modalities: ['image'] }) // live search, populates cache - // seed the cache entry with 1 valid + 1 malformed item + // seed the cache entry with 1 valid + 1 malformed item (keep the {q, refs} envelope) for (const k of cache.store.keys()) { - const valid = JSON.parse(cache.store.get(k)!) - cache.store.set(k, JSON.stringify([...valid, { id: '', modality: 'image' }])) + const payload = JSON.parse(cache.store.get(k)!) + cache.store.set(k, JSON.stringify({ ...payload, refs: [...payload.refs, { id: '', modality: 'image' }] })) } const out = await rk.searchWithMeta({ query: 'x', modalities: ['image'], onProviderError }) expect(out.references.map(r => r.id)).toEqual(['c-1']) diff --git a/packages/core/src/__tests__/merge.test.ts b/packages/core/src/__tests__/merge.test.ts index fede5b8..1b1bb84 100644 --- a/packages/core/src/__tests__/merge.test.ts +++ b/packages/core/src/__tests__/merge.test.ts @@ -93,6 +93,20 @@ describe('mergeReferences (RRF)', () => { expect(same.rights.license).toBe('CC-BY') }) + it('reports a 3-source conflict ONCE with only source-declared licenses (no phantom unknown)', () => { + const seen: RightsConflict[] = [] + const out = mergeReferences([ + [withLicense(make('a-1', 'https://shared/1'), 'CC-BY-NC')], + [withLicense(make('b-1', 'https://shared/1'), 'CC-BY-SA')], + [withLicense(make('c-1', 'https://shared/1'), 'CC-BY-NC')], // re-declares an already-seen license + ], { onRightsConflict: (c) => seen.push(c) }) + expect(seen).toHaveLength(1) + expect(seen[0].licenses.sort()).toEqual(['CC-BY-NC', 'CC-BY-SA']) + expect(seen[0].licenses).not.toContain('unknown') // resolution value never reported as a claim + expect(seen[0].resolvedLicense).toBe('unknown') // NC vs SA are incomparable → strict-deny + expect(out[0].rights.license).toBe('unknown') + }) + it('stricterLicense: dominance picks the stricter; incomparable pairs return undefined', () => { expect(stricterLicense('CC-BY', 'CC-BY-NC')).toBe('CC-BY-NC') expect(stricterLicense('CC0-1.0', 'proprietary')).toBe('proprietary') diff --git a/packages/core/src/__tests__/provider-run.test.ts b/packages/core/src/__tests__/provider-run.test.ts index fce8b37..bb59d60 100644 --- a/packages/core/src/__tests__/provider-run.test.ts +++ b/packages/core/src/__tests__/provider-run.test.ts @@ -30,11 +30,12 @@ function memoryCache(): KeyValueCache & { store: Map } { } describe('providerCacheKey', () => { - it('embeds the full normalized query — distinct queries can never collide', () => { - const a = providerCacheKey('p', { text: 'lion', modalities: ['image'] }) + it('is short, fixed-shape, and free of raw query characters', () => { + const a = providerCacheKey('p', { text: 'lion cub "quoted" \n spaced', modalities: ['image'] }) const b = providerCacheKey('p', { text: 'tiger', modalities: ['image'] }) expect(a).not.toBe(b) - expect(a).toContain('lion') + expect(a).toMatch(/^refkit:v2:p:[a-z0-9]+$/) // no spaces/quotes → safe for strict KV backends + expect(a.length).toBeLessThan(64) }) it('is insensitive to object key order', () => { @@ -52,7 +53,7 @@ describe('runProviderSearch cacheRaw', () => { await runProviderSearch(provider([ref('https://a/1')]), { text: 'q', modalities: ['image'] }, { ...deps, cache, cacheRaw: true }) await new Promise(r => setTimeout(r)) // cache write is fire-and-forget const [payload] = [...cache.store.values()] - expect(JSON.parse(payload)[0].raw).toEqual({ upstream: 'payload' }) + expect(JSON.parse(payload).refs[0].raw).toEqual({ upstream: 'payload' }) }) it('cacheRaw: false strips raw from the cached payload but not from live results', async () => { @@ -61,6 +62,6 @@ describe('runProviderSearch cacheRaw', () => { expect(run.ok && run.valid[0].raw).toEqual({ upstream: 'payload' }) await new Promise(r => setTimeout(r)) const [payload] = [...cache.store.values()] - expect(JSON.parse(payload)[0].raw).toBeUndefined() + expect(JSON.parse(payload).refs[0].raw).toBeUndefined() }) }) diff --git a/packages/core/src/__tests__/query.test.ts b/packages/core/src/__tests__/query.test.ts index 055edf9..04f4ded 100644 --- a/packages/core/src/__tests__/query.test.ts +++ b/packages/core/src/__tests__/query.test.ts @@ -17,6 +17,37 @@ describe('normalizeQuery', () => { expect(nq.controls).toEqual({ color: 'red' }) // derived channel stays consistent }) + it('legacy compat: a capabilities-less provider declaring only queryFeatures still receives its filters', () => { + const legacy: ReferenceProvider = { + id: 'p', + modalities: ['image'], + queryFeatures: ['keyword', 'orientation'], // pre-capabilities third-party shape + search: async () => [], + } + const nq = normalizeQuery( + { query: 'cat', modalities: ['image'], filters: { orientation: 'landscape', color: 'red' } }, + legacy, + ) + expect(nq.filters).toEqual({ orientation: 'landscape' }) // color not declared → dropped + expect(nq.controls).toEqual({ orientation: 'landscape' }) + }) + + it('capabilities, once declared, win over queryFeatures', () => { + const both: ReferenceProvider = { + id: 'p', + modalities: ['image'], + queryFeatures: ['keyword', 'orientation'], + capabilities: { controls: [] }, // explicit: supports nothing + search: async () => [], + } + const nq = normalizeQuery( + { query: 'cat', modalities: ['image'], filters: { orientation: 'landscape' } }, + both, + ) + expect(nq.filters).toBeUndefined() + expect(nq.controls).toBeUndefined() + }) + it('omits filters entirely when none survive', () => { const nq = normalizeQuery( { query: 'cat', modalities: ['image'], filters: { color: 'red' } }, diff --git a/packages/core/src/client.ts b/packages/core/src/client.ts index b3a749b..b3e40a1 100644 --- a/packages/core/src/client.ts +++ b/packages/core/src/client.ts @@ -16,7 +16,7 @@ import type { import { mergeReferences, type MergeOptions, type RightsConflict } from './merge' import { mergeSearchControls, normalizeQuery, requestedControlKeys, supportedControlKeys, unsupportedControlKeys } from './query' import { retryingFetch } from './resilience' -import { runProviderSearch, type ProviderRun } from './provider-run' +import { runProviderSearch } from './provider-run' import { cursorSeenKey, decodeCursor, encodeCursor } from './cursor' export interface ResilienceOptions { @@ -89,8 +89,8 @@ export interface SearchMeta { providers: ProviderSearchStatus[] gate?: SearchGateMeta /** Opaque "load more" cursor: pass as `SearchInput.cursor` to fetch the next - * page with cross-page dedup handled internally. Present when this page - * returned at least one result. */ + * batch with cross-page dedup handled internally. Present when this call + * returned at least one result; absent = the stream is exhausted. */ nextCursor?: string warnings: string[] } @@ -111,10 +111,11 @@ export interface SearchInput { * matching entry to each provider; providers whitelist what they translate. */ providerOptions?: ProviderOptionsById limit?: number - /** Opaque cursor from a previous search's `meta.nextCursor`. Sets the - * provider-local page (overriding `controls.page`) and filters out results - * already returned on earlier pages, so "load more" needs no caller-side - * dedup. Throws on a string that did not come from `meta.nextCursor`. */ + /** Opaque cursor from a previous search's `meta.nextCursor`. Resumes the + * provider-local page (overriding `controls.page`), filters out results + * already returned on earlier calls, and advances the page automatically once + * the current page's pool is exhausted — "load more" needs no caller-side + * bookkeeping. Throws on a string that did not come from `meta.nextCursor`. */ cursor?: string /** Overfetch this many × `limit` candidates per provider before merge/rerank/gate, * then narrow to `limit` — a wider pool means better dedup + ranking. Default 4 @@ -142,6 +143,12 @@ const MAX_POOL_LIMIT = 100 // never ask a single source for more than this, even const DEFAULT_TIMEOUT_MS = 10_000 const DEFAULT_RETRIES = 1 const DEFAULT_CACHE_TTL_MS = 300_000 +// Cursor: how many further provider pages one load-more call may try when the +// current page's pool is fully consumed, before reporting an empty batch. +const MAX_CURSOR_ADVANCES = 3 +// Cursor: cap on remembered already-returned keys (most recent kept). Bounds +// cursor size (~7 bytes/key); overflowing just risks re-showing very old results. +const MAX_CURSOR_SEEN = 500 function errorSummary(error: unknown): string { if (error instanceof Error) return error.message @@ -164,8 +171,11 @@ async function mapBounded(items: readonly T[], limit: number, fn: (item: T } export function createRefkit(options: RefkitOptions): RefkitClient { - if (!options.providers || options.providers.length === 0) { - throw new Error('createRefkit: at least one provider is required') + // Array.isArray (not just truthiness): a Promise — e.g. an + // un-awaited async factory — must fail here with a clear message, not pass + // construction and crash cryptically on the first search. + if (!Array.isArray(options.providers) || options.providers.length === 0) { + throw new Error('createRefkit: providers must be a non-empty array (did you forget to await an async provider factory?)') } async function searchInternal(input: SearchInput): Promise { @@ -182,21 +192,8 @@ export function createRefkit(options: RefkitOptions): RefkitClient { // Overfetch a wider candidate pool per provider, then narrow to `limit` after // merge/rerank/gate — you can't rank or dedup candidates you never fetched. const fetchLimit = Math.max(limit, Math.min(Math.ceil(limit * poolFactor), MAX_POOL_LIMIT)) - // A cursor overrides controls.page and carries the already-seen set; the - // effective controls are what routing, providers, and meta all see. const cursorState = input.cursor !== undefined ? decodeCursor(input.cursor) : undefined - const controls = cursorState ? { ...input.controls, page: cursorState.page } : input.controls - const requestedControlsSource = mergeSearchControls(controls, input.filters) - const requestedControls = requestedControlKeys(requestedControlsSource) - const controlsMeta = requestedControls.length > 0 ? { - requested: requestedControls, - appliedByProvider: Object.fromEntries(options.providers.map(p => [p.id, supportedControlKeys(p, requestedControlsSource)])), - ignoredByProvider: Object.fromEntries(options.providers.map(p => [p.id, unsupportedControlKeys(p, requestedControlsSource)])), - } : undefined - const statusByProvider = new Map() - for (const p of options.providers) { - if (!chosen.includes(p)) statusByProvider.set(p.id, { providerId: p.id, status: 'skipped', reason: 'unsupported-modality' }) - } + const seenSet = cursorState ? new Set(cursorState.seen) : undefined const resilience = options.resilience === false ? undefined : { timeoutMs: options.resilience?.timeoutMs ?? DEFAULT_TIMEOUT_MS, retries: options.resilience?.retries ?? DEFAULT_RETRIES, @@ -205,106 +202,156 @@ export function createRefkit(options: RefkitOptions): RefkitClient { // shared across every provider in the fan-out below, instead of allocating // a fresh wrapper per provider. const sharedFetch = resilience && resilience.retries > 0 ? retryingFetch(doFetch, { retries: resilience.retries }) : doFetch - - const runProvider = (p: ReferenceProvider): Promise => { - const query = normalizeQuery({ - query: input.query, - modalities: input.modalities, - filters: input.filters, - controls, - providerOptions: input.providerOptions, - limit: fetchLimit, - }, p) - return runProviderSearch(p, query, { - fetch: sharedFetch, - cache: options.cache, - cacheTtlMs: options.cacheTtlMs ?? DEFAULT_CACHE_TTL_MS, - cacheRaw: options.cacheRaw ?? true, - timeoutMs: resilience?.timeoutMs, - signal: input.signal ?? options.signal, - onError: (error) => input.onProviderError?.({ providerId: p.id, error }), - }) - } - const concurrency = options.concurrency !== undefined && options.concurrency >= 1 ? Math.floor(options.concurrency) : undefined - const runs = concurrency - ? await mapBounded(chosen, concurrency, runProvider) - : await Promise.all(chosen.map(runProvider)) - const perSource: Reference[][] = [] - let anyOk = false - runs.forEach((run, i) => { - const provider = chosen[i] - if (run.ok) { - anyOk = true - statusByProvider.set(provider.id, { - providerId: provider.id, - status: 'fulfilled', - returned: run.returned, - accepted: run.valid.length, - rejected: run.returned - run.valid.length, - latencyMs: run.latencyMs, - ...(run.cached ? { cached: true } : {}), + interface PassOutcome { + refs: Reference[] // post merge/rerank/gate/seen-filter, best-first + controlsMeta?: SearchControlsMeta + statusByProvider: Map + gate?: SearchGateMeta + rightsConflicts: RightsConflict[] + totalReturned: number // raw items across fulfilled providers (pre-parse) + } + + // One full fan-out → merge → rerank → gate → seen-filter pass at the given + // provider-local page. The cursor path may run several passes per call. + const runPass = async (page: number | undefined): Promise => { + const controls = page !== undefined ? { ...input.controls, page } : input.controls + const requestedControlsSource = mergeSearchControls(controls, input.filters) + const requestedControls = requestedControlKeys(requestedControlsSource) + const controlsMeta = requestedControls.length > 0 ? { + requested: requestedControls, + appliedByProvider: Object.fromEntries(options.providers.map(p => [p.id, supportedControlKeys(p, requestedControlsSource)])), + ignoredByProvider: Object.fromEntries(options.providers.map(p => [p.id, unsupportedControlKeys(p, requestedControlsSource)])), + } : undefined + const statusByProvider = new Map() + for (const p of options.providers) { + if (!chosen.includes(p)) statusByProvider.set(p.id, { providerId: p.id, status: 'skipped', reason: 'unsupported-modality' }) + } + + const runProvider = (p: ReferenceProvider) => { + const query = normalizeQuery({ + query: input.query, + modalities: input.modalities, + filters: input.filters, + controls, + providerOptions: input.providerOptions, + limit: fetchLimit, + }, p) + return runProviderSearch(p, query, { + fetch: sharedFetch, + cache: options.cache, + cacheTtlMs: options.cacheTtlMs ?? DEFAULT_CACHE_TTL_MS, + cacheRaw: options.cacheRaw ?? true, + timeoutMs: resilience?.timeoutMs, + signal: input.signal ?? options.signal, + onError: (error) => input.onProviderError?.({ providerId: p.id, error }), }) - perSource.push(run.valid) - } else { - statusByProvider.set(provider.id, { providerId: provider.id, status: 'failed', error: errorSummary(run.error), latencyMs: run.latencyMs }) } - }) - if (!anyOk) { - throw new AggregateError(runs.filter(r => !r.ok).map(r => (r as { error: unknown }).error), 'refkit.search: all providers failed') - } + const runs = concurrency + ? await mapBounded(chosen, concurrency, runProvider) + : await Promise.all(chosen.map(runProvider)) - // Collect cross-source license conflicts for meta.warnings while still - // forwarding them to a host-supplied observer. - const rightsConflicts: RightsConflict[] = [] - let refs = mergeReferences(perSource, { - ...options.merge, - onRightsConflict: (c) => { - rightsConflicts.push(c) - options.merge?.onRightsConflict?.(c) - }, - }) - // Rerank runs over the FULL merged pool, before the license gate — ordering - // (and a reranker's batch-relative scoring, e.g. quality normalised across - // the pool) is computed against every candidate, then the gate drops denied - // ones while preserving order. Core does not re-validate the returned refs; - // a reranker is trusted to honour the Reranker contract. - if (input.rerank) { - refs = await input.rerank({ query: input.query, refs, signal: input.signal ?? options.signal }) - } - const beforeGate = refs.length - let gate: SearchGateMeta | undefined - if (input.gateFor) { - const intent = input.gateFor - refs = refs.filter(r => evaluateUse(r.rights, intent).decision.startsWith('allowed')) - gate = { intent, before: beforeGate, after: refs.length, dropped: beforeGate - refs.length } + const perSource: Reference[][] = [] + let anyOk = false + let totalReturned = 0 + runs.forEach((run, i) => { + const provider = chosen[i] + if (run.ok) { + anyOk = true + totalReturned += run.returned + statusByProvider.set(provider.id, { + providerId: provider.id, + status: 'fulfilled', + returned: run.returned, + accepted: run.valid.length, + rejected: run.returned - run.valid.length, + latencyMs: run.latencyMs, + ...(run.cached ? { cached: true } : {}), + }) + perSource.push(run.valid) + } else { + statusByProvider.set(provider.id, { providerId: provider.id, status: 'failed', error: errorSummary(run.error), latencyMs: run.latencyMs }) + } + }) + + if (!anyOk) { + throw new AggregateError(runs.filter(r => !r.ok).map(r => (r as { error: unknown }).error), 'refkit.search: all providers failed') + } + + // Collect cross-source license conflicts for meta.warnings while still + // forwarding them to a host-supplied observer. + const rightsConflicts: RightsConflict[] = [] + let refs = mergeReferences(perSource, { + ...options.merge, + onRightsConflict: (c) => { + rightsConflicts.push(c) + options.merge?.onRightsConflict?.(c) + }, + }) + // Rerank runs over the FULL merged pool, before the license gate — ordering + // (and a reranker's batch-relative scoring, e.g. quality normalised across + // the pool) is computed against every candidate, then the gate drops denied + // ones while preserving order. Core does not re-validate the returned refs; + // a reranker is trusted to honour the Reranker contract. + if (input.rerank) { + refs = await input.rerank({ query: input.query, refs, signal: input.signal ?? options.signal }) + } + const beforeGate = refs.length + let gate: SearchGateMeta | undefined + if (input.gateFor) { + const intent = input.gateFor + refs = refs.filter(r => evaluateUse(r.rights, intent).decision.startsWith('allowed')) + gate = { intent, before: beforeGate, after: refs.length, dropped: beforeGate - refs.length } + } + // Cursor pagination: drop results already returned on earlier calls (RRF + // pages overlap by design), AFTER rank/gate so ordering is batch-consistent + // but BEFORE the limit so repeats don't consume the batch budget. + if (seenSet) { + refs = refs.filter(r => !seenSet.has(cursorSeenKey(r.canonicalUrl))) + } + return { refs, controlsMeta, statusByProvider, gate, rightsConflicts, totalReturned } } - // Cursor pagination: drop results already returned on earlier pages (RRF - // pages overlap by design), AFTER rank/gate so ordering is batch-consistent - // but BEFORE the limit so repeats don't consume the page budget. + + // Providers fetch fetchLimit (≥ limit) candidates per page, but each call + // returns only `limit` — so the cursor must NOT advance the provider page + // per call, or the unreturned overfetch remainder would be skipped forever. + // Instead nextCursor keeps pointing at the SAME page (the seen-filter makes + // repeats free) and the page advances here, internally, only once a page's + // pool yields nothing new — up to MAX_CURSOR_ADVANCES pages per call. + let page = cursorState ? cursorState.page : input.controls?.page + let pass = await runPass(page) if (cursorState) { - const seen = new Set(cursorState.seen) - refs = refs.filter(r => !seen.has(cursorSeenKey(r.canonicalUrl))) + for ( + let advances = 0; + pass.refs.length === 0 && pass.totalReturned > 0 && advances < MAX_CURSOR_ADVANCES; + advances++ + ) { + page = (page ?? 1) + 1 + pass = await runPass(page) + } } - const references = refs.slice(0, limit) + + const references = pass.refs.slice(0, limit) const nextCursor = references.length > 0 ? encodeCursor({ v: 1, - page: (controls?.page ?? 1) + 1, - seen: [...(cursorState?.seen ?? []), ...references.map(r => cursorSeenKey(r.canonicalUrl))], + // Same page on purpose — its overfetched pool may still hold + // unreturned results; the next call advances internally if not. + page: page ?? 1, + seen: [...(cursorState?.seen ?? []), ...references.map(r => cursorSeenKey(r.canonicalUrl))].slice(-MAX_CURSOR_SEEN), }) : undefined const warnings: string[] = [] - const failedCount = [...statusByProvider.values()].filter(s => s.status === 'failed').length + const failedCount = [...pass.statusByProvider.values()].filter(s => s.status === 'failed').length if (failedCount > 0) warnings.push(`${failedCount} provider(s) failed; returning partial results.`) - for (const c of rightsConflicts) { + for (const c of pass.rightsConflicts) { warnings.push(`cross-source license conflict for ${c.canonicalUrl}: ${c.licenses.join(' vs ')} → resolved to ${c.resolvedLicense}.`) } - if (gate && gate.dropped > 0) warnings.push(`${gate.dropped} result(s) dropped by ${gate.intent} gate.`) + if (pass.gate && pass.gate.dropped > 0) warnings.push(`${pass.gate.dropped} result(s) dropped by ${pass.gate.intent} gate.`) return { references, meta: { @@ -314,10 +361,10 @@ export function createRefkit(options: RefkitOptions): RefkitClient { poolFactor, fetchLimit, ...(input.filters ? { appliedFilters: input.filters } : {}), - ...(controlsMeta ? { controls: controlsMeta } : {}), + ...(pass.controlsMeta ? { controls: pass.controlsMeta } : {}), ...(input.providerOptions ? { providerOptions: Object.keys(input.providerOptions) } : {}), - providers: options.providers.map(p => statusByProvider.get(p.id) ?? { providerId: p.id, status: 'skipped', reason: 'unsupported-modality' }), - ...(gate ? { gate } : {}), + providers: options.providers.map(p => pass.statusByProvider.get(p.id) ?? { providerId: p.id, status: 'skipped', reason: 'unsupported-modality' }), + ...(pass.gate ? { gate: pass.gate } : {}), ...(nextCursor ? { nextCursor } : {}), warnings, }, diff --git a/packages/core/src/cursor.ts b/packages/core/src/cursor.ts index 34d1e02..aff274f 100644 --- a/packages/core/src/cursor.ts +++ b/packages/core/src/cursor.ts @@ -1,22 +1,32 @@ -// Unified "load more" cursor (v1). RRF fusion means provider page N+1 overlaps -// page N, so raw `controls.page` pushes cross-page dedup onto every caller. The -// cursor internalizes that: it carries the next provider-local page plus compact -// hashes of every already-returned result, and the client filters repeats out -// before applying `limit`. The string is an implementation detail — treat it as -// opaque; only `meta.nextCursor` from a previous search is a valid input. +// Unified "load more" cursor (v1). Providers fetch an overfetched pool per page +// (fetchLimit ≥ limit) while each call returns only `limit`, and RRF fusion makes +// raw provider pages overlap — so the cursor carries the CURRENT provider-local +// page plus compact hashes of every already-returned result. The client filters +// repeats out and advances the page internally only once a page's pool is +// exhausted. The string is an implementation detail — treat it as opaque; only +// `meta.nextCursor` from a previous search is a valid input. +import { z } from 'zod' import { fnv1a } from './hash' import { canonicalizeUrl } from './dedup-key' export interface SearchCursorState { v: 1 - /** Provider-local page to request next (routed as controls.page; 1-based). */ + /** Provider-local page the current pool comes from (routed as controls.page; + * 1-based). Advanced by the client, not per call. */ page: number - /** {@link cursorSeenKey} hashes of results returned on previous pages. Grows - * linearly with pages consumed — a 32-bit hash keeps entries compact; the - * worst case of a collision is one new result suppressed as already-seen. */ + /** {@link cursorSeenKey} hashes of results returned on previous calls. Capped + * by the client (most recent kept) so cursor size stays bounded; a 32-bit + * hash keeps entries compact — the worst case of a collision or an evicted + * entry is one result suppressed or repeated. */ seen: string[] } +const cursorSchema = z.object({ + v: z.literal(1), + page: z.number().int().min(1), + seen: z.array(z.string()), +}) + /** Compact already-seen key for a result — same URL canonicalization as merge/dedup. */ export function cursorSeenKey(canonicalUrl: string): string { return fnv1a(canonicalizeUrl(canonicalUrl)) @@ -36,13 +46,9 @@ export function decodeCursor(cursor: string): SearchCursorState { } catch { throw new Error('refkit.search: invalid cursor (not produced by meta.nextCursor)') } - const s = parsed as Partial | null - if ( - !s || typeof s !== 'object' || s.v !== 1 - || typeof s.page !== 'number' || !Number.isInteger(s.page) || s.page < 1 - || !Array.isArray(s.seen) || !s.seen.every(k => typeof k === 'string') - ) { + const result = cursorSchema.safeParse(parsed) + if (!result.success) { throw new Error('refkit.search: invalid cursor (not produced by meta.nextCursor)') } - return { v: 1, page: s.page, seen: s.seen } + return result.data } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index e902967..2c579b1 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -45,7 +45,7 @@ export type { } from './provider' export { setIfString, setIfBoolean, setIfStringList, - setIfInt, setIfPositiveInt, setIfNonNegativeInt, setIfNumber, + setIfInt, setIfPositiveInt, setIfNonNegativeInt, setIfNumber, offsetForPage, first, mapCcDeedUrl, mapRightsUrl, ccVersionFor, CC_FAMILY_BY_TOKEN, CC_VERSIONED_FAMILIES, isLikelyImageUrl, imageMediaType, IMAGE_EXT, } from './provider-helpers' diff --git a/packages/core/src/merge.ts b/packages/core/src/merge.ts index 2d8202e..1702f60 100644 --- a/packages/core/src/merge.ts +++ b/packages/core/src/merge.ts @@ -5,13 +5,14 @@ import { canonicalizeUrl } from './dedup-key' import { dedupeReferences, type DedupeOptions } from './dedup' /** A cross-source disagreement about the license of the same canonical URL, - * reported once per conflicting pair as the merge encounters it. */ + * reported once per URL after the merge pass. */ export interface RightsConflict { canonicalUrl: string - /** The two license ids that disagreed (already-resolved vs newly seen). */ - licenses: [LicenseId, LicenseId] - /** What the merge resolved to: the stricter of the two, or 'unknown' when - * neither is stricter on every axis (strict-deny → needs-review). */ + /** Every distinct SOURCE-DECLARED license id for this URL (never includes a + * synthetic resolution value a source didn't claim). */ + licenses: LicenseId[] + /** What the merge resolved to: the strictest comparable claim, or 'unknown' + * when claims are incomparable (strict-deny → needs-review). */ resolvedLicense: LicenseId } @@ -80,7 +81,11 @@ export function mergeReferences(perSource: Reference[][], opts: MergeOptions = { const score = new Map() // dedup key -> accumulated RRF score const rep = new Map() // dedup key -> best representative const rights = new Map() // dedup key -> conservatively-resolved rights - const conflicted = new Set() // keys whose rights were resolved across sources + // Allocated only on an actual conflict: dedup key -> distinct SOURCE-DECLARED + // license ids. Comparing new refs against this set (not against the resolved + // record, which may already be a synthetic 'unknown') keeps a third source + // re-declaring an already-seen license from re-triggering a phantom conflict. + const conflictLicenses = new Map>() for (const list of perSource) { list.forEach((ref, rank) => { @@ -96,19 +101,32 @@ export function mergeReferences(perSource: Reference[][], opts: MergeOptions = { const known = rights.get(key) if (known === undefined) { rights.set(key, ref.rights) + return + } + const declared = conflictLicenses.get(key) + if (declared) { + if (!declared.has(ref.rights.license)) { + declared.add(ref.rights.license) + rights.set(key, resolveRightsConflict(known, ref.rights)) + } } else if (known.license !== ref.rights.license) { - const resolved = resolveRightsConflict(known, ref.rights) - rights.set(key, resolved) - conflicted.add(key) - opts.onRightsConflict?.({ - canonicalUrl: ref.canonicalUrl, - licenses: [known.license, ref.rights.license], - resolvedLicense: resolved.license, - }) + conflictLicenses.set(key, new Set([known.license, ref.rights.license])) + rights.set(key, resolveRightsConflict(known, ref.rights)) } }) } + // Report each conflicted URL once, with the full set of source-declared claims. + if (opts.onRightsConflict) { + for (const [key, declared] of conflictLicenses) { + opts.onRightsConflict({ + canonicalUrl: rep.get(key)!.canonicalUrl, + licenses: [...declared], + resolvedLicense: rights.get(key)!.license, + }) + } + } + // Normalize by the actual max so the top result's relevance is exactly 1.0. // Reduce, not Math.max(...score.values()) — the merged pool can be large and a // spread of that many args overflows the call stack. RRF scores are fractional @@ -122,7 +140,7 @@ export function mergeReferences(perSource: Reference[][], opts: MergeOptions = { ...rep.get(key)!, // A conflicted key carries the conservatively-resolved rights instead of // whichever source happened to supply the representative. - ...(conflicted.has(key) ? { rights: rights.get(key)! } : {}), + ...(conflictLicenses.has(key) ? { rights: rights.get(key)! } : {}), relevance: s / maxScore, })) .sort((a, b) => b.relevance - a.relevance) diff --git a/packages/core/src/provider-helpers.ts b/packages/core/src/provider-helpers.ts index 83036d3..43dfbb2 100644 --- a/packages/core/src/provider-helpers.ts +++ b/packages/core/src/provider-helpers.ts @@ -61,6 +61,15 @@ export function setIfNumber(url: URL, key: string, value: unknown, opts?: { min? url.searchParams.set(key, String(value)) } +/** Translate the 1-based `controls.page` into a row offset for offset-based APIs. + * Single-sourced so every provider derives page windows with the same (validated) + * math: undefined for page 1 / missing / non-integer input — callers then omit + * the param. `base` shifts the origin (e.g. Europeana's 1-based `start`). */ +export function offsetForPage(page: unknown, pageSize: number, base = 0): number | undefined { + if (typeof page !== 'number' || !Number.isInteger(page) || page <= 1) return undefined + return (page - 1) * pageSize + base +} + // — array helper — /** First element of an array-typed field, or undefined. */ diff --git a/packages/core/src/provider-run.ts b/packages/core/src/provider-run.ts index c28f61e..58641b1 100644 --- a/packages/core/src/provider-run.ts +++ b/packages/core/src/provider-run.ts @@ -6,6 +6,7 @@ import type { Reference } from './reference' import { parseReference } from './reference' import type { KeyValueCache, NormalizedQuery, ProviderContext, ReferenceProvider } from './provider' import { withTimeout } from './resilience' +import { fnv1a } from './hash' // Deterministic JSON for cache keys: object keys sorted recursively, so a // caller's providerOptions key order can't split otherwise-identical keys. @@ -20,14 +21,25 @@ export function stableStringify(value: unknown): string { return JSON.stringify(value) ?? 'null' } -/** Cache key for one provider's slice of a search. Embeds the FULL normalized - * query (not a truncated hash): a 32-bit digest would let two different queries - * silently collide and serve each other's cached results. Keys can therefore be - * a few hundred bytes — KeyValueCache implementations must tolerate that. */ +/** Cache key for one provider's slice of a search: short, fixed-shape, and safe + * for restrictive backends (no raw query characters, no unbounded length — a + * memcached-style 250-byte/no-whitespace key contract holds). Collisions are + * made harmless rather than merely improbable: the cached VALUE embeds the full + * normalized-query string and the read path verifies it before trusting a hit + * (see runProviderSearch), so a colliding key degrades to a cache miss. The two + * hash passes (raw + length-salted) exist only to make that degradation rare. */ export function providerCacheKey(providerId: string, query: NormalizedQuery): string { - return `refkit:v1:${providerId}:${stableStringify(query)}` + return keyForFingerprint(providerId, stableStringify(query)) } +function keyForFingerprint(providerId: string, fingerprint: string): string { + return `refkit:v2:${providerId}:${fnv1a(fingerprint)}${fnv1a(`${fingerprint.length}:${fingerprint}`)}` +} + +/** Shape of a cached entry: the query fingerprint (verified on read so a key + * collision degrades to a miss, never to another query's results) + the refs. */ +interface CachePayload { q: string; refs: unknown[] } + export interface ProviderRunDeps { /** Already retry-wrapped by the orchestrator and shared across the fan-out. */ fetch: typeof fetch @@ -59,7 +71,8 @@ export async function runProviderSearch( cache: deps.cache, signal: timeout?.signal ?? deps.signal, } - const cacheKey = deps.cache ? providerCacheKey(provider.id, query) : undefined + const fingerprint = deps.cache ? stableStringify(query) : undefined + const cacheKey = fingerprint !== undefined ? keyForFingerprint(provider.id, fingerprint) : undefined // Race a promise against the deadline without leaking an unhandled rejection // for whichever side loses the race. const raceDeadline = (p: Promise): Promise => { @@ -95,21 +108,24 @@ export async function runProviderSearch( try { // cached refs keep their original verifiedAt — staleness is bounded // by the TTL when the cache honors ttlMs. Only a whole-payload - // failure (not JSON, not an array) falls through to live; a single - // bad item within an otherwise-valid array is reported and dropped, - // same as the live path. - const raw = JSON.parse(hit) as unknown[] - if (!Array.isArray(raw)) throw new Error('cached payload is not an array') - const valid = parseItems(raw) - return { ok: true, valid, returned: raw.length, latencyMs: Date.now() - started, cached: true } + // failure (bad JSON, wrong shape, fingerprint mismatch) falls through + // to live; a single bad item within an otherwise-valid entry is + // reported and dropped, same as the live path. + const payload = JSON.parse(hit) as CachePayload + if (!payload || payload.q !== fingerprint || !Array.isArray(payload.refs)) { + throw new Error('cached payload mismatch') // hash collision or format drift → miss + } + const valid = parseItems(payload.refs) + return { ok: true, valid, returned: payload.refs.length, latencyMs: Date.now() - started, cached: true } } catch { /* fall through to live */ } } } const searching = provider.search(query, ctx) const raw = await raceDeadline(searching) const valid = parseItems(raw) - if (deps.cache && cacheKey) { - const payload = deps.cacheRaw ? valid : valid.map(({ raw: _raw, ...rest }) => rest) + if (deps.cache && cacheKey && fingerprint !== undefined) { + const refsPayload = deps.cacheRaw ? valid : valid.map(({ raw: _raw, ...rest }) => rest) + const payload: CachePayload = { q: fingerprint, refs: refsPayload } // fire-and-forget: cache write failure must never fail the search void deps.cache.set(cacheKey, JSON.stringify(payload), deps.cacheTtlMs).catch(() => {}) } diff --git a/packages/core/src/query.ts b/packages/core/src/query.ts index 3a0f33c..45017a2 100644 --- a/packages/core/src/query.ts +++ b/packages/core/src/query.ts @@ -2,12 +2,31 @@ import type { Modality } from './modality' import type { NormalizedQuery, ProviderOptionsById, + QueryFeature, ReferenceProvider, SearchControlKey, SearchControls, SearchFilters, } from './provider' +// Legacy-compat routing: a provider with NO capabilities but with deprecated +// queryFeatures keeps receiving the filter-ish controls those features implied — +// a pre-capabilities third-party provider must degrade loudly (deprecation), +// never silently (unfiltered results). Providers that declare capabilities are +// routed by capabilities alone. +const LEGACY_FEATURE_CONTROLS: Partial> = { + color: 'color', + orientation: 'orientation', + language: 'language', +} + +function effectiveControlCaps(provider: ReferenceProvider): readonly SearchControlKey[] { + if (provider.capabilities) return provider.capabilities.controls + return (provider.queryFeatures ?? []) + .map(f => LEGACY_FEATURE_CONTROLS[f]) + .filter((k): k is SearchControlKey => k !== undefined) +} + function controlsFromFilters(filters: SearchFilters | undefined): SearchControls { if (!filters) return {} return { @@ -76,13 +95,12 @@ export function requestedControlKeys(controls: SearchControls): SearchControlKey } export function supportedControlKeys(provider: ReferenceProvider, controls: SearchControls): SearchControlKey[] { - const caps = provider.capabilities?.controls ?? [] - return caps.filter(key => hasControl(controls, key)) + return effectiveControlCaps(provider).filter(key => hasControl(controls, key)) } export function unsupportedControlKeys(provider: ReferenceProvider, controls: SearchControls): SearchControlKey[] { const requested = requestedControlKeys(controls) - const supported = new Set(provider.capabilities?.controls ?? []) + const supported = new Set(effectiveControlCaps(provider)) return requested.filter(key => !supported.has(key)) } diff --git a/packages/mcp/src/cli.ts b/packages/mcp/src/cli.ts index 3f10bca..817efea 100644 --- a/packages/mcp/src/cli.ts +++ b/packages/mcp/src/cli.ts @@ -91,16 +91,28 @@ export async function defaultProviders(env: NodeJS.ProcessEnv = process.env): Pr openverse(), openverseAudio(), wikimediaCommons(), met(), artic(), gutendex(), poetrydb(), rijksmuseum(), polyhaven(), ambientcg(), internetArchive(), ] - for (const source of BYOK_SOURCES) { + // Independent module loads — run them concurrently (startup cost = max, not sum) + // and keep BYOK_SOURCES order in the provider list. + const loaded = await Promise.all(BYOK_SOURCES.map(async (source) => { const key = source.key(env) - if (!key) continue + if (!key) return [] try { - providers.push(...await source.load(key)) - } catch { - // stderr only — stdout is the MCP transport. - console.error(`[refkit-mcp] key for ${source.pkg} is set but the package is not installed — skipping this source. Reinstall @refkit/mcp with optional dependencies (or add ${source.pkg}) to enable it.`) + return await source.load(key) + } catch (err) { + // stderr only — stdout is the MCP transport. Only a genuinely missing + // module gets the "not installed" hint; anything else (factory throw, + // broken transitive import) surfaces the REAL error so the operator + // doesn't chase the wrong remediation. + const code = (err as { code?: string } | null)?.code + if (code === 'ERR_MODULE_NOT_FOUND' || code === 'MODULE_NOT_FOUND') { + console.error(`[refkit-mcp] key for ${source.pkg} is set but the package is not installed — skipping this source. Reinstall @refkit/mcp with optional dependencies (or add ${source.pkg}) to enable it.`) + } else { + console.error(`[refkit-mcp] failed to initialize ${source.pkg} — skipping this source.`, err) + } + return [] } - } + })) + for (const group of loaded) providers.push(...group) return providers } diff --git a/packages/mcp/src/index.ts b/packages/mcp/src/index.ts index 33c7ca0..122388f 100644 --- a/packages/mcp/src/index.ts +++ b/packages/mcp/src/index.ts @@ -173,12 +173,16 @@ export function createRefkitMcpServer(refkit: RefkitClient): McpServer { providerOptions: providerOptionsSchema.optional().describe('provider-specific search controls keyed by provider id; each provider whitelists supported keys'), explain: z.boolean().optional().describe('include provider status, applied and ignored controls, warnings, gate/drop metadata, and the load-more cursor'), limit: z.number().int().positive().optional(), - cursor: z.string().optional().describe('opaque cursor from a previous result\'s meta.nextCursor — fetches the next page, deduped against earlier pages (requires explain to read the next cursor)'), + cursor: z.string().optional().describe('opaque cursor from a previous result\'s nextCursor — fetches the next batch, deduped against earlier batches'), rerank: z.boolean().optional().describe('re-rank results by query relevance (term coverage incl. CJK, resolution, source diversity) instead of raw cross-source rank fusion'), intent: z.enum(INTENTS).optional().describe('annotate each result with a use-verdict for this intended use (no filtering)'), gateFor: z.enum(INTENTS).optional().describe('only return results whose license allows this intended use'), }, - outputSchema: { references: z.array(agentRefSchema), meta: searchMetaSchema.optional() }, + outputSchema: { + references: z.array(agentRefSchema), + nextCursor: z.string().optional().describe('pass back as `cursor` for the next batch; absent = exhausted'), + meta: searchMetaSchema.optional(), + }, }, async ({ query, modalities, filters, controls, providerOptions, explain, limit, cursor, rerank, intent, gateFor }) => { const searchInput = { @@ -192,7 +196,9 @@ export function createRefkitMcpServer(refkit: RefkitClient): McpServer { ...(rerank ? { rerank: lexicalReranker() } : {}), gateFor, } - const result = explain ? await refkit.searchWithMeta(searchInput) : { references: await refkit.search(searchInput), meta: undefined } + // Always searchWithMeta: the continuation token (meta.nextCursor) must not + // depend on the explain diagnostics flag — only the meta DUMP is gated. + const result = await refkit.searchWithMeta(searchInput) const refs = result.references const assessIntent = intent ?? gateFor const references = refs.map(r => @@ -202,7 +208,11 @@ export function createRefkitMcpServer(refkit: RefkitClient): McpServer { ) return { content: [{ type: 'text', text: `${references.length} reference(s) for "${query}".` }], - structuredContent: { references, ...(result.meta ? { meta: result.meta } : {}) }, + structuredContent: { + references, + ...(result.meta.nextCursor ? { nextCursor: result.meta.nextCursor } : {}), + ...(explain ? { meta: result.meta } : {}), + }, } }, ) diff --git a/packages/mcp/tsup.config.ts b/packages/mcp/tsup.config.ts index 5355d0a..b9dc3a5 100644 --- a/packages/mcp/tsup.config.ts +++ b/packages/mcp/tsup.config.ts @@ -7,4 +7,10 @@ export default defineConfig({ clean: true, outDir: 'dist', sourcemap: true, + // tsup externalizes dependencies + peerDependencies but NOT optionalDependencies, + // so without this the dynamically-imported BYOK providers get bundled INTO dist — + // defeating --omit=optional, snapshotting provider code at publish time, and making + // the "package not installed" fallback unreachable. Every @refkit/* package must + // resolve at runtime from node_modules. + external: [/^@refkit\//], }) diff --git a/packages/provider-artic/src/index.ts b/packages/provider-artic/src/index.ts index 70845bb..0f6ce5a 100644 --- a/packages/provider-artic/src/index.ts +++ b/packages/provider-artic/src/index.ts @@ -2,6 +2,7 @@ import { defineProvider, referenceId, setIfString, setIfNonNegativeInt, setIfStringList, type Reference, type RightsRecord, type NormalizedQuery, type ProviderContext, + setIfPositiveInt, } from '@refkit/core' interface ArticArtwork { @@ -79,7 +80,7 @@ export function artic() { url.searchParams.set('query[term][is_public_domain]', 'true') url.searchParams.set('fields', articFields(opts?.fields)) url.searchParams.set('limit', String(q.limit ?? 20)) - if (q.controls?.page) url.searchParams.set('page', String(q.controls.page)) + setIfPositiveInt(url, 'page', q.controls?.page) setIfString(url, 'sort', opts?.sort) setIfNonNegativeInt(url, 'from', opts?.from) setIfNonNegativeInt(url, 'size', opts?.size) diff --git a/packages/provider-europeana/src/index.ts b/packages/provider-europeana/src/index.ts index 4df0f35..2d9cd10 100644 --- a/packages/provider-europeana/src/index.ts +++ b/packages/provider-europeana/src/index.ts @@ -3,6 +3,7 @@ import { first, isLikelyImageUrl, imageMediaType, mapRightsUrl, ccVersionFor, type Reference, type RightsRecord, type NormalizedQuery, type ProviderContext, + offsetForPage, setIfNonNegativeInt, } from '@refkit/core' const BASE = 'https://api.europeana.eu/record/v2/search.json' @@ -103,7 +104,7 @@ export function europeana(config: EuropeanaConfig) { url.searchParams.set('query', q.text) url.searchParams.set('rows', String(q.limit ?? 20)) // 1-based `start` offset: item index of the first result, not a page number - if (q.controls?.page && q.controls.page > 1) url.searchParams.set('start', String((q.controls.page - 1) * (q.limit ?? 20) + 1)) + setIfNonNegativeInt(url, 'start', offsetForPage(q.controls?.page, q.limit ?? 20, 1)) url.searchParams.set('media', 'true') // only items that actually carry media url.searchParams.set('qf', 'TYPE:IMAGE') // v1 image-only scope (D1) const res = await ctx.fetch(url.toString(), { signal: ctx.signal }) diff --git a/packages/provider-flickr/src/index.ts b/packages/provider-flickr/src/index.ts index 57ae268..9d440d6 100644 --- a/packages/provider-flickr/src/index.ts +++ b/packages/provider-flickr/src/index.ts @@ -3,6 +3,7 @@ import { setIfString, setIfInt, setIfStringList, type Reference, type RightsRecord, type LicenseId, type SearchLicenseControls, type NormalizedQuery, type ProviderContext, + setIfPositiveInt, } from '@refkit/core' export interface FlickrConfig { @@ -216,7 +217,7 @@ export function flickr(config: FlickrConfig) { setBooleanFlag(url, 'in_gallery', opts?.inGallery) setBooleanFlag(url, 'is_getty', opts?.isGetty) url.searchParams.set('extras', flickrExtras(opts?.extras)) - if (q.controls?.page) url.searchParams.set('page', String(q.controls.page)) + setIfPositiveInt(url, 'page', q.controls?.page) setIfInt(url, 'page', opts?.page, { min: 1 }) url.searchParams.set('per_page', String(q.limit ?? 20)) setIfInt(url, 'per_page', opts?.perPage, { min: 1, max: 500 }) diff --git a/packages/provider-freesound/src/index.ts b/packages/provider-freesound/src/index.ts index 7be4f55..9cc662f 100644 --- a/packages/provider-freesound/src/index.ts +++ b/packages/provider-freesound/src/index.ts @@ -104,7 +104,7 @@ export function freesound(config: FreesoundConfig) { url.searchParams.set('page_size', String(opts?.pageSize ?? q.limit ?? 20)) setIfString(url, 'sort', opts?.sort) setIfString(url, 'filter', opts?.filter) - if (q.controls?.page) url.searchParams.set('page', String(q.controls.page)) + setIfPositiveInt(url, 'page', q.controls?.page) setIfPositiveInt(url, 'page', opts?.page) const res = await ctx.fetch(url.toString(), { signal: ctx.signal }) if (!res.ok) throw new Error(`freesound search failed: ${res.status}`) diff --git a/packages/provider-gutendex/src/index.ts b/packages/provider-gutendex/src/index.ts index 97da611..57a809d 100644 --- a/packages/provider-gutendex/src/index.ts +++ b/packages/provider-gutendex/src/index.ts @@ -80,7 +80,7 @@ export function gutendex(config: GutendexConfig = {}) { if (q.controls?.language) url.searchParams.set('languages', q.controls.language) if (q.controls?.text?.copyright === 'public-domain') url.searchParams.set('copyright', 'false') if (q.controls?.text?.copyright === 'copyrighted') url.searchParams.set('copyright', 'true') - if (q.controls?.page) url.searchParams.set('page', String(q.controls.page)) + setIfPositiveInt(url, 'page', q.controls?.page) const opts = q.providerOptions as GutendexSearchOptions | undefined setIfInt(url, 'author_year_start', opts?.authorYearStart) setIfInt(url, 'author_year_end', opts?.authorYearEnd) diff --git a/packages/provider-internet-archive/src/__tests__/internet-archive.test.ts b/packages/provider-internet-archive/src/__tests__/internet-archive.test.ts index 0cb8c02..d607471 100644 --- a/packages/provider-internet-archive/src/__tests__/internet-archive.test.ts +++ b/packages/provider-internet-archive/src/__tests__/internet-archive.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest' -import { mapIaLicense, mediatypeToModality } from '../index' +import { escapeLucene, mapIaLicense, mediatypeToModality } from '../index' describe('mapIaLicense', () => { it('maps CC0 / PD mark / PD dedication URLs', () => { @@ -184,4 +184,17 @@ describe('internetArchive search', () => { expect.arrayContaining(['identifier', 'title', 'creator', 'licenseurl', 'mediatype']), ) }) + + it('escapes Lucene syntax in the user query so it cannot break or escape the mediatype scope', async () => { + expect(escapeLucene('smile :)')).toBe('smile \\:\\)') + expect(escapeLucene('x) OR (mediatype:audio')).toBe('x\\) OR \\(mediatype\\:audio') + expect(escapeLucene('a && b || c')).toBe('a \\&\\& b \\|\\| c') + let seen = '' + await internetArchive().search( + { text: 'x) OR (mediatype:audio', modalities: ['video'] }, + ctxResponding({ response: { numFound: 0, docs: [] } }, u => { seen = u }), + ) + const q = new URL(seen).searchParams.get('q')! + expect(q).toBe('(x\\) OR \\(mediatype\\:audio) AND mediatype:(movies OR texts)') + }) }) diff --git a/packages/provider-internet-archive/src/index.ts b/packages/provider-internet-archive/src/index.ts index 6818e4c..6b53fa0 100644 --- a/packages/provider-internet-archive/src/index.ts +++ b/packages/provider-internet-archive/src/index.ts @@ -2,6 +2,7 @@ import { defineProvider, referenceId, mapRightsUrl, ccVersionFor, type Reference, type RightsRecord, type Modality, type NormalizedQuery, type ProviderContext, + setIfPositiveInt, } from '@refkit/core' const BASE = 'https://archive.org/advancedsearch.php' @@ -21,6 +22,13 @@ export interface InternetArchiveConfig { * declaration mapped faithfully (NoC-US → PD is the source's word, not a guess). */ export const mapIaLicense = mapRightsUrl +/** Escape Lucene reserved syntax in the user query so it stays a literal term + * inside the composed expression — unescaped `)` would break the grouping and + * `) OR (mediatype:...` could escape the movies/texts scope entirely. */ +export function escapeLucene(text: string): string { + return text.replace(/&&|\|\||[+\-!(){}[\]^"~*?:\\/]/g, (m) => [...m].map(c => `\\${c}`).join('')) +} + const MEDIATYPE_MODALITY: Record = { movies: 'video', texts: 'text' } /** v1 scope (D1): only `movies`→video and `texts`→text. Everything else → null @@ -87,12 +95,13 @@ export function internetArchive(config: InternetArchiveConfig = {}) { capabilities: { controls: ['page'] }, async search(q: NormalizedQuery, ctx: ProviderContext): Promise { const url = new URL(BASE) - url.searchParams.set('q', `(${q.text}) AND mediatype:(movies OR texts)`) + url.searchParams.set('q', `(${escapeLucene(q.text)}) AND mediatype:(movies OR texts)`) for (const f of ['identifier', 'title', 'creator', 'licenseurl', 'mediatype']) { url.searchParams.append('fl[]', f) } url.searchParams.set('output', 'json') - url.searchParams.set('page', String(q.controls?.page ?? 1)) + url.searchParams.set('page', '1') + setIfPositiveInt(url, 'page', q.controls?.page) const rows = Math.min(config.maxRows ?? q.limit ?? 20, 100) url.searchParams.set('rows', String(rows)) const res = await ctx.fetch(url.toString(), { signal: ctx.signal }) diff --git a/packages/provider-jamendo/src/index.ts b/packages/provider-jamendo/src/index.ts index 0df5566..1cf1c9e 100644 --- a/packages/provider-jamendo/src/index.ts +++ b/packages/provider-jamendo/src/index.ts @@ -3,6 +3,7 @@ import { setIfString, setIfStringList, setIfBoolean, setIfNonNegativeInt, mapCcDeedUrl, ccVersionFor, type Reference, type RightsRecord, type NormalizedQuery, type ProviderContext, + offsetForPage, } from '@refkit/core' export interface JamendoConfig { @@ -97,7 +98,8 @@ export function jamendo(config: JamendoConfig) { url.searchParams.set('client_id', config.clientId) url.searchParams.set('format', 'json') url.searchParams.set('search', q.text) - url.searchParams.set('limit', String(Math.min(q.limit ?? 20, 200))) + const pageSize = Math.min(q.limit ?? 20, 200) + url.searchParams.set('limit', String(pageSize)) const opts = q.providerOptions as JamendoSearchOptions | undefined setIfString(url, 'audioformat', opts?.audioformat, ['mp31', 'mp32', 'ogg', 'flac']) setIfString(url, 'order', opts?.order, ['relevance', 'popularity_total', 'popularity_month', 'popularity_week', 'releasedate_asc', 'releasedate_desc', 'buzzrate']) @@ -108,7 +110,7 @@ export function jamendo(config: JamendoConfig) { setIfStringList(url, 'tags', opts?.tags, { separator: ' ' }) setIfString(url, 'artist_name', opts?.artist_name) // offset-based API: translate the 1-based page control (providerOptions.offset overrides below) - if (q.controls?.page && q.controls.page > 1) url.searchParams.set('offset', String((q.controls.page - 1) * Math.min(q.limit ?? 20, 200))) + setIfNonNegativeInt(url, 'offset', offsetForPage(q.controls?.page, pageSize)) // jamendo's offset is non-negative (0 is valid) → setIfNonNegativeInt, not PositiveInt. setIfNonNegativeInt(url, 'offset', opts?.offset) const res = await ctx.fetch(url.toString(), { signal: ctx.signal }) diff --git a/packages/provider-met/src/index.ts b/packages/provider-met/src/index.ts index 2d7daf7..7c086a7 100644 --- a/packages/provider-met/src/index.ts +++ b/packages/provider-met/src/index.ts @@ -2,6 +2,7 @@ import { defineProvider, referenceId, setIfBoolean, setIfInt, setIfString, type Reference, type RightsRecord, type NormalizedQuery, type ProviderContext, + offsetForPage, } from '@refkit/core' export interface MetConfig { @@ -91,7 +92,7 @@ export function met(config: MetConfig = {}) { if (!objectIDs || objectIDs.length === 0) return [] const n = Math.min(config.maxObjects ?? q.limit ?? 12, 30) // the search endpoint returns every matching id — page = a window over that list - const from = ((q.controls?.page ?? 1) - 1) * n + const from = offsetForPage(q.controls?.page, n) ?? 0 const objects = await Promise.all(objectIDs.slice(from, from + n).map(async (id) => { try { const r = await ctx.fetch(`${BASE}/objects/${id}`, { signal: ctx.signal }) diff --git a/packages/provider-openverse/src/index.ts b/packages/provider-openverse/src/index.ts index e05de15..e8d0f15 100644 --- a/packages/provider-openverse/src/index.ts +++ b/packages/provider-openverse/src/index.ts @@ -158,7 +158,7 @@ export function openverse(config: OpenverseConfig = {}) { url.searchParams.set('q', q.text) url.searchParams.set('license_type', openverseLicenseType(q.controls?.license)) // performance/relevance hint only — the AUTHORITATIVE rights gate is mapOpenverseLicense below, not this filter url.searchParams.set('page_size', String(q.limit ?? 20)) - if (q.controls?.page) url.searchParams.set('page', String(q.controls.page)) + setIfPositiveInt(url, 'page', q.controls?.page) const opts = q.providerOptions as OpenverseImageSearchOptions | undefined applyOpenverseSearchOptions(url, opts) setIfStringList(url, 'aspect_ratio', opts?.aspectRatio) @@ -229,7 +229,7 @@ export function openverseAudio(config: OpenverseConfig = {}) { url.searchParams.set('q', q.text) url.searchParams.set('license_type', openverseLicenseType(q.controls?.license)) // relevance hint; mapOpenverseLicense authoritative url.searchParams.set('page_size', String(q.limit ?? 20)) - if (q.controls?.page) url.searchParams.set('page', String(q.controls.page)) + setIfPositiveInt(url, 'page', q.controls?.page) const opts = q.providerOptions as OpenverseAudioSearchOptions | undefined applyOpenverseSearchOptions(url, opts) setIfStringList(url, 'length', opts?.length) diff --git a/packages/provider-pexels/src/index.ts b/packages/provider-pexels/src/index.ts index 63a1445..815b813 100644 --- a/packages/provider-pexels/src/index.ts +++ b/packages/provider-pexels/src/index.ts @@ -28,22 +28,12 @@ interface PexelsPhoto { } interface PexelsResponse { photos: PexelsPhoto[] } -function useLegacyFilter(control: T | undefined, legacy: T | undefined): T | undefined { - return control === undefined ? legacy : undefined -} - function applyPexelsSearchParams(url: URL, q: NormalizedQuery, options?: { allowColor?: boolean }) { if (q.controls?.orientation) url.searchParams.set('orientation', q.controls.orientation) if (options?.allowColor && q.controls?.color) url.searchParams.set('color', q.controls.color) if (q.controls?.language) url.searchParams.set('locale', q.controls.language) if (q.controls?.media?.size) url.searchParams.set('size', q.controls.media.size) - if (q.controls?.page) url.searchParams.set('page', String(q.controls.page)) - const legacyOrientation = useLegacyFilter(q.controls?.orientation, q.filters?.orientation) - if (legacyOrientation) url.searchParams.set('orientation', legacyOrientation) - const legacyColor = useLegacyFilter(q.controls?.color, q.filters?.color) - if (options?.allowColor && legacyColor) url.searchParams.set('color', legacyColor) - const legacyLanguage = useLegacyFilter(q.controls?.language, q.filters?.language) - if (legacyLanguage) url.searchParams.set('locale', legacyLanguage) + setIfPositiveInt(url, 'page', q.controls?.page) const opts = q.providerOptions as PexelsSearchOptions | undefined setIfString(url, 'orientation', opts?.orientation, ['landscape', 'portrait', 'square']) if (options?.allowColor) setIfString(url, 'color', opts?.color) diff --git a/packages/provider-pixabay/src/index.ts b/packages/provider-pixabay/src/index.ts index 39a54fb..73a4ee3 100644 --- a/packages/provider-pixabay/src/index.ts +++ b/packages/provider-pixabay/src/index.ts @@ -66,10 +66,6 @@ function setColorsList(url: URL, key: string, value: unknown, allowed?: readonly } } -function useLegacyFilter(control: T | undefined, legacy: T | undefined): T | undefined { - return control === undefined ? legacy : undefined -} - function pixabayOrientation(orientation: string | undefined): string | undefined { if (orientation === 'landscape') return 'horizontal' if (orientation === 'portrait') return 'vertical' @@ -109,7 +105,7 @@ export function pixabay(config: PixabayConfig) { url.searchParams.set('q', q.text) url.searchParams.set('image_type', 'photo') url.searchParams.set('per_page', String(Math.min(Math.max(q.limit ?? 20, 3), 200))) - if (q.controls?.page) url.searchParams.set('page', String(q.controls.page)) + setIfPositiveInt(url, 'page', q.controls?.page) if (q.controls?.language) url.searchParams.set('lang', q.controls.language) if (q.controls?.color) url.searchParams.set('colors', q.controls.color) const controlsOrientation = pixabayOrientation(q.controls?.orientation) @@ -122,12 +118,6 @@ export function pixabay(config: PixabayConfig) { } if (q.controls?.media?.minWidth !== undefined) url.searchParams.set('min_width', String(q.controls.media.minWidth)) if (q.controls?.media?.minHeight !== undefined) url.searchParams.set('min_height', String(q.controls.media.minHeight)) - const legacyLanguage = useLegacyFilter(q.controls?.language, q.filters?.language) - if (legacyLanguage) url.searchParams.set('lang', legacyLanguage) - const legacyColor = useLegacyFilter(q.controls?.color, q.filters?.color) - if (legacyColor) url.searchParams.set('colors', legacyColor) - const orientation = pixabayOrientation(useLegacyFilter(q.controls?.orientation, q.filters?.orientation)) - if (orientation) url.searchParams.set('orientation', orientation) const opts = q.providerOptions as PixabayImageSearchOptions | undefined setIfString(url, 'lang', opts?.lang) setIfString(url, 'id', opts?.id) @@ -197,7 +187,7 @@ export function pixabayVideo(config: PixabayConfig) { url.searchParams.set('key', config.key) url.searchParams.set('q', q.text) url.searchParams.set('per_page', String(Math.min(Math.max(q.limit ?? 20, 3), 200))) - if (q.controls?.page) url.searchParams.set('page', String(q.controls.page)) + setIfPositiveInt(url, 'page', q.controls?.page) if (q.controls?.language) url.searchParams.set('lang', q.controls.language) if (q.controls?.sort === 'latest' || q.controls?.sort === 'popular') url.searchParams.set('order', q.controls.sort) if (q.controls?.safety === 'strict') url.searchParams.set('safesearch', 'true') @@ -207,8 +197,6 @@ export function pixabayVideo(config: PixabayConfig) { } if (q.controls?.media?.minWidth !== undefined) url.searchParams.set('min_width', String(q.controls.media.minWidth)) if (q.controls?.media?.minHeight !== undefined) url.searchParams.set('min_height', String(q.controls.media.minHeight)) - const legacyLanguage = useLegacyFilter(q.controls?.language, q.filters?.language) - if (legacyLanguage) url.searchParams.set('lang', legacyLanguage) const opts = q.providerOptions as PixabayVideoSearchOptions | undefined setIfString(url, 'lang', opts?.lang) setIfString(url, 'id', opts?.id) diff --git a/packages/provider-polyhaven/src/index.ts b/packages/provider-polyhaven/src/index.ts index 8f006e6..b965cfc 100644 --- a/packages/provider-polyhaven/src/index.ts +++ b/packages/provider-polyhaven/src/index.ts @@ -1,6 +1,7 @@ import { defineProvider, referenceId, imageMediaType, type Reference, type RightsRecord, type NormalizedQuery, type ProviderContext, + offsetForPage, setIfNonNegativeInt, } from '@refkit/core' const PH_BASE = 'https://api.polyhaven.com' @@ -102,7 +103,7 @@ export function polyhaven(config: PolyHavenConfig = {}) { } const n = Math.min(config.maxAssets ?? q.limit ?? 12, 30) // the list endpoint returns everything — page = a window over the filtered list - const from = ((q.controls?.page ?? 1) - 1) * n + const from = offsetForPage(q.controls?.page, n) ?? 0 const picked = entries.slice(from, from + n) const refs = await Promise.all(picked.map(async ([id, asset]) => { try { @@ -177,9 +178,10 @@ export function ambientcg(config: AmbientCgConfig = {}) { const url = new URL(ACG_BASE) url.searchParams.set('type', 'Material') // image-based PBR materials only (D1) url.searchParams.set('include', 'displayData,imageData') - url.searchParams.set('limit', String(Math.min(config.limit ?? q.limit ?? 12, 30))) + const pageSize = Math.min(config.limit ?? q.limit ?? 12, 30) + url.searchParams.set('limit', String(pageSize)) // offset-based API: translate the 1-based page control - if (q.controls?.page && q.controls.page > 1) url.searchParams.set('offset', String((q.controls.page - 1) * Math.min(config.limit ?? q.limit ?? 12, 30))) + setIfNonNegativeInt(url, 'offset', offsetForPage(q.controls?.page, pageSize)) if (q.text?.trim()) url.searchParams.set('q', q.text.trim()) const res = await ctx.fetch(url.toString(), { signal: ctx.signal }) if (!res.ok) throw new Error(`ambientcg search failed: ${res.status}`) diff --git a/packages/provider-smithsonian/src/index.ts b/packages/provider-smithsonian/src/index.ts index b8c83e3..50a577a 100644 --- a/packages/provider-smithsonian/src/index.ts +++ b/packages/provider-smithsonian/src/index.ts @@ -2,6 +2,7 @@ import { defineProvider, referenceId, setIfString, setIfNonNegativeInt, type Reference, type RightsRecord, type NormalizedQuery, type ProviderContext, + offsetForPage, } from '@refkit/core' export interface SmithsonianConfig { @@ -74,7 +75,7 @@ export function smithsonian(config: SmithsonianConfig) { url.searchParams.set('q', q.text) url.searchParams.set('rows', String(q.limit ?? 20)) // offset-based API: translate the 1-based page control (providerOptions.start overrides below) - if (q.controls?.page && q.controls.page > 1) url.searchParams.set('start', String((q.controls.page - 1) * (q.limit ?? 20))) + setIfNonNegativeInt(url, 'start', offsetForPage(q.controls?.page, q.limit ?? 20)) // bias toward CC0 image records; toReference stays authoritative per media url.searchParams.set('fq', 'online_media_type:"Images" AND media_usage:"CC0"') const opts = q.providerOptions as SmithsonianSearchOptions | undefined diff --git a/packages/provider-unsplash/src/__tests__/unsplash.test.ts b/packages/provider-unsplash/src/__tests__/unsplash.test.ts index d262968..c5f6238 100644 --- a/packages/provider-unsplash/src/__tests__/unsplash.test.ts +++ b/packages/provider-unsplash/src/__tests__/unsplash.test.ts @@ -48,7 +48,9 @@ describe('unsplash provider', () => { await unsplash({ accessKey: 'k' }).search({ text: 'coffee', modalities: ['image'], - filters: { color: 'blue', orientation: 'square', language: 'zh-Hans' }, + // NormalizedQuery.filters is deprecated (a mirror derived from controls); + // providers read controls — this exercises the current contract. + controls: { color: 'blue', orientation: 'square', language: 'zh-Hans' }, providerOptions: { orderBy: 'latest', contentFilter: 'high', collections: ['abc', 'def'], page: 3, perPage: 12 }, }, ctx) const url = new URL(calledUrl) diff --git a/packages/provider-unsplash/src/index.ts b/packages/provider-unsplash/src/index.ts index 2c8c28c..74d5cf4 100644 --- a/packages/provider-unsplash/src/index.ts +++ b/packages/provider-unsplash/src/index.ts @@ -33,10 +33,6 @@ function setCollections(url: URL, value: unknown) { if (Array.isArray(value) && value.every(v => typeof v === 'string')) url.searchParams.set('collections', value.join(',')) } -function useLegacyFilter(control: T | undefined, legacy: T | undefined): T | undefined { - return control === undefined ? legacy : undefined -} - function toReference(r: UnsplashResult): Reference { const rights: RightsRecord = { license: 'unsplash', @@ -69,7 +65,7 @@ export function unsplash(config: UnsplashConfig) { url.searchParams.set('query', q.text) url.searchParams.set('per_page', String(Math.min(q.limit ?? 10, 30))) // Unsplash hard-caps per_page at 30; default kept low for free-tier rate limits const controls = q.controls - if (controls?.page) url.searchParams.set('page', String(controls.page)) + setIfPositiveInt(url, 'page', controls?.page) if (controls?.color) url.searchParams.set('color', controls.color) if (controls?.orientation) url.searchParams.set('orientation', controls.orientation === 'square' ? 'squarish' : controls.orientation) if (controls?.language) url.searchParams.set('lang', controls.language) @@ -78,12 +74,6 @@ export function unsplash(config: UnsplashConfig) { } if (controls?.safety === 'strict') url.searchParams.set('content_filter', 'high') if (controls?.safety === 'moderate') url.searchParams.set('content_filter', 'low') - const legacyColor = useLegacyFilter(controls?.color, q.filters?.color) - if (legacyColor) url.searchParams.set('color', legacyColor) - const legacyOrientation = useLegacyFilter(controls?.orientation, q.filters?.orientation) - if (legacyOrientation) url.searchParams.set('orientation', legacyOrientation === 'square' ? 'squarish' : legacyOrientation) - const legacyLanguage = useLegacyFilter(controls?.language, q.filters?.language) - if (legacyLanguage) url.searchParams.set('lang', legacyLanguage) const opts = q.providerOptions as UnsplashSearchOptions | undefined setIfString(url, 'order_by', opts?.orderBy, ['latest', 'relevant']) setIfString(url, 'content_filter', opts?.contentFilter, ['low', 'high']) diff --git a/packages/provider-wikimedia-commons/src/index.ts b/packages/provider-wikimedia-commons/src/index.ts index b55e9af..b3068d6 100644 --- a/packages/provider-wikimedia-commons/src/index.ts +++ b/packages/provider-wikimedia-commons/src/index.ts @@ -4,6 +4,7 @@ import { CC_FAMILY_BY_TOKEN, ccVersionFor, type Reference, type RightsRecord, type LicenseId, type NormalizedQuery, type ProviderContext, + offsetForPage, } from '@refkit/core' export interface WikimediaCommonsConfig { @@ -152,7 +153,7 @@ export function wikimediaCommons(config: WikimediaCommonsConfig = {}) { url.searchParams.set('gsrnamespace', '6') // File: url.searchParams.set('gsrlimit', String(q.limit ?? 20)) // offset-based API: translate the 1-based page control (providerOptions.gsroffset overrides below) - if (q.controls?.page && q.controls.page > 1) url.searchParams.set('gsroffset', String((q.controls.page - 1) * (q.limit ?? 20))) + setIfNonNegativeInt(url, 'gsroffset', offsetForPage(q.controls?.page, q.limit ?? 20)) url.searchParams.set('prop', 'imageinfo') url.searchParams.set('iiprop', 'url|mime|size|extmetadata') url.searchParams.set('iiurlwidth', String(config.thumbWidth ?? 1024)) From a1795ac2df96bf05d867415aee4df1fc2651fe70 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 14 Jul 2026 14:50:32 +0000 Subject: [PATCH 4/5] test(mcp): lock in cursor-without-explain contract Verified the three release-blocking findings A/B against the pre-fix commit (a88f105) with runtime repros: overfetch gap (20/60 vs 60/60 results returned), false exhaustion (page-3 unreachable vs reached), and optional-provider bundling (9 bundled chunks/220K vs 0/80K with imports resolving from node_modules); built cli answers the MCP initialize handshake with a BYOK key set. This adds the missing unit test for top-level nextCursor without explain + cursor round-trip. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_018hoN6wKWxcs8WbUNCM9rtt --- packages/mcp/src/__tests__/mcp.test.ts | 39 ++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/packages/mcp/src/__tests__/mcp.test.ts b/packages/mcp/src/__tests__/mcp.test.ts index 20980d3..d79e166 100644 --- a/packages/mcp/src/__tests__/mcp.test.ts +++ b/packages/mcp/src/__tests__/mcp.test.ts @@ -84,6 +84,45 @@ describe('@refkit/mcp', () => { await client.close() }) + it('returns nextCursor at the top level WITHOUT explain, and the cursor round-trips', async () => { + const item = (i: number) => ({ + id: `pg:${i}`, + modality: 'image' as const, + source: { providerId: 'pg', sourceUrl: `https://pg/${i}` }, + canonicalUrl: `https://pg/${i}`, + rights: { license: 'CC0-1.0' as const, rehostPolicy: 'cache-allowed' as const, raw: { sourceTerms: 't', sourceUrl: `https://pg/${i}` } }, + verifiedAt: '2026-06-22T00:00:00.000Z', + relevance: 0, + }) + const pool = Array.from({ length: 6 }, (_, i) => item(i + 1)) + const paging = defineProvider({ + id: 'pg', + modalities: ['image'], + capabilities: { controls: ['page'] }, + search: async (q) => { + const size = q.limit ?? 20 + const page = q.controls?.page ?? 1 + return pool.slice((page - 1) * size, page * size) + }, + }) + const server = createRefkitMcpServer(createRefkit({ providers: [paging] })) + const [clientT, serverT] = InMemoryTransport.createLinkedPair() + const client = new Client({ name: 'test', version: '1.0.0' }) + await Promise.all([client.connect(clientT), server.connect(serverT)]) + + type Out = { references: Array<{ canonicalUrl: string }>; nextCursor?: string; meta?: unknown } + const batch1 = (await client.callTool({ name: 'search_references', arguments: { query: 'x', modalities: ['image'], limit: 2 } })).structuredContent as Out + expect(batch1.references).toHaveLength(2) + expect(batch1.nextCursor).toBeDefined() // present WITHOUT explain + expect(batch1.meta).toBeUndefined() // meta dump still gated by explain + + const batch2 = (await client.callTool({ name: 'search_references', arguments: { query: 'x', modalities: ['image'], limit: 2, cursor: batch1.nextCursor } })).structuredContent as Out + expect(batch2.references).toHaveLength(2) + const urls1 = batch1.references.map(r => r.canonicalUrl) + expect(batch2.references.every(r => !urls1.includes(r.canonicalUrl))).toBe(true) // no repeats + await client.close() + }) + it('accepts filters and providerOptions for provider-specific search controls', async () => { let seen: { filters?: unknown; providerOptions?: unknown } = {} const fakeProvider = defineProvider({ From 3ebf35c0c290e3cca98547848ba019c48d92d043 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 14 Jul 2026 15:10:14 +0000 Subject: [PATCH 5/5] test: add artifact-level smoke to CI + close remaining test gaps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - scripts/smoke-artifact.mjs: pack every package (publishConfig applied, like a real publish), install the tarballs into a clean project, and boot the installed @refkit/mcp cli through three scenarios — keyless handshake, BYOK dynamic import from node_modules, and graceful skip when the optional package is removed. Guards the layer source suites cannot see (a bundler regression re-inlining optionalDependencies fails scenario 3). Wired into CI after build as `pnpm smoke:artifact`. - mcp: BYOK_SOURCES exported + test pinning it 1:1 to package.json optionalDependencies (and out of hard dependencies) - core: offsetForPage unit tests (offsets, 1-based origin, rejection of page 1 / non-integer / negative input) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_018hoN6wKWxcs8WbUNCM9rtt --- .github/workflows/ci.yml | 3 + package.json | 1 + .../src/__tests__/provider-helpers.test.ts | 14 +++ packages/mcp/src/__tests__/mcp.test.ts | 17 +++- packages/mcp/src/cli.ts | 5 +- scripts/smoke-artifact.mjs | 97 +++++++++++++++++++ 6 files changed, 135 insertions(+), 2 deletions(-) create mode 100644 scripts/smoke-artifact.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 548777c..8011e97 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,3 +20,6 @@ jobs: - run: pnpm lint - run: pnpm build - run: pnpm test:run + # source suites resolve workspace src/*.ts — only this exercises the bits we publish + - name: Artifact smoke (pack → clean install → boot MCP cli) + run: pnpm smoke:artifact diff --git a/package.json b/package.json index 96ad51f..e6e2d09 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "test:run": "vitest run", "test:watch": "vitest watch", "test:live": "REFKIT_LIVE=1 vitest run", + "smoke:artifact": "node scripts/smoke-artifact.mjs", "changeset": "changeset", "version-packages": "changeset version", "release": "pnpm -r --if-present build && changeset publish" diff --git a/packages/core/src/__tests__/provider-helpers.test.ts b/packages/core/src/__tests__/provider-helpers.test.ts index 5fa094d..e6d4239 100644 --- a/packages/core/src/__tests__/provider-helpers.test.ts +++ b/packages/core/src/__tests__/provider-helpers.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest' import { + offsetForPage, setIfString, setIfBoolean, setIfStringList, setIfInt, setIfPositiveInt, setIfNonNegativeInt, setIfNumber, first, mapCcDeedUrl, mapRightsUrl, ccVersionFor, isLikelyImageUrl, imageMediaType, @@ -63,6 +64,19 @@ describe('int/number setters', () => { }) }) +describe('offsetForPage', () => { + it('translates a 1-based page into a row offset; page 1 / missing / invalid → undefined', () => { + expect(offsetForPage(2, 20)).toBe(20) + expect(offsetForPage(3, 12)).toBe(24) + expect(offsetForPage(2, 20, 1)).toBe(21) // 1-based origins (e.g. Europeana `start`) + expect(offsetForPage(1, 20)).toBeUndefined() // first page → omit the param + expect(offsetForPage(undefined, 20)).toBeUndefined() + expect(offsetForPage(2.5, 20)).toBeUndefined() // non-integer rejected, not truncated + expect(offsetForPage(0, 20)).toBeUndefined() + expect(offsetForPage(-3, 20)).toBeUndefined() + }) +}) + describe('first', () => { it('returns the first element or undefined', () => { expect(first(['a', 'b'])).toBe('a') diff --git a/packages/mcp/src/__tests__/mcp.test.ts b/packages/mcp/src/__tests__/mcp.test.ts index d79e166..7a82f59 100644 --- a/packages/mcp/src/__tests__/mcp.test.ts +++ b/packages/mcp/src/__tests__/mcp.test.ts @@ -3,8 +3,9 @@ import { Client } from '@modelcontextprotocol/sdk/client/index.js' import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js' import { createRefkit, defineProvider } from '@refkit/core' import { openverse } from '@refkit/provider-openverse' +import { readFileSync } from 'node:fs' import { createRefkitMcpServer } from '../index' -import { defaultProviders } from '../cli' +import { defaultProviders, BYOK_SOURCES } from '../cli' const OPENVERSE = { results: [ { id: 'aaa', title: 'cc0 sky', creator: 'Alice', foreign_landing_url: 'https://ov/aaa', url: 'https://cdn/aaa.jpg', thumbnail: 'https://ov/aaa/thumb', width: 10, height: 10, license: 'cc0', license_version: '1.0', license_url: 'https://cc/cc0' }, @@ -435,4 +436,18 @@ describe('defaultProviders (zero-config CLI wiring)', () => { expect((await defaultProviders({})).map(p => p.id)).not.toContain('smithsonian') expect((await defaultProviders({ SI_KEY: 'k' })).map(p => p.id)).toContain('smithsonian') }) + + it('BYOK_SOURCES and package.json optionalDependencies stay in sync', () => { + const pkg = JSON.parse(readFileSync(new URL('../../package.json', import.meta.url), 'utf8')) as { + optionalDependencies?: Record + dependencies?: Record + } + const optional = Object.keys(pkg.optionalDependencies ?? {}).sort() + const table = BYOK_SOURCES.map(s => s.pkg).sort() + // an entry in one but not the other ships a source that either can never + // load (missing dep) or never installs for a purpose (dep without loader) + expect(table).toEqual(optional) + // and no BYOK package may ALSO be a hard dependency + for (const p of table) expect(pkg.dependencies ?? {}).not.toHaveProperty(p) + }) }) diff --git a/packages/mcp/src/cli.ts b/packages/mcp/src/cli.ts index 817efea..49c2619 100644 --- a/packages/mcp/src/cli.ts +++ b/packages/mcp/src/cli.ts @@ -28,7 +28,10 @@ interface ByokSource { // Env var convention: each BYOK key is read as `REFKIT__KEY` first (the // unified name), falling back to the provider's legacy env var name — both are // honored indefinitely, the unified name is just preferred going forward. -const BYOK_SOURCES: ByokSource[] = [ +// Exported so tests can assert this table stays in sync with the package's +// optionalDependencies — an entry in one but not the other ships a source that +// either can never load or never installs. +export const BYOK_SOURCES: ByokSource[] = [ { pkg: '@refkit/provider-unsplash', key: (env) => env.REFKIT_UNSPLASH_KEY ?? env.UNSPLASH_KEY, diff --git a/scripts/smoke-artifact.mjs b/scripts/smoke-artifact.mjs new file mode 100644 index 0000000..7f30b9c --- /dev/null +++ b/scripts/smoke-artifact.mjs @@ -0,0 +1,97 @@ +#!/usr/bin/env node +// Artifact-level smoke test: verifies the bits we PUBLISH, not the source we test. +// +// Source-level suites (vitest) resolve workspace exports to src/*.ts, so nothing +// there can catch a broken dist — e.g. a bundler config change that inlines +// optionalDependencies (the exact regression this guards against). This script +// mirrors the real release path: `pnpm pack` every package (applies publishConfig +// exports→dist and rewrites workspace:* — same as `pnpm publish`), installs the +// tarballs into a clean temp project with npm, then boots the installed +// @refkit/mcp CLI through three scenarios: +// 1. keyless boot → MCP initialize handshake answers +// 2. BYOK key set → optional provider dynamically imports from node_modules +// 3. key set, package REMOVED → boots anyway with the graceful "not installed" warning +// +// Requires registry access for third-party deps (zod, MCP SDK). Run after `pnpm build`. +import { execFileSync, spawn } from 'node:child_process' +import { mkdtempSync, readdirSync, writeFileSync, rmSync, existsSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +const root = join(import.meta.dirname, '..') +const work = mkdtempSync(join(tmpdir(), 'refkit-smoke-')) +const tarballs = join(work, 'tarballs') +const app = join(work, 'app') + +const run = (cmd, args, opts = {}) => execFileSync(cmd, args, { stdio: ['ignore', 'pipe', 'inherit'], encoding: 'utf8', ...opts }) + +console.log('[smoke] packing workspace packages (publishConfig applied, like a real publish)…') +run('pnpm', ['-r', '--filter', './packages/*', 'exec', 'pnpm', 'pack', '--pack-destination', tarballs], { cwd: root }) +const tgz = readdirSync(tarballs).filter(f => f.endsWith('.tgz')) +if (tgz.length < 20) throw new Error(`[smoke] expected ~21 tarballs, got ${tgz.length}`) + +console.log(`[smoke] installing ${tgz.length} tarballs into a clean project…`) +run('mkdir', ['-p', app]) +writeFileSync(join(app, 'package.json'), JSON.stringify({ + name: 'refkit-smoke-app', + private: true, + // every tarball is a direct file: dep, so @refkit/* inter-deps resolve to the + // packed local versions instead of the (older) registry releases + dependencies: Object.fromEntries(tgz.map(f => { + const name = '@refkit/' + f.replace(/^refkit-/, '').replace(/-\d+\.\d+\.\d+\.tgz$/, '') + return [name, `file:${join(tarballs, f)}`] + })), +}, null, 2)) +run('npm', ['install', '--no-audit', '--no-fund', '--loglevel=error'], { cwd: app }) + +const cli = join(app, 'node_modules', '@refkit', 'mcp', 'dist', 'cli.js') +if (!existsSync(cli)) throw new Error('[smoke] installed @refkit/mcp has no dist/cli.js') + +const INIT = JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'initialize', params: { protocolVersion: '2024-11-05', capabilities: {}, clientInfo: { name: 'smoke', version: '0' } } }) + '\n' + +/** Boot the installed CLI, send initialize, capture first stdout line + stderr. */ +function boot(env) { + return new Promise((resolve, reject) => { + const child = spawn('node', [cli], { cwd: app, env: { ...process.env, ...env }, stdio: ['pipe', 'pipe', 'pipe'] }) + let out = '' + let err = '' + const timer = setTimeout(() => { child.kill('SIGKILL'); reject(new Error(`[smoke] boot timed out; stderr:\n${err}`)) }, 30_000) + child.stdout.on('data', d => { + out += d + if (out.includes('\n')) { clearTimeout(timer); child.kill(); resolve({ out, err }) } + }) + child.stderr.on('data', d => { err += d }) + child.on('error', reject) + child.on('exit', (code) => { + if (!out.includes('\n')) { clearTimeout(timer); reject(new Error(`[smoke] exited (${code}) before responding; stderr:\n${err}`)) } + }) + child.stdin.write(INIT) + }) +} + +const assert = (cond, msg) => { if (!cond) throw new Error(`[smoke] FAIL: ${msg}`) } + +console.log('[smoke] scenario 1: keyless boot answers the MCP initialize handshake…') +{ + const { out } = await boot({}) + const res = JSON.parse(out.slice(0, out.indexOf('\n'))) + assert(res.result?.serverInfo?.name === 'refkit', `unexpected initialize response: ${out.slice(0, 200)}`) +} + +console.log('[smoke] scenario 2: BYOK key set — optional provider imports from node_modules…') +{ + const { out, err } = await boot({ REFKIT_UNSPLASH_KEY: 'dummy' }) + assert(out.includes('serverInfo'), 'no handshake with BYOK key set') + assert(!err.includes('not installed'), `unexpected missing-package warning:\n${err}`) +} + +console.log('[smoke] scenario 3: key set but package removed — graceful skip, not a crash…') +{ + rmSync(join(app, 'node_modules', '@refkit', 'provider-unsplash'), { recursive: true, force: true }) + const { out, err } = await boot({ REFKIT_UNSPLASH_KEY: 'dummy' }) + assert(out.includes('serverInfo'), 'server failed to boot with a missing optional provider') + assert(err.includes('not installed'), `expected the graceful "not installed" warning, got:\n${err || '(empty)'}`) +} + +rmSync(work, { recursive: true, force: true }) +console.log('[smoke] PASS: published artifact boots, optional providers resolve at runtime and degrade gracefully')