From 3211131096fae9b750f675d0e031b0638bbd52fb Mon Sep 17 00:00:00 2001 From: MyPrototypeWhat Date: Mon, 13 Jul 2026 14:57:23 +0800 Subject: [PATCH 1/6] docs: plan live-smoke provider fixes --- .../2026-07-13-live-smoke-provider-fixes.md | 262 ++++++++++++++++++ ...-07-13-live-smoke-provider-fixes-design.md | 26 ++ 2 files changed, 288 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-13-live-smoke-provider-fixes.md create mode 100644 docs/superpowers/specs/2026-07-13-live-smoke-provider-fixes-design.md diff --git a/docs/superpowers/plans/2026-07-13-live-smoke-provider-fixes.md b/docs/superpowers/plans/2026-07-13-live-smoke-provider-fixes.md new file mode 100644 index 0000000..19bf9bd --- /dev/null +++ b/docs/superpowers/plans/2026-07-13-live-smoke-provider-fixes.md @@ -0,0 +1,262 @@ +# Live Smoke Provider Fixes Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make the deterministic keyless provider live-smoke checks pass against current upstream contracts. + +**Architecture:** Keep each fix inside its provider package. Bound queries at the upstream API, use Rijksmuseum's one-hop EDM JSON-LD profile, and preserve existing `Reference` semantics and local result caps. + +**Tech Stack:** TypeScript 5.9, Vitest 3.2, pnpm 10, native `fetch`/`URL`. + +## Global Constraints + +- No production behavior change for Gutendex in this plan. +- No new dependencies. +- Preserve provider IDs, `referenceId` construction, rights mapping, and normalized `q.limit` semantics. +- Write each regression test first and observe the expected failure before production edits. +- Do not modify or stage the pre-existing untracked files under `docs/superpowers/plans/`. + +--- + +### Task 1: Bound Internet Archive to supported media types + +**Files:** +- Modify: `packages/provider-internet-archive/src/index.ts:89-105` +- Test: `packages/provider-internet-archive/src/__tests__/internet-archive.test.ts:171-186` + +**Interfaces:** +- Consumes: `NormalizedQuery.text`, `NormalizedQuery.limit`, `InternetArchiveConfig.maxRows`. +- Produces: an Advanced Search request whose `q` is `() AND mediatype:(movies OR texts)`. + +- [ ] **Step 1: Write the failing request-construction assertion** + +```ts +expect(url.searchParams.get('q')).toBe('(jazz) AND mediatype:(movies OR texts)') +``` + +- [ ] **Step 2: Run the test and verify RED** + +Run: `pnpm vitest run packages/provider-internet-archive/src/__tests__/internet-archive.test.ts` + +Expected: the request test fails because the current value is `jazz`. + +- [ ] **Step 3: Implement the server-side media filter** + +```ts +url.searchParams.set('q', `(${q.text}) AND mediatype:(movies OR texts)`) +``` + +- [ ] **Step 4: Run the suite and verify GREEN** + +Run: `pnpm vitest run packages/provider-internet-archive/src/__tests__/internet-archive.test.ts` + +Expected: all Internet Archive unit tests pass. + +- [ ] **Step 5: Commit only Task 1 files** + +```bash +git add packages/provider-internet-archive/src/index.ts packages/provider-internet-archive/src/__tests__/internet-archive.test.ts +git commit -m "fix(provider-internet-archive): filter supported media upstream" +``` + +--- + +### Task 2: Apply PoetryDB query limits upstream + +**Files:** +- Modify: `packages/provider-poetrydb/src/index.ts:55-104` +- Test: `packages/provider-poetrydb/src/__tests__/poetrydb.test.ts` + +**Interfaces:** +- Consumes: `NormalizedQuery.text`, `NormalizedQuery.limit`, `PoetryDbSearchOptions`. +- Produces: a default bounded URL such as `https://poetrydb.org/lines,poemcount/love;5`. + +- [ ] **Step 1: Add failing URL tests** + +```ts +it('maps q.limit to poemcount for the default line search', async () => { + let calledUrl = '' + const ctx = responding([], url => { calledUrl = url }) + await poetrydb().search({ text: 'love', modalities: ['text'], limit: 5 }, ctx) + expect(calledUrl).toBe('https://poetrydb.org/lines,poemcount/love;5') +}) +``` + +Also cover that an explicit positive `poemCount` overrides `q.limit`, while an explicit positive `random` suppresses the implicit `poemcount`. + +```ts +expect(urlFor({ poemCount: 3 }, 5)).toBe('https://poetrydb.org/lines,poemcount/love;3') +expect(urlFor({ random: 2 }, 5)).toBe('https://poetrydb.org/lines,random/love;2') +``` + +- [ ] **Step 2: Run the tests and verify RED** + +Run: `pnpm vitest run packages/provider-poetrydb/src/__tests__/poetrydb.test.ts` + +Expected: the default URL is currently `https://poetrydb.org/lines/love`. + +- [ ] **Step 3: Implement bounded URL construction** + +Change `poetrydbUrl` to accept `limit`. Seed default input with `fields=['lines']` and `terms=[text]`, then append one count control: + +```ts +function positiveInt(value: unknown): number | undefined { + return typeof value === 'number' && Number.isInteger(value) && value > 0 + ? value + : undefined +} + +const explicitRandom = positiveInt(options?.random) +const explicitCount = positiveInt(options?.poemCount) +const implicitCount = positiveInt(limit) + +if (fields.length === 0 && terms.length === 0) { + fields.push('lines') + terms.push(text) +} +if (explicitRandom !== undefined && !fields.includes('random')) { + fields.push('random') + terms.push(String(explicitRandom)) +} else if (!fields.includes('poemcount')) { + const count = explicitCount ?? implicitCount + if (count !== undefined) { + fields.push('poemcount') + terms.push(String(count)) + } +} +``` + +Pass `q.limit` from `search`. Preserve an already explicit `poemcount` field and its paired search term. + +- [ ] **Step 4: Run the suite and verify GREEN** + +Run: `pnpm vitest run packages/provider-poetrydb/src/__tests__/poetrydb.test.ts` + +Expected: all PoetryDB unit tests pass. + +- [ ] **Step 5: Commit only Task 2 files** + +```bash +git add packages/provider-poetrydb/src/index.ts packages/provider-poetrydb/src/__tests__/poetrydb.test.ts +git commit -m "fix(provider-poetrydb): bound searches with poemcount" +``` + +--- + +### Task 3: Adapt Rijksmuseum to the current search and EDM contracts + +**Files:** +- Modify: `packages/provider-rijksmuseum/src/index.ts` +- Test: `packages/provider-rijksmuseum/src/__tests__/rijksmuseum.test.ts` + +**Interfaces:** +- Consumes: the current collection search page and one `edm-framed` aggregation per selected ID. +- Produces: image references using `aggregatedCHO.id`, localized title/creator metadata, `edmRights`, `isShownAt.id`, and `isShownBy.id`. + +- [ ] **Step 1: Add failing contract tests** + +Update the search URL assertion: + +```ts +expect(url.searchParams.get('pageSize')).toBeNull() +``` + +Use an EDM fixture with this minimum shape and assert the record URL ends in `_profile=edm-framed`: + +```ts +const EDM = { + id: 'https://id.rijksmuseum.nl/1#aggregation', + edmRights: 'http://creativecommons.org/publicdomain/mark/1.0/', + isShownAt: { id: 'https://www.rijksmuseum.nl/en/collection/object-1' }, + isShownBy: { id: 'https://iiif.micr.io/example/full/max/0/default.jpg' }, + aggregatedCHO: { + id: 'https://id.rijksmuseum.nl/1', + title: { en: ['Landscape'], nl: ['Landschap'] }, + creator: [{ + 'http://www.w3.org/2004/02/skos/core#prefLabel': [ + { '@language': 'en', '@value': 'Example Maker' }, + ], + }], + }, +} +``` + +Assert title `Landscape`, author `Example Maker`, PD rights, canonical ID URL, and IIIF thumbnail/preview. + +- [ ] **Step 2: Run the tests and verify RED** + +Run: `pnpm vitest run packages/provider-rijksmuseum/src/__tests__/rijksmuseum.test.ts` + +Expected: `pageSize` is present, the profile is `la`, and the Linked Art mapper cannot map the EDM fixture. + +- [ ] **Step 3: Implement the current contract** + +Remove `pageSize`. Request each selected record with `_profile=edm-framed`. Replace the Linked Art-specific mapper with focused helpers that: + +```ts +const canonicalUrl = rec.aggregatedCHO?.id +const imageUrl = rec.isShownBy?.id ?? rec.object?.id +const title = firstLocalized(rec.aggregatedCHO?.title, ['en', 'nl']) +const author = firstCreatorLabel(rec.aggregatedCHO?.creator, ['en', 'nl']) +const { license, version, jurisdiction } = mapRightsUrl(rec.edmRights) +``` + +Return `null` when canonical ID or image is absent. Keep one-record failure isolation and `.slice(0, n)`. + +- [ ] **Step 4: Run the suite and verify GREEN** + +Run: `pnpm vitest run packages/provider-rijksmuseum/src/__tests__/rijksmuseum.test.ts` + +Expected: all Rijksmuseum unit tests pass. + +- [ ] **Step 5: Commit only Task 3 files** + +```bash +git add packages/provider-rijksmuseum/src/index.ts packages/provider-rijksmuseum/src/__tests__/rijksmuseum.test.ts +git commit -m "fix(provider-rijksmuseum): use current EDM API contract" +``` + +--- + +### Task 4: Integration verification and live recheck + +**Files:** +- Verify only; no planned production files. + +**Interfaces:** +- Consumes: Tasks 1-3. +- Produces: fresh local evidence for unit, type, build, and real upstream behavior. + +- [ ] **Step 1: Run targeted unit suites** + +Run: `pnpm vitest run packages/provider-internet-archive/src/__tests__/internet-archive.test.ts packages/provider-poetrydb/src/__tests__/poetrydb.test.ts packages/provider-rijksmuseum/src/__tests__/rijksmuseum.test.ts` + +Expected: all targeted tests pass. + +- [ ] **Step 2: Run the three keyless live suites** + +Run: `REFKIT_LIVE=1 pnpm vitest run packages/provider-internet-archive/src/__tests__/live.test.ts packages/provider-poetrydb/src/__tests__/live.test.ts packages/provider-rijksmuseum/src/__tests__/live.test.ts` + +Expected: all three live tests pass. + +- [ ] **Step 3: Re-probe Gutendex without changing code** + +Run: `REFKIT_LIVE=1 pnpm vitest run packages/provider-gutendex/src/__tests__/live.test.ts` + +Expected: record pass/fail as external evidence; a repeated 403 is a follow-up diagnostics task, not permission to alter UA behavior here. + +- [ ] **Step 4: Run repository gates** + +Run: `pnpm typecheck` + +Run: `pnpm test:run` + +Run: `pnpm build` + +Run: `git diff --check` + +Expected: all commands exit 0. + +- [ ] **Step 5: Review the full branch diff** + +Compare the branch against its starting commit, verify only the six provider source/test files and approved plan/spec artifacts changed, and resolve all Critical or Important findings before handoff. diff --git a/docs/superpowers/specs/2026-07-13-live-smoke-provider-fixes-design.md b/docs/superpowers/specs/2026-07-13-live-smoke-provider-fixes-design.md new file mode 100644 index 0000000..1f2b516 --- /dev/null +++ b/docs/superpowers/specs/2026-07-13-live-smoke-provider-fixes-design.md @@ -0,0 +1,26 @@ +# Live Smoke Provider Fixes Design + +## Goal + +Restore the weekly keyless live-smoke workflow by fixing the three deterministic provider failures revealed by run `29229284714`, without changing Gutendex behavior on the basis of a single non-reproducible 403. + +## Scope + +- Internet Archive must ask the upstream search API only for the `movies` and `texts` media types that the provider can map, before applying the requested row limit. +- PoetryDB must translate the normalized query limit into the upstream `poemcount` input so broad line searches remain bounded. Explicit provider options continue to take precedence, and `random` must not be combined with an implicit `poemcount`. +- Rijksmuseum must stop sending the unsupported `pageSize` parameter and must consume the current one-hop `edm-framed` JSON-LD representation instead of parsing the N-Triples `la` profile as JSON. +- Gutendex receives no code change in this implementation. Re-run evidence decides whether a later diagnostics or self-hosting change is warranted. + +## Data Flow + +Each provider continues to receive a `NormalizedQuery` and injected `ProviderContext.fetch`. The fix stays inside each provider package: construct a valid bounded upstream request, map the returned provider-native payload to `Reference`, and preserve the existing provider IDs, canonical IDs, rights mapping, and result limits. + +For Rijksmuseum, collection search still returns object IDs. At most `n` IDs are selected locally, then each ID is fetched once with `_profile=edm-framed`. The aggregation supplies `aggregatedCHO` metadata, `edmRights`, `isShownAt`, and `isShownBy`; no additional Linked Art graph traversal is introduced. + +## Error Handling + +Existing provider-level non-2xx errors remain errors. Rijksmuseum continues to tolerate one bad record fetch without dropping successful siblings. Live smoke remains the integration guard for upstream contract drift. + +## Verification + +Every production change follows a red-green unit test. Final verification runs the three targeted unit suites, the three keyless live suites, `pnpm typecheck`, `pnpm test:run`, `pnpm build`, and `git diff --check`. Gutendex is re-probed separately and is reported as residual external risk if its 403 cannot be reproduced. From 4c6ae5c01ac356594d0791d566754c2ef99c9ccb Mon Sep 17 00:00:00 2001 From: MyPrototypeWhat Date: Mon, 13 Jul 2026 14:59:47 +0800 Subject: [PATCH 2/6] fix(provider-internet-archive): filter supported media upstream --- .../src/__tests__/internet-archive.test.ts | 2 +- packages/provider-internet-archive/src/index.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/provider-internet-archive/src/__tests__/internet-archive.test.ts b/packages/provider-internet-archive/src/__tests__/internet-archive.test.ts index 1d20923..0cb8c02 100644 --- a/packages/provider-internet-archive/src/__tests__/internet-archive.test.ts +++ b/packages/provider-internet-archive/src/__tests__/internet-archive.test.ts @@ -176,7 +176,7 @@ describe('internetArchive search', () => { ) const url = new URL(seen) expect(url.pathname).toBe('/advancedsearch.php') - expect(url.searchParams.get('q')).toBe('jazz') + expect(url.searchParams.get('q')).toBe('(jazz) AND mediatype:(movies OR texts)') expect(url.searchParams.get('output')).toBe('json') expect(url.searchParams.get('rows')).toBe('7') expect(url.searchParams.get('page')).toBe('1') diff --git a/packages/provider-internet-archive/src/index.ts b/packages/provider-internet-archive/src/index.ts index 3b42e5e..af10186 100644 --- a/packages/provider-internet-archive/src/index.ts +++ b/packages/provider-internet-archive/src/index.ts @@ -88,7 +88,7 @@ export function internetArchive(config: InternetArchiveConfig = {}) { capabilities: { controls: [] }, async search(q: NormalizedQuery, ctx: ProviderContext): Promise { const url = new URL(BASE) - url.searchParams.set('q', q.text) + url.searchParams.set('q', `(${q.text}) AND mediatype:(movies OR texts)`) for (const f of ['identifier', 'title', 'creator', 'licenseurl', 'mediatype']) { url.searchParams.append('fl[]', f) } From 7365ea6a06959f3d69ff0dbab62711be8c39f3a1 Mon Sep 17 00:00:00 2001 From: MyPrototypeWhat Date: Mon, 13 Jul 2026 15:06:41 +0800 Subject: [PATCH 3/6] fix(provider-poetrydb): bound searches with poemcount --- .../src/__tests__/poetrydb.test.ts | 36 +++++++++++++++++- packages/provider-poetrydb/src/index.ts | 38 +++++++++++++------ 2 files changed, 61 insertions(+), 13 deletions(-) diff --git a/packages/provider-poetrydb/src/__tests__/poetrydb.test.ts b/packages/provider-poetrydb/src/__tests__/poetrydb.test.ts index c889897..2ac3f16 100644 --- a/packages/provider-poetrydb/src/__tests__/poetrydb.test.ts +++ b/packages/provider-poetrydb/src/__tests__/poetrydb.test.ts @@ -20,7 +20,12 @@ const FIXTURE = [ linecount: '14', }, ] -const ctxWith = (body: unknown): ProviderContext => ({ fetch: (async () => new Response(JSON.stringify(body), { status: 200 })) as typeof fetch }) +const ctxWith = (body: unknown, onFetch?: (url: string) => void): ProviderContext => ({ + fetch: (async (input: Parameters[0]) => { + onFetch?.(String(input)) + return new Response(JSON.stringify(body), { status: 200 }) + }) as typeof fetch, +}) describe('poetrydb provider', () => { it('maps a poem to a full-text passage Reference (PD inferred)', async () => { @@ -43,6 +48,34 @@ describe('poetrydb provider', () => { expect(refs).toEqual([]) }) + it('maps q.limit to poemcount for the default line search', async () => { + let calledUrl = '' + await poetrydb().search({ text: 'love', modalities: ['text'], limit: 5 }, ctxWith([], url => { calledUrl = url })) + expect(calledUrl).toBe('https://poetrydb.org/lines,poemcount/love;5') + }) + + it('prefers an explicit positive poemCount over q.limit', async () => { + let calledUrl = '' + await poetrydb().search({ + text: 'love', + modalities: ['text'], + limit: 5, + providerOptions: { poemCount: 3 }, + }, ctxWith([], url => { calledUrl = url })) + expect(calledUrl).toBe('https://poetrydb.org/lines,poemcount/love;3') + }) + + it('uses an explicit positive random instead of an implicit poemcount', async () => { + let calledUrl = '' + await poetrydb().search({ + text: 'love', + modalities: ['text'], + limit: 5, + providerOptions: { random: 2 }, + }, ctxWith([], url => { calledUrl = url })) + expect(calledUrl).toBe('https://poetrydb.org/lines,random/love;2') + }) + it('builds documented PoetryDB routes from providerOptions', async () => { let calledUrl = '' const ctx: ProviderContext = { @@ -54,6 +87,7 @@ describe('poetrydb provider', () => { await poetrydb().search({ text: 'ignored', modalities: ['text'], + limit: 5, providerOptions: { inputFields: ['title', 'author', 'poemcount'], searchTerms: ['Winter', 'William Shakespeare', '2'], diff --git a/packages/provider-poetrydb/src/index.ts b/packages/provider-poetrydb/src/index.ts index 48b5fc8..2b8a28c 100644 --- a/packages/provider-poetrydb/src/index.ts +++ b/packages/provider-poetrydb/src/index.ts @@ -57,27 +57,41 @@ function searchTerms(value: unknown): string[] { return [] } -function poetrydbUrl(text: string, options: PoetryDbSearchOptions | undefined): string { - if (!options) return `https://poetrydb.org/lines/${encodeURIComponent(text)}` +function positiveInt(value: unknown): number | undefined { + return typeof value === 'number' && Number.isInteger(value) && value > 0 + ? value + : undefined +} +function poetrydbUrl(text: string, options: PoetryDbSearchOptions | undefined, limit: number | undefined): string { const allowedInput = ['author', 'title', 'lines', 'linecount', 'poemcount', 'random'] - const fields = stringList(options.inputFields, allowedInput) - const terms = searchTerms(options.searchTerms) - if (options.poemCount !== undefined && Number.isInteger(options.poemCount) && options.poemCount > 0 && !fields.includes('poemcount')) { - fields.push('poemcount') - terms.push(String(options.poemCount)) + const fields = stringList(options?.inputFields, allowedInput) + const terms = searchTerms(options?.searchTerms) + if (fields.length === 0 && terms.length === 0) { + fields.push('lines') + terms.push(text) } - if (options.random !== undefined && Number.isInteger(options.random) && options.random > 0 && !fields.includes('random')) { + + const explicitRandom = positiveInt(options?.random) + const explicitCount = positiveInt(options?.poemCount) + const implicitCount = positiveInt(limit) + if (explicitRandom !== undefined && !fields.includes('random')) { fields.push('random') - terms.push(String(options.random)) + terms.push(String(explicitRandom)) + } else if (!fields.includes('random') && !fields.includes('poemcount')) { + const count = explicitCount ?? implicitCount + if (count !== undefined) { + fields.push('poemcount') + terms.push(String(count)) + } } const inputFields = fields.length > 0 ? fields : ['lines'] const inputTerms = terms.length > 0 ? terms : [text] if (inputFields.length !== inputTerms.length) return `https://poetrydb.org/lines/${encodeURIComponent(text)}` const encodedTerms = inputTerms.map(term => encodeURIComponent(term)).join(';') - const exact = options.matchExact ? ':abs' : '' - const output = stringList(options.outputFields, ['author', 'title', 'lines', 'linecount', 'all']) + const exact = options?.matchExact ? ':abs' : '' + const output = stringList(options?.outputFields, ['author', 'title', 'lines', 'linecount', 'all']) if (output.length > 0 && !output.includes('all')) { const required = ['author', 'title', 'lines', 'linecount'] const extras = output.filter(field => !required.includes(field)) @@ -95,7 +109,7 @@ export function poetrydb() { capabilities: { controls: [] }, async search(q: NormalizedQuery, ctx: ProviderContext): Promise { // /lines/ finds poems whose line content contains the term (closest to keyword search) - const url = poetrydbUrl(q.text, q.providerOptions as PoetryDbSearchOptions | undefined) + const url = poetrydbUrl(q.text, q.providerOptions as PoetryDbSearchOptions | undefined, q.limit) const res = await ctx.fetch(url, { signal: ctx.signal }) if (!res.ok) throw new Error(`poetrydb search failed: ${res.status}`) const json = (await res.json()) as PoetryDbPoem[] | { status: number } From 1964a900c2cbcf89d568cd13fdd718b5971b1bcd Mon Sep 17 00:00:00 2001 From: MyPrototypeWhat Date: Mon, 13 Jul 2026 15:15:49 +0800 Subject: [PATCH 4/6] fix(provider-poetrydb): preserve one-sided route options --- .../src/__tests__/poetrydb.test.ts | 40 +++++++++++++++++++ packages/provider-poetrydb/src/index.ts | 6 +-- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/packages/provider-poetrydb/src/__tests__/poetrydb.test.ts b/packages/provider-poetrydb/src/__tests__/poetrydb.test.ts index 2ac3f16..9301e67 100644 --- a/packages/provider-poetrydb/src/__tests__/poetrydb.test.ts +++ b/packages/provider-poetrydb/src/__tests__/poetrydb.test.ts @@ -54,6 +54,28 @@ describe('poetrydb provider', () => { expect(calledUrl).toBe('https://poetrydb.org/lines,poemcount/love;5') }) + it('preserves inputFields when searchTerms are omitted', async () => { + let calledUrl = '' + await poetrydb().search({ + text: 'love', + modalities: ['text'], + limit: 5, + providerOptions: { inputFields: ['title'] }, + }, ctxWith([], url => { calledUrl = url })) + expect(calledUrl).toBe('https://poetrydb.org/title,poemcount/love;5') + }) + + it('preserves searchTerms when inputFields are omitted', async () => { + let calledUrl = '' + await poetrydb().search({ + text: 'ignored', + modalities: ['text'], + limit: 5, + providerOptions: { searchTerms: ['Winter'] }, + }, ctxWith([], url => { calledUrl = url })) + expect(calledUrl).toBe('https://poetrydb.org/lines,poemcount/Winter;5') + }) + it('prefers an explicit positive poemCount over q.limit', async () => { let calledUrl = '' await poetrydb().search({ @@ -76,6 +98,24 @@ describe('poetrydb provider', () => { expect(calledUrl).toBe('https://poetrydb.org/lines,random/love;2') }) + it.each([ + ['poemCount', 0], + ['poemCount', -1], + ['poemCount', 1.5], + ['random', 0], + ['random', -1], + ['random', 1.5], + ] as const)('ignores invalid %s=%s and falls back to q.limit', async (option, value) => { + let calledUrl = '' + await poetrydb().search({ + text: 'love', + modalities: ['text'], + limit: 5, + providerOptions: { [option]: value }, + }, ctxWith([], url => { calledUrl = url })) + expect(calledUrl).toBe('https://poetrydb.org/lines,poemcount/love;5') + }) + it('builds documented PoetryDB routes from providerOptions', async () => { let calledUrl = '' const ctx: ProviderContext = { diff --git a/packages/provider-poetrydb/src/index.ts b/packages/provider-poetrydb/src/index.ts index 2b8a28c..c2f399a 100644 --- a/packages/provider-poetrydb/src/index.ts +++ b/packages/provider-poetrydb/src/index.ts @@ -67,10 +67,8 @@ function poetrydbUrl(text: string, options: PoetryDbSearchOptions | undefined, l const allowedInput = ['author', 'title', 'lines', 'linecount', 'poemcount', 'random'] const fields = stringList(options?.inputFields, allowedInput) const terms = searchTerms(options?.searchTerms) - if (fields.length === 0 && terms.length === 0) { - fields.push('lines') - terms.push(text) - } + if (fields.length === 0) fields.push('lines') + if (terms.length === 0) terms.push(text) const explicitRandom = positiveInt(options?.random) const explicitCount = positiveInt(options?.poemCount) From 2cc0aba5a1bbc17cf7d8ffbf8f415a3c71325d89 Mon Sep 17 00:00:00 2001 From: MyPrototypeWhat Date: Mon, 13 Jul 2026 15:22:56 +0800 Subject: [PATCH 5/6] fix(provider-poetrydb): keep fallback searches bounded --- .../src/__tests__/poetrydb.test.ts | 37 +++++++++++++++++++ packages/provider-poetrydb/src/index.ts | 8 +++- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/packages/provider-poetrydb/src/__tests__/poetrydb.test.ts b/packages/provider-poetrydb/src/__tests__/poetrydb.test.ts index 9301e67..2142f1b 100644 --- a/packages/provider-poetrydb/src/__tests__/poetrydb.test.ts +++ b/packages/provider-poetrydb/src/__tests__/poetrydb.test.ts @@ -76,6 +76,43 @@ describe('poetrydb provider', () => { expect(calledUrl).toBe('https://poetrydb.org/lines,poemcount/Winter;5') }) + it('falls back to a bounded line search for multiple inputFields without searchTerms', async () => { + let calledUrl = '' + await poetrydb().search({ + text: 'love', + modalities: ['text'], + limit: 5, + providerOptions: { inputFields: ['title', 'author'] }, + }, ctxWith([], url => { calledUrl = url })) + expect(calledUrl).toBe('https://poetrydb.org/lines,poemcount/love;5') + }) + + it('falls back to a bounded line search for multiple searchTerms without inputFields', async () => { + let calledUrl = '' + await poetrydb().search({ + text: 'love', + modalities: ['text'], + limit: 5, + providerOptions: { searchTerms: ['Winter', 'Sonnet'] }, + }, ctxWith([], url => { calledUrl = url })) + expect(calledUrl).toBe('https://poetrydb.org/lines,poemcount/love;5') + }) + + it('keeps explicit random when mismatched route options fall back', async () => { + let calledUrl = '' + await poetrydb().search({ + text: 'love', + modalities: ['text'], + limit: 5, + providerOptions: { + inputFields: ['title', 'author'], + searchTerms: ['Winter'], + random: 2, + }, + }, ctxWith([], url => { calledUrl = url })) + expect(calledUrl).toBe('https://poetrydb.org/lines,random/love;2') + }) + it('prefers an explicit positive poemCount over q.limit', async () => { let calledUrl = '' await poetrydb().search({ diff --git a/packages/provider-poetrydb/src/index.ts b/packages/provider-poetrydb/src/index.ts index c2f399a..3770407 100644 --- a/packages/provider-poetrydb/src/index.ts +++ b/packages/provider-poetrydb/src/index.ts @@ -65,10 +65,14 @@ function positiveInt(value: unknown): number | undefined { function poetrydbUrl(text: string, options: PoetryDbSearchOptions | undefined, limit: number | undefined): string { const allowedInput = ['author', 'title', 'lines', 'linecount', 'poemcount', 'random'] - const fields = stringList(options?.inputFields, allowedInput) - const terms = searchTerms(options?.searchTerms) + let fields = stringList(options?.inputFields, allowedInput) + let terms = searchTerms(options?.searchTerms) if (fields.length === 0) fields.push('lines') if (terms.length === 0) terms.push(text) + if (fields.length !== terms.length) { + fields = ['lines'] + terms = [text] + } const explicitRandom = positiveInt(options?.random) const explicitCount = positiveInt(options?.poemCount) From d591c553d6b198086e6cd585f9b9b90878103e2e Mon Sep 17 00:00:00 2001 From: MyPrototypeWhat Date: Mon, 13 Jul 2026 15:34:59 +0800 Subject: [PATCH 6/6] fix(provider-rijksmuseum): use current EDM API contract --- .../src/__tests__/rijksmuseum.test.ts | 385 ++++++++++++------ packages/provider-rijksmuseum/src/index.ts | 207 ++++------ 2 files changed, 340 insertions(+), 252 deletions(-) diff --git a/packages/provider-rijksmuseum/src/__tests__/rijksmuseum.test.ts b/packages/provider-rijksmuseum/src/__tests__/rijksmuseum.test.ts index bb43916..4b6ebd8 100644 --- a/packages/provider-rijksmuseum/src/__tests__/rijksmuseum.test.ts +++ b/packages/provider-rijksmuseum/src/__tests__/rijksmuseum.test.ts @@ -1,22 +1,30 @@ import { describe, expect, it } from 'vitest' -import { evaluateUse, type ProviderContext } from '@refkit/core' +import { evaluateUse, referenceId, type ProviderContext } from '@refkit/core' import { rijksmuseum } from '../index' -// Search returns IDs only → N+1 record fetch. Route /search/collection to the -// list body, and each /{id} (with ?_profile=la) to its record body. +interface Captures { + search?: (url: string) => void + record?: (url: string) => void +} + +// Search returns IDs only, so route the collection page separately from each +// one-hop EDM aggregation response. const ctxRouting = ( list: unknown, records: Record, - capture?: (searchUrl: string) => void, + captures: Captures = {}, ): ProviderContext => ({ fetch: (async (input: Parameters[0]) => { - const u = String(input) - if (u.includes('/search/collection')) { - capture?.(u) + const url = String(input) + if (url.includes('/search/collection')) { + captures.search?.(url) return new Response(JSON.stringify(list), { status: 200 }) } - const m = u.match(/\/(\d+)(?:\?|$)/) - if (m && records[m[1]]) return new Response(JSON.stringify(records[m[1]]), { status: 200 }) + captures.record?.(url) + const match = url.match(/\/(\d+)(?:\?|$)/) + if (match && Object.hasOwn(records, match[1])) { + return new Response(JSON.stringify(records[match[1]]), { status: 200 }) + } return new Response('null', { status: 404 }) }) as typeof fetch, }) @@ -26,88 +34,166 @@ const LIST = { type: 'OrderedCollectionPage', partOf: { type: 'OrderedCollection', totalItems: 3 }, orderedItems: [ - { id: 'https://id.rijksmuseum.nl/200100988', type: 'HumanMadeObject' }, - { id: 'https://id.rijksmuseum.nl/200100777', type: 'HumanMadeObject' }, - { id: 'https://id.rijksmuseum.nl/200100666', type: 'HumanMadeObject' }, + { id: 'https://id.rijksmuseum.nl/1', type: 'HumanMadeObject' }, + { id: 'https://id.rijksmuseum.nl/2', type: 'HumanMadeObject' }, + { id: 'https://id.rijksmuseum.nl/3', type: 'HumanMadeObject' }, ], next: { id: 'https://data.rijksmuseum.nl/search/collection?title=sea&pageToken=abc', type: 'OrderedCollectionPage' }, } -// CC0 record (verified shape: title=identified_by[].content of type Name; creator -// via produced_by.carried_out_by; rights URI under subject_to.classified_as.id; -// image under digitally_carried_by.access_point.id). -const REC_CC0 = { - id: 'https://id.rijksmuseum.nl/200100988', - type: 'HumanMadeObject', - identified_by: [ - { type: 'Name', classified_as: [{ id: 'http://vocab.getty.edu/aat/300404670', _label: 'preferred terms' }], content: 'Misty Sea' }, - ], - produced_by: { - type: 'Production', - carried_out_by: [{ id: 'https://id.rijksmuseum.nl/person/toorop', type: 'Person', _label: 'Jan Toorop' }], +const EDM = { + id: 'https://id.rijksmuseum.nl/1#aggregation', + edmRights: 'http://creativecommons.org/publicdomain/mark/1.0/', + isShownAt: { id: 'https://www.rijksmuseum.nl/en/collection/object-1' }, + isShownBy: { id: 'https://iiif.micr.io/example/full/max/0/default.jpg' }, + aggregatedCHO: { + id: 'https://id.rijksmuseum.nl/1', + title: { en: ['Landscape'], nl: ['Landschap'] }, + creator: [{ + 'http://www.w3.org/2004/02/skos/core#prefLabel': [ + { '@language': 'en', '@value': 'Example Maker' }, + ], + }], }, - subject_to: [ - { type: 'Right', classified_as: [{ id: 'https://creativecommons.org/publicdomain/zero/1.0/', _label: 'CC0 1.0' }] }, - ], - subject_of: [ - { type: 'VisualItem', digitally_carried_by: [{ type: 'DigitalObject', access_point: [{ id: 'https://lh3.googleusercontent.com/cc0-image=s0', type: 'DigitalObject' }] }] }, - ], } -// Public Domain Mark record. -const REC_PDM = { - id: 'https://id.rijksmuseum.nl/200100777', - type: 'HumanMadeObject', - identified_by: [{ type: 'Name', content: 'Old Engraving' }], - produced_by: { type: 'Production', carried_out_by: [{ type: 'Person', _label: 'Anonymous' }] }, - subject_to: [{ type: 'Right', classified_as: [{ id: 'https://creativecommons.org/publicdomain/mark/1.0/', _label: 'PDM' }] }], - subject_of: [{ type: 'VisualItem', digitally_carried_by: [{ type: 'DigitalObject', access_point: [{ id: 'https://lh3.googleusercontent.com/pdm-image=s0' }] }] }], +const REC_CC0 = { + id: 'https://id.rijksmuseum.nl/2#aggregation', + edmRights: 'https://creativecommons.org/publicdomain/zero/1.0/', + isShownAt: { id: 'https://www.rijksmuseum.nl/en/collection/object-2' }, + isShownBy: { id: 'https://iiif.micr.io/cc0/full/max/0/default.jpg' }, + aggregatedCHO: { + id: 'https://id.rijksmuseum.nl/2', + title: { en: ['Misty Sea'] }, + creator: [{ + 'http://www.w3.org/2004/02/skos/core#prefLabel': [ + { '@language': 'en', '@value': 'Jan Toorop' }, + ], + }], + }, } -// Rights-less record: no creativecommons/rightsstatements URI anywhere → unknown. const REC_NO_RIGHTS = { - id: 'https://id.rijksmuseum.nl/200100666', - type: 'HumanMadeObject', - identified_by: [{ type: 'Name', content: 'Untitled (rights unclear)' }], - produced_by: { type: 'Production', carried_out_by: [{ type: 'Person', _label: 'Unknown Maker' }] }, - subject_of: [{ type: 'VisualItem', digitally_carried_by: [{ type: 'DigitalObject', access_point: [{ id: 'https://lh3.googleusercontent.com/mystery=s0' }] }] }], + id: 'https://id.rijksmuseum.nl/3#aggregation', + isShownAt: { id: 'https://www.rijksmuseum.nl/en/collection/object-3' }, + isShownBy: { id: 'https://iiif.micr.io/mystery/full/max/0/default.jpg' }, + aggregatedCHO: { + id: 'https://id.rijksmuseum.nl/3', + title: { en: ['Untitled (rights unclear)'] }, + creator: [{ + 'http://www.w3.org/2004/02/skos/core#prefLabel': [ + { '@language': 'en', '@value': 'Unknown Maker' }, + ], + }], + }, } describe('rijksmuseum provider', () => { - it('maps a CC0 record to a CC0 reference that clears a commercial-product use', async () => { + it('fetches edm-framed JSON-LD and maps its canonical, page, image, metadata, and rights fields', async () => { + const recordUrls: string[] = [] const refs = await rijksmuseum().search( - { text: 'sea', modalities: ['image'], limit: 10 }, - ctxRouting(LIST, { '200100988': REC_CC0, '200100777': REC_PDM, '200100666': REC_NO_RIGHTS }), + { text: 'landscape', modalities: ['image'], limit: 1 }, + ctxRouting(LIST, { '1': EDM }, { record: url => recordUrls.push(url) }), ) - const cc0 = refs.find(r => r.title === 'Misty Sea')! - expect(cc0.modality).toBe('image') - expect(cc0.rights.license).toBe('CC0-1.0') - expect(cc0.rights.author).toBe('Jan Toorop') - expect(cc0.canonicalUrl).toBe('https://id.rijksmuseum.nl/200100988') - expect(cc0.preview?.url).toContain('googleusercontent') - expect(cc0.rights.licenseVersion).toBeUndefined() // CC0/PD never set version - expect(evaluateUse(cc0.rights, 'commercial-product').decision).toBe('allowed') + + expect(recordUrls).toHaveLength(1) + expect(new URL(recordUrls[0]).searchParams.get('_profile')).toBe('edm-framed') + expect(refs).toHaveLength(1) + + const landscape = refs[0] + expect(landscape.id).toBe(referenceId('rijksmuseum', 'https://id.rijksmuseum.nl/1')) + expect(landscape.modality).toBe('image') + expect(landscape.title).toBe('Landscape') + expect(landscape.rights.author).toBe('Example Maker') + expect(landscape.rights.license).toBe('PD') + expect(landscape.rights.licenseVersion).toBeUndefined() + expect(landscape.source).toEqual({ + providerId: 'rijksmuseum', + sourceUrl: 'https://www.rijksmuseum.nl/en/collection/object-1', + }) + expect(landscape.rights.raw.sourceUrl).toBe('https://www.rijksmuseum.nl/en/collection/object-1') + expect(landscape.canonicalUrl).toBe('https://id.rijksmuseum.nl/1') + expect(landscape.thumbnail?.url).toBe('https://iiif.micr.io/example/full/max/0/default.jpg') + expect(landscape.preview).toEqual({ + url: 'https://iiif.micr.io/example/full/max/0/default.jpg', + mediaType: 'image/jpeg', + }) + expect(evaluateUse(landscape.rights, 'commercial-product').decision).toBe('allowed') }) - it('maps a Public Domain Mark record to PD', async () => { + it('maps a CC0 record to a CC0 reference that clears a commercial-product use', async () => { const refs = await rijksmuseum().search( { text: 'sea', modalities: ['image'] }, - ctxRouting(LIST, { '200100988': REC_CC0, '200100777': REC_PDM, '200100666': REC_NO_RIGHTS }), + ctxRouting(LIST, { '1': EDM, '2': REC_CC0, '3': REC_NO_RIGHTS }), ) - const pd = refs.find(r => r.title === 'Old Engraving')! - expect(pd.rights.license).toBe('PD') - expect(evaluateUse(pd.rights, 'commercial-product').decision).toBe('allowed') + const cc0 = refs.find(ref => ref.title === 'Misty Sea') + expect(cc0).toBeDefined() + expect(cc0?.rights.license).toBe('CC0-1.0') + expect(cc0?.rights.author).toBe('Jan Toorop') + expect(cc0?.rights.licenseVersion).toBeUndefined() + expect(evaluateUse(cc0!.rights, 'commercial-product').decision).toBe('allowed') }) - it('marks a record with no parseable open-rights URI as unknown → needs-review (not dropped)', async () => { + it('marks a record with no rights URI as unknown and keeps it for review', async () => { const refs = await rijksmuseum().search( { text: 'sea', modalities: ['image'] }, - ctxRouting(LIST, { '200100988': REC_CC0, '200100777': REC_PDM, '200100666': REC_NO_RIGHTS }), + ctxRouting(LIST, { '1': EDM, '2': REC_CC0, '3': REC_NO_RIGHTS }), + ) + const mystery = refs.find(ref => ref.title === 'Untitled (rights unclear)') + expect(mystery).toBeDefined() + expect(mystery?.rights.license).toBe('unknown') + expect(evaluateUse(mystery!.rights, 'commercial-product').decision).toBe('needs-review') + }) + + it('prefers English localized metadata, then Dutch, then the first available value', async () => { + const records = { + '4': { + edmRights: 'http://creativecommons.org/publicdomain/mark/1.0/', + isShownAt: { id: 'https://www.rijksmuseum.nl/nl/collectie/object-4' }, + isShownBy: { id: 'https://iiif.micr.io/4/full/max/0/default.jpg' }, + aggregatedCHO: { + id: 'https://id.rijksmuseum.nl/4', + title: { fr: ['Paysage'], nl: ['Landschap'] }, + creator: [{ + 'http://www.w3.org/2004/02/skos/core#prefLabel': [ + { '@language': 'fr', '@value': 'Créateur français' }, + { '@language': 'nl', '@value': 'Nederlandse maker' }, + ], + }], + }, + }, + '5': { + edmRights: 'http://creativecommons.org/publicdomain/mark/1.0/', + isShownAt: { id: 'https://www.rijksmuseum.nl/en/collection/object-5' }, + isShownBy: { id: 'https://iiif.micr.io/5/full/max/0/default.jpg' }, + aggregatedCHO: { + id: 'https://id.rijksmuseum.nl/5', + title: { de: ['Landschaft'], fr: ['Paysage'] }, + creator: [{ + 'http://www.w3.org/2004/02/skos/core#prefLabel': [ + { '@language': 'de', '@value': 'Deutscher Künstler' }, + { '@language': 'fr', '@value': 'Artiste français' }, + ], + }], + }, + }, + } + const list = { + type: 'OrderedCollectionPage', + orderedItems: [ + { id: 'https://id.rijksmuseum.nl/4' }, + { id: 'https://id.rijksmuseum.nl/5' }, + ], + } + const refs = await rijksmuseum().search( + { text: 'landscape', modalities: ['image'] }, + ctxRouting(list, records), ) - const mystery = refs.find(r => r.title === 'Untitled (rights unclear)')! - expect(mystery).toBeDefined() // kept, not silently dropped - expect(mystery.rights.license).toBe('unknown') - expect(evaluateUse(mystery.rights, 'commercial-product').decision).toBe('needs-review') + + expect(refs.map(ref => [ref.title, ref.rights.author])).toEqual([ + ['Landschap', 'Nederlandse maker'], + ['Landschaft', 'Deutscher Künstler'], + ]) }) it('returns [] when the search finds nothing', async () => { @@ -118,112 +204,149 @@ describe('rijksmuseum provider', () => { expect(refs).toEqual([]) }) - it('survives a single failed per-item fetch without dropping the batch', async () => { + it('survives a single failed per-item fetch without dropping successful siblings', async () => { const refs = await rijksmuseum().search( { text: 'sea', modalities: ['image'] }, - // 200100777 record omitted → its fetch 404s; the other two must still map. - ctxRouting(LIST, { '200100988': REC_CC0, '200100666': REC_NO_RIGHTS }), + // Record 2 is omitted, so its fetch returns 404. Records 1 and 3 still map. + ctxRouting(LIST, { '1': EDM, '3': REC_NO_RIGHTS }), ) - expect(refs.map(r => r.title).sort()).toEqual(['Misty Sea', 'Untitled (rights unclear)']) + expect(refs.map(ref => ref.title).sort()).toEqual(['Landscape', 'Untitled (rights unclear)']) }) - it('drops a record whose only access_point is a viewer/collection page (never a non-image preview)', async () => { - // No `format`/IIIF on the DigitalObject and the access_point is a web page, not an - // image → findImage() returns undefined → the item is dropped (not surfaced with a - // webpage in preview.url). - const REC_PAGE_ONLY = { - id: 'https://id.rijksmuseum.nl/200100555', - type: 'HumanMadeObject', - identified_by: [{ type: 'Name', content: 'Viewer Only' }], - subject_to: [{ type: 'Right', classified_as: [{ id: 'https://creativecommons.org/publicdomain/zero/1.0/' }] }], - subject_of: [{ type: 'VisualItem', digitally_carried_by: [{ type: 'DigitalObject', access_point: [{ id: 'https://www.rijksmuseum.nl/en/collection/SK-A-1' }] }] }], - } - const ONE = { + it('filters records missing either the canonical ID or an image URL', async () => { + const list = { type: 'OrderedCollectionPage', - orderedItems: [{ id: 'https://id.rijksmuseum.nl/200100555', type: 'HumanMadeObject' }], + orderedItems: [ + { id: 'https://id.rijksmuseum.nl/6' }, + { id: 'https://id.rijksmuseum.nl/7' }, + ], + } + const records = { + '6': { + edmRights: 'http://creativecommons.org/publicdomain/mark/1.0/', + isShownBy: { id: 'https://iiif.micr.io/6/full/max/0/default.jpg' }, + aggregatedCHO: { title: { en: ['Missing canonical ID'] } }, + }, + '7': { + edmRights: 'http://creativecommons.org/publicdomain/mark/1.0/', + isShownAt: { id: 'https://www.rijksmuseum.nl/en/collection/object-7' }, + aggregatedCHO: { + id: 'https://id.rijksmuseum.nl/7', + title: { en: ['Missing image'] }, + }, + }, } + const refs = await rijksmuseum().search( { text: 'x', modalities: ['image'] }, - ctxRouting(ONE, { '200100555': REC_PAGE_ONLY }), + ctxRouting(list, records), ) expect(refs).toEqual([]) }) - it('prefers an image-typed (format/IIIF) DigitalObject over a non-image access_point', async () => { - // The first access_point is a page; a second DigitalObject is typed image/jpeg → - // findImage() must pick the typed one and carry its mediaType. - const REC_TYPED = { - id: 'https://id.rijksmuseum.nl/200100444', - type: 'HumanMadeObject', - identified_by: [{ type: 'Name', content: 'Typed Image' }], - subject_to: [{ type: 'Right', classified_as: [{ id: 'https://creativecommons.org/publicdomain/zero/1.0/' }] }], - subject_of: [ - { type: 'VisualItem', digitally_carried_by: [{ type: 'DigitalObject', access_point: [{ id: 'https://www.rijksmuseum.nl/en/collection/SK-A-2' }] }] }, - { type: 'VisualItem', digitally_carried_by: [{ type: 'DigitalObject', format: 'image/jpeg', access_point: [{ id: 'https://iiif.example.org/image/abc/full/full/0/default.jpg' }] }] }, - ], + it('falls back to object.id when isShownBy is absent', async () => { + const record = { + edmRights: 'http://creativecommons.org/publicdomain/mark/1.0/', + isShownAt: { id: 'https://www.rijksmuseum.nl/en/collection/object-8' }, + object: { id: 'https://iiif.micr.io/8/full/max/0/default.jpg' }, + aggregatedCHO: { + id: 'https://id.rijksmuseum.nl/8', + title: { en: ['Object image fallback'] }, + }, + } + const list = { + type: 'OrderedCollectionPage', + orderedItems: [{ id: 'https://id.rijksmuseum.nl/8' }], } - const ONE = { type: 'OrderedCollectionPage', orderedItems: [{ id: 'https://id.rijksmuseum.nl/200100444', type: 'HumanMadeObject' }] } - const refs = await rijksmuseum().search({ text: 'x', modalities: ['image'] }, ctxRouting(ONE, { '200100444': REC_TYPED })) + + const refs = await rijksmuseum().search( + { text: 'x', modalities: ['image'] }, + ctxRouting(list, { '8': record }), + ) expect(refs).toHaveLength(1) - expect(refs[0].preview?.url).toBe('https://iiif.example.org/image/abc/full/full/0/default.jpg') - expect(refs[0].preview?.mediaType).toBe('image/jpeg') + expect(refs[0].thumbnail?.url).toBe('https://iiif.micr.io/8/full/max/0/default.jpg') + expect(refs[0].preview?.url).toBe('https://iiif.micr.io/8/full/max/0/default.jpg') }) - it('maps a found rightsstatements.org URI faithfully (InC→proprietary, NoC-US→PD+US)', async () => { - // findRightsUrl matches rightsstatements.org; mapping must honor it, not collapse to unknown. - const REC_INC = { - id: 'https://id.rijksmuseum.nl/200100333', - type: 'HumanMadeObject', - identified_by: [{ type: 'Name', content: 'In Copyright' }], - subject_to: [{ type: 'Right', classified_as: [{ id: 'http://rightsstatements.org/vocab/InC/1.0/' }] }], - subject_of: [{ type: 'VisualItem', digitally_carried_by: [{ type: 'DigitalObject', format: 'image/jpeg', access_point: [{ id: 'https://iiif.example.org/inc/full/full/0/default.jpg' }] }] }], - } - const REC_NOC_US = { - id: 'https://id.rijksmuseum.nl/200100222', - type: 'HumanMadeObject', - identified_by: [{ type: 'Name', content: 'No Copyright US' }], - subject_to: [{ type: 'Right', classified_as: [{ id: 'http://rightsstatements.org/vocab/NoC-US/1.0/' }] }], - subject_of: [{ type: 'VisualItem', digitally_carried_by: [{ type: 'DigitalObject', format: 'image/jpeg', access_point: [{ id: 'https://iiif.example.org/noc/full/full/0/default.jpg' }] }] }], + it('maps edmRights rightsstatements.org URIs faithfully', async () => { + const records = { + '9': { + edmRights: 'http://rightsstatements.org/vocab/InC/1.0/', + isShownAt: { id: 'https://www.rijksmuseum.nl/en/collection/object-9' }, + isShownBy: { id: 'https://iiif.micr.io/9/full/max/0/default.jpg' }, + aggregatedCHO: { + id: 'https://id.rijksmuseum.nl/9', + title: { en: ['In Copyright'] }, + }, + }, + '10': { + edmRights: 'http://rightsstatements.org/vocab/NoC-US/1.0/', + isShownAt: { id: 'https://www.rijksmuseum.nl/en/collection/object-10' }, + isShownBy: { id: 'https://iiif.micr.io/10/full/max/0/default.jpg' }, + aggregatedCHO: { + id: 'https://id.rijksmuseum.nl/10', + title: { en: ['No Copyright US'] }, + }, + }, } - const TWO = { + const list = { type: 'OrderedCollectionPage', orderedItems: [ - { id: 'https://id.rijksmuseum.nl/200100333', type: 'HumanMadeObject' }, - { id: 'https://id.rijksmuseum.nl/200100222', type: 'HumanMadeObject' }, + { id: 'https://id.rijksmuseum.nl/9' }, + { id: 'https://id.rijksmuseum.nl/10' }, ], } + const refs = await rijksmuseum().search( { text: 'x', modalities: ['image'] }, - ctxRouting(TWO, { '200100333': REC_INC, '200100222': REC_NOC_US }), + ctxRouting(list, records), ) - const inc = refs.find(r => r.title === 'In Copyright')! - expect(inc.rights.license).toBe('proprietary') - const nocUs = refs.find(r => r.title === 'No Copyright US')! - expect(nocUs.rights.license).toBe('PD') - expect(nocUs.rights.jurisdiction).toBe('US') + const inc = refs.find(ref => ref.title === 'In Copyright') + expect(inc?.rights.license).toBe('proprietary') + const nocUs = refs.find(ref => ref.title === 'No Copyright US') + expect(nocUs?.rights.license).toBe('PD') + expect(nocUs?.rights.jurisdiction).toBe('US') }) - it('forwards the keyword and documented search options + caps the page size to the limit', async () => { + it('forwards documented search options without pageSize and limits record fetches locally', async () => { let searchUrl = '' + const recordUrls: string[] = [] await rijksmuseum().search( { text: 'vermeer', modalities: ['image'], - limit: 5, - providerOptions: { type: 'painting', material: 'canvas', technique: 'oil paint', creator: 'Johannes Vermeer', imageAvailable: true }, + limit: 2, + providerOptions: { + type: 'painting', + material: 'canvas', + technique: 'oil paint', + creator: 'Johannes Vermeer', + description: 'interior', + imageAvailable: true, + }, }, - ctxRouting({ type: 'OrderedCollectionPage', orderedItems: [] }, {}, (u) => { searchUrl = u }), + ctxRouting( + LIST, + { '1': EDM, '2': REC_CC0 }, + { + search: url => { searchUrl = url }, + record: url => recordUrls.push(url), + }, + ), ) + const url = new URL(searchUrl) expect(url.origin + url.pathname).toBe('https://data.rijksmuseum.nl/search/collection') - expect(url.searchParams.get('title')).toBe('vermeer') // primary keyword param + expect(url.searchParams.get('title')).toBe('vermeer') expect(url.searchParams.get('type')).toBe('painting') expect(url.searchParams.get('material')).toBe('canvas') expect(url.searchParams.get('technique')).toBe('oil paint') expect(url.searchParams.get('creator')).toBe('Johannes Vermeer') + expect(url.searchParams.get('description')).toBe('interior') expect(url.searchParams.get('imageAvailable')).toBe('true') - expect(url.searchParams.get('pageSize')).toBe('5') // limit → page size cap - // keyless: never a key param + expect(url.searchParams.get('pageSize')).toBeNull() expect(url.searchParams.get('key')).toBeNull() + expect(recordUrls).toHaveLength(2) + expect(recordUrls.some(recordUrl => recordUrl.includes('/3?'))).toBe(false) }) }) diff --git a/packages/provider-rijksmuseum/src/index.ts b/packages/provider-rijksmuseum/src/index.ts index 25c5df8..82e83dd 100644 --- a/packages/provider-rijksmuseum/src/index.ts +++ b/packages/provider-rijksmuseum/src/index.ts @@ -1,13 +1,13 @@ import { defineProvider, referenceId, - setIfString, setIfBoolean, mapRightsUrl, isLikelyImageUrl, ccVersionFor, + setIfString, setIfBoolean, mapRightsUrl, ccVersionFor, type Reference, type RightsRecord, type NormalizedQuery, type ProviderContext, } from '@refkit/core' export interface RijksmuseumConfig { /** Max records fetched per search. Search returns only IDs, so each result - * costs one extra Linked-Art fetch — this bounds that N+1 fan-out. Default 12. */ + * costs one extra EDM fetch, bounding the N+1 fan-out. Default 12. */ maxObjects?: number } @@ -18,7 +18,7 @@ export interface RijksmuseumSearchOptions { material?: string /** Technique, e.g. 'oil paint'. */ technique?: string - /** Maker/artist (maps to `creator`). */ + /** Maker/artist (maps to creator). */ creator?: string /** Free-text description match. */ description?: string @@ -28,144 +28,111 @@ export interface RijksmuseumSearchOptions { const SEARCH = 'https://data.rijksmuseum.nl/search/collection' const RIJKS_TERMS = 'https://www.rijksmuseum.nl/en/data/policy' +const SKOS_PREF_LABEL = 'http://www.w3.org/2004/02/skos/core#prefLabel' -// Rijksmuseum open-access rights are usually CC deed URLs (effectively CC0/PDM; BY/BY-SA -// possible), but `findRightsUrl` also matches rightsstatements.org URIs — so we map via core -// `mapRightsUrl` (CC deeds + faithful rightsstatements.org). Mapping via the CC-only -// `mapCcDeedUrl` would collapse a found rightsstatements URI to `unknown`, contradicting the -// matcher. mapRightsUrl delegates CC deeds to mapCcDeedUrl, so CC handling is identical. +interface EdmLink { + id?: string +} -// The Linked-Art graph is deeply nested and varies per record, so we extract by -// shape, not by fixed index paths (see plan Open Questions). +interface EdmCreatorLabel { + '@language'?: string + '@value'?: string +} -/** First string anywhere in the record matching a known rights-deed host. */ -function findRightsUrl(node: unknown, depth = 0): string | undefined { - if (depth > 8 || node == null) return undefined - if (typeof node === 'string') { - return /creativecommons\.org\/(publicdomain|licenses)|rightsstatements\.org/.test(node) ? node : undefined - } - if (Array.isArray(node)) { - for (const v of node) { const hit = findRightsUrl(v, depth + 1); if (hit) return hit } - return undefined - } - if (typeof node === 'object') { - for (const v of Object.values(node as Record)) { - const hit = findRightsUrl(v, depth + 1); if (hit) return hit - } - } - return undefined +interface EdmCreator { + [SKOS_PREF_LABEL]?: EdmCreatorLabel[] } -// We must not put a NON-image URL (a viewer/collection web page) into preview.url. -// The API carries the answer: a DigitalObject's `format` (a MIME type) and IIIF -// `conforms_to` say which access_point is the image. So: read the type first, then -// fall back to a cheap URL heuristic (core `isLikelyImageUrl`, no network probe — `core` -// never fetches bytes, and that would add an extra request per item). See Open Questions #1. +interface EdmAggregatedCho { + id?: string + title?: Record + creator?: EdmCreator[] +} -interface LaDigitalObject { - type?: string - format?: string - conforms_to?: Array<{ id?: string }> - access_point?: Array<{ id?: string }> +interface EdmRecord { + edmRights?: string + isShownAt?: EdmLink + isShownBy?: EdmLink + object?: EdmLink + aggregatedCHO?: EdmAggregatedCho } -/** Collect every node that carries an `access_point` (the DigitalObjects) anywhere. */ -function collectDigitalObjects(node: unknown, out: LaDigitalObject[] = [], depth = 0): LaDigitalObject[] { - if (depth > 8 || node == null) return out - if (Array.isArray(node)) { for (const v of node) collectDigitalObjects(v, out, depth + 1); return out } - if (typeof node === 'object') { - const o = node as Record - if (Array.isArray(o.access_point)) out.push(o as LaDigitalObject) - for (const v of Object.values(o)) collectDigitalObjects(v, out, depth + 1) - } - return out +function firstString(value: unknown): string | undefined { + if (typeof value === 'string') return value || undefined + if (!Array.isArray(value)) return undefined + return value.find((item): item is string => typeof item === 'string' && item.length > 0) } -/** Best usable IMAGE url + its mediaType, or undefined. - * Tier 1: a DigitalObject explicitly typed `image/*` or IIIF → trust it. - * Tier 2: any access_point whose URL heuristically looks like an image. - * Otherwise undefined → the item is dropped (an image provider with no image is useless). */ -function findImage(rec: Record): { url: string; mediaType: string } | undefined { - const objs = collectDigitalObjects(rec) - // Tier 1 — explicit type from the data. - for (const o of objs) { - const fmt = typeof o.format === 'string' ? o.format : undefined - const isIiif = Array.isArray(o.conforms_to) && o.conforms_to.some(c => typeof c?.id === 'string' && /iiif/i.test(c.id)) - if ((fmt && fmt.startsWith('image/')) || isIiif) { - const url = o.access_point?.find(a => typeof a?.id === 'string')?.id - if (url) return { url, mediaType: fmt && fmt.startsWith('image/') ? fmt : 'image/jpeg' } - } +function firstLocalized(value: unknown, preferredLanguages: string[]): string | undefined { + if (!value || typeof value !== 'object' || Array.isArray(value)) return firstString(value) + const localized = value as Record + for (const language of preferredLanguages) { + const text = firstString(localized[language]) + if (text) return text } - // Tier 2 — URL heuristic fallback. - for (const o of objs) { - const hit = o.access_point?.find(a => typeof a?.id === 'string' && isLikelyImageUrl(a.id))?.id - if (hit) return { url: hit, mediaType: 'image/jpeg' } + for (const candidate of Object.values(localized)) { + const text = firstString(candidate) + if (text) return text } return undefined } -interface LaName { type?: string; content?: string } -function findTitle(rec: Record): string | undefined { - const names = rec.identified_by - if (Array.isArray(names)) { - for (const n of names as LaName[]) { - if (n?.type === 'Name' && typeof n.content === 'string' && n.content) return n.content +function firstCreatorLabel( + creators: EdmCreator[] | undefined, + preferredLanguages: string[], +): string | undefined { + const labels = (creators ?? []).flatMap(creator => creator[SKOS_PREF_LABEL] ?? []) + for (const language of preferredLanguages) { + for (const label of labels) { + if (label['@language'] !== language) continue + const text = firstString(label['@value']) + if (text) return text } } - return undefined -} - -function findCreator(rec: Record): string | undefined { - const prod = rec.produced_by as Record | undefined - if (!prod) return undefined - const direct = prod.carried_out_by - const parts = Array.isArray(prod.part) ? (prod.part as Record[]) : [] - const actors = [ - ...(Array.isArray(direct) ? (direct as Record[]) : []), - ...parts.flatMap(p => (Array.isArray(p.carried_out_by) ? (p.carried_out_by as Record[]) : [])), - ] - for (const a of actors) { - const label = a._label ?? (a as { notation?: unknown }).notation - if (typeof label === 'string' && label) return label + for (const label of labels) { + const text = firstString(label['@value']) + if (text) return text } return undefined } -function toReference(rec: Record): Reference | null { - const id = typeof rec.id === 'string' ? rec.id : undefined - if (!id) return null - const img = findImage(rec) - if (!img) return null // no usable IMAGE url (e.g. only a viewer/collection page) → drop - const { license, version, jurisdiction } = mapRightsUrl(findRightsUrl(rec)) +function toReference(rec: EdmRecord): Reference | null { + const canonicalUrl = rec.aggregatedCHO?.id + const imageUrl = rec.isShownBy?.id ?? rec.object?.id + if (typeof canonicalUrl !== 'string' || !canonicalUrl) return null + if (typeof imageUrl !== 'string' || !imageUrl) return null + + const shownAt = rec.isShownAt?.id + const sourceUrl = typeof shownAt === 'string' && shownAt ? shownAt : canonicalUrl + const rightsUrl = typeof rec.edmRights === 'string' ? rec.edmRights : undefined + const { license, version, jurisdiction } = mapRightsUrl(rightsUrl) const rights: RightsRecord = { license, - // CC version is metadata only (attribution/audit), kept for every versioned CC family - // (BY/BY-SA/NC/ND variants). Rijksmuseum open-access items are typically CC0/PD, so this - // is behavior-neutral here in practice — kept for consistency with the other URL-mapped - // providers so no stale guard survives. licenseVersion: ccVersionFor(license, version), - // jurisdiction-scoped status (e.g. rightsstatements NoC-US → PD in the US) ...(jurisdiction ? { jurisdiction } : {}), - author: findCreator(rec) || undefined, + author: firstCreatorLabel(rec.aggregatedCHO?.creator, ['en', 'nl']), rehostPolicy: 'cache-allowed', - raw: { sourceTerms: RIJKS_TERMS, sourceUrl: id }, + raw: { sourceTerms: RIJKS_TERMS, sourceUrl }, } + return { - id: referenceId('rijksmuseum', id), + id: referenceId('rijksmuseum', canonicalUrl), modality: 'image', - title: findTitle(rec), - source: { providerId: 'rijksmuseum', sourceUrl: id }, - canonicalUrl: id, + title: firstLocalized(rec.aggregatedCHO?.title, ['en', 'nl']), + source: { providerId: 'rijksmuseum', sourceUrl }, + canonicalUrl, rights, verifiedAt: new Date().toISOString(), - thumbnail: { url: img.url }, - preview: { url: img.url, mediaType: img.mediaType }, + thumbnail: { url: imageUrl }, + preview: { url: imageUrl, mediaType: 'image/jpeg' }, relevance: 0, raw: rec, } } -interface SearchPage { orderedItems?: Array<{ id?: string }> } +interface SearchPage { + orderedItems?: Array<{ id?: string }> +} export function rijksmuseum(config: RijksmuseumConfig = {}) { return defineProvider({ @@ -177,7 +144,7 @@ export function rijksmuseum(config: RijksmuseumConfig = {}) { const opts = q.providerOptions as RijksmuseumSearchOptions | undefined const n = Math.min(config.maxObjects ?? q.limit ?? 12, 30) const searchUrl = new URL(SEARCH) - // No global free-text param; `title` is a partial keyword match → use it as the keyword. + // The API has no global free-text parameter; title is its partial keyword match. if (q.text) searchUrl.searchParams.set('title', q.text) setIfString(searchUrl, 'type', opts?.type) setIfString(searchUrl, 'material', opts?.material) @@ -185,32 +152,30 @@ export function rijksmuseum(config: RijksmuseumConfig = {}) { setIfString(searchUrl, 'creator', opts?.creator) setIfString(searchUrl, 'description', opts?.description) setIfBoolean(searchUrl, 'imageAvailable', opts?.imageAvailable) - searchUrl.searchParams.set('pageSize', String(n)) // best-effort cap; server caps at 100 const res = await ctx.fetch(searchUrl.toString(), { signal: ctx.signal }) - if (!res.ok) throw new Error(`rijksmuseum search failed: ${res.status}`) + if (!res.ok) throw new Error('rijksmuseum search failed: ' + res.status) const page = (await res.json()) as SearchPage const ids = (page.orderedItems ?? []) - .map(it => it.id) - .filter((u): u is string => typeof u === 'string') + .map(item => item.id) + .filter((url): url is string => typeof url === 'string') .slice(0, n) if (ids.length === 0) return [] - const records = await Promise.all(ids.map(async (idUrl) => { + const records = await Promise.all(ids.map(async (idUrl): Promise => { try { - // Content-negotiate the Linked-Art JSON-LD. id.rijksmuseum.nl 303s to - // data.rijksmuseum.nl; ?_profile=la selects the Linked-Art profile. - const recUrl = `${idUrl}${idUrl.includes('?') ? '&' : '?'}_profile=la` - const r = await ctx.fetch(recUrl, { signal: ctx.signal }) - if (!r.ok) return null - return (await r.json()) as Record + const separator = idUrl.includes('?') ? '&' : '?' + const recordUrl = idUrl + separator + '_profile=edm-framed' + const response = await ctx.fetch(recordUrl, { signal: ctx.signal }) + if (!response.ok) return null + const record = (await response.json()) as EdmRecord + return toReference(record) } catch { - return null // one bad record fetch must not drop the whole batch + return null } })) - return records - .map(rec => (rec ? toReference(rec) : null)) - .filter((r): r is Reference => r !== null) + + return records.filter((record): record is Reference => record !== null) }, }) }