From cf1773d77001a12036dde8c2922e7c9a15a20c58 Mon Sep 17 00:00:00 2001 From: elkaix Date: Mon, 17 Aug 2026 04:16:34 -0400 Subject: [PATCH] feat(core): derive model capabilities in the catalog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `capabilities` was passed straight through from the model alias in the user's config file, and nothing ever derived it. A user who had not hand-written a capability list got nothing, which is almost everyone — so any client rendering capabilities showed an empty result. Derive the list from `getModelCapability` when the alias declares none. An explicit list in the config still wins, including an explicit empty one. A wire type that reports unknown capabilities keeps omitting the field, because omission says "unknown" while an empty list would claim the model can do nothing. `max_context_tokens` and `cost` are not capabilities and are excluded; the context size already has its own field. An unresolvable provider falls back to the previous behaviour rather than throwing, so a model that cannot be classified still appears. --- .changeset/derive-model-capabilities.md | 5 ++ .../src/services/modelCatalog/modelCatalog.ts | 23 +++++- .../modelCatalog/modelCatalogService.ts | 4 +- .../services/model-catalog-service.test.ts | 75 +++++++++++++++++++ .../server/test/model-catalog.e2e.test.ts | 4 + 5 files changed, 108 insertions(+), 3 deletions(-) create mode 100644 .changeset/derive-model-capabilities.md diff --git a/.changeset/derive-model-capabilities.md b/.changeset/derive-model-capabilities.md new file mode 100644 index 00000000..41498147 --- /dev/null +++ b/.changeset/derive-model-capabilities.md @@ -0,0 +1,5 @@ +--- +"@pymodel/pythinker-code": minor +--- + +Report each model's real capabilities in the catalog. Until now `capabilities` carried only what a user had typed into their config file by hand, so for almost every model it was empty. It is now derived from the model itself when the config says nothing, while an explicit list in the config still wins. A provider whose capabilities are genuinely unknown keeps omitting the field rather than claiming the model can do nothing. diff --git a/packages/agent-core/src/services/modelCatalog/modelCatalog.ts b/packages/agent-core/src/services/modelCatalog/modelCatalog.ts index 0524cf62..497725d8 100644 --- a/packages/agent-core/src/services/modelCatalog/modelCatalog.ts +++ b/packages/agent-core/src/services/modelCatalog/modelCatalog.ts @@ -1,3 +1,4 @@ +import { getModelCapability, isUnknownCapability } from '@pymodel/kosong'; import { createDecorator } from '../../di'; import type { PythinkerConfig, ModelAlias, ProviderConfig } from '../../config'; import type { @@ -43,18 +44,38 @@ export class ModelNotFoundError extends Error { export function toProtocolModel( modelId: string, alias: ModelAlias, + provider?: ProviderConfig, ): ModelCatalogItem { return { provider: alias.provider, model: modelId, display_name: alias.displayName ?? alias.model, max_context_size: alias.maxContextSize, - capabilities: alias.capabilities, + capabilities: alias.capabilities ?? derivedCapabilities(alias, provider), support_efforts: alias.supportEfforts, adaptive_thinking: alias.adaptiveThinking, }; } +function derivedCapabilities( + alias: ModelAlias, + provider: ProviderConfig | undefined, +): string[] | undefined { + if (provider === undefined) return undefined; + try { + const capability = getModelCapability(provider.type, alias.model); + if (isUnknownCapability(capability)) return undefined; + return Object.entries(capability) + .filter( + ([key, value]) => + value === true && key !== 'max_context_tokens' && key !== 'cost', + ) + .map(([key]) => key); + } catch { + return undefined; + } +} + export interface ProviderCredentialState { readonly hasApiKey: boolean; readonly hasOAuthToken: boolean; diff --git a/packages/agent-core/src/services/modelCatalog/modelCatalogService.ts b/packages/agent-core/src/services/modelCatalog/modelCatalogService.ts index 0e3513f1..ad0849bc 100644 --- a/packages/agent-core/src/services/modelCatalog/modelCatalogService.ts +++ b/packages/agent-core/src/services/modelCatalog/modelCatalogService.ts @@ -30,7 +30,7 @@ export class ModelCatalogService async listModels(): Promise { const config = await this._readConfig(); return Object.entries(config.models ?? {}).map(([modelId, alias]) => - toProtocolModel(modelId, alias), + toProtocolModel(modelId, alias, config.providers[alias.provider]), ); } @@ -63,7 +63,7 @@ export class ModelCatalogService const updatedAlias = updated.models?.[modelId] ?? alias; return { default_model: modelId, - model: toProtocolModel(modelId, updatedAlias), + model: toProtocolModel(modelId, updatedAlias, updated.providers[updatedAlias.provider]), }; } diff --git a/packages/agent-core/test/services/model-catalog-service.test.ts b/packages/agent-core/test/services/model-catalog-service.test.ts index 8534aae5..71e8fd48 100644 --- a/packages/agent-core/test/services/model-catalog-service.test.ts +++ b/packages/agent-core/test/services/model-catalog-service.test.ts @@ -1,4 +1,5 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; +import { getModelCapability } from '@pymodel/kosong'; import type { CoreRPC, @@ -116,6 +117,80 @@ function catalogConfig(): PythinkerConfig { } describe('model catalog adapters', () => { + it('derives capabilities from the configured provider wire type', async () => { + const config = catalogConfig(); + const alias = { + provider: 'openai', + model: 'gpt-5.4', + maxContextSize: 200000, + }; + config.models = { gpt54: alias }; + const { core } = makeCore({ current: config }); + + const [model] = await new ModelCatalogService(core).listModels(); + const detected = getModelCapability('openai', alias.model); + expect(model?.capabilities).toEqual( + Object.entries(detected) + .filter(([, value]) => value === true) + .map(([key]) => key), + ); + }); + + it('keeps an explicit capability list exactly', () => { + const config = catalogConfig(); + const alias = { + ...config.models!['gpt4o']!, + capabilities: ['custom_capability', 'always_thinking'], + }; + + expect(toProtocolModel('gpt4o', alias, config.providers['openai']).capabilities).toEqual( + alias.capabilities, + ); + }); + + it('emits only true capability flags, excluding context and cost metadata', () => { + const config = catalogConfig(); + const alias = config.models!['gpt4o']!; + const capabilities = toProtocolModel('gpt4o', alias, config.providers['openai']).capabilities; + + expect(capabilities).toEqual(['image_in', 'tool_use']); + expect(capabilities).not.toContain('video_in'); + expect(capabilities).not.toContain('audio_in'); + expect(capabilities).not.toContain('thinking'); + expect(capabilities).not.toContain('max_context_tokens'); + expect(capabilities).not.toContain('cost'); + }); + + it('omits capabilities when the provider reports unknown capability data', () => { + const config = catalogConfig(); + const alias = { ...config.models!['turbo']!, capabilities: undefined }; + + expect(toProtocolModel('turbo', alias, config.providers['pythinker']).capabilities).toBeUndefined(); + }); + + it('keeps a model entry when its provider cannot be resolved', async () => { + const config: PythinkerConfig = { + providers: {}, + models: { + orphan: { + provider: 'missing', + model: 'gpt-4o', + maxContextSize: 128000, + }, + }, + }; + const { core } = makeCore({ current: config }); + + await expect(new ModelCatalogService(core).listModels()).resolves.toMatchObject([ + { + provider: 'missing', + model: 'orphan', + max_context_size: 128000, + capabilities: undefined, + }, + ]); + }); + it('maps model aliases to selectable wire ids', () => { const alias = catalogConfig().models!['k2']!; expect(toProtocolModel('k2', alias)).toEqual({ diff --git a/packages/server/test/model-catalog.e2e.test.ts b/packages/server/test/model-catalog.e2e.test.ts index e183e44f..d2b364e0 100644 --- a/packages/server/test/model-catalog.e2e.test.ts +++ b/packages/server/test/model-catalog.e2e.test.ts @@ -136,6 +136,10 @@ describe('model/provider catalog routes', () => { model: 'gpt4o', display_name: 'gpt-4o', max_context_size: 128000, + // Declares no capabilities in config, so they are derived from the + // model itself. `k2` above keeps the list its config states, and + // `turbo` omits the field because the pythinker wire reports unknown. + capabilities: ['image_in', 'tool_use'], }, ]); });