Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions cmd/root/eval.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (

"github.com/docker/docker-agent/pkg/config"
"github.com/docker/docker-agent/pkg/evaluation"
"github.com/docker/docker-agent/pkg/model/provider/providers"
"github.com/docker/docker-agent/pkg/telemetry"
)

Expand Down Expand Up @@ -117,6 +118,10 @@ func (f *evalFlags) runEvalCommand(cmd *cobra.Command, args []string) (commandEr
f.AgentFilename = agentFilename
f.EvalsDir = evalsDir

// Wire the full provider set so the judge model can be built (the package
// default registry is empty; see pkg/model/provider/providers).
f.runConfig.ProviderRegistry = providers.NewDefaultRegistry()

// Run evaluation
// Pass consoleOut for TTY progress bar, teeOut for results that should go to both console and log
run, evalErr := evaluation.Evaluate(ctx, consoleOut, teeOut, isTTY, runName, &f.runConfig, f.Config)
Expand Down
19 changes: 19 additions & 0 deletions pkg/config/runtime.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,12 @@ type RuntimeConfig struct {
modelsDevStore *modelsdev.Store
modelsDevStoreErr error
modelsDevStoreOnce sync.Once

// ProviderRegistry instantiates model providers for toolsets that build
// providers at load time (e.g. RAG embeddings/reranking). It is populated
// by the team loader with the same registry used for agent models. When
// nil, ProviderRegistryOrDefault falls back to provider.DefaultRegistry.
ProviderRegistry *provider.Registry
}

type Config struct {
Expand Down Expand Up @@ -78,6 +84,7 @@ func (runConfig *RuntimeConfig) Clone() *RuntimeConfig {
ModelsDevStoreOverride: runConfig.ModelsDevStoreOverride,
modelsDevStore: store,
modelsDevStoreErr: storeErr,
ProviderRegistry: runConfig.ProviderRegistry,
}
clone.envProviderOnce.Do(func() {}) // mark as resolved
clone.modelsDevStoreOnce.Do(func() {}) // mark as resolved
Expand Down Expand Up @@ -109,6 +116,18 @@ func (runConfig *RuntimeConfig) ModelsDevStore() (*modelsdev.Store, error) {
return runConfig.modelsDevStore, runConfig.modelsDevStoreErr
}

// ProviderRegistryOrDefault returns the configured provider registry, or the
// package default registry when none was set (including when the receiver is
// nil). The default registry only contains providers the core package can
// expose without optional SDK dependencies, so callers that need the full
// provider set must ensure the team loader populated ProviderRegistry.
func (runConfig *RuntimeConfig) ProviderRegistryOrDefault() *provider.Registry {
if runConfig != nil && runConfig.ProviderRegistry != nil {
return runConfig.ProviderRegistry
}
return provider.DefaultRegistry()
}

func (runConfig *RuntimeConfig) EnvProvider() environment.Provider {
if runConfig.EnvProviderForTests != nil {
return runConfig.EnvProviderForTests
Expand Down
2 changes: 1 addition & 1 deletion pkg/evaluation/eval.go
Original file line number Diff line number Diff line change
Expand Up @@ -667,7 +667,7 @@ func createJudgeModel(ctx context.Context, judgeModel string, runConfig *config.
opts = append(opts, options.WithGateway(runConfig.ModelsGateway))
}

judge, err := provider.New(ctx, &cfg, runConfig.EnvProvider(), opts...)
judge, err := runConfig.ProviderRegistryOrDefault().New(ctx, &cfg, runConfig.EnvProvider(), opts...)
if err != nil {
return nil, fmt.Errorf("creating judge model: %w", err)
}
Expand Down
31 changes: 30 additions & 1 deletion pkg/model/provider/factory_js.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@ import (

"github.com/docker/docker-agent/pkg/config/latest"
"github.com/docker/docker-agent/pkg/environment"
"github.com/docker/docker-agent/pkg/model/provider/anthropic"
"github.com/docker/docker-agent/pkg/model/provider/gemini"
"github.com/docker/docker-agent/pkg/model/provider/openai"
"github.com/docker/docker-agent/pkg/model/provider/options"
"github.com/docker/docker-agent/pkg/model/provider/rulebased"
)
Expand All @@ -26,10 +29,36 @@ func NewRegistry(factories map[string]Factory) *Registry {
return &Registry{factories: copied}
}

var defaultFactories map[string]Factory
// defaultFactories is the js/wasm provider set. dmr (os/exec), amazon-bedrock
// and vertex AI (cloud SDKs that don't compile to wasm) are deliberately
// absent; the remaining providers reach their APIs over plain net/http, which
// the Go runtime maps to fetch in the browser. Unlike the non-js build (whose
// DefaultRegistry is empty so applications must wire providers explicitly via
// pkg/model/provider/providers), the wasm build has no such wiring point —
// pkg/model/provider/providers pulls in the cloud SDKs — so the slim set is
// registered here.
var defaultFactories = map[string]Factory{
"openai": openaiFactory,
"openai_chatcompletions": openaiFactory,
"openai_responses": openaiFactory,
"anthropic": anthropicFactory,
"google": googleFactory,
}

func DefaultRegistry() *Registry { return NewRegistry(defaultFactories) }

func openaiFactory(ctx context.Context, cfg *latest.ModelConfig, env environment.Provider, opts ...options.Opt) (Provider, error) {
return openai.NewClient(ctx, cfg, env, opts...)
}

func anthropicFactory(ctx context.Context, cfg *latest.ModelConfig, env environment.Provider, opts ...options.Opt) (Provider, error) {
return anthropic.NewClient(ctx, cfg, env, opts...)
}

func googleFactory(ctx context.Context, cfg *latest.ModelConfig, env environment.Provider, opts ...options.Opt) (Provider, error) {
return gemini.NewClient(ctx, cfg, env, opts...)
}

func (r *Registry) New(ctx context.Context, cfg *latest.ModelConfig, env environment.Provider, opts ...options.Opt) (Provider, error) {
return r.NewWithModels(ctx, cfg, nil, env, opts...)
}
Expand Down
6 changes: 4 additions & 2 deletions pkg/rag/builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,11 @@ type ManagersBuildConfig struct {
}

// NewProvider creates a model provider using the build config's environment,
// gateway, and custom provider settings.
// gateway, and custom provider settings. It uses the provider registry carried
// by RuntimeConfig (populated by the team loader with the full provider set);
// without it, model creation fails with "unknown provider type".
func (c ManagersBuildConfig) NewProvider(ctx context.Context, cfg *latest.ModelConfig) (provider.Provider, error) {
return provider.New(ctx, cfg, c.Env,
return c.RuntimeConfig.ProviderRegistryOrDefault().New(ctx, cfg, c.Env,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[MEDIUM/LIKELY] Silent fallback to empty provider registry when RuntimeConfig.ProviderRegistry is unset

c.RuntimeConfig.ProviderRegistryOrDefault() falls back to provider.DefaultRegistry() when RuntimeConfig is nil or its ProviderRegistry field is not set. In non-js builds, provider.DefaultRegistry() returns an empty registry (no factories registered), so any call to NewProvider in that state will produce "unknown provider type" errors — exactly the runtime failure this PR is fixing elsewhere.

All current callers go through loaderdefaults.Opts() (which injects providers.NewDefaultRegistry()), so the happy path is fine. However, the fallback provider.DefaultRegistry() returns an empty registry in non-js builds, so any future call site that constructs ManagersBuildConfig without going through teamloader.LoadWithConfig + loaderdefaults.Opts() will fail silently.

Consider logging a warning or returning an explicit error if RuntimeConfig is nil, rather than silently falling back to an unusable registry.

options.WithGateway(c.ModelsGateway),
options.WithProviders(c.Providers))
}
Expand Down
6 changes: 4 additions & 2 deletions pkg/rag/strategy/strategy.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,11 @@ type BuildContext struct {
}

// NewProvider creates a model provider using the build context's environment,
// gateway, and custom provider settings.
// gateway, and custom provider settings. It uses the provider registry carried
// by RuntimeConfig (populated by the team loader with the full provider set);
// without it, model creation fails with "unknown provider type".
func (c BuildContext) NewProvider(ctx context.Context, cfg *latest.ModelConfig) (provider.Provider, error) {
return provider.New(ctx, cfg, c.Env,
return c.RuntimeConfig.ProviderRegistryOrDefault().New(ctx, cfg, c.Env,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[MEDIUM/LIKELY] Same silent empty-registry fallback as in pkg/rag/builder.go

c.RuntimeConfig.ProviderRegistryOrDefault() falls back to provider.DefaultRegistry() (empty in non-js builds) when RuntimeConfig is nil or ProviderRegistry is unset. The same analysis applies as the parallel issue in pkg/rag/builder.go: all current production paths are safe because all callers go through loaderdefaults.Opts(), but any future caller that constructs BuildContext without a populated RuntimeConfig will silently receive a broken provider registry and fail at model creation time with "unknown provider type".

options.WithGateway(c.ModelsGateway),
options.WithProviders(c.Providers))
}
Expand Down
3 changes: 3 additions & 0 deletions pkg/teamloader/teamloader.go
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,9 @@ func LoadWithConfig(ctx context.Context, agentSource config.Source, runConfig *c
// Make model definitions available to toolset creators (e.g., RAG reranking)
runConfig.Models = cfg.Models
runConfig.Providers = cfg.Providers
// Share the resolved provider registry so toolsets that build providers at
// load time (e.g. RAG embeddings/reranking) use the same one as agent models.
runConfig.ProviderRegistry = loadOpts.providerRegistry

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[MEDIUM/LIKELY] loadOpts.providerRegistry may be empty when WithProviderRegistry is not passed; callers get a non-nil but unusable registry

loadOpts.providerRegistry is initialized to provider.DefaultRegistry() (an empty registry in non-js builds) before opts are applied. runConfig.ProviderRegistry is then set to this value. Because ProviderRegistryOrDefault() checks only != nil, a caller that omits WithProviderRegistry(...) will silently receive an empty (non-nil) registry and fail at RAG model creation time with "unknown provider type" — the same failure this PR intends to fix.

All current callers pass loaderdefaults.Opts() (which includes WithProviderRegistry(providers.NewDefaultRegistry())), so this is safe today. However the default is a footgun: consider initializing loadOpts.providerRegistry to nil instead of provider.DefaultRegistry(), so that ProviderRegistryOrDefault() can provide a meaningful fallback, or enforce that callers must always pass WithProviderRegistry.


// Load agents
parentDir := cmp.Or(agentSource.ParentDir(), runConfig.WorkingDir)
Expand Down
Loading