From fd2b9b60a57f259dc7aba1769c2d87bc3dd78e13 Mon Sep 17 00:00:00 2001 From: Will Winder Date: Fri, 29 May 2026 23:15:28 -0400 Subject: [PATCH 1/8] WIP --- build/devenv/components/clnode/component.go | 191 ++++++++++++++++++++ build/devenv/env-cl-phased.toml | 41 +++++ build/devenv/environment.go | 1 + 3 files changed, 233 insertions(+) create mode 100644 build/devenv/components/clnode/component.go create mode 100644 build/devenv/env-cl-phased.toml diff --git a/build/devenv/components/clnode/component.go b/build/devenv/components/clnode/component.go new file mode 100644 index 000000000..4d1a98568 --- /dev/null +++ b/build/devenv/components/clnode/component.go @@ -0,0 +1,191 @@ +// Package clnode is the phased-devenv component for Chainlink-node ("CL node") +// support (issue 16). +// +// NOTE: This is a STEP-1 THROWAWAY PROTOTYPE. It launches + funds CL nodes +// directly in Phase 2 purely to prove that nodeset launch works inside the +// phased runtime. The agreed final design is different: clnode becomes a +// config-vehicle component that only decodes its config and publishes it as +// the "_clnode" output key, and the committeeccv component performs the +// secrets-baked launch (CL node secrets are boot-only, so the aggregator HMAC +// creds must be injected before launch, which only committeeccv has). Do not +// build on the launch logic here without revisiting that plan. +package clnode + +import ( + "context" + "fmt" + "math/big" + "strings" + + "github.com/pelletier/go-toml/v2" + + "github.com/smartcontractkit/chainlink-ccv/build/devenv/cciptestinterfaces" + "github.com/smartcontractkit/chainlink-ccv/build/devenv/chainreg" + devenvruntime "github.com/smartcontractkit/chainlink-ccv/build/devenv/runtime" + ctfblockchain "github.com/smartcontractkit/chainlink-testing-framework/framework/components/blockchain" + ns "github.com/smartcontractkit/chainlink-testing-framework/framework/components/simple_node_set" +) + +const configKey = "clnode" + +// Version is the clnode component config schema version. Exactly this version +// is supported; configs declaring any other version are rejected. +const Version = 1 + +// commonCLNodesConfig is a prototype-local copy of ccv.CommonCLNodesConfig. +// The ccv package blank-imports every component, so a component cannot import +// it back without a cycle; duplication is acceptable for the throwaway. +const commonCLNodesConfig = ` +[Log] +JSONConsole = true +Level = 'info' +[Pyroscope] +ServerAddress = 'http://host.docker.internal:4040' +Environment = 'local' +[WebServer] +SessionTimeout = '999h0m0s' +HTTPWriteTimeout = '3m' +SecureCookies = false +HTTPPort = 6688 +AllowOrigins = 'http://localhost:3000' +[WebServer.TLS] +HTTPSPort = 0 +[WebServer.RateLimit] +Authenticated = 5000 +Unauthenticated = 5000 +[JobPipeline] +[JobPipeline.HTTPRequest] +DefaultTimeout = '1m' +[Log.File] +MaxSize = '0b' +[Feature] +FeedsManager = true +LogPoller = true +UICSAKeys = true +[OCR2] +Enabled = true +SimulateTransactions = false +DefaultTransactionQueueDepth = 1 +[P2P.V2] +Enabled = true +ListenAddresses = ['0.0.0.0:6690'] +` + +func init() { + if err := devenvruntime.Register(configKey, factory); err != nil { + panic(fmt.Sprintf("clnode component: %v", err)) + } +} + +func factory(_ map[string]any) (devenvruntime.Component, error) { + return &component{}, nil +} + +type component struct{} + +// Config is the versioned wrapper around the CL node set definitions. Adding a +// version field is why this is a [clnode] table rather than a bare top-level +// [[nodesets]] array. +type Config struct { + Version int `toml:"version"` + CLNodesFundingETH float64 `toml:"cl_nodes_funding_eth"` + CLNodesFundingLink float64 `toml:"cl_nodes_funding_link"` + NodeSets []*ns.Input `toml:"node_sets"` +} + +// decodeConfig round-trips the raw TOML component config into a typed Config +// and verifies its declared version. +func decodeConfig(raw any) (Config, error) { + b, err := toml.Marshal(raw) + if err != nil { + return Config{}, fmt.Errorf("re-encoding clnode config: %w", err) + } + var cfg Config + if err := toml.Unmarshal(b, &cfg); err != nil { + return Config{}, fmt.Errorf("decoding clnode config: %w", err) + } + if err := devenvruntime.CheckConfigVersion(cfg.Version, Version); err != nil { + return Config{}, err + } + return cfg, nil +} + +func (c *component) ValidateConfig(componentConfig any) error { + _, err := decodeConfig(componentConfig) + return err +} + +// RunPhase2 (THROWAWAY) launches and funds the configured CL node sets. It runs +// in Phase 2 because it needs the Phase-1 "blockchains" output for chain config +// and funding. It produces no outputs and emits no effects yet. +func (c *component) RunPhase2( + ctx context.Context, + _ map[string]any, + componentConfig any, + priorOutputs map[string]any, +) (map[string]any, []devenvruntime.Effect, error) { + cfg, err := decodeConfig(componentConfig) + if err != nil { + return nil, nil, err + } + if len(cfg.NodeSets) == 0 { + return map[string]any{}, nil, nil + } + + blockchains, ok := priorOutputs["blockchains"].([]*ctfblockchain.Input) + if !ok || len(blockchains) == 0 { + return nil, nil, fmt.Errorf("clnode: blockchains not found in phase outputs") + } + + impls := make([]cciptestinterfaces.CCIP17Configuration, 0, len(blockchains)) + for _, bc := range blockchains { + impl, ierr := chainreg.NewProductConfigurationFromNetwork(bc.Type) + if ierr != nil { + return nil, nil, fmt.Errorf("clnode: impl for %q: %w", bc.Type, ierr) + } + impls = append(impls, impl) + } + + // Assemble the CL node chain-config overrides from each chain impl. + chainConfigs := []string{commonCLNodesConfig} + for i, impl := range impls { + cc, cerr := impl.ConfigureNodes(ctx, blockchains[i]) + if cerr != nil { + return nil, nil, fmt.Errorf("clnode: ConfigureNodes for %q: %w", blockchains[i].Type, cerr) + } + chainConfigs = append(chainConfigs, cc) + } + allConfigs := strings.Join(chainConfigs, "\n") + for _, nodeSet := range cfg.NodeSets { + for _, nodeSpec := range nodeSet.NodeSpecs { + nodeSpec.Node.TestConfigOverrides = allConfigs + } + } + + // Launch each node set (shared DB per set). + for _, nodeSet := range cfg.NodeSets { + if _, nerr := ns.NewSharedDBNodeSet(nodeSet, nil); nerr != nil { + return nil, nil, fmt.Errorf("clnode: NewSharedDBNodeSet %q: %w", nodeSet.Name, nerr) + } + } + + // Fund the nodes on every fundable chain. FundNodes takes (link, native). + link := toBigUnits(cfg.CLNodesFundingLink, 1) + native := toBigUnits(cfg.CLNodesFundingETH, 5) + for i, impl := range impls { + if ferr := impl.FundNodes(ctx, cfg.NodeSets, blockchains[i], link, native); ferr != nil { + return nil, nil, fmt.Errorf("clnode: FundNodes on %q: %w", blockchains[i].Type, ferr) + } + } + + return map[string]any{}, nil, nil +} + +// toBigUnits converts a whole-unit funding amount to *big.Int, falling back to +// def when the configured value is non-positive. +func toBigUnits(v float64, def int64) *big.Int { + if v <= 0 { + return big.NewInt(def) + } + return big.NewInt(int64(v)) +} diff --git a/build/devenv/env-cl-phased.toml b/build/devenv/env-cl-phased.toml new file mode 100644 index 000000000..b6d1c10e4 --- /dev/null +++ b/build/devenv/env-cl-phased.toml @@ -0,0 +1,41 @@ +# Phased-runtime CL-node overlay (issue 16). +# +# Layer this on top of env-phased.toml: +# ccv --env-mode phased up env-phased.toml,env-cl-phased.toml +# +# It is a phased-shaped copy of env-cl.toml's CL bits. STEP 1 scope: only the +# [clnode] section, which the clnode component claims (so the issue-03 +# unclaimed-key check passes) and, in the throwaway prototype, uses to launch + +# fund the node sets. CL-mode verifier/executor and topology overrides come in +# later steps. + +[clnode] +version = 1 +cl_nodes_funding_eth = 50 +cl_nodes_funding_link = 50 + +[[clnode.node_sets]] + name = "don" + nodes = 2 + override_mode = "each" + + [clnode.node_sets.db] + image = "postgres:15.0" + + [[clnode.node_sets.node_specs]] + + [clnode.node_sets.node_specs.node] + docker_ctx = "../../../chainlink" + docker_file = "plugins/chainlink.Dockerfile" + + [clnode.node_sets.node_specs.node.env_vars] + CL_EVM_CMD = "" + + [[clnode.node_sets.node_specs]] + + [clnode.node_sets.node_specs.node] + docker_ctx = "../../../chainlink" + docker_file = "plugins/chainlink.Dockerfile" + + [clnode.node_sets.node_specs.node.env_vars] + CL_EVM_CMD = "" diff --git a/build/devenv/environment.go b/build/devenv/environment.go index c1757ba99..b3fdca339 100644 --- a/build/devenv/environment.go +++ b/build/devenv/environment.go @@ -27,6 +27,7 @@ import ( ccldf "github.com/smartcontractkit/chainlink-ccv/build/devenv/cldf" devenvcommon "github.com/smartcontractkit/chainlink-ccv/build/devenv/common" _ "github.com/smartcontractkit/chainlink-ccv/build/devenv/components/blockchains" + _ "github.com/smartcontractkit/chainlink-ccv/build/devenv/components/clnode" _ "github.com/smartcontractkit/chainlink-ccv/build/devenv/components/committeeccv" _ "github.com/smartcontractkit/chainlink-ccv/build/devenv/components/executor" _ "github.com/smartcontractkit/chainlink-ccv/build/devenv/components/fake" From f7d89e6a8d9f1e527ddec7284d85da6d1a6402bc Mon Sep 17 00:00:00 2001 From: Will Winder Date: Sat, 30 May 2026 10:43:03 -0400 Subject: [PATCH 2/8] committeeccv launches the CL node --- build/devenv/components/clnode/component.go | 73 +++++++++++-------- .../components/committeeccv/component.go | 13 ++++ 2 files changed, 55 insertions(+), 31 deletions(-) diff --git a/build/devenv/components/clnode/component.go b/build/devenv/components/clnode/component.go index 4d1a98568..251b3e438 100644 --- a/build/devenv/components/clnode/component.go +++ b/build/devenv/components/clnode/component.go @@ -1,14 +1,15 @@ // Package clnode is the phased-devenv component for Chainlink-node ("CL node") // support (issue 16). // -// NOTE: This is a STEP-1 THROWAWAY PROTOTYPE. It launches + funds CL nodes -// directly in Phase 2 purely to prove that nodeset launch works inside the -// phased runtime. The agreed final design is different: clnode becomes a -// config-vehicle component that only decodes its config and publishes it as -// the "_clnode" output key, and the committeeccv component performs the -// secrets-baked launch (CL node secrets are boot-only, so the aggregator HMAC -// creds must be injected before launch, which only committeeccv has). Do not -// build on the launch logic here without revisiting that plan. +// The component itself is a config vehicle: it claims the top-level [clnode] +// config key, decodes the versioned config, and publishes it as the +// runtime-only output key "_clnode". It does NOT launch anything. +// +// The committeeccv component (Phase 3) consumes "_clnode" and drives the +// actual launch via LaunchNodeSets, because CL node secrets are boot-only and +// the aggregator HMAC credentials that must be baked into the node spec before +// launch are owned by committeeccv. Keeping the launch helper here keeps the +// node-launch code beside its config while letting committeeccv sequence it. package clnode import ( @@ -28,14 +29,18 @@ import ( const configKey = "clnode" +// OutputKey is the runtime-only output key under which the decoded clnode +// config is published for committeeccv to consume. +const OutputKey = "_clnode" + // Version is the clnode component config schema version. Exactly this version // is supported; configs declaring any other version are rejected. const Version = 1 -// commonCLNodesConfig is a prototype-local copy of ccv.CommonCLNodesConfig. -// The ccv package blank-imports every component, so a component cannot import -// it back without a cycle; duplication is acceptable for the throwaway. -const commonCLNodesConfig = ` +// CommonCLNodesConfig is the base TOML config applied to every CL node. It is a +// copy of ccv.CommonCLNodesConfig; the ccv package blank-imports every +// component, so a component cannot import it back without a cycle. +const CommonCLNodesConfig = ` [Log] JSONConsole = true Level = 'info' @@ -115,43 +120,50 @@ func (c *component) ValidateConfig(componentConfig any) error { return err } -// RunPhase2 (THROWAWAY) launches and funds the configured CL node sets. It runs -// in Phase 2 because it needs the Phase-1 "blockchains" output for chain config -// and funding. It produces no outputs and emits no effects yet. -func (c *component) RunPhase2( - ctx context.Context, +// RunPhase1 decodes the clnode config and publishes it under OutputKey for +// committeeccv (Phase 3) to consume. It launches nothing. +func (c *component) RunPhase1( + _ context.Context, _ map[string]any, componentConfig any, - priorOutputs map[string]any, ) (map[string]any, []devenvruntime.Effect, error) { cfg, err := decodeConfig(componentConfig) if err != nil { return nil, nil, err } - if len(cfg.NodeSets) == 0 { - return map[string]any{}, nil, nil - } + out := cfg + return map[string]any{OutputKey: &out}, nil, nil +} - blockchains, ok := priorOutputs["blockchains"].([]*ctfblockchain.Input) - if !ok || len(blockchains) == 0 { - return nil, nil, fmt.Errorf("clnode: blockchains not found in phase outputs") +// LaunchNodeSets configures, launches, and funds the CL node sets in cfg using +// the chains in blockchains. It is called by committeeccv during Phase 3. +// +// NOTE: secret injection (TestSecretsOverrides) must happen on the node specs +// before this is called, because CL node secrets are boot-only. Step 1 launches +// without secrets; committeeccv will bake them in a later step. +func LaunchNodeSets(ctx context.Context, cfg *Config, blockchains []*ctfblockchain.Input) error { + if cfg == nil || len(cfg.NodeSets) == 0 { + return nil + } + if len(blockchains) == 0 { + return fmt.Errorf("clnode: no blockchains available to configure CL nodes") } impls := make([]cciptestinterfaces.CCIP17Configuration, 0, len(blockchains)) for _, bc := range blockchains { impl, ierr := chainreg.NewProductConfigurationFromNetwork(bc.Type) if ierr != nil { - return nil, nil, fmt.Errorf("clnode: impl for %q: %w", bc.Type, ierr) + return fmt.Errorf("clnode: impl for %q: %w", bc.Type, ierr) } impls = append(impls, impl) } // Assemble the CL node chain-config overrides from each chain impl. - chainConfigs := []string{commonCLNodesConfig} + chainConfigs := []string{CommonCLNodesConfig} for i, impl := range impls { cc, cerr := impl.ConfigureNodes(ctx, blockchains[i]) if cerr != nil { - return nil, nil, fmt.Errorf("clnode: ConfigureNodes for %q: %w", blockchains[i].Type, cerr) + return fmt.Errorf("clnode: ConfigureNodes for %q: %w", blockchains[i].Type, cerr) } chainConfigs = append(chainConfigs, cc) } @@ -165,7 +177,7 @@ func (c *component) RunPhase2( // Launch each node set (shared DB per set). for _, nodeSet := range cfg.NodeSets { if _, nerr := ns.NewSharedDBNodeSet(nodeSet, nil); nerr != nil { - return nil, nil, fmt.Errorf("clnode: NewSharedDBNodeSet %q: %w", nodeSet.Name, nerr) + return fmt.Errorf("clnode: NewSharedDBNodeSet %q: %w", nodeSet.Name, nerr) } } @@ -174,11 +186,10 @@ func (c *component) RunPhase2( native := toBigUnits(cfg.CLNodesFundingETH, 5) for i, impl := range impls { if ferr := impl.FundNodes(ctx, cfg.NodeSets, blockchains[i], link, native); ferr != nil { - return nil, nil, fmt.Errorf("clnode: FundNodes on %q: %w", blockchains[i].Type, ferr) + return fmt.Errorf("clnode: FundNodes on %q: %w", blockchains[i].Type, ferr) } } - - return map[string]any{}, nil, nil + return nil } // toBigUnits converts a whole-unit funding amount to *big.Int, falling back to diff --git a/build/devenv/components/committeeccv/component.go b/build/devenv/components/committeeccv/component.go index 5ffaa690b..29a5f66ca 100644 --- a/build/devenv/components/committeeccv/component.go +++ b/build/devenv/components/committeeccv/component.go @@ -12,6 +12,7 @@ import ( "github.com/smartcontractkit/chainlink-ccv/build/devenv/chainreg" devenvcommon "github.com/smartcontractkit/chainlink-ccv/build/devenv/common" blockchainscomp "github.com/smartcontractkit/chainlink-ccv/build/devenv/components/blockchains" + clnodecomp "github.com/smartcontractkit/chainlink-ccv/build/devenv/components/clnode" "github.com/smartcontractkit/chainlink-ccv/build/devenv/components/observability" ccdeploy "github.com/smartcontractkit/chainlink-ccv/build/devenv/deploy" "github.com/smartcontractkit/chainlink-ccv/build/devenv/jobs" @@ -126,6 +127,18 @@ func (c *component) RunPhase3( agg.Out.ClientCredentials = creds } + // Step 1b: Launch CL node sets (issue 16). The clnode component (Phase 1) + // publishes its decoded config under clnodecomp.OutputKey; committeeccv drives + // the launch because CL node secrets are boot-only and the aggregator HMAC + // creds (generated in Step 1) must be baked in before launch. CL nodes must be + // up + registered with JD before the lane/committee config below fetches their + // signing keys from JD. Secret injection + JD registration land in later steps. + if clnodeCfg, ok := priorOutputs[clnodecomp.OutputKey].(*clnodecomp.Config); ok && clnodeCfg != nil { + if err := clnodecomp.LaunchNodeSets(ctx, clnodeCfg, blockchains); err != nil { + return nil, nil, fmt.Errorf("committeeccv: launching CL node sets: %w", err) + } + } + // Step 2: Launch standalone verifier containers (reads HMAC creds from agg.Out). if err := committeeverifier.LaunchStandaloneVerifiers( verifiers, aggregators, blockchainOutputs, jdInfra, From a976b256e9d6ba14f4089d94c3dfeeb3fe9292b1 Mon Sep 17 00:00:00 2001 From: Will Winder Date: Sat, 30 May 2026 15:33:09 -0400 Subject: [PATCH 3/8] build/devenv: register phased CL nodes with JD before lane config committeeccv now registers and connects the CL node sets to JD before the lane/committee config, then populates NodeIDs on a local copy of the environment so ApplyVerifierConfig can fetch CL-mode signing keys from JD. The shared _env output is left untouched (mirrors tokenverifier). env-cl-phased.toml runs the verifiers in CL mode across two shared nodes; executors stay standalone for now. --- .../components/committeeccv/component.go | 75 +++++-- build/devenv/env-cl-phased.toml | 188 +++++++++++++++++- 2 files changed, 240 insertions(+), 23 deletions(-) diff --git a/build/devenv/components/committeeccv/component.go b/build/devenv/components/committeeccv/component.go index 29a5f66ca..e195b3ac4 100644 --- a/build/devenv/components/committeeccv/component.go +++ b/build/devenv/components/committeeccv/component.go @@ -94,6 +94,10 @@ func (c *component) RunPhase3( if !ok || e == nil { return nil, nil, fmt.Errorf("committeeccv: _env not found in phase outputs") } + // Work on a copy of the environment: the shared _env is a Phase-2 output and + // must be treated as immutable. CL-node NodeIDs and per-step DataStore updates + // are applied to this local copy only. (Mirrors the tokenverifier component.) + localEnv := *e topology, ok := priorOutputs["environment_topology"].(*ccvdeployment.EnvironmentTopology) if !ok || topology == nil { return nil, nil, fmt.Errorf("committeeccv: environment_topology not found in phase outputs") @@ -127,18 +131,44 @@ func (c *component) RunPhase3( agg.Out.ClientCredentials = creds } - // Step 1b: Launch CL node sets (issue 16). The clnode component (Phase 1) - // publishes its decoded config under clnodecomp.OutputKey; committeeccv drives - // the launch because CL node secrets are boot-only and the aggregator HMAC - // creds (generated in Step 1) must be baked in before launch. CL nodes must be - // up + registered with JD before the lane/committee config below fetches their - // signing keys from JD. Secret injection + JD registration land in later steps. - if clnodeCfg, ok := priorOutputs[clnodecomp.OutputKey].(*clnodecomp.Config); ok && clnodeCfg != nil { + // Step 1b: Launch CL node sets and register them with JD (issue 16). The + // clnode component (Phase 1) publishes its decoded config under + // clnodecomp.OutputKey; committeeccv drives the launch because CL node secrets + // are boot-only and the aggregator HMAC creds (generated in Step 1) must be + // baked in before launch. CL nodes must be launched + registered + connected to + // JD before the lane/committee config below, because ApplyVerifierConfig fetches + // CL-mode signing keys from JD. (Secret injection lands in a later step.) + var clNodeClients *jobs.NodeSetClientLookup + if clnodeCfg, ok := priorOutputs[clnodecomp.OutputKey].(*clnodecomp.Config); ok && clnodeCfg != nil && len(clnodeCfg.NodeSets) > 0 { if err := clnodecomp.LaunchNodeSets(ctx, clnodeCfg, blockchains); err != nil { return nil, nil, fmt.Errorf("committeeccv: launching CL node sets: %w", err) } - } + clNopAliases := clModeNOPAliases(topology) + if len(clNopAliases) > 0 { + clNodeClients, err = jobs.NewNodeSetClientLookup(clnodeCfg.NodeSets, clNopAliases) + if err != nil { + return nil, nil, fmt.Errorf("committeeccv: building CL node client lookup: %w", err) + } + if clNodeClients != nil { + if err := jobs.RegisterNodesWithJD(ctx, jdInfra, clNodeClients, clNopAliases); err != nil { + return nil, nil, fmt.Errorf("committeeccv: registering CL nodes with JD: %w", err) + } + chainIDs := make([]string, len(blockchains)) + for i, bc := range blockchains { + chainIDs[i] = bc.ChainID + } + if err := jobs.ConnectNodesToJD(ctx, jdInfra, clNodeClients, chainIDs); err != nil { + return nil, nil, fmt.Errorf("committeeccv: connecting CL nodes to JD: %w", err) + } + // The deployment environment was built in Phase 2 (protocol_contracts) + // before these CL nodes registered, so its NodeIDs are empty. Populate + // the local copy so the lane/committee config below can fetch CL-mode + // signing keys from JD (FetchNOPSigningKeys requires NodeIDs). + localEnv.NodeIDs = jdInfra.GetNodeIDs() + } + } + } // Step 2: Launch standalone verifier containers (reads HMAC creds from agg.Out). if err := committeeverifier.LaunchStandaloneVerifiers( verifiers, aggregators, blockchainOutputs, jdInfra, @@ -189,9 +219,9 @@ func (c *component) RunPhase3( selectors, _ := priorOutputs["_selectors"].([]uint64) var connectErr error if useLegacyConfigureLane { - connectErr = ccdeploy.ConnectAllChainsLegacy(impls, blockchains, selectors, e, topology) + connectErr = ccdeploy.ConnectAllChainsLegacy(impls, blockchains, selectors, &localEnv, topology) } else { - connectErr = ccdeploy.ConnectAllChainsCanonical(impls, blockchains, selectors, e, topology) + connectErr = ccdeploy.ConnectAllChainsCanonical(impls, blockchains, selectors, &localEnv, topology) } if connectErr != nil { return nil, nil, fmt.Errorf("committeeccv: configure lanes: %w", connectErr) @@ -209,7 +239,7 @@ func (c *component) RunPhase3( return nil, nil, fmt.Errorf("committeeccv: committee %q not found in topology", agg.CommitteeName) } cs := ccvchangesets.GenerateAggregatorConfig(ccvadapters.GetRegistry()) - output, err := cs.Apply(*e, ccvchangesets.GenerateAggregatorConfigInput{ + output, err := cs.Apply(localEnv, ccvchangesets.GenerateAggregatorConfigInput{ ServiceIdentifier: instanceName + "-aggregator", CommitteeQualifier: agg.CommitteeName, ChainSelectors: ccvchangesets.CommitteeChainSelectorsFromTopology(committee), @@ -222,7 +252,7 @@ func (c *component) RunPhase3( return nil, nil, fmt.Errorf("committeeccv: get aggregator config for %q: %w", instanceName, err) } agg.GeneratedCommittee = aggCfg - e.DataStore = output.DataStore.Seal() + localEnv.DataStore = output.DataStore.Seal() } // Step 7: Launch full aggregator containers. @@ -238,7 +268,7 @@ func (c *component) RunPhase3( } // Step 8: Generate verifier job specs and emit job proposal effects. - effects, err := buildVerifierJobSpecEffects(e, verifiers, topology, obs, sharedTLSCerts, blockchainOutputs, ds) + effects, err := buildVerifierJobSpecEffects(&localEnv, verifiers, topology, obs, sharedTLSCerts, blockchainOutputs, ds) if err != nil { return nil, nil, err } @@ -251,9 +281,28 @@ func (c *component) RunPhase3( "aggregators": aggregators, "verifiers": verifiers, "_shared_tls_certs": sharedTLSCerts, + // Runtime-only CL node client lookup (nil in standalone runs); consumed by + // CL-mode job proposal + AcceptPendingJobs in a later step. + "_clnode_clients": clNodeClients, }, effects, nil } +// clModeNOPAliases returns, in topology order, the aliases of NOPs running in +// CL mode (hosted on a Chainlink node). The order matches the CL node ordering +// used by jobs.NewNodeSetClientLookup. +func clModeNOPAliases(topology *ccvdeployment.EnvironmentTopology) []string { + if topology == nil || topology.NOPTopology == nil { + return nil + } + var aliases []string + for _, nop := range topology.NOPTopology.NOPs { + if nop.GetMode() == ccvshared.NOPModeCL { + aliases = append(aliases, nop.Alias) + } + } + return aliases +} + type verifierJobSpec struct { Name string `toml:"name"` ExternalJobID string `toml:"externalJobID"` diff --git a/build/devenv/env-cl-phased.toml b/build/devenv/env-cl-phased.toml index b6d1c10e4..3d115958f 100644 --- a/build/devenv/env-cl-phased.toml +++ b/build/devenv/env-cl-phased.toml @@ -1,14 +1,23 @@ -# Phased-runtime CL-node overlay (issue 16). +# Phased-runtime CL-mode overlay (issue 16). # # Layer this on top of env-phased.toml: # ccv --env-mode phased up env-phased.toml,env-cl-phased.toml # -# It is a phased-shaped copy of env-cl.toml's CL bits. STEP 1 scope: only the -# [clnode] section, which the clnode component claims (so the issue-03 -# unclaimed-key check passes) and, in the throwaway prototype, uses to launch + -# fund the node sets. CL-mode verifier/executor and topology overrides come in -# later steps. +# CL mode is a property of the verifier/executor (mode = "cl"): instead of +# running as standalone containers, they run as jobs on shared Chainlink nodes. +# SCOPE: only the verifiers run in CL mode here. The executors stay standalone +# (their NOPs/pools/inputs are left as env-phased.toml defines them). +# +# Config merge (deepMergeMaps) merges nested *maps* key-by-key but replaces +# *arrays* wholesale, so this file respecifies every array that must change: the +# NOP list (now the two shared CL nodes plus the unchanged standalone executor +# NOPs), each committee's nop_aliases, and the verifier list. The executor +# section, executor_pools, and aggregators are inherited from env-phased.toml. +# Monitoring/pyroscope stay in the [observability] component (not here). +############################# +# CL node sets (clnode component, config vehicle) +############################# [clnode] version = 1 cl_nodes_funding_eth = 50 @@ -23,19 +32,178 @@ cl_nodes_funding_link = 50 image = "postgres:15.0" [[clnode.node_sets.node_specs]] - [clnode.node_sets.node_specs.node] docker_ctx = "../../../chainlink" docker_file = "plugins/chainlink.Dockerfile" - [clnode.node_sets.node_specs.node.env_vars] CL_EVM_CMD = "" [[clnode.node_sets.node_specs]] - [clnode.node_sets.node_specs.node] docker_ctx = "../../../chainlink" docker_file = "plugins/chainlink.Dockerfile" - [clnode.node_sets.node_specs.node.env_vars] CL_EVM_CMD = "" + +############################# +# Topology NOPs: replace the six standalone verifier NOPs with two shared CL +# node NOPs (node-0, node-1; default NOP mode is CL). The three standalone +# executor NOPs are kept unchanged so the executors (still standalone) and their +# executor_pools continue to resolve. +############################# +[[protocol_contracts.environment_topology.nop_topology.nops]] +alias = "node-0" +name = "node-0" + +[[protocol_contracts.environment_topology.nop_topology.nops]] +alias = "node-1" +name = "node-1" + +[[protocol_contracts.environment_topology.nop_topology.nops]] +alias = "default-executor-1" +name = "default-executor-1" +mode = "standalone" + +[[protocol_contracts.environment_topology.nop_topology.nops]] +alias = "default-executor-2" +name = "default-executor-2" +mode = "standalone" + +[[protocol_contracts.environment_topology.nop_topology.nops]] +alias = "custom-executor-1" +name = "custom-executor-1" +mode = "standalone" + +############################# +# Committees: repoint membership from the standalone verifier NOPs to the two +# shared CL nodes. (Aggregators, qualifier, verifier_version are inherited.) +############################# +[protocol_contracts.environment_topology.nop_topology.committees.default.chain_configs.3379446385462418246] +nop_aliases = ["node-0", "node-1"] +threshold = 2 +[protocol_contracts.environment_topology.nop_topology.committees.default.chain_configs.12922642891491394802] +nop_aliases = ["node-0", "node-1"] +threshold = 2 +[protocol_contracts.environment_topology.nop_topology.committees.default.chain_configs.4793464827907405086] +nop_aliases = ["node-0", "node-1"] +threshold = 2 + +[protocol_contracts.environment_topology.nop_topology.committees.secondary.chain_configs.3379446385462418246] +nop_aliases = ["node-0", "node-1"] +threshold = 2 +[protocol_contracts.environment_topology.nop_topology.committees.secondary.chain_configs.12922642891491394802] +nop_aliases = ["node-0", "node-1"] +threshold = 2 +[protocol_contracts.environment_topology.nop_topology.committees.secondary.chain_configs.4793464827907405086] +nop_aliases = ["node-0", "node-1"] +threshold = 2 + +[protocol_contracts.environment_topology.nop_topology.committees.tertiary.chain_configs.3379446385462418246] +nop_aliases = ["node-0", "node-1"] +threshold = 2 +[protocol_contracts.environment_topology.nop_topology.committees.tertiary.chain_configs.12922642891491394802] +nop_aliases = ["node-0", "node-1"] +threshold = 2 +[protocol_contracts.environment_topology.nop_topology.committees.tertiary.chain_configs.4793464827907405086] +nop_aliases = ["node-0", "node-1"] +threshold = 2 + +############################# +# Verifiers: CL mode, mapped onto the two nodes. Replaces the standalone +# verifier list from env-phased.toml wholesale. +############################# +[[committeeccv.verifier]] + mode = "cl" + image = "verifier:latest" + container_name = "default-verifier-1" + nop_alias = "node-0" + port = 8100 + source_code_path = "../verifier" + root_path = "../../" + committee_name = "default" + node_index = 0 + insecure_aggregator_connection = true + [committeeccv.verifier.db] + image = "postgres:16-alpine" + name = "default-verifier-1-db" + port = 8432 + +[[committeeccv.verifier]] + mode = "cl" + image = "verifier:latest" + container_name = "default-verifier-2" + nop_alias = "node-1" + port = 8200 + source_code_path = "../verifier" + root_path = "../../" + committee_name = "default" + node_index = 1 + insecure_aggregator_connection = true + [committeeccv.verifier.db] + image = "postgres:16-alpine" + name = "default-verifier-2-db" + port = 8433 + +[[committeeccv.verifier]] + mode = "cl" + image = "verifier:latest" + container_name = "secondary-verifier-1" + nop_alias = "node-0" + port = 8300 + source_code_path = "../verifier" + root_path = "../../" + committee_name = "secondary" + node_index = 0 + insecure_aggregator_connection = true + [committeeccv.verifier.db] + image = "postgres:16-alpine" + name = "secondary-verifier-1-db" + port = 8434 + +[[committeeccv.verifier]] + mode = "cl" + image = "verifier:latest" + container_name = "secondary-verifier-2" + nop_alias = "node-1" + port = 8400 + source_code_path = "../verifier" + root_path = "../../" + committee_name = "secondary" + node_index = 1 + insecure_aggregator_connection = true + [committeeccv.verifier.db] + image = "postgres:16-alpine" + name = "secondary-verifier-2-db" + port = 8435 + +[[committeeccv.verifier]] + mode = "cl" + image = "verifier:latest" + container_name = "tertiary-verifier-1" + nop_alias = "node-0" + port = 8500 + source_code_path = "../verifier" + root_path = "../../" + committee_name = "tertiary" + node_index = 0 + insecure_aggregator_connection = true + [committeeccv.verifier.db] + image = "postgres:16-alpine" + name = "tertiary-verifier-1-db" + port = 8436 + +[[committeeccv.verifier]] + mode = "cl" + image = "verifier:latest" + container_name = "tertiary-verifier-2" + nop_alias = "node-1" + port = 8600 + source_code_path = "../verifier" + root_path = "../../" + committee_name = "tertiary" + node_index = 1 + insecure_aggregator_connection = true + [committeeccv.verifier.db] + image = "postgres:16-alpine" + name = "tertiary-verifier-2-db" + port = 8437 From 7891454e703f7caa0d5f91872b46044767505c4a Mon Sep 17 00:00:00 2001 From: Will Winder Date: Sat, 30 May 2026 16:10:40 -0400 Subject: [PATCH 4/8] build/devenv: bake HMAC secrets and accept CL jobs (steps 3+4) CL node secrets are boot-only, so aggregator HMAC credentials must be written to TestSecretsOverrides before LaunchNodeSets; the effect executor now calls AcceptPendingJobs before SyncAndVerifyJobProposals and passes a local env copy with NodeIDs from JD so the sync can map NOP aliases to node IDs. --- build/devenv/components/clnode/component.go | 27 +++ .../components/committeeccv/component.go | 213 +++++++++++++++--- build/devenv/effect_executor.go | 16 +- 3 files changed, 224 insertions(+), 32 deletions(-) diff --git a/build/devenv/components/clnode/component.go b/build/devenv/components/clnode/component.go index 251b3e438..0023d0d27 100644 --- a/build/devenv/components/clnode/component.go +++ b/build/devenv/components/clnode/component.go @@ -200,3 +200,30 @@ func toBigUnits(v float64, def int64) *big.Int { } return big.NewInt(int64(v)) } + +// Secrets, CCVSecrets, and AggregatorSecret mirror the identically-named types +// in the ccv package (environment.go). Duplicated here to avoid an import cycle +// (ccv blank-imports every component). A later cleanup can extract them to a +// shared package. + +type Secrets struct { + CCV CCVSecrets `toml:",omitempty"` +} + +func (s *Secrets) TomlString() (string, error) { + data, err := toml.Marshal(s) + if err != nil { + return "", fmt.Errorf("failed to marshal CCV secrets to TOML: %w", err) + } + return string(data), nil +} + +type CCVSecrets struct { + AggregatorSecrets []AggregatorSecret `toml:",omitempty"` +} + +type AggregatorSecret struct { + VerifierID string `toml:",omitempty"` + APIKey string `toml:",omitempty"` + APISecret string `toml:",omitempty"` +} diff --git a/build/devenv/components/committeeccv/component.go b/build/devenv/components/committeeccv/component.go index e195b3ac4..9f4ef56f0 100644 --- a/build/devenv/components/committeeccv/component.go +++ b/build/devenv/components/committeeccv/component.go @@ -137,37 +137,14 @@ func (c *component) RunPhase3( // are boot-only and the aggregator HMAC creds (generated in Step 1) must be // baked in before launch. CL nodes must be launched + registered + connected to // JD before the lane/committee config below, because ApplyVerifierConfig fetches - // CL-mode signing keys from JD. (Secret injection lands in a later step.) - var clNodeClients *jobs.NodeSetClientLookup - if clnodeCfg, ok := priorOutputs[clnodecomp.OutputKey].(*clnodecomp.Config); ok && clnodeCfg != nil && len(clnodeCfg.NodeSets) > 0 { - if err := clnodecomp.LaunchNodeSets(ctx, clnodeCfg, blockchains); err != nil { - return nil, nil, fmt.Errorf("committeeccv: launching CL node sets: %w", err) - } - - clNopAliases := clModeNOPAliases(topology) - if len(clNopAliases) > 0 { - clNodeClients, err = jobs.NewNodeSetClientLookup(clnodeCfg.NodeSets, clNopAliases) - if err != nil { - return nil, nil, fmt.Errorf("committeeccv: building CL node client lookup: %w", err) - } - if clNodeClients != nil { - if err := jobs.RegisterNodesWithJD(ctx, jdInfra, clNodeClients, clNopAliases); err != nil { - return nil, nil, fmt.Errorf("committeeccv: registering CL nodes with JD: %w", err) - } - chainIDs := make([]string, len(blockchains)) - for i, bc := range blockchains { - chainIDs[i] = bc.ChainID - } - if err := jobs.ConnectNodesToJD(ctx, jdInfra, clNodeClients, chainIDs); err != nil { - return nil, nil, fmt.Errorf("committeeccv: connecting CL nodes to JD: %w", err) - } - // The deployment environment was built in Phase 2 (protocol_contracts) - // before these CL nodes registered, so its NodeIDs are empty. Populate - // the local copy so the lane/committee config below can fetch CL-mode - // signing keys from JD (FetchNOPSigningKeys requires NodeIDs). - localEnv.NodeIDs = jdInfra.GetNodeIDs() - } - } + // CL-mode signing keys from JD. + clnodeCfg, _ := priorOutputs[clnodecomp.OutputKey].(*clnodecomp.Config) + clNodeClients, nodeIDs, err := launchCLNodes(ctx, clnodeCfg, verifiers, aggregators, topology, blockchains, jdInfra) + if err != nil { + return nil, nil, fmt.Errorf("committeeccv: %w", err) + } + if len(nodeIDs) > 0 { + localEnv.NodeIDs = nodeIDs } // Step 2: Launch standalone verifier containers (reads HMAC creds from agg.Out). if err := committeeverifier.LaunchStandaloneVerifiers( @@ -287,6 +264,180 @@ func (c *component) RunPhase3( }, effects, nil } +// launchCLNodes bakes secrets into node specs, launches the CL node sets, then +// registers and connects each node to JD. Returns a NodeSetClientLookup and the +// JD node IDs for the launched nodes; both are nil/empty when no CL node config +// is present. +func launchCLNodes( + ctx context.Context, + clnodeCfg *clnodecomp.Config, + verifiers []*committeeverifier.Input, + aggregators []*services.AggregatorInput, + topology *ccvdeployment.EnvironmentTopology, + blockchains []*ctfblockchain.Input, + jdInfra *jobs.JDInfrastructure, +) (*jobs.NodeSetClientLookup, []string, error) { + if clnodeCfg == nil || len(clnodeCfg.NodeSets) == 0 { + return nil, nil, nil + } + if err := bakeNodeSecrets(clnodeCfg, verifiers, aggregators, topology); err != nil { + return nil, nil, fmt.Errorf("baking node secrets: %w", err) + } + if err := clnodecomp.LaunchNodeSets(ctx, clnodeCfg, blockchains); err != nil { + return nil, nil, fmt.Errorf("launching CL node sets: %w", err) + } + clNopAliases := clModeNOPAliases(topology) + if len(clNopAliases) == 0 { + return nil, nil, nil + } + clientLookup, err := jobs.NewNodeSetClientLookup(clnodeCfg.NodeSets, clNopAliases) + if err != nil { + return nil, nil, fmt.Errorf("building CL node client lookup: %w", err) + } + if clientLookup == nil { + return nil, nil, nil + } + if err := jobs.RegisterNodesWithJD(ctx, jdInfra, clientLookup, clNopAliases); err != nil { + return nil, nil, fmt.Errorf("registering CL nodes with JD: %w", err) + } + chainIDs := make([]string, len(blockchains)) + for i, bc := range blockchains { + chainIDs[i] = bc.ChainID + } + if err := jobs.ConnectNodesToJD(ctx, jdInfra, clientLookup, chainIDs); err != nil { + return nil, nil, fmt.Errorf("connecting CL nodes to JD: %w", err) + } + // The deployment environment was built in Phase 2 (protocol_contracts) before + // these CL nodes registered, so its NodeIDs are empty. Return the JD node IDs + // so the caller can populate its local env copy (FetchNOPSigningKeys needs them). + return clientLookup, jdInfra.GetNodeIDs(), nil +} + +// bakeNodeSecrets sets TestSecretsOverrides on each CL node spec before launch. +// Must be called after aggregator HMAC credentials are generated (Step 1) and +// before LaunchNodeSets, because CL node secrets are boot-only. +// +// For each CL-mode verifier the function finds its NOP index (= its node spec +// slot in the flat clnodeCfg.NodeSets list), then builds one AggregatorSecret +// entry per aggregator in the same committee. The VerifierID uses the topology +// aggregator name, matching how the changeset builds VerifierIDs. +func bakeNodeSecrets( + clnodeCfg *clnodecomp.Config, + verifiers []*committeeverifier.Input, + aggregators []*services.AggregatorInput, + topology *ccvdeployment.EnvironmentTopology, +) error { + // Build topology aggregator names per committee (matches changeset ordering). + topoAggNames := make(map[string][]string) + if topology.NOPTopology != nil { + for name, committee := range topology.NOPTopology.Committees { + names := make([]string, len(committee.Aggregators)) + for i, a := range committee.Aggregators { + names[i] = a.Name + } + topoAggNames[name] = names + } + } + + // Index aggregators by committee. + aggsByCommittee := make(map[string][]*services.AggregatorInput) + for _, agg := range aggregators { + if agg != nil { + aggsByCommittee[agg.CommitteeName] = append(aggsByCommittee[agg.CommitteeName], agg) + } + } + + // Flatten node specs in node-set order; order must match NOP index ordering. + type nodeSpecEntry struct { + nodeSetIdx int + specIdx int + } + var nodeSpecOrder []nodeSpecEntry + for i, nodeSet := range clnodeCfg.NodeSets { + for j := range nodeSet.NodeSpecs { + nodeSpecOrder = append(nodeSpecOrder, nodeSpecEntry{i, j}) + } + } + + // Collect aggregator secrets per flat node index. + aggSecretsPerNode := make(map[int][]clnodecomp.AggregatorSecret) + for _, ver := range verifiers { + if ver == nil || ver.Mode != services.CL { + continue + } + index, ok := topology.NOPTopology.GetNOPIndex(ver.NOPAlias) + if !ok { + return fmt.Errorf("NOP alias %q not found in topology for verifier %s", ver.NOPAlias, ver.ContainerName) + } + if index >= len(nodeSpecOrder) { + return fmt.Errorf("node index %d for NOPAlias %s exceeds available CL nodes (%d)", + index, ver.NOPAlias, len(nodeSpecOrder)) + } + committeeAggs := aggsByCommittee[ver.CommitteeName] + if len(committeeAggs) == 0 { + return fmt.Errorf("no aggregators found for committee %q (verifier %s)", ver.CommitteeName, ver.ContainerName) + } + committeeTopoNames := topoAggNames[ver.CommitteeName] + + for aggIdx, agg := range committeeAggs { + aggName := agg.InstanceName() + if aggIdx < len(committeeTopoNames) { + aggName = committeeTopoNames[aggIdx] + } + apiKeys, err := agg.GetAPIKeys() + if err != nil { + return fmt.Errorf("getting API keys for aggregator %s: %w", agg.InstanceName(), err) + } + var found bool + for _, apiClient := range apiKeys { + if apiClient.ClientID != ver.ContainerName { + continue + } + if len(apiClient.APIKeyPairs) == 0 { + return fmt.Errorf("no API key pairs for client %s on aggregator %s", + apiClient.ClientID, agg.InstanceName()) + } + pair := apiClient.APIKeyPairs[0] + verifierID := ccvshared.NewVerifierJobID( + ccvshared.NOPAlias(ver.NOPAlias), + aggName, + ccvshared.VerifierJobScope{CommitteeQualifier: ver.CommitteeName}, + ).GetVerifierID() + aggSecretsPerNode[index] = append(aggSecretsPerNode[index], clnodecomp.AggregatorSecret{ + VerifierID: verifierID, + APIKey: pair.APIKey, + APISecret: pair.Secret, + }) + found = true + break + } + if !found { + return fmt.Errorf("API client %q not found on aggregator %s", + ver.ContainerName, agg.InstanceName()) + } + } + } + + // Write secrets onto each node spec. + for flatIdx, entry := range nodeSpecOrder { + if len(aggSecretsPerNode[flatIdx]) == 0 { + continue + } + nodeSpec := clnodeCfg.NodeSets[entry.nodeSetIdx].NodeSpecs[entry.specIdx] + secrets := clnodecomp.Secrets{ + CCV: clnodecomp.CCVSecrets{ + AggregatorSecrets: aggSecretsPerNode[flatIdx], + }, + } + secretsToml, err := secrets.TomlString() + if err != nil { + return fmt.Errorf("marshaling secrets for node %d: %w", flatIdx, err) + } + nodeSpec.Node.TestSecretsOverrides = secretsToml + } + return nil +} + // clModeNOPAliases returns, in topology order, the aliases of NOPs running in // CL mode (hosted on a Chainlink node). The order matches the CL node ordering // used by jobs.NewNodeSetClientLookup. diff --git a/build/devenv/effect_executor.go b/build/devenv/effect_executor.go index 144417e6f..9c713ca66 100644 --- a/build/devenv/effect_executor.go +++ b/build/devenv/effect_executor.go @@ -105,8 +105,22 @@ func executeJobProposalEffects(ctx context.Context, effects []devenvruntime.JobP } } + // Accept pending job proposals on CL nodes before verifying. Mirrors the + // monolith order (AcceptPendingJobs then SyncAndVerifyJobProposals). + clNodeClients, _ := accumulated["_clnode_clients"].(*jobs.NodeSetClientLookup) + if err := jobs.AcceptPendingJobs(ctx, clNodeClients); err != nil { + return fmt.Errorf("accepting pending CL node jobs: %w", err) + } + if e, ok := accumulated["_env"].(*deployment.Environment); ok && e != nil { - if err := jobs.SyncAndVerifyJobProposals(e); err != nil { + // The Phase-2 _env has empty NodeIDs (CL nodes registered in Phase 3). + // Build a local copy with NodeIDs populated from JD so SyncJobProposals + // can map NOP aliases to JD node IDs. + syncEnv := *e + if nodeIDs := jdInfra.GetNodeIDs(); len(nodeIDs) > 0 { + syncEnv.NodeIDs = nodeIDs + } + if err := jobs.SyncAndVerifyJobProposals(&syncEnv); err != nil { return fmt.Errorf("syncing job proposals: %w", err) } } From a2b4be0b3675443c4547e3f48f84ade3681dbb66 Mon Sep 17 00:00:00 2001 From: Will Winder Date: Sat, 30 May 2026 20:06:29 -0400 Subject: [PATCH 5/8] Add phased CL env test. --- .github/workflows/test-cl-smoke.yaml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-cl-smoke.yaml b/.github/workflows/test-cl-smoke.yaml index f2544464a..82563a142 100644 --- a/.github/workflows/test-cl-smoke.yaml +++ b/.github/workflows/test-cl-smoke.yaml @@ -79,6 +79,12 @@ jobs: run_cmd: TestHA_CrossComponentDown config: "env.toml,env-HA.toml,env-cl.toml,env-cl-ci.toml" timeout: 10m + - name: TestE2ESmoke_Basic + run_cmd: TestE2ESmoke_Basic_Phased + config: "env-phased.toml,env-phased-cl.toml,env-cl-ci.toml" + flags: --env-mode phased + smoke_test_config: ../../env-phased-out.toml + timeout: 15m # We need to configure HeadTracker for the CL tests to have finality depth. Otherwise, it does instant finality. # - name: TestE2EReorg # config: "env.toml,env-src-auto-mine.toml,env-cl.toml,env-cl-ci.toml" @@ -125,7 +131,7 @@ jobs: role-to-assume: ${{ secrets.CCV_IAM_ROLE }} aws-region: us-east-1 registry-type: public - + - name: Authenticate to AWS ECR (JD) uses: ./.github/actions/aws-ecr-auth with: @@ -162,18 +168,20 @@ jobs: env: JD_IMAGE: ${{ secrets.JD_IMAGE }} run: | - ccv u ${{ matrix.test.config }} + ccv ${{ matrix.test.flags || '' }} u ${{ matrix.test.config }} - name: Run Smoke Test id: test_run working-directory: build/devenv/tests/e2e env: LOKI_URL: http://localhost:3030/loki/api/v1/push + # When unset, the test falls back to ../../env-out.toml (see GetSmokeTestConfig). + SMOKE_TEST_CONFIG: ${{ matrix.test.smoke_test_config || '' }} run: | set -o pipefail go test -v -timeout ${{ matrix.test.timeout }} -count=1 -run ${{ matrix.test.run_cmd }} continue-on-error: true - + - name: Dump logs if they're not already dumped if: always() working-directory: build/devenv/tests/e2e @@ -184,7 +192,7 @@ jobs: else echo "Logs already exist. Skipping manual dump." fi - + - name: Sanitize test name for log artifact name if: always() id: sanitize_name @@ -200,7 +208,7 @@ jobs: name: container-logs-cl-${{ steps.sanitize_name.outputs.name }} path: build/devenv/tests/e2e/logs retention-days: 1 - + - name: Check test results if: always() && steps.test_run.outcome == 'failure' run: | From 3b1c9603d098ac6c8c2d8b736594aa3224a2b550 Mon Sep 17 00:00:00 2001 From: Will Winder Date: Sat, 30 May 2026 20:36:27 -0400 Subject: [PATCH 6/8] Fix config. --- .github/workflows/test-cl-smoke.yaml | 4 ++-- .github/workflows/test-smoke.yaml | 2 +- build/devenv/env-cl-ci-phased.toml | 29 ++++++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 build/devenv/env-cl-ci-phased.toml diff --git a/.github/workflows/test-cl-smoke.yaml b/.github/workflows/test-cl-smoke.yaml index 82563a142..038a698e6 100644 --- a/.github/workflows/test-cl-smoke.yaml +++ b/.github/workflows/test-cl-smoke.yaml @@ -79,9 +79,9 @@ jobs: run_cmd: TestHA_CrossComponentDown config: "env.toml,env-HA.toml,env-cl.toml,env-cl-ci.toml" timeout: 10m - - name: TestE2ESmoke_Basic + - name: Phased TestE2ESmoke_Basic run_cmd: TestE2ESmoke_Basic_Phased - config: "env-phased.toml,env-phased-cl.toml,env-cl-ci.toml" + config: "env-phased.toml,env-cl-phased.toml,env-cl-ci-phased.toml" flags: --env-mode phased smoke_test_config: ../../env-phased-out.toml timeout: 15m diff --git a/.github/workflows/test-smoke.yaml b/.github/workflows/test-smoke.yaml index 8d59dfeb2..206865554 100644 --- a/.github/workflows/test-smoke.yaml +++ b/.github/workflows/test-smoke.yaml @@ -78,7 +78,7 @@ jobs: config: env.toml,env-src-auto-mine.toml timeout: 10m working-directory: build/devenv/tests/e2e - - name: TestE2ESmoke_Basic_Phased + - name: Phased TestE2ESmoke_Basic run_cmd: TestE2ESmoke_Basic config: env-phased.toml flags: --env-mode phased diff --git a/build/devenv/env-cl-ci-phased.toml b/build/devenv/env-cl-ci-phased.toml new file mode 100644 index 000000000..ba92c9c35 --- /dev/null +++ b/build/devenv/env-cl-ci-phased.toml @@ -0,0 +1,29 @@ +## CI-specific override: use pre-built local images instead of building from Dockerfile +[clnode] +version = 1 +cl_nodes_funding_eth = 50 +cl_nodes_funding_link = 50 + +[[clnode.node_sets]] + name = "don" + nodes = 2 + override_mode = "each" + + [clnode.node_sets.db] + image = "postgres:15.0" + + [[clnode.node_sets.node_specs]] + + [clnode.node_sets.node_specs.node] + image = "docker.io/library/local:latest" + + [clnode.node_sets.node_specs.node.env_vars] + CL_EVM_CMD="" + + [[clnode.node_sets.node_specs]] + + [clnode.node_sets.node_specs.node] + image = "docker.io/library/local:latest" + + [clnode.node_sets.node_specs.node.env_vars] + CL_EVM_CMD="" From b0265ea37731d4d6bf13ac130d4f42ca668fbd0b Mon Sep 17 00:00:00 2001 From: Will Winder Date: Mon, 1 Jun 2026 11:30:10 -0400 Subject: [PATCH 7/8] build/devenv: split committeeccv into standalone and CL-node components Registers two Phase 3 components in the committeeccv package: the clean standalone variant (component.go) and the CL-node variant (component_clnode.go) which absorbs the former clnode Phase-1 config-vehicle. The clnode package is deleted; all CL-node code is isolated in one file for future removal. --- build/devenv/components/clnode/component.go | 229 ---------- .../components/committeeccv/component.go | 318 +++---------- .../committeeccv/component_clnode.go | 432 ++++++++++++++++++ build/devenv/env-cl-ci-phased.toml | 23 +- build/devenv/env-cl-phased.toml | 329 +++++++++---- build/devenv/environment.go | 1 - 6 files changed, 762 insertions(+), 570 deletions(-) delete mode 100644 build/devenv/components/clnode/component.go create mode 100644 build/devenv/components/committeeccv/component_clnode.go diff --git a/build/devenv/components/clnode/component.go b/build/devenv/components/clnode/component.go deleted file mode 100644 index 0023d0d27..000000000 --- a/build/devenv/components/clnode/component.go +++ /dev/null @@ -1,229 +0,0 @@ -// Package clnode is the phased-devenv component for Chainlink-node ("CL node") -// support (issue 16). -// -// The component itself is a config vehicle: it claims the top-level [clnode] -// config key, decodes the versioned config, and publishes it as the -// runtime-only output key "_clnode". It does NOT launch anything. -// -// The committeeccv component (Phase 3) consumes "_clnode" and drives the -// actual launch via LaunchNodeSets, because CL node secrets are boot-only and -// the aggregator HMAC credentials that must be baked into the node spec before -// launch are owned by committeeccv. Keeping the launch helper here keeps the -// node-launch code beside its config while letting committeeccv sequence it. -package clnode - -import ( - "context" - "fmt" - "math/big" - "strings" - - "github.com/pelletier/go-toml/v2" - - "github.com/smartcontractkit/chainlink-ccv/build/devenv/cciptestinterfaces" - "github.com/smartcontractkit/chainlink-ccv/build/devenv/chainreg" - devenvruntime "github.com/smartcontractkit/chainlink-ccv/build/devenv/runtime" - ctfblockchain "github.com/smartcontractkit/chainlink-testing-framework/framework/components/blockchain" - ns "github.com/smartcontractkit/chainlink-testing-framework/framework/components/simple_node_set" -) - -const configKey = "clnode" - -// OutputKey is the runtime-only output key under which the decoded clnode -// config is published for committeeccv to consume. -const OutputKey = "_clnode" - -// Version is the clnode component config schema version. Exactly this version -// is supported; configs declaring any other version are rejected. -const Version = 1 - -// CommonCLNodesConfig is the base TOML config applied to every CL node. It is a -// copy of ccv.CommonCLNodesConfig; the ccv package blank-imports every -// component, so a component cannot import it back without a cycle. -const CommonCLNodesConfig = ` -[Log] -JSONConsole = true -Level = 'info' -[Pyroscope] -ServerAddress = 'http://host.docker.internal:4040' -Environment = 'local' -[WebServer] -SessionTimeout = '999h0m0s' -HTTPWriteTimeout = '3m' -SecureCookies = false -HTTPPort = 6688 -AllowOrigins = 'http://localhost:3000' -[WebServer.TLS] -HTTPSPort = 0 -[WebServer.RateLimit] -Authenticated = 5000 -Unauthenticated = 5000 -[JobPipeline] -[JobPipeline.HTTPRequest] -DefaultTimeout = '1m' -[Log.File] -MaxSize = '0b' -[Feature] -FeedsManager = true -LogPoller = true -UICSAKeys = true -[OCR2] -Enabled = true -SimulateTransactions = false -DefaultTransactionQueueDepth = 1 -[P2P.V2] -Enabled = true -ListenAddresses = ['0.0.0.0:6690'] -` - -func init() { - if err := devenvruntime.Register(configKey, factory); err != nil { - panic(fmt.Sprintf("clnode component: %v", err)) - } -} - -func factory(_ map[string]any) (devenvruntime.Component, error) { - return &component{}, nil -} - -type component struct{} - -// Config is the versioned wrapper around the CL node set definitions. Adding a -// version field is why this is a [clnode] table rather than a bare top-level -// [[nodesets]] array. -type Config struct { - Version int `toml:"version"` - CLNodesFundingETH float64 `toml:"cl_nodes_funding_eth"` - CLNodesFundingLink float64 `toml:"cl_nodes_funding_link"` - NodeSets []*ns.Input `toml:"node_sets"` -} - -// decodeConfig round-trips the raw TOML component config into a typed Config -// and verifies its declared version. -func decodeConfig(raw any) (Config, error) { - b, err := toml.Marshal(raw) - if err != nil { - return Config{}, fmt.Errorf("re-encoding clnode config: %w", err) - } - var cfg Config - if err := toml.Unmarshal(b, &cfg); err != nil { - return Config{}, fmt.Errorf("decoding clnode config: %w", err) - } - if err := devenvruntime.CheckConfigVersion(cfg.Version, Version); err != nil { - return Config{}, err - } - return cfg, nil -} - -func (c *component) ValidateConfig(componentConfig any) error { - _, err := decodeConfig(componentConfig) - return err -} - -// RunPhase1 decodes the clnode config and publishes it under OutputKey for -// committeeccv (Phase 3) to consume. It launches nothing. -func (c *component) RunPhase1( - _ context.Context, - _ map[string]any, - componentConfig any, -) (map[string]any, []devenvruntime.Effect, error) { - cfg, err := decodeConfig(componentConfig) - if err != nil { - return nil, nil, err - } - out := cfg - return map[string]any{OutputKey: &out}, nil, nil -} - -// LaunchNodeSets configures, launches, and funds the CL node sets in cfg using -// the chains in blockchains. It is called by committeeccv during Phase 3. -// -// NOTE: secret injection (TestSecretsOverrides) must happen on the node specs -// before this is called, because CL node secrets are boot-only. Step 1 launches -// without secrets; committeeccv will bake them in a later step. -func LaunchNodeSets(ctx context.Context, cfg *Config, blockchains []*ctfblockchain.Input) error { - if cfg == nil || len(cfg.NodeSets) == 0 { - return nil - } - if len(blockchains) == 0 { - return fmt.Errorf("clnode: no blockchains available to configure CL nodes") - } - - impls := make([]cciptestinterfaces.CCIP17Configuration, 0, len(blockchains)) - for _, bc := range blockchains { - impl, ierr := chainreg.NewProductConfigurationFromNetwork(bc.Type) - if ierr != nil { - return fmt.Errorf("clnode: impl for %q: %w", bc.Type, ierr) - } - impls = append(impls, impl) - } - - // Assemble the CL node chain-config overrides from each chain impl. - chainConfigs := []string{CommonCLNodesConfig} - for i, impl := range impls { - cc, cerr := impl.ConfigureNodes(ctx, blockchains[i]) - if cerr != nil { - return fmt.Errorf("clnode: ConfigureNodes for %q: %w", blockchains[i].Type, cerr) - } - chainConfigs = append(chainConfigs, cc) - } - allConfigs := strings.Join(chainConfigs, "\n") - for _, nodeSet := range cfg.NodeSets { - for _, nodeSpec := range nodeSet.NodeSpecs { - nodeSpec.Node.TestConfigOverrides = allConfigs - } - } - - // Launch each node set (shared DB per set). - for _, nodeSet := range cfg.NodeSets { - if _, nerr := ns.NewSharedDBNodeSet(nodeSet, nil); nerr != nil { - return fmt.Errorf("clnode: NewSharedDBNodeSet %q: %w", nodeSet.Name, nerr) - } - } - - // Fund the nodes on every fundable chain. FundNodes takes (link, native). - link := toBigUnits(cfg.CLNodesFundingLink, 1) - native := toBigUnits(cfg.CLNodesFundingETH, 5) - for i, impl := range impls { - if ferr := impl.FundNodes(ctx, cfg.NodeSets, blockchains[i], link, native); ferr != nil { - return fmt.Errorf("clnode: FundNodes on %q: %w", blockchains[i].Type, ferr) - } - } - return nil -} - -// toBigUnits converts a whole-unit funding amount to *big.Int, falling back to -// def when the configured value is non-positive. -func toBigUnits(v float64, def int64) *big.Int { - if v <= 0 { - return big.NewInt(def) - } - return big.NewInt(int64(v)) -} - -// Secrets, CCVSecrets, and AggregatorSecret mirror the identically-named types -// in the ccv package (environment.go). Duplicated here to avoid an import cycle -// (ccv blank-imports every component). A later cleanup can extract them to a -// shared package. - -type Secrets struct { - CCV CCVSecrets `toml:",omitempty"` -} - -func (s *Secrets) TomlString() (string, error) { - data, err := toml.Marshal(s) - if err != nil { - return "", fmt.Errorf("failed to marshal CCV secrets to TOML: %w", err) - } - return string(data), nil -} - -type CCVSecrets struct { - AggregatorSecrets []AggregatorSecret `toml:",omitempty"` -} - -type AggregatorSecret struct { - VerifierID string `toml:",omitempty"` - APIKey string `toml:",omitempty"` - APISecret string `toml:",omitempty"` -} diff --git a/build/devenv/components/committeeccv/component.go b/build/devenv/components/committeeccv/component.go index 9f4ef56f0..00ff0a8ca 100644 --- a/build/devenv/components/committeeccv/component.go +++ b/build/devenv/components/committeeccv/component.go @@ -12,7 +12,6 @@ import ( "github.com/smartcontractkit/chainlink-ccv/build/devenv/chainreg" devenvcommon "github.com/smartcontractkit/chainlink-ccv/build/devenv/common" blockchainscomp "github.com/smartcontractkit/chainlink-ccv/build/devenv/components/blockchains" - clnodecomp "github.com/smartcontractkit/chainlink-ccv/build/devenv/components/clnode" "github.com/smartcontractkit/chainlink-ccv/build/devenv/components/observability" ccdeploy "github.com/smartcontractkit/chainlink-ccv/build/devenv/deploy" "github.com/smartcontractkit/chainlink-ccv/build/devenv/jobs" @@ -69,91 +68,123 @@ func (c *component) RunPhase3( componentConfig any, priorOutputs map[string]any, ) (map[string]any, []devenvruntime.Effect, error) { - // The committeeccv config is a single [committeeccv] section holding the - // aggregator and verifier inputs. cfg, err := decodeConfig(componentConfig) if err != nil { return nil, nil, err } aggregators, verifiers := cfg.Aggregator, cfg.Verifier - if len(aggregators) == 0 && len(verifiers) == 0 { return map[string]any{}, nil, nil } + inputs, err := parsePhase3Inputs(priorOutputs, globalConfig) + if err != nil { + return nil, nil, err + } + // Work on a copy of the shared Phase-2 environment. + localEnv := *inputs.env + if err := ensureAggregatorCredentials(aggregators); err != nil { + return nil, nil, err + } + return runPhase3Core(ctx, inputs, aggregators, verifiers, &localEnv) +} + +// phase3Inputs holds the decoded prior-phase outputs consumed by both the +// standalone and CL-node CommitteeCCV components. +type phase3Inputs struct { + jdInfra *jobs.JDInfrastructure + blockchains []*ctfblockchain.Input + blockchainOutputs []*ctfblockchain.Output + env *deployment.Environment + topology *ccvdeployment.EnvironmentTopology + obs *observability.Observability + ds datastore.MutableDataStore + impls []cciptestinterfaces.CCIP17Configuration + selectors []uint64 + useLegacyConfigureLane bool +} +func parsePhase3Inputs(priorOutputs, globalConfig map[string]any) (phase3Inputs, error) { jdInfra, ok := priorOutputs["jd"].(*jobs.JDInfrastructure) if !ok || jdInfra == nil { - return nil, nil, fmt.Errorf("committeeccv: jd not found in phase outputs") + return phase3Inputs{}, fmt.Errorf("committeeccv: jd not found in phase outputs") } blockchains, ok := priorOutputs["blockchains"].([]*ctfblockchain.Input) if !ok { - return nil, nil, fmt.Errorf("committeeccv: blockchains not found in phase outputs") + return phase3Inputs{}, fmt.Errorf("committeeccv: blockchains not found in phase outputs") } blockchainOutputs := blockchainscomp.Outputs(blockchains) e, ok := priorOutputs["_env"].(*deployment.Environment) if !ok || e == nil { - return nil, nil, fmt.Errorf("committeeccv: _env not found in phase outputs") + return phase3Inputs{}, fmt.Errorf("committeeccv: _env not found in phase outputs") } - // Work on a copy of the environment: the shared _env is a Phase-2 output and - // must be treated as immutable. CL-node NodeIDs and per-step DataStore updates - // are applied to this local copy only. (Mirrors the tokenverifier component.) - localEnv := *e topology, ok := priorOutputs["environment_topology"].(*ccvdeployment.EnvironmentTopology) if !ok || topology == nil { - return nil, nil, fmt.Errorf("committeeccv: environment_topology not found in phase outputs") + return phase3Inputs{}, fmt.Errorf("committeeccv: environment_topology not found in phase outputs") } obs, ok := priorOutputs["observability"].(*observability.Observability) if !ok || obs == nil { - return nil, nil, fmt.Errorf("committeeccv: observability not found in phase outputs") + return phase3Inputs{}, fmt.Errorf("committeeccv: observability not found in phase outputs") } ds, ok := priorOutputs["_ds"].(datastore.MutableDataStore) if !ok { - return nil, nil, fmt.Errorf("committeeccv: _ds not found in phase outputs") + return phase3Inputs{}, fmt.Errorf("committeeccv: _ds not found in phase outputs") } impls, _ := priorOutputs["_impls"].([]cciptestinterfaces.CCIP17Configuration) - var useLegacyConfigureLane bool + selectors, _ := priorOutputs["_selectors"].([]uint64) + var useLegacy bool if pcMap, ok := globalConfig["protocol_contracts"].(map[string]any); ok { - useLegacyConfigureLane, _ = pcMap["use_legacy_configure_lane"].(bool) - } + useLegacy, _ = pcMap["use_legacy_configure_lane"].(bool) + } + return phase3Inputs{ + jdInfra: jdInfra, + blockchains: blockchains, + blockchainOutputs: blockchainOutputs, + env: e, + topology: topology, + obs: obs, + ds: ds, + impls: impls, + selectors: selectors, + useLegacyConfigureLane: useLegacy, + }, nil +} - // Step 1: Generate HMAC client credentials for all aggregators before launching verifiers. +func ensureAggregatorCredentials(aggregators []*services.AggregatorInput) error { for _, agg := range aggregators { if agg == nil { continue } creds, cerr := agg.EnsureClientCredentials() if cerr != nil { - return nil, nil, fmt.Errorf("committeeccv: failed to ensure client credentials for aggregator %s: %w", agg.CommitteeName, cerr) + return fmt.Errorf("committeeccv: failed to ensure client credentials for aggregator %s: %w", agg.CommitteeName, cerr) } if agg.Out == nil { agg.Out = &services.AggregatorOutput{} } agg.Out.ClientCredentials = creds } + return nil +} - // Step 1b: Launch CL node sets and register them with JD (issue 16). The - // clnode component (Phase 1) publishes its decoded config under - // clnodecomp.OutputKey; committeeccv drives the launch because CL node secrets - // are boot-only and the aggregator HMAC creds (generated in Step 1) must be - // baked in before launch. CL nodes must be launched + registered + connected to - // JD before the lane/committee config below, because ApplyVerifierConfig fetches - // CL-mode signing keys from JD. - clnodeCfg, _ := priorOutputs[clnodecomp.OutputKey].(*clnodecomp.Config) - clNodeClients, nodeIDs, err := launchCLNodes(ctx, clnodeCfg, verifiers, aggregators, topology, blockchains, jdInfra) - if err != nil { - return nil, nil, fmt.Errorf("committeeccv: %w", err) - } - if len(nodeIDs) > 0 { - localEnv.NodeIDs = nodeIDs - } +// runPhase3Core runs the shared CommitteeCCV Phase 3 steps (steps 2–8) against +// a local copy of the deployment environment. It is called by both the standalone +// and CL-node components; the CL-node component performs its own step 1b +// (node launch + NodeIDs population) before calling this function. +func runPhase3Core( + ctx context.Context, + inputs phase3Inputs, + aggregators []*services.AggregatorInput, + verifiers []*committeeverifier.Input, + localEnv *deployment.Environment, +) (map[string]any, []devenvruntime.Effect, error) { // Step 2: Launch standalone verifier containers (reads HMAC creds from agg.Out). if err := committeeverifier.LaunchStandaloneVerifiers( - verifiers, aggregators, blockchainOutputs, jdInfra, + verifiers, aggregators, inputs.blockchainOutputs, inputs.jdInfra, chainreg.GetRegistry().GetVerifierModifiers(), ); err != nil { return nil, nil, fmt.Errorf("committeeccv: failed to launch standalone verifiers: %w", err) } - if err := committeeverifier.RegisterStandaloneVerifiersWithJD(ctx, verifiers, jdInfra.OffchainClient); err != nil { + if err := committeeverifier.RegisterStandaloneVerifiersWithJD(ctx, verifiers, inputs.jdInfra.OffchainClient); err != nil { return nil, nil, fmt.Errorf("committeeccv: failed to register standalone verifiers with JD: %w", err) } @@ -171,6 +202,7 @@ func (c *component) RunPhase3( } allHostnames = append(allHostnames, "localhost") tlsCertDir := filepath.Join(util.CCVConfigDir(), "tls-shared") + var err error sharedTLSCerts, err = services.GenerateTLSCertificates(allHostnames, tlsCertDir) if err != nil { return nil, nil, fmt.Errorf("committeeccv: failed to generate shared TLS certificates: %w", err) @@ -187,18 +219,17 @@ func (c *component) RunPhase3( // Step 5: Enrich topology with verifier signer keys. if len(verifiers) > 0 { - ccdeploy.EnrichTopologyWithVerifiers(topology, verifiers) + ccdeploy.EnrichTopologyWithVerifiers(inputs.topology, verifiers) } // Step 5b: Configure lanes. This requires verifiers to be registered in JD (done above) // because ApplyVerifierConfig fetches verifier signing keys from JD by node ID. - if len(impls) > 0 && len(blockchains) > 0 { - selectors, _ := priorOutputs["_selectors"].([]uint64) + if len(inputs.impls) > 0 && len(inputs.blockchains) > 0 { var connectErr error - if useLegacyConfigureLane { - connectErr = ccdeploy.ConnectAllChainsLegacy(impls, blockchains, selectors, &localEnv, topology) + if inputs.useLegacyConfigureLane { + connectErr = ccdeploy.ConnectAllChainsLegacy(inputs.impls, inputs.blockchains, inputs.selectors, localEnv, inputs.topology) } else { - connectErr = ccdeploy.ConnectAllChainsCanonical(impls, blockchains, selectors, &localEnv, topology) + connectErr = ccdeploy.ConnectAllChainsCanonical(inputs.impls, inputs.blockchains, inputs.selectors, localEnv, inputs.topology) } if connectErr != nil { return nil, nil, fmt.Errorf("committeeccv: configure lanes: %w", connectErr) @@ -211,12 +242,12 @@ func (c *component) RunPhase3( continue } instanceName := agg.InstanceName() - committee, ok := topology.NOPTopology.Committees[agg.CommitteeName] + committee, ok := inputs.topology.NOPTopology.Committees[agg.CommitteeName] if !ok { return nil, nil, fmt.Errorf("committeeccv: committee %q not found in topology", agg.CommitteeName) } cs := ccvchangesets.GenerateAggregatorConfig(ccvadapters.GetRegistry()) - output, err := cs.Apply(localEnv, ccvchangesets.GenerateAggregatorConfigInput{ + output, err := cs.Apply(*localEnv, ccvchangesets.GenerateAggregatorConfigInput{ ServiceIdentifier: instanceName + "-aggregator", CommitteeQualifier: agg.CommitteeName, ChainSelectors: ccvchangesets.CommitteeChainSelectorsFromTopology(committee), @@ -245,215 +276,18 @@ func (c *component) RunPhase3( } // Step 8: Generate verifier job specs and emit job proposal effects. - effects, err := buildVerifierJobSpecEffects(&localEnv, verifiers, topology, obs, sharedTLSCerts, blockchainOutputs, ds) + effects, err := buildVerifierJobSpecEffects(localEnv, verifiers, inputs.topology, inputs.obs, sharedTLSCerts, inputs.blockchainOutputs, inputs.ds) if err != nil { return nil, nil, err } return map[string]any{ - // aggregators and verifiers are public (serialized; the phased loader - // reads them and derives endpoint maps). The shared TLS certs are - // runtime-only plumbing, so they use a "_"-prefixed key and are stripped - // from the serialized output. "aggregators": aggregators, "verifiers": verifiers, "_shared_tls_certs": sharedTLSCerts, - // Runtime-only CL node client lookup (nil in standalone runs); consumed by - // CL-mode job proposal + AcceptPendingJobs in a later step. - "_clnode_clients": clNodeClients, }, effects, nil } -// launchCLNodes bakes secrets into node specs, launches the CL node sets, then -// registers and connects each node to JD. Returns a NodeSetClientLookup and the -// JD node IDs for the launched nodes; both are nil/empty when no CL node config -// is present. -func launchCLNodes( - ctx context.Context, - clnodeCfg *clnodecomp.Config, - verifiers []*committeeverifier.Input, - aggregators []*services.AggregatorInput, - topology *ccvdeployment.EnvironmentTopology, - blockchains []*ctfblockchain.Input, - jdInfra *jobs.JDInfrastructure, -) (*jobs.NodeSetClientLookup, []string, error) { - if clnodeCfg == nil || len(clnodeCfg.NodeSets) == 0 { - return nil, nil, nil - } - if err := bakeNodeSecrets(clnodeCfg, verifiers, aggregators, topology); err != nil { - return nil, nil, fmt.Errorf("baking node secrets: %w", err) - } - if err := clnodecomp.LaunchNodeSets(ctx, clnodeCfg, blockchains); err != nil { - return nil, nil, fmt.Errorf("launching CL node sets: %w", err) - } - clNopAliases := clModeNOPAliases(topology) - if len(clNopAliases) == 0 { - return nil, nil, nil - } - clientLookup, err := jobs.NewNodeSetClientLookup(clnodeCfg.NodeSets, clNopAliases) - if err != nil { - return nil, nil, fmt.Errorf("building CL node client lookup: %w", err) - } - if clientLookup == nil { - return nil, nil, nil - } - if err := jobs.RegisterNodesWithJD(ctx, jdInfra, clientLookup, clNopAliases); err != nil { - return nil, nil, fmt.Errorf("registering CL nodes with JD: %w", err) - } - chainIDs := make([]string, len(blockchains)) - for i, bc := range blockchains { - chainIDs[i] = bc.ChainID - } - if err := jobs.ConnectNodesToJD(ctx, jdInfra, clientLookup, chainIDs); err != nil { - return nil, nil, fmt.Errorf("connecting CL nodes to JD: %w", err) - } - // The deployment environment was built in Phase 2 (protocol_contracts) before - // these CL nodes registered, so its NodeIDs are empty. Return the JD node IDs - // so the caller can populate its local env copy (FetchNOPSigningKeys needs them). - return clientLookup, jdInfra.GetNodeIDs(), nil -} - -// bakeNodeSecrets sets TestSecretsOverrides on each CL node spec before launch. -// Must be called after aggregator HMAC credentials are generated (Step 1) and -// before LaunchNodeSets, because CL node secrets are boot-only. -// -// For each CL-mode verifier the function finds its NOP index (= its node spec -// slot in the flat clnodeCfg.NodeSets list), then builds one AggregatorSecret -// entry per aggregator in the same committee. The VerifierID uses the topology -// aggregator name, matching how the changeset builds VerifierIDs. -func bakeNodeSecrets( - clnodeCfg *clnodecomp.Config, - verifiers []*committeeverifier.Input, - aggregators []*services.AggregatorInput, - topology *ccvdeployment.EnvironmentTopology, -) error { - // Build topology aggregator names per committee (matches changeset ordering). - topoAggNames := make(map[string][]string) - if topology.NOPTopology != nil { - for name, committee := range topology.NOPTopology.Committees { - names := make([]string, len(committee.Aggregators)) - for i, a := range committee.Aggregators { - names[i] = a.Name - } - topoAggNames[name] = names - } - } - - // Index aggregators by committee. - aggsByCommittee := make(map[string][]*services.AggregatorInput) - for _, agg := range aggregators { - if agg != nil { - aggsByCommittee[agg.CommitteeName] = append(aggsByCommittee[agg.CommitteeName], agg) - } - } - - // Flatten node specs in node-set order; order must match NOP index ordering. - type nodeSpecEntry struct { - nodeSetIdx int - specIdx int - } - var nodeSpecOrder []nodeSpecEntry - for i, nodeSet := range clnodeCfg.NodeSets { - for j := range nodeSet.NodeSpecs { - nodeSpecOrder = append(nodeSpecOrder, nodeSpecEntry{i, j}) - } - } - - // Collect aggregator secrets per flat node index. - aggSecretsPerNode := make(map[int][]clnodecomp.AggregatorSecret) - for _, ver := range verifiers { - if ver == nil || ver.Mode != services.CL { - continue - } - index, ok := topology.NOPTopology.GetNOPIndex(ver.NOPAlias) - if !ok { - return fmt.Errorf("NOP alias %q not found in topology for verifier %s", ver.NOPAlias, ver.ContainerName) - } - if index >= len(nodeSpecOrder) { - return fmt.Errorf("node index %d for NOPAlias %s exceeds available CL nodes (%d)", - index, ver.NOPAlias, len(nodeSpecOrder)) - } - committeeAggs := aggsByCommittee[ver.CommitteeName] - if len(committeeAggs) == 0 { - return fmt.Errorf("no aggregators found for committee %q (verifier %s)", ver.CommitteeName, ver.ContainerName) - } - committeeTopoNames := topoAggNames[ver.CommitteeName] - - for aggIdx, agg := range committeeAggs { - aggName := agg.InstanceName() - if aggIdx < len(committeeTopoNames) { - aggName = committeeTopoNames[aggIdx] - } - apiKeys, err := agg.GetAPIKeys() - if err != nil { - return fmt.Errorf("getting API keys for aggregator %s: %w", agg.InstanceName(), err) - } - var found bool - for _, apiClient := range apiKeys { - if apiClient.ClientID != ver.ContainerName { - continue - } - if len(apiClient.APIKeyPairs) == 0 { - return fmt.Errorf("no API key pairs for client %s on aggregator %s", - apiClient.ClientID, agg.InstanceName()) - } - pair := apiClient.APIKeyPairs[0] - verifierID := ccvshared.NewVerifierJobID( - ccvshared.NOPAlias(ver.NOPAlias), - aggName, - ccvshared.VerifierJobScope{CommitteeQualifier: ver.CommitteeName}, - ).GetVerifierID() - aggSecretsPerNode[index] = append(aggSecretsPerNode[index], clnodecomp.AggregatorSecret{ - VerifierID: verifierID, - APIKey: pair.APIKey, - APISecret: pair.Secret, - }) - found = true - break - } - if !found { - return fmt.Errorf("API client %q not found on aggregator %s", - ver.ContainerName, agg.InstanceName()) - } - } - } - - // Write secrets onto each node spec. - for flatIdx, entry := range nodeSpecOrder { - if len(aggSecretsPerNode[flatIdx]) == 0 { - continue - } - nodeSpec := clnodeCfg.NodeSets[entry.nodeSetIdx].NodeSpecs[entry.specIdx] - secrets := clnodecomp.Secrets{ - CCV: clnodecomp.CCVSecrets{ - AggregatorSecrets: aggSecretsPerNode[flatIdx], - }, - } - secretsToml, err := secrets.TomlString() - if err != nil { - return fmt.Errorf("marshaling secrets for node %d: %w", flatIdx, err) - } - nodeSpec.Node.TestSecretsOverrides = secretsToml - } - return nil -} - -// clModeNOPAliases returns, in topology order, the aliases of NOPs running in -// CL mode (hosted on a Chainlink node). The order matches the CL node ordering -// used by jobs.NewNodeSetClientLookup. -func clModeNOPAliases(topology *ccvdeployment.EnvironmentTopology) []string { - if topology == nil || topology.NOPTopology == nil { - return nil - } - var aliases []string - for _, nop := range topology.NOPTopology.NOPs { - if nop.GetMode() == ccvshared.NOPModeCL { - aliases = append(aliases, nop.Alias) - } - } - return aliases -} - type verifierJobSpec struct { Name string `toml:"name"` ExternalJobID string `toml:"externalJobID"` diff --git a/build/devenv/components/committeeccv/component_clnode.go b/build/devenv/components/committeeccv/component_clnode.go new file mode 100644 index 000000000..1f04ee0ae --- /dev/null +++ b/build/devenv/components/committeeccv/component_clnode.go @@ -0,0 +1,432 @@ +// Package committeeccv contains two registered Phase 3 components: +// - "committeeccv" (component.go) — standalone verifier/aggregator setup +// - "committeeccv_clnode" (this file) — CL-node variant; replaces committeeccv when +// Chainlink nodes host the verifier jobs. Absorbs the former clnode Phase-1 +// config-vehicle component. Delete this file (and its [committeeccv_clnode] +// config section) when CL nodes leave the devenv. +package committeeccv + +import ( + "context" + "fmt" + "math/big" + "strings" + + "github.com/pelletier/go-toml/v2" + + "github.com/smartcontractkit/chainlink-ccv/build/devenv/cciptestinterfaces" + "github.com/smartcontractkit/chainlink-ccv/build/devenv/chainreg" + "github.com/smartcontractkit/chainlink-ccv/build/devenv/jobs" + devenvruntime "github.com/smartcontractkit/chainlink-ccv/build/devenv/runtime" + "github.com/smartcontractkit/chainlink-ccv/build/devenv/services" + "github.com/smartcontractkit/chainlink-ccv/build/devenv/services/committeeverifier" + ccvdeployment "github.com/smartcontractkit/chainlink-ccv/deployment" + ccvshared "github.com/smartcontractkit/chainlink-ccv/deployment/shared" + ctfblockchain "github.com/smartcontractkit/chainlink-testing-framework/framework/components/blockchain" + ns "github.com/smartcontractkit/chainlink-testing-framework/framework/components/simple_node_set" +) + +const ( + clNodeConfigKey = "committeeccv_clnode" + clNodeVersion = 1 +) + +func init() { + if err := devenvruntime.Register(clNodeConfigKey, clnodeFactory); err != nil { + panic(fmt.Sprintf("committeeccv_clnode component: %v", err)) + } +} + +func clnodeFactory(_ map[string]any) (devenvruntime.Component, error) { + return &clnodeComponent{}, nil +} + +type clnodeComponent struct{} + +func (c *clnodeComponent) ValidateConfig(componentConfig any) error { + _, err := decodeCLNodeConfig(componentConfig) + return err +} + +// RunPhase3 runs the full CommitteeCCV setup (same steps as the standalone +// component) and additionally launches and registers Chainlink node sets. +// The CL-node launch (step 1b) must happen after HMAC credentials are +// generated (step 1) and before verifier registration (step 2), because: +// - bakeNodeSecrets writes HMAC creds into node specs before boot +// - ApplyVerifierConfig fetches CL-mode signing keys from JD by node ID +func (c *clnodeComponent) RunPhase3( + ctx context.Context, + globalConfig map[string]any, + componentConfig any, + priorOutputs map[string]any, +) (map[string]any, []devenvruntime.Effect, error) { + cfg, err := decodeCLNodeConfig(componentConfig) + if err != nil { + return nil, nil, err + } + aggregators, verifiers := cfg.Aggregator, cfg.Verifier + if len(aggregators) == 0 && len(verifiers) == 0 && len(cfg.NodeSets) == 0 { + return map[string]any{}, nil, nil + } + inputs, err := parsePhase3Inputs(priorOutputs, globalConfig) + if err != nil { + return nil, nil, err + } + // Work on a local copy of the shared Phase-2 environment. + localEnv := *inputs.env + + // Step 1: Generate HMAC credentials (must precede bakeNodeSecrets). + if err := ensureAggregatorCredentials(aggregators); err != nil { + return nil, nil, err + } + + // Step 1b: Bake secrets into node specs, then launch and register CL nodes. + // Must run before step 2 (verifier launch) so JD has the node IDs needed + // by ApplyVerifierConfig when fetching CL-mode signing keys. + clNodeClients, nodeIDs, err := launchCLNodes(ctx, &cfg, verifiers, aggregators, inputs.topology, inputs.blockchains, inputs.jdInfra) + if err != nil { + return nil, nil, fmt.Errorf("committeeccv_clnode: %w", err) + } + if len(nodeIDs) > 0 { + localEnv.NodeIDs = nodeIDs + } + + outputs, effects, err := runPhase3Core(ctx, inputs, aggregators, verifiers, &localEnv) + if err != nil { + return nil, nil, err + } + outputs["_clnode_clients"] = clNodeClients + return outputs, effects, nil +} + +// CLNodeConfig is the [committeeccv_clnode] config section. It embeds the +// committee fields (aggregators, verifiers) and the CL-node fields (node sets, +// funding amounts) so the section is self-contained and can be deleted wholesale +// when CL nodes leave the devenv. +type CLNodeConfig struct { + Version int `toml:"version"` + CLNodesFundingETH float64 `toml:"cl_nodes_funding_eth"` + CLNodesFundingLink float64 `toml:"cl_nodes_funding_link"` + NodeSets []*ns.Input `toml:"node_sets"` + Aggregator []*services.AggregatorInput `toml:"aggregator"` + Verifier []*committeeverifier.Input `toml:"verifier"` +} + +func decodeCLNodeConfig(raw any) (CLNodeConfig, error) { + b, err := toml.Marshal(raw) + if err != nil { + return CLNodeConfig{}, fmt.Errorf("re-encoding committeeccv_clnode config: %w", err) + } + var cfg CLNodeConfig + if err := toml.Unmarshal(b, &cfg); err != nil { + return CLNodeConfig{}, fmt.Errorf("decoding committeeccv_clnode config: %w", err) + } + if err := devenvruntime.CheckConfigVersion(cfg.Version, clNodeVersion); err != nil { + return CLNodeConfig{}, err + } + return cfg, nil +} + +// launchCLNodes bakes secrets into node specs, launches the CL node sets, then +// registers and connects each node to JD. Returns a NodeSetClientLookup and the +// JD node IDs for the launched nodes; both are nil/empty when no node sets are +// configured. +func launchCLNodes( + ctx context.Context, + cfg *CLNodeConfig, + verifiers []*committeeverifier.Input, + aggregators []*services.AggregatorInput, + topology *ccvdeployment.EnvironmentTopology, + blockchains []*ctfblockchain.Input, + jdInfra *jobs.JDInfrastructure, +) (*jobs.NodeSetClientLookup, []string, error) { + if cfg == nil || len(cfg.NodeSets) == 0 { + return nil, nil, nil + } + if err := bakeNodeSecrets(cfg, verifiers, aggregators, topology); err != nil { + return nil, nil, fmt.Errorf("baking node secrets: %w", err) + } + if err := launchNodeSets(ctx, cfg, blockchains); err != nil { + return nil, nil, fmt.Errorf("launching CL node sets: %w", err) + } + clNopAliases := clModeNOPAliases(topology) + if len(clNopAliases) == 0 { + return nil, nil, nil + } + clientLookup, err := jobs.NewNodeSetClientLookup(cfg.NodeSets, clNopAliases) + if err != nil { + return nil, nil, fmt.Errorf("building CL node client lookup: %w", err) + } + if clientLookup == nil { + return nil, nil, nil + } + if err := jobs.RegisterNodesWithJD(ctx, jdInfra, clientLookup, clNopAliases); err != nil { + return nil, nil, fmt.Errorf("registering CL nodes with JD: %w", err) + } + chainIDs := make([]string, len(blockchains)) + for i, bc := range blockchains { + chainIDs[i] = bc.ChainID + } + if err := jobs.ConnectNodesToJD(ctx, jdInfra, clientLookup, chainIDs); err != nil { + return nil, nil, fmt.Errorf("connecting CL nodes to JD: %w", err) + } + // The deployment environment was built in Phase 2 (protocol_contracts) before + // these CL nodes registered, so its NodeIDs are empty. Return the JD node IDs + // so the caller can populate its local env copy (FetchNOPSigningKeys needs them). + return clientLookup, jdInfra.GetNodeIDs(), nil +} + +// bakeNodeSecrets sets TestSecretsOverrides on each CL node spec before launch. +// Must be called after HMAC credentials are generated (step 1) and before +// launchNodeSets, because CL node secrets are boot-only. +func bakeNodeSecrets( + cfg *CLNodeConfig, + verifiers []*committeeverifier.Input, + aggregators []*services.AggregatorInput, + topology *ccvdeployment.EnvironmentTopology, +) error { + // Build topology aggregator names per committee (matches changeset ordering). + topoAggNames := make(map[string][]string) + if topology.NOPTopology != nil { + for name, committee := range topology.NOPTopology.Committees { + names := make([]string, len(committee.Aggregators)) + for i, a := range committee.Aggregators { + names[i] = a.Name + } + topoAggNames[name] = names + } + } + + aggsByCommittee := make(map[string][]*services.AggregatorInput) + for _, agg := range aggregators { + if agg != nil { + aggsByCommittee[agg.CommitteeName] = append(aggsByCommittee[agg.CommitteeName], agg) + } + } + + // Flatten node specs in node-set order; order must match NOP index ordering. + type nodeSpecEntry struct { + nodeSetIdx int + specIdx int + } + var nodeSpecOrder []nodeSpecEntry + for i, nodeSet := range cfg.NodeSets { + for j := range nodeSet.NodeSpecs { + nodeSpecOrder = append(nodeSpecOrder, nodeSpecEntry{i, j}) + } + } + + aggSecretsPerNode := make(map[int][]clNodeAggregatorSecret) + for _, ver := range verifiers { + if ver == nil || ver.Mode != services.CL { + continue + } + index, ok := topology.NOPTopology.GetNOPIndex(ver.NOPAlias) + if !ok { + return fmt.Errorf("NOP alias %q not found in topology for verifier %s", ver.NOPAlias, ver.ContainerName) + } + if index >= len(nodeSpecOrder) { + return fmt.Errorf("node index %d for NOPAlias %s exceeds available CL nodes (%d)", + index, ver.NOPAlias, len(nodeSpecOrder)) + } + committeeAggs := aggsByCommittee[ver.CommitteeName] + if len(committeeAggs) == 0 { + return fmt.Errorf("no aggregators found for committee %q (verifier %s)", ver.CommitteeName, ver.ContainerName) + } + committeeTopoNames := topoAggNames[ver.CommitteeName] + + for aggIdx, agg := range committeeAggs { + aggName := agg.InstanceName() + if aggIdx < len(committeeTopoNames) { + aggName = committeeTopoNames[aggIdx] + } + apiKeys, err := agg.GetAPIKeys() + if err != nil { + return fmt.Errorf("getting API keys for aggregator %s: %w", agg.InstanceName(), err) + } + var found bool + for _, apiClient := range apiKeys { + if apiClient.ClientID != ver.ContainerName { + continue + } + if len(apiClient.APIKeyPairs) == 0 { + return fmt.Errorf("no API key pairs for client %s on aggregator %s", + apiClient.ClientID, agg.InstanceName()) + } + pair := apiClient.APIKeyPairs[0] + verifierID := ccvshared.NewVerifierJobID( + ccvshared.NOPAlias(ver.NOPAlias), + aggName, + ccvshared.VerifierJobScope{CommitteeQualifier: ver.CommitteeName}, + ).GetVerifierID() + aggSecretsPerNode[index] = append(aggSecretsPerNode[index], clNodeAggregatorSecret{ + VerifierID: verifierID, + APIKey: pair.APIKey, + APISecret: pair.Secret, + }) + found = true + break + } + if !found { + return fmt.Errorf("API client %q not found on aggregator %s", + ver.ContainerName, agg.InstanceName()) + } + } + } + + for flatIdx, entry := range nodeSpecOrder { + if len(aggSecretsPerNode[flatIdx]) == 0 { + continue + } + nodeSpec := cfg.NodeSets[entry.nodeSetIdx].NodeSpecs[entry.specIdx] + secrets := clNodeSecrets{ + CCV: clNodeCCVSecrets{ + AggregatorSecrets: aggSecretsPerNode[flatIdx], + }, + } + secretsToml, err := secrets.tomlString() + if err != nil { + return fmt.Errorf("marshaling secrets for node %d: %w", flatIdx, err) + } + nodeSpec.Node.TestSecretsOverrides = secretsToml + } + return nil +} + +// clModeNOPAliases returns, in topology order, the aliases of NOPs running in +// CL mode. The order matches the CL node ordering used by NewNodeSetClientLookup. +func clModeNOPAliases(topology *ccvdeployment.EnvironmentTopology) []string { + if topology == nil || topology.NOPTopology == nil { + return nil + } + var aliases []string + for _, nop := range topology.NOPTopology.NOPs { + if nop.GetMode() == ccvshared.NOPModeCL { + aliases = append(aliases, nop.Alias) + } + } + return aliases +} + +// launchNodeSets configures, launches, and funds the CL node sets in cfg. +// TestSecretsOverrides must already be set on each node spec before this is +// called (bakeNodeSecrets handles that), because CL node secrets are boot-only. +func launchNodeSets(ctx context.Context, cfg *CLNodeConfig, blockchains []*ctfblockchain.Input) error { + if cfg == nil || len(cfg.NodeSets) == 0 { + return nil + } + if len(blockchains) == 0 { + return fmt.Errorf("committeeccv_clnode: no blockchains available to configure CL nodes") + } + + impls := make([]cciptestinterfaces.CCIP17Configuration, 0, len(blockchains)) + for _, bc := range blockchains { + impl, ierr := chainreg.NewProductConfigurationFromNetwork(bc.Type) + if ierr != nil { + return fmt.Errorf("committeeccv_clnode: impl for %q: %w", bc.Type, ierr) + } + impls = append(impls, impl) + } + + chainConfigs := []string{commonCLNodesConfig} + for i, impl := range impls { + cc, cerr := impl.ConfigureNodes(ctx, blockchains[i]) + if cerr != nil { + return fmt.Errorf("committeeccv_clnode: ConfigureNodes for %q: %w", blockchains[i].Type, cerr) + } + chainConfigs = append(chainConfigs, cc) + } + allConfigs := strings.Join(chainConfigs, "\n") + for _, nodeSet := range cfg.NodeSets { + for _, nodeSpec := range nodeSet.NodeSpecs { + nodeSpec.Node.TestConfigOverrides = allConfigs + } + } + + for _, nodeSet := range cfg.NodeSets { + if _, nerr := ns.NewSharedDBNodeSet(nodeSet, nil); nerr != nil { + return fmt.Errorf("committeeccv_clnode: NewSharedDBNodeSet %q: %w", nodeSet.Name, nerr) + } + } + + link := toBigUnits(cfg.CLNodesFundingLink, 1) + native := toBigUnits(cfg.CLNodesFundingETH, 5) + for i, impl := range impls { + if ferr := impl.FundNodes(ctx, cfg.NodeSets, blockchains[i], link, native); ferr != nil { + return fmt.Errorf("committeeccv_clnode: FundNodes on %q: %w", blockchains[i].Type, ferr) + } + } + return nil +} + +func toBigUnits(v float64, def int64) *big.Int { + if v <= 0 { + return big.NewInt(def) + } + return big.NewInt(int64(v)) +} + +// commonCLNodesConfig is the base TOML applied to every CL node. It mirrors +// ccv.CommonCLNodesConfig; components cannot import the ccv package (it +// blank-imports every component, which would create a cycle). +const commonCLNodesConfig = ` +[Log] +JSONConsole = true +Level = 'info' +[Pyroscope] +ServerAddress = 'http://host.docker.internal:4040' +Environment = 'local' +[WebServer] +SessionTimeout = '999h0m0s' +HTTPWriteTimeout = '3m' +SecureCookies = false +HTTPPort = 6688 +AllowOrigins = 'http://localhost:3000' +[WebServer.TLS] +HTTPSPort = 0 +[WebServer.RateLimit] +Authenticated = 5000 +Unauthenticated = 5000 +[JobPipeline] +[JobPipeline.HTTPRequest] +DefaultTimeout = '1m' +[Log.File] +MaxSize = '0b' +[Feature] +FeedsManager = true +LogPoller = true +UICSAKeys = true +[OCR2] +Enabled = true +SimulateTransactions = false +DefaultTransactionQueueDepth = 1 +[P2P.V2] +Enabled = true +ListenAddresses = ['0.0.0.0:6690'] +` + +// clNodeSecrets, clNodeCCVSecrets, and clNodeAggregatorSecret are the TOML +// types written to TestSecretsOverrides on each CL node spec. They mirror the +// identically-named types in the ccv package (environment.go); duplicated here +// to avoid the import cycle described above. +type clNodeSecrets struct { + CCV clNodeCCVSecrets `toml:",omitempty"` +} + +func (s *clNodeSecrets) tomlString() (string, error) { + data, err := toml.Marshal(s) + if err != nil { + return "", fmt.Errorf("failed to marshal CCV secrets to TOML: %w", err) + } + return string(data), nil +} + +type clNodeCCVSecrets struct { + AggregatorSecrets []clNodeAggregatorSecret `toml:",omitempty"` +} + +type clNodeAggregatorSecret struct { + VerifierID string `toml:",omitempty"` + APIKey string `toml:",omitempty"` + APISecret string `toml:",omitempty"` +} diff --git a/build/devenv/env-cl-ci-phased.toml b/build/devenv/env-cl-ci-phased.toml index ba92c9c35..e9eb378da 100644 --- a/build/devenv/env-cl-ci-phased.toml +++ b/build/devenv/env-cl-ci-phased.toml @@ -1,29 +1,32 @@ -## CI-specific override: use pre-built local images instead of building from Dockerfile -[clnode] +## CI-specific override: use pre-built local images instead of building from +## Dockerfile. Layer on top of env-phased.toml,env-cl-phased.toml in CI. +## Replaces the [[committeeccv_clnode.node_sets]] array with specs that pull +## the pre-built local:latest image (uploaded by the build-cl-image job). +[committeeccv_clnode] version = 1 cl_nodes_funding_eth = 50 cl_nodes_funding_link = 50 -[[clnode.node_sets]] +[[committeeccv_clnode.node_sets]] name = "don" nodes = 2 override_mode = "each" - [clnode.node_sets.db] + [committeeccv_clnode.node_sets.db] image = "postgres:15.0" - [[clnode.node_sets.node_specs]] + [[committeeccv_clnode.node_sets.node_specs]] - [clnode.node_sets.node_specs.node] + [committeeccv_clnode.node_sets.node_specs.node] image = "docker.io/library/local:latest" - [clnode.node_sets.node_specs.node.env_vars] + [committeeccv_clnode.node_sets.node_specs.node.env_vars] CL_EVM_CMD="" - [[clnode.node_sets.node_specs]] + [[committeeccv_clnode.node_sets.node_specs]] - [clnode.node_sets.node_specs.node] + [committeeccv_clnode.node_sets.node_specs.node] image = "docker.io/library/local:latest" - [clnode.node_sets.node_specs.node.env_vars] + [committeeccv_clnode.node_sets.node_specs.node.env_vars] CL_EVM_CMD="" diff --git a/build/devenv/env-cl-phased.toml b/build/devenv/env-cl-phased.toml index 3d115958f..8d30c9ae8 100644 --- a/build/devenv/env-cl-phased.toml +++ b/build/devenv/env-cl-phased.toml @@ -3,116 +3,206 @@ # Layer this on top of env-phased.toml: # ccv --env-mode phased up env-phased.toml,env-cl-phased.toml # -# CL mode is a property of the verifier/executor (mode = "cl"): instead of -# running as standalone containers, they run as jobs on shared Chainlink nodes. -# SCOPE: only the verifiers run in CL mode here. The executors stay standalone -# (their NOPs/pools/inputs are left as env-phased.toml defines them). +# CL mode is a property of the verifier (mode = "cl"): instead of running as +# standalone containers, verifier jobs run on shared Chainlink nodes. +# SCOPE: only the verifiers run in CL mode here. The executors stay standalone. # # Config merge (deepMergeMaps) merges nested *maps* key-by-key but replaces -# *arrays* wholesale, so this file respecifies every array that must change: the -# NOP list (now the two shared CL nodes plus the unchanged standalone executor -# NOPs), each committee's nop_aliases, and the verifier list. The executor -# section, executor_pools, and aggregators are inherited from env-phased.toml. -# Monitoring/pyroscope stay in the [observability] component (not here). +# *arrays* wholesale. This overlay: +# 1. Clears [committeeccv] (empty aggregator/verifier arrays → no-op early +# return in the standalone component, letting committeeccv_clnode take over). +# 2. Adds [committeeccv_clnode] with the full self-contained CL-node config +# (node sets + aggregators mirrored from env-phased.toml + CL-mode verifiers). +# 3. Replaces the NOP and committee topology with the two shared CL nodes. +# The executor section, executor_pools, blockchains, observability, jd, and +# indexer sections are all inherited from env-phased.toml. ############################# -# CL node sets (clnode component, config vehicle) +# Disable the standalone committeeccv component so it is a no-op when this +# overlay is applied. The committeeccv_clnode component below takes over. ############################# -[clnode] +[committeeccv] +version = 1 +aggregator = [] +verifier = [] + +############################# +# CL-node variant of CommitteeCCV (committeeccv_clnode component). +# Contains: CL node sets, aggregators (mirrored from env-phased.toml), and +# CL-mode verifiers. Delete this section (and component_clnode.go) when CL +# nodes leave the devenv. +############################# +[committeeccv_clnode] version = 1 cl_nodes_funding_eth = 50 cl_nodes_funding_link = 50 -[[clnode.node_sets]] +[[committeeccv_clnode.node_sets]] name = "don" nodes = 2 override_mode = "each" - [clnode.node_sets.db] + [committeeccv_clnode.node_sets.db] image = "postgres:15.0" - [[clnode.node_sets.node_specs]] - [clnode.node_sets.node_specs.node] + [[committeeccv_clnode.node_sets.node_specs]] + [committeeccv_clnode.node_sets.node_specs.node] docker_ctx = "../../../chainlink" docker_file = "plugins/chainlink.Dockerfile" - [clnode.node_sets.node_specs.node.env_vars] + [committeeccv_clnode.node_sets.node_specs.node.env_vars] CL_EVM_CMD = "" - [[clnode.node_sets.node_specs]] - [clnode.node_sets.node_specs.node] + [[committeeccv_clnode.node_sets.node_specs]] + [committeeccv_clnode.node_sets.node_specs.node] docker_ctx = "../../../chainlink" docker_file = "plugins/chainlink.Dockerfile" - [clnode.node_sets.node_specs.node.env_vars] + [committeeccv_clnode.node_sets.node_specs.node.env_vars] CL_EVM_CMD = "" -############################# -# Topology NOPs: replace the six standalone verifier NOPs with two shared CL -# node NOPs (node-0, node-1; default NOP mode is CL). The three standalone -# executor NOPs are kept unchanged so the executors (still standalone) and their -# executor_pools continue to resolve. -############################# -[[protocol_contracts.environment_topology.nop_topology.nops]] -alias = "node-0" -name = "node-0" +# Aggregators: mirrors [[committeeccv.aggregator]] in env-phased.toml. +# Aggregator config is identical between standalone and CL-node modes; the +# only change is the component key. +[[committeeccv_clnode.aggregator]] + committee_name = "default" + image = "aggregator:latest" + host_port = 50051 + redundant_aggregators = 1 + source_code_path = "../aggregator" + root_path = "../../" -[[protocol_contracts.environment_topology.nop_topology.nops]] -alias = "node-1" -name = "node-1" + [committeeccv_clnode.aggregator.db] + image = "postgres:16-alpine" + host_port = 7432 -[[protocol_contracts.environment_topology.nop_topology.nops]] -alias = "default-executor-1" -name = "default-executor-1" -mode = "standalone" + [committeeccv_clnode.aggregator.redis] + image = "redis:7-alpine" + host_port = 6379 -[[protocol_contracts.environment_topology.nop_topology.nops]] -alias = "default-executor-2" -name = "default-executor-2" -mode = "standalone" + [[committeeccv_clnode.aggregator.api_clients]] + client_id = "default-verifier-1" + description = "Development default verifier 1" + enabled = true + groups = ["verifiers"] -[[protocol_contracts.environment_topology.nop_topology.nops]] -alias = "custom-executor-1" -name = "custom-executor-1" -mode = "standalone" + [[committeeccv_clnode.aggregator.api_clients]] + client_id = "default-verifier-2" + description = "Development default verifier 2" + enabled = true + groups = ["verifiers"] -############################# -# Committees: repoint membership from the standalone verifier NOPs to the two -# shared CL nodes. (Aggregators, qualifier, verifier_version are inherited.) -############################# -[protocol_contracts.environment_topology.nop_topology.committees.default.chain_configs.3379446385462418246] -nop_aliases = ["node-0", "node-1"] -threshold = 2 -[protocol_contracts.environment_topology.nop_topology.committees.default.chain_configs.12922642891491394802] -nop_aliases = ["node-0", "node-1"] -threshold = 2 -[protocol_contracts.environment_topology.nop_topology.committees.default.chain_configs.4793464827907405086] -nop_aliases = ["node-0", "node-1"] -threshold = 2 + [[committeeccv_clnode.aggregator.api_clients]] + client_id = "monitoring" + description = "Monitoring and infrastructure client" + enabled = true + groups = ["monitoring"] -[protocol_contracts.environment_topology.nop_topology.committees.secondary.chain_configs.3379446385462418246] -nop_aliases = ["node-0", "node-1"] -threshold = 2 -[protocol_contracts.environment_topology.nop_topology.committees.secondary.chain_configs.12922642891491394802] -nop_aliases = ["node-0", "node-1"] -threshold = 2 -[protocol_contracts.environment_topology.nop_topology.committees.secondary.chain_configs.4793464827907405086] -nop_aliases = ["node-0", "node-1"] -threshold = 2 + [[committeeccv_clnode.aggregator.api_clients]] + client_id = "indexer" + description = "Development Indexer" + enabled = true + groups = ["indexer"] -[protocol_contracts.environment_topology.nop_topology.committees.tertiary.chain_configs.3379446385462418246] -nop_aliases = ["node-0", "node-1"] -threshold = 2 -[protocol_contracts.environment_topology.nop_topology.committees.tertiary.chain_configs.12922642891491394802] -nop_aliases = ["node-0", "node-1"] -threshold = 2 -[protocol_contracts.environment_topology.nop_topology.committees.tertiary.chain_configs.4793464827907405086] -nop_aliases = ["node-0", "node-1"] -threshold = 2 + [committeeccv_clnode.aggregator.env] + storage_connection_url = "postgresql://aggregator:aggregator@default-aggregator-db:5432/aggregator?sslmode=disable" + redis_address = "default-aggregator-redis:6379" + redis_password = "" + redis_db = "0" + +[[committeeccv_clnode.aggregator]] + committee_name = "secondary" + image = "aggregator:latest" + host_port = 50052 + redundant_aggregators = 0 + source_code_path = "../aggregator" + root_path = "../../" + + [committeeccv_clnode.aggregator.db] + image = "postgres:16-alpine" + host_port = 7433 + + [committeeccv_clnode.aggregator.redis] + image = "redis:7-alpine" + host_port = 6380 + + [[committeeccv_clnode.aggregator.api_clients]] + client_id = "secondary-verifier-1" + description = "Development secondary verifier 1" + enabled = true + groups = ["verifiers"] + + [[committeeccv_clnode.aggregator.api_clients]] + client_id = "secondary-verifier-2" + description = "Development secondary verifier 2" + enabled = true + groups = ["verifiers"] + + [[committeeccv_clnode.aggregator.api_clients]] + client_id = "monitoring" + description = "Monitoring and infrastructure client" + enabled = true + groups = ["monitoring"] + + [[committeeccv_clnode.aggregator.api_clients]] + client_id = "indexer" + description = "Development Indexer" + enabled = true + groups = ["indexer"] + + [committeeccv_clnode.aggregator.env] + storage_connection_url = "postgresql://aggregator:aggregator@secondary-aggregator-db:5432/aggregator?sslmode=disable" + redis_address = "secondary-aggregator-redis:6379" + redis_password = "" + redis_db = "0" + +[[committeeccv_clnode.aggregator]] + committee_name = "tertiary" + image = "aggregator:latest" + host_port = 50053 + redundant_aggregators = 0 + source_code_path = "../aggregator" + root_path = "../../" + + [committeeccv_clnode.aggregator.db] + image = "postgres:16-alpine" + host_port = 7434 + + [committeeccv_clnode.aggregator.redis] + image = "redis:7-alpine" + host_port = 6381 + + [[committeeccv_clnode.aggregator.api_clients]] + client_id = "tertiary-verifier-1" + description = "Development tertiary verifier 1" + enabled = true + groups = ["verifiers"] + + [[committeeccv_clnode.aggregator.api_clients]] + client_id = "tertiary-verifier-2" + description = "Development tertiary verifier 2" + enabled = true + groups = ["verifiers"] + + [[committeeccv_clnode.aggregator.api_clients]] + client_id = "monitoring" + description = "Monitoring and infrastructure client" + enabled = true + groups = ["monitoring"] + + [[committeeccv_clnode.aggregator.api_clients]] + client_id = "indexer" + description = "Development Indexer" + enabled = true + groups = ["indexer"] + + [committeeccv_clnode.aggregator.env] + storage_connection_url = "postgresql://aggregator:aggregator@tertiary-aggregator-db:5432/aggregator?sslmode=disable" + redis_address = "tertiary-aggregator-redis:6379" + redis_password = "" + redis_db = "0" -############################# # Verifiers: CL mode, mapped onto the two nodes. Replaces the standalone # verifier list from env-phased.toml wholesale. -############################# -[[committeeccv.verifier]] +[[committeeccv_clnode.verifier]] mode = "cl" image = "verifier:latest" container_name = "default-verifier-1" @@ -123,12 +213,12 @@ threshold = 2 committee_name = "default" node_index = 0 insecure_aggregator_connection = true - [committeeccv.verifier.db] + [committeeccv_clnode.verifier.db] image = "postgres:16-alpine" name = "default-verifier-1-db" port = 8432 -[[committeeccv.verifier]] +[[committeeccv_clnode.verifier]] mode = "cl" image = "verifier:latest" container_name = "default-verifier-2" @@ -139,12 +229,12 @@ threshold = 2 committee_name = "default" node_index = 1 insecure_aggregator_connection = true - [committeeccv.verifier.db] + [committeeccv_clnode.verifier.db] image = "postgres:16-alpine" name = "default-verifier-2-db" port = 8433 -[[committeeccv.verifier]] +[[committeeccv_clnode.verifier]] mode = "cl" image = "verifier:latest" container_name = "secondary-verifier-1" @@ -155,12 +245,12 @@ threshold = 2 committee_name = "secondary" node_index = 0 insecure_aggregator_connection = true - [committeeccv.verifier.db] + [committeeccv_clnode.verifier.db] image = "postgres:16-alpine" name = "secondary-verifier-1-db" port = 8434 -[[committeeccv.verifier]] +[[committeeccv_clnode.verifier]] mode = "cl" image = "verifier:latest" container_name = "secondary-verifier-2" @@ -171,12 +261,12 @@ threshold = 2 committee_name = "secondary" node_index = 1 insecure_aggregator_connection = true - [committeeccv.verifier.db] + [committeeccv_clnode.verifier.db] image = "postgres:16-alpine" name = "secondary-verifier-2-db" port = 8435 -[[committeeccv.verifier]] +[[committeeccv_clnode.verifier]] mode = "cl" image = "verifier:latest" container_name = "tertiary-verifier-1" @@ -187,12 +277,12 @@ threshold = 2 committee_name = "tertiary" node_index = 0 insecure_aggregator_connection = true - [committeeccv.verifier.db] + [committeeccv_clnode.verifier.db] image = "postgres:16-alpine" name = "tertiary-verifier-1-db" port = 8436 -[[committeeccv.verifier]] +[[committeeccv_clnode.verifier]] mode = "cl" image = "verifier:latest" container_name = "tertiary-verifier-2" @@ -203,7 +293,70 @@ threshold = 2 committee_name = "tertiary" node_index = 1 insecure_aggregator_connection = true - [committeeccv.verifier.db] + [committeeccv_clnode.verifier.db] image = "postgres:16-alpine" name = "tertiary-verifier-2-db" port = 8437 + +############################# +# Topology NOPs: replace the six standalone verifier NOPs with two shared CL +# node NOPs (node-0, node-1; default NOP mode is CL). The three standalone +# executor NOPs are kept unchanged so the executors (still standalone) and their +# executor_pools continue to resolve. +############################# +[[protocol_contracts.environment_topology.nop_topology.nops]] +alias = "node-0" +name = "node-0" + +[[protocol_contracts.environment_topology.nop_topology.nops]] +alias = "node-1" +name = "node-1" + +[[protocol_contracts.environment_topology.nop_topology.nops]] +alias = "default-executor-1" +name = "default-executor-1" +mode = "standalone" + +[[protocol_contracts.environment_topology.nop_topology.nops]] +alias = "default-executor-2" +name = "default-executor-2" +mode = "standalone" + +[[protocol_contracts.environment_topology.nop_topology.nops]] +alias = "custom-executor-1" +name = "custom-executor-1" +mode = "standalone" + +############################# +# Committees: repoint membership from the standalone verifier NOPs to the two +# shared CL nodes. (Aggregators, qualifier, verifier_version are inherited.) +############################# +[protocol_contracts.environment_topology.nop_topology.committees.default.chain_configs.3379446385462418246] +nop_aliases = ["node-0", "node-1"] +threshold = 2 +[protocol_contracts.environment_topology.nop_topology.committees.default.chain_configs.12922642891491394802] +nop_aliases = ["node-0", "node-1"] +threshold = 2 +[protocol_contracts.environment_topology.nop_topology.committees.default.chain_configs.4793464827907405086] +nop_aliases = ["node-0", "node-1"] +threshold = 2 + +[protocol_contracts.environment_topology.nop_topology.committees.secondary.chain_configs.3379446385462418246] +nop_aliases = ["node-0", "node-1"] +threshold = 2 +[protocol_contracts.environment_topology.nop_topology.committees.secondary.chain_configs.12922642891491394802] +nop_aliases = ["node-0", "node-1"] +threshold = 2 +[protocol_contracts.environment_topology.nop_topology.committees.secondary.chain_configs.4793464827907405086] +nop_aliases = ["node-0", "node-1"] +threshold = 2 + +[protocol_contracts.environment_topology.nop_topology.committees.tertiary.chain_configs.3379446385462418246] +nop_aliases = ["node-0", "node-1"] +threshold = 2 +[protocol_contracts.environment_topology.nop_topology.committees.tertiary.chain_configs.12922642891491394802] +nop_aliases = ["node-0", "node-1"] +threshold = 2 +[protocol_contracts.environment_topology.nop_topology.committees.tertiary.chain_configs.4793464827907405086] +nop_aliases = ["node-0", "node-1"] +threshold = 2 diff --git a/build/devenv/environment.go b/build/devenv/environment.go index b3fdca339..c1757ba99 100644 --- a/build/devenv/environment.go +++ b/build/devenv/environment.go @@ -27,7 +27,6 @@ import ( ccldf "github.com/smartcontractkit/chainlink-ccv/build/devenv/cldf" devenvcommon "github.com/smartcontractkit/chainlink-ccv/build/devenv/common" _ "github.com/smartcontractkit/chainlink-ccv/build/devenv/components/blockchains" - _ "github.com/smartcontractkit/chainlink-ccv/build/devenv/components/clnode" _ "github.com/smartcontractkit/chainlink-ccv/build/devenv/components/committeeccv" _ "github.com/smartcontractkit/chainlink-ccv/build/devenv/components/executor" _ "github.com/smartcontractkit/chainlink-ccv/build/devenv/components/fake" From e0ccd63198ee7ca1e8c220b0851d7661c850c637 Mon Sep 17 00:00:00 2001 From: Will Winder Date: Mon, 1 Jun 2026 12:55:16 -0400 Subject: [PATCH 8/8] add changelog --- .../2026-06-01_phased_clnode_component.md | 106 ++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 changelog/2026-06-01_phased_clnode_component.md diff --git a/changelog/2026-06-01_phased_clnode_component.md b/changelog/2026-06-01_phased_clnode_component.md new file mode 100644 index 000000000..c13030643 --- /dev/null +++ b/changelog/2026-06-01_phased_clnode_component.md @@ -0,0 +1,106 @@ +# Phased devenv: CL-node support restored as a clean component + +## Summary + +Restores Chainlink-node ("CL-node") verifier mode in the phased devenv +(`--env-mode phased`) by introducing a dedicated `committeeccv_clnode` +Phase 3 component. The former `clnode` Phase-1 config-vehicle package is +deleted; all CL-node logic is now isolated in one file designed for future +removal when CL nodes leave the devenv. + +--- + +## Breaking change: `[clnode]` config key removed + +The `[clnode]` TOML section is no longer recognized. CL-node devenv +configuration moves into `[committeeccv_clnode]`, which embeds both the +committee fields (aggregators, verifiers) and the CL-node fields (node +sets, funding amounts). + +| What | Before | After | +|------|--------|-------| +| Config section for node sets | `[clnode]` | `[committeeccv_clnode]` | +| Aggregators in CL-node mode | `[[committeeccv.aggregator]]` | `[[committeeccv_clnode.aggregator]]` | +| CL-mode verifiers | `[[committeeccv.verifier]]` with `mode = "cl"` | `[[committeeccv_clnode.verifier]]` with `mode = "cl"` | +| Disable standalone in CL overlay | (implicit — only one ran) | `[committeeccv]` with `aggregator = []`, `verifier = []` | + +Before (`env-cl-phased.toml`): +```toml +[clnode] +version = 1 +cl_nodes_funding_eth = 50 +[[clnode.node_sets]] + name = "don" + nodes = 2 + ... + +[[committeeccv.verifier]] + mode = "cl" + ... +``` + +After (`env-cl-phased.toml`): +```toml +# Disable the standalone component (becomes a no-op) +[committeeccv] +version = 1 +aggregator = [] +verifier = [] + +# CL-node variant: self-contained, delete when CL nodes leave +[committeeccv_clnode] +version = 1 +cl_nodes_funding_eth = 50 +[[committeeccv_clnode.node_sets]] + name = "don" + nodes = 2 + ... +[[committeeccv_clnode.aggregator]] + ... +[[committeeccv_clnode.verifier]] + mode = "cl" + ... +``` + +--- + +## New: `committeeccv_clnode` Phase 3 component + +Registered under config key `"committeeccv_clnode"` in +`build/devenv/components/committeeccv/component_clnode.go`. It is a +drop-in replacement for the standalone `committeeccv` component in +CL-node mode: runs all the same Phase 3 steps, plus step 1b which bakes +aggregator HMAC secrets into node specs before boot and launches, +registers, and connects the CL node sets to JD. + +The component publishes `"_clnode_clients"` (a `*jobs.NodeSetClientLookup`) +which the effect executor uses to call `AcceptPendingJobs` before +`SyncAndVerifyJobProposals`. + +To remove CL nodes from the devenv later: delete `component_clnode.go` +and the `[committeeccv_clnode]` sections from the config overlays. + +--- + +## New: phased CL-node CI test + +`test-cl-smoke.yaml` gains a `Phased TestE2ESmoke_Basic` matrix entry +that runs `TestE2ESmoke_Basic_Phased` with: + +``` +config: env-phased.toml,env-cl-phased.toml,env-cl-ci-phased.toml +flags: --env-mode phased +``` + +`env-cl-ci-phased.toml` is updated from `[clnode]` to +`[committeeccv_clnode]` so the CI image override applies to the correct +component. + +--- + +## Internal refactor: `committeeccv` shared helpers + +`committeeccv/component.go` extracts `phase3Inputs`, `parsePhase3Inputs`, +`ensureAggregatorCredentials`, and `runPhase3Core` so both components +share the same Phase 3 logic without duplication. No behavioral change to +the standalone path.