diff --git a/.github/workflows/test-cl-smoke.yaml b/.github/workflows/test-cl-smoke.yaml index f2544464a..038a698e6 100644 --- a/.github/workflows/test-cl-smoke.yaml +++ b/.github/workflows/test-cl-smoke.yaml @@ -79,6 +79,12 @@ jobs: run_cmd: TestHA_CrossComponentDown config: "env.toml,env-HA.toml,env-cl.toml,env-cl-ci.toml" timeout: 10m + - name: Phased TestE2ESmoke_Basic + run_cmd: TestE2ESmoke_Basic_Phased + config: "env-phased.toml,env-cl-phased.toml,env-cl-ci-phased.toml" + flags: --env-mode phased + smoke_test_config: ../../env-phased-out.toml + timeout: 15m # We need to configure HeadTracker for the CL tests to have finality depth. Otherwise, it does instant finality. # - name: TestE2EReorg # config: "env.toml,env-src-auto-mine.toml,env-cl.toml,env-cl-ci.toml" @@ -125,7 +131,7 @@ jobs: role-to-assume: ${{ secrets.CCV_IAM_ROLE }} aws-region: us-east-1 registry-type: public - + - name: Authenticate to AWS ECR (JD) uses: ./.github/actions/aws-ecr-auth with: @@ -162,18 +168,20 @@ jobs: env: JD_IMAGE: ${{ secrets.JD_IMAGE }} run: | - ccv u ${{ matrix.test.config }} + ccv ${{ matrix.test.flags || '' }} u ${{ matrix.test.config }} - name: Run Smoke Test id: test_run working-directory: build/devenv/tests/e2e env: LOKI_URL: http://localhost:3030/loki/api/v1/push + # When unset, the test falls back to ../../env-out.toml (see GetSmokeTestConfig). + SMOKE_TEST_CONFIG: ${{ matrix.test.smoke_test_config || '' }} run: | set -o pipefail go test -v -timeout ${{ matrix.test.timeout }} -count=1 -run ${{ matrix.test.run_cmd }} continue-on-error: true - + - name: Dump logs if they're not already dumped if: always() working-directory: build/devenv/tests/e2e @@ -184,7 +192,7 @@ jobs: else echo "Logs already exist. Skipping manual dump." fi - + - name: Sanitize test name for log artifact name if: always() id: sanitize_name @@ -200,7 +208,7 @@ jobs: name: container-logs-cl-${{ steps.sanitize_name.outputs.name }} path: build/devenv/tests/e2e/logs retention-days: 1 - + - name: Check test results if: always() && steps.test_run.outcome == 'failure' run: | diff --git a/.github/workflows/test-smoke.yaml b/.github/workflows/test-smoke.yaml index 8d59dfeb2..206865554 100644 --- a/.github/workflows/test-smoke.yaml +++ b/.github/workflows/test-smoke.yaml @@ -78,7 +78,7 @@ jobs: config: env.toml,env-src-auto-mine.toml timeout: 10m working-directory: build/devenv/tests/e2e - - name: TestE2ESmoke_Basic_Phased + - name: Phased TestE2ESmoke_Basic run_cmd: TestE2ESmoke_Basic config: env-phased.toml flags: --env-mode phased diff --git a/build/devenv/components/committeeccv/component.go b/build/devenv/components/committeeccv/component.go index 5ffaa690b..00ff0a8ca 100644 --- a/build/devenv/components/committeeccv/component.go +++ b/build/devenv/components/committeeccv/component.go @@ -68,72 +68,123 @@ func (c *component) RunPhase3( componentConfig any, priorOutputs map[string]any, ) (map[string]any, []devenvruntime.Effect, error) { - // The committeeccv config is a single [committeeccv] section holding the - // aggregator and verifier inputs. cfg, err := decodeConfig(componentConfig) if err != nil { return nil, nil, err } aggregators, verifiers := cfg.Aggregator, cfg.Verifier - if len(aggregators) == 0 && len(verifiers) == 0 { return map[string]any{}, nil, nil } + inputs, err := parsePhase3Inputs(priorOutputs, globalConfig) + if err != nil { + return nil, nil, err + } + // Work on a copy of the shared Phase-2 environment. + localEnv := *inputs.env + if err := ensureAggregatorCredentials(aggregators); err != nil { + return nil, nil, err + } + return runPhase3Core(ctx, inputs, aggregators, verifiers, &localEnv) +} + +// phase3Inputs holds the decoded prior-phase outputs consumed by both the +// standalone and CL-node CommitteeCCV components. +type phase3Inputs struct { + jdInfra *jobs.JDInfrastructure + blockchains []*ctfblockchain.Input + blockchainOutputs []*ctfblockchain.Output + env *deployment.Environment + topology *ccvdeployment.EnvironmentTopology + obs *observability.Observability + ds datastore.MutableDataStore + impls []cciptestinterfaces.CCIP17Configuration + selectors []uint64 + useLegacyConfigureLane bool +} +func parsePhase3Inputs(priorOutputs, globalConfig map[string]any) (phase3Inputs, error) { jdInfra, ok := priorOutputs["jd"].(*jobs.JDInfrastructure) if !ok || jdInfra == nil { - return nil, nil, fmt.Errorf("committeeccv: jd not found in phase outputs") + return phase3Inputs{}, fmt.Errorf("committeeccv: jd not found in phase outputs") } blockchains, ok := priorOutputs["blockchains"].([]*ctfblockchain.Input) if !ok { - return nil, nil, fmt.Errorf("committeeccv: blockchains not found in phase outputs") + return phase3Inputs{}, fmt.Errorf("committeeccv: blockchains not found in phase outputs") } blockchainOutputs := blockchainscomp.Outputs(blockchains) e, ok := priorOutputs["_env"].(*deployment.Environment) if !ok || e == nil { - return nil, nil, fmt.Errorf("committeeccv: _env not found in phase outputs") + return phase3Inputs{}, fmt.Errorf("committeeccv: _env not found in phase outputs") } topology, ok := priorOutputs["environment_topology"].(*ccvdeployment.EnvironmentTopology) if !ok || topology == nil { - return nil, nil, fmt.Errorf("committeeccv: environment_topology not found in phase outputs") + return phase3Inputs{}, fmt.Errorf("committeeccv: environment_topology not found in phase outputs") } obs, ok := priorOutputs["observability"].(*observability.Observability) if !ok || obs == nil { - return nil, nil, fmt.Errorf("committeeccv: observability not found in phase outputs") + return phase3Inputs{}, fmt.Errorf("committeeccv: observability not found in phase outputs") } ds, ok := priorOutputs["_ds"].(datastore.MutableDataStore) if !ok { - return nil, nil, fmt.Errorf("committeeccv: _ds not found in phase outputs") + return phase3Inputs{}, fmt.Errorf("committeeccv: _ds not found in phase outputs") } impls, _ := priorOutputs["_impls"].([]cciptestinterfaces.CCIP17Configuration) - var useLegacyConfigureLane bool + selectors, _ := priorOutputs["_selectors"].([]uint64) + var useLegacy bool if pcMap, ok := globalConfig["protocol_contracts"].(map[string]any); ok { - useLegacyConfigureLane, _ = pcMap["use_legacy_configure_lane"].(bool) - } + useLegacy, _ = pcMap["use_legacy_configure_lane"].(bool) + } + return phase3Inputs{ + jdInfra: jdInfra, + blockchains: blockchains, + blockchainOutputs: blockchainOutputs, + env: e, + topology: topology, + obs: obs, + ds: ds, + impls: impls, + selectors: selectors, + useLegacyConfigureLane: useLegacy, + }, nil +} - // Step 1: Generate HMAC client credentials for all aggregators before launching verifiers. +func ensureAggregatorCredentials(aggregators []*services.AggregatorInput) error { for _, agg := range aggregators { if agg == nil { continue } creds, cerr := agg.EnsureClientCredentials() if cerr != nil { - return nil, nil, fmt.Errorf("committeeccv: failed to ensure client credentials for aggregator %s: %w", agg.CommitteeName, cerr) + return fmt.Errorf("committeeccv: failed to ensure client credentials for aggregator %s: %w", agg.CommitteeName, cerr) } if agg.Out == nil { agg.Out = &services.AggregatorOutput{} } agg.Out.ClientCredentials = creds } + return nil +} +// runPhase3Core runs the shared CommitteeCCV Phase 3 steps (steps 2–8) against +// a local copy of the deployment environment. It is called by both the standalone +// and CL-node components; the CL-node component performs its own step 1b +// (node launch + NodeIDs population) before calling this function. +func runPhase3Core( + ctx context.Context, + inputs phase3Inputs, + aggregators []*services.AggregatorInput, + verifiers []*committeeverifier.Input, + localEnv *deployment.Environment, +) (map[string]any, []devenvruntime.Effect, error) { // Step 2: Launch standalone verifier containers (reads HMAC creds from agg.Out). if err := committeeverifier.LaunchStandaloneVerifiers( - verifiers, aggregators, blockchainOutputs, jdInfra, + verifiers, aggregators, inputs.blockchainOutputs, inputs.jdInfra, chainreg.GetRegistry().GetVerifierModifiers(), ); err != nil { return nil, nil, fmt.Errorf("committeeccv: failed to launch standalone verifiers: %w", err) } - if err := committeeverifier.RegisterStandaloneVerifiersWithJD(ctx, verifiers, jdInfra.OffchainClient); err != nil { + if err := committeeverifier.RegisterStandaloneVerifiersWithJD(ctx, verifiers, inputs.jdInfra.OffchainClient); err != nil { return nil, nil, fmt.Errorf("committeeccv: failed to register standalone verifiers with JD: %w", err) } @@ -151,6 +202,7 @@ func (c *component) RunPhase3( } allHostnames = append(allHostnames, "localhost") tlsCertDir := filepath.Join(util.CCVConfigDir(), "tls-shared") + var err error sharedTLSCerts, err = services.GenerateTLSCertificates(allHostnames, tlsCertDir) if err != nil { return nil, nil, fmt.Errorf("committeeccv: failed to generate shared TLS certificates: %w", err) @@ -167,18 +219,17 @@ func (c *component) RunPhase3( // Step 5: Enrich topology with verifier signer keys. if len(verifiers) > 0 { - ccdeploy.EnrichTopologyWithVerifiers(topology, verifiers) + ccdeploy.EnrichTopologyWithVerifiers(inputs.topology, verifiers) } // Step 5b: Configure lanes. This requires verifiers to be registered in JD (done above) // because ApplyVerifierConfig fetches verifier signing keys from JD by node ID. - if len(impls) > 0 && len(blockchains) > 0 { - selectors, _ := priorOutputs["_selectors"].([]uint64) + if len(inputs.impls) > 0 && len(inputs.blockchains) > 0 { var connectErr error - if useLegacyConfigureLane { - connectErr = ccdeploy.ConnectAllChainsLegacy(impls, blockchains, selectors, e, topology) + if inputs.useLegacyConfigureLane { + connectErr = ccdeploy.ConnectAllChainsLegacy(inputs.impls, inputs.blockchains, inputs.selectors, localEnv, inputs.topology) } else { - connectErr = ccdeploy.ConnectAllChainsCanonical(impls, blockchains, selectors, e, topology) + connectErr = ccdeploy.ConnectAllChainsCanonical(inputs.impls, inputs.blockchains, inputs.selectors, localEnv, inputs.topology) } if connectErr != nil { return nil, nil, fmt.Errorf("committeeccv: configure lanes: %w", connectErr) @@ -191,12 +242,12 @@ func (c *component) RunPhase3( continue } instanceName := agg.InstanceName() - committee, ok := topology.NOPTopology.Committees[agg.CommitteeName] + committee, ok := inputs.topology.NOPTopology.Committees[agg.CommitteeName] if !ok { return nil, nil, fmt.Errorf("committeeccv: committee %q not found in topology", agg.CommitteeName) } cs := ccvchangesets.GenerateAggregatorConfig(ccvadapters.GetRegistry()) - output, err := cs.Apply(*e, ccvchangesets.GenerateAggregatorConfigInput{ + output, err := cs.Apply(*localEnv, ccvchangesets.GenerateAggregatorConfigInput{ ServiceIdentifier: instanceName + "-aggregator", CommitteeQualifier: agg.CommitteeName, ChainSelectors: ccvchangesets.CommitteeChainSelectorsFromTopology(committee), @@ -209,7 +260,7 @@ func (c *component) RunPhase3( return nil, nil, fmt.Errorf("committeeccv: get aggregator config for %q: %w", instanceName, err) } agg.GeneratedCommittee = aggCfg - e.DataStore = output.DataStore.Seal() + localEnv.DataStore = output.DataStore.Seal() } // Step 7: Launch full aggregator containers. @@ -225,16 +276,12 @@ func (c *component) RunPhase3( } // Step 8: Generate verifier job specs and emit job proposal effects. - effects, err := buildVerifierJobSpecEffects(e, verifiers, topology, obs, sharedTLSCerts, blockchainOutputs, ds) + effects, err := buildVerifierJobSpecEffects(localEnv, verifiers, inputs.topology, inputs.obs, sharedTLSCerts, inputs.blockchainOutputs, inputs.ds) if err != nil { return nil, nil, err } return map[string]any{ - // aggregators and verifiers are public (serialized; the phased loader - // reads them and derives endpoint maps). The shared TLS certs are - // runtime-only plumbing, so they use a "_"-prefixed key and are stripped - // from the serialized output. "aggregators": aggregators, "verifiers": verifiers, "_shared_tls_certs": sharedTLSCerts, diff --git a/build/devenv/components/committeeccv/component_clnode.go b/build/devenv/components/committeeccv/component_clnode.go new file mode 100644 index 000000000..1f04ee0ae --- /dev/null +++ b/build/devenv/components/committeeccv/component_clnode.go @@ -0,0 +1,432 @@ +// Package committeeccv contains two registered Phase 3 components: +// - "committeeccv" (component.go) — standalone verifier/aggregator setup +// - "committeeccv_clnode" (this file) — CL-node variant; replaces committeeccv when +// Chainlink nodes host the verifier jobs. Absorbs the former clnode Phase-1 +// config-vehicle component. Delete this file (and its [committeeccv_clnode] +// config section) when CL nodes leave the devenv. +package committeeccv + +import ( + "context" + "fmt" + "math/big" + "strings" + + "github.com/pelletier/go-toml/v2" + + "github.com/smartcontractkit/chainlink-ccv/build/devenv/cciptestinterfaces" + "github.com/smartcontractkit/chainlink-ccv/build/devenv/chainreg" + "github.com/smartcontractkit/chainlink-ccv/build/devenv/jobs" + devenvruntime "github.com/smartcontractkit/chainlink-ccv/build/devenv/runtime" + "github.com/smartcontractkit/chainlink-ccv/build/devenv/services" + "github.com/smartcontractkit/chainlink-ccv/build/devenv/services/committeeverifier" + ccvdeployment "github.com/smartcontractkit/chainlink-ccv/deployment" + ccvshared "github.com/smartcontractkit/chainlink-ccv/deployment/shared" + ctfblockchain "github.com/smartcontractkit/chainlink-testing-framework/framework/components/blockchain" + ns "github.com/smartcontractkit/chainlink-testing-framework/framework/components/simple_node_set" +) + +const ( + clNodeConfigKey = "committeeccv_clnode" + clNodeVersion = 1 +) + +func init() { + if err := devenvruntime.Register(clNodeConfigKey, clnodeFactory); err != nil { + panic(fmt.Sprintf("committeeccv_clnode component: %v", err)) + } +} + +func clnodeFactory(_ map[string]any) (devenvruntime.Component, error) { + return &clnodeComponent{}, nil +} + +type clnodeComponent struct{} + +func (c *clnodeComponent) ValidateConfig(componentConfig any) error { + _, err := decodeCLNodeConfig(componentConfig) + return err +} + +// RunPhase3 runs the full CommitteeCCV setup (same steps as the standalone +// component) and additionally launches and registers Chainlink node sets. +// The CL-node launch (step 1b) must happen after HMAC credentials are +// generated (step 1) and before verifier registration (step 2), because: +// - bakeNodeSecrets writes HMAC creds into node specs before boot +// - ApplyVerifierConfig fetches CL-mode signing keys from JD by node ID +func (c *clnodeComponent) RunPhase3( + ctx context.Context, + globalConfig map[string]any, + componentConfig any, + priorOutputs map[string]any, +) (map[string]any, []devenvruntime.Effect, error) { + cfg, err := decodeCLNodeConfig(componentConfig) + if err != nil { + return nil, nil, err + } + aggregators, verifiers := cfg.Aggregator, cfg.Verifier + if len(aggregators) == 0 && len(verifiers) == 0 && len(cfg.NodeSets) == 0 { + return map[string]any{}, nil, nil + } + inputs, err := parsePhase3Inputs(priorOutputs, globalConfig) + if err != nil { + return nil, nil, err + } + // Work on a local copy of the shared Phase-2 environment. + localEnv := *inputs.env + + // Step 1: Generate HMAC credentials (must precede bakeNodeSecrets). + if err := ensureAggregatorCredentials(aggregators); err != nil { + return nil, nil, err + } + + // Step 1b: Bake secrets into node specs, then launch and register CL nodes. + // Must run before step 2 (verifier launch) so JD has the node IDs needed + // by ApplyVerifierConfig when fetching CL-mode signing keys. + clNodeClients, nodeIDs, err := launchCLNodes(ctx, &cfg, verifiers, aggregators, inputs.topology, inputs.blockchains, inputs.jdInfra) + if err != nil { + return nil, nil, fmt.Errorf("committeeccv_clnode: %w", err) + } + if len(nodeIDs) > 0 { + localEnv.NodeIDs = nodeIDs + } + + outputs, effects, err := runPhase3Core(ctx, inputs, aggregators, verifiers, &localEnv) + if err != nil { + return nil, nil, err + } + outputs["_clnode_clients"] = clNodeClients + return outputs, effects, nil +} + +// CLNodeConfig is the [committeeccv_clnode] config section. It embeds the +// committee fields (aggregators, verifiers) and the CL-node fields (node sets, +// funding amounts) so the section is self-contained and can be deleted wholesale +// when CL nodes leave the devenv. +type CLNodeConfig struct { + Version int `toml:"version"` + CLNodesFundingETH float64 `toml:"cl_nodes_funding_eth"` + CLNodesFundingLink float64 `toml:"cl_nodes_funding_link"` + NodeSets []*ns.Input `toml:"node_sets"` + Aggregator []*services.AggregatorInput `toml:"aggregator"` + Verifier []*committeeverifier.Input `toml:"verifier"` +} + +func decodeCLNodeConfig(raw any) (CLNodeConfig, error) { + b, err := toml.Marshal(raw) + if err != nil { + return CLNodeConfig{}, fmt.Errorf("re-encoding committeeccv_clnode config: %w", err) + } + var cfg CLNodeConfig + if err := toml.Unmarshal(b, &cfg); err != nil { + return CLNodeConfig{}, fmt.Errorf("decoding committeeccv_clnode config: %w", err) + } + if err := devenvruntime.CheckConfigVersion(cfg.Version, clNodeVersion); err != nil { + return CLNodeConfig{}, err + } + return cfg, nil +} + +// launchCLNodes bakes secrets into node specs, launches the CL node sets, then +// registers and connects each node to JD. Returns a NodeSetClientLookup and the +// JD node IDs for the launched nodes; both are nil/empty when no node sets are +// configured. +func launchCLNodes( + ctx context.Context, + cfg *CLNodeConfig, + verifiers []*committeeverifier.Input, + aggregators []*services.AggregatorInput, + topology *ccvdeployment.EnvironmentTopology, + blockchains []*ctfblockchain.Input, + jdInfra *jobs.JDInfrastructure, +) (*jobs.NodeSetClientLookup, []string, error) { + if cfg == nil || len(cfg.NodeSets) == 0 { + return nil, nil, nil + } + if err := bakeNodeSecrets(cfg, verifiers, aggregators, topology); err != nil { + return nil, nil, fmt.Errorf("baking node secrets: %w", err) + } + if err := launchNodeSets(ctx, cfg, blockchains); err != nil { + return nil, nil, fmt.Errorf("launching CL node sets: %w", err) + } + clNopAliases := clModeNOPAliases(topology) + if len(clNopAliases) == 0 { + return nil, nil, nil + } + clientLookup, err := jobs.NewNodeSetClientLookup(cfg.NodeSets, clNopAliases) + if err != nil { + return nil, nil, fmt.Errorf("building CL node client lookup: %w", err) + } + if clientLookup == nil { + return nil, nil, nil + } + if err := jobs.RegisterNodesWithJD(ctx, jdInfra, clientLookup, clNopAliases); err != nil { + return nil, nil, fmt.Errorf("registering CL nodes with JD: %w", err) + } + chainIDs := make([]string, len(blockchains)) + for i, bc := range blockchains { + chainIDs[i] = bc.ChainID + } + if err := jobs.ConnectNodesToJD(ctx, jdInfra, clientLookup, chainIDs); err != nil { + return nil, nil, fmt.Errorf("connecting CL nodes to JD: %w", err) + } + // The deployment environment was built in Phase 2 (protocol_contracts) before + // these CL nodes registered, so its NodeIDs are empty. Return the JD node IDs + // so the caller can populate its local env copy (FetchNOPSigningKeys needs them). + return clientLookup, jdInfra.GetNodeIDs(), nil +} + +// bakeNodeSecrets sets TestSecretsOverrides on each CL node spec before launch. +// Must be called after HMAC credentials are generated (step 1) and before +// launchNodeSets, because CL node secrets are boot-only. +func bakeNodeSecrets( + cfg *CLNodeConfig, + verifiers []*committeeverifier.Input, + aggregators []*services.AggregatorInput, + topology *ccvdeployment.EnvironmentTopology, +) error { + // Build topology aggregator names per committee (matches changeset ordering). + topoAggNames := make(map[string][]string) + if topology.NOPTopology != nil { + for name, committee := range topology.NOPTopology.Committees { + names := make([]string, len(committee.Aggregators)) + for i, a := range committee.Aggregators { + names[i] = a.Name + } + topoAggNames[name] = names + } + } + + aggsByCommittee := make(map[string][]*services.AggregatorInput) + for _, agg := range aggregators { + if agg != nil { + aggsByCommittee[agg.CommitteeName] = append(aggsByCommittee[agg.CommitteeName], agg) + } + } + + // Flatten node specs in node-set order; order must match NOP index ordering. + type nodeSpecEntry struct { + nodeSetIdx int + specIdx int + } + var nodeSpecOrder []nodeSpecEntry + for i, nodeSet := range cfg.NodeSets { + for j := range nodeSet.NodeSpecs { + nodeSpecOrder = append(nodeSpecOrder, nodeSpecEntry{i, j}) + } + } + + aggSecretsPerNode := make(map[int][]clNodeAggregatorSecret) + for _, ver := range verifiers { + if ver == nil || ver.Mode != services.CL { + continue + } + index, ok := topology.NOPTopology.GetNOPIndex(ver.NOPAlias) + if !ok { + return fmt.Errorf("NOP alias %q not found in topology for verifier %s", ver.NOPAlias, ver.ContainerName) + } + if index >= len(nodeSpecOrder) { + return fmt.Errorf("node index %d for NOPAlias %s exceeds available CL nodes (%d)", + index, ver.NOPAlias, len(nodeSpecOrder)) + } + committeeAggs := aggsByCommittee[ver.CommitteeName] + if len(committeeAggs) == 0 { + return fmt.Errorf("no aggregators found for committee %q (verifier %s)", ver.CommitteeName, ver.ContainerName) + } + committeeTopoNames := topoAggNames[ver.CommitteeName] + + for aggIdx, agg := range committeeAggs { + aggName := agg.InstanceName() + if aggIdx < len(committeeTopoNames) { + aggName = committeeTopoNames[aggIdx] + } + apiKeys, err := agg.GetAPIKeys() + if err != nil { + return fmt.Errorf("getting API keys for aggregator %s: %w", agg.InstanceName(), err) + } + var found bool + for _, apiClient := range apiKeys { + if apiClient.ClientID != ver.ContainerName { + continue + } + if len(apiClient.APIKeyPairs) == 0 { + return fmt.Errorf("no API key pairs for client %s on aggregator %s", + apiClient.ClientID, agg.InstanceName()) + } + pair := apiClient.APIKeyPairs[0] + verifierID := ccvshared.NewVerifierJobID( + ccvshared.NOPAlias(ver.NOPAlias), + aggName, + ccvshared.VerifierJobScope{CommitteeQualifier: ver.CommitteeName}, + ).GetVerifierID() + aggSecretsPerNode[index] = append(aggSecretsPerNode[index], clNodeAggregatorSecret{ + VerifierID: verifierID, + APIKey: pair.APIKey, + APISecret: pair.Secret, + }) + found = true + break + } + if !found { + return fmt.Errorf("API client %q not found on aggregator %s", + ver.ContainerName, agg.InstanceName()) + } + } + } + + for flatIdx, entry := range nodeSpecOrder { + if len(aggSecretsPerNode[flatIdx]) == 0 { + continue + } + nodeSpec := cfg.NodeSets[entry.nodeSetIdx].NodeSpecs[entry.specIdx] + secrets := clNodeSecrets{ + CCV: clNodeCCVSecrets{ + AggregatorSecrets: aggSecretsPerNode[flatIdx], + }, + } + secretsToml, err := secrets.tomlString() + if err != nil { + return fmt.Errorf("marshaling secrets for node %d: %w", flatIdx, err) + } + nodeSpec.Node.TestSecretsOverrides = secretsToml + } + return nil +} + +// clModeNOPAliases returns, in topology order, the aliases of NOPs running in +// CL mode. The order matches the CL node ordering used by NewNodeSetClientLookup. +func clModeNOPAliases(topology *ccvdeployment.EnvironmentTopology) []string { + if topology == nil || topology.NOPTopology == nil { + return nil + } + var aliases []string + for _, nop := range topology.NOPTopology.NOPs { + if nop.GetMode() == ccvshared.NOPModeCL { + aliases = append(aliases, nop.Alias) + } + } + return aliases +} + +// launchNodeSets configures, launches, and funds the CL node sets in cfg. +// TestSecretsOverrides must already be set on each node spec before this is +// called (bakeNodeSecrets handles that), because CL node secrets are boot-only. +func launchNodeSets(ctx context.Context, cfg *CLNodeConfig, blockchains []*ctfblockchain.Input) error { + if cfg == nil || len(cfg.NodeSets) == 0 { + return nil + } + if len(blockchains) == 0 { + return fmt.Errorf("committeeccv_clnode: no blockchains available to configure CL nodes") + } + + impls := make([]cciptestinterfaces.CCIP17Configuration, 0, len(blockchains)) + for _, bc := range blockchains { + impl, ierr := chainreg.NewProductConfigurationFromNetwork(bc.Type) + if ierr != nil { + return fmt.Errorf("committeeccv_clnode: impl for %q: %w", bc.Type, ierr) + } + impls = append(impls, impl) + } + + chainConfigs := []string{commonCLNodesConfig} + for i, impl := range impls { + cc, cerr := impl.ConfigureNodes(ctx, blockchains[i]) + if cerr != nil { + return fmt.Errorf("committeeccv_clnode: ConfigureNodes for %q: %w", blockchains[i].Type, cerr) + } + chainConfigs = append(chainConfigs, cc) + } + allConfigs := strings.Join(chainConfigs, "\n") + for _, nodeSet := range cfg.NodeSets { + for _, nodeSpec := range nodeSet.NodeSpecs { + nodeSpec.Node.TestConfigOverrides = allConfigs + } + } + + for _, nodeSet := range cfg.NodeSets { + if _, nerr := ns.NewSharedDBNodeSet(nodeSet, nil); nerr != nil { + return fmt.Errorf("committeeccv_clnode: NewSharedDBNodeSet %q: %w", nodeSet.Name, nerr) + } + } + + link := toBigUnits(cfg.CLNodesFundingLink, 1) + native := toBigUnits(cfg.CLNodesFundingETH, 5) + for i, impl := range impls { + if ferr := impl.FundNodes(ctx, cfg.NodeSets, blockchains[i], link, native); ferr != nil { + return fmt.Errorf("committeeccv_clnode: FundNodes on %q: %w", blockchains[i].Type, ferr) + } + } + return nil +} + +func toBigUnits(v float64, def int64) *big.Int { + if v <= 0 { + return big.NewInt(def) + } + return big.NewInt(int64(v)) +} + +// commonCLNodesConfig is the base TOML applied to every CL node. It mirrors +// ccv.CommonCLNodesConfig; components cannot import the ccv package (it +// blank-imports every component, which would create a cycle). +const commonCLNodesConfig = ` +[Log] +JSONConsole = true +Level = 'info' +[Pyroscope] +ServerAddress = 'http://host.docker.internal:4040' +Environment = 'local' +[WebServer] +SessionTimeout = '999h0m0s' +HTTPWriteTimeout = '3m' +SecureCookies = false +HTTPPort = 6688 +AllowOrigins = 'http://localhost:3000' +[WebServer.TLS] +HTTPSPort = 0 +[WebServer.RateLimit] +Authenticated = 5000 +Unauthenticated = 5000 +[JobPipeline] +[JobPipeline.HTTPRequest] +DefaultTimeout = '1m' +[Log.File] +MaxSize = '0b' +[Feature] +FeedsManager = true +LogPoller = true +UICSAKeys = true +[OCR2] +Enabled = true +SimulateTransactions = false +DefaultTransactionQueueDepth = 1 +[P2P.V2] +Enabled = true +ListenAddresses = ['0.0.0.0:6690'] +` + +// clNodeSecrets, clNodeCCVSecrets, and clNodeAggregatorSecret are the TOML +// types written to TestSecretsOverrides on each CL node spec. They mirror the +// identically-named types in the ccv package (environment.go); duplicated here +// to avoid the import cycle described above. +type clNodeSecrets struct { + CCV clNodeCCVSecrets `toml:",omitempty"` +} + +func (s *clNodeSecrets) tomlString() (string, error) { + data, err := toml.Marshal(s) + if err != nil { + return "", fmt.Errorf("failed to marshal CCV secrets to TOML: %w", err) + } + return string(data), nil +} + +type clNodeCCVSecrets struct { + AggregatorSecrets []clNodeAggregatorSecret `toml:",omitempty"` +} + +type clNodeAggregatorSecret struct { + VerifierID string `toml:",omitempty"` + APIKey string `toml:",omitempty"` + APISecret string `toml:",omitempty"` +} diff --git a/build/devenv/effect_executor.go b/build/devenv/effect_executor.go index 144417e6f..9c713ca66 100644 --- a/build/devenv/effect_executor.go +++ b/build/devenv/effect_executor.go @@ -105,8 +105,22 @@ func executeJobProposalEffects(ctx context.Context, effects []devenvruntime.JobP } } + // Accept pending job proposals on CL nodes before verifying. Mirrors the + // monolith order (AcceptPendingJobs then SyncAndVerifyJobProposals). + clNodeClients, _ := accumulated["_clnode_clients"].(*jobs.NodeSetClientLookup) + if err := jobs.AcceptPendingJobs(ctx, clNodeClients); err != nil { + return fmt.Errorf("accepting pending CL node jobs: %w", err) + } + if e, ok := accumulated["_env"].(*deployment.Environment); ok && e != nil { - if err := jobs.SyncAndVerifyJobProposals(e); err != nil { + // The Phase-2 _env has empty NodeIDs (CL nodes registered in Phase 3). + // Build a local copy with NodeIDs populated from JD so SyncJobProposals + // can map NOP aliases to JD node IDs. + syncEnv := *e + if nodeIDs := jdInfra.GetNodeIDs(); len(nodeIDs) > 0 { + syncEnv.NodeIDs = nodeIDs + } + if err := jobs.SyncAndVerifyJobProposals(&syncEnv); err != nil { return fmt.Errorf("syncing job proposals: %w", err) } } diff --git a/build/devenv/env-cl-ci-phased.toml b/build/devenv/env-cl-ci-phased.toml new file mode 100644 index 000000000..e9eb378da --- /dev/null +++ b/build/devenv/env-cl-ci-phased.toml @@ -0,0 +1,32 @@ +## CI-specific override: use pre-built local images instead of building from +## Dockerfile. Layer on top of env-phased.toml,env-cl-phased.toml in CI. +## Replaces the [[committeeccv_clnode.node_sets]] array with specs that pull +## the pre-built local:latest image (uploaded by the build-cl-image job). +[committeeccv_clnode] +version = 1 +cl_nodes_funding_eth = 50 +cl_nodes_funding_link = 50 + +[[committeeccv_clnode.node_sets]] + name = "don" + nodes = 2 + override_mode = "each" + + [committeeccv_clnode.node_sets.db] + image = "postgres:15.0" + + [[committeeccv_clnode.node_sets.node_specs]] + + [committeeccv_clnode.node_sets.node_specs.node] + image = "docker.io/library/local:latest" + + [committeeccv_clnode.node_sets.node_specs.node.env_vars] + CL_EVM_CMD="" + + [[committeeccv_clnode.node_sets.node_specs]] + + [committeeccv_clnode.node_sets.node_specs.node] + image = "docker.io/library/local:latest" + + [committeeccv_clnode.node_sets.node_specs.node.env_vars] + CL_EVM_CMD="" diff --git a/build/devenv/env-cl-phased.toml b/build/devenv/env-cl-phased.toml new file mode 100644 index 000000000..8d30c9ae8 --- /dev/null +++ b/build/devenv/env-cl-phased.toml @@ -0,0 +1,362 @@ +# Phased-runtime CL-mode overlay (issue 16). +# +# Layer this on top of env-phased.toml: +# ccv --env-mode phased up env-phased.toml,env-cl-phased.toml +# +# CL mode is a property of the verifier (mode = "cl"): instead of running as +# standalone containers, verifier jobs run on shared Chainlink nodes. +# SCOPE: only the verifiers run in CL mode here. The executors stay standalone. +# +# Config merge (deepMergeMaps) merges nested *maps* key-by-key but replaces +# *arrays* wholesale. This overlay: +# 1. Clears [committeeccv] (empty aggregator/verifier arrays → no-op early +# return in the standalone component, letting committeeccv_clnode take over). +# 2. Adds [committeeccv_clnode] with the full self-contained CL-node config +# (node sets + aggregators mirrored from env-phased.toml + CL-mode verifiers). +# 3. Replaces the NOP and committee topology with the two shared CL nodes. +# The executor section, executor_pools, blockchains, observability, jd, and +# indexer sections are all inherited from env-phased.toml. + +############################# +# Disable the standalone committeeccv component so it is a no-op when this +# overlay is applied. The committeeccv_clnode component below takes over. +############################# +[committeeccv] +version = 1 +aggregator = [] +verifier = [] + +############################# +# CL-node variant of CommitteeCCV (committeeccv_clnode component). +# Contains: CL node sets, aggregators (mirrored from env-phased.toml), and +# CL-mode verifiers. Delete this section (and component_clnode.go) when CL +# nodes leave the devenv. +############################# +[committeeccv_clnode] +version = 1 +cl_nodes_funding_eth = 50 +cl_nodes_funding_link = 50 + +[[committeeccv_clnode.node_sets]] + name = "don" + nodes = 2 + override_mode = "each" + + [committeeccv_clnode.node_sets.db] + image = "postgres:15.0" + + [[committeeccv_clnode.node_sets.node_specs]] + [committeeccv_clnode.node_sets.node_specs.node] + docker_ctx = "../../../chainlink" + docker_file = "plugins/chainlink.Dockerfile" + [committeeccv_clnode.node_sets.node_specs.node.env_vars] + CL_EVM_CMD = "" + + [[committeeccv_clnode.node_sets.node_specs]] + [committeeccv_clnode.node_sets.node_specs.node] + docker_ctx = "../../../chainlink" + docker_file = "plugins/chainlink.Dockerfile" + [committeeccv_clnode.node_sets.node_specs.node.env_vars] + CL_EVM_CMD = "" + +# Aggregators: mirrors [[committeeccv.aggregator]] in env-phased.toml. +# Aggregator config is identical between standalone and CL-node modes; the +# only change is the component key. +[[committeeccv_clnode.aggregator]] + committee_name = "default" + image = "aggregator:latest" + host_port = 50051 + redundant_aggregators = 1 + source_code_path = "../aggregator" + root_path = "../../" + + [committeeccv_clnode.aggregator.db] + image = "postgres:16-alpine" + host_port = 7432 + + [committeeccv_clnode.aggregator.redis] + image = "redis:7-alpine" + host_port = 6379 + + [[committeeccv_clnode.aggregator.api_clients]] + client_id = "default-verifier-1" + description = "Development default verifier 1" + enabled = true + groups = ["verifiers"] + + [[committeeccv_clnode.aggregator.api_clients]] + client_id = "default-verifier-2" + description = "Development default verifier 2" + enabled = true + groups = ["verifiers"] + + [[committeeccv_clnode.aggregator.api_clients]] + client_id = "monitoring" + description = "Monitoring and infrastructure client" + enabled = true + groups = ["monitoring"] + + [[committeeccv_clnode.aggregator.api_clients]] + client_id = "indexer" + description = "Development Indexer" + enabled = true + groups = ["indexer"] + + [committeeccv_clnode.aggregator.env] + storage_connection_url = "postgresql://aggregator:aggregator@default-aggregator-db:5432/aggregator?sslmode=disable" + redis_address = "default-aggregator-redis:6379" + redis_password = "" + redis_db = "0" + +[[committeeccv_clnode.aggregator]] + committee_name = "secondary" + image = "aggregator:latest" + host_port = 50052 + redundant_aggregators = 0 + source_code_path = "../aggregator" + root_path = "../../" + + [committeeccv_clnode.aggregator.db] + image = "postgres:16-alpine" + host_port = 7433 + + [committeeccv_clnode.aggregator.redis] + image = "redis:7-alpine" + host_port = 6380 + + [[committeeccv_clnode.aggregator.api_clients]] + client_id = "secondary-verifier-1" + description = "Development secondary verifier 1" + enabled = true + groups = ["verifiers"] + + [[committeeccv_clnode.aggregator.api_clients]] + client_id = "secondary-verifier-2" + description = "Development secondary verifier 2" + enabled = true + groups = ["verifiers"] + + [[committeeccv_clnode.aggregator.api_clients]] + client_id = "monitoring" + description = "Monitoring and infrastructure client" + enabled = true + groups = ["monitoring"] + + [[committeeccv_clnode.aggregator.api_clients]] + client_id = "indexer" + description = "Development Indexer" + enabled = true + groups = ["indexer"] + + [committeeccv_clnode.aggregator.env] + storage_connection_url = "postgresql://aggregator:aggregator@secondary-aggregator-db:5432/aggregator?sslmode=disable" + redis_address = "secondary-aggregator-redis:6379" + redis_password = "" + redis_db = "0" + +[[committeeccv_clnode.aggregator]] + committee_name = "tertiary" + image = "aggregator:latest" + host_port = 50053 + redundant_aggregators = 0 + source_code_path = "../aggregator" + root_path = "../../" + + [committeeccv_clnode.aggregator.db] + image = "postgres:16-alpine" + host_port = 7434 + + [committeeccv_clnode.aggregator.redis] + image = "redis:7-alpine" + host_port = 6381 + + [[committeeccv_clnode.aggregator.api_clients]] + client_id = "tertiary-verifier-1" + description = "Development tertiary verifier 1" + enabled = true + groups = ["verifiers"] + + [[committeeccv_clnode.aggregator.api_clients]] + client_id = "tertiary-verifier-2" + description = "Development tertiary verifier 2" + enabled = true + groups = ["verifiers"] + + [[committeeccv_clnode.aggregator.api_clients]] + client_id = "monitoring" + description = "Monitoring and infrastructure client" + enabled = true + groups = ["monitoring"] + + [[committeeccv_clnode.aggregator.api_clients]] + client_id = "indexer" + description = "Development Indexer" + enabled = true + groups = ["indexer"] + + [committeeccv_clnode.aggregator.env] + storage_connection_url = "postgresql://aggregator:aggregator@tertiary-aggregator-db:5432/aggregator?sslmode=disable" + redis_address = "tertiary-aggregator-redis:6379" + redis_password = "" + redis_db = "0" + +# Verifiers: CL mode, mapped onto the two nodes. Replaces the standalone +# verifier list from env-phased.toml wholesale. +[[committeeccv_clnode.verifier]] + mode = "cl" + image = "verifier:latest" + container_name = "default-verifier-1" + nop_alias = "node-0" + port = 8100 + source_code_path = "../verifier" + root_path = "../../" + committee_name = "default" + node_index = 0 + insecure_aggregator_connection = true + [committeeccv_clnode.verifier.db] + image = "postgres:16-alpine" + name = "default-verifier-1-db" + port = 8432 + +[[committeeccv_clnode.verifier]] + mode = "cl" + image = "verifier:latest" + container_name = "default-verifier-2" + nop_alias = "node-1" + port = 8200 + source_code_path = "../verifier" + root_path = "../../" + committee_name = "default" + node_index = 1 + insecure_aggregator_connection = true + [committeeccv_clnode.verifier.db] + image = "postgres:16-alpine" + name = "default-verifier-2-db" + port = 8433 + +[[committeeccv_clnode.verifier]] + mode = "cl" + image = "verifier:latest" + container_name = "secondary-verifier-1" + nop_alias = "node-0" + port = 8300 + source_code_path = "../verifier" + root_path = "../../" + committee_name = "secondary" + node_index = 0 + insecure_aggregator_connection = true + [committeeccv_clnode.verifier.db] + image = "postgres:16-alpine" + name = "secondary-verifier-1-db" + port = 8434 + +[[committeeccv_clnode.verifier]] + mode = "cl" + image = "verifier:latest" + container_name = "secondary-verifier-2" + nop_alias = "node-1" + port = 8400 + source_code_path = "../verifier" + root_path = "../../" + committee_name = "secondary" + node_index = 1 + insecure_aggregator_connection = true + [committeeccv_clnode.verifier.db] + image = "postgres:16-alpine" + name = "secondary-verifier-2-db" + port = 8435 + +[[committeeccv_clnode.verifier]] + mode = "cl" + image = "verifier:latest" + container_name = "tertiary-verifier-1" + nop_alias = "node-0" + port = 8500 + source_code_path = "../verifier" + root_path = "../../" + committee_name = "tertiary" + node_index = 0 + insecure_aggregator_connection = true + [committeeccv_clnode.verifier.db] + image = "postgres:16-alpine" + name = "tertiary-verifier-1-db" + port = 8436 + +[[committeeccv_clnode.verifier]] + mode = "cl" + image = "verifier:latest" + container_name = "tertiary-verifier-2" + nop_alias = "node-1" + port = 8600 + source_code_path = "../verifier" + root_path = "../../" + committee_name = "tertiary" + node_index = 1 + insecure_aggregator_connection = true + [committeeccv_clnode.verifier.db] + image = "postgres:16-alpine" + name = "tertiary-verifier-2-db" + port = 8437 + +############################# +# Topology NOPs: replace the six standalone verifier NOPs with two shared CL +# node NOPs (node-0, node-1; default NOP mode is CL). The three standalone +# executor NOPs are kept unchanged so the executors (still standalone) and their +# executor_pools continue to resolve. +############################# +[[protocol_contracts.environment_topology.nop_topology.nops]] +alias = "node-0" +name = "node-0" + +[[protocol_contracts.environment_topology.nop_topology.nops]] +alias = "node-1" +name = "node-1" + +[[protocol_contracts.environment_topology.nop_topology.nops]] +alias = "default-executor-1" +name = "default-executor-1" +mode = "standalone" + +[[protocol_contracts.environment_topology.nop_topology.nops]] +alias = "default-executor-2" +name = "default-executor-2" +mode = "standalone" + +[[protocol_contracts.environment_topology.nop_topology.nops]] +alias = "custom-executor-1" +name = "custom-executor-1" +mode = "standalone" + +############################# +# Committees: repoint membership from the standalone verifier NOPs to the two +# shared CL nodes. (Aggregators, qualifier, verifier_version are inherited.) +############################# +[protocol_contracts.environment_topology.nop_topology.committees.default.chain_configs.3379446385462418246] +nop_aliases = ["node-0", "node-1"] +threshold = 2 +[protocol_contracts.environment_topology.nop_topology.committees.default.chain_configs.12922642891491394802] +nop_aliases = ["node-0", "node-1"] +threshold = 2 +[protocol_contracts.environment_topology.nop_topology.committees.default.chain_configs.4793464827907405086] +nop_aliases = ["node-0", "node-1"] +threshold = 2 + +[protocol_contracts.environment_topology.nop_topology.committees.secondary.chain_configs.3379446385462418246] +nop_aliases = ["node-0", "node-1"] +threshold = 2 +[protocol_contracts.environment_topology.nop_topology.committees.secondary.chain_configs.12922642891491394802] +nop_aliases = ["node-0", "node-1"] +threshold = 2 +[protocol_contracts.environment_topology.nop_topology.committees.secondary.chain_configs.4793464827907405086] +nop_aliases = ["node-0", "node-1"] +threshold = 2 + +[protocol_contracts.environment_topology.nop_topology.committees.tertiary.chain_configs.3379446385462418246] +nop_aliases = ["node-0", "node-1"] +threshold = 2 +[protocol_contracts.environment_topology.nop_topology.committees.tertiary.chain_configs.12922642891491394802] +nop_aliases = ["node-0", "node-1"] +threshold = 2 +[protocol_contracts.environment_topology.nop_topology.committees.tertiary.chain_configs.4793464827907405086] +nop_aliases = ["node-0", "node-1"] +threshold = 2 diff --git a/changelog/2026-06-01_phased_clnode_component.md b/changelog/2026-06-01_phased_clnode_component.md new file mode 100644 index 000000000..c13030643 --- /dev/null +++ b/changelog/2026-06-01_phased_clnode_component.md @@ -0,0 +1,106 @@ +# Phased devenv: CL-node support restored as a clean component + +## Summary + +Restores Chainlink-node ("CL-node") verifier mode in the phased devenv +(`--env-mode phased`) by introducing a dedicated `committeeccv_clnode` +Phase 3 component. The former `clnode` Phase-1 config-vehicle package is +deleted; all CL-node logic is now isolated in one file designed for future +removal when CL nodes leave the devenv. + +--- + +## Breaking change: `[clnode]` config key removed + +The `[clnode]` TOML section is no longer recognized. CL-node devenv +configuration moves into `[committeeccv_clnode]`, which embeds both the +committee fields (aggregators, verifiers) and the CL-node fields (node +sets, funding amounts). + +| What | Before | After | +|------|--------|-------| +| Config section for node sets | `[clnode]` | `[committeeccv_clnode]` | +| Aggregators in CL-node mode | `[[committeeccv.aggregator]]` | `[[committeeccv_clnode.aggregator]]` | +| CL-mode verifiers | `[[committeeccv.verifier]]` with `mode = "cl"` | `[[committeeccv_clnode.verifier]]` with `mode = "cl"` | +| Disable standalone in CL overlay | (implicit — only one ran) | `[committeeccv]` with `aggregator = []`, `verifier = []` | + +Before (`env-cl-phased.toml`): +```toml +[clnode] +version = 1 +cl_nodes_funding_eth = 50 +[[clnode.node_sets]] + name = "don" + nodes = 2 + ... + +[[committeeccv.verifier]] + mode = "cl" + ... +``` + +After (`env-cl-phased.toml`): +```toml +# Disable the standalone component (becomes a no-op) +[committeeccv] +version = 1 +aggregator = [] +verifier = [] + +# CL-node variant: self-contained, delete when CL nodes leave +[committeeccv_clnode] +version = 1 +cl_nodes_funding_eth = 50 +[[committeeccv_clnode.node_sets]] + name = "don" + nodes = 2 + ... +[[committeeccv_clnode.aggregator]] + ... +[[committeeccv_clnode.verifier]] + mode = "cl" + ... +``` + +--- + +## New: `committeeccv_clnode` Phase 3 component + +Registered under config key `"committeeccv_clnode"` in +`build/devenv/components/committeeccv/component_clnode.go`. It is a +drop-in replacement for the standalone `committeeccv` component in +CL-node mode: runs all the same Phase 3 steps, plus step 1b which bakes +aggregator HMAC secrets into node specs before boot and launches, +registers, and connects the CL node sets to JD. + +The component publishes `"_clnode_clients"` (a `*jobs.NodeSetClientLookup`) +which the effect executor uses to call `AcceptPendingJobs` before +`SyncAndVerifyJobProposals`. + +To remove CL nodes from the devenv later: delete `component_clnode.go` +and the `[committeeccv_clnode]` sections from the config overlays. + +--- + +## New: phased CL-node CI test + +`test-cl-smoke.yaml` gains a `Phased TestE2ESmoke_Basic` matrix entry +that runs `TestE2ESmoke_Basic_Phased` with: + +``` +config: env-phased.toml,env-cl-phased.toml,env-cl-ci-phased.toml +flags: --env-mode phased +``` + +`env-cl-ci-phased.toml` is updated from `[clnode]` to +`[committeeccv_clnode]` so the CI image override applies to the correct +component. + +--- + +## Internal refactor: `committeeccv` shared helpers + +`committeeccv/component.go` extracts `phase3Inputs`, `parsePhase3Inputs`, +`ensureAggregatorCredentials`, and `runPhase3Core` so both components +share the same Phase 3 logic without duplication. No behavioral change to +the standalone path.