diff --git a/commit/metrics/prom.go b/commit/metrics/prom.go index d12a6c12c1..6adcdf33db 100644 --- a/commit/metrics/prom.go +++ b/commit/metrics/prom.go @@ -92,6 +92,10 @@ var ( Name: "ccip_commit_loopp_ccip_provider_supported", Help: "Tracks whether LOOPP CCIP provider is supported for each chain family (1 = supported, 0 = not supported)", }, []string{"chain_family"}) + promCommitConfigDigestMismatch = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "ccip_commit_config_digest_mismatch", + Help: "Reports whether the home chain config digest differs from the offramp config digest (1 = mismatch, 0 = match)", + }, []string{"chain_family", "chain_id"}) ) type PromReporter struct { @@ -116,6 +120,9 @@ type PromReporter struct { bhSequenceNumbers metric.Int64Gauge bhCommitLatestRound metric.Int64Gauge bhLooppProviderSupported metric.Int64Gauge + + configDigestMismatch *prometheus.GaugeVec + bhConfigDigestMismatch metric.Int64Gauge } func NewPromReporter( @@ -149,6 +156,10 @@ func NewPromReporter( if err != nil { return nil, fmt.Errorf("failed to register ccip_commit_loopp_ccip_provider_supported gauge: %w", err) } + configDigestMismatch, err := bhClient.Meter.Int64Gauge("ccip_commit_config_digest_mismatch") + if err != nil { + return nil, fmt.Errorf("failed to register ccip_commit_config_digest_mismatch gauge: %w", err) + } return &PromReporter{ lggr: lggr, @@ -162,6 +173,7 @@ func NewPromReporter( sequenceNumbers: promSequenceNumbers, commitLatestRound: promCommitLatestRoundID, looppProviderSupported: promLooppCCIPProviderSupported, + configDigestMismatch: promCommitConfigDigestMismatch, processorLatencyHistogram: promProcessorLatencyHistogram, processorOutputCounter: promProcessorOutputCounter, @@ -173,6 +185,7 @@ func NewPromReporter( bhSequenceNumbers: sequenceNumbers, bhCommitLatestRound: commitLatestRoundID, bhLooppProviderSupported: looppProviderSupported, + bhConfigDigestMismatch: configDigestMismatch, }, nil } @@ -340,3 +353,15 @@ func (p *PromReporter) TrackLooppProviderSupported(looppCCIPProviderSupported ma )) } } + +func (p *PromReporter) TrackConfigDigestMismatch(mismatch bool) { + var value float64 + if mismatch { + value = 1 + } + p.configDigestMismatch.WithLabelValues(p.chainFamily, p.chainID).Set(value) + p.bhConfigDigestMismatch.Record(context.Background(), int64(value), metric.WithAttributes( + attribute.String("chain_family", p.chainFamily), + attribute.String("chain_id", p.chainID), + )) +} diff --git a/commit/metrics/reporter.go b/commit/metrics/reporter.go index 1e006098ac..e5200b3ada 100644 --- a/commit/metrics/reporter.go +++ b/commit/metrics/reporter.go @@ -27,11 +27,14 @@ type Reporter interface { TrackProcessorLatency(processor string, method plugincommon.MethodType, latency time.Duration, err error) TrackProcessorOutput(processor string, method plugincommon.MethodType, obs plugintypes.Trackable) + + TrackConfigDigestMismatch(mismatch bool) } type CommitPluginReporter interface { TrackObservation(obs committypes.Observation, round uint64) TrackOutcome(outcome committypes.Outcome, round uint64) + TrackConfigDigestMismatch(mismatch bool) } type Noop struct{} @@ -48,6 +51,8 @@ func (n *Noop) TrackProcessorLatency(string, plugincommon.MethodType, time.Durat func (n *Noop) TrackProcessorOutput(string, plugincommon.MethodType, plugintypes.Trackable) {} +func (n *Noop) TrackConfigDigestMismatch(bool) {} + var _ Reporter = &PromReporter{} var _ CommitPluginReporter = &PromReporter{} var _ merkleroot.MetricsReporter = &PromReporter{} diff --git a/commit/plugin.go b/commit/plugin.go index d3aaac7df2..40c2bf3d36 100644 --- a/commit/plugin.go +++ b/commit/plugin.go @@ -291,6 +291,9 @@ func (p *Plugin) Observation( return encoded, nil } + // Check config digest every round and emit a mismatch metric. + p.trackConfigDigestMismatch(ctx, lggr) + prevOutcome, err := p.ocrTypeCodec.DecodeOutcome(outCtx.PreviousOutcome) if err != nil { return nil, fmt.Errorf("decode previous outcome: %w", err) @@ -412,6 +415,21 @@ func (p *Plugin) ObserveFChain(lggr logger.Logger) map[cciptypes.ChainSelector]i return fChain } +func (p *Plugin) trackConfigDigestMismatch(ctx context.Context, lggr logger.Logger) { + configMatch, _, err := plugincommon.ConfigDigestsMatch( + ctx, p.ccipReader, consts.PluginTypeCommit, p.reportingCfg.ConfigDigest, + ) + if err != nil { + lggr.Errorw("failed to check for config digest mismatch", + "err", err, + "homeChainConfigDigest", p.reportingCfg.ConfigDigest, + "pluginType", consts.PluginTypeCommit, + ) + return + } + p.metricsReporter.TrackConfigDigestMismatch(!configMatch) +} + //nolint:gocyclo func (p *Plugin) Outcome( ctx context.Context, outCtx ocr3types.OutcomeContext, q types.Query, aos []types.AttributedObservation, diff --git a/commit/plugin_roledon_e2e_test.go b/commit/plugin_roledon_e2e_test.go index ce3fc99e26..2ac85525a7 100644 --- a/commit/plugin_roledon_e2e_test.go +++ b/commit/plugin_roledon_e2e_test.go @@ -92,6 +92,8 @@ func TestPlugin_RoleDonE2E_NoPrevOutcome(t *testing.T) { { deps.ccipReader.EXPECT().DiscoverContracts(mock.Anything, mock.Anything, mock.Anything).Return(nil, nil) deps.ccipReader.EXPECT().Sync(mock.Anything, mock.Anything).Return(nil) + deps.ccipReader.EXPECT().GetOffRampConfigDigest(mock.Anything, mock.AnythingOfType("uint8")). + Return([32]byte{}, nil).Maybe() } // Source Chain Expectations - Makes sure only oracles that support specific source chains are reading them. @@ -209,6 +211,8 @@ func TestPlugin_RoleDonE2E_RangesAndPricesSelectedPreviously(t *testing.T) { { deps.ccipReader.EXPECT().DiscoverContracts(mock.Anything, mock.Anything, mock.Anything).Return(nil, nil) deps.ccipReader.EXPECT().Sync(mock.Anything, mock.Anything).Return(nil) + deps.ccipReader.EXPECT().GetOffRampConfigDigest(mock.Anything, mock.AnythingOfType("uint8")). + Return([32]byte{}, nil).Maybe() } // Source Chain Expectations - Makes sure only oracles that support specific source chains are reading them. @@ -372,6 +376,8 @@ func TestPlugin_RoleDonE2E_Discovery(t *testing.T) { }, addresses) return nil }) + deps.ccipReader.EXPECT().GetOffRampConfigDigest(mock.Anything, mock.AnythingOfType("uint8")). + Return([32]byte{}, nil).Maybe() } p := s.newRoleDonTestPlugin(oracleID, true) diff --git a/commit/plugin_test.go b/commit/plugin_test.go index 15238608f5..ba9eae4f6e 100644 --- a/commit/plugin_test.go +++ b/commit/plugin_test.go @@ -205,6 +205,9 @@ func TestObservation_prices(t *testing.T) { ccr.EXPECT().GetLatestPriceSeqNr(mock.Anything).Return(tc.onchainOcrSeqNum, tc.rpcErr).Maybe() + ccr.EXPECT().GetOffRampConfigDigest(mock.Anything, mock.AnythingOfType("uint8")). + Return([32]byte{}, nil).Maybe() + tokenPriceObs := tokenprice.Observation{} if tc.expObservedPrices { tokenPriceObs.FeedTokenPrices = map[ccipocr3.UnknownEncodedAddress]ccipocr3.BigInt{ diff --git a/commit/report.go b/commit/report.go index 9c0e547fbd..1cc591f891 100644 --- a/commit/report.go +++ b/commit/report.go @@ -1,9 +1,7 @@ package commit import ( - "bytes" "context" - "encoding/hex" "errors" "fmt" "maps" @@ -212,16 +210,18 @@ func (p *Plugin) validateReport( return cciptypes.CommitPluginReport{}, plugincommon.NewErrInvalidReport("dest chain not supported") } - offRampConfigDigest, err := p.ccipReader.GetOffRampConfigDigest(ctx, consts.PluginTypeCommit) + match, offRampDigest, err := plugincommon.ConfigDigestsMatch( + ctx, p.ccipReader, consts.PluginTypeCommit, p.reportingCfg.ConfigDigest, + ) if err != nil { - err = plugincommon.NewErrValidatingReport(fmt.Errorf("get offramp config digest: %w", err)) - return cciptypes.CommitPluginReport{}, plugincommon.NewErrValidatingReport(err) + return cciptypes.CommitPluginReport{}, + plugincommon.NewErrValidatingReport(fmt.Errorf("check config digest: %w", err)) } - if !bytes.Equal(offRampConfigDigest[:], p.reportingCfg.ConfigDigest[:]) { + if !match { lggr.Warnw("my config digest doesn't match offramp's config digest, not accepting report", "myConfigDigest", p.reportingCfg.ConfigDigest, - "offRampConfigDigest", hex.EncodeToString(offRampConfigDigest[:]), + "offRampConfigDigest", plugincommon.FormatConfigDigest(offRampDigest), ) return cciptypes.CommitPluginReport{}, plugincommon.NewErrInvalidReport("config digest mismatch") } diff --git a/devenv/tests/config_digest.go b/devenv/tests/config_digest.go new file mode 100644 index 0000000000..662f54f928 --- /dev/null +++ b/devenv/tests/config_digest.go @@ -0,0 +1,44 @@ +package tests + +import ( + "testing" + + "github.com/Masterminds/semver/v3" + chainsel "github.com/smartcontractkit/chain-selectors" + "github.com/smartcontractkit/chainlink-ccip/deployment/deploy" + "github.com/smartcontractkit/chainlink-common/pkg/types/ccipocr3" + "github.com/smartcontractkit/chainlink-deployments-framework/deployment" + "github.com/smartcontractkit/chainlink-deployments-framework/operations" + "github.com/stretchr/testify/require" +) + +func RunUpdateConfigDigestTests(t *testing.T, e *deployment.Environment, selectors []uint64) { + selectorsToImpl := buildImplsMap(t, e, selectors) + + // get two distinct selectors + fromImpl, toImpl := selectorsToImpl[selectors[0]], selectorsToImpl[selectors[1]] + require.NotEqual(t, fromImpl, toImpl) + + destFamily, err := chainsel.GetSelectorFamily(toImpl.ChainSelector()) + require.NoError(t, err) + + deployAdapter, ok := deploy.GetRegistry().GetDeployer(destFamily, semver.MustParse("1.6.0")) + require.True(t, ok) + require.NotNil(t, deployAdapter) + + _, err = operations.ExecuteSequence(e.OperationsBundle, deployAdapter.SetOCR3Config(), e.BlockChains, deploy.SetOCR3ConfigInput{ + ChainSelector: toImpl.ChainSelector(), + Datastore: e.DataStore, + Configs: map[ccipocr3.PluginType]deploy.OCR3ConfigArgs{ + ccipocr3.PluginTypeCCIPCommit: { + ConfigDigest: [32]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32}, + PluginType: ccipocr3.PluginTypeCCIPCommit, + F: 1, + IsSignatureVerificationEnabled: true, + Signers: [][]byte{{1, 2, 3}, {4, 5, 6}, {7, 8, 9}, {10, 11, 12}, {13, 14, 15}}, + Transmitters: [][]byte{{1, 2, 3}, {4, 5, 6}, {7, 8, 9}, {10, 11, 12}, {13, 14, 15}}, + }, + }, + }) + require.NoError(t, err) +} diff --git a/devenv/tests/e2e/update_cfg_digest_test.go b/devenv/tests/e2e/update_cfg_digest_test.go new file mode 100644 index 0000000000..5088196369 --- /dev/null +++ b/devenv/tests/e2e/update_cfg_digest_test.go @@ -0,0 +1,29 @@ +package e2e + +import ( + "fmt" + "testing" + + ccip "github.com/smartcontractkit/chainlink-ccip/devenv" + "github.com/smartcontractkit/chainlink-ccip/devenv/tests" + "github.com/smartcontractkit/chainlink-testing-framework/framework" + "github.com/stretchr/testify/require" +) + +func TestUpdateConfigDigest(t *testing.T) { + in, err := ccip.LoadOutput[ccip.Cfg]("../../env-out.toml") + require.NoError(t, err) + if in.ForkedEnvConfig != nil { + t.Skip("Skipping E2E tests on forked environments, not supported yet") + } + + selectors, e, err := ccip.NewCLDFOperationsEnvironment(in.Blockchains, in.CLDF.DataStore) + require.NoError(t, err) + + t.Cleanup(func() { + _, err := framework.SaveContainerLogs(fmt.Sprintf("%s-%s", framework.DefaultCTFLogsDir, t.Name())) + require.NoError(t, err) + }) + + tests.RunUpdateConfigDigestTests(t, e, selectors) +} diff --git a/execute/metrics/prom.go b/execute/metrics/prom.go index 72bc25ec5b..2afde1887e 100644 --- a/execute/metrics/prom.go +++ b/execute/metrics/prom.go @@ -101,6 +101,10 @@ var ( Name: "ccip_exec_loopp_ccip_provider_supported", Help: "Tracks whether LOOPP CCIP provider is supported for each chain family (1 = supported, 0 = not supported)", }, []string{"chain_family"}) + promExecConfigDigestMismatch = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "ccip_exec_config_digest_mismatch", + Help: "Reports whether the home chain config digest differs from the offramp config digest (1 = mismatch, 0 = match)", + }, []string{"chain_family", "chain_id"}) ) type PromReporter struct { @@ -127,6 +131,9 @@ type PromReporter struct { beholderProcessorErrors metric.Int64Counter bhExecLatestRound metric.Int64Gauge bhLooppProviderSupported metric.Int64Gauge + + configDigestMismatch *prometheus.GaugeVec + bhConfigDigestMismatch metric.Int64Gauge } func NewPromReporter( @@ -169,6 +176,10 @@ func NewPromReporter( if err != nil { return nil, fmt.Errorf("failed to register ccip_exec_loopp_ccip_provider_supported gauge: %w", err) } + configDigestMismatch, err := bhClient.Meter.Int64Gauge("ccip_exec_config_digest_mismatch") + if err != nil { + return nil, fmt.Errorf("failed to register ccip_exec_config_digest_mismatch gauge: %w", err) + } return &PromReporter{ lggr: lggr, @@ -184,6 +195,7 @@ func NewPromReporter( processorErrors: PromExecProcessorErrors, latestRoundID: PromExecLatestRoundID, looppProviderSupported: promLooppCCIPProviderSupported, + configDigestMismatch: promExecConfigDigestMismatch, bhLatencyHistogram: latencyHistogram, bhProcessorLatencyHistogram: processorLatencyHistogram, @@ -193,6 +205,7 @@ func NewPromReporter( beholderProcessorErrors: processorErrors, bhExecLatestRound: execLatestRoundID, bhLooppProviderSupported: looppProviderSupported, + bhConfigDigestMismatch: configDigestMismatch, }, nil } @@ -399,3 +412,15 @@ func (p *PromReporter) TrackLooppProviderSupported(looppCCIPProviderSupported ma )) } } + +func (p *PromReporter) TrackConfigDigestMismatch(mismatch bool) { + var value float64 + if mismatch { + value = 1 + } + p.configDigestMismatch.WithLabelValues(p.chainFamily, p.chainID).Set(value) + p.bhConfigDigestMismatch.Record(context.Background(), int64(value), metric.WithAttributes( + attribute.String("chain_family", p.chainFamily), + attribute.String("chain_id", p.chainID), + )) +} diff --git a/execute/metrics/reporter.go b/execute/metrics/reporter.go index 67c9c2e666..8b569f9bff 100644 --- a/execute/metrics/reporter.go +++ b/execute/metrics/reporter.go @@ -19,6 +19,7 @@ type Reporter interface { TrackLatency(state exectypes.PluginState, method plugincommon.MethodType, latency time.Duration, err error) TrackProcessorOutput(string, plugincommon.MethodType, plugintypes.Trackable) TrackProcessorLatency(processor string, method plugincommon.MethodType, latency time.Duration, err error) + TrackConfigDigestMismatch(mismatch bool) } type Noop struct{} @@ -33,5 +34,7 @@ func (n *Noop) TrackProcessorOutput(string, plugincommon.MethodType, plugintypes func (n *Noop) TrackProcessorLatency(string, plugincommon.MethodType, time.Duration, error) {} +func (n *Noop) TrackConfigDigestMismatch(bool) {} + var _ Reporter = &Noop{} var _ Reporter = &PromReporter{} diff --git a/execute/observation.go b/execute/observation.go index 52279c1247..dfe2e63ae3 100644 --- a/execute/observation.go +++ b/execute/observation.go @@ -1,9 +1,7 @@ package execute import ( - "bytes" "context" - "encoding/hex" "errors" "fmt" "slices" @@ -19,6 +17,7 @@ import ( cciptypes "github.com/smartcontractkit/chainlink-common/pkg/types/ccipocr3" "github.com/smartcontractkit/chainlink-ccip/execute/exectypes" + "github.com/smartcontractkit/chainlink-ccip/internal/plugincommon" dt "github.com/smartcontractkit/chainlink-ccip/internal/plugincommon/discovery/discoverytypes" "github.com/smartcontractkit/chainlink-ccip/pkg/logutil" ) @@ -56,6 +55,21 @@ func (p *Plugin) Observation( } lggr.Infow("decoded previous outcome", "previousOutcome", previousOutcome) + // Check config digest every round and emit a mismatch metric. + // Both home chain and offramp config digest reads are cached, safe to call every round. + configMatch, _, configDigestErr := plugincommon.ConfigDigestsMatch( + ctx, p.ccipReader, consts.PluginTypeExecute, p.reportingCfg.ConfigDigest, + ) + if configDigestErr != nil { + lggr.Errorw("failed to check for config digest mismatch", + "err", configDigestErr, + "homeChainConfigDigest", p.reportingCfg.ConfigDigest, + "pluginType", consts.PluginTypeExecute, + ) + } else { + p.observer.TrackConfigDigestMismatch(!configMatch) + } + // If the previous outcome was the filter state, and reports were built, mark the messages as inflight. if previousOutcome.State == exectypes.Filter { // the lane is invalid due to a config digest mismatch, skip updating @@ -544,15 +558,17 @@ func (p *Plugin) getFilterObservation( } func (p *Plugin) checkConfigDigest(ctx context.Context) error { - offRampConfigDigest, err := p.ccipReader.GetOffRampConfigDigest(ctx, consts.PluginTypeExecute) + match, offRampDigest, err := plugincommon.ConfigDigestsMatch( + ctx, p.ccipReader, consts.PluginTypeExecute, p.reportingCfg.ConfigDigest, + ) if err != nil { - return fmt.Errorf("get offramp config digest: %w", err) + return err } - if !bytes.Equal(offRampConfigDigest[:], p.reportingCfg.ConfigDigest[:]) { + if !match { p.lggr.Warnw("home chain config digest doesn't match offramp's config digest, not starting", "homeChainConfigDigest", p.reportingCfg.ConfigDigest, - "offRampConfigDigest", hex.EncodeToString(offRampConfigDigest[:]), + "offRampConfigDigest", plugincommon.FormatConfigDigest(offRampDigest), ) return errOffRampConfigMismatch } diff --git a/execute/observation_test.go b/execute/observation_test.go index 5382662e71..7490690bd3 100644 --- a/execute/observation_test.go +++ b/execute/observation_test.go @@ -19,6 +19,7 @@ import ( "github.com/smartcontractkit/chainlink-ccip/execute/exectypes" "github.com/smartcontractkit/chainlink-ccip/execute/internal/cache" + execmetrics "github.com/smartcontractkit/chainlink-ccip/execute/metrics" "github.com/smartcontractkit/chainlink-ccip/execute/tokendata/observer" "github.com/smartcontractkit/chainlink-ccip/internal/mocks" "github.com/smartcontractkit/chainlink-ccip/mocks/chainlink_common/ccipocr3" @@ -58,6 +59,7 @@ func Test_Observation_CacheUpdate(t *testing.T) { ocrTypeCodec: ocrTypeCodec, inflightMessageCache: cache.NewInflightMessageCache(10 * time.Minute), ccipReader: ccipReaderMock, + observer: &execmetrics.Noop{}, reportingCfg: ocr3types.ReportingPluginConfig{ OracleID: commontypes.OracleID(1), ConfigDigest: configDigest, diff --git a/execute/plugin_test.go b/execute/plugin_test.go index ddcc4d79ca..01aea4c51e 100644 --- a/execute/plugin_test.go +++ b/execute/plugin_test.go @@ -1215,6 +1215,10 @@ func TestPlugin_Observation_EligibilityCheckFailure(t *testing.T) { GetRmnCurseInfo(mock.Anything). Return(cciptypes.CurseInfo{}, nil).Maybe() + mockCCIPReader.EXPECT(). + GetOffRampConfigDigest(mock.Anything, mock.AnythingOfType("uint8")). + Return([32]byte{}, nil).Maybe() + // Create a simplified plugin structure that will test the eligibility failure // This removes the dependency on actual cache implementations p := &Plugin{ @@ -1223,6 +1227,7 @@ func TestPlugin_Observation_EligibilityCheckFailure(t *testing.T) { lggr: lggr, ocrTypeCodec: ocrTypeCodec, ccipReader: mockCCIPReader, + observer: &metrics.Noop{}, commitRootsCache: cache.NewCommitRootsCache( lggr, 8*time.Hour, diff --git a/internal/plugincommon/config_digest.go b/internal/plugincommon/config_digest.go new file mode 100644 index 0000000000..b0d5d157a9 --- /dev/null +++ b/internal/plugincommon/config_digest.go @@ -0,0 +1,32 @@ +package plugincommon + +import ( + "bytes" + "context" + "encoding/hex" + "fmt" + + readerpkg "github.com/smartcontractkit/chainlink-ccip/pkg/reader" +) + +// ConfigDigestsMatch compares the expected config digest (from the home chain / OCR config) against the +// offramp's on-chain config digest for the given plugin type. +// Returns true when the digests match, false when they differ. +func ConfigDigestsMatch( + ctx context.Context, + ccipReader readerpkg.CCIPReader, + pluginType uint8, + expectedDigest [32]byte, +) (match bool, offRampDigest [32]byte, err error) { + offRampDigest, err = ccipReader.GetOffRampConfigDigest(ctx, pluginType) + if err != nil { + return false, offRampDigest, fmt.Errorf("get offramp config digest: %w", err) + } + + return bytes.Equal(offRampDigest[:], expectedDigest[:]), offRampDigest, nil +} + +// FormatConfigDigest returns a hex-encoded string representation of a config digest. +func FormatConfigDigest(digest [32]byte) string { + return hex.EncodeToString(digest[:]) +}