diff --git a/features/__snapshots__/validate_image.snap b/features/__snapshots__/validate_image.snap index ef941cc58..31c064735 100644 --- a/features/__snapshots__/validate_image.snap +++ b/features/__snapshots__/validate_image.snap @@ -357,7 +357,7 @@ Error: success criteria not met "source": {}, "warnings": [ { - "msg": "Fails in 2099", + "msg": "Fails in 2099. This will become a failure starting on ${TIMESTAMP}", "metadata": { "effective_on": "${TIMESTAMP}" } diff --git a/internal/evaluator/conftest_evaluator_unit_core_test.go b/internal/evaluator/conftest_evaluator_unit_core_test.go index 9793ccd16..41e66a7e2 100644 --- a/internal/evaluator/conftest_evaluator_unit_core_test.go +++ b/internal/evaluator/conftest_evaluator_unit_core_test.go @@ -221,7 +221,7 @@ func TestConftestEvaluatorEvaluateSeverity(t *testing.T) { }, { - Message: "not yet effective", + Message: "not yet effective. This will become a failure starting on 3021-01-01T00:00:00Z", Metadata: map[string]any{ "effective_on": "3021-01-01T00:00:00Z", }, diff --git a/internal/evaluator/filters.go b/internal/evaluator/filters.go index d6d2d9448..82a8a14d4 100644 --- a/internal/evaluator/filters.go +++ b/internal/evaluator/filters.go @@ -392,6 +392,13 @@ func extractStringArrayFromRuleData(src ecc.Source, key string) []string { } } +// formatEffectiveOnMessage returns the message with an appended notice +// indicating when the rule will begin enforcement. +func formatEffectiveOnMessage(message, effectiveOn string) string { + return fmt.Sprintf("%s. This will become a failure starting on %s", + strings.TrimSuffix(message, "."), effectiveOn) +} + ////////////////////////////////////////////////////////////////////////////// // Comprehensive Policy Resolution ////////////////////////////////////////////////////////////////////////////// @@ -996,7 +1003,10 @@ func (f *LegacyPostEvaluationFilter) CategorizeResults( warnings = append(warnings, result) } case "failure": - if getSeverity(result) == severityWarning || !isResultEffective(result, effectiveTime) { + if getSeverity(result) == severityWarning { + warnings = append(warnings, result) + } else if !isResultEffective(result, effectiveTime) { + result.Message = formatEffectiveOnMessage(result.Message, result.Metadata[metadataEffectiveOn].(string)) warnings = append(warnings, result) } else { failures = append(failures, result) @@ -1119,7 +1129,10 @@ func (f *UnifiedPostEvaluationFilter) CategorizeResults( warnings = append(warnings, filteredResult) } case "failure": - if getSeverity(filteredResult) == severityWarning || !isResultEffective(filteredResult, effectiveTime) { + if getSeverity(filteredResult) == severityWarning { + warnings = append(warnings, filteredResult) + } else if !isResultEffective(filteredResult, effectiveTime) { + filteredResult.Message = formatEffectiveOnMessage(filteredResult.Message, filteredResult.Metadata[metadataEffectiveOn].(string)) warnings = append(warnings, filteredResult) } else { failures = append(failures, filteredResult) diff --git a/internal/evaluator/filters_test.go b/internal/evaluator/filters_test.go index 293b2eb59..fbb13252d 100644 --- a/internal/evaluator/filters_test.go +++ b/internal/evaluator/filters_test.go @@ -758,6 +758,169 @@ func TestUnifiedPostEvaluationFilter(t *testing.T) { }) } +func TestUnifiedCategorizeResultsFutureEffectiveOn(t *testing.T) { + now := time.Date(2025, 6, 15, 0, 0, 0, 0, time.UTC) + futureDate := "2099-01-01T00:00:00Z" + pastDate := "2020-01-01T00:00:00Z" + + source := ecc.Source{ + Config: &ecc.SourceConfig{ + Include: []string{"*"}, + }, + } + configProvider := &simpleConfigProvider{ + effectiveTime: now, + } + filter := NewUnifiedPostEvaluationFilter(NewECPolicyResolver(source, configProvider)) + + tests := []struct { + name string + effectiveTime time.Time + effectiveOn string + severity string + expectWarning bool + expectEnhanced bool + expectedMessage string + }{ + { + name: "severity override demotes failure to warning", + effectiveTime: now, + effectiveOn: futureDate, + severity: "warning", + expectWarning: true, + }, + { + name: "future effective_on demoted to warning with enhanced message", + effectiveTime: now, + effectiveOn: futureDate, + expectWarning: true, + expectEnhanced: true, + expectedMessage: "some failure. This will become a failure starting on " + futureDate, + }, + { + name: "past effective_on stays as failure", + effectiveTime: now, + effectiveOn: pastDate, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + metadata := map[string]interface{}{ + metadataEffectiveOn: tc.effectiveOn, + } + if tc.severity != "" { + metadata["severity"] = tc.severity + } + + result := Result{ + Message: "some failure", + Metadata: metadata, + } + + originalResult := Outcome{ + Failures: []Result{result}, + } + + warnings, failures, _, _ := filter.CategorizeResults( + []Result{result}, originalResult, tc.effectiveTime) + + if tc.expectWarning { + assert.Len(t, warnings, 1) + assert.Empty(t, failures) + if tc.expectEnhanced { + assert.Equal(t, tc.expectedMessage, warnings[0].Message) + } + } else { + assert.Empty(t, warnings) + assert.Len(t, failures, 1) + } + }) + } +} + +func TestLegacyCategorizeResultsFutureEffectiveOn(t *testing.T) { + now := time.Date(2025, 6, 15, 0, 0, 0, 0, time.UTC) + futureDate := "2099-01-01T00:00:00Z" + pastDate := "2020-01-01T00:00:00Z" + + source := ecc.Source{ + Config: &ecc.SourceConfig{ + Include: []string{"*"}, + }, + } + configProvider := &simpleConfigProvider{ + effectiveTime: now, + } + filter := NewLegacyPostEvaluationFilter(source, configProvider) + + tests := []struct { + name string + effectiveTime time.Time + effectiveOn string + severity string + expectWarning bool + expectEnhanced bool + expectedMessage string + }{ + { + name: "severity override demotes failure to warning", + effectiveTime: now, + effectiveOn: futureDate, + severity: "warning", + expectWarning: true, + }, + { + name: "future effective_on demoted to warning with enhanced message", + effectiveTime: now, + effectiveOn: futureDate, + expectWarning: true, + expectEnhanced: true, + expectedMessage: "some failure. This will become a failure starting on " + futureDate, + }, + { + name: "past effective_on stays as failure", + effectiveTime: now, + effectiveOn: pastDate, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + metadata := map[string]interface{}{ + metadataCode: "test.future_rule", + metadataEffectiveOn: tc.effectiveOn, + } + if tc.severity != "" { + metadata["severity"] = tc.severity + } + + result := Result{ + Message: "some failure", + Metadata: metadata, + } + + originalResult := Outcome{ + Failures: []Result{result}, + } + + warnings, failures, _, _ := filter.CategorizeResults( + []Result{result}, originalResult, tc.effectiveTime) + + if tc.expectWarning { + assert.Len(t, warnings, 1) + assert.Empty(t, failures) + if tc.expectEnhanced { + assert.Equal(t, tc.expectedMessage, warnings[0].Message) + } + } else { + assert.Empty(t, warnings) + assert.Len(t, failures, 1) + } + }) + } +} + func TestUnifiedPostEvaluationFilterVsLegacy(t *testing.T) { // Test that the new comprehensive post-evaluation filter produces // the same results as the legacy filtering approach @@ -1796,3 +1959,36 @@ func TestPipelineIntentionWithMultipleValues(t *testing.T) { "security package should be included (has included rules)") }) } + +////////////////////////////////////////////////////////////////////////////// +// formatEffectiveOnMessage tests +////////////////////////////////////////////////////////////////////////////// + +func TestFormatFutureEnforcementMessage(t *testing.T) { + tests := []struct { + name string + message string + effectiveOn string + expected string + }{ + { + name: "appends enforcement date", + message: "The required 'cpe' label is missing", + effectiveOn: "2024-12-01T00:00:00Z", + expected: "The required 'cpe' label is missing. This will become a failure starting on 2024-12-01T00:00:00Z", + }, + { + name: "trims trailing period before appending", + message: "The required 'cpe' label is missing.", + effectiveOn: "2025-01-15T00:00:00Z", + expected: "The required 'cpe' label is missing. This will become a failure starting on 2025-01-15T00:00:00Z", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := formatEffectiveOnMessage(tc.message, tc.effectiveOn) + assert.Equal(t, tc.expected, got) + }) + } +}