Add support for running policy error cases in conformance tests

conformance/src/test/java/dev/cel/conformance/policy/BUILD.bazel

@@ -11,7 +11,11 @@ java_library(
     srcs = glob(["*.java"]),
     deps = [
         "//:auto_value",
+        "//:java_truth",
         "//bundle:cel",
+        "//policy:parser_factory",
+        "//policy:validation_exception",
+        "//policy/testing:k8s_test_tag_handler",
         "//runtime:function_binding",
         "//testing/testrunner:cel_expression_source",
         "//testing/testrunner:cel_test_context",

conformance/src/test/java/dev/cel/conformance/policy/PolicyConformanceTest.java

@@ -14,10 +14,15 @@
 
 package dev.cel.conformance.policy;
 
+import static com.google.common.truth.Truth.assertThat;
+
 import com.google.protobuf.Struct;
 import dev.cel.bundle.Cel;
 import dev.cel.bundle.CelFactory;
 import dev.cel.expr.conformance.proto3.TestAllTypes;
+import dev.cel.policy.CelPolicyParserFactory;
+import dev.cel.policy.CelPolicyValidationException;
+import dev.cel.policy.testing.K8sTagHandler;
 import dev.cel.runtime.CelFunctionBinding;
 import dev.cel.testing.testrunner.CelExpressionSource;
 import dev.cel.testing.testrunner.CelTestContext;
@@ -26,6 +31,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.Locale;
 import org.junit.runners.model.Statement;
 
 /** Statement representing a single CEL policy conformance test case. */
@@ -77,6 +83,13 @@ public void evaluate() throws Throwable {
                 TestAllTypes.getDescriptor().getFile(),
                 Struct.getDescriptor().getFile());
 
+    // Scopes the custom Kubernetes tag visitor exclusively to k8s tests to prevent non-standard
+    // grammar leakage.
+    if (name.startsWith("k8s/")) {
+      contextBuilder.setCelPolicyParser(
+          CelPolicyParserFactory.newYamlParserBuilder().addTagVisitor(new K8sTagHandler()).build());
+    }
+
     Path yamlConfigPath = Paths.get(dirPath, "config.yaml");
     Path textprotoConfigPath = Paths.get(dirPath, "config.textproto");
 
@@ -86,6 +99,16 @@ public void evaluate() throws Throwable {
       contextBuilder.setConfigFile(textprotoConfigPath.toString());
     }
 
-    TestRunnerLibrary.runTest(testCase, contextBuilder.build());
+    try {
+      TestRunnerLibrary.runTest(testCase, contextBuilder.build());
+    } catch (CelPolicyValidationException e) {
+      if (testCase.output().kind() == CelTestCase.Output.Kind.EVAL_ERROR) {
+        String expectedError = testCase.output().evalError().get(0).toString();
+        assertThat(e.getMessage().toLowerCase(Locale.US))
+            .contains(expectedError.toLowerCase(Locale.US));
+      } else {
+        throw e;
+      }
+    }
   }
 }

conformance/src/test/java/dev/cel/conformance/policy/PolicyConformanceTestRunner.java

@@ -44,6 +44,7 @@ public final class PolicyConformanceTestRunner extends ParentRunner<PolicyConfor
   private static final Splitter SPLITTER = Splitter.on(",").omitEmptyStrings();
   private static final String TESTS_YAML_FILE_NAME = "tests.yaml";
   private static final String TESTS_TEXTPROTO_FILE_NAME = "tests.textproto";
+  private static final String POLICY_YAML_FILE_NAME = "policy.yaml";
   private static final TypeRegistry TYPE_REGISTRY =
       TypeRegistry.newBuilder()
           .add(Struct.getDescriptor())
@@ -73,12 +74,40 @@ private static ImmutableList<String> discoverTestDirs(String testdataDir) {
     if (!dir.exists() || !dir.isDirectory()) {
       return ImmutableList.of();
     }
-    String[] directories = dir.list((current, name) -> new File(current, name).isDirectory());
-    if (directories == null) {
+    File[] topLevelDirs = dir.listFiles(File::isDirectory);
+    if (topLevelDirs == null) {
       return ImmutableList.of();
     }
-    Arrays.sort(directories);
-    return ImmutableList.copyOf(directories);
+
+    ImmutableList.Builder<String> testDirsBuilder = ImmutableList.builder();
+    Arrays.sort(topLevelDirs);
+    for (File topLevelDir : topLevelDirs) {
+      if (hasTestSuite(topLevelDir)) {
+        testDirsBuilder.add(topLevelDir.getName());
+        continue;
+      }
+
+      // Check one level deeper to support nested tests like compile_errors/unreachable
+      File[] subDirs = topLevelDir.listFiles(File::isDirectory);
+      if (subDirs == null) {
+        continue;
+      }
+
+      Arrays.sort(subDirs);
+      for (File subDir : subDirs) {
+        if (hasTestSuite(subDir)) {
+          testDirsBuilder.add(topLevelDir.getName() + "/" + subDir.getName());
+        }
+      }
+    }
+
+    return testDirsBuilder.build();
+  }
+
+  private static boolean hasTestSuite(File dir) {
+    return (new File(dir, TESTS_YAML_FILE_NAME).exists()
+            || new File(dir, TESTS_TEXTPROTO_FILE_NAME).exists())
+        && new File(dir, POLICY_YAML_FILE_NAME).exists();
   }
 
   private final ImmutableList<PolicyConformanceTest> tests;

policy/src/main/java/dev/cel/policy/CelPolicy.java

@@ -252,7 +252,7 @@ public abstract static class Builder implements RequiredFieldsChecker {
 
       abstract Optional<Long> id();
 
-      abstract Optional<Result> result();
+      public abstract Optional<Result> result();
 
       abstract Optional<ValueString> explanation();
 

policy/src/main/java/dev/cel/policy/RuleComposer.java

@@ -93,7 +93,7 @@ private Step optimizeRule(Cel cel, CelCompiledRule compiledRule) {
           assertComposedAstIsValid(
               cel,
               output.expr,
-              "conflicting output types found.",
+              "incompatible output types found.",
               matchOutput.sourceId(),
               lastOutputId);
           lastOutputId = matchOutput.sourceId();
@@ -115,7 +115,7 @@ private Step optimizeRule(Cel cel, CelCompiledRule compiledRule) {
               cel,
               output.expr,
               String.format(
-                  "failed composing the subrule '%s' due to conflicting output types.",
+                  "failed composing the subrule '%s' due to incompatible output types.",
                   matchNestedRule.ruleId().map(ValueString::value).orElse("")),
               lastOutputId);
           break;

policy/src/main/java/dev/cel/policy/testing/BUILD.bazel

@@ -0,0 +1,27 @@
+load("@rules_java//java:defs.bzl", "java_library")
+
+package(
+    default_applicable_licenses = [
+        "//:license",
+    ],
+    default_testonly = True,
+    default_visibility = [
+        "//policy/testing:__pkg__",
+    ],
+)
+
+java_library(
+    name = "k8s_tag_handler",
+    srcs = ["K8sTagHandler.java"],
+    tags = [
+    ],
+    deps = [
+        "//common/formats:value_string",
+        "//common/formats:yaml_helper",
+        "//policy",
+        "//policy:parser",
+        "//policy:policy_parser_context",
+        "@maven//:com_google_guava_guava",
+        "@maven//:org_yaml_snakeyaml",
+    ],
+)

policy/src/main/java/dev/cel/policy/testing/K8sTagHandler.java

@@ -0,0 +1,117 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dev.cel.policy.testing;
+
+import com.google.common.annotations.VisibleForTesting;
+import dev.cel.common.formats.ValueString;
+import dev.cel.common.formats.YamlHelper;
+import dev.cel.common.formats.YamlHelper.YamlNodeType;
+import dev.cel.policy.CelPolicy;
+import dev.cel.policy.CelPolicy.Match;
+import dev.cel.policy.CelPolicyParser.TagVisitor;
+import dev.cel.policy.PolicyParserContext;
+import org.yaml.snakeyaml.nodes.Node;
+import org.yaml.snakeyaml.nodes.SequenceNode;
+
+/**
+ * K8sTagHandler is a {@link TagVisitor} implementation to support parsing Kubernetes
+ * ValidatingAdmissionPolicy structures in testing and conformance environments.
+ */
+@VisibleForTesting
+public final class K8sTagHandler implements TagVisitor<Node> {
+
+  @Override
+  public void visitPolicyTag(
+      PolicyParserContext<Node> ctx,
+      long id,
+      String tagName,
+      Node node,
+      CelPolicy.Builder policyBuilder) {
+    switch (tagName) {
+      case "kind":
+        policyBuilder.putMetadata("kind", ctx.newYamlString(node).value());
+        break;
+      case "metadata":
+        YamlHelper.assertYamlType(ctx, id, node, YamlNodeType.MAP);
+        break;
+      case "spec":
+        CelPolicy.Rule spec = ctx.parseRule(ctx, policyBuilder, node);
+        policyBuilder.setRule(spec);
+        break;
+      default:
+        TagVisitor.super.visitPolicyTag(ctx, id, tagName, node, policyBuilder);
+        break;
+    }
+  }
+
+  @Override
+  public void visitRuleTag(
+      PolicyParserContext<Node> ctx,
+      long id,
+      String tagName,
+      Node node,
+      CelPolicy.Builder policyBuilder,
+      CelPolicy.Rule.Builder ruleBuilder) {
+    switch (tagName) {
+      case "failurePolicy":
+        policyBuilder.putMetadata(tagName, ctx.newYamlString(node).value());
+        break;
+      case "matchConstraints":
+        YamlHelper.assertYamlType(ctx, id, node, YamlNodeType.MAP);
+        break;
+      case "validations":
+        if (!YamlHelper.assertYamlType(ctx, id, node, YamlNodeType.LIST)) {
+          return;
+        }
+        SequenceNode seqNode = (SequenceNode) node;
+        for (Node valNode : seqNode.getValue()) {
+          ruleBuilder.addMatches(ctx.parseMatch(ctx, policyBuilder, valNode));
+        }
+        break;
+      default:
+        TagVisitor.super.visitRuleTag(ctx, id, tagName, node, policyBuilder, ruleBuilder);
+        break;
+    }
+  }
+
+  @Override
+  public void visitMatchTag(
+      PolicyParserContext<Node> ctx,
+      long id,
+      String tagName,
+      Node node,
+      CelPolicy.Builder policyBuilder,
+      CelPolicy.Match.Builder matchBuilder) {
+    if (!matchBuilder.result().isPresent()) {
+      matchBuilder.setResult(
+          Match.Result.ofOutput(ValueString.of(ctx.nextId(), "'invalid admission request'")));
+    }
+    switch (tagName) {
+      case "expression":
+        // The K8s expression to validate must return false in order to generate a violation
+        // message.
+        ValueString condition = ctx.newSourceString(node);
+        String invertedCondition = "!(" + condition.value() + ")";
+        matchBuilder.setCondition(ValueString.of(condition.id(), invertedCondition));
+        break;
+      case "messageExpression":
+        matchBuilder.setResult(Match.Result.ofOutput(ctx.newSourceString(node)));
+        break;
+      default:
+        TagVisitor.super.visitMatchTag(ctx, id, tagName, node, policyBuilder, matchBuilder);
+        break;
+    }
+  }
+}

policy/src/test/java/dev/cel/policy/BUILD.bazel

@@ -30,9 +30,9 @@ java_library(
         "//policy:compiler_factory",
         "//policy:parser",
         "//policy:parser_factory",
-        "//policy:policy_parser_context",
         "//policy:source",
         "//policy:validation_exception",
+        "//policy/testing:k8s_test_tag_handler",
         "//runtime",
         "//runtime:function_binding",
         "//testing:cel_runtime_flavor",

policy/src/test/java/dev/cel/policy/CelPolicyCompilerImplTest.java

@@ -37,12 +37,12 @@
 import dev.cel.extensions.CelOptionalLibrary;
 import dev.cel.parser.CelStandardMacro;
 import dev.cel.parser.CelUnparserFactory;
-import dev.cel.policy.PolicyTestHelper.K8sTagHandler;
 import dev.cel.policy.PolicyTestHelper.PolicyTestSuite;
 import dev.cel.policy.PolicyTestHelper.PolicyTestSuite.PolicyTestSection;
 import dev.cel.policy.PolicyTestHelper.PolicyTestSuite.PolicyTestSection.PolicyTestCase;
 import dev.cel.policy.PolicyTestHelper.PolicyTestSuite.PolicyTestSection.PolicyTestCase.PolicyTestInput;
 import dev.cel.policy.PolicyTestHelper.TestYamlPolicy;
+import dev.cel.policy.testing.K8sTagHandler;
 import dev.cel.runtime.CelFunctionBinding;
 import dev.cel.runtime.CelLateFunctionBindings;
 import dev.cel.testing.CelRuntimeFlavor;

policy/src/test/java/dev/cel/policy/CelPolicyYamlParserTest.java

@@ -22,8 +22,8 @@
 import com.google.testing.junit.testparameterinjector.TestParameterInjector;
 import dev.cel.common.formats.ValueString;
 import dev.cel.policy.CelPolicy.Import;
-import dev.cel.policy.PolicyTestHelper.K8sTagHandler;
 import dev.cel.policy.PolicyTestHelper.TestYamlPolicy;
+import dev.cel.policy.testing.K8sTagHandler;
 import org.junit.Test;
 import org.junit.runner.RunWith;