Skip to content

Commit 4e940e8

Browse files
authored
Run conformance tests via pytest (#484)
1 parent 7f13120 commit 4e940e8

7 files changed

Lines changed: 110 additions & 39 deletions

File tree

.github/workflows/ci.yaml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@ jobs:
3434
python-version: ${{ matrix.python-version }}
3535
- run: uv sync
3636
- run: uv run poe test
37-
- run: uv run poe test-conformance
3837

3938
lint:
4039
runs-on: ubuntu-latest

poe_tasks.toml

Lines changed: 1 addition & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,5 @@
11
#:schema https://json.schemastore.org/partial-poe.json
22

3-
[env]
4-
PROTOVALIDATE_VERSION.default = "v1.2.0"
5-
63
[tasks.add-license-header]
74
help = "Add license header to all source files"
85
cmd = """
@@ -28,8 +25,6 @@ help = "Run code checks"
2825
sequence = [
2926
"lint",
3027
"test",
31-
"test-conformance",
32-
"test-conformance-legacy",
3328
]
3429

3530
[tasks.diffcheck]
@@ -49,7 +44,7 @@ sequence = [
4944
script = "scripts.generate_cel:main"
5045

5146
[tasks.generate-protovalidate]
52-
script = "scripts.generate_protovalidate:main(environ['PROTOVALIDATE_VERSION'])"
47+
script = "scripts.generate_protovalidate:main"
5348

5449
[tasks.generate-test]
5550
sequence = [
@@ -116,27 +111,6 @@ sequence = [
116111
{ cmd = "tombi lint" },
117112
]
118113

119-
[tasks.test-conformance]
120-
help = "Run the CEL conformance tests"
121-
cmd = """
122-
go run github.com/bufbuild/protovalidate/tools/protovalidate-conformance@${PROTOVALIDATE_VERSION}
123-
--strict_message
124-
--expected_failures=test/conformance/nonconforming.yaml
125-
--timeout 10s
126-
python -- -m test.conformance.runner
127-
"""
128-
129-
[tasks.test-conformance-legacy]
130-
help = "Run the CEL conformance tests through the legacy google.protobuf message path"
131-
env = { PROTOVALIDATE_CONFORMANCE_LEGACY = "1" }
132-
cmd = """
133-
go run github.com/bufbuild/protovalidate/tools/protovalidate-conformance@${PROTOVALIDATE_VERSION}
134-
--strict_message
135-
--expected_failures=test/conformance/nonconforming.yaml
136-
--timeout 10s
137-
python -- -m test.conformance.runner
138-
"""
139-
140114
[tasks.test]
141115
help = "Run unit tests"
142116
cmd = "pytest"

scripts/generate_cel.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919

2020
from fix_protobuf_imports.fix_protobuf_imports import fix_protobuf_imports
2121

22-
from test.test_format import CEL_SPEC_VERSION
22+
from test.versions import CEL_SPEC_VERSION
2323

2424
test_dir = Path(__file__).parent.parent / "test"
2525

scripts/generate_protovalidate.py

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,18 +17,20 @@
1717
import subprocess
1818
from pathlib import Path
1919

20+
from test.versions import PROTOVALIDATE_VERSION
2021

21-
def main(version: str) -> None:
22-
if re.match(r"^v\d+\.\d+\.\d+(\-.+)?$", version):
22+
23+
def main() -> None:
24+
if re.match(r"^v\d+\.\d+\.\d+(\-.+)?$", PROTOVALIDATE_VERSION):
2325
# Version tag, fetch from BSR
24-
protovalidate_path = f"buf.build/bufbuild/protovalidate:{version}"
25-
protovalidate_testing_path = f"buf.build/bufbuild/protovalidate-testing:{version}"
26+
protovalidate_path = f"buf.build/bufbuild/protovalidate:{PROTOVALIDATE_VERSION}"
27+
protovalidate_testing_path = f"buf.build/bufbuild/protovalidate-testing:{PROTOVALIDATE_VERSION}"
2628
else:
2729
# Not a tag, generally an unreleased commit, fetch directly from git
28-
protovalidate_path = f"https://github.com/bufbuild/protovalidate.git#subdir=proto/protovalidate,ref={version}"
29-
protovalidate_testing_path = (
30-
f"https://github.com/bufbuild/protovalidate.git#subdir=proto/protovalidate-testing,ref={version}"
30+
protovalidate_path = (
31+
f"https://github.com/bufbuild/protovalidate.git#subdir=proto/protovalidate,ref={PROTOVALIDATE_VERSION}"
3132
)
33+
protovalidate_testing_path = f"https://github.com/bufbuild/protovalidate.git#subdir=proto/protovalidate-testing,ref={PROTOVALIDATE_VERSION}"
3234

3335
repo = Path(__file__).parent.parent
3436

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
# Copyright 2023-2026 Buf Technologies, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from __future__ import annotations
16+
17+
import os
18+
import subprocess
19+
import sys
20+
from pathlib import Path
21+
from textwrap import dedent
22+
23+
import pytest
24+
25+
from test.versions import PROTOVALIDATE_VERSION
26+
27+
28+
def maybe_patch_args_with_debug(args: list[str]) -> list[str]:
29+
# Do a best effort to invoke the child with debugging.
30+
# This invokes internal methods from bundles provided by the IDE
31+
# and may not always work.
32+
try:
33+
from pydevd import ( # ty: ignore[unresolved-import] - provided by IDE # pyright: ignore[reportMissingImports] # noqa: PLC0415
34+
_pydev_bundle,
35+
)
36+
37+
return _pydev_bundle.pydev_monkey.patch_args(args)
38+
except Exception:
39+
return args
40+
41+
42+
@pytest.mark.parametrize("legacy", [False, True], ids=["py", "legacy"])
43+
def test_conformance(*, legacy: bool) -> None:
44+
# Workaround pydevd monkeypatching of -m invocation not being compatible
45+
# with Python 3.14 yet by executing a script that uses runpy itself.
46+
# pydevd does monkeypatch -c form correctly.
47+
script = dedent(
48+
"""
49+
import runpy
50+
runpy.run_module(
51+
'test.conformance.runner',
52+
run_name='__main__',
53+
alter_sys=True
54+
)
55+
"""
56+
)
57+
command = [sys.executable, "--", "-c", script]
58+
command = maybe_patch_args_with_debug(command)
59+
60+
env = os.environ.copy()
61+
if legacy:
62+
env["PROTOVALIDATE_CONFORMANCE_LEGACY"] = "1"
63+
64+
subprocess.run( # noqa: S603
65+
[ # noqa: S607
66+
"go",
67+
"run",
68+
f"github.com/bufbuild/protovalidate/tools/protovalidate-conformance@{PROTOVALIDATE_VERSION}",
69+
"--strict_message",
70+
f"--expected_failures={Path(__file__).parent / 'nonconforming.yaml'}",
71+
"--timeout",
72+
"10s",
73+
*command,
74+
],
75+
env=env,
76+
check=True,
77+
)

test/test_format.py

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,7 @@
2727

2828
from .gen.cel.expr import eval_pb2
2929
from .gen.cel.expr.conformance.test import simple_pb2
30-
31-
# Version of the cel-spec that this implementation is conformant with.
32-
CEL_SPEC_VERSION = "v0.25.1"
30+
from .versions import CEL_SPEC_VERSION
3331

3432
skipped_tests = [
3533
# cel-python seems to have a bug with ints and booleans in the same map which evaluate to the same value

test/versions.py

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# Copyright 2023-2026 Buf Technologies, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import os
16+
17+
# Version of the cel-spec that this implementation is conformant with.
18+
CEL_SPEC_VERSION = os.getenv("CEL_SPEC_VERSION", "v0.25.1")
19+
20+
# Version of protovalidate this implementation targets.
21+
PROTOVALIDATE_VERSION = os.getenv("PROTOVALIDATE_VERSION", "v1.2.0")

0 commit comments

Comments
 (0)