[FEATURE](pyspark) PySpark validation generated from the Pydantic schema

.github/workflows/check-python-code.yaml

@@ -31,7 +31,8 @@ jobs:
         # Default resolution exercises the committed lock against every
         # supported Python minor version. The lowest-direct cell pins each
         # direct dependency to its declared floor (see UV_RESOLUTION below)
-        # and runs only on the Python floor.
+        # and runs only on the Python floor, since the resolved-low pyspark
+        # 3.4 wheels exist for 3.10/3.11 only.
         python: ["3.10", "3.11", "3.12", "3.13", "3.14"]
         resolution: [default]
         include:
@@ -56,9 +57,19 @@ jobs:
         with:
           python-version: ${{ matrix.python }}
 
+      # PySpark 3.4 (the declared minimum) does not support Java 21, which is
+      # the default JDK on ubuntu-latest runners. Pin to Java 17 for the
+      # lowest-direct cell so the resolved pyspark==3.4.0 can actually start.
+      - name: Set up JDK 17
+        if: matrix.resolution == 'lowest-direct'
+        uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
+        with:
+          distribution: temurin
+          java-version: '17'
+
       # UV_RESOLUTION=lowest-direct makes `uv sync` re-resolve every direct
       # dependency to the lowest version permitted by pyproject.toml. This
-      # exercises the declared floor (e.g. pydantic==2.12.0) instead of
+      # exercises the declared floor (e.g. pyspark==3.4.0) instead of
       # whatever the committed lock happens to point at. Failures here mean
       # a direct dep's minimum needs to be bumped. Set via GITHUB_ENV only
       # in the relevant cell so default cells run with no UV_RESOLUTION at

.gitignore

@@ -5,3 +5,4 @@
 docs/docusaurus
 __pycache__/
 .coverage
+.testmondata*

Makefile

@@ -1,23 +1,38 @@
-.PHONY: default uv-sync check test-all test test-only docformat doctest doctest-only mypy mypy-only lint-only update-baselines
+.PHONY: default uv-sync clean-pyspark generate-pyspark check test-all test test-only docformat doctest doctest-only mypy mypy-only lint-only update-baselines
+
+TESTMON ?= --testmon
 
 default: test-all
 
 install: uv-sync
 
 uv-sync:
-	@uv sync --all-packages 2> /dev/null
+	@uv sync --all-packages --all-extras 2> /dev/null
+
+PYSPARK_EXPRESSIONS := packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated
+PYSPARK_GENERATED_TESTS := packages/overture-schema-pyspark/tests/generated
+
+clean-pyspark:
+	@rm -rf $(PYSPARK_EXPRESSIONS) $(PYSPARK_GENERATED_TESTS)
+
+generate-pyspark: uv-sync clean-pyspark
+	@uv run overture-codegen generate --format pyspark \
+		--output-dir $(PYSPARK_EXPRESSIONS) \
+		--test-output-dir $(PYSPARK_GENERATED_TESTS)
+	@uv run ruff check --fix --quiet $(PYSPARK_EXPRESSIONS) $(PYSPARK_GENERATED_TESTS)
+	@uv run ruff format --quiet $(PYSPARK_EXPRESSIONS) $(PYSPARK_GENERATED_TESTS)
 
-check: uv-sync
+check: uv-sync generate-pyspark
 	@$(MAKE) -j test-only doctest-only lint-only mypy-only
 
 test-all: uv-sync
-	@uv run pytest -W error packages/
+	@uv run pytest -W error $(TESTMON) packages/
 
 test: uv-sync
-	@uv run pytest -W error packages/ -x -q --tb=short
+	@uv run pytest -W error $(TESTMON) packages/ -x -q --tb=short
 
 test-only:
-	@uv run pytest -W error packages/ -x -q --tb=short
+	@uv run pytest -W error $(TESTMON) packages/ -x -q --tb=short
 
 coverage: uv-sync
 	@uv run pytest packages/ --cov overture.schema --cov-report=term --cov-report=html && open htmlcov/index.html