From 8879b80ef64ac954731f7dd4effa0dcefc350dc0 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Sun, 26 Apr 2026 18:00:23 +0200 Subject: [PATCH 01/14] Add ASV backend benchmarks --- .github/workflows/asv-benchmarks.yml | 48 ++++++++ .gitignore | 3 + asv.conf.json | 23 ++++ benchmarks/README.md | 64 ++++++++++ benchmarks/__init__.py | 1 + benchmarks/benchmark_backends.py | 147 +++++++++++++++++++++++ tools/asv_speedup_summary.py | 173 +++++++++++++++++++++++++++ 7 files changed, 459 insertions(+) create mode 100644 .github/workflows/asv-benchmarks.yml create mode 100644 asv.conf.json create mode 100644 benchmarks/README.md create mode 100644 benchmarks/__init__.py create mode 100644 benchmarks/benchmark_backends.py create mode 100644 tools/asv_speedup_summary.py diff --git a/.github/workflows/asv-benchmarks.yml b/.github/workflows/asv-benchmarks.yml new file mode 100644 index 000000000..294dbff42 --- /dev/null +++ b/.github/workflows/asv-benchmarks.yml @@ -0,0 +1,48 @@ +name: ASV Benchmarks + +on: + workflow_dispatch: + +jobs: + benchmark: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: asv + python-version: "3.12" + channels: conda-forge + auto-activate-base: false + + - name: Install ASV + shell: bash -l {0} + run: | + conda install -y -c conda-forge asv + + - name: Configure ASV machine + shell: bash -l {0} + run: | + asv machine --yes + + - name: Run ASV benchmarks + shell: bash -l {0} + run: | + asv run + + - name: Publish ASV report + shell: bash -l {0} + run: | + asv publish + + - name: Upload ASV results + uses: actions/upload-artifact@v4 + with: + name: asv-results + path: | + .asv/results/ + .asv/html/ diff --git a/.gitignore b/.gitignore index bcdc980be..aa7b7f9fd 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,9 @@ htmlcov/ .coverage .coverage.* .cache +.asv/env/ +.asv/results/ +.asv/html/ nosetests.xml coverage.xml *.cover diff --git a/asv.conf.json b/asv.conf.json new file mode 100644 index 000000000..211c4f51c --- /dev/null +++ b/asv.conf.json @@ -0,0 +1,23 @@ +{ + "version": 1, + "project": "GSTools", + "project_url": "https://github.com/jeilealr/GSTools", + "repo": ".", + "branches": ["main"], + "benchmark_dir": "benchmarks", + "env_dir": ".asv/env", + "results_dir": ".asv/results", + "html_dir": ".asv/html", + "show_commit_url": "https://github.com/jeilealr/GSTools/commit/", + "environment_type": "conda", + "conda_channels": ["conda-forge"], + "pythons": ["3.12"], + "matrix": { + "req": { + "numpy": [""] + } + }, + "install_command": [ + "in-dir={env_dir} python -m pip install {build_dir}[rust]" + ] +} diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 000000000..1c36b5be3 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,64 @@ +# GSTools ASV Benchmarks + +This directory contains performance benchmarks for GSTools. Unit tests in +`tests/` should remain focused on correctness; ASV benchmarks in this +directory measure runtime and peak memory only. + +For a beginner-friendly explanation of ASV and every file added here, read +[`ASV_TUTORIAL.md`](ASV_TUTORIAL.md). + +## Setup + +Run ASV commands from the GSTools repository root, where `asv.conf.json` +is located: + +```bash +cd /Users/lealroja/Documents/UFZ/MPS-Tools/GSTools +conda install -c conda-forge asv +asv machine +``` + +## Common Commands + +```bash +asv run --quick +asv run HEAD^! +asv run +asv publish +asv preview +asv compare HEAD~1 HEAD +``` + +`asv run --quick` is the quick development check for ASV 0.6.x. It runs each +benchmark only once and does not save useful performance results. + +`asv run HEAD^!` benchmarks only the current commit. Plain `asv run` follows +the branches configured in `asv.conf.json`. + +If you need to run ASV from another directory, pass the config explicitly: + +```bash +asv --config /Users/lealroja/Documents/UFZ/MPS-Tools/GSTools/asv.conf.json run --quick +``` + +## Backend Comparison + +Benchmarks are parameterized with readable backend labels: + +- `cython_fallback` +- `rust_core` + +ASV tracks each backend separately. Interpret Rust speedup on the same machine +and same benchmark as: + +```text +speedup = cython_fallback_time / rust_core_time +``` + +So: + +- `speedup > 1.0` means Rust is faster +- `speedup = 1.0` means similar performance +- `speedup < 1.0` means Rust is slower + +Do not compare absolute benchmark times across different machines. diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 000000000..c94e5d288 --- /dev/null +++ b/benchmarks/__init__.py @@ -0,0 +1 @@ +"""ASV benchmarks for GSTools.""" diff --git a/benchmarks/benchmark_backends.py b/benchmarks/benchmark_backends.py new file mode 100644 index 000000000..2469fbd0c --- /dev/null +++ b/benchmarks/benchmark_backends.py @@ -0,0 +1,147 @@ +"""Backend benchmarks for GSTools. + +Usage: + cd /my/path/to/GSTools + conda install -c conda-forge asv + asv machine + asv run --quick + asv run HEAD^! + asv run + asv publish + asv preview + asv compare HEAD~1 HEAD + +Backend speedup should be interpreted as: + speedup = cython_fallback_time / rust_core_time + +Values greater than 1.0 mean the Rust backend is faster on the same machine +for the same benchmark and commit. +""" + +from __future__ import annotations + +import contextlib + +import numpy as np + +import gstools as gs + + +BACKENDS = ("cython_fallback", "rust_core") + + +@contextlib.contextmanager +def gstools_backend(use_core): + """Temporarily force either gstools-core or the Cython fallback.""" + previous = (gs.config._GSTOOLS_CORE_AVAIL, gs.config.USE_GSTOOLS_CORE) + try: + if use_core: + if not previous[0]: + raise NotImplementedError("gstools_core is not available") + gs.config._GSTOOLS_CORE_AVAIL = True + gs.config.USE_GSTOOLS_CORE = True + else: + gs.config._GSTOOLS_CORE_AVAIL = False + gs.config.USE_GSTOOLS_CORE = False + yield + finally: + gs.config._GSTOOLS_CORE_AVAIL, gs.config.USE_GSTOOLS_CORE = previous + + +def _use_core(backend): + if backend == "rust_core": + return True + if backend == "cython_fallback": + return False + raise ValueError(f"Unknown backend: {backend}") + + +class BackendBenchmarks: + """Runtime and peak-memory benchmarks for backend-dispatched operations.""" + + params = BACKENDS + param_names = ["backend"] + + def setup_cache(self): + """Create deterministic data once per benchmark environment.""" + srf_x = np.random.RandomState(20220425).rand(2000) * 100.0 + srf_y = np.random.RandomState(20220426).rand(2000) * 100.0 + + vario_x = np.random.RandomState(20220427).rand(900) * 100.0 + vario_y = np.random.RandomState(20220428).rand(900) * 100.0 + vario_field = np.sin(vario_x / 10.0) + np.cos(vario_y / 15.0) + vario_bins = np.linspace(0.0, 60.0, 16) + + rng = np.random.RandomState(20220429) + cond_x = rng.rand(40) * 50.0 + cond_y = rng.rand(40) * 50.0 + cond_val = np.sin(cond_x / 8.0) + np.cos(cond_y / 9.0) + target_pos = (rng.rand(1000) * 50.0, rng.rand(1000) * 50.0) + + return { + "srf": (srf_x, srf_y), + "variogram": ((vario_x, vario_y), vario_field, vario_bins), + "krige": ((cond_x, cond_y), cond_val, target_pos), + } + + def setup(self, data, backend): + """Skip only the Rust parameter when gstools-core is unavailable.""" + if backend == "rust_core" and not gs.config._GSTOOLS_CORE_AVAIL: + raise NotImplementedError("gstools_core is not available") + + def time_srf(self, data, backend): + with gstools_backend(_use_core(backend)): + self._run_srf(data) + + def peakmem_srf(self, data, backend): + with gstools_backend(_use_core(backend)): + self._run_srf(data) + + def time_variogram(self, data, backend): + with gstools_backend(_use_core(backend)): + self._run_variogram(data) + + def peakmem_variogram(self, data, backend): + with gstools_backend(_use_core(backend)): + self._run_variogram(data) + + def time_krige(self, data, backend): + with gstools_backend(_use_core(backend)): + self._run_krige(data) + + def peakmem_krige(self, data, backend): + with gstools_backend(_use_core(backend)): + self._run_krige(data) + + def _run_srf(self, data): + x, y = data["srf"] + model = gs.Exponential(dim=2, var=2.0, len_scale=8.0) + srf = gs.SRF(model, mean=1.0, seed=20220425, mode_no=512) + return srf((x, y), mesh_type="unstructured") + + def _run_variogram(self, data): + pos, field, bins = data["variogram"] + return gs.vario_estimate( + pos, + field, + bins, + mesh_type="unstructured", + return_counts=True, + ) + + def _run_krige(self, data): + cond_pos, cond_val, target_pos = data["krige"] + model = gs.Exponential(dim=2, var=1.5, len_scale=12.0, nugget=0.05) + krige = gs.Krige( + model, + cond_pos, + cond_val, + exact=False, + cond_err=0.05, + ) + return krige( + target_pos, + mesh_type="unstructured", + return_var=True, + store=False, + ) diff --git a/tools/asv_speedup_summary.py b/tools/asv_speedup_summary.py new file mode 100644 index 000000000..01ebf3700 --- /dev/null +++ b/tools/asv_speedup_summary.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python +"""Print Rust-vs-Cython speedups from local ASV result files. + +The summary is optional. ASV itself remains the source of truth for benchmark +storage and visualization. + +Usage: + python tools/asv_speedup_summary.py + python tools/asv_speedup_summary.py --results-dir .asv/results + +Speedup is calculated as: + cython_fallback_time / rust_core_time + +Values greater than 1.0 mean Rust was faster on the same machine, commit, +environment, and benchmark. +""" + +from __future__ import annotations + +import argparse +import itertools +import json +import math +from pathlib import Path + + +BACKENDS = ("cython_fallback", "rust_core") + + +def parse_args(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--results-dir", + default=".asv/results", + type=Path, + help="Path to the ASV results directory.", + ) + parser.add_argument( + "--all", + action="store_true", + help="Include non-time benchmarks as ratios too.", + ) + return parser.parse_args() + + +def iter_result_files(results_dir): + for path in sorted(results_dir.glob("**/*.json")): + if path.name in {"benchmarks.json", "machine.json"}: + continue + yield path + + +def load_json(path): + try: + with path.open(encoding="utf8") as handle: + return json.load(handle) + except json.JSONDecodeError: + return None + + +def result_entry(raw_result, result_columns): + if isinstance(raw_result, dict): + return raw_result + if isinstance(raw_result, list) and result_columns: + return dict(zip(result_columns, raw_result)) + return {"result": raw_result, "params": []} + + +def is_number(value): + return isinstance(value, (int, float)) and not math.isnan(value) + + +def backend_values(entry): + result = entry.get("result") + params = entry.get("params") or [] + if not isinstance(result, list) or not params: + return {} + + values = {} + combinations = itertools.product(*params) + for combo, value in zip(combinations, result): + if not is_number(value): + continue + combo_values = [str(item).strip("'\"") for item in combo] + for backend in BACKENDS: + if backend in combo_values: + values[backend] = float(value) + return values + + +def short_benchmark_name(name): + return name.rsplit(".", maxsplit=1)[-1] + + +def collect_speedups(results_dir, include_all): + rows = [] + for path in iter_result_files(results_dir): + data = load_json(path) + if not data: + continue + result_columns = data.get("result_columns", []) + commit = data.get("commit_hash", "unknown")[:8] + env_name = data.get("env_name", path.stem) + results = data.get("results", {}) + for benchmark, raw_result in results.items(): + if not include_all and ".time_" not in benchmark: + continue + values = backend_values(result_entry(raw_result, result_columns)) + cython = values.get("cython_fallback") + rust = values.get("rust_core") + if not is_number(cython) or not is_number(rust) or rust == 0: + continue + rows.append( + { + "commit": commit, + "env": env_name, + "benchmark": short_benchmark_name(benchmark), + "cython": cython, + "rust": rust, + "speedup": cython / rust, + } + ) + return rows + + +def print_table(rows): + if not rows: + print("No matching Rust-vs-Cython ASV results found.") + return + + headers = [ + "commit", + "env", + "benchmark", + "cython", + "rust", + "speedup", + ] + table = [ + [ + row["commit"], + row["env"], + row["benchmark"], + f"{row['cython']:.6g}", + f"{row['rust']:.6g}", + f"{row['speedup']:.3f}x", + ] + for row in rows + ] + widths = [ + max(len(str(item)) for item in column) + for column in zip(headers, *table) + ] + + def fmt(row): + return " ".join( + str(item).ljust(width) for item, width in zip(row, widths) + ) + + print(fmt(headers)) + print(fmt(["-" * width for width in widths])) + for row in table: + print(fmt(row)) + + +def main(): + args = parse_args() + rows = collect_speedups(args.results_dir, args.all) + print_table(rows) + + +if __name__ == "__main__": + main() From bfba184514a13fdcad8037b1948afd88dc26fbef Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Thu, 7 May 2026 16:36:15 +0200 Subject: [PATCH 02/14] First Bench marking: current status, both backends, ASV and cProfile --- benchmarks/README.md | 451 ++++++++++++++++-- benchmarks/benchmark_backends.py | 226 ++++++--- .../tools}/asv_speedup_summary.py | 104 +++- .../tools/profile_benchmark_workflows.py | 192 ++++++++ pyproject.toml | 1 + 5 files changed, 867 insertions(+), 107 deletions(-) rename {tools => benchmarks/tools}/asv_speedup_summary.py (56%) create mode 100644 benchmarks/tools/profile_benchmark_workflows.py diff --git a/benchmarks/README.md b/benchmarks/README.md index 1c36b5be3..061062be4 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -1,64 +1,455 @@ -# GSTools ASV Benchmarks +# GSTools Benchmark Guide -This directory contains performance benchmarks for GSTools. Unit tests in -`tests/` should remain focused on correctness; ASV benchmarks in this -directory measure runtime and peak memory only. +This directory contains the Airspeed Velocity ([ASV](https://github.com/airspeed-velocity/asv/)) benchmark suite for GSTools and a complementary profiling helper implemented with cProfile (part of the Python standard library). -For a beginner-friendly explanation of ASV and every file added here, read -[`ASV_TUTORIAL.md`](ASV_TUTORIAL.md). +This is a measurement-first guide: benchmark real workflows, inspect the +results, profile the slow paths, and then decide what to optimize. + +Unit tests in `tests/` answer "is the code correct?". The ASV benchmarks in +`benchmarks/` answer "how fast is this workflow, how much memory does it use, +and did that change across commits?". The complementary cProfile helper +answers "inside this workflow, which Python functions are taking most of the +time right now?". + +The benchmarks compare two GSTools backends, which gives more context for +deciding where optimization work should go: + +- `cython_fallback`: the default Cython-backed fallback implementation from + [gstools-cython](https://github.com/GeoStat-Framework/GSTools-Cython). +- `rust_core`: the Rust-backed implementation from + [gstools_core](https://github.com/GeoStat-Framework/GSTools-Core). + +## Index + +- [Setup](#setup) +- [Benchmarking Scripts](#benchmarking-scripts) +- [ASV Configuration](#asv-configuration) +- [Benchmark Naming](#benchmark-naming) +- [Benchmark Coverage](#benchmark-coverage) +- [Benchmark Classes](#benchmark-classes) +- [VariogramWorkflowBenchmarks](#variogramworkflowbenchmarks) +- [KrigingWorkflowBenchmarks](#krigingworkflowbenchmarks) +- [RandomFieldWorkflowBenchmarks](#randomfieldworkflowbenchmarks) +- [Running The Benchmarks](#running-the-benchmarks) +- [Profiling With cProfile](#profiling-with-cprofile) +- [More ASV Commands](#more-asv-commands) +- [External Reference](#external-reference) ## Setup -Run ASV commands from the GSTools repository root, where `asv.conf.json` -is located: +The regular installation commands in the main `README.md` install GSTools for +normal use. For benchmark work, install this local checkout with the optional +benchmark dependencies. + +1. Move to the GSTools repository root: ```bash -cd /Users/lealroja/Documents/UFZ/MPS-Tools/GSTools -conda install -c conda-forge asv -asv machine +cd /path/to/GSTools ``` -## Common Commands +2. Install GSTools in editable mode with the benchmark tooling and Rust backend: ```bash -asv run --quick -asv run HEAD^! -asv run -asv publish -asv preview -asv compare HEAD~1 HEAD +python -m pip install -e ".[benchmark,rust]" +``` + +3. Create a machine profile once per computer: + +```bash +asv machine --yes +``` + +Notes: + +- The machine profile records local hardware information so ASV can label + results correctly. Do not compare absolute times across different machines. +- You can also install ASV with conda or pip, and you can install the Rust + backend package from + [gstools_core](https://github.com/GeoStat-Framework/GSTools-Core) directly. + +## Benchmarking Scripts + +The benchmarking setup currently consists of: + +- `asv.conf.json`: tells ASV how to build GSTools, where benchmarks live, where + to store results, and which Python/environment matrix to use. +- `benchmarks/benchmark_backends.py`: contains the ASV benchmark classes. +- `benchmarks/README.md`: this practical guide. +- `benchmarks/tools/asv_speedup_summary.py`: reads `.asv/results/` and prints + Rust-vs-Cython speedup ratios. +- `benchmarks/tools/profile_benchmark_workflows.py`: runs one representative + workflow from `benchmark_backends.py` under Python's built-in `cProfile`, so + you can see which functions take time in the current checkout. + +Do not run `benchmarks/benchmark_backends.py` directly with Python. ASV loads +that file, discovers benchmark classes and methods, and runs them inside +isolated benchmark environments. The scripts in `benchmarks/tools/` are +different: run them directly with Python. The profiling helper can run against +the current checkout at any time; the speedup-summary helper needs saved ASV +results in `.asv/results/`. + +### ASV Configuration + +The repo root `asv.conf.json` is tailored to this GSTools checkout: + +```json +{ + "repo": ".", + "branches": ["main"], + "benchmark_dir": "benchmarks", + "env_dir": ".asv/env", + "results_dir": ".asv/results", + "html_dir": ".asv/html", + "environment_type": "conda", + "pythons": ["3.12"], + "install_command": [ + "in-dir={env_dir} python -m pip install {build_dir}[rust]" + ] +} +``` + +Important details: + +- `install_command` installs the checked-out GSTools revision with the `[rust]` + extra, so `gstools_core` should be available for Rust backend measurements. + ASV still needs its own `install_command` because it creates isolated + environments for the commits it benchmarks. +- This is separate from any editable install in your active development + environment, such as `python -m pip install -e ".[benchmark,rust]"`. + The editable install is only needed when you want your active environment to + import the current checkout directly, for example when running + `benchmarks/tools/profile_benchmark_workflows.py` with `--backend rust_core`. +- ASV and the cProfile helper use different environments. ASV runs + `benchmarks/benchmark_backends.py` inside `.asv/env/`; the cProfile helper + imports the same benchmark classes but runs them in your active Python + environment. + +ASV creates these generated directories: + +```text +.asv/env/ benchmark environments +.asv/results/ local benchmark result JSON files +.asv/html/ generated local benchmark website ``` -`asv run --quick` is the quick development check for ASV 0.6.x. It runs each -benchmark only once and does not save useful performance results. +Those directories are machine-specific generated artifacts. They should +normally stay out of git. + +If needed, users can list more than one branch, Python version, benchmark +directory, and so on. For example: -`asv run HEAD^!` benchmarks only the current commit. Plain `asv run` follows -the branches configured in `asv.conf.json`. +```json +"branches": ["main", "my-feature-branch"] +``` -If you need to run ASV from another directory, pass the config explicitly: +Users can also benchmark any explicit branch, commit, tag, or range without +changing `asv.conf.json`: ```bash -asv --config /Users/lealroja/Documents/UFZ/MPS-Tools/GSTools/asv.conf.json run --quick +asv run my-feature-branch^! --bench benchmark_backends +asv run main..my-feature-branch --bench benchmark_backends +``` + +ASV checks out package code at each git commit being benchmarked. Commit source +changes before benchmarking them with ASV. Otherwise ASV may benchmark the last +committed package code rather than your uncommitted source changes. + + +### Benchmark Naming + +ASV recognizes benchmark methods by name: + +- methods starting with `time_` measure runtime +- methods starting with `peakmem_` measure peak memory +- `setup_cache()` creates reusable data once per benchmark environment +- `setup()` can skip or prepare individual parameter combinations + +## Benchmark Coverage + +### Shared Constants + +```python +BACKENDS = ("cython_fallback", "rust_core") +VARIOGRAM_CASES = ( + "full_900", + "sampled_5000_to_1500", + "sampled_15000_to_4500", +) +KRIGE_CASES = ("small_30x500", "large_120x2000", "extra_large_360x6000") +FIELD_CASES = ( + "srf_unstructured_randmeth", + "srf_structured_randmeth", + "srf_structured_fourier", + "condsrf_unstructured", +) ``` -## Backend Comparison +These constants define parameter labels shown in ASV results. -Benchmarks are parameterized with readable backend labels: +`BACKENDS` compares: - `cython_fallback` - `rust_core` -ASV tracks each backend separately. Interpret Rust speedup on the same machine -and same benchmark as: +### Shared Helpers + +`gstools_backend(use_core)` temporarily forces GSTools to use either the Cython +fallback backend or the Rust `gstools_core` backend. + +`_random_points(seed, count, scale)` creates deterministic 2D point clouds. + +`_smooth_field(x, y)` creates deterministic synthetic values: + +```python +np.sin(x / 10.0) + np.cos(y / 15.0) +``` + +`_make_variogram_data(...)` creates positions, field values, and bins for +variogram estimation. + +`_make_krige_data(...)` creates conditioning points, conditioning values, and +target points for kriging and conditioned random fields. + +The fixed random seeds are intentional. They keep benchmark inputs stable so +changes in results are more likely to come from code changes, not new random +data. + +### Benchmark Classes + +The ASV benchmarking is organized around workflow classes. Each workflow class +compares `cython_fallback` and `rust_core`, and each class includes both +runtime and peak-memory methods. + +The suite currently measures: + +- `VariogramWorkflowBenchmarks`: full pairwise work vs sampled large work +- `KrigingWorkflowBenchmarks`: small vs larger global kriging systems +- `RandomFieldWorkflowBenchmarks`: unstructured SRF, structured SRF, Fourier + SRF, and conditioned SRF + +This keeps the ASV suite focused on representative workflows rather than +separate duplicate backend checks. + +#### VariogramWorkflowBenchmarks + +This class measures variogram estimation cases: + +```text +full_900 +sampled_5000_to_1500 +sampled_15000_to_4500 +``` + +The labels mean: + +- `full_900`: create 900 scattered points and use all 900 points for the + variogram calculation. +- `sampled_5000_to_1500`: create 5,000 scattered points, then randomly select + 1,500 of those points for the variogram calculation. +- `sampled_15000_to_4500`: create 15,000 scattered points, then randomly select + 4,500 of those points for the variogram calculation. + +The sampled cases still represent larger input datasets, but the variogram +calculation is done on the randomly selected subset so the pairwise work stays +practical. + +#### KrigingWorkflowBenchmarks + +This class measures global kriging at three scales: + +```text +small_30x500 +large_120x2000 +extra_large_360x6000 +``` + +The labels mean: + +- `small_30x500`: 30 conditioning points, 500 target points +- `large_120x2000`: 120 conditioning points, 2,000 target points +- `extra_large_360x6000`: 360 conditioning points, 6,000 target points + +#### RandomFieldWorkflowBenchmarks + +This class measures SRF and CondSRF generation workflows: + +```text +srf_unstructured_randmeth +srf_structured_randmeth +srf_structured_fourier +condsrf_unstructured +``` + +The cases are: + +- `srf_unstructured_randmeth`: SRF using RandMeth on 2,000 unstructured points +- `srf_structured_randmeth`: SRF using RandMeth on a 64 by 64 structured grid +- `srf_structured_fourier`: SRF using the Fourier generator on a 64 by 64 + structured grid +- `condsrf_unstructured`: conditioned SRF with 40 conditioning points and 1,000 + target points + +## Running The Benchmarks + +Check that the benchmark module imports and runs: + +```bash +asv run --quick --show-stderr --bench benchmark_backends +``` + +Save a baseline for the current commit: + +```bash +asv run HEAD^! --bench benchmark_backends +``` + +Run the last five commits on a linear branch: + +```bash +asv run HEAD~5..HEAD --bench benchmark_backends +``` + +Build and open the local website: + +```bash +asv publish +asv preview +``` + +Then open the printed local URL, for example: + +```text +http://127.0.0.1:8082/#/ +``` +(or any other `http://127.0.0.1:/#/` URL shown by the running preview). + +After ASV has saved results, print explicit Rust-vs-Cython speedup ratios: + +```bash +python benchmarks/tools/asv_speedup_summary.py +``` + +The helper reads `.asv/results/` and reports: ```text speedup = cython_fallback_time / rust_core_time ``` -So: +Interpret the ratio as: - `speedup > 1.0` means Rust is faster - `speedup = 1.0` means similar performance - `speedup < 1.0` means Rust is slower -Do not compare absolute benchmark times across different machines. +The browser report shows ASV plots and trends. The speedup helper prints the +backend ratio explicitly in the terminal. By default, the helper skips removed +legacy duplicate rows from older saved results. + +## Profiling With cProfile + +`cProfile` is useful for the current checkout. It does not update the ASV +browser report. Instead, it prints a table in the terminal showing which Python +functions consumed time while one workflow ran. + +The helper script is: + +```text +benchmarks/tools/profile_benchmark_workflows.py +``` + +It imports the ASV benchmark classes from `benchmark_backends.py`, selects one +case, forces one backend, and runs that case under `cProfile`. + +List available cases: + +```bash +python benchmarks/tools/profile_benchmark_workflows.py --list +``` + +Profile selected cases: + +```bash +python benchmarks/tools/profile_benchmark_workflows.py --case variogram-sampled --backend rust_core --limit 10 +python benchmarks/tools/profile_benchmark_workflows.py --case variogram-extra-large --backend rust_core --limit 10 +python benchmarks/tools/profile_benchmark_workflows.py --case krige-large --backend rust_core --limit 10 +python benchmarks/tools/profile_benchmark_workflows.py --case krige-extra-large --backend rust_core --limit 10 +python benchmarks/tools/profile_benchmark_workflows.py --case condsrf --backend rust_core --limit 10 +``` + +Useful options: + +- `--case`: choose one workflow, or use `all` +- `--backend`: choose `cython_fallback` or `rust_core` +- `--limit`: number of function rows to print from the cProfile table +- `--sort cumtime`: sort by cumulative time, usually the best first view +- `--sort tottime`: sort by time spent directly in each function +- `--repeat`: repeat a workflow inside the profiler + +For example, `--limit 10` means "print the top 10 function rows after sorting". + +## More ASV Commands + +Save results for only the current commit: + +```bash +asv run HEAD^! --bench benchmark_backends +``` + +Compare current commit with previous commit: + +```bash +asv run HEAD~1^! --bench benchmark_backends +asv run HEAD^! --bench benchmark_backends +asv compare HEAD~1 HEAD +``` + +Compare local `main` with the current branch tip: + +```bash +asv run main^! --bench benchmark_backends +asv run HEAD^! --bench benchmark_backends +asv compare main HEAD +``` + +Compare remote `main` with the current branch tip: + +```bash +git fetch origin main +asv run origin/main^! --bench benchmark_backends +asv run HEAD^! --bench benchmark_backends +asv compare origin/main HEAD +``` + +On a linear branch, `HEAD~5..HEAD` benchmarks: + +```text +HEAD~4 +HEAD~3 +HEAD~2 +HEAD~1 +HEAD +``` + +Run a selected list of commits: + +```bash +git rev-parse HEAD HEAD~3 main e20c88f7 > /tmp/gstools-asv-commits.txt +asv run HASHFILE:/tmp/gstools-asv-commits.txt --bench benchmark_backends +``` + +Use full commit hashes when sharing results. Short hashes and branch names are +fine locally but can become ambiguous later. + +If running ASV from outside the repo root, pass the config explicitly: + +```bash +asv --config /path/to/MPS-Tools/GSTools/asv.conf.json run --quick --bench benchmark_backends +``` + +## External Reference + +For complete ASV command syntax, see: + +```text +https://asv.readthedocs.io/en/stable/commands.html +``` diff --git a/benchmarks/benchmark_backends.py b/benchmarks/benchmark_backends.py index 2469fbd0c..517c61893 100644 --- a/benchmarks/benchmark_backends.py +++ b/benchmarks/benchmark_backends.py @@ -1,11 +1,11 @@ -"""Backend benchmarks for GSTools. +"""Workflow benchmarks for GSTools backends. Usage: - cd /my/path/to/GSTools - conda install -c conda-forge asv - asv machine - asv run --quick - asv run HEAD^! + cd /path/to/MPS-Tools/GSTools + python -m pip install -e ".[benchmark]" + asv machine --yes + asv run --quick --show-stderr --bench benchmark_backends + asv run HEAD^! --bench benchmark_backends asv run asv publish asv preview @@ -28,6 +28,18 @@ BACKENDS = ("cython_fallback", "rust_core") +VARIOGRAM_CASES = ( + "full_900", + "sampled_5000_to_1500", + "sampled_15000_to_4500", +) +KRIGE_CASES = ("small_30x500", "large_120x2000", "extra_large_360x6000") +FIELD_CASES = ( + "srf_unstructured_randmeth", + "srf_structured_randmeth", + "srf_structured_fourier", + "condsrf_unstructured", +) @contextlib.contextmanager @@ -56,81 +68,181 @@ def _use_core(backend): raise ValueError(f"Unknown backend: {backend}") -class BackendBenchmarks: - """Runtime and peak-memory benchmarks for backend-dispatched operations.""" +def _random_points(seed, count, scale): + rng = np.random.RandomState(seed) + return rng.rand(count) * scale, rng.rand(count) * scale - params = BACKENDS - param_names = ["backend"] - def setup_cache(self): - """Create deterministic data once per benchmark environment.""" - srf_x = np.random.RandomState(20220425).rand(2000) * 100.0 - srf_y = np.random.RandomState(20220426).rand(2000) * 100.0 +def _smooth_field(x, y): + return np.sin(x / 10.0) + np.cos(y / 15.0) + + +def _make_variogram_data(seed, count, scale=100.0): + x, y = _random_points(seed, count, scale) + field = _smooth_field(x, y) + bins = np.linspace(0.0, scale * 0.6, 16) + return (x, y), field, bins + - vario_x = np.random.RandomState(20220427).rand(900) * 100.0 - vario_y = np.random.RandomState(20220428).rand(900) * 100.0 - vario_field = np.sin(vario_x / 10.0) + np.cos(vario_y / 15.0) - vario_bins = np.linspace(0.0, 60.0, 16) +def _make_krige_data(seed, cond_count, target_count, scale=50.0): + rng = np.random.RandomState(seed) + cond_x = rng.rand(cond_count) * scale + cond_y = rng.rand(cond_count) * scale + cond_val = _smooth_field(cond_x, cond_y) + target_pos = ( + rng.rand(target_count) * scale, + rng.rand(target_count) * scale, + ) + return (cond_x, cond_y), cond_val, target_pos - rng = np.random.RandomState(20220429) - cond_x = rng.rand(40) * 50.0 - cond_y = rng.rand(40) * 50.0 - cond_val = np.sin(cond_x / 8.0) + np.cos(cond_y / 9.0) - target_pos = (rng.rand(1000) * 50.0, rng.rand(1000) * 50.0) +class VariogramWorkflowBenchmarks: + """Variogram workflow benchmarks by case and backend.""" + + params = [VARIOGRAM_CASES, BACKENDS] + param_names = ["case", "backend"] + + def setup_cache(self): return { - "srf": (srf_x, srf_y), - "variogram": ((vario_x, vario_y), vario_field, vario_bins), - "krige": ((cond_x, cond_y), cond_val, target_pos), + "full_900": _make_variogram_data(20220501, 900), + "sampled_5000_to_1500": _make_variogram_data(20220502, 5000), + "sampled_15000_to_4500": _make_variogram_data(20220503, 15000), } - def setup(self, data, backend): - """Skip only the Rust parameter when gstools-core is unavailable.""" + def setup(self, data, case, backend): if backend == "rust_core" and not gs.config._GSTOOLS_CORE_AVAIL: raise NotImplementedError("gstools_core is not available") - def time_srf(self, data, backend): + def time_variogram_estimate(self, data, case, backend): with gstools_backend(_use_core(backend)): - self._run_srf(data) + self._run_variogram(data, case) - def peakmem_srf(self, data, backend): + def peakmem_variogram_estimate(self, data, case, backend): with gstools_backend(_use_core(backend)): - self._run_srf(data) + self._run_variogram(data, case) + + def _run_variogram(self, data, case): + pos, field, bins = data[case] + kwargs = {} + if case == "sampled_5000_to_1500": + kwargs = {"sampling_size": 1500, "sampling_seed": 20220504} + if case == "sampled_15000_to_4500": + kwargs = {"sampling_size": 4500, "sampling_seed": 20220505} + return gs.vario_estimate( + pos, + field, + bins, + mesh_type="unstructured", + return_counts=True, + **kwargs, + ) + - def time_variogram(self, data, backend): +class KrigingWorkflowBenchmarks: + """Global kriging workflow benchmarks by case and backend.""" + + params = [KRIGE_CASES, BACKENDS] + param_names = ["case", "backend"] + + def setup_cache(self): + return { + "small_30x500": _make_krige_data(20220506, 30, 500), + "large_120x2000": _make_krige_data(20220507, 120, 2000), + "extra_large_360x6000": _make_krige_data(20220508, 360, 6000), + } + + def setup(self, data, case, backend): + if backend == "rust_core" and not gs.config._GSTOOLS_CORE_AVAIL: + raise NotImplementedError("gstools_core is not available") + + def time_global_krige(self, data, case, backend): with gstools_backend(_use_core(backend)): - self._run_variogram(data) + self._run_krige(data, case) - def peakmem_variogram(self, data, backend): + def peakmem_global_krige(self, data, case, backend): with gstools_backend(_use_core(backend)): - self._run_variogram(data) + self._run_krige(data, case) + + def _run_krige(self, data, case): + cond_pos, cond_val, target_pos = data[case] + model = gs.Exponential(dim=2, var=1.5, len_scale=12.0, nugget=0.05) + krige = gs.Krige( + model, + cond_pos, + cond_val, + exact=False, + cond_err=0.05, + ) + return krige( + target_pos, + mesh_type="unstructured", + return_var=True, + store=False, + ) + + +class RandomFieldWorkflowBenchmarks: + """SRF and CondSRF workflow benchmarks by case and backend.""" + + params = [FIELD_CASES, BACKENDS] + param_names = ["case", "backend"] + + def setup_cache(self): + return { + "unstructured_pos": _random_points(20220509, 2000, 100.0), + "structured_pos": ( + np.linspace(0.0, 100.0, 64), + np.linspace(0.0, 100.0, 64), + ), + "condsrf": _make_krige_data(20220510, 40, 1000), + } - def time_krige(self, data, backend): + def setup(self, data, case, backend): + if backend == "rust_core" and not gs.config._GSTOOLS_CORE_AVAIL: + raise NotImplementedError("gstools_core is not available") + + def time_field_generation(self, data, case, backend): with gstools_backend(_use_core(backend)): - self._run_krige(data) + self._run_field(data, case) - def peakmem_krige(self, data, backend): + def peakmem_field_generation(self, data, case, backend): with gstools_backend(_use_core(backend)): - self._run_krige(data) + self._run_field(data, case) + + def _run_field(self, data, case): + if case == "srf_unstructured_randmeth": + return self._run_srf_unstructured(data) + if case == "srf_structured_randmeth": + return self._run_srf_structured(data) + if case == "srf_structured_fourier": + return self._run_srf_fourier(data) + if case == "condsrf_unstructured": + return self._run_condsrf(data) + raise ValueError(f"Unknown field benchmark case: {case}") - def _run_srf(self, data): - x, y = data["srf"] + def _run_srf_unstructured(self, data): model = gs.Exponential(dim=2, var=2.0, len_scale=8.0) - srf = gs.SRF(model, mean=1.0, seed=20220425, mode_no=512) - return srf((x, y), mesh_type="unstructured") + srf = gs.SRF(model, mean=1.0, seed=20220508, mode_no=512) + return srf(data["unstructured_pos"], mesh_type="unstructured") - def _run_variogram(self, data): - pos, field, bins = data["variogram"] - return gs.vario_estimate( - pos, - field, - bins, - mesh_type="unstructured", - return_counts=True, + def _run_srf_structured(self, data): + model = gs.Exponential(dim=2, var=2.0, len_scale=8.0) + srf = gs.SRF(model, mean=1.0, seed=20220509, mode_no=512) + return srf(data["structured_pos"], mesh_type="structured") + + def _run_srf_fourier(self, data): + model = gs.Gaussian(dim=2, var=2.0, len_scale=30.0) + srf = gs.SRF( + model, + generator="Fourier", + period=[100.0, 100.0], + mode_no=[32, 32], + seed=20220510, ) + return srf(data["structured_pos"], mesh_type="structured") - def _run_krige(self, data): - cond_pos, cond_val, target_pos = data["krige"] + def _run_condsrf(self, data): + cond_pos, cond_val, target_pos = data["condsrf"] model = gs.Exponential(dim=2, var=1.5, len_scale=12.0, nugget=0.05) krige = gs.Krige( model, @@ -139,9 +251,11 @@ def _run_krige(self, data): exact=False, cond_err=0.05, ) - return krige( + cond_srf = gs.CondSRF(krige, seed=20220511, mode_no=512) + return cond_srf( target_pos, mesh_type="unstructured", - return_var=True, + seed=20220512, store=False, + krige_store=False, ) diff --git a/tools/asv_speedup_summary.py b/benchmarks/tools/asv_speedup_summary.py similarity index 56% rename from tools/asv_speedup_summary.py rename to benchmarks/tools/asv_speedup_summary.py index 01ebf3700..d341d2178 100644 --- a/tools/asv_speedup_summary.py +++ b/benchmarks/tools/asv_speedup_summary.py @@ -5,14 +5,15 @@ storage and visualization. Usage: - python tools/asv_speedup_summary.py - python tools/asv_speedup_summary.py --results-dir .asv/results + python benchmarks/tools/asv_speedup_summary.py + python benchmarks/tools/asv_speedup_summary.py --results-dir .asv/results + python benchmarks/tools/asv_speedup_summary.py --include-legacy Speedup is calculated as: cython_fallback_time / rust_core_time Values greater than 1.0 mean Rust was faster on the same machine, commit, -environment, and benchmark. +environment, benchmark, and non-backend parameter combination. """ from __future__ import annotations @@ -25,6 +26,14 @@ BACKENDS = ("cython_fallback", "rust_core") +LEGACY_BENCHMARKS = { + "time_srf", + "peakmem_srf", + "time_variogram", + "peakmem_variogram", + "time_krige", + "peakmem_krige", +} def parse_args(): @@ -40,6 +49,11 @@ def parse_args(): action="store_true", help="Include non-time benchmarks as ratios too.", ) + parser.add_argument( + "--include-legacy", + action="store_true", + help="Include removed BackendBenchmarks rows from older saved results.", + ) return parser.parse_args() @@ -70,6 +84,14 @@ def is_number(value): return isinstance(value, (int, float)) and not math.isnan(value) +def flatten_values(values): + if isinstance(values, list): + for value in values: + yield from flatten_values(value) + return + yield values + + def backend_values(entry): result = entry.get("result") params = entry.get("params") or [] @@ -78,7 +100,7 @@ def backend_values(entry): values = {} combinations = itertools.product(*params) - for combo, value in zip(combinations, result): + for combo, value in zip(combinations, flatten_values(result)): if not is_number(value): continue combo_values = [str(item).strip("'\"") for item in combo] @@ -88,11 +110,40 @@ def backend_values(entry): return values +def backend_rows(entry): + result = entry.get("result") + params = entry.get("params") or [] + if not isinstance(result, list) or not params: + return [] + + rows = [] + combinations = itertools.product(*params) + for combo, value in zip(combinations, flatten_values(result)): + if not is_number(value): + continue + combo_values = [str(item).strip("'\"") for item in combo] + backend = next( + (candidate for candidate in BACKENDS if candidate in combo_values), + None, + ) + if backend is None: + continue + case_values = [item for item in combo_values if item not in BACKENDS] + rows.append( + { + "backend": backend, + "case": "/".join(case_values) if case_values else "-", + "value": float(value), + } + ) + return rows + + def short_benchmark_name(name): return name.rsplit(".", maxsplit=1)[-1] -def collect_speedups(results_dir, include_all): +def collect_speedups(results_dir, include_all, include_legacy): rows = [] for path in iter_result_files(results_dir): data = load_json(path) @@ -103,23 +154,32 @@ def collect_speedups(results_dir, include_all): env_name = data.get("env_name", path.stem) results = data.get("results", {}) for benchmark, raw_result in results.items(): - if not include_all and ".time_" not in benchmark: + benchmark_name = short_benchmark_name(benchmark) + if not include_legacy and benchmark_name in LEGACY_BENCHMARKS: continue - values = backend_values(result_entry(raw_result, result_columns)) - cython = values.get("cython_fallback") - rust = values.get("rust_core") - if not is_number(cython) or not is_number(rust) or rust == 0: + if not include_all and ".time_" not in benchmark: continue - rows.append( - { - "commit": commit, - "env": env_name, - "benchmark": short_benchmark_name(benchmark), - "cython": cython, - "rust": rust, - "speedup": cython / rust, - } - ) + by_case = {} + for row in backend_rows(result_entry(raw_result, result_columns)): + by_case.setdefault(row["case"], {})[row["backend"]] = row[ + "value" + ] + for case, values in by_case.items(): + cython = values.get("cython_fallback") + rust = values.get("rust_core") + if not is_number(cython) or not is_number(rust) or rust == 0: + continue + rows.append( + { + "commit": commit, + "env": env_name, + "benchmark": benchmark_name, + "case": case, + "cython": cython, + "rust": rust, + "speedup": cython / rust, + } + ) return rows @@ -132,6 +192,7 @@ def print_table(rows): "commit", "env", "benchmark", + "case", "cython", "rust", "speedup", @@ -141,6 +202,7 @@ def print_table(rows): row["commit"], row["env"], row["benchmark"], + row["case"], f"{row['cython']:.6g}", f"{row['rust']:.6g}", f"{row['speedup']:.3f}x", @@ -165,7 +227,7 @@ def fmt(row): def main(): args = parse_args() - rows = collect_speedups(args.results_dir, args.all) + rows = collect_speedups(args.results_dir, args.all, args.include_legacy) print_table(rows) diff --git a/benchmarks/tools/profile_benchmark_workflows.py b/benchmarks/tools/profile_benchmark_workflows.py new file mode 100644 index 000000000..7f953e0dd --- /dev/null +++ b/benchmarks/tools/profile_benchmark_workflows.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python +"""Profile the representative GSTools benchmark workflows with cProfile. + +This is a quick measurement helper. ASV remains the source of truth for saved +benchmark results, while this script helps identify the top cumulative Python +call sites before making algorithmic changes. + +Usage: + cd /path/to/MPS-Tools/GSTools + python benchmarks/tools/profile_benchmark_workflows.py --list + python benchmarks/tools/profile_benchmark_workflows.py --case variogram-sampled + python benchmarks/tools/profile_benchmark_workflows.py --case krige-large \ + --backend rust_core --limit 30 +""" + +from __future__ import annotations + +import argparse +import cProfile +from pathlib import Path +import pstats +import sys + + +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO_ROOT)) +sys.path.insert(0, str(REPO_ROOT / "src")) + + +CASES = { + "variogram-full": ( + "VariogramWorkflowBenchmarks", + "time_variogram_estimate", + ("full_900",), + ), + "variogram-sampled": ( + "VariogramWorkflowBenchmarks", + "time_variogram_estimate", + ("sampled_5000_to_1500",), + ), + "variogram-extra-large": ( + "VariogramWorkflowBenchmarks", + "time_variogram_estimate", + ("sampled_15000_to_4500",), + ), + "krige-small": ( + "KrigingWorkflowBenchmarks", + "time_global_krige", + ("small_30x500",), + ), + "krige-large": ( + "KrigingWorkflowBenchmarks", + "time_global_krige", + ("large_120x2000",), + ), + "krige-extra-large": ( + "KrigingWorkflowBenchmarks", + "time_global_krige", + ("extra_large_360x6000",), + ), + "srf-unstructured": ( + "RandomFieldWorkflowBenchmarks", + "time_field_generation", + ("srf_unstructured_randmeth",), + ), + "srf-structured": ( + "RandomFieldWorkflowBenchmarks", + "time_field_generation", + ("srf_structured_randmeth",), + ), + "srf-fourier": ( + "RandomFieldWorkflowBenchmarks", + "time_field_generation", + ("srf_structured_fourier",), + ), + "condsrf": ( + "RandomFieldWorkflowBenchmarks", + "time_field_generation", + ("condsrf_unstructured",), + ), +} + + +def parse_args(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--case", + default="all", + choices=["all", *CASES], + help="Workflow to profile. Defaults to all workflows.", + ) + parser.add_argument( + "--repeat", + default=1, + type=int, + help="Number of times to run each selected workflow.", + ) + parser.add_argument( + "--limit", + default=25, + type=int, + help="Number of cProfile rows to print per workflow.", + ) + parser.add_argument( + "--sort", + default="cumtime", + choices=["cumtime", "tottime", "calls"], + help="pstats sort key.", + ) + parser.add_argument( + "--backend", + default="rust_core", + choices=["cython_fallback", "rust_core"], + help="Backend label to force while profiling.", + ) + parser.add_argument( + "--list", + action="store_true", + help="List available workflow cases and exit.", + ) + return parser.parse_args() + + +def iter_selected(case): + if case == "all": + yield from CASES.items() + return + yield case, CASES[case] + + +def load_suite_class(class_name): + try: + from benchmarks import benchmark_backends + except ModuleNotFoundError as err: + print( + "Could not import GSTools benchmark dependencies. Activate the " + "GSTools benchmark environment or install the project dependencies " + f"first. Original error: {err}", + file=sys.stderr, + ) + raise SystemExit(1) from err + return getattr(benchmark_backends, class_name) + + +def run_case( + name, + class_name, + method_name, + params, + repeat, + limit, + sort, + backend, +): + suite_cls = load_suite_class(class_name) + suite = suite_cls() + data = suite.setup_cache() + method = getattr(suite, method_name) + + profiler = cProfile.Profile() + profiler.enable() + for _ in range(repeat): + method(data, *params, backend) + profiler.disable() + + print(f"\n== {name} [{backend}] ==") + stats = pstats.Stats(profiler, stream=sys.stdout) + stats.strip_dirs().sort_stats(sort).print_stats(limit) + + +def main(): + args = parse_args() + if args.list: + for name in CASES: + print(name) + return + + for name, (suite_cls, method_name, params) in iter_selected(args.case): + run_case( + name, + suite_cls, + method_name, + params, + args.repeat, + args.limit, + args.sort, + args.backend, + ) + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index 885bcd774..f39ea2eab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,6 +66,7 @@ doc = [ ] plotting = ["matplotlib>=3.7", "pyvista>=0.40"] rust = ["gstools_core>=1.0.0"] +benchmark = ["asv"] test = ["pytest-cov>=3"] lint = ["ruff"] From f8870650ec99193498736a3a5061a28920b0d726 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Fri, 15 May 2026 16:53:34 +0200 Subject: [PATCH 03/14] fix parallelisation in benchmarking, adding macos parallelisation --- .gitignore | 5 +- asv.conf.json | 11 +- asv.macos-openmp.conf.json | 36 ++ benchmarks/README.md | 462 +++++++++++++++--- benchmarks/benchmark_backends.py | 100 +++- benchmarks/tools/asv_speedup_summary.py | 28 +- benchmarks/tools/check_cython_openmp.py | 103 ++++ .../tools/install_macos_openmp_cython.py | 139 ++++++ .../tools/profile_benchmark_workflows.py | 36 +- 9 files changed, 798 insertions(+), 122 deletions(-) create mode 100644 asv.macos-openmp.conf.json create mode 100644 benchmarks/tools/check_cython_openmp.py create mode 100644 benchmarks/tools/install_macos_openmp_cython.py diff --git a/.gitignore b/.gitignore index aa7b7f9fd..1cec6e510 100644 --- a/.gitignore +++ b/.gitignore @@ -41,9 +41,8 @@ htmlcov/ .coverage .coverage.* .cache -.asv/env/ -.asv/results/ -.asv/html/ +.asv/* +.asv-openmp/* nosetests.xml coverage.xml *.cover diff --git a/asv.conf.json b/asv.conf.json index 211c4f51c..289cc426a 100644 --- a/asv.conf.json +++ b/asv.conf.json @@ -14,10 +14,17 @@ "pythons": ["3.12"], "matrix": { "req": { - "numpy": [""] + "emcee": [""], + "hankel": [""], + "meshio": [""], + "numpy": [""], + "pyevtk": [""], + "scipy": [""], + "gstools-cython": [""] } }, "install_command": [ - "in-dir={env_dir} python -m pip install {build_dir}[rust]" + "in-dir={env_dir} python -m pip install gstools_core>=1.0.0", + "in-dir={env_dir} python -m pip install --no-deps {build_dir}" ] } diff --git a/asv.macos-openmp.conf.json b/asv.macos-openmp.conf.json new file mode 100644 index 000000000..9df8067ab --- /dev/null +++ b/asv.macos-openmp.conf.json @@ -0,0 +1,36 @@ +{ + "version": 1, + "project": "GSTools", + "project_url": "https://github.com/jeilealr/GSTools", + "repo": ".", + "branches": ["main"], + "benchmark_dir": "benchmarks", + "env_dir": ".asv-openmp/env", + "results_dir": ".asv-openmp/results", + "html_dir": ".asv-openmp/html", + "show_commit_url": "https://github.com/jeilealr/GSTools/commit/", + "environment_type": "conda", + "conda_channels": ["conda-forge"], + "pythons": ["3.12"], + "matrix": { + "req": { + "cython": [""], + "emcee": [""], + "extension-helpers": [""], + "hankel": [""], + "llvm-openmp": [""], + "meshio": [""], + "numpy": [""], + "pyevtk": [""], + "scipy": [""], + "setuptools": [""], + "wheel": [""] + } + }, + "install_command": [ + "in-dir={env_dir} python -m pip install gstools_core>=1.0.0", + "in-dir={env_dir} python {conf_dir}/benchmarks/tools/install_macos_openmp_cython.py {env_dir}", + "in-dir={env_dir} python {conf_dir}/benchmarks/tools/check_cython_openmp.py --fail-if-no-openmp", + "in-dir={env_dir} python -m pip install --no-deps {build_dir}" + ] +} diff --git a/benchmarks/README.md b/benchmarks/README.md index 061062be4..b4df15b6a 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -2,8 +2,8 @@ This directory contains the Airspeed Velocity ([ASV](https://github.com/airspeed-velocity/asv/)) benchmark suite for GSTools and a complementary profiling helper implemented with cProfile (part of the Python standard library). -This is a measurement-first guide: benchmark real workflows, inspect the -results, profile the slow paths, and then decide what to optimize. +This guide benchmarks GSTools, inspects the +results, profiles where runtime is spent, and then decides what to optimize. Unit tests in `tests/` answer "is the code correct?". The ASV benchmarks in `benchmarks/` answer "how fast is this workflow, how much memory does it use, @@ -23,23 +23,54 @@ deciding where optimization work should go: - [Setup](#setup) - [Benchmarking Scripts](#benchmarking-scripts) -- [ASV Configuration](#asv-configuration) -- [Benchmark Naming](#benchmark-naming) + - [ASV Configuration](#asv-configuration) + - [Benchmark Naming](#benchmark-naming) - [Benchmark Coverage](#benchmark-coverage) -- [Benchmark Classes](#benchmark-classes) -- [VariogramWorkflowBenchmarks](#variogramworkflowbenchmarks) -- [KrigingWorkflowBenchmarks](#krigingworkflowbenchmarks) -- [RandomFieldWorkflowBenchmarks](#randomfieldworkflowbenchmarks) + - [Shared Constants](#shared-constants) + - [Shared Helpers](#shared-helpers) + - [Benchmark Classes](#benchmark-classes) + - [VariogramWorkflowBenchmarks](#variogramworkflowbenchmarks) + - [KrigingWorkflowBenchmarks](#krigingworkflowbenchmarks) + - [RandomFieldWorkflowBenchmarks](#randomfieldworkflowbenchmarks) - [Running The Benchmarks](#running-the-benchmarks) -- [Profiling With cProfile](#profiling-with-cprofile) + - [Baseline Benchmark](#baseline-benchmark) + - [Current Commit Baseline](#current-commit-baseline) + - [Several Commits Baseline](#several-commits-baseline) + - [Summary of Results](#summary-of-results) + - [Visualization of Results](#visualization-of-results) + - [Profiling With cProfile](#profiling-with-cprofile) +- [Optional Parallelisation with OpenMP](#optional-parallelisation-with-openmp) + - [Shared OpenMP Rule](#shared-openmp-rule) + - [macOS Example](#macos-example) + - [What The macOS OpenMP Config Does](#what-the-macos-openmp-config-does) + - [Run On macOS](#run-on-macos) + - [Interpreting The macOS OpenMP Run](#interpreting-the-macos-openmp-run) + - [Windows Example](#windows-example) + - [Linux Example](#linux-example) + - [HPC Example](#hpc-example) + - [Profiling With cProfile for Multiple Threads](#profiling-with-cprofile-for-multiple-threads) - [More ASV Commands](#more-asv-commands) - [External Reference](#external-reference) ## Setup The regular installation commands in the main `README.md` install GSTools for -normal use. For benchmark work, install this local checkout with the optional -benchmark dependencies. +normal use. This benchmark guide uses conda because ASV creates isolated +benchmark environments for the commits it measures. + +The default benchmark configuration intentionally compares both backends with +one GSTools thread: + +```text +gstools.config.NUM_THREADS = 1 +``` + +That keeps the first comparison simple: Cython fallback vs Rust core without +parallelism as a confounding factor. Parallel/OpenMP scaling is treated as a +separate optional experiment because the correct Cython OpenMP build depends on +the user's operating system, compiler, and runtime environment. + +To run the benchmark and the optional cProfile helper, follow these steps: 1. Move to the GSTools repository root: @@ -47,25 +78,28 @@ benchmark dependencies. cd /path/to/GSTools ``` -2. Install GSTools in editable mode with the benchmark tooling and Rust backend: +2. Create and activate a conda environment for local benchmark work: ```bash -python -m pip install -e ".[benchmark,rust]" +conda create -n gstools-benchmark -c conda-forge python=3.12 asv packaging +conda activate gstools-benchmark ``` -3. Create a machine profile once per computer: +If you already have a suitable conda environment, activate that instead. + +3. If you use an existing environment, make sure ASV is installed: ```bash -asv machine --yes +conda install -c conda-forge asv ``` -Notes: +4. Create a machine profile once per computer: -- The machine profile records local hardware information so ASV can label - results correctly. Do not compare absolute times across different machines. -- You can also install ASV with conda or pip, and you can install the Rust - backend package from - [gstools_core](https://github.com/GeoStat-Framework/GSTools-Core) directly. +```bash +asv machine --yes +``` + +The machine profile records local hardware information so ASV can label results correctly. Do not compare absolute times across different machines. ## Benchmarking Scripts @@ -73,6 +107,8 @@ The benchmarking setup currently consists of: - `asv.conf.json`: tells ASV how to build GSTools, where benchmarks live, where to store results, and which Python/environment matrix to use. +- `asv.macos-openmp.conf.json`: optional macOS-specific ASV configuration that + builds `gstools-cython` from source with OpenMP inside ASV's own environment. - `benchmarks/benchmark_backends.py`: contains the ASV benchmark classes. - `benchmarks/README.md`: this practical guide. - `benchmarks/tools/asv_speedup_summary.py`: reads `.asv/results/` and prints @@ -80,13 +116,13 @@ The benchmarking setup currently consists of: - `benchmarks/tools/profile_benchmark_workflows.py`: runs one representative workflow from `benchmark_backends.py` under Python's built-in `cProfile`, so you can see which functions take time in the current checkout. +- `benchmarks/tools/check_cython_openmp.py`: optional helper for checking + whether the active Python environment's GSTools-Cython extensions detect + OpenMP parallel support. +- `benchmarks/tools/install_macos_openmp_cython.py`: helper used only by + `asv.macos-openmp.conf.json` to compile `gstools-cython` with `llvm-openmp` + on macOS. -Do not run `benchmarks/benchmark_backends.py` directly with Python. ASV loads -that file, discovers benchmark classes and methods, and runs them inside -isolated benchmark environments. The scripts in `benchmarks/tools/` are -different: run them directly with Python. The profiling helper can run against -the current checkout at any time; the speedup-summary helper needs saved ASV -results in `.asv/results/`. ### ASV Configuration @@ -102,27 +138,46 @@ The repo root `asv.conf.json` is tailored to this GSTools checkout: "html_dir": ".asv/html", "environment_type": "conda", "pythons": ["3.12"], + "matrix": { + "req": { + "emcee": [""], + "hankel": [""], + "meshio": [""], + "numpy": [""], + "pyevtk": [""], + "scipy": [""], + "gstools-cython": [""] + } + }, "install_command": [ - "in-dir={env_dir} python -m pip install {build_dir}[rust]" + "in-dir={env_dir} python -m pip install gstools_core>=1.0.0", + "in-dir={env_dir} python -m pip install --no-deps {build_dir}" ] } ``` Important details: -- `install_command` installs the checked-out GSTools revision with the `[rust]` - extra, so `gstools_core` should be available for Rust backend measurements. - ASV still needs its own `install_command` because it creates isolated +- `environment_type: "conda"` means conda is required for the ASV workflow in + this guide. ASV creates isolated conda environments for the commits it + benchmarks. +- `pythons: ["3.12"]` means ASV creates Python 3.12 benchmark environments. + Keep this pinned unless you intentionally validate a newer Python/GSTools + backend stack. +- `matrix.req` asks ASV to install GSTools runtime dependencies before + installing the checked-out GSTools source. It includes `gstools-cython` + explicitly because the GSTools commit is installed with `--no-deps`. +- `{build_dir}` is ASV's temporary checkout/build directory for the exact + GSTools commit being benchmarked. +- `install_command` installs the checked-out GSTools revision with `--no-deps`. + It also installs `gstools_core` with pip because `gstools-core` is not + available as a conda package in every solver/platform combination. +- ASV still needs its own `install_command` because it creates isolated environments for the commits it benchmarks. -- This is separate from any editable install in your active development - environment, such as `python -m pip install -e ".[benchmark,rust]"`. - The editable install is only needed when you want your active environment to - import the current checkout directly, for example when running - `benchmarks/tools/profile_benchmark_workflows.py` with `--backend rust_core`. -- ASV and the cProfile helper use different environments. ASV runs - `benchmarks/benchmark_backends.py` inside `.asv/env/`; the cProfile helper - imports the same benchmark classes but runs them in your active Python - environment. +- Run the cProfile helper with the Python executable from ASV's isolated + environment, for example `.asv/env//bin/python`. In that mode, the + ASV environment provides dependencies while the helper imports the current + checkout through the repo `src/` path. ASV creates these generated directories: @@ -154,7 +209,6 @@ ASV checks out package code at each git commit being benchmarked. Commit source changes before benchmarking them with ASV. Otherwise ASV may benchmark the last committed package code rather than your uncommitted source changes. - ### Benchmark Naming ASV recognizes benchmark methods by name: @@ -166,10 +220,21 @@ ASV recognizes benchmark methods by name: ## Benchmark Coverage +This section describes what is measured by the ASV suite and how the benchmark +labels map to real GSTools workflows. The goal is to cover representative +operations that are relevant for geostatistical work, not isolated +micro-functions. + +The current suite measures runtime and peak memory for variogram estimation, +global kriging, spatial random field generation, and conditioned random field +generation. Each workflow is run with both backends so the results can show +both absolute performance and Rust-vs-Cython differences. + ### Shared Constants ```python BACKENDS = ("cython_fallback", "rust_core") +THREAD_COUNTS = _configured_thread_counts() VARIOGRAM_CASES = ( "full_900", "sampled_5000_to_1500", @@ -191,10 +256,18 @@ These constants define parameter labels shown in ASV results. - `cython_fallback` - `rust_core` +`THREAD_COUNTS` defaults to: + +- `threads_1`: force `gstools.config.NUM_THREADS = 1` + +That is the default because the first benchmark target is a clean Cython-vs-Rust +backend comparison without parallelism. + ### Shared Helpers -`gstools_backend(use_core)` temporarily forces GSTools to use either the Cython -fallback backend or the Rust `gstools_core` backend. +`gstools_backend(use_core, num_threads)` temporarily forces GSTools to use +either the Cython fallback backend or the Rust `gstools_core` backend, and +sets `gstools.config.NUM_THREADS` for that benchmark run. `_random_points(seed, count, scale)` creates deterministic 2D point clouds. @@ -291,45 +364,39 @@ The cases are: ## Running The Benchmarks -Check that the benchmark module imports and runs: +### Baseline Benchmark -```bash -asv run --quick --show-stderr --bench benchmark_backends -``` +The baseline benchmark is the first result set to create before doing any +optimization work. It uses the default ASV configuration, so each workflow is +measured with `threads_1` for both `cython_fallback` and `rust_core`. + +#### Current Commit Baseline -Save a baseline for the current commit: +- Save a baseline for the current commit: ```bash asv run HEAD^! --bench benchmark_backends ``` -Run the last five commits on a linear branch: +#### Several Commits Baseline -```bash -asv run HEAD~5..HEAD --bench benchmark_backends -``` +As mentioned previously, ASV can also compare several commits, here we will run the last five commits: -Build and open the local website: +- Run the last five commits on main branch: ```bash -asv publish -asv preview +asv run HEAD~5..HEAD --bench benchmark_backends ``` -Then open the printed local URL, for example: - -```text -http://127.0.0.1:8082/#/ -``` -(or any other `http://127.0.0.1:/#/` URL shown by the running preview). +#### Summary of Results -After ASV has saved results, print explicit Rust-vs-Cython speedup ratios: +After running ASV, inspect the explicit Rust-vs-Cython speedup ratios: ```bash python benchmarks/tools/asv_speedup_summary.py ``` -The helper reads `.asv/results/` and reports: +The helper reads `.asv/results/` and reports ratios per case and thread label: ```text speedup = cython_fallback_time / rust_core_time @@ -341,14 +408,39 @@ Interpret the ratio as: - `speedup = 1.0` means similar performance - `speedup < 1.0` means Rust is slower -The browser report shows ASV plots and trends. The speedup helper prints the -backend ratio explicitly in the terminal. By default, the helper skips removed -legacy duplicate rows from older saved results. +The speedup helper prints the backend ratio explicitly in the terminal. By +default, the helper skips removed legacy duplicate rows from older saved +results. -## Profiling With cProfile +#### Visualization of Results -`cProfile` is useful for the current checkout. It does not update the ASV -browser report. Instead, it prints a table in the terminal showing which Python +You can inspect the results in the ASV browser report by building and opening +the local website: + +```bash +asv publish +asv preview +``` + +Then open the printed local URL, for example: + +```text +http://127.0.0.1:8082/#/ +``` +(or any other `http://127.0.0.1:/#/` URL shown by the running preview). + +The browser report shows ASV plots and trends. ASV plot views do not draw a line/graph when there is only one x-axis point, therefore running `asv run HEAD^! --bench benchmark_backends` will most likely not load any graphs. + +For the default benchmark run, the `threads` column should show `threads_1`. +If you later run the +[optional OpenMP scaling experiment](#optional-parallelisation-with-openmp), +the same column can be used to compare several threads. + + +### Profiling With cProfile + +`cProfile` does not update the ASV results shown in the browser report. +Instead, it prints a table in the terminal showing which Python functions consumed time while one workflow ran. The helper script is: @@ -360,26 +452,244 @@ benchmarks/tools/profile_benchmark_workflows.py It imports the ASV benchmark classes from `benchmark_backends.py`, selects one case, forces one backend, and runs that case under `cProfile`. +Since ASV has already created an isolated Python environment, select that +environment to execute the profiling helper: + +```bash +ASV_ENV="$(ls -td .asv/env/* | head -n 1)" +ASV_PYTHON="$ASV_ENV/bin/python" +``` + +The helper still profiles the current checkout because +`profile_benchmark_workflows.py` adds the repository `src/` directory to +`sys.path`. The ASV environment provides the installed dependencies, including +`gstools-cython` and `gstools_core`. + List available cases: ```bash -python benchmarks/tools/profile_benchmark_workflows.py --list +"$ASV_PYTHON" benchmarks/tools/profile_benchmark_workflows.py --list ``` -Profile selected cases: +Possible profile selected cases: ```bash -python benchmarks/tools/profile_benchmark_workflows.py --case variogram-sampled --backend rust_core --limit 10 -python benchmarks/tools/profile_benchmark_workflows.py --case variogram-extra-large --backend rust_core --limit 10 -python benchmarks/tools/profile_benchmark_workflows.py --case krige-large --backend rust_core --limit 10 -python benchmarks/tools/profile_benchmark_workflows.py --case krige-extra-large --backend rust_core --limit 10 -python benchmarks/tools/profile_benchmark_workflows.py --case condsrf --backend rust_core --limit 10 +"$ASV_PYTHON" benchmarks/tools/profile_benchmark_workflows.py --case variogram-sampled --backend rust_core --threads threads_1 --limit 10 +"$ASV_PYTHON" benchmarks/tools/profile_benchmark_workflows.py --case variogram-extra-large --backend rust_core --threads threads_1 --limit 10 +"$ASV_PYTHON" benchmarks/tools/profile_benchmark_workflows.py --case krige-large --backend rust_core --threads threads_1 --limit 10 +"$ASV_PYTHON" benchmarks/tools/profile_benchmark_workflows.py --case krige-extra-large --backend rust_core --threads threads_1 --limit 10 +"$ASV_PYTHON" benchmarks/tools/profile_benchmark_workflows.py --case condsrf --backend rust_core --threads threads_1 --limit 10 +``` + +## Optional Parallelisation with OpenMP + +This section collects optional workflows for testing Cython and Rust with +several thread counts. OpenMP setup is platform-dependent, so each operating +system should have its own tested instructions. + +The default setup above remains the recommended baseline: one thread, normal +ASV environment, and no extra OpenMP build steps. Use this section only when +you explicitly want to measure backend scaling with multiple thread counts. + +### Shared OpenMP Rule + +The benchmark code can be run with several thread labels by setting for example +`GSTOOLS_BENCHMARK_THREADS=1,2,4,8,16`. That only passes different +`gstools.config.NUM_THREADS` values to GSTools. It does not, by itself, make +the Cython backend parallel. + +For Cython OpenMP scaling, the Cython extension must be compiled with OpenMP +support inside the same ASV environment that runs the benchmark. Always verify +that environment before interpreting Cython scaling results: + +```bash +ASV_ENV="$(ls -td .asv-openmp/env/* | head -n 1)" +"$ASV_ENV/bin/python" benchmarks/tools/check_cython_openmp.py --fail-if-no-openmp +``` + +If the check fails, the benchmark may still run, but the Cython backend should +not be interpreted as an OpenMP-enabled Cython run. + +### macOS Example + +This is the currently tested OpenMP workflow. It is separate from the +default setup above. + +The default ASV configuration, `asv.conf.json`, stays conservative: it is the +one-thread baseline and uses the normal conda-forge `gstools-cython` package. +The default `.asv/env/` environment does not provide Cython OpenMP support. That is why this section uses a second ASV configuration: + +```text +asv.macos-openmp.conf.json +``` + +This OpenMP config creates separate generated directories: + +```text +.asv-openmp/env/ +.asv-openmp/results/ +.asv-openmp/html/ +``` + +That keeps the OpenMP experiment separate from the default `.asv/` baseline. + +#### What The macOS OpenMP Config Does + +`asv.macos-openmp.conf.json` asks conda to install the build/runtime pieces +needed for the macOS OpenMP experiment: + +```text +llvm-openmp +cython +extension-helpers +setuptools +wheel +``` + +During ASV installation, it runs: + +```bash +benchmarks/tools/install_macos_openmp_cython.py +``` + +That helper compiles `gstools-cython` from source inside ASV's own environment, +not inside your active conda environment. This matters because ASV benchmarks +the packages installed under `.asv-openmp/env/`. + +Internally, the helper sets: + +```text +GSTOOLS_BUILD_PARALLEL=1 +CC=/bin/gstools-asv-clang-openmp +CXX=/bin/gstools-asv-clang-openmp++ +``` + +The wrapper translates the plain `-fopenmp` flag used by the Cython build into +Apple-clang-compatible compiler and linker arguments that use conda's +`llvm-openmp`. + +#### Run On macOS + +In the previous section, the default config gives a quick overview for both +backends with `threads_1`. In this section, the OpenMP config runs several +thread labels: `threads_1`, `threads_2`, `threads_4`, `threads_8`, and +`threads_16`. + +Start from the GSTools repository root: + +```bash +cd /path/to/GSTools +``` + +Create a clean driver environment. This environment only runs ASV; ASV will +create the real benchmark environment under `.asv-openmp/env/`. + +```bash +conda create -n gstools-benchmark -c conda-forge python=3.12 asv +conda activate gstools-benchmark +``` + +Create the ASV machine profile once: + +```bash +asv --config asv.macos-openmp.conf.json machine --yes +``` + +Run a quick current-commit OpenMP check. This builds the OpenMP-enabled +`gstools-cython` package inside `.asv-openmp/env/` and runs the benchmark suite: + +```bash +GSTOOLS_BENCHMARK_THREADS=1,2,4,8,16 \ +asv --config asv.macos-openmp.conf.json run HEAD^! --quick --bench benchmark_backends --show-stderr +``` + +Verify that the ASV OpenMP environment really uses Cython OpenMP: + +```bash +ASV_OPENMP_ENV="$(ls -td .asv-openmp/env/* | head -n 1)" +"$ASV_OPENMP_ENV/bin/python" benchmarks/tools/check_cython_openmp.py --verbose +"$ASV_OPENMP_ENV/bin/python" benchmarks/tools/check_cython_openmp.py --fail-if-no-openmp +``` + +Expected result on the tested Mac M2 setup: + +```text +variogram default None -> 10 +field default None -> 10 +krige default None -> 10 +OpenMP check: PASS +``` + +If that check passes, run the last-five-commits OpenMP benchmark: + +```bash +GSTOOLS_BENCHMARK_THREADS=1,2,4,8,16 \ +asv --config asv.macos-openmp.conf.json run HEAD~5..HEAD --bench benchmark_backends --show-stderr +``` + +Print Rust-vs-Cython ratios from the OpenMP result folder: + +```bash +python benchmarks/tools/asv_speedup_summary.py --results-dir .asv-openmp/results +``` + +Build and preview the OpenMP browser report: + +```bash +asv --config asv.macos-openmp.conf.json publish +asv --config asv.macos-openmp.conf.json preview +``` + +#### Interpreting The macOS OpenMP Run + +- Use default `asv.conf.json` for the reproducible one-thread baseline. +- Use `asv.macos-openmp.conf.json` for the macOS OpenMP experiment. +- Only claim Cython OpenMP scaling if `check_cython_openmp.py` passes inside + `.asv-openmp/env/...`. +- The active `gstools-benchmark` conda environment does not need `gstools` + installed. It only needs ASV. The benchmarked GSTools packages live inside + `.asv-openmp/env/...`. + +This workflow is intended for macOS systems that use Apple clang with conda's +`llvm-openmp`. It should be portable across many macOS machines, including +Apple Silicon and Intel Macs, but it is not guaranteed for every macOS setup. + +It is not guaranteed to run without local changes on: + +- older macOS versions +- systems missing Xcode command-line tools +- systems with a nonstandard compiler setup +- HPC or managed macOS environments +- unusual conda installations + +Do not assume this exact OpenMP setup applies to Linux, Windows, or HPC systems. + +### Windows Example + +### Linux Example + +### HPC Example + +### Profiling With cProfile for Multiple Threads + +To profile how a workflow changes across configured thread counts, run the +same cProfile case several times with the OpenMP ASV environment: + +```bash +ASV_OPENMP_ENV="$(ls -td .asv-openmp/env/* | head -n 1)" +ASV_OPENMP_PYTHON="$ASV_OPENMP_ENV/bin/python" + +for threads in threads_1 threads_2 threads_4 threads_8 threads_16; do + "$ASV_OPENMP_PYTHON" benchmarks/tools/profile_benchmark_workflows.py --case krige-extra-large --backend rust_core --threads "$threads" --limit 10 +done ``` Useful options: - `--case`: choose one workflow, or use `all` - `--backend`: choose `cython_fallback` or `rust_core` +- `--threads`: choose `threads_1`, `threads_2`, `threads_4`, `threads_8`, + or `threads_16` - `--limit`: number of function rows to print from the cProfile table - `--sort cumtime`: sort by cumulative time, usually the best first view - `--sort tottime`: sort by time spent directly in each function diff --git a/benchmarks/benchmark_backends.py b/benchmarks/benchmark_backends.py index 517c61893..c9526eb79 100644 --- a/benchmarks/benchmark_backends.py +++ b/benchmarks/benchmark_backends.py @@ -2,7 +2,7 @@ Usage: cd /path/to/MPS-Tools/GSTools - python -m pip install -e ".[benchmark]" + # See benchmarks/README.md for ASV and optional cProfile setup. asv machine --yes asv run --quick --show-stderr --bench benchmark_backends asv run HEAD^! --bench benchmark_backends @@ -15,12 +15,17 @@ speedup = cython_fallback_time / rust_core_time Values greater than 1.0 mean the Rust backend is faster on the same machine -for the same benchmark and commit. +for the same benchmark, commit, and thread label. + +By default the suite uses one GSTools thread. For local OpenMP scaling +experiments, set GSTOOLS_BENCHMARK_THREADS, for example: + GSTOOLS_BENCHMARK_THREADS=1,2,4,8,16 asv run HEAD^! """ from __future__ import annotations import contextlib +import os import numpy as np @@ -28,6 +33,29 @@ BACKENDS = ("cython_fallback", "rust_core") + + +def _configured_thread_counts(): + raw = os.environ.get("GSTOOLS_BENCHMARK_THREADS", "1") + thread_counts = [] + for item in raw.split(","): + item = item.strip() + if not item: + continue + if item.startswith("threads_"): + label = item + value = item.removeprefix("threads_") + else: + label = f"threads_{item}" + value = item + int(value) + thread_counts.append(label) + if not thread_counts: + raise ValueError("GSTOOLS_BENCHMARK_THREADS did not define threads") + return tuple(thread_counts) + + +THREAD_COUNTS = _configured_thread_counts() VARIOGRAM_CASES = ( "full_900", "sampled_5000_to_1500", @@ -43,9 +71,13 @@ @contextlib.contextmanager -def gstools_backend(use_core): - """Temporarily force either gstools-core or the Cython fallback.""" - previous = (gs.config._GSTOOLS_CORE_AVAIL, gs.config.USE_GSTOOLS_CORE) +def gstools_backend(use_core, num_threads): + """Temporarily force backend and GSTools thread count.""" + previous = ( + gs.config._GSTOOLS_CORE_AVAIL, + gs.config.USE_GSTOOLS_CORE, + gs.config.NUM_THREADS, + ) try: if use_core: if not previous[0]: @@ -55,9 +87,14 @@ def gstools_backend(use_core): else: gs.config._GSTOOLS_CORE_AVAIL = False gs.config.USE_GSTOOLS_CORE = False + gs.config.NUM_THREADS = num_threads yield finally: - gs.config._GSTOOLS_CORE_AVAIL, gs.config.USE_GSTOOLS_CORE = previous + ( + gs.config._GSTOOLS_CORE_AVAIL, + gs.config.USE_GSTOOLS_CORE, + gs.config.NUM_THREADS, + ) = previous def _use_core(backend): @@ -68,6 +105,12 @@ def _use_core(backend): raise ValueError(f"Unknown backend: {backend}") +def _num_threads(thread_count): + if thread_count.startswith("threads_"): + return int(thread_count.removeprefix("threads_")) + raise ValueError(f"Unknown thread count: {thread_count}") + + def _random_points(seed, count, scale): rng = np.random.RandomState(seed) return rng.rand(count) * scale, rng.rand(count) * scale @@ -99,8 +142,8 @@ def _make_krige_data(seed, cond_count, target_count, scale=50.0): class VariogramWorkflowBenchmarks: """Variogram workflow benchmarks by case and backend.""" - params = [VARIOGRAM_CASES, BACKENDS] - param_names = ["case", "backend"] + params = [VARIOGRAM_CASES, BACKENDS, THREAD_COUNTS] + param_names = ["case", "backend", "threads"] def setup_cache(self): return { @@ -109,16 +152,17 @@ def setup_cache(self): "sampled_15000_to_4500": _make_variogram_data(20220503, 15000), } - def setup(self, data, case, backend): + def setup(self, data, case, backend, threads): if backend == "rust_core" and not gs.config._GSTOOLS_CORE_AVAIL: raise NotImplementedError("gstools_core is not available") + _num_threads(threads) - def time_variogram_estimate(self, data, case, backend): - with gstools_backend(_use_core(backend)): + def time_variogram_estimate(self, data, case, backend, threads): + with gstools_backend(_use_core(backend), _num_threads(threads)): self._run_variogram(data, case) - def peakmem_variogram_estimate(self, data, case, backend): - with gstools_backend(_use_core(backend)): + def peakmem_variogram_estimate(self, data, case, backend, threads): + with gstools_backend(_use_core(backend), _num_threads(threads)): self._run_variogram(data, case) def _run_variogram(self, data, case): @@ -141,8 +185,8 @@ def _run_variogram(self, data, case): class KrigingWorkflowBenchmarks: """Global kriging workflow benchmarks by case and backend.""" - params = [KRIGE_CASES, BACKENDS] - param_names = ["case", "backend"] + params = [KRIGE_CASES, BACKENDS, THREAD_COUNTS] + param_names = ["case", "backend", "threads"] def setup_cache(self): return { @@ -151,16 +195,17 @@ def setup_cache(self): "extra_large_360x6000": _make_krige_data(20220508, 360, 6000), } - def setup(self, data, case, backend): + def setup(self, data, case, backend, threads): if backend == "rust_core" and not gs.config._GSTOOLS_CORE_AVAIL: raise NotImplementedError("gstools_core is not available") + _num_threads(threads) - def time_global_krige(self, data, case, backend): - with gstools_backend(_use_core(backend)): + def time_global_krige(self, data, case, backend, threads): + with gstools_backend(_use_core(backend), _num_threads(threads)): self._run_krige(data, case) - def peakmem_global_krige(self, data, case, backend): - with gstools_backend(_use_core(backend)): + def peakmem_global_krige(self, data, case, backend, threads): + with gstools_backend(_use_core(backend), _num_threads(threads)): self._run_krige(data, case) def _run_krige(self, data, case): @@ -184,8 +229,8 @@ def _run_krige(self, data, case): class RandomFieldWorkflowBenchmarks: """SRF and CondSRF workflow benchmarks by case and backend.""" - params = [FIELD_CASES, BACKENDS] - param_names = ["case", "backend"] + params = [FIELD_CASES, BACKENDS, THREAD_COUNTS] + param_names = ["case", "backend", "threads"] def setup_cache(self): return { @@ -197,16 +242,17 @@ def setup_cache(self): "condsrf": _make_krige_data(20220510, 40, 1000), } - def setup(self, data, case, backend): + def setup(self, data, case, backend, threads): if backend == "rust_core" and not gs.config._GSTOOLS_CORE_AVAIL: raise NotImplementedError("gstools_core is not available") + _num_threads(threads) - def time_field_generation(self, data, case, backend): - with gstools_backend(_use_core(backend)): + def time_field_generation(self, data, case, backend, threads): + with gstools_backend(_use_core(backend), _num_threads(threads)): self._run_field(data, case) - def peakmem_field_generation(self, data, case, backend): - with gstools_backend(_use_core(backend)): + def peakmem_field_generation(self, data, case, backend, threads): + with gstools_backend(_use_core(backend), _num_threads(threads)): self._run_field(data, case) def _run_field(self, data, case): diff --git a/benchmarks/tools/asv_speedup_summary.py b/benchmarks/tools/asv_speedup_summary.py index d341d2178..b3239d702 100644 --- a/benchmarks/tools/asv_speedup_summary.py +++ b/benchmarks/tools/asv_speedup_summary.py @@ -13,7 +13,7 @@ cython_fallback_time / rust_core_time Values greater than 1.0 mean Rust was faster on the same machine, commit, -environment, benchmark, and non-backend parameter combination. +environment, benchmark, case, and thread-count combination. """ from __future__ import annotations @@ -26,6 +26,7 @@ BACKENDS = ("cython_fallback", "rust_core") +THREAD_PREFIX = "threads_" LEGACY_BENCHMARKS = { "time_srf", "peakmem_srf", @@ -128,11 +129,24 @@ def backend_rows(entry): ) if backend is None: continue - case_values = [item for item in combo_values if item not in BACKENDS] + case_values = [ + item + for item in combo_values + if item not in BACKENDS and not item.startswith(THREAD_PREFIX) + ] + threads = next( + ( + item + for item in combo_values + if item.startswith(THREAD_PREFIX) + ), + "-", + ) rows.append( { "backend": backend, "case": "/".join(case_values) if case_values else "-", + "threads": threads, "value": float(value), } ) @@ -161,10 +175,9 @@ def collect_speedups(results_dir, include_all, include_legacy): continue by_case = {} for row in backend_rows(result_entry(raw_result, result_columns)): - by_case.setdefault(row["case"], {})[row["backend"]] = row[ - "value" - ] - for case, values in by_case.items(): + key = (row["case"], row["threads"]) + by_case.setdefault(key, {})[row["backend"]] = row["value"] + for (case, threads), values in by_case.items(): cython = values.get("cython_fallback") rust = values.get("rust_core") if not is_number(cython) or not is_number(rust) or rust == 0: @@ -175,6 +188,7 @@ def collect_speedups(results_dir, include_all, include_legacy): "env": env_name, "benchmark": benchmark_name, "case": case, + "threads": threads, "cython": cython, "rust": rust, "speedup": cython / rust, @@ -193,6 +207,7 @@ def print_table(rows): "env", "benchmark", "case", + "threads", "cython", "rust", "speedup", @@ -203,6 +218,7 @@ def print_table(rows): row["env"], row["benchmark"], row["case"], + row["threads"], f"{row['cython']:.6g}", f"{row['rust']:.6g}", f"{row['speedup']:.3f}x", diff --git a/benchmarks/tools/check_cython_openmp.py b/benchmarks/tools/check_cython_openmp.py new file mode 100644 index 000000000..02fe73db3 --- /dev/null +++ b/benchmarks/tools/check_cython_openmp.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +"""Check whether GSTools-Cython detects OpenMP parallel support. + +This script verifies the active Python environment. Use it with the editable +development environment or with an ASV-created environment. + +Examples: + python benchmarks/tools/check_cython_openmp.py + python benchmarks/tools/check_cython_openmp.py --fail-if-no-openmp + python benchmarks/tools/check_cython_openmp.py --verbose + .asv/env//bin/python3 benchmarks/tools/check_cython_openmp.py +""" + +from __future__ import annotations + +import argparse +import importlib +import sys + + +MODULES = { + "variogram": "gstools_cython.variogram", + "field": "gstools_cython.field", + "krige": "gstools_cython.krige", +} +EXPLICIT_THREAD_COUNTS = (1, 2, 4, 8, 16) + + +def parse_args(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--fail-if-no-openmp", + action="store_true", + help="Exit with status 1 if OpenMP thread detection reports <= 1.", + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Print per-module default and explicit thread-count values.", + ) + return parser.parse_args() + + +def package_version(package_name): + try: + package = importlib.import_module(package_name) + except ModuleNotFoundError: + return "not installed" + return getattr(package, "__version__", "unknown") + + +def check_module(label, module_name): + module = importlib.import_module(module_name) + default_threads = module.set_num_threads(None) + explicit = { + count: module.set_num_threads(count) + for count in EXPLICIT_THREAD_COUNTS + } + return label, default_threads, explicit + + +def main(): + args = parse_args() + + print(f"python: {sys.executable}") + print(f"gstools: {package_version('gstools')}") + print(f"gstools_cython: {package_version('gstools_cython')}") + print(f"gstools_core: {package_version('gstools_core')}") + if args.verbose: + print( + "OpenMP evidence: default None should be >1. " + "Explicit values only prove the wrapper accepts the requested count." + ) + + default_values = [] + for label, module_name in MODULES.items(): + try: + label, default_threads, explicit = check_module(label, module_name) + except ModuleNotFoundError as err: + print(f"OpenMP check: FAIL. Missing module: {err.name}") + return 1 + default_values.append(default_threads) + if args.verbose: + explicit_text = ", ".join( + f"{request}->{actual}" for request, actual in explicit.items() + ) + print(f"{label} default None -> {default_threads}") + print(f"{label} explicit -> {explicit_text}") + + if min(default_values) > 1: + print("OpenMP check: PASS") + return 0 + + print( + "OpenMP check: FAIL. GSTools-Cython reports one default thread. " + "Explicit thread values may be accepted by the wrapper, but this does " + "not prove that the compiled extension is using OpenMP." + ) + return 1 if args.fail_if_no_openmp else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/tools/install_macos_openmp_cython.py b/benchmarks/tools/install_macos_openmp_cython.py new file mode 100644 index 000000000..09f0840cc --- /dev/null +++ b/benchmarks/tools/install_macos_openmp_cython.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python +"""Install GSTools-Cython with OpenMP inside a macOS ASV environment. + +This helper is intentionally macOS-specific. It is called from +``asv.macos-openmp.conf.json`` after ASV has created a conda environment that +contains ``llvm-openmp``. +""" + +from __future__ import annotations + +import os +import platform +import stat +import subprocess +import sys +from pathlib import Path + + +def run(command, env=None, check=True): + print("+ " + " ".join(str(part) for part in command), flush=True) + return subprocess.run(command, check=check, env=env) + + +def write_wrapper(path, force_cxx=False): + text = """#!/bin/bash +set -e +prefix="${GSTOOLS_OPENMP_PREFIX:-${CONDA_PREFIX:-}}" +name="$(basename "$0")" +if [[ "${GSTOOLS_FORCE_CXX:-0}" == "1" || "$name" == *++* ]]; then + real="${GSTOOLS_REAL_CXX:-/usr/bin/clang++}" +else + real="${GSTOOLS_REAL_CC:-/usr/bin/clang}" +fi +is_compile=0 +for arg in "$@"; do + [[ "$arg" == "-c" ]] && is_compile=1 +done +args=() +for arg in "$@"; do + if [[ "$arg" == "-fopenmp" ]]; then + if [[ "$is_compile" == "1" ]]; then + args+=("-Xpreprocessor" "-fopenmp" "-I${prefix}/include") + else + args+=("-L${prefix}/lib" "-lomp" "-Wl,-rpath,${prefix}/lib") + fi + else + args+=("$arg") + fi +done +exec "$real" "${args[@]}" +""" + if force_cxx: + text = """#!/bin/bash +GSTOOLS_FORCE_CXX=1 exec "$(dirname "$0")/gstools-asv-clang-openmp" "$@" +""" + path.write_text(text, encoding="utf8") + path.chmod(path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + +def main(): + if len(sys.argv) != 2: + print( + "Usage: install_macos_openmp_cython.py ", + file=sys.stderr, + ) + return 2 + + if platform.system() != "Darwin": + print( + "This helper is macOS-specific. Use the default ASV config or " + "write an OpenMP setup for this platform.", + file=sys.stderr, + ) + return 2 + + env_dir = Path(sys.argv[1]).resolve() + include_dir = env_dir / "include" + lib_dir = env_dir / "lib" + omp_header = include_dir / "omp.h" + omp_lib = lib_dir / "libomp.dylib" + + if not omp_header.exists() or not omp_lib.exists(): + print( + "llvm-openmp was not found in the ASV environment. Expected " + f"{omp_header} and {omp_lib}.", + file=sys.stderr, + ) + return 2 + + cc_wrapper = env_dir / "bin" / "gstools-asv-clang-openmp" + cxx_wrapper = env_dir / "bin" / "gstools-asv-clang-openmp++" + write_wrapper(cc_wrapper) + write_wrapper(cxx_wrapper, force_cxx=True) + + build_env = os.environ.copy() + build_env.update( + { + "GSTOOLS_BUILD_PARALLEL": "1", + "GSTOOLS_OPENMP_PREFIX": str(env_dir), + "CC": str(cc_wrapper), + "CXX": str(cxx_wrapper), + "CFLAGS": f"-I{include_dir}", + "LDFLAGS": f"-L{lib_dir}", + } + ) + + run( + [ + sys.executable, + "-m", + "pip", + "uninstall", + "-y", + "gstools-cython", + "gstools_cython", + ], + env=build_env, + check=False, + ) + run( + [ + sys.executable, + "-m", + "pip", + "install", + "--no-build-isolation", + "--no-cache-dir", + "--force-reinstall", + "--no-binary=gstools-cython", + "--no-deps", + "gstools-cython", + ], + env=build_env, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/tools/profile_benchmark_workflows.py b/benchmarks/tools/profile_benchmark_workflows.py index 7f953e0dd..2fd2fdff5 100644 --- a/benchmarks/tools/profile_benchmark_workflows.py +++ b/benchmarks/tools/profile_benchmark_workflows.py @@ -7,10 +7,13 @@ Usage: cd /path/to/MPS-Tools/GSTools - python benchmarks/tools/profile_benchmark_workflows.py --list - python benchmarks/tools/profile_benchmark_workflows.py --case variogram-sampled - python benchmarks/tools/profile_benchmark_workflows.py --case krige-large \ - --backend rust_core --limit 30 + ASV_ENV="$(ls -td .asv/env/* | head -n 1)" + "$ASV_ENV/bin/python" benchmarks/tools/profile_benchmark_workflows.py --list + "$ASV_ENV/bin/python" benchmarks/tools/profile_benchmark_workflows.py \ + --case variogram-sampled + "$ASV_ENV/bin/python" benchmarks/tools/profile_benchmark_workflows.py \ + --case krige-large \ + --backend rust_core --threads threads_1 --limit 30 """ from __future__ import annotations @@ -80,6 +83,14 @@ ), } +THREAD_COUNTS = ( + "threads_1", + "threads_2", + "threads_4", + "threads_8", + "threads_16", +) + def parse_args(): parser = argparse.ArgumentParser(description=__doc__) @@ -113,6 +124,12 @@ def parse_args(): choices=["cython_fallback", "rust_core"], help="Backend label to force while profiling.", ) + parser.add_argument( + "--threads", + default="threads_1", + choices=THREAD_COUNTS, + help="GSTools thread count label.", + ) parser.add_argument( "--list", action="store_true", @@ -134,8 +151,9 @@ def load_suite_class(class_name): except ModuleNotFoundError as err: print( "Could not import GSTools benchmark dependencies. Activate the " - "GSTools benchmark environment or install the project dependencies " - f"first. Original error: {err}", + "GSTools benchmark environment, run this script with an ASV env " + "Python from .asv/env//bin/python, or install the project " + f"dependencies first. Original error: {err}", file=sys.stderr, ) raise SystemExit(1) from err @@ -151,6 +169,7 @@ def run_case( limit, sort, backend, + threads, ): suite_cls = load_suite_class(class_name) suite = suite_cls() @@ -160,10 +179,10 @@ def run_case( profiler = cProfile.Profile() profiler.enable() for _ in range(repeat): - method(data, *params, backend) + method(data, *params, backend, threads) profiler.disable() - print(f"\n== {name} [{backend}] ==") + print(f"\n== {name} [{backend}, {threads}] ==") stats = pstats.Stats(profiler, stream=sys.stdout) stats.strip_dirs().sort_stats(sort).print_stats(limit) @@ -185,6 +204,7 @@ def main(): args.limit, args.sort, args.backend, + args.threads, ) From 806e3868c3599f680ed91105059b7f745af196ee Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Wed, 27 May 2026 15:16:08 +0200 Subject: [PATCH 04/14] Add cross-platform benchmark availability CI --- .github/dependabot.yml | 20 + .github/linters/.markdown-lint.yml | 2 + .github/linters/.yaml-lint.yml | 10 + .github/workflows/asv-benchmarks.yml | 262 +++++++++++++- .github/workflows/super-linter.yml | 44 +++ .gitignore | 1 + ...s-openmp.conf.json => asv.openmp.conf.json | 7 +- benchmarks/README.md | 341 ++++++++++++------ benchmarks/benchmark_backends.py | 7 +- .../tools/check_backend_parallel_ready.py | 130 +++++++ benchmarks/tools/check_cython_openmp.py | 1 - .../tools/install_macos_openmp_cython.py | 139 ------- benchmarks/tools/install_openmp_cython.py | 298 +++++++++++++++ benchmarks/tools/write_asv_ci_config.py | 52 +++ 14 files changed, 1027 insertions(+), 287 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/linters/.markdown-lint.yml create mode 100644 .github/linters/.yaml-lint.yml create mode 100644 .github/workflows/super-linter.yml rename asv.macos-openmp.conf.json => asv.openmp.conf.json (89%) create mode 100644 benchmarks/tools/check_backend_parallel_ready.py delete mode 100644 benchmarks/tools/install_macos_openmp_cython.py create mode 100644 benchmarks/tools/install_openmp_cython.py create mode 100644 benchmarks/tools/write_asv_ci_config.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..4dfddfd9b --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,20 @@ +version: 2 +updates: + # Dependabot version updates do not run benchmark CI every week. They only + # check whether workflow actions such as actions/checkout or Super-Linter + # have newer versions, then open PRs for review. + - package-ecosystem: "github-actions" + directory: "/" + schedule: + # Weekly is a low-noise maintenance cadence. Change to monthly if the + # update PRs become distracting. + interval: "weekly" + day: "monday" + time: "05:00" + groups: + # Group action bumps into one PR where possible, instead of one PR per + # action version. + github-actions: + patterns: + - "*" + open-pull-requests-limit: 5 diff --git a/.github/linters/.markdown-lint.yml b/.github/linters/.markdown-lint.yml new file mode 100644 index 000000000..ee0ceb79a --- /dev/null +++ b/.github/linters/.markdown-lint.yml @@ -0,0 +1,2 @@ +MD013: false +MD033: false diff --git a/.github/linters/.yaml-lint.yml b/.github/linters/.yaml-lint.yml new file mode 100644 index 000000000..b55f2aa14 --- /dev/null +++ b/.github/linters/.yaml-lint.yml @@ -0,0 +1,10 @@ +extends: default + +rules: + document-start: disable + line-length: disable + truthy: + allowed-values: + - "true" + - "false" + - "on" diff --git a/.github/workflows/asv-benchmarks.yml b/.github/workflows/asv-benchmarks.yml index 294dbff42..63c495504 100644 --- a/.github/workflows/asv-benchmarks.yml +++ b/.github/workflows/asv-benchmarks.yml @@ -1,48 +1,270 @@ -name: ASV Benchmarks +name: Benchmark Availability Checks on: + # This workflow checks benchmark installability, not package correctness. + # It is intentionally path-filtered because ASV creates conda environments + # and the OpenMP job compiles extensions. Normal source/test validation is + # handled by main.yml; this workflow runs when benchmark-related files or + # package dependency metadata change. + pull_request: + branches: + - "main" + paths: + - ".github/workflows/asv-benchmarks.yml" + - "asv*.json" + - "benchmarks/**" + - "pyproject.toml" + push: + branches: + - "main" + paths: + - ".github/workflows/asv-benchmarks.yml" + - "asv*.json" + - "benchmarks/**" + - "pyproject.toml" workflow_dispatch: +# The jobs only need to read the repository. Keep the default token minimal. +permissions: + contents: read + +defaults: + run: + # setup-miniconda initializes conda for bash login shells on all runners. + shell: bash -el {0} + jobs: - benchmark: - runs-on: ubuntu-latest + benchmark_1_thread_availability: + # First stage: prove the normal one-thread ASV benchmark can be installed + # and started across the supported runner/dependency combinations. + name: 1-thread ASV on ${{ matrix.os }} py${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + timeout-minutes: 75 + strategy: + # Keep the rest of the OS/dependency matrix running after one failure. + # That gives a full compatibility picture from a single workflow run. + fail-fast: false + matrix: + # This list is hand-picked, not imported from main.yml or GSTools-Core. + # Each entry below becomes one independent GitHub Actions job. The goal + # is representative coverage: each OS runner gets older/current/latest + # Python/NumPy/SciPy stacks without running every possible cross-product. + include: + - case-id: ubuntu-py38-oldest + os: ubuntu-latest + python-version: "3.8" + numpy: "==1.20.0" + scipy: "==1.5.4" + - case-id: ubuntu-py312-current + os: ubuntu-latest + python-version: "3.12" + numpy: "==1.26.2" + scipy: "==1.11.2" + - case-id: ubuntu-py313-latest + os: ubuntu-latest + python-version: "3.13" + numpy: ">=2.1.0" + scipy: ">=1.14.1" + - case-id: windows-py38-oldest + os: windows-latest + python-version: "3.8" + numpy: "==1.20.0" + scipy: "==1.5.4" + - case-id: windows-py312-current + os: windows-latest + python-version: "3.12" + numpy: "==1.26.2" + scipy: "==1.11.2" + - case-id: windows-py313-latest + os: windows-latest + python-version: "3.13" + numpy: ">=2.1.0" + scipy: ">=1.14.1" + - case-id: macos15-py311-older + os: macos-15 + python-version: "3.11" + numpy: "==1.23.2" + scipy: "==1.9.2" + - case-id: macos15-py312-current + os: macos-15 + python-version: "3.12" + numpy: "==1.26.2" + scipy: "==1.11.2" + - case-id: macos15-py313-latest + os: macos-15 + python-version: "3.13" + numpy: ">=2.1.0" + scipy: ">=1.14.1" + - case-id: macos15-intel-py38-oldest + os: macos-15-intel + python-version: "3.8" + numpy: "==1.20.0" + scipy: "==1.5.4" + - case-id: macos15-intel-py312-current + os: macos-15-intel + python-version: "3.12" + numpy: "==1.26.2" + scipy: "==1.11.2" + - case-id: macos15-intel-py313-latest + os: macos-15-intel + python-version: "3.13" + numpy: ">=2.1.0" + scipy: ">=1.14.1" steps: - uses: actions/checkout@v4 with: + # ASV needs git history for revision expressions such as HEAD^!. fetch-depth: 0 - uses: conda-incubator/setup-miniconda@v3 with: - activate-environment: asv + # Driver environment: only runs ASV and helper scripts. The packages + # being benchmarked are installed inside ASV-created environments. + activate-environment: gstools-benchmark-driver python-version: "3.12" channels: conda-forge auto-activate-base: false - - name: Install ASV - shell: bash -l {0} + - name: Install ASV driver run: | - conda install -y -c conda-forge asv + conda install -n gstools-benchmark-driver -y -c conda-forge asv - - name: Configure ASV machine - shell: bash -l {0} + - name: Write per-job ASV config run: | - asv machine --yes + # The committed asv.conf.json stays simple for local users. CI writes + # a temporary config so each matrix child can pin one dependency set. + conda run -n gstools-benchmark-driver python benchmarks/tools/write_asv_ci_config.py \ + --base-config asv.conf.json \ + --output ".asv-ci/config/${{ matrix.case-id }}.json" \ + --python-version "${{ matrix.python-version }}" \ + --numpy "${{ matrix.numpy }}" \ + --scipy "${{ matrix.scipy }}" \ + --env-dir ".asv-ci/env/${{ matrix.case-id }}" \ + --results-dir ".asv-ci/results/${{ matrix.case-id }}" \ + --html-dir ".asv-ci/html/${{ matrix.case-id }}" - - name: Run ASV benchmarks - shell: bash -l {0} + - name: Configure ASV machine run: | - asv run + conda run -n gstools-benchmark-driver asv \ + --config ".asv-ci/config/${{ matrix.case-id }}.json" \ + machine --yes - - name: Publish ASV report - shell: bash -l {0} + - name: Run 1-thread ASV availability check + env: + # Public benchmark-thread interface. Baseline CI intentionally checks + # the non-parallel path first. + GSTOOLS_BENCHMARK_THREADS: "1" run: | - asv publish + conda run -n gstools-benchmark-driver asv \ + --config ".asv-ci/config/${{ matrix.case-id }}.json" \ + run 'HEAD^!' --quick --bench benchmark_backends --show-stderr - - name: Upload ASV results + - name: Upload ASV availability results + # Upload even on failure so environment/config details are available + # when debugging resolver, compiler, or benchmark discovery problems. + if: always() uses: actions/upload-artifact@v4 with: - name: asv-results + name: asv-1-thread-${{ matrix.case-id }} path: | - .asv/results/ - .asv/html/ + .asv-ci/config/${{ matrix.case-id }}.json + .asv-ci/results/${{ matrix.case-id }} + .asv-ci/html/${{ matrix.case-id }} + if-no-files-found: ignore + + benchmark_parallel_backend_availability: + # Second stage: after every 1-thread ASV matrix child succeeds, prove that + # the parallel backends can be installed and used on each OS runner. + name: Parallel backend readiness on ${{ matrix.os }} py${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + # For a matrix job, this waits for all children of the upstream matrix. + needs: benchmark_1_thread_availability + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + # OpenMP builds are compiler-heavy. Check them on every OS, but only + # with the latest dependency stack to avoid multiplying CI cost. + include: + - case-id: ubuntu-py313-latest + os: ubuntu-latest + python-version: "3.13" + numpy: ">=2.1.0" + scipy: ">=1.14.1" + - case-id: windows-py313-latest + os: windows-latest + python-version: "3.13" + numpy: ">=2.1.0" + scipy: ">=1.14.1" + - case-id: macos15-py313-latest + os: macos-15 + python-version: "3.13" + numpy: ">=2.1.0" + scipy: ">=1.14.1" + - case-id: macos15-intel-py313-latest + os: macos-15-intel + python-version: "3.13" + numpy: ">=2.1.0" + scipy: ">=1.14.1" + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: conda-incubator/setup-miniconda@v3 + with: + # OpenMP environment: unlike gstools-benchmark-driver, this env gets + # GSTools, gstools-cython, gstools_core, compilers, and runtime deps. + activate-environment: gstools-benchmark-openmp + python-version: ${{ matrix.python-version }} + channels: conda-forge + auto-activate-base: false + + - name: Install benchmark and build requirements + run: | + # Install runtime deps plus compilers/build tools before compiling + # gstools-cython from source with GSTOOLS_BUILD_PARALLEL=1. + conda install -n gstools-benchmark-openmp -y -c conda-forge \ + "numpy${{ matrix.numpy }}" \ + "scipy${{ matrix.scipy }}" \ + c-compiler \ + cxx-compiler \ + cython \ + emcee \ + extension-helpers \ + hankel \ + meshio \ + pip \ + pyevtk \ + "setuptools>=77" \ + wheel + + - name: Install Rust backend package + run: | + # gstools_core is installed with pip because conda availability is not + # uniform across every runner/platform combination. + conda run -n gstools-benchmark-openmp python -m pip install \ + "gstools_core>=1.0.0" + + - name: Build GSTools-Cython with OpenMP + run: | + # The helper dispatches by platform: llvm-openmp wrapper on macOS, + # conda compiler toolchain on Linux, and native MSVC on Windows. + conda run -n gstools-benchmark-openmp python \ + benchmarks/tools/install_openmp_cython.py + + - name: Install local GSTools checkout + run: | + # Dependencies are already pinned by the matrix. Install the checkout + # without letting pip resolve or replace them. + conda run -n gstools-benchmark-openmp python -m pip install \ + --no-deps --editable . + + - name: Check Cython OpenMP and Rust backend readiness + run: | + # Fast readiness check, not a performance benchmark. It fails if + # Cython does not report OpenMP or Rust cannot run with NUM_THREADS=2. + conda run -n gstools-benchmark-openmp python \ + benchmarks/tools/check_backend_parallel_ready.py --verbose diff --git a/.github/workflows/super-linter.yml b/.github/workflows/super-linter.yml new file mode 100644 index 000000000..17ab77687 --- /dev/null +++ b/.github/workflows/super-linter.yml @@ -0,0 +1,44 @@ +name: Super-Linter + +on: + # This workflow is cheap compared with ASV/compiler checks, so run it on + # every PR and push to main to catch metadata/documentation issues early. + pull_request: + push: + branches: + - "main" + workflow_dispatch: + +# Super-Linter needs read access and a status check permission so it can report +# results back to the commit/PR. It does not need write access to repository +# contents. +permissions: + contents: read + packages: read + statuses: write + +jobs: + lint_repository_metadata: + name: Lint GitHub Actions, YAML, JSON, and Markdown + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Run Super-Linter + uses: super-linter/super-linter@v8.6.0 + env: + DEFAULT_BRANCH: main + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Lint changed files only. This avoids introducing unrelated failures + # from old Markdown/YAML while still checking new PR changes. + VALIDATE_ALL_CODEBASE: false + # Opt-in mode: these true values make Super-Linter skip Python. Ruff + # already owns Python linting in main.yml. + VALIDATE_GITHUB_ACTIONS: true + VALIDATE_JSON: true + VALIDATE_MARKDOWN: true + VALIDATE_YAML: true diff --git a/.gitignore b/.gitignore index 1cec6e510..b154c9fac 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,7 @@ htmlcov/ .cache .asv/* .asv-openmp/* +.asv-ci/* nosetests.xml coverage.xml *.cover diff --git a/asv.macos-openmp.conf.json b/asv.openmp.conf.json similarity index 89% rename from asv.macos-openmp.conf.json rename to asv.openmp.conf.json index 9df8067ab..274e2d3cd 100644 --- a/asv.macos-openmp.conf.json +++ b/asv.openmp.conf.json @@ -14,22 +14,23 @@ "pythons": ["3.12"], "matrix": { "req": { + "c-compiler": [""], + "cxx-compiler": [""], "cython": [""], "emcee": [""], "extension-helpers": [""], "hankel": [""], - "llvm-openmp": [""], "meshio": [""], "numpy": [""], "pyevtk": [""], "scipy": [""], - "setuptools": [""], + "setuptools>=77": [""], "wheel": [""] } }, "install_command": [ "in-dir={env_dir} python -m pip install gstools_core>=1.0.0", - "in-dir={env_dir} python {conf_dir}/benchmarks/tools/install_macos_openmp_cython.py {env_dir}", + "in-dir={env_dir} python {conf_dir}/benchmarks/tools/install_openmp_cython.py {env_dir}", "in-dir={env_dir} python {conf_dir}/benchmarks/tools/check_cython_openmp.py --fail-if-no-openmp", "in-dir={env_dir} python -m pip install --no-deps {build_dir}" ] diff --git a/benchmarks/README.md b/benchmarks/README.md index b4df15b6a..013077232 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -41,14 +41,13 @@ deciding where optimization work should go: - [Profiling With cProfile](#profiling-with-cprofile) - [Optional Parallelisation with OpenMP](#optional-parallelisation-with-openmp) - [Shared OpenMP Rule](#shared-openmp-rule) - - [macOS Example](#macos-example) - - [What The macOS OpenMP Config Does](#what-the-macos-openmp-config-does) - - [Run On macOS](#run-on-macos) - - [Interpreting The macOS OpenMP Run](#interpreting-the-macos-openmp-run) - - [Windows Example](#windows-example) - - [Linux Example](#linux-example) - - [HPC Example](#hpc-example) + - [OpenMP ASV Configuration](#openmp-asv-configuration) + - [Verify Cython OpenMP](#verify-cython-openmp) + - [Run On macOS And Linux](#run-on-macos-and-linux) + - [Run On Windows](#run-on-windows) + - [HPC Notes](#hpc-notes) - [Profiling With cProfile for Multiple Threads](#profiling-with-cprofile-for-multiple-threads) +- [GitHub Action Benchmark Availability Checks](#github-action-benchmark-availability-checks) - [More ASV Commands](#more-asv-commands) - [External Reference](#external-reference) @@ -81,7 +80,7 @@ cd /path/to/GSTools 2. Create and activate a conda environment for local benchmark work: ```bash -conda create -n gstools-benchmark -c conda-forge python=3.12 asv packaging +conda create -n gstools-benchmark -c conda-forge python=3.12 asv conda activate gstools-benchmark ``` @@ -107,8 +106,8 @@ The benchmarking setup currently consists of: - `asv.conf.json`: tells ASV how to build GSTools, where benchmarks live, where to store results, and which Python/environment matrix to use. -- `asv.macos-openmp.conf.json`: optional macOS-specific ASV configuration that - builds `gstools-cython` from source with OpenMP inside ASV's own environment. +- `asv.openmp.conf.json`: optional cross-platform ASV configuration that builds + `gstools-cython` from source with OpenMP inside ASV's own environment. - `benchmarks/benchmark_backends.py`: contains the ASV benchmark classes. - `benchmarks/README.md`: this practical guide. - `benchmarks/tools/asv_speedup_summary.py`: reads `.asv/results/` and prints @@ -119,10 +118,14 @@ The benchmarking setup currently consists of: - `benchmarks/tools/check_cython_openmp.py`: optional helper for checking whether the active Python environment's GSTools-Cython extensions detect OpenMP parallel support. -- `benchmarks/tools/install_macos_openmp_cython.py`: helper used only by - `asv.macos-openmp.conf.json` to compile `gstools-cython` with `llvm-openmp` - on macOS. - +- `benchmarks/tools/check_backend_parallel_ready.py`: CI helper that verifies + Cython OpenMP detection and Rust backend execution with more than one GSTools + thread. +- `benchmarks/tools/write_asv_ci_config.py`: CI helper that writes a temporary + per-job ASV config for one Python/NumPy/SciPy combination. +- `benchmarks/tools/install_openmp_cython.py`: helper used by + `asv.openmp.conf.json` to compile `gstools-cython` with OpenMP on macOS, + Linux, and native Windows. ### ASV Configuration @@ -190,6 +193,22 @@ ASV creates these generated directories: Those directories are machine-specific generated artifacts. They should normally stay out of git. +The optional `asv.openmp.conf.json` uses the same benchmark suite, but stores +its generated files separately: + +```text +.asv-openmp/env/ OpenMP benchmark environments +.asv-openmp/results/ OpenMP result JSON files +.asv-openmp/html/ OpenMP benchmark website +``` + +It also installs build tools needed to compile `gstools-cython` from source +with `GSTOOLS_BUILD_PARALLEL=1`. The platform-specific OpenMP setup is handled +by `benchmarks/tools/install_openmp_cython.py`: macOS uses conda's +`llvm-openmp` with an Apple-clang wrapper, Linux uses the compiler toolchain +from the ASV conda environment when available, and native Windows uses the +installed MSVC Build Tools. + If needed, users can list more than one branch, Python version, benchmark directory, and so on. For example: @@ -201,7 +220,7 @@ Users can also benchmark any explicit branch, commit, tag, or range without changing `asv.conf.json`: ```bash -asv run my-feature-branch^! --bench benchmark_backends +asv run 'my-feature-branch^!' --bench benchmark_backends asv run main..my-feature-branch --bench benchmark_backends ``` @@ -375,7 +394,7 @@ measured with `threads_1` for both `cython_fallback` and `rust_core`. - Save a baseline for the current commit: ```bash -asv run HEAD^! --bench benchmark_backends +asv run 'HEAD^!' --bench benchmark_backends ``` #### Several Commits Baseline @@ -429,14 +448,13 @@ http://127.0.0.1:8082/#/ ``` (or any other `http://127.0.0.1:/#/` URL shown by the running preview). -The browser report shows ASV plots and trends. ASV plot views do not draw a line/graph when there is only one x-axis point, therefore running `asv run HEAD^! --bench benchmark_backends` will most likely not load any graphs. +The browser report shows ASV plots and trends. ASV plot views do not draw a line/graph when there is only one x-axis point, therefore running `asv run 'HEAD^!' --bench benchmark_backends` will most likely not load any graphs. For the default benchmark run, the `threads` column should show `threads_1`. If you later run the [optional OpenMP scaling experiment](#optional-parallelisation-with-openmp), the same column can be used to compare several threads. - ### Profiling With cProfile `cProfile` does not update the ASV results shown in the browser report. @@ -460,6 +478,15 @@ ASV_ENV="$(ls -td .asv/env/* | head -n 1)" ASV_PYTHON="$ASV_ENV/bin/python" ``` +On Windows PowerShell, use: + +```powershell +$asvEnv = Get-ChildItem .asv\env -Directory | + Sort-Object LastWriteTime -Descending | + Select-Object -First 1 +$asvPython = Join-Path $asvEnv.FullName 'python.exe' +``` + The helper still profiles the current checkout because `profile_benchmark_workflows.py` adds the repository `src/` directory to `sys.path`. The ASV environment provides the installed dependencies, including @@ -471,6 +498,8 @@ List available cases: "$ASV_PYTHON" benchmarks/tools/profile_benchmark_workflows.py --list ``` +On Windows PowerShell, replace `"$ASV_PYTHON"` with `& $asvPython`. + Possible profile selected cases: ```bash @@ -485,7 +514,7 @@ Possible profile selected cases: This section collects optional workflows for testing Cython and Rust with several thread counts. OpenMP setup is platform-dependent, so each operating -system should have its own tested instructions. +system must be verified on the machine that produces the results. The default setup above remains the recommended baseline: one thread, normal ASV environment, and no extra OpenMP build steps. Use this section only when @@ -510,20 +539,15 @@ ASV_ENV="$(ls -td .asv-openmp/env/* | head -n 1)" If the check fails, the benchmark may still run, but the Cython backend should not be interpreted as an OpenMP-enabled Cython run. -### macOS Example - -This is the currently tested OpenMP workflow. It is separate from the -default setup above. +### OpenMP ASV Configuration -The default ASV configuration, `asv.conf.json`, stays conservative: it is the -one-thread baseline and uses the normal conda-forge `gstools-cython` package. -The default `.asv/env/` environment does not provide Cython OpenMP support. That is why this section uses a second ASV configuration: +Use one OpenMP ASV config on all supported desktop platforms: ```text -asv.macos-openmp.conf.json +asv.openmp.conf.json ``` -This OpenMP config creates separate generated directories: +This config keeps the OpenMP experiment separate from the default baseline: ```text .asv-openmp/env/ @@ -531,144 +555,157 @@ This OpenMP config creates separate generated directories: .asv-openmp/html/ ``` -That keeps the OpenMP experiment separate from the default `.asv/` baseline. +During ASV installation, it runs: + +```bash +benchmarks/tools/install_openmp_cython.py +``` + +That helper compiles `gstools-cython` from source inside ASV's own +environment, not inside your active `gstools-benchmark` driver environment. +The helper sets `GSTOOLS_BUILD_PARALLEL=1` and then uses platform-specific +compiler handling: -#### What The macOS OpenMP Config Does +- macOS: uses Apple clang through wrapper scripts and conda's `llvm-openmp`. +- Linux: uses the ASV conda compiler toolchain when available. +- Windows: uses native MSVC Build Tools. -`asv.macos-openmp.conf.json` asks conda to install the build/runtime pieces -needed for the macOS OpenMP experiment: +### Verify Cython OpenMP -```text -llvm-openmp -cython -extension-helpers -setuptools -wheel -``` +Only interpret the Cython rows as OpenMP-enabled after this check passes inside +the `.asv-openmp/env/...` environment. The active `gstools-benchmark` conda +environment is only the ASV driver environment; it is normal for +`python benchmarks/tools/check_backend_parallel_ready.py --verbose` to fail +there with `gstools: not installed`, `gstools_cython: not installed`, or +`gstools_core: not installed`. -During ASV installation, it runs: +On macOS and Linux: ```bash -benchmarks/tools/install_macos_openmp_cython.py +ASV_OPENMP_ENV="$(ls -td .asv-openmp/env/* | head -n 1)" +"$ASV_OPENMP_ENV/bin/python" benchmarks/tools/check_cython_openmp.py --verbose +"$ASV_OPENMP_ENV/bin/python" benchmarks/tools/check_cython_openmp.py --fail-if-no-openmp +"$ASV_OPENMP_ENV/bin/python" benchmarks/tools/check_backend_parallel_ready.py --verbose ``` -That helper compiles `gstools-cython` from source inside ASV's own environment, -not inside your active conda environment. This matters because ASV benchmarks -the packages installed under `.asv-openmp/env/`. +On Windows PowerShell: -Internally, the helper sets: +```powershell +$asvOpenmpEnv = Get-ChildItem .asv-openmp\env -Directory | + Sort-Object LastWriteTime -Descending | + Select-Object -First 1 +$asvOpenmpPython = Join-Path $asvOpenmpEnv.FullName 'python.exe' +& $asvOpenmpPython benchmarks\tools\check_cython_openmp.py --verbose +& $asvOpenmpPython benchmarks\tools\check_cython_openmp.py --fail-if-no-openmp +& $asvOpenmpPython benchmarks\tools\check_backend_parallel_ready.py --verbose +``` + +Expected passing output contains: ```text -GSTOOLS_BUILD_PARALLEL=1 -CC=/bin/gstools-asv-clang-openmp -CXX=/bin/gstools-asv-clang-openmp++ +OpenMP check: PASS +Cython OpenMP readiness: PASS +Rust backend readiness: PASS with NUM_THREADS=2 ``` -The wrapper translates the plain `-fopenmp` flag used by the Cython build into -Apple-clang-compatible compiler and linker arguments that use conda's -`llvm-openmp`. +### Run On macOS And Linux -#### Run On macOS +Use these commands from a POSIX shell on macOS or Linux. On macOS, install +Xcode command-line tools first. On Linux, make sure the conda compiler packages +from `asv.openmp.conf.json` solve for your platform. -In the previous section, the default config gives a quick overview for both -backends with `threads_1`. In this section, the OpenMP config runs several -thread labels: `threads_1`, `threads_2`, `threads_4`, `threads_8`, and -`threads_16`. - -Start from the GSTools repository root: +Create the ASV machine profile once: ```bash -cd /path/to/GSTools +asv --config asv.openmp.conf.json machine --yes ``` -Create a clean driver environment. This environment only runs ASV; ASV will -create the real benchmark environment under `.asv-openmp/env/`. +Run a quick current-commit OpenMP smoke run: ```bash -conda create -n gstools-benchmark -c conda-forge python=3.12 asv -conda activate gstools-benchmark +GSTOOLS_BENCHMARK_THREADS=1,2,4,8,16 \ +asv --config asv.openmp.conf.json run 'HEAD^!' --quick --bench benchmark_backends --show-stderr ``` -Create the ASV machine profile once: +Verify Cython OpenMP with the commands in +[Verify Cython OpenMP](#verify-cython-openmp). If it passes, run the last-five +commits: ```bash -asv --config asv.macos-openmp.conf.json machine --yes +GSTOOLS_BENCHMARK_THREADS=1,2,4,8,16 \ +asv --config asv.openmp.conf.json run 'HEAD~5..HEAD' --bench benchmark_backends --show-stderr ``` -Run a quick current-commit OpenMP check. This builds the OpenMP-enabled -`gstools-cython` package inside `.asv-openmp/env/` and runs the benchmark suite: +Print Rust-vs-Cython ratios from the OpenMP result folder: ```bash -GSTOOLS_BENCHMARK_THREADS=1,2,4,8,16 \ -asv --config asv.macos-openmp.conf.json run HEAD^! --quick --bench benchmark_backends --show-stderr +python benchmarks/tools/asv_speedup_summary.py --results-dir .asv-openmp/results ``` -Verify that the ASV OpenMP environment really uses Cython OpenMP: +Build and preview the OpenMP browser report: ```bash -ASV_OPENMP_ENV="$(ls -td .asv-openmp/env/* | head -n 1)" -"$ASV_OPENMP_ENV/bin/python" benchmarks/tools/check_cython_openmp.py --verbose -"$ASV_OPENMP_ENV/bin/python" benchmarks/tools/check_cython_openmp.py --fail-if-no-openmp +asv --config asv.openmp.conf.json publish +asv --config asv.openmp.conf.json preview ``` -Expected result on the tested Mac M2 setup: +### Run On Windows -```text -variogram default None -> 10 -field default None -> 10 -krige default None -> 10 -OpenMP check: PASS -``` +Use native Windows, not WSL, when you want Windows benchmark results. Install +Microsoft C++ Build Tools first, including the C++ build tools workload and a +Windows SDK. Run the commands from PowerShell or Anaconda Prompt after +activating the `gstools-benchmark` conda environment. -If that check passes, run the last-five-commits OpenMP benchmark: +Create the ASV machine profile once: -```bash -GSTOOLS_BENCHMARK_THREADS=1,2,4,8,16 \ -asv --config asv.macos-openmp.conf.json run HEAD~5..HEAD --bench benchmark_backends --show-stderr +```powershell +asv --config asv.openmp.conf.json machine --yes ``` -Print Rust-vs-Cython ratios from the OpenMP result folder: +Run a quick current-commit OpenMP smoke run: -```bash -python benchmarks/tools/asv_speedup_summary.py --results-dir .asv-openmp/results +```powershell +$env:GSTOOLS_BENCHMARK_THREADS = '1,2,4,8,16' +asv --config asv.openmp.conf.json run 'HEAD^!' --quick --bench benchmark_backends --show-stderr ``` -Build and preview the OpenMP browser report: +Verify Cython OpenMP with the PowerShell commands in +[Verify Cython OpenMP](#verify-cython-openmp). If it passes, run the last-five +commits: -```bash -asv --config asv.macos-openmp.conf.json publish -asv --config asv.macos-openmp.conf.json preview +```powershell +$env:GSTOOLS_BENCHMARK_THREADS = '1,2,4,8,16' +asv --config asv.openmp.conf.json run 'HEAD~5..HEAD' --bench benchmark_backends --show-stderr ``` -#### Interpreting The macOS OpenMP Run - -- Use default `asv.conf.json` for the reproducible one-thread baseline. -- Use `asv.macos-openmp.conf.json` for the macOS OpenMP experiment. -- Only claim Cython OpenMP scaling if `check_cython_openmp.py` passes inside - `.asv-openmp/env/...`. -- The active `gstools-benchmark` conda environment does not need `gstools` - installed. It only needs ASV. The benchmarked GSTools packages live inside - `.asv-openmp/env/...`. +Print Rust-vs-Cython ratios from the OpenMP result folder: -This workflow is intended for macOS systems that use Apple clang with conda's -`llvm-openmp`. It should be portable across many macOS machines, including -Apple Silicon and Intel Macs, but it is not guaranteed for every macOS setup. +```powershell +python benchmarks\tools\asv_speedup_summary.py --results-dir .asv-openmp\results +``` -It is not guaranteed to run without local changes on: +Build and preview the OpenMP browser report: -- older macOS versions -- systems missing Xcode command-line tools -- systems with a nonstandard compiler setup -- HPC or managed macOS environments -- unusual conda installations +```powershell +asv --config asv.openmp.conf.json publish +asv --config asv.openmp.conf.json preview +``` -Do not assume this exact OpenMP setup applies to Linux, Windows, or HPC systems. +When finished, clear the thread-count override if the PowerShell session will +be reused: -### Windows Example +```powershell +Remove-Item Env:\GSTOOLS_BENCHMARK_THREADS +``` -### Linux Example +### HPC Notes -### HPC Example +The `asv.openmp.conf.json` workflow is intended for local macOS, Linux, and +native Windows machines. Managed HPC systems often use custom compiler modules, +MPI/OpenMP runtimes, scheduler pinning, and CPU affinity rules. Use the same +validation rule there: only interpret Cython as OpenMP-enabled if +`check_cython_openmp.py --fail-if-no-openmp` passes inside the exact ASV +environment used for the benchmark run. ### Profiling With cProfile for Multiple Threads @@ -684,6 +721,19 @@ for threads in threads_1 threads_2 threads_4 threads_8 threads_16; do done ``` +On Windows PowerShell: + +```powershell +$asvOpenmpEnv = Get-ChildItem .asv-openmp\env -Directory | + Sort-Object LastWriteTime -Descending | + Select-Object -First 1 +$asvOpenmpPython = Join-Path $asvOpenmpEnv.FullName 'python.exe' + +foreach ($threads in 'threads_1', 'threads_2', 'threads_4', 'threads_8', 'threads_16') { + & $asvOpenmpPython benchmarks\tools\profile_benchmark_workflows.py --case krige-extra-large --backend rust_core --threads $threads --limit 10 +} +``` + Useful options: - `--case`: choose one workflow, or use `all` @@ -697,27 +747,78 @@ Useful options: For example, `--limit 10` means "print the top 10 function rows after sorting". +## GitHub Action Benchmark Availability Checks + +The repository includes `.github/workflows/asv-benchmarks.yml` to check that +the benchmark tooling can be installed and started on GitHub-hosted Linux, +Windows, macOS Apple Silicon, and macOS Intel runners. This workflow is an +availability check, not a performance benchmark run. + +The workflow has two dependent stages: + +- `benchmark_1_thread_availability`: runs `asv.conf.json` with + `GSTOOLS_BENCHMARK_THREADS=1` on a representative Python/NumPy/SciPy matrix. +- `benchmark_parallel_backend_availability`: waits for the 1-thread stage, then + verifies Cython OpenMP and Rust backend readiness on the latest dependency + combo for each OS runner. + +The 1-thread stage runs a quick ASV command: + +```bash +asv run "HEAD^!" --quick --bench benchmark_backends --show-stderr +``` + +The parallel-readiness stage intentionally does not run a full ASV OpenMP +benchmark. Instead, it builds `gstools-cython` with OpenMP, installs +`gstools_core`, installs the local GSTools checkout, and runs: + +```bash +conda run -n gstools-benchmark-openmp python \ + benchmarks/tools/check_backend_parallel_ready.py --verbose +``` + +Locally, run the same helper with the Python executable from the ASV OpenMP +environment: + +```bash +ASV_OPENMP_ENV="$(ls -td .asv-openmp/env/* | head -n 1)" +"$ASV_OPENMP_ENV/bin/python" benchmarks/tools/check_backend_parallel_ready.py --verbose +``` + +That fast check proves that Cython reports OpenMP support and that the Rust +backend can run a small workflow with `gstools.config.NUM_THREADS=2`. For real +OpenMP scaling measurements, use `asv.openmp.conf.json` locally with +`GSTOOLS_BENCHMARK_THREADS=1,2,4,8,16`. + +The workflow runs automatically on pull requests and pushes that change ASV +configs, benchmark files, package metadata, or the workflow itself. It can also +be started from the GitHub Actions tab with `workflow_dispatch`. + +For a short explanation of GitHub Actions contexts, matrix jobs, `needs`, +runner labels, secrets, variables, and why this workflow is path-filtered, see +[`benchmarks/github_actions_guide.md`](github_actions_guide.md). + ## More ASV Commands Save results for only the current commit: ```bash -asv run HEAD^! --bench benchmark_backends +asv run 'HEAD^!' --bench benchmark_backends ``` Compare current commit with previous commit: ```bash -asv run HEAD~1^! --bench benchmark_backends -asv run HEAD^! --bench benchmark_backends +asv run 'HEAD~1^!' --bench benchmark_backends +asv run 'HEAD^!' --bench benchmark_backends asv compare HEAD~1 HEAD ``` Compare local `main` with the current branch tip: ```bash -asv run main^! --bench benchmark_backends -asv run HEAD^! --bench benchmark_backends +asv run 'main^!' --bench benchmark_backends +asv run 'HEAD^!' --bench benchmark_backends asv compare main HEAD ``` @@ -725,8 +826,8 @@ Compare remote `main` with the current branch tip: ```bash git fetch origin main -asv run origin/main^! --bench benchmark_backends -asv run HEAD^! --bench benchmark_backends +asv run 'origin/main^!' --bench benchmark_backends +asv run 'HEAD^!' --bench benchmark_backends asv compare origin/main HEAD ``` diff --git a/benchmarks/benchmark_backends.py b/benchmarks/benchmark_backends.py index c9526eb79..cc588c1e7 100644 --- a/benchmarks/benchmark_backends.py +++ b/benchmarks/benchmark_backends.py @@ -19,7 +19,8 @@ By default the suite uses one GSTools thread. For local OpenMP scaling experiments, set GSTOOLS_BENCHMARK_THREADS, for example: - GSTOOLS_BENCHMARK_THREADS=1,2,4,8,16 asv run HEAD^! + GSTOOLS_BENCHMARK_THREADS=1,2,4,8,16 \ + asv --config asv.openmp.conf.json run 'HEAD^!' """ from __future__ import annotations @@ -27,10 +28,8 @@ import contextlib import os -import numpy as np - import gstools as gs - +import numpy as np BACKENDS = ("cython_fallback", "rust_core") diff --git a/benchmarks/tools/check_backend_parallel_ready.py b/benchmarks/tools/check_backend_parallel_ready.py new file mode 100644 index 000000000..8a6c7aa38 --- /dev/null +++ b/benchmarks/tools/check_backend_parallel_ready.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python +"""Check that GSTools benchmark backends are ready for parallel runs. + +This is a fast CI probe. It verifies that GSTools-Cython reports OpenMP support +and that the Rust backend can run a small workflow while GSTools is configured +with more than one thread. +""" + +from __future__ import annotations + +import argparse +import importlib +import sys + +import numpy as np +from check_cython_openmp import MODULES, check_module, package_version + + +def parse_args(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--threads", + default=2, + type=int, + help="Thread count to request for the Rust backend smoke workflow.", + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Print per-module Cython OpenMP thread details.", + ) + return parser.parse_args() + + +def check_cython_openmp(verbose=False): + default_values = [] + for label, module_name in MODULES.items(): + try: + label, default_threads, explicit = check_module(label, module_name) + except ModuleNotFoundError as err: + print(f"Cython OpenMP readiness: FAIL. Missing module: {err.name}") + return False + default_values.append(default_threads) + if verbose: + explicit_text = ", ".join( + f"{request}->{actual}" for request, actual in explicit.items() + ) + print(f"{label} default None -> {default_threads}") + print(f"{label} explicit -> {explicit_text}") + if explicit.get(2) != 2: + print( + "Cython OpenMP readiness: FAIL. " + f"{label} did not accept an explicit 2-thread request." + ) + return False + + if min(default_values) <= 1: + print( + "Cython OpenMP readiness: FAIL. GSTools-Cython reports one " + "default thread." + ) + return False + + print("Cython OpenMP readiness: PASS") + return True + + +def check_rust_backend(threads): + try: + import gstools as gs + except ModuleNotFoundError as err: + print(f"Rust backend readiness: FAIL. Missing module: {err.name}") + return False + + try: + importlib.import_module("gstools_core") + except ModuleNotFoundError as err: + print(f"Rust backend readiness: FAIL. Missing module: {err.name}") + return False + + if not gs.config._GSTOOLS_CORE_AVAIL: + print("Rust backend readiness: FAIL. GSTools did not detect gstools_core.") + return False + + previous = ( + gs.config._GSTOOLS_CORE_AVAIL, + gs.config.USE_GSTOOLS_CORE, + gs.config.NUM_THREADS, + ) + try: + gs.config._GSTOOLS_CORE_AVAIL = True + gs.config.USE_GSTOOLS_CORE = True + gs.config.NUM_THREADS = threads + + x = np.linspace(0.0, 10.0, 12) + y = np.linspace(0.0, 5.0, 12) + field = np.sin(x) + np.cos(y) + bins = np.linspace(0.0, 8.0, 6) + gs.vario_estimate( + (x, y), + field, + bins, + mesh_type="unstructured", + return_counts=True, + ) + finally: + ( + gs.config._GSTOOLS_CORE_AVAIL, + gs.config.USE_GSTOOLS_CORE, + gs.config.NUM_THREADS, + ) = previous + + print(f"Rust backend readiness: PASS with NUM_THREADS={threads}") + return True + + +def main(): + args = parse_args() + print(f"python: {sys.executable}") + print(f"gstools: {package_version('gstools')}") + print(f"gstools_cython: {package_version('gstools_cython')}") + print(f"gstools_core: {package_version('gstools_core')}") + + cython_ready = check_cython_openmp(verbose=args.verbose) + rust_ready = check_rust_backend(args.threads) + return 0 if cython_ready and rust_ready else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/tools/check_cython_openmp.py b/benchmarks/tools/check_cython_openmp.py index 02fe73db3..b3a410cc9 100644 --- a/benchmarks/tools/check_cython_openmp.py +++ b/benchmarks/tools/check_cython_openmp.py @@ -17,7 +17,6 @@ import importlib import sys - MODULES = { "variogram": "gstools_cython.variogram", "field": "gstools_cython.field", diff --git a/benchmarks/tools/install_macos_openmp_cython.py b/benchmarks/tools/install_macos_openmp_cython.py deleted file mode 100644 index 09f0840cc..000000000 --- a/benchmarks/tools/install_macos_openmp_cython.py +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env python -"""Install GSTools-Cython with OpenMP inside a macOS ASV environment. - -This helper is intentionally macOS-specific. It is called from -``asv.macos-openmp.conf.json`` after ASV has created a conda environment that -contains ``llvm-openmp``. -""" - -from __future__ import annotations - -import os -import platform -import stat -import subprocess -import sys -from pathlib import Path - - -def run(command, env=None, check=True): - print("+ " + " ".join(str(part) for part in command), flush=True) - return subprocess.run(command, check=check, env=env) - - -def write_wrapper(path, force_cxx=False): - text = """#!/bin/bash -set -e -prefix="${GSTOOLS_OPENMP_PREFIX:-${CONDA_PREFIX:-}}" -name="$(basename "$0")" -if [[ "${GSTOOLS_FORCE_CXX:-0}" == "1" || "$name" == *++* ]]; then - real="${GSTOOLS_REAL_CXX:-/usr/bin/clang++}" -else - real="${GSTOOLS_REAL_CC:-/usr/bin/clang}" -fi -is_compile=0 -for arg in "$@"; do - [[ "$arg" == "-c" ]] && is_compile=1 -done -args=() -for arg in "$@"; do - if [[ "$arg" == "-fopenmp" ]]; then - if [[ "$is_compile" == "1" ]]; then - args+=("-Xpreprocessor" "-fopenmp" "-I${prefix}/include") - else - args+=("-L${prefix}/lib" "-lomp" "-Wl,-rpath,${prefix}/lib") - fi - else - args+=("$arg") - fi -done -exec "$real" "${args[@]}" -""" - if force_cxx: - text = """#!/bin/bash -GSTOOLS_FORCE_CXX=1 exec "$(dirname "$0")/gstools-asv-clang-openmp" "$@" -""" - path.write_text(text, encoding="utf8") - path.chmod(path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) - - -def main(): - if len(sys.argv) != 2: - print( - "Usage: install_macos_openmp_cython.py ", - file=sys.stderr, - ) - return 2 - - if platform.system() != "Darwin": - print( - "This helper is macOS-specific. Use the default ASV config or " - "write an OpenMP setup for this platform.", - file=sys.stderr, - ) - return 2 - - env_dir = Path(sys.argv[1]).resolve() - include_dir = env_dir / "include" - lib_dir = env_dir / "lib" - omp_header = include_dir / "omp.h" - omp_lib = lib_dir / "libomp.dylib" - - if not omp_header.exists() or not omp_lib.exists(): - print( - "llvm-openmp was not found in the ASV environment. Expected " - f"{omp_header} and {omp_lib}.", - file=sys.stderr, - ) - return 2 - - cc_wrapper = env_dir / "bin" / "gstools-asv-clang-openmp" - cxx_wrapper = env_dir / "bin" / "gstools-asv-clang-openmp++" - write_wrapper(cc_wrapper) - write_wrapper(cxx_wrapper, force_cxx=True) - - build_env = os.environ.copy() - build_env.update( - { - "GSTOOLS_BUILD_PARALLEL": "1", - "GSTOOLS_OPENMP_PREFIX": str(env_dir), - "CC": str(cc_wrapper), - "CXX": str(cxx_wrapper), - "CFLAGS": f"-I{include_dir}", - "LDFLAGS": f"-L{lib_dir}", - } - ) - - run( - [ - sys.executable, - "-m", - "pip", - "uninstall", - "-y", - "gstools-cython", - "gstools_cython", - ], - env=build_env, - check=False, - ) - run( - [ - sys.executable, - "-m", - "pip", - "install", - "--no-build-isolation", - "--no-cache-dir", - "--force-reinstall", - "--no-binary=gstools-cython", - "--no-deps", - "gstools-cython", - ], - env=build_env, - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmarks/tools/install_openmp_cython.py b/benchmarks/tools/install_openmp_cython.py new file mode 100644 index 000000000..2883fe0ca --- /dev/null +++ b/benchmarks/tools/install_openmp_cython.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python +"""Install GSTools-Cython with OpenMP inside an ASV environment. + +This helper is called by ``asv.openmp.conf.json`` after ASV has created a +conda environment with the common build requirements. GitHub Actions can also +call it inside an already-active conda environment. It keeps the ASV config +portable and handles platform-specific compiler/OpenMP details here. +""" + +from __future__ import annotations + +import argparse +import os +import platform +import shutil +import stat +import subprocess +import sys +from pathlib import Path + + +def run(command, env=None, check=True): + print("+ " + " ".join(str(part) for part in command), flush=True) + return subprocess.run(command, check=check, env=env) + + +def prepend_path(build_env, *paths): + current = build_env.get("PATH", "") + existing = [str(path) for path in paths if path.exists()] + if existing: + build_env["PATH"] = os.pathsep.join(existing + [current]) + + +def first_match(directory, patterns): + for pattern in patterns: + matches = sorted(path for path in directory.glob(pattern) if path.is_file()) + if matches: + return matches[0] + return None + + +def require_file(path, message): + if not path.exists(): + print(message, file=sys.stderr) + return False + return True + + +def conda_executable(): + conda = os.environ.get("CONDA_EXE") or shutil.which("conda") + if conda is None: + print("No conda executable was found.", file=sys.stderr) + return conda + + +def conda_install(env_dir, *packages): + conda = conda_executable() + if conda is None: + return False + run( + [ + conda, + "install", + "-y", + "-p", + str(env_dir), + "-c", + "conda-forge", + *packages, + ] + ) + return True + + +def ensure_macos_llvm_openmp(env_dir): + include_dir = env_dir / "include" + lib_dir = env_dir / "lib" + omp_header = include_dir / "omp.h" + omp_lib = lib_dir / "libomp.dylib" + + if omp_header.exists() and omp_lib.exists(): + return True + + if not conda_install(env_dir, "llvm-openmp"): + print( + "llvm-openmp is required on macOS but was not found in the ASV " + "environment, and it could not be installed with conda.", + file=sys.stderr, + ) + return False + + return require_file( + omp_header, + f"llvm-openmp install did not create expected header: {omp_header}", + ) and require_file( + omp_lib, + f"llvm-openmp install did not create expected library: {omp_lib}", + ) + + +def ensure_linux_libgomp(env_dir): + lib_dir = env_dir / "lib" + if first_match(lib_dir, ["libgomp.so*", "libomp.so*"]) is not None: + return True + + if not conda_install(env_dir, "libgomp"): + print( + "libgomp is required on Linux but could not be installed with conda.", + file=sys.stderr, + ) + return False + + if first_match(lib_dir, ["libgomp.so*", "libomp.so*"]) is not None: + return True + + print( + f"libgomp install did not create an OpenMP runtime under {lib_dir}.", + file=sys.stderr, + ) + return False + + +def write_macos_wrapper(path, force_cxx=False): + text = """#!/bin/bash +set -e +prefix="${GSTOOLS_OPENMP_PREFIX:-${CONDA_PREFIX:-}}" +name="$(basename "$0")" +if [[ "${GSTOOLS_FORCE_CXX:-0}" == "1" || "$name" == *++* ]]; then + real="${GSTOOLS_REAL_CXX:-/usr/bin/clang++}" +else + real="${GSTOOLS_REAL_CC:-/usr/bin/clang}" +fi +is_compile=0 +for arg in "$@"; do + [[ "$arg" == "-c" ]] && is_compile=1 +done +args=() +for arg in "$@"; do + if [[ "$arg" == "-fopenmp" ]]; then + if [[ "$is_compile" == "1" ]]; then + args+=("-Xpreprocessor" "-fopenmp" "-I${prefix}/include") + else + args+=("-L${prefix}/lib" "-lomp" "-Wl,-rpath,${prefix}/lib") + fi + else + args+=("$arg") + fi +done +exec "$real" "${args[@]}" +""" + if force_cxx: + text = """#!/bin/bash +GSTOOLS_FORCE_CXX=1 exec "$(dirname "$0")/gstools-asv-clang-openmp" "$@" +""" + path.write_text(text, encoding="utf8") + path.chmod(path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + +def macos_build_env(env_dir): + if not ensure_macos_llvm_openmp(env_dir): + return None + + bin_dir = env_dir / "bin" + include_dir = env_dir / "include" + lib_dir = env_dir / "lib" + cc_wrapper = bin_dir / "gstools-asv-clang-openmp" + cxx_wrapper = bin_dir / "gstools-asv-clang-openmp++" + write_macos_wrapper(cc_wrapper) + write_macos_wrapper(cxx_wrapper, force_cxx=True) + + build_env = os.environ.copy() + prepend_path(build_env, bin_dir) + build_env.update( + { + "GSTOOLS_BUILD_PARALLEL": "1", + "GSTOOLS_OPENMP_PREFIX": str(env_dir), + "CC": str(cc_wrapper), + "CXX": str(cxx_wrapper), + "CFLAGS": f"-I{include_dir}", + "LDFLAGS": f"-L{lib_dir}", + } + ) + return build_env + + +def linux_build_env(env_dir): + if not ensure_linux_libgomp(env_dir): + return None + + bin_dir = env_dir / "bin" + cc = first_match( + bin_dir, + [ + "*-conda-linux-gnu-cc", + "*-conda-linux-gnu-gcc", + "gcc", + "cc", + ], + ) + cxx = first_match( + bin_dir, + [ + "*-conda-linux-gnu-c++", + "*-conda-linux-gnu-g++", + "g++", + "c++", + ], + ) + + build_env = os.environ.copy() + prepend_path(build_env, bin_dir) + build_env["GSTOOLS_BUILD_PARALLEL"] = "1" + if cc is not None: + build_env["CC"] = str(cc) + if cxx is not None: + build_env["CXX"] = str(cxx) + return build_env + + +def windows_build_env(env_dir): + build_env = os.environ.copy() + prepend_path( + build_env, + env_dir, + env_dir / "Scripts", + env_dir / "Library" / "bin", + ) + build_env["GSTOOLS_BUILD_PARALLEL"] = "1" + return build_env + + +def install_gstools_cython(build_env): + run( + [ + sys.executable, + "-m", + "pip", + "uninstall", + "-y", + "gstools-cython", + "gstools_cython", + ], + env=build_env, + check=False, + ) + run( + [ + sys.executable, + "-m", + "pip", + "install", + "--no-build-isolation", + "--no-cache-dir", + "--force-reinstall", + "--no-binary=gstools-cython", + "--no-deps", + "gstools-cython", + ], + env=build_env, + ) + + +def parse_args(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "env_dir", + nargs="?", + default=sys.prefix, + help="ASV or active conda environment directory. Defaults to sys.prefix.", + ) + return parser.parse_args() + + +def main(): + args = parse_args() + env_dir = Path(args.env_dir).resolve() + system = platform.system() + print(f"Preparing GSTools-Cython OpenMP build for {system}") + + if system == "Darwin": + build_env = macos_build_env(env_dir) + elif system == "Linux": + build_env = linux_build_env(env_dir) + elif system == "Windows": + build_env = windows_build_env(env_dir) + else: + print(f"Unsupported platform for OpenMP ASV build: {system}", file=sys.stderr) + return 2 + + if build_env is None: + return 2 + + install_gstools_cython(build_env) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/tools/write_asv_ci_config.py b/benchmarks/tools/write_asv_ci_config.py new file mode 100644 index 000000000..9ec5b2b8a --- /dev/null +++ b/benchmarks/tools/write_asv_ci_config.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +"""Write a per-job ASV configuration for GitHub Actions. + +The committed ASV configs stay convenient for local benchmarking. CI uses this +helper to pin one Python/NumPy/SciPy combination per matrix job without turning +normal local ``asv run`` commands into a large dependency matrix. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + + +def parse_args(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-config", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--python-version", required=True) + parser.add_argument("--numpy", required=True) + parser.add_argument("--scipy", required=True) + parser.add_argument("--env-dir", required=True) + parser.add_argument("--results-dir", required=True) + parser.add_argument("--html-dir", required=True) + return parser.parse_args() + + +def main(): + args = parse_args() + with args.base_config.open(encoding="utf8") as config_file: + config = json.load(config_file) + + config["pythons"] = [args.python_version] + config["env_dir"] = args.env_dir + config["results_dir"] = args.results_dir + config["html_dir"] = args.html_dir + + req = config.setdefault("matrix", {}).setdefault("req", {}) + req["numpy"] = [args.numpy] + req["scipy"] = [args.scipy] + + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open("w", encoding="utf8") as config_file: + json.dump(config, config_file, indent=2) + config_file.write("\n") + + print(f"Wrote {args.output}") + + +if __name__ == "__main__": + main() From 253114655aea41d57aa6add2b57b39fe638e7771 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Wed, 27 May 2026 22:55:05 +0200 Subject: [PATCH 05/14] Fix benchmark CI config resolution and linter scope --- .github/linters/.markdown-lint.yml | 2 -- .github/workflows/asv-benchmarks.yml | 12 ++++++------ .github/workflows/super-linter.yml | 10 +++++----- .gitignore | 1 + 4 files changed, 12 insertions(+), 13 deletions(-) delete mode 100644 .github/linters/.markdown-lint.yml diff --git a/.github/linters/.markdown-lint.yml b/.github/linters/.markdown-lint.yml deleted file mode 100644 index ee0ceb79a..000000000 --- a/.github/linters/.markdown-lint.yml +++ /dev/null @@ -1,2 +0,0 @@ -MD013: false -MD033: false diff --git a/.github/workflows/asv-benchmarks.yml b/.github/workflows/asv-benchmarks.yml index 63c495504..e03f1bde3 100644 --- a/.github/workflows/asv-benchmarks.yml +++ b/.github/workflows/asv-benchmarks.yml @@ -124,7 +124,7 @@ jobs: activate-environment: gstools-benchmark-driver python-version: "3.12" channels: conda-forge - auto-activate-base: false + conda-remove-defaults: true - name: Install ASV driver run: | @@ -136,7 +136,7 @@ jobs: # a temporary config so each matrix child can pin one dependency set. conda run -n gstools-benchmark-driver python benchmarks/tools/write_asv_ci_config.py \ --base-config asv.conf.json \ - --output ".asv-ci/config/${{ matrix.case-id }}.json" \ + --output "${GITHUB_WORKSPACE}/asv.ci.${{ matrix.case-id }}.json" \ --python-version "${{ matrix.python-version }}" \ --numpy "${{ matrix.numpy }}" \ --scipy "${{ matrix.scipy }}" \ @@ -147,7 +147,7 @@ jobs: - name: Configure ASV machine run: | conda run -n gstools-benchmark-driver asv \ - --config ".asv-ci/config/${{ matrix.case-id }}.json" \ + --config "${GITHUB_WORKSPACE}/asv.ci.${{ matrix.case-id }}.json" \ machine --yes - name: Run 1-thread ASV availability check @@ -157,7 +157,7 @@ jobs: GSTOOLS_BENCHMARK_THREADS: "1" run: | conda run -n gstools-benchmark-driver asv \ - --config ".asv-ci/config/${{ matrix.case-id }}.json" \ + --config "${GITHUB_WORKSPACE}/asv.ci.${{ matrix.case-id }}.json" \ run 'HEAD^!' --quick --bench benchmark_backends --show-stderr - name: Upload ASV availability results @@ -168,7 +168,7 @@ jobs: with: name: asv-1-thread-${{ matrix.case-id }} path: | - .asv-ci/config/${{ matrix.case-id }}.json + asv.ci.${{ matrix.case-id }}.json .asv-ci/results/${{ matrix.case-id }} .asv-ci/html/${{ matrix.case-id }} if-no-files-found: ignore @@ -220,7 +220,7 @@ jobs: activate-environment: gstools-benchmark-openmp python-version: ${{ matrix.python-version }} channels: conda-forge - auto-activate-base: false + conda-remove-defaults: true - name: Install benchmark and build requirements run: | diff --git a/.github/workflows/super-linter.yml b/.github/workflows/super-linter.yml index 17ab77687..9bac92ae9 100644 --- a/.github/workflows/super-linter.yml +++ b/.github/workflows/super-linter.yml @@ -2,7 +2,7 @@ name: Super-Linter on: # This workflow is cheap compared with ASV/compiler checks, so run it on - # every PR and push to main to catch metadata/documentation issues early. + # every PR and push to main to catch workflow/configuration issues early. pull_request: push: branches: @@ -19,7 +19,7 @@ permissions: jobs: lint_repository_metadata: - name: Lint GitHub Actions, YAML, JSON, and Markdown + name: Lint GitHub Actions, YAML, and JSON runs-on: ubuntu-latest timeout-minutes: 15 @@ -36,9 +36,9 @@ jobs: # Lint changed files only. This avoids introducing unrelated failures # from old Markdown/YAML while still checking new PR changes. VALIDATE_ALL_CODEBASE: false - # Opt-in mode: these true values make Super-Linter skip Python. Ruff - # already owns Python linting in main.yml. + # Opt-in mode: these true values make Super-Linter skip Python and + # Markdown. Ruff owns Python linting in main.yml. Prose/style checks + # should use a dedicated tool such as Vale or textlint if added later. VALIDATE_GITHUB_ACTIONS: true VALIDATE_JSON: true - VALIDATE_MARKDOWN: true VALIDATE_YAML: true diff --git a/.gitignore b/.gitignore index b154c9fac..460490cd6 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,7 @@ htmlcov/ .asv/* .asv-openmp/* .asv-ci/* +asv.ci.*.json nosetests.xml coverage.xml *.cover From 99e53286bc17da5a1ff807d93c1c1154595a53bc Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Thu, 28 May 2026 08:43:33 +0200 Subject: [PATCH 06/14] Add Vale Markdown terminology checks --- .github/styles/GSTools/Terms.yml | 14 ++++++ .../config/vocabularies/GSTools/accept.txt | 11 ++++ .../config/vocabularies/GSTools/reject.txt | 1 + .github/workflows/super-linter.yml | 4 +- .github/workflows/vale.yml | 50 +++++++++++++++++++ .vale.ini | 6 +++ README.md | 4 +- 7 files changed, 86 insertions(+), 4 deletions(-) create mode 100644 .github/styles/GSTools/Terms.yml create mode 100644 .github/styles/config/vocabularies/GSTools/accept.txt create mode 100644 .github/styles/config/vocabularies/GSTools/reject.txt create mode 100644 .github/workflows/vale.yml create mode 100644 .vale.ini diff --git a/.github/styles/GSTools/Terms.yml b/.github/styles/GSTools/Terms.yml new file mode 100644 index 000000000..9d528e238 --- /dev/null +++ b/.github/styles/GSTools/Terms.yml @@ -0,0 +1,14 @@ +extends: substitution +message: "Use '%s' instead of '%s'." +level: error +ignorecase: false +swap: + "Github": "GitHub" + "MacOS": "macOS" + "MacOs": "macOS" + "Macos": "macOS" + "OpenMp": "OpenMP" + "openMP": "OpenMP" + "Powershell": "PowerShell" + "Power Shell": "PowerShell" + "conda forge": "conda-forge" diff --git a/.github/styles/config/vocabularies/GSTools/accept.txt b/.github/styles/config/vocabularies/GSTools/accept.txt new file mode 100644 index 000000000..581358563 --- /dev/null +++ b/.github/styles/config/vocabularies/GSTools/accept.txt @@ -0,0 +1,11 @@ +ASV +Cython +GeoStat +GSTools +OpenMP +PowerShell +NumPy +SciPy +conda-forge +gstools-cython +gstools_core diff --git a/.github/styles/config/vocabularies/GSTools/reject.txt b/.github/styles/config/vocabularies/GSTools/reject.txt new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/.github/styles/config/vocabularies/GSTools/reject.txt @@ -0,0 +1 @@ + diff --git a/.github/workflows/super-linter.yml b/.github/workflows/super-linter.yml index 9bac92ae9..83eab46eb 100644 --- a/.github/workflows/super-linter.yml +++ b/.github/workflows/super-linter.yml @@ -37,8 +37,8 @@ jobs: # from old Markdown/YAML while still checking new PR changes. VALIDATE_ALL_CODEBASE: false # Opt-in mode: these true values make Super-Linter skip Python and - # Markdown. Ruff owns Python linting in main.yml. Prose/style checks - # should use a dedicated tool such as Vale or textlint if added later. + # Markdown. Ruff owns Python linting in main.yml, and vale.yml owns + # Markdown prose/terminology checks. VALIDATE_GITHUB_ACTIONS: true VALIDATE_JSON: true VALIDATE_YAML: true diff --git a/.github/workflows/vale.yml b/.github/workflows/vale.yml new file mode 100644 index 000000000..b0da9d3e5 --- /dev/null +++ b/.github/workflows/vale.yml @@ -0,0 +1,50 @@ +name: Vale + +on: + # Vale is for documentation/prose quality. It runs only when Markdown or Vale + # configuration changes, so it complements Super-Linter without adding noise + # to source-only changes. + pull_request: + paths: + - "**/*.md" + - ".vale.ini" + - ".github/styles/**" + - ".github/workflows/vale.yml" + push: + branches: + - "main" + paths: + - "**/*.md" + - ".vale.ini" + - ".github/styles/**" + - ".github/workflows/vale.yml" + workflow_dispatch: + +# The Vale action reads repository content and reports findings back to the PR +# check. It does not need permission to modify files. +permissions: + contents: read + pull-requests: read + checks: write + +jobs: + lint_markdown_prose: + name: Lint Markdown prose and terminology + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - uses: actions/checkout@v4 + + - name: Run Vale + uses: vale-cli/vale-action@v2.1.2 + with: + # Pin the Vale CLI separately from the GitHub Action wrapper so prose + # checks do not change behavior unexpectedly between runs. + version: "3.14.1" + # Check tracked documentation paths, not generated local ASV + # environments such as .asv/ or .asv-openmp/. + files: '["README.md", "CONTRIBUTING.md", "AUTHORS.md", "CHANGELOG.md", "benchmarks"]' + filter_mode: nofilter + reporter: github-check + fail_on_error: true diff --git a/.vale.ini b/.vale.ini new file mode 100644 index 000000000..045ef69f8 --- /dev/null +++ b/.vale.ini @@ -0,0 +1,6 @@ +StylesPath = .github/styles +MinAlertLevel = warning +Vocab = GSTools + +[*.md] +BasedOnStyles = GSTools diff --git a/README.md b/README.md index 1947df0bf..237f7587a 100644 --- a/README.md +++ b/README.md @@ -54,8 +54,8 @@ Install the package by typing the following command in a command terminal: conda install gstools -In case conda forge is not set up for your system yet, see the easy to follow -instructions on [conda forge][conda_forge_link]. Using conda, the parallelized +In case conda-forge is not set up for your system yet, see the easy to follow +instructions on [conda-forge][conda_forge_link]. Using conda, the parallelized version of GSTools should be installed. From bddc2dffeba6c1f4efd044c76dd8e92cd8381285 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Thu, 28 May 2026 09:24:31 +0200 Subject: [PATCH 07/14] Second iteration to fix GitHub actions + pin GitHub actions to commit SHAs --- .github/workflows/asv-benchmarks.yml | 119 ++++++++---------- .github/workflows/main.yml | 34 +++-- .github/workflows/super-linter.yml | 4 +- .github/workflows/vale.yml | 4 +- .../tools/check_backend_parallel_ready.py | 4 +- benchmarks/tools/install_openmp_cython.py | 13 +- pyproject.toml | 1 + 7 files changed, 93 insertions(+), 86 deletions(-) diff --git a/.github/workflows/asv-benchmarks.yml b/.github/workflows/asv-benchmarks.yml index e03f1bde3..97ae3e9be 100644 --- a/.github/workflows/asv-benchmarks.yml +++ b/.github/workflows/asv-benchmarks.yml @@ -37,7 +37,7 @@ jobs: benchmark_1_thread_availability: # First stage: prove the normal one-thread ASV benchmark can be installed # and started across the supported runner/dependency combinations. - name: 1-thread ASV on ${{ matrix.os }} py${{ matrix.python-version }} + name: 1-thread ASV on ${{ matrix.os }} py${{ matrix.ver.py }} runs-on: ${{ matrix.os }} timeout-minutes: 75 strategy: @@ -52,72 +52,60 @@ jobs: include: - case-id: ubuntu-py38-oldest os: ubuntu-latest - python-version: "3.8" - numpy: "==1.20.0" - scipy: "==1.5.4" + ver: { py: "3.8", np: "==1.20.0", sp: "==1.5.4" } - case-id: ubuntu-py312-current os: ubuntu-latest - python-version: "3.12" - numpy: "==1.26.2" - scipy: "==1.11.2" + ver: { py: "3.12", np: "==1.26.2", sp: "==1.11.2" } - case-id: ubuntu-py313-latest os: ubuntu-latest - python-version: "3.13" - numpy: ">=2.1.0" - scipy: ">=1.14.1" + ver: { py: "3.13", np: ">=2.1.0", sp: ">=1.14.1" } + - case-id: ubuntu-py314-latest + os: ubuntu-latest + ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } - case-id: windows-py38-oldest os: windows-latest - python-version: "3.8" - numpy: "==1.20.0" - scipy: "==1.5.4" + ver: { py: "3.8", np: "==1.20.0", sp: "==1.5.4" } - case-id: windows-py312-current os: windows-latest - python-version: "3.12" - numpy: "==1.26.2" - scipy: "==1.11.2" + ver: { py: "3.12", np: "==1.26.2", sp: "==1.11.2" } - case-id: windows-py313-latest os: windows-latest - python-version: "3.13" - numpy: ">=2.1.0" - scipy: ">=1.14.1" + ver: { py: "3.13", np: ">=2.1.0", sp: ">=1.14.1" } + - case-id: windows-py314-latest + os: windows-latest + ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } - case-id: macos15-py311-older os: macos-15 - python-version: "3.11" - numpy: "==1.23.2" - scipy: "==1.9.2" + ver: { py: "3.11", np: "==1.23.2", sp: "==1.9.2" } - case-id: macos15-py312-current os: macos-15 - python-version: "3.12" - numpy: "==1.26.2" - scipy: "==1.11.2" + ver: { py: "3.12", np: "==1.26.2", sp: "==1.11.2" } - case-id: macos15-py313-latest os: macos-15 - python-version: "3.13" - numpy: ">=2.1.0" - scipy: ">=1.14.1" + ver: { py: "3.13", np: ">=2.1.0", sp: ">=1.14.1" } + - case-id: macos15-py314-latest + os: macos-15 + ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } - case-id: macos15-intel-py38-oldest os: macos-15-intel - python-version: "3.8" - numpy: "==1.20.0" - scipy: "==1.5.4" + ver: { py: "3.8", np: "==1.20.0", sp: "==1.5.4" } - case-id: macos15-intel-py312-current os: macos-15-intel - python-version: "3.12" - numpy: "==1.26.2" - scipy: "==1.11.2" + ver: { py: "3.12", np: "==1.26.2", sp: "==1.11.2" } - case-id: macos15-intel-py313-latest os: macos-15-intel - python-version: "3.13" - numpy: ">=2.1.0" - scipy: ">=1.14.1" + ver: { py: "3.13", np: ">=2.1.0", sp: ">=1.14.1" } + - case-id: macos15-intel-py314-latest + os: macos-15-intel + ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: # ASV needs git history for revision expressions such as HEAD^!. fetch-depth: 0 - - uses: conda-incubator/setup-miniconda@v3 + - uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3 with: # Driver environment: only runs ASV and helper scripts. The packages # being benchmarked are installed inside ASV-created environments. @@ -128,7 +116,14 @@ jobs: - name: Install ASV driver run: | - conda install -n gstools-benchmark-driver -y -c conda-forge asv + # ASV loads environment backends as optional plugins. Installing + # packaging/virtualenv explicitly prevents CI from falling back to + # the "existing" backend only, which would make environment_type: + # conda unavailable. + conda install -n gstools-benchmark-driver -y -c conda-forge \ + asv \ + packaging \ + virtualenv - name: Write per-job ASV config run: | @@ -137,9 +132,9 @@ jobs: conda run -n gstools-benchmark-driver python benchmarks/tools/write_asv_ci_config.py \ --base-config asv.conf.json \ --output "${GITHUB_WORKSPACE}/asv.ci.${{ matrix.case-id }}.json" \ - --python-version "${{ matrix.python-version }}" \ - --numpy "${{ matrix.numpy }}" \ - --scipy "${{ matrix.scipy }}" \ + --python-version "${{ matrix.ver.py }}" \ + --numpy "${{ matrix.ver.np }}" \ + --scipy "${{ matrix.ver.sp }}" \ --env-dir ".asv-ci/env/${{ matrix.case-id }}" \ --results-dir ".asv-ci/results/${{ matrix.case-id }}" \ --html-dir ".asv-ci/html/${{ matrix.case-id }}" @@ -164,7 +159,7 @@ jobs: # Upload even on failure so environment/config details are available # when debugging resolver, compiler, or benchmark discovery problems. if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: asv-1-thread-${{ matrix.case-id }} path: | @@ -176,7 +171,7 @@ jobs: benchmark_parallel_backend_availability: # Second stage: after every 1-thread ASV matrix child succeeds, prove that # the parallel backends can be installed and used on each OS runner. - name: Parallel backend readiness on ${{ matrix.os }} py${{ matrix.python-version }} + name: Parallel backend readiness on ${{ matrix.os }} py${{ matrix.ver.py }} runs-on: ${{ matrix.os }} # For a matrix job, this waits for all children of the upstream matrix. needs: benchmark_1_thread_availability @@ -187,38 +182,30 @@ jobs: # OpenMP builds are compiler-heavy. Check them on every OS, but only # with the latest dependency stack to avoid multiplying CI cost. include: - - case-id: ubuntu-py313-latest + - case-id: ubuntu-py314-latest os: ubuntu-latest - python-version: "3.13" - numpy: ">=2.1.0" - scipy: ">=1.14.1" - - case-id: windows-py313-latest + ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } + - case-id: windows-py314-latest os: windows-latest - python-version: "3.13" - numpy: ">=2.1.0" - scipy: ">=1.14.1" - - case-id: macos15-py313-latest + ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } + - case-id: macos15-py314-latest os: macos-15 - python-version: "3.13" - numpy: ">=2.1.0" - scipy: ">=1.14.1" - - case-id: macos15-intel-py313-latest + ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } + - case-id: macos15-intel-py314-latest os: macos-15-intel - python-version: "3.13" - numpy: ">=2.1.0" - scipy: ">=1.14.1" + ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: fetch-depth: 0 - - uses: conda-incubator/setup-miniconda@v3 + - uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3 with: # OpenMP environment: unlike gstools-benchmark-driver, this env gets # GSTools, gstools-cython, gstools_core, compilers, and runtime deps. activate-environment: gstools-benchmark-openmp - python-version: ${{ matrix.python-version }} + python-version: ${{ matrix.ver.py }} channels: conda-forge conda-remove-defaults: true @@ -227,8 +214,8 @@ jobs: # Install runtime deps plus compilers/build tools before compiling # gstools-cython from source with GSTOOLS_BUILD_PARALLEL=1. conda install -n gstools-benchmark-openmp -y -c conda-forge \ - "numpy${{ matrix.numpy }}" \ - "scipy${{ matrix.scipy }}" \ + "numpy${{ matrix.ver.np }}" \ + "scipy${{ matrix.ver.sp }}" \ c-compiler \ cxx-compiler \ cython \ diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 79993388a..6e4683e4f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,12 +18,12 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: fetch-depth: "0" - name: Set up Python 3.11 - uses: actions/setup-python@v5 + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: python-version: 3.11 @@ -55,7 +55,13 @@ jobs: fail-fast: false matrix: os: - [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-13, macos-14] + [ + ubuntu-latest, + ubuntu-24.04-arm, + windows-latest, + macos-15, + macos-15-intel, + ] # https://github.com/scipy/oldest-supported-numpy/blob/main/setup.cfg ver: - { py: "3.8", np: "==1.20.0", sp: "==1.5.4" } @@ -65,22 +71,26 @@ jobs: - { py: "3.12", np: "==1.26.2", sp: "==1.11.2" } - { py: "3.13", np: "==2.1.0", sp: "==1.14.1" } - { py: "3.13", np: ">=2.1.0", sp: ">=1.14.1" } + - { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } exclude: - os: ubuntu-24.04-arm ver: { py: "3.8", np: "==1.20.0", sp: "==1.5.4" } - - os: macos-14 + # macos-15 is the Apple Silicon runner. Old Python/NumPy/SciPy + # combinations do not have reliable arm64 wheels there, so keep + # legacy dependency coverage on Linux, Windows, and macOS Intel. + - os: macos-15 ver: { py: "3.8", np: "==1.20.0", sp: "==1.5.4" } - - os: macos-14 + - os: macos-15 ver: { py: "3.9", np: "==1.20.0", sp: "==1.5.4" } - - os: macos-14 + - os: macos-15 ver: { py: "3.10", np: "==1.21.6", sp: "==1.7.2" } steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: fetch-depth: "0" - name: Set up Python ${{ matrix.ver.py }} - uses: actions/setup-python@v5 + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: python-version: ${{ matrix.ver.py }} @@ -103,7 +113,7 @@ jobs: # PEP 517 package builder from pypa python -m build . - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 if: matrix.os == 'ubuntu-latest' && matrix.ver.py == '3.11' with: name: dist @@ -114,7 +124,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 with: name: dist path: dist @@ -122,7 +132,7 @@ jobs: - name: Publish to Test PyPI # only if working on main if: github.ref == 'refs/heads/main' - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 with: user: __token__ password: ${{ secrets.test_pypi_password }} @@ -132,7 +142,7 @@ jobs: - name: Publish to PyPI # only if tagged if: startsWith(github.ref, 'refs/tags') - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 with: user: __token__ password: ${{ secrets.pypi_password }} diff --git a/.github/workflows/super-linter.yml b/.github/workflows/super-linter.yml index 83eab46eb..044f4251b 100644 --- a/.github/workflows/super-linter.yml +++ b/.github/workflows/super-linter.yml @@ -24,12 +24,12 @@ jobs: timeout-minutes: 15 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: fetch-depth: 0 - name: Run Super-Linter - uses: super-linter/super-linter@v8.6.0 + uses: super-linter/super-linter@9e863354e3ff62e0727d37183162c4a88873df41 # v8.6.0 env: DEFAULT_BRANCH: main GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/vale.yml b/.github/workflows/vale.yml index b0da9d3e5..606028117 100644 --- a/.github/workflows/vale.yml +++ b/.github/workflows/vale.yml @@ -34,10 +34,10 @@ jobs: timeout-minutes: 10 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Run Vale - uses: vale-cli/vale-action@v2.1.2 + uses: vale-cli/vale-action@d89dee975228ae261d22c15adcd03578634d429c # v2 with: # Pin the Vale CLI separately from the GitHub Action wrapper so prose # checks do not change behavior unexpectedly between runs. diff --git a/benchmarks/tools/check_backend_parallel_ready.py b/benchmarks/tools/check_backend_parallel_ready.py index 8a6c7aa38..cf56845be 100644 --- a/benchmarks/tools/check_backend_parallel_ready.py +++ b/benchmarks/tools/check_backend_parallel_ready.py @@ -79,7 +79,9 @@ def check_rust_backend(threads): return False if not gs.config._GSTOOLS_CORE_AVAIL: - print("Rust backend readiness: FAIL. GSTools did not detect gstools_core.") + print( + "Rust backend readiness: FAIL. GSTools did not detect gstools_core." + ) return False previous = ( diff --git a/benchmarks/tools/install_openmp_cython.py b/benchmarks/tools/install_openmp_cython.py index 2883fe0ca..6c720218d 100644 --- a/benchmarks/tools/install_openmp_cython.py +++ b/benchmarks/tools/install_openmp_cython.py @@ -33,7 +33,9 @@ def prepend_path(build_env, *paths): def first_match(directory, patterns): for pattern in patterns: - matches = sorted(path for path in directory.glob(pattern) if path.is_file()) + matches = sorted( + path for path in directory.glob(pattern) if path.is_file() + ) if matches: return matches[0] return None @@ -153,7 +155,9 @@ def write_macos_wrapper(path, force_cxx=False): GSTOOLS_FORCE_CXX=1 exec "$(dirname "$0")/gstools-asv-clang-openmp" "$@" """ path.write_text(text, encoding="utf8") - path.chmod(path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + path.chmod( + path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH + ) def macos_build_env(env_dir): @@ -284,7 +288,10 @@ def main(): elif system == "Windows": build_env = windows_build_env(env_dir) else: - print(f"Unsupported platform for OpenMP ASV build: {system}", file=sys.stderr) + print( + f"Unsupported platform for OpenMP ASV build: {system}", + file=sys.stderr, + ) return 2 if build_env is None: diff --git a/pyproject.toml b/pyproject.toml index f39ea2eab..ce43bdb71 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Topic :: Scientific/Engineering", "Topic :: Scientific/Engineering :: GIS", "Topic :: Scientific/Engineering :: Hydrology", From fca014eab7b31d7bba4babd27ff01af30d0bf828 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Thu, 28 May 2026 10:16:34 +0200 Subject: [PATCH 08/14] Fix ASV benchmark CI matrix and lint config --- .github/linters/.yaml-lint.yml | 5 +++++ .github/workflows/asv-benchmarks.yml | 25 +++++++++++++------------ .github/workflows/main.yml | 7 +++++-- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/.github/linters/.yaml-lint.yml b/.github/linters/.yaml-lint.yml index b55f2aa14..c9872e0bd 100644 --- a/.github/linters/.yaml-lint.yml +++ b/.github/linters/.yaml-lint.yml @@ -1,6 +1,11 @@ extends: default rules: + braces: + max-spaces-inside: 1 + min-spaces-inside: 0 + comments: + min-spaces-from-content: 1 document-start: disable line-length: disable truthy: diff --git a/.github/workflows/asv-benchmarks.yml b/.github/workflows/asv-benchmarks.yml index 97ae3e9be..9e24f6fe1 100644 --- a/.github/workflows/asv-benchmarks.yml +++ b/.github/workflows/asv-benchmarks.yml @@ -48,26 +48,27 @@ jobs: # This list is hand-picked, not imported from main.yml or GSTools-Core. # Each entry below becomes one independent GitHub Actions job. The goal # is representative coverage: each OS runner gets older/current/latest - # Python/NumPy/SciPy stacks without running every possible cross-product. + # conda-solvable Python/NumPy/SciPy stacks without running every + # possible cross-product. include: - - case-id: ubuntu-py38-oldest + - case-id: ubuntu-py311-older os: ubuntu-latest - ver: { py: "3.8", np: "==1.20.0", sp: "==1.5.4" } + ver: { py: "3.11", np: "==1.23.5", sp: "==1.9.3" } - case-id: ubuntu-py312-current os: ubuntu-latest - ver: { py: "3.12", np: "==1.26.2", sp: "==1.11.2" } + ver: { py: "3.12", np: "==1.26.4", sp: "==1.11.4" } - case-id: ubuntu-py313-latest os: ubuntu-latest ver: { py: "3.13", np: ">=2.1.0", sp: ">=1.14.1" } - case-id: ubuntu-py314-latest os: ubuntu-latest ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } - - case-id: windows-py38-oldest + - case-id: windows-py311-older os: windows-latest - ver: { py: "3.8", np: "==1.20.0", sp: "==1.5.4" } + ver: { py: "3.11", np: "==1.23.5", sp: "==1.9.3" } - case-id: windows-py312-current os: windows-latest - ver: { py: "3.12", np: "==1.26.2", sp: "==1.11.2" } + ver: { py: "3.12", np: "==1.26.4", sp: "==1.11.4" } - case-id: windows-py313-latest os: windows-latest ver: { py: "3.13", np: ">=2.1.0", sp: ">=1.14.1" } @@ -76,22 +77,22 @@ jobs: ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } - case-id: macos15-py311-older os: macos-15 - ver: { py: "3.11", np: "==1.23.2", sp: "==1.9.2" } + ver: { py: "3.11", np: "==1.23.5", sp: "==1.9.3" } - case-id: macos15-py312-current os: macos-15 - ver: { py: "3.12", np: "==1.26.2", sp: "==1.11.2" } + ver: { py: "3.12", np: "==1.26.4", sp: "==1.11.4" } - case-id: macos15-py313-latest os: macos-15 ver: { py: "3.13", np: ">=2.1.0", sp: ">=1.14.1" } - case-id: macos15-py314-latest os: macos-15 ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } - - case-id: macos15-intel-py38-oldest + - case-id: macos15-intel-py311-older os: macos-15-intel - ver: { py: "3.8", np: "==1.20.0", sp: "==1.5.4" } + ver: { py: "3.11", np: "==1.23.5", sp: "==1.9.3" } - case-id: macos15-intel-py312-current os: macos-15-intel - ver: { py: "3.12", np: "==1.26.2", sp: "==1.11.2" } + ver: { py: "3.12", np: "==1.26.4", sp: "==1.11.4" } - case-id: macos15-intel-py313-latest os: macos-15-intel ver: { py: "3.13", np: ">=2.1.0", sp: ">=1.14.1" } diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6e4683e4f..b09c61712 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -101,11 +101,14 @@ jobs: - name: Install GSTools run: | - pip install -v --editable .[test] + pip install -v --editable '.[test]' - name: Run tests + env: + NUMPY_SPEC: ${{ matrix.ver.np }} + SCIPY_SPEC: ${{ matrix.ver.sp }} run: | - pip install "numpy${{ matrix.ver.np }}" "scipy${{ matrix.ver.sp }}" + pip install "numpy${NUMPY_SPEC}" "scipy${SCIPY_SPEC}" python -m pytest -v tests/ - name: Build dist From 00d19093a21a29605965fb86c3568271dd42d8e5 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Thu, 28 May 2026 13:27:55 +0200 Subject: [PATCH 09/14] third iteration GitHub actions --- .github/workflows/asv-benchmarks.yml | 127 +++++++++++++----------- benchmarks/README.md | 26 +++-- benchmarks/tools/write_asv_ci_config.py | 16 ++- 3 files changed, 99 insertions(+), 70 deletions(-) diff --git a/.github/workflows/asv-benchmarks.yml b/.github/workflows/asv-benchmarks.yml index 9e24f6fe1..356764762 100644 --- a/.github/workflows/asv-benchmarks.yml +++ b/.github/workflows/asv-benchmarks.yml @@ -34,10 +34,13 @@ defaults: shell: bash -el {0} jobs: - benchmark_1_thread_availability: - # First stage: prove the normal one-thread ASV benchmark can be installed - # and started across the supported runner/dependency combinations. - name: 1-thread ASV on ${{ matrix.os }} py${{ matrix.ver.py }} + benchmark_availability: + # Each matrix child is its own small chain: first run the normal one-thread + # ASV smoke check, then run parallel readiness immediately for entries + # marked with parallel_check. GitHub Actions cannot make one matrix child + # depend only on the matching child of another matrix job, so keeping both + # phases in one job avoids waiting for unrelated OS/Python combinations. + name: Benchmark availability on ${{ matrix.os }} py${{ matrix.ver.py }} runs-on: ${{ matrix.os }} timeout-minutes: 75 strategy: @@ -49,56 +52,73 @@ jobs: # Each entry below becomes one independent GitHub Actions job. The goal # is representative coverage: each OS runner gets older/current/latest # conda-solvable Python/NumPy/SciPy stacks without running every - # possible cross-product. + # possible cross-product. Use exact package pins here because ASV's + # conda backend does not handle open-ended version ranges reliably. include: - case-id: ubuntu-py311-older os: ubuntu-latest ver: { py: "3.11", np: "==1.23.5", sp: "==1.9.3" } + parallel_check: "false" - case-id: ubuntu-py312-current os: ubuntu-latest ver: { py: "3.12", np: "==1.26.4", sp: "==1.11.4" } + parallel_check: "false" - case-id: ubuntu-py313-latest os: ubuntu-latest - ver: { py: "3.13", np: ">=2.1.0", sp: ">=1.14.1" } + ver: { py: "3.13", np: "==2.1.3", sp: "==1.14.1" } + parallel_check: "false" - case-id: ubuntu-py314-latest os: ubuntu-latest - ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } + ver: { py: "3.14", np: "==2.3.3", sp: "==1.16.1" } + parallel_check: "true" - case-id: windows-py311-older os: windows-latest ver: { py: "3.11", np: "==1.23.5", sp: "==1.9.3" } + parallel_check: "false" - case-id: windows-py312-current os: windows-latest ver: { py: "3.12", np: "==1.26.4", sp: "==1.11.4" } + parallel_check: "false" - case-id: windows-py313-latest os: windows-latest - ver: { py: "3.13", np: ">=2.1.0", sp: ">=1.14.1" } + ver: { py: "3.13", np: "==2.1.3", sp: "==1.14.1" } + parallel_check: "false" - case-id: windows-py314-latest os: windows-latest - ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } + ver: { py: "3.14", np: "==2.3.3", sp: "==1.16.1" } + parallel_check: "true" - case-id: macos15-py311-older os: macos-15 ver: { py: "3.11", np: "==1.23.5", sp: "==1.9.3" } + parallel_check: "false" - case-id: macos15-py312-current os: macos-15 ver: { py: "3.12", np: "==1.26.4", sp: "==1.11.4" } + parallel_check: "false" - case-id: macos15-py313-latest os: macos-15 - ver: { py: "3.13", np: ">=2.1.0", sp: ">=1.14.1" } + ver: { py: "3.13", np: "==2.1.3", sp: "==1.14.1" } + parallel_check: "false" - case-id: macos15-py314-latest os: macos-15 - ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } + ver: { py: "3.14", np: "==2.3.3", sp: "==1.16.1" } + parallel_check: "true" - case-id: macos15-intel-py311-older os: macos-15-intel ver: { py: "3.11", np: "==1.23.5", sp: "==1.9.3" } + parallel_check: "false" - case-id: macos15-intel-py312-current os: macos-15-intel ver: { py: "3.12", np: "==1.26.4", sp: "==1.11.4" } + parallel_check: "false" - case-id: macos15-intel-py313-latest os: macos-15-intel - ver: { py: "3.13", np: ">=2.1.0", sp: ">=1.14.1" } + ver: { py: "3.13", np: "==2.1.3", sp: "==1.14.1" } + parallel_check: "false" - case-id: macos15-intel-py314-latest os: macos-15-intel - ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } + ver: { py: "3.14", np: "==2.3.3", sp: "==1.16.1" } + parallel_check: "true" steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 @@ -142,9 +162,20 @@ jobs: - name: Configure ASV machine run: | + # Supply explicit machine metadata for CI. ASV can infer this on + # Linux/macOS ARM, but Windows runner detection is less reliable. + # The case-id is also passed to asv run below so both commands use + # the same machine entry. conda run -n gstools-benchmark-driver asv \ --config "${GITHUB_WORKSPACE}/asv.ci.${{ matrix.case-id }}.json" \ - machine --yes + machine \ + --machine "${{ matrix.case-id }}" \ + --os "${{ runner.os }}" \ + --arch "${{ runner.arch }}" \ + --cpu "GitHub Actions ${{ matrix.os }}" \ + --num_cpu "2" \ + --ram "unknown" \ + --yes - name: Run 1-thread ASV availability check env: @@ -152,9 +183,18 @@ jobs: # the non-parallel path first. GSTOOLS_BENCHMARK_THREADS: "1" run: | + # Run one lightweight ASV benchmark method. Its parameters still + # exercise both backend values, but CI avoids running the full + # benchmark suite because this workflow checks availability, not + # performance quality. conda run -n gstools-benchmark-driver asv \ --config "${GITHUB_WORKSPACE}/asv.ci.${{ matrix.case-id }}.json" \ - run 'HEAD^!' --quick --bench benchmark_backends --show-stderr + run \ + --machine "${{ matrix.case-id }}" \ + --quick \ + --bench benchmark_backends.VariogramWorkflowBenchmarks.time_variogram_estimate \ + --show-stderr \ + 'HEAD^!' - name: Upload ASV availability results # Upload even on failure so environment/config details are available @@ -169,52 +209,15 @@ jobs: .asv-ci/html/${{ matrix.case-id }} if-no-files-found: ignore - benchmark_parallel_backend_availability: - # Second stage: after every 1-thread ASV matrix child succeeds, prove that - # the parallel backends can be installed and used on each OS runner. - name: Parallel backend readiness on ${{ matrix.os }} py${{ matrix.ver.py }} - runs-on: ${{ matrix.os }} - # For a matrix job, this waits for all children of the upstream matrix. - needs: benchmark_1_thread_availability - timeout-minutes: 75 - strategy: - fail-fast: false - matrix: - # OpenMP builds are compiler-heavy. Check them on every OS, but only - # with the latest dependency stack to avoid multiplying CI cost. - include: - - case-id: ubuntu-py314-latest - os: ubuntu-latest - ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } - - case-id: windows-py314-latest - os: windows-latest - ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } - - case-id: macos15-py314-latest - os: macos-15 - ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } - - case-id: macos15-intel-py314-latest - os: macos-15-intel - ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } - - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - fetch-depth: 0 - - - uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3 - with: - # OpenMP environment: unlike gstools-benchmark-driver, this env gets - # GSTools, gstools-cython, gstools_core, compilers, and runtime deps. - activate-environment: gstools-benchmark-openmp - python-version: ${{ matrix.ver.py }} - channels: conda-forge - conda-remove-defaults: true - - - name: Install benchmark and build requirements + - name: Create OpenMP readiness environment + if: matrix.parallel_check == 'true' run: | - # Install runtime deps plus compilers/build tools before compiling - # gstools-cython from source with GSTOOLS_BUILD_PARALLEL=1. - conda install -n gstools-benchmark-openmp -y -c conda-forge \ + # OpenMP builds are compiler-heavy. Check them on every OS, but only + # with the latest dependency stack to avoid multiplying CI cost. This + # step runs only after this exact matrix child's 1-thread ASV check + # has passed. + conda create -n gstools-benchmark-openmp -y -c conda-forge \ + "python=${{ matrix.ver.py }}" \ "numpy${{ matrix.ver.np }}" \ "scipy${{ matrix.ver.sp }}" \ c-compiler \ @@ -230,6 +233,7 @@ jobs: wheel - name: Install Rust backend package + if: matrix.parallel_check == 'true' run: | # gstools_core is installed with pip because conda availability is not # uniform across every runner/platform combination. @@ -237,6 +241,7 @@ jobs: "gstools_core>=1.0.0" - name: Build GSTools-Cython with OpenMP + if: matrix.parallel_check == 'true' run: | # The helper dispatches by platform: llvm-openmp wrapper on macOS, # conda compiler toolchain on Linux, and native MSVC on Windows. @@ -244,6 +249,7 @@ jobs: benchmarks/tools/install_openmp_cython.py - name: Install local GSTools checkout + if: matrix.parallel_check == 'true' run: | # Dependencies are already pinned by the matrix. Install the checkout # without letting pip resolve or replace them. @@ -251,6 +257,7 @@ jobs: --no-deps --editable . - name: Check Cython OpenMP and Rust backend readiness + if: matrix.parallel_check == 'true' run: | # Fast readiness check, not a performance benchmark. It fails if # Cython does not report OpenMP or Rust cannot run with NUM_THREADS=2. diff --git a/benchmarks/README.md b/benchmarks/README.md index 013077232..0e323e977 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -754,21 +754,31 @@ the benchmark tooling can be installed and started on GitHub-hosted Linux, Windows, macOS Apple Silicon, and macOS Intel runners. This workflow is an availability check, not a performance benchmark run. -The workflow has two dependent stages: +Each matrix job is its own small chain: -- `benchmark_1_thread_availability`: runs `asv.conf.json` with - `GSTOOLS_BENCHMARK_THREADS=1` on a representative Python/NumPy/SciPy matrix. -- `benchmark_parallel_backend_availability`: waits for the 1-thread stage, then - verifies Cython OpenMP and Rust backend readiness on the latest dependency - combo for each OS runner. +- first, it runs `asv.conf.json` with `GSTOOLS_BENCHMARK_THREADS=1` on one + representative Python/NumPy/SciPy combination. +- then, for entries marked as the latest dependency combo for an OS runner, it + builds the OpenMP environment and verifies Cython OpenMP plus Rust backend + readiness. + +This layout keeps the required order within each runner: the parallel check for +one OS/dependency setup starts only after that setup's 1-thread ASV check has +passed. It does not wait for unrelated matrix entries on other operating +systems. The 1-thread stage runs a quick ASV command: ```bash -asv run "HEAD^!" --quick --bench benchmark_backends --show-stderr +asv run "HEAD^!" --quick \ + --bench benchmark_backends.VariogramWorkflowBenchmarks.time_variogram_estimate \ + --show-stderr ``` -The parallel-readiness stage intentionally does not run a full ASV OpenMP +That benchmark method still exercises both configured backend values through +ASV parameters, but avoids running the full performance suite in CI. + +The parallel-readiness steps intentionally do not run a full ASV OpenMP benchmark. Instead, it builds `gstools-cython` with OpenMP, installs `gstools_core`, installs the local GSTools checkout, and runs: diff --git a/benchmarks/tools/write_asv_ci_config.py b/benchmarks/tools/write_asv_ci_config.py index 9ec5b2b8a..0999ada01 100644 --- a/benchmarks/tools/write_asv_ci_config.py +++ b/benchmarks/tools/write_asv_ci_config.py @@ -13,6 +13,15 @@ from pathlib import Path +def validate_asv_req_spec(package, spec): + if spec.startswith((">", "<", "~", "!=")) or "," in spec: + raise ValueError( + f"{package} uses open-ended CI spec {spec!r}. ASV's conda " + "backend expects exact matrix pins here, for example '==2.1.3'." + ) + return spec + + def parse_args(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--base-config", required=True, type=Path) @@ -37,8 +46,11 @@ def main(): config["html_dir"] = args.html_dir req = config.setdefault("matrix", {}).setdefault("req", {}) - req["numpy"] = [args.numpy] - req["scipy"] = [args.scipy] + try: + req["numpy"] = [validate_asv_req_spec("numpy", args.numpy)] + req["scipy"] = [validate_asv_req_spec("scipy", args.scipy)] + except ValueError as err: + raise SystemExit(str(err)) from err args.output.parent.mkdir(parents=True, exist_ok=True) with args.output.open("w", encoding="utf8") as config_file: From db86d44d3914fb861acde67c15f84c81e235b110 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Thu, 28 May 2026 15:30:23 +0200 Subject: [PATCH 10/14] removing not needed dependencies --- .github/dependabot.yml | 20 -------- .github/linters/.yaml-lint.yml | 15 ------ .github/styles/GSTools/Terms.yml | 14 ------ .../config/vocabularies/GSTools/accept.txt | 11 ---- .../config/vocabularies/GSTools/reject.txt | 1 - .github/workflows/asv-benchmarks.yml | 16 +++--- .github/workflows/main.yml | 44 ++++++---------- .github/workflows/super-linter.yml | 44 ---------------- .github/workflows/vale.yml | 50 ------------------- .vale.ini | 6 --- benchmarks/README.md | 6 +-- 11 files changed, 25 insertions(+), 202 deletions(-) delete mode 100644 .github/dependabot.yml delete mode 100644 .github/linters/.yaml-lint.yml delete mode 100644 .github/styles/GSTools/Terms.yml delete mode 100644 .github/styles/config/vocabularies/GSTools/accept.txt delete mode 100644 .github/styles/config/vocabularies/GSTools/reject.txt delete mode 100644 .github/workflows/super-linter.yml delete mode 100644 .github/workflows/vale.yml delete mode 100644 .vale.ini diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 4dfddfd9b..000000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,20 +0,0 @@ -version: 2 -updates: - # Dependabot version updates do not run benchmark CI every week. They only - # check whether workflow actions such as actions/checkout or Super-Linter - # have newer versions, then open PRs for review. - - package-ecosystem: "github-actions" - directory: "/" - schedule: - # Weekly is a low-noise maintenance cadence. Change to monthly if the - # update PRs become distracting. - interval: "weekly" - day: "monday" - time: "05:00" - groups: - # Group action bumps into one PR where possible, instead of one PR per - # action version. - github-actions: - patterns: - - "*" - open-pull-requests-limit: 5 diff --git a/.github/linters/.yaml-lint.yml b/.github/linters/.yaml-lint.yml deleted file mode 100644 index c9872e0bd..000000000 --- a/.github/linters/.yaml-lint.yml +++ /dev/null @@ -1,15 +0,0 @@ -extends: default - -rules: - braces: - max-spaces-inside: 1 - min-spaces-inside: 0 - comments: - min-spaces-from-content: 1 - document-start: disable - line-length: disable - truthy: - allowed-values: - - "true" - - "false" - - "on" diff --git a/.github/styles/GSTools/Terms.yml b/.github/styles/GSTools/Terms.yml deleted file mode 100644 index 9d528e238..000000000 --- a/.github/styles/GSTools/Terms.yml +++ /dev/null @@ -1,14 +0,0 @@ -extends: substitution -message: "Use '%s' instead of '%s'." -level: error -ignorecase: false -swap: - "Github": "GitHub" - "MacOS": "macOS" - "MacOs": "macOS" - "Macos": "macOS" - "OpenMp": "OpenMP" - "openMP": "OpenMP" - "Powershell": "PowerShell" - "Power Shell": "PowerShell" - "conda forge": "conda-forge" diff --git a/.github/styles/config/vocabularies/GSTools/accept.txt b/.github/styles/config/vocabularies/GSTools/accept.txt deleted file mode 100644 index 581358563..000000000 --- a/.github/styles/config/vocabularies/GSTools/accept.txt +++ /dev/null @@ -1,11 +0,0 @@ -ASV -Cython -GeoStat -GSTools -OpenMP -PowerShell -NumPy -SciPy -conda-forge -gstools-cython -gstools_core diff --git a/.github/styles/config/vocabularies/GSTools/reject.txt b/.github/styles/config/vocabularies/GSTools/reject.txt deleted file mode 100644 index 8b1378917..000000000 --- a/.github/styles/config/vocabularies/GSTools/reject.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/.github/workflows/asv-benchmarks.yml b/.github/workflows/asv-benchmarks.yml index 356764762..757a331a2 100644 --- a/.github/workflows/asv-benchmarks.yml +++ b/.github/workflows/asv-benchmarks.yml @@ -87,20 +87,20 @@ jobs: os: windows-latest ver: { py: "3.14", np: "==2.3.3", sp: "==1.16.1" } parallel_check: "true" - - case-id: macos15-py311-older - os: macos-15 + - case-id: macos-latest-py311-older + os: macos-latest ver: { py: "3.11", np: "==1.23.5", sp: "==1.9.3" } parallel_check: "false" - - case-id: macos15-py312-current - os: macos-15 + - case-id: macos-latest-py312-current + os: macos-latest ver: { py: "3.12", np: "==1.26.4", sp: "==1.11.4" } parallel_check: "false" - - case-id: macos15-py313-latest - os: macos-15 + - case-id: macos-latest-py313-latest + os: macos-latest ver: { py: "3.13", np: "==2.1.3", sp: "==1.14.1" } parallel_check: "false" - - case-id: macos15-py314-latest - os: macos-15 + - case-id: macos-latest-py314-latest + os: macos-latest ver: { py: "3.14", np: "==2.3.3", sp: "==1.16.1" } parallel_check: "true" - case-id: macos15-intel-py311-older diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b09c61712..da78b6457 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,12 +18,12 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@v4 with: fetch-depth: "0" - name: Set up Python 3.11 - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + uses: actions/setup-python@v5 with: python-version: 3.11 @@ -55,13 +55,7 @@ jobs: fail-fast: false matrix: os: - [ - ubuntu-latest, - ubuntu-24.04-arm, - windows-latest, - macos-15, - macos-15-intel, - ] + [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-15-intel, macos-latest] # https://github.com/scipy/oldest-supported-numpy/blob/main/setup.cfg ver: - { py: "3.8", np: "==1.20.0", sp: "==1.5.4" } @@ -70,27 +64,24 @@ jobs: - { py: "3.11", np: "==1.23.2", sp: "==1.9.2" } - { py: "3.12", np: "==1.26.2", sp: "==1.11.2" } - { py: "3.13", np: "==2.1.0", sp: "==1.14.1" } - - { py: "3.13", np: ">=2.1.0", sp: ">=1.14.1" } - - { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } + - { py: "3.14", np: "==2.3.2", sp: "==1.16.1" } + - { py: "3.14", np: ">=2.3.2", sp: ">=1.16.1" } exclude: - os: ubuntu-24.04-arm ver: { py: "3.8", np: "==1.20.0", sp: "==1.5.4" } - # macos-15 is the Apple Silicon runner. Old Python/NumPy/SciPy - # combinations do not have reliable arm64 wheels there, so keep - # legacy dependency coverage on Linux, Windows, and macOS Intel. - - os: macos-15 + - os: macos-latest ver: { py: "3.8", np: "==1.20.0", sp: "==1.5.4" } - - os: macos-15 + - os: macos-latest ver: { py: "3.9", np: "==1.20.0", sp: "==1.5.4" } - - os: macos-15 + - os: macos-latest ver: { py: "3.10", np: "==1.21.6", sp: "==1.7.2" } steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@v4 with: fetch-depth: "0" - name: Set up Python ${{ matrix.ver.py }} - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.ver.py }} @@ -101,14 +92,11 @@ jobs: - name: Install GSTools run: | - pip install -v --editable '.[test]' + pip install -v --editable .[test] - name: Run tests - env: - NUMPY_SPEC: ${{ matrix.ver.np }} - SCIPY_SPEC: ${{ matrix.ver.sp }} run: | - pip install "numpy${NUMPY_SPEC}" "scipy${SCIPY_SPEC}" + pip install "numpy${{ matrix.ver.np }}" "scipy${{ matrix.ver.sp }}" python -m pytest -v tests/ - name: Build dist @@ -116,7 +104,7 @@ jobs: # PEP 517 package builder from pypa python -m build . - - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + - uses: actions/upload-artifact@v4 if: matrix.os == 'ubuntu-latest' && matrix.ver.py == '3.11' with: name: dist @@ -127,7 +115,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + - uses: actions/download-artifact@v4 with: name: dist path: dist @@ -135,7 +123,7 @@ jobs: - name: Publish to Test PyPI # only if working on main if: github.ref == 'refs/heads/main' - uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 + uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ password: ${{ secrets.test_pypi_password }} @@ -145,7 +133,7 @@ jobs: - name: Publish to PyPI # only if tagged if: startsWith(github.ref, 'refs/tags') - uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 + uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ password: ${{ secrets.pypi_password }} diff --git a/.github/workflows/super-linter.yml b/.github/workflows/super-linter.yml deleted file mode 100644 index 044f4251b..000000000 --- a/.github/workflows/super-linter.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: Super-Linter - -on: - # This workflow is cheap compared with ASV/compiler checks, so run it on - # every PR and push to main to catch workflow/configuration issues early. - pull_request: - push: - branches: - - "main" - workflow_dispatch: - -# Super-Linter needs read access and a status check permission so it can report -# results back to the commit/PR. It does not need write access to repository -# contents. -permissions: - contents: read - packages: read - statuses: write - -jobs: - lint_repository_metadata: - name: Lint GitHub Actions, YAML, and JSON - runs-on: ubuntu-latest - timeout-minutes: 15 - - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - fetch-depth: 0 - - - name: Run Super-Linter - uses: super-linter/super-linter@9e863354e3ff62e0727d37183162c4a88873df41 # v8.6.0 - env: - DEFAULT_BRANCH: main - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # Lint changed files only. This avoids introducing unrelated failures - # from old Markdown/YAML while still checking new PR changes. - VALIDATE_ALL_CODEBASE: false - # Opt-in mode: these true values make Super-Linter skip Python and - # Markdown. Ruff owns Python linting in main.yml, and vale.yml owns - # Markdown prose/terminology checks. - VALIDATE_GITHUB_ACTIONS: true - VALIDATE_JSON: true - VALIDATE_YAML: true diff --git a/.github/workflows/vale.yml b/.github/workflows/vale.yml deleted file mode 100644 index 606028117..000000000 --- a/.github/workflows/vale.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Vale - -on: - # Vale is for documentation/prose quality. It runs only when Markdown or Vale - # configuration changes, so it complements Super-Linter without adding noise - # to source-only changes. - pull_request: - paths: - - "**/*.md" - - ".vale.ini" - - ".github/styles/**" - - ".github/workflows/vale.yml" - push: - branches: - - "main" - paths: - - "**/*.md" - - ".vale.ini" - - ".github/styles/**" - - ".github/workflows/vale.yml" - workflow_dispatch: - -# The Vale action reads repository content and reports findings back to the PR -# check. It does not need permission to modify files. -permissions: - contents: read - pull-requests: read - checks: write - -jobs: - lint_markdown_prose: - name: Lint Markdown prose and terminology - runs-on: ubuntu-latest - timeout-minutes: 10 - - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - name: Run Vale - uses: vale-cli/vale-action@d89dee975228ae261d22c15adcd03578634d429c # v2 - with: - # Pin the Vale CLI separately from the GitHub Action wrapper so prose - # checks do not change behavior unexpectedly between runs. - version: "3.14.1" - # Check tracked documentation paths, not generated local ASV - # environments such as .asv/ or .asv-openmp/. - files: '["README.md", "CONTRIBUTING.md", "AUTHORS.md", "CHANGELOG.md", "benchmarks"]' - filter_mode: nofilter - reporter: github-check - fail_on_error: true diff --git a/.vale.ini b/.vale.ini deleted file mode 100644 index 045ef69f8..000000000 --- a/.vale.ini +++ /dev/null @@ -1,6 +0,0 @@ -StylesPath = .github/styles -MinAlertLevel = warning -Vocab = GSTools - -[*.md] -BasedOnStyles = GSTools diff --git a/benchmarks/README.md b/benchmarks/README.md index 0e323e977..382b87c83 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -751,7 +751,7 @@ For example, `--limit 10` means "print the top 10 function rows after sorting". The repository includes `.github/workflows/asv-benchmarks.yml` to check that the benchmark tooling can be installed and started on GitHub-hosted Linux, -Windows, macOS Apple Silicon, and macOS Intel runners. This workflow is an +Windows, `macos-latest`, and `macos-15-intel` runners. This workflow is an availability check, not a performance benchmark run. Each matrix job is its own small chain: @@ -804,10 +804,6 @@ The workflow runs automatically on pull requests and pushes that change ASV configs, benchmark files, package metadata, or the workflow itself. It can also be started from the GitHub Actions tab with `workflow_dispatch`. -For a short explanation of GitHub Actions contexts, matrix jobs, `needs`, -runner labels, secrets, variables, and why this workflow is path-filtered, see -[`benchmarks/github_actions_guide.md`](github_actions_guide.md). - ## More ASV Commands Save results for only the current commit: From bd58de319362a8d4e95688bdc915a0571a974992 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Thu, 28 May 2026 22:42:43 +0200 Subject: [PATCH 11/14] 4th iteration benchmarking GitHub Actions --- .github/workflows/asv-benchmarks.yml | 96 +++++++++-------- asv.openmp.conf.json | 4 +- benchmarks/README.md | 85 ++++++++------- benchmarks/benchmark_backends.py | 23 +++- .../tools/check_backend_parallel_ready.py | 51 +++++++-- benchmarks/tools/check_cython_openmp.py | 102 ------------------ .../tools/profile_benchmark_workflows.py | 4 +- benchmarks/tools/write_asv_ci_config.py | 20 ++-- benchmarks/tools/write_asv_ci_machine.py | 55 ++++++++++ 9 files changed, 237 insertions(+), 203 deletions(-) delete mode 100644 benchmarks/tools/check_cython_openmp.py create mode 100644 benchmarks/tools/write_asv_ci_machine.py diff --git a/.github/workflows/asv-benchmarks.yml b/.github/workflows/asv-benchmarks.yml index 757a331a2..3987bbd85 100644 --- a/.github/workflows/asv-benchmarks.yml +++ b/.github/workflows/asv-benchmarks.yml @@ -52,73 +52,77 @@ jobs: # Each entry below becomes one independent GitHub Actions job. The goal # is representative coverage: each OS runner gets older/current/latest # conda-solvable Python/NumPy/SciPy stacks without running every - # possible cross-product. Use exact package pins here because ASV's - # conda backend does not handle open-ended version ranges reliably. + # possible cross-product. Use bare exact package versions here; the + # helper writes conda-style pins into ASV's generated config. include: - case-id: ubuntu-py311-older os: ubuntu-latest - ver: { py: "3.11", np: "==1.23.5", sp: "==1.9.3" } + ver: { py: "3.11", np: "1.23.5", sp: "1.9.3" } parallel_check: "false" - case-id: ubuntu-py312-current os: ubuntu-latest - ver: { py: "3.12", np: "==1.26.4", sp: "==1.11.4" } + ver: { py: "3.12", np: "1.26.4", sp: "1.11.4" } parallel_check: "false" - case-id: ubuntu-py313-latest os: ubuntu-latest - ver: { py: "3.13", np: "==2.1.3", sp: "==1.14.1" } - parallel_check: "false" + ver: { py: "3.13", np: "2.1.3", sp: "1.14.1" } + parallel_check: "true" - case-id: ubuntu-py314-latest os: ubuntu-latest - ver: { py: "3.14", np: "==2.3.3", sp: "==1.16.1" } - parallel_check: "true" + ver: { py: "3.14", np: "2.3.2", sp: "1.16.1" } + asv_backends: "cython_fallback" + parallel_check: "false" - case-id: windows-py311-older os: windows-latest - ver: { py: "3.11", np: "==1.23.5", sp: "==1.9.3" } + ver: { py: "3.11", np: "1.23.5", sp: "1.9.3" } parallel_check: "false" - case-id: windows-py312-current os: windows-latest - ver: { py: "3.12", np: "==1.26.4", sp: "==1.11.4" } + ver: { py: "3.12", np: "1.26.4", sp: "1.11.4" } parallel_check: "false" - case-id: windows-py313-latest os: windows-latest - ver: { py: "3.13", np: "==2.1.3", sp: "==1.14.1" } - parallel_check: "false" + ver: { py: "3.13", np: "2.1.3", sp: "1.14.1" } + parallel_check: "true" - case-id: windows-py314-latest os: windows-latest - ver: { py: "3.14", np: "==2.3.3", sp: "==1.16.1" } - parallel_check: "true" + ver: { py: "3.14", np: "2.3.2", sp: "1.16.1" } + asv_backends: "cython_fallback" + parallel_check: "false" - case-id: macos-latest-py311-older os: macos-latest - ver: { py: "3.11", np: "==1.23.5", sp: "==1.9.3" } + ver: { py: "3.11", np: "1.23.5", sp: "1.9.3" } parallel_check: "false" - case-id: macos-latest-py312-current os: macos-latest - ver: { py: "3.12", np: "==1.26.4", sp: "==1.11.4" } + ver: { py: "3.12", np: "1.26.4", sp: "1.11.4" } parallel_check: "false" - case-id: macos-latest-py313-latest os: macos-latest - ver: { py: "3.13", np: "==2.1.3", sp: "==1.14.1" } - parallel_check: "false" + ver: { py: "3.13", np: "2.1.3", sp: "1.14.1" } + parallel_check: "true" - case-id: macos-latest-py314-latest os: macos-latest - ver: { py: "3.14", np: "==2.3.3", sp: "==1.16.1" } - parallel_check: "true" + ver: { py: "3.14", np: "2.3.2", sp: "1.16.1" } + asv_backends: "cython_fallback" + parallel_check: "false" - case-id: macos15-intel-py311-older os: macos-15-intel - ver: { py: "3.11", np: "==1.23.5", sp: "==1.9.3" } + ver: { py: "3.11", np: "1.23.5", sp: "1.9.3" } parallel_check: "false" - case-id: macos15-intel-py312-current os: macos-15-intel - ver: { py: "3.12", np: "==1.26.4", sp: "==1.11.4" } + ver: { py: "3.12", np: "1.26.4", sp: "1.11.4" } parallel_check: "false" - case-id: macos15-intel-py313-latest os: macos-15-intel - ver: { py: "3.13", np: "==2.1.3", sp: "==1.14.1" } - parallel_check: "false" + ver: { py: "3.13", np: "2.1.3", sp: "1.14.1" } + parallel_check: "true" - case-id: macos15-intel-py314-latest os: macos-15-intel - ver: { py: "3.14", np: "==2.3.3", sp: "==1.16.1" } - parallel_check: "true" + ver: { py: "3.14", np: "2.3.2", sp: "1.16.1" } + asv_backends: "cython_fallback" + parallel_check: "false" steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 @@ -162,31 +166,33 @@ jobs: - name: Configure ASV machine run: | - # Supply explicit machine metadata for CI. ASV can infer this on - # Linux/macOS ARM, but Windows runner detection is less reliable. - # The case-id is also passed to asv run below so both commands use - # the same machine entry. - conda run -n gstools-benchmark-driver asv \ - --config "${GITHUB_WORKSPACE}/asv.ci.${{ matrix.case-id }}.json" \ - machine \ + # Supply explicit machine metadata for CI. Writing the ASV machine + # file directly avoids platform-specific issues with `asv machine` + # under the Windows conda/bash wrapper. + conda run -n gstools-benchmark-driver python \ + benchmarks/tools/write_asv_ci_machine.py \ --machine "${{ matrix.case-id }}" \ --os "${{ runner.os }}" \ --arch "${{ runner.arch }}" \ --cpu "GitHub Actions ${{ matrix.os }}" \ - --num_cpu "2" \ - --ram "unknown" \ - --yes + --num-cpu "2" \ + --ram "unknown" - name: Run 1-thread ASV availability check env: # Public benchmark-thread interface. Baseline CI intentionally checks # the non-parallel path first. GSTOOLS_BENCHMARK_THREADS: "1" + # Keep the default benchmark behavior unless a matrix entry narrows + # the smoke run. Python 3.14 currently runs Cython-only here because + # gstools_core 1.2.0 can segfault during the variogram smoke case. + GSTOOLS_BENCHMARK_BACKENDS: ${{ matrix.asv_backends || '' }} run: | - # Run one lightweight ASV benchmark method. Its parameters still - # exercise both backend values, but CI avoids running the full - # benchmark suite because this workflow checks availability, not - # performance quality. + # Run one lightweight ASV benchmark method. Most matrix entries + # exercise both backend values; Python 3.14 is limited to Cython + # until gstools_core is stable there. CI avoids the full benchmark + # suite because this workflow checks availability, not performance + # quality. conda run -n gstools-benchmark-driver asv \ --config "${GITHUB_WORKSPACE}/asv.ci.${{ matrix.case-id }}.json" \ run \ @@ -213,13 +219,13 @@ jobs: if: matrix.parallel_check == 'true' run: | # OpenMP builds are compiler-heavy. Check them on every OS, but only - # with the latest dependency stack to avoid multiplying CI cost. This - # step runs only after this exact matrix child's 1-thread ASV check - # has passed. + # with the newest Rust-compatible dependency stack to avoid + # multiplying CI cost. This step runs only after this exact matrix + # child's 1-thread ASV check has passed. conda create -n gstools-benchmark-openmp -y -c conda-forge \ "python=${{ matrix.ver.py }}" \ - "numpy${{ matrix.ver.np }}" \ - "scipy${{ matrix.ver.sp }}" \ + "numpy=${{ matrix.ver.np }}" \ + "scipy=${{ matrix.ver.sp }}" \ c-compiler \ cxx-compiler \ cython \ diff --git a/asv.openmp.conf.json b/asv.openmp.conf.json index 274e2d3cd..6d707b994 100644 --- a/asv.openmp.conf.json +++ b/asv.openmp.conf.json @@ -31,7 +31,7 @@ "install_command": [ "in-dir={env_dir} python -m pip install gstools_core>=1.0.0", "in-dir={env_dir} python {conf_dir}/benchmarks/tools/install_openmp_cython.py {env_dir}", - "in-dir={env_dir} python {conf_dir}/benchmarks/tools/check_cython_openmp.py --fail-if-no-openmp", - "in-dir={env_dir} python -m pip install --no-deps {build_dir}" + "in-dir={env_dir} python -m pip install --no-deps {build_dir}", + "in-dir={env_dir} python {conf_dir}/benchmarks/tools/check_backend_parallel_ready.py --verbose" ] } diff --git a/benchmarks/README.md b/benchmarks/README.md index 382b87c83..04d095e04 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -42,7 +42,7 @@ deciding where optimization work should go: - [Optional Parallelisation with OpenMP](#optional-parallelisation-with-openmp) - [Shared OpenMP Rule](#shared-openmp-rule) - [OpenMP ASV Configuration](#openmp-asv-configuration) - - [Verify Cython OpenMP](#verify-cython-openmp) + - [Verify Parallel Backends](#verify-parallel-backends) - [Run On macOS And Linux](#run-on-macos-and-linux) - [Run On Windows](#run-on-windows) - [HPC Notes](#hpc-notes) @@ -115,14 +115,13 @@ The benchmarking setup currently consists of: - `benchmarks/tools/profile_benchmark_workflows.py`: runs one representative workflow from `benchmark_backends.py` under Python's built-in `cProfile`, so you can see which functions take time in the current checkout. -- `benchmarks/tools/check_cython_openmp.py`: optional helper for checking - whether the active Python environment's GSTools-Cython extensions detect - OpenMP parallel support. - `benchmarks/tools/check_backend_parallel_ready.py`: CI helper that verifies - Cython OpenMP detection and Rust backend execution with more than one GSTools - thread. + Cython OpenMP detection and Rust backend execution with more than one + GSTools thread. - `benchmarks/tools/write_asv_ci_config.py`: CI helper that writes a temporary per-job ASV config for one Python/NumPy/SciPy combination. +- `benchmarks/tools/write_asv_ci_machine.py`: CI helper that writes ASV machine + metadata non-interactively for GitHub-hosted runners. - `benchmarks/tools/install_openmp_cython.py`: helper used by `asv.openmp.conf.json` to compile `gstools-cython` with OpenMP on macOS, Linux, and native Windows. @@ -522,8 +521,8 @@ you explicitly want to measure backend scaling with multiple thread counts. ### Shared OpenMP Rule -The benchmark code can be run with several thread labels by setting for example -`GSTOOLS_BENCHMARK_THREADS=1,2,4,8,16`. That only passes different +The benchmark code can be run with several thread labels by setting, for +example, `GSTOOLS_BENCHMARK_THREADS=2,4,8`. That only passes different `gstools.config.NUM_THREADS` values to GSTools. It does not, by itself, make the Cython backend parallel. @@ -533,7 +532,7 @@ that environment before interpreting Cython scaling results: ```bash ASV_ENV="$(ls -td .asv-openmp/env/* | head -n 1)" -"$ASV_ENV/bin/python" benchmarks/tools/check_cython_openmp.py --fail-if-no-openmp +"$ASV_ENV/bin/python" benchmarks/tools/check_backend_parallel_ready.py --verbose ``` If the check fails, the benchmark may still run, but the Cython backend should @@ -570,11 +569,12 @@ compiler handling: - Linux: uses the ASV conda compiler toolchain when available. - Windows: uses native MSVC Build Tools. -### Verify Cython OpenMP +### Verify Parallel Backends -Only interpret the Cython rows as OpenMP-enabled after this check passes inside -the `.asv-openmp/env/...` environment. The active `gstools-benchmark` conda -environment is only the ASV driver environment; it is normal for +Only interpret the Cython rows as OpenMP-enabled, and the Rust rows as +parallel-ready, after this check passes inside the `.asv-openmp/env/...` +environment. The active `gstools-benchmark` conda environment is only the ASV +driver environment; it is normal for `python benchmarks/tools/check_backend_parallel_ready.py --verbose` to fail there with `gstools: not installed`, `gstools_cython: not installed`, or `gstools_core: not installed`. @@ -583,8 +583,6 @@ On macOS and Linux: ```bash ASV_OPENMP_ENV="$(ls -td .asv-openmp/env/* | head -n 1)" -"$ASV_OPENMP_ENV/bin/python" benchmarks/tools/check_cython_openmp.py --verbose -"$ASV_OPENMP_ENV/bin/python" benchmarks/tools/check_cython_openmp.py --fail-if-no-openmp "$ASV_OPENMP_ENV/bin/python" benchmarks/tools/check_backend_parallel_ready.py --verbose ``` @@ -595,15 +593,12 @@ $asvOpenmpEnv = Get-ChildItem .asv-openmp\env -Directory | Sort-Object LastWriteTime -Descending | Select-Object -First 1 $asvOpenmpPython = Join-Path $asvOpenmpEnv.FullName 'python.exe' -& $asvOpenmpPython benchmarks\tools\check_cython_openmp.py --verbose -& $asvOpenmpPython benchmarks\tools\check_cython_openmp.py --fail-if-no-openmp & $asvOpenmpPython benchmarks\tools\check_backend_parallel_ready.py --verbose ``` Expected passing output contains: ```text -OpenMP check: PASS Cython OpenMP readiness: PASS Rust backend readiness: PASS with NUM_THREADS=2 ``` @@ -623,16 +618,16 @@ asv --config asv.openmp.conf.json machine --yes Run a quick current-commit OpenMP smoke run: ```bash -GSTOOLS_BENCHMARK_THREADS=1,2,4,8,16 \ +GSTOOLS_BENCHMARK_THREADS=2,4,8 \ asv --config asv.openmp.conf.json run 'HEAD^!' --quick --bench benchmark_backends --show-stderr ``` -Verify Cython OpenMP with the commands in -[Verify Cython OpenMP](#verify-cython-openmp). If it passes, run the last-five +Verify the parallel backends with the commands in +[Verify Parallel Backends](#verify-parallel-backends). If it passes, run the last-five commits: ```bash -GSTOOLS_BENCHMARK_THREADS=1,2,4,8,16 \ +GSTOOLS_BENCHMARK_THREADS=2,4,8 \ asv --config asv.openmp.conf.json run 'HEAD~5..HEAD' --bench benchmark_backends --show-stderr ``` @@ -665,16 +660,16 @@ asv --config asv.openmp.conf.json machine --yes Run a quick current-commit OpenMP smoke run: ```powershell -$env:GSTOOLS_BENCHMARK_THREADS = '1,2,4,8,16' +$env:GSTOOLS_BENCHMARK_THREADS = '2,4,8' asv --config asv.openmp.conf.json run 'HEAD^!' --quick --bench benchmark_backends --show-stderr ``` -Verify Cython OpenMP with the PowerShell commands in -[Verify Cython OpenMP](#verify-cython-openmp). If it passes, run the last-five +Verify the parallel backends with the PowerShell commands in +[Verify Parallel Backends](#verify-parallel-backends). If it passes, run the last-five commits: ```powershell -$env:GSTOOLS_BENCHMARK_THREADS = '1,2,4,8,16' +$env:GSTOOLS_BENCHMARK_THREADS = '2,4,8' asv --config asv.openmp.conf.json run 'HEAD~5..HEAD' --bench benchmark_backends --show-stderr ``` @@ -704,7 +699,7 @@ The `asv.openmp.conf.json` workflow is intended for local macOS, Linux, and native Windows machines. Managed HPC systems often use custom compiler modules, MPI/OpenMP runtimes, scheduler pinning, and CPU affinity rules. Use the same validation rule there: only interpret Cython as OpenMP-enabled if -`check_cython_openmp.py --fail-if-no-openmp` passes inside the exact ASV +`check_backend_parallel_ready.py --verbose` passes inside the exact ASV environment used for the benchmark run. ### Profiling With cProfile for Multiple Threads @@ -716,7 +711,7 @@ same cProfile case several times with the OpenMP ASV environment: ASV_OPENMP_ENV="$(ls -td .asv-openmp/env/* | head -n 1)" ASV_OPENMP_PYTHON="$ASV_OPENMP_ENV/bin/python" -for threads in threads_1 threads_2 threads_4 threads_8 threads_16; do +for threads in threads_2 threads_4 threads_8; do "$ASV_OPENMP_PYTHON" benchmarks/tools/profile_benchmark_workflows.py --case krige-extra-large --backend rust_core --threads "$threads" --limit 10 done ``` @@ -729,7 +724,7 @@ $asvOpenmpEnv = Get-ChildItem .asv-openmp\env -Directory | Select-Object -First 1 $asvOpenmpPython = Join-Path $asvOpenmpEnv.FullName 'python.exe' -foreach ($threads in 'threads_1', 'threads_2', 'threads_4', 'threads_8', 'threads_16') { +foreach ($threads in 'threads_2', 'threads_4', 'threads_8') { & $asvOpenmpPython benchmarks\tools\profile_benchmark_workflows.py --case krige-extra-large --backend rust_core --threads $threads --limit 10 } ``` @@ -738,8 +733,8 @@ Useful options: - `--case`: choose one workflow, or use `all` - `--backend`: choose `cython_fallback` or `rust_core` -- `--threads`: choose `threads_1`, `threads_2`, `threads_4`, `threads_8`, - or `threads_16` +- `--threads`: choose `threads_1` for a baseline profile, or `threads_2`, + `threads_4`, or `threads_8` for parallel profiles - `--limit`: number of function rows to print from the cProfile table - `--sort cumtime`: sort by cumulative time, usually the best first view - `--sort tottime`: sort by time spent directly in each function @@ -758,9 +753,9 @@ Each matrix job is its own small chain: - first, it runs `asv.conf.json` with `GSTOOLS_BENCHMARK_THREADS=1` on one representative Python/NumPy/SciPy combination. -- then, for entries marked as the latest dependency combo for an OS runner, it - builds the OpenMP environment and verifies Cython OpenMP plus Rust backend - readiness. +- then, for entries marked as the newest Rust-compatible dependency combo for + an OS runner, it builds the OpenMP environment and verifies Cython OpenMP + plus Rust backend readiness. This layout keeps the required order within each runner: the parallel check for one OS/dependency setup starts only after that setup's 1-thread ASV check has @@ -775,8 +770,11 @@ asv run "HEAD^!" --quick \ --show-stderr ``` -That benchmark method still exercises both configured backend values through -ASV parameters, but avoids running the full performance suite in CI. +That benchmark method exercises both configured backend values through ASV +parameters, but avoids running the full performance suite in CI. Python 3.14 +currently runs this ASV smoke check with the Cython fallback backend only, +because `gstools_core` 1.2.0 can fail during the Rust variogram smoke case on +that interpreter. The parallel-readiness steps intentionally do not run a full ASV OpenMP benchmark. Instead, it builds `gstools-cython` with OpenMP, installs @@ -798,7 +796,7 @@ ASV_OPENMP_ENV="$(ls -td .asv-openmp/env/* | head -n 1)" That fast check proves that Cython reports OpenMP support and that the Rust backend can run a small workflow with `gstools.config.NUM_THREADS=2`. For real OpenMP scaling measurements, use `asv.openmp.conf.json` locally with -`GSTOOLS_BENCHMARK_THREADS=1,2,4,8,16`. +`GSTOOLS_BENCHMARK_THREADS=2,4,8`. The workflow runs automatically on pull requests and pushes that change ASV configs, benchmark files, package metadata, or the workflow itself. It can also @@ -837,6 +835,19 @@ asv run 'HEAD^!' --bench benchmark_backends asv compare origin/main HEAD ``` +Run the last three commits on local `main`: + +```bash +asv run 'main~3..main' --bench benchmark_backends --show-stderr +``` + +Run the last three commits on the latest remote `main`: + +```bash +git fetch origin main +asv run 'origin/main~3..origin/main' --bench benchmark_backends --show-stderr +``` + On a linear branch, `HEAD~5..HEAD` benchmarks: ```text diff --git a/benchmarks/benchmark_backends.py b/benchmarks/benchmark_backends.py index cc588c1e7..f4c853f39 100644 --- a/benchmarks/benchmark_backends.py +++ b/benchmarks/benchmark_backends.py @@ -19,7 +19,7 @@ By default the suite uses one GSTools thread. For local OpenMP scaling experiments, set GSTOOLS_BENCHMARK_THREADS, for example: - GSTOOLS_BENCHMARK_THREADS=1,2,4,8,16 \ + GSTOOLS_BENCHMARK_THREADS=2,4,8 \ asv --config asv.openmp.conf.json run 'HEAD^!' """ @@ -31,7 +31,26 @@ import gstools as gs import numpy as np -BACKENDS = ("cython_fallback", "rust_core") +AVAILABLE_BACKENDS = ("cython_fallback", "rust_core") + + +def _configured_backends(): + raw = os.environ.get("GSTOOLS_BENCHMARK_BACKENDS", "") + if not raw.strip(): + return AVAILABLE_BACKENDS + backends = tuple(item.strip() for item in raw.split(",") if item.strip()) + unknown = sorted(set(backends) - set(AVAILABLE_BACKENDS)) + if unknown: + raise ValueError( + "GSTOOLS_BENCHMARK_BACKENDS contains unknown backends: " + + ", ".join(unknown) + ) + if not backends: + raise ValueError("GSTOOLS_BENCHMARK_BACKENDS did not define backends") + return backends + + +BACKENDS = _configured_backends() def _configured_thread_counts(): diff --git a/benchmarks/tools/check_backend_parallel_ready.py b/benchmarks/tools/check_backend_parallel_ready.py index cf56845be..01ad8f374 100644 --- a/benchmarks/tools/check_backend_parallel_ready.py +++ b/benchmarks/tools/check_backend_parallel_ready.py @@ -13,7 +13,13 @@ import sys import numpy as np -from check_cython_openmp import MODULES, check_module, package_version + +MODULES = { + "variogram": "gstools_cython.variogram", + "field": "gstools_cython.field", + "krige": "gstools_cython.krige", +} +EXPLICIT_THREAD_COUNTS = (2, 4, 8) def parse_args(): @@ -29,10 +35,33 @@ def parse_args(): action="store_true", help="Print per-module Cython OpenMP thread details.", ) + parser.add_argument( + "--cython-only", + action="store_true", + help="Only check GSTools-Cython OpenMP readiness.", + ) return parser.parse_args() -def check_cython_openmp(verbose=False): +def package_version(package_name): + try: + package = importlib.import_module(package_name) + except ModuleNotFoundError: + return "not installed" + return getattr(package, "__version__", "unknown") + + +def check_module(label, module_name): + module = importlib.import_module(module_name) + default_threads = module.set_num_threads(None) + explicit = { + count: module.set_num_threads(count) + for count in EXPLICIT_THREAD_COUNTS + } + return label, default_threads, explicit + + +def check_cython_parallel(verbose=False): default_values = [] for label, module_name in MODULES.items(): try: @@ -47,10 +76,20 @@ def check_cython_openmp(verbose=False): ) print(f"{label} default None -> {default_threads}") print(f"{label} explicit -> {explicit_text}") - if explicit.get(2) != 2: + mismatches = { + request: actual + for request, actual in explicit.items() + if actual != request + } + if mismatches: + mismatch_text = ", ".join( + f"{request}->{actual}" + for request, actual in mismatches.items() + ) print( "Cython OpenMP readiness: FAIL. " - f"{label} did not accept an explicit 2-thread request." + f"{label} did not accept requested thread counts: " + f"{mismatch_text}." ) return False @@ -123,8 +162,8 @@ def main(): print(f"gstools_cython: {package_version('gstools_cython')}") print(f"gstools_core: {package_version('gstools_core')}") - cython_ready = check_cython_openmp(verbose=args.verbose) - rust_ready = check_rust_backend(args.threads) + cython_ready = check_cython_parallel(verbose=args.verbose) + rust_ready = True if args.cython_only else check_rust_backend(args.threads) return 0 if cython_ready and rust_ready else 1 diff --git a/benchmarks/tools/check_cython_openmp.py b/benchmarks/tools/check_cython_openmp.py deleted file mode 100644 index b3a410cc9..000000000 --- a/benchmarks/tools/check_cython_openmp.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python -"""Check whether GSTools-Cython detects OpenMP parallel support. - -This script verifies the active Python environment. Use it with the editable -development environment or with an ASV-created environment. - -Examples: - python benchmarks/tools/check_cython_openmp.py - python benchmarks/tools/check_cython_openmp.py --fail-if-no-openmp - python benchmarks/tools/check_cython_openmp.py --verbose - .asv/env//bin/python3 benchmarks/tools/check_cython_openmp.py -""" - -from __future__ import annotations - -import argparse -import importlib -import sys - -MODULES = { - "variogram": "gstools_cython.variogram", - "field": "gstools_cython.field", - "krige": "gstools_cython.krige", -} -EXPLICIT_THREAD_COUNTS = (1, 2, 4, 8, 16) - - -def parse_args(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--fail-if-no-openmp", - action="store_true", - help="Exit with status 1 if OpenMP thread detection reports <= 1.", - ) - parser.add_argument( - "--verbose", - action="store_true", - help="Print per-module default and explicit thread-count values.", - ) - return parser.parse_args() - - -def package_version(package_name): - try: - package = importlib.import_module(package_name) - except ModuleNotFoundError: - return "not installed" - return getattr(package, "__version__", "unknown") - - -def check_module(label, module_name): - module = importlib.import_module(module_name) - default_threads = module.set_num_threads(None) - explicit = { - count: module.set_num_threads(count) - for count in EXPLICIT_THREAD_COUNTS - } - return label, default_threads, explicit - - -def main(): - args = parse_args() - - print(f"python: {sys.executable}") - print(f"gstools: {package_version('gstools')}") - print(f"gstools_cython: {package_version('gstools_cython')}") - print(f"gstools_core: {package_version('gstools_core')}") - if args.verbose: - print( - "OpenMP evidence: default None should be >1. " - "Explicit values only prove the wrapper accepts the requested count." - ) - - default_values = [] - for label, module_name in MODULES.items(): - try: - label, default_threads, explicit = check_module(label, module_name) - except ModuleNotFoundError as err: - print(f"OpenMP check: FAIL. Missing module: {err.name}") - return 1 - default_values.append(default_threads) - if args.verbose: - explicit_text = ", ".join( - f"{request}->{actual}" for request, actual in explicit.items() - ) - print(f"{label} default None -> {default_threads}") - print(f"{label} explicit -> {explicit_text}") - - if min(default_values) > 1: - print("OpenMP check: PASS") - return 0 - - print( - "OpenMP check: FAIL. GSTools-Cython reports one default thread. " - "Explicit thread values may be accepted by the wrapper, but this does " - "not prove that the compiled extension is using OpenMP." - ) - return 1 if args.fail_if_no_openmp else 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmarks/tools/profile_benchmark_workflows.py b/benchmarks/tools/profile_benchmark_workflows.py index 2fd2fdff5..ebd8b925d 100644 --- a/benchmarks/tools/profile_benchmark_workflows.py +++ b/benchmarks/tools/profile_benchmark_workflows.py @@ -20,10 +20,9 @@ import argparse import cProfile -from pathlib import Path import pstats import sys - +from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(REPO_ROOT)) @@ -88,7 +87,6 @@ "threads_2", "threads_4", "threads_8", - "threads_16", ) diff --git a/benchmarks/tools/write_asv_ci_config.py b/benchmarks/tools/write_asv_ci_config.py index 0999ada01..edf6f5206 100644 --- a/benchmarks/tools/write_asv_ci_config.py +++ b/benchmarks/tools/write_asv_ci_config.py @@ -13,13 +13,21 @@ from pathlib import Path -def validate_asv_req_spec(package, spec): - if spec.startswith((">", "<", "~", "!=")) or "," in spec: +def conda_exact_pin(package, spec): + """Return an ASV/conda exact package pin from a CI matrix value.""" + spec = spec.strip() + if spec.startswith((">", "<", "~", "!")) or "," in spec: raise ValueError( f"{package} uses open-ended CI spec {spec!r}. ASV's conda " - "backend expects exact matrix pins here, for example '==2.1.3'." + "backend expects exact matrix pins here, for example '2.1.3'." ) - return spec + if spec.startswith("=="): + spec = spec[2:] + elif spec.startswith("="): + spec = spec[1:] + if not spec: + raise ValueError(f"{package} has an empty CI version pin.") + return f"={spec}" def parse_args(): @@ -47,8 +55,8 @@ def main(): req = config.setdefault("matrix", {}).setdefault("req", {}) try: - req["numpy"] = [validate_asv_req_spec("numpy", args.numpy)] - req["scipy"] = [validate_asv_req_spec("scipy", args.scipy)] + req["numpy"] = [conda_exact_pin("numpy", args.numpy)] + req["scipy"] = [conda_exact_pin("scipy", args.scipy)] except ValueError as err: raise SystemExit(str(err)) from err diff --git a/benchmarks/tools/write_asv_ci_machine.py b/benchmarks/tools/write_asv_ci_machine.py new file mode 100644 index 000000000..fd6ef25a5 --- /dev/null +++ b/benchmarks/tools/write_asv_ci_machine.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +"""Write ASV machine metadata for a non-interactive CI runner.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + + +def parse_args(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--machine", required=True) + parser.add_argument("--os", required=True) + parser.add_argument("--arch", required=True) + parser.add_argument("--cpu", required=True) + parser.add_argument("--num-cpu", required=True) + parser.add_argument("--ram", required=True) + parser.add_argument( + "--path", + type=Path, + default=Path.home() / ".asv-machine.json", + help="Machine metadata file written by 'asv machine'.", + ) + return parser.parse_args() + + +def main(): + args = parse_args() + if args.path.exists(): + with args.path.open(encoding="utf8") as machine_file: + machines = json.load(machine_file) + else: + machines = {"version": 1} + + machines["version"] = 1 + machines[args.machine] = { + "machine": args.machine, + "os": args.os, + "arch": args.arch, + "cpu": args.cpu, + "num_cpu": str(args.num_cpu), + "ram": args.ram, + } + + args.path.parent.mkdir(parents=True, exist_ok=True) + with args.path.open("w", encoding="utf8") as machine_file: + json.dump(machines, machine_file, indent=2) + machine_file.write("\n") + + print(f"Wrote ASV machine metadata for {args.machine} to {args.path}") + + +if __name__ == "__main__": + main() From 9bdb2c1c1c38e24528cb6d2e903ada4d8ddc119e Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Thu, 28 May 2026 23:54:16 +0200 Subject: [PATCH 12/14] simplified version GitHub actions for a pracmatic approach --- .github/workflows/asv-benchmarks.yml | 290 +++++++++-------------- benchmarks/README.md | 37 ++- benchmarks/tools/asv_speedup_summary.py | 1 - benchmarks/tools/write_asv_ci_config.py | 72 ------ benchmarks/tools/write_asv_ci_machine.py | 55 ----- 5 files changed, 121 insertions(+), 334 deletions(-) delete mode 100644 benchmarks/tools/write_asv_ci_config.py delete mode 100644 benchmarks/tools/write_asv_ci_machine.py diff --git a/.github/workflows/asv-benchmarks.yml b/.github/workflows/asv-benchmarks.yml index 3987bbd85..6a15c0977 100644 --- a/.github/workflows/asv-benchmarks.yml +++ b/.github/workflows/asv-benchmarks.yml @@ -1,11 +1,9 @@ name: Benchmark Availability Checks on: - # This workflow checks benchmark installability, not package correctness. - # It is intentionally path-filtered because ASV creates conda environments - # and the OpenMP job compiles extensions. Normal source/test validation is - # handled by main.yml; this workflow runs when benchmark-related files or - # package dependency metadata change. + # This workflow checks whether the benchmark code can be installed and + # started on the supported GitHub-hosted runners. It is a smoke check, not a + # performance benchmark suite. pull_request: branches: - "main" @@ -24,248 +22,172 @@ on: - "pyproject.toml" workflow_dispatch: -# The jobs only need to read the repository. Keep the default token minimal. permissions: contents: read -defaults: - run: - # setup-miniconda initializes conda for bash login shells on all runners. - shell: bash -el {0} - jobs: - benchmark_availability: - # Each matrix child is its own small chain: first run the normal one-thread - # ASV smoke check, then run parallel readiness immediately for entries - # marked with parallel_check. GitHub Actions cannot make one matrix child - # depend only on the matching child of another matrix job, so keeping both - # phases in one job avoids waiting for unrelated OS/Python combinations. - name: Benchmark availability on ${{ matrix.os }} py${{ matrix.ver.py }} + benchmark_asv_existing_env: + # Use the same dependency combinations as main.yml, but run only one + # lightweight benchmark method in the already-created Python environment. + # That avoids ASV's conda environment management in CI while still proving + # that users can start the benchmark suite on each runner family. + name: ASV smoke on ${{ matrix.os }} py${{ matrix.ver.py }} runs-on: ${{ matrix.os }} - timeout-minutes: 75 + timeout-minutes: 30 strategy: - # Keep the rest of the OS/dependency matrix running after one failure. - # That gives a full compatibility picture from a single workflow run. fail-fast: false matrix: - # This list is hand-picked, not imported from main.yml or GSTools-Core. - # Each entry below becomes one independent GitHub Actions job. The goal - # is representative coverage: each OS runner gets older/current/latest - # conda-solvable Python/NumPy/SciPy stacks without running every - # possible cross-product. Use bare exact package versions here; the - # helper writes conda-style pins into ASV's generated config. include: - - case-id: ubuntu-py311-older + - case-id: ubuntu-py311 os: ubuntu-latest - ver: { py: "3.11", np: "1.23.5", sp: "1.9.3" } - parallel_check: "false" - - case-id: ubuntu-py312-current + ver: { py: "3.11", np: "==1.23.2", sp: "==1.9.2" } + - case-id: ubuntu-py312 os: ubuntu-latest - ver: { py: "3.12", np: "1.26.4", sp: "1.11.4" } - parallel_check: "false" - - case-id: ubuntu-py313-latest + ver: { py: "3.12", np: "==1.26.2", sp: "==1.11.2" } + - case-id: ubuntu-py313 os: ubuntu-latest - ver: { py: "3.13", np: "2.1.3", sp: "1.14.1" } - parallel_check: "true" - - case-id: ubuntu-py314-latest + ver: { py: "3.13", np: "==2.1.0", sp: "==1.14.1" } + - case-id: ubuntu-py314 os: ubuntu-latest - ver: { py: "3.14", np: "2.3.2", sp: "1.16.1" } + ver: { py: "3.14", np: "==2.3.2", sp: "==1.16.1" } asv_backends: "cython_fallback" - parallel_check: "false" - - case-id: windows-py311-older + - case-id: windows-py311 os: windows-latest - ver: { py: "3.11", np: "1.23.5", sp: "1.9.3" } - parallel_check: "false" - - case-id: windows-py312-current + ver: { py: "3.11", np: "==1.23.2", sp: "==1.9.2" } + - case-id: windows-py312 os: windows-latest - ver: { py: "3.12", np: "1.26.4", sp: "1.11.4" } - parallel_check: "false" - - case-id: windows-py313-latest + ver: { py: "3.12", np: "==1.26.2", sp: "==1.11.2" } + - case-id: windows-py313 os: windows-latest - ver: { py: "3.13", np: "2.1.3", sp: "1.14.1" } - parallel_check: "true" - - case-id: windows-py314-latest + ver: { py: "3.13", np: "==2.1.0", sp: "==1.14.1" } + - case-id: windows-py314 os: windows-latest - ver: { py: "3.14", np: "2.3.2", sp: "1.16.1" } + ver: { py: "3.14", np: "==2.3.2", sp: "==1.16.1" } asv_backends: "cython_fallback" - parallel_check: "false" - - case-id: macos-latest-py311-older + - case-id: macos-latest-py311 os: macos-latest - ver: { py: "3.11", np: "1.23.5", sp: "1.9.3" } - parallel_check: "false" - - case-id: macos-latest-py312-current + ver: { py: "3.11", np: "==1.23.2", sp: "==1.9.2" } + - case-id: macos-latest-py312 os: macos-latest - ver: { py: "3.12", np: "1.26.4", sp: "1.11.4" } - parallel_check: "false" - - case-id: macos-latest-py313-latest + ver: { py: "3.12", np: "==1.26.2", sp: "==1.11.2" } + - case-id: macos-latest-py313 os: macos-latest - ver: { py: "3.13", np: "2.1.3", sp: "1.14.1" } - parallel_check: "true" - - case-id: macos-latest-py314-latest + ver: { py: "3.13", np: "==2.1.0", sp: "==1.14.1" } + - case-id: macos-latest-py314 os: macos-latest - ver: { py: "3.14", np: "2.3.2", sp: "1.16.1" } + ver: { py: "3.14", np: "==2.3.2", sp: "==1.16.1" } asv_backends: "cython_fallback" - parallel_check: "false" - - case-id: macos15-intel-py311-older + - case-id: macos15-intel-py311 os: macos-15-intel - ver: { py: "3.11", np: "1.23.5", sp: "1.9.3" } - parallel_check: "false" - - case-id: macos15-intel-py312-current + ver: { py: "3.11", np: "==1.23.2", sp: "==1.9.2" } + - case-id: macos15-intel-py312 os: macos-15-intel - ver: { py: "3.12", np: "1.26.4", sp: "1.11.4" } - parallel_check: "false" - - case-id: macos15-intel-py313-latest + ver: { py: "3.12", np: "==1.26.2", sp: "==1.11.2" } + - case-id: macos15-intel-py313 os: macos-15-intel - ver: { py: "3.13", np: "2.1.3", sp: "1.14.1" } - parallel_check: "true" - - case-id: macos15-intel-py314-latest + ver: { py: "3.13", np: "==2.1.0", sp: "==1.14.1" } + - case-id: macos15-intel-py314 os: macos-15-intel - ver: { py: "3.14", np: "2.3.2", sp: "1.16.1" } + ver: { py: "3.14", np: "==2.3.2", sp: "==1.16.1" } asv_backends: "cython_fallback" - parallel_check: "false" + + defaults: + run: + # Use one shell syntax for Linux, Windows, and macOS smoke commands. + shell: bash steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: - # ASV needs git history for revision expressions such as HEAD^!. fetch-depth: 0 - - uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3 + - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: - # Driver environment: only runs ASV and helper scripts. The packages - # being benchmarked are installed inside ASV-created environments. - activate-environment: gstools-benchmark-driver - python-version: "3.12" - channels: conda-forge - conda-remove-defaults: true + python-version: ${{ matrix.ver.py }} - - name: Install ASV driver + - name: Install benchmark smoke dependencies run: | - # ASV loads environment backends as optional plugins. Installing - # packaging/virtualenv explicitly prevents CI from falling back to - # the "existing" backend only, which would make environment_type: - # conda unavailable. - conda install -n gstools-benchmark-driver -y -c conda-forge \ + python -m pip install --upgrade pip + python -m pip install \ asv \ - packaging \ - virtualenv - - - name: Write per-job ASV config - run: | - # The committed asv.conf.json stays simple for local users. CI writes - # a temporary config so each matrix child can pin one dependency set. - conda run -n gstools-benchmark-driver python benchmarks/tools/write_asv_ci_config.py \ - --base-config asv.conf.json \ - --output "${GITHUB_WORKSPACE}/asv.ci.${{ matrix.case-id }}.json" \ - --python-version "${{ matrix.ver.py }}" \ - --numpy "${{ matrix.ver.np }}" \ - --scipy "${{ matrix.ver.sp }}" \ - --env-dir ".asv-ci/env/${{ matrix.case-id }}" \ - --results-dir ".asv-ci/results/${{ matrix.case-id }}" \ - --html-dir ".asv-ci/html/${{ matrix.case-id }}" + emcee \ + "gstools-cython>=1,<2" \ + "gstools_core>=1.0.0" \ + hankel \ + meshio \ + "numpy${{ matrix.ver.np }}" \ + pyevtk \ + "scipy${{ matrix.ver.sp }}" + python -m pip install --no-deps --editable . - name: Configure ASV machine run: | - # Supply explicit machine metadata for CI. Writing the ASV machine - # file directly avoids platform-specific issues with `asv machine` - # under the Windows conda/bash wrapper. - conda run -n gstools-benchmark-driver python \ - benchmarks/tools/write_asv_ci_machine.py \ - --machine "${{ matrix.case-id }}" \ - --os "${{ runner.os }}" \ - --arch "${{ runner.arch }}" \ - --cpu "GitHub Actions ${{ matrix.os }}" \ - --num-cpu "2" \ - --ram "unknown" + # existing environments still need a machine profile for ASV result + # metadata. --yes keeps the command non-interactive in CI. + asv machine --yes - - name: Run 1-thread ASV availability check + - name: Run ASV smoke check in existing environment env: - # Public benchmark-thread interface. Baseline CI intentionally checks - # the non-parallel path first. GSTOOLS_BENCHMARK_THREADS: "1" - # Keep the default benchmark behavior unless a matrix entry narrows - # the smoke run. Python 3.14 currently runs Cython-only here because - # gstools_core 1.2.0 can segfault during the variogram smoke case. GSTOOLS_BENCHMARK_BACKENDS: ${{ matrix.asv_backends || '' }} run: | - # Run one lightweight ASV benchmark method. Most matrix entries - # exercise both backend values; Python 3.14 is limited to Cython - # until gstools_core is stable there. CI avoids the full benchmark - # suite because this workflow checks availability, not performance - # quality. - conda run -n gstools-benchmark-driver asv \ - --config "${GITHUB_WORKSPACE}/asv.ci.${{ matrix.case-id }}.json" \ - run \ - --machine "${{ matrix.case-id }}" \ + # -E existing uses the Python environment prepared above. Python 3.14 + # is currently Cython-only because gstools_core 1.2.0 can fail during + # the Rust variogram smoke case. + asv run \ + -E existing \ --quick \ --bench benchmark_backends.VariogramWorkflowBenchmarks.time_variogram_estimate \ --show-stderr \ 'HEAD^!' - - name: Upload ASV availability results - # Upload even on failure so environment/config details are available - # when debugging resolver, compiler, or benchmark discovery problems. - if: always() - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + benchmark_parallel_readiness: + # Keep this separate from the ASV smoke job. It checks the smallest useful + # runtime proof for parallel backends without running a full ASV benchmark + # and without building gstools-cython from the PyPI source distribution. + name: Parallel readiness on ${{ matrix.os }} py3.13 + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + - windows-latest + - macos-latest + - macos-15-intel + + defaults: + run: + shell: bash -el {0} + + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3 with: - name: asv-1-thread-${{ matrix.case-id }} - path: | - asv.ci.${{ matrix.case-id }}.json - .asv-ci/results/${{ matrix.case-id }} - .asv-ci/html/${{ matrix.case-id }} - if-no-files-found: ignore + activate-environment: gstools-benchmark-readiness + python-version: "3.13" + channels: conda-forge + conda-remove-defaults: true - - name: Create OpenMP readiness environment - if: matrix.parallel_check == 'true' + - name: Install parallel readiness dependencies run: | - # OpenMP builds are compiler-heavy. Check them on every OS, but only - # with the newest Rust-compatible dependency stack to avoid - # multiplying CI cost. This step runs only after this exact matrix - # child's 1-thread ASV check has passed. - conda create -n gstools-benchmark-openmp -y -c conda-forge \ - "python=${{ matrix.ver.py }}" \ - "numpy=${{ matrix.ver.np }}" \ - "scipy=${{ matrix.ver.sp }}" \ - c-compiler \ - cxx-compiler \ - cython \ + conda install -n gstools-benchmark-readiness -y -c conda-forge \ emcee \ - extension-helpers \ + gstools-cython \ hankel \ meshio \ + "numpy=2.1.0" \ pip \ pyevtk \ - "setuptools>=77" \ - wheel - - - name: Install Rust backend package - if: matrix.parallel_check == 'true' - run: | - # gstools_core is installed with pip because conda availability is not - # uniform across every runner/platform combination. - conda run -n gstools-benchmark-openmp python -m pip install \ + "scipy=1.14.1" + conda run -n gstools-benchmark-readiness python -m pip install \ "gstools_core>=1.0.0" - - - name: Build GSTools-Cython with OpenMP - if: matrix.parallel_check == 'true' - run: | - # The helper dispatches by platform: llvm-openmp wrapper on macOS, - # conda compiler toolchain on Linux, and native MSVC on Windows. - conda run -n gstools-benchmark-openmp python \ - benchmarks/tools/install_openmp_cython.py - - - name: Install local GSTools checkout - if: matrix.parallel_check == 'true' - run: | - # Dependencies are already pinned by the matrix. Install the checkout - # without letting pip resolve or replace them. - conda run -n gstools-benchmark-openmp python -m pip install \ + conda run -n gstools-benchmark-readiness python -m pip install \ --no-deps --editable . - name: Check Cython OpenMP and Rust backend readiness - if: matrix.parallel_check == 'true' run: | - # Fast readiness check, not a performance benchmark. It fails if - # Cython does not report OpenMP or Rust cannot run with NUM_THREADS=2. - conda run -n gstools-benchmark-openmp python \ + conda run -n gstools-benchmark-readiness python \ benchmarks/tools/check_backend_parallel_ready.py --verbose diff --git a/benchmarks/README.md b/benchmarks/README.md index 04d095e04..3651dad0c 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -118,10 +118,6 @@ The benchmarking setup currently consists of: - `benchmarks/tools/check_backend_parallel_ready.py`: CI helper that verifies Cython OpenMP detection and Rust backend execution with more than one GSTools thread. -- `benchmarks/tools/write_asv_ci_config.py`: CI helper that writes a temporary - per-job ASV config for one Python/NumPy/SciPy combination. -- `benchmarks/tools/write_asv_ci_machine.py`: CI helper that writes ASV machine - metadata non-interactively for GitHub-hosted runners. - `benchmarks/tools/install_openmp_cython.py`: helper used by `asv.openmp.conf.json` to compile `gstools-cython` with OpenMP on macOS, Linux, and native Windows. @@ -749,23 +745,18 @@ the benchmark tooling can be installed and started on GitHub-hosted Linux, Windows, `macos-latest`, and `macos-15-intel` runners. This workflow is an availability check, not a performance benchmark run. -Each matrix job is its own small chain: +The workflow has two independent matrix jobs: -- first, it runs `asv.conf.json` with `GSTOOLS_BENCHMARK_THREADS=1` on one - representative Python/NumPy/SciPy combination. -- then, for entries marked as the newest Rust-compatible dependency combo for - an OS runner, it builds the OpenMP environment and verifies Cython OpenMP - plus Rust backend readiness. +- `benchmark_asv_existing_env` installs GSTools and the benchmark + dependencies with pip, then runs one ASV benchmark method with + `-E existing`. +- `benchmark_parallel_readiness` creates a small conda-forge environment and + runs `check_backend_parallel_ready.py --verbose`. -This layout keeps the required order within each runner: the parallel check for -one OS/dependency setup starts only after that setup's 1-thread ASV check has -passed. It does not wait for unrelated matrix entries on other operating -systems. - -The 1-thread stage runs a quick ASV command: +The ASV smoke stage runs a quick command in the existing Python environment: ```bash -asv run "HEAD^!" --quick \ +asv run -E existing "HEAD^!" --quick \ --bench benchmark_backends.VariogramWorkflowBenchmarks.time_variogram_estimate \ --show-stderr ``` @@ -777,16 +768,18 @@ because `gstools_core` 1.2.0 can fail during the Rust variogram smoke case on that interpreter. The parallel-readiness steps intentionally do not run a full ASV OpenMP -benchmark. Instead, it builds `gstools-cython` with OpenMP, installs -`gstools_core`, installs the local GSTools checkout, and runs: +benchmark. They install `gstools-cython` from conda-forge, install +`gstools_core`, install the local GSTools checkout, and run: ```bash -conda run -n gstools-benchmark-openmp python \ +conda run -n gstools-benchmark-readiness python \ benchmarks/tools/check_backend_parallel_ready.py --verbose ``` -Locally, run the same helper with the Python executable from the ASV OpenMP -environment: +CI does not call `benchmarks/tools/install_openmp_cython.py`; that helper is +kept for local/manual OpenMP ASV builds through `asv.openmp.conf.json`. +Locally, run the readiness helper with the Python executable from the ASV +OpenMP environment: ```bash ASV_OPENMP_ENV="$(ls -td .asv-openmp/env/* | head -n 1)" diff --git a/benchmarks/tools/asv_speedup_summary.py b/benchmarks/tools/asv_speedup_summary.py index b3239d702..f50e7a542 100644 --- a/benchmarks/tools/asv_speedup_summary.py +++ b/benchmarks/tools/asv_speedup_summary.py @@ -24,7 +24,6 @@ import math from pathlib import Path - BACKENDS = ("cython_fallback", "rust_core") THREAD_PREFIX = "threads_" LEGACY_BENCHMARKS = { diff --git a/benchmarks/tools/write_asv_ci_config.py b/benchmarks/tools/write_asv_ci_config.py deleted file mode 100644 index edf6f5206..000000000 --- a/benchmarks/tools/write_asv_ci_config.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python -"""Write a per-job ASV configuration for GitHub Actions. - -The committed ASV configs stay convenient for local benchmarking. CI uses this -helper to pin one Python/NumPy/SciPy combination per matrix job without turning -normal local ``asv run`` commands into a large dependency matrix. -""" - -from __future__ import annotations - -import argparse -import json -from pathlib import Path - - -def conda_exact_pin(package, spec): - """Return an ASV/conda exact package pin from a CI matrix value.""" - spec = spec.strip() - if spec.startswith((">", "<", "~", "!")) or "," in spec: - raise ValueError( - f"{package} uses open-ended CI spec {spec!r}. ASV's conda " - "backend expects exact matrix pins here, for example '2.1.3'." - ) - if spec.startswith("=="): - spec = spec[2:] - elif spec.startswith("="): - spec = spec[1:] - if not spec: - raise ValueError(f"{package} has an empty CI version pin.") - return f"={spec}" - - -def parse_args(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--base-config", required=True, type=Path) - parser.add_argument("--output", required=True, type=Path) - parser.add_argument("--python-version", required=True) - parser.add_argument("--numpy", required=True) - parser.add_argument("--scipy", required=True) - parser.add_argument("--env-dir", required=True) - parser.add_argument("--results-dir", required=True) - parser.add_argument("--html-dir", required=True) - return parser.parse_args() - - -def main(): - args = parse_args() - with args.base_config.open(encoding="utf8") as config_file: - config = json.load(config_file) - - config["pythons"] = [args.python_version] - config["env_dir"] = args.env_dir - config["results_dir"] = args.results_dir - config["html_dir"] = args.html_dir - - req = config.setdefault("matrix", {}).setdefault("req", {}) - try: - req["numpy"] = [conda_exact_pin("numpy", args.numpy)] - req["scipy"] = [conda_exact_pin("scipy", args.scipy)] - except ValueError as err: - raise SystemExit(str(err)) from err - - args.output.parent.mkdir(parents=True, exist_ok=True) - with args.output.open("w", encoding="utf8") as config_file: - json.dump(config, config_file, indent=2) - config_file.write("\n") - - print(f"Wrote {args.output}") - - -if __name__ == "__main__": - main() diff --git a/benchmarks/tools/write_asv_ci_machine.py b/benchmarks/tools/write_asv_ci_machine.py deleted file mode 100644 index fd6ef25a5..000000000 --- a/benchmarks/tools/write_asv_ci_machine.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python -"""Write ASV machine metadata for a non-interactive CI runner.""" - -from __future__ import annotations - -import argparse -import json -from pathlib import Path - - -def parse_args(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--machine", required=True) - parser.add_argument("--os", required=True) - parser.add_argument("--arch", required=True) - parser.add_argument("--cpu", required=True) - parser.add_argument("--num-cpu", required=True) - parser.add_argument("--ram", required=True) - parser.add_argument( - "--path", - type=Path, - default=Path.home() / ".asv-machine.json", - help="Machine metadata file written by 'asv machine'.", - ) - return parser.parse_args() - - -def main(): - args = parse_args() - if args.path.exists(): - with args.path.open(encoding="utf8") as machine_file: - machines = json.load(machine_file) - else: - machines = {"version": 1} - - machines["version"] = 1 - machines[args.machine] = { - "machine": args.machine, - "os": args.os, - "arch": args.arch, - "cpu": args.cpu, - "num_cpu": str(args.num_cpu), - "ram": args.ram, - } - - args.path.parent.mkdir(parents=True, exist_ok=True) - with args.path.open("w", encoding="utf8") as machine_file: - json.dump(machines, machine_file, indent=2) - machine_file.write("\n") - - print(f"Wrote ASV machine metadata for {args.machine} to {args.path}") - - -if __name__ == "__main__": - main() From 9c4f46b0ee137ec99a2a309b690245ddd5fedf02 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Fri, 29 May 2026 07:02:23 +0200 Subject: [PATCH 13/14] fifth iteration GitHub Actions for benchmarking --- .github/workflows/asv-benchmarks.yml | 7 ++++--- benchmarks/README.md | 7 ++++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/asv-benchmarks.yml b/.github/workflows/asv-benchmarks.yml index 6a15c0977..e469b1d29 100644 --- a/.github/workflows/asv-benchmarks.yml +++ b/.github/workflows/asv-benchmarks.yml @@ -133,13 +133,14 @@ jobs: run: | # -E existing uses the Python environment prepared above. Python 3.14 # is currently Cython-only because gstools_core 1.2.0 can fail during - # the Rust variogram smoke case. + # the Rust variogram smoke case. Existing environments cannot accept + # a commit range, so the checked-out commit is recorded explicitly. asv run \ -E existing \ + --set-commit-hash "$GITHUB_SHA" \ --quick \ --bench benchmark_backends.VariogramWorkflowBenchmarks.time_variogram_estimate \ - --show-stderr \ - 'HEAD^!' + --show-stderr benchmark_parallel_readiness: # Keep this separate from the ASV smoke job. It checks the smallest useful diff --git a/benchmarks/README.md b/benchmarks/README.md index 3651dad0c..a1a891ae7 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -756,11 +756,16 @@ The workflow has two independent matrix jobs: The ASV smoke stage runs a quick command in the existing Python environment: ```bash -asv run -E existing "HEAD^!" --quick \ +asv run -E existing --set-commit-hash "$(git rev-parse HEAD)" --quick \ --bench benchmark_backends.VariogramWorkflowBenchmarks.time_variogram_estimate \ --show-stderr ``` +Do not pass a range such as `HEAD^!` with `-E existing`: ASV cannot checkout +revisions inside an existing environment. The workflow uses +`--set-commit-hash "$GITHUB_SHA"` to label the result with the commit already +checked out by GitHub Actions. + That benchmark method exercises both configured backend values through ASV parameters, but avoids running the full performance suite in CI. Python 3.14 currently runs this ASV smoke check with the Cython fallback backend only, From c8947091b89dfa23ffcd75b8426ff9340f963e05 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Fri, 29 May 2026 07:20:14 +0200 Subject: [PATCH 14/14] Simplify benchmark CI smoke checks --- .github/workflows/asv-benchmarks.yml | 18 ++++++++++++ benchmarks/README.md | 28 +++++++++++++++---- .../tools/check_backend_parallel_ready.py | 24 ++++++++++++---- 3 files changed, 58 insertions(+), 12 deletions(-) diff --git a/.github/workflows/asv-benchmarks.yml b/.github/workflows/asv-benchmarks.yml index e469b1d29..e6a8d1e98 100644 --- a/.github/workflows/asv-benchmarks.yml +++ b/.github/workflows/asv-benchmarks.yml @@ -126,6 +126,22 @@ jobs: # metadata. --yes keeps the command non-interactive in CI. asv machine --yes + - name: Write ASV smoke config for checked-out commit + run: | + # With -E existing, ASV cannot receive a range like HEAD^!. If no + # range is supplied, ASV resolves the configured branches. PR + # checkouts do not necessarily have a local "main" branch, so this + # temporary config points ASV at the exact checked-out commit. + python - <<'PY' + import json + import os + from pathlib import Path + + config = json.loads(Path("asv.conf.json").read_text()) + config["branches"] = [os.environ["GITHUB_SHA"]] + Path("asv.ci-existing.json").write_text(json.dumps(config, indent=2)) + PY + - name: Run ASV smoke check in existing environment env: GSTOOLS_BENCHMARK_THREADS: "1" @@ -136,8 +152,10 @@ jobs: # the Rust variogram smoke case. Existing environments cannot accept # a commit range, so the checked-out commit is recorded explicitly. asv run \ + --config asv.ci-existing.json \ -E existing \ --set-commit-hash "$GITHUB_SHA" \ + --no-pull \ --quick \ --bench benchmark_backends.VariogramWorkflowBenchmarks.time_variogram_estimate \ --show-stderr diff --git a/benchmarks/README.md b/benchmarks/README.md index a1a891ae7..2187eeb07 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -756,15 +756,29 @@ The workflow has two independent matrix jobs: The ASV smoke stage runs a quick command in the existing Python environment: ```bash -asv run -E existing --set-commit-hash "$(git rev-parse HEAD)" --quick \ +python - <<'PY' +import json +from pathlib import Path + +config = json.loads(Path("asv.conf.json").read_text()) +config["branches"] = ["HEAD"] +Path("asv.ci-existing.json").write_text(json.dumps(config, indent=2)) +PY + +asv run --config asv.ci-existing.json \ + -E existing \ + --set-commit-hash "$(git rev-parse HEAD)" \ + --no-pull \ + --quick \ --bench benchmark_backends.VariogramWorkflowBenchmarks.time_variogram_estimate \ --show-stderr ``` Do not pass a range such as `HEAD^!` with `-E existing`: ASV cannot checkout -revisions inside an existing environment. The workflow uses -`--set-commit-hash "$GITHUB_SHA"` to label the result with the commit already -checked out by GitHub Actions. +revisions inside an existing environment. If no range is supplied, ASV resolves +the branches from the config file. The workflow writes a temporary config that +points `branches` at `$GITHUB_SHA`, then uses `--set-commit-hash "$GITHUB_SHA"` +to label the result with the commit already checked out by GitHub Actions. That benchmark method exercises both configured backend values through ASV parameters, but avoids running the full performance suite in CI. Python 3.14 @@ -792,8 +806,10 @@ ASV_OPENMP_ENV="$(ls -td .asv-openmp/env/* | head -n 1)" ``` That fast check proves that Cython reports OpenMP support and that the Rust -backend can run a small workflow with `gstools.config.NUM_THREADS=2`. For real -OpenMP scaling measurements, use `asv.openmp.conf.json` locally with +backend can run a small workflow with `gstools.config.NUM_THREADS=2`. The CI +probe requires Cython to accept explicit thread counts `2,4,8`; it prints the +default thread count but does not fail when a runner defaults to one thread. For +real OpenMP scaling measurements, use `asv.openmp.conf.json` locally with `GSTOOLS_BENCHMARK_THREADS=2,4,8`. The workflow runs automatically on pull requests and pushes that change ASV diff --git a/benchmarks/tools/check_backend_parallel_ready.py b/benchmarks/tools/check_backend_parallel_ready.py index 01ad8f374..2542702f4 100644 --- a/benchmarks/tools/check_backend_parallel_ready.py +++ b/benchmarks/tools/check_backend_parallel_ready.py @@ -1,9 +1,9 @@ #!/usr/bin/env python """Check that GSTools benchmark backends are ready for parallel runs. -This is a fast CI probe. It verifies that GSTools-Cython reports OpenMP support -and that the Rust backend can run a small workflow while GSTools is configured -with more than one thread. +This is a fast CI probe. It verifies that GSTools-Cython accepts explicit +OpenMP thread counts and that the Rust backend can run a small workflow while +GSTools is configured with more than one thread. """ from __future__ import annotations @@ -40,6 +40,15 @@ def parse_args(): action="store_true", help="Only check GSTools-Cython OpenMP readiness.", ) + parser.add_argument( + "--require-default-openmp", + action="store_true", + help=( + "Also fail when GSTools-Cython reports only one default thread. " + "CI does not use this because some runners default to one thread " + "while still accepting explicit OpenMP thread counts." + ), + ) return parser.parse_args() @@ -61,7 +70,7 @@ def check_module(label, module_name): return label, default_threads, explicit -def check_cython_parallel(verbose=False): +def check_cython_parallel(verbose=False, require_default_openmp=False): default_values = [] for label, module_name in MODULES.items(): try: @@ -93,7 +102,7 @@ def check_cython_parallel(verbose=False): ) return False - if min(default_values) <= 1: + if require_default_openmp and min(default_values) <= 1: print( "Cython OpenMP readiness: FAIL. GSTools-Cython reports one " "default thread." @@ -162,7 +171,10 @@ def main(): print(f"gstools_cython: {package_version('gstools_cython')}") print(f"gstools_core: {package_version('gstools_core')}") - cython_ready = check_cython_parallel(verbose=args.verbose) + cython_ready = check_cython_parallel( + verbose=args.verbose, + require_default_openmp=args.require_default_openmp, + ) rust_ready = True if args.cython_only else check_rust_backend(args.threads) return 0 if cython_ready and rust_ready else 1