diff --git a/.gitignore b/.gitignore index 7f6619314..b141b3217 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,10 @@ pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ +.asv/ +benchmarks/env/ +benchmarks/results/ +benchmarks/html/ .coverage .coverage.* .cache diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 000000000..a1b68f811 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,35 @@ +# ASV benchmarks + +These benchmarks measure `codecarbon` performance across git commits. + +Run a local smoke check: + +```bash +uvx asv --config benchmarks/asv.conf.json run --quick --show-stderr +``` + + +Run across releases: + +```bash +# Get the commits of the releases +./benchmarks/get_release_hashes.sh +# Run only against those commits +uvx asv --config benchmarks/asv.conf.json run HASHFILE:release_hashes.txt --show-stderr +``` + +Then when you see a big jump between releases, go and run the intermediate commits to see the diff: + +```bash +# Imagine the jump is between 3.2.1 and 3.2.2 +uvx asv --config benchmarks/asv.conf.json run v3.2.1..v3.2.2 --show-stderr +``` + +Build and preview the report: + +```bash +uvx asv --config benchmarks/asv.conf.json publish +uvx asv --config benchmarks/asv.conf.json preview +``` + +Generated ASV environments, results, and HTML are written under `benchmarks/`. diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json new file mode 100644 index 000000000..3a0c7a163 --- /dev/null +++ b/benchmarks/asv.conf.json @@ -0,0 +1,17 @@ +{ + "version": 1, + "project": "CodeCarbon", + "project_url": "https://codecarbon.io/", + "repo": "..", + "build_command": [ + "python -m pip wheel --no-deps --no-build-isolation -w {build_cache_dir} {build_dir}" + ], + "branches": ["master"], + "pythons": ["3.11"], + "benchmark_dir": "benchmarks", + "env_dir": "env", + "results_dir": "results", + "html_dir": "html", + "show_commit_url": "https://github.com/mlco2/codecarbon/commit/", + "environment_type": "virtualenv" +} diff --git a/benchmarks/benchmarks/__init__.py b/benchmarks/benchmarks/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/benchmarks/benchmarks/bench_emissions_tracker.py b/benchmarks/benchmarks/bench_emissions_tracker.py new file mode 100644 index 000000000..2448b7221 --- /dev/null +++ b/benchmarks/benchmarks/bench_emissions_tracker.py @@ -0,0 +1,63 @@ +TRACKER_SETUP = """ +import os +from tempfile import TemporaryDirectory + +from codecarbon import OfflineEmissionsTracker + +tmpdir = TemporaryDirectory() +home = TemporaryDirectory() +os.environ["HOME"] = home.name +os.chdir(tmpdir.name) +tracker = OfflineEmissionsTracker( + country_iso_code="FRA", + output_dir=tmpdir.name, + output_file="emissions.csv", + measure_power_secs=3600, + save_to_file=False, + allow_multiple_runs=True, + log_level="critical", +) +""" + + +class TrackerStartTime: + """Measure the time it takes to start the tracker.""" + + number = 1 + repeat = 5 + timeout = 120 + + def timeraw_start(self): + return "tracker.start()", TRACKER_SETUP + + +class TrackerStopTime: + """Measure the time it takes to stop the tracker.""" + + number = 1 + repeat = 5 + timeout = 120 + + def timeraw_stop(self): + return "tracker.stop()", TRACKER_SETUP + "\ntracker.start()\n" + + +class TrackerLifecycleTime: + """Measure the time it takes to start and stop the tracker. + + This is useful to see if there is some overhead between the start and stop + of the tracker. + """ + + number = 1 + repeat = 5 + timeout = 120 + + def timeraw_lifecycle(self): + return ( + """ +tracker.start() +tracker.stop() +""", + TRACKER_SETUP, + ) diff --git a/benchmarks/get_release_hashes.sh b/benchmarks/get_release_hashes.sh new file mode 100755 index 000000000..ddfc185a4 --- /dev/null +++ b/benchmarks/get_release_hashes.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" + +# Get all release hashes from git tags and write them to a file +# The first tag to include is the v3.0.4 because it is the first +# that uses hatchling.build as the build backend. +git tag --list --sort=v:refname \ +| grep -Ev '(a|b|rc|alpha|beta)' \ +| sed -n '/^v3\.0\.4$/,$p' \ +| while read -r tag; do + git rev-list -n1 "$tag" +done > "${SCRIPT_DIR}/release_hashes.txt" \ No newline at end of file diff --git a/benchmarks/release_hashes.txt b/benchmarks/release_hashes.txt new file mode 100644 index 000000000..e9a458a4f --- /dev/null +++ b/benchmarks/release_hashes.txt @@ -0,0 +1,17 @@ +a30fe4d0f66f0db046bfb22007c9d5464e5be544 +6546c74be8da855266badf2796aa7ed1b627e267 +0c33c7cefa805fe652f4ccdf25f3db01ad11f506 +bcf998c16e05fc895ed0b8832cbd6eb0fb997824 +3d5db3b941d3c2ee9fd35320ae29a29d7c32c563 +329d2a333512343161bf7b701b671ba7212998a7 +f4191bd1cedfe35a25df7140231afc4ee39eaf43 +f44672f745ad5ebde623e37551bfa74c42e79ef0 +78baaa73699abb7d6439cd9e11913393c5dfb050 +37f7b7c9d1370a402aa8f26b21bdfdce4bcac187 +efedba874bfc3feeeaca600f6aef9cb034e5abb6 +f5fa2905ecf8b0f44cd376387f77b4ccea4bd1c0 +31b4dacc65185bb985b6a6fed2b945ac8961b164 +7ea496584e3c0ff19e84e35e0db22cfed4872ad4 +7f099366452510af5f166e119b8bbe97b94a9c08 +08dcc1b33097363349fdf759e54fa07d7697624c +75a74beeaf63d8c040171442f1fa80de9e09745d