diff --git a/Cargo.lock b/Cargo.lock index 68a124b8b..3833e7bb9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloca" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" +dependencies = [ + "cc", +] + [[package]] name = "android_system_properties" version = "0.1.5" @@ -35,6 +44,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anstream" version = "1.0.0" @@ -186,6 +201,12 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.4.0" @@ -226,11 +247,38 @@ dependencies = [ "windows-link", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "clang-sys" -version = "1.8.1" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a" dependencies = [ "glob", "libc", @@ -363,6 +411,72 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "criterion" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" +dependencies = [ + "alloca", + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools 0.13.0", + "num-traits", + "oorandom", + "page_size", + "plotters", + "rayon", + "regex", + "serde", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" +dependencies = [ + "cast", + "itertools 0.13.0", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "csv" version = "1.4.0" @@ -623,6 +737,17 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -936,6 +1061,7 @@ dependencies = [ "clap-markdown", "colored 3.1.1", "context_manager", + "criterion", "csv", "derive_more", "dirs", @@ -1033,6 +1159,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "optfield" version = "0.4.0" @@ -1059,6 +1191,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "petgraph" version = "0.8.3" @@ -1135,6 +1277,34 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "predicates" version = "3.1.4" @@ -1205,6 +1375,26 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_users" version = "0.5.2" @@ -1337,6 +1527,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "semver" version = "1.0.28" @@ -1373,6 +1572,19 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + [[package]] name = "serde_spanned" version = "1.1.1" @@ -1517,6 +1729,16 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "toml" version = "1.1.4+spec-1.1.0" @@ -1628,6 +1850,16 @@ dependencies = [ "libc", ] +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -1679,6 +1911,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "web-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "which" version = "4.4.2" @@ -1707,6 +1949,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" @@ -1923,3 +2174,29 @@ dependencies = [ "arraydeque", "hashlink", ] + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml index f76e169ed..f7c2b73f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,6 +48,7 @@ context_manager = "0.1.3" [dev-dependencies] assert_cmd = "2.2.2" colored = "3.0.0" +criterion = "0.8.2" logtest = "2.0.0" map-macro = "0.3.0" ordered-float = "5.1.0" @@ -85,3 +86,10 @@ implicit_hasher = "allow" cast_lossless = "allow" return_self_not_must_use = "allow" from_iter_instead_of_collect = "allow" + +[[bench]] +name = "examples" +harness = false + +[features] +bench = [] diff --git a/benches/examples.rs b/benches/examples.rs new file mode 100644 index 000000000..4762fb6b8 --- /dev/null +++ b/benches/examples.rs @@ -0,0 +1,35 @@ +use criterion::{Criterion, criterion_group, criterion_main}; +use muse2::cli::RunOpts; +use muse2::cli::example::handle_example_run_command; +use muse2::example::get_example_names; +use std::hint::black_box; + +/// Benchmark the example run command for all examples +fn criterion_benchmark(c: &mut Criterion) { + let example_names = get_example_names(); + let output_root = tempfile::TempDir::new() + .expect("Failed to create temporary output directory for benchmarks"); + let output_dir = output_root.path().join("muse2-example-bench-output"); + let options = RunOpts { + output_dir: Some(output_dir), + overwrite: true, + debug_model: false, + no_copy_input_files: false, + }; + let mut group = c.benchmark_group("example_run"); + group + .noise_threshold(0.05) // Set a noise threshold of 5% + .sample_size(20); // Set the sample size to 20 iterations + + for name in example_names { + group.bench_function(format!("{name} example"), |b| { + b.iter(|| { + handle_example_run_command(black_box(name), black_box(false), black_box(&options)) + .unwrap_or(()); + }); + }); + } +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); diff --git a/docs/developer_guide/coding.md b/docs/developer_guide/coding.md index 4496219be..bb55ac0b2 100644 --- a/docs/developer_guide/coding.md +++ b/docs/developer_guide/coding.md @@ -74,3 +74,57 @@ just coverage --open This will generate a report in HTML format showing which lines are not currently covered by tests and open it in your default browser. + +## Working with the benchmarking framework + +MUSE2 includes a benchmarking framework to measure the performance of various components using the +[Criterion.rs] library. To run all the available benchmarks, use the following command: + +```sh +cargo bench --features bench +``` + +The `--features bench` flag disables some aspects of the code that are incompatible with the +benchmarking, like singleton objects that can be only instantiated once and throw an error otherwise. +See `rust` documentation for more information about [features]. + +You can also run specific benchmarks by specifying their names: + +```sh +cargo bench --features bench --bench +``` + +For example, to run the `examples` benchmark, you would use: + +```sh +cargo bench --features bench --bench examples +``` + +If run more than once, the benchmarking framework will compare the results of the current run with the +previous run and report the differences. This is useful for checking whether a change has +introduced a performance regression. + +[Criterion.rs] automatically generates an HTML report that can be found in the +`target/criterion/report` directory. You can open it with your web browser to see the results and +navigate through the different benchmarks. + +### Adding new benchmarks + +To add a new benchmark, i.e. a collection of related performance tests, create a new file in the +`benches` directory with the name of your benchmark. For example, to add a benchmark for the `foo` +functionality, you would create a file called `benches/foo.rs`. Then `foo` will need to be added to +`cargo.toml` under the `[bench]` section, like so: + +```toml +[[bench]] +name = "foo" +harness = false +``` + +The `harness = false` line is necessary because Criterion.rs provides its own test harness, which +is incompatible with the default Rust test harness. Within the new benchmark file, you can then add +your performance tests using the Criterion.rs API. Check `benches/examples.rs` for an example of how +to do this. More information about writing benchmarks can be found in the [Criterion.rs] documentation. + +[Criterion.rs]: https://bheisler.github.io/criterion.rs/book/index.html +[features]: https://doc.rust-lang.org/cargo/reference/features.html diff --git a/src/cli.rs b/src/cli.rs index 416b602fc..67f1645a8 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -182,8 +182,11 @@ pub fn handle_run_command(model_path: &Path, opts: &RunOpts) -> Result<()> { .context("Failed to copy input files to output directory.")?; } // Initialise program logger - log::init(&settings.log_level, Some(output_path)).context("Failed to initialise logging.")?; - + if !cfg!(feature = "bench") { + // Don't initialise logging when running bench, as it will interfere with the process + log::init(&settings.log_level, Some(output_path)) + .context("Failed to initialise logging.")?; + } info!("Starting MUSE2 v{}", env!("CARGO_PKG_VERSION")); // Load the model to run diff --git a/src/model/parameters.rs b/src/model/parameters.rs index ef542d3cb..284a913d7 100644 --- a/src/model/parameters.rs +++ b/src/model/parameters.rs @@ -53,7 +53,7 @@ pub fn dangerous_model_options_enabled() -> bool { fn set_dangerous_model_options_flag(enabled: bool) { let result = DANGEROUS_OPTIONS_ENABLED.set(enabled); if result.is_err() { - if cfg!(test) { + if cfg!(test) || cfg!(feature = "bench") { // Sanity check assert_eq!(enabled, dangerous_model_options_enabled()); } else {