From 9e7244b81bea553cd376f610409cc91c4ea462de Mon Sep 17 00:00:00 2001 From: Mark Dittmer Date: Thu, 21 May 2026 19:05:00 +0000 Subject: [PATCH] SQUASH ME: Add vendored dependencies; might have been required by previous iterations; oops gherrit-pr-id: Gn2jbsncbrry4u7ogkmwjhmuupzdeh4fh --- .../crossbeam-deque/.cargo-checksum.json | 1 + .../crossbeam-deque/.cargo_vcs_info.json | 6 + anneal/v2/vendor/crossbeam-deque/CHANGELOG.md | 141 + anneal/v2/vendor/crossbeam-deque/Cargo.toml | 92 + .../v2/vendor/crossbeam-deque/Cargo.toml.orig | 34 + .../v2/vendor/crossbeam-deque/LICENSE-APACHE | 201 + anneal/v2/vendor/crossbeam-deque/LICENSE-MIT | 27 + anneal/v2/vendor/crossbeam-deque/README.md | 46 + anneal/v2/vendor/crossbeam-deque/src/deque.rs | 2216 ++++++++++ anneal/v2/vendor/crossbeam-deque/src/lib.rs | 106 + .../v2/vendor/crossbeam-deque/tests/fifo.rs | 357 ++ .../vendor/crossbeam-deque/tests/injector.rs | 391 ++ .../v2/vendor/crossbeam-deque/tests/lifo.rs | 359 ++ .../v2/vendor/crossbeam-deque/tests/steal.rs | 212 + .../crossbeam-epoch/.cargo-checksum.json | 1 + .../crossbeam-epoch/.cargo_vcs_info.json | 6 + anneal/v2/vendor/crossbeam-epoch/CHANGELOG.md | 204 + anneal/v2/vendor/crossbeam-epoch/Cargo.lock | 457 +++ anneal/v2/vendor/crossbeam-epoch/Cargo.toml | 57 + .../v2/vendor/crossbeam-epoch/Cargo.toml.orig | 56 + .../v2/vendor/crossbeam-epoch/LICENSE-APACHE | 201 + anneal/v2/vendor/crossbeam-epoch/LICENSE-MIT | 27 + anneal/v2/vendor/crossbeam-epoch/README.md | 53 + .../vendor/crossbeam-epoch/benches/defer.rs | 69 + .../vendor/crossbeam-epoch/benches/flush.rs | 52 + .../v2/vendor/crossbeam-epoch/benches/pin.rs | 31 + .../crossbeam-epoch/examples/sanitize.rs | 66 + .../v2/vendor/crossbeam-epoch/src/atomic.rs | 1702 ++++++++ .../vendor/crossbeam-epoch/src/collector.rs | 464 +++ .../v2/vendor/crossbeam-epoch/src/default.rs | 93 + .../v2/vendor/crossbeam-epoch/src/deferred.rs | 146 + anneal/v2/vendor/crossbeam-epoch/src/epoch.rs | 132 + anneal/v2/vendor/crossbeam-epoch/src/guard.rs | 523 +++ .../v2/vendor/crossbeam-epoch/src/internal.rs | 600 +++ anneal/v2/vendor/crossbeam-epoch/src/lib.rs | 166 + .../vendor/crossbeam-epoch/src/sync/list.rs | 487 +++ .../v2/vendor/crossbeam-epoch/src/sync/mod.rs | 7 + .../crossbeam-epoch/src/sync/once_lock.rs | 88 + .../vendor/crossbeam-epoch/src/sync/queue.rs | 468 +++ .../v2/vendor/crossbeam-epoch/tests/loom.rs | 157 + .../crossbeam-utils/.cargo-checksum.json | 1 + .../crossbeam-utils/.cargo_vcs_info.json | 6 + anneal/v2/vendor/crossbeam-utils/CHANGELOG.md | 243 ++ anneal/v2/vendor/crossbeam-utils/Cargo.toml | 101 + .../v2/vendor/crossbeam-utils/Cargo.toml.orig | 46 + .../v2/vendor/crossbeam-utils/LICENSE-APACHE | 201 + anneal/v2/vendor/crossbeam-utils/LICENSE-MIT | 27 + anneal/v2/vendor/crossbeam-utils/README.md | 73 + .../crossbeam-utils/benches/atomic_cell.rs | 156 + .../v2/vendor/crossbeam-utils/build-common.rs | 13 + anneal/v2/vendor/crossbeam-utils/build.rs | 48 + anneal/v2/vendor/crossbeam-utils/no_atomic.rs | 9 + .../crossbeam-utils/src/atomic/atomic_cell.rs | 1182 ++++++ .../crossbeam-utils/src/atomic/consume.rs | 111 + .../vendor/crossbeam-utils/src/atomic/mod.rs | 32 + .../crossbeam-utils/src/atomic/seq_lock.rs | 112 + .../src/atomic/seq_lock_wide.rs | 155 + .../v2/vendor/crossbeam-utils/src/backoff.rs | 287 ++ .../crossbeam-utils/src/cache_padded.rs | 217 + anneal/v2/vendor/crossbeam-utils/src/lib.rs | 110 + .../v2/vendor/crossbeam-utils/src/sync/mod.rs | 17 + .../crossbeam-utils/src/sync/once_lock.rs | 88 + .../vendor/crossbeam-utils/src/sync/parker.rs | 415 ++ .../crossbeam-utils/src/sync/sharded_lock.rs | 638 +++ .../crossbeam-utils/src/sync/wait_group.rs | 145 + .../v2/vendor/crossbeam-utils/src/thread.rs | 611 +++ .../crossbeam-utils/tests/atomic_cell.rs | 374 ++ .../crossbeam-utils/tests/cache_padded.rs | 113 + .../v2/vendor/crossbeam-utils/tests/parker.rs | 41 + .../crossbeam-utils/tests/sharded_lock.rs | 252 ++ .../v2/vendor/crossbeam-utils/tests/thread.rs | 215 + .../crossbeam-utils/tests/wait_group.rs | 67 + anneal/v2/vendor/either/.cargo-checksum.json | 1 + anneal/v2/vendor/either/.cargo_vcs_info.json | 6 + .../v2/vendor/either/.github/workflows/ci.yml | 90 + anneal/v2/vendor/either/Cargo.lock | 89 + anneal/v2/vendor/either/Cargo.toml | 69 + anneal/v2/vendor/either/Cargo.toml.orig | 39 + anneal/v2/vendor/either/LICENSE-APACHE | 201 + anneal/v2/vendor/either/LICENSE-MIT | 25 + anneal/v2/vendor/either/README-crates.io.md | 10 + anneal/v2/vendor/either/README.rst | 220 + anneal/v2/vendor/either/src/into_either.rs | 64 + anneal/v2/vendor/either/src/iterator.rs | 315 ++ anneal/v2/vendor/either/src/lib.rs | 1826 +++++++++ anneal/v2/vendor/either/src/serde_untagged.rs | 69 + .../either/src/serde_untagged_optional.rs | 74 + .../v2/vendor/rayon-core/.cargo-checksum.json | 1 + .../v2/vendor/rayon-core/.cargo_vcs_info.json | 6 + anneal/v2/vendor/rayon-core/Cargo.lock | 309 ++ anneal/v2/vendor/rayon-core/Cargo.toml | 93 + anneal/v2/vendor/rayon-core/Cargo.toml.orig | 35 + anneal/v2/vendor/rayon-core/LICENSE-APACHE | 201 + anneal/v2/vendor/rayon-core/LICENSE-MIT | 25 + anneal/v2/vendor/rayon-core/README.md | 11 + anneal/v2/vendor/rayon-core/build.rs | 7 + .../v2/vendor/rayon-core/src/broadcast/mod.rs | 150 + .../vendor/rayon-core/src/broadcast/test.rs | 263 ++ .../vendor/rayon-core/src/compile_fail/mod.rs | 7 + .../src/compile_fail/quicksort_race1.rs | 28 + .../src/compile_fail/quicksort_race2.rs | 28 + .../src/compile_fail/quicksort_race3.rs | 28 + .../rayon-core/src/compile_fail/rc_return.rs | 17 + .../rayon-core/src/compile_fail/rc_upvar.rs | 9 + .../src/compile_fail/scope_join_bad.rs | 24 + anneal/v2/vendor/rayon-core/src/job.rs | 270 ++ anneal/v2/vendor/rayon-core/src/join/mod.rs | 186 + anneal/v2/vendor/rayon-core/src/join/test.rs | 150 + anneal/v2/vendor/rayon-core/src/latch.rs | 457 +++ anneal/v2/vendor/rayon-core/src/lib.rs | 864 ++++ anneal/v2/vendor/rayon-core/src/private.rs | 26 + anneal/v2/vendor/rayon-core/src/registry.rs | 1002 +++++ anneal/v2/vendor/rayon-core/src/scope/mod.rs | 773 ++++ anneal/v2/vendor/rayon-core/src/scope/test.rs | 621 +++ .../v2/vendor/rayon-core/src/sleep/README.md | 219 + .../vendor/rayon-core/src/sleep/counters.rs | 273 ++ anneal/v2/vendor/rayon-core/src/sleep/mod.rs | 324 ++ anneal/v2/vendor/rayon-core/src/spawn/mod.rs | 163 + anneal/v2/vendor/rayon-core/src/spawn/test.rs | 255 ++ anneal/v2/vendor/rayon-core/src/test.rs | 200 + .../vendor/rayon-core/src/thread_pool/mod.rs | 502 +++ .../vendor/rayon-core/src/thread_pool/test.rs | 418 ++ anneal/v2/vendor/rayon-core/src/unwind.rs | 31 + .../rayon-core/tests/double_init_fail.rs | 15 + .../rayon-core/tests/init_zero_threads.rs | 10 + .../v2/vendor/rayon-core/tests/scope_join.rs | 45 + .../rayon-core/tests/scoped_threadpool.rs | 99 + .../vendor/rayon-core/tests/simple_panic.rs | 7 + .../rayon-core/tests/stack_overflow_crash.rs | 97 + .../rayon-core/tests/use_current_thread.rs | 57 + anneal/v2/vendor/rayon/.cargo-checksum.json | 1 + anneal/v2/vendor/rayon/.cargo_vcs_info.json | 6 + anneal/v2/vendor/rayon/Cargo.lock | 297 ++ anneal/v2/vendor/rayon/Cargo.toml | 130 + anneal/v2/vendor/rayon/Cargo.toml.orig | 57 + anneal/v2/vendor/rayon/FAQ.md | 213 + anneal/v2/vendor/rayon/LICENSE-APACHE | 201 + anneal/v2/vendor/rayon/LICENSE-MIT | 25 + anneal/v2/vendor/rayon/README.md | 150 + anneal/v2/vendor/rayon/RELEASES.md | 929 +++++ anneal/v2/vendor/rayon/src/array.rs | 85 + .../rayon/src/collections/binary_heap.rs | 129 + .../vendor/rayon/src/collections/btree_map.rs | 66 + .../vendor/rayon/src/collections/btree_set.rs | 52 + .../vendor/rayon/src/collections/hash_map.rs | 93 + .../vendor/rayon/src/collections/hash_set.rs | 79 + .../rayon/src/collections/linked_list.rs | 66 + anneal/v2/vendor/rayon/src/collections/mod.rs | 84 + .../vendor/rayon/src/collections/vec_deque.rs | 159 + .../cannot_collect_filtermap_data.rs | 14 + .../compile_fail/cannot_zip_filtered_data.rs | 14 + .../rayon/src/compile_fail/cell_par_iter.rs | 13 + .../v2/vendor/rayon/src/compile_fail/mod.rs | 7 + .../vendor/rayon/src/compile_fail/must_use.rs | 71 + .../src/compile_fail/no_send_par_iter.rs | 58 + .../rayon/src/compile_fail/rc_par_iter.rs | 15 + anneal/v2/vendor/rayon/src/delegate.rs | 109 + anneal/v2/vendor/rayon/src/iter/blocks.rs | 129 + anneal/v2/vendor/rayon/src/iter/chain.rs | 258 ++ anneal/v2/vendor/rayon/src/iter/chunks.rs | 216 + anneal/v2/vendor/rayon/src/iter/cloned.rs | 219 + .../vendor/rayon/src/iter/collect/consumer.rs | 185 + .../v2/vendor/rayon/src/iter/collect/mod.rs | 114 + .../v2/vendor/rayon/src/iter/collect/test.rs | 368 ++ anneal/v2/vendor/rayon/src/iter/copied.rs | 219 + anneal/v2/vendor/rayon/src/iter/empty.rs | 108 + anneal/v2/vendor/rayon/src/iter/enumerate.rs | 128 + anneal/v2/vendor/rayon/src/iter/extend.rs | 619 +++ anneal/v2/vendor/rayon/src/iter/filter.rs | 137 + anneal/v2/vendor/rayon/src/iter/filter_map.rs | 141 + anneal/v2/vendor/rayon/src/iter/find.rs | 120 + .../rayon/src/iter/find_first_last/mod.rs | 230 ++ .../rayon/src/iter/find_first_last/test.rs | 102 + anneal/v2/vendor/rayon/src/iter/flat_map.rs | 153 + .../v2/vendor/rayon/src/iter/flat_map_iter.rs | 145 + anneal/v2/vendor/rayon/src/iter/flatten.rs | 134 + .../v2/vendor/rayon/src/iter/flatten_iter.rs | 124 + anneal/v2/vendor/rayon/src/iter/fold.rs | 289 ++ .../v2/vendor/rayon/src/iter/fold_chunks.rs | 224 + .../vendor/rayon/src/iter/fold_chunks_with.rs | 220 + anneal/v2/vendor/rayon/src/iter/for_each.rs | 77 + .../v2/vendor/rayon/src/iter/from_par_iter.rs | 280 ++ anneal/v2/vendor/rayon/src/iter/inspect.rs | 253 ++ anneal/v2/vendor/rayon/src/iter/interleave.rs | 326 ++ .../rayon/src/iter/interleave_shortest.rs | 80 + .../v2/vendor/rayon/src/iter/intersperse.rs | 401 ++ anneal/v2/vendor/rayon/src/iter/len.rs | 262 ++ anneal/v2/vendor/rayon/src/iter/map.rs | 255 ++ anneal/v2/vendor/rayon/src/iter/map_with.rs | 565 +++ anneal/v2/vendor/rayon/src/iter/mod.rs | 3628 +++++++++++++++++ anneal/v2/vendor/rayon/src/iter/multizip.rs | 335 ++ anneal/v2/vendor/rayon/src/iter/noop.rs | 59 + anneal/v2/vendor/rayon/src/iter/once.rs | 70 + anneal/v2/vendor/rayon/src/iter/panic_fuse.rs | 338 ++ anneal/v2/vendor/rayon/src/iter/par_bridge.rs | 157 + .../vendor/rayon/src/iter/plumbing/README.md | 315 ++ .../v2/vendor/rayon/src/iter/plumbing/mod.rs | 476 +++ anneal/v2/vendor/rayon/src/iter/positions.rs | 133 + anneal/v2/vendor/rayon/src/iter/product.rs | 114 + anneal/v2/vendor/rayon/src/iter/reduce.rs | 116 + anneal/v2/vendor/rayon/src/iter/repeat.rs | 295 ++ anneal/v2/vendor/rayon/src/iter/rev.rs | 119 + anneal/v2/vendor/rayon/src/iter/skip.rs | 93 + anneal/v2/vendor/rayon/src/iter/skip_any.rs | 140 + .../vendor/rayon/src/iter/skip_any_while.rs | 162 + anneal/v2/vendor/rayon/src/iter/splitter.rs | 172 + anneal/v2/vendor/rayon/src/iter/step_by.rs | 135 + anneal/v2/vendor/rayon/src/iter/sum.rs | 110 + anneal/v2/vendor/rayon/src/iter/take.rs | 84 + anneal/v2/vendor/rayon/src/iter/take_any.rs | 140 + .../vendor/rayon/src/iter/take_any_while.rs | 162 + anneal/v2/vendor/rayon/src/iter/test.rs | 2392 +++++++++++ anneal/v2/vendor/rayon/src/iter/try_fold.rs | 282 ++ anneal/v2/vendor/rayon/src/iter/try_reduce.rs | 131 + .../vendor/rayon/src/iter/try_reduce_with.rs | 132 + anneal/v2/vendor/rayon/src/iter/unzip.rs | 524 +++ anneal/v2/vendor/rayon/src/iter/update.rs | 323 ++ anneal/v2/vendor/rayon/src/iter/walk_tree.rs | 524 +++ anneal/v2/vendor/rayon/src/iter/while_some.rs | 150 + anneal/v2/vendor/rayon/src/iter/zip.rs | 153 + anneal/v2/vendor/rayon/src/iter/zip_eq.rs | 67 + anneal/v2/vendor/rayon/src/lib.rs | 156 + anneal/v2/vendor/rayon/src/math.rs | 25 + anneal/v2/vendor/rayon/src/option.rs | 197 + anneal/v2/vendor/rayon/src/par_either.rs | 74 + anneal/v2/vendor/rayon/src/prelude.rs | 17 + anneal/v2/vendor/rayon/src/private.rs | 26 + anneal/v2/vendor/rayon/src/range.rs | 474 +++ anneal/v2/vendor/rayon/src/range_inclusive.rs | 398 ++ anneal/v2/vendor/rayon/src/result.rs | 132 + anneal/v2/vendor/rayon/src/slice/chunk_by.rs | 239 ++ anneal/v2/vendor/rayon/src/slice/chunks.rs | 387 ++ anneal/v2/vendor/rayon/src/slice/mod.rs | 1173 ++++++ anneal/v2/vendor/rayon/src/slice/rchunks.rs | 385 ++ anneal/v2/vendor/rayon/src/slice/sort.rs | 1686 ++++++++ anneal/v2/vendor/rayon/src/slice/test.rs | 216 + anneal/v2/vendor/rayon/src/slice/windows.rs | 147 + anneal/v2/vendor/rayon/src/split_producer.rs | 152 + anneal/v2/vendor/rayon/src/str.rs | 1006 +++++ anneal/v2/vendor/rayon/src/string.rs | 48 + anneal/v2/vendor/rayon/src/vec.rs | 292 ++ anneal/v2/vendor/rayon/tests/chars.rs | 38 + anneal/v2/vendor/rayon/tests/clones.rs | 223 + anneal/v2/vendor/rayon/tests/collect.rs | 113 + anneal/v2/vendor/rayon/tests/cross-pool.rs | 22 + anneal/v2/vendor/rayon/tests/debug.rs | 234 ++ anneal/v2/vendor/rayon/tests/drain_vec.rs | 41 + anneal/v2/vendor/rayon/tests/intersperse.rs | 60 + .../v2/vendor/rayon/tests/issue671-unzip.rs | 17 + anneal/v2/vendor/rayon/tests/issue671.rs | 16 + anneal/v2/vendor/rayon/tests/iter_panic.rs | 53 + anneal/v2/vendor/rayon/tests/named-threads.rs | 25 + anneal/v2/vendor/rayon/tests/octillion.rs | 156 + .../rayon/tests/par_bridge_recursion.rs | 31 + .../vendor/rayon/tests/producer_split_at.rs | 393 ++ .../v2/vendor/rayon/tests/sort-panic-safe.rs | 164 + anneal/v2/vendor/rayon/tests/str.rs | 122 + 257 files changed, 59185 insertions(+) create mode 100644 anneal/v2/vendor/crossbeam-deque/.cargo-checksum.json create mode 100644 anneal/v2/vendor/crossbeam-deque/.cargo_vcs_info.json create mode 100644 anneal/v2/vendor/crossbeam-deque/CHANGELOG.md create mode 100644 anneal/v2/vendor/crossbeam-deque/Cargo.toml create mode 100644 anneal/v2/vendor/crossbeam-deque/Cargo.toml.orig create mode 100644 anneal/v2/vendor/crossbeam-deque/LICENSE-APACHE create mode 100644 anneal/v2/vendor/crossbeam-deque/LICENSE-MIT create mode 100644 anneal/v2/vendor/crossbeam-deque/README.md create mode 100644 anneal/v2/vendor/crossbeam-deque/src/deque.rs create mode 100644 anneal/v2/vendor/crossbeam-deque/src/lib.rs create mode 100644 anneal/v2/vendor/crossbeam-deque/tests/fifo.rs create mode 100644 anneal/v2/vendor/crossbeam-deque/tests/injector.rs create mode 100644 anneal/v2/vendor/crossbeam-deque/tests/lifo.rs create mode 100644 anneal/v2/vendor/crossbeam-deque/tests/steal.rs create mode 100644 anneal/v2/vendor/crossbeam-epoch/.cargo-checksum.json create mode 100644 anneal/v2/vendor/crossbeam-epoch/.cargo_vcs_info.json create mode 100644 anneal/v2/vendor/crossbeam-epoch/CHANGELOG.md create mode 100644 anneal/v2/vendor/crossbeam-epoch/Cargo.lock create mode 100644 anneal/v2/vendor/crossbeam-epoch/Cargo.toml create mode 100644 anneal/v2/vendor/crossbeam-epoch/Cargo.toml.orig create mode 100644 anneal/v2/vendor/crossbeam-epoch/LICENSE-APACHE create mode 100644 anneal/v2/vendor/crossbeam-epoch/LICENSE-MIT create mode 100644 anneal/v2/vendor/crossbeam-epoch/README.md create mode 100644 anneal/v2/vendor/crossbeam-epoch/benches/defer.rs create mode 100644 anneal/v2/vendor/crossbeam-epoch/benches/flush.rs create mode 100644 anneal/v2/vendor/crossbeam-epoch/benches/pin.rs create mode 100644 anneal/v2/vendor/crossbeam-epoch/examples/sanitize.rs create mode 100644 anneal/v2/vendor/crossbeam-epoch/src/atomic.rs create mode 100644 anneal/v2/vendor/crossbeam-epoch/src/collector.rs create mode 100644 anneal/v2/vendor/crossbeam-epoch/src/default.rs create mode 100644 anneal/v2/vendor/crossbeam-epoch/src/deferred.rs create mode 100644 anneal/v2/vendor/crossbeam-epoch/src/epoch.rs create mode 100644 anneal/v2/vendor/crossbeam-epoch/src/guard.rs create mode 100644 anneal/v2/vendor/crossbeam-epoch/src/internal.rs create mode 100644 anneal/v2/vendor/crossbeam-epoch/src/lib.rs create mode 100644 anneal/v2/vendor/crossbeam-epoch/src/sync/list.rs create mode 100644 anneal/v2/vendor/crossbeam-epoch/src/sync/mod.rs create mode 100644 anneal/v2/vendor/crossbeam-epoch/src/sync/once_lock.rs create mode 100644 anneal/v2/vendor/crossbeam-epoch/src/sync/queue.rs create mode 100644 anneal/v2/vendor/crossbeam-epoch/tests/loom.rs create mode 100644 anneal/v2/vendor/crossbeam-utils/.cargo-checksum.json create mode 100644 anneal/v2/vendor/crossbeam-utils/.cargo_vcs_info.json create mode 100644 anneal/v2/vendor/crossbeam-utils/CHANGELOG.md create mode 100644 anneal/v2/vendor/crossbeam-utils/Cargo.toml create mode 100644 anneal/v2/vendor/crossbeam-utils/Cargo.toml.orig create mode 100644 anneal/v2/vendor/crossbeam-utils/LICENSE-APACHE create mode 100644 anneal/v2/vendor/crossbeam-utils/LICENSE-MIT create mode 100644 anneal/v2/vendor/crossbeam-utils/README.md create mode 100644 anneal/v2/vendor/crossbeam-utils/benches/atomic_cell.rs create mode 100644 anneal/v2/vendor/crossbeam-utils/build-common.rs create mode 100644 anneal/v2/vendor/crossbeam-utils/build.rs create mode 100644 anneal/v2/vendor/crossbeam-utils/no_atomic.rs create mode 100644 anneal/v2/vendor/crossbeam-utils/src/atomic/atomic_cell.rs create mode 100644 anneal/v2/vendor/crossbeam-utils/src/atomic/consume.rs create mode 100644 anneal/v2/vendor/crossbeam-utils/src/atomic/mod.rs create mode 100644 anneal/v2/vendor/crossbeam-utils/src/atomic/seq_lock.rs create mode 100644 anneal/v2/vendor/crossbeam-utils/src/atomic/seq_lock_wide.rs create mode 100644 anneal/v2/vendor/crossbeam-utils/src/backoff.rs create mode 100644 anneal/v2/vendor/crossbeam-utils/src/cache_padded.rs create mode 100644 anneal/v2/vendor/crossbeam-utils/src/lib.rs create mode 100644 anneal/v2/vendor/crossbeam-utils/src/sync/mod.rs create mode 100644 anneal/v2/vendor/crossbeam-utils/src/sync/once_lock.rs create mode 100644 anneal/v2/vendor/crossbeam-utils/src/sync/parker.rs create mode 100644 anneal/v2/vendor/crossbeam-utils/src/sync/sharded_lock.rs create mode 100644 anneal/v2/vendor/crossbeam-utils/src/sync/wait_group.rs create mode 100644 anneal/v2/vendor/crossbeam-utils/src/thread.rs create mode 100644 anneal/v2/vendor/crossbeam-utils/tests/atomic_cell.rs create mode 100644 anneal/v2/vendor/crossbeam-utils/tests/cache_padded.rs create mode 100644 anneal/v2/vendor/crossbeam-utils/tests/parker.rs create mode 100644 anneal/v2/vendor/crossbeam-utils/tests/sharded_lock.rs create mode 100644 anneal/v2/vendor/crossbeam-utils/tests/thread.rs create mode 100644 anneal/v2/vendor/crossbeam-utils/tests/wait_group.rs create mode 100644 anneal/v2/vendor/either/.cargo-checksum.json create mode 100644 anneal/v2/vendor/either/.cargo_vcs_info.json create mode 100644 anneal/v2/vendor/either/.github/workflows/ci.yml create mode 100644 anneal/v2/vendor/either/Cargo.lock create mode 100644 anneal/v2/vendor/either/Cargo.toml create mode 100644 anneal/v2/vendor/either/Cargo.toml.orig create mode 100644 anneal/v2/vendor/either/LICENSE-APACHE create mode 100644 anneal/v2/vendor/either/LICENSE-MIT create mode 100644 anneal/v2/vendor/either/README-crates.io.md create mode 100644 anneal/v2/vendor/either/README.rst create mode 100644 anneal/v2/vendor/either/src/into_either.rs create mode 100644 anneal/v2/vendor/either/src/iterator.rs create mode 100644 anneal/v2/vendor/either/src/lib.rs create mode 100644 anneal/v2/vendor/either/src/serde_untagged.rs create mode 100644 anneal/v2/vendor/either/src/serde_untagged_optional.rs create mode 100644 anneal/v2/vendor/rayon-core/.cargo-checksum.json create mode 100644 anneal/v2/vendor/rayon-core/.cargo_vcs_info.json create mode 100644 anneal/v2/vendor/rayon-core/Cargo.lock create mode 100644 anneal/v2/vendor/rayon-core/Cargo.toml create mode 100644 anneal/v2/vendor/rayon-core/Cargo.toml.orig create mode 100644 anneal/v2/vendor/rayon-core/LICENSE-APACHE create mode 100644 anneal/v2/vendor/rayon-core/LICENSE-MIT create mode 100644 anneal/v2/vendor/rayon-core/README.md create mode 100644 anneal/v2/vendor/rayon-core/build.rs create mode 100644 anneal/v2/vendor/rayon-core/src/broadcast/mod.rs create mode 100644 anneal/v2/vendor/rayon-core/src/broadcast/test.rs create mode 100644 anneal/v2/vendor/rayon-core/src/compile_fail/mod.rs create mode 100644 anneal/v2/vendor/rayon-core/src/compile_fail/quicksort_race1.rs create mode 100644 anneal/v2/vendor/rayon-core/src/compile_fail/quicksort_race2.rs create mode 100644 anneal/v2/vendor/rayon-core/src/compile_fail/quicksort_race3.rs create mode 100644 anneal/v2/vendor/rayon-core/src/compile_fail/rc_return.rs create mode 100644 anneal/v2/vendor/rayon-core/src/compile_fail/rc_upvar.rs create mode 100644 anneal/v2/vendor/rayon-core/src/compile_fail/scope_join_bad.rs create mode 100644 anneal/v2/vendor/rayon-core/src/job.rs create mode 100644 anneal/v2/vendor/rayon-core/src/join/mod.rs create mode 100644 anneal/v2/vendor/rayon-core/src/join/test.rs create mode 100644 anneal/v2/vendor/rayon-core/src/latch.rs create mode 100644 anneal/v2/vendor/rayon-core/src/lib.rs create mode 100644 anneal/v2/vendor/rayon-core/src/private.rs create mode 100644 anneal/v2/vendor/rayon-core/src/registry.rs create mode 100644 anneal/v2/vendor/rayon-core/src/scope/mod.rs create mode 100644 anneal/v2/vendor/rayon-core/src/scope/test.rs create mode 100644 anneal/v2/vendor/rayon-core/src/sleep/README.md create mode 100644 anneal/v2/vendor/rayon-core/src/sleep/counters.rs create mode 100644 anneal/v2/vendor/rayon-core/src/sleep/mod.rs create mode 100644 anneal/v2/vendor/rayon-core/src/spawn/mod.rs create mode 100644 anneal/v2/vendor/rayon-core/src/spawn/test.rs create mode 100644 anneal/v2/vendor/rayon-core/src/test.rs create mode 100644 anneal/v2/vendor/rayon-core/src/thread_pool/mod.rs create mode 100644 anneal/v2/vendor/rayon-core/src/thread_pool/test.rs create mode 100644 anneal/v2/vendor/rayon-core/src/unwind.rs create mode 100644 anneal/v2/vendor/rayon-core/tests/double_init_fail.rs create mode 100644 anneal/v2/vendor/rayon-core/tests/init_zero_threads.rs create mode 100644 anneal/v2/vendor/rayon-core/tests/scope_join.rs create mode 100644 anneal/v2/vendor/rayon-core/tests/scoped_threadpool.rs create mode 100644 anneal/v2/vendor/rayon-core/tests/simple_panic.rs create mode 100644 anneal/v2/vendor/rayon-core/tests/stack_overflow_crash.rs create mode 100644 anneal/v2/vendor/rayon-core/tests/use_current_thread.rs create mode 100644 anneal/v2/vendor/rayon/.cargo-checksum.json create mode 100644 anneal/v2/vendor/rayon/.cargo_vcs_info.json create mode 100644 anneal/v2/vendor/rayon/Cargo.lock create mode 100644 anneal/v2/vendor/rayon/Cargo.toml create mode 100644 anneal/v2/vendor/rayon/Cargo.toml.orig create mode 100644 anneal/v2/vendor/rayon/FAQ.md create mode 100644 anneal/v2/vendor/rayon/LICENSE-APACHE create mode 100644 anneal/v2/vendor/rayon/LICENSE-MIT create mode 100644 anneal/v2/vendor/rayon/README.md create mode 100644 anneal/v2/vendor/rayon/RELEASES.md create mode 100644 anneal/v2/vendor/rayon/src/array.rs create mode 100644 anneal/v2/vendor/rayon/src/collections/binary_heap.rs create mode 100644 anneal/v2/vendor/rayon/src/collections/btree_map.rs create mode 100644 anneal/v2/vendor/rayon/src/collections/btree_set.rs create mode 100644 anneal/v2/vendor/rayon/src/collections/hash_map.rs create mode 100644 anneal/v2/vendor/rayon/src/collections/hash_set.rs create mode 100644 anneal/v2/vendor/rayon/src/collections/linked_list.rs create mode 100644 anneal/v2/vendor/rayon/src/collections/mod.rs create mode 100644 anneal/v2/vendor/rayon/src/collections/vec_deque.rs create mode 100644 anneal/v2/vendor/rayon/src/compile_fail/cannot_collect_filtermap_data.rs create mode 100644 anneal/v2/vendor/rayon/src/compile_fail/cannot_zip_filtered_data.rs create mode 100644 anneal/v2/vendor/rayon/src/compile_fail/cell_par_iter.rs create mode 100644 anneal/v2/vendor/rayon/src/compile_fail/mod.rs create mode 100644 anneal/v2/vendor/rayon/src/compile_fail/must_use.rs create mode 100644 anneal/v2/vendor/rayon/src/compile_fail/no_send_par_iter.rs create mode 100644 anneal/v2/vendor/rayon/src/compile_fail/rc_par_iter.rs create mode 100644 anneal/v2/vendor/rayon/src/delegate.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/blocks.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/chain.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/chunks.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/cloned.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/collect/consumer.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/collect/mod.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/collect/test.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/copied.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/empty.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/enumerate.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/extend.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/filter.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/filter_map.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/find.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/find_first_last/mod.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/find_first_last/test.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/flat_map.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/flat_map_iter.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/flatten.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/flatten_iter.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/fold.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/fold_chunks.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/fold_chunks_with.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/for_each.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/from_par_iter.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/inspect.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/interleave.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/interleave_shortest.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/intersperse.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/len.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/map.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/map_with.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/mod.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/multizip.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/noop.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/once.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/panic_fuse.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/par_bridge.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/plumbing/README.md create mode 100644 anneal/v2/vendor/rayon/src/iter/plumbing/mod.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/positions.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/product.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/reduce.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/repeat.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/rev.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/skip.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/skip_any.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/skip_any_while.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/splitter.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/step_by.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/sum.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/take.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/take_any.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/take_any_while.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/test.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/try_fold.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/try_reduce.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/try_reduce_with.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/unzip.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/update.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/walk_tree.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/while_some.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/zip.rs create mode 100644 anneal/v2/vendor/rayon/src/iter/zip_eq.rs create mode 100644 anneal/v2/vendor/rayon/src/lib.rs create mode 100644 anneal/v2/vendor/rayon/src/math.rs create mode 100644 anneal/v2/vendor/rayon/src/option.rs create mode 100644 anneal/v2/vendor/rayon/src/par_either.rs create mode 100644 anneal/v2/vendor/rayon/src/prelude.rs create mode 100644 anneal/v2/vendor/rayon/src/private.rs create mode 100644 anneal/v2/vendor/rayon/src/range.rs create mode 100644 anneal/v2/vendor/rayon/src/range_inclusive.rs create mode 100644 anneal/v2/vendor/rayon/src/result.rs create mode 100644 anneal/v2/vendor/rayon/src/slice/chunk_by.rs create mode 100644 anneal/v2/vendor/rayon/src/slice/chunks.rs create mode 100644 anneal/v2/vendor/rayon/src/slice/mod.rs create mode 100644 anneal/v2/vendor/rayon/src/slice/rchunks.rs create mode 100644 anneal/v2/vendor/rayon/src/slice/sort.rs create mode 100644 anneal/v2/vendor/rayon/src/slice/test.rs create mode 100644 anneal/v2/vendor/rayon/src/slice/windows.rs create mode 100644 anneal/v2/vendor/rayon/src/split_producer.rs create mode 100644 anneal/v2/vendor/rayon/src/str.rs create mode 100644 anneal/v2/vendor/rayon/src/string.rs create mode 100644 anneal/v2/vendor/rayon/src/vec.rs create mode 100644 anneal/v2/vendor/rayon/tests/chars.rs create mode 100644 anneal/v2/vendor/rayon/tests/clones.rs create mode 100644 anneal/v2/vendor/rayon/tests/collect.rs create mode 100644 anneal/v2/vendor/rayon/tests/cross-pool.rs create mode 100644 anneal/v2/vendor/rayon/tests/debug.rs create mode 100644 anneal/v2/vendor/rayon/tests/drain_vec.rs create mode 100644 anneal/v2/vendor/rayon/tests/intersperse.rs create mode 100644 anneal/v2/vendor/rayon/tests/issue671-unzip.rs create mode 100644 anneal/v2/vendor/rayon/tests/issue671.rs create mode 100644 anneal/v2/vendor/rayon/tests/iter_panic.rs create mode 100644 anneal/v2/vendor/rayon/tests/named-threads.rs create mode 100644 anneal/v2/vendor/rayon/tests/octillion.rs create mode 100644 anneal/v2/vendor/rayon/tests/par_bridge_recursion.rs create mode 100644 anneal/v2/vendor/rayon/tests/producer_split_at.rs create mode 100644 anneal/v2/vendor/rayon/tests/sort-panic-safe.rs create mode 100644 anneal/v2/vendor/rayon/tests/str.rs diff --git a/anneal/v2/vendor/crossbeam-deque/.cargo-checksum.json b/anneal/v2/vendor/crossbeam-deque/.cargo-checksum.json new file mode 100644 index 0000000000..219fd0230e --- /dev/null +++ b/anneal/v2/vendor/crossbeam-deque/.cargo-checksum.json @@ -0,0 +1 @@ +{"files":{".cargo_vcs_info.json":"d9ccc496cf1bd0f02d77bac0221c8ff040efca5156f835d0adf491d8ebdb89ca","CHANGELOG.md":"0291dc92112ba4d6fe899162cded7f2b7150ed2d06d23e08e5d02a0b53a469f8","Cargo.toml":"93550ef824cb400b7a92af6b2ebe9ef24aa5a37567d80be5e4ba149bdc651be1","Cargo.toml.orig":"2b164c815b5ae4e596a6dc8b5f7b99b9eed7681d8b3372267638ac0d73ddcbc5","LICENSE-APACHE":"a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2","LICENSE-MIT":"5734ed989dfca1f625b40281ee9f4530f91b2411ec01cb748223e7eb87e201ab","README.md":"86445da156ad68ea1d1f2dc49a3cef942ccc377ff56316aefe89732ded763aba","src/deque.rs":"f3c0f92946eb0f71a42a5de2549cf2da137f0a07f31b79def69451fd85c15776","src/lib.rs":"3427a4468253e91e09b264e1f8fd6fc67cfda54be5aecbd38f1b0cb7e0d928de","tests/fifo.rs":"3d98e0d4ca7cfddf10708b71642cf1ff05543d067ad837e48401d63cc31c0a18","tests/injector.rs":"859a0e6fd306e844e8fd660104a994aa894a165f20fa326f4b807e660eaf7878","tests/lifo.rs":"57abdb3fc5920a422f785ba308b658bdc5400947532eeffb799f2395a2061549","tests/steal.rs":"cdf588cc13eeb275ef1231eb18e3245faca7a2d054fa6527bfdba2a34bc8f7bf"},"package":"9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"} \ No newline at end of file diff --git a/anneal/v2/vendor/crossbeam-deque/.cargo_vcs_info.json b/anneal/v2/vendor/crossbeam-deque/.cargo_vcs_info.json new file mode 100644 index 0000000000..0c63ec5c82 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-deque/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "ccd83ac4108a2a1b41e9c6e79c87267167d18dfa" + }, + "path_in_vcs": "crossbeam-deque" +} \ No newline at end of file diff --git a/anneal/v2/vendor/crossbeam-deque/CHANGELOG.md b/anneal/v2/vendor/crossbeam-deque/CHANGELOG.md new file mode 100644 index 0000000000..5bc6e9f47a --- /dev/null +++ b/anneal/v2/vendor/crossbeam-deque/CHANGELOG.md @@ -0,0 +1,141 @@ +# Version 0.8.6 + +- Fix stack overflow when pushing large value to `Injector`. (#1146, #1147, #1159) + +# Version 0.8.5 + +- Remove dependency on `cfg-if`. (#1072) + +# Version 0.8.4 + +- Bump the minimum supported Rust version to 1.61. (#1037) + +# Version 0.8.3 + +- Add `Stealer::{steal_batch_with_limit, steal_batch_with_limit_and_pop}` methods. (#903) +- Add `Injector::{steal_batch_with_limit, steal_batch_with_limit_and_pop}` methods. (#903) + +# Version 0.8.2 + +- Bump the minimum supported Rust version to 1.38. (#877) + +# Version 0.8.1 + +- Fix deque steal race condition. (#726) +- Add `Stealer::len` method. (#708) + +# Version 0.8.0 + +**Note:** This release has been yanked. See [GHSA-pqqp-xmhj-wgcw](https://github.com/crossbeam-rs/crossbeam/security/advisories/GHSA-pqqp-xmhj-wgcw) for details. + +- Bump the minimum supported Rust version to 1.36. +- Add `Worker::len()` and `Injector::len()` methods. +- Add `std` (enabled by default) feature for forward compatibility. + +# Version 0.7.4 + +- Fix deque steal race condition. + +# Version 0.7.3 + +**Note:** This release has been yanked. See [GHSA-pqqp-xmhj-wgcw](https://github.com/crossbeam-rs/crossbeam/security/advisories/GHSA-pqqp-xmhj-wgcw) for details. + +- Stop stealing from the same deque. (#448) +- Fix unsoundness issues by adopting `MaybeUninit`. (#458) + +# Version 0.7.2 + +**Note:** This release has been yanked. See [GHSA-pqqp-xmhj-wgcw](https://github.com/crossbeam-rs/crossbeam/security/advisories/GHSA-pqqp-xmhj-wgcw) for details. + +- Bump `crossbeam-epoch` to `0.8`. +- Bump `crossbeam-utils` to `0.7`. + +# Version 0.7.1 + +**Note:** This release has been yanked. See [GHSA-pqqp-xmhj-wgcw](https://github.com/crossbeam-rs/crossbeam/security/advisories/GHSA-pqqp-xmhj-wgcw) for details. + +- Bump the minimum required version of `crossbeam-utils`. + +# Version 0.7.0 + +**Note:** This release has been yanked. See [GHSA-pqqp-xmhj-wgcw](https://github.com/crossbeam-rs/crossbeam/security/advisories/GHSA-pqqp-xmhj-wgcw) for details. + +- Make `Worker::pop()` faster in the FIFO case. +- Replace `fifo()` nad `lifo()` with `Worker::new_fifo()` and `Worker::new_lifo()`. +- Add more batched steal methods. +- Introduce `Injector`, a MPMC queue. +- Rename `Steal::Data` to `Steal::Success`. +- Add `Steal::or_else()` and implement `FromIterator` for `Steal`. +- Add `#[must_use]` to `Steal`. + +# Version 0.6.3 + +- Bump `crossbeam-epoch` to `0.7`. + +# Version 0.6.2 + +- Update `crosbeam-utils` to `0.6`. + +# Version 0.6.1 + +- Change a few `Relaxed` orderings to `Release` in order to fix false positives by tsan. + +# Version 0.6.0 + +- Add `Stealer::steal_many` for batched stealing. +- Change the return type of `pop` to `Pop` so that spinning can be handled manually. + +# Version 0.5.2 + +- Update `crossbeam-utils` to `0.5.0`. + +# Version 0.5.1 + +- Minor optimizations. + +# Version 0.5.0 + +- Add two deque constructors : `fifo()` and `lifo()`. +- Update `rand` to `0.5.3`. +- Rename `Deque` to `Worker`. +- Return `Option` from `Stealer::steal`. +- Remove methods `Deque::len` and `Stealer::len`. +- Remove method `Deque::stealer`. +- Remove method `Deque::steal`. + +# Version 0.4.1 + +- Update `crossbeam-epoch` to `0.5.0`. + +# Version 0.4.0 + +- Update `crossbeam-epoch` to `0.4.2`. +- Update `crossbeam-utils` to `0.4.0`. +- Require minimum Rust version 1.25. + +# Version 0.3.1 + +- Add `Deque::capacity`. +- Add `Deque::min_capacity`. +- Add `Deque::shrink_to_fit`. +- Update `crossbeam-epoch` to `0.3.0`. +- Support Rust 1.20. +- Shrink the buffer in `Deque::push` if necessary. + +# Version 0.3.0 + +- Update `crossbeam-epoch` to `0.4.0`. +- Drop support for Rust 1.13. + +# Version 0.2.0 + +- Update `crossbeam-epoch` to `0.3.0`. +- Support Rust 1.13. + +# Version 0.1.1 + +- Update `crossbeam-epoch` to `0.2.0`. + +# Version 0.1.0 + +- First implementation of the Chase-Lev deque. diff --git a/anneal/v2/vendor/crossbeam-deque/Cargo.toml b/anneal/v2/vendor/crossbeam-deque/Cargo.toml new file mode 100644 index 0000000000..79278183e0 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-deque/Cargo.toml @@ -0,0 +1,92 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +rust-version = "1.61" +name = "crossbeam-deque" +version = "0.8.6" +build = false +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Concurrent work-stealing deque" +homepage = "https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-deque" +readme = "README.md" +keywords = [ + "chase-lev", + "lock-free", + "scheduler", + "scheduling", +] +categories = [ + "algorithms", + "concurrency", + "data-structures", +] +license = "MIT OR Apache-2.0" +repository = "https://github.com/crossbeam-rs/crossbeam" + +[lib] +name = "crossbeam_deque" +path = "src/lib.rs" + +[[test]] +name = "fifo" +path = "tests/fifo.rs" + +[[test]] +name = "injector" +path = "tests/injector.rs" + +[[test]] +name = "lifo" +path = "tests/lifo.rs" + +[[test]] +name = "steal" +path = "tests/steal.rs" + +[dependencies.crossbeam-epoch] +version = "0.9.17" +default-features = false + +[dependencies.crossbeam-utils] +version = "0.8.18" +default-features = false + +[dev-dependencies.rand] +version = "0.8" + +[features] +default = ["std"] +std = [ + "crossbeam-epoch/std", + "crossbeam-utils/std", +] + +[lints.clippy.declare_interior_mutable_const] +level = "allow" +priority = 1 + +[lints.clippy.lint_groups_priority] +level = "allow" +priority = 1 + +[lints.rust.unexpected_cfgs] +level = "warn" +priority = 0 +check-cfg = [ + "cfg(crossbeam_loom)", + "cfg(crossbeam_sanitize)", +] diff --git a/anneal/v2/vendor/crossbeam-deque/Cargo.toml.orig b/anneal/v2/vendor/crossbeam-deque/Cargo.toml.orig new file mode 100644 index 0000000000..3c6d446396 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-deque/Cargo.toml.orig @@ -0,0 +1,34 @@ +[package] +name = "crossbeam-deque" +# When publishing a new version: +# - Update CHANGELOG.md +# - Update README.md (when increasing major or minor version) +# - Run './tools/publish.sh crossbeam-deque ' +version = "0.8.6" +edition = "2021" +rust-version = "1.61" +license = "MIT OR Apache-2.0" +repository = "https://github.com/crossbeam-rs/crossbeam" +homepage = "https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-deque" +description = "Concurrent work-stealing deque" +keywords = ["chase-lev", "lock-free", "scheduler", "scheduling"] +categories = ["algorithms", "concurrency", "data-structures"] + +[features] +default = ["std"] + +# Enable to use APIs that require `std`. +# This is enabled by default. +# +# NOTE: Disabling `std` feature is not supported yet. +std = ["crossbeam-epoch/std", "crossbeam-utils/std"] + +[dependencies] +crossbeam-epoch = { version = "0.9.17", path = "../crossbeam-epoch", default-features = false } +crossbeam-utils = { version = "0.8.18", path = "../crossbeam-utils", default-features = false } + +[dev-dependencies] +rand = "0.8" + +[lints] +workspace = true diff --git a/anneal/v2/vendor/crossbeam-deque/LICENSE-APACHE b/anneal/v2/vendor/crossbeam-deque/LICENSE-APACHE new file mode 100644 index 0000000000..16fe87b06e --- /dev/null +++ b/anneal/v2/vendor/crossbeam-deque/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/anneal/v2/vendor/crossbeam-deque/LICENSE-MIT b/anneal/v2/vendor/crossbeam-deque/LICENSE-MIT new file mode 100644 index 0000000000..068d491fd5 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-deque/LICENSE-MIT @@ -0,0 +1,27 @@ +The MIT License (MIT) + +Copyright (c) 2019 The Crossbeam Project Developers + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/anneal/v2/vendor/crossbeam-deque/README.md b/anneal/v2/vendor/crossbeam-deque/README.md new file mode 100644 index 0000000000..4eae1447ec --- /dev/null +++ b/anneal/v2/vendor/crossbeam-deque/README.md @@ -0,0 +1,46 @@ +# Crossbeam Deque + +[![Build Status](https://github.com/crossbeam-rs/crossbeam/workflows/CI/badge.svg)]( +https://github.com/crossbeam-rs/crossbeam/actions) +[![License](https://img.shields.io/badge/license-MIT_OR_Apache--2.0-blue.svg)]( +https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-deque#license) +[![Cargo](https://img.shields.io/crates/v/crossbeam-deque.svg)]( +https://crates.io/crates/crossbeam-deque) +[![Documentation](https://docs.rs/crossbeam-deque/badge.svg)]( +https://docs.rs/crossbeam-deque) +[![Rust 1.61+](https://img.shields.io/badge/rust-1.61+-lightgray.svg)]( +https://www.rust-lang.org) +[![chat](https://img.shields.io/discord/569610676205781012.svg?logo=discord)](https://discord.com/invite/JXYwgWZ) + +This crate provides work-stealing deques, which are primarily intended for +building task schedulers. + +## Usage + +Add this to your `Cargo.toml`: + +```toml +[dependencies] +crossbeam-deque = "0.8" +``` + +## Compatibility + +Crossbeam Deque supports stable Rust releases going back at least six months, +and every time the minimum supported Rust version is increased, a new minor +version is released. Currently, the minimum supported Rust version is 1.61. + +## License + +Licensed under either of + + * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) + * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) + +at your option. + +#### Contribution + +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in the work by you, as defined in the Apache-2.0 license, shall be +dual licensed as above, without any additional terms or conditions. diff --git a/anneal/v2/vendor/crossbeam-deque/src/deque.rs b/anneal/v2/vendor/crossbeam-deque/src/deque.rs new file mode 100644 index 0000000000..84ba0b8c04 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-deque/src/deque.rs @@ -0,0 +1,2216 @@ +use std::alloc::{alloc_zeroed, handle_alloc_error, Layout}; +use std::boxed::Box; +use std::cell::{Cell, UnsafeCell}; +use std::cmp; +use std::fmt; +use std::marker::PhantomData; +use std::mem::{self, MaybeUninit}; +use std::ptr; +use std::sync::atomic::{self, AtomicIsize, AtomicPtr, AtomicUsize, Ordering}; +use std::sync::Arc; + +use crossbeam_epoch::{self as epoch, Atomic, Owned}; +use crossbeam_utils::{Backoff, CachePadded}; + +// Minimum buffer capacity. +const MIN_CAP: usize = 64; +// Maximum number of tasks that can be stolen in `steal_batch()` and `steal_batch_and_pop()`. +const MAX_BATCH: usize = 32; +// If a buffer of at least this size is retired, thread-local garbage is flushed so that it gets +// deallocated as soon as possible. +const FLUSH_THRESHOLD_BYTES: usize = 1 << 10; + +/// A buffer that holds tasks in a worker queue. +/// +/// This is just a pointer to the buffer and its length - dropping an instance of this struct will +/// *not* deallocate the buffer. +struct Buffer { + /// Pointer to the allocated memory. + ptr: *mut T, + + /// Capacity of the buffer. Always a power of two. + cap: usize, +} + +unsafe impl Send for Buffer {} + +impl Buffer { + /// Allocates a new buffer with the specified capacity. + fn alloc(cap: usize) -> Buffer { + debug_assert_eq!(cap, cap.next_power_of_two()); + + let ptr = Box::into_raw( + (0..cap) + .map(|_| MaybeUninit::::uninit()) + .collect::>(), + ) + .cast::(); + + Buffer { ptr, cap } + } + + /// Deallocates the buffer. + unsafe fn dealloc(self) { + drop(Box::from_raw(ptr::slice_from_raw_parts_mut( + self.ptr.cast::>(), + self.cap, + ))); + } + + /// Returns a pointer to the task at the specified `index`. + unsafe fn at(&self, index: isize) -> *mut T { + // `self.cap` is always a power of two. + // We do all the loads at `MaybeUninit` because we might realize, after loading, that we + // don't actually have the right to access this memory. + self.ptr.offset(index & (self.cap - 1) as isize) + } + + /// Writes `task` into the specified `index`. + /// + /// This method might be concurrently called with another `read` at the same index, which is + /// technically speaking a data race and therefore UB. We should use an atomic store here, but + /// that would be more expensive and difficult to implement generically for all types `T`. + /// Hence, as a hack, we use a volatile write instead. + unsafe fn write(&self, index: isize, task: MaybeUninit) { + ptr::write_volatile(self.at(index).cast::>(), task) + } + + /// Reads a task from the specified `index`. + /// + /// This method might be concurrently called with another `write` at the same index, which is + /// technically speaking a data race and therefore UB. We should use an atomic load here, but + /// that would be more expensive and difficult to implement generically for all types `T`. + /// Hence, as a hack, we use a volatile load instead. + unsafe fn read(&self, index: isize) -> MaybeUninit { + ptr::read_volatile(self.at(index).cast::>()) + } +} + +impl Clone for Buffer { + fn clone(&self) -> Buffer { + *self + } +} + +impl Copy for Buffer {} + +/// Internal queue data shared between the worker and stealers. +/// +/// The implementation is based on the following work: +/// +/// 1. [Chase and Lev. Dynamic circular work-stealing deque. SPAA 2005.][chase-lev] +/// 2. [Le, Pop, Cohen, and Nardelli. Correct and efficient work-stealing for weak memory models. +/// PPoPP 2013.][weak-mem] +/// 3. [Norris and Demsky. CDSchecker: checking concurrent data structures written with C/C++ +/// atomics. OOPSLA 2013.][checker] +/// +/// [chase-lev]: https://dl.acm.org/citation.cfm?id=1073974 +/// [weak-mem]: https://dl.acm.org/citation.cfm?id=2442524 +/// [checker]: https://dl.acm.org/citation.cfm?id=2509514 +struct Inner { + /// The front index. + front: AtomicIsize, + + /// The back index. + back: AtomicIsize, + + /// The underlying buffer. + buffer: CachePadded>>, +} + +impl Drop for Inner { + fn drop(&mut self) { + // Load the back index, front index, and buffer. + let b = *self.back.get_mut(); + let f = *self.front.get_mut(); + + unsafe { + let buffer = self.buffer.load(Ordering::Relaxed, epoch::unprotected()); + + // Go through the buffer from front to back and drop all tasks in the queue. + let mut i = f; + while i != b { + buffer.deref().at(i).drop_in_place(); + i = i.wrapping_add(1); + } + + // Free the memory allocated by the buffer. + buffer.into_owned().into_box().dealloc(); + } + } +} + +/// Worker queue flavor: FIFO or LIFO. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum Flavor { + /// The first-in first-out flavor. + Fifo, + + /// The last-in first-out flavor. + Lifo, +} + +/// A worker queue. +/// +/// This is a FIFO or LIFO queue that is owned by a single thread, but other threads may steal +/// tasks from it. Task schedulers typically create a single worker queue per thread. +/// +/// # Examples +/// +/// A FIFO worker: +/// +/// ``` +/// use crossbeam_deque::{Steal, Worker}; +/// +/// let w = Worker::new_fifo(); +/// let s = w.stealer(); +/// +/// w.push(1); +/// w.push(2); +/// w.push(3); +/// +/// assert_eq!(s.steal(), Steal::Success(1)); +/// assert_eq!(w.pop(), Some(2)); +/// assert_eq!(w.pop(), Some(3)); +/// ``` +/// +/// A LIFO worker: +/// +/// ``` +/// use crossbeam_deque::{Steal, Worker}; +/// +/// let w = Worker::new_lifo(); +/// let s = w.stealer(); +/// +/// w.push(1); +/// w.push(2); +/// w.push(3); +/// +/// assert_eq!(s.steal(), Steal::Success(1)); +/// assert_eq!(w.pop(), Some(3)); +/// assert_eq!(w.pop(), Some(2)); +/// ``` +pub struct Worker { + /// A reference to the inner representation of the queue. + inner: Arc>>, + + /// A copy of `inner.buffer` for quick access. + buffer: Cell>, + + /// The flavor of the queue. + flavor: Flavor, + + /// Indicates that the worker cannot be shared among threads. + _marker: PhantomData<*mut ()>, // !Send + !Sync +} + +unsafe impl Send for Worker {} + +impl Worker { + /// Creates a FIFO worker queue. + /// + /// Tasks are pushed and popped from opposite ends. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_deque::Worker; + /// + /// let w = Worker::::new_fifo(); + /// ``` + pub fn new_fifo() -> Worker { + let buffer = Buffer::alloc(MIN_CAP); + + let inner = Arc::new(CachePadded::new(Inner { + front: AtomicIsize::new(0), + back: AtomicIsize::new(0), + buffer: CachePadded::new(Atomic::new(buffer)), + })); + + Worker { + inner, + buffer: Cell::new(buffer), + flavor: Flavor::Fifo, + _marker: PhantomData, + } + } + + /// Creates a LIFO worker queue. + /// + /// Tasks are pushed and popped from the same end. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_deque::Worker; + /// + /// let w = Worker::::new_lifo(); + /// ``` + pub fn new_lifo() -> Worker { + let buffer = Buffer::alloc(MIN_CAP); + + let inner = Arc::new(CachePadded::new(Inner { + front: AtomicIsize::new(0), + back: AtomicIsize::new(0), + buffer: CachePadded::new(Atomic::new(buffer)), + })); + + Worker { + inner, + buffer: Cell::new(buffer), + flavor: Flavor::Lifo, + _marker: PhantomData, + } + } + + /// Creates a stealer for this queue. + /// + /// The returned stealer can be shared among threads and cloned. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_deque::Worker; + /// + /// let w = Worker::::new_lifo(); + /// let s = w.stealer(); + /// ``` + pub fn stealer(&self) -> Stealer { + Stealer { + inner: self.inner.clone(), + flavor: self.flavor, + } + } + + /// Resizes the internal buffer to the new capacity of `new_cap`. + #[cold] + unsafe fn resize(&self, new_cap: usize) { + // Load the back index, front index, and buffer. + let b = self.inner.back.load(Ordering::Relaxed); + let f = self.inner.front.load(Ordering::Relaxed); + let buffer = self.buffer.get(); + + // Allocate a new buffer and copy data from the old buffer to the new one. + let new = Buffer::alloc(new_cap); + let mut i = f; + while i != b { + ptr::copy_nonoverlapping(buffer.at(i), new.at(i), 1); + i = i.wrapping_add(1); + } + + let guard = &epoch::pin(); + + // Replace the old buffer with the new one. + self.buffer.replace(new); + let old = + self.inner + .buffer + .swap(Owned::new(new).into_shared(guard), Ordering::Release, guard); + + // Destroy the old buffer later. + guard.defer_unchecked(move || old.into_owned().into_box().dealloc()); + + // If the buffer is very large, then flush the thread-local garbage in order to deallocate + // it as soon as possible. + if mem::size_of::() * new_cap >= FLUSH_THRESHOLD_BYTES { + guard.flush(); + } + } + + /// Reserves enough capacity so that `reserve_cap` tasks can be pushed without growing the + /// buffer. + fn reserve(&self, reserve_cap: usize) { + if reserve_cap > 0 { + // Compute the current length. + let b = self.inner.back.load(Ordering::Relaxed); + let f = self.inner.front.load(Ordering::SeqCst); + let len = b.wrapping_sub(f) as usize; + + // The current capacity. + let cap = self.buffer.get().cap; + + // Is there enough capacity to push `reserve_cap` tasks? + if cap - len < reserve_cap { + // Keep doubling the capacity as much as is needed. + let mut new_cap = cap * 2; + while new_cap - len < reserve_cap { + new_cap *= 2; + } + + // Resize the buffer. + unsafe { + self.resize(new_cap); + } + } + } + } + + /// Returns `true` if the queue is empty. + /// + /// ``` + /// use crossbeam_deque::Worker; + /// + /// let w = Worker::new_lifo(); + /// + /// assert!(w.is_empty()); + /// w.push(1); + /// assert!(!w.is_empty()); + /// ``` + pub fn is_empty(&self) -> bool { + let b = self.inner.back.load(Ordering::Relaxed); + let f = self.inner.front.load(Ordering::SeqCst); + b.wrapping_sub(f) <= 0 + } + + /// Returns the number of tasks in the deque. + /// + /// ``` + /// use crossbeam_deque::Worker; + /// + /// let w = Worker::new_lifo(); + /// + /// assert_eq!(w.len(), 0); + /// w.push(1); + /// assert_eq!(w.len(), 1); + /// w.push(1); + /// assert_eq!(w.len(), 2); + /// ``` + pub fn len(&self) -> usize { + let b = self.inner.back.load(Ordering::Relaxed); + let f = self.inner.front.load(Ordering::SeqCst); + b.wrapping_sub(f).max(0) as usize + } + + /// Pushes a task into the queue. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_deque::Worker; + /// + /// let w = Worker::new_lifo(); + /// w.push(1); + /// w.push(2); + /// ``` + pub fn push(&self, task: T) { + // Load the back index, front index, and buffer. + let b = self.inner.back.load(Ordering::Relaxed); + let f = self.inner.front.load(Ordering::Acquire); + let mut buffer = self.buffer.get(); + + // Calculate the length of the queue. + let len = b.wrapping_sub(f); + + // Is the queue full? + if len >= buffer.cap as isize { + // Yes. Grow the underlying buffer. + unsafe { + self.resize(2 * buffer.cap); + } + buffer = self.buffer.get(); + } + + // Write `task` into the slot. + unsafe { + buffer.write(b, MaybeUninit::new(task)); + } + + atomic::fence(Ordering::Release); + + // Increment the back index. + // + // This ordering could be `Relaxed`, but then thread sanitizer would falsely report data + // races because it doesn't understand fences. + self.inner.back.store(b.wrapping_add(1), Ordering::Release); + } + + /// Pops a task from the queue. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_deque::Worker; + /// + /// let w = Worker::new_fifo(); + /// w.push(1); + /// w.push(2); + /// + /// assert_eq!(w.pop(), Some(1)); + /// assert_eq!(w.pop(), Some(2)); + /// assert_eq!(w.pop(), None); + /// ``` + pub fn pop(&self) -> Option { + // Load the back and front index. + let b = self.inner.back.load(Ordering::Relaxed); + let f = self.inner.front.load(Ordering::Relaxed); + + // Calculate the length of the queue. + let len = b.wrapping_sub(f); + + // Is the queue empty? + if len <= 0 { + return None; + } + + match self.flavor { + // Pop from the front of the queue. + Flavor::Fifo => { + // Try incrementing the front index to pop the task. + let f = self.inner.front.fetch_add(1, Ordering::SeqCst); + let new_f = f.wrapping_add(1); + + if b.wrapping_sub(new_f) < 0 { + self.inner.front.store(f, Ordering::Relaxed); + return None; + } + + unsafe { + // Read the popped task. + let buffer = self.buffer.get(); + let task = buffer.read(f).assume_init(); + + // Shrink the buffer if `len - 1` is less than one fourth of the capacity. + if buffer.cap > MIN_CAP && len <= buffer.cap as isize / 4 { + self.resize(buffer.cap / 2); + } + + Some(task) + } + } + + // Pop from the back of the queue. + Flavor::Lifo => { + // Decrement the back index. + let b = b.wrapping_sub(1); + self.inner.back.store(b, Ordering::Relaxed); + + atomic::fence(Ordering::SeqCst); + + // Load the front index. + let f = self.inner.front.load(Ordering::Relaxed); + + // Compute the length after the back index was decremented. + let len = b.wrapping_sub(f); + + if len < 0 { + // The queue is empty. Restore the back index to the original task. + self.inner.back.store(b.wrapping_add(1), Ordering::Relaxed); + None + } else { + // Read the task to be popped. + let buffer = self.buffer.get(); + let mut task = unsafe { Some(buffer.read(b)) }; + + // Are we popping the last task from the queue? + if len == 0 { + // Try incrementing the front index. + if self + .inner + .front + .compare_exchange( + f, + f.wrapping_add(1), + Ordering::SeqCst, + Ordering::Relaxed, + ) + .is_err() + { + // Failed. We didn't pop anything. Reset to `None`. + task.take(); + } + + // Restore the back index to the original task. + self.inner.back.store(b.wrapping_add(1), Ordering::Relaxed); + } else { + // Shrink the buffer if `len` is less than one fourth of the capacity. + if buffer.cap > MIN_CAP && len < buffer.cap as isize / 4 { + unsafe { + self.resize(buffer.cap / 2); + } + } + } + + task.map(|t| unsafe { t.assume_init() }) + } + } + } + } +} + +impl fmt::Debug for Worker { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad("Worker { .. }") + } +} + +/// A stealer handle of a worker queue. +/// +/// Stealers can be shared among threads. +/// +/// Task schedulers typically have a single worker queue per worker thread. +/// +/// # Examples +/// +/// ``` +/// use crossbeam_deque::{Steal, Worker}; +/// +/// let w = Worker::new_lifo(); +/// w.push(1); +/// w.push(2); +/// +/// let s = w.stealer(); +/// assert_eq!(s.steal(), Steal::Success(1)); +/// assert_eq!(s.steal(), Steal::Success(2)); +/// assert_eq!(s.steal(), Steal::Empty); +/// ``` +pub struct Stealer { + /// A reference to the inner representation of the queue. + inner: Arc>>, + + /// The flavor of the queue. + flavor: Flavor, +} + +unsafe impl Send for Stealer {} +unsafe impl Sync for Stealer {} + +impl Stealer { + /// Returns `true` if the queue is empty. + /// + /// ``` + /// use crossbeam_deque::Worker; + /// + /// let w = Worker::new_lifo(); + /// let s = w.stealer(); + /// + /// assert!(s.is_empty()); + /// w.push(1); + /// assert!(!s.is_empty()); + /// ``` + pub fn is_empty(&self) -> bool { + let f = self.inner.front.load(Ordering::Acquire); + atomic::fence(Ordering::SeqCst); + let b = self.inner.back.load(Ordering::Acquire); + b.wrapping_sub(f) <= 0 + } + + /// Returns the number of tasks in the deque. + /// + /// ``` + /// use crossbeam_deque::Worker; + /// + /// let w = Worker::new_lifo(); + /// let s = w.stealer(); + /// + /// assert_eq!(s.len(), 0); + /// w.push(1); + /// assert_eq!(s.len(), 1); + /// w.push(2); + /// assert_eq!(s.len(), 2); + /// ``` + pub fn len(&self) -> usize { + let f = self.inner.front.load(Ordering::Acquire); + atomic::fence(Ordering::SeqCst); + let b = self.inner.back.load(Ordering::Acquire); + b.wrapping_sub(f).max(0) as usize + } + + /// Steals a task from the queue. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_deque::{Steal, Worker}; + /// + /// let w = Worker::new_lifo(); + /// w.push(1); + /// w.push(2); + /// + /// let s = w.stealer(); + /// assert_eq!(s.steal(), Steal::Success(1)); + /// assert_eq!(s.steal(), Steal::Success(2)); + /// ``` + pub fn steal(&self) -> Steal { + // Load the front index. + let f = self.inner.front.load(Ordering::Acquire); + + // A SeqCst fence is needed here. + // + // If the current thread is already pinned (reentrantly), we must manually issue the + // fence. Otherwise, the following pinning will issue the fence anyway, so we don't + // have to. + if epoch::is_pinned() { + atomic::fence(Ordering::SeqCst); + } + + let guard = &epoch::pin(); + + // Load the back index. + let b = self.inner.back.load(Ordering::Acquire); + + // Is the queue empty? + if b.wrapping_sub(f) <= 0 { + return Steal::Empty; + } + + // Load the buffer and read the task at the front. + let buffer = self.inner.buffer.load(Ordering::Acquire, guard); + let task = unsafe { buffer.deref().read(f) }; + + // Try incrementing the front index to steal the task. + // If the buffer has been swapped or the increment fails, we retry. + if self.inner.buffer.load(Ordering::Acquire, guard) != buffer + || self + .inner + .front + .compare_exchange(f, f.wrapping_add(1), Ordering::SeqCst, Ordering::Relaxed) + .is_err() + { + // We didn't steal this task, forget it. + return Steal::Retry; + } + + // Return the stolen task. + Steal::Success(unsafe { task.assume_init() }) + } + + /// Steals a batch of tasks and pushes them into another worker. + /// + /// How many tasks exactly will be stolen is not specified. That said, this method will try to + /// steal around half of the tasks in the queue, but also not more than some constant limit. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_deque::Worker; + /// + /// let w1 = Worker::new_fifo(); + /// w1.push(1); + /// w1.push(2); + /// w1.push(3); + /// w1.push(4); + /// + /// let s = w1.stealer(); + /// let w2 = Worker::new_fifo(); + /// + /// let _ = s.steal_batch(&w2); + /// assert_eq!(w2.pop(), Some(1)); + /// assert_eq!(w2.pop(), Some(2)); + /// ``` + pub fn steal_batch(&self, dest: &Worker) -> Steal<()> { + self.steal_batch_with_limit(dest, MAX_BATCH) + } + + /// Steals no more than `limit` of tasks and pushes them into another worker. + /// + /// How many tasks exactly will be stolen is not specified. That said, this method will try to + /// steal around half of the tasks in the queue, but also not more than the given limit. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_deque::Worker; + /// + /// let w1 = Worker::new_fifo(); + /// w1.push(1); + /// w1.push(2); + /// w1.push(3); + /// w1.push(4); + /// w1.push(5); + /// w1.push(6); + /// + /// let s = w1.stealer(); + /// let w2 = Worker::new_fifo(); + /// + /// let _ = s.steal_batch_with_limit(&w2, 2); + /// assert_eq!(w2.pop(), Some(1)); + /// assert_eq!(w2.pop(), Some(2)); + /// assert_eq!(w2.pop(), None); + /// + /// w1.push(7); + /// w1.push(8); + /// // Setting a large limit does not guarantee that all elements will be popped. In this case, + /// // half of the elements are currently popped, but the number of popped elements is considered + /// // an implementation detail that may be changed in the future. + /// let _ = s.steal_batch_with_limit(&w2, std::usize::MAX); + /// assert_eq!(w2.len(), 3); + /// ``` + pub fn steal_batch_with_limit(&self, dest: &Worker, limit: usize) -> Steal<()> { + assert!(limit > 0); + if Arc::ptr_eq(&self.inner, &dest.inner) { + if dest.is_empty() { + return Steal::Empty; + } else { + return Steal::Success(()); + } + } + + // Load the front index. + let mut f = self.inner.front.load(Ordering::Acquire); + + // A SeqCst fence is needed here. + // + // If the current thread is already pinned (reentrantly), we must manually issue the + // fence. Otherwise, the following pinning will issue the fence anyway, so we don't + // have to. + if epoch::is_pinned() { + atomic::fence(Ordering::SeqCst); + } + + let guard = &epoch::pin(); + + // Load the back index. + let b = self.inner.back.load(Ordering::Acquire); + + // Is the queue empty? + let len = b.wrapping_sub(f); + if len <= 0 { + return Steal::Empty; + } + + // Reserve capacity for the stolen batch. + let batch_size = cmp::min((len as usize + 1) / 2, limit); + dest.reserve(batch_size); + let mut batch_size = batch_size as isize; + + // Get the destination buffer and back index. + let dest_buffer = dest.buffer.get(); + let mut dest_b = dest.inner.back.load(Ordering::Relaxed); + + // Load the buffer. + let buffer = self.inner.buffer.load(Ordering::Acquire, guard); + + match self.flavor { + // Steal a batch of tasks from the front at once. + Flavor::Fifo => { + // Copy the batch from the source to the destination buffer. + match dest.flavor { + Flavor::Fifo => { + for i in 0..batch_size { + unsafe { + let task = buffer.deref().read(f.wrapping_add(i)); + dest_buffer.write(dest_b.wrapping_add(i), task); + } + } + } + Flavor::Lifo => { + for i in 0..batch_size { + unsafe { + let task = buffer.deref().read(f.wrapping_add(i)); + dest_buffer.write(dest_b.wrapping_add(batch_size - 1 - i), task); + } + } + } + } + + // Try incrementing the front index to steal the batch. + // If the buffer has been swapped or the increment fails, we retry. + if self.inner.buffer.load(Ordering::Acquire, guard) != buffer + || self + .inner + .front + .compare_exchange( + f, + f.wrapping_add(batch_size), + Ordering::SeqCst, + Ordering::Relaxed, + ) + .is_err() + { + return Steal::Retry; + } + + dest_b = dest_b.wrapping_add(batch_size); + } + + // Steal a batch of tasks from the front one by one. + Flavor::Lifo => { + // This loop may modify the batch_size, which triggers a clippy lint warning. + // Use a new variable to avoid the warning, and to make it clear we aren't + // modifying the loop exit condition during iteration. + let original_batch_size = batch_size; + + for i in 0..original_batch_size { + // If this is not the first steal, check whether the queue is empty. + if i > 0 { + // We've already got the current front index. Now execute the fence to + // synchronize with other threads. + atomic::fence(Ordering::SeqCst); + + // Load the back index. + let b = self.inner.back.load(Ordering::Acquire); + + // Is the queue empty? + if b.wrapping_sub(f) <= 0 { + batch_size = i; + break; + } + } + + // Read the task at the front. + let task = unsafe { buffer.deref().read(f) }; + + // Try incrementing the front index to steal the task. + // If the buffer has been swapped or the increment fails, we retry. + if self.inner.buffer.load(Ordering::Acquire, guard) != buffer + || self + .inner + .front + .compare_exchange( + f, + f.wrapping_add(1), + Ordering::SeqCst, + Ordering::Relaxed, + ) + .is_err() + { + // We didn't steal this task, forget it and break from the loop. + batch_size = i; + break; + } + + // Write the stolen task into the destination buffer. + unsafe { + dest_buffer.write(dest_b, task); + } + + // Move the source front index and the destination back index one step forward. + f = f.wrapping_add(1); + dest_b = dest_b.wrapping_add(1); + } + + // If we didn't steal anything, the operation needs to be retried. + if batch_size == 0 { + return Steal::Retry; + } + + // If stealing into a FIFO queue, stolen tasks need to be reversed. + if dest.flavor == Flavor::Fifo { + for i in 0..batch_size / 2 { + unsafe { + let i1 = dest_b.wrapping_sub(batch_size - i); + let i2 = dest_b.wrapping_sub(i + 1); + let t1 = dest_buffer.read(i1); + let t2 = dest_buffer.read(i2); + dest_buffer.write(i1, t2); + dest_buffer.write(i2, t1); + } + } + } + } + } + + atomic::fence(Ordering::Release); + + // Update the back index in the destination queue. + // + // This ordering could be `Relaxed`, but then thread sanitizer would falsely report data + // races because it doesn't understand fences. + dest.inner.back.store(dest_b, Ordering::Release); + + // Return with success. + Steal::Success(()) + } + + /// Steals a batch of tasks, pushes them into another worker, and pops a task from that worker. + /// + /// How many tasks exactly will be stolen is not specified. That said, this method will try to + /// steal around half of the tasks in the queue, but also not more than some constant limit. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_deque::{Steal, Worker}; + /// + /// let w1 = Worker::new_fifo(); + /// w1.push(1); + /// w1.push(2); + /// w1.push(3); + /// w1.push(4); + /// + /// let s = w1.stealer(); + /// let w2 = Worker::new_fifo(); + /// + /// assert_eq!(s.steal_batch_and_pop(&w2), Steal::Success(1)); + /// assert_eq!(w2.pop(), Some(2)); + /// ``` + pub fn steal_batch_and_pop(&self, dest: &Worker) -> Steal { + self.steal_batch_with_limit_and_pop(dest, MAX_BATCH) + } + + /// Steals no more than `limit` of tasks, pushes them into another worker, and pops a task from + /// that worker. + /// + /// How many tasks exactly will be stolen is not specified. That said, this method will try to + /// steal around half of the tasks in the queue, but also not more than the given limit. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_deque::{Steal, Worker}; + /// + /// let w1 = Worker::new_fifo(); + /// w1.push(1); + /// w1.push(2); + /// w1.push(3); + /// w1.push(4); + /// w1.push(5); + /// w1.push(6); + /// + /// let s = w1.stealer(); + /// let w2 = Worker::new_fifo(); + /// + /// assert_eq!(s.steal_batch_with_limit_and_pop(&w2, 2), Steal::Success(1)); + /// assert_eq!(w2.pop(), Some(2)); + /// assert_eq!(w2.pop(), None); + /// + /// w1.push(7); + /// w1.push(8); + /// // Setting a large limit does not guarantee that all elements will be popped. In this case, + /// // half of the elements are currently popped, but the number of popped elements is considered + /// // an implementation detail that may be changed in the future. + /// assert_eq!(s.steal_batch_with_limit_and_pop(&w2, std::usize::MAX), Steal::Success(3)); + /// assert_eq!(w2.pop(), Some(4)); + /// assert_eq!(w2.pop(), Some(5)); + /// assert_eq!(w2.pop(), None); + /// ``` + pub fn steal_batch_with_limit_and_pop(&self, dest: &Worker, limit: usize) -> Steal { + assert!(limit > 0); + if Arc::ptr_eq(&self.inner, &dest.inner) { + match dest.pop() { + None => return Steal::Empty, + Some(task) => return Steal::Success(task), + } + } + + // Load the front index. + let mut f = self.inner.front.load(Ordering::Acquire); + + // A SeqCst fence is needed here. + // + // If the current thread is already pinned (reentrantly), we must manually issue the + // fence. Otherwise, the following pinning will issue the fence anyway, so we don't + // have to. + if epoch::is_pinned() { + atomic::fence(Ordering::SeqCst); + } + + let guard = &epoch::pin(); + + // Load the back index. + let b = self.inner.back.load(Ordering::Acquire); + + // Is the queue empty? + let len = b.wrapping_sub(f); + if len <= 0 { + return Steal::Empty; + } + + // Reserve capacity for the stolen batch. + let batch_size = cmp::min((len as usize - 1) / 2, limit - 1); + dest.reserve(batch_size); + let mut batch_size = batch_size as isize; + + // Get the destination buffer and back index. + let dest_buffer = dest.buffer.get(); + let mut dest_b = dest.inner.back.load(Ordering::Relaxed); + + // Load the buffer + let buffer = self.inner.buffer.load(Ordering::Acquire, guard); + + // Read the task at the front. + let mut task = unsafe { buffer.deref().read(f) }; + + match self.flavor { + // Steal a batch of tasks from the front at once. + Flavor::Fifo => { + // Copy the batch from the source to the destination buffer. + match dest.flavor { + Flavor::Fifo => { + for i in 0..batch_size { + unsafe { + let task = buffer.deref().read(f.wrapping_add(i + 1)); + dest_buffer.write(dest_b.wrapping_add(i), task); + } + } + } + Flavor::Lifo => { + for i in 0..batch_size { + unsafe { + let task = buffer.deref().read(f.wrapping_add(i + 1)); + dest_buffer.write(dest_b.wrapping_add(batch_size - 1 - i), task); + } + } + } + } + + // Try incrementing the front index to steal the task. + // If the buffer has been swapped or the increment fails, we retry. + if self.inner.buffer.load(Ordering::Acquire, guard) != buffer + || self + .inner + .front + .compare_exchange( + f, + f.wrapping_add(batch_size + 1), + Ordering::SeqCst, + Ordering::Relaxed, + ) + .is_err() + { + // We didn't steal this task, forget it. + return Steal::Retry; + } + + dest_b = dest_b.wrapping_add(batch_size); + } + + // Steal a batch of tasks from the front one by one. + Flavor::Lifo => { + // Try incrementing the front index to steal the task. + if self + .inner + .front + .compare_exchange(f, f.wrapping_add(1), Ordering::SeqCst, Ordering::Relaxed) + .is_err() + { + // We didn't steal this task, forget it. + return Steal::Retry; + } + + // Move the front index one step forward. + f = f.wrapping_add(1); + + // Repeat the same procedure for the batch steals. + // + // This loop may modify the batch_size, which triggers a clippy lint warning. + // Use a new variable to avoid the warning, and to make it clear we aren't + // modifying the loop exit condition during iteration. + let original_batch_size = batch_size; + for i in 0..original_batch_size { + // We've already got the current front index. Now execute the fence to + // synchronize with other threads. + atomic::fence(Ordering::SeqCst); + + // Load the back index. + let b = self.inner.back.load(Ordering::Acquire); + + // Is the queue empty? + if b.wrapping_sub(f) <= 0 { + batch_size = i; + break; + } + + // Read the task at the front. + let tmp = unsafe { buffer.deref().read(f) }; + + // Try incrementing the front index to steal the task. + // If the buffer has been swapped or the increment fails, we retry. + if self.inner.buffer.load(Ordering::Acquire, guard) != buffer + || self + .inner + .front + .compare_exchange( + f, + f.wrapping_add(1), + Ordering::SeqCst, + Ordering::Relaxed, + ) + .is_err() + { + // We didn't steal this task, forget it and break from the loop. + batch_size = i; + break; + } + + // Write the previously stolen task into the destination buffer. + unsafe { + dest_buffer.write(dest_b, mem::replace(&mut task, tmp)); + } + + // Move the source front index and the destination back index one step forward. + f = f.wrapping_add(1); + dest_b = dest_b.wrapping_add(1); + } + + // If stealing into a FIFO queue, stolen tasks need to be reversed. + if dest.flavor == Flavor::Fifo { + for i in 0..batch_size / 2 { + unsafe { + let i1 = dest_b.wrapping_sub(batch_size - i); + let i2 = dest_b.wrapping_sub(i + 1); + let t1 = dest_buffer.read(i1); + let t2 = dest_buffer.read(i2); + dest_buffer.write(i1, t2); + dest_buffer.write(i2, t1); + } + } + } + } + } + + atomic::fence(Ordering::Release); + + // Update the back index in the destination queue. + // + // This ordering could be `Relaxed`, but then thread sanitizer would falsely report data + // races because it doesn't understand fences. + dest.inner.back.store(dest_b, Ordering::Release); + + // Return with success. + Steal::Success(unsafe { task.assume_init() }) + } +} + +impl Clone for Stealer { + fn clone(&self) -> Stealer { + Stealer { + inner: self.inner.clone(), + flavor: self.flavor, + } + } +} + +impl fmt::Debug for Stealer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad("Stealer { .. }") + } +} + +// Bits indicating the state of a slot: +// * If a task has been written into the slot, `WRITE` is set. +// * If a task has been read from the slot, `READ` is set. +// * If the block is being destroyed, `DESTROY` is set. +const WRITE: usize = 1; +const READ: usize = 2; +const DESTROY: usize = 4; + +// Each block covers one "lap" of indices. +const LAP: usize = 64; +// The maximum number of values a block can hold. +const BLOCK_CAP: usize = LAP - 1; +// How many lower bits are reserved for metadata. +const SHIFT: usize = 1; +// Indicates that the block is not the last one. +const HAS_NEXT: usize = 1; + +/// A slot in a block. +struct Slot { + /// The task. + task: UnsafeCell>, + + /// The state of the slot. + state: AtomicUsize, +} + +impl Slot { + /// Waits until a task is written into the slot. + fn wait_write(&self) { + let backoff = Backoff::new(); + while self.state.load(Ordering::Acquire) & WRITE == 0 { + backoff.snooze(); + } + } +} + +/// A block in a linked list. +/// +/// Each block in the list can hold up to `BLOCK_CAP` values. +struct Block { + /// The next block in the linked list. + next: AtomicPtr>, + + /// Slots for values. + slots: [Slot; BLOCK_CAP], +} + +impl Block { + const LAYOUT: Layout = { + let layout = Layout::new::(); + assert!( + layout.size() != 0, + "Block should never be zero-sized, as it has an AtomicPtr field" + ); + layout + }; + + /// Creates an empty block. + fn new() -> Box { + // SAFETY: layout is not zero-sized + let ptr = unsafe { alloc_zeroed(Self::LAYOUT) }; + // Handle allocation failure + if ptr.is_null() { + handle_alloc_error(Self::LAYOUT) + } + // SAFETY: This is safe because: + // [1] `Block::next` (AtomicPtr) may be safely zero initialized. + // [2] `Block::slots` (Array) may be safely zero initialized because of [3, 4]. + // [3] `Slot::task` (UnsafeCell) may be safely zero initialized because it + // holds a MaybeUninit. + // [4] `Slot::state` (AtomicUsize) may be safely zero initialized. + // TODO: unsafe { Box::new_zeroed().assume_init() } + unsafe { Box::from_raw(ptr.cast()) } + } + + /// Waits until the next pointer is set. + fn wait_next(&self) -> *mut Block { + let backoff = Backoff::new(); + loop { + let next = self.next.load(Ordering::Acquire); + if !next.is_null() { + return next; + } + backoff.snooze(); + } + } + + /// Sets the `DESTROY` bit in slots starting from `start` and destroys the block. + unsafe fn destroy(this: *mut Block, count: usize) { + // It is not necessary to set the `DESTROY` bit in the last slot because that slot has + // begun destruction of the block. + for i in (0..count).rev() { + let slot = (*this).slots.get_unchecked(i); + + // Mark the `DESTROY` bit if a thread is still using the slot. + if slot.state.load(Ordering::Acquire) & READ == 0 + && slot.state.fetch_or(DESTROY, Ordering::AcqRel) & READ == 0 + { + // If a thread is still using the slot, it will continue destruction of the block. + return; + } + } + + // No thread is using the block, now it is safe to destroy it. + drop(Box::from_raw(this)); + } +} + +/// A position in a queue. +struct Position { + /// The index in the queue. + index: AtomicUsize, + + /// The block in the linked list. + block: AtomicPtr>, +} + +/// An injector queue. +/// +/// This is a FIFO queue that can be shared among multiple threads. Task schedulers typically have +/// a single injector queue, which is the entry point for new tasks. +/// +/// # Examples +/// +/// ``` +/// use crossbeam_deque::{Injector, Steal}; +/// +/// let q = Injector::new(); +/// q.push(1); +/// q.push(2); +/// +/// assert_eq!(q.steal(), Steal::Success(1)); +/// assert_eq!(q.steal(), Steal::Success(2)); +/// assert_eq!(q.steal(), Steal::Empty); +/// ``` +pub struct Injector { + /// The head of the queue. + head: CachePadded>, + + /// The tail of the queue. + tail: CachePadded>, + + /// Indicates that dropping a `Injector` may drop values of type `T`. + _marker: PhantomData, +} + +unsafe impl Send for Injector {} +unsafe impl Sync for Injector {} + +impl Default for Injector { + fn default() -> Self { + let block = Box::into_raw(Block::::new()); + Self { + head: CachePadded::new(Position { + block: AtomicPtr::new(block), + index: AtomicUsize::new(0), + }), + tail: CachePadded::new(Position { + block: AtomicPtr::new(block), + index: AtomicUsize::new(0), + }), + _marker: PhantomData, + } + } +} + +impl Injector { + /// Creates a new injector queue. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_deque::Injector; + /// + /// let q = Injector::::new(); + /// ``` + pub fn new() -> Injector { + Self::default() + } + + /// Pushes a task into the queue. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_deque::Injector; + /// + /// let w = Injector::new(); + /// w.push(1); + /// w.push(2); + /// ``` + pub fn push(&self, task: T) { + let backoff = Backoff::new(); + let mut tail = self.tail.index.load(Ordering::Acquire); + let mut block = self.tail.block.load(Ordering::Acquire); + let mut next_block = None; + + loop { + // Calculate the offset of the index into the block. + let offset = (tail >> SHIFT) % LAP; + + // If we reached the end of the block, wait until the next one is installed. + if offset == BLOCK_CAP { + backoff.snooze(); + tail = self.tail.index.load(Ordering::Acquire); + block = self.tail.block.load(Ordering::Acquire); + continue; + } + + // If we're going to have to install the next block, allocate it in advance in order to + // make the wait for other threads as short as possible. + if offset + 1 == BLOCK_CAP && next_block.is_none() { + next_block = Some(Block::::new()); + } + + let new_tail = tail + (1 << SHIFT); + + // Try advancing the tail forward. + match self.tail.index.compare_exchange_weak( + tail, + new_tail, + Ordering::SeqCst, + Ordering::Acquire, + ) { + Ok(_) => unsafe { + // If we've reached the end of the block, install the next one. + if offset + 1 == BLOCK_CAP { + let next_block = Box::into_raw(next_block.unwrap()); + let next_index = new_tail.wrapping_add(1 << SHIFT); + + self.tail.block.store(next_block, Ordering::Release); + self.tail.index.store(next_index, Ordering::Release); + (*block).next.store(next_block, Ordering::Release); + } + + // Write the task into the slot. + let slot = (*block).slots.get_unchecked(offset); + slot.task.get().write(MaybeUninit::new(task)); + slot.state.fetch_or(WRITE, Ordering::Release); + + return; + }, + Err(t) => { + tail = t; + block = self.tail.block.load(Ordering::Acquire); + backoff.spin(); + } + } + } + } + + /// Steals a task from the queue. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_deque::{Injector, Steal}; + /// + /// let q = Injector::new(); + /// q.push(1); + /// q.push(2); + /// + /// assert_eq!(q.steal(), Steal::Success(1)); + /// assert_eq!(q.steal(), Steal::Success(2)); + /// assert_eq!(q.steal(), Steal::Empty); + /// ``` + pub fn steal(&self) -> Steal { + let mut head; + let mut block; + let mut offset; + + let backoff = Backoff::new(); + loop { + head = self.head.index.load(Ordering::Acquire); + block = self.head.block.load(Ordering::Acquire); + + // Calculate the offset of the index into the block. + offset = (head >> SHIFT) % LAP; + + // If we reached the end of the block, wait until the next one is installed. + if offset == BLOCK_CAP { + backoff.snooze(); + } else { + break; + } + } + + let mut new_head = head + (1 << SHIFT); + + if new_head & HAS_NEXT == 0 { + atomic::fence(Ordering::SeqCst); + let tail = self.tail.index.load(Ordering::Relaxed); + + // If the tail equals the head, that means the queue is empty. + if head >> SHIFT == tail >> SHIFT { + return Steal::Empty; + } + + // If head and tail are not in the same block, set `HAS_NEXT` in head. + if (head >> SHIFT) / LAP != (tail >> SHIFT) / LAP { + new_head |= HAS_NEXT; + } + } + + // Try moving the head index forward. + if self + .head + .index + .compare_exchange_weak(head, new_head, Ordering::SeqCst, Ordering::Acquire) + .is_err() + { + return Steal::Retry; + } + + unsafe { + // If we've reached the end of the block, move to the next one. + if offset + 1 == BLOCK_CAP { + let next = (*block).wait_next(); + let mut next_index = (new_head & !HAS_NEXT).wrapping_add(1 << SHIFT); + if !(*next).next.load(Ordering::Relaxed).is_null() { + next_index |= HAS_NEXT; + } + + self.head.block.store(next, Ordering::Release); + self.head.index.store(next_index, Ordering::Release); + } + + // Read the task. + let slot = (*block).slots.get_unchecked(offset); + slot.wait_write(); + let task = slot.task.get().read().assume_init(); + + // Destroy the block if we've reached the end, or if another thread wanted to destroy + // but couldn't because we were busy reading from the slot. + if (offset + 1 == BLOCK_CAP) + || (slot.state.fetch_or(READ, Ordering::AcqRel) & DESTROY != 0) + { + Block::destroy(block, offset); + } + + Steal::Success(task) + } + } + + /// Steals a batch of tasks and pushes them into a worker. + /// + /// How many tasks exactly will be stolen is not specified. That said, this method will try to + /// steal around half of the tasks in the queue, but also not more than some constant limit. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_deque::{Injector, Worker}; + /// + /// let q = Injector::new(); + /// q.push(1); + /// q.push(2); + /// q.push(3); + /// q.push(4); + /// + /// let w = Worker::new_fifo(); + /// let _ = q.steal_batch(&w); + /// assert_eq!(w.pop(), Some(1)); + /// assert_eq!(w.pop(), Some(2)); + /// ``` + pub fn steal_batch(&self, dest: &Worker) -> Steal<()> { + self.steal_batch_with_limit(dest, MAX_BATCH) + } + + /// Steals no more than of tasks and pushes them into a worker. + /// + /// How many tasks exactly will be stolen is not specified. That said, this method will try to + /// steal around half of the tasks in the queue, but also not more than some constant limit. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_deque::{Injector, Worker}; + /// + /// let q = Injector::new(); + /// q.push(1); + /// q.push(2); + /// q.push(3); + /// q.push(4); + /// q.push(5); + /// q.push(6); + /// + /// let w = Worker::new_fifo(); + /// let _ = q.steal_batch_with_limit(&w, 2); + /// assert_eq!(w.pop(), Some(1)); + /// assert_eq!(w.pop(), Some(2)); + /// assert_eq!(w.pop(), None); + /// + /// q.push(7); + /// q.push(8); + /// // Setting a large limit does not guarantee that all elements will be popped. In this case, + /// // half of the elements are currently popped, but the number of popped elements is considered + /// // an implementation detail that may be changed in the future. + /// let _ = q.steal_batch_with_limit(&w, std::usize::MAX); + /// assert_eq!(w.len(), 3); + /// ``` + pub fn steal_batch_with_limit(&self, dest: &Worker, limit: usize) -> Steal<()> { + assert!(limit > 0); + let mut head; + let mut block; + let mut offset; + + let backoff = Backoff::new(); + loop { + head = self.head.index.load(Ordering::Acquire); + block = self.head.block.load(Ordering::Acquire); + + // Calculate the offset of the index into the block. + offset = (head >> SHIFT) % LAP; + + // If we reached the end of the block, wait until the next one is installed. + if offset == BLOCK_CAP { + backoff.snooze(); + } else { + break; + } + } + + let mut new_head = head; + let advance; + + if new_head & HAS_NEXT == 0 { + atomic::fence(Ordering::SeqCst); + let tail = self.tail.index.load(Ordering::Relaxed); + + // If the tail equals the head, that means the queue is empty. + if head >> SHIFT == tail >> SHIFT { + return Steal::Empty; + } + + // If head and tail are not in the same block, set `HAS_NEXT` in head. Also, calculate + // the right batch size to steal. + if (head >> SHIFT) / LAP != (tail >> SHIFT) / LAP { + new_head |= HAS_NEXT; + // We can steal all tasks till the end of the block. + advance = (BLOCK_CAP - offset).min(limit); + } else { + let len = (tail - head) >> SHIFT; + // Steal half of the available tasks. + advance = ((len + 1) / 2).min(limit); + } + } else { + // We can steal all tasks till the end of the block. + advance = (BLOCK_CAP - offset).min(limit); + } + + new_head += advance << SHIFT; + let new_offset = offset + advance; + + // Try moving the head index forward. + if self + .head + .index + .compare_exchange_weak(head, new_head, Ordering::SeqCst, Ordering::Acquire) + .is_err() + { + return Steal::Retry; + } + + // Reserve capacity for the stolen batch. + let batch_size = new_offset - offset; + dest.reserve(batch_size); + + // Get the destination buffer and back index. + let dest_buffer = dest.buffer.get(); + let dest_b = dest.inner.back.load(Ordering::Relaxed); + + unsafe { + // If we've reached the end of the block, move to the next one. + if new_offset == BLOCK_CAP { + let next = (*block).wait_next(); + let mut next_index = (new_head & !HAS_NEXT).wrapping_add(1 << SHIFT); + if !(*next).next.load(Ordering::Relaxed).is_null() { + next_index |= HAS_NEXT; + } + + self.head.block.store(next, Ordering::Release); + self.head.index.store(next_index, Ordering::Release); + } + + // Copy values from the injector into the destination queue. + match dest.flavor { + Flavor::Fifo => { + for i in 0..batch_size { + // Read the task. + let slot = (*block).slots.get_unchecked(offset + i); + slot.wait_write(); + let task = slot.task.get().read(); + + // Write it into the destination queue. + dest_buffer.write(dest_b.wrapping_add(i as isize), task); + } + } + + Flavor::Lifo => { + for i in 0..batch_size { + // Read the task. + let slot = (*block).slots.get_unchecked(offset + i); + slot.wait_write(); + let task = slot.task.get().read(); + + // Write it into the destination queue. + dest_buffer.write(dest_b.wrapping_add((batch_size - 1 - i) as isize), task); + } + } + } + + atomic::fence(Ordering::Release); + + // Update the back index in the destination queue. + // + // This ordering could be `Relaxed`, but then thread sanitizer would falsely report + // data races because it doesn't understand fences. + dest.inner + .back + .store(dest_b.wrapping_add(batch_size as isize), Ordering::Release); + + // Destroy the block if we've reached the end, or if another thread wanted to destroy + // but couldn't because we were busy reading from the slot. + if new_offset == BLOCK_CAP { + Block::destroy(block, offset); + } else { + for i in offset..new_offset { + let slot = (*block).slots.get_unchecked(i); + + if slot.state.fetch_or(READ, Ordering::AcqRel) & DESTROY != 0 { + Block::destroy(block, offset); + break; + } + } + } + + Steal::Success(()) + } + } + + /// Steals a batch of tasks, pushes them into a worker, and pops a task from that worker. + /// + /// How many tasks exactly will be stolen is not specified. That said, this method will try to + /// steal around half of the tasks in the queue, but also not more than some constant limit. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_deque::{Injector, Steal, Worker}; + /// + /// let q = Injector::new(); + /// q.push(1); + /// q.push(2); + /// q.push(3); + /// q.push(4); + /// + /// let w = Worker::new_fifo(); + /// assert_eq!(q.steal_batch_and_pop(&w), Steal::Success(1)); + /// assert_eq!(w.pop(), Some(2)); + /// ``` + pub fn steal_batch_and_pop(&self, dest: &Worker) -> Steal { + // TODO: we use `MAX_BATCH + 1` as the hard limit for Injecter as the performance is slightly + // better, but we may change it in the future to be compatible with the same method in Stealer. + self.steal_batch_with_limit_and_pop(dest, MAX_BATCH + 1) + } + + /// Steals no more than `limit` of tasks, pushes them into a worker, and pops a task from that worker. + /// + /// How many tasks exactly will be stolen is not specified. That said, this method will try to + /// steal around half of the tasks in the queue, but also not more than the given limit. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_deque::{Injector, Steal, Worker}; + /// + /// let q = Injector::new(); + /// q.push(1); + /// q.push(2); + /// q.push(3); + /// q.push(4); + /// q.push(5); + /// q.push(6); + /// + /// let w = Worker::new_fifo(); + /// assert_eq!(q.steal_batch_with_limit_and_pop(&w, 2), Steal::Success(1)); + /// assert_eq!(w.pop(), Some(2)); + /// assert_eq!(w.pop(), None); + /// + /// q.push(7); + /// // Setting a large limit does not guarantee that all elements will be popped. In this case, + /// // half of the elements are currently popped, but the number of popped elements is considered + /// // an implementation detail that may be changed in the future. + /// assert_eq!(q.steal_batch_with_limit_and_pop(&w, std::usize::MAX), Steal::Success(3)); + /// assert_eq!(w.pop(), Some(4)); + /// assert_eq!(w.pop(), Some(5)); + /// assert_eq!(w.pop(), None); + /// ``` + pub fn steal_batch_with_limit_and_pop(&self, dest: &Worker, limit: usize) -> Steal { + assert!(limit > 0); + let mut head; + let mut block; + let mut offset; + + let backoff = Backoff::new(); + loop { + head = self.head.index.load(Ordering::Acquire); + block = self.head.block.load(Ordering::Acquire); + + // Calculate the offset of the index into the block. + offset = (head >> SHIFT) % LAP; + + // If we reached the end of the block, wait until the next one is installed. + if offset == BLOCK_CAP { + backoff.snooze(); + } else { + break; + } + } + + let mut new_head = head; + let advance; + + if new_head & HAS_NEXT == 0 { + atomic::fence(Ordering::SeqCst); + let tail = self.tail.index.load(Ordering::Relaxed); + + // If the tail equals the head, that means the queue is empty. + if head >> SHIFT == tail >> SHIFT { + return Steal::Empty; + } + + // If head and tail are not in the same block, set `HAS_NEXT` in head. + if (head >> SHIFT) / LAP != (tail >> SHIFT) / LAP { + new_head |= HAS_NEXT; + // We can steal all tasks till the end of the block. + advance = (BLOCK_CAP - offset).min(limit); + } else { + let len = (tail - head) >> SHIFT; + // Steal half of the available tasks. + advance = ((len + 1) / 2).min(limit); + } + } else { + // We can steal all tasks till the end of the block. + advance = (BLOCK_CAP - offset).min(limit); + } + + new_head += advance << SHIFT; + let new_offset = offset + advance; + + // Try moving the head index forward. + if self + .head + .index + .compare_exchange_weak(head, new_head, Ordering::SeqCst, Ordering::Acquire) + .is_err() + { + return Steal::Retry; + } + + // Reserve capacity for the stolen batch. + let batch_size = new_offset - offset - 1; + dest.reserve(batch_size); + + // Get the destination buffer and back index. + let dest_buffer = dest.buffer.get(); + let dest_b = dest.inner.back.load(Ordering::Relaxed); + + unsafe { + // If we've reached the end of the block, move to the next one. + if new_offset == BLOCK_CAP { + let next = (*block).wait_next(); + let mut next_index = (new_head & !HAS_NEXT).wrapping_add(1 << SHIFT); + if !(*next).next.load(Ordering::Relaxed).is_null() { + next_index |= HAS_NEXT; + } + + self.head.block.store(next, Ordering::Release); + self.head.index.store(next_index, Ordering::Release); + } + + // Read the task. + let slot = (*block).slots.get_unchecked(offset); + slot.wait_write(); + let task = slot.task.get().read(); + + match dest.flavor { + Flavor::Fifo => { + // Copy values from the injector into the destination queue. + for i in 0..batch_size { + // Read the task. + let slot = (*block).slots.get_unchecked(offset + i + 1); + slot.wait_write(); + let task = slot.task.get().read(); + + // Write it into the destination queue. + dest_buffer.write(dest_b.wrapping_add(i as isize), task); + } + } + + Flavor::Lifo => { + // Copy values from the injector into the destination queue. + for i in 0..batch_size { + // Read the task. + let slot = (*block).slots.get_unchecked(offset + i + 1); + slot.wait_write(); + let task = slot.task.get().read(); + + // Write it into the destination queue. + dest_buffer.write(dest_b.wrapping_add((batch_size - 1 - i) as isize), task); + } + } + } + + atomic::fence(Ordering::Release); + + // Update the back index in the destination queue. + // + // This ordering could be `Relaxed`, but then thread sanitizer would falsely report + // data races because it doesn't understand fences. + dest.inner + .back + .store(dest_b.wrapping_add(batch_size as isize), Ordering::Release); + + // Destroy the block if we've reached the end, or if another thread wanted to destroy + // but couldn't because we were busy reading from the slot. + if new_offset == BLOCK_CAP { + Block::destroy(block, offset); + } else { + for i in offset..new_offset { + let slot = (*block).slots.get_unchecked(i); + + if slot.state.fetch_or(READ, Ordering::AcqRel) & DESTROY != 0 { + Block::destroy(block, offset); + break; + } + } + } + + Steal::Success(task.assume_init()) + } + } + + /// Returns `true` if the queue is empty. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_deque::Injector; + /// + /// let q = Injector::new(); + /// + /// assert!(q.is_empty()); + /// q.push(1); + /// assert!(!q.is_empty()); + /// ``` + pub fn is_empty(&self) -> bool { + let head = self.head.index.load(Ordering::SeqCst); + let tail = self.tail.index.load(Ordering::SeqCst); + head >> SHIFT == tail >> SHIFT + } + + /// Returns the number of tasks in the queue. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_deque::Injector; + /// + /// let q = Injector::new(); + /// + /// assert_eq!(q.len(), 0); + /// q.push(1); + /// assert_eq!(q.len(), 1); + /// q.push(1); + /// assert_eq!(q.len(), 2); + /// ``` + pub fn len(&self) -> usize { + loop { + // Load the tail index, then load the head index. + let mut tail = self.tail.index.load(Ordering::SeqCst); + let mut head = self.head.index.load(Ordering::SeqCst); + + // If the tail index didn't change, we've got consistent indices to work with. + if self.tail.index.load(Ordering::SeqCst) == tail { + // Erase the lower bits. + tail &= !((1 << SHIFT) - 1); + head &= !((1 << SHIFT) - 1); + + // Fix up indices if they fall onto block ends. + if (tail >> SHIFT) & (LAP - 1) == LAP - 1 { + tail = tail.wrapping_add(1 << SHIFT); + } + if (head >> SHIFT) & (LAP - 1) == LAP - 1 { + head = head.wrapping_add(1 << SHIFT); + } + + // Rotate indices so that head falls into the first block. + let lap = (head >> SHIFT) / LAP; + tail = tail.wrapping_sub((lap * LAP) << SHIFT); + head = head.wrapping_sub((lap * LAP) << SHIFT); + + // Remove the lower bits. + tail >>= SHIFT; + head >>= SHIFT; + + // Return the difference minus the number of blocks between tail and head. + return tail - head - tail / LAP; + } + } + } +} + +impl Drop for Injector { + fn drop(&mut self) { + let mut head = *self.head.index.get_mut(); + let mut tail = *self.tail.index.get_mut(); + let mut block = *self.head.block.get_mut(); + + // Erase the lower bits. + head &= !((1 << SHIFT) - 1); + tail &= !((1 << SHIFT) - 1); + + unsafe { + // Drop all values between `head` and `tail` and deallocate the heap-allocated blocks. + while head != tail { + let offset = (head >> SHIFT) % LAP; + + if offset < BLOCK_CAP { + // Drop the task in the slot. + let slot = (*block).slots.get_unchecked(offset); + (*slot.task.get()).assume_init_drop(); + } else { + // Deallocate the block and move to the next one. + let next = *(*block).next.get_mut(); + drop(Box::from_raw(block)); + block = next; + } + + head = head.wrapping_add(1 << SHIFT); + } + + // Deallocate the last remaining block. + drop(Box::from_raw(block)); + } + } +} + +impl fmt::Debug for Injector { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad("Worker { .. }") + } +} + +/// Possible outcomes of a steal operation. +/// +/// # Examples +/// +/// There are lots of ways to chain results of steal operations together: +/// +/// ``` +/// use crossbeam_deque::Steal::{self, Empty, Retry, Success}; +/// +/// let collect = |v: Vec>| v.into_iter().collect::>(); +/// +/// assert_eq!(collect(vec![Empty, Empty, Empty]), Empty); +/// assert_eq!(collect(vec![Empty, Retry, Empty]), Retry); +/// assert_eq!(collect(vec![Retry, Success(1), Empty]), Success(1)); +/// +/// assert_eq!(collect(vec![Empty, Empty]).or_else(|| Retry), Retry); +/// assert_eq!(collect(vec![Retry, Empty]).or_else(|| Success(1)), Success(1)); +/// ``` +#[must_use] +#[derive(PartialEq, Eq, Copy, Clone)] +pub enum Steal { + /// The queue was empty at the time of stealing. + Empty, + + /// At least one task was successfully stolen. + Success(T), + + /// The steal operation needs to be retried. + Retry, +} + +impl Steal { + /// Returns `true` if the queue was empty at the time of stealing. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_deque::Steal::{Empty, Retry, Success}; + /// + /// assert!(!Success(7).is_empty()); + /// assert!(!Retry::.is_empty()); + /// + /// assert!(Empty::.is_empty()); + /// ``` + pub fn is_empty(&self) -> bool { + match self { + Steal::Empty => true, + _ => false, + } + } + + /// Returns `true` if at least one task was stolen. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_deque::Steal::{Empty, Retry, Success}; + /// + /// assert!(!Empty::.is_success()); + /// assert!(!Retry::.is_success()); + /// + /// assert!(Success(7).is_success()); + /// ``` + pub fn is_success(&self) -> bool { + match self { + Steal::Success(_) => true, + _ => false, + } + } + + /// Returns `true` if the steal operation needs to be retried. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_deque::Steal::{Empty, Retry, Success}; + /// + /// assert!(!Empty::.is_retry()); + /// assert!(!Success(7).is_retry()); + /// + /// assert!(Retry::.is_retry()); + /// ``` + pub fn is_retry(&self) -> bool { + match self { + Steal::Retry => true, + _ => false, + } + } + + /// Returns the result of the operation, if successful. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_deque::Steal::{Empty, Retry, Success}; + /// + /// assert_eq!(Empty::.success(), None); + /// assert_eq!(Retry::.success(), None); + /// + /// assert_eq!(Success(7).success(), Some(7)); + /// ``` + pub fn success(self) -> Option { + match self { + Steal::Success(res) => Some(res), + _ => None, + } + } + + /// If no task was stolen, attempts another steal operation. + /// + /// Returns this steal result if it is `Success`. Otherwise, closure `f` is invoked and then: + /// + /// * If the second steal resulted in `Success`, it is returned. + /// * If both steals were unsuccessful but any resulted in `Retry`, then `Retry` is returned. + /// * If both resulted in `None`, then `None` is returned. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_deque::Steal::{Empty, Retry, Success}; + /// + /// assert_eq!(Success(1).or_else(|| Success(2)), Success(1)); + /// assert_eq!(Retry.or_else(|| Success(2)), Success(2)); + /// + /// assert_eq!(Retry.or_else(|| Empty), Retry::); + /// assert_eq!(Empty.or_else(|| Retry), Retry::); + /// + /// assert_eq!(Empty.or_else(|| Empty), Empty::); + /// ``` + pub fn or_else(self, f: F) -> Steal + where + F: FnOnce() -> Steal, + { + match self { + Steal::Empty => f(), + Steal::Success(_) => self, + Steal::Retry => { + if let Steal::Success(res) = f() { + Steal::Success(res) + } else { + Steal::Retry + } + } + } + } +} + +impl fmt::Debug for Steal { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Steal::Empty => f.pad("Empty"), + Steal::Success(_) => f.pad("Success(..)"), + Steal::Retry => f.pad("Retry"), + } + } +} + +impl FromIterator> for Steal { + /// Consumes items until a `Success` is found and returns it. + /// + /// If no `Success` was found, but there was at least one `Retry`, then returns `Retry`. + /// Otherwise, `Empty` is returned. + fn from_iter(iter: I) -> Steal + where + I: IntoIterator>, + { + let mut retry = false; + for s in iter { + match &s { + Steal::Empty => {} + Steal::Success(_) => return s, + Steal::Retry => retry = true, + } + } + + if retry { + Steal::Retry + } else { + Steal::Empty + } + } +} diff --git a/anneal/v2/vendor/crossbeam-deque/src/lib.rs b/anneal/v2/vendor/crossbeam-deque/src/lib.rs new file mode 100644 index 0000000000..0d856c8541 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-deque/src/lib.rs @@ -0,0 +1,106 @@ +//! Concurrent work-stealing deques. +//! +//! These data structures are most commonly used in work-stealing schedulers. The typical setup +//! involves a number of threads, each having its own FIFO or LIFO queue (*worker*). There is also +//! one global FIFO queue (*injector*) and a list of references to *worker* queues that are able to +//! steal tasks (*stealers*). +//! +//! We spawn a new task onto the scheduler by pushing it into the *injector* queue. Each worker +//! thread waits in a loop until it finds the next task to run and then runs it. To find a task, it +//! first looks into its local *worker* queue, and then into the *injector* and *stealers*. +//! +//! # Queues +//! +//! [`Injector`] is a FIFO queue, where tasks are pushed and stolen from opposite ends. It is +//! shared among threads and is usually the entry point for new tasks. +//! +//! [`Worker`] has two constructors: +//! +//! * [`new_fifo()`] - Creates a FIFO queue, in which tasks are pushed and popped from opposite +//! ends. +//! * [`new_lifo()`] - Creates a LIFO queue, in which tasks are pushed and popped from the same +//! end. +//! +//! Each [`Worker`] is owned by a single thread and supports only push and pop operations. +//! +//! Method [`stealer()`] creates a [`Stealer`] that may be shared among threads and can only steal +//! tasks from its [`Worker`]. Tasks are stolen from the end opposite to where they get pushed. +//! +//! # Stealing +//! +//! Steal operations come in three flavors: +//! +//! 1. [`steal()`] - Steals one task. +//! 2. [`steal_batch()`] - Steals a batch of tasks and moves them into another worker. +//! 3. [`steal_batch_and_pop()`] - Steals a batch of tasks, moves them into another queue, and pops +//! one task from that worker. +//! +//! In contrast to push and pop operations, stealing can spuriously fail with [`Steal::Retry`], in +//! which case the steal operation needs to be retried. +//! +//! # Examples +//! +//! Suppose a thread in a work-stealing scheduler is idle and looking for the next task to run. To +//! find an available task, it might do the following: +//! +//! 1. Try popping one task from the local worker queue. +//! 2. Try stealing a batch of tasks from the global injector queue. +//! 3. Try stealing one task from another thread using the stealer list. +//! +//! An implementation of this work-stealing strategy: +//! +//! ``` +//! use crossbeam_deque::{Injector, Stealer, Worker}; +//! use std::iter; +//! +//! fn find_task( +//! local: &Worker, +//! global: &Injector, +//! stealers: &[Stealer], +//! ) -> Option { +//! // Pop a task from the local queue, if not empty. +//! local.pop().or_else(|| { +//! // Otherwise, we need to look for a task elsewhere. +//! iter::repeat_with(|| { +//! // Try stealing a batch of tasks from the global queue. +//! global.steal_batch_and_pop(local) +//! // Or try stealing a task from one of the other threads. +//! .or_else(|| stealers.iter().map(|s| s.steal()).collect()) +//! }) +//! // Loop while no task was stolen and any steal operation needs to be retried. +//! .find(|s| !s.is_retry()) +//! // Extract the stolen task, if there is one. +//! .and_then(|s| s.success()) +//! }) +//! } +//! ``` +//! +//! [`new_fifo()`]: Worker::new_fifo +//! [`new_lifo()`]: Worker::new_lifo +//! [`stealer()`]: Worker::stealer +//! [`steal()`]: Stealer::steal +//! [`steal_batch()`]: Stealer::steal_batch +//! [`steal_batch_and_pop()`]: Stealer::steal_batch_and_pop + +#![no_std] +#![doc(test( + no_crate_inject, + attr( + deny(warnings, rust_2018_idioms), + allow(dead_code, unused_assignments, unused_variables) + ) +))] +#![warn( + missing_docs, + missing_debug_implementations, + rust_2018_idioms, + unreachable_pub +)] + +#[cfg(feature = "std")] +extern crate std; + +#[cfg(feature = "std")] +mod deque; +#[cfg(feature = "std")] +pub use crate::deque::{Injector, Steal, Stealer, Worker}; diff --git a/anneal/v2/vendor/crossbeam-deque/tests/fifo.rs b/anneal/v2/vendor/crossbeam-deque/tests/fifo.rs new file mode 100644 index 0000000000..f98737b58d --- /dev/null +++ b/anneal/v2/vendor/crossbeam-deque/tests/fifo.rs @@ -0,0 +1,357 @@ +use std::sync::atomic::Ordering::SeqCst; +use std::sync::atomic::{AtomicBool, AtomicUsize}; +use std::sync::{Arc, Mutex}; + +use crossbeam_deque::Steal::{Empty, Success}; +use crossbeam_deque::Worker; +use crossbeam_utils::thread::scope; +use rand::Rng; + +#[test] +fn smoke() { + let w = Worker::new_fifo(); + let s = w.stealer(); + assert_eq!(w.pop(), None); + assert_eq!(s.steal(), Empty); + + w.push(1); + assert_eq!(w.pop(), Some(1)); + assert_eq!(w.pop(), None); + assert_eq!(s.steal(), Empty); + + w.push(2); + assert_eq!(s.steal(), Success(2)); + assert_eq!(s.steal(), Empty); + assert_eq!(w.pop(), None); + + w.push(3); + w.push(4); + w.push(5); + assert_eq!(s.steal(), Success(3)); + assert_eq!(s.steal(), Success(4)); + assert_eq!(s.steal(), Success(5)); + assert_eq!(s.steal(), Empty); + + w.push(6); + w.push(7); + w.push(8); + w.push(9); + assert_eq!(w.pop(), Some(6)); + assert_eq!(s.steal(), Success(7)); + assert_eq!(w.pop(), Some(8)); + assert_eq!(w.pop(), Some(9)); + assert_eq!(w.pop(), None); +} + +#[test] +fn is_empty() { + let w = Worker::new_fifo(); + let s = w.stealer(); + + assert!(w.is_empty()); + w.push(1); + assert!(!w.is_empty()); + w.push(2); + assert!(!w.is_empty()); + let _ = w.pop(); + assert!(!w.is_empty()); + let _ = w.pop(); + assert!(w.is_empty()); + + assert!(s.is_empty()); + w.push(1); + assert!(!s.is_empty()); + w.push(2); + assert!(!s.is_empty()); + let _ = s.steal(); + assert!(!s.is_empty()); + let _ = s.steal(); + assert!(s.is_empty()); +} + +#[test] +fn spsc() { + #[cfg(miri)] + const STEPS: usize = 500; + #[cfg(not(miri))] + const STEPS: usize = 50_000; + + let w = Worker::new_fifo(); + let s = w.stealer(); + + scope(|scope| { + scope.spawn(|_| { + for i in 0..STEPS { + loop { + if let Success(v) = s.steal() { + assert_eq!(i, v); + break; + } + } + } + + assert_eq!(s.steal(), Empty); + }); + + for i in 0..STEPS { + w.push(i); + } + }) + .unwrap(); +} + +#[test] +fn stampede() { + const THREADS: usize = 8; + #[cfg(miri)] + const COUNT: usize = 500; + #[cfg(not(miri))] + const COUNT: usize = 50_000; + + let w = Worker::new_fifo(); + + for i in 0..COUNT { + w.push(Box::new(i + 1)); + } + let remaining = Arc::new(AtomicUsize::new(COUNT)); + + scope(|scope| { + for _ in 0..THREADS { + let s = w.stealer(); + let remaining = remaining.clone(); + + scope.spawn(move |_| { + let mut last = 0; + while remaining.load(SeqCst) > 0 { + if let Success(x) = s.steal() { + assert!(last < *x); + last = *x; + remaining.fetch_sub(1, SeqCst); + } + } + }); + } + + let mut last = 0; + while remaining.load(SeqCst) > 0 { + if let Some(x) = w.pop() { + assert!(last < *x); + last = *x; + remaining.fetch_sub(1, SeqCst); + } + } + }) + .unwrap(); +} + +#[test] +fn stress() { + const THREADS: usize = 8; + #[cfg(miri)] + const COUNT: usize = 500; + #[cfg(not(miri))] + const COUNT: usize = 50_000; + + let w = Worker::new_fifo(); + let done = Arc::new(AtomicBool::new(false)); + let hits = Arc::new(AtomicUsize::new(0)); + + scope(|scope| { + for _ in 0..THREADS { + let s = w.stealer(); + let done = done.clone(); + let hits = hits.clone(); + + scope.spawn(move |_| { + let w2 = Worker::new_fifo(); + + while !done.load(SeqCst) { + if let Success(_) = s.steal() { + hits.fetch_add(1, SeqCst); + } + + let _ = s.steal_batch(&w2); + + if let Success(_) = s.steal_batch_and_pop(&w2) { + hits.fetch_add(1, SeqCst); + } + + while w2.pop().is_some() { + hits.fetch_add(1, SeqCst); + } + } + }); + } + + let mut rng = rand::thread_rng(); + let mut expected = 0; + while expected < COUNT { + if rng.gen_range(0..3) == 0 { + while w.pop().is_some() { + hits.fetch_add(1, SeqCst); + } + } else { + w.push(expected); + expected += 1; + } + } + + while hits.load(SeqCst) < COUNT { + while w.pop().is_some() { + hits.fetch_add(1, SeqCst); + } + } + done.store(true, SeqCst); + }) + .unwrap(); +} + +#[cfg_attr(miri, ignore)] // Miri is too slow +#[test] +fn no_starvation() { + const THREADS: usize = 8; + const COUNT: usize = 50_000; + + let w = Worker::new_fifo(); + let done = Arc::new(AtomicBool::new(false)); + let mut all_hits = Vec::new(); + + scope(|scope| { + for _ in 0..THREADS { + let s = w.stealer(); + let done = done.clone(); + let hits = Arc::new(AtomicUsize::new(0)); + all_hits.push(hits.clone()); + + scope.spawn(move |_| { + let w2 = Worker::new_fifo(); + + while !done.load(SeqCst) { + if let Success(_) = s.steal() { + hits.fetch_add(1, SeqCst); + } + + let _ = s.steal_batch(&w2); + + if let Success(_) = s.steal_batch_and_pop(&w2) { + hits.fetch_add(1, SeqCst); + } + + while w2.pop().is_some() { + hits.fetch_add(1, SeqCst); + } + } + }); + } + + let mut rng = rand::thread_rng(); + let mut my_hits = 0; + loop { + for i in 0..rng.gen_range(0..COUNT) { + if rng.gen_range(0..3) == 0 && my_hits == 0 { + while w.pop().is_some() { + my_hits += 1; + } + } else { + w.push(i); + } + } + + if my_hits > 0 && all_hits.iter().all(|h| h.load(SeqCst) > 0) { + break; + } + } + done.store(true, SeqCst); + }) + .unwrap(); +} + +#[test] +fn destructors() { + #[cfg(miri)] + const THREADS: usize = 2; + #[cfg(not(miri))] + const THREADS: usize = 8; + #[cfg(miri)] + const COUNT: usize = 500; + #[cfg(not(miri))] + const COUNT: usize = 50_000; + #[cfg(miri)] + const STEPS: usize = 100; + #[cfg(not(miri))] + const STEPS: usize = 1000; + + struct Elem(usize, Arc>>); + + impl Drop for Elem { + fn drop(&mut self) { + self.1.lock().unwrap().push(self.0); + } + } + + let w = Worker::new_fifo(); + let dropped = Arc::new(Mutex::new(Vec::new())); + let remaining = Arc::new(AtomicUsize::new(COUNT)); + + for i in 0..COUNT { + w.push(Elem(i, dropped.clone())); + } + + scope(|scope| { + for _ in 0..THREADS { + let remaining = remaining.clone(); + let s = w.stealer(); + + scope.spawn(move |_| { + let w2 = Worker::new_fifo(); + let mut cnt = 0; + + while cnt < STEPS { + if let Success(_) = s.steal() { + cnt += 1; + remaining.fetch_sub(1, SeqCst); + } + + let _ = s.steal_batch(&w2); + + if let Success(_) = s.steal_batch_and_pop(&w2) { + cnt += 1; + remaining.fetch_sub(1, SeqCst); + } + + while w2.pop().is_some() { + cnt += 1; + remaining.fetch_sub(1, SeqCst); + } + } + }); + } + + for _ in 0..STEPS { + if w.pop().is_some() { + remaining.fetch_sub(1, SeqCst); + } + } + }) + .unwrap(); + + let rem = remaining.load(SeqCst); + assert!(rem > 0); + + { + let mut v = dropped.lock().unwrap(); + assert_eq!(v.len(), COUNT - rem); + v.clear(); + } + + drop(w); + + { + let mut v = dropped.lock().unwrap(); + assert_eq!(v.len(), rem); + v.sort_unstable(); + for pair in v.windows(2) { + assert_eq!(pair[0] + 1, pair[1]); + } + } +} diff --git a/anneal/v2/vendor/crossbeam-deque/tests/injector.rs b/anneal/v2/vendor/crossbeam-deque/tests/injector.rs new file mode 100644 index 0000000000..5f6c3e98e1 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-deque/tests/injector.rs @@ -0,0 +1,391 @@ +use std::sync::atomic::Ordering::SeqCst; +use std::sync::atomic::{AtomicBool, AtomicUsize}; +use std::sync::{Arc, Mutex}; + +use crossbeam_deque::Steal::{Empty, Success}; +use crossbeam_deque::{Injector, Worker}; +use crossbeam_utils::thread::scope; +use rand::Rng; + +#[test] +fn smoke() { + let q = Injector::new(); + assert_eq!(q.steal(), Empty); + + q.push(1); + q.push(2); + assert_eq!(q.steal(), Success(1)); + assert_eq!(q.steal(), Success(2)); + assert_eq!(q.steal(), Empty); + + q.push(3); + assert_eq!(q.steal(), Success(3)); + assert_eq!(q.steal(), Empty); +} + +#[test] +fn is_empty() { + let q = Injector::new(); + assert!(q.is_empty()); + + q.push(1); + assert!(!q.is_empty()); + q.push(2); + assert!(!q.is_empty()); + + let _ = q.steal(); + assert!(!q.is_empty()); + let _ = q.steal(); + assert!(q.is_empty()); + + q.push(3); + assert!(!q.is_empty()); + let _ = q.steal(); + assert!(q.is_empty()); +} + +#[test] +fn spsc() { + #[cfg(miri)] + const COUNT: usize = 500; + #[cfg(not(miri))] + const COUNT: usize = 100_000; + + let q = Injector::new(); + + scope(|scope| { + scope.spawn(|_| { + for i in 0..COUNT { + loop { + if let Success(v) = q.steal() { + assert_eq!(i, v); + break; + } + #[cfg(miri)] + std::hint::spin_loop(); + } + } + + assert_eq!(q.steal(), Empty); + }); + + for i in 0..COUNT { + q.push(i); + } + }) + .unwrap(); +} + +#[test] +fn mpmc() { + #[cfg(miri)] + const COUNT: usize = 500; + #[cfg(not(miri))] + const COUNT: usize = 25_000; + const THREADS: usize = 4; + + let q = Injector::new(); + let v = (0..COUNT).map(|_| AtomicUsize::new(0)).collect::>(); + + scope(|scope| { + for _ in 0..THREADS { + scope.spawn(|_| { + for i in 0..COUNT { + q.push(i); + } + }); + } + + for _ in 0..THREADS { + scope.spawn(|_| { + for _ in 0..COUNT { + loop { + if let Success(n) = q.steal() { + v[n].fetch_add(1, SeqCst); + break; + } + #[cfg(miri)] + std::hint::spin_loop(); + } + } + }); + } + }) + .unwrap(); + + for c in v { + assert_eq!(c.load(SeqCst), THREADS); + } +} + +#[test] +fn stampede() { + const THREADS: usize = 8; + #[cfg(miri)] + const COUNT: usize = 500; + #[cfg(not(miri))] + const COUNT: usize = 50_000; + + let q = Injector::new(); + + for i in 0..COUNT { + q.push(Box::new(i + 1)); + } + let remaining = Arc::new(AtomicUsize::new(COUNT)); + + scope(|scope| { + for _ in 0..THREADS { + let remaining = remaining.clone(); + let q = &q; + + scope.spawn(move |_| { + let mut last = 0; + while remaining.load(SeqCst) > 0 { + if let Success(x) = q.steal() { + assert!(last < *x); + last = *x; + remaining.fetch_sub(1, SeqCst); + } + } + }); + } + + let mut last = 0; + while remaining.load(SeqCst) > 0 { + if let Success(x) = q.steal() { + assert!(last < *x); + last = *x; + remaining.fetch_sub(1, SeqCst); + } + } + }) + .unwrap(); +} + +#[test] +fn stress() { + const THREADS: usize = 8; + #[cfg(miri)] + const COUNT: usize = 500; + #[cfg(not(miri))] + const COUNT: usize = 50_000; + + let q = Injector::new(); + let done = Arc::new(AtomicBool::new(false)); + let hits = Arc::new(AtomicUsize::new(0)); + + scope(|scope| { + for _ in 0..THREADS { + let done = done.clone(); + let hits = hits.clone(); + let q = &q; + + scope.spawn(move |_| { + let w2 = Worker::new_fifo(); + + while !done.load(SeqCst) { + if let Success(_) = q.steal() { + hits.fetch_add(1, SeqCst); + } + + let _ = q.steal_batch(&w2); + + if let Success(_) = q.steal_batch_and_pop(&w2) { + hits.fetch_add(1, SeqCst); + } + + while w2.pop().is_some() { + hits.fetch_add(1, SeqCst); + } + } + }); + } + + let mut rng = rand::thread_rng(); + let mut expected = 0; + while expected < COUNT { + if rng.gen_range(0..3) == 0 { + while let Success(_) = q.steal() { + hits.fetch_add(1, SeqCst); + } + } else { + q.push(expected); + expected += 1; + } + } + + while hits.load(SeqCst) < COUNT { + while let Success(_) = q.steal() { + hits.fetch_add(1, SeqCst); + } + } + done.store(true, SeqCst); + }) + .unwrap(); +} + +#[cfg_attr(miri, ignore)] // Miri is too slow +#[test] +fn no_starvation() { + const THREADS: usize = 8; + const COUNT: usize = 50_000; + + let q = Injector::new(); + let done = Arc::new(AtomicBool::new(false)); + let mut all_hits = Vec::new(); + + scope(|scope| { + for _ in 0..THREADS { + let done = done.clone(); + let hits = Arc::new(AtomicUsize::new(0)); + all_hits.push(hits.clone()); + let q = &q; + + scope.spawn(move |_| { + let w2 = Worker::new_fifo(); + + while !done.load(SeqCst) { + if let Success(_) = q.steal() { + hits.fetch_add(1, SeqCst); + } + + let _ = q.steal_batch(&w2); + + if let Success(_) = q.steal_batch_and_pop(&w2) { + hits.fetch_add(1, SeqCst); + } + + while w2.pop().is_some() { + hits.fetch_add(1, SeqCst); + } + } + }); + } + + let mut rng = rand::thread_rng(); + let mut my_hits = 0; + loop { + for i in 0..rng.gen_range(0..COUNT) { + if rng.gen_range(0..3) == 0 && my_hits == 0 { + while let Success(_) = q.steal() { + my_hits += 1; + } + } else { + q.push(i); + } + } + + if my_hits > 0 && all_hits.iter().all(|h| h.load(SeqCst) > 0) { + break; + } + } + done.store(true, SeqCst); + }) + .unwrap(); +} + +#[test] +fn destructors() { + #[cfg(miri)] + const THREADS: usize = 2; + #[cfg(not(miri))] + const THREADS: usize = 8; + #[cfg(miri)] + const COUNT: usize = 500; + #[cfg(not(miri))] + const COUNT: usize = 50_000; + #[cfg(miri)] + const STEPS: usize = 100; + #[cfg(not(miri))] + const STEPS: usize = 1000; + + struct Elem(usize, Arc>>); + + impl Drop for Elem { + fn drop(&mut self) { + self.1.lock().unwrap().push(self.0); + } + } + + let q = Injector::new(); + let dropped = Arc::new(Mutex::new(Vec::new())); + let remaining = Arc::new(AtomicUsize::new(COUNT)); + + for i in 0..COUNT { + q.push(Elem(i, dropped.clone())); + } + + scope(|scope| { + for _ in 0..THREADS { + let remaining = remaining.clone(); + let q = &q; + + scope.spawn(move |_| { + let w2 = Worker::new_fifo(); + let mut cnt = 0; + + while cnt < STEPS { + if let Success(_) = q.steal() { + cnt += 1; + remaining.fetch_sub(1, SeqCst); + } + + let _ = q.steal_batch(&w2); + + if let Success(_) = q.steal_batch_and_pop(&w2) { + cnt += 1; + remaining.fetch_sub(1, SeqCst); + } + + while w2.pop().is_some() { + cnt += 1; + remaining.fetch_sub(1, SeqCst); + } + } + }); + } + + for _ in 0..STEPS { + if let Success(_) = q.steal() { + remaining.fetch_sub(1, SeqCst); + } + } + }) + .unwrap(); + + let rem = remaining.load(SeqCst); + assert!(rem > 0); + + { + let mut v = dropped.lock().unwrap(); + assert_eq!(v.len(), COUNT - rem); + v.clear(); + } + + drop(q); + + { + let mut v = dropped.lock().unwrap(); + assert_eq!(v.len(), rem); + v.sort_unstable(); + for pair in v.windows(2) { + assert_eq!(pair[0] + 1, pair[1]); + } + } +} + +// If `Block` is created on the stack, the array of slots will multiply this `BigStruct` and +// probably overflow the thread stack. It's now directly created on the heap to avoid this. +#[test] +fn stack_overflow() { + const N: usize = 32_768; + struct BigStruct { + _data: [u8; N], + } + + let q = Injector::new(); + + q.push(BigStruct { _data: [0u8; N] }); + + while !matches!(q.steal(), Empty) {} +} diff --git a/anneal/v2/vendor/crossbeam-deque/tests/lifo.rs b/anneal/v2/vendor/crossbeam-deque/tests/lifo.rs new file mode 100644 index 0000000000..c1a65cd2ef --- /dev/null +++ b/anneal/v2/vendor/crossbeam-deque/tests/lifo.rs @@ -0,0 +1,359 @@ +use std::sync::atomic::Ordering::SeqCst; +use std::sync::atomic::{AtomicBool, AtomicUsize}; +use std::sync::{Arc, Mutex}; + +use crossbeam_deque::Steal::{Empty, Success}; +use crossbeam_deque::Worker; +use crossbeam_utils::thread::scope; +use rand::Rng; + +#[test] +fn smoke() { + let w = Worker::new_lifo(); + let s = w.stealer(); + assert_eq!(w.pop(), None); + assert_eq!(s.steal(), Empty); + + w.push(1); + assert_eq!(w.pop(), Some(1)); + assert_eq!(w.pop(), None); + assert_eq!(s.steal(), Empty); + + w.push(2); + assert_eq!(s.steal(), Success(2)); + assert_eq!(s.steal(), Empty); + assert_eq!(w.pop(), None); + + w.push(3); + w.push(4); + w.push(5); + assert_eq!(s.steal(), Success(3)); + assert_eq!(s.steal(), Success(4)); + assert_eq!(s.steal(), Success(5)); + assert_eq!(s.steal(), Empty); + + w.push(6); + w.push(7); + w.push(8); + w.push(9); + assert_eq!(w.pop(), Some(9)); + assert_eq!(s.steal(), Success(6)); + assert_eq!(w.pop(), Some(8)); + assert_eq!(w.pop(), Some(7)); + assert_eq!(w.pop(), None); +} + +#[test] +fn is_empty() { + let w = Worker::new_lifo(); + let s = w.stealer(); + + assert!(w.is_empty()); + w.push(1); + assert!(!w.is_empty()); + w.push(2); + assert!(!w.is_empty()); + let _ = w.pop(); + assert!(!w.is_empty()); + let _ = w.pop(); + assert!(w.is_empty()); + + assert!(s.is_empty()); + w.push(1); + assert!(!s.is_empty()); + w.push(2); + assert!(!s.is_empty()); + let _ = s.steal(); + assert!(!s.is_empty()); + let _ = s.steal(); + assert!(s.is_empty()); +} + +#[test] +fn spsc() { + #[cfg(miri)] + const STEPS: usize = 500; + #[cfg(not(miri))] + const STEPS: usize = 50_000; + + let w = Worker::new_lifo(); + let s = w.stealer(); + + scope(|scope| { + scope.spawn(|_| { + for i in 0..STEPS { + loop { + if let Success(v) = s.steal() { + assert_eq!(i, v); + break; + } + #[cfg(miri)] + std::hint::spin_loop(); + } + } + + assert_eq!(s.steal(), Empty); + }); + + for i in 0..STEPS { + w.push(i); + } + }) + .unwrap(); +} + +#[test] +fn stampede() { + const THREADS: usize = 8; + #[cfg(miri)] + const COUNT: usize = 500; + #[cfg(not(miri))] + const COUNT: usize = 50_000; + + let w = Worker::new_lifo(); + + for i in 0..COUNT { + w.push(Box::new(i + 1)); + } + let remaining = Arc::new(AtomicUsize::new(COUNT)); + + scope(|scope| { + for _ in 0..THREADS { + let s = w.stealer(); + let remaining = remaining.clone(); + + scope.spawn(move |_| { + let mut last = 0; + while remaining.load(SeqCst) > 0 { + if let Success(x) = s.steal() { + assert!(last < *x); + last = *x; + remaining.fetch_sub(1, SeqCst); + } + } + }); + } + + let mut last = COUNT + 1; + while remaining.load(SeqCst) > 0 { + if let Some(x) = w.pop() { + assert!(last > *x); + last = *x; + remaining.fetch_sub(1, SeqCst); + } + } + }) + .unwrap(); +} + +#[test] +fn stress() { + const THREADS: usize = 8; + #[cfg(miri)] + const COUNT: usize = 500; + #[cfg(not(miri))] + const COUNT: usize = 50_000; + + let w = Worker::new_lifo(); + let done = Arc::new(AtomicBool::new(false)); + let hits = Arc::new(AtomicUsize::new(0)); + + scope(|scope| { + for _ in 0..THREADS { + let s = w.stealer(); + let done = done.clone(); + let hits = hits.clone(); + + scope.spawn(move |_| { + let w2 = Worker::new_lifo(); + + while !done.load(SeqCst) { + if let Success(_) = s.steal() { + hits.fetch_add(1, SeqCst); + } + + let _ = s.steal_batch(&w2); + + if let Success(_) = s.steal_batch_and_pop(&w2) { + hits.fetch_add(1, SeqCst); + } + + while w2.pop().is_some() { + hits.fetch_add(1, SeqCst); + } + } + }); + } + + let mut rng = rand::thread_rng(); + let mut expected = 0; + while expected < COUNT { + if rng.gen_range(0..3) == 0 { + while w.pop().is_some() { + hits.fetch_add(1, SeqCst); + } + } else { + w.push(expected); + expected += 1; + } + } + + while hits.load(SeqCst) < COUNT { + while w.pop().is_some() { + hits.fetch_add(1, SeqCst); + } + } + done.store(true, SeqCst); + }) + .unwrap(); +} + +#[cfg_attr(miri, ignore)] // Miri is too slow +#[test] +fn no_starvation() { + const THREADS: usize = 8; + const COUNT: usize = 50_000; + + let w = Worker::new_lifo(); + let done = Arc::new(AtomicBool::new(false)); + let mut all_hits = Vec::new(); + + scope(|scope| { + for _ in 0..THREADS { + let s = w.stealer(); + let done = done.clone(); + let hits = Arc::new(AtomicUsize::new(0)); + all_hits.push(hits.clone()); + + scope.spawn(move |_| { + let w2 = Worker::new_lifo(); + + while !done.load(SeqCst) { + if let Success(_) = s.steal() { + hits.fetch_add(1, SeqCst); + } + + let _ = s.steal_batch(&w2); + + if let Success(_) = s.steal_batch_and_pop(&w2) { + hits.fetch_add(1, SeqCst); + } + + while w2.pop().is_some() { + hits.fetch_add(1, SeqCst); + } + } + }); + } + + let mut rng = rand::thread_rng(); + let mut my_hits = 0; + loop { + for i in 0..rng.gen_range(0..COUNT) { + if rng.gen_range(0..3) == 0 && my_hits == 0 { + while w.pop().is_some() { + my_hits += 1; + } + } else { + w.push(i); + } + } + + if my_hits > 0 && all_hits.iter().all(|h| h.load(SeqCst) > 0) { + break; + } + } + done.store(true, SeqCst); + }) + .unwrap(); +} + +#[test] +fn destructors() { + #[cfg(miri)] + const THREADS: usize = 2; + #[cfg(not(miri))] + const THREADS: usize = 8; + #[cfg(miri)] + const COUNT: usize = 500; + #[cfg(not(miri))] + const COUNT: usize = 50_000; + #[cfg(miri)] + const STEPS: usize = 100; + #[cfg(not(miri))] + const STEPS: usize = 1000; + + struct Elem(usize, Arc>>); + + impl Drop for Elem { + fn drop(&mut self) { + self.1.lock().unwrap().push(self.0); + } + } + + let w = Worker::new_lifo(); + let dropped = Arc::new(Mutex::new(Vec::new())); + let remaining = Arc::new(AtomicUsize::new(COUNT)); + + for i in 0..COUNT { + w.push(Elem(i, dropped.clone())); + } + + scope(|scope| { + for _ in 0..THREADS { + let remaining = remaining.clone(); + let s = w.stealer(); + + scope.spawn(move |_| { + let w2 = Worker::new_lifo(); + let mut cnt = 0; + + while cnt < STEPS { + if let Success(_) = s.steal() { + cnt += 1; + remaining.fetch_sub(1, SeqCst); + } + + let _ = s.steal_batch(&w2); + + if let Success(_) = s.steal_batch_and_pop(&w2) { + cnt += 1; + remaining.fetch_sub(1, SeqCst); + } + + while w2.pop().is_some() { + cnt += 1; + remaining.fetch_sub(1, SeqCst); + } + } + }); + } + + for _ in 0..STEPS { + if w.pop().is_some() { + remaining.fetch_sub(1, SeqCst); + } + } + }) + .unwrap(); + + let rem = remaining.load(SeqCst); + assert!(rem > 0); + + { + let mut v = dropped.lock().unwrap(); + assert_eq!(v.len(), COUNT - rem); + v.clear(); + } + + drop(w); + + { + let mut v = dropped.lock().unwrap(); + assert_eq!(v.len(), rem); + v.sort_unstable(); + for pair in v.windows(2) { + assert_eq!(pair[0] + 1, pair[1]); + } + } +} diff --git a/anneal/v2/vendor/crossbeam-deque/tests/steal.rs b/anneal/v2/vendor/crossbeam-deque/tests/steal.rs new file mode 100644 index 0000000000..af2499856c --- /dev/null +++ b/anneal/v2/vendor/crossbeam-deque/tests/steal.rs @@ -0,0 +1,212 @@ +use crossbeam_deque::Steal::Success; +use crossbeam_deque::{Injector, Worker}; + +#[test] +fn steal_fifo() { + let w = Worker::new_fifo(); + for i in 1..=3 { + w.push(i); + } + + let s = w.stealer(); + assert_eq!(s.steal(), Success(1)); + assert_eq!(s.steal(), Success(2)); + assert_eq!(s.steal(), Success(3)); +} + +#[test] +fn steal_lifo() { + let w = Worker::new_lifo(); + for i in 1..=3 { + w.push(i); + } + + let s = w.stealer(); + assert_eq!(s.steal(), Success(1)); + assert_eq!(s.steal(), Success(2)); + assert_eq!(s.steal(), Success(3)); +} + +#[test] +fn steal_injector() { + let q = Injector::new(); + for i in 1..=3 { + q.push(i); + } + + assert_eq!(q.steal(), Success(1)); + assert_eq!(q.steal(), Success(2)); + assert_eq!(q.steal(), Success(3)); +} + +#[test] +fn steal_batch_fifo_fifo() { + let w = Worker::new_fifo(); + for i in 1..=4 { + w.push(i); + } + + let s = w.stealer(); + let w2 = Worker::new_fifo(); + + assert_eq!(s.steal_batch(&w2), Success(())); + assert_eq!(w2.pop(), Some(1)); + assert_eq!(w2.pop(), Some(2)); +} + +#[test] +fn steal_batch_lifo_lifo() { + let w = Worker::new_lifo(); + for i in 1..=4 { + w.push(i); + } + + let s = w.stealer(); + let w2 = Worker::new_lifo(); + + assert_eq!(s.steal_batch(&w2), Success(())); + assert_eq!(w2.pop(), Some(2)); + assert_eq!(w2.pop(), Some(1)); +} + +#[test] +fn steal_batch_fifo_lifo() { + let w = Worker::new_fifo(); + for i in 1..=4 { + w.push(i); + } + + let s = w.stealer(); + let w2 = Worker::new_lifo(); + + assert_eq!(s.steal_batch(&w2), Success(())); + assert_eq!(w2.pop(), Some(1)); + assert_eq!(w2.pop(), Some(2)); +} + +#[test] +fn steal_batch_lifo_fifo() { + let w = Worker::new_lifo(); + for i in 1..=4 { + w.push(i); + } + + let s = w.stealer(); + let w2 = Worker::new_fifo(); + + assert_eq!(s.steal_batch(&w2), Success(())); + assert_eq!(w2.pop(), Some(2)); + assert_eq!(w2.pop(), Some(1)); +} + +#[test] +fn steal_batch_injector_fifo() { + let q = Injector::new(); + for i in 1..=4 { + q.push(i); + } + + let w2 = Worker::new_fifo(); + assert_eq!(q.steal_batch(&w2), Success(())); + assert_eq!(w2.pop(), Some(1)); + assert_eq!(w2.pop(), Some(2)); +} + +#[test] +fn steal_batch_injector_lifo() { + let q = Injector::new(); + for i in 1..=4 { + q.push(i); + } + + let w2 = Worker::new_lifo(); + assert_eq!(q.steal_batch(&w2), Success(())); + assert_eq!(w2.pop(), Some(1)); + assert_eq!(w2.pop(), Some(2)); +} + +#[test] +fn steal_batch_and_pop_fifo_fifo() { + let w = Worker::new_fifo(); + for i in 1..=6 { + w.push(i); + } + + let s = w.stealer(); + let w2 = Worker::new_fifo(); + + assert_eq!(s.steal_batch_and_pop(&w2), Success(1)); + assert_eq!(w2.pop(), Some(2)); + assert_eq!(w2.pop(), Some(3)); +} + +#[test] +fn steal_batch_and_pop_lifo_lifo() { + let w = Worker::new_lifo(); + for i in 1..=6 { + w.push(i); + } + + let s = w.stealer(); + let w2 = Worker::new_lifo(); + + assert_eq!(s.steal_batch_and_pop(&w2), Success(3)); + assert_eq!(w2.pop(), Some(2)); + assert_eq!(w2.pop(), Some(1)); +} + +#[test] +fn steal_batch_and_pop_fifo_lifo() { + let w = Worker::new_fifo(); + for i in 1..=6 { + w.push(i); + } + + let s = w.stealer(); + let w2 = Worker::new_lifo(); + + assert_eq!(s.steal_batch_and_pop(&w2), Success(1)); + assert_eq!(w2.pop(), Some(2)); + assert_eq!(w2.pop(), Some(3)); +} + +#[test] +fn steal_batch_and_pop_lifo_fifo() { + let w = Worker::new_lifo(); + for i in 1..=6 { + w.push(i); + } + + let s = w.stealer(); + let w2 = Worker::new_fifo(); + + assert_eq!(s.steal_batch_and_pop(&w2), Success(3)); + assert_eq!(w2.pop(), Some(2)); + assert_eq!(w2.pop(), Some(1)); +} + +#[test] +fn steal_batch_and_pop_injector_fifo() { + let q = Injector::new(); + for i in 1..=6 { + q.push(i); + } + + let w2 = Worker::new_fifo(); + assert_eq!(q.steal_batch_and_pop(&w2), Success(1)); + assert_eq!(w2.pop(), Some(2)); + assert_eq!(w2.pop(), Some(3)); +} + +#[test] +fn steal_batch_and_pop_injector_lifo() { + let q = Injector::new(); + for i in 1..=6 { + q.push(i); + } + + let w2 = Worker::new_lifo(); + assert_eq!(q.steal_batch_and_pop(&w2), Success(1)); + assert_eq!(w2.pop(), Some(2)); + assert_eq!(w2.pop(), Some(3)); +} diff --git a/anneal/v2/vendor/crossbeam-epoch/.cargo-checksum.json b/anneal/v2/vendor/crossbeam-epoch/.cargo-checksum.json new file mode 100644 index 0000000000..7f83164d8d --- /dev/null +++ b/anneal/v2/vendor/crossbeam-epoch/.cargo-checksum.json @@ -0,0 +1 @@ +{"files":{".cargo_vcs_info.json":"49529baffd2fd52cbf26931156570b752c8fc679c09a5091ba11c48645c77754","CHANGELOG.md":"678c7c5b4e522345076d63e8f24f7ab4dc9f6b7428ca2f665e4017f7ef24a087","Cargo.lock":"fd85f51f6b4a2dabbb41d9f96775abd21d89221fa01c154afac970c539022f17","Cargo.toml":"cfbceb820c7a1519351826839decd3ff1b1ad54ef2c4dfc4d2c9f173e4726046","Cargo.toml.orig":"b3c80c139498aac55f4caabab81579a87169f6b92156312ebcaec61f5578af9d","LICENSE-APACHE":"a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2","LICENSE-MIT":"5734ed989dfca1f625b40281ee9f4530f91b2411ec01cb748223e7eb87e201ab","README.md":"6ba897c52496a66705df72da33dea5f6e0ce5caa87c4ff073b0faf4e05516dad","benches/defer.rs":"c330b704d96b2ad1aed29f72c37a99da534adef8cb06a3976d5f93bf567abb20","benches/flush.rs":"0389ac6c473632f0e93c962f223404cc360257f6699b4ec90b9b3be16bb6d74f","benches/pin.rs":"2f649a5153745c7930efdb32a52f9dc522f7b8cf548a251c5e2c82ee25dc3fff","examples/sanitize.rs":"a39d1635fa61e643e59192d7a63becc97ff81f03c1f4e03d38cedefb1525026a","src/atomic.rs":"48b8b02d1e0235b2d87342c13c09d778fba076f79addef32294bed5b8f67b21a","src/collector.rs":"df05c7573413a8f3ac933de7cf941d24bd0ca7341f5923dcad2f811a020c49eb","src/default.rs":"8196e9a2a7a43fdd668177585ba1d4deaec2d16a8a9532f819e4d9afd64ca73d","src/deferred.rs":"092c49e65d5f0ccad8c868b9bcaf431b580c98b7efed98c3797d82d0b9d0c471","src/epoch.rs":"e6813975198df667423c7e1911f7a0f5cb3a917e56080eecd6250d9cca7af950","src/guard.rs":"8db7a20503f55e9e29fc1cf33f99522ec0a5873683ab16638e0e55c917bfc30a","src/internal.rs":"74a15b34b235ab428ffa41cb3a01930e29e3f91e35a288f8f6e0c3c2f56e63f6","src/lib.rs":"3f81f1727c3f74114fbd2f9225a4899834fc254f1444f7c7355901c8fd755494","src/sync/list.rs":"10aa4c59845ab9ff1d8bcb6f594b70bbe23c320fa7a2b125fdf85df88b9d61e2","src/sync/mod.rs":"326e32489d467e974c441120640a8338aa55da55c24b20276075ce9053997326","src/sync/once_lock.rs":"aa8f957604d1119c4fc7038a18c14a6281230e81005f31201c099acff284ad4b","src/sync/queue.rs":"d4ad500501c52a90b6624dc31196793be09bd19e9c298d5dd7b3ae37bee6b6a8","tests/loom.rs":"db772f4478966de6ec98774ca4093171dc942da635822a0d2d3257d31188cb9b"},"package":"5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"} \ No newline at end of file diff --git a/anneal/v2/vendor/crossbeam-epoch/.cargo_vcs_info.json b/anneal/v2/vendor/crossbeam-epoch/.cargo_vcs_info.json new file mode 100644 index 0000000000..d24fe6d54a --- /dev/null +++ b/anneal/v2/vendor/crossbeam-epoch/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "9c3182abebb36bdc9446d75d4644190fef70fa01" + }, + "path_in_vcs": "crossbeam-epoch" +} \ No newline at end of file diff --git a/anneal/v2/vendor/crossbeam-epoch/CHANGELOG.md b/anneal/v2/vendor/crossbeam-epoch/CHANGELOG.md new file mode 100644 index 0000000000..d5ca3a0717 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-epoch/CHANGELOG.md @@ -0,0 +1,204 @@ +# Version 0.9.18 + +- Remove dependency on `cfg-if`. (#1072) +- Remove dependency on `autocfg`. (#1071) + +# Version 0.9.17 + +- Remove dependency on `memoffset`. (#1058) + +# Version 0.9.16 + +- Bump the minimum supported Rust version to 1.61. (#1037) +- Improve support for targets without atomic CAS. (#1037) +- Remove build script. (#1037) +- Remove dependency on `scopeguard`. (#1045) +- Update `loom` dependency to 0.7. + +# Version 0.9.15 + +- Update `memoffset` to 0.9. (#981) + +# Version 0.9.14 + +- Update `memoffset` to 0.8. (#955) + +# Version 0.9.13 + +- Fix build script bug introduced in 0.9.12. (#932) + +# Version 0.9.12 + +**Note:** This release has been yanked due to regression fixed in 0.9.13. + +- Update `memoffset` to 0.7. (#926) +- Improve support for custom targets. (#922) + +# Version 0.9.11 + +- Removes the dependency on the `once_cell` crate to restore the MSRV. (#913) +- Work around [rust-lang#98302](https://github.com/rust-lang/rust/issues/98302), which causes compile error on windows-gnu when LTO is enabled. (#913) + +# Version 0.9.10 + +- Bump the minimum supported Rust version to 1.38. (#877) +- Mitigate the risk of segmentation faults in buggy downstream implementations. (#879) +- Add `{Atomic, Shared}::try_into_owned` (#701) + +# Version 0.9.9 + +- Replace lazy_static with once_cell. (#817) + +# Version 0.9.8 + +- Make `Atomic::null()` const function at 1.61+. (#797) + +# Version 0.9.7 + +- Fix Miri error when `-Zmiri-check-number-validity` is enabled. (#779) + +# Version 0.9.6 + +- Add `Atomic::fetch_update`. (#706) + +# Version 0.9.5 + +- Fix UB in `Pointable` impl of `[MaybeUninit]`. (#694) +- Support targets that do not have atomic CAS on stable Rust. (#698) +- Fix breakage with nightly feature due to rust-lang/rust#84510. (#692) + +# Version 0.9.4 + +**Note**: This release has been yanked. See [#693](https://github.com/crossbeam-rs/crossbeam/issues/693) for details. + +- Fix UB in `<[MaybeUninit] as Pointable>::init` when global allocator failed allocation. (#690) +- Bump `loom` dependency to version 0.5. (#686) + +# Version 0.9.3 + +**Note**: This release has been yanked. See [#693](https://github.com/crossbeam-rs/crossbeam/issues/693) for details. + +- Make `loom` dependency optional. (#666) + +# Version 0.9.2 + +**Note**: This release has been yanked. See [#693](https://github.com/crossbeam-rs/crossbeam/issues/693) for details. + +- Add `Atomic::compare_exchange` and `Atomic::compare_exchange_weak`. (#628) +- Deprecate `Atomic::compare_and_set` and `Atomic::compare_and_set_weak`. Use `Atomic::compare_exchange` or `Atomic::compare_exchange_weak` instead. (#628) +- Make `const_fn` dependency optional. (#611) +- Add unstable support for `loom`. (#487) + +# Version 0.9.1 + +**Note**: This release has been yanked. See [#693](https://github.com/crossbeam-rs/crossbeam/issues/693) for details. + +- Bump `memoffset` dependency to version 0.6. (#592) + +# Version 0.9.0 + +**Note**: This release has been yanked. See [#693](https://github.com/crossbeam-rs/crossbeam/issues/693) for details. + +- Bump the minimum supported Rust version to 1.36. +- Support dynamically sized types. + +# Version 0.8.2 + +- Fix bug in release (yanking 0.8.1) + +# Version 0.8.1 + +- Bump `autocfg` dependency to version 1.0. (#460) +- Reduce stall in list iteration. (#376) +- Stop stealing from the same deque. (#448) +- Fix unsoundness issues by adopting `MaybeUninit`. (#458) +- Fix use-after-free in lock-free queue. (#466) + +# Version 0.8.0 + +- Bump the minimum required version to 1.28. +- Fix breakage with nightly feature due to rust-lang/rust#65214. +- Make `Atomic::null()` const function at 1.31+. +- Bump `crossbeam-utils` to `0.7`. + +# Version 0.7.2 + +- Add `Atomic::into_owned()`. +- Update `memoffset` dependency. + +# Version 0.7.1 + +- Add `Shared::deref_mut()`. +- Add a Treiber stack to examples. + +# Version 0.7.0 + +- Remove `Guard::clone()`. +- Bump dependencies. + +# Version 0.6.1 + +- Update `crossbeam-utils` to `0.6`. + +# Version 0.6.0 + +- `defer` now requires `F: Send + 'static`. +- Bump the minimum Rust version to 1.26. +- Pinning while TLS is tearing down does not fail anymore. +- Rename `Handle` to `LocalHandle`. +- Add `defer_unchecked` and `defer_destroy`. +- Remove `Clone` impl for `LocalHandle`. + +# Version 0.5.2 + +- Update `crossbeam-utils` to `0.5`. + +# Version 0.5.1 + +- Fix compatibility with the latest Rust nightly. + +# Version 0.5.0 + +- Update `crossbeam-utils` to `0.4`. +- Specify the minimum Rust version to `1.25.0`. + +# Version 0.4.3 + +- Downgrade `crossbeam-utils` to `0.3` because it was a breaking change. + +# Version 0.4.2 + +- Expose the `Pointer` trait. +- Warn missing docs and missing debug impls. +- Update `crossbeam-utils` to `0.4`. + +# Version 0.4.1 + +- Add `Debug` impls for `Collector`, `Handle`, and `Guard`. +- Add `load_consume` to `Atomic`. +- Rename `Collector::handle` to `Collector::register`. +- Remove the `Send` implementation for `Handle` (this was a bug). Only + `Collector`s can be shared among multiple threads, while `Handle`s and + `Guard`s must stay within the thread in which they were created. + +# Version 0.4.0 + +- Update dependencies. +- Remove support for Rust 1.13. + +# Version 0.3.0 + +- Add support for Rust 1.13. +- Improve documentation for CAS. + +# Version 0.2.0 + +- Add method `Owned::into_box`. +- Fix a use-after-free bug in `Local::finalize`. +- Fix an ordering bug in `Global::push_bag`. +- Fix a bug in calculating distance between epochs. +- Remove `impl Into> for Owned`. + +# Version 0.1.0 + +- First version of the new epoch-based GC. diff --git a/anneal/v2/vendor/crossbeam-epoch/Cargo.lock b/anneal/v2/vendor/crossbeam-epoch/Cargo.lock new file mode 100644 index 0000000000..3a2c62417b --- /dev/null +++ b/anneal/v2/vendor/crossbeam-epoch/Cargo.lock @@ -0,0 +1,457 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +dependencies = [ + "memchr", +] + +[[package]] +name = "cc" +version = "1.0.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "libc", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +dependencies = [ + "crossbeam-utils", + "loom", + "rand", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3a430a770ebd84726f584a90ee7f020d28db52c6d02138900f22341f866d39c" +dependencies = [ + "cfg-if", + "loom", +] + +[[package]] +name = "generator" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc16584ff22b460a382b7feec54b23d2908d858152e5739a120b949293bd74e" +dependencies = [ + "cc", + "libc", + "log", + "rustversion", + "windows", +] + +[[package]] +name = "getrandom" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.152" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" + +[[package]] +name = "log" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" + +[[package]] +name = "loom" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e045d70ddfbc984eacfa964ded019534e8f6cbf36f6410aee0ed5cefa5a9175" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + +[[package]] +name = "memchr" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" + +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "proc-macro2" +version = "1.0.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "regex" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata 0.4.3", + "regex-syntax 0.8.2", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", +] + +[[package]] +name = "regex-automata" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax 0.8.2", +] + +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + +[[package]] +name = "regex-syntax" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" + +[[package]] +name = "rustversion" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "smallvec" +version = "1.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" + +[[package]] +name = "syn" +version = "2.0.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thread_local" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +dependencies = [ + "cfg-if", + "once_cell", +] + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" diff --git a/anneal/v2/vendor/crossbeam-epoch/Cargo.toml b/anneal/v2/vendor/crossbeam-epoch/Cargo.toml new file mode 100644 index 0000000000..0a8fc81cf5 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-epoch/Cargo.toml @@ -0,0 +1,57 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +rust-version = "1.61" +name = "crossbeam-epoch" +version = "0.9.18" +description = "Epoch-based garbage collection" +homepage = "https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-epoch" +readme = "README.md" +keywords = [ + "lock-free", + "rcu", + "atomic", + "garbage", +] +categories = [ + "concurrency", + "memory-management", + "no-std", +] +license = "MIT OR Apache-2.0" +repository = "https://github.com/crossbeam-rs/crossbeam" + +[dependencies.crossbeam-utils] +version = "0.8.18" +default-features = false + +[dev-dependencies.rand] +version = "0.8" + +[features] +alloc = [] +default = ["std"] +loom = [ + "loom-crate", + "crossbeam-utils/loom", +] +nightly = ["crossbeam-utils/nightly"] +std = [ + "alloc", + "crossbeam-utils/std", +] + +[target."cfg(crossbeam_loom)".dependencies.loom-crate] +version = "0.7.1" +optional = true +package = "loom" diff --git a/anneal/v2/vendor/crossbeam-epoch/Cargo.toml.orig b/anneal/v2/vendor/crossbeam-epoch/Cargo.toml.orig new file mode 100644 index 0000000000..0f87aed708 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-epoch/Cargo.toml.orig @@ -0,0 +1,56 @@ +[package] +name = "crossbeam-epoch" +# When publishing a new version: +# - Update CHANGELOG.md +# - Update README.md +# - Create "crossbeam-epoch-X.Y.Z" git tag +version = "0.9.18" +edition = "2021" +rust-version = "1.61" +license = "MIT OR Apache-2.0" +repository = "https://github.com/crossbeam-rs/crossbeam" +homepage = "https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-epoch" +description = "Epoch-based garbage collection" +keywords = ["lock-free", "rcu", "atomic", "garbage"] +categories = ["concurrency", "memory-management", "no-std"] + +[features] +default = ["std"] + +# Enable to use APIs that require `std`. +# This is enabled by default. +std = ["alloc", "crossbeam-utils/std"] + +# Enable to use APIs that require `alloc`. +# This is enabled by default and also enabled if the `std` feature is enabled. +# +# NOTE: Disabling both `std` *and* `alloc` features is not supported yet. +alloc = [] + +# These features are no longer used. +# TODO: remove in the next major version. +# Enable to use of unstable functionality. +# This is disabled by default and requires recent nightly compiler. +# +# NOTE: This feature is outside of the normal semver guarantees and minor or +# patch versions of crossbeam may make breaking changes to them at any time. +nightly = ["crossbeam-utils/nightly"] + +# Enable the use of loom for concurrency testing. +# +# NOTE: This feature is outside of the normal semver guarantees and minor or +# patch versions of crossbeam may make breaking changes to them at any time. +loom = ["loom-crate", "crossbeam-utils/loom"] + +[dependencies] +crossbeam-utils = { version = "0.8.18", path = "../crossbeam-utils", default-features = false } + +# Enable the use of loom for concurrency testing. +# +# NOTE: This feature is outside of the normal semver guarantees and minor or +# patch versions of crossbeam may make breaking changes to them at any time. +[target.'cfg(crossbeam_loom)'.dependencies] +loom-crate = { package = "loom", version = "0.7.1", optional = true } + +[dev-dependencies] +rand = "0.8" diff --git a/anneal/v2/vendor/crossbeam-epoch/LICENSE-APACHE b/anneal/v2/vendor/crossbeam-epoch/LICENSE-APACHE new file mode 100644 index 0000000000..16fe87b06e --- /dev/null +++ b/anneal/v2/vendor/crossbeam-epoch/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/anneal/v2/vendor/crossbeam-epoch/LICENSE-MIT b/anneal/v2/vendor/crossbeam-epoch/LICENSE-MIT new file mode 100644 index 0000000000..068d491fd5 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-epoch/LICENSE-MIT @@ -0,0 +1,27 @@ +The MIT License (MIT) + +Copyright (c) 2019 The Crossbeam Project Developers + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/anneal/v2/vendor/crossbeam-epoch/README.md b/anneal/v2/vendor/crossbeam-epoch/README.md new file mode 100644 index 0000000000..ba74c7c75b --- /dev/null +++ b/anneal/v2/vendor/crossbeam-epoch/README.md @@ -0,0 +1,53 @@ +# Crossbeam Epoch + +[![Build Status](https://github.com/crossbeam-rs/crossbeam/workflows/CI/badge.svg)]( +https://github.com/crossbeam-rs/crossbeam/actions) +[![License](https://img.shields.io/badge/license-MIT_OR_Apache--2.0-blue.svg)]( +https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-epoch#license) +[![Cargo](https://img.shields.io/crates/v/crossbeam-epoch.svg)]( +https://crates.io/crates/crossbeam-epoch) +[![Documentation](https://docs.rs/crossbeam-epoch/badge.svg)]( +https://docs.rs/crossbeam-epoch) +[![Rust 1.61+](https://img.shields.io/badge/rust-1.61+-lightgray.svg)]( +https://www.rust-lang.org) +[![chat](https://img.shields.io/discord/569610676205781012.svg?logo=discord)](https://discord.com/invite/JXYwgWZ) + +This crate provides epoch-based garbage collection for building concurrent data structures. + +When a thread removes an object from a concurrent data structure, other threads +may be still using pointers to it at the same time, so it cannot be destroyed +immediately. Epoch-based GC is an efficient mechanism for deferring destruction of +shared objects until no pointers to them can exist. + +Everything in this crate except the global GC can be used in `no_std` environments, provided that +`alloc` feature is enabled. + +## Usage + +Add this to your `Cargo.toml`: + +```toml +[dependencies] +crossbeam-epoch = "0.9" +``` + +## Compatibility + +Crossbeam Epoch supports stable Rust releases going back at least six months, +and every time the minimum supported Rust version is increased, a new minor +version is released. Currently, the minimum supported Rust version is 1.61. + +## License + +Licensed under either of + + * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) + * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) + +at your option. + +#### Contribution + +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in the work by you, as defined in the Apache-2.0 license, shall be +dual licensed as above, without any additional terms or conditions. diff --git a/anneal/v2/vendor/crossbeam-epoch/benches/defer.rs b/anneal/v2/vendor/crossbeam-epoch/benches/defer.rs new file mode 100644 index 0000000000..246f907988 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-epoch/benches/defer.rs @@ -0,0 +1,69 @@ +#![feature(test)] + +extern crate test; + +use crossbeam_epoch::{self as epoch, Owned}; +use crossbeam_utils::thread::scope; +use test::Bencher; + +#[bench] +fn single_alloc_defer_free(b: &mut Bencher) { + b.iter(|| { + let guard = &epoch::pin(); + let p = Owned::new(1).into_shared(guard); + unsafe { + guard.defer_destroy(p); + } + }); +} + +#[bench] +fn single_defer(b: &mut Bencher) { + b.iter(|| { + let guard = &epoch::pin(); + guard.defer(move || ()); + }); +} + +#[bench] +fn multi_alloc_defer_free(b: &mut Bencher) { + const THREADS: usize = 16; + const STEPS: usize = 10_000; + + b.iter(|| { + scope(|s| { + for _ in 0..THREADS { + s.spawn(|_| { + for _ in 0..STEPS { + let guard = &epoch::pin(); + let p = Owned::new(1).into_shared(guard); + unsafe { + guard.defer_destroy(p); + } + } + }); + } + }) + .unwrap(); + }); +} + +#[bench] +fn multi_defer(b: &mut Bencher) { + const THREADS: usize = 16; + const STEPS: usize = 10_000; + + b.iter(|| { + scope(|s| { + for _ in 0..THREADS { + s.spawn(|_| { + for _ in 0..STEPS { + let guard = &epoch::pin(); + guard.defer(move || ()); + } + }); + } + }) + .unwrap(); + }); +} diff --git a/anneal/v2/vendor/crossbeam-epoch/benches/flush.rs b/anneal/v2/vendor/crossbeam-epoch/benches/flush.rs new file mode 100644 index 0000000000..99aab19e1e --- /dev/null +++ b/anneal/v2/vendor/crossbeam-epoch/benches/flush.rs @@ -0,0 +1,52 @@ +#![feature(test)] + +extern crate test; + +use std::sync::Barrier; + +use crossbeam_epoch as epoch; +use crossbeam_utils::thread::scope; +use test::Bencher; + +#[bench] +fn single_flush(b: &mut Bencher) { + const THREADS: usize = 16; + + let start = Barrier::new(THREADS + 1); + let end = Barrier::new(THREADS + 1); + + scope(|s| { + for _ in 0..THREADS { + s.spawn(|_| { + epoch::pin(); + start.wait(); + end.wait(); + }); + } + + start.wait(); + b.iter(|| epoch::pin().flush()); + end.wait(); + }) + .unwrap(); +} + +#[bench] +fn multi_flush(b: &mut Bencher) { + const THREADS: usize = 16; + const STEPS: usize = 10_000; + + b.iter(|| { + scope(|s| { + for _ in 0..THREADS { + s.spawn(|_| { + for _ in 0..STEPS { + let guard = &epoch::pin(); + guard.flush(); + } + }); + } + }) + .unwrap(); + }); +} diff --git a/anneal/v2/vendor/crossbeam-epoch/benches/pin.rs b/anneal/v2/vendor/crossbeam-epoch/benches/pin.rs new file mode 100644 index 0000000000..8bf87e9b73 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-epoch/benches/pin.rs @@ -0,0 +1,31 @@ +#![feature(test)] + +extern crate test; + +use crossbeam_epoch as epoch; +use crossbeam_utils::thread::scope; +use test::Bencher; + +#[bench] +fn single_pin(b: &mut Bencher) { + b.iter(epoch::pin); +} + +#[bench] +fn multi_pin(b: &mut Bencher) { + const THREADS: usize = 16; + const STEPS: usize = 100_000; + + b.iter(|| { + scope(|s| { + for _ in 0..THREADS { + s.spawn(|_| { + for _ in 0..STEPS { + epoch::pin(); + } + }); + } + }) + .unwrap(); + }); +} diff --git a/anneal/v2/vendor/crossbeam-epoch/examples/sanitize.rs b/anneal/v2/vendor/crossbeam-epoch/examples/sanitize.rs new file mode 100644 index 0000000000..4109c34a8c --- /dev/null +++ b/anneal/v2/vendor/crossbeam-epoch/examples/sanitize.rs @@ -0,0 +1,66 @@ +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering::{AcqRel, Acquire, Relaxed}; +use std::sync::Arc; +use std::thread; +use std::time::{Duration, Instant}; + +use crossbeam_epoch::{self as epoch, Atomic, Collector, LocalHandle, Owned, Shared}; +use rand::Rng; + +fn worker(a: Arc>, handle: LocalHandle) -> usize { + let mut rng = rand::thread_rng(); + let mut sum = 0; + + if rng.gen() { + thread::sleep(Duration::from_millis(1)); + } + let timeout = Duration::from_millis(rng.gen_range(0..10)); + let now = Instant::now(); + + while now.elapsed() < timeout { + for _ in 0..100 { + let guard = &handle.pin(); + guard.flush(); + + let val = if rng.gen() { + let p = a.swap(Owned::new(AtomicUsize::new(sum)), AcqRel, guard); + unsafe { + guard.defer_destroy(p); + guard.flush(); + p.deref().load(Relaxed) + } + } else { + let p = a.load(Acquire, guard); + unsafe { p.deref().fetch_add(sum, Relaxed) } + }; + + sum = sum.wrapping_add(val); + } + } + + sum +} + +fn main() { + for _ in 0..100 { + let collector = Collector::new(); + let a = Arc::new(Atomic::new(AtomicUsize::new(777))); + + let threads = (0..16) + .map(|_| { + let a = a.clone(); + let c = collector.clone(); + thread::spawn(move || worker(a, c.register())) + }) + .collect::>(); + + for t in threads { + t.join().unwrap(); + } + + unsafe { + a.swap(Shared::null(), AcqRel, epoch::unprotected()) + .into_owned(); + } + } +} diff --git a/anneal/v2/vendor/crossbeam-epoch/src/atomic.rs b/anneal/v2/vendor/crossbeam-epoch/src/atomic.rs new file mode 100644 index 0000000000..41b4cd9102 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-epoch/src/atomic.rs @@ -0,0 +1,1702 @@ +use alloc::boxed::Box; +use core::alloc::Layout; +use core::borrow::{Borrow, BorrowMut}; +use core::cmp; +use core::fmt; +use core::marker::PhantomData; +use core::mem::{self, MaybeUninit}; +use core::ops::{Deref, DerefMut}; +use core::ptr; +use core::slice; + +use crate::guard::Guard; +use crate::primitive::sync::atomic::{AtomicUsize, Ordering}; +use crossbeam_utils::atomic::AtomicConsume; + +/// Given ordering for the success case in a compare-exchange operation, returns the strongest +/// appropriate ordering for the failure case. +#[inline] +fn strongest_failure_ordering(ord: Ordering) -> Ordering { + use self::Ordering::*; + match ord { + Relaxed | Release => Relaxed, + Acquire | AcqRel => Acquire, + _ => SeqCst, + } +} + +/// The error returned on failed compare-and-set operation. +// TODO: remove in the next major version. +#[deprecated(note = "Use `CompareExchangeError` instead")] +pub type CompareAndSetError<'g, T, P> = CompareExchangeError<'g, T, P>; + +/// The error returned on failed compare-and-swap operation. +pub struct CompareExchangeError<'g, T: ?Sized + Pointable, P: Pointer> { + /// The value in the atomic pointer at the time of the failed operation. + pub current: Shared<'g, T>, + + /// The new value, which the operation failed to store. + pub new: P, +} + +impl + fmt::Debug> fmt::Debug for CompareExchangeError<'_, T, P> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("CompareExchangeError") + .field("current", &self.current) + .field("new", &self.new) + .finish() + } +} + +/// Memory orderings for compare-and-set operations. +/// +/// A compare-and-set operation can have different memory orderings depending on whether it +/// succeeds or fails. This trait generalizes different ways of specifying memory orderings. +/// +/// The two ways of specifying orderings for compare-and-set are: +/// +/// 1. Just one `Ordering` for the success case. In case of failure, the strongest appropriate +/// ordering is chosen. +/// 2. A pair of `Ordering`s. The first one is for the success case, while the second one is +/// for the failure case. +// TODO: remove in the next major version. +#[deprecated( + note = "`compare_and_set` and `compare_and_set_weak` that use this trait are deprecated, \ + use `compare_exchange` or `compare_exchange_weak instead`" +)] +pub trait CompareAndSetOrdering { + /// The ordering of the operation when it succeeds. + fn success(&self) -> Ordering; + + /// The ordering of the operation when it fails. + /// + /// The failure ordering can't be `Release` or `AcqRel` and must be equivalent or weaker than + /// the success ordering. + fn failure(&self) -> Ordering; +} + +#[allow(deprecated)] +impl CompareAndSetOrdering for Ordering { + #[inline] + fn success(&self) -> Ordering { + *self + } + + #[inline] + fn failure(&self) -> Ordering { + strongest_failure_ordering(*self) + } +} + +#[allow(deprecated)] +impl CompareAndSetOrdering for (Ordering, Ordering) { + #[inline] + fn success(&self) -> Ordering { + self.0 + } + + #[inline] + fn failure(&self) -> Ordering { + self.1 + } +} + +/// Returns a bitmask containing the unused least significant bits of an aligned pointer to `T`. +#[inline] +fn low_bits() -> usize { + (1 << T::ALIGN.trailing_zeros()) - 1 +} + +/// Panics if the pointer is not properly unaligned. +#[inline] +fn ensure_aligned(raw: usize) { + assert_eq!(raw & low_bits::(), 0, "unaligned pointer"); +} + +/// Given a tagged pointer `data`, returns the same pointer, but tagged with `tag`. +/// +/// `tag` is truncated to fit into the unused bits of the pointer to `T`. +#[inline] +fn compose_tag(data: usize, tag: usize) -> usize { + (data & !low_bits::()) | (tag & low_bits::()) +} + +/// Decomposes a tagged pointer `data` into the pointer and the tag. +#[inline] +fn decompose_tag(data: usize) -> (usize, usize) { + (data & !low_bits::(), data & low_bits::()) +} + +/// Types that are pointed to by a single word. +/// +/// In concurrent programming, it is necessary to represent an object within a word because atomic +/// operations (e.g., reads, writes, read-modify-writes) support only single words. This trait +/// qualifies such types that are pointed to by a single word. +/// +/// The trait generalizes `Box` for a sized type `T`. In a box, an object of type `T` is +/// allocated in heap and it is owned by a single-word pointer. This trait is also implemented for +/// `[MaybeUninit]` by storing its size along with its elements and pointing to the pair of array +/// size and elements. +/// +/// Pointers to `Pointable` types can be stored in [`Atomic`], [`Owned`], and [`Shared`]. In +/// particular, Crossbeam supports dynamically sized slices as follows. +/// +/// ``` +/// use std::mem::MaybeUninit; +/// use crossbeam_epoch::Owned; +/// +/// let o = Owned::<[MaybeUninit]>::init(10); // allocating [i32; 10] +/// ``` +pub trait Pointable { + /// The alignment of pointer. + const ALIGN: usize; + + /// The type for initializers. + type Init; + + /// Initializes a with the given initializer. + /// + /// # Safety + /// + /// The result should be a multiple of `ALIGN`. + unsafe fn init(init: Self::Init) -> usize; + + /// Dereferences the given pointer. + /// + /// # Safety + /// + /// - The given `ptr` should have been initialized with [`Pointable::init`]. + /// - `ptr` should not have yet been dropped by [`Pointable::drop`]. + /// - `ptr` should not be mutably dereferenced by [`Pointable::deref_mut`] concurrently. + unsafe fn deref<'a>(ptr: usize) -> &'a Self; + + /// Mutably dereferences the given pointer. + /// + /// # Safety + /// + /// - The given `ptr` should have been initialized with [`Pointable::init`]. + /// - `ptr` should not have yet been dropped by [`Pointable::drop`]. + /// - `ptr` should not be dereferenced by [`Pointable::deref`] or [`Pointable::deref_mut`] + /// concurrently. + unsafe fn deref_mut<'a>(ptr: usize) -> &'a mut Self; + + /// Drops the object pointed to by the given pointer. + /// + /// # Safety + /// + /// - The given `ptr` should have been initialized with [`Pointable::init`]. + /// - `ptr` should not have yet been dropped by [`Pointable::drop`]. + /// - `ptr` should not be dereferenced by [`Pointable::deref`] or [`Pointable::deref_mut`] + /// concurrently. + unsafe fn drop(ptr: usize); +} + +impl Pointable for T { + const ALIGN: usize = mem::align_of::(); + + type Init = T; + + unsafe fn init(init: Self::Init) -> usize { + Box::into_raw(Box::new(init)) as usize + } + + unsafe fn deref<'a>(ptr: usize) -> &'a Self { + &*(ptr as *const T) + } + + unsafe fn deref_mut<'a>(ptr: usize) -> &'a mut Self { + &mut *(ptr as *mut T) + } + + unsafe fn drop(ptr: usize) { + drop(Box::from_raw(ptr as *mut T)); + } +} + +/// Array with size. +/// +/// # Memory layout +/// +/// An array consisting of size and elements: +/// +/// ```text +/// elements +/// | +/// | +/// ------------------------------------ +/// | size | 0 | 1 | 2 | 3 | 4 | 5 | 6 | +/// ------------------------------------ +/// ``` +/// +/// Its memory layout is different from that of `Box<[T]>` in that size is in the allocation (not +/// along with pointer as in `Box<[T]>`). +/// +/// Elements are not present in the type, but they will be in the allocation. +/// ``` +#[repr(C)] +struct Array { + /// The number of elements (not the number of bytes). + len: usize, + elements: [MaybeUninit; 0], +} + +impl Array { + fn layout(len: usize) -> Layout { + Layout::new::() + .extend(Layout::array::>(len).unwrap()) + .unwrap() + .0 + .pad_to_align() + } +} + +impl Pointable for [MaybeUninit] { + const ALIGN: usize = mem::align_of::>(); + + type Init = usize; + + unsafe fn init(len: Self::Init) -> usize { + let layout = Array::::layout(len); + let ptr = alloc::alloc::alloc(layout).cast::>(); + if ptr.is_null() { + alloc::alloc::handle_alloc_error(layout); + } + ptr::addr_of_mut!((*ptr).len).write(len); + ptr as usize + } + + unsafe fn deref<'a>(ptr: usize) -> &'a Self { + let array = &*(ptr as *const Array); + slice::from_raw_parts(array.elements.as_ptr() as *const _, array.len) + } + + unsafe fn deref_mut<'a>(ptr: usize) -> &'a mut Self { + let array = &*(ptr as *mut Array); + slice::from_raw_parts_mut(array.elements.as_ptr() as *mut _, array.len) + } + + unsafe fn drop(ptr: usize) { + let len = (*(ptr as *mut Array)).len; + let layout = Array::::layout(len); + alloc::alloc::dealloc(ptr as *mut u8, layout); + } +} + +/// An atomic pointer that can be safely shared between threads. +/// +/// The pointer must be properly aligned. Since it is aligned, a tag can be stored into the unused +/// least significant bits of the address. For example, the tag for a pointer to a sized type `T` +/// should be less than `(1 << mem::align_of::().trailing_zeros())`. +/// +/// Any method that loads the pointer must be passed a reference to a [`Guard`]. +/// +/// Crossbeam supports dynamically sized types. See [`Pointable`] for details. +pub struct Atomic { + data: AtomicUsize, + _marker: PhantomData<*mut T>, +} + +unsafe impl Send for Atomic {} +unsafe impl Sync for Atomic {} + +impl Atomic { + /// Allocates `value` on the heap and returns a new atomic pointer pointing to it. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::Atomic; + /// + /// let a = Atomic::new(1234); + /// # unsafe { drop(a.into_owned()); } // avoid leak + /// ``` + pub fn new(init: T) -> Atomic { + Self::init(init) + } +} + +impl Atomic { + /// Allocates `value` on the heap and returns a new atomic pointer pointing to it. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::Atomic; + /// + /// let a = Atomic::::init(1234); + /// # unsafe { drop(a.into_owned()); } // avoid leak + /// ``` + pub fn init(init: T::Init) -> Atomic { + Self::from(Owned::init(init)) + } + + /// Returns a new atomic pointer pointing to the tagged pointer `data`. + fn from_usize(data: usize) -> Self { + Self { + data: AtomicUsize::new(data), + _marker: PhantomData, + } + } + + /// Returns a new null atomic pointer. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::Atomic; + /// + /// let a = Atomic::::null(); + /// ``` + #[cfg(not(crossbeam_loom))] + pub const fn null() -> Atomic { + Self { + data: AtomicUsize::new(0), + _marker: PhantomData, + } + } + /// Returns a new null atomic pointer. + #[cfg(crossbeam_loom)] + pub fn null() -> Atomic { + Self { + data: AtomicUsize::new(0), + _marker: PhantomData, + } + } + + /// Loads a `Shared` from the atomic pointer. + /// + /// This method takes an [`Ordering`] argument which describes the memory ordering of this + /// operation. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::{self as epoch, Atomic}; + /// use std::sync::atomic::Ordering::SeqCst; + /// + /// let a = Atomic::new(1234); + /// let guard = &epoch::pin(); + /// let p = a.load(SeqCst, guard); + /// # unsafe { drop(a.into_owned()); } // avoid leak + /// ``` + pub fn load<'g>(&self, ord: Ordering, _: &'g Guard) -> Shared<'g, T> { + unsafe { Shared::from_usize(self.data.load(ord)) } + } + + /// Loads a `Shared` from the atomic pointer using a "consume" memory ordering. + /// + /// This is similar to the "acquire" ordering, except that an ordering is + /// only guaranteed with operations that "depend on" the result of the load. + /// However consume loads are usually much faster than acquire loads on + /// architectures with a weak memory model since they don't require memory + /// fence instructions. + /// + /// The exact definition of "depend on" is a bit vague, but it works as you + /// would expect in practice since a lot of software, especially the Linux + /// kernel, rely on this behavior. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::{self as epoch, Atomic}; + /// + /// let a = Atomic::new(1234); + /// let guard = &epoch::pin(); + /// let p = a.load_consume(guard); + /// # unsafe { drop(a.into_owned()); } // avoid leak + /// ``` + pub fn load_consume<'g>(&self, _: &'g Guard) -> Shared<'g, T> { + unsafe { Shared::from_usize(self.data.load_consume()) } + } + + /// Stores a `Shared` or `Owned` pointer into the atomic pointer. + /// + /// This method takes an [`Ordering`] argument which describes the memory ordering of this + /// operation. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::{Atomic, Owned, Shared}; + /// use std::sync::atomic::Ordering::SeqCst; + /// + /// let a = Atomic::new(1234); + /// # unsafe { drop(a.load(SeqCst, &crossbeam_epoch::pin()).into_owned()); } // avoid leak + /// a.store(Shared::null(), SeqCst); + /// a.store(Owned::new(1234), SeqCst); + /// # unsafe { drop(a.into_owned()); } // avoid leak + /// ``` + pub fn store>(&self, new: P, ord: Ordering) { + self.data.store(new.into_usize(), ord); + } + + /// Stores a `Shared` or `Owned` pointer into the atomic pointer, returning the previous + /// `Shared`. + /// + /// This method takes an [`Ordering`] argument which describes the memory ordering of this + /// operation. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::{self as epoch, Atomic, Shared}; + /// use std::sync::atomic::Ordering::SeqCst; + /// + /// let a = Atomic::new(1234); + /// let guard = &epoch::pin(); + /// let p = a.swap(Shared::null(), SeqCst, guard); + /// # unsafe { drop(p.into_owned()); } // avoid leak + /// ``` + pub fn swap<'g, P: Pointer>(&self, new: P, ord: Ordering, _: &'g Guard) -> Shared<'g, T> { + unsafe { Shared::from_usize(self.data.swap(new.into_usize(), ord)) } + } + + /// Stores the pointer `new` (either `Shared` or `Owned`) into the atomic pointer if the current + /// value is the same as `current`. The tag is also taken into account, so two pointers to the + /// same object, but with different tags, will not be considered equal. + /// + /// The return value is a result indicating whether the new pointer was written. On success the + /// pointer that was written is returned. On failure the actual current value and `new` are + /// returned. + /// + /// This method takes two `Ordering` arguments to describe the memory + /// ordering of this operation. `success` describes the required ordering for the + /// read-modify-write operation that takes place if the comparison with `current` succeeds. + /// `failure` describes the required ordering for the load operation that takes place when + /// the comparison fails. Using `Acquire` as success ordering makes the store part + /// of this operation `Relaxed`, and using `Release` makes the successful load + /// `Relaxed`. The failure ordering can only be `SeqCst`, `Acquire` or `Relaxed` + /// and must be equivalent to or weaker than the success ordering. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::{self as epoch, Atomic, Owned, Shared}; + /// use std::sync::atomic::Ordering::SeqCst; + /// + /// let a = Atomic::new(1234); + /// + /// let guard = &epoch::pin(); + /// let curr = a.load(SeqCst, guard); + /// let res1 = a.compare_exchange(curr, Shared::null(), SeqCst, SeqCst, guard); + /// let res2 = a.compare_exchange(curr, Owned::new(5678), SeqCst, SeqCst, guard); + /// # unsafe { drop(curr.into_owned()); } // avoid leak + /// ``` + pub fn compare_exchange<'g, P>( + &self, + current: Shared<'_, T>, + new: P, + success: Ordering, + failure: Ordering, + _: &'g Guard, + ) -> Result, CompareExchangeError<'g, T, P>> + where + P: Pointer, + { + let new = new.into_usize(); + self.data + .compare_exchange(current.into_usize(), new, success, failure) + .map(|_| unsafe { Shared::from_usize(new) }) + .map_err(|current| unsafe { + CompareExchangeError { + current: Shared::from_usize(current), + new: P::from_usize(new), + } + }) + } + + /// Stores the pointer `new` (either `Shared` or `Owned`) into the atomic pointer if the current + /// value is the same as `current`. The tag is also taken into account, so two pointers to the + /// same object, but with different tags, will not be considered equal. + /// + /// Unlike [`compare_exchange`], this method is allowed to spuriously fail even when comparison + /// succeeds, which can result in more efficient code on some platforms. The return value is a + /// result indicating whether the new pointer was written. On success the pointer that was + /// written is returned. On failure the actual current value and `new` are returned. + /// + /// This method takes two `Ordering` arguments to describe the memory + /// ordering of this operation. `success` describes the required ordering for the + /// read-modify-write operation that takes place if the comparison with `current` succeeds. + /// `failure` describes the required ordering for the load operation that takes place when + /// the comparison fails. Using `Acquire` as success ordering makes the store part + /// of this operation `Relaxed`, and using `Release` makes the successful load + /// `Relaxed`. The failure ordering can only be `SeqCst`, `Acquire` or `Relaxed` + /// and must be equivalent to or weaker than the success ordering. + /// + /// [`compare_exchange`]: Atomic::compare_exchange + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::{self as epoch, Atomic, Owned, Shared}; + /// use std::sync::atomic::Ordering::SeqCst; + /// + /// let a = Atomic::new(1234); + /// let guard = &epoch::pin(); + /// + /// let mut new = Owned::new(5678); + /// let mut ptr = a.load(SeqCst, guard); + /// # unsafe { drop(a.load(SeqCst, guard).into_owned()); } // avoid leak + /// loop { + /// match a.compare_exchange_weak(ptr, new, SeqCst, SeqCst, guard) { + /// Ok(p) => { + /// ptr = p; + /// break; + /// } + /// Err(err) => { + /// ptr = err.current; + /// new = err.new; + /// } + /// } + /// } + /// + /// let mut curr = a.load(SeqCst, guard); + /// loop { + /// match a.compare_exchange_weak(curr, Shared::null(), SeqCst, SeqCst, guard) { + /// Ok(_) => break, + /// Err(err) => curr = err.current, + /// } + /// } + /// # unsafe { drop(curr.into_owned()); } // avoid leak + /// ``` + pub fn compare_exchange_weak<'g, P>( + &self, + current: Shared<'_, T>, + new: P, + success: Ordering, + failure: Ordering, + _: &'g Guard, + ) -> Result, CompareExchangeError<'g, T, P>> + where + P: Pointer, + { + let new = new.into_usize(); + self.data + .compare_exchange_weak(current.into_usize(), new, success, failure) + .map(|_| unsafe { Shared::from_usize(new) }) + .map_err(|current| unsafe { + CompareExchangeError { + current: Shared::from_usize(current), + new: P::from_usize(new), + } + }) + } + + /// Fetches the pointer, and then applies a function to it that returns a new value. + /// Returns a `Result` of `Ok(previous_value)` if the function returned `Some`, else `Err(_)`. + /// + /// Note that the given function may be called multiple times if the value has been changed by + /// other threads in the meantime, as long as the function returns `Some(_)`, but the function + /// will have been applied only once to the stored value. + /// + /// `fetch_update` takes two [`Ordering`] arguments to describe the memory + /// ordering of this operation. The first describes the required ordering for + /// when the operation finally succeeds while the second describes the + /// required ordering for loads. These correspond to the success and failure + /// orderings of [`Atomic::compare_exchange`] respectively. + /// + /// Using [`Acquire`] as success ordering makes the store part of this + /// operation [`Relaxed`], and using [`Release`] makes the final successful + /// load [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], + /// [`Acquire`] or [`Relaxed`] and must be equivalent to or weaker than the + /// success ordering. + /// + /// [`Relaxed`]: Ordering::Relaxed + /// [`Acquire`]: Ordering::Acquire + /// [`Release`]: Ordering::Release + /// [`SeqCst`]: Ordering::SeqCst + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::{self as epoch, Atomic}; + /// use std::sync::atomic::Ordering::SeqCst; + /// + /// let a = Atomic::new(1234); + /// let guard = &epoch::pin(); + /// + /// let res1 = a.fetch_update(SeqCst, SeqCst, guard, |x| Some(x.with_tag(1))); + /// assert!(res1.is_ok()); + /// + /// let res2 = a.fetch_update(SeqCst, SeqCst, guard, |x| None); + /// assert!(res2.is_err()); + /// # unsafe { drop(a.into_owned()); } // avoid leak + /// ``` + pub fn fetch_update<'g, F>( + &self, + set_order: Ordering, + fail_order: Ordering, + guard: &'g Guard, + mut func: F, + ) -> Result, Shared<'g, T>> + where + F: FnMut(Shared<'g, T>) -> Option>, + { + let mut prev = self.load(fail_order, guard); + while let Some(next) = func(prev) { + match self.compare_exchange_weak(prev, next, set_order, fail_order, guard) { + Ok(shared) => return Ok(shared), + Err(next_prev) => prev = next_prev.current, + } + } + Err(prev) + } + + /// Stores the pointer `new` (either `Shared` or `Owned`) into the atomic pointer if the current + /// value is the same as `current`. The tag is also taken into account, so two pointers to the + /// same object, but with different tags, will not be considered equal. + /// + /// The return value is a result indicating whether the new pointer was written. On success the + /// pointer that was written is returned. On failure the actual current value and `new` are + /// returned. + /// + /// This method takes a [`CompareAndSetOrdering`] argument which describes the memory + /// ordering of this operation. + /// + /// # Migrating to `compare_exchange` + /// + /// `compare_and_set` is equivalent to `compare_exchange` with the following mapping for + /// memory orderings: + /// + /// Original | Success | Failure + /// -------- | ------- | ------- + /// Relaxed | Relaxed | Relaxed + /// Acquire | Acquire | Acquire + /// Release | Release | Relaxed + /// AcqRel | AcqRel | Acquire + /// SeqCst | SeqCst | SeqCst + /// + /// # Examples + /// + /// ``` + /// # #![allow(deprecated)] + /// use crossbeam_epoch::{self as epoch, Atomic, Owned, Shared}; + /// use std::sync::atomic::Ordering::SeqCst; + /// + /// let a = Atomic::new(1234); + /// + /// let guard = &epoch::pin(); + /// let curr = a.load(SeqCst, guard); + /// let res1 = a.compare_and_set(curr, Shared::null(), SeqCst, guard); + /// let res2 = a.compare_and_set(curr, Owned::new(5678), SeqCst, guard); + /// # unsafe { drop(curr.into_owned()); } // avoid leak + /// ``` + // TODO: remove in the next major version. + #[allow(deprecated)] + #[deprecated(note = "Use `compare_exchange` instead")] + pub fn compare_and_set<'g, O, P>( + &self, + current: Shared<'_, T>, + new: P, + ord: O, + guard: &'g Guard, + ) -> Result, CompareAndSetError<'g, T, P>> + where + O: CompareAndSetOrdering, + P: Pointer, + { + self.compare_exchange(current, new, ord.success(), ord.failure(), guard) + } + + /// Stores the pointer `new` (either `Shared` or `Owned`) into the atomic pointer if the current + /// value is the same as `current`. The tag is also taken into account, so two pointers to the + /// same object, but with different tags, will not be considered equal. + /// + /// Unlike [`compare_and_set`], this method is allowed to spuriously fail even when comparison + /// succeeds, which can result in more efficient code on some platforms. The return value is a + /// result indicating whether the new pointer was written. On success the pointer that was + /// written is returned. On failure the actual current value and `new` are returned. + /// + /// This method takes a [`CompareAndSetOrdering`] argument which describes the memory + /// ordering of this operation. + /// + /// [`compare_and_set`]: Atomic::compare_and_set + /// + /// # Migrating to `compare_exchange_weak` + /// + /// `compare_and_set_weak` is equivalent to `compare_exchange_weak` with the following mapping for + /// memory orderings: + /// + /// Original | Success | Failure + /// -------- | ------- | ------- + /// Relaxed | Relaxed | Relaxed + /// Acquire | Acquire | Acquire + /// Release | Release | Relaxed + /// AcqRel | AcqRel | Acquire + /// SeqCst | SeqCst | SeqCst + /// + /// # Examples + /// + /// ``` + /// # #![allow(deprecated)] + /// use crossbeam_epoch::{self as epoch, Atomic, Owned, Shared}; + /// use std::sync::atomic::Ordering::SeqCst; + /// + /// let a = Atomic::new(1234); + /// let guard = &epoch::pin(); + /// + /// let mut new = Owned::new(5678); + /// let mut ptr = a.load(SeqCst, guard); + /// # unsafe { drop(a.load(SeqCst, guard).into_owned()); } // avoid leak + /// loop { + /// match a.compare_and_set_weak(ptr, new, SeqCst, guard) { + /// Ok(p) => { + /// ptr = p; + /// break; + /// } + /// Err(err) => { + /// ptr = err.current; + /// new = err.new; + /// } + /// } + /// } + /// + /// let mut curr = a.load(SeqCst, guard); + /// loop { + /// match a.compare_and_set_weak(curr, Shared::null(), SeqCst, guard) { + /// Ok(_) => break, + /// Err(err) => curr = err.current, + /// } + /// } + /// # unsafe { drop(curr.into_owned()); } // avoid leak + /// ``` + // TODO: remove in the next major version. + #[allow(deprecated)] + #[deprecated(note = "Use `compare_exchange_weak` instead")] + pub fn compare_and_set_weak<'g, O, P>( + &self, + current: Shared<'_, T>, + new: P, + ord: O, + guard: &'g Guard, + ) -> Result, CompareAndSetError<'g, T, P>> + where + O: CompareAndSetOrdering, + P: Pointer, + { + self.compare_exchange_weak(current, new, ord.success(), ord.failure(), guard) + } + + /// Bitwise "and" with the current tag. + /// + /// Performs a bitwise "and" operation on the current tag and the argument `val`, and sets the + /// new tag to the result. Returns the previous pointer. + /// + /// This method takes an [`Ordering`] argument which describes the memory ordering of this + /// operation. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::{self as epoch, Atomic, Shared}; + /// use std::sync::atomic::Ordering::SeqCst; + /// + /// let a = Atomic::::from(Shared::null().with_tag(3)); + /// let guard = &epoch::pin(); + /// assert_eq!(a.fetch_and(2, SeqCst, guard).tag(), 3); + /// assert_eq!(a.load(SeqCst, guard).tag(), 2); + /// ``` + pub fn fetch_and<'g>(&self, val: usize, ord: Ordering, _: &'g Guard) -> Shared<'g, T> { + unsafe { Shared::from_usize(self.data.fetch_and(val | !low_bits::(), ord)) } + } + + /// Bitwise "or" with the current tag. + /// + /// Performs a bitwise "or" operation on the current tag and the argument `val`, and sets the + /// new tag to the result. Returns the previous pointer. + /// + /// This method takes an [`Ordering`] argument which describes the memory ordering of this + /// operation. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::{self as epoch, Atomic, Shared}; + /// use std::sync::atomic::Ordering::SeqCst; + /// + /// let a = Atomic::::from(Shared::null().with_tag(1)); + /// let guard = &epoch::pin(); + /// assert_eq!(a.fetch_or(2, SeqCst, guard).tag(), 1); + /// assert_eq!(a.load(SeqCst, guard).tag(), 3); + /// ``` + pub fn fetch_or<'g>(&self, val: usize, ord: Ordering, _: &'g Guard) -> Shared<'g, T> { + unsafe { Shared::from_usize(self.data.fetch_or(val & low_bits::(), ord)) } + } + + /// Bitwise "xor" with the current tag. + /// + /// Performs a bitwise "xor" operation on the current tag and the argument `val`, and sets the + /// new tag to the result. Returns the previous pointer. + /// + /// This method takes an [`Ordering`] argument which describes the memory ordering of this + /// operation. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::{self as epoch, Atomic, Shared}; + /// use std::sync::atomic::Ordering::SeqCst; + /// + /// let a = Atomic::::from(Shared::null().with_tag(1)); + /// let guard = &epoch::pin(); + /// assert_eq!(a.fetch_xor(3, SeqCst, guard).tag(), 1); + /// assert_eq!(a.load(SeqCst, guard).tag(), 2); + /// ``` + pub fn fetch_xor<'g>(&self, val: usize, ord: Ordering, _: &'g Guard) -> Shared<'g, T> { + unsafe { Shared::from_usize(self.data.fetch_xor(val & low_bits::(), ord)) } + } + + /// Takes ownership of the pointee. + /// + /// This consumes the atomic and converts it into [`Owned`]. As [`Atomic`] doesn't have a + /// destructor and doesn't drop the pointee while [`Owned`] does, this is suitable for + /// destructors of data structures. + /// + /// # Panics + /// + /// Panics if this pointer is null, but only in debug mode. + /// + /// # Safety + /// + /// This method may be called only if the pointer is valid and nobody else is holding a + /// reference to the same object. + /// + /// # Examples + /// + /// ```rust + /// # use std::mem; + /// # use crossbeam_epoch::Atomic; + /// struct DataStructure { + /// ptr: Atomic, + /// } + /// + /// impl Drop for DataStructure { + /// fn drop(&mut self) { + /// // By now the DataStructure lives only in our thread and we are sure we don't hold + /// // any Shared or & to it ourselves. + /// unsafe { + /// drop(mem::replace(&mut self.ptr, Atomic::null()).into_owned()); + /// } + /// } + /// } + /// ``` + pub unsafe fn into_owned(self) -> Owned { + Owned::from_usize(self.data.into_inner()) + } + + /// Takes ownership of the pointee if it is non-null. + /// + /// This consumes the atomic and converts it into [`Owned`]. As [`Atomic`] doesn't have a + /// destructor and doesn't drop the pointee while [`Owned`] does, this is suitable for + /// destructors of data structures. + /// + /// # Safety + /// + /// This method may be called only if the pointer is valid and nobody else is holding a + /// reference to the same object, or the pointer is null. + /// + /// # Examples + /// + /// ```rust + /// # use std::mem; + /// # use crossbeam_epoch::Atomic; + /// struct DataStructure { + /// ptr: Atomic, + /// } + /// + /// impl Drop for DataStructure { + /// fn drop(&mut self) { + /// // By now the DataStructure lives only in our thread and we are sure we don't hold + /// // any Shared or & to it ourselves, but it may be null, so we have to be careful. + /// let old = mem::replace(&mut self.ptr, Atomic::null()); + /// unsafe { + /// if let Some(x) = old.try_into_owned() { + /// drop(x) + /// } + /// } + /// } + /// } + /// ``` + pub unsafe fn try_into_owned(self) -> Option> { + let data = self.data.into_inner(); + if decompose_tag::(data).0 == 0 { + None + } else { + Some(Owned::from_usize(data)) + } + } +} + +impl fmt::Debug for Atomic { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let data = self.data.load(Ordering::SeqCst); + let (raw, tag) = decompose_tag::(data); + + f.debug_struct("Atomic") + .field("raw", &raw) + .field("tag", &tag) + .finish() + } +} + +impl fmt::Pointer for Atomic { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let data = self.data.load(Ordering::SeqCst); + let (raw, _) = decompose_tag::(data); + fmt::Pointer::fmt(&(unsafe { T::deref(raw) as *const _ }), f) + } +} + +impl Clone for Atomic { + /// Returns a copy of the atomic value. + /// + /// Note that a `Relaxed` load is used here. If you need synchronization, use it with other + /// atomics or fences. + fn clone(&self) -> Self { + let data = self.data.load(Ordering::Relaxed); + Atomic::from_usize(data) + } +} + +impl Default for Atomic { + fn default() -> Self { + Atomic::null() + } +} + +impl From> for Atomic { + /// Returns a new atomic pointer pointing to `owned`. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::{Atomic, Owned}; + /// + /// let a = Atomic::::from(Owned::new(1234)); + /// # unsafe { drop(a.into_owned()); } // avoid leak + /// ``` + fn from(owned: Owned) -> Self { + let data = owned.data; + mem::forget(owned); + Self::from_usize(data) + } +} + +impl From> for Atomic { + fn from(b: Box) -> Self { + Self::from(Owned::from(b)) + } +} + +impl From for Atomic { + fn from(t: T) -> Self { + Self::new(t) + } +} + +impl<'g, T: ?Sized + Pointable> From> for Atomic { + /// Returns a new atomic pointer pointing to `ptr`. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::{Atomic, Shared}; + /// + /// let a = Atomic::::from(Shared::::null()); + /// ``` + fn from(ptr: Shared<'g, T>) -> Self { + Self::from_usize(ptr.data) + } +} + +impl From<*const T> for Atomic { + /// Returns a new atomic pointer pointing to `raw`. + /// + /// # Examples + /// + /// ``` + /// use std::ptr; + /// use crossbeam_epoch::Atomic; + /// + /// let a = Atomic::::from(ptr::null::()); + /// ``` + fn from(raw: *const T) -> Self { + Self::from_usize(raw as usize) + } +} + +/// A trait for either `Owned` or `Shared` pointers. +pub trait Pointer { + /// Returns the machine representation of the pointer. + fn into_usize(self) -> usize; + + /// Returns a new pointer pointing to the tagged pointer `data`. + /// + /// # Safety + /// + /// The given `data` should have been created by `Pointer::into_usize()`, and one `data` should + /// not be converted back by `Pointer::from_usize()` multiple times. + unsafe fn from_usize(data: usize) -> Self; +} + +/// An owned heap-allocated object. +/// +/// This type is very similar to `Box`. +/// +/// The pointer must be properly aligned. Since it is aligned, a tag can be stored into the unused +/// least significant bits of the address. +pub struct Owned { + data: usize, + _marker: PhantomData>, +} + +impl Pointer for Owned { + #[inline] + fn into_usize(self) -> usize { + let data = self.data; + mem::forget(self); + data + } + + /// Returns a new pointer pointing to the tagged pointer `data`. + /// + /// # Panics + /// + /// Panics if the data is zero in debug mode. + #[inline] + unsafe fn from_usize(data: usize) -> Self { + debug_assert!(data != 0, "converting zero into `Owned`"); + Owned { + data, + _marker: PhantomData, + } + } +} + +impl Owned { + /// Returns a new owned pointer pointing to `raw`. + /// + /// This function is unsafe because improper use may lead to memory problems. Argument `raw` + /// must be a valid pointer. Also, a double-free may occur if the function is called twice on + /// the same raw pointer. + /// + /// # Panics + /// + /// Panics if `raw` is not properly aligned. + /// + /// # Safety + /// + /// The given `raw` should have been derived from `Owned`, and one `raw` should not be converted + /// back by `Owned::from_raw()` multiple times. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::Owned; + /// + /// let o = unsafe { Owned::from_raw(Box::into_raw(Box::new(1234))) }; + /// ``` + pub unsafe fn from_raw(raw: *mut T) -> Owned { + let raw = raw as usize; + ensure_aligned::(raw); + Self::from_usize(raw) + } + + /// Converts the owned pointer into a `Box`. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::Owned; + /// + /// let o = Owned::new(1234); + /// let b: Box = o.into_box(); + /// assert_eq!(*b, 1234); + /// ``` + pub fn into_box(self) -> Box { + let (raw, _) = decompose_tag::(self.data); + mem::forget(self); + unsafe { Box::from_raw(raw as *mut _) } + } + + /// Allocates `value` on the heap and returns a new owned pointer pointing to it. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::Owned; + /// + /// let o = Owned::new(1234); + /// ``` + pub fn new(init: T) -> Owned { + Self::init(init) + } +} + +impl Owned { + /// Allocates `value` on the heap and returns a new owned pointer pointing to it. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::Owned; + /// + /// let o = Owned::::init(1234); + /// ``` + pub fn init(init: T::Init) -> Owned { + unsafe { Self::from_usize(T::init(init)) } + } + + /// Converts the owned pointer into a [`Shared`]. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::{self as epoch, Owned}; + /// + /// let o = Owned::new(1234); + /// let guard = &epoch::pin(); + /// let p = o.into_shared(guard); + /// # unsafe { drop(p.into_owned()); } // avoid leak + /// ``` + #[allow(clippy::needless_lifetimes)] + pub fn into_shared<'g>(self, _: &'g Guard) -> Shared<'g, T> { + unsafe { Shared::from_usize(self.into_usize()) } + } + + /// Returns the tag stored within the pointer. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::Owned; + /// + /// assert_eq!(Owned::new(1234).tag(), 0); + /// ``` + pub fn tag(&self) -> usize { + let (_, tag) = decompose_tag::(self.data); + tag + } + + /// Returns the same pointer, but tagged with `tag`. `tag` is truncated to be fit into the + /// unused bits of the pointer to `T`. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::Owned; + /// + /// let o = Owned::new(0u64); + /// assert_eq!(o.tag(), 0); + /// let o = o.with_tag(2); + /// assert_eq!(o.tag(), 2); + /// ``` + pub fn with_tag(self, tag: usize) -> Owned { + let data = self.into_usize(); + unsafe { Self::from_usize(compose_tag::(data, tag)) } + } +} + +impl Drop for Owned { + fn drop(&mut self) { + let (raw, _) = decompose_tag::(self.data); + unsafe { + T::drop(raw); + } + } +} + +impl fmt::Debug for Owned { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let (raw, tag) = decompose_tag::(self.data); + + f.debug_struct("Owned") + .field("raw", &raw) + .field("tag", &tag) + .finish() + } +} + +impl Clone for Owned { + fn clone(&self) -> Self { + Owned::new((**self).clone()).with_tag(self.tag()) + } +} + +impl Deref for Owned { + type Target = T; + + fn deref(&self) -> &T { + let (raw, _) = decompose_tag::(self.data); + unsafe { T::deref(raw) } + } +} + +impl DerefMut for Owned { + fn deref_mut(&mut self) -> &mut T { + let (raw, _) = decompose_tag::(self.data); + unsafe { T::deref_mut(raw) } + } +} + +impl From for Owned { + fn from(t: T) -> Self { + Owned::new(t) + } +} + +impl From> for Owned { + /// Returns a new owned pointer pointing to `b`. + /// + /// # Panics + /// + /// Panics if the pointer (the `Box`) is not properly aligned. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::Owned; + /// + /// let o = unsafe { Owned::from_raw(Box::into_raw(Box::new(1234))) }; + /// ``` + fn from(b: Box) -> Self { + unsafe { Self::from_raw(Box::into_raw(b)) } + } +} + +impl Borrow for Owned { + fn borrow(&self) -> &T { + self.deref() + } +} + +impl BorrowMut for Owned { + fn borrow_mut(&mut self) -> &mut T { + self.deref_mut() + } +} + +impl AsRef for Owned { + fn as_ref(&self) -> &T { + self.deref() + } +} + +impl AsMut for Owned { + fn as_mut(&mut self) -> &mut T { + self.deref_mut() + } +} + +/// A pointer to an object protected by the epoch GC. +/// +/// The pointer is valid for use only during the lifetime `'g`. +/// +/// The pointer must be properly aligned. Since it is aligned, a tag can be stored into the unused +/// least significant bits of the address. +pub struct Shared<'g, T: 'g + ?Sized + Pointable> { + data: usize, + _marker: PhantomData<(&'g (), *const T)>, +} + +impl Clone for Shared<'_, T> { + fn clone(&self) -> Self { + *self + } +} + +impl Copy for Shared<'_, T> {} + +impl Pointer for Shared<'_, T> { + #[inline] + fn into_usize(self) -> usize { + self.data + } + + #[inline] + unsafe fn from_usize(data: usize) -> Self { + Shared { + data, + _marker: PhantomData, + } + } +} + +impl<'g, T> Shared<'g, T> { + /// Converts the pointer to a raw pointer (without the tag). + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::{self as epoch, Atomic, Owned}; + /// use std::sync::atomic::Ordering::SeqCst; + /// + /// let o = Owned::new(1234); + /// let raw = &*o as *const _; + /// let a = Atomic::from(o); + /// + /// let guard = &epoch::pin(); + /// let p = a.load(SeqCst, guard); + /// assert_eq!(p.as_raw(), raw); + /// # unsafe { drop(a.into_owned()); } // avoid leak + /// ``` + pub fn as_raw(&self) -> *const T { + let (raw, _) = decompose_tag::(self.data); + raw as *const _ + } +} + +impl<'g, T: ?Sized + Pointable> Shared<'g, T> { + /// Returns a new null pointer. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::Shared; + /// + /// let p = Shared::::null(); + /// assert!(p.is_null()); + /// ``` + pub fn null() -> Shared<'g, T> { + Shared { + data: 0, + _marker: PhantomData, + } + } + + /// Returns `true` if the pointer is null. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::{self as epoch, Atomic, Owned}; + /// use std::sync::atomic::Ordering::SeqCst; + /// + /// let a = Atomic::null(); + /// let guard = &epoch::pin(); + /// assert!(a.load(SeqCst, guard).is_null()); + /// a.store(Owned::new(1234), SeqCst); + /// assert!(!a.load(SeqCst, guard).is_null()); + /// # unsafe { drop(a.into_owned()); } // avoid leak + /// ``` + pub fn is_null(&self) -> bool { + let (raw, _) = decompose_tag::(self.data); + raw == 0 + } + + /// Dereferences the pointer. + /// + /// Returns a reference to the pointee that is valid during the lifetime `'g`. + /// + /// # Safety + /// + /// Dereferencing a pointer is unsafe because it could be pointing to invalid memory. + /// + /// Another concern is the possibility of data races due to lack of proper synchronization. + /// For example, consider the following scenario: + /// + /// 1. A thread creates a new object: `a.store(Owned::new(10), Relaxed)` + /// 2. Another thread reads it: `*a.load(Relaxed, guard).as_ref().unwrap()` + /// + /// The problem is that relaxed orderings don't synchronize initialization of the object with + /// the read from the second thread. This is a data race. A possible solution would be to use + /// `Release` and `Acquire` orderings. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::{self as epoch, Atomic}; + /// use std::sync::atomic::Ordering::SeqCst; + /// + /// let a = Atomic::new(1234); + /// let guard = &epoch::pin(); + /// let p = a.load(SeqCst, guard); + /// unsafe { + /// assert_eq!(p.deref(), &1234); + /// } + /// # unsafe { drop(a.into_owned()); } // avoid leak + /// ``` + pub unsafe fn deref(&self) -> &'g T { + let (raw, _) = decompose_tag::(self.data); + T::deref(raw) + } + + /// Dereferences the pointer. + /// + /// Returns a mutable reference to the pointee that is valid during the lifetime `'g`. + /// + /// # Safety + /// + /// * There is no guarantee that there are no more threads attempting to read/write from/to the + /// actual object at the same time. + /// + /// The user must know that there are no concurrent accesses towards the object itself. + /// + /// * Other than the above, all safety concerns of `deref()` applies here. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::{self as epoch, Atomic}; + /// use std::sync::atomic::Ordering::SeqCst; + /// + /// let a = Atomic::new(vec![1, 2, 3, 4]); + /// let guard = &epoch::pin(); + /// + /// let mut p = a.load(SeqCst, guard); + /// unsafe { + /// assert!(!p.is_null()); + /// let b = p.deref_mut(); + /// assert_eq!(b, &vec![1, 2, 3, 4]); + /// b.push(5); + /// assert_eq!(b, &vec![1, 2, 3, 4, 5]); + /// } + /// + /// let p = a.load(SeqCst, guard); + /// unsafe { + /// assert_eq!(p.deref(), &vec![1, 2, 3, 4, 5]); + /// } + /// # unsafe { drop(a.into_owned()); } // avoid leak + /// ``` + pub unsafe fn deref_mut(&mut self) -> &'g mut T { + let (raw, _) = decompose_tag::(self.data); + T::deref_mut(raw) + } + + /// Converts the pointer to a reference. + /// + /// Returns `None` if the pointer is null, or else a reference to the object wrapped in `Some`. + /// + /// # Safety + /// + /// Dereferencing a pointer is unsafe because it could be pointing to invalid memory. + /// + /// Another concern is the possibility of data races due to lack of proper synchronization. + /// For example, consider the following scenario: + /// + /// 1. A thread creates a new object: `a.store(Owned::new(10), Relaxed)` + /// 2. Another thread reads it: `*a.load(Relaxed, guard).as_ref().unwrap()` + /// + /// The problem is that relaxed orderings don't synchronize initialization of the object with + /// the read from the second thread. This is a data race. A possible solution would be to use + /// `Release` and `Acquire` orderings. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::{self as epoch, Atomic}; + /// use std::sync::atomic::Ordering::SeqCst; + /// + /// let a = Atomic::new(1234); + /// let guard = &epoch::pin(); + /// let p = a.load(SeqCst, guard); + /// unsafe { + /// assert_eq!(p.as_ref(), Some(&1234)); + /// } + /// # unsafe { drop(a.into_owned()); } // avoid leak + /// ``` + pub unsafe fn as_ref(&self) -> Option<&'g T> { + let (raw, _) = decompose_tag::(self.data); + if raw == 0 { + None + } else { + Some(T::deref(raw)) + } + } + + /// Takes ownership of the pointee. + /// + /// # Panics + /// + /// Panics if this pointer is null, but only in debug mode. + /// + /// # Safety + /// + /// This method may be called only if the pointer is valid and nobody else is holding a + /// reference to the same object. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::{self as epoch, Atomic}; + /// use std::sync::atomic::Ordering::SeqCst; + /// + /// let a = Atomic::new(1234); + /// unsafe { + /// let guard = &epoch::unprotected(); + /// let p = a.load(SeqCst, guard); + /// drop(p.into_owned()); + /// } + /// ``` + pub unsafe fn into_owned(self) -> Owned { + debug_assert!(!self.is_null(), "converting a null `Shared` into `Owned`"); + Owned::from_usize(self.data) + } + + /// Takes ownership of the pointee if it is not null. + /// + /// # Safety + /// + /// This method may be called only if the pointer is valid and nobody else is holding a + /// reference to the same object, or if the pointer is null. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::{self as epoch, Atomic}; + /// use std::sync::atomic::Ordering::SeqCst; + /// + /// let a = Atomic::new(1234); + /// unsafe { + /// let guard = &epoch::unprotected(); + /// let p = a.load(SeqCst, guard); + /// if let Some(x) = p.try_into_owned() { + /// drop(x); + /// } + /// } + /// ``` + pub unsafe fn try_into_owned(self) -> Option> { + if self.is_null() { + None + } else { + Some(Owned::from_usize(self.data)) + } + } + + /// Returns the tag stored within the pointer. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::{self as epoch, Atomic, Owned}; + /// use std::sync::atomic::Ordering::SeqCst; + /// + /// let a = Atomic::::from(Owned::new(0u64).with_tag(2)); + /// let guard = &epoch::pin(); + /// let p = a.load(SeqCst, guard); + /// assert_eq!(p.tag(), 2); + /// # unsafe { drop(a.into_owned()); } // avoid leak + /// ``` + pub fn tag(&self) -> usize { + let (_, tag) = decompose_tag::(self.data); + tag + } + + /// Returns the same pointer, but tagged with `tag`. `tag` is truncated to be fit into the + /// unused bits of the pointer to `T`. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::{self as epoch, Atomic}; + /// use std::sync::atomic::Ordering::SeqCst; + /// + /// let a = Atomic::new(0u64); + /// let guard = &epoch::pin(); + /// let p1 = a.load(SeqCst, guard); + /// let p2 = p1.with_tag(2); + /// + /// assert_eq!(p1.tag(), 0); + /// assert_eq!(p2.tag(), 2); + /// assert_eq!(p1.as_raw(), p2.as_raw()); + /// # unsafe { drop(a.into_owned()); } // avoid leak + /// ``` + pub fn with_tag(&self, tag: usize) -> Shared<'g, T> { + unsafe { Self::from_usize(compose_tag::(self.data, tag)) } + } +} + +impl From<*const T> for Shared<'_, T> { + /// Returns a new pointer pointing to `raw`. + /// + /// # Panics + /// + /// Panics if `raw` is not properly aligned. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::Shared; + /// + /// let p = Shared::from(Box::into_raw(Box::new(1234)) as *const _); + /// assert!(!p.is_null()); + /// # unsafe { drop(p.into_owned()); } // avoid leak + /// ``` + fn from(raw: *const T) -> Self { + let raw = raw as usize; + ensure_aligned::(raw); + unsafe { Self::from_usize(raw) } + } +} + +impl<'g, T: ?Sized + Pointable> PartialEq> for Shared<'g, T> { + fn eq(&self, other: &Self) -> bool { + self.data == other.data + } +} + +impl Eq for Shared<'_, T> {} + +impl<'g, T: ?Sized + Pointable> PartialOrd> for Shared<'g, T> { + fn partial_cmp(&self, other: &Self) -> Option { + self.data.partial_cmp(&other.data) + } +} + +impl Ord for Shared<'_, T> { + fn cmp(&self, other: &Self) -> cmp::Ordering { + self.data.cmp(&other.data) + } +} + +impl fmt::Debug for Shared<'_, T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let (raw, tag) = decompose_tag::(self.data); + + f.debug_struct("Shared") + .field("raw", &raw) + .field("tag", &tag) + .finish() + } +} + +impl fmt::Pointer for Shared<'_, T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Pointer::fmt(&(unsafe { self.deref() as *const _ }), f) + } +} + +impl Default for Shared<'_, T> { + fn default() -> Self { + Shared::null() + } +} + +#[cfg(all(test, not(crossbeam_loom)))] +mod tests { + use super::{Owned, Shared}; + use std::mem::MaybeUninit; + + #[test] + fn valid_tag_i8() { + Shared::::null().with_tag(0); + } + + #[test] + fn valid_tag_i64() { + Shared::::null().with_tag(7); + } + + #[test] + fn const_atomic_null() { + use super::Atomic; + static _U: Atomic = Atomic::::null(); + } + + #[test] + fn array_init() { + let owned = Owned::<[MaybeUninit]>::init(10); + let arr: &[MaybeUninit] = &owned; + assert_eq!(arr.len(), 10); + } +} diff --git a/anneal/v2/vendor/crossbeam-epoch/src/collector.rs b/anneal/v2/vendor/crossbeam-epoch/src/collector.rs new file mode 100644 index 0000000000..12655d6cdb --- /dev/null +++ b/anneal/v2/vendor/crossbeam-epoch/src/collector.rs @@ -0,0 +1,464 @@ +/// Epoch-based garbage collector. +/// +/// # Examples +/// +/// ``` +/// use crossbeam_epoch::Collector; +/// +/// let collector = Collector::new(); +/// +/// let handle = collector.register(); +/// drop(collector); // `handle` still works after dropping `collector` +/// +/// handle.pin().flush(); +/// ``` +use core::fmt; + +use crate::guard::Guard; +use crate::internal::{Global, Local}; +use crate::primitive::sync::Arc; + +/// An epoch-based garbage collector. +pub struct Collector { + pub(crate) global: Arc, +} + +unsafe impl Send for Collector {} +unsafe impl Sync for Collector {} + +impl Default for Collector { + fn default() -> Self { + Self { + global: Arc::new(Global::new()), + } + } +} + +impl Collector { + /// Creates a new collector. + pub fn new() -> Self { + Self::default() + } + + /// Registers a new handle for the collector. + pub fn register(&self) -> LocalHandle { + Local::register(self) + } +} + +impl Clone for Collector { + /// Creates another reference to the same garbage collector. + fn clone(&self) -> Self { + Collector { + global: self.global.clone(), + } + } +} + +impl fmt::Debug for Collector { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad("Collector { .. }") + } +} + +impl PartialEq for Collector { + /// Checks if both handles point to the same collector. + fn eq(&self, rhs: &Collector) -> bool { + Arc::ptr_eq(&self.global, &rhs.global) + } +} +impl Eq for Collector {} + +/// A handle to a garbage collector. +pub struct LocalHandle { + pub(crate) local: *const Local, +} + +impl LocalHandle { + /// Pins the handle. + #[inline] + pub fn pin(&self) -> Guard { + unsafe { (*self.local).pin() } + } + + /// Returns `true` if the handle is pinned. + #[inline] + pub fn is_pinned(&self) -> bool { + unsafe { (*self.local).is_pinned() } + } + + /// Returns the `Collector` associated with this handle. + #[inline] + pub fn collector(&self) -> &Collector { + unsafe { (*self.local).collector() } + } +} + +impl Drop for LocalHandle { + #[inline] + fn drop(&mut self) { + unsafe { + Local::release_handle(&*self.local); + } + } +} + +impl fmt::Debug for LocalHandle { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad("LocalHandle { .. }") + } +} + +#[cfg(all(test, not(crossbeam_loom)))] +mod tests { + use std::mem::ManuallyDrop; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use crossbeam_utils::thread; + + use crate::{Collector, Owned}; + + const NUM_THREADS: usize = 8; + + #[test] + fn pin_reentrant() { + let collector = Collector::new(); + let handle = collector.register(); + drop(collector); + + assert!(!handle.is_pinned()); + { + let _guard = &handle.pin(); + assert!(handle.is_pinned()); + { + let _guard = &handle.pin(); + assert!(handle.is_pinned()); + } + assert!(handle.is_pinned()); + } + assert!(!handle.is_pinned()); + } + + #[test] + fn flush_local_bag() { + let collector = Collector::new(); + let handle = collector.register(); + drop(collector); + + for _ in 0..100 { + let guard = &handle.pin(); + unsafe { + let a = Owned::new(7).into_shared(guard); + guard.defer_destroy(a); + + assert!(!(*guard.local).bag.with(|b| (*b).is_empty())); + + while !(*guard.local).bag.with(|b| (*b).is_empty()) { + guard.flush(); + } + } + } + } + + #[test] + fn garbage_buffering() { + let collector = Collector::new(); + let handle = collector.register(); + drop(collector); + + let guard = &handle.pin(); + unsafe { + for _ in 0..10 { + let a = Owned::new(7).into_shared(guard); + guard.defer_destroy(a); + } + assert!(!(*guard.local).bag.with(|b| (*b).is_empty())); + } + } + + #[test] + fn pin_holds_advance() { + #[cfg(miri)] + const N: usize = 500; + #[cfg(not(miri))] + const N: usize = 500_000; + + let collector = Collector::new(); + + thread::scope(|scope| { + for _ in 0..NUM_THREADS { + scope.spawn(|_| { + let handle = collector.register(); + for _ in 0..N { + let guard = &handle.pin(); + + let before = collector.global.epoch.load(Ordering::Relaxed); + collector.global.collect(guard); + let after = collector.global.epoch.load(Ordering::Relaxed); + + assert!(after.wrapping_sub(before) <= 2); + } + }); + } + }) + .unwrap(); + } + + #[cfg(not(crossbeam_sanitize))] // TODO: assertions failed due to `cfg(crossbeam_sanitize)` reduce `internal::MAX_OBJECTS` + #[test] + fn incremental() { + #[cfg(miri)] + const COUNT: usize = 500; + #[cfg(not(miri))] + const COUNT: usize = 100_000; + static DESTROYS: AtomicUsize = AtomicUsize::new(0); + + let collector = Collector::new(); + let handle = collector.register(); + + unsafe { + let guard = &handle.pin(); + for _ in 0..COUNT { + let a = Owned::new(7i32).into_shared(guard); + guard.defer_unchecked(move || { + drop(a.into_owned()); + DESTROYS.fetch_add(1, Ordering::Relaxed); + }); + } + guard.flush(); + } + + let mut last = 0; + + while last < COUNT { + let curr = DESTROYS.load(Ordering::Relaxed); + assert!(curr - last <= 1024); + last = curr; + + let guard = &handle.pin(); + collector.global.collect(guard); + } + assert!(DESTROYS.load(Ordering::Relaxed) == COUNT); + } + + #[test] + fn buffering() { + const COUNT: usize = 10; + #[cfg(miri)] + const N: usize = 500; + #[cfg(not(miri))] + const N: usize = 100_000; + static DESTROYS: AtomicUsize = AtomicUsize::new(0); + + let collector = Collector::new(); + let handle = collector.register(); + + unsafe { + let guard = &handle.pin(); + for _ in 0..COUNT { + let a = Owned::new(7i32).into_shared(guard); + guard.defer_unchecked(move || { + drop(a.into_owned()); + DESTROYS.fetch_add(1, Ordering::Relaxed); + }); + } + } + + for _ in 0..N { + collector.global.collect(&handle.pin()); + } + assert!(DESTROYS.load(Ordering::Relaxed) < COUNT); + + handle.pin().flush(); + + while DESTROYS.load(Ordering::Relaxed) < COUNT { + let guard = &handle.pin(); + collector.global.collect(guard); + } + assert_eq!(DESTROYS.load(Ordering::Relaxed), COUNT); + } + + #[test] + fn count_drops() { + #[cfg(miri)] + const COUNT: usize = 500; + #[cfg(not(miri))] + const COUNT: usize = 100_000; + static DROPS: AtomicUsize = AtomicUsize::new(0); + + struct Elem(#[allow(dead_code)] i32); + + impl Drop for Elem { + fn drop(&mut self) { + DROPS.fetch_add(1, Ordering::Relaxed); + } + } + + let collector = Collector::new(); + let handle = collector.register(); + + unsafe { + let guard = &handle.pin(); + + for _ in 0..COUNT { + let a = Owned::new(Elem(7i32)).into_shared(guard); + guard.defer_destroy(a); + } + guard.flush(); + } + + while DROPS.load(Ordering::Relaxed) < COUNT { + let guard = &handle.pin(); + collector.global.collect(guard); + } + assert_eq!(DROPS.load(Ordering::Relaxed), COUNT); + } + + #[test] + fn count_destroy() { + #[cfg(miri)] + const COUNT: usize = 500; + #[cfg(not(miri))] + const COUNT: usize = 100_000; + static DESTROYS: AtomicUsize = AtomicUsize::new(0); + + let collector = Collector::new(); + let handle = collector.register(); + + unsafe { + let guard = &handle.pin(); + + for _ in 0..COUNT { + let a = Owned::new(7i32).into_shared(guard); + guard.defer_unchecked(move || { + drop(a.into_owned()); + DESTROYS.fetch_add(1, Ordering::Relaxed); + }); + } + guard.flush(); + } + + while DESTROYS.load(Ordering::Relaxed) < COUNT { + let guard = &handle.pin(); + collector.global.collect(guard); + } + assert_eq!(DESTROYS.load(Ordering::Relaxed), COUNT); + } + + #[test] + fn drop_array() { + const COUNT: usize = 700; + static DROPS: AtomicUsize = AtomicUsize::new(0); + + struct Elem(#[allow(dead_code)] i32); + + impl Drop for Elem { + fn drop(&mut self) { + DROPS.fetch_add(1, Ordering::Relaxed); + } + } + + let collector = Collector::new(); + let handle = collector.register(); + + let mut guard = handle.pin(); + + let mut v = Vec::with_capacity(COUNT); + for i in 0..COUNT { + v.push(Elem(i as i32)); + } + + { + let a = Owned::new(v).into_shared(&guard); + unsafe { + guard.defer_destroy(a); + } + guard.flush(); + } + + while DROPS.load(Ordering::Relaxed) < COUNT { + guard.repin(); + collector.global.collect(&guard); + } + assert_eq!(DROPS.load(Ordering::Relaxed), COUNT); + } + + #[test] + fn destroy_array() { + #[cfg(miri)] + const COUNT: usize = 500; + #[cfg(not(miri))] + const COUNT: usize = 100_000; + static DESTROYS: AtomicUsize = AtomicUsize::new(0); + + let collector = Collector::new(); + let handle = collector.register(); + + unsafe { + let guard = &handle.pin(); + + let mut v = Vec::with_capacity(COUNT); + for i in 0..COUNT { + v.push(i as i32); + } + + let len = v.len(); + let cap = v.capacity(); + let ptr = ManuallyDrop::new(v).as_mut_ptr(); + guard.defer_unchecked(move || { + drop(Vec::from_raw_parts(ptr, len, cap)); + DESTROYS.fetch_add(len, Ordering::Relaxed); + }); + guard.flush(); + } + + while DESTROYS.load(Ordering::Relaxed) < COUNT { + let guard = &handle.pin(); + collector.global.collect(guard); + } + assert_eq!(DESTROYS.load(Ordering::Relaxed), COUNT); + } + + #[test] + fn stress() { + const THREADS: usize = 8; + #[cfg(miri)] + const COUNT: usize = 500; + #[cfg(not(miri))] + const COUNT: usize = 100_000; + static DROPS: AtomicUsize = AtomicUsize::new(0); + + struct Elem(#[allow(dead_code)] i32); + + impl Drop for Elem { + fn drop(&mut self) { + DROPS.fetch_add(1, Ordering::Relaxed); + } + } + + let collector = Collector::new(); + + thread::scope(|scope| { + for _ in 0..THREADS { + scope.spawn(|_| { + let handle = collector.register(); + for _ in 0..COUNT { + let guard = &handle.pin(); + unsafe { + let a = Owned::new(Elem(7i32)).into_shared(guard); + guard.defer_destroy(a); + } + } + }); + } + }) + .unwrap(); + + let handle = collector.register(); + while DROPS.load(Ordering::Relaxed) < COUNT * THREADS { + let guard = &handle.pin(); + collector.global.collect(guard); + } + assert_eq!(DROPS.load(Ordering::Relaxed), COUNT * THREADS); + } +} diff --git a/anneal/v2/vendor/crossbeam-epoch/src/default.rs b/anneal/v2/vendor/crossbeam-epoch/src/default.rs new file mode 100644 index 0000000000..a9790a373a --- /dev/null +++ b/anneal/v2/vendor/crossbeam-epoch/src/default.rs @@ -0,0 +1,93 @@ +//! The default garbage collector. +//! +//! For each thread, a participant is lazily initialized on its first use, when the current thread +//! is registered in the default collector. If initialized, the thread's participant will get +//! destructed on thread exit, which in turn unregisters the thread. + +use crate::collector::{Collector, LocalHandle}; +use crate::guard::Guard; +use crate::primitive::thread_local; +#[cfg(not(crossbeam_loom))] +use crate::sync::once_lock::OnceLock; + +fn collector() -> &'static Collector { + #[cfg(not(crossbeam_loom))] + { + /// The global data for the default garbage collector. + static COLLECTOR: OnceLock = OnceLock::new(); + COLLECTOR.get_or_init(Collector::new) + } + // FIXME: loom does not currently provide the equivalent of Lazy: + // https://github.com/tokio-rs/loom/issues/263 + #[cfg(crossbeam_loom)] + { + loom::lazy_static! { + /// The global data for the default garbage collector. + static ref COLLECTOR: Collector = Collector::new(); + } + &COLLECTOR + } +} + +thread_local! { + /// The per-thread participant for the default garbage collector. + static HANDLE: LocalHandle = collector().register(); +} + +/// Pins the current thread. +#[inline] +pub fn pin() -> Guard { + with_handle(|handle| handle.pin()) +} + +/// Returns `true` if the current thread is pinned. +#[inline] +pub fn is_pinned() -> bool { + with_handle(|handle| handle.is_pinned()) +} + +/// Returns the default global collector. +pub fn default_collector() -> &'static Collector { + collector() +} + +#[inline] +fn with_handle(mut f: F) -> R +where + F: FnMut(&LocalHandle) -> R, +{ + HANDLE + .try_with(|h| f(h)) + .unwrap_or_else(|_| f(&collector().register())) +} + +#[cfg(all(test, not(crossbeam_loom)))] +mod tests { + use crossbeam_utils::thread; + + #[test] + fn pin_while_exiting() { + struct Foo; + + impl Drop for Foo { + fn drop(&mut self) { + // Pin after `HANDLE` has been dropped. This must not panic. + super::pin(); + } + } + + thread_local! { + static FOO: Foo = const { Foo }; + } + + thread::scope(|scope| { + scope.spawn(|_| { + // Initialize `FOO` and then `HANDLE`. + FOO.with(|_| ()); + super::pin(); + // At thread exit, `HANDLE` gets dropped first and `FOO` second. + }); + }) + .unwrap(); + } +} diff --git a/anneal/v2/vendor/crossbeam-epoch/src/deferred.rs b/anneal/v2/vendor/crossbeam-epoch/src/deferred.rs new file mode 100644 index 0000000000..041955f52b --- /dev/null +++ b/anneal/v2/vendor/crossbeam-epoch/src/deferred.rs @@ -0,0 +1,146 @@ +use alloc::boxed::Box; +use core::fmt; +use core::marker::PhantomData; +use core::mem::{self, MaybeUninit}; +use core::ptr; + +/// Number of words a piece of `Data` can hold. +/// +/// Three words should be enough for the majority of cases. For example, you can fit inside it the +/// function pointer together with a fat pointer representing an object that needs to be destroyed. +const DATA_WORDS: usize = 3; + +/// Some space to keep a `FnOnce()` object on the stack. +type Data = [usize; DATA_WORDS]; + +/// A `FnOnce()` that is stored inline if small, or otherwise boxed on the heap. +/// +/// This is a handy way of keeping an unsized `FnOnce()` within a sized structure. +pub(crate) struct Deferred { + call: unsafe fn(*mut u8), + data: MaybeUninit, + _marker: PhantomData<*mut ()>, // !Send + !Sync +} + +impl fmt::Debug for Deferred { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { + f.pad("Deferred { .. }") + } +} + +impl Deferred { + pub(crate) const NO_OP: Self = { + fn no_op_call(_raw: *mut u8) {} + Self { + call: no_op_call, + data: MaybeUninit::uninit(), + _marker: PhantomData, + } + }; + + /// Constructs a new `Deferred` from a `FnOnce()`. + pub(crate) fn new(f: F) -> Self { + let size = mem::size_of::(); + let align = mem::align_of::(); + + unsafe { + if size <= mem::size_of::() && align <= mem::align_of::() { + let mut data = MaybeUninit::::uninit(); + ptr::write(data.as_mut_ptr().cast::(), f); + + unsafe fn call(raw: *mut u8) { + let f: F = ptr::read(raw.cast::()); + f(); + } + + Deferred { + call: call::, + data, + _marker: PhantomData, + } + } else { + let b: Box = Box::new(f); + let mut data = MaybeUninit::::uninit(); + ptr::write(data.as_mut_ptr().cast::>(), b); + + unsafe fn call(raw: *mut u8) { + // It's safe to cast `raw` from `*mut u8` to `*mut Box`, because `raw` is + // originally derived from `*mut Box`. + let b: Box = ptr::read(raw.cast::>()); + (*b)(); + } + + Deferred { + call: call::, + data, + _marker: PhantomData, + } + } + } + } + + /// Calls the function. + #[inline] + pub(crate) fn call(mut self) { + let call = self.call; + unsafe { call(self.data.as_mut_ptr().cast::()) }; + } +} + +#[cfg(all(test, not(crossbeam_loom)))] +mod tests { + use super::Deferred; + use std::cell::Cell; + use std::convert::identity; + + #[test] + fn on_stack() { + let fired = &Cell::new(false); + let a = [0usize; 1]; + + let d = Deferred::new(move || { + let _ = identity(a); + fired.set(true); + }); + + assert!(!fired.get()); + d.call(); + assert!(fired.get()); + } + + #[test] + fn on_heap() { + let fired = &Cell::new(false); + let a = [0usize; 10]; + + let d = Deferred::new(move || { + let _ = identity(a); + fired.set(true); + }); + + assert!(!fired.get()); + d.call(); + assert!(fired.get()); + } + + #[test] + fn string() { + let a = "hello".to_string(); + let d = Deferred::new(move || assert_eq!(a, "hello")); + d.call(); + } + + #[test] + fn boxed_slice_i32() { + let a: Box<[i32]> = vec![2, 3, 5, 7].into_boxed_slice(); + let d = Deferred::new(move || assert_eq!(*a, [2, 3, 5, 7])); + d.call(); + } + + #[test] + fn long_slice_usize() { + let a: [usize; 5] = [2, 3, 5, 7, 11]; + let d = Deferred::new(move || assert_eq!(a, [2, 3, 5, 7, 11])); + d.call(); + } +} diff --git a/anneal/v2/vendor/crossbeam-epoch/src/epoch.rs b/anneal/v2/vendor/crossbeam-epoch/src/epoch.rs new file mode 100644 index 0000000000..18d7418a10 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-epoch/src/epoch.rs @@ -0,0 +1,132 @@ +//! The global epoch +//! +//! The last bit in this number is unused and is always zero. Every so often the global epoch is +//! incremented, i.e. we say it "advances". A pinned participant may advance the global epoch only +//! if all currently pinned participants have been pinned in the current epoch. +//! +//! If an object became garbage in some epoch, then we can be sure that after two advancements no +//! participant will hold a reference to it. That is the crux of safe memory reclamation. + +use crate::primitive::sync::atomic::{AtomicUsize, Ordering}; + +/// An epoch that can be marked as pinned or unpinned. +/// +/// Internally, the epoch is represented as an integer that wraps around at some unspecified point +/// and a flag that represents whether it is pinned or unpinned. +#[derive(Copy, Clone, Default, Debug, Eq, PartialEq)] +pub(crate) struct Epoch { + /// The least significant bit is set if pinned. The rest of the bits hold the epoch. + data: usize, +} + +impl Epoch { + /// Returns the starting epoch in unpinned state. + #[inline] + pub(crate) fn starting() -> Self { + Self::default() + } + + /// Returns the number of epochs `self` is ahead of `rhs`. + /// + /// Internally, epochs are represented as numbers in the range `(isize::MIN / 2) .. (isize::MAX + /// / 2)`, so the returned distance will be in the same interval. + pub(crate) fn wrapping_sub(self, rhs: Self) -> isize { + // The result is the same with `(self.data & !1).wrapping_sub(rhs.data & !1) as isize >> 1`, + // because the possible difference of LSB in `(self.data & !1).wrapping_sub(rhs.data & !1)` + // will be ignored in the shift operation. + self.data.wrapping_sub(rhs.data & !1) as isize >> 1 + } + + /// Returns `true` if the epoch is marked as pinned. + #[inline] + pub(crate) fn is_pinned(self) -> bool { + (self.data & 1) == 1 + } + + /// Returns the same epoch, but marked as pinned. + #[inline] + pub(crate) fn pinned(self) -> Epoch { + Epoch { + data: self.data | 1, + } + } + + /// Returns the same epoch, but marked as unpinned. + #[inline] + pub(crate) fn unpinned(self) -> Epoch { + Epoch { + data: self.data & !1, + } + } + + /// Returns the successor epoch. + /// + /// The returned epoch will be marked as pinned only if the previous one was as well. + #[inline] + pub(crate) fn successor(self) -> Epoch { + Epoch { + data: self.data.wrapping_add(2), + } + } +} + +/// An atomic value that holds an `Epoch`. +#[derive(Default, Debug)] +pub(crate) struct AtomicEpoch { + /// Since `Epoch` is just a wrapper around `usize`, an `AtomicEpoch` is similarly represented + /// using an `AtomicUsize`. + data: AtomicUsize, +} + +impl AtomicEpoch { + /// Creates a new atomic epoch. + #[inline] + pub(crate) fn new(epoch: Epoch) -> Self { + let data = AtomicUsize::new(epoch.data); + AtomicEpoch { data } + } + + /// Loads a value from the atomic epoch. + #[inline] + pub(crate) fn load(&self, ord: Ordering) -> Epoch { + Epoch { + data: self.data.load(ord), + } + } + + /// Stores a value into the atomic epoch. + #[inline] + pub(crate) fn store(&self, epoch: Epoch, ord: Ordering) { + self.data.store(epoch.data, ord); + } + + /// Stores a value into the atomic epoch if the current value is the same as `current`. + /// + /// The return value is a result indicating whether the new value was written and containing + /// the previous value. On success this value is guaranteed to be equal to `current`. + /// + /// This method takes two `Ordering` arguments to describe the memory + /// ordering of this operation. `success` describes the required ordering for the + /// read-modify-write operation that takes place if the comparison with `current` succeeds. + /// `failure` describes the required ordering for the load operation that takes place when + /// the comparison fails. Using `Acquire` as success ordering makes the store part + /// of this operation `Relaxed`, and using `Release` makes the successful load + /// `Relaxed`. The failure ordering can only be `SeqCst`, `Acquire` or `Relaxed` + /// and must be equivalent to or weaker than the success ordering. + #[inline] + pub(crate) fn compare_exchange( + &self, + current: Epoch, + new: Epoch, + success: Ordering, + failure: Ordering, + ) -> Result { + match self + .data + .compare_exchange(current.data, new.data, success, failure) + { + Ok(data) => Ok(Epoch { data }), + Err(data) => Err(Epoch { data }), + } + } +} diff --git a/anneal/v2/vendor/crossbeam-epoch/src/guard.rs b/anneal/v2/vendor/crossbeam-epoch/src/guard.rs new file mode 100644 index 0000000000..5fe33807c5 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-epoch/src/guard.rs @@ -0,0 +1,523 @@ +use core::fmt; +use core::mem; + +use crate::atomic::Shared; +use crate::collector::Collector; +use crate::deferred::Deferred; +use crate::internal::Local; + +/// A guard that keeps the current thread pinned. +/// +/// # Pinning +/// +/// The current thread is pinned by calling [`pin`], which returns a new guard: +/// +/// ``` +/// use crossbeam_epoch as epoch; +/// +/// // It is often convenient to prefix a call to `pin` with a `&` in order to create a reference. +/// // This is not really necessary, but makes passing references to the guard a bit easier. +/// let guard = &epoch::pin(); +/// ``` +/// +/// When a guard gets dropped, the current thread is automatically unpinned. +/// +/// # Pointers on the stack +/// +/// Having a guard allows us to create pointers on the stack to heap-allocated objects. +/// For example: +/// +/// ``` +/// use crossbeam_epoch::{self as epoch, Atomic}; +/// use std::sync::atomic::Ordering::SeqCst; +/// +/// // Create a heap-allocated number. +/// let a = Atomic::new(777); +/// +/// // Pin the current thread. +/// let guard = &epoch::pin(); +/// +/// // Load the heap-allocated object and create pointer `p` on the stack. +/// let p = a.load(SeqCst, guard); +/// +/// // Dereference the pointer and print the value: +/// if let Some(num) = unsafe { p.as_ref() } { +/// println!("The number is {}.", num); +/// } +/// # unsafe { drop(a.into_owned()); } // avoid leak +/// ``` +/// +/// # Multiple guards +/// +/// Pinning is reentrant and it is perfectly legal to create multiple guards. In that case, the +/// thread will actually be pinned only when the first guard is created and unpinned when the last +/// one is dropped: +/// +/// ``` +/// use crossbeam_epoch as epoch; +/// +/// let guard1 = epoch::pin(); +/// let guard2 = epoch::pin(); +/// assert!(epoch::is_pinned()); +/// drop(guard1); +/// assert!(epoch::is_pinned()); +/// drop(guard2); +/// assert!(!epoch::is_pinned()); +/// ``` +/// +/// [`pin`]: super::pin +pub struct Guard { + pub(crate) local: *const Local, +} + +impl Guard { + /// Stores a function so that it can be executed at some point after all currently pinned + /// threads get unpinned. + /// + /// This method first stores `f` into the thread-local (or handle-local) cache. If this cache + /// becomes full, some functions are moved into the global cache. At the same time, some + /// functions from both local and global caches may get executed in order to incrementally + /// clean up the caches as they fill up. + /// + /// There is no guarantee when exactly `f` will be executed. The only guarantee is that it + /// won't be executed until all currently pinned threads get unpinned. In theory, `f` might + /// never run, but the epoch-based garbage collection will make an effort to execute it + /// reasonably soon. + /// + /// If this method is called from an [`unprotected`] guard, the function will simply be + /// executed immediately. + pub fn defer(&self, f: F) + where + F: FnOnce() -> R, + F: Send + 'static, + { + unsafe { + self.defer_unchecked(f); + } + } + + /// Stores a function so that it can be executed at some point after all currently pinned + /// threads get unpinned. + /// + /// This method first stores `f` into the thread-local (or handle-local) cache. If this cache + /// becomes full, some functions are moved into the global cache. At the same time, some + /// functions from both local and global caches may get executed in order to incrementally + /// clean up the caches as they fill up. + /// + /// There is no guarantee when exactly `f` will be executed. The only guarantee is that it + /// won't be executed until all currently pinned threads get unpinned. In theory, `f` might + /// never run, but the epoch-based garbage collection will make an effort to execute it + /// reasonably soon. + /// + /// If this method is called from an [`unprotected`] guard, the function will simply be + /// executed immediately. + /// + /// # Safety + /// + /// The given function must not hold reference onto the stack. It is highly recommended that + /// the passed function is **always** marked with `move` in order to prevent accidental + /// borrows. + /// + /// ``` + /// use crossbeam_epoch as epoch; + /// + /// let guard = &epoch::pin(); + /// let message = "Hello!"; + /// unsafe { + /// // ALWAYS use `move` when sending a closure into `defer_unchecked`. + /// guard.defer_unchecked(move || { + /// println!("{}", message); + /// }); + /// } + /// ``` + /// + /// Apart from that, keep in mind that another thread may execute `f`, so anything accessed by + /// the closure must be `Send`. + /// + /// We intentionally didn't require `F: Send`, because Rust's type systems usually cannot prove + /// `F: Send` for typical use cases. For example, consider the following code snippet, which + /// exemplifies the typical use case of deferring the deallocation of a shared reference: + /// + /// ```ignore + /// let shared = Owned::new(7i32).into_shared(guard); + /// guard.defer_unchecked(move || shared.into_owned()); // `Shared` is not `Send`! + /// ``` + /// + /// While `Shared` is not `Send`, it's safe for another thread to call the deferred function, + /// because it's called only after the grace period and `shared` is no longer shared with other + /// threads. But we don't expect type systems to prove this. + /// + /// # Examples + /// + /// When a heap-allocated object in a data structure becomes unreachable, it has to be + /// deallocated. However, the current thread and other threads may be still holding references + /// on the stack to that same object. Therefore it cannot be deallocated before those references + /// get dropped. This method can defer deallocation until all those threads get unpinned and + /// consequently drop all their references on the stack. + /// + /// ``` + /// use crossbeam_epoch::{self as epoch, Atomic, Owned}; + /// use std::sync::atomic::Ordering::SeqCst; + /// + /// let a = Atomic::new("foo"); + /// + /// // Now suppose that `a` is shared among multiple threads and concurrently + /// // accessed and modified... + /// + /// // Pin the current thread. + /// let guard = &epoch::pin(); + /// + /// // Steal the object currently stored in `a` and swap it with another one. + /// let p = a.swap(Owned::new("bar").into_shared(guard), SeqCst, guard); + /// + /// if !p.is_null() { + /// // The object `p` is pointing to is now unreachable. + /// // Defer its deallocation until all currently pinned threads get unpinned. + /// unsafe { + /// // ALWAYS use `move` when sending a closure into `defer_unchecked`. + /// guard.defer_unchecked(move || { + /// println!("{} is now being deallocated.", p.deref()); + /// // Now we have unique access to the object pointed to by `p` and can turn it + /// // into an `Owned`. Dropping the `Owned` will deallocate the object. + /// drop(p.into_owned()); + /// }); + /// } + /// } + /// # unsafe { drop(a.into_owned()); } // avoid leak + /// ``` + pub unsafe fn defer_unchecked(&self, f: F) + where + F: FnOnce() -> R, + { + if let Some(local) = self.local.as_ref() { + local.defer(Deferred::new(move || drop(f())), self); + } else { + drop(f()); + } + } + + /// Stores a destructor for an object so that it can be deallocated and dropped at some point + /// after all currently pinned threads get unpinned. + /// + /// This method first stores the destructor into the thread-local (or handle-local) cache. If + /// this cache becomes full, some destructors are moved into the global cache. At the same + /// time, some destructors from both local and global caches may get executed in order to + /// incrementally clean up the caches as they fill up. + /// + /// There is no guarantee when exactly the destructor will be executed. The only guarantee is + /// that it won't be executed until all currently pinned threads get unpinned. In theory, the + /// destructor might never run, but the epoch-based garbage collection will make an effort to + /// execute it reasonably soon. + /// + /// If this method is called from an [`unprotected`] guard, the destructor will simply be + /// executed immediately. + /// + /// # Safety + /// + /// The object must not be reachable by other threads anymore, otherwise it might be still in + /// use when the destructor runs. + /// + /// Apart from that, keep in mind that another thread may execute the destructor, so the object + /// must be sendable to other threads. + /// + /// We intentionally didn't require `T: Send`, because Rust's type systems usually cannot prove + /// `T: Send` for typical use cases. For example, consider the following code snippet, which + /// exemplifies the typical use case of deferring the deallocation of a shared reference: + /// + /// ```ignore + /// let shared = Owned::new(7i32).into_shared(guard); + /// guard.defer_destroy(shared); // `Shared` is not `Send`! + /// ``` + /// + /// While `Shared` is not `Send`, it's safe for another thread to call the destructor, because + /// it's called only after the grace period and `shared` is no longer shared with other + /// threads. But we don't expect type systems to prove this. + /// + /// # Examples + /// + /// When a heap-allocated object in a data structure becomes unreachable, it has to be + /// deallocated. However, the current thread and other threads may be still holding references + /// on the stack to that same object. Therefore it cannot be deallocated before those references + /// get dropped. This method can defer deallocation until all those threads get unpinned and + /// consequently drop all their references on the stack. + /// + /// ``` + /// use crossbeam_epoch::{self as epoch, Atomic, Owned}; + /// use std::sync::atomic::Ordering::SeqCst; + /// + /// let a = Atomic::new("foo"); + /// + /// // Now suppose that `a` is shared among multiple threads and concurrently + /// // accessed and modified... + /// + /// // Pin the current thread. + /// let guard = &epoch::pin(); + /// + /// // Steal the object currently stored in `a` and swap it with another one. + /// let p = a.swap(Owned::new("bar").into_shared(guard), SeqCst, guard); + /// + /// if !p.is_null() { + /// // The object `p` is pointing to is now unreachable. + /// // Defer its deallocation until all currently pinned threads get unpinned. + /// unsafe { + /// guard.defer_destroy(p); + /// } + /// } + /// # unsafe { drop(a.into_owned()); } // avoid leak + /// ``` + pub unsafe fn defer_destroy(&self, ptr: Shared<'_, T>) { + self.defer_unchecked(move || ptr.into_owned()); + } + + /// Clears up the thread-local cache of deferred functions by executing them or moving into the + /// global cache. + /// + /// Call this method after deferring execution of a function if you want to get it executed as + /// soon as possible. Flushing will make sure it is residing in in the global cache, so that + /// any thread has a chance of taking the function and executing it. + /// + /// If this method is called from an [`unprotected`] guard, it is a no-op (nothing happens). + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch as epoch; + /// + /// let guard = &epoch::pin(); + /// guard.defer(move || { + /// println!("This better be printed as soon as possible!"); + /// }); + /// guard.flush(); + /// ``` + pub fn flush(&self) { + if let Some(local) = unsafe { self.local.as_ref() } { + local.flush(self); + } + } + + /// Unpins and then immediately re-pins the thread. + /// + /// This method is useful when you don't want delay the advancement of the global epoch by + /// holding an old epoch. For safety, you should not maintain any guard-based reference across + /// the call (the latter is enforced by `&mut self`). The thread will only be repinned if this + /// is the only active guard for the current thread. + /// + /// If this method is called from an [`unprotected`] guard, then the call will be just no-op. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::{self as epoch, Atomic}; + /// use std::sync::atomic::Ordering::SeqCst; + /// + /// let a = Atomic::new(777); + /// let mut guard = epoch::pin(); + /// { + /// let p = a.load(SeqCst, &guard); + /// assert_eq!(unsafe { p.as_ref() }, Some(&777)); + /// } + /// guard.repin(); + /// { + /// let p = a.load(SeqCst, &guard); + /// assert_eq!(unsafe { p.as_ref() }, Some(&777)); + /// } + /// # unsafe { drop(a.into_owned()); } // avoid leak + /// ``` + pub fn repin(&mut self) { + if let Some(local) = unsafe { self.local.as_ref() } { + local.repin(); + } + } + + /// Temporarily unpins the thread, executes the given function and then re-pins the thread. + /// + /// This method is useful when you need to perform a long-running operation (e.g. sleeping) + /// and don't need to maintain any guard-based reference across the call (the latter is enforced + /// by `&mut self`). The thread will only be unpinned if this is the only active guard for the + /// current thread. + /// + /// If this method is called from an [`unprotected`] guard, then the passed function is called + /// directly without unpinning the thread. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch::{self as epoch, Atomic}; + /// use std::sync::atomic::Ordering::SeqCst; + /// use std::thread; + /// use std::time::Duration; + /// + /// let a = Atomic::new(777); + /// let mut guard = epoch::pin(); + /// { + /// let p = a.load(SeqCst, &guard); + /// assert_eq!(unsafe { p.as_ref() }, Some(&777)); + /// } + /// guard.repin_after(|| thread::sleep(Duration::from_millis(50))); + /// { + /// let p = a.load(SeqCst, &guard); + /// assert_eq!(unsafe { p.as_ref() }, Some(&777)); + /// } + /// # unsafe { drop(a.into_owned()); } // avoid leak + /// ``` + pub fn repin_after(&mut self, f: F) -> R + where + F: FnOnce() -> R, + { + // Ensure the Guard is re-pinned even if the function panics + struct ScopeGuard(*const Local); + impl Drop for ScopeGuard { + fn drop(&mut self) { + if let Some(local) = unsafe { self.0.as_ref() } { + mem::forget(local.pin()); + local.release_handle(); + } + } + } + + if let Some(local) = unsafe { self.local.as_ref() } { + // We need to acquire a handle here to ensure the Local doesn't + // disappear from under us. + local.acquire_handle(); + local.unpin(); + } + + let _guard = ScopeGuard(self.local); + + f() + } + + /// Returns the `Collector` associated with this guard. + /// + /// This method is useful when you need to ensure that all guards used with + /// a data structure come from the same collector. + /// + /// If this method is called from an [`unprotected`] guard, then `None` is returned. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_epoch as epoch; + /// + /// let guard1 = epoch::pin(); + /// let guard2 = epoch::pin(); + /// assert!(guard1.collector() == guard2.collector()); + /// ``` + pub fn collector(&self) -> Option<&Collector> { + unsafe { self.local.as_ref().map(|local| local.collector()) } + } +} + +impl Drop for Guard { + #[inline] + fn drop(&mut self) { + if let Some(local) = unsafe { self.local.as_ref() } { + local.unpin(); + } + } +} + +impl fmt::Debug for Guard { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad("Guard { .. }") + } +} + +/// Returns a reference to a dummy guard that allows unprotected access to [`Atomic`]s. +/// +/// This guard should be used in special occasions only. Note that it doesn't actually keep any +/// thread pinned - it's just a fake guard that allows loading from [`Atomic`]s unsafely. +/// +/// Note that calling [`defer`] with a dummy guard will not defer the function - it will just +/// execute the function immediately. +/// +/// If necessary, it's possible to create more dummy guards by cloning: `unprotected().clone()`. +/// +/// # Safety +/// +/// Loading and dereferencing data from an [`Atomic`] using this guard is safe only if the +/// [`Atomic`] is not being concurrently modified by other threads. +/// +/// # Examples +/// +/// ``` +/// use crossbeam_epoch::{self as epoch, Atomic}; +/// use std::sync::atomic::Ordering::Relaxed; +/// +/// let a = Atomic::new(7); +/// +/// unsafe { +/// // Load `a` without pinning the current thread. +/// a.load(Relaxed, epoch::unprotected()); +/// +/// // It's possible to create more dummy guards. +/// let dummy = epoch::unprotected(); +/// +/// dummy.defer(move || { +/// println!("This gets executed immediately."); +/// }); +/// +/// // Dropping `dummy` doesn't affect the current thread - it's just a noop. +/// } +/// # unsafe { drop(a.into_owned()); } // avoid leak +/// ``` +/// +/// The most common use of this function is when constructing or destructing a data structure. +/// +/// For example, we can use a dummy guard in the destructor of a Treiber stack because at that +/// point no other thread could concurrently modify the [`Atomic`]s we are accessing. +/// +/// If we were to actually pin the current thread during destruction, that would just unnecessarily +/// delay garbage collection and incur some performance cost, so in cases like these `unprotected` +/// is very helpful. +/// +/// ``` +/// use crossbeam_epoch::{self as epoch, Atomic}; +/// use std::mem::ManuallyDrop; +/// use std::sync::atomic::Ordering::Relaxed; +/// +/// struct Stack { +/// head: Atomic>, +/// } +/// +/// struct Node { +/// data: ManuallyDrop, +/// next: Atomic>, +/// } +/// +/// impl Drop for Stack { +/// fn drop(&mut self) { +/// unsafe { +/// // Unprotected load. +/// let mut node = self.head.load(Relaxed, epoch::unprotected()); +/// +/// while let Some(n) = node.as_ref() { +/// // Unprotected load. +/// let next = n.next.load(Relaxed, epoch::unprotected()); +/// +/// // Take ownership of the node, then drop its data and deallocate it. +/// let mut o = node.into_owned(); +/// ManuallyDrop::drop(&mut o.data); +/// drop(o); +/// +/// node = next; +/// } +/// } +/// } +/// } +/// ``` +/// +/// [`Atomic`]: super::Atomic +/// [`defer`]: Guard::defer +#[inline] +pub unsafe fn unprotected() -> &'static Guard { + // An unprotected guard is just a `Guard` with its field `local` set to null. + // We make a newtype over `Guard` because `Guard` isn't `Sync`, so can't be directly stored in + // a `static` + struct GuardWrapper(Guard); + unsafe impl Sync for GuardWrapper {} + static UNPROTECTED: GuardWrapper = GuardWrapper(Guard { + local: core::ptr::null(), + }); + &UNPROTECTED.0 +} diff --git a/anneal/v2/vendor/crossbeam-epoch/src/internal.rs b/anneal/v2/vendor/crossbeam-epoch/src/internal.rs new file mode 100644 index 0000000000..b2e9e71bcb --- /dev/null +++ b/anneal/v2/vendor/crossbeam-epoch/src/internal.rs @@ -0,0 +1,600 @@ +//! The global data and participant for garbage collection. +//! +//! # Registration +//! +//! In order to track all participants in one place, we need some form of participant +//! registration. When a participant is created, it is registered to a global lock-free +//! singly-linked list of registries; and when a participant is leaving, it is unregistered from the +//! list. +//! +//! # Pinning +//! +//! Every participant contains an integer that tells whether the participant is pinned and if so, +//! what was the global epoch at the time it was pinned. Participants also hold a pin counter that +//! aids in periodic global epoch advancement. +//! +//! When a participant is pinned, a `Guard` is returned as a witness that the participant is pinned. +//! Guards are necessary for performing atomic operations, and for freeing/dropping locations. +//! +//! # Thread-local bag +//! +//! Objects that get unlinked from concurrent data structures must be stashed away until the global +//! epoch sufficiently advances so that they become safe for destruction. Pointers to such objects +//! are pushed into a thread-local bag, and when it becomes full, the bag is marked with the current +//! global epoch and pushed into the global queue of bags. We store objects in thread-local storages +//! for amortizing the synchronization cost of pushing the garbages to a global queue. +//! +//! # Global queue +//! +//! Whenever a bag is pushed into a queue, the objects in some bags in the queue are collected and +//! destroyed along the way. This design reduces contention on data structures. The global queue +//! cannot be explicitly accessed: the only way to interact with it is by calling functions +//! `defer()` that adds an object to the thread-local bag, or `collect()` that manually triggers +//! garbage collection. +//! +//! Ideally each instance of concurrent data structure may have its own queue that gets fully +//! destroyed as soon as the data structure gets dropped. + +use crate::primitive::cell::UnsafeCell; +use crate::primitive::sync::atomic::{self, Ordering}; +use core::cell::Cell; +use core::mem::{self, ManuallyDrop}; +use core::num::Wrapping; +use core::{fmt, ptr}; + +use crossbeam_utils::CachePadded; + +use crate::atomic::{Owned, Shared}; +use crate::collector::{Collector, LocalHandle}; +use crate::deferred::Deferred; +use crate::epoch::{AtomicEpoch, Epoch}; +use crate::guard::{unprotected, Guard}; +use crate::sync::list::{Entry, IsElement, IterError, List}; +use crate::sync::queue::Queue; + +/// Maximum number of objects a bag can contain. +#[cfg(not(any(crossbeam_sanitize, miri)))] +const MAX_OBJECTS: usize = 64; +// Makes it more likely to trigger any potential data races. +#[cfg(any(crossbeam_sanitize, miri))] +const MAX_OBJECTS: usize = 4; + +/// A bag of deferred functions. +pub(crate) struct Bag { + /// Stashed objects. + deferreds: [Deferred; MAX_OBJECTS], + len: usize, +} + +/// `Bag::try_push()` requires that it is safe for another thread to execute the given functions. +unsafe impl Send for Bag {} + +impl Bag { + /// Returns a new, empty bag. + pub(crate) fn new() -> Self { + Self::default() + } + + /// Returns `true` if the bag is empty. + pub(crate) fn is_empty(&self) -> bool { + self.len == 0 + } + + /// Attempts to insert a deferred function into the bag. + /// + /// Returns `Ok(())` if successful, and `Err(deferred)` for the given `deferred` if the bag is + /// full. + /// + /// # Safety + /// + /// It should be safe for another thread to execute the given function. + pub(crate) unsafe fn try_push(&mut self, deferred: Deferred) -> Result<(), Deferred> { + if self.len < MAX_OBJECTS { + self.deferreds[self.len] = deferred; + self.len += 1; + Ok(()) + } else { + Err(deferred) + } + } + + /// Seals the bag with the given epoch. + fn seal(self, epoch: Epoch) -> SealedBag { + SealedBag { epoch, _bag: self } + } +} + +impl Default for Bag { + fn default() -> Self { + Bag { + len: 0, + deferreds: [Deferred::NO_OP; MAX_OBJECTS], + } + } +} + +impl Drop for Bag { + fn drop(&mut self) { + // Call all deferred functions. + for deferred in &mut self.deferreds[..self.len] { + let no_op = Deferred::NO_OP; + let owned_deferred = mem::replace(deferred, no_op); + owned_deferred.call(); + } + } +} + +// can't #[derive(Debug)] because Debug is not implemented for arrays 64 items long +impl fmt::Debug for Bag { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Bag") + .field("deferreds", &&self.deferreds[..self.len]) + .finish() + } +} + +/// A pair of an epoch and a bag. +#[derive(Default, Debug)] +struct SealedBag { + epoch: Epoch, + _bag: Bag, +} + +/// It is safe to share `SealedBag` because `is_expired` only inspects the epoch. +unsafe impl Sync for SealedBag {} + +impl SealedBag { + /// Checks if it is safe to drop the bag w.r.t. the given global epoch. + fn is_expired(&self, global_epoch: Epoch) -> bool { + // A pinned participant can witness at most one epoch advancement. Therefore, any bag that + // is within one epoch of the current one cannot be destroyed yet. + global_epoch.wrapping_sub(self.epoch) >= 2 + } +} + +/// The global data for a garbage collector. +pub(crate) struct Global { + /// The intrusive linked list of `Local`s. + locals: List, + + /// The global queue of bags of deferred functions. + queue: Queue, + + /// The global epoch. + pub(crate) epoch: CachePadded, +} + +impl Global { + /// Number of bags to destroy. + const COLLECT_STEPS: usize = 8; + + /// Creates a new global data for garbage collection. + #[inline] + pub(crate) fn new() -> Self { + Self { + locals: List::new(), + queue: Queue::new(), + epoch: CachePadded::new(AtomicEpoch::new(Epoch::starting())), + } + } + + /// Pushes the bag into the global queue and replaces the bag with a new empty bag. + pub(crate) fn push_bag(&self, bag: &mut Bag, guard: &Guard) { + let bag = mem::replace(bag, Bag::new()); + + atomic::fence(Ordering::SeqCst); + + let epoch = self.epoch.load(Ordering::Relaxed); + self.queue.push(bag.seal(epoch), guard); + } + + /// Collects several bags from the global queue and executes deferred functions in them. + /// + /// Note: This may itself produce garbage and in turn allocate new bags. + /// + /// `pin()` rarely calls `collect()`, so we want the compiler to place that call on a cold + /// path. In other words, we want the compiler to optimize branching for the case when + /// `collect()` is not called. + #[cold] + pub(crate) fn collect(&self, guard: &Guard) { + let global_epoch = self.try_advance(guard); + + let steps = if cfg!(crossbeam_sanitize) { + usize::max_value() + } else { + Self::COLLECT_STEPS + }; + + for _ in 0..steps { + match self.queue.try_pop_if( + &|sealed_bag: &SealedBag| sealed_bag.is_expired(global_epoch), + guard, + ) { + None => break, + Some(sealed_bag) => drop(sealed_bag), + } + } + } + + /// Attempts to advance the global epoch. + /// + /// The global epoch can advance only if all currently pinned participants have been pinned in + /// the current epoch. + /// + /// Returns the current global epoch. + /// + /// `try_advance()` is annotated `#[cold]` because it is rarely called. + #[cold] + pub(crate) fn try_advance(&self, guard: &Guard) -> Epoch { + let global_epoch = self.epoch.load(Ordering::Relaxed); + atomic::fence(Ordering::SeqCst); + + // TODO(stjepang): `Local`s are stored in a linked list because linked lists are fairly + // easy to implement in a lock-free manner. However, traversal can be slow due to cache + // misses and data dependencies. We should experiment with other data structures as well. + for local in self.locals.iter(guard) { + match local { + Err(IterError::Stalled) => { + // A concurrent thread stalled this iteration. That thread might also try to + // advance the epoch, in which case we leave the job to it. Otherwise, the + // epoch will not be advanced. + return global_epoch; + } + Ok(local) => { + let local_epoch = local.epoch.load(Ordering::Relaxed); + + // If the participant was pinned in a different epoch, we cannot advance the + // global epoch just yet. + if local_epoch.is_pinned() && local_epoch.unpinned() != global_epoch { + return global_epoch; + } + } + } + } + atomic::fence(Ordering::Acquire); + + // All pinned participants were pinned in the current global epoch. + // Now let's advance the global epoch... + // + // Note that if another thread already advanced it before us, this store will simply + // overwrite the global epoch with the same value. This is true because `try_advance` was + // called from a thread that was pinned in `global_epoch`, and the global epoch cannot be + // advanced two steps ahead of it. + let new_epoch = global_epoch.successor(); + self.epoch.store(new_epoch, Ordering::Release); + new_epoch + } +} + +/// Participant for garbage collection. +#[repr(C)] // Note: `entry` must be the first field +pub(crate) struct Local { + /// A node in the intrusive linked list of `Local`s. + entry: Entry, + + /// A reference to the global data. + /// + /// When all guards and handles get dropped, this reference is destroyed. + collector: UnsafeCell>, + + /// The local bag of deferred functions. + pub(crate) bag: UnsafeCell, + + /// The number of guards keeping this participant pinned. + guard_count: Cell, + + /// The number of active handles. + handle_count: Cell, + + /// Total number of pinnings performed. + /// + /// This is just an auxiliary counter that sometimes kicks off collection. + pin_count: Cell>, + + /// The local epoch. + epoch: CachePadded, +} + +// Make sure `Local` is less than or equal to 2048 bytes. +// https://github.com/crossbeam-rs/crossbeam/issues/551 +#[cfg(not(any(crossbeam_sanitize, miri)))] // `crossbeam_sanitize` and `miri` reduce the size of `Local` +#[test] +fn local_size() { + // TODO: https://github.com/crossbeam-rs/crossbeam/issues/869 + // assert!( + // core::mem::size_of::() <= 2048, + // "An allocation of `Local` should be <= 2048 bytes." + // ); +} + +impl Local { + /// Number of pinnings after which a participant will execute some deferred functions from the + /// global queue. + const PINNINGS_BETWEEN_COLLECT: usize = 128; + + /// Registers a new `Local` in the provided `Global`. + pub(crate) fn register(collector: &Collector) -> LocalHandle { + unsafe { + // Since we dereference no pointers in this block, it is safe to use `unprotected`. + + let local = Owned::new(Local { + entry: Entry::default(), + collector: UnsafeCell::new(ManuallyDrop::new(collector.clone())), + bag: UnsafeCell::new(Bag::new()), + guard_count: Cell::new(0), + handle_count: Cell::new(1), + pin_count: Cell::new(Wrapping(0)), + epoch: CachePadded::new(AtomicEpoch::new(Epoch::starting())), + }) + .into_shared(unprotected()); + collector.global.locals.insert(local, unprotected()); + LocalHandle { + local: local.as_raw(), + } + } + } + + /// Returns a reference to the `Global` in which this `Local` resides. + #[inline] + pub(crate) fn global(&self) -> &Global { + &self.collector().global + } + + /// Returns a reference to the `Collector` in which this `Local` resides. + #[inline] + pub(crate) fn collector(&self) -> &Collector { + self.collector.with(|c| unsafe { &**c }) + } + + /// Returns `true` if the current participant is pinned. + #[inline] + pub(crate) fn is_pinned(&self) -> bool { + self.guard_count.get() > 0 + } + + /// Adds `deferred` to the thread-local bag. + /// + /// # Safety + /// + /// It should be safe for another thread to execute the given function. + pub(crate) unsafe fn defer(&self, mut deferred: Deferred, guard: &Guard) { + let bag = self.bag.with_mut(|b| &mut *b); + + while let Err(d) = bag.try_push(deferred) { + self.global().push_bag(bag, guard); + deferred = d; + } + } + + pub(crate) fn flush(&self, guard: &Guard) { + let bag = self.bag.with_mut(|b| unsafe { &mut *b }); + + if !bag.is_empty() { + self.global().push_bag(bag, guard); + } + + self.global().collect(guard); + } + + /// Pins the `Local`. + #[inline] + pub(crate) fn pin(&self) -> Guard { + let guard = Guard { local: self }; + + let guard_count = self.guard_count.get(); + self.guard_count.set(guard_count.checked_add(1).unwrap()); + + if guard_count == 0 { + let global_epoch = self.global().epoch.load(Ordering::Relaxed); + let new_epoch = global_epoch.pinned(); + + // Now we must store `new_epoch` into `self.epoch` and execute a `SeqCst` fence. + // The fence makes sure that any future loads from `Atomic`s will not happen before + // this store. + if cfg!(all( + any(target_arch = "x86", target_arch = "x86_64"), + not(miri) + )) { + // HACK(stjepang): On x86 architectures there are two different ways of executing + // a `SeqCst` fence. + // + // 1. `atomic::fence(SeqCst)`, which compiles into a `mfence` instruction. + // 2. `_.compare_exchange(_, _, SeqCst, SeqCst)`, which compiles into a `lock cmpxchg` + // instruction. + // + // Both instructions have the effect of a full barrier, but benchmarks have shown + // that the second one makes pinning faster in this particular case. It is not + // clear that this is permitted by the C++ memory model (SC fences work very + // differently from SC accesses), but experimental evidence suggests that this + // works fine. Using inline assembly would be a viable (and correct) alternative, + // but alas, that is not possible on stable Rust. + let current = Epoch::starting(); + let res = self.epoch.compare_exchange( + current, + new_epoch, + Ordering::SeqCst, + Ordering::SeqCst, + ); + debug_assert!(res.is_ok(), "participant was expected to be unpinned"); + // We add a compiler fence to make it less likely for LLVM to do something wrong + // here. Formally, this is not enough to get rid of data races; practically, + // it should go a long way. + atomic::compiler_fence(Ordering::SeqCst); + } else { + self.epoch.store(new_epoch, Ordering::Relaxed); + atomic::fence(Ordering::SeqCst); + } + + // Increment the pin counter. + let count = self.pin_count.get(); + self.pin_count.set(count + Wrapping(1)); + + // After every `PINNINGS_BETWEEN_COLLECT` try advancing the epoch and collecting + // some garbage. + if count.0 % Self::PINNINGS_BETWEEN_COLLECT == 0 { + self.global().collect(&guard); + } + } + + guard + } + + /// Unpins the `Local`. + #[inline] + pub(crate) fn unpin(&self) { + let guard_count = self.guard_count.get(); + self.guard_count.set(guard_count - 1); + + if guard_count == 1 { + self.epoch.store(Epoch::starting(), Ordering::Release); + + if self.handle_count.get() == 0 { + self.finalize(); + } + } + } + + /// Unpins and then pins the `Local`. + #[inline] + pub(crate) fn repin(&self) { + let guard_count = self.guard_count.get(); + + // Update the local epoch only if there's only one guard. + if guard_count == 1 { + let epoch = self.epoch.load(Ordering::Relaxed); + let global_epoch = self.global().epoch.load(Ordering::Relaxed).pinned(); + + // Update the local epoch only if the global epoch is greater than the local epoch. + if epoch != global_epoch { + // We store the new epoch with `Release` because we need to ensure any memory + // accesses from the previous epoch do not leak into the new one. + self.epoch.store(global_epoch, Ordering::Release); + + // However, we don't need a following `SeqCst` fence, because it is safe for memory + // accesses from the new epoch to be executed before updating the local epoch. At + // worse, other threads will see the new epoch late and delay GC slightly. + } + } + } + + /// Increments the handle count. + #[inline] + pub(crate) fn acquire_handle(&self) { + let handle_count = self.handle_count.get(); + debug_assert!(handle_count >= 1); + self.handle_count.set(handle_count + 1); + } + + /// Decrements the handle count. + #[inline] + pub(crate) fn release_handle(&self) { + let guard_count = self.guard_count.get(); + let handle_count = self.handle_count.get(); + debug_assert!(handle_count >= 1); + self.handle_count.set(handle_count - 1); + + if guard_count == 0 && handle_count == 1 { + self.finalize(); + } + } + + /// Removes the `Local` from the global linked list. + #[cold] + fn finalize(&self) { + debug_assert_eq!(self.guard_count.get(), 0); + debug_assert_eq!(self.handle_count.get(), 0); + + // Temporarily increment handle count. This is required so that the following call to `pin` + // doesn't call `finalize` again. + self.handle_count.set(1); + unsafe { + // Pin and move the local bag into the global queue. It's important that `push_bag` + // doesn't defer destruction on any new garbage. + let guard = &self.pin(); + self.global() + .push_bag(self.bag.with_mut(|b| &mut *b), guard); + } + // Revert the handle count back to zero. + self.handle_count.set(0); + + unsafe { + // Take the reference to the `Global` out of this `Local`. Since we're not protected + // by a guard at this time, it's crucial that the reference is read before marking the + // `Local` as deleted. + let collector: Collector = ptr::read(self.collector.with(|c| &*(*c))); + + // Mark this node in the linked list as deleted. + self.entry.delete(unprotected()); + + // Finally, drop the reference to the global. Note that this might be the last reference + // to the `Global`. If so, the global data will be destroyed and all deferred functions + // in its queue will be executed. + drop(collector); + } + } +} + +impl IsElement for Local { + fn entry_of(local: &Self) -> &Entry { + // SAFETY: `Local` is `repr(C)` and `entry` is the first field of it. + unsafe { + let entry_ptr = (local as *const Self).cast::(); + &*entry_ptr + } + } + + unsafe fn element_of(entry: &Entry) -> &Self { + // SAFETY: `Local` is `repr(C)` and `entry` is the first field of it. + let local_ptr = (entry as *const Entry).cast::(); + &*local_ptr + } + + unsafe fn finalize(entry: &Entry, guard: &Guard) { + guard.defer_destroy(Shared::from(Self::element_of(entry) as *const _)); + } +} + +#[cfg(all(test, not(crossbeam_loom)))] +mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + + use super::*; + + #[test] + fn check_defer() { + static FLAG: AtomicUsize = AtomicUsize::new(0); + fn set() { + FLAG.store(42, Ordering::Relaxed); + } + + let d = Deferred::new(set); + assert_eq!(FLAG.load(Ordering::Relaxed), 0); + d.call(); + assert_eq!(FLAG.load(Ordering::Relaxed), 42); + } + + #[test] + fn check_bag() { + static FLAG: AtomicUsize = AtomicUsize::new(0); + fn incr() { + FLAG.fetch_add(1, Ordering::Relaxed); + } + + let mut bag = Bag::new(); + assert!(bag.is_empty()); + + for _ in 0..MAX_OBJECTS { + assert!(unsafe { bag.try_push(Deferred::new(incr)).is_ok() }); + assert!(!bag.is_empty()); + assert_eq!(FLAG.load(Ordering::Relaxed), 0); + } + + let result = unsafe { bag.try_push(Deferred::new(incr)) }; + assert!(result.is_err()); + assert!(!bag.is_empty()); + assert_eq!(FLAG.load(Ordering::Relaxed), 0); + + drop(bag); + assert_eq!(FLAG.load(Ordering::Relaxed), MAX_OBJECTS); + } +} diff --git a/anneal/v2/vendor/crossbeam-epoch/src/lib.rs b/anneal/v2/vendor/crossbeam-epoch/src/lib.rs new file mode 100644 index 0000000000..fd4d74bedb --- /dev/null +++ b/anneal/v2/vendor/crossbeam-epoch/src/lib.rs @@ -0,0 +1,166 @@ +//! Epoch-based memory reclamation. +//! +//! An interesting problem concurrent collections deal with comes from the remove operation. +//! Suppose that a thread removes an element from a lock-free map, while another thread is reading +//! that same element at the same time. The first thread must wait until the second thread stops +//! reading the element. Only then it is safe to destruct it. +//! +//! Programming languages that come with garbage collectors solve this problem trivially. The +//! garbage collector will destruct the removed element when no thread can hold a reference to it +//! anymore. +//! +//! This crate implements a basic memory reclamation mechanism, which is based on epochs. When an +//! element gets removed from a concurrent collection, it is inserted into a pile of garbage and +//! marked with the current epoch. Every time a thread accesses a collection, it checks the current +//! epoch, attempts to increment it, and destructs some garbage that became so old that no thread +//! can be referencing it anymore. +//! +//! That is the general mechanism behind epoch-based memory reclamation, but the details are a bit +//! more complicated. Anyhow, memory reclamation is designed to be fully automatic and something +//! users of concurrent collections don't have to worry much about. +//! +//! # Pointers +//! +//! Concurrent collections are built using atomic pointers. This module provides [`Atomic`], which +//! is just a shared atomic pointer to a heap-allocated object. Loading an [`Atomic`] yields a +//! [`Shared`], which is an epoch-protected pointer through which the loaded object can be safely +//! read. +//! +//! # Pinning +//! +//! Before an [`Atomic`] can be loaded, a participant must be [`pin`]ned. By pinning a participant +//! we declare that any object that gets removed from now on must not be destructed just +//! yet. Garbage collection of newly removed objects is suspended until the participant gets +//! unpinned. +//! +//! # Garbage +//! +//! Objects that get removed from concurrent collections must be stashed away until all currently +//! pinned participants get unpinned. Such objects can be stored into a thread-local or global +//! storage, where they are kept until the right time for their destruction comes. +//! +//! There is a global shared instance of garbage queue. You can [`defer`](Guard::defer) the execution of an +//! arbitrary function until the global epoch is advanced enough. Most notably, concurrent data +//! structures may defer the deallocation of an object. +//! +//! # APIs +//! +//! For majority of use cases, just use the default garbage collector by invoking [`pin`]. If you +//! want to create your own garbage collector, use the [`Collector`] API. + +#![doc(test( + no_crate_inject, + attr( + deny(warnings, rust_2018_idioms), + allow(dead_code, unused_assignments, unused_variables) + ) +))] +#![warn( + missing_docs, + missing_debug_implementations, + rust_2018_idioms, + unreachable_pub +)] +#![cfg_attr(not(feature = "std"), no_std)] + +#[cfg(crossbeam_loom)] +extern crate loom_crate as loom; + +#[cfg(crossbeam_loom)] +#[allow(unused_imports, dead_code)] +mod primitive { + pub(crate) mod cell { + pub(crate) use loom::cell::UnsafeCell; + } + pub(crate) mod sync { + pub(crate) mod atomic { + pub(crate) use loom::sync::atomic::{fence, AtomicPtr, AtomicUsize, Ordering}; + + // FIXME: loom does not support compiler_fence at the moment. + // https://github.com/tokio-rs/loom/issues/117 + // we use fence as a stand-in for compiler_fence for the time being. + // this may miss some races since fence is stronger than compiler_fence, + // but it's the best we can do for the time being. + pub(crate) use self::fence as compiler_fence; + } + pub(crate) use loom::sync::Arc; + } + pub(crate) use loom::thread_local; +} +#[cfg(target_has_atomic = "ptr")] +#[cfg(not(crossbeam_loom))] +#[allow(unused_imports, dead_code)] +mod primitive { + pub(crate) mod cell { + #[derive(Debug)] + #[repr(transparent)] + pub(crate) struct UnsafeCell(::core::cell::UnsafeCell); + + // loom's UnsafeCell has a slightly different API than the standard library UnsafeCell. + // Since we want the rest of the code to be agnostic to whether it's running under loom or + // not, we write this small wrapper that provides the loom-supported API for the standard + // library UnsafeCell. This is also what the loom documentation recommends: + // https://github.com/tokio-rs/loom#handling-loom-api-differences + impl UnsafeCell { + #[inline] + pub(crate) const fn new(data: T) -> UnsafeCell { + UnsafeCell(::core::cell::UnsafeCell::new(data)) + } + + #[inline] + pub(crate) fn with(&self, f: impl FnOnce(*const T) -> R) -> R { + f(self.0.get()) + } + + #[inline] + pub(crate) fn with_mut(&self, f: impl FnOnce(*mut T) -> R) -> R { + f(self.0.get()) + } + } + } + pub(crate) mod sync { + pub(crate) mod atomic { + pub(crate) use core::sync::atomic::{ + compiler_fence, fence, AtomicPtr, AtomicUsize, Ordering, + }; + } + #[cfg(feature = "alloc")] + pub(crate) use alloc::sync::Arc; + } + + #[cfg(feature = "std")] + pub(crate) use std::thread_local; +} + +#[cfg(all(feature = "alloc", target_has_atomic = "ptr"))] +extern crate alloc; + +#[cfg(all(feature = "alloc", target_has_atomic = "ptr"))] +mod atomic; +#[cfg(all(feature = "alloc", target_has_atomic = "ptr"))] +mod collector; +#[cfg(all(feature = "alloc", target_has_atomic = "ptr"))] +mod deferred; +#[cfg(all(feature = "alloc", target_has_atomic = "ptr"))] +mod epoch; +#[cfg(all(feature = "alloc", target_has_atomic = "ptr"))] +mod guard; +#[cfg(all(feature = "alloc", target_has_atomic = "ptr"))] +mod internal; +#[cfg(all(feature = "alloc", target_has_atomic = "ptr"))] +mod sync; + +#[cfg(all(feature = "alloc", target_has_atomic = "ptr"))] +#[allow(deprecated)] +pub use crate::atomic::{CompareAndSetError, CompareAndSetOrdering}; +#[cfg(all(feature = "alloc", target_has_atomic = "ptr"))] +pub use crate::{ + atomic::{Atomic, CompareExchangeError, Owned, Pointable, Pointer, Shared}, + collector::{Collector, LocalHandle}, + guard::{unprotected, Guard}, +}; + +#[cfg(feature = "std")] +mod default; +#[cfg(feature = "std")] +pub use crate::default::{default_collector, is_pinned, pin}; diff --git a/anneal/v2/vendor/crossbeam-epoch/src/sync/list.rs b/anneal/v2/vendor/crossbeam-epoch/src/sync/list.rs new file mode 100644 index 0000000000..52ffd6fca4 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-epoch/src/sync/list.rs @@ -0,0 +1,487 @@ +//! Lock-free intrusive linked list. +//! +//! Ideas from Michael. High Performance Dynamic Lock-Free Hash Tables and List-Based Sets. SPAA +//! 2002. + +use core::marker::PhantomData; +use core::sync::atomic::Ordering::{Acquire, Relaxed, Release}; + +use crate::{unprotected, Atomic, Guard, Shared}; + +/// An entry in a linked list. +/// +/// An Entry is accessed from multiple threads, so it would be beneficial to put it in a different +/// cache-line than thread-local data in terms of performance. +#[derive(Debug)] +pub(crate) struct Entry { + /// The next entry in the linked list. + /// If the tag is 1, this entry is marked as deleted. + next: Atomic, +} + +/// Implementing this trait asserts that the type `T` can be used as an element in the intrusive +/// linked list defined in this module. `T` has to contain (or otherwise be linked to) an instance +/// of `Entry`. +/// +/// # Example +/// +/// ```ignore +/// struct A { +/// entry: Entry, +/// data: usize, +/// } +/// +/// impl IsElement for A { +/// fn entry_of(a: &A) -> &Entry { +/// let entry_ptr = ((a as usize) + offset_of!(A, entry)) as *const Entry; +/// unsafe { &*entry_ptr } +/// } +/// +/// unsafe fn element_of(entry: &Entry) -> &T { +/// let elem_ptr = ((entry as usize) - offset_of!(A, entry)) as *const T; +/// &*elem_ptr +/// } +/// +/// unsafe fn finalize(entry: &Entry, guard: &Guard) { +/// guard.defer_destroy(Shared::from(Self::element_of(entry) as *const _)); +/// } +/// } +/// ``` +/// +/// This trait is implemented on a type separate from `T` (although it can be just `T`), because +/// one type might be placeable into multiple lists, in which case it would require multiple +/// implementations of `IsElement`. In such cases, each struct implementing `IsElement` +/// represents a distinct `Entry` in `T`. +/// +/// For example, we can insert the following struct into two lists using `entry1` for one +/// and `entry2` for the other: +/// +/// ```ignore +/// struct B { +/// entry1: Entry, +/// entry2: Entry, +/// data: usize, +/// } +/// ``` +/// +pub(crate) trait IsElement { + /// Returns a reference to this element's `Entry`. + fn entry_of(_: &T) -> &Entry; + + /// Given a reference to an element's entry, returns that element. + /// + /// ```ignore + /// let elem = ListElement::new(); + /// assert_eq!(elem.entry_of(), + /// unsafe { ListElement::element_of(elem.entry_of()) } ); + /// ``` + /// + /// # Safety + /// + /// The caller has to guarantee that the `Entry` is called with was retrieved from an instance + /// of the element type (`T`). + unsafe fn element_of(_: &Entry) -> &T; + + /// The function that is called when an entry is unlinked from list. + /// + /// # Safety + /// + /// The caller has to guarantee that the `Entry` is called with was retrieved from an instance + /// of the element type (`T`). + unsafe fn finalize(_: &Entry, _: &Guard); +} + +/// A lock-free, intrusive linked list of type `T`. +#[derive(Debug)] +pub(crate) struct List = T> { + /// The head of the linked list. + head: Atomic, + + /// The phantom data for using `T` and `C`. + _marker: PhantomData<(T, C)>, +} + +/// An iterator used for retrieving values from the list. +pub(crate) struct Iter<'g, T, C: IsElement> { + /// The guard that protects the iteration. + guard: &'g Guard, + + /// Pointer from the predecessor to the current entry. + pred: &'g Atomic, + + /// The current entry. + curr: Shared<'g, Entry>, + + /// The list head, needed for restarting iteration. + head: &'g Atomic, + + /// Logically, we store a borrow of an instance of `T` and + /// use the type information from `C`. + _marker: PhantomData<(&'g T, C)>, +} + +/// An error that occurs during iteration over the list. +#[derive(PartialEq, Debug)] +pub(crate) enum IterError { + /// A concurrent thread modified the state of the list at the same place that this iterator + /// was inspecting. Subsequent iteration will restart from the beginning of the list. + Stalled, +} + +impl Default for Entry { + /// Returns the empty entry. + fn default() -> Self { + Self { + next: Atomic::null(), + } + } +} + +impl Entry { + /// Marks this entry as deleted, deferring the actual deallocation to a later iteration. + /// + /// # Safety + /// + /// The entry should be a member of a linked list, and it should not have been deleted. + /// It should be safe to call `C::finalize` on the entry after the `guard` is dropped, where `C` + /// is the associated helper for the linked list. + pub(crate) unsafe fn delete(&self, guard: &Guard) { + self.next.fetch_or(1, Release, guard); + } +} + +impl> List { + /// Returns a new, empty linked list. + pub(crate) fn new() -> Self { + Self { + head: Atomic::null(), + _marker: PhantomData, + } + } + + /// Inserts `entry` into the head of the list. + /// + /// # Safety + /// + /// You should guarantee that: + /// + /// - `container` is not null + /// - `container` is immovable, e.g. inside an `Owned` + /// - the same `Entry` is not inserted more than once + /// - the inserted object will be removed before the list is dropped + pub(crate) unsafe fn insert<'g>(&'g self, container: Shared<'g, T>, guard: &'g Guard) { + // Insert right after head, i.e. at the beginning of the list. + let to = &self.head; + // Get the intrusively stored Entry of the new element to insert. + let entry: &Entry = C::entry_of(container.deref()); + // Make a Shared ptr to that Entry. + let entry_ptr = Shared::from(entry as *const _); + // Read the current successor of where we want to insert. + let mut next = to.load(Relaxed, guard); + + loop { + // Set the Entry of the to-be-inserted element to point to the previous successor of + // `to`. + entry.next.store(next, Relaxed); + match to.compare_exchange_weak(next, entry_ptr, Release, Relaxed, guard) { + Ok(_) => break, + // We lost the race or weak CAS failed spuriously. Update the successor and try + // again. + Err(err) => next = err.current, + } + } + } + + /// Returns an iterator over all objects. + /// + /// # Caveat + /// + /// Every object that is inserted at the moment this function is called and persists at least + /// until the end of iteration will be returned. Since this iterator traverses a lock-free + /// linked list that may be concurrently modified, some additional caveats apply: + /// + /// 1. If a new object is inserted during iteration, it may or may not be returned. + /// 2. If an object is deleted during iteration, it may or may not be returned. + /// 3. The iteration may be aborted when it lost in a race condition. In this case, the winning + /// thread will continue to iterate over the same list. + pub(crate) fn iter<'g>(&'g self, guard: &'g Guard) -> Iter<'g, T, C> { + Iter { + guard, + pred: &self.head, + curr: self.head.load(Acquire, guard), + head: &self.head, + _marker: PhantomData, + } + } +} + +impl> Drop for List { + fn drop(&mut self) { + unsafe { + let guard = unprotected(); + let mut curr = self.head.load(Relaxed, guard); + while let Some(c) = curr.as_ref() { + let succ = c.next.load(Relaxed, guard); + // Verify that all elements have been removed from the list. + assert_eq!(succ.tag(), 1); + + C::finalize(curr.deref(), guard); + curr = succ; + } + } + } +} + +impl<'g, T: 'g, C: IsElement> Iterator for Iter<'g, T, C> { + type Item = Result<&'g T, IterError>; + + fn next(&mut self) -> Option { + while let Some(c) = unsafe { self.curr.as_ref() } { + let succ = c.next.load(Acquire, self.guard); + + if succ.tag() == 1 { + // This entry was removed. Try unlinking it from the list. + let succ = succ.with_tag(0); + + // The tag should always be zero, because removing a node after a logically deleted + // node leaves the list in an invalid state. + debug_assert!(self.curr.tag() == 0); + + // Try to unlink `curr` from the list, and get the new value of `self.pred`. + let succ = match self + .pred + .compare_exchange(self.curr, succ, Acquire, Acquire, self.guard) + { + Ok(_) => { + // We succeeded in unlinking `curr`, so we have to schedule + // deallocation. Deferred drop is okay, because `list.delete()` can only be + // called if `T: 'static`. + unsafe { + C::finalize(self.curr.deref(), self.guard); + } + + // `succ` is the new value of `self.pred`. + succ + } + Err(e) => { + // `e.current` is the current value of `self.pred`. + e.current + } + }; + + // If the predecessor node is already marked as deleted, we need to restart from + // `head`. + if succ.tag() != 0 { + self.pred = self.head; + self.curr = self.head.load(Acquire, self.guard); + + return Some(Err(IterError::Stalled)); + } + + // Move over the removed by only advancing `curr`, not `pred`. + self.curr = succ; + continue; + } + + // Move one step forward. + self.pred = &c.next; + self.curr = succ; + + return Some(Ok(unsafe { C::element_of(c) })); + } + + // We reached the end of the list. + None + } +} + +#[cfg(all(test, not(crossbeam_loom)))] +mod tests { + use super::*; + use crate::{Collector, Owned}; + use crossbeam_utils::thread; + use std::sync::Barrier; + + impl IsElement for Entry { + fn entry_of(entry: &Entry) -> &Entry { + entry + } + + unsafe fn element_of(entry: &Entry) -> &Entry { + entry + } + + unsafe fn finalize(entry: &Entry, guard: &Guard) { + guard.defer_destroy(Shared::from(Self::element_of(entry) as *const _)); + } + } + + /// Checks whether the list retains inserted elements + /// and returns them in the correct order. + #[test] + fn insert() { + let collector = Collector::new(); + let handle = collector.register(); + let guard = handle.pin(); + + let l: List = List::new(); + + let e1 = Owned::new(Entry::default()).into_shared(&guard); + let e2 = Owned::new(Entry::default()).into_shared(&guard); + let e3 = Owned::new(Entry::default()).into_shared(&guard); + + unsafe { + l.insert(e1, &guard); + l.insert(e2, &guard); + l.insert(e3, &guard); + } + + let mut iter = l.iter(&guard); + let maybe_e3 = iter.next(); + assert!(maybe_e3.is_some()); + assert!(maybe_e3.unwrap().unwrap() as *const Entry == e3.as_raw()); + let maybe_e2 = iter.next(); + assert!(maybe_e2.is_some()); + assert!(maybe_e2.unwrap().unwrap() as *const Entry == e2.as_raw()); + let maybe_e1 = iter.next(); + assert!(maybe_e1.is_some()); + assert!(maybe_e1.unwrap().unwrap() as *const Entry == e1.as_raw()); + assert!(iter.next().is_none()); + + unsafe { + e1.as_ref().unwrap().delete(&guard); + e2.as_ref().unwrap().delete(&guard); + e3.as_ref().unwrap().delete(&guard); + } + } + + /// Checks whether elements can be removed from the list and whether + /// the correct elements are removed. + #[test] + fn delete() { + let collector = Collector::new(); + let handle = collector.register(); + let guard = handle.pin(); + + let l: List = List::new(); + + let e1 = Owned::new(Entry::default()).into_shared(&guard); + let e2 = Owned::new(Entry::default()).into_shared(&guard); + let e3 = Owned::new(Entry::default()).into_shared(&guard); + unsafe { + l.insert(e1, &guard); + l.insert(e2, &guard); + l.insert(e3, &guard); + e2.as_ref().unwrap().delete(&guard); + } + + let mut iter = l.iter(&guard); + let maybe_e3 = iter.next(); + assert!(maybe_e3.is_some()); + assert!(maybe_e3.unwrap().unwrap() as *const Entry == e3.as_raw()); + let maybe_e1 = iter.next(); + assert!(maybe_e1.is_some()); + assert!(maybe_e1.unwrap().unwrap() as *const Entry == e1.as_raw()); + assert!(iter.next().is_none()); + + unsafe { + e1.as_ref().unwrap().delete(&guard); + e3.as_ref().unwrap().delete(&guard); + } + + let mut iter = l.iter(&guard); + assert!(iter.next().is_none()); + } + + const THREADS: usize = 8; + const ITERS: usize = 512; + + /// Contends the list on insert and delete operations to make sure they can run concurrently. + #[test] + fn insert_delete_multi() { + let collector = Collector::new(); + + let l: List = List::new(); + let b = Barrier::new(THREADS); + + thread::scope(|s| { + for _ in 0..THREADS { + s.spawn(|_| { + b.wait(); + + let handle = collector.register(); + let guard: Guard = handle.pin(); + let mut v = Vec::with_capacity(ITERS); + + for _ in 0..ITERS { + let e = Owned::new(Entry::default()).into_shared(&guard); + v.push(e); + unsafe { + l.insert(e, &guard); + } + } + + for e in v { + unsafe { + e.as_ref().unwrap().delete(&guard); + } + } + }); + } + }) + .unwrap(); + + let handle = collector.register(); + let guard = handle.pin(); + + let mut iter = l.iter(&guard); + assert!(iter.next().is_none()); + } + + /// Contends the list on iteration to make sure that it can be iterated over concurrently. + #[test] + fn iter_multi() { + let collector = Collector::new(); + + let l: List = List::new(); + let b = Barrier::new(THREADS); + + thread::scope(|s| { + for _ in 0..THREADS { + s.spawn(|_| { + b.wait(); + + let handle = collector.register(); + let guard: Guard = handle.pin(); + let mut v = Vec::with_capacity(ITERS); + + for _ in 0..ITERS { + let e = Owned::new(Entry::default()).into_shared(&guard); + v.push(e); + unsafe { + l.insert(e, &guard); + } + } + + let mut iter = l.iter(&guard); + for _ in 0..ITERS { + assert!(iter.next().is_some()); + } + + for e in v { + unsafe { + e.as_ref().unwrap().delete(&guard); + } + } + }); + } + }) + .unwrap(); + + let handle = collector.register(); + let guard = handle.pin(); + + let mut iter = l.iter(&guard); + assert!(iter.next().is_none()); + } +} diff --git a/anneal/v2/vendor/crossbeam-epoch/src/sync/mod.rs b/anneal/v2/vendor/crossbeam-epoch/src/sync/mod.rs new file mode 100644 index 0000000000..08981be257 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-epoch/src/sync/mod.rs @@ -0,0 +1,7 @@ +//! Synchronization primitives. + +pub(crate) mod list; +#[cfg(feature = "std")] +#[cfg(not(crossbeam_loom))] +pub(crate) mod once_lock; +pub(crate) mod queue; diff --git a/anneal/v2/vendor/crossbeam-epoch/src/sync/once_lock.rs b/anneal/v2/vendor/crossbeam-epoch/src/sync/once_lock.rs new file mode 100644 index 0000000000..e057aca7d5 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-epoch/src/sync/once_lock.rs @@ -0,0 +1,88 @@ +// Based on unstable std::sync::OnceLock. +// +// Source: https://github.com/rust-lang/rust/blob/8e9c93df464b7ada3fc7a1c8ccddd9dcb24ee0a0/library/std/src/sync/once_lock.rs + +use core::cell::UnsafeCell; +use core::mem::MaybeUninit; +use std::sync::Once; + +pub(crate) struct OnceLock { + once: Once, + value: UnsafeCell>, + // Unlike std::sync::OnceLock, we don't need PhantomData here because + // we don't use #[may_dangle]. +} + +unsafe impl Sync for OnceLock {} +unsafe impl Send for OnceLock {} + +impl OnceLock { + /// Creates a new empty cell. + #[must_use] + pub(crate) const fn new() -> Self { + Self { + once: Once::new(), + value: UnsafeCell::new(MaybeUninit::uninit()), + } + } + + /// Gets the contents of the cell, initializing it with `f` if the cell + /// was empty. + /// + /// Many threads may call `get_or_init` concurrently with different + /// initializing functions, but it is guaranteed that only one function + /// will be executed. + /// + /// # Panics + /// + /// If `f` panics, the panic is propagated to the caller, and the cell + /// remains uninitialized. + /// + /// It is an error to reentrantly initialize the cell from `f`. The + /// exact outcome is unspecified. Current implementation deadlocks, but + /// this may be changed to a panic in the future. + pub(crate) fn get_or_init(&self, f: F) -> &T + where + F: FnOnce() -> T, + { + // Fast path check + if self.once.is_completed() { + // SAFETY: The inner value has been initialized + return unsafe { self.get_unchecked() }; + } + self.initialize(f); + + // SAFETY: The inner value has been initialized + unsafe { self.get_unchecked() } + } + + #[cold] + fn initialize(&self, f: F) + where + F: FnOnce() -> T, + { + let slot = self.value.get(); + + self.once.call_once(|| { + let value = f(); + unsafe { slot.write(MaybeUninit::new(value)) } + }); + } + + /// # Safety + /// + /// The value must be initialized + unsafe fn get_unchecked(&self) -> &T { + debug_assert!(self.once.is_completed()); + &*self.value.get().cast::() + } +} + +impl Drop for OnceLock { + fn drop(&mut self) { + if self.once.is_completed() { + // SAFETY: The inner value has been initialized + unsafe { (*self.value.get()).assume_init_drop() }; + } + } +} diff --git a/anneal/v2/vendor/crossbeam-epoch/src/sync/queue.rs b/anneal/v2/vendor/crossbeam-epoch/src/sync/queue.rs new file mode 100644 index 0000000000..76c326beb3 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-epoch/src/sync/queue.rs @@ -0,0 +1,468 @@ +//! Michael-Scott lock-free queue. +//! +//! Usable with any number of producers and consumers. +//! +//! Michael and Scott. Simple, Fast, and Practical Non-Blocking and Blocking Concurrent Queue +//! Algorithms. PODC 1996. +//! +//! Simon Doherty, Lindsay Groves, Victor Luchangco, and Mark Moir. 2004b. Formal Verification of a +//! Practical Lock-Free Queue Algorithm. + +use core::mem::MaybeUninit; +use core::sync::atomic::Ordering::{Acquire, Relaxed, Release}; + +use crossbeam_utils::CachePadded; + +use crate::{unprotected, Atomic, Guard, Owned, Shared}; + +// The representation here is a singly-linked list, with a sentinel node at the front. In general +// the `tail` pointer may lag behind the actual tail. Non-sentinel nodes are either all `Data` or +// all `Blocked` (requests for data from blocked threads). +#[derive(Debug)] +pub(crate) struct Queue { + head: CachePadded>>, + tail: CachePadded>>, +} + +struct Node { + /// The slot in which a value of type `T` can be stored. + /// + /// The type of `data` is `MaybeUninit` because a `Node` doesn't always contain a `T`. + /// For example, the sentinel node in a queue never contains a value: its slot is always empty. + /// Other nodes start their life with a push operation and contain a value until it gets popped + /// out. After that such empty nodes get added to the collector for destruction. + data: MaybeUninit, + + next: Atomic>, +} + +// Any particular `T` should never be accessed concurrently, so no need for `Sync`. +unsafe impl Sync for Queue {} +unsafe impl Send for Queue {} + +impl Queue { + /// Create a new, empty queue. + pub(crate) fn new() -> Queue { + let q = Queue { + head: CachePadded::new(Atomic::null()), + tail: CachePadded::new(Atomic::null()), + }; + let sentinel = Owned::new(Node { + data: MaybeUninit::uninit(), + next: Atomic::null(), + }); + unsafe { + let guard = unprotected(); + let sentinel = sentinel.into_shared(guard); + q.head.store(sentinel, Relaxed); + q.tail.store(sentinel, Relaxed); + q + } + } + + /// Attempts to atomically place `n` into the `next` pointer of `onto`, and returns `true` on + /// success. The queue's `tail` pointer may be updated. + #[inline(always)] + fn push_internal( + &self, + onto: Shared<'_, Node>, + new: Shared<'_, Node>, + guard: &Guard, + ) -> bool { + // is `onto` the actual tail? + let o = unsafe { onto.deref() }; + let next = o.next.load(Acquire, guard); + if unsafe { next.as_ref().is_some() } { + // if not, try to "help" by moving the tail pointer forward + let _ = self + .tail + .compare_exchange(onto, next, Release, Relaxed, guard); + false + } else { + // looks like the actual tail; attempt to link in `n` + let result = o + .next + .compare_exchange(Shared::null(), new, Release, Relaxed, guard) + .is_ok(); + if result { + // try to move the tail pointer forward + let _ = self + .tail + .compare_exchange(onto, new, Release, Relaxed, guard); + } + result + } + } + + /// Adds `t` to the back of the queue, possibly waking up threads blocked on `pop`. + pub(crate) fn push(&self, t: T, guard: &Guard) { + let new = Owned::new(Node { + data: MaybeUninit::new(t), + next: Atomic::null(), + }); + let new = Owned::into_shared(new, guard); + + loop { + // We push onto the tail, so we'll start optimistically by looking there first. + let tail = self.tail.load(Acquire, guard); + + // Attempt to push onto the `tail` snapshot; fails if `tail.next` has changed. + if self.push_internal(tail, new, guard) { + break; + } + } + } + + /// Attempts to pop a data node. `Ok(None)` if queue is empty; `Err(())` if lost race to pop. + #[inline(always)] + fn pop_internal(&self, guard: &Guard) -> Result, ()> { + let head = self.head.load(Acquire, guard); + let h = unsafe { head.deref() }; + let next = h.next.load(Acquire, guard); + match unsafe { next.as_ref() } { + Some(n) => unsafe { + self.head + .compare_exchange(head, next, Release, Relaxed, guard) + .map(|_| { + let tail = self.tail.load(Relaxed, guard); + // Advance the tail so that we don't retire a pointer to a reachable node. + if head == tail { + let _ = self + .tail + .compare_exchange(tail, next, Release, Relaxed, guard); + } + guard.defer_destroy(head); + Some(n.data.assume_init_read()) + }) + .map_err(|_| ()) + }, + None => Ok(None), + } + } + + /// Attempts to pop a data node, if the data satisfies the given condition. `Ok(None)` if queue + /// is empty or the data does not satisfy the condition; `Err(())` if lost race to pop. + #[inline(always)] + fn pop_if_internal(&self, condition: F, guard: &Guard) -> Result, ()> + where + T: Sync, + F: Fn(&T) -> bool, + { + let head = self.head.load(Acquire, guard); + let h = unsafe { head.deref() }; + let next = h.next.load(Acquire, guard); + match unsafe { next.as_ref() } { + Some(n) if condition(unsafe { &*n.data.as_ptr() }) => unsafe { + self.head + .compare_exchange(head, next, Release, Relaxed, guard) + .map(|_| { + let tail = self.tail.load(Relaxed, guard); + // Advance the tail so that we don't retire a pointer to a reachable node. + if head == tail { + let _ = self + .tail + .compare_exchange(tail, next, Release, Relaxed, guard); + } + guard.defer_destroy(head); + Some(n.data.assume_init_read()) + }) + .map_err(|_| ()) + }, + None | Some(_) => Ok(None), + } + } + + /// Attempts to dequeue from the front. + /// + /// Returns `None` if the queue is observed to be empty. + pub(crate) fn try_pop(&self, guard: &Guard) -> Option { + loop { + if let Ok(head) = self.pop_internal(guard) { + return head; + } + } + } + + /// Attempts to dequeue from the front, if the item satisfies the given condition. + /// + /// Returns `None` if the queue is observed to be empty, or the head does not satisfy the given + /// condition. + pub(crate) fn try_pop_if(&self, condition: F, guard: &Guard) -> Option + where + T: Sync, + F: Fn(&T) -> bool, + { + loop { + if let Ok(head) = self.pop_if_internal(&condition, guard) { + return head; + } + } + } +} + +impl Drop for Queue { + fn drop(&mut self) { + unsafe { + let guard = unprotected(); + + while self.try_pop(guard).is_some() {} + + // Destroy the remaining sentinel node. + let sentinel = self.head.load(Relaxed, guard); + drop(sentinel.into_owned()); + } + } +} + +#[cfg(all(test, not(crossbeam_loom)))] +mod test { + use super::*; + use crate::pin; + use crossbeam_utils::thread; + + struct Queue { + queue: super::Queue, + } + + impl Queue { + pub(crate) fn new() -> Queue { + Queue { + queue: super::Queue::new(), + } + } + + pub(crate) fn push(&self, t: T) { + let guard = &pin(); + self.queue.push(t, guard); + } + + pub(crate) fn is_empty(&self) -> bool { + let guard = &pin(); + let head = self.queue.head.load(Acquire, guard); + let h = unsafe { head.deref() }; + h.next.load(Acquire, guard).is_null() + } + + pub(crate) fn try_pop(&self) -> Option { + let guard = &pin(); + self.queue.try_pop(guard) + } + + pub(crate) fn pop(&self) -> T { + loop { + match self.try_pop() { + None => continue, + Some(t) => return t, + } + } + } + } + + #[cfg(miri)] + const CONC_COUNT: i64 = 1000; + #[cfg(not(miri))] + const CONC_COUNT: i64 = 1000000; + + #[test] + fn push_try_pop_1() { + let q: Queue = Queue::new(); + assert!(q.is_empty()); + q.push(37); + assert!(!q.is_empty()); + assert_eq!(q.try_pop(), Some(37)); + assert!(q.is_empty()); + } + + #[test] + fn push_try_pop_2() { + let q: Queue = Queue::new(); + assert!(q.is_empty()); + q.push(37); + q.push(48); + assert_eq!(q.try_pop(), Some(37)); + assert!(!q.is_empty()); + assert_eq!(q.try_pop(), Some(48)); + assert!(q.is_empty()); + } + + #[test] + fn push_try_pop_many_seq() { + let q: Queue = Queue::new(); + assert!(q.is_empty()); + for i in 0..200 { + q.push(i) + } + assert!(!q.is_empty()); + for i in 0..200 { + assert_eq!(q.try_pop(), Some(i)); + } + assert!(q.is_empty()); + } + + #[test] + fn push_pop_1() { + let q: Queue = Queue::new(); + assert!(q.is_empty()); + q.push(37); + assert!(!q.is_empty()); + assert_eq!(q.pop(), 37); + assert!(q.is_empty()); + } + + #[test] + fn push_pop_2() { + let q: Queue = Queue::new(); + q.push(37); + q.push(48); + assert_eq!(q.pop(), 37); + assert_eq!(q.pop(), 48); + } + + #[test] + fn push_pop_many_seq() { + let q: Queue = Queue::new(); + assert!(q.is_empty()); + for i in 0..200 { + q.push(i) + } + assert!(!q.is_empty()); + for i in 0..200 { + assert_eq!(q.pop(), i); + } + assert!(q.is_empty()); + } + + #[test] + fn push_try_pop_many_spsc() { + let q: Queue = Queue::new(); + assert!(q.is_empty()); + + thread::scope(|scope| { + scope.spawn(|_| { + let mut next = 0; + + while next < CONC_COUNT { + if let Some(elem) = q.try_pop() { + assert_eq!(elem, next); + next += 1; + } + } + }); + + for i in 0..CONC_COUNT { + q.push(i) + } + }) + .unwrap(); + } + + #[test] + fn push_try_pop_many_spmc() { + fn recv(_t: i32, q: &Queue) { + let mut cur = -1; + for _i in 0..CONC_COUNT { + if let Some(elem) = q.try_pop() { + assert!(elem > cur); + cur = elem; + + if cur == CONC_COUNT - 1 { + break; + } + } + } + } + + let q: Queue = Queue::new(); + assert!(q.is_empty()); + thread::scope(|scope| { + for i in 0..3 { + let q = &q; + scope.spawn(move |_| recv(i, q)); + } + + scope.spawn(|_| { + for i in 0..CONC_COUNT { + q.push(i); + } + }); + }) + .unwrap(); + } + + #[test] + fn push_try_pop_many_mpmc() { + enum LR { + Left(i64), + Right(i64), + } + + let q: Queue = Queue::new(); + assert!(q.is_empty()); + + thread::scope(|scope| { + for _t in 0..2 { + scope.spawn(|_| { + for i in CONC_COUNT - 1..CONC_COUNT { + q.push(LR::Left(i)) + } + }); + scope.spawn(|_| { + for i in CONC_COUNT - 1..CONC_COUNT { + q.push(LR::Right(i)) + } + }); + scope.spawn(|_| { + let mut vl = vec![]; + let mut vr = vec![]; + for _i in 0..CONC_COUNT { + match q.try_pop() { + Some(LR::Left(x)) => vl.push(x), + Some(LR::Right(x)) => vr.push(x), + _ => {} + } + } + + let mut vl2 = vl.clone(); + let mut vr2 = vr.clone(); + vl2.sort_unstable(); + vr2.sort_unstable(); + + assert_eq!(vl, vl2); + assert_eq!(vr, vr2); + }); + } + }) + .unwrap(); + } + + #[test] + fn push_pop_many_spsc() { + let q: Queue = Queue::new(); + + thread::scope(|scope| { + scope.spawn(|_| { + let mut next = 0; + while next < CONC_COUNT { + assert_eq!(q.pop(), next); + next += 1; + } + }); + + for i in 0..CONC_COUNT { + q.push(i) + } + }) + .unwrap(); + assert!(q.is_empty()); + } + + #[test] + fn is_empty_dont_pop() { + let q: Queue = Queue::new(); + q.push(20); + q.push(20); + assert!(!q.is_empty()); + assert!(!q.is_empty()); + assert!(q.try_pop().is_some()); + } +} diff --git a/anneal/v2/vendor/crossbeam-epoch/tests/loom.rs b/anneal/v2/vendor/crossbeam-epoch/tests/loom.rs new file mode 100644 index 0000000000..4e56acdbca --- /dev/null +++ b/anneal/v2/vendor/crossbeam-epoch/tests/loom.rs @@ -0,0 +1,157 @@ +#![cfg(crossbeam_loom)] + +use crossbeam_epoch as epoch; +use loom_crate as loom; + +use epoch::*; +use epoch::{Atomic, Owned}; +use loom::sync::atomic::Ordering::{self, Acquire, Relaxed, Release}; +use loom::sync::Arc; +use loom::thread::spawn; +use std::mem::ManuallyDrop; +use std::ptr; + +#[test] +fn it_works() { + loom::model(|| { + let collector = Collector::new(); + let item: Atomic = Atomic::from(Owned::new(String::from("boom"))); + let item2 = item.clone(); + let collector2 = collector.clone(); + let guard = collector.register().pin(); + + let jh = loom::thread::spawn(move || { + let guard = collector2.register().pin(); + guard.defer(move || { + // this isn't really safe, since other threads may still have pointers to the + // value, but in this limited test scenario it's okay, since we know the test won't + // access item after all the pins are released. + let mut item = unsafe { item2.into_owned() }; + // mutate it as a second measure to make sure the assert_eq below would fail + item.retain(|c| c == 'o'); + drop(item); + }); + }); + + let item = item.load(Ordering::SeqCst, &guard); + // we pinned strictly before the call to defer_destroy, + // so item cannot have been dropped yet + assert_eq!(*unsafe { item.deref() }, "boom"); + drop(guard); + + jh.join().unwrap(); + + drop(collector); + }) +} + +#[test] +fn treiber_stack() { + /// Treiber's lock-free stack. + /// + /// Usable with any number of producers and consumers. + #[derive(Debug)] + pub struct TreiberStack { + head: Atomic>, + } + + #[derive(Debug)] + struct Node { + data: ManuallyDrop, + next: Atomic>, + } + + impl TreiberStack { + /// Creates a new, empty stack. + pub fn new() -> TreiberStack { + TreiberStack { + head: Atomic::null(), + } + } + + /// Pushes a value on top of the stack. + pub fn push(&self, t: T) { + let mut n = Owned::new(Node { + data: ManuallyDrop::new(t), + next: Atomic::null(), + }); + + let guard = epoch::pin(); + + loop { + let head = self.head.load(Relaxed, &guard); + n.next.store(head, Relaxed); + + match self + .head + .compare_exchange(head, n, Release, Relaxed, &guard) + { + Ok(_) => break, + Err(e) => n = e.new, + } + } + } + + /// Attempts to pop the top element from the stack. + /// + /// Returns `None` if the stack is empty. + pub fn pop(&self) -> Option { + let guard = epoch::pin(); + loop { + let head = self.head.load(Acquire, &guard); + + match unsafe { head.as_ref() } { + Some(h) => { + let next = h.next.load(Relaxed, &guard); + + if self + .head + .compare_exchange(head, next, Relaxed, Relaxed, &guard) + .is_ok() + { + unsafe { + guard.defer_destroy(head); + return Some(ManuallyDrop::into_inner(ptr::read(&(*h).data))); + } + } + } + None => return None, + } + } + } + + /// Returns `true` if the stack is empty. + pub fn is_empty(&self) -> bool { + let guard = epoch::pin(); + self.head.load(Acquire, &guard).is_null() + } + } + + impl Drop for TreiberStack { + fn drop(&mut self) { + while self.pop().is_some() {} + } + } + + loom::model(|| { + let stack1 = Arc::new(TreiberStack::new()); + let stack2 = Arc::clone(&stack1); + + // use 5 since it's greater than the 4 used for the sanitize feature + let jh = spawn(move || { + for i in 0..5 { + stack2.push(i); + assert!(stack2.pop().is_some()); + } + }); + + for i in 0..5 { + stack1.push(i); + assert!(stack1.pop().is_some()); + } + + jh.join().unwrap(); + assert!(stack1.pop().is_none()); + assert!(stack1.is_empty()); + }); +} diff --git a/anneal/v2/vendor/crossbeam-utils/.cargo-checksum.json b/anneal/v2/vendor/crossbeam-utils/.cargo-checksum.json new file mode 100644 index 0000000000..32c3420a24 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-utils/.cargo-checksum.json @@ -0,0 +1 @@ +{"files":{".cargo_vcs_info.json":"d43f49dbbe655cb91dcd62c862552f2ca4520eaff2e4bee391e01b0df968e358","CHANGELOG.md":"366caba01b88f421c71b97f61b9806abbf05e1ba0d24e4bf034191c1f8aa03b8","Cargo.toml":"961aa297754d8fdbae9e23d15bbbcfdcd2b50b2db56becddb84e8cba4f730713","Cargo.toml.orig":"6697cafee3a273f8c04e25c8606621b2fff8a779e3e1d01f9c08d225a0f36fc5","LICENSE-APACHE":"a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2","LICENSE-MIT":"5734ed989dfca1f625b40281ee9f4530f91b2411ec01cb748223e7eb87e201ab","README.md":"3c82bbb994f54ab76a9ed30a42dfd095c6e636258d379b9be3fbf66324310e71","benches/atomic_cell.rs":"c927eb3cd1e5ecc4b91adbc3bde98af15ffab4086190792ba64d5cde0e24df3d","build-common.rs":"502cb7494549bed6fa10ac7bea36e880eeb60290dc69b679ac5c92b376469562","build.rs":"7a7f9e56ea7fb4f78c4e532b84b9d27be719d600e85eaeb3a2f4b79a4f0b419c","no_atomic.rs":"fc1baa4489d9842988bacaaa545a7d7d0e2f8b93cfa0b7d1ae31f21256e4cb0a","src/atomic/atomic_cell.rs":"6d8b83b65c73644abc10ec88a1442c8db531ae140de79197901510fcaea45966","src/atomic/consume.rs":"381c2a8b13312ca0525d53ca1b7d0d4f525ddb154951fa3e216b061ad22012ff","src/atomic/mod.rs":"712e2337e710c07116e977154ea4247a1c065bf5599e6bf368138e715b403f6d","src/atomic/seq_lock.rs":"27182e6b87a9db73c5f6831759f8625f9fcdec3c2828204c444aef04f427735a","src/atomic/seq_lock_wide.rs":"9888dd03116bb89ca36d4ab8d5a0b5032107a2983a7eb8024454263b09080088","src/backoff.rs":"8715f0303ec91d1847c8ac3fc24bcc002a22a7284ade610e5eff4181f85827c7","src/cache_padded.rs":"b6ff04ecf6de9124c0069c014d35f37de543cff1c4bfc1f260586aa49a5af6d8","src/lib.rs":"060dabc6dc07de92a7afa57dcbc47222a95ef5819d543ad854858c3b329d6637","src/sync/mod.rs":"eca73c04f821859b8434d2b93db87d160dc6a3f65498ca201cd40d732ca4c134","src/sync/once_lock.rs":"aa8f957604d1119c4fc7038a18c14a6281230e81005f31201c099acff284ad4b","src/sync/parker.rs":"698996e7530da1f3815df11c89df7d916155229cbfd022cccbd555f1d1d31985","src/sync/sharded_lock.rs":"f96d536f5622fe2a0a0f7d8117be31e4b1ed607544c52c7e2ffcd1f51a6b93a1","src/sync/wait_group.rs":"3e339aab014f50e214fea535c841755113ea058153378ed54e50a4acb403c937","src/thread.rs":"04610787ba88f1f59549874a13fc037f2dcf4d8b5f1daaf08378f05c2b3c0039","tests/atomic_cell.rs":"716c864d4e103039dc5cd8bf6110da4cbabafc7e4e03819aa197828e8fb0a9c7","tests/cache_padded.rs":"1bfaff8354c8184e1ee1f902881ca9400b60effb273b0d3f752801a483d2b66d","tests/parker.rs":"6def4721287d9d70b1cfd63ebb34e1c83fbb3376edbad2bc8aac6ef69dd99d20","tests/sharded_lock.rs":"314adeb8a651a28935f7a49c9a261b8fa1fd82bf6a16c865a5aced6216d7e40b","tests/thread.rs":"9a7d7d3028c552fd834c68598b04a1cc252a816bc20ab62cec060d6cd09cab10","tests/wait_group.rs":"2a41533a5f7f113d19cd2bdafcc2abf86509109652274156efdd74abd00896b6"},"package":"d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"} \ No newline at end of file diff --git a/anneal/v2/vendor/crossbeam-utils/.cargo_vcs_info.json b/anneal/v2/vendor/crossbeam-utils/.cargo_vcs_info.json new file mode 100644 index 0000000000..87108da095 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-utils/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "ccd83ac4108a2a1b41e9c6e79c87267167d18dfa" + }, + "path_in_vcs": "crossbeam-utils" +} \ No newline at end of file diff --git a/anneal/v2/vendor/crossbeam-utils/CHANGELOG.md b/anneal/v2/vendor/crossbeam-utils/CHANGELOG.md new file mode 100644 index 0000000000..5aa1967e71 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-utils/CHANGELOG.md @@ -0,0 +1,243 @@ +# Version 0.8.21 + +- Improve implementation of `CachePadded`. (#1152) + +# Version 0.8.20 + +- Implement `Display` for `CachePadded`. (#1097) + +# Version 0.8.19 + +- Remove dependency on `cfg-if`. (#1072) + +# Version 0.8.18 + +- Relax the minimum supported Rust version to 1.60. (#1056) +- Improve scalability of `AtomicCell` fallback. (#1055) + +# Version 0.8.17 + +- Bump the minimum supported Rust version to 1.61. (#1037) +- Improve support for targets without atomic CAS or 64-bit atomic. (#1037) +- Always implement `UnwindSafe` and `RefUnwindSafe` for `AtomicCell`. (#1045) +- Improve compatibility with Miri, TSan, and loom. (#995, #1003) +- Improve compatibility with unstable `oom=panic`. (#1045) +- Improve implementation of `CachePadded`. (#1014, #1025) +- Update `loom` dependency to 0.7. + +# Version 0.8.16 + +- Improve implementation of `CachePadded`. (#967) + +# Version 0.8.15 + +- Add `#[clippy::has_significant_drop]` to `ShardedLock{Read,Write}Guard`. (#958) +- Improve handling of very large timeout. (#953) +- Soft-deprecate `thread::scope()` in favor of the more efficient `std::thread::scope` that stabilized in Rust 1.63. (#954) + +# Version 0.8.14 + +- Fix build script bug introduced in 0.8.13. (#932) + +# Version 0.8.13 + +**Note:** This release has been yanked due to regression fixed in 0.8.14. + +- Improve support for custom targets. (#922) + +# Version 0.8.12 + +- Removes the dependency on the `once_cell` crate to restore the MSRV. (#913) +- Work around [rust-lang#98302](https://github.com/rust-lang/rust/issues/98302), which causes compile error on windows-gnu when LTO is enabled. (#913) + +# Version 0.8.11 + +- Bump the minimum supported Rust version to 1.38. (#877) + +# Version 0.8.10 + +- Fix unsoundness of `AtomicCell` on types containing niches. (#834) + This fix contains breaking changes, but they are allowed because this is a soundness bug fix. See #834 for more. + +# Version 0.8.9 + +- Replace lazy_static with once_cell. (#817) + +# Version 0.8.8 + +- Fix a bug when unstable `loom` support is enabled. (#787) + +# Version 0.8.7 + +- Add `AtomicCell<{i*,u*}>::{fetch_max,fetch_min}`. (#785) +- Add `AtomicCell<{i*,u*,bool}>::fetch_nand`. (#785) +- Fix unsoundness of `AtomicCell<{i,u}64>` arithmetics on 32-bit targets that support `Atomic{I,U}64` (#781) + +# Version 0.8.6 + +**Note:** This release has been yanked. See [GHSA-qc84-gqf4-9926](https://github.com/crossbeam-rs/crossbeam/security/advisories/GHSA-qc84-gqf4-9926) for details. + +- Re-add `AtomicCell<{i,u}64>::{fetch_add,fetch_sub,fetch_and,fetch_or,fetch_xor}` that were accidentally removed in 0.8.0 on targets that do not support `Atomic{I,U}64`. (#767) +- Re-add `AtomicCell<{i,u}128>::{fetch_add,fetch_sub,fetch_and,fetch_or,fetch_xor}` that were accidentally removed in 0.8.0. (#767) + +# Version 0.8.5 + +**Note:** This release has been yanked. See [GHSA-qc84-gqf4-9926](https://github.com/crossbeam-rs/crossbeam/security/advisories/GHSA-qc84-gqf4-9926) for details. + +- Add `AtomicCell::fetch_update`. (#704) +- Support targets that do not have atomic CAS on stable Rust. (#698) + +# Version 0.8.4 + +**Note:** This release has been yanked. See [GHSA-qc84-gqf4-9926](https://github.com/crossbeam-rs/crossbeam/security/advisories/GHSA-qc84-gqf4-9926) for details. + +- Bump `loom` dependency to version 0.5. (#686) + +# Version 0.8.3 + +**Note:** This release has been yanked. See [GHSA-qc84-gqf4-9926](https://github.com/crossbeam-rs/crossbeam/security/advisories/GHSA-qc84-gqf4-9926) for details. + +- Make `loom` dependency optional. (#666) + +# Version 0.8.2 + +**Note:** This release has been yanked. See [GHSA-qc84-gqf4-9926](https://github.com/crossbeam-rs/crossbeam/security/advisories/GHSA-qc84-gqf4-9926) for details. + +- Deprecate `AtomicCell::compare_and_swap`. Use `AtomicCell::compare_exchange` instead. (#619) +- Add `Parker::park_deadline`. (#563) +- Improve implementation of `CachePadded`. (#636) +- Add unstable support for `loom`. (#487) + +# Version 0.8.1 + +**Note:** This release has been yanked. See [GHSA-qc84-gqf4-9926](https://github.com/crossbeam-rs/crossbeam/security/advisories/GHSA-qc84-gqf4-9926) for details. + +- Make `AtomicCell::is_lock_free` always const fn. (#600) +- Fix a bug in `seq_lock_wide`. (#596) +- Remove `const_fn` dependency. (#600) +- `crossbeam-utils` no longer fails to compile if unable to determine rustc version. Instead, it now displays a warning. (#604) + +# Version 0.8.0 + +**Note:** This release has been yanked. See [GHSA-qc84-gqf4-9926](https://github.com/crossbeam-rs/crossbeam/security/advisories/GHSA-qc84-gqf4-9926) for details. + +- Bump the minimum supported Rust version to 1.36. +- Remove deprecated `AtomicCell::get_mut()` and `Backoff::is_complete()` methods. +- Remove `alloc` feature. +- Make `CachePadded::new()` const function. +- Make `AtomicCell::is_lock_free()` const function at 1.46+. +- Implement `From` for `AtomicCell`. + +# Version 0.7.2 + +- Fix bug in release (yanking 0.7.1) + +# Version 0.7.1 + +- Bump `autocfg` dependency to version 1.0. (#460) +- Make `AtomicCell` lockfree for u8, u16, u32, u64 sized values at 1.34+. (#454) + +# Version 0.7.0 + +- Bump the minimum required version to 1.28. +- Fix breakage with nightly feature due to rust-lang/rust#65214. +- Apply `#[repr(transparent)]` to `AtomicCell`. +- Make `AtomicCell::new()` const function at 1.31+. + +# Version 0.6.6 + +- Add `UnwindSafe` and `RefUnwindSafe` impls for `AtomicCell`. +- Add `AtomicCell::as_ptr()`. +- Add `AtomicCell::take()`. +- Fix a bug in `AtomicCell::compare_exchange()` and `AtomicCell::compare_and_swap()`. +- Various documentation improvements. + +# Version 0.6.5 + +- Rename `Backoff::is_complete()` to `Backoff::is_completed()`. + +# Version 0.6.4 + +- Add `WaitGroup`, `ShardedLock`, and `Backoff`. +- Add `fetch_*` methods for `AtomicCell` and `AtomicCell`. +- Expand documentation. + +# Version 0.6.3 + +- Add `AtomicCell`. +- Improve documentation. + +# Version 0.6.2 + +- Add `Parker`. +- Improve documentation. + +# Version 0.6.1 + +- Fix a soundness bug in `Scope::spawn()`. +- Remove the `T: 'scope` bound on `ScopedJoinHandle`. + +# Version 0.6.0 + +- Move `AtomicConsume` to `atomic` module. +- `scope()` returns a `Result` of thread joins. +- Remove `spawn_unchecked`. +- Fix a soundness bug due to incorrect lifetimes. +- Improve documentation. +- Support nested scoped spawns. +- Implement `Copy`, `Hash`, `PartialEq`, and `Eq` for `CachePadded`. +- Add `CachePadded::into_inner()`. + +# Version 0.5.0 + +- Reorganize sub-modules and rename functions. + +# Version 0.4.1 + +- Fix a documentation link. + +# Version 0.4.0 + +- `CachePadded` supports types bigger than 64 bytes. +- Fix a bug in scoped threads where unitialized memory was being dropped. +- Minimum required Rust version is now 1.25. + +# Version 0.3.2 + +- Mark `load_consume` with `#[inline]`. + +# Version 0.3.1 + +- `load_consume` on ARM and AArch64. + +# Version 0.3.0 + +- Add `join` for scoped thread API. +- Add `load_consume` for atomic load-consume memory ordering. +- Remove `AtomicOption`. + +# Version 0.2.2 + +- Support Rust 1.12.1. +- Call `T::clone` when cloning a `CachePadded`. + +# Version 0.2.1 + +- Add `use_std` feature. + +# Version 0.2.0 + +- Add `nightly` feature. +- Use `repr(align(64))` on `CachePadded` with the `nightly` feature. +- Implement `Drop` for `CachePadded`. +- Implement `Clone` for `CachePadded`. +- Implement `From` for `CachePadded`. +- Implement better `Debug` for `CachePadded`. +- Write more tests. +- Add this changelog. +- Change cache line length to 64 bytes. +- Remove `ZerosValid`. + +# Version 0.1.0 + +- Old implementation of `CachePadded` from `crossbeam` version 0.3.0 diff --git a/anneal/v2/vendor/crossbeam-utils/Cargo.toml b/anneal/v2/vendor/crossbeam-utils/Cargo.toml new file mode 100644 index 0000000000..c93dc2f275 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-utils/Cargo.toml @@ -0,0 +1,101 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +rust-version = "1.60" +name = "crossbeam-utils" +version = "0.8.21" +build = "build.rs" +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Utilities for concurrent programming" +homepage = "https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-utils" +readme = "README.md" +keywords = [ + "scoped", + "thread", + "atomic", + "cache", +] +categories = [ + "algorithms", + "concurrency", + "data-structures", + "no-std", +] +license = "MIT OR Apache-2.0" +repository = "https://github.com/crossbeam-rs/crossbeam" + +[lib] +name = "crossbeam_utils" +path = "src/lib.rs" + +[[test]] +name = "atomic_cell" +path = "tests/atomic_cell.rs" + +[[test]] +name = "cache_padded" +path = "tests/cache_padded.rs" + +[[test]] +name = "parker" +path = "tests/parker.rs" + +[[test]] +name = "sharded_lock" +path = "tests/sharded_lock.rs" + +[[test]] +name = "thread" +path = "tests/thread.rs" + +[[test]] +name = "wait_group" +path = "tests/wait_group.rs" + +[[bench]] +name = "atomic_cell" +path = "benches/atomic_cell.rs" + +[dependencies] + +[dev-dependencies.rand] +version = "0.8" + +[features] +default = ["std"] +nightly = [] +std = [] + +[target."cfg(crossbeam_loom)".dependencies.loom] +version = "0.7.1" +optional = true + +[lints.clippy.declare_interior_mutable_const] +level = "allow" +priority = 1 + +[lints.clippy.lint_groups_priority] +level = "allow" +priority = 1 + +[lints.rust.unexpected_cfgs] +level = "warn" +priority = 0 +check-cfg = [ + "cfg(crossbeam_loom)", + "cfg(crossbeam_sanitize)", +] diff --git a/anneal/v2/vendor/crossbeam-utils/Cargo.toml.orig b/anneal/v2/vendor/crossbeam-utils/Cargo.toml.orig new file mode 100644 index 0000000000..3a95baea25 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-utils/Cargo.toml.orig @@ -0,0 +1,46 @@ +[package] +name = "crossbeam-utils" +# When publishing a new version: +# - Update CHANGELOG.md +# - Update README.md (when increasing major or minor version) +# - Run './tools/publish.sh crossbeam-utils ' +version = "0.8.21" +edition = "2021" +rust-version = "1.60" +license = "MIT OR Apache-2.0" +repository = "https://github.com/crossbeam-rs/crossbeam" +homepage = "https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-utils" +description = "Utilities for concurrent programming" +keywords = ["scoped", "thread", "atomic", "cache"] +categories = ["algorithms", "concurrency", "data-structures", "no-std"] + +[features] +default = ["std"] + +# Enable to use APIs that require `std`. +# This is enabled by default. +std = [] + +# These features are no longer used. +# TODO: remove in the next major version. +# Enable to use of unstable functionality. +# This is disabled by default and requires recent nightly compiler. +# +# NOTE: This feature is outside of the normal semver guarantees and minor or +# patch versions of crossbeam may make breaking changes to them at any time. +nightly = [] + +[dependencies] + +# Enable the use of loom for concurrency testing. +# +# NOTE: This feature is outside of the normal semver guarantees and minor or +# patch versions of crossbeam may make breaking changes to them at any time. +[target.'cfg(crossbeam_loom)'.dependencies] +loom = { version = "0.7.1", optional = true } + +[dev-dependencies] +rand = "0.8" + +[lints] +workspace = true diff --git a/anneal/v2/vendor/crossbeam-utils/LICENSE-APACHE b/anneal/v2/vendor/crossbeam-utils/LICENSE-APACHE new file mode 100644 index 0000000000..16fe87b06e --- /dev/null +++ b/anneal/v2/vendor/crossbeam-utils/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/anneal/v2/vendor/crossbeam-utils/LICENSE-MIT b/anneal/v2/vendor/crossbeam-utils/LICENSE-MIT new file mode 100644 index 0000000000..068d491fd5 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-utils/LICENSE-MIT @@ -0,0 +1,27 @@ +The MIT License (MIT) + +Copyright (c) 2019 The Crossbeam Project Developers + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/anneal/v2/vendor/crossbeam-utils/README.md b/anneal/v2/vendor/crossbeam-utils/README.md new file mode 100644 index 0000000000..7d6a679487 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-utils/README.md @@ -0,0 +1,73 @@ +# Crossbeam Utils + +[![Build Status](https://github.com/crossbeam-rs/crossbeam/workflows/CI/badge.svg)]( +https://github.com/crossbeam-rs/crossbeam/actions) +[![License](https://img.shields.io/badge/license-MIT_OR_Apache--2.0-blue.svg)]( +https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-utils#license) +[![Cargo](https://img.shields.io/crates/v/crossbeam-utils.svg)]( +https://crates.io/crates/crossbeam-utils) +[![Documentation](https://docs.rs/crossbeam-utils/badge.svg)]( +https://docs.rs/crossbeam-utils) +[![Rust 1.60+](https://img.shields.io/badge/rust-1.60+-lightgray.svg)]( +https://www.rust-lang.org) +[![chat](https://img.shields.io/discord/569610676205781012.svg?logo=discord)](https://discord.com/invite/JXYwgWZ) + +This crate provides miscellaneous tools for concurrent programming: + +#### Atomics + +* [`AtomicCell`], a thread-safe mutable memory location.(no_std) +* [`AtomicConsume`], for reading from primitive atomic types with "consume" ordering.(no_std) + +#### Thread synchronization + +* [`Parker`], a thread parking primitive. +* [`ShardedLock`], a sharded reader-writer lock with fast concurrent reads. +* [`WaitGroup`], for synchronizing the beginning or end of some computation. + +#### Utilities + +* [`Backoff`], for exponential backoff in spin loops.(no_std) +* [`CachePadded`], for padding and aligning a value to the length of a cache line.(no_std) +* [`scope`], for spawning threads that borrow local variables from the stack. + +*Features marked with (no_std) can be used in `no_std` environments.*
+ +[`AtomicCell`]: https://docs.rs/crossbeam-utils/*/crossbeam_utils/atomic/struct.AtomicCell.html +[`AtomicConsume`]: https://docs.rs/crossbeam-utils/*/crossbeam_utils/atomic/trait.AtomicConsume.html +[`Parker`]: https://docs.rs/crossbeam-utils/*/crossbeam_utils/sync/struct.Parker.html +[`ShardedLock`]: https://docs.rs/crossbeam-utils/*/crossbeam_utils/sync/struct.ShardedLock.html +[`WaitGroup`]: https://docs.rs/crossbeam-utils/*/crossbeam_utils/sync/struct.WaitGroup.html +[`Backoff`]: https://docs.rs/crossbeam-utils/*/crossbeam_utils/struct.Backoff.html +[`CachePadded`]: https://docs.rs/crossbeam-utils/*/crossbeam_utils/struct.CachePadded.html +[`scope`]: https://docs.rs/crossbeam-utils/*/crossbeam_utils/thread/fn.scope.html + +## Usage + +Add this to your `Cargo.toml`: + +```toml +[dependencies] +crossbeam-utils = "0.8" +``` + +## Compatibility + +Crossbeam Utils supports stable Rust releases going back at least six months, +and every time the minimum supported Rust version is increased, a new minor +version is released. Currently, the minimum supported Rust version is 1.60. + +## License + +Licensed under either of + + * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) + * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) + +at your option. + +#### Contribution + +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in the work by you, as defined in the Apache-2.0 license, shall be +dual licensed as above, without any additional terms or conditions. diff --git a/anneal/v2/vendor/crossbeam-utils/benches/atomic_cell.rs b/anneal/v2/vendor/crossbeam-utils/benches/atomic_cell.rs new file mode 100644 index 0000000000..844f7c02b6 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-utils/benches/atomic_cell.rs @@ -0,0 +1,156 @@ +#![feature(test)] + +extern crate test; + +use std::sync::Barrier; + +use crossbeam_utils::atomic::AtomicCell; +use crossbeam_utils::thread; + +#[bench] +fn load_u8(b: &mut test::Bencher) { + let a = AtomicCell::new(0u8); + let mut sum = 0; + b.iter(|| sum += a.load()); + test::black_box(sum); +} + +#[bench] +fn store_u8(b: &mut test::Bencher) { + let a = AtomicCell::new(0u8); + b.iter(|| a.store(1)); +} + +#[bench] +fn fetch_add_u8(b: &mut test::Bencher) { + let a = AtomicCell::new(0u8); + b.iter(|| a.fetch_add(1)); +} + +#[bench] +fn compare_exchange_u8(b: &mut test::Bencher) { + let a = AtomicCell::new(0u8); + let mut i = 0; + b.iter(|| { + let _ = a.compare_exchange(i, i.wrapping_add(1)); + i = i.wrapping_add(1); + }); +} + +#[bench] +fn concurrent_load_u8(b: &mut test::Bencher) { + const THREADS: usize = 2; + const STEPS: usize = 1_000_000; + + let start = Barrier::new(THREADS + 1); + let end = Barrier::new(THREADS + 1); + let exit = AtomicCell::new(false); + + let a = AtomicCell::new(0u8); + + thread::scope(|scope| { + for _ in 0..THREADS { + scope.spawn(|_| loop { + start.wait(); + + let mut sum = 0; + for _ in 0..STEPS { + sum += a.load(); + } + test::black_box(sum); + + end.wait(); + if exit.load() { + break; + } + }); + } + + start.wait(); + end.wait(); + + b.iter(|| { + start.wait(); + end.wait(); + }); + + start.wait(); + exit.store(true); + end.wait(); + }) + .unwrap(); +} + +#[bench] +fn load_usize(b: &mut test::Bencher) { + let a = AtomicCell::new(0usize); + let mut sum = 0; + b.iter(|| sum += a.load()); + test::black_box(sum); +} + +#[bench] +fn store_usize(b: &mut test::Bencher) { + let a = AtomicCell::new(0usize); + b.iter(|| a.store(1)); +} + +#[bench] +fn fetch_add_usize(b: &mut test::Bencher) { + let a = AtomicCell::new(0usize); + b.iter(|| a.fetch_add(1)); +} + +#[bench] +fn compare_exchange_usize(b: &mut test::Bencher) { + let a = AtomicCell::new(0usize); + let mut i = 0; + b.iter(|| { + let _ = a.compare_exchange(i, i.wrapping_add(1)); + i = i.wrapping_add(1); + }); +} + +#[bench] +fn concurrent_load_usize(b: &mut test::Bencher) { + const THREADS: usize = 2; + const STEPS: usize = 1_000_000; + + let start = Barrier::new(THREADS + 1); + let end = Barrier::new(THREADS + 1); + let exit = AtomicCell::new(false); + + let a = AtomicCell::new(0usize); + + thread::scope(|scope| { + for _ in 0..THREADS { + scope.spawn(|_| loop { + start.wait(); + + let mut sum = 0; + for _ in 0..STEPS { + sum += a.load(); + } + test::black_box(sum); + + end.wait(); + if exit.load() { + break; + } + }); + } + + start.wait(); + end.wait(); + + b.iter(|| { + start.wait(); + end.wait(); + }); + + start.wait(); + exit.store(true); + end.wait(); + }) + .unwrap(); +} diff --git a/anneal/v2/vendor/crossbeam-utils/build-common.rs b/anneal/v2/vendor/crossbeam-utils/build-common.rs new file mode 100644 index 0000000000..e91bb4d471 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-utils/build-common.rs @@ -0,0 +1,13 @@ +// The target triplets have the form of 'arch-vendor-system'. +// +// When building for Linux (e.g. the 'system' part is +// 'linux-something'), replace the vendor with 'unknown' +// so that mapping to rust standard targets happens correctly. +fn convert_custom_linux_target(target: String) -> String { + let mut parts: Vec<&str> = target.split('-').collect(); + let system = parts.get(2); + if system == Some(&"linux") { + parts[1] = "unknown"; + }; + parts.join("-") +} diff --git a/anneal/v2/vendor/crossbeam-utils/build.rs b/anneal/v2/vendor/crossbeam-utils/build.rs new file mode 100644 index 0000000000..ff7e81f949 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-utils/build.rs @@ -0,0 +1,48 @@ +// The rustc-cfg listed below are considered public API, but it is *unstable* +// and outside of the normal semver guarantees: +// +// - `crossbeam_no_atomic` +// Assume the target does *not* support any atomic operations. +// This is usually detected automatically by the build script, but you may +// need to enable it manually when building for custom targets or using +// non-cargo build systems that don't run the build script. +// +// With the exceptions mentioned above, the rustc-cfg emitted by the build +// script are *not* public API. + +#![warn(rust_2018_idioms)] + +use std::env; + +include!("no_atomic.rs"); +include!("build-common.rs"); + +fn main() { + println!("cargo:rerun-if-changed=no_atomic.rs"); + println!("cargo:rustc-check-cfg=cfg(crossbeam_no_atomic,crossbeam_sanitize_thread)"); + + let target = match env::var("TARGET") { + Ok(target) => convert_custom_linux_target(target), + Err(e) => { + println!( + "cargo:warning={}: unable to get TARGET environment variable: {}", + env!("CARGO_PKG_NAME"), + e + ); + return; + } + }; + + // Note that this is `no_`*, not `has_*`. This allows treating as the latest + // stable rustc is used when the build script doesn't run. This is useful + // for non-cargo build systems that don't run the build script. + if NO_ATOMIC.contains(&&*target) { + println!("cargo:rustc-cfg=crossbeam_no_atomic"); + } + + // `cfg(sanitize = "..")` is not stabilized. + let sanitize = env::var("CARGO_CFG_SANITIZE").unwrap_or_default(); + if sanitize.contains("thread") { + println!("cargo:rustc-cfg=crossbeam_sanitize_thread"); + } +} diff --git a/anneal/v2/vendor/crossbeam-utils/no_atomic.rs b/anneal/v2/vendor/crossbeam-utils/no_atomic.rs new file mode 100644 index 0000000000..f7e6d2fa42 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-utils/no_atomic.rs @@ -0,0 +1,9 @@ +// This file is @generated by no_atomic.sh. +// It is not intended for manual editing. + +const NO_ATOMIC: &[&str] = &[ + "bpfeb-unknown-none", + "bpfel-unknown-none", + "mipsel-sony-psx", + "msp430-none-elf", +]; diff --git a/anneal/v2/vendor/crossbeam-utils/src/atomic/atomic_cell.rs b/anneal/v2/vendor/crossbeam-utils/src/atomic/atomic_cell.rs new file mode 100644 index 0000000000..47472534c8 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-utils/src/atomic/atomic_cell.rs @@ -0,0 +1,1182 @@ +// Necessary for implementing atomic methods for `AtomicUnit` +#![allow(clippy::unit_arg)] + +use crate::primitive::sync::atomic::{self, Ordering}; +use crate::CachePadded; +use core::cell::UnsafeCell; +use core::cmp; +use core::fmt; +use core::mem::{self, ManuallyDrop, MaybeUninit}; +use core::panic::{RefUnwindSafe, UnwindSafe}; +use core::ptr; + +use super::seq_lock::SeqLock; + +/// A thread-safe mutable memory location. +/// +/// This type is equivalent to [`Cell`], except it can also be shared among multiple threads. +/// +/// Operations on `AtomicCell`s use atomic instructions whenever possible, and synchronize using +/// global locks otherwise. You can call [`AtomicCell::::is_lock_free()`] to check whether +/// atomic instructions or locks will be used. +/// +/// Atomic loads use the [`Acquire`] ordering and atomic stores use the [`Release`] ordering. +/// +/// [`Cell`]: std::cell::Cell +/// [`AtomicCell::::is_lock_free()`]: AtomicCell::is_lock_free +/// [`Acquire`]: std::sync::atomic::Ordering::Acquire +/// [`Release`]: std::sync::atomic::Ordering::Release +#[repr(transparent)] +pub struct AtomicCell { + /// The inner value. + /// + /// If this value can be transmuted into a primitive atomic type, it will be treated as such. + /// Otherwise, all potentially concurrent operations on this data will be protected by a global + /// lock. + /// + /// Using MaybeUninit to prevent code outside the cell from observing partially initialized state: + /// + /// (This rustc bug has been fixed in Rust 1.64.) + /// + /// Note: + /// - we'll never store uninitialized `T` due to our API only using initialized `T`. + /// - this `MaybeUninit` does *not* fix . + value: UnsafeCell>, +} + +unsafe impl Send for AtomicCell {} +unsafe impl Sync for AtomicCell {} + +impl UnwindSafe for AtomicCell {} +impl RefUnwindSafe for AtomicCell {} + +impl AtomicCell { + /// Creates a new atomic cell initialized with `val`. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + /// let a = AtomicCell::new(7); + /// ``` + pub const fn new(val: T) -> AtomicCell { + AtomicCell { + value: UnsafeCell::new(MaybeUninit::new(val)), + } + } + + /// Consumes the atomic and returns the contained value. + /// + /// This is safe because passing `self` by value guarantees that no other threads are + /// concurrently accessing the atomic data. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + /// let a = AtomicCell::new(7); + /// let v = a.into_inner(); + /// + /// assert_eq!(v, 7); + /// ``` + pub fn into_inner(self) -> T { + let this = ManuallyDrop::new(self); + // SAFETY: + // - passing `self` by value guarantees that no other threads are concurrently + // accessing the atomic data + // - the raw pointer passed in is valid because we got it from an owned value. + // - `ManuallyDrop` prevents double dropping `T` + unsafe { this.as_ptr().read() } + } + + /// Returns `true` if operations on values of this type are lock-free. + /// + /// If the compiler or the platform doesn't support the necessary atomic instructions, + /// `AtomicCell` will use global locks for every potentially concurrent atomic operation. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + /// // This type is internally represented as `AtomicUsize` so we can just use atomic + /// // operations provided by it. + /// assert_eq!(AtomicCell::::is_lock_free(), true); + /// + /// // A wrapper struct around `isize`. + /// struct Foo { + /// bar: isize, + /// } + /// // `AtomicCell` will be internally represented as `AtomicIsize`. + /// assert_eq!(AtomicCell::::is_lock_free(), true); + /// + /// // Operations on zero-sized types are always lock-free. + /// assert_eq!(AtomicCell::<()>::is_lock_free(), true); + /// + /// // Very large types cannot be represented as any of the standard atomic types, so atomic + /// // operations on them will have to use global locks for synchronization. + /// assert_eq!(AtomicCell::<[u8; 1000]>::is_lock_free(), false); + /// ``` + pub const fn is_lock_free() -> bool { + atomic_is_lock_free::() + } + + /// Stores `val` into the atomic cell. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + /// let a = AtomicCell::new(7); + /// + /// assert_eq!(a.load(), 7); + /// a.store(8); + /// assert_eq!(a.load(), 8); + /// ``` + pub fn store(&self, val: T) { + if mem::needs_drop::() { + drop(self.swap(val)); + } else { + unsafe { + atomic_store(self.as_ptr(), val); + } + } + } + + /// Stores `val` into the atomic cell and returns the previous value. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + /// let a = AtomicCell::new(7); + /// + /// assert_eq!(a.load(), 7); + /// assert_eq!(a.swap(8), 7); + /// assert_eq!(a.load(), 8); + /// ``` + pub fn swap(&self, val: T) -> T { + unsafe { atomic_swap(self.as_ptr(), val) } + } + + /// Returns a raw pointer to the underlying data in this atomic cell. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + /// let a = AtomicCell::new(5); + /// + /// let ptr = a.as_ptr(); + /// ``` + #[inline] + pub fn as_ptr(&self) -> *mut T { + self.value.get().cast::() + } +} + +impl AtomicCell { + /// Takes the value of the atomic cell, leaving `Default::default()` in its place. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + /// let a = AtomicCell::new(5); + /// let five = a.take(); + /// + /// assert_eq!(five, 5); + /// assert_eq!(a.into_inner(), 0); + /// ``` + pub fn take(&self) -> T { + self.swap(Default::default()) + } +} + +impl AtomicCell { + /// Loads a value from the atomic cell. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + /// let a = AtomicCell::new(7); + /// + /// assert_eq!(a.load(), 7); + /// ``` + pub fn load(&self) -> T { + unsafe { atomic_load(self.as_ptr()) } + } +} + +impl AtomicCell { + /// If the current value equals `current`, stores `new` into the atomic cell. + /// + /// The return value is always the previous value. If it is equal to `current`, then the value + /// was updated. + /// + /// # Examples + /// + /// ``` + /// # #![allow(deprecated)] + /// use crossbeam_utils::atomic::AtomicCell; + /// + /// let a = AtomicCell::new(1); + /// + /// assert_eq!(a.compare_and_swap(2, 3), 1); + /// assert_eq!(a.load(), 1); + /// + /// assert_eq!(a.compare_and_swap(1, 2), 1); + /// assert_eq!(a.load(), 2); + /// ``` + // TODO: remove in the next major version. + #[deprecated(note = "Use `compare_exchange` instead")] + pub fn compare_and_swap(&self, current: T, new: T) -> T { + match self.compare_exchange(current, new) { + Ok(v) => v, + Err(v) => v, + } + } + + /// If the current value equals `current`, stores `new` into the atomic cell. + /// + /// The return value is a result indicating whether the new value was written and containing + /// the previous value. On success this value is guaranteed to be equal to `current`. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + /// let a = AtomicCell::new(1); + /// + /// assert_eq!(a.compare_exchange(2, 3), Err(1)); + /// assert_eq!(a.load(), 1); + /// + /// assert_eq!(a.compare_exchange(1, 2), Ok(1)); + /// assert_eq!(a.load(), 2); + /// ``` + pub fn compare_exchange(&self, current: T, new: T) -> Result { + unsafe { atomic_compare_exchange_weak(self.as_ptr(), current, new) } + } + + /// Fetches the value, and applies a function to it that returns an optional + /// new value. Returns a `Result` of `Ok(previous_value)` if the function returned `Some(_)`, else + /// `Err(previous_value)`. + /// + /// Note: This may call the function multiple times if the value has been changed from other threads in + /// the meantime, as long as the function returns `Some(_)`, but the function will have been applied + /// only once to the stored value. + /// + /// # Examples + /// + /// ```rust + /// use crossbeam_utils::atomic::AtomicCell; + /// + /// let a = AtomicCell::new(7); + /// assert_eq!(a.fetch_update(|_| None), Err(7)); + /// assert_eq!(a.fetch_update(|a| Some(a + 1)), Ok(7)); + /// assert_eq!(a.fetch_update(|a| Some(a + 1)), Ok(8)); + /// assert_eq!(a.load(), 9); + /// ``` + #[inline] + pub fn fetch_update(&self, mut f: F) -> Result + where + F: FnMut(T) -> Option, + { + let mut prev = self.load(); + while let Some(next) = f(prev) { + match self.compare_exchange(prev, next) { + x @ Ok(_) => return x, + Err(next_prev) => prev = next_prev, + } + } + Err(prev) + } +} + +// `MaybeUninit` prevents `T` from being dropped, so we need to implement `Drop` +// for `AtomicCell` to avoid leaks of non-`Copy` types. +impl Drop for AtomicCell { + fn drop(&mut self) { + if mem::needs_drop::() { + // SAFETY: + // - the mutable reference guarantees that no other threads are concurrently accessing the atomic data + // - the raw pointer passed in is valid because we got it from a reference + // - `MaybeUninit` prevents double dropping `T` + unsafe { + self.as_ptr().drop_in_place(); + } + } + } +} + +macro_rules! atomic { + // If values of type `$t` can be transmuted into values of the primitive atomic type `$atomic`, + // declares variable `$a` of type `$atomic` and executes `$atomic_op`, breaking out of the loop. + (@check, $t:ty, $atomic:ty, $a:ident, $atomic_op:expr) => { + if can_transmute::<$t, $atomic>() { + let $a: &$atomic; + break $atomic_op; + } + }; + + // If values of type `$t` can be transmuted into values of a primitive atomic type, declares + // variable `$a` of that type and executes `$atomic_op`. Otherwise, just executes + // `$fallback_op`. + ($t:ty, $a:ident, $atomic_op:expr, $fallback_op:expr) => { + loop { + atomic!(@check, $t, AtomicUnit, $a, $atomic_op); + + atomic!(@check, $t, atomic::AtomicU8, $a, $atomic_op); + atomic!(@check, $t, atomic::AtomicU16, $a, $atomic_op); + atomic!(@check, $t, atomic::AtomicU32, $a, $atomic_op); + #[cfg(target_has_atomic = "64")] + atomic!(@check, $t, atomic::AtomicU64, $a, $atomic_op); + // TODO: AtomicU128 is unstable + // atomic!(@check, $t, atomic::AtomicU128, $a, $atomic_op); + + break $fallback_op; + } + }; +} + +macro_rules! impl_arithmetic { + ($t:ty, fallback, $example:tt) => { + impl AtomicCell<$t> { + /// Increments the current value by `val` and returns the previous value. + /// + /// The addition wraps on overflow. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + #[doc = $example] + /// + /// assert_eq!(a.fetch_add(3), 7); + /// assert_eq!(a.load(), 10); + /// ``` + #[inline] + pub fn fetch_add(&self, val: $t) -> $t { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value = value.wrapping_add(val); + old + } + + /// Decrements the current value by `val` and returns the previous value. + /// + /// The subtraction wraps on overflow. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + #[doc = $example] + /// + /// assert_eq!(a.fetch_sub(3), 7); + /// assert_eq!(a.load(), 4); + /// ``` + #[inline] + pub fn fetch_sub(&self, val: $t) -> $t { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value = value.wrapping_sub(val); + old + } + + /// Applies bitwise "and" to the current value and returns the previous value. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + #[doc = $example] + /// + /// assert_eq!(a.fetch_and(3), 7); + /// assert_eq!(a.load(), 3); + /// ``` + #[inline] + pub fn fetch_and(&self, val: $t) -> $t { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value &= val; + old + } + + /// Applies bitwise "nand" to the current value and returns the previous value. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + #[doc = $example] + /// + /// assert_eq!(a.fetch_nand(3), 7); + /// assert_eq!(a.load(), !(7 & 3)); + /// ``` + #[inline] + pub fn fetch_nand(&self, val: $t) -> $t { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value = !(old & val); + old + } + + /// Applies bitwise "or" to the current value and returns the previous value. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + #[doc = $example] + /// + /// assert_eq!(a.fetch_or(16), 7); + /// assert_eq!(a.load(), 23); + /// ``` + #[inline] + pub fn fetch_or(&self, val: $t) -> $t { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value |= val; + old + } + + /// Applies bitwise "xor" to the current value and returns the previous value. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + #[doc = $example] + /// + /// assert_eq!(a.fetch_xor(2), 7); + /// assert_eq!(a.load(), 5); + /// ``` + #[inline] + pub fn fetch_xor(&self, val: $t) -> $t { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value ^= val; + old + } + + /// Compares and sets the maximum of the current value and `val`, + /// and returns the previous value. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + #[doc = $example] + /// + /// assert_eq!(a.fetch_max(2), 7); + /// assert_eq!(a.load(), 7); + /// ``` + #[inline] + pub fn fetch_max(&self, val: $t) -> $t { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value = cmp::max(old, val); + old + } + + /// Compares and sets the minimum of the current value and `val`, + /// and returns the previous value. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + #[doc = $example] + /// + /// assert_eq!(a.fetch_min(2), 7); + /// assert_eq!(a.load(), 2); + /// ``` + #[inline] + pub fn fetch_min(&self, val: $t) -> $t { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value = cmp::min(old, val); + old + } + } + }; + ($t:ty, $atomic:ident, $example:tt) => { + impl AtomicCell<$t> { + /// Increments the current value by `val` and returns the previous value. + /// + /// The addition wraps on overflow. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + #[doc = $example] + /// + /// assert_eq!(a.fetch_add(3), 7); + /// assert_eq!(a.load(), 10); + /// ``` + #[inline] + pub fn fetch_add(&self, val: $t) -> $t { + atomic! { + $t, _a, + { + let a = unsafe { &*(self.as_ptr() as *const atomic::$atomic) }; + a.fetch_add(val, Ordering::AcqRel) + }, + { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value = value.wrapping_add(val); + old + } + } + } + + /// Decrements the current value by `val` and returns the previous value. + /// + /// The subtraction wraps on overflow. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + #[doc = $example] + /// + /// assert_eq!(a.fetch_sub(3), 7); + /// assert_eq!(a.load(), 4); + /// ``` + #[inline] + pub fn fetch_sub(&self, val: $t) -> $t { + atomic! { + $t, _a, + { + let a = unsafe { &*(self.as_ptr() as *const atomic::$atomic) }; + a.fetch_sub(val, Ordering::AcqRel) + }, + { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value = value.wrapping_sub(val); + old + } + } + } + + /// Applies bitwise "and" to the current value and returns the previous value. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + #[doc = $example] + /// + /// assert_eq!(a.fetch_and(3), 7); + /// assert_eq!(a.load(), 3); + /// ``` + #[inline] + pub fn fetch_and(&self, val: $t) -> $t { + atomic! { + $t, _a, + { + let a = unsafe { &*(self.as_ptr() as *const atomic::$atomic) }; + a.fetch_and(val, Ordering::AcqRel) + }, + { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value &= val; + old + } + } + } + + /// Applies bitwise "nand" to the current value and returns the previous value. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + #[doc = $example] + /// + /// assert_eq!(a.fetch_nand(3), 7); + /// assert_eq!(a.load(), !(7 & 3)); + /// ``` + #[inline] + pub fn fetch_nand(&self, val: $t) -> $t { + atomic! { + $t, _a, + { + let a = unsafe { &*(self.as_ptr() as *const atomic::$atomic) }; + a.fetch_nand(val, Ordering::AcqRel) + }, + { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value = !(old & val); + old + } + } + } + + /// Applies bitwise "or" to the current value and returns the previous value. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + #[doc = $example] + /// + /// assert_eq!(a.fetch_or(16), 7); + /// assert_eq!(a.load(), 23); + /// ``` + #[inline] + pub fn fetch_or(&self, val: $t) -> $t { + atomic! { + $t, _a, + { + let a = unsafe { &*(self.as_ptr() as *const atomic::$atomic) }; + a.fetch_or(val, Ordering::AcqRel) + }, + { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value |= val; + old + } + } + } + + /// Applies bitwise "xor" to the current value and returns the previous value. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + #[doc = $example] + /// + /// assert_eq!(a.fetch_xor(2), 7); + /// assert_eq!(a.load(), 5); + /// ``` + #[inline] + pub fn fetch_xor(&self, val: $t) -> $t { + atomic! { + $t, _a, + { + let a = unsafe { &*(self.as_ptr() as *const atomic::$atomic) }; + a.fetch_xor(val, Ordering::AcqRel) + }, + { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value ^= val; + old + } + } + } + + /// Compares and sets the maximum of the current value and `val`, + /// and returns the previous value. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + #[doc = $example] + /// + /// assert_eq!(a.fetch_max(9), 7); + /// assert_eq!(a.load(), 9); + /// ``` + #[inline] + pub fn fetch_max(&self, val: $t) -> $t { + atomic! { + $t, _a, + { + let a = unsafe { &*(self.as_ptr() as *const atomic::$atomic) }; + a.fetch_max(val, Ordering::AcqRel) + }, + { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value = cmp::max(old, val); + old + } + } + } + + /// Compares and sets the minimum of the current value and `val`, + /// and returns the previous value. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + #[doc = $example] + /// + /// assert_eq!(a.fetch_min(2), 7); + /// assert_eq!(a.load(), 2); + /// ``` + #[inline] + pub fn fetch_min(&self, val: $t) -> $t { + atomic! { + $t, _a, + { + let a = unsafe { &*(self.as_ptr() as *const atomic::$atomic) }; + a.fetch_min(val, Ordering::AcqRel) + }, + { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value = cmp::min(old, val); + old + } + } + } + } + }; +} + +impl_arithmetic!(u8, AtomicU8, "let a = AtomicCell::new(7u8);"); +impl_arithmetic!(i8, AtomicI8, "let a = AtomicCell::new(7i8);"); +impl_arithmetic!(u16, AtomicU16, "let a = AtomicCell::new(7u16);"); +impl_arithmetic!(i16, AtomicI16, "let a = AtomicCell::new(7i16);"); + +impl_arithmetic!(u32, AtomicU32, "let a = AtomicCell::new(7u32);"); +impl_arithmetic!(i32, AtomicI32, "let a = AtomicCell::new(7i32);"); + +#[cfg(target_has_atomic = "64")] +impl_arithmetic!(u64, AtomicU64, "let a = AtomicCell::new(7u64);"); +#[cfg(target_has_atomic = "64")] +impl_arithmetic!(i64, AtomicI64, "let a = AtomicCell::new(7i64);"); +#[cfg(not(target_has_atomic = "64"))] +impl_arithmetic!(u64, fallback, "let a = AtomicCell::new(7u64);"); +#[cfg(not(target_has_atomic = "64"))] +impl_arithmetic!(i64, fallback, "let a = AtomicCell::new(7i64);"); + +// TODO: AtomicU128 is unstable +// impl_arithmetic!(u128, AtomicU128, "let a = AtomicCell::new(7u128);"); +// impl_arithmetic!(i128, AtomicI128, "let a = AtomicCell::new(7i128);"); +impl_arithmetic!(u128, fallback, "let a = AtomicCell::new(7u128);"); +impl_arithmetic!(i128, fallback, "let a = AtomicCell::new(7i128);"); + +impl_arithmetic!(usize, AtomicUsize, "let a = AtomicCell::new(7usize);"); +impl_arithmetic!(isize, AtomicIsize, "let a = AtomicCell::new(7isize);"); + +impl AtomicCell { + /// Applies logical "and" to the current value and returns the previous value. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + /// let a = AtomicCell::new(true); + /// + /// assert_eq!(a.fetch_and(true), true); + /// assert_eq!(a.load(), true); + /// + /// assert_eq!(a.fetch_and(false), true); + /// assert_eq!(a.load(), false); + /// ``` + #[inline] + pub fn fetch_and(&self, val: bool) -> bool { + atomic! { + bool, _a, + { + let a = unsafe { &*(self.as_ptr() as *const atomic::AtomicBool) }; + a.fetch_and(val, Ordering::AcqRel) + }, + { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value &= val; + old + } + } + } + + /// Applies logical "nand" to the current value and returns the previous value. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + /// let a = AtomicCell::new(true); + /// + /// assert_eq!(a.fetch_nand(false), true); + /// assert_eq!(a.load(), true); + /// + /// assert_eq!(a.fetch_nand(true), true); + /// assert_eq!(a.load(), false); + /// + /// assert_eq!(a.fetch_nand(false), false); + /// assert_eq!(a.load(), true); + /// ``` + #[inline] + pub fn fetch_nand(&self, val: bool) -> bool { + atomic! { + bool, _a, + { + let a = unsafe { &*(self.as_ptr() as *const atomic::AtomicBool) }; + a.fetch_nand(val, Ordering::AcqRel) + }, + { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value = !(old & val); + old + } + } + } + + /// Applies logical "or" to the current value and returns the previous value. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + /// let a = AtomicCell::new(false); + /// + /// assert_eq!(a.fetch_or(false), false); + /// assert_eq!(a.load(), false); + /// + /// assert_eq!(a.fetch_or(true), false); + /// assert_eq!(a.load(), true); + /// ``` + #[inline] + pub fn fetch_or(&self, val: bool) -> bool { + atomic! { + bool, _a, + { + let a = unsafe { &*(self.as_ptr() as *const atomic::AtomicBool) }; + a.fetch_or(val, Ordering::AcqRel) + }, + { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value |= val; + old + } + } + } + + /// Applies logical "xor" to the current value and returns the previous value. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + /// let a = AtomicCell::new(true); + /// + /// assert_eq!(a.fetch_xor(false), true); + /// assert_eq!(a.load(), true); + /// + /// assert_eq!(a.fetch_xor(true), true); + /// assert_eq!(a.load(), false); + /// ``` + #[inline] + pub fn fetch_xor(&self, val: bool) -> bool { + atomic! { + bool, _a, + { + let a = unsafe { &*(self.as_ptr() as *const atomic::AtomicBool) }; + a.fetch_xor(val, Ordering::AcqRel) + }, + { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value ^= val; + old + } + } + } +} + +impl Default for AtomicCell { + fn default() -> AtomicCell { + AtomicCell::new(T::default()) + } +} + +impl From for AtomicCell { + #[inline] + fn from(val: T) -> AtomicCell { + AtomicCell::new(val) + } +} + +impl fmt::Debug for AtomicCell { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("AtomicCell") + .field("value", &self.load()) + .finish() + } +} + +/// Returns `true` if values of type `A` can be transmuted into values of type `B`. +const fn can_transmute() -> bool { + // Sizes must be equal, but alignment of `A` must be greater or equal than that of `B`. + (mem::size_of::() == mem::size_of::()) & (mem::align_of::() >= mem::align_of::()) +} + +/// Returns a reference to the global lock associated with the `AtomicCell` at address `addr`. +/// +/// This function is used to protect atomic data which doesn't fit into any of the primitive atomic +/// types in `std::sync::atomic`. Operations on such atomics must therefore use a global lock. +/// +/// However, there is not only one global lock but an array of many locks, and one of them is +/// picked based on the given address. Having many locks reduces contention and improves +/// scalability. +#[inline] +#[must_use] +fn lock(addr: usize) -> &'static SeqLock { + // The number of locks is a prime number because we want to make sure `addr % LEN` gets + // dispersed across all locks. + // + // Note that addresses are always aligned to some power of 2, depending on type `T` in + // `AtomicCell`. If `LEN` was an even number, then `addr % LEN` would be an even number, + // too, which means only half of the locks would get utilized! + // + // It is also possible for addresses to accidentally get aligned to a number that is not a + // power of 2. Consider this example: + // + // ``` + // #[repr(C)] + // struct Foo { + // a: AtomicCell, + // b: u8, + // c: u8, + // } + // ``` + // + // Now, if we have a slice of type `&[Foo]`, it is possible that field `a` in all items gets + // stored at addresses that are multiples of 3. It'd be too bad if `LEN` was divisible by 3. + // In order to protect from such cases, we simply choose a large prime number for `LEN`. + const LEN: usize = 67; + const L: CachePadded = CachePadded::new(SeqLock::new()); + static LOCKS: [CachePadded; LEN] = [L; LEN]; + + // If the modulus is a constant number, the compiler will use crazy math to transform this into + // a sequence of cheap arithmetic operations rather than using the slow modulo instruction. + &LOCKS[addr % LEN] +} + +/// An atomic `()`. +/// +/// All operations are noops. +struct AtomicUnit; + +impl AtomicUnit { + #[inline] + fn load(&self, _order: Ordering) {} + + #[inline] + fn store(&self, _val: (), _order: Ordering) {} + + #[inline] + fn swap(&self, _val: (), _order: Ordering) {} + + #[inline] + fn compare_exchange_weak( + &self, + _current: (), + _new: (), + _success: Ordering, + _failure: Ordering, + ) -> Result<(), ()> { + Ok(()) + } +} + +/// Returns `true` if operations on `AtomicCell` are lock-free. +const fn atomic_is_lock_free() -> bool { + atomic! { T, _a, true, false } +} + +/// Atomically reads data from `src`. +/// +/// This operation uses the `Acquire` ordering. If possible, an atomic instructions is used, and a +/// global lock otherwise. +unsafe fn atomic_load(src: *mut T) -> T +where + T: Copy, +{ + atomic! { + T, a, + { + a = &*(src as *const _ as *const _); + mem::transmute_copy(&a.load(Ordering::Acquire)) + }, + { + let lock = lock(src as usize); + + // Try doing an optimistic read first. + if let Some(stamp) = lock.optimistic_read() { + // We need a volatile read here because other threads might concurrently modify the + // value. In theory, data races are *always* UB, even if we use volatile reads and + // discard the data when a data race is detected. The proper solution would be to + // do atomic reads and atomic writes, but we can't atomically read and write all + // kinds of data since `AtomicU8` is not available on stable Rust yet. + // Load as `MaybeUninit` because we may load a value that is not valid as `T`. + let val = ptr::read_volatile(src.cast::>()); + + if lock.validate_read(stamp) { + return val.assume_init(); + } + } + + // Grab a regular write lock so that writers don't starve this load. + let guard = lock.write(); + let val = ptr::read(src); + // The value hasn't been changed. Drop the guard without incrementing the stamp. + guard.abort(); + val + } + } +} + +/// Atomically writes `val` to `dst`. +/// +/// This operation uses the `Release` ordering. If possible, an atomic instructions is used, and a +/// global lock otherwise. +unsafe fn atomic_store(dst: *mut T, val: T) { + atomic! { + T, a, + { + a = &*(dst as *const _ as *const _); + a.store(mem::transmute_copy(&val), Ordering::Release); + mem::forget(val); + }, + { + let _guard = lock(dst as usize).write(); + ptr::write(dst, val); + } + } +} + +/// Atomically swaps data at `dst` with `val`. +/// +/// This operation uses the `AcqRel` ordering. If possible, an atomic instructions is used, and a +/// global lock otherwise. +unsafe fn atomic_swap(dst: *mut T, val: T) -> T { + atomic! { + T, a, + { + a = &*(dst as *const _ as *const _); + let res = mem::transmute_copy(&a.swap(mem::transmute_copy(&val), Ordering::AcqRel)); + mem::forget(val); + res + }, + { + let _guard = lock(dst as usize).write(); + ptr::replace(dst, val) + } + } +} + +/// Atomically compares data at `dst` to `current` and, if equal byte-for-byte, exchanges data at +/// `dst` with `new`. +/// +/// Returns the old value on success, or the current value at `dst` on failure. +/// +/// This operation uses the `AcqRel` ordering. If possible, an atomic instructions is used, and a +/// global lock otherwise. +#[allow(clippy::let_unit_value)] +unsafe fn atomic_compare_exchange_weak(dst: *mut T, mut current: T, new: T) -> Result +where + T: Copy + Eq, +{ + atomic! { + T, a, + { + a = &*(dst as *const _ as *const _); + let mut current_raw = mem::transmute_copy(¤t); + let new_raw = mem::transmute_copy(&new); + + loop { + match a.compare_exchange_weak( + current_raw, + new_raw, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => break Ok(current), + Err(previous_raw) => { + let previous = mem::transmute_copy(&previous_raw); + + if !T::eq(&previous, ¤t) { + break Err(previous); + } + + // The compare-exchange operation has failed and didn't store `new`. The + // failure is either spurious, or `previous` was semantically equal to + // `current` but not byte-equal. Let's retry with `previous` as the new + // `current`. + current = previous; + current_raw = previous_raw; + } + } + } + }, + { + let guard = lock(dst as usize).write(); + + if T::eq(&*dst, ¤t) { + Ok(ptr::replace(dst, new)) + } else { + let val = ptr::read(dst); + // The value hasn't been changed. Drop the guard without incrementing the stamp. + guard.abort(); + Err(val) + } + } + } +} diff --git a/anneal/v2/vendor/crossbeam-utils/src/atomic/consume.rs b/anneal/v2/vendor/crossbeam-utils/src/atomic/consume.rs new file mode 100644 index 0000000000..ff8e316b2c --- /dev/null +++ b/anneal/v2/vendor/crossbeam-utils/src/atomic/consume.rs @@ -0,0 +1,111 @@ +#[cfg(not(crossbeam_no_atomic))] +use core::sync::atomic::Ordering; + +/// Trait which allows reading from primitive atomic types with "consume" ordering. +pub trait AtomicConsume { + /// Type returned by `load_consume`. + type Val; + + /// Loads a value from the atomic using a "consume" memory ordering. + /// + /// This is similar to the "acquire" ordering, except that an ordering is + /// only guaranteed with operations that "depend on" the result of the load. + /// However consume loads are usually much faster than acquire loads on + /// architectures with a weak memory model since they don't require memory + /// fence instructions. + /// + /// The exact definition of "depend on" is a bit vague, but it works as you + /// would expect in practice since a lot of software, especially the Linux + /// kernel, rely on this behavior. + /// + /// This is currently only implemented on ARM and AArch64, where a fence + /// can be avoided. On other architectures this will fall back to a simple + /// `load(Ordering::Acquire)`. + fn load_consume(&self) -> Self::Val; +} + +#[cfg(not(crossbeam_no_atomic))] +// Miri and Loom don't support "consume" ordering and ThreadSanitizer doesn't treat +// load(Relaxed) + compiler_fence(Acquire) as "consume" load. +// LLVM generates machine code equivalent to fence(Acquire) in compiler_fence(Acquire) +// on PowerPC, MIPS, etc. (https://godbolt.org/z/hffvjvW7h), so for now the fence +// can be actually avoided here only on ARM and AArch64. See also +// https://github.com/rust-lang/rust/issues/62256. +#[cfg(all( + any(target_arch = "arm", target_arch = "aarch64"), + not(any(miri, crossbeam_loom, crossbeam_sanitize_thread)), +))] +macro_rules! impl_consume { + () => { + #[inline] + fn load_consume(&self) -> Self::Val { + use crate::primitive::sync::atomic::compiler_fence; + let result = self.load(Ordering::Relaxed); + compiler_fence(Ordering::Acquire); + result + } + }; +} + +#[cfg(not(crossbeam_no_atomic))] +#[cfg(not(all( + any(target_arch = "arm", target_arch = "aarch64"), + not(any(miri, crossbeam_loom, crossbeam_sanitize_thread)), +)))] +macro_rules! impl_consume { + () => { + #[inline] + fn load_consume(&self) -> Self::Val { + self.load(Ordering::Acquire) + } + }; +} + +macro_rules! impl_atomic { + ($atomic:ident, $val:ty) => { + #[cfg(not(crossbeam_no_atomic))] + impl AtomicConsume for core::sync::atomic::$atomic { + type Val = $val; + impl_consume!(); + } + #[cfg(crossbeam_loom)] + impl AtomicConsume for loom::sync::atomic::$atomic { + type Val = $val; + impl_consume!(); + } + }; +} + +impl_atomic!(AtomicBool, bool); +impl_atomic!(AtomicUsize, usize); +impl_atomic!(AtomicIsize, isize); +impl_atomic!(AtomicU8, u8); +impl_atomic!(AtomicI8, i8); +impl_atomic!(AtomicU16, u16); +impl_atomic!(AtomicI16, i16); +#[cfg(any(target_has_atomic = "32", not(target_pointer_width = "16")))] +impl_atomic!(AtomicU32, u32); +#[cfg(any(target_has_atomic = "32", not(target_pointer_width = "16")))] +impl_atomic!(AtomicI32, i32); +#[cfg(any( + target_has_atomic = "64", + not(any(target_pointer_width = "16", target_pointer_width = "32")), +))] +impl_atomic!(AtomicU64, u64); +#[cfg(any( + target_has_atomic = "64", + not(any(target_pointer_width = "16", target_pointer_width = "32")), +))] +impl_atomic!(AtomicI64, i64); + +#[cfg(not(crossbeam_no_atomic))] +impl AtomicConsume for core::sync::atomic::AtomicPtr { + type Val = *mut T; + impl_consume!(); +} + +#[cfg(crossbeam_loom)] +impl AtomicConsume for loom::sync::atomic::AtomicPtr { + type Val = *mut T; + impl_consume!(); +} diff --git a/anneal/v2/vendor/crossbeam-utils/src/atomic/mod.rs b/anneal/v2/vendor/crossbeam-utils/src/atomic/mod.rs new file mode 100644 index 0000000000..8662ded564 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-utils/src/atomic/mod.rs @@ -0,0 +1,32 @@ +//! Atomic types. +//! +//! * [`AtomicCell`], a thread-safe mutable memory location. +//! * [`AtomicConsume`], for reading from primitive atomic types with "consume" ordering. + +#[cfg(target_has_atomic = "ptr")] +#[cfg(not(crossbeam_loom))] +// Use "wide" sequence lock if the pointer width <= 32 for preventing its counter against wrap +// around. +// +// In narrow architectures (pointer width <= 16), the counter is still <= 32-bit and may be +// vulnerable to wrap around. But it's mostly okay, since in such a primitive hardware, the +// counter will not be increased that fast. +// Note that Rust (and C99) pointers must be at least 16-bit (i.e., 8-bit targets are impossible): https://github.com/rust-lang/rust/pull/49305 +#[cfg_attr( + any(target_pointer_width = "16", target_pointer_width = "32"), + path = "seq_lock_wide.rs" +)] +mod seq_lock; + +#[cfg(target_has_atomic = "ptr")] +// We cannot provide AtomicCell under cfg(crossbeam_loom) because loom's atomic +// types have a different in-memory representation than the underlying type. +// TODO: The latest loom supports fences, so fallback using seqlock may be available. +#[cfg(not(crossbeam_loom))] +mod atomic_cell; +#[cfg(target_has_atomic = "ptr")] +#[cfg(not(crossbeam_loom))] +pub use atomic_cell::AtomicCell; + +mod consume; +pub use consume::AtomicConsume; diff --git a/anneal/v2/vendor/crossbeam-utils/src/atomic/seq_lock.rs b/anneal/v2/vendor/crossbeam-utils/src/atomic/seq_lock.rs new file mode 100644 index 0000000000..ff8defd26d --- /dev/null +++ b/anneal/v2/vendor/crossbeam-utils/src/atomic/seq_lock.rs @@ -0,0 +1,112 @@ +use core::mem; +use core::sync::atomic::{self, AtomicUsize, Ordering}; + +use crate::Backoff; + +/// A simple stamped lock. +pub(crate) struct SeqLock { + /// The current state of the lock. + /// + /// All bits except the least significant one hold the current stamp. When locked, the state + /// equals 1 and doesn't contain a valid stamp. + state: AtomicUsize, +} + +impl SeqLock { + pub(crate) const fn new() -> Self { + Self { + state: AtomicUsize::new(0), + } + } + + /// If not locked, returns the current stamp. + /// + /// This method should be called before optimistic reads. + #[inline] + pub(crate) fn optimistic_read(&self) -> Option { + let state = self.state.load(Ordering::Acquire); + if state == 1 { + None + } else { + Some(state) + } + } + + /// Returns `true` if the current stamp is equal to `stamp`. + /// + /// This method should be called after optimistic reads to check whether they are valid. The + /// argument `stamp` should correspond to the one returned by method `optimistic_read`. + #[inline] + pub(crate) fn validate_read(&self, stamp: usize) -> bool { + atomic::fence(Ordering::Acquire); + self.state.load(Ordering::Relaxed) == stamp + } + + /// Grabs the lock for writing. + #[inline] + pub(crate) fn write(&'static self) -> SeqLockWriteGuard { + let backoff = Backoff::new(); + loop { + let previous = self.state.swap(1, Ordering::Acquire); + + if previous != 1 { + atomic::fence(Ordering::Release); + + return SeqLockWriteGuard { + lock: self, + state: previous, + }; + } + + backoff.snooze(); + } + } +} + +/// An RAII guard that releases the lock and increments the stamp when dropped. +pub(crate) struct SeqLockWriteGuard { + /// The parent lock. + lock: &'static SeqLock, + + /// The stamp before locking. + state: usize, +} + +impl SeqLockWriteGuard { + /// Releases the lock without incrementing the stamp. + #[inline] + pub(crate) fn abort(self) { + self.lock.state.store(self.state, Ordering::Release); + + // We specifically don't want to call drop(), since that's + // what increments the stamp. + mem::forget(self); + } +} + +impl Drop for SeqLockWriteGuard { + #[inline] + fn drop(&mut self) { + // Release the lock and increment the stamp. + self.lock + .state + .store(self.state.wrapping_add(2), Ordering::Release); + } +} + +#[cfg(test)] +mod tests { + use super::SeqLock; + + #[test] + fn test_abort() { + static LK: SeqLock = SeqLock::new(); + let before = LK.optimistic_read().unwrap(); + { + let guard = LK.write(); + guard.abort(); + } + let after = LK.optimistic_read().unwrap(); + assert_eq!(before, after, "aborted write does not update the stamp"); + } +} diff --git a/anneal/v2/vendor/crossbeam-utils/src/atomic/seq_lock_wide.rs b/anneal/v2/vendor/crossbeam-utils/src/atomic/seq_lock_wide.rs new file mode 100644 index 0000000000..ef5d94a454 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-utils/src/atomic/seq_lock_wide.rs @@ -0,0 +1,155 @@ +use core::mem; +use core::sync::atomic::{self, AtomicUsize, Ordering}; + +use crate::Backoff; + +/// A simple stamped lock. +/// +/// The state is represented as two `AtomicUsize`: `state_hi` for high bits and `state_lo` for low +/// bits. +pub(crate) struct SeqLock { + /// The high bits of the current state of the lock. + state_hi: AtomicUsize, + + /// The low bits of the current state of the lock. + /// + /// All bits except the least significant one hold the current stamp. When locked, the state_lo + /// equals 1 and doesn't contain a valid stamp. + state_lo: AtomicUsize, +} + +impl SeqLock { + pub(crate) const fn new() -> Self { + Self { + state_hi: AtomicUsize::new(0), + state_lo: AtomicUsize::new(0), + } + } + + /// If not locked, returns the current stamp. + /// + /// This method should be called before optimistic reads. + #[inline] + pub(crate) fn optimistic_read(&self) -> Option<(usize, usize)> { + // The acquire loads from `state_hi` and `state_lo` synchronize with the release stores in + // `SeqLockWriteGuard::drop`. + // + // As a consequence, we can make sure that (1) all writes within the era of `state_hi - 1` + // happens before now; and therefore, (2) if `state_lo` is even, all writes within the + // critical section of (`state_hi`, `state_lo`) happens before now. + let state_hi = self.state_hi.load(Ordering::Acquire); + let state_lo = self.state_lo.load(Ordering::Acquire); + if state_lo == 1 { + None + } else { + Some((state_hi, state_lo)) + } + } + + /// Returns `true` if the current stamp is equal to `stamp`. + /// + /// This method should be called after optimistic reads to check whether they are valid. The + /// argument `stamp` should correspond to the one returned by method `optimistic_read`. + #[inline] + pub(crate) fn validate_read(&self, stamp: (usize, usize)) -> bool { + // Thanks to the fence, if we're noticing any modification to the data at the critical + // section of `(a, b)`, then the critical section's write of 1 to state_lo should be + // visible. + atomic::fence(Ordering::Acquire); + + // So if `state_lo` coincides with `stamp.1`, then either (1) we're noticing no modification + // to the data after the critical section of `(stamp.0, stamp.1)`, or (2) `state_lo` wrapped + // around. + // + // If (2) is the case, the acquire ordering ensures we see the new value of `state_hi`. + let state_lo = self.state_lo.load(Ordering::Acquire); + + // If (2) is the case and `state_hi` coincides with `stamp.0`, then `state_hi` also wrapped + // around, which we give up to correctly validate the read. + let state_hi = self.state_hi.load(Ordering::Relaxed); + + // Except for the case that both `state_hi` and `state_lo` wrapped around, the following + // condition implies that we're noticing no modification to the data after the critical + // section of `(stamp.0, stamp.1)`. + (state_hi, state_lo) == stamp + } + + /// Grabs the lock for writing. + #[inline] + pub(crate) fn write(&'static self) -> SeqLockWriteGuard { + let backoff = Backoff::new(); + loop { + let previous = self.state_lo.swap(1, Ordering::Acquire); + + if previous != 1 { + // To synchronize with the acquire fence in `validate_read` via any modification to + // the data at the critical section of `(state_hi, previous)`. + atomic::fence(Ordering::Release); + + return SeqLockWriteGuard { + lock: self, + state_lo: previous, + }; + } + + backoff.snooze(); + } + } +} + +/// An RAII guard that releases the lock and increments the stamp when dropped. +pub(crate) struct SeqLockWriteGuard { + /// The parent lock. + lock: &'static SeqLock, + + /// The stamp before locking. + state_lo: usize, +} + +impl SeqLockWriteGuard { + /// Releases the lock without incrementing the stamp. + #[inline] + pub(crate) fn abort(self) { + self.lock.state_lo.store(self.state_lo, Ordering::Release); + mem::forget(self); + } +} + +impl Drop for SeqLockWriteGuard { + #[inline] + fn drop(&mut self) { + let state_lo = self.state_lo.wrapping_add(2); + + // Increase the high bits if the low bits wrap around. + // + // Release ordering for synchronizing with `optimistic_read`. + if state_lo == 0 { + let state_hi = self.lock.state_hi.load(Ordering::Relaxed); + self.lock + .state_hi + .store(state_hi.wrapping_add(1), Ordering::Release); + } + + // Release the lock and increment the stamp. + // + // Release ordering for synchronizing with `optimistic_read`. + self.lock.state_lo.store(state_lo, Ordering::Release); + } +} + +#[cfg(test)] +mod tests { + use super::SeqLock; + + #[test] + fn test_abort() { + static LK: SeqLock = SeqLock::new(); + let before = LK.optimistic_read().unwrap(); + { + let guard = LK.write(); + guard.abort(); + } + let after = LK.optimistic_read().unwrap(); + assert_eq!(before, after, "aborted write does not update the stamp"); + } +} diff --git a/anneal/v2/vendor/crossbeam-utils/src/backoff.rs b/anneal/v2/vendor/crossbeam-utils/src/backoff.rs new file mode 100644 index 0000000000..7a505ed614 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-utils/src/backoff.rs @@ -0,0 +1,287 @@ +use crate::primitive::hint; +use core::cell::Cell; +use core::fmt; + +const SPIN_LIMIT: u32 = 6; +const YIELD_LIMIT: u32 = 10; + +/// Performs exponential backoff in spin loops. +/// +/// Backing off in spin loops reduces contention and improves overall performance. +/// +/// This primitive can execute *YIELD* and *PAUSE* instructions, yield the current thread to the OS +/// scheduler, and tell when is a good time to block the thread using a different synchronization +/// mechanism. Each step of the back off procedure takes roughly twice as long as the previous +/// step. +/// +/// # Examples +/// +/// Backing off in a lock-free loop: +/// +/// ``` +/// use crossbeam_utils::Backoff; +/// use std::sync::atomic::AtomicUsize; +/// use std::sync::atomic::Ordering::SeqCst; +/// +/// fn fetch_mul(a: &AtomicUsize, b: usize) -> usize { +/// let backoff = Backoff::new(); +/// loop { +/// let val = a.load(SeqCst); +/// if a.compare_exchange(val, val.wrapping_mul(b), SeqCst, SeqCst).is_ok() { +/// return val; +/// } +/// backoff.spin(); +/// } +/// } +/// ``` +/// +/// Waiting for an [`AtomicBool`] to become `true`: +/// +/// ``` +/// use crossbeam_utils::Backoff; +/// use std::sync::atomic::AtomicBool; +/// use std::sync::atomic::Ordering::SeqCst; +/// +/// fn spin_wait(ready: &AtomicBool) { +/// let backoff = Backoff::new(); +/// while !ready.load(SeqCst) { +/// backoff.snooze(); +/// } +/// } +/// ``` +/// +/// Waiting for an [`AtomicBool`] to become `true` and parking the thread after a long wait. +/// Note that whoever sets the atomic variable to `true` must notify the parked thread by calling +/// [`unpark()`]: +/// +/// ``` +/// use crossbeam_utils::Backoff; +/// use std::sync::atomic::AtomicBool; +/// use std::sync::atomic::Ordering::SeqCst; +/// use std::thread; +/// +/// fn blocking_wait(ready: &AtomicBool) { +/// let backoff = Backoff::new(); +/// while !ready.load(SeqCst) { +/// if backoff.is_completed() { +/// thread::park(); +/// } else { +/// backoff.snooze(); +/// } +/// } +/// } +/// ``` +/// +/// [`is_completed`]: Backoff::is_completed +/// [`std::thread::park()`]: std::thread::park +/// [`Condvar`]: std::sync::Condvar +/// [`AtomicBool`]: std::sync::atomic::AtomicBool +/// [`unpark()`]: std::thread::Thread::unpark +pub struct Backoff { + step: Cell, +} + +impl Backoff { + /// Creates a new `Backoff`. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::Backoff; + /// + /// let backoff = Backoff::new(); + /// ``` + #[inline] + pub fn new() -> Self { + Backoff { step: Cell::new(0) } + } + + /// Resets the `Backoff`. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::Backoff; + /// + /// let backoff = Backoff::new(); + /// backoff.reset(); + /// ``` + #[inline] + pub fn reset(&self) { + self.step.set(0); + } + + /// Backs off in a lock-free loop. + /// + /// This method should be used when we need to retry an operation because another thread made + /// progress. + /// + /// The processor may yield using the *YIELD* or *PAUSE* instruction. + /// + /// # Examples + /// + /// Backing off in a lock-free loop: + /// + /// ``` + /// use crossbeam_utils::Backoff; + /// use std::sync::atomic::AtomicUsize; + /// use std::sync::atomic::Ordering::SeqCst; + /// + /// fn fetch_mul(a: &AtomicUsize, b: usize) -> usize { + /// let backoff = Backoff::new(); + /// loop { + /// let val = a.load(SeqCst); + /// if a.compare_exchange(val, val.wrapping_mul(b), SeqCst, SeqCst).is_ok() { + /// return val; + /// } + /// backoff.spin(); + /// } + /// } + /// + /// let a = AtomicUsize::new(7); + /// assert_eq!(fetch_mul(&a, 8), 7); + /// assert_eq!(a.load(SeqCst), 56); + /// ``` + #[inline] + pub fn spin(&self) { + for _ in 0..1 << self.step.get().min(SPIN_LIMIT) { + hint::spin_loop(); + } + + if self.step.get() <= SPIN_LIMIT { + self.step.set(self.step.get() + 1); + } + } + + /// Backs off in a blocking loop. + /// + /// This method should be used when we need to wait for another thread to make progress. + /// + /// The processor may yield using the *YIELD* or *PAUSE* instruction and the current thread + /// may yield by giving up a timeslice to the OS scheduler. + /// + /// In `#[no_std]` environments, this method is equivalent to [`spin`]. + /// + /// If possible, use [`is_completed`] to check when it is advised to stop using backoff and + /// block the current thread using a different synchronization mechanism instead. + /// + /// [`spin`]: Backoff::spin + /// [`is_completed`]: Backoff::is_completed + /// + /// # Examples + /// + /// Waiting for an [`AtomicBool`] to become `true`: + /// + /// ``` + /// use crossbeam_utils::Backoff; + /// use std::sync::Arc; + /// use std::sync::atomic::AtomicBool; + /// use std::sync::atomic::Ordering::SeqCst; + /// use std::thread; + /// use std::time::Duration; + /// + /// fn spin_wait(ready: &AtomicBool) { + /// let backoff = Backoff::new(); + /// while !ready.load(SeqCst) { + /// backoff.snooze(); + /// } + /// } + /// + /// let ready = Arc::new(AtomicBool::new(false)); + /// let ready2 = ready.clone(); + /// + /// thread::spawn(move || { + /// thread::sleep(Duration::from_millis(100)); + /// ready2.store(true, SeqCst); + /// }); + /// + /// assert_eq!(ready.load(SeqCst), false); + /// spin_wait(&ready); + /// assert_eq!(ready.load(SeqCst), true); + /// # std::thread::sleep(std::time::Duration::from_millis(500)); // wait for background threads closed: https://github.com/rust-lang/miri/issues/1371 + /// ``` + /// + /// [`AtomicBool`]: std::sync::atomic::AtomicBool + #[inline] + pub fn snooze(&self) { + if self.step.get() <= SPIN_LIMIT { + for _ in 0..1 << self.step.get() { + hint::spin_loop(); + } + } else { + #[cfg(not(feature = "std"))] + for _ in 0..1 << self.step.get() { + hint::spin_loop(); + } + + #[cfg(feature = "std")] + ::std::thread::yield_now(); + } + + if self.step.get() <= YIELD_LIMIT { + self.step.set(self.step.get() + 1); + } + } + + /// Returns `true` if exponential backoff has completed and blocking the thread is advised. + /// + /// # Examples + /// + /// Waiting for an [`AtomicBool`] to become `true` and parking the thread after a long wait: + /// + /// ``` + /// use crossbeam_utils::Backoff; + /// use std::sync::Arc; + /// use std::sync::atomic::AtomicBool; + /// use std::sync::atomic::Ordering::SeqCst; + /// use std::thread; + /// use std::time::Duration; + /// + /// fn blocking_wait(ready: &AtomicBool) { + /// let backoff = Backoff::new(); + /// while !ready.load(SeqCst) { + /// if backoff.is_completed() { + /// thread::park(); + /// } else { + /// backoff.snooze(); + /// } + /// } + /// } + /// + /// let ready = Arc::new(AtomicBool::new(false)); + /// let ready2 = ready.clone(); + /// let waiter = thread::current(); + /// + /// thread::spawn(move || { + /// thread::sleep(Duration::from_millis(100)); + /// ready2.store(true, SeqCst); + /// waiter.unpark(); + /// }); + /// + /// assert_eq!(ready.load(SeqCst), false); + /// blocking_wait(&ready); + /// assert_eq!(ready.load(SeqCst), true); + /// # std::thread::sleep(std::time::Duration::from_millis(500)); // wait for background threads closed: https://github.com/rust-lang/miri/issues/1371 + /// ``` + /// + /// [`AtomicBool`]: std::sync::atomic::AtomicBool + #[inline] + pub fn is_completed(&self) -> bool { + self.step.get() > YIELD_LIMIT + } +} + +impl fmt::Debug for Backoff { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Backoff") + .field("step", &self.step) + .field("is_completed", &self.is_completed()) + .finish() + } +} + +impl Default for Backoff { + fn default() -> Backoff { + Backoff::new() + } +} diff --git a/anneal/v2/vendor/crossbeam-utils/src/cache_padded.rs b/anneal/v2/vendor/crossbeam-utils/src/cache_padded.rs new file mode 100644 index 0000000000..6c930c6f3f --- /dev/null +++ b/anneal/v2/vendor/crossbeam-utils/src/cache_padded.rs @@ -0,0 +1,217 @@ +use core::fmt; +use core::ops::{Deref, DerefMut}; + +/// Pads and aligns a value to the length of a cache line. +/// +/// In concurrent programming, sometimes it is desirable to make sure commonly accessed pieces of +/// data are not placed into the same cache line. Updating an atomic value invalidates the whole +/// cache line it belongs to, which makes the next access to the same cache line slower for other +/// CPU cores. Use `CachePadded` to ensure updating one piece of data doesn't invalidate other +/// cached data. +/// +/// # Size and alignment +/// +/// Cache lines are assumed to be N bytes long, depending on the architecture: +/// +/// * On x86-64, aarch64, and powerpc64, N = 128. +/// * On arm, mips, mips64, sparc, and hexagon, N = 32. +/// * On m68k, N = 16. +/// * On s390x, N = 256. +/// * On all others, N = 64. +/// +/// Note that N is just a reasonable guess and is not guaranteed to match the actual cache line +/// length of the machine the program is running on. On modern Intel architectures, spatial +/// prefetcher is pulling pairs of 64-byte cache lines at a time, so we pessimistically assume that +/// cache lines are 128 bytes long. +/// +/// The size of `CachePadded` is the smallest multiple of N bytes large enough to accommodate +/// a value of type `T`. +/// +/// The alignment of `CachePadded` is the maximum of N bytes and the alignment of `T`. +/// +/// # Examples +/// +/// Alignment and padding: +/// +/// ``` +/// use crossbeam_utils::CachePadded; +/// +/// let array = [CachePadded::new(1i8), CachePadded::new(2i8)]; +/// let addr1 = &*array[0] as *const i8 as usize; +/// let addr2 = &*array[1] as *const i8 as usize; +/// +/// assert!(addr2 - addr1 >= 32); +/// assert_eq!(addr1 % 32, 0); +/// assert_eq!(addr2 % 32, 0); +/// ``` +/// +/// When building a concurrent queue with a head and a tail index, it is wise to place them in +/// different cache lines so that concurrent threads pushing and popping elements don't invalidate +/// each other's cache lines: +/// +/// ``` +/// use crossbeam_utils::CachePadded; +/// use std::sync::atomic::AtomicUsize; +/// +/// struct Queue { +/// head: CachePadded, +/// tail: CachePadded, +/// buffer: *mut T, +/// } +/// ``` +#[derive(Clone, Copy, Default, Hash, PartialEq, Eq)] +// Starting from Intel's Sandy Bridge, spatial prefetcher is now pulling pairs of 64-byte cache +// lines at a time, so we have to align to 128 bytes rather than 64. +// +// Sources: +// - https://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-optimization-manual.pdf +// - https://github.com/facebook/folly/blob/1b5288e6eea6df074758f877c849b6e73bbb9fbb/folly/lang/Align.h#L107 +// +// aarch64/arm64ec's big.LITTLE architecture has asymmetric cores and "big" cores have 128-byte cache line size. +// +// Sources: +// - https://www.mono-project.com/news/2016/09/12/arm64-icache/ +// +// powerpc64 has 128-byte cache line size. +// +// Sources: +// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_ppc64x.go#L9 +// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/powerpc/include/asm/cache.h#L26 +#[cfg_attr( + any( + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "arm64ec", + target_arch = "powerpc64", + ), + repr(align(128)) +)] +// arm, mips, mips64, sparc, and hexagon have 32-byte cache line size. +// +// Sources: +// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_arm.go#L7 +// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mips.go#L7 +// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mipsle.go#L7 +// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mips64x.go#L9 +// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/sparc/include/asm/cache.h#L17 +// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/hexagon/include/asm/cache.h#L12 +#[cfg_attr( + any( + target_arch = "arm", + target_arch = "mips", + target_arch = "mips32r6", + target_arch = "mips64", + target_arch = "mips64r6", + target_arch = "sparc", + target_arch = "hexagon", + ), + repr(align(32)) +)] +// m68k has 16-byte cache line size. +// +// Sources: +// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/m68k/include/asm/cache.h#L9 +#[cfg_attr(target_arch = "m68k", repr(align(16)))] +// s390x has 256-byte cache line size. +// +// Sources: +// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_s390x.go#L7 +// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/s390/include/asm/cache.h#L13 +#[cfg_attr(target_arch = "s390x", repr(align(256)))] +// x86, wasm, riscv, and sparc64 have 64-byte cache line size. +// +// Sources: +// - https://github.com/golang/go/blob/dda2991c2ea0c5914714469c4defc2562a907230/src/internal/cpu/cpu_x86.go#L9 +// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_wasm.go#L7 +// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/riscv/include/asm/cache.h#L10 +// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/sparc/include/asm/cache.h#L19 +// +// All others are assumed to have 64-byte cache line size. +#[cfg_attr( + not(any( + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "arm64ec", + target_arch = "powerpc64", + target_arch = "arm", + target_arch = "mips", + target_arch = "mips32r6", + target_arch = "mips64", + target_arch = "mips64r6", + target_arch = "sparc", + target_arch = "hexagon", + target_arch = "m68k", + target_arch = "s390x", + )), + repr(align(64)) +)] +pub struct CachePadded { + value: T, +} + +unsafe impl Send for CachePadded {} +unsafe impl Sync for CachePadded {} + +impl CachePadded { + /// Pads and aligns a value to the length of a cache line. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::CachePadded; + /// + /// let padded_value = CachePadded::new(1); + /// ``` + pub const fn new(t: T) -> CachePadded { + CachePadded:: { value: t } + } + + /// Returns the inner value. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::CachePadded; + /// + /// let padded_value = CachePadded::new(7); + /// let value = padded_value.into_inner(); + /// assert_eq!(value, 7); + /// ``` + pub fn into_inner(self) -> T { + self.value + } +} + +impl Deref for CachePadded { + type Target = T; + + fn deref(&self) -> &T { + &self.value + } +} + +impl DerefMut for CachePadded { + fn deref_mut(&mut self) -> &mut T { + &mut self.value + } +} + +impl fmt::Debug for CachePadded { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("CachePadded") + .field("value", &self.value) + .finish() + } +} + +impl From for CachePadded { + fn from(t: T) -> Self { + CachePadded::new(t) + } +} + +impl fmt::Display for CachePadded { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.value, f) + } +} diff --git a/anneal/v2/vendor/crossbeam-utils/src/lib.rs b/anneal/v2/vendor/crossbeam-utils/src/lib.rs new file mode 100644 index 0000000000..6f124f9700 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-utils/src/lib.rs @@ -0,0 +1,110 @@ +//! Miscellaneous tools for concurrent programming. +//! +//! ## Atomics +//! +//! * [`AtomicCell`], a thread-safe mutable memory location. +//! * [`AtomicConsume`], for reading from primitive atomic types with "consume" ordering. +//! +//! ## Thread synchronization +//! +//! * [`Parker`], a thread parking primitive. +//! * [`ShardedLock`], a sharded reader-writer lock with fast concurrent reads. +//! * [`WaitGroup`], for synchronizing the beginning or end of some computation. +//! +//! ## Utilities +//! +//! * [`Backoff`], for exponential backoff in spin loops. +//! * [`CachePadded`], for padding and aligning a value to the length of a cache line. +//! * [`scope`], for spawning threads that borrow local variables from the stack. +//! +//! [`AtomicCell`]: atomic::AtomicCell +//! [`AtomicConsume`]: atomic::AtomicConsume +//! [`Parker`]: sync::Parker +//! [`ShardedLock`]: sync::ShardedLock +//! [`WaitGroup`]: sync::WaitGroup +//! [`scope`]: thread::scope + +#![no_std] +#![doc(test( + no_crate_inject, + attr( + deny(warnings, rust_2018_idioms), + allow(dead_code, unused_assignments, unused_variables) + ) +))] +#![warn( + missing_docs, + missing_debug_implementations, + rust_2018_idioms, + unreachable_pub +)] + +#[cfg(feature = "std")] +extern crate std; + +#[cfg(crossbeam_loom)] +#[allow(unused_imports)] +mod primitive { + pub(crate) mod hint { + pub(crate) use loom::hint::spin_loop; + } + pub(crate) mod sync { + pub(crate) mod atomic { + pub(crate) use loom::sync::atomic::{ + AtomicBool, AtomicI16, AtomicI32, AtomicI64, AtomicI8, AtomicIsize, AtomicU16, + AtomicU32, AtomicU64, AtomicU8, AtomicUsize, Ordering, + }; + + // FIXME: loom does not support compiler_fence at the moment. + // https://github.com/tokio-rs/loom/issues/117 + // we use fence as a stand-in for compiler_fence for the time being. + // this may miss some races since fence is stronger than compiler_fence, + // but it's the best we can do for the time being. + pub(crate) use loom::sync::atomic::fence as compiler_fence; + } + pub(crate) use loom::sync::{Arc, Condvar, Mutex}; + } +} +#[cfg(not(crossbeam_loom))] +#[allow(unused_imports)] +mod primitive { + pub(crate) mod hint { + pub(crate) use core::hint::spin_loop; + } + pub(crate) mod sync { + pub(crate) mod atomic { + pub(crate) use core::sync::atomic::{compiler_fence, Ordering}; + #[cfg(not(crossbeam_no_atomic))] + pub(crate) use core::sync::atomic::{ + AtomicBool, AtomicI16, AtomicI8, AtomicIsize, AtomicU16, AtomicU8, AtomicUsize, + }; + #[cfg(not(crossbeam_no_atomic))] + #[cfg(any(target_has_atomic = "32", not(target_pointer_width = "16")))] + pub(crate) use core::sync::atomic::{AtomicI32, AtomicU32}; + #[cfg(not(crossbeam_no_atomic))] + #[cfg(any( + target_has_atomic = "64", + not(any(target_pointer_width = "16", target_pointer_width = "32")), + ))] + pub(crate) use core::sync::atomic::{AtomicI64, AtomicU64}; + } + + #[cfg(feature = "std")] + pub(crate) use std::sync::{Arc, Condvar, Mutex}; + } +} + +pub mod atomic; + +mod cache_padded; +pub use crate::cache_padded::CachePadded; + +mod backoff; +pub use crate::backoff::Backoff; + +#[cfg(feature = "std")] +pub mod sync; + +#[cfg(feature = "std")] +#[cfg(not(crossbeam_loom))] +pub mod thread; diff --git a/anneal/v2/vendor/crossbeam-utils/src/sync/mod.rs b/anneal/v2/vendor/crossbeam-utils/src/sync/mod.rs new file mode 100644 index 0000000000..f9eec71fb3 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-utils/src/sync/mod.rs @@ -0,0 +1,17 @@ +//! Thread synchronization primitives. +//! +//! * [`Parker`], a thread parking primitive. +//! * [`ShardedLock`], a sharded reader-writer lock with fast concurrent reads. +//! * [`WaitGroup`], for synchronizing the beginning or end of some computation. + +#[cfg(not(crossbeam_loom))] +mod once_lock; +mod parker; +#[cfg(not(crossbeam_loom))] +mod sharded_lock; +mod wait_group; + +pub use self::parker::{Parker, Unparker}; +#[cfg(not(crossbeam_loom))] +pub use self::sharded_lock::{ShardedLock, ShardedLockReadGuard, ShardedLockWriteGuard}; +pub use self::wait_group::WaitGroup; diff --git a/anneal/v2/vendor/crossbeam-utils/src/sync/once_lock.rs b/anneal/v2/vendor/crossbeam-utils/src/sync/once_lock.rs new file mode 100644 index 0000000000..e057aca7d5 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-utils/src/sync/once_lock.rs @@ -0,0 +1,88 @@ +// Based on unstable std::sync::OnceLock. +// +// Source: https://github.com/rust-lang/rust/blob/8e9c93df464b7ada3fc7a1c8ccddd9dcb24ee0a0/library/std/src/sync/once_lock.rs + +use core::cell::UnsafeCell; +use core::mem::MaybeUninit; +use std::sync::Once; + +pub(crate) struct OnceLock { + once: Once, + value: UnsafeCell>, + // Unlike std::sync::OnceLock, we don't need PhantomData here because + // we don't use #[may_dangle]. +} + +unsafe impl Sync for OnceLock {} +unsafe impl Send for OnceLock {} + +impl OnceLock { + /// Creates a new empty cell. + #[must_use] + pub(crate) const fn new() -> Self { + Self { + once: Once::new(), + value: UnsafeCell::new(MaybeUninit::uninit()), + } + } + + /// Gets the contents of the cell, initializing it with `f` if the cell + /// was empty. + /// + /// Many threads may call `get_or_init` concurrently with different + /// initializing functions, but it is guaranteed that only one function + /// will be executed. + /// + /// # Panics + /// + /// If `f` panics, the panic is propagated to the caller, and the cell + /// remains uninitialized. + /// + /// It is an error to reentrantly initialize the cell from `f`. The + /// exact outcome is unspecified. Current implementation deadlocks, but + /// this may be changed to a panic in the future. + pub(crate) fn get_or_init(&self, f: F) -> &T + where + F: FnOnce() -> T, + { + // Fast path check + if self.once.is_completed() { + // SAFETY: The inner value has been initialized + return unsafe { self.get_unchecked() }; + } + self.initialize(f); + + // SAFETY: The inner value has been initialized + unsafe { self.get_unchecked() } + } + + #[cold] + fn initialize(&self, f: F) + where + F: FnOnce() -> T, + { + let slot = self.value.get(); + + self.once.call_once(|| { + let value = f(); + unsafe { slot.write(MaybeUninit::new(value)) } + }); + } + + /// # Safety + /// + /// The value must be initialized + unsafe fn get_unchecked(&self) -> &T { + debug_assert!(self.once.is_completed()); + &*self.value.get().cast::() + } +} + +impl Drop for OnceLock { + fn drop(&mut self) { + if self.once.is_completed() { + // SAFETY: The inner value has been initialized + unsafe { (*self.value.get()).assume_init_drop() }; + } + } +} diff --git a/anneal/v2/vendor/crossbeam-utils/src/sync/parker.rs b/anneal/v2/vendor/crossbeam-utils/src/sync/parker.rs new file mode 100644 index 0000000000..971981d2b7 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-utils/src/sync/parker.rs @@ -0,0 +1,415 @@ +use crate::primitive::sync::atomic::{AtomicUsize, Ordering::SeqCst}; +use crate::primitive::sync::{Arc, Condvar, Mutex}; +use std::fmt; +use std::marker::PhantomData; +use std::time::{Duration, Instant}; + +/// A thread parking primitive. +/// +/// Conceptually, each `Parker` has an associated token which is initially not present: +/// +/// * The [`park`] method blocks the current thread unless or until the token is available, at +/// which point it automatically consumes the token. +/// +/// * The [`park_timeout`] and [`park_deadline`] methods work the same as [`park`], but block for +/// a specified maximum time. +/// +/// * The [`unpark`] method atomically makes the token available if it wasn't already. Because the +/// token is initially absent, [`unpark`] followed by [`park`] will result in the second call +/// returning immediately. +/// +/// In other words, each `Parker` acts a bit like a spinlock that can be locked and unlocked using +/// [`park`] and [`unpark`]. +/// +/// # Examples +/// +/// ``` +/// use std::thread; +/// use std::time::Duration; +/// use crossbeam_utils::sync::Parker; +/// +/// let p = Parker::new(); +/// let u = p.unparker().clone(); +/// +/// // Make the token available. +/// u.unpark(); +/// // Wakes up immediately and consumes the token. +/// p.park(); +/// +/// thread::spawn(move || { +/// thread::sleep(Duration::from_millis(500)); +/// u.unpark(); +/// }); +/// +/// // Wakes up when `u.unpark()` provides the token. +/// p.park(); +/// # std::thread::sleep(std::time::Duration::from_millis(500)); // wait for background threads closed: https://github.com/rust-lang/miri/issues/1371 +/// ``` +/// +/// [`park`]: Parker::park +/// [`park_timeout`]: Parker::park_timeout +/// [`park_deadline`]: Parker::park_deadline +/// [`unpark`]: Unparker::unpark +pub struct Parker { + unparker: Unparker, + _marker: PhantomData<*const ()>, +} + +unsafe impl Send for Parker {} + +impl Default for Parker { + fn default() -> Self { + Self { + unparker: Unparker { + inner: Arc::new(Inner { + state: AtomicUsize::new(EMPTY), + lock: Mutex::new(()), + cvar: Condvar::new(), + }), + }, + _marker: PhantomData, + } + } +} + +impl Parker { + /// Creates a new `Parker`. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::sync::Parker; + /// + /// let p = Parker::new(); + /// ``` + /// + pub fn new() -> Parker { + Self::default() + } + + /// Blocks the current thread until the token is made available. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::sync::Parker; + /// + /// let p = Parker::new(); + /// let u = p.unparker().clone(); + /// + /// // Make the token available. + /// u.unpark(); + /// + /// // Wakes up immediately and consumes the token. + /// p.park(); + /// ``` + pub fn park(&self) { + self.unparker.inner.park(None); + } + + /// Blocks the current thread until the token is made available, but only for a limited time. + /// + /// # Examples + /// + /// ``` + /// use std::time::Duration; + /// use crossbeam_utils::sync::Parker; + /// + /// let p = Parker::new(); + /// + /// // Waits for the token to become available, but will not wait longer than 500 ms. + /// p.park_timeout(Duration::from_millis(500)); + /// ``` + pub fn park_timeout(&self, timeout: Duration) { + match Instant::now().checked_add(timeout) { + Some(deadline) => self.park_deadline(deadline), + None => self.park(), + } + } + + /// Blocks the current thread until the token is made available, or until a certain deadline. + /// + /// # Examples + /// + /// ``` + /// use std::time::{Duration, Instant}; + /// use crossbeam_utils::sync::Parker; + /// + /// let p = Parker::new(); + /// let deadline = Instant::now() + Duration::from_millis(500); + /// + /// // Waits for the token to become available, but will not wait longer than 500 ms. + /// p.park_deadline(deadline); + /// ``` + pub fn park_deadline(&self, deadline: Instant) { + self.unparker.inner.park(Some(deadline)) + } + + /// Returns a reference to an associated [`Unparker`]. + /// + /// The returned [`Unparker`] doesn't have to be used by reference - it can also be cloned. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::sync::Parker; + /// + /// let p = Parker::new(); + /// let u = p.unparker().clone(); + /// + /// // Make the token available. + /// u.unpark(); + /// // Wakes up immediately and consumes the token. + /// p.park(); + /// ``` + /// + /// [`park`]: Parker::park + /// [`park_timeout`]: Parker::park_timeout + pub fn unparker(&self) -> &Unparker { + &self.unparker + } + + /// Converts a `Parker` into a raw pointer. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::sync::Parker; + /// + /// let p = Parker::new(); + /// let raw = Parker::into_raw(p); + /// # let _ = unsafe { Parker::from_raw(raw) }; + /// ``` + pub fn into_raw(this: Parker) -> *const () { + Unparker::into_raw(this.unparker) + } + + /// Converts a raw pointer into a `Parker`. + /// + /// # Safety + /// + /// This method is safe to use only with pointers returned by [`Parker::into_raw`]. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::sync::Parker; + /// + /// let p = Parker::new(); + /// let raw = Parker::into_raw(p); + /// let p = unsafe { Parker::from_raw(raw) }; + /// ``` + pub unsafe fn from_raw(ptr: *const ()) -> Parker { + Parker { + unparker: Unparker::from_raw(ptr), + _marker: PhantomData, + } + } +} + +impl fmt::Debug for Parker { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad("Parker { .. }") + } +} + +/// Unparks a thread parked by the associated [`Parker`]. +pub struct Unparker { + inner: Arc, +} + +unsafe impl Send for Unparker {} +unsafe impl Sync for Unparker {} + +impl Unparker { + /// Atomically makes the token available if it is not already. + /// + /// This method will wake up the thread blocked on [`park`] or [`park_timeout`], if there is + /// any. + /// + /// # Examples + /// + /// ``` + /// use std::thread; + /// use std::time::Duration; + /// use crossbeam_utils::sync::Parker; + /// + /// let p = Parker::new(); + /// let u = p.unparker().clone(); + /// + /// thread::spawn(move || { + /// thread::sleep(Duration::from_millis(500)); + /// u.unpark(); + /// }); + /// + /// // Wakes up when `u.unpark()` provides the token. + /// p.park(); + /// # std::thread::sleep(std::time::Duration::from_millis(500)); // wait for background threads closed: https://github.com/rust-lang/miri/issues/1371 + /// ``` + /// + /// [`park`]: Parker::park + /// [`park_timeout`]: Parker::park_timeout + pub fn unpark(&self) { + self.inner.unpark() + } + + /// Converts an `Unparker` into a raw pointer. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::sync::{Parker, Unparker}; + /// + /// let p = Parker::new(); + /// let u = p.unparker().clone(); + /// let raw = Unparker::into_raw(u); + /// # let _ = unsafe { Unparker::from_raw(raw) }; + /// ``` + pub fn into_raw(this: Unparker) -> *const () { + Arc::into_raw(this.inner).cast::<()>() + } + + /// Converts a raw pointer into an `Unparker`. + /// + /// # Safety + /// + /// This method is safe to use only with pointers returned by [`Unparker::into_raw`]. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::sync::{Parker, Unparker}; + /// + /// let p = Parker::new(); + /// let u = p.unparker().clone(); + /// + /// let raw = Unparker::into_raw(u); + /// let u = unsafe { Unparker::from_raw(raw) }; + /// ``` + pub unsafe fn from_raw(ptr: *const ()) -> Unparker { + Unparker { + inner: Arc::from_raw(ptr.cast::()), + } + } +} + +impl fmt::Debug for Unparker { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad("Unparker { .. }") + } +} + +impl Clone for Unparker { + fn clone(&self) -> Unparker { + Unparker { + inner: self.inner.clone(), + } + } +} + +const EMPTY: usize = 0; +const PARKED: usize = 1; +const NOTIFIED: usize = 2; + +struct Inner { + state: AtomicUsize, + lock: Mutex<()>, + cvar: Condvar, +} + +impl Inner { + fn park(&self, deadline: Option) { + // If we were previously notified then we consume this notification and return quickly. + if self + .state + .compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst) + .is_ok() + { + return; + } + + // If the timeout is zero, then there is no need to actually block. + if let Some(deadline) = deadline { + if deadline <= Instant::now() { + return; + } + } + + // Otherwise we need to coordinate going to sleep. + let mut m = self.lock.lock().unwrap(); + + match self.state.compare_exchange(EMPTY, PARKED, SeqCst, SeqCst) { + Ok(_) => {} + // Consume this notification to avoid spurious wakeups in the next park. + Err(NOTIFIED) => { + // We must read `state` here, even though we know it will be `NOTIFIED`. This is + // because `unpark` may have been called again since we read `NOTIFIED` in the + // `compare_exchange` above. We must perform an acquire operation that synchronizes + // with that `unpark` to observe any writes it made before the call to `unpark`. To + // do that we must read from the write it made to `state`. + let old = self.state.swap(EMPTY, SeqCst); + assert_eq!(old, NOTIFIED, "park state changed unexpectedly"); + return; + } + Err(n) => panic!("inconsistent park_timeout state: {}", n), + } + + loop { + // Block the current thread on the conditional variable. + m = match deadline { + None => self.cvar.wait(m).unwrap(), + Some(deadline) => { + let now = Instant::now(); + if now < deadline { + // We could check for a timeout here, in the return value of wait_timeout, + // but in the case that a timeout and an unpark arrive simultaneously, we + // prefer to report the former. + self.cvar.wait_timeout(m, deadline - now).unwrap().0 + } else { + // We've timed out; swap out the state back to empty on our way out + match self.state.swap(EMPTY, SeqCst) { + NOTIFIED | PARKED => return, + n => panic!("inconsistent park_timeout state: {}", n), + }; + } + } + }; + + if self + .state + .compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst) + .is_ok() + { + // got a notification + return; + } + + // Spurious wakeup, go back to sleep. Alternatively, if we timed out, it will be caught + // in the branch above, when we discover the deadline is in the past + } + } + + pub(crate) fn unpark(&self) { + // To ensure the unparked thread will observe any writes we made before this call, we must + // perform a release operation that `park` can synchronize with. To do that we must write + // `NOTIFIED` even if `state` is already `NOTIFIED`. That is why this must be a swap rather + // than a compare-and-swap that returns if it reads `NOTIFIED` on failure. + match self.state.swap(NOTIFIED, SeqCst) { + EMPTY => return, // no one was waiting + NOTIFIED => return, // already unparked + PARKED => {} // gotta go wake someone up + _ => panic!("inconsistent state in unpark"), + } + + // There is a period between when the parked thread sets `state` to `PARKED` (or last + // checked `state` in the case of a spurious wakeup) and when it actually waits on `cvar`. + // If we were to notify during this period it would be ignored and then when the parked + // thread went to sleep it would never wake up. Fortunately, it has `lock` locked at this + // stage so we can acquire `lock` to wait until it is ready to receive the notification. + // + // Releasing `lock` before the call to `notify_one` means that when the parked thread wakes + // it doesn't get woken only to have to wait for us to release `lock`. + drop(self.lock.lock().unwrap()); + self.cvar.notify_one(); + } +} diff --git a/anneal/v2/vendor/crossbeam-utils/src/sync/sharded_lock.rs b/anneal/v2/vendor/crossbeam-utils/src/sync/sharded_lock.rs new file mode 100644 index 0000000000..629b97598e --- /dev/null +++ b/anneal/v2/vendor/crossbeam-utils/src/sync/sharded_lock.rs @@ -0,0 +1,638 @@ +use std::boxed::Box; +use std::cell::UnsafeCell; +use std::collections::HashMap; +use std::fmt; +use std::marker::PhantomData; +use std::mem; +use std::ops::{Deref, DerefMut}; +use std::panic::{RefUnwindSafe, UnwindSafe}; +use std::sync::{LockResult, PoisonError, TryLockError, TryLockResult}; +use std::sync::{Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard}; +use std::thread::{self, ThreadId}; +use std::vec::Vec; + +use crate::sync::once_lock::OnceLock; +use crate::CachePadded; + +/// The number of shards per sharded lock. Must be a power of two. +const NUM_SHARDS: usize = 8; + +/// A shard containing a single reader-writer lock. +struct Shard { + /// The inner reader-writer lock. + lock: RwLock<()>, + + /// The write-guard keeping this shard locked. + /// + /// Write operations will lock each shard and store the guard here. These guards get dropped at + /// the same time the big guard is dropped. + write_guard: UnsafeCell>>, +} + +/// A sharded reader-writer lock. +/// +/// This lock is equivalent to [`RwLock`], except read operations are faster and write operations +/// are slower. +/// +/// A `ShardedLock` is internally made of a list of *shards*, each being a [`RwLock`] occupying a +/// single cache line. Read operations will pick one of the shards depending on the current thread +/// and lock it. Write operations need to lock all shards in succession. +/// +/// By splitting the lock into shards, concurrent read operations will in most cases choose +/// different shards and thus update different cache lines, which is good for scalability. However, +/// write operations need to do more work and are therefore slower than usual. +/// +/// The priority policy of the lock is dependent on the underlying operating system's +/// implementation, and this type does not guarantee that any particular policy will be used. +/// +/// # Poisoning +/// +/// A `ShardedLock`, like [`RwLock`], will become poisoned on a panic. Note that it may only be +/// poisoned if a panic occurs while a write operation is in progress. If a panic occurs in any +/// read operation, the lock will not be poisoned. +/// +/// # Examples +/// +/// ``` +/// use crossbeam_utils::sync::ShardedLock; +/// +/// let lock = ShardedLock::new(5); +/// +/// // Any number of read locks can be held at once. +/// { +/// let r1 = lock.read().unwrap(); +/// let r2 = lock.read().unwrap(); +/// assert_eq!(*r1, 5); +/// assert_eq!(*r2, 5); +/// } // Read locks are dropped at this point. +/// +/// // However, only one write lock may be held. +/// { +/// let mut w = lock.write().unwrap(); +/// *w += 1; +/// assert_eq!(*w, 6); +/// } // Write lock is dropped here. +/// ``` +/// +/// [`RwLock`]: std::sync::RwLock +pub struct ShardedLock { + /// A list of locks protecting the internal data. + shards: Box<[CachePadded]>, + + /// The internal data. + value: UnsafeCell, +} + +unsafe impl Send for ShardedLock {} +unsafe impl Sync for ShardedLock {} + +impl UnwindSafe for ShardedLock {} +impl RefUnwindSafe for ShardedLock {} + +impl ShardedLock { + /// Creates a new sharded reader-writer lock. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::sync::ShardedLock; + /// + /// let lock = ShardedLock::new(5); + /// ``` + pub fn new(value: T) -> ShardedLock { + ShardedLock { + shards: (0..NUM_SHARDS) + .map(|_| { + CachePadded::new(Shard { + lock: RwLock::new(()), + write_guard: UnsafeCell::new(None), + }) + }) + .collect::>(), + value: UnsafeCell::new(value), + } + } + + /// Consumes this lock, returning the underlying data. + /// + /// # Errors + /// + /// This method will return an error if the lock is poisoned. A lock gets poisoned when a write + /// operation panics. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::sync::ShardedLock; + /// + /// let lock = ShardedLock::new(String::new()); + /// { + /// let mut s = lock.write().unwrap(); + /// *s = "modified".to_owned(); + /// } + /// assert_eq!(lock.into_inner().unwrap(), "modified"); + /// ``` + pub fn into_inner(self) -> LockResult { + let is_poisoned = self.is_poisoned(); + let inner = self.value.into_inner(); + + if is_poisoned { + Err(PoisonError::new(inner)) + } else { + Ok(inner) + } + } +} + +impl ShardedLock { + /// Returns `true` if the lock is poisoned. + /// + /// If another thread can still access the lock, it may become poisoned at any time. A `false` + /// result should not be trusted without additional synchronization. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::sync::ShardedLock; + /// use std::sync::Arc; + /// use std::thread; + /// + /// let lock = Arc::new(ShardedLock::new(0)); + /// let c_lock = lock.clone(); + /// + /// let _ = thread::spawn(move || { + /// let _lock = c_lock.write().unwrap(); + /// panic!(); // the lock gets poisoned + /// }).join(); + /// assert_eq!(lock.is_poisoned(), true); + /// ``` + pub fn is_poisoned(&self) -> bool { + self.shards[0].lock.is_poisoned() + } + + /// Returns a mutable reference to the underlying data. + /// + /// Since this call borrows the lock mutably, no actual locking needs to take place. + /// + /// # Errors + /// + /// This method will return an error if the lock is poisoned. A lock gets poisoned when a write + /// operation panics. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::sync::ShardedLock; + /// + /// let mut lock = ShardedLock::new(0); + /// *lock.get_mut().unwrap() = 10; + /// assert_eq!(*lock.read().unwrap(), 10); + /// ``` + pub fn get_mut(&mut self) -> LockResult<&mut T> { + let is_poisoned = self.is_poisoned(); + let inner = unsafe { &mut *self.value.get() }; + + if is_poisoned { + Err(PoisonError::new(inner)) + } else { + Ok(inner) + } + } + + /// Attempts to acquire this lock with shared read access. + /// + /// If the access could not be granted at this time, an error is returned. Otherwise, a guard + /// is returned which will release the shared access when it is dropped. This method does not + /// provide any guarantees with respect to the ordering of whether contentious readers or + /// writers will acquire the lock first. + /// + /// # Errors + /// + /// This method will return an error if the lock is poisoned. A lock gets poisoned when a write + /// operation panics. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::sync::ShardedLock; + /// + /// let lock = ShardedLock::new(1); + /// + /// match lock.try_read() { + /// Ok(n) => assert_eq!(*n, 1), + /// Err(_) => unreachable!(), + /// }; + /// ``` + pub fn try_read(&self) -> TryLockResult> { + // Take the current thread index and map it to a shard index. Thread indices will tend to + // distribute shards among threads equally, thus reducing contention due to read-locking. + let current_index = current_index().unwrap_or(0); + let shard_index = current_index & (self.shards.len() - 1); + + match self.shards[shard_index].lock.try_read() { + Ok(guard) => Ok(ShardedLockReadGuard { + lock: self, + _guard: guard, + _marker: PhantomData, + }), + Err(TryLockError::Poisoned(err)) => { + let guard = ShardedLockReadGuard { + lock: self, + _guard: err.into_inner(), + _marker: PhantomData, + }; + Err(TryLockError::Poisoned(PoisonError::new(guard))) + } + Err(TryLockError::WouldBlock) => Err(TryLockError::WouldBlock), + } + } + + /// Locks with shared read access, blocking the current thread until it can be acquired. + /// + /// The calling thread will be blocked until there are no more writers which hold the lock. + /// There may be other readers currently inside the lock when this method returns. This method + /// does not provide any guarantees with respect to the ordering of whether contentious readers + /// or writers will acquire the lock first. + /// + /// Returns a guard which will release the shared access when dropped. + /// + /// # Errors + /// + /// This method will return an error if the lock is poisoned. A lock gets poisoned when a write + /// operation panics. + /// + /// # Panics + /// + /// This method might panic when called if the lock is already held by the current thread. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::sync::ShardedLock; + /// use std::sync::Arc; + /// use std::thread; + /// + /// let lock = Arc::new(ShardedLock::new(1)); + /// let c_lock = lock.clone(); + /// + /// let n = lock.read().unwrap(); + /// assert_eq!(*n, 1); + /// + /// thread::spawn(move || { + /// let r = c_lock.read(); + /// assert!(r.is_ok()); + /// }).join().unwrap(); + /// ``` + pub fn read(&self) -> LockResult> { + // Take the current thread index and map it to a shard index. Thread indices will tend to + // distribute shards among threads equally, thus reducing contention due to read-locking. + let current_index = current_index().unwrap_or(0); + let shard_index = current_index & (self.shards.len() - 1); + + match self.shards[shard_index].lock.read() { + Ok(guard) => Ok(ShardedLockReadGuard { + lock: self, + _guard: guard, + _marker: PhantomData, + }), + Err(err) => Err(PoisonError::new(ShardedLockReadGuard { + lock: self, + _guard: err.into_inner(), + _marker: PhantomData, + })), + } + } + + /// Attempts to acquire this lock with exclusive write access. + /// + /// If the access could not be granted at this time, an error is returned. Otherwise, a guard + /// is returned which will release the exclusive access when it is dropped. This method does + /// not provide any guarantees with respect to the ordering of whether contentious readers or + /// writers will acquire the lock first. + /// + /// # Errors + /// + /// This method will return an error if the lock is poisoned. A lock gets poisoned when a write + /// operation panics. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::sync::ShardedLock; + /// + /// let lock = ShardedLock::new(1); + /// + /// let n = lock.read().unwrap(); + /// assert_eq!(*n, 1); + /// + /// assert!(lock.try_write().is_err()); + /// ``` + pub fn try_write(&self) -> TryLockResult> { + let mut poisoned = false; + let mut blocked = None; + + // Write-lock each shard in succession. + for (i, shard) in self.shards.iter().enumerate() { + let guard = match shard.lock.try_write() { + Ok(guard) => guard, + Err(TryLockError::Poisoned(err)) => { + poisoned = true; + err.into_inner() + } + Err(TryLockError::WouldBlock) => { + blocked = Some(i); + break; + } + }; + + // Store the guard into the shard. + unsafe { + let guard: RwLockWriteGuard<'static, ()> = mem::transmute(guard); + let dest: *mut _ = shard.write_guard.get(); + *dest = Some(guard); + } + } + + if let Some(i) = blocked { + // Unlock the shards in reverse order of locking. + for shard in self.shards[0..i].iter().rev() { + unsafe { + let dest: *mut _ = shard.write_guard.get(); + let guard = (*dest).take(); + drop(guard); + } + } + Err(TryLockError::WouldBlock) + } else if poisoned { + let guard = ShardedLockWriteGuard { + lock: self, + _marker: PhantomData, + }; + Err(TryLockError::Poisoned(PoisonError::new(guard))) + } else { + Ok(ShardedLockWriteGuard { + lock: self, + _marker: PhantomData, + }) + } + } + + /// Locks with exclusive write access, blocking the current thread until it can be acquired. + /// + /// The calling thread will be blocked until there are no more writers which hold the lock. + /// There may be other readers currently inside the lock when this method returns. This method + /// does not provide any guarantees with respect to the ordering of whether contentious readers + /// or writers will acquire the lock first. + /// + /// Returns a guard which will release the exclusive access when dropped. + /// + /// # Errors + /// + /// This method will return an error if the lock is poisoned. A lock gets poisoned when a write + /// operation panics. + /// + /// # Panics + /// + /// This method might panic when called if the lock is already held by the current thread. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::sync::ShardedLock; + /// + /// let lock = ShardedLock::new(1); + /// + /// let mut n = lock.write().unwrap(); + /// *n = 2; + /// + /// assert!(lock.try_read().is_err()); + /// ``` + pub fn write(&self) -> LockResult> { + let mut poisoned = false; + + // Write-lock each shard in succession. + for shard in self.shards.iter() { + let guard = match shard.lock.write() { + Ok(guard) => guard, + Err(err) => { + poisoned = true; + err.into_inner() + } + }; + + // Store the guard into the shard. + unsafe { + let guard: RwLockWriteGuard<'_, ()> = guard; + let guard: RwLockWriteGuard<'static, ()> = mem::transmute(guard); + let dest: *mut _ = shard.write_guard.get(); + *dest = Some(guard); + } + } + + if poisoned { + Err(PoisonError::new(ShardedLockWriteGuard { + lock: self, + _marker: PhantomData, + })) + } else { + Ok(ShardedLockWriteGuard { + lock: self, + _marker: PhantomData, + }) + } + } +} + +impl fmt::Debug for ShardedLock { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.try_read() { + Ok(guard) => f + .debug_struct("ShardedLock") + .field("data", &&*guard) + .finish(), + Err(TryLockError::Poisoned(err)) => f + .debug_struct("ShardedLock") + .field("data", &&**err.get_ref()) + .finish(), + Err(TryLockError::WouldBlock) => { + struct LockedPlaceholder; + impl fmt::Debug for LockedPlaceholder { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("") + } + } + f.debug_struct("ShardedLock") + .field("data", &LockedPlaceholder) + .finish() + } + } + } +} + +impl Default for ShardedLock { + fn default() -> ShardedLock { + ShardedLock::new(Default::default()) + } +} + +impl From for ShardedLock { + fn from(t: T) -> Self { + ShardedLock::new(t) + } +} + +/// A guard used to release the shared read access of a [`ShardedLock`] when dropped. +#[clippy::has_significant_drop] +pub struct ShardedLockReadGuard<'a, T: ?Sized> { + lock: &'a ShardedLock, + _guard: RwLockReadGuard<'a, ()>, + _marker: PhantomData>, +} + +unsafe impl Sync for ShardedLockReadGuard<'_, T> {} + +impl Deref for ShardedLockReadGuard<'_, T> { + type Target = T; + + fn deref(&self) -> &T { + unsafe { &*self.lock.value.get() } + } +} + +impl fmt::Debug for ShardedLockReadGuard<'_, T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ShardedLockReadGuard") + .field("lock", &self.lock) + .finish() + } +} + +impl fmt::Display for ShardedLockReadGuard<'_, T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + (**self).fmt(f) + } +} + +/// A guard used to release the exclusive write access of a [`ShardedLock`] when dropped. +#[clippy::has_significant_drop] +pub struct ShardedLockWriteGuard<'a, T: ?Sized> { + lock: &'a ShardedLock, + _marker: PhantomData>, +} + +unsafe impl Sync for ShardedLockWriteGuard<'_, T> {} + +impl Drop for ShardedLockWriteGuard<'_, T> { + fn drop(&mut self) { + // Unlock the shards in reverse order of locking. + for shard in self.lock.shards.iter().rev() { + unsafe { + let dest: *mut _ = shard.write_guard.get(); + let guard = (*dest).take(); + drop(guard); + } + } + } +} + +impl fmt::Debug for ShardedLockWriteGuard<'_, T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ShardedLockWriteGuard") + .field("lock", &self.lock) + .finish() + } +} + +impl fmt::Display for ShardedLockWriteGuard<'_, T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + (**self).fmt(f) + } +} + +impl Deref for ShardedLockWriteGuard<'_, T> { + type Target = T; + + fn deref(&self) -> &T { + unsafe { &*self.lock.value.get() } + } +} + +impl DerefMut for ShardedLockWriteGuard<'_, T> { + fn deref_mut(&mut self) -> &mut T { + unsafe { &mut *self.lock.value.get() } + } +} + +/// Returns a `usize` that identifies the current thread. +/// +/// Each thread is associated with an 'index'. While there are no particular guarantees, indices +/// usually tend to be consecutive numbers between 0 and the number of running threads. +/// +/// Since this function accesses TLS, `None` might be returned if the current thread's TLS is +/// tearing down. +#[inline] +fn current_index() -> Option { + REGISTRATION.try_with(|reg| reg.index).ok() +} + +/// The global registry keeping track of registered threads and indices. +struct ThreadIndices { + /// Mapping from `ThreadId` to thread index. + mapping: HashMap, + + /// A list of free indices. + free_list: Vec, + + /// The next index to allocate if the free list is empty. + next_index: usize, +} + +fn thread_indices() -> &'static Mutex { + static THREAD_INDICES: OnceLock> = OnceLock::new(); + fn init() -> Mutex { + Mutex::new(ThreadIndices { + mapping: HashMap::new(), + free_list: Vec::new(), + next_index: 0, + }) + } + THREAD_INDICES.get_or_init(init) +} + +/// A registration of a thread with an index. +/// +/// When dropped, unregisters the thread and frees the reserved index. +struct Registration { + index: usize, + thread_id: ThreadId, +} + +impl Drop for Registration { + fn drop(&mut self) { + let mut indices = thread_indices().lock().unwrap(); + indices.mapping.remove(&self.thread_id); + indices.free_list.push(self.index); + } +} + +std::thread_local! { + static REGISTRATION: Registration = { + let thread_id = thread::current().id(); + let mut indices = thread_indices().lock().unwrap(); + + let index = match indices.free_list.pop() { + Some(i) => i, + None => { + let i = indices.next_index; + indices.next_index += 1; + i + } + }; + indices.mapping.insert(thread_id, index); + + Registration { + index, + thread_id, + } + }; +} diff --git a/anneal/v2/vendor/crossbeam-utils/src/sync/wait_group.rs b/anneal/v2/vendor/crossbeam-utils/src/sync/wait_group.rs new file mode 100644 index 0000000000..19d6074157 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-utils/src/sync/wait_group.rs @@ -0,0 +1,145 @@ +use crate::primitive::sync::{Arc, Condvar, Mutex}; +use std::fmt; + +/// Enables threads to synchronize the beginning or end of some computation. +/// +/// # Wait groups vs barriers +/// +/// `WaitGroup` is very similar to [`Barrier`], but there are a few differences: +/// +/// * [`Barrier`] needs to know the number of threads at construction, while `WaitGroup` is cloned to +/// register more threads. +/// +/// * A [`Barrier`] can be reused even after all threads have synchronized, while a `WaitGroup` +/// synchronizes threads only once. +/// +/// * All threads wait for others to reach the [`Barrier`]. With `WaitGroup`, each thread can choose +/// to either wait for other threads or to continue without blocking. +/// +/// # Examples +/// +/// ``` +/// use crossbeam_utils::sync::WaitGroup; +/// use std::thread; +/// +/// // Create a new wait group. +/// let wg = WaitGroup::new(); +/// +/// for _ in 0..4 { +/// // Create another reference to the wait group. +/// let wg = wg.clone(); +/// +/// thread::spawn(move || { +/// // Do some work. +/// +/// // Drop the reference to the wait group. +/// drop(wg); +/// }); +/// } +/// +/// // Block until all threads have finished their work. +/// wg.wait(); +/// # std::thread::sleep(std::time::Duration::from_millis(500)); // wait for background threads closed: https://github.com/rust-lang/miri/issues/1371 +/// ``` +/// +/// [`Barrier`]: std::sync::Barrier +pub struct WaitGroup { + inner: Arc, +} + +/// Inner state of a `WaitGroup`. +struct Inner { + cvar: Condvar, + count: Mutex, +} + +impl Default for WaitGroup { + fn default() -> Self { + Self { + inner: Arc::new(Inner { + cvar: Condvar::new(), + count: Mutex::new(1), + }), + } + } +} + +impl WaitGroup { + /// Creates a new wait group and returns the single reference to it. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::sync::WaitGroup; + /// + /// let wg = WaitGroup::new(); + /// ``` + pub fn new() -> Self { + Self::default() + } + + /// Drops this reference and waits until all other references are dropped. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::sync::WaitGroup; + /// use std::thread; + /// + /// let wg = WaitGroup::new(); + /// + /// thread::spawn({ + /// let wg = wg.clone(); + /// move || { + /// // Block until both threads have reached `wait()`. + /// wg.wait(); + /// } + /// }); + /// + /// // Block until both threads have reached `wait()`. + /// wg.wait(); + /// # std::thread::sleep(std::time::Duration::from_millis(500)); // wait for background threads closed: https://github.com/rust-lang/miri/issues/1371 + /// ``` + pub fn wait(self) { + if *self.inner.count.lock().unwrap() == 1 { + return; + } + + let inner = self.inner.clone(); + drop(self); + + let mut count = inner.count.lock().unwrap(); + while *count > 0 { + count = inner.cvar.wait(count).unwrap(); + } + } +} + +impl Drop for WaitGroup { + fn drop(&mut self) { + let mut count = self.inner.count.lock().unwrap(); + *count -= 1; + + if *count == 0 { + self.inner.cvar.notify_all(); + } + } +} + +impl Clone for WaitGroup { + fn clone(&self) -> WaitGroup { + let mut count = self.inner.count.lock().unwrap(); + *count += 1; + + WaitGroup { + inner: self.inner.clone(), + } + } +} + +impl fmt::Debug for WaitGroup { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let count: &usize = &*self.inner.count.lock().unwrap(); + f.debug_struct("WaitGroup").field("count", count).finish() + } +} diff --git a/anneal/v2/vendor/crossbeam-utils/src/thread.rs b/anneal/v2/vendor/crossbeam-utils/src/thread.rs new file mode 100644 index 0000000000..847f4cf112 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-utils/src/thread.rs @@ -0,0 +1,611 @@ +//! Threads that can borrow variables from the stack. +//! +//! Create a scope when spawned threads need to access variables on the stack: +//! +//! ``` +//! use crossbeam_utils::thread; +//! +//! let people = vec![ +//! "Alice".to_string(), +//! "Bob".to_string(), +//! "Carol".to_string(), +//! ]; +//! +//! thread::scope(|s| { +//! for person in &people { +//! s.spawn(move |_| { +//! println!("Hello, {}!", person); +//! }); +//! } +//! }).unwrap(); +//! ``` +//! +//! # Why scoped threads? +//! +//! Suppose we wanted to re-write the previous example using plain threads: +//! +//! ```compile_fail,E0597 +//! use std::thread; +//! +//! let people = vec![ +//! "Alice".to_string(), +//! "Bob".to_string(), +//! "Carol".to_string(), +//! ]; +//! +//! let mut threads = Vec::new(); +//! +//! for person in &people { +//! threads.push(thread::spawn(move || { +//! println!("Hello, {}!", person); +//! })); +//! } +//! +//! for thread in threads { +//! thread.join().unwrap(); +//! } +//! ``` +//! +//! This doesn't work because the borrow checker complains about `people` not living long enough: +//! +//! ```text +//! error[E0597]: `people` does not live long enough +//! --> src/main.rs:12:20 +//! | +//! 12 | for person in &people { +//! | ^^^^^^ borrowed value does not live long enough +//! ... +//! 21 | } +//! | - borrowed value only lives until here +//! | +//! = note: borrowed value must be valid for the static lifetime... +//! ``` +//! +//! The problem here is that spawned threads are not allowed to borrow variables on stack because +//! the compiler cannot prove they will be joined before `people` is destroyed. +//! +//! Scoped threads are a mechanism to guarantee to the compiler that spawned threads will be joined +//! before the scope ends. +//! +//! # How scoped threads work +//! +//! If a variable is borrowed by a thread, the thread must complete before the variable is +//! destroyed. Threads spawned using [`std::thread::spawn`] can only borrow variables with the +//! `'static` lifetime because the borrow checker cannot be sure when the thread will complete. +//! +//! A scope creates a clear boundary between variables outside the scope and threads inside the +//! scope. Whenever a scope spawns a thread, it promises to join the thread before the scope ends. +//! This way we guarantee to the borrow checker that scoped threads only live within the scope and +//! can safely access variables outside it. +//! +//! # Nesting scoped threads +//! +//! Sometimes scoped threads need to spawn more threads within the same scope. This is a little +//! tricky because argument `s` lives *inside* the invocation of `thread::scope()` and as such +//! cannot be borrowed by scoped threads: +//! +//! ```compile_fail,E0521 +//! use crossbeam_utils::thread; +//! +//! thread::scope(|s| { +//! s.spawn(|_| { +//! // Not going to compile because we're trying to borrow `s`, +//! // which lives *inside* the scope! :( +//! s.spawn(|_| println!("nested thread")); +//! }); +//! }); +//! ``` +//! +//! Fortunately, there is a solution. Every scoped thread is passed a reference to its scope as an +//! argument, which can be used for spawning nested threads: +//! +//! ``` +//! use crossbeam_utils::thread; +//! +//! thread::scope(|s| { +//! // Note the `|s|` here. +//! s.spawn(|s| { +//! // Yay, this works because we're using a fresh argument `s`! :) +//! s.spawn(|_| println!("nested thread")); +//! }); +//! }).unwrap(); +//! ``` + +use std::boxed::Box; +use std::fmt; +use std::io; +use std::marker::PhantomData; +use std::mem; +use std::panic; +use std::string::String; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::vec::Vec; + +use crate::sync::WaitGroup; + +type SharedVec = Arc>>; +type SharedOption = Arc>>; + +/// Creates a new scope for spawning threads. +/// +/// All child threads that haven't been manually joined will be automatically joined just before +/// this function invocation ends. If all joined threads have successfully completed, `Ok` is +/// returned with the return value of `f`. If any of the joined threads has panicked, an `Err` is +/// returned containing errors from panicked threads. Note that if panics are implemented by +/// aborting the process, no error is returned; see the notes of [std::panic::catch_unwind]. +/// +/// **Note:** Since Rust 1.63, this function is soft-deprecated in favor of the more efficient [`std::thread::scope`]. +/// +/// # Examples +/// +/// ``` +/// use crossbeam_utils::thread; +/// +/// let var = vec![1, 2, 3]; +/// +/// thread::scope(|s| { +/// s.spawn(|_| { +/// println!("A child thread borrowing `var`: {:?}", var); +/// }); +/// }).unwrap(); +/// ``` +pub fn scope<'env, F, R>(f: F) -> thread::Result +where + F: FnOnce(&Scope<'env>) -> R, +{ + struct AbortOnPanic; + impl Drop for AbortOnPanic { + fn drop(&mut self) { + if thread::panicking() { + std::process::abort(); + } + } + } + + let wg = WaitGroup::new(); + let scope = Scope::<'env> { + handles: SharedVec::default(), + wait_group: wg.clone(), + _marker: PhantomData, + }; + + // Execute the scoped function, but catch any panics. + let result = panic::catch_unwind(panic::AssertUnwindSafe(|| f(&scope))); + + // If an unwinding panic occurs before all threads are joined + // promote it to an aborting panic to prevent any threads from escaping the scope. + let guard = AbortOnPanic; + + // Wait until all nested scopes are dropped. + drop(scope.wait_group); + wg.wait(); + + // Join all remaining spawned threads. + let panics: Vec<_> = scope + .handles + .lock() + .unwrap() + // Filter handles that haven't been joined, join them, and collect errors. + .drain(..) + .filter_map(|handle| handle.lock().unwrap().take()) + .filter_map(|handle| handle.join().err()) + .collect(); + + mem::forget(guard); + + // If `f` has panicked, resume unwinding. + // If any of the child threads have panicked, return the panic errors. + // Otherwise, everything is OK and return the result of `f`. + match result { + Err(err) => panic::resume_unwind(err), + Ok(res) => { + if panics.is_empty() { + Ok(res) + } else { + Err(Box::new(panics)) + } + } + } +} + +/// A scope for spawning threads. +pub struct Scope<'env> { + /// The list of the thread join handles. + handles: SharedVec>>, + + /// Used to wait until all subscopes all dropped. + wait_group: WaitGroup, + + /// Borrows data with invariant lifetime `'env`. + _marker: PhantomData<&'env mut &'env ()>, +} + +unsafe impl Sync for Scope<'_> {} + +impl<'env> Scope<'env> { + /// Spawns a scoped thread. + /// + /// This method is similar to the [`spawn`] function in Rust's standard library. The difference + /// is that this thread is scoped, meaning it's guaranteed to terminate before the scope exits, + /// allowing it to reference variables outside the scope. + /// + /// The scoped thread is passed a reference to this scope as an argument, which can be used for + /// spawning nested threads. + /// + /// The returned [handle](ScopedJoinHandle) can be used to manually + /// [join](ScopedJoinHandle::join) the thread before the scope exits. + /// + /// This will create a thread using default parameters of [`ScopedThreadBuilder`], if you want to specify the + /// stack size or the name of the thread, use this API instead. + /// + /// [`spawn`]: std::thread::spawn + /// + /// # Panics + /// + /// Panics if the OS fails to create a thread; use [`ScopedThreadBuilder::spawn`] + /// to recover from such errors. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::thread; + /// + /// thread::scope(|s| { + /// let handle = s.spawn(|_| { + /// println!("A child thread is running"); + /// 42 + /// }); + /// + /// // Join the thread and retrieve its result. + /// let res = handle.join().unwrap(); + /// assert_eq!(res, 42); + /// }).unwrap(); + /// ``` + pub fn spawn<'scope, F, T>(&'scope self, f: F) -> ScopedJoinHandle<'scope, T> + where + F: FnOnce(&Scope<'env>) -> T, + F: Send + 'env, + T: Send + 'env, + { + self.builder() + .spawn(f) + .expect("failed to spawn scoped thread") + } + + /// Creates a builder that can configure a thread before spawning. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::thread; + /// + /// thread::scope(|s| { + /// s.builder() + /// .spawn(|_| println!("A child thread is running")) + /// .unwrap(); + /// }).unwrap(); + /// ``` + pub fn builder<'scope>(&'scope self) -> ScopedThreadBuilder<'scope, 'env> { + ScopedThreadBuilder { + scope: self, + builder: thread::Builder::new(), + } + } +} + +impl fmt::Debug for Scope<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad("Scope { .. }") + } +} + +/// Configures the properties of a new thread. +/// +/// The two configurable properties are: +/// +/// - [`name`]: Specifies an [associated name for the thread][naming-threads]. +/// - [`stack_size`]: Specifies the [desired stack size for the thread][stack-size]. +/// +/// The [`spawn`] method will take ownership of the builder and return an [`io::Result`] of the +/// thread handle with the given configuration. +/// +/// The [`Scope::spawn`] method uses a builder with default configuration and unwraps its return +/// value. You may want to use this builder when you want to recover from a failure to launch a +/// thread. +/// +/// # Examples +/// +/// ``` +/// use crossbeam_utils::thread; +/// +/// thread::scope(|s| { +/// s.builder() +/// .spawn(|_| println!("Running a child thread")) +/// .unwrap(); +/// }).unwrap(); +/// ``` +/// +/// [`name`]: ScopedThreadBuilder::name +/// [`stack_size`]: ScopedThreadBuilder::stack_size +/// [`spawn`]: ScopedThreadBuilder::spawn +/// [`io::Result`]: std::io::Result +/// [naming-threads]: std::thread#naming-threads +/// [stack-size]: std::thread#stack-size +#[derive(Debug)] +pub struct ScopedThreadBuilder<'scope, 'env> { + scope: &'scope Scope<'env>, + builder: thread::Builder, +} + +impl<'scope, 'env> ScopedThreadBuilder<'scope, 'env> { + /// Sets the name for the new thread. + /// + /// The name must not contain null bytes (`\0`). + /// + /// For more information about named threads, see [here][naming-threads]. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::thread; + /// use std::thread::current; + /// + /// thread::scope(|s| { + /// s.builder() + /// .name("my thread".to_string()) + /// .spawn(|_| assert_eq!(current().name(), Some("my thread"))) + /// .unwrap(); + /// }).unwrap(); + /// ``` + /// + /// [naming-threads]: std::thread#naming-threads + pub fn name(mut self, name: String) -> ScopedThreadBuilder<'scope, 'env> { + self.builder = self.builder.name(name); + self + } + + /// Sets the size of the stack for the new thread. + /// + /// The stack size is measured in bytes. + /// + /// For more information about the stack size for threads, see [here][stack-size]. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::thread; + /// + /// thread::scope(|s| { + /// s.builder() + /// .stack_size(32 * 1024) + /// .spawn(|_| println!("Running a child thread")) + /// .unwrap(); + /// }).unwrap(); + /// ``` + /// + /// [stack-size]: std::thread#stack-size + pub fn stack_size(mut self, size: usize) -> ScopedThreadBuilder<'scope, 'env> { + self.builder = self.builder.stack_size(size); + self + } + + /// Spawns a scoped thread with this configuration. + /// + /// The scoped thread is passed a reference to this scope as an argument, which can be used for + /// spawning nested threads. + /// + /// The returned handle can be used to manually join the thread before the scope exits. + /// + /// # Errors + /// + /// Unlike the [`Scope::spawn`] method, this method yields an + /// [`io::Result`] to capture any failure to create the thread at + /// the OS level. + /// + /// [`io::Result`]: std::io::Result + /// + /// # Panics + /// + /// Panics if a thread name was set and it contained null bytes. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::thread; + /// + /// thread::scope(|s| { + /// let handle = s.builder() + /// .spawn(|_| { + /// println!("A child thread is running"); + /// 42 + /// }) + /// .unwrap(); + /// + /// // Join the thread and retrieve its result. + /// let res = handle.join().unwrap(); + /// assert_eq!(res, 42); + /// }).unwrap(); + /// ``` + pub fn spawn(self, f: F) -> io::Result> + where + F: FnOnce(&Scope<'env>) -> T, + F: Send + 'env, + T: Send + 'env, + { + // The result of `f` will be stored here. + let result = SharedOption::default(); + + // Spawn the thread and grab its join handle and thread handle. + let (handle, thread) = { + let result = Arc::clone(&result); + + // A clone of the scope that will be moved into the new thread. + let scope = Scope::<'env> { + handles: Arc::clone(&self.scope.handles), + wait_group: self.scope.wait_group.clone(), + _marker: PhantomData, + }; + + // Spawn the thread. + let handle = { + let closure = move || { + // Make sure the scope is inside the closure with the proper `'env` lifetime. + let scope: Scope<'env> = scope; + + // Run the closure. + let res = f(&scope); + + // Store the result if the closure didn't panic. + *result.lock().unwrap() = Some(res); + }; + + // Allocate `closure` on the heap and erase the `'env` bound. + let closure: Box = Box::new(closure); + let closure: Box = + unsafe { mem::transmute(closure) }; + + // Finally, spawn the closure. + self.builder.spawn(closure)? + }; + + let thread = handle.thread().clone(); + let handle = Arc::new(Mutex::new(Some(handle))); + (handle, thread) + }; + + // Add the handle to the shared list of join handles. + self.scope.handles.lock().unwrap().push(Arc::clone(&handle)); + + Ok(ScopedJoinHandle { + handle, + result, + thread, + _marker: PhantomData, + }) + } +} + +unsafe impl Send for ScopedJoinHandle<'_, T> {} +unsafe impl Sync for ScopedJoinHandle<'_, T> {} + +/// A handle that can be used to join its scoped thread. +/// +/// This struct is created by the [`Scope::spawn`] method and the +/// [`ScopedThreadBuilder::spawn`] method. +pub struct ScopedJoinHandle<'scope, T> { + /// A join handle to the spawned thread. + handle: SharedOption>, + + /// Holds the result of the inner closure. + result: SharedOption, + + /// A handle to the spawned thread. + thread: thread::Thread, + + /// Borrows the parent scope with lifetime `'scope`. + _marker: PhantomData<&'scope ()>, +} + +impl ScopedJoinHandle<'_, T> { + /// Waits for the thread to finish and returns its result. + /// + /// If the child thread panics, an error is returned. Note that if panics are implemented by + /// aborting the process, no error is returned; see the notes of [std::panic::catch_unwind]. + /// + /// # Panics + /// + /// This function may panic on some platforms if a thread attempts to join itself or otherwise + /// may create a deadlock with joining threads. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::thread; + /// + /// thread::scope(|s| { + /// let handle1 = s.spawn(|_| println!("I'm a happy thread :)")); + /// let handle2 = s.spawn(|_| panic!("I'm a sad thread :(")); + /// + /// // Join the first thread and verify that it succeeded. + /// let res = handle1.join(); + /// assert!(res.is_ok()); + /// + /// // Join the second thread and verify that it panicked. + /// let res = handle2.join(); + /// assert!(res.is_err()); + /// }).unwrap(); + /// ``` + pub fn join(self) -> thread::Result { + // Take out the handle. The handle will surely be available because the root scope waits + // for nested scopes before joining remaining threads. + let handle = self.handle.lock().unwrap().take().unwrap(); + + // Join the thread and then take the result out of its inner closure. + handle + .join() + .map(|()| self.result.lock().unwrap().take().unwrap()) + } + + /// Returns a handle to the underlying thread. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::thread; + /// + /// thread::scope(|s| { + /// let handle = s.spawn(|_| println!("A child thread is running")); + /// println!("The child thread ID: {:?}", handle.thread().id()); + /// }).unwrap(); + /// ``` + pub fn thread(&self) -> &thread::Thread { + &self.thread + } +} + +/// Unix-specific extensions. +#[cfg(unix)] +mod unix { + use super::ScopedJoinHandle; + use std::os::unix::thread::{JoinHandleExt, RawPthread}; + + impl JoinHandleExt for ScopedJoinHandle<'_, T> { + fn as_pthread_t(&self) -> RawPthread { + // Borrow the handle. The handle will surely be available because the root scope waits + // for nested scopes before joining remaining threads. + let handle = self.handle.lock().unwrap(); + handle.as_ref().unwrap().as_pthread_t() + } + fn into_pthread_t(self) -> RawPthread { + self.as_pthread_t() + } + } +} +/// Windows-specific extensions. +#[cfg(windows)] +mod windows { + use super::ScopedJoinHandle; + use std::os::windows::io::{AsRawHandle, IntoRawHandle, RawHandle}; + + impl AsRawHandle for ScopedJoinHandle<'_, T> { + fn as_raw_handle(&self) -> RawHandle { + // Borrow the handle. The handle will surely be available because the root scope waits + // for nested scopes before joining remaining threads. + let handle = self.handle.lock().unwrap(); + handle.as_ref().unwrap().as_raw_handle() + } + } + + impl IntoRawHandle for ScopedJoinHandle<'_, T> { + fn into_raw_handle(self) -> RawHandle { + self.as_raw_handle() + } + } +} + +impl fmt::Debug for ScopedJoinHandle<'_, T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad("ScopedJoinHandle { .. }") + } +} diff --git a/anneal/v2/vendor/crossbeam-utils/tests/atomic_cell.rs b/anneal/v2/vendor/crossbeam-utils/tests/atomic_cell.rs new file mode 100644 index 0000000000..9fe69328df --- /dev/null +++ b/anneal/v2/vendor/crossbeam-utils/tests/atomic_cell.rs @@ -0,0 +1,374 @@ +use std::mem; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering::SeqCst; + +use crossbeam_utils::atomic::AtomicCell; + +#[test] +fn is_lock_free() { + struct UsizeWrap(#[allow(dead_code)] usize); + struct U8Wrap(#[allow(dead_code)] bool); + struct I16Wrap(#[allow(dead_code)] i16); + #[repr(align(8))] + struct U64Align8(#[allow(dead_code)] u64); + + assert!(AtomicCell::::is_lock_free()); + assert!(AtomicCell::::is_lock_free()); + assert!(AtomicCell::::is_lock_free()); + + assert!(AtomicCell::<()>::is_lock_free()); + + assert!(AtomicCell::::is_lock_free()); + assert!(AtomicCell::::is_lock_free()); + assert!(AtomicCell::::is_lock_free()); + assert!(AtomicCell::::is_lock_free()); + + assert!(AtomicCell::::is_lock_free()); + assert!(AtomicCell::::is_lock_free()); + assert!(AtomicCell::::is_lock_free()); + + assert!(AtomicCell::::is_lock_free()); + assert!(AtomicCell::::is_lock_free()); + + // Sizes of both types must be equal, and the alignment of `u64` must be greater or equal than + // that of `AtomicU64`. In i686-unknown-linux-gnu, the alignment of `u64` is `4` and alignment + // of `AtomicU64` is `8`, so `AtomicCell` is not lock-free. + assert_eq!( + AtomicCell::::is_lock_free(), + cfg!(target_has_atomic = "64") && std::mem::align_of::() == 8 + ); + assert_eq!(mem::size_of::(), 8); + assert_eq!(mem::align_of::(), 8); + assert_eq!( + AtomicCell::::is_lock_free(), + cfg!(target_has_atomic = "64") + ); + + // AtomicU128 is unstable + assert!(!AtomicCell::::is_lock_free()); +} + +#[test] +fn const_is_lock_free() { + const _U: bool = AtomicCell::::is_lock_free(); + const _I: bool = AtomicCell::::is_lock_free(); +} + +#[test] +fn drops_unit() { + static CNT: AtomicUsize = AtomicUsize::new(0); + CNT.store(0, SeqCst); + + #[derive(Debug, PartialEq, Eq)] + struct Foo(); + + impl Foo { + fn new() -> Foo { + CNT.fetch_add(1, SeqCst); + Foo() + } + } + + impl Drop for Foo { + fn drop(&mut self) { + CNT.fetch_sub(1, SeqCst); + } + } + + impl Default for Foo { + fn default() -> Foo { + Foo::new() + } + } + + let a = AtomicCell::new(Foo::new()); + + assert_eq!(a.swap(Foo::new()), Foo::new()); + assert_eq!(CNT.load(SeqCst), 1); + + a.store(Foo::new()); + assert_eq!(CNT.load(SeqCst), 1); + + assert_eq!(a.swap(Foo::default()), Foo::new()); + assert_eq!(CNT.load(SeqCst), 1); + + drop(a); + assert_eq!(CNT.load(SeqCst), 0); +} + +#[test] +fn drops_u8() { + static CNT: AtomicUsize = AtomicUsize::new(0); + CNT.store(0, SeqCst); + + #[derive(Debug, PartialEq, Eq)] + struct Foo(u8); + + impl Foo { + fn new(val: u8) -> Foo { + CNT.fetch_add(1, SeqCst); + Foo(val) + } + } + + impl Drop for Foo { + fn drop(&mut self) { + CNT.fetch_sub(1, SeqCst); + } + } + + impl Default for Foo { + fn default() -> Foo { + Foo::new(0) + } + } + + let a = AtomicCell::new(Foo::new(5)); + + assert_eq!(a.swap(Foo::new(6)), Foo::new(5)); + assert_eq!(a.swap(Foo::new(1)), Foo::new(6)); + assert_eq!(CNT.load(SeqCst), 1); + + a.store(Foo::new(2)); + assert_eq!(CNT.load(SeqCst), 1); + + assert_eq!(a.swap(Foo::default()), Foo::new(2)); + assert_eq!(CNT.load(SeqCst), 1); + + assert_eq!(a.swap(Foo::default()), Foo::new(0)); + assert_eq!(CNT.load(SeqCst), 1); + + drop(a); + assert_eq!(CNT.load(SeqCst), 0); +} + +#[test] +fn drops_usize() { + static CNT: AtomicUsize = AtomicUsize::new(0); + CNT.store(0, SeqCst); + + #[derive(Debug, PartialEq, Eq)] + struct Foo(usize); + + impl Foo { + fn new(val: usize) -> Foo { + CNT.fetch_add(1, SeqCst); + Foo(val) + } + } + + impl Drop for Foo { + fn drop(&mut self) { + CNT.fetch_sub(1, SeqCst); + } + } + + impl Default for Foo { + fn default() -> Foo { + Foo::new(0) + } + } + + let a = AtomicCell::new(Foo::new(5)); + + assert_eq!(a.swap(Foo::new(6)), Foo::new(5)); + assert_eq!(a.swap(Foo::new(1)), Foo::new(6)); + assert_eq!(CNT.load(SeqCst), 1); + + a.store(Foo::new(2)); + assert_eq!(CNT.load(SeqCst), 1); + + assert_eq!(a.swap(Foo::default()), Foo::new(2)); + assert_eq!(CNT.load(SeqCst), 1); + + assert_eq!(a.swap(Foo::default()), Foo::new(0)); + assert_eq!(CNT.load(SeqCst), 1); + + drop(a); + assert_eq!(CNT.load(SeqCst), 0); +} + +#[test] +fn modular_u8() { + #[derive(Clone, Copy, Eq, Debug, Default)] + struct Foo(u8); + + impl PartialEq for Foo { + fn eq(&self, other: &Foo) -> bool { + self.0 % 5 == other.0 % 5 + } + } + + let a = AtomicCell::new(Foo(1)); + + assert_eq!(a.load(), Foo(1)); + assert_eq!(a.swap(Foo(2)), Foo(11)); + assert_eq!(a.load(), Foo(52)); + + a.store(Foo(0)); + assert_eq!(a.compare_exchange(Foo(0), Foo(5)), Ok(Foo(100))); + assert_eq!(a.load().0, 5); + assert_eq!(a.compare_exchange(Foo(10), Foo(15)), Ok(Foo(100))); + assert_eq!(a.load().0, 15); +} + +#[test] +fn modular_usize() { + #[derive(Clone, Copy, Eq, Debug, Default)] + struct Foo(usize); + + impl PartialEq for Foo { + fn eq(&self, other: &Foo) -> bool { + self.0 % 5 == other.0 % 5 + } + } + + let a = AtomicCell::new(Foo(1)); + + assert_eq!(a.load(), Foo(1)); + assert_eq!(a.swap(Foo(2)), Foo(11)); + assert_eq!(a.load(), Foo(52)); + + a.store(Foo(0)); + assert_eq!(a.compare_exchange(Foo(0), Foo(5)), Ok(Foo(100))); + assert_eq!(a.load().0, 5); + assert_eq!(a.compare_exchange(Foo(10), Foo(15)), Ok(Foo(100))); + assert_eq!(a.load().0, 15); +} + +#[test] +fn garbage_padding() { + #[derive(Copy, Clone, Eq, PartialEq)] + struct Object { + a: i64, + b: i32, + } + + let cell = AtomicCell::new(Object { a: 0, b: 0 }); + let _garbage = [0xfe, 0xfe, 0xfe, 0xfe, 0xfe]; // Needed + let next = Object { a: 0, b: 0 }; + + let prev = cell.load(); + assert!(cell.compare_exchange(prev, next).is_ok()); + println!(); +} + +#[test] +fn const_atomic_cell_new() { + static CELL: AtomicCell = AtomicCell::new(0); + + CELL.store(1); + assert_eq!(CELL.load(), 1); +} + +// https://github.com/crossbeam-rs/crossbeam/pull/767 +macro_rules! test_arithmetic { + ($test_name:ident, $ty:ident) => { + #[test] + fn $test_name() { + let a: AtomicCell<$ty> = AtomicCell::new(7); + + assert_eq!(a.fetch_add(3), 7); + assert_eq!(a.load(), 10); + + assert_eq!(a.fetch_sub(3), 10); + assert_eq!(a.load(), 7); + + assert_eq!(a.fetch_and(3), 7); + assert_eq!(a.load(), 3); + + assert_eq!(a.fetch_or(16), 3); + assert_eq!(a.load(), 19); + + assert_eq!(a.fetch_xor(2), 19); + assert_eq!(a.load(), 17); + + assert_eq!(a.fetch_max(18), 17); + assert_eq!(a.load(), 18); + + assert_eq!(a.fetch_min(17), 18); + assert_eq!(a.load(), 17); + + assert_eq!(a.fetch_nand(7), 17); + assert_eq!(a.load(), !(17 & 7)); + } + }; +} +test_arithmetic!(arithmetic_u8, u8); +test_arithmetic!(arithmetic_i8, i8); +test_arithmetic!(arithmetic_u16, u16); +test_arithmetic!(arithmetic_i16, i16); +test_arithmetic!(arithmetic_u32, u32); +test_arithmetic!(arithmetic_i32, i32); +test_arithmetic!(arithmetic_u64, u64); +test_arithmetic!(arithmetic_i64, i64); +test_arithmetic!(arithmetic_u128, u128); +test_arithmetic!(arithmetic_i128, i128); + +// https://github.com/crossbeam-rs/crossbeam/issues/748 +#[cfg_attr(miri, ignore)] // TODO +#[test] +fn issue_748() { + #[allow(dead_code)] + #[repr(align(8))] + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + enum Test { + Field(u32), + FieldLess, + } + + assert_eq!(mem::size_of::(), 8); + assert_eq!( + AtomicCell::::is_lock_free(), + cfg!(target_has_atomic = "64") + ); + let x = AtomicCell::new(Test::FieldLess); + assert_eq!(x.load(), Test::FieldLess); +} + +// https://github.com/crossbeam-rs/crossbeam/issues/833 +#[test] +fn issue_833() { + use std::num::NonZeroU128; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::thread; + + #[cfg(miri)] + const N: usize = 10_000; + #[cfg(not(miri))] + const N: usize = 1_000_000; + + #[allow(dead_code)] + enum Enum { + NeverConstructed, + Cell(AtomicCell), + } + + static STATIC: Enum = Enum::Cell(AtomicCell::new(match NonZeroU128::new(1) { + Some(nonzero) => nonzero, + None => unreachable!(), + })); + static FINISHED: AtomicBool = AtomicBool::new(false); + + let handle = thread::spawn(|| { + let cell = match &STATIC { + Enum::NeverConstructed => unreachable!(), + Enum::Cell(cell) => cell, + }; + let x = NonZeroU128::new(0xFFFF_FFFF_FFFF_FFFF_0000_0000_0000_0000).unwrap(); + let y = NonZeroU128::new(0x0000_0000_0000_0000_FFFF_FFFF_FFFF_FFFF).unwrap(); + while !FINISHED.load(Ordering::Relaxed) { + cell.store(x); + cell.store(y); + } + }); + + for _ in 0..N { + if let Enum::NeverConstructed = STATIC { + unreachable!(":("); + } + } + + FINISHED.store(true, Ordering::Relaxed); + handle.join().unwrap(); +} diff --git a/anneal/v2/vendor/crossbeam-utils/tests/cache_padded.rs b/anneal/v2/vendor/crossbeam-utils/tests/cache_padded.rs new file mode 100644 index 0000000000..86e9a7709c --- /dev/null +++ b/anneal/v2/vendor/crossbeam-utils/tests/cache_padded.rs @@ -0,0 +1,113 @@ +use std::cell::Cell; +use std::mem; + +use crossbeam_utils::CachePadded; + +#[test] +fn default() { + let x: CachePadded = Default::default(); + assert_eq!(*x, 0); +} + +#[test] +fn store_u64() { + let x: CachePadded = CachePadded::new(17); + assert_eq!(*x, 17); +} + +#[test] +fn store_pair() { + let x: CachePadded<(u64, u64)> = CachePadded::new((17, 37)); + assert_eq!(x.0, 17); + assert_eq!(x.1, 37); +} + +#[test] +fn distance() { + let arr = [CachePadded::new(17u8), CachePadded::new(37u8)]; + let a = &*arr[0] as *const u8; + let b = &*arr[1] as *const u8; + let align = mem::align_of::>(); + assert!(align >= 32); + assert_eq!(unsafe { a.add(align) }, b); +} + +#[test] +fn different_sizes() { + CachePadded::new(17u8); + CachePadded::new(17u16); + CachePadded::new(17u32); + CachePadded::new([17u64; 0]); + CachePadded::new([17u64; 1]); + CachePadded::new([17u64; 2]); + CachePadded::new([17u64; 3]); + CachePadded::new([17u64; 4]); + CachePadded::new([17u64; 5]); + CachePadded::new([17u64; 6]); + CachePadded::new([17u64; 7]); + CachePadded::new([17u64; 8]); +} + +#[test] +fn large() { + let a = [17u64; 9]; + let b = CachePadded::new(a); + assert!(mem::size_of_val(&a) <= mem::size_of_val(&b)); +} + +#[test] +fn debug() { + assert_eq!( + format!("{:?}", CachePadded::new(17u64)), + "CachePadded { value: 17 }" + ); +} + +#[test] +fn drops() { + let count = Cell::new(0); + + struct Foo<'a>(&'a Cell); + + impl<'a> Drop for Foo<'a> { + fn drop(&mut self) { + self.0.set(self.0.get() + 1); + } + } + + let a = CachePadded::new(Foo(&count)); + let b = CachePadded::new(Foo(&count)); + + assert_eq!(count.get(), 0); + drop(a); + assert_eq!(count.get(), 1); + drop(b); + assert_eq!(count.get(), 2); +} + +#[allow(clippy::clone_on_copy)] // This is intentional. +#[test] +fn clone() { + let a = CachePadded::new(17); + let b = a.clone(); + assert_eq!(*a, *b); +} + +#[test] +fn runs_custom_clone() { + let count = Cell::new(0); + + struct Foo<'a>(&'a Cell); + + impl<'a> Clone for Foo<'a> { + fn clone(&self) -> Foo<'a> { + self.0.set(self.0.get() + 1); + Foo::<'a>(self.0) + } + } + + let a = CachePadded::new(Foo(&count)); + let _ = a.clone(); + + assert_eq!(count.get(), 1); +} diff --git a/anneal/v2/vendor/crossbeam-utils/tests/parker.rs b/anneal/v2/vendor/crossbeam-utils/tests/parker.rs new file mode 100644 index 0000000000..2bf9c37d49 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-utils/tests/parker.rs @@ -0,0 +1,41 @@ +use std::thread::sleep; +use std::time::Duration; +use std::u32; + +use crossbeam_utils::sync::Parker; +use crossbeam_utils::thread; + +#[test] +fn park_timeout_unpark_before() { + let p = Parker::new(); + for _ in 0..10 { + p.unparker().unpark(); + p.park_timeout(Duration::from_millis(u32::MAX as u64)); + } +} + +#[test] +fn park_timeout_unpark_not_called() { + let p = Parker::new(); + for _ in 0..10 { + p.park_timeout(Duration::from_millis(10)) + } +} + +#[test] +fn park_timeout_unpark_called_other_thread() { + for _ in 0..10 { + let p = Parker::new(); + let u = p.unparker().clone(); + + thread::scope(|scope| { + scope.spawn(move |_| { + sleep(Duration::from_millis(50)); + u.unpark(); + }); + + p.park_timeout(Duration::from_millis(u32::MAX as u64)) + }) + .unwrap(); + } +} diff --git a/anneal/v2/vendor/crossbeam-utils/tests/sharded_lock.rs b/anneal/v2/vendor/crossbeam-utils/tests/sharded_lock.rs new file mode 100644 index 0000000000..002f7f5e19 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-utils/tests/sharded_lock.rs @@ -0,0 +1,252 @@ +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::mpsc::channel; +use std::sync::{Arc, TryLockError}; +use std::thread; + +use crossbeam_utils::sync::ShardedLock; +use rand::Rng; + +#[derive(Eq, PartialEq, Debug)] +struct NonCopy(i32); + +#[test] +fn smoke() { + let l = ShardedLock::new(()); + drop(l.read().unwrap()); + drop(l.write().unwrap()); + drop((l.read().unwrap(), l.read().unwrap())); + drop(l.write().unwrap()); +} + +#[test] +fn frob() { + const N: u32 = 10; + #[cfg(miri)] + const M: usize = 50; + #[cfg(not(miri))] + const M: usize = 1000; + + let r = Arc::new(ShardedLock::new(())); + + let (tx, rx) = channel::<()>(); + for _ in 0..N { + let tx = tx.clone(); + let r = r.clone(); + thread::spawn(move || { + let mut rng = rand::thread_rng(); + for _ in 0..M { + if rng.gen_bool(1.0 / (N as f64)) { + drop(r.write().unwrap()); + } else { + drop(r.read().unwrap()); + } + } + drop(tx); + }); + } + drop(tx); + let _ = rx.recv(); +} + +#[test] +fn arc_poison_wr() { + let arc = Arc::new(ShardedLock::new(1)); + let arc2 = arc.clone(); + let _: Result<(), _> = thread::spawn(move || { + let _lock = arc2.write().unwrap(); + panic!(); + }) + .join(); + assert!(arc.read().is_err()); +} + +#[test] +fn arc_poison_ww() { + let arc = Arc::new(ShardedLock::new(1)); + assert!(!arc.is_poisoned()); + let arc2 = arc.clone(); + let _: Result<(), _> = thread::spawn(move || { + let _lock = arc2.write().unwrap(); + panic!(); + }) + .join(); + assert!(arc.write().is_err()); + assert!(arc.is_poisoned()); +} + +#[test] +fn arc_no_poison_rr() { + let arc = Arc::new(ShardedLock::new(1)); + let arc2 = arc.clone(); + let _: Result<(), _> = thread::spawn(move || { + let _lock = arc2.read().unwrap(); + panic!(); + }) + .join(); + let lock = arc.read().unwrap(); + assert_eq!(*lock, 1); +} +#[test] +fn arc_no_poison_sl() { + let arc = Arc::new(ShardedLock::new(1)); + let arc2 = arc.clone(); + let _: Result<(), _> = thread::spawn(move || { + let _lock = arc2.read().unwrap(); + panic!() + }) + .join(); + let lock = arc.write().unwrap(); + assert_eq!(*lock, 1); +} + +#[test] +fn arc() { + let arc = Arc::new(ShardedLock::new(0)); + let arc2 = arc.clone(); + let (tx, rx) = channel(); + + thread::spawn(move || { + let mut lock = arc2.write().unwrap(); + for _ in 0..10 { + let tmp = *lock; + *lock = -1; + thread::yield_now(); + *lock = tmp + 1; + } + tx.send(()).unwrap(); + }); + + // Readers try to catch the writer in the act + let mut children = Vec::new(); + for _ in 0..5 { + let arc3 = arc.clone(); + children.push(thread::spawn(move || { + let lock = arc3.read().unwrap(); + assert!(*lock >= 0); + })); + } + + // Wait for children to pass their asserts + for r in children { + assert!(r.join().is_ok()); + } + + // Wait for writer to finish + rx.recv().unwrap(); + let lock = arc.read().unwrap(); + assert_eq!(*lock, 10); +} + +#[test] +fn arc_access_in_unwind() { + let arc = Arc::new(ShardedLock::new(1)); + let arc2 = arc.clone(); + let _ = thread::spawn(move || { + struct Unwinder { + i: Arc>, + } + impl Drop for Unwinder { + fn drop(&mut self) { + let mut lock = self.i.write().unwrap(); + *lock += 1; + } + } + let _u = Unwinder { i: arc2 }; + panic!(); + }) + .join(); + let lock = arc.read().unwrap(); + assert_eq!(*lock, 2); +} + +#[test] +fn unsized_type() { + let sl: &ShardedLock<[i32]> = &ShardedLock::new([1, 2, 3]); + { + let b = &mut *sl.write().unwrap(); + b[0] = 4; + b[2] = 5; + } + let comp: &[i32] = &[4, 2, 5]; + assert_eq!(&*sl.read().unwrap(), comp); +} + +#[test] +fn try_write() { + let lock = ShardedLock::new(0isize); + let read_guard = lock.read().unwrap(); + + let write_result = lock.try_write(); + match write_result { + Err(TryLockError::WouldBlock) => (), + Ok(_) => panic!("try_write should not succeed while read_guard is in scope"), + Err(_) => panic!("unexpected error"), + } + + drop(read_guard); +} + +#[test] +fn test_into_inner() { + let m = ShardedLock::new(NonCopy(10)); + assert_eq!(m.into_inner().unwrap(), NonCopy(10)); +} + +#[test] +fn test_into_inner_drop() { + struct Foo(Arc); + impl Drop for Foo { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + let num_drops = Arc::new(AtomicUsize::new(0)); + let m = ShardedLock::new(Foo(num_drops.clone())); + assert_eq!(num_drops.load(Ordering::SeqCst), 0); + { + let _inner = m.into_inner().unwrap(); + assert_eq!(num_drops.load(Ordering::SeqCst), 0); + } + assert_eq!(num_drops.load(Ordering::SeqCst), 1); +} + +#[test] +fn test_into_inner_poison() { + let m = Arc::new(ShardedLock::new(NonCopy(10))); + let m2 = m.clone(); + let _ = thread::spawn(move || { + let _lock = m2.write().unwrap(); + panic!("test panic in inner thread to poison ShardedLock"); + }) + .join(); + + assert!(m.is_poisoned()); + match Arc::try_unwrap(m).unwrap().into_inner() { + Err(e) => assert_eq!(e.into_inner(), NonCopy(10)), + Ok(x) => panic!("into_inner of poisoned ShardedLock is Ok: {:?}", x), + } +} + +#[test] +fn test_get_mut() { + let mut m = ShardedLock::new(NonCopy(10)); + *m.get_mut().unwrap() = NonCopy(20); + assert_eq!(m.into_inner().unwrap(), NonCopy(20)); +} + +#[test] +fn test_get_mut_poison() { + let m = Arc::new(ShardedLock::new(NonCopy(10))); + let m2 = m.clone(); + let _ = thread::spawn(move || { + let _lock = m2.write().unwrap(); + panic!("test panic in inner thread to poison ShardedLock"); + }) + .join(); + + assert!(m.is_poisoned()); + match Arc::try_unwrap(m).unwrap().get_mut() { + Err(e) => assert_eq!(*e.into_inner(), NonCopy(10)), + Ok(x) => panic!("get_mut of poisoned ShardedLock is Ok: {:?}", x), + } +} diff --git a/anneal/v2/vendor/crossbeam-utils/tests/thread.rs b/anneal/v2/vendor/crossbeam-utils/tests/thread.rs new file mode 100644 index 0000000000..0dfad90bd6 --- /dev/null +++ b/anneal/v2/vendor/crossbeam-utils/tests/thread.rs @@ -0,0 +1,215 @@ +use std::any::Any; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::thread::sleep; +use std::time::Duration; + +use crossbeam_utils::thread; + +const THREADS: usize = 10; +const SMALL_STACK_SIZE: usize = 20; + +#[test] +fn join() { + let counter = AtomicUsize::new(0); + thread::scope(|scope| { + let handle = scope.spawn(|_| { + counter.store(1, Ordering::Relaxed); + }); + assert!(handle.join().is_ok()); + + let panic_handle = scope.spawn(|_| { + panic!("\"My honey is running out!\", said Pooh."); + }); + assert!(panic_handle.join().is_err()); + }) + .unwrap(); + + // There should be sufficient synchronization. + assert_eq!(1, counter.load(Ordering::Relaxed)); +} + +#[test] +fn counter() { + let counter = AtomicUsize::new(0); + thread::scope(|scope| { + for _ in 0..THREADS { + scope.spawn(|_| { + counter.fetch_add(1, Ordering::Relaxed); + }); + } + }) + .unwrap(); + + assert_eq!(THREADS, counter.load(Ordering::Relaxed)); +} + +#[test] +fn counter_builder() { + let counter = AtomicUsize::new(0); + thread::scope(|scope| { + for i in 0..THREADS { + scope + .builder() + .name(format!("child-{}", i)) + .stack_size(SMALL_STACK_SIZE) + .spawn(|_| { + counter.fetch_add(1, Ordering::Relaxed); + }) + .unwrap(); + } + }) + .unwrap(); + + assert_eq!(THREADS, counter.load(Ordering::Relaxed)); +} + +#[test] +fn counter_panic() { + let counter = AtomicUsize::new(0); + let result = thread::scope(|scope| { + scope.spawn(|_| { + panic!("\"My honey is running out!\", said Pooh."); + }); + sleep(Duration::from_millis(100)); + + for _ in 0..THREADS { + scope.spawn(|_| { + counter.fetch_add(1, Ordering::Relaxed); + }); + } + }); + + assert_eq!(THREADS, counter.load(Ordering::Relaxed)); + assert!(result.is_err()); +} + +#[test] +fn panic_twice() { + let result = thread::scope(|scope| { + scope.spawn(|_| { + sleep(Duration::from_millis(500)); + panic!("thread #1"); + }); + scope.spawn(|_| { + panic!("thread #2"); + }); + }); + + let err = result.unwrap_err(); + let vec = err + .downcast_ref::>>() + .unwrap(); + assert_eq!(2, vec.len()); + + let first = vec[0].downcast_ref::<&str>().unwrap(); + let second = vec[1].downcast_ref::<&str>().unwrap(); + assert_eq!("thread #1", *first); + assert_eq!("thread #2", *second) +} + +#[test] +fn panic_many() { + let result = thread::scope(|scope| { + scope.spawn(|_| panic!("deliberate panic #1")); + scope.spawn(|_| panic!("deliberate panic #2")); + scope.spawn(|_| panic!("deliberate panic #3")); + }); + + let err = result.unwrap_err(); + let vec = err + .downcast_ref::>>() + .unwrap(); + assert_eq!(3, vec.len()); + + for panic in vec.iter() { + let panic = panic.downcast_ref::<&str>().unwrap(); + assert!( + *panic == "deliberate panic #1" + || *panic == "deliberate panic #2" + || *panic == "deliberate panic #3" + ); + } +} + +#[test] +fn nesting() { + let var = "foo".to_string(); + + struct Wrapper<'a> { + var: &'a String, + } + + impl<'a> Wrapper<'a> { + fn recurse(&'a self, scope: &thread::Scope<'a>, depth: usize) { + assert_eq!(self.var, "foo"); + + if depth > 0 { + scope.spawn(move |scope| { + self.recurse(scope, depth - 1); + }); + } + } + } + + let wrapper = Wrapper { var: &var }; + + thread::scope(|scope| { + scope.spawn(|scope| { + scope.spawn(|scope| { + wrapper.recurse(scope, 5); + }); + }); + }) + .unwrap(); +} + +#[test] +fn join_nested() { + thread::scope(|scope| { + scope.spawn(|scope| { + let handle = scope.spawn(|_| 7); + + sleep(Duration::from_millis(200)); + handle.join().unwrap(); + }); + + sleep(Duration::from_millis(100)); + }) + .unwrap(); +} + +#[test] +fn scope_returns_ok() { + let result = thread::scope(|scope| scope.spawn(|_| 1234).join().unwrap()).unwrap(); + assert_eq!(result, 1234); +} + +#[cfg(unix)] +#[test] +fn as_pthread_t() { + use std::os::unix::thread::JoinHandleExt; + thread::scope(|scope| { + let handle = scope.spawn(|_scope| { + sleep(Duration::from_millis(100)); + 42 + }); + let _pthread_t = handle.as_pthread_t(); + handle.join().unwrap(); + }) + .unwrap(); +} + +#[cfg(windows)] +#[test] +fn as_raw_handle() { + use std::os::windows::io::AsRawHandle; + thread::scope(|scope| { + let handle = scope.spawn(|_scope| { + sleep(Duration::from_millis(100)); + 42 + }); + let _raw_handle = handle.as_raw_handle(); + handle.join().unwrap(); + }) + .unwrap(); +} diff --git a/anneal/v2/vendor/crossbeam-utils/tests/wait_group.rs b/anneal/v2/vendor/crossbeam-utils/tests/wait_group.rs new file mode 100644 index 0000000000..5b549b849c --- /dev/null +++ b/anneal/v2/vendor/crossbeam-utils/tests/wait_group.rs @@ -0,0 +1,67 @@ +use std::sync::mpsc; +use std::thread; +use std::time::Duration; + +use crossbeam_utils::sync::WaitGroup; + +const THREADS: usize = 10; + +#[test] +fn wait() { + let wg = WaitGroup::new(); + let (tx, rx) = mpsc::channel(); + + for _ in 0..THREADS { + let wg = wg.clone(); + let tx = tx.clone(); + + thread::spawn(move || { + wg.wait(); + tx.send(()).unwrap(); + }); + } + + thread::sleep(Duration::from_millis(100)); + + // At this point, all spawned threads should be blocked, so we shouldn't get anything from the + // channel. + assert!(rx.try_recv().is_err()); + + wg.wait(); + + // Now, the wait group is cleared and we should receive messages. + for _ in 0..THREADS { + rx.recv().unwrap(); + } +} + +#[test] +fn wait_and_drop() { + let wg = WaitGroup::new(); + let wg2 = WaitGroup::new(); + let (tx, rx) = mpsc::channel(); + + for _ in 0..THREADS { + let wg = wg.clone(); + let wg2 = wg2.clone(); + let tx = tx.clone(); + + thread::spawn(move || { + wg2.wait(); + tx.send(()).unwrap(); + drop(wg); + }); + } + + // At this point, no thread has gotten past `wg2.wait()`, so we shouldn't get anything from the + // channel. + assert!(rx.try_recv().is_err()); + drop(wg2); + + wg.wait(); + + // Now, the wait group is cleared and we should receive messages. + for _ in 0..THREADS { + rx.try_recv().unwrap(); + } +} diff --git a/anneal/v2/vendor/either/.cargo-checksum.json b/anneal/v2/vendor/either/.cargo-checksum.json new file mode 100644 index 0000000000..9915491767 --- /dev/null +++ b/anneal/v2/vendor/either/.cargo-checksum.json @@ -0,0 +1 @@ +{"files":{".cargo_vcs_info.json":"3f0041c1992d886f8f19441bd5fd859e3a68cbc9f91d006494ec4bd587653bfc",".github/workflows/ci.yml":"d2da89285c13788a6a355b9ad5fdd84941b3982e395270cb809b64d9be8c9c1f","Cargo.lock":"47e4ddc10d96119e10d5172d19a7eef277f18e93c7a9e7a48e28f0e822b422fa","Cargo.toml":"1f4d971b97901d1420a508ca4c9cc7ec3e2114c782500e5192801509cf043725","Cargo.toml.orig":"f82350f4effc4946b40f8d37e3db28488319cf4def755d00f5687f7cf768621a","LICENSE-APACHE":"a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2","LICENSE-MIT":"7576269ea71f767b99297934c0b2367532690f8c4badc695edf8e04ab6a1e545","README-crates.io.md":"b775991a01ab4a0a8de6169f597775319d9ce8178f5c74ccdc634f13a286b20c","README.rst":"01c248679713f4109b8aedb904bb79edef054e477fce9ab260921b1c0861c70a","src/into_either.rs":"0477f226bbba78ef017de08b87d421d3cd99fbc95b90ba4e6e3e803e3d15254e","src/iterator.rs":"3dd8a1255e1448eeee20b24de849523a13ca0d3c178b81d1d5adc6220ca692c4","src/lib.rs":"481ca250233811a5ccbff9a9fc7c2acb15287aede47d8ec280484fe5209e2ad7","src/serde_untagged.rs":"5f7a4035ca164125a433f42ac0949185d1732e2fcf5b1ac6dfca1e5aede439bb","src/serde_untagged_optional.rs":"ba5d6c2213388b69e127e763b6e394de6388bdd682ffb5d5696b068f1938ec79"},"package":"91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e"} \ No newline at end of file diff --git a/anneal/v2/vendor/either/.cargo_vcs_info.json b/anneal/v2/vendor/either/.cargo_vcs_info.json new file mode 100644 index 0000000000..148bd94dd5 --- /dev/null +++ b/anneal/v2/vendor/either/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "8f4ecd9964ea909a7f989a3668a2710310a0e62f" + }, + "path_in_vcs": "" +} \ No newline at end of file diff --git a/anneal/v2/vendor/either/.github/workflows/ci.yml b/anneal/v2/vendor/either/.github/workflows/ci.yml new file mode 100644 index 0000000000..1c6dbb0a5e --- /dev/null +++ b/anneal/v2/vendor/either/.github/workflows/ci.yml @@ -0,0 +1,90 @@ +on: + push: + branches: [ main ] + pull_request: + merge_group: + +name: CI + +jobs: + ci: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + rust: + - 1.63.0 # MSRV + - stable + - beta + - nightly + features: + - "" + - "serde" + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Cache the registry + uses: actions/cache@v4 + if: startsWith(matrix.rust, '1') + with: + path: ~/.cargo/registry/index + key: cargo-${{ matrix.rust }}-git-index + + - name: Lock MSRV-compatible dependencies + if: startsWith(matrix.rust, '1') + env: + CARGO_RESOLVER_INCOMPATIBLE_RUST_VERSIONS: fallback + # Note that this uses the runner's pre-installed stable cargo + run: cargo generate-lockfile + + - name: Set up Rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ matrix.rust }} + + - name: Build (no_std) + run: cargo build --no-default-features --features "${{ matrix.features }}" + + - name: Build + run: cargo build --features "${{ matrix.features }}" + + - name: Test + run: cargo test --features "${{ matrix.features }}" + + - name: Doc + run: cargo doc --features "${{ matrix.features }}" + + clippy: + name: Rustfmt and Clippy + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up nightly Rust + uses: dtolnay/rust-toolchain@nightly + with: + components: rustfmt, clippy + + - name: Rustfmt + run: cargo fmt --all -- --check + + - name: Clippy + run: cargo clippy # -- -D warnings + + # One job that "summarizes" the success state of this pipeline. This can then be added to branch + # protection, rather than having to add each job separately. + success: + name: Success + runs-on: ubuntu-latest + needs: [ci, clippy] + # Github branch protection is exceedingly silly and treats "jobs skipped because a dependency + # failed" as success. So we have to do some contortions to ensure the job fails if any of its + # dependencies fails. + if: always() # make sure this is never "skipped" + steps: + # Manually check the status of all dependencies. `if: failure()` does not work. + - name: check if any dependency failed + run: jq --exit-status 'all(.result == "success")' <<< '${{ toJson(needs) }}' diff --git a/anneal/v2/vendor/either/Cargo.lock b/anneal/v2/vendor/either/Cargo.lock new file mode 100644 index 0000000000..3064f8b7e5 --- /dev/null +++ b/anneal/v2/vendor/either/Cargo.lock @@ -0,0 +1,89 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "either" +version = "1.16.0" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "itoa" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" + +[[package]] +name = "proc-macro2" +version = "1.0.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75cb1540fadbd5b8fbccc4dddad2734eba435053f725621c070711a14bb5f4b8" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ryu" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" + +[[package]] +name = "serde" +version = "1.0.193" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.193" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "syn" +version = "2.0.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee659fb5f3d355364e1f3e5bc10fb82068efbf824a1e9d1c9504244a6469ad53" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" diff --git a/anneal/v2/vendor/either/Cargo.toml b/anneal/v2/vendor/either/Cargo.toml new file mode 100644 index 0000000000..15de8104b8 --- /dev/null +++ b/anneal/v2/vendor/either/Cargo.toml @@ -0,0 +1,69 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +rust-version = "1.63.0" +name = "either" +version = "1.16.0" +build = false +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = """ +The enum `Either` with variants `Left` and `Right` is a general purpose sum type with two cases. +""" +documentation = "https://docs.rs/either/1/" +readme = "README-crates.io.md" +keywords = [ + "data-structure", + "no_std", +] +categories = [ + "data-structures", + "no-std", +] +license = "MIT OR Apache-2.0" +repository = "https://github.com/rayon-rs/either" + +[package.metadata.release] +allow-branch = ["main"] +sign-tag = true +tag-name = "{{version}}" + +[package.metadata.docs.rs] +features = ["serde"] + +[package.metadata.playground] +features = ["serde"] + +[features] +default = ["std"] +std = [] +use_std = ["std"] + +[lib] +name = "either" +path = "src/lib.rs" + +[dependencies.serde] +version = "1.0.95" +features = [ + "alloc", + "derive", +] +optional = true +default-features = false + +[dev-dependencies.serde_json] +version = "1.0.0" diff --git a/anneal/v2/vendor/either/Cargo.toml.orig b/anneal/v2/vendor/either/Cargo.toml.orig new file mode 100644 index 0000000000..fc46d174ec --- /dev/null +++ b/anneal/v2/vendor/either/Cargo.toml.orig @@ -0,0 +1,39 @@ +[package] +name = "either" +version = "1.16.0" +edition = "2021" +rust-version = "1.63.0" + +license = "MIT OR Apache-2.0" +repository = "https://github.com/rayon-rs/either" +documentation = "https://docs.rs/either/1/" +readme = "README-crates.io.md" + +description = """ +The enum `Either` with variants `Left` and `Right` is a general purpose sum type with two cases. +""" + +keywords = ["data-structure", "no_std"] +categories = ["data-structures", "no-std"] + +[dependencies] +serde = { version = "1.0.95", optional = true, default-features = false, features = ["alloc", "derive"] } + +[features] +default = ["std"] +std = [] +use_std = ["std"] # deprecated alias + +[dev-dependencies] +serde_json = "1.0.0" + +[package.metadata.release] +allow-branch = ["main"] +sign-tag = true +tag-name = "{{version}}" + +[package.metadata.docs.rs] +features = ["serde"] + +[package.metadata.playground] +features = ["serde"] diff --git a/anneal/v2/vendor/either/LICENSE-APACHE b/anneal/v2/vendor/either/LICENSE-APACHE new file mode 100644 index 0000000000..16fe87b06e --- /dev/null +++ b/anneal/v2/vendor/either/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/anneal/v2/vendor/either/LICENSE-MIT b/anneal/v2/vendor/either/LICENSE-MIT new file mode 100644 index 0000000000..9203baa055 --- /dev/null +++ b/anneal/v2/vendor/either/LICENSE-MIT @@ -0,0 +1,25 @@ +Copyright (c) 2015 + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/anneal/v2/vendor/either/README-crates.io.md b/anneal/v2/vendor/either/README-crates.io.md new file mode 100644 index 0000000000..d36890278b --- /dev/null +++ b/anneal/v2/vendor/either/README-crates.io.md @@ -0,0 +1,10 @@ +The enum `Either` with variants `Left` and `Right` is a general purpose +sum type with two cases. + +Either has methods that are similar to Option and Result, and it also implements +traits like `Iterator`. + +Includes macros `try_left!()` and `try_right!()` to use for +short-circuiting logic, similar to how the `?` operator is used with `Result`. +Note that `Either` is general purpose. For describing success or error, use the +regular `Result`. diff --git a/anneal/v2/vendor/either/README.rst b/anneal/v2/vendor/either/README.rst new file mode 100644 index 0000000000..10f392a75b --- /dev/null +++ b/anneal/v2/vendor/either/README.rst @@ -0,0 +1,220 @@ + +Either +====== + +The enum ``Either`` with variants ``Left`` and ``Right`` and trait +implementations including Iterator, Read, Write. + +Either has methods that are similar to Option and Result. + +Includes convenience macros ``try_left!()`` and ``try_right!()`` to use for +short-circuiting logic. + +Please read the `API documentation here`__ + +__ https://docs.rs/either/ + +|build_status|_ |crates|_ + +.. |build_status| image:: https://github.com/rayon-rs/either/workflows/CI/badge.svg?branch=main +.. _build_status: https://github.com/rayon-rs/either/actions + +.. |crates| image:: https://img.shields.io/crates/v/either.svg +.. _crates: https://crates.io/crates/either + +How to use with cargo:: + + [dependencies] + either = "1" + + +Recent Changes +-------------- + +- 1.16.0 + + - Add many new methods dealing with each side, by @A4-Tacks: + + - ``inspect_left`` and ``inspect_right`` (#124) + + - ``left_and`` and ``right_and`` (#125) + + - ``is_left_and`` and ``is_right_and`` (#126) + + - ``map_left_or`` and ``map_right_or`` (#127) + + - Add a version of ``for_both!`` with a single ``ident``, by @A4-Tacks (#128) + + - Add a ``map_both!`` macro, by @JohnScience (#109) and @ronnodas (#137) + +- 1.15.0 + + - Fix ``serde`` support when building without ``std``, by @klkvr (#119) + + - Use a more common ``std`` feature for default enablement, deprecating + the ``use_std`` feature as a mere alias of the new name. + +- 1.14.0 + + - **MSRV**: ``either`` now requires Rust 1.63 or later. + + - Implement ``fmt::Write`` for ``Either``, by @yotamofek (#113) + + - Replace ``Into for Either`` with ``From for Result``, by @cuviper (#118) + +- 1.13.0 + + - Add new methods ``.cloned()`` and ``.copied()``, by @ColonelThirtyTwo (#107) + +- 1.12.0 + + - **MSRV**: ``either`` now requires Rust 1.37 or later. + + - Specialize ``nth_back`` for ``Either`` and ``IterEither``, by @cuviper (#106) + +- 1.11.0 + + - Add new trait ``IntoEither`` that is useful to convert to ``Either`` in method chains, + by @SFM61319 (#101) + +- 1.10.0 + + - Add new methods ``.factor_iter()``, ``.factor_iter_mut()``, and ``.factor_into_iter()`` + that return ``Either`` items, plus ``.iter()`` and ``.iter_mut()`` to convert to direct + reference iterators; by @aj-bagwell and @cuviper (#91) + +- 1.9.0 + + - Add new methods ``.map_either()`` and ``.map_either_with()``, by @nasadorian (#82) + +- 1.8.1 + + - Clarified that the multiple licenses are combined with OR. + +- 1.8.0 + + - **MSRV**: ``either`` now requires Rust 1.36 or later. + + - Add new methods ``.as_pin_ref()`` and ``.as_pin_mut()`` to project a + pinned ``Either`` as inner ``Pin`` variants, by @cuviper (#77) + + - Implement the ``Future`` trait, by @cuviper (#77) + + - Specialize more methods of the ``io`` traits, by @Kixunil and @cuviper (#75) + +- 1.7.0 + + - **MSRV**: ``either`` now requires Rust 1.31 or later. + + - Export the macro ``for_both!``, by @thomaseizinger (#58) + + - Implement the ``io::Seek`` trait, by @Kerollmops (#60) + + - Add new method ``.either_into()`` for ``Into`` conversion, by @TonalidadeHidrica (#63) + + - Add new methods ``.factor_ok()``, ``.factor_err()``, and ``.factor_none()``, + by @zachs18 (#67) + + - Specialize ``source`` in the ``Error`` implementation, by @thomaseizinger (#69) + + - Specialize more iterator methods and implement the ``FusedIterator`` trait, + by @Ten0 (#66) and @cuviper (#71) + + - Specialize ``Clone::clone_from``, by @cuviper (#72) + +- 1.6.1 + + - Add new methods ``.expect_left()``, ``.unwrap_left()``, + and equivalents on the right, by @spenserblack (#51) + +- 1.6.0 + + - Add new modules ``serde_untagged`` and ``serde_untagged_optional`` to customize + how ``Either`` fields are serialized in other types, by @MikailBag (#49) + +- 1.5.3 + + - Add new method ``.map()`` for ``Either`` by @nvzqz (#40). + +- 1.5.2 + + - Add new methods ``.left_or()``, ``.left_or_default()``, ``.left_or_else()``, + and equivalents on the right, by @DCjanus (#36) + +- 1.5.1 + + - Add ``AsRef`` and ``AsMut`` implementations for common unsized types: + ``str``, ``[T]``, ``CStr``, ``OsStr``, and ``Path``, by @mexus (#29) + +- 1.5.0 + + - Add new methods ``.factor_first()``, ``.factor_second()`` and ``.into_inner()`` + by @mathstuf (#19) + +- 1.4.0 + + - Add inherent method ``.into_iter()`` by @cuviper (#12) + +- 1.3.0 + + - Add opt-in serde support by @hcpl + +- 1.2.0 + + - Add method ``.either_with()`` by @Twey (#13) + +- 1.1.0 + + - Add methods ``left_and_then``, ``right_and_then`` by @rampantmonkey + - Include license files in the repository and released crate + +- 1.0.3 + + - Add crate categories + +- 1.0.2 + + - Forward more ``Iterator`` methods + - Implement ``Extend`` for ``Either`` if ``L, R`` do. + +- 1.0.1 + + - Fix ``Iterator`` impl for ``Either`` to forward ``.fold()``. + +- 1.0.0 + + - Add default crate feature ``use_std`` so that you can opt out of linking to + std. + +- 0.1.7 + + - Add methods ``.map_left()``, ``.map_right()`` and ``.either()``. + - Add more documentation + +- 0.1.3 + + - Implement Display, Error + +- 0.1.2 + + - Add macros ``try_left!`` and ``try_right!``. + +- 0.1.1 + + - Implement Deref, DerefMut + +- 0.1.0 + + - Initial release + - Support Iterator, Read, Write + +License +------- + +Dual-licensed to be compatible with the Rust project. + +Licensed under the Apache License, Version 2.0 +https://www.apache.org/licenses/LICENSE-2.0 or the MIT license +https://opensource.org/licenses/MIT, at your +option. This file may not be copied, modified, or distributed +except according to those terms. diff --git a/anneal/v2/vendor/either/src/into_either.rs b/anneal/v2/vendor/either/src/into_either.rs new file mode 100644 index 0000000000..73746c80f1 --- /dev/null +++ b/anneal/v2/vendor/either/src/into_either.rs @@ -0,0 +1,64 @@ +//! The trait [`IntoEither`] provides methods for converting a type `Self`, whose +//! size is constant and known at compile-time, into an [`Either`] variant. + +use super::{Either, Left, Right}; + +/// Provides methods for converting a type `Self` into either a [`Left`] or [`Right`] +/// variant of [`Either`](Either). +/// +/// The [`into_either`](IntoEither::into_either) method takes a [`bool`] to determine +/// whether to convert to [`Left`] or [`Right`]. +/// +/// The [`into_either_with`](IntoEither::into_either_with) method takes a +/// [predicate function](FnOnce) to determine whether to convert to [`Left`] or [`Right`]. +pub trait IntoEither: Sized { + /// Converts `self` into a [`Left`] variant of [`Either`](Either) + /// if `into_left` is `true`. + /// Converts `self` into a [`Right`] variant of [`Either`](Either) + /// otherwise. + /// + /// # Examples + /// + /// ``` + /// use either::{IntoEither, Left, Right}; + /// + /// let x = 0; + /// assert_eq!(x.into_either(true), Left(x)); + /// assert_eq!(x.into_either(false), Right(x)); + /// ``` + fn into_either(self, into_left: bool) -> Either { + if into_left { + Left(self) + } else { + Right(self) + } + } + + /// Converts `self` into a [`Left`] variant of [`Either`](Either) + /// if `into_left(&self)` returns `true`. + /// Converts `self` into a [`Right`] variant of [`Either`](Either) + /// otherwise. + /// + /// # Examples + /// + /// ``` + /// use either::{IntoEither, Left, Right}; + /// + /// fn is_even(x: &u8) -> bool { + /// x % 2 == 0 + /// } + /// + /// let x = 0; + /// assert_eq!(x.into_either_with(is_even), Left(x)); + /// assert_eq!(x.into_either_with(|x| !is_even(x)), Right(x)); + /// ``` + fn into_either_with(self, into_left: F) -> Either + where + F: FnOnce(&Self) -> bool, + { + let into_left = into_left(&self); + self.into_either(into_left) + } +} + +impl IntoEither for T {} diff --git a/anneal/v2/vendor/either/src/iterator.rs b/anneal/v2/vendor/either/src/iterator.rs new file mode 100644 index 0000000000..a5535f0530 --- /dev/null +++ b/anneal/v2/vendor/either/src/iterator.rs @@ -0,0 +1,315 @@ +use super::{Either, Left, Right}; +use core::iter; + +macro_rules! wrap_either { + ($value:expr => $( $tail:tt )*) => { + match $value { + Left(inner) => inner.map(Left) $($tail)*, + Right(inner) => inner.map(Right) $($tail)*, + } + }; +} + +/// Iterator that maps left or right iterators to corresponding `Either`-wrapped items. +/// +/// This struct is created by the [`Either::factor_into_iter`], +/// [`factor_iter`][Either::factor_iter], +/// and [`factor_iter_mut`][Either::factor_iter_mut] methods. +#[derive(Clone, Debug)] +pub struct IterEither { + inner: Either, +} + +impl IterEither { + pub(crate) fn new(inner: Either) -> Self { + IterEither { inner } + } +} + +impl Extend for Either +where + L: Extend, + R: Extend, +{ + fn extend(&mut self, iter: T) + where + T: IntoIterator, + { + for_both!(self, inner => inner.extend(iter)) + } +} + +/// `Either` is an iterator if both `L` and `R` are iterators. +impl Iterator for Either +where + L: Iterator, + R: Iterator, +{ + type Item = L::Item; + + fn next(&mut self) -> Option { + for_both!(self, inner => inner.next()) + } + + fn size_hint(&self) -> (usize, Option) { + for_both!(self, inner => inner.size_hint()) + } + + fn fold(self, init: Acc, f: G) -> Acc + where + G: FnMut(Acc, Self::Item) -> Acc, + { + for_both!(self, inner => inner.fold(init, f)) + } + + fn for_each(self, f: F) + where + F: FnMut(Self::Item), + { + for_both!(self, inner => inner.for_each(f)) + } + + fn count(self) -> usize { + for_both!(self, inner => inner.count()) + } + + fn last(self) -> Option { + for_both!(self, inner => inner.last()) + } + + fn nth(&mut self, n: usize) -> Option { + for_both!(self, inner => inner.nth(n)) + } + + fn collect(self) -> B + where + B: iter::FromIterator, + { + for_both!(self, inner => inner.collect()) + } + + fn partition(self, f: F) -> (B, B) + where + B: Default + Extend, + F: FnMut(&Self::Item) -> bool, + { + for_both!(self, inner => inner.partition(f)) + } + + fn all(&mut self, f: F) -> bool + where + F: FnMut(Self::Item) -> bool, + { + for_both!(self, inner => inner.all(f)) + } + + fn any(&mut self, f: F) -> bool + where + F: FnMut(Self::Item) -> bool, + { + for_both!(self, inner => inner.any(f)) + } + + fn find

(&mut self, predicate: P) -> Option + where + P: FnMut(&Self::Item) -> bool, + { + for_both!(self, inner => inner.find(predicate)) + } + + fn find_map(&mut self, f: F) -> Option + where + F: FnMut(Self::Item) -> Option, + { + for_both!(self, inner => inner.find_map(f)) + } + + fn position

(&mut self, predicate: P) -> Option + where + P: FnMut(Self::Item) -> bool, + { + for_both!(self, inner => inner.position(predicate)) + } +} + +impl DoubleEndedIterator for Either +where + L: DoubleEndedIterator, + R: DoubleEndedIterator, +{ + fn next_back(&mut self) -> Option { + for_both!(self, inner => inner.next_back()) + } + + fn nth_back(&mut self, n: usize) -> Option { + for_both!(self, inner => inner.nth_back(n)) + } + + fn rfold(self, init: Acc, f: G) -> Acc + where + G: FnMut(Acc, Self::Item) -> Acc, + { + for_both!(self, inner => inner.rfold(init, f)) + } + + fn rfind

(&mut self, predicate: P) -> Option + where + P: FnMut(&Self::Item) -> bool, + { + for_both!(self, inner => inner.rfind(predicate)) + } +} + +impl ExactSizeIterator for Either +where + L: ExactSizeIterator, + R: ExactSizeIterator, +{ + fn len(&self) -> usize { + for_both!(self, inner => inner.len()) + } +} + +impl iter::FusedIterator for Either +where + L: iter::FusedIterator, + R: iter::FusedIterator, +{ +} + +impl Iterator for IterEither +where + L: Iterator, + R: Iterator, +{ + type Item = Either; + + fn next(&mut self) -> Option { + Some(map_both!(self.inner, ref mut inner => inner.next()?)) + } + + fn size_hint(&self) -> (usize, Option) { + for_both!(self.inner, ref inner => inner.size_hint()) + } + + fn fold(self, init: Acc, f: G) -> Acc + where + G: FnMut(Acc, Self::Item) -> Acc, + { + wrap_either!(self.inner => .fold(init, f)) + } + + fn for_each(self, f: F) + where + F: FnMut(Self::Item), + { + wrap_either!(self.inner => .for_each(f)) + } + + fn count(self) -> usize { + for_both!(self.inner, inner => inner.count()) + } + + fn last(self) -> Option { + Some(map_both!(self.inner, inner => inner.last()?)) + } + + fn nth(&mut self, n: usize) -> Option { + Some(map_both!(self.inner, ref mut inner => inner.nth(n)?)) + } + + fn collect(self) -> B + where + B: iter::FromIterator, + { + wrap_either!(self.inner => .collect()) + } + + fn partition(self, f: F) -> (B, B) + where + B: Default + Extend, + F: FnMut(&Self::Item) -> bool, + { + wrap_either!(self.inner => .partition(f)) + } + + fn all(&mut self, f: F) -> bool + where + F: FnMut(Self::Item) -> bool, + { + wrap_either!(&mut self.inner => .all(f)) + } + + fn any(&mut self, f: F) -> bool + where + F: FnMut(Self::Item) -> bool, + { + wrap_either!(&mut self.inner => .any(f)) + } + + fn find

(&mut self, predicate: P) -> Option + where + P: FnMut(&Self::Item) -> bool, + { + wrap_either!(&mut self.inner => .find(predicate)) + } + + fn find_map(&mut self, f: F) -> Option + where + F: FnMut(Self::Item) -> Option, + { + wrap_either!(&mut self.inner => .find_map(f)) + } + + fn position

(&mut self, predicate: P) -> Option + where + P: FnMut(Self::Item) -> bool, + { + wrap_either!(&mut self.inner => .position(predicate)) + } +} + +impl DoubleEndedIterator for IterEither +where + L: DoubleEndedIterator, + R: DoubleEndedIterator, +{ + fn next_back(&mut self) -> Option { + Some(map_both!(self.inner, ref mut inner => inner.next_back()?)) + } + + fn nth_back(&mut self, n: usize) -> Option { + Some(map_both!(self.inner, ref mut inner => inner.nth_back(n)?)) + } + + fn rfold(self, init: Acc, f: G) -> Acc + where + G: FnMut(Acc, Self::Item) -> Acc, + { + wrap_either!(self.inner => .rfold(init, f)) + } + + fn rfind

(&mut self, predicate: P) -> Option + where + P: FnMut(&Self::Item) -> bool, + { + wrap_either!(&mut self.inner => .rfind(predicate)) + } +} + +impl ExactSizeIterator for IterEither +where + L: ExactSizeIterator, + R: ExactSizeIterator, +{ + fn len(&self) -> usize { + for_both!(self.inner, ref inner => inner.len()) + } +} + +impl iter::FusedIterator for IterEither +where + L: iter::FusedIterator, + R: iter::FusedIterator, +{ +} diff --git a/anneal/v2/vendor/either/src/lib.rs b/anneal/v2/vendor/either/src/lib.rs new file mode 100644 index 0000000000..ec16cee001 --- /dev/null +++ b/anneal/v2/vendor/either/src/lib.rs @@ -0,0 +1,1826 @@ +//! The enum [`Either`] with variants `Left` and `Right` is a general purpose +//! sum type with two cases. +//! +//! [`Either`]: enum.Either.html +//! +//! **Crate features:** +//! +//! * `"std"` +//! Enabled by default. Disable to make the library `#![no_std]`. +//! +//! * `"serde"` +//! Disabled by default. Enable to `#[derive(Serialize, Deserialize)]` for `Either` +//! + +#![doc(html_root_url = "https://docs.rs/either/1/")] +#![no_std] + +#[cfg(any(test, feature = "std"))] +extern crate std; + +#[cfg(feature = "serde")] +pub mod serde_untagged; + +#[cfg(feature = "serde")] +pub mod serde_untagged_optional; + +use core::convert::{AsMut, AsRef}; +use core::fmt; +use core::future::Future; +use core::ops::Deref; +use core::ops::DerefMut; +use core::pin::Pin; + +#[cfg(any(test, feature = "std"))] +use std::error::Error; +#[cfg(any(test, feature = "std"))] +use std::io::{self, BufRead, Read, Seek, SeekFrom, Write}; + +pub use crate::Either::{Left, Right}; + +/// The enum `Either` with variants `Left` and `Right` is a general purpose +/// sum type with two cases. +/// +/// The `Either` type is symmetric and treats its variants the same way, without +/// preference. +/// (For representing success or error, use the regular `Result` enum instead.) +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[derive(Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +pub enum Either { + /// A value of type `L`. + Left(L), + /// A value of type `R`. + Right(R), +} + +/// Evaluate the provided expression for both [`Either::Left`] and [`Either::Right`]. +/// +/// This macro is useful in cases where both sides of [`Either`] can be interacted with +/// in the same way even though the don't share the same type. +/// +/// Syntax: +/// +/// - `either::for_both!(` *expression* `,` *pattern* `=>` *expression* `)` +/// - `either::for_both!(` *ident* `=>` *expression* `)` +/// +/// Unlike [`map_both!`], this macro converges both variants to the type returned by the expression. +/// +/// # Example +/// +/// ``` +/// use either::Either; +/// +/// fn length(owned_or_borrowed: Either) -> usize { +/// either::for_both!(owned_or_borrowed, s => s.len()) +/// } +/// +/// fn main() { +/// let borrowed = Either::Right("Hello world!"); +/// let owned = Either::Left("Hello world!".to_owned()); +/// +/// assert_eq!(length(borrowed), 12); +/// assert_eq!(length(owned), 12); +/// } +/// ``` +/// +/// ``` +/// use either::Either; +/// +/// fn length(s: Either>) -> usize { +/// either::for_both!(s => s.len()) +/// } +/// +/// fn main() { +/// let string = Either::Left("Hello world!".to_owned()); +/// let bytes = Either::Right(b"Hello world!".to_vec()); +/// +/// assert_eq!(length(string), 12); +/// assert_eq!(length(bytes), 12); +/// } +/// ``` +#[macro_export] +macro_rules! for_both { + ($value:expr, $pattern:pat => $result:expr) => { + match $value { + $crate::Either::Left($pattern) => $result, + $crate::Either::Right($pattern) => $result, + } + }; + ($name:ident => $result:expr) => { + match $name { + $crate::Either::Left($name) => $result, + $crate::Either::Right($name) => $result, + } + }; +} + +/// Evaluate the provided expression for both [`Either::Left`] and [`Either::Right`], +/// returning an [`Either`] with the results. +/// +/// This macro is useful in cases where both sides of [`Either`] can be interacted with +/// in the same way even though the don't share the same type. +/// +/// Syntax: `either::map_both!(` *expression* `,` *pattern* `=>` *expression* `)` +/// +/// Unlike [`for_both!`], this macro returns an [`Either`] with the results of the expressions. +/// +/// # Example +/// +/// ``` +/// use either::Either; +/// +/// struct Wrapper(T); +/// +/// fn wrap( +/// owned_or_borrowed: Either, +/// ) -> Either, Wrapper<&'static str>> { +/// either::map_both!(owned_or_borrowed, s => Wrapper(s)) +/// } +/// ``` +/// +/// ``` +/// use either::Either; +/// +/// fn widen(x: Either) -> Either { +/// either::map_both!(x => x.into()) +/// } +/// ``` +#[macro_export] +macro_rules! map_both { + ($value:expr, $pattern:pat => $result:expr) => { + match $value { + $crate::Either::Left($pattern) => $crate::Either::Left($result), + $crate::Either::Right($pattern) => $crate::Either::Right($result), + } + }; + ($name:ident => $result:expr) => { + match $name { + $crate::Either::Left($name) => $crate::Either::Left($result), + $crate::Either::Right($name) => $crate::Either::Right($result), + } + }; +} + +/// Macro for unwrapping the left side of an [`Either`], which fails early +/// with the opposite side. Can only be used in functions that return +/// `Either` because of the early return of `Right` that it provides. +/// +/// See also [`try_right!`] for its dual, which applies the same just to the +/// right side. +/// +/// # Example +/// +/// ``` +/// use either::{Either, Left, Right}; +/// +/// fn twice(wrapper: Either) -> Either { +/// let value = either::try_left!(wrapper); +/// Left(value * 2) +/// } +/// +/// fn main() { +/// assert_eq!(twice(Left(2)), Left(4)); +/// assert_eq!(twice(Right("ups")), Right("ups")); +/// } +/// ``` +#[macro_export] +macro_rules! try_left { + ($expr:expr) => { + match $expr { + $crate::Left(val) => val, + $crate::Right(err) => return $crate::Right(::core::convert::From::from(err)), + } + }; +} + +/// Dual to [`try_left!`], see its documentation for more information. +#[macro_export] +macro_rules! try_right { + ($expr:expr) => { + match $expr { + $crate::Left(err) => return $crate::Left(::core::convert::From::from(err)), + $crate::Right(val) => val, + } + }; +} + +mod iterator; +pub use self::iterator::IterEither; + +mod into_either; +pub use self::into_either::IntoEither; + +impl Clone for Either { + fn clone(&self) -> Self { + match self { + Left(inner) => Left(inner.clone()), + Right(inner) => Right(inner.clone()), + } + } + + fn clone_from(&mut self, source: &Self) { + match (self, source) { + (Left(dest), Left(source)) => dest.clone_from(source), + (Right(dest), Right(source)) => dest.clone_from(source), + (dest, source) => *dest = source.clone(), + } + } +} + +impl Either { + /// Return true if the value is the `Left` variant. + /// + /// ``` + /// use either::*; + /// + /// let values = [Left(1), Right("the right value")]; + /// assert_eq!(values[0].is_left(), true); + /// assert_eq!(values[1].is_left(), false); + /// ``` + pub fn is_left(&self) -> bool { + match self { + Left(_) => true, + Right(_) => false, + } + } + + /// Returns `true` if the value is [`Left`] and the value inside of it matches a predicate. + /// + /// # Examples + /// + /// ``` + /// use either::*; + /// + /// let left0: Either = Left(0); + /// let left2: Either = Left(2); + /// let right: Either = Right(2); + /// + /// assert_eq!(left2.is_left_and(|n| n > 1), true); + /// assert_eq!(left0.is_left_and(|n| n > 1), false); + /// assert_eq!(right.is_left_and(|n| n > 1), false); + /// ``` + pub fn is_left_and(self, f: F) -> bool + where + F: FnOnce(L) -> bool, + { + match self { + Left(left) => f(left), + Right(_) => false, + } + } + + /// Returns `true` if the value is [`Right`] and the value inside of it matches a predicate. + /// + /// # Examples + /// + /// ``` + /// use either::*; + /// + /// let right0: Either = Right(0); + /// let right2: Either = Right(2); + /// let left: Either = Left(2); + /// + /// assert_eq!(right2.is_right_and(|n| n > 1), true); + /// assert_eq!(right0.is_right_and(|n| n > 1), false); + /// assert_eq!(left.is_right_and(|n| n > 1), false); + /// ``` + pub fn is_right_and(self, f: F) -> bool + where + F: FnOnce(R) -> bool, + { + match self { + Left(_) => false, + Right(right) => f(right), + } + } + + /// Return true if the value is the `Right` variant. + /// + /// ``` + /// use either::*; + /// + /// let values = [Left(1), Right("the right value")]; + /// assert_eq!(values[0].is_right(), false); + /// assert_eq!(values[1].is_right(), true); + /// ``` + pub fn is_right(&self) -> bool { + !self.is_left() + } + + /// Convert the left side of `Either` to an `Option`. + /// + /// ``` + /// use either::*; + /// + /// let left: Either<_, ()> = Left("some value"); + /// assert_eq!(left.left(), Some("some value")); + /// + /// let right: Either<(), _> = Right(321); + /// assert_eq!(right.left(), None); + /// ``` + pub fn left(self) -> Option { + match self { + Left(l) => Some(l), + Right(_) => None, + } + } + + /// Convert the right side of `Either` to an `Option`. + /// + /// ``` + /// use either::*; + /// + /// let left: Either<_, ()> = Left("some value"); + /// assert_eq!(left.right(), None); + /// + /// let right: Either<(), _> = Right(321); + /// assert_eq!(right.right(), Some(321)); + /// ``` + pub fn right(self) -> Option { + match self { + Left(_) => None, + Right(r) => Some(r), + } + } + + /// Convert `&Either` to `Either<&L, &R>`. + /// + /// ``` + /// use either::*; + /// + /// let left: Either<_, ()> = Left("some value"); + /// assert_eq!(left.as_ref(), Left(&"some value")); + /// + /// let right: Either<(), _> = Right("some value"); + /// assert_eq!(right.as_ref(), Right(&"some value")); + /// ``` + pub fn as_ref(&self) -> Either<&L, &R> { + map_both!(self, inner => inner) + } + + /// Convert `&mut Either` to `Either<&mut L, &mut R>`. + /// + /// ``` + /// use either::*; + /// + /// fn mutate_left(value: &mut Either) { + /// if let Some(l) = value.as_mut().left() { + /// *l = 999; + /// } + /// } + /// + /// let mut left = Left(123); + /// let mut right = Right(123); + /// mutate_left(&mut left); + /// mutate_left(&mut right); + /// assert_eq!(left, Left(999)); + /// assert_eq!(right, Right(123)); + /// ``` + pub fn as_mut(&mut self) -> Either<&mut L, &mut R> { + map_both!(self, inner => inner) + } + + /// Convert `Pin<&Either>` to `Either, Pin<&R>>`, + /// pinned projections of the inner variants. + pub fn as_pin_ref(self: Pin<&Self>) -> Either, Pin<&R>> { + // SAFETY: We can use `new_unchecked` because the `inner` parts are + // guaranteed to be pinned, as they come from `self` which is pinned. + unsafe { map_both!(Pin::get_ref(self), inner => Pin::new_unchecked(inner)) } + } + + /// Convert `Pin<&mut Either>` to `Either, Pin<&mut R>>`, + /// pinned projections of the inner variants. + pub fn as_pin_mut(self: Pin<&mut Self>) -> Either, Pin<&mut R>> { + // SAFETY: `get_unchecked_mut` is fine because we don't move anything. + // We can use `new_unchecked` because the `inner` parts are guaranteed + // to be pinned, as they come from `self` which is pinned, and we never + // offer an unpinned `&mut L` or `&mut R` through `Pin<&mut Self>`. We + // also don't have an implementation of `Drop`, nor manual `Unpin`. + unsafe { map_both!(Pin::get_unchecked_mut(self), inner => Pin::new_unchecked(inner)) } + } + + /// Convert `Either` to `Either`. + /// + /// ``` + /// use either::*; + /// + /// let left: Either<_, ()> = Left(123); + /// assert_eq!(left.flip(), Right(123)); + /// + /// let right: Either<(), _> = Right("some value"); + /// assert_eq!(right.flip(), Left("some value")); + /// ``` + pub fn flip(self) -> Either { + match self { + Left(l) => Right(l), + Right(r) => Left(r), + } + } + + /// Apply the function `f` on the value in the `Left` variant if it is present rewrapping the + /// result in `Left`. + /// + /// ``` + /// use either::*; + /// + /// let left: Either<_, u32> = Left(123); + /// assert_eq!(left.map_left(|x| x * 2), Left(246)); + /// + /// let right: Either = Right(123); + /// assert_eq!(right.map_left(|x| x * 2), Right(123)); + /// ``` + pub fn map_left(self, f: F) -> Either + where + F: FnOnce(L) -> M, + { + match self { + Left(l) => Left(f(l)), + Right(r) => Right(r), + } + } + + /// Apply the function `f` on the value in the `Right` variant if it is present rewrapping the + /// result in `Right`. + /// + /// ``` + /// use either::*; + /// + /// let left: Either<_, u32> = Left(123); + /// assert_eq!(left.map_right(|x| x * 2), Left(123)); + /// + /// let right: Either = Right(123); + /// assert_eq!(right.map_right(|x| x * 2), Right(246)); + /// ``` + pub fn map_right(self, f: F) -> Either + where + F: FnOnce(R) -> S, + { + match self { + Left(l) => Left(l), + Right(r) => Right(f(r)), + } + } + + /// Returns the provided default (if [`Right`]), or + /// applies a function to the contained value (if [`Left`]). + /// + /// # Examples + /// + /// ``` + /// use either::*; + /// + /// let x: Either<_, i32> = Left(123); + /// assert_eq!(x.map_left_or(0, |n| n * 2), 246); + /// + /// let x: Either = Right(123); + /// assert_eq!(x.map_left_or(0, |n| n * 2), 0); + /// ``` + pub fn map_left_or(self, default: S, f: F) -> S + where + F: FnOnce(L) -> S, + { + match self { + Left(left) => f(left), + Right(_) => default, + } + } + + /// Returns the provided default (if [`Left`]), or + /// applies a function to the contained value (if [`Right`]). + /// + /// # Examples + /// + /// ``` + /// use either::*; + /// + /// let x: Either<_, i32> = Left(123); + /// assert_eq!(x.map_right_or(0, |n| n * 2), 0); + /// + /// let x: Either = Right(123); + /// assert_eq!(x.map_right_or(0, |n| n * 2), 246); + /// ``` + pub fn map_right_or(self, default: S, f: F) -> S + where + F: FnOnce(R) -> S, + { + match self { + Left(_) => default, + Right(right) => f(right), + } + } + + /// Apply the functions `f` and `g` to the `Left` and `Right` variants + /// respectively. This is equivalent to + /// [bimap](https://hackage.haskell.org/package/bifunctors-5/docs/Data-Bifunctor.html) + /// in functional programming. + /// + /// ``` + /// use either::*; + /// + /// let f = |s: String| s.len(); + /// let g = |u: u8| u.to_string(); + /// + /// let left: Either = Left("loopy".into()); + /// assert_eq!(left.map_either(f, g), Left(5)); + /// + /// let right: Either = Right(42); + /// assert_eq!(right.map_either(f, g), Right("42".into())); + /// ``` + pub fn map_either(self, f: F, g: G) -> Either + where + F: FnOnce(L) -> M, + G: FnOnce(R) -> S, + { + match self { + Left(l) => Left(f(l)), + Right(r) => Right(g(r)), + } + } + + /// Similar to [`map_either`][Self::map_either], with an added context `ctx` accessible to + /// both functions. + /// + /// ``` + /// use either::*; + /// + /// let mut sum = 0; + /// + /// // Both closures want to update the same value, so pass it as context. + /// let mut f = |sum: &mut usize, s: String| { *sum += s.len(); s.to_uppercase() }; + /// let mut g = |sum: &mut usize, u: usize| { *sum += u; u.to_string() }; + /// + /// let left: Either = Left("loopy".into()); + /// assert_eq!(left.map_either_with(&mut sum, &mut f, &mut g), Left("LOOPY".into())); + /// + /// let right: Either = Right(42); + /// assert_eq!(right.map_either_with(&mut sum, &mut f, &mut g), Right("42".into())); + /// + /// assert_eq!(sum, 47); + /// ``` + pub fn map_either_with(self, ctx: Ctx, f: F, g: G) -> Either + where + F: FnOnce(Ctx, L) -> M, + G: FnOnce(Ctx, R) -> S, + { + match self { + Left(l) => Left(f(ctx, l)), + Right(r) => Right(g(ctx, r)), + } + } + + /// Apply one of two functions depending on contents, unifying their result. If the value is + /// `Left(L)` then the first function `f` is applied; if it is `Right(R)` then the second + /// function `g` is applied. + /// + /// ``` + /// use either::*; + /// + /// fn square(n: u32) -> i32 { (n * n) as i32 } + /// fn negate(n: i32) -> i32 { -n } + /// + /// let left: Either = Left(4); + /// assert_eq!(left.either(square, negate), 16); + /// + /// let right: Either = Right(-4); + /// assert_eq!(right.either(square, negate), 4); + /// ``` + pub fn either(self, f: F, g: G) -> T + where + F: FnOnce(L) -> T, + G: FnOnce(R) -> T, + { + match self { + Left(l) => f(l), + Right(r) => g(r), + } + } + + /// Like [`either`][Self::either], but provide some context to whichever of the + /// functions ends up being called. + /// + /// ``` + /// // In this example, the context is a mutable reference + /// use either::*; + /// + /// let mut result = Vec::new(); + /// + /// let values = vec![Left(2), Right(2.7)]; + /// + /// for value in values { + /// value.either_with(&mut result, + /// |ctx, integer| ctx.push(integer), + /// |ctx, real| ctx.push(f64::round(real) as i32)); + /// } + /// + /// assert_eq!(result, vec![2, 3]); + /// ``` + pub fn either_with(self, ctx: Ctx, f: F, g: G) -> T + where + F: FnOnce(Ctx, L) -> T, + G: FnOnce(Ctx, R) -> T, + { + match self { + Left(l) => f(ctx, l), + Right(r) => g(ctx, r), + } + } + + /// Returns `other` if the value is [`Left`], otherwise returns the [`Right`] value of `self`. + /// + /// Arguments passed to `left_and` are eagerly evaluated; if you are passing the + /// result of a function call, it is recommended to use [`left_and_then`], which is + /// lazily evaluated. + /// + /// [`left_and_then`]: Either::left_and_then + /// + /// # Examples + /// + /// ``` + /// use either::*; + /// + /// let left: Either<_, u32> = Left(123); + /// assert_eq!(left.left_and::<()>(Right(246)), Right(246)); + /// assert_eq!(left.left_and(Left(246)), Left(246)); + /// + /// let right: Either = Right(123); + /// assert_eq!(right.left_and::<()>(Right(246)), Right(123)); + /// assert_eq!(right.left_and(Left(246)), Right(123)); + /// ``` + pub fn left_and(self, other: Either) -> Either { + match self { + Left(_) => other, + Right(r) => Right(r), + } + } + + /// Returns `other` if the value is [`Right`], otherwise returns the [`Left`] value of `self`. + /// + /// Arguments passed to `right_and` are eagerly evaluated; if you are passing the + /// result of a function call, it is recommended to use [`right_and_then`], which is + /// lazily evaluated. + /// + /// [`right_and_then`]: Either::right_and_then + /// + /// # Examples + /// + /// ``` + /// use either::*; + /// + /// let left: Either<_, u32> = Left(123); + /// assert_eq!(left.right_and(Right(246)), Left(123)); + /// assert_eq!(left.right_and::<()>(Left(246)), Left(123)); + /// + /// let right: Either = Right(123); + /// assert_eq!(right.right_and(Right(246)), Right(246)); + /// assert_eq!(right.right_and::<()>(Left(246)), Left(246)); + /// ``` + pub fn right_and(self, other: Either) -> Either { + match self { + Left(l) => Left(l), + Right(_) => other, + } + } + + /// Apply the function `f` on the value in the `Left` variant if it is present. + /// + /// ``` + /// use either::*; + /// + /// let left: Either<_, u32> = Left(123); + /// assert_eq!(left.left_and_then::<_,()>(|x| Right(x * 2)), Right(246)); + /// + /// let right: Either = Right(123); + /// assert_eq!(right.left_and_then(|x| Right::<(), _>(x * 2)), Right(123)); + /// ``` + pub fn left_and_then(self, f: F) -> Either + where + F: FnOnce(L) -> Either, + { + match self { + Left(l) => f(l), + Right(r) => Right(r), + } + } + + /// Apply the function `f` on the value in the `Right` variant if it is present. + /// + /// ``` + /// use either::*; + /// + /// let left: Either<_, u32> = Left(123); + /// assert_eq!(left.right_and_then(|x| Right(x * 2)), Left(123)); + /// + /// let right: Either = Right(123); + /// assert_eq!(right.right_and_then(|x| Right(x * 2)), Right(246)); + /// ``` + pub fn right_and_then(self, f: F) -> Either + where + F: FnOnce(R) -> Either, + { + match self { + Left(l) => Left(l), + Right(r) => f(r), + } + } + + /// Convert the inner value to an iterator. + /// + /// This requires the `Left` and `Right` iterators to have the same item type. + /// See [`factor_into_iter`][Either::factor_into_iter] to iterate different types. + /// + /// ``` + /// use either::*; + /// + /// let left: Either<_, Vec> = Left(vec![1, 2, 3, 4, 5]); + /// let mut right: Either, _> = Right(vec![]); + /// right.extend(left.into_iter()); + /// assert_eq!(right, Right(vec![1, 2, 3, 4, 5])); + /// ``` + #[allow(clippy::should_implement_trait)] + pub fn into_iter(self) -> Either + where + L: IntoIterator, + R: IntoIterator, + { + map_both!(self, inner => inner.into_iter()) + } + + /// Borrow the inner value as an iterator. + /// + /// This requires the `Left` and `Right` iterators to have the same item type. + /// See [`factor_iter`][Either::factor_iter] to iterate different types. + /// + /// ``` + /// use either::*; + /// + /// let left: Either<_, &[u32]> = Left(vec![2, 3]); + /// let mut right: Either, _> = Right(&[4, 5][..]); + /// let mut all = vec![1]; + /// all.extend(left.iter()); + /// all.extend(right.iter()); + /// assert_eq!(all, vec![1, 2, 3, 4, 5]); + /// ``` + pub fn iter(&self) -> Either<<&L as IntoIterator>::IntoIter, <&R as IntoIterator>::IntoIter> + where + for<'a> &'a L: IntoIterator, + for<'a> &'a R: IntoIterator::Item>, + { + map_both!(self, inner => inner.into_iter()) + } + + /// Mutably borrow the inner value as an iterator. + /// + /// This requires the `Left` and `Right` iterators to have the same item type. + /// See [`factor_iter_mut`][Either::factor_iter_mut] to iterate different types. + /// + /// ``` + /// use either::*; + /// + /// let mut left: Either<_, &mut [u32]> = Left(vec![2, 3]); + /// for l in left.iter_mut() { + /// *l *= *l + /// } + /// assert_eq!(left, Left(vec![4, 9])); + /// + /// let mut inner = [4, 5]; + /// let mut right: Either, _> = Right(&mut inner[..]); + /// for r in right.iter_mut() { + /// *r *= *r + /// } + /// assert_eq!(inner, [16, 25]); + /// ``` + pub fn iter_mut( + &mut self, + ) -> Either<<&mut L as IntoIterator>::IntoIter, <&mut R as IntoIterator>::IntoIter> + where + for<'a> &'a mut L: IntoIterator, + for<'a> &'a mut R: IntoIterator::Item>, + { + map_both!(self, inner => inner.into_iter()) + } + + /// Converts an `Either` of `Iterator`s to be an `Iterator` of `Either`s + /// + /// Unlike [`into_iter`][Either::into_iter], this does not require the + /// `Left` and `Right` iterators to have the same item type. + /// + /// ``` + /// use either::*; + /// let left: Either<_, Vec> = Left(&["hello"]); + /// assert_eq!(left.factor_into_iter().next(), Some(Left(&"hello"))); + /// + /// let right: Either<&[&str], _> = Right(vec![0, 1]); + /// assert_eq!(right.factor_into_iter().collect::>(), vec![Right(0), Right(1)]); + /// + /// ``` + pub fn factor_into_iter(self) -> IterEither + where + L: IntoIterator, + R: IntoIterator, + { + IterEither::new(map_both!(self, inner => inner.into_iter())) + } + + /// Borrows an `Either` of `Iterator`s to be an `Iterator` of `Either`s + /// + /// Unlike [`iter`][Either::iter], this does not require the + /// `Left` and `Right` iterators to have the same item type. + /// + /// ``` + /// use either::*; + /// let left: Either<_, Vec> = Left(["hello"]); + /// assert_eq!(left.factor_iter().next(), Some(Left(&"hello"))); + /// + /// let right: Either<[&str; 2], _> = Right(vec![0, 1]); + /// assert_eq!(right.factor_iter().collect::>(), vec![Right(&0), Right(&1)]); + /// + /// ``` + pub fn factor_iter( + &self, + ) -> IterEither<<&L as IntoIterator>::IntoIter, <&R as IntoIterator>::IntoIter> + where + for<'a> &'a L: IntoIterator, + for<'a> &'a R: IntoIterator, + { + IterEither::new(map_both!(self, inner => inner.into_iter())) + } + + /// Mutably borrows an `Either` of `Iterator`s to be an `Iterator` of `Either`s + /// + /// Unlike [`iter_mut`][Either::iter_mut], this does not require the + /// `Left` and `Right` iterators to have the same item type. + /// + /// ``` + /// use either::*; + /// let mut left: Either<_, Vec> = Left(["hello"]); + /// left.factor_iter_mut().for_each(|x| *x.unwrap_left() = "goodbye"); + /// assert_eq!(left, Left(["goodbye"])); + /// + /// let mut right: Either<[&str; 2], _> = Right(vec![0, 1, 2]); + /// right.factor_iter_mut().for_each(|x| if let Right(r) = x { *r = -*r; }); + /// assert_eq!(right, Right(vec![0, -1, -2])); + /// + /// ``` + pub fn factor_iter_mut( + &mut self, + ) -> IterEither<<&mut L as IntoIterator>::IntoIter, <&mut R as IntoIterator>::IntoIter> + where + for<'a> &'a mut L: IntoIterator, + for<'a> &'a mut R: IntoIterator, + { + IterEither::new(map_both!(self, inner => inner.into_iter())) + } + + /// Return left value or given value + /// + /// Arguments passed to `left_or` are eagerly evaluated; if you are passing + /// the result of a function call, it is recommended to use + /// [`left_or_else`][Self::left_or_else], which is lazily evaluated. + /// + /// # Examples + /// + /// ``` + /// # use either::*; + /// let left: Either<&str, &str> = Left("left"); + /// assert_eq!(left.left_or("foo"), "left"); + /// + /// let right: Either<&str, &str> = Right("right"); + /// assert_eq!(right.left_or("left"), "left"); + /// ``` + pub fn left_or(self, other: L) -> L { + match self { + Either::Left(l) => l, + Either::Right(_) => other, + } + } + + /// Return left or a default + /// + /// # Examples + /// + /// ``` + /// # use either::*; + /// let left: Either = Left("left".to_string()); + /// assert_eq!(left.left_or_default(), "left"); + /// + /// let right: Either = Right(42); + /// assert_eq!(right.left_or_default(), String::default()); + /// ``` + pub fn left_or_default(self) -> L + where + L: Default, + { + match self { + Either::Left(l) => l, + Either::Right(_) => L::default(), + } + } + + /// Returns left value or computes it from a closure + /// + /// # Examples + /// + /// ``` + /// # use either::*; + /// let left: Either = Left("3".to_string()); + /// assert_eq!(left.left_or_else(|_| unreachable!()), "3"); + /// + /// let right: Either = Right(3); + /// assert_eq!(right.left_or_else(|x| x.to_string()), "3"); + /// ``` + pub fn left_or_else(self, f: F) -> L + where + F: FnOnce(R) -> L, + { + match self { + Either::Left(l) => l, + Either::Right(r) => f(r), + } + } + + /// Return right value or given value + /// + /// Arguments passed to `right_or` are eagerly evaluated; if you are passing + /// the result of a function call, it is recommended to use + /// [`right_or_else`][Self::right_or_else], which is lazily evaluated. + /// + /// # Examples + /// + /// ``` + /// # use either::*; + /// let right: Either<&str, &str> = Right("right"); + /// assert_eq!(right.right_or("foo"), "right"); + /// + /// let left: Either<&str, &str> = Left("left"); + /// assert_eq!(left.right_or("right"), "right"); + /// ``` + pub fn right_or(self, other: R) -> R { + match self { + Either::Left(_) => other, + Either::Right(r) => r, + } + } + + /// Return right or a default + /// + /// # Examples + /// + /// ``` + /// # use either::*; + /// let left: Either = Left("left".to_string()); + /// assert_eq!(left.right_or_default(), u32::default()); + /// + /// let right: Either = Right(42); + /// assert_eq!(right.right_or_default(), 42); + /// ``` + pub fn right_or_default(self) -> R + where + R: Default, + { + match self { + Either::Left(_) => R::default(), + Either::Right(r) => r, + } + } + + /// Returns right value or computes it from a closure + /// + /// # Examples + /// + /// ``` + /// # use either::*; + /// let left: Either = Left("3".to_string()); + /// assert_eq!(left.right_or_else(|x| x.parse().unwrap()), 3); + /// + /// let right: Either = Right(3); + /// assert_eq!(right.right_or_else(|_| unreachable!()), 3); + /// ``` + pub fn right_or_else(self, f: F) -> R + where + F: FnOnce(L) -> R, + { + match self { + Either::Left(l) => f(l), + Either::Right(r) => r, + } + } + + /// Returns the left value + /// + /// # Examples + /// + /// ``` + /// # use either::*; + /// let left: Either<_, ()> = Left(3); + /// assert_eq!(left.unwrap_left(), 3); + /// ``` + /// + /// # Panics + /// + /// When `Either` is a `Right` value + /// + /// ```should_panic + /// # use either::*; + /// let right: Either<(), _> = Right(3); + /// right.unwrap_left(); + /// ``` + #[track_caller] + pub fn unwrap_left(self) -> L + where + R: core::fmt::Debug, + { + match self { + Either::Left(l) => l, + Either::Right(r) => { + panic!("called `Either::unwrap_left()` on a `Right` value: {:?}", r) + } + } + } + + /// Returns the right value + /// + /// # Examples + /// + /// ``` + /// # use either::*; + /// let right: Either<(), _> = Right(3); + /// assert_eq!(right.unwrap_right(), 3); + /// ``` + /// + /// # Panics + /// + /// When `Either` is a `Left` value + /// + /// ```should_panic + /// # use either::*; + /// let left: Either<_, ()> = Left(3); + /// left.unwrap_right(); + /// ``` + #[track_caller] + pub fn unwrap_right(self) -> R + where + L: core::fmt::Debug, + { + match self { + Either::Right(r) => r, + Either::Left(l) => panic!("called `Either::unwrap_right()` on a `Left` value: {:?}", l), + } + } + + /// Returns the left value + /// + /// # Examples + /// + /// ``` + /// # use either::*; + /// let left: Either<_, ()> = Left(3); + /// assert_eq!(left.expect_left("value was Right"), 3); + /// ``` + /// + /// # Panics + /// + /// When `Either` is a `Right` value + /// + /// ```should_panic + /// # use either::*; + /// let right: Either<(), _> = Right(3); + /// right.expect_left("value was Right"); + /// ``` + #[track_caller] + pub fn expect_left(self, msg: &str) -> L + where + R: core::fmt::Debug, + { + match self { + Either::Left(l) => l, + Either::Right(r) => panic!("{}: {:?}", msg, r), + } + } + + /// Returns the right value + /// + /// # Examples + /// + /// ``` + /// # use either::*; + /// let right: Either<(), _> = Right(3); + /// assert_eq!(right.expect_right("value was Left"), 3); + /// ``` + /// + /// # Panics + /// + /// When `Either` is a `Left` value + /// + /// ```should_panic + /// # use either::*; + /// let left: Either<_, ()> = Left(3); + /// left.expect_right("value was Right"); + /// ``` + #[track_caller] + pub fn expect_right(self, msg: &str) -> R + where + L: core::fmt::Debug, + { + match self { + Either::Right(r) => r, + Either::Left(l) => panic!("{}: {:?}", msg, l), + } + } + + /// Calls a function with a reference to the contained value if [`Left`]. + /// + /// Returns the original self. + /// + /// # Examples + /// + /// ``` + /// use either::*; + /// + /// # fn foo() -> Either { Right(2) } + /// let x = foo() + /// .inspect_left(|n| println!("left: {n}")) + /// .left_or(0); + /// ``` + pub fn inspect_left(self, f: F) -> Self + where + F: FnOnce(&L), + { + if let Left(ref left) = self { + f(left); + } + + self + } + + /// Calls a function with a reference to the contained value if [`Right`]. + /// + /// Returns the original self. + /// + /// # Examples + /// + /// ``` + /// use either::*; + /// + /// # fn foo() -> Either { Right(2) } + /// let x = foo() + /// .inspect_right(|n| println!("right: {n}")) + /// .left_or(0); + /// ``` + pub fn inspect_right(self, f: F) -> Self + where + F: FnOnce(&R), + { + if let Right(ref right) = self { + f(right); + } + + self + } + + /// Convert the contained value into `T` + /// + /// # Examples + /// + /// ``` + /// # use either::*; + /// // Both u16 and u32 can be converted to u64. + /// let left: Either = Left(3u16); + /// assert_eq!(left.either_into::(), 3u64); + /// let right: Either = Right(7u32); + /// assert_eq!(right.either_into::(), 7u64); + /// ``` + pub fn either_into(self) -> T + where + L: Into, + R: Into, + { + for_both!(self, inner => inner.into()) + } +} + +impl Either, Option> { + /// Factors out `None` from an `Either` of [`Option`]. + /// + /// ``` + /// use either::*; + /// let left: Either<_, Option> = Left(Some(vec![0])); + /// assert_eq!(left.factor_none(), Some(Left(vec![0]))); + /// + /// let right: Either>, _> = Right(Some(String::new())); + /// assert_eq!(right.factor_none(), Some(Right(String::new()))); + /// ``` + #[doc(alias = "transpose")] + pub fn factor_none(self) -> Option> { + match self { + Left(l) => l.map(Either::Left), + Right(r) => r.map(Either::Right), + } + } +} + +impl Either, Result> { + /// Factors out a homogeneous type from an `Either` of [`Result`]. + /// + /// Here, the homogeneous type is the `Err` type of the [`Result`]. + /// + /// ``` + /// use either::*; + /// let left: Either<_, Result> = Left(Ok(vec![0])); + /// assert_eq!(left.factor_err(), Ok(Left(vec![0]))); + /// + /// let right: Either, u32>, _> = Right(Ok(String::new())); + /// assert_eq!(right.factor_err(), Ok(Right(String::new()))); + /// ``` + #[doc(alias = "transpose")] + pub fn factor_err(self) -> Result, E> { + match self { + Left(l) => l.map(Either::Left), + Right(r) => r.map(Either::Right), + } + } +} + +impl Either, Result> { + /// Factors out a homogeneous type from an `Either` of [`Result`]. + /// + /// Here, the homogeneous type is the `Ok` type of the [`Result`]. + /// + /// ``` + /// use either::*; + /// let left: Either<_, Result> = Left(Err(vec![0])); + /// assert_eq!(left.factor_ok(), Err(Left(vec![0]))); + /// + /// let right: Either>, _> = Right(Err(String::new())); + /// assert_eq!(right.factor_ok(), Err(Right(String::new()))); + /// ``` + #[doc(alias = "transpose")] + pub fn factor_ok(self) -> Result> { + match self { + Left(l) => l.map_err(Either::Left), + Right(r) => r.map_err(Either::Right), + } + } +} + +impl Either<(T, L), (T, R)> { + /// Factor out a homogeneous type from an either of pairs. + /// + /// Here, the homogeneous type is the first element of the pairs. + /// + /// ``` + /// use either::*; + /// let left: Either<_, (u32, String)> = Left((123, vec![0])); + /// assert_eq!(left.factor_first().0, 123); + /// + /// let right: Either<(u32, Vec), _> = Right((123, String::new())); + /// assert_eq!(right.factor_first().0, 123); + /// ``` + pub fn factor_first(self) -> (T, Either) { + match self { + Left((t, l)) => (t, Left(l)), + Right((t, r)) => (t, Right(r)), + } + } +} + +impl Either<(L, T), (R, T)> { + /// Factor out a homogeneous type from an either of pairs. + /// + /// Here, the homogeneous type is the second element of the pairs. + /// + /// ``` + /// use either::*; + /// let left: Either<_, (String, u32)> = Left((vec![0], 123)); + /// assert_eq!(left.factor_second().1, 123); + /// + /// let right: Either<(Vec, u32), _> = Right((String::new(), 123)); + /// assert_eq!(right.factor_second().1, 123); + /// ``` + pub fn factor_second(self) -> (Either, T) { + match self { + Left((l, t)) => (Left(l), t), + Right((r, t)) => (Right(r), t), + } + } +} + +impl Either { + /// Extract the value of an either over two equivalent types. + /// + /// ``` + /// use either::*; + /// + /// let left: Either<_, u32> = Left(123); + /// assert_eq!(left.into_inner(), 123); + /// + /// let right: Either = Right(123); + /// assert_eq!(right.into_inner(), 123); + /// ``` + pub fn into_inner(self) -> T { + for_both!(self, inner => inner) + } + + /// Map `f` over the contained value and return the result in the + /// corresponding variant. + /// + /// ``` + /// use either::*; + /// + /// let value: Either<_, i32> = Right(42); + /// + /// let other = value.map(|x| x * 2); + /// assert_eq!(other, Right(84)); + /// ``` + pub fn map(self, f: F) -> Either + where + F: FnOnce(T) -> M, + { + map_both!(self, t => f(t)) + } +} + +impl Either<&L, &R> { + /// Maps an `Either<&L, &R>` to an `Either` by cloning the contents of + /// either branch. + pub fn cloned(self) -> Either + where + L: Clone, + R: Clone, + { + map_both!(self, inner => inner.clone()) + } + + /// Maps an `Either<&L, &R>` to an `Either` by copying the contents of + /// either branch. + pub fn copied(self) -> Either + where + L: Copy, + R: Copy, + { + map_both!(self, inner => *inner) + } +} + +impl Either<&mut L, &mut R> { + /// Maps an `Either<&mut L, &mut R>` to an `Either` by cloning the contents of + /// either branch. + pub fn cloned(self) -> Either + where + L: Clone, + R: Clone, + { + map_both!(self, inner => inner.clone()) + } + + /// Maps an `Either<&mut L, &mut R>` to an `Either` by copying the contents of + /// either branch. + pub fn copied(self) -> Either + where + L: Copy, + R: Copy, + { + map_both!(self, inner => *inner) + } +} + +/// Convert from `Result` to `Either` with `Ok => Right` and `Err => Left`. +impl From> for Either { + fn from(r: Result) -> Self { + match r { + Err(e) => Left(e), + Ok(o) => Right(o), + } + } +} + +/// Convert from `Either` to `Result` with `Right => Ok` and `Left => Err`. +impl From> for Result { + fn from(val: Either) -> Self { + match val { + Left(l) => Err(l), + Right(r) => Ok(r), + } + } +} + +/// `Either` is a future if both `L` and `R` are futures. +impl Future for Either +where + L: Future, + R: Future, +{ + type Output = L::Output; + + fn poll( + self: Pin<&mut Self>, + cx: &mut core::task::Context<'_>, + ) -> core::task::Poll { + for_both!(self.as_pin_mut(), inner => inner.poll(cx)) + } +} + +#[cfg(any(test, feature = "std"))] +/// `Either` implements `Read` if both `L` and `R` do. +/// +/// Requires crate feature `"std"` +impl Read for Either +where + L: Read, + R: Read, +{ + fn read(&mut self, buf: &mut [u8]) -> io::Result { + for_both!(self, inner => inner.read(buf)) + } + + fn read_exact(&mut self, buf: &mut [u8]) -> io::Result<()> { + for_both!(self, inner => inner.read_exact(buf)) + } + + fn read_to_end(&mut self, buf: &mut std::vec::Vec) -> io::Result { + for_both!(self, inner => inner.read_to_end(buf)) + } + + fn read_to_string(&mut self, buf: &mut std::string::String) -> io::Result { + for_both!(self, inner => inner.read_to_string(buf)) + } +} + +#[cfg(any(test, feature = "std"))] +/// `Either` implements `Seek` if both `L` and `R` do. +/// +/// Requires crate feature `"std"` +impl Seek for Either +where + L: Seek, + R: Seek, +{ + fn seek(&mut self, pos: SeekFrom) -> io::Result { + for_both!(self, inner => inner.seek(pos)) + } +} + +#[cfg(any(test, feature = "std"))] +/// Requires crate feature `"std"` +impl BufRead for Either +where + L: BufRead, + R: BufRead, +{ + fn fill_buf(&mut self) -> io::Result<&[u8]> { + for_both!(self, inner => inner.fill_buf()) + } + + fn consume(&mut self, amt: usize) { + for_both!(self, inner => inner.consume(amt)) + } + + fn read_until(&mut self, byte: u8, buf: &mut std::vec::Vec) -> io::Result { + for_both!(self, inner => inner.read_until(byte, buf)) + } + + fn read_line(&mut self, buf: &mut std::string::String) -> io::Result { + for_both!(self, inner => inner.read_line(buf)) + } +} + +#[cfg(any(test, feature = "std"))] +/// `Either` implements `Write` if both `L` and `R` do. +/// +/// Requires crate feature `"std"` +impl Write for Either +where + L: Write, + R: Write, +{ + fn write(&mut self, buf: &[u8]) -> io::Result { + for_both!(self, inner => inner.write(buf)) + } + + fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { + for_both!(self, inner => inner.write_all(buf)) + } + + fn write_fmt(&mut self, fmt: fmt::Arguments<'_>) -> io::Result<()> { + for_both!(self, inner => inner.write_fmt(fmt)) + } + + fn flush(&mut self) -> io::Result<()> { + for_both!(self, inner => inner.flush()) + } +} + +impl AsRef for Either +where + L: AsRef, + R: AsRef, +{ + fn as_ref(&self) -> &Target { + for_both!(self, inner => inner.as_ref()) + } +} + +macro_rules! impl_specific_ref_and_mut { + ($t:ty, $($attr:meta),* ) => { + $(#[$attr])* + impl AsRef<$t> for Either + where L: AsRef<$t>, R: AsRef<$t> + { + fn as_ref(&self) -> &$t { + for_both!(self, inner => inner.as_ref()) + } + } + + $(#[$attr])* + impl AsMut<$t> for Either + where L: AsMut<$t>, R: AsMut<$t> + { + fn as_mut(&mut self) -> &mut $t { + for_both!(self, inner => inner.as_mut()) + } + } + }; +} + +impl_specific_ref_and_mut!(str,); +impl_specific_ref_and_mut!( + ::std::path::Path, + cfg(feature = "std"), + doc = "Requires crate feature `std`." +); +impl_specific_ref_and_mut!( + ::std::ffi::OsStr, + cfg(feature = "std"), + doc = "Requires crate feature `std`." +); +impl_specific_ref_and_mut!( + ::std::ffi::CStr, + cfg(feature = "std"), + doc = "Requires crate feature `std`." +); + +impl AsRef<[Target]> for Either +where + L: AsRef<[Target]>, + R: AsRef<[Target]>, +{ + fn as_ref(&self) -> &[Target] { + for_both!(self, inner => inner.as_ref()) + } +} + +impl AsMut for Either +where + L: AsMut, + R: AsMut, +{ + fn as_mut(&mut self) -> &mut Target { + for_both!(self, inner => inner.as_mut()) + } +} + +impl AsMut<[Target]> for Either +where + L: AsMut<[Target]>, + R: AsMut<[Target]>, +{ + fn as_mut(&mut self) -> &mut [Target] { + for_both!(self, inner => inner.as_mut()) + } +} + +impl Deref for Either +where + L: Deref, + R: Deref, +{ + type Target = L::Target; + + fn deref(&self) -> &Self::Target { + for_both!(self, inner => &**inner) + } +} + +impl DerefMut for Either +where + L: DerefMut, + R: DerefMut, +{ + fn deref_mut(&mut self) -> &mut Self::Target { + for_both!(self, inner => &mut *inner) + } +} + +#[cfg(any(test, feature = "std"))] +/// `Either` implements `Error` if *both* `L` and `R` implement it. +/// +/// Requires crate feature `"std"` +impl Error for Either +where + L: Error, + R: Error, +{ + fn source(&self) -> Option<&(dyn Error + 'static)> { + for_both!(self, inner => inner.source()) + } + + #[allow(deprecated)] + fn description(&self) -> &str { + for_both!(self, inner => inner.description()) + } + + #[allow(deprecated)] + fn cause(&self) -> Option<&dyn Error> { + for_both!(self, inner => inner.cause()) + } +} + +impl fmt::Display for Either +where + L: fmt::Display, + R: fmt::Display, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for_both!(self, inner => inner.fmt(f)) + } +} + +impl fmt::Write for Either +where + L: fmt::Write, + R: fmt::Write, +{ + fn write_str(&mut self, s: &str) -> fmt::Result { + for_both!(self, inner => inner.write_str(s)) + } + + fn write_char(&mut self, c: char) -> fmt::Result { + for_both!(self, inner => inner.write_char(c)) + } + + fn write_fmt(&mut self, args: fmt::Arguments<'_>) -> fmt::Result { + for_both!(self, inner => inner.write_fmt(args)) + } +} + +#[test] +fn basic() { + let mut e = Left(2); + let r = Right(2); + assert_eq!(e, Left(2)); + e = r; + assert_eq!(e, Right(2)); + assert_eq!(e.left(), None); + assert_eq!(e.right(), Some(2)); + assert_eq!(e.as_ref().right(), Some(&2)); + assert_eq!(e.as_mut().right(), Some(&mut 2)); +} + +#[test] +fn macros() { + use std::string::String; + + fn a() -> Either { + let x: u32 = try_left!(Right(1337u32)); + Left(x * 2) + } + assert_eq!(a(), Right(1337)); + + fn b() -> Either { + Right(try_right!(Left("foo bar"))) + } + assert_eq!(b(), Left(String::from("foo bar"))); +} + +#[test] +fn deref() { + use std::string::String; + + fn is_str(_: &str) {} + let value: Either = Left(String::from("test")); + is_str(&value); +} + +#[test] +fn iter() { + let x = 3; + let mut iter = match x { + 3 => Left(0..10), + _ => Right(17..), + }; + + assert_eq!(iter.next(), Some(0)); + assert_eq!(iter.count(), 9); +} + +#[test] +fn seek() { + use std::io; + + let use_empty = false; + let mut mockdata = [0x00; 256]; + for (i, data) in mockdata.iter_mut().enumerate() { + *data = i as u8; + } + + let mut reader = if use_empty { + // Empty didn't impl Seek until Rust 1.51 + Left(io::Cursor::new([])) + } else { + Right(io::Cursor::new(&mockdata[..])) + }; + + let mut buf = [0u8; 16]; + assert_eq!(reader.read(&mut buf).unwrap(), buf.len()); + assert_eq!(buf, mockdata[..buf.len()]); + + // the first read should advance the cursor and return the next 16 bytes thus the `ne` + assert_eq!(reader.read(&mut buf).unwrap(), buf.len()); + assert_ne!(buf, mockdata[..buf.len()]); + + // if the seek operation fails it should read 16..31 instead of 0..15 + reader.seek(io::SeekFrom::Start(0)).unwrap(); + assert_eq!(reader.read(&mut buf).unwrap(), buf.len()); + assert_eq!(buf, mockdata[..buf.len()]); +} + +#[test] +fn read_write() { + use std::io; + + let use_stdio = false; + let mockdata = [0xff; 256]; + + let mut reader = if use_stdio { + Left(io::stdin()) + } else { + Right(&mockdata[..]) + }; + + let mut buf = [0u8; 16]; + assert_eq!(reader.read(&mut buf).unwrap(), buf.len()); + assert_eq!(&buf, &mockdata[..buf.len()]); + + let mut mockbuf = [0u8; 256]; + let mut writer = if use_stdio { + Left(io::stdout()) + } else { + Right(&mut mockbuf[..]) + }; + + let buf = [1u8; 16]; + assert_eq!(writer.write(&buf).unwrap(), buf.len()); +} + +#[test] +fn error() { + let invalid_utf8 = b"\xff"; + #[allow(invalid_from_utf8)] + let res = if let Err(error) = ::std::str::from_utf8(invalid_utf8) { + Err(Left(error)) + } else if let Err(error) = "x".parse::() { + Err(Right(error)) + } else { + Ok(()) + }; + assert!(res.is_err()); + #[allow(deprecated)] + res.unwrap_err().description(); // make sure this can be called +} + +/// A helper macro to check if AsRef and AsMut are implemented for a given type. +macro_rules! check_t { + ($t:ty) => {{ + fn check_ref>() {} + fn propagate_ref, T2: AsRef<$t>>() { + check_ref::>() + } + fn check_mut>() {} + fn propagate_mut, T2: AsMut<$t>>() { + check_mut::>() + } + }}; +} + +// This "unused" method is here to ensure that compilation doesn't fail on given types. +fn _unsized_ref_propagation() { + check_t!(str); + + fn check_array_ref, Item>() {} + fn check_array_mut, Item>() {} + + fn propagate_array_ref, T2: AsRef<[Item]>, Item>() { + check_array_ref::, _>() + } + + fn propagate_array_mut, T2: AsMut<[Item]>, Item>() { + check_array_mut::, _>() + } +} + +// This "unused" method is here to ensure that compilation doesn't fail on given types. +#[cfg(feature = "std")] +fn _unsized_std_propagation() { + check_t!(::std::path::Path); + check_t!(::std::ffi::OsStr); + check_t!(::std::ffi::CStr); +} diff --git a/anneal/v2/vendor/either/src/serde_untagged.rs b/anneal/v2/vendor/either/src/serde_untagged.rs new file mode 100644 index 0000000000..d62dd7d065 --- /dev/null +++ b/anneal/v2/vendor/either/src/serde_untagged.rs @@ -0,0 +1,69 @@ +//! Untagged serialization/deserialization support for `Either`. +//! +//! `Either` uses default, externally-tagged representation. +//! However, sometimes it is useful to support several alternative types. +//! For example, we may have a field which is generally a `HashMap` +//! but in typical cases `Vec` would suffice, too. +//! +//! ```rust +//! # fn main() -> Result<(), Box> { +//! use either::Either; +//! use std::collections::HashMap; +//! +//! #[derive(serde::Serialize, serde::Deserialize, Debug)] +//! #[serde(transparent)] +//! struct IntOrString { +//! #[serde(with = "either::serde_untagged")] +//! inner: Either, HashMap> +//! }; +//! +//! // serialization +//! let data = IntOrString { +//! inner: Either::Left(vec!["Hello".to_string()]) +//! }; +//! // notice: no tags are emitted. +//! assert_eq!(serde_json::to_string(&data)?, r#"["Hello"]"#); +//! +//! // deserialization +//! let data: IntOrString = serde_json::from_str( +//! r#"{"a": 0, "b": 14}"# +//! )?; +//! println!("found {:?}", data); +//! # Ok(()) +//! # } +//! ``` + +use serde::{Deserialize, Deserializer, Serialize, Serializer}; + +#[derive(serde::Serialize, serde::Deserialize)] +#[serde(untagged)] +enum Either { + Left(L), + Right(R), +} + +pub fn serialize(this: &super::Either, serializer: S) -> Result +where + S: Serializer, + L: Serialize, + R: Serialize, +{ + let untagged = match this { + super::Either::Left(left) => Either::Left(left), + super::Either::Right(right) => Either::Right(right), + }; + untagged.serialize(serializer) +} + +pub fn deserialize<'de, L, R, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, + L: Deserialize<'de>, + R: Deserialize<'de>, +{ + match Either::deserialize(deserializer) { + Ok(Either::Left(left)) => Ok(super::Either::Left(left)), + Ok(Either::Right(right)) => Ok(super::Either::Right(right)), + Err(error) => Err(error), + } +} diff --git a/anneal/v2/vendor/either/src/serde_untagged_optional.rs b/anneal/v2/vendor/either/src/serde_untagged_optional.rs new file mode 100644 index 0000000000..d1316935c4 --- /dev/null +++ b/anneal/v2/vendor/either/src/serde_untagged_optional.rs @@ -0,0 +1,74 @@ +//! Untagged serialization/deserialization support for `Option>`. +//! +//! `Either` uses default, externally-tagged representation. +//! However, sometimes it is useful to support several alternative types. +//! For example, we may have a field which is generally a `HashMap` +//! but in typical cases `Vec` would suffice, too. +//! +//! ```rust +//! # fn main() -> Result<(), Box> { +//! use either::Either; +//! use std::collections::HashMap; +//! +//! #[derive(serde::Serialize, serde::Deserialize, Debug)] +//! #[serde(transparent)] +//! struct IntOrString { +//! #[serde(with = "either::serde_untagged_optional")] +//! inner: Option, HashMap>> +//! }; +//! +//! // serialization +//! let data = IntOrString { +//! inner: Some(Either::Left(vec!["Hello".to_string()])) +//! }; +//! // notice: no tags are emitted. +//! assert_eq!(serde_json::to_string(&data)?, r#"["Hello"]"#); +//! +//! // deserialization +//! let data: IntOrString = serde_json::from_str( +//! r#"{"a": 0, "b": 14}"# +//! )?; +//! println!("found {:?}", data); +//! # Ok(()) +//! # } +//! ``` + +use serde::{Deserialize, Deserializer, Serialize, Serializer}; + +#[derive(Serialize, Deserialize)] +#[serde(untagged)] +enum Either { + Left(L), + Right(R), +} + +pub fn serialize( + this: &Option>, + serializer: S, +) -> Result +where + S: Serializer, + L: Serialize, + R: Serialize, +{ + let untagged = match this { + Some(super::Either::Left(left)) => Some(Either::Left(left)), + Some(super::Either::Right(right)) => Some(Either::Right(right)), + None => None, + }; + untagged.serialize(serializer) +} + +pub fn deserialize<'de, L, R, D>(deserializer: D) -> Result>, D::Error> +where + D: Deserializer<'de>, + L: Deserialize<'de>, + R: Deserialize<'de>, +{ + match Option::deserialize(deserializer) { + Ok(Some(Either::Left(left))) => Ok(Some(super::Either::Left(left))), + Ok(Some(Either::Right(right))) => Ok(Some(super::Either::Right(right))), + Ok(None) => Ok(None), + Err(error) => Err(error), + } +} diff --git a/anneal/v2/vendor/rayon-core/.cargo-checksum.json b/anneal/v2/vendor/rayon-core/.cargo-checksum.json new file mode 100644 index 0000000000..27934bb572 --- /dev/null +++ b/anneal/v2/vendor/rayon-core/.cargo-checksum.json @@ -0,0 +1 @@ +{"files":{".cargo_vcs_info.json":"c6bfcaeeeb689844bb154055fe696ced313b9b5f3523528346dbbe46806c35dd","Cargo.lock":"e30b2aa3d7e6f59dda0ef717580cec3d787d3f13a0cd3da46969874736e48684","Cargo.toml":"ab50ad4709436f6bda04e5345444f1197b1eda2bd3b49f9412c0cb318111b5a2","Cargo.toml.orig":"2fb5b9f5a2ec91a1e3f2d9c26cf611bb5a9eeb30fcccb68cf6d087ad522176ab","LICENSE-APACHE":"a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2","LICENSE-MIT":"0621878e61f0d0fda054bcbe02df75192c28bde1ecc8289cbd86aeba2dd72720","README.md":"37a9a028cb507dbaebe7bb60fce9194f0f5b51fb3a52b2ac616969cf6ff4ef95","build.rs":"fa31cb198b772600d100a7c403ddedccef637d2e6b2da431fa7f02ca41307fc6","src/broadcast/mod.rs":"850859640276aefefba3493ad6f07a3122b869c64fd617212f2c1e178f3c39ee","src/broadcast/test.rs":"fe50fc868e67d855a9f71e078b0c3a7780e789652abb4b586accb4ccf035e872","src/compile_fail/mod.rs":"4d70256295bd64691a8c1994b323559cda1888e85f0b45ca55711541c257dcb6","src/compile_fail/quicksort_race1.rs":"35f498cda38f4eb6e00117f78ed68e0fe5a3fa61c25303d9c08a19bda345bc6c","src/compile_fail/quicksort_race2.rs":"cbb40030c7867cae34bb373b6ec5d97c2ac6de39bc917f47879b30eb423924bc","src/compile_fail/quicksort_race3.rs":"8403643e64c969306b1a9b1243378e6ccdd313b57e1878dbd31393618082fd35","src/compile_fail/rc_return.rs":"197894803d8df58fc8005d90c86b90cd98f1972f1a4b57438516a565df35903f","src/compile_fail/rc_upvar.rs":"42d110429621f407ef0dada1306dab116583d2c782a99894204dd8e0ccd2312f","src/compile_fail/scope_join_bad.rs":"892959949f77cadfc07458473e7a290301182027ca64428df5a8ce887be0892b","src/job.rs":"06de0c2add2e303b6383bf11f5f0d75775c1efe6aa7bc16de3992117f1012f09","src/join/mod.rs":"0d1d6ba860c65fd991a81459110dfb9ca03e590d830b515e5afc9c7b6bae6b08","src/join/test.rs":"b853c3b99a1cca585069c0ccc9eb3f962533d5b3c7875dba6b905aae3880c473","src/latch.rs":"e749d6d15e685ca164cd7f2dcad6b2701ae04bc53cb5144be0c43f012406eddd","src/lib.rs":"5da845890b55010d8f5e243ac9d112c60c710a0f412dce8ebf6f8b6987bd261c","src/private.rs":"152f6d65ce4741616a1dec796b9442f78a018d38bb040f76c4cd85008333a3bb","src/registry.rs":"61674f4cdf39edb26df75435de6925d54c5f864f9e1dcd0fb114ad2cc553a1d7","src/scope/mod.rs":"b7720b853328173764b33e2d866c33c49148d777254a8d4a51e766b692537914","src/scope/test.rs":"f358307efb3c0f5dbd8a7f7380aaf23603aae946bf2eb01f8adc35122e6b46d0","src/sleep/README.md":"8790c7795d1a37e0c3016d4ac220ff2fc37f3995d2b5f251bbcb2216362fb9cd","src/sleep/counters.rs":"3e7880534822abbbd59210c378ca611b75759e8b566b167263fd843cd638dd22","src/sleep/mod.rs":"937bf9fc0610bf526522e6ef2efeca9b75f65d87cfaf69375277e161a4130e5e","src/spawn/mod.rs":"3520ce5822787a592e5c7da9d0dd8a4a4d9eb08235e598ab63a50bf43fef56f4","src/spawn/test.rs":"b4850681aa5ff607d927b1d3f4ba2b4a3199ddaa283a95ea44547b997cf17e61","src/test.rs":"b82d95f84314305ff3268d341f731851e4d33fc895c34219ed8ff08b8d864575","src/thread_pool/mod.rs":"f071a1bc766cde10ec844cc3b53e53479c3f1ade97ceaff1cc9e3f9913ade9f3","src/thread_pool/test.rs":"e9377afa0d5727a9f6874b5abf4589403d8dc8938e4e2c5dc26be4106daa77b9","src/unwind.rs":"7baa4511467d008b14856ea8de7ced92b9251c9df4854f69c81f7efc3cf0cd6c","tests/double_init_fail.rs":"8c208ce45e83ab1dfc5890353d5b2f06fc8005684ae622827a65d05abb35a072","tests/init_zero_threads.rs":"5c7f7e0e13e9ead3733253e30d6b52ac5ee66fd6c105999d096bdf31cfccaf95","tests/scope_join.rs":"56f570c4b6a01704aacf93e7f17f89fe0f40f46ed6f9ede517abfe9adaf91f83","tests/scoped_threadpool.rs":"24d1293fe65ad5f194bbff9d1ef0486c3440d0a3783f04eaaaae4929adef5cb8","tests/simple_panic.rs":"916d40d36c1a0fad3e1dfb31550f0672641feab4b03d480f039143dbe2f2445f","tests/stack_overflow_crash.rs":"87b962c66f301ac44f808d992d4e8b861305db0c282f256761a5075c9f018243","tests/use_current_thread.rs":"552ec922f123aafdaeafef8f31c802781ebed16bf35cb5eac128bef0c29199cc"},"package":"22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"} \ No newline at end of file diff --git a/anneal/v2/vendor/rayon-core/.cargo_vcs_info.json b/anneal/v2/vendor/rayon-core/.cargo_vcs_info.json new file mode 100644 index 0000000000..002271ae89 --- /dev/null +++ b/anneal/v2/vendor/rayon-core/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "7af20d7692d5decbcb4adcaa079cd607e3e50814" + }, + "path_in_vcs": "rayon-core" +} \ No newline at end of file diff --git a/anneal/v2/vendor/rayon-core/Cargo.lock b/anneal/v2/vendor/rayon-core/Cargo.lock new file mode 100644 index 0000000000..01fff34cd3 --- /dev/null +++ b/anneal/v2/vendor/rayon-core/Cargo.lock @@ -0,0 +1,309 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "bitflags" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" + +[[package]] +name = "bumpalo" +version = "3.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" + +[[package]] +name = "cfg-if" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "getrandom" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi", +] + +[[package]] +name = "js-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.174" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" + +[[package]] +name = "log" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", + "libc", + "rand", + "rand_xorshift", + "scoped-tls", + "wasm_sync", +] + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "syn" +version = "2.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" + +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +dependencies = [ + "cfg-if", + "once_cell", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm_sync" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cff360cade7fec41ff0e9d2cda57fe58258c5f16def0e21302394659e6bbb0ea" +dependencies = [ + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags", +] + +[[package]] +name = "zerocopy" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/anneal/v2/vendor/rayon-core/Cargo.toml b/anneal/v2/vendor/rayon-core/Cargo.toml new file mode 100644 index 0000000000..7a521f87be --- /dev/null +++ b/anneal/v2/vendor/rayon-core/Cargo.toml @@ -0,0 +1,93 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +rust-version = "1.80" +name = "rayon-core" +version = "1.13.0" +build = "build.rs" +links = "rayon-core" +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Core APIs for Rayon" +documentation = "https://docs.rs/rayon-core/" +readme = "README.md" +keywords = [ + "parallel", + "thread", + "concurrency", + "join", + "performance", +] +categories = ["concurrency"] +license = "MIT OR Apache-2.0" +repository = "https://github.com/rayon-rs/rayon" + +[features] +web_spin_lock = ["dep:wasm_sync"] + +[lib] +name = "rayon_core" +path = "src/lib.rs" + +[[test]] +name = "double_init_fail" +path = "tests/double_init_fail.rs" + +[[test]] +name = "init_zero_threads" +path = "tests/init_zero_threads.rs" + +[[test]] +name = "scope_join" +path = "tests/scope_join.rs" + +[[test]] +name = "scoped_threadpool" +path = "tests/scoped_threadpool.rs" + +[[test]] +name = "simple_panic" +path = "tests/simple_panic.rs" + +[[test]] +name = "stack_overflow_crash" +path = "tests/stack_overflow_crash.rs" + +[[test]] +name = "use_current_thread" +path = "tests/use_current_thread.rs" + +[dependencies.crossbeam-deque] +version = "0.8.1" + +[dependencies.crossbeam-utils] +version = "0.8.0" + +[dependencies.wasm_sync] +version = "0.1.0" +optional = true + +[dev-dependencies.rand] +version = "0.9" + +[dev-dependencies.rand_xorshift] +version = "0.4" + +[dev-dependencies.scoped-tls] +version = "1.0" + +[target."cfg(unix)".dev-dependencies.libc] +version = "0.2" diff --git a/anneal/v2/vendor/rayon-core/Cargo.toml.orig b/anneal/v2/vendor/rayon-core/Cargo.toml.orig new file mode 100644 index 0000000000..5f34e257bd --- /dev/null +++ b/anneal/v2/vendor/rayon-core/Cargo.toml.orig @@ -0,0 +1,35 @@ +[package] +name = "rayon-core" +version = "1.13.0" +description = "Core APIs for Rayon" +documentation = "https://docs.rs/rayon-core/" +links = "rayon-core" +build = "build.rs" + +categories.workspace = true +edition.workspace = true +keywords.workspace = true +license.workspace = true +readme.workspace = true +repository.workspace = true +rust-version.workspace = true + +[features] +# This feature switches to a spin-lock implementation on the browser's +# main thread to avoid the forbidden `atomics.wait`. +# +# Only useful on the `wasm32-unknown-unknown` target. +web_spin_lock = ["dep:wasm_sync"] + +[dependencies] +crossbeam-deque.workspace = true +crossbeam-utils.workspace = true +wasm_sync = { workspace = true, optional = true } + +[dev-dependencies] +rand.workspace = true +rand_xorshift.workspace = true +scoped-tls.workspace = true + +[target.'cfg(unix)'.dev-dependencies] +libc.workspace = true diff --git a/anneal/v2/vendor/rayon-core/LICENSE-APACHE b/anneal/v2/vendor/rayon-core/LICENSE-APACHE new file mode 100644 index 0000000000..16fe87b06e --- /dev/null +++ b/anneal/v2/vendor/rayon-core/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/anneal/v2/vendor/rayon-core/LICENSE-MIT b/anneal/v2/vendor/rayon-core/LICENSE-MIT new file mode 100644 index 0000000000..25597d5838 --- /dev/null +++ b/anneal/v2/vendor/rayon-core/LICENSE-MIT @@ -0,0 +1,25 @@ +Copyright (c) 2010 The Rust Project Developers + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/anneal/v2/vendor/rayon-core/README.md b/anneal/v2/vendor/rayon-core/README.md new file mode 100644 index 0000000000..ed2aded1ba --- /dev/null +++ b/anneal/v2/vendor/rayon-core/README.md @@ -0,0 +1,11 @@ +Rayon-core represents the "core, stable" APIs of Rayon: join, scope, and so forth, as well as the ability to create custom thread pools with ThreadPool. + +Maybe worth mentioning: users are not necessarily intended to directly access rayon-core; all its APIs are mirrored in the rayon crate. To that end, the examples in the docs use rayon::join and so forth rather than rayon_core::join. + +rayon-core aims to never, or almost never, have a breaking change to its API, because each revision of rayon-core also houses the global thread pool (and hence if you have two simultaneous versions of rayon-core, you have two thread pools). + +Please see [Rayon Docs] for details about using Rayon. + +[Rayon Docs]: https://docs.rs/rayon/ + +Rayon-core currently requires `rustc 1.80.0` or greater. diff --git a/anneal/v2/vendor/rayon-core/build.rs b/anneal/v2/vendor/rayon-core/build.rs new file mode 100644 index 0000000000..8771b63fcf --- /dev/null +++ b/anneal/v2/vendor/rayon-core/build.rs @@ -0,0 +1,7 @@ +// We need a build script to use `link = "rayon-core"`. But we're not +// *actually* linking to anything, just making sure that we're the only +// rayon-core in use. +fn main() { + // we don't need to rebuild for anything else + println!("cargo:rerun-if-changed=build.rs"); +} diff --git a/anneal/v2/vendor/rayon-core/src/broadcast/mod.rs b/anneal/v2/vendor/rayon-core/src/broadcast/mod.rs new file mode 100644 index 0000000000..17526e3434 --- /dev/null +++ b/anneal/v2/vendor/rayon-core/src/broadcast/mod.rs @@ -0,0 +1,150 @@ +use crate::job::{ArcJob, StackJob}; +use crate::latch::{CountLatch, LatchRef}; +use crate::registry::{Registry, WorkerThread}; +use std::fmt; +use std::marker::PhantomData; +use std::sync::Arc; + +mod test; + +/// Executes `op` within every thread in the current thread pool. If this is +/// called from a non-Rayon thread, it will execute in the global thread pool. +/// Any attempts to use `join`, `scope`, or parallel iterators will then operate +/// within that thread pool. When the call has completed on each thread, returns +/// a vector containing all of their return values. +/// +/// For more information, see the [`ThreadPool::broadcast()`] method. +/// +/// [`ThreadPool::broadcast()`]: crate::ThreadPool::broadcast() +pub fn broadcast(op: OP) -> Vec +where + OP: Fn(BroadcastContext<'_>) -> R + Sync, + R: Send, +{ + // We assert that current registry has not terminated. + unsafe { broadcast_in(op, &Registry::current()) } +} + +/// Spawns an asynchronous task on every thread in this thread pool. This task +/// will run in the implicit, global scope, which means that it may outlast the +/// current stack frame -- therefore, it cannot capture any references onto the +/// stack (you will likely need a `move` closure). +/// +/// For more information, see the [`ThreadPool::spawn_broadcast()`] method. +/// +/// [`ThreadPool::spawn_broadcast()`]: crate::ThreadPool::spawn_broadcast() +pub fn spawn_broadcast(op: OP) +where + OP: Fn(BroadcastContext<'_>) + Send + Sync + 'static, +{ + // We assert that current registry has not terminated. + unsafe { spawn_broadcast_in(op, &Registry::current()) } +} + +/// Provides context to a closure called by `broadcast`. +pub struct BroadcastContext<'a> { + worker: &'a WorkerThread, + + /// Make sure to prevent auto-traits like `Send` and `Sync`. + _marker: PhantomData<&'a mut dyn Fn()>, +} + +impl<'a> BroadcastContext<'a> { + pub(super) fn with(f: impl FnOnce(BroadcastContext<'_>) -> R) -> R { + let worker_thread = WorkerThread::current(); + assert!(!worker_thread.is_null()); + f(BroadcastContext { + worker: unsafe { &*worker_thread }, + _marker: PhantomData, + }) + } + + /// Our index amongst the broadcast threads (ranges from `0..self.num_threads()`). + #[inline] + pub fn index(&self) -> usize { + self.worker.index() + } + + /// The number of threads receiving the broadcast in the thread pool. + /// + /// # Future compatibility note + /// + /// Future versions of Rayon might vary the number of threads over time, but + /// this method will always return the number of threads which are actually + /// receiving your particular `broadcast` call. + #[inline] + pub fn num_threads(&self) -> usize { + self.worker.registry().num_threads() + } +} + +impl<'a> fmt::Debug for BroadcastContext<'a> { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.debug_struct("BroadcastContext") + .field("index", &self.index()) + .field("num_threads", &self.num_threads()) + .field("pool_id", &self.worker.registry().id()) + .finish() + } +} + +/// Execute `op` on every thread in the pool. It will be executed on each +/// thread when they have nothing else to do locally, before they try to +/// steal work from other threads. This function will not return until all +/// threads have completed the `op`. +/// +/// Unsafe because `registry` must not yet have terminated. +pub(super) unsafe fn broadcast_in(op: OP, registry: &Arc) -> Vec +where + OP: Fn(BroadcastContext<'_>) -> R + Sync, + R: Send, +{ + let f = move |injected: bool| { + debug_assert!(injected); + BroadcastContext::with(&op) + }; + + let n_threads = registry.num_threads(); + let current_thread = WorkerThread::current().as_ref(); + let latch = CountLatch::with_count(n_threads, current_thread); + let jobs: Vec<_> = (0..n_threads) + .map(|_| StackJob::new(&f, LatchRef::new(&latch))) + .collect(); + let job_refs = jobs.iter().map(|job| job.as_job_ref()); + + registry.inject_broadcast(job_refs); + + // Wait for all jobs to complete, then collect the results, maybe propagating a panic. + latch.wait(current_thread); + jobs.into_iter().map(|job| job.into_result()).collect() +} + +/// Execute `op` on every thread in the pool. It will be executed on each +/// thread when they have nothing else to do locally, before they try to +/// steal work from other threads. This function returns immediately after +/// injecting the jobs. +/// +/// Unsafe because `registry` must not yet have terminated. +pub(super) unsafe fn spawn_broadcast_in(op: OP, registry: &Arc) +where + OP: Fn(BroadcastContext<'_>) + Send + Sync + 'static, +{ + let job = ArcJob::new({ + let registry = Arc::clone(registry); + move || { + registry.catch_unwind(|| BroadcastContext::with(&op)); + registry.terminate(); // (*) permit registry to terminate now + } + }); + + let n_threads = registry.num_threads(); + let job_refs = (0..n_threads).map(|_| { + // Ensure that registry cannot terminate until this job has executed + // on each thread. This ref is decremented at the (*) above. + registry.increment_terminate_count(); + + ArcJob::as_static_job_ref(&job) + }); + + registry.inject_broadcast(job_refs); +} diff --git a/anneal/v2/vendor/rayon-core/src/broadcast/test.rs b/anneal/v2/vendor/rayon-core/src/broadcast/test.rs new file mode 100644 index 0000000000..00ab4ad7fe --- /dev/null +++ b/anneal/v2/vendor/rayon-core/src/broadcast/test.rs @@ -0,0 +1,263 @@ +#![cfg(test)] + +use crate::ThreadPoolBuilder; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::mpsc::channel; +use std::sync::Arc; +use std::{thread, time}; + +#[test] +fn broadcast_global() { + let v = crate::broadcast(|ctx| ctx.index()); + assert!(v.into_iter().eq(0..crate::current_num_threads())); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn spawn_broadcast_global() { + let (tx, rx) = channel(); + crate::spawn_broadcast(move |ctx| tx.send(ctx.index()).unwrap()); + + let mut v: Vec<_> = rx.into_iter().collect(); + v.sort_unstable(); + assert!(v.into_iter().eq(0..crate::current_num_threads())); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn broadcast_pool() { + let pool = ThreadPoolBuilder::new().num_threads(7).build().unwrap(); + let v = pool.broadcast(|ctx| ctx.index()); + assert!(v.into_iter().eq(0..7)); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn spawn_broadcast_pool() { + let (tx, rx) = channel(); + let pool = ThreadPoolBuilder::new().num_threads(7).build().unwrap(); + pool.spawn_broadcast(move |ctx| tx.send(ctx.index()).unwrap()); + + let mut v: Vec<_> = rx.into_iter().collect(); + v.sort_unstable(); + assert!(v.into_iter().eq(0..7)); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn broadcast_self() { + let pool = ThreadPoolBuilder::new().num_threads(7).build().unwrap(); + let v = pool.install(|| crate::broadcast(|ctx| ctx.index())); + assert!(v.into_iter().eq(0..7)); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn spawn_broadcast_self() { + let (tx, rx) = channel(); + let pool = ThreadPoolBuilder::new().num_threads(7).build().unwrap(); + pool.spawn(|| crate::spawn_broadcast(move |ctx| tx.send(ctx.index()).unwrap())); + + let mut v: Vec<_> = rx.into_iter().collect(); + v.sort_unstable(); + assert!(v.into_iter().eq(0..7)); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn broadcast_mutual() { + let count = AtomicUsize::new(0); + let pool1 = ThreadPoolBuilder::new().num_threads(3).build().unwrap(); + let pool2 = ThreadPoolBuilder::new().num_threads(7).build().unwrap(); + pool1.install(|| { + pool2.broadcast(|_| { + pool1.broadcast(|_| { + count.fetch_add(1, Ordering::Relaxed); + }) + }) + }); + assert_eq!(count.into_inner(), 3 * 7); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn spawn_broadcast_mutual() { + let (tx, rx) = channel(); + let pool1 = Arc::new(ThreadPoolBuilder::new().num_threads(3).build().unwrap()); + let pool2 = ThreadPoolBuilder::new().num_threads(7).build().unwrap(); + pool1.spawn({ + let pool1 = Arc::clone(&pool1); + move || { + pool2.spawn_broadcast(move |_| { + let tx = tx.clone(); + pool1.spawn_broadcast(move |_| tx.send(()).unwrap()) + }) + } + }); + assert_eq!(rx.into_iter().count(), 3 * 7); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn broadcast_mutual_sleepy() { + let count = AtomicUsize::new(0); + let pool1 = ThreadPoolBuilder::new().num_threads(3).build().unwrap(); + let pool2 = ThreadPoolBuilder::new().num_threads(7).build().unwrap(); + pool1.install(|| { + thread::sleep(time::Duration::from_secs(1)); + pool2.broadcast(|_| { + thread::sleep(time::Duration::from_secs(1)); + pool1.broadcast(|_| { + thread::sleep(time::Duration::from_millis(100)); + count.fetch_add(1, Ordering::Relaxed); + }) + }) + }); + assert_eq!(count.into_inner(), 3 * 7); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn spawn_broadcast_mutual_sleepy() { + let (tx, rx) = channel(); + let pool1 = Arc::new(ThreadPoolBuilder::new().num_threads(3).build().unwrap()); + let pool2 = ThreadPoolBuilder::new().num_threads(7).build().unwrap(); + pool1.spawn({ + let pool1 = Arc::clone(&pool1); + move || { + thread::sleep(time::Duration::from_secs(1)); + pool2.spawn_broadcast(move |_| { + let tx = tx.clone(); + thread::sleep(time::Duration::from_secs(1)); + pool1.spawn_broadcast(move |_| { + thread::sleep(time::Duration::from_millis(100)); + tx.send(()).unwrap(); + }) + }) + } + }); + assert_eq!(rx.into_iter().count(), 3 * 7); +} + +#[test] +#[cfg_attr(not(panic = "unwind"), ignore)] +fn broadcast_panic_one() { + let count = AtomicUsize::new(0); + let pool = ThreadPoolBuilder::new().num_threads(7).build().unwrap(); + let result = crate::unwind::halt_unwinding(|| { + pool.broadcast(|ctx| { + count.fetch_add(1, Ordering::Relaxed); + if ctx.index() == 3 { + panic!("Hello, world!"); + } + }) + }); + assert_eq!(count.into_inner(), 7); + assert!(result.is_err(), "broadcast panic should propagate!"); +} + +#[test] +#[cfg_attr(not(panic = "unwind"), ignore)] +fn spawn_broadcast_panic_one() { + let (tx, rx) = channel(); + let (panic_tx, panic_rx) = channel(); + let pool = ThreadPoolBuilder::new() + .num_threads(7) + .panic_handler(move |e| panic_tx.send(e).unwrap()) + .build() + .unwrap(); + pool.spawn_broadcast(move |ctx| { + tx.send(()).unwrap(); + if ctx.index() == 3 { + panic!("Hello, world!"); + } + }); + drop(pool); // including panic_tx + assert_eq!(rx.into_iter().count(), 7); + assert_eq!(panic_rx.into_iter().count(), 1); +} + +#[test] +#[cfg_attr(not(panic = "unwind"), ignore)] +fn broadcast_panic_many() { + let count = AtomicUsize::new(0); + let pool = ThreadPoolBuilder::new().num_threads(7).build().unwrap(); + let result = crate::unwind::halt_unwinding(|| { + pool.broadcast(|ctx| { + count.fetch_add(1, Ordering::Relaxed); + if ctx.index() % 2 == 0 { + panic!("Hello, world!"); + } + }) + }); + assert_eq!(count.into_inner(), 7); + assert!(result.is_err(), "broadcast panic should propagate!"); +} + +#[test] +#[cfg_attr(not(panic = "unwind"), ignore)] +fn spawn_broadcast_panic_many() { + let (tx, rx) = channel(); + let (panic_tx, panic_rx) = channel(); + let pool = ThreadPoolBuilder::new() + .num_threads(7) + .panic_handler(move |e| panic_tx.send(e).unwrap()) + .build() + .unwrap(); + pool.spawn_broadcast(move |ctx| { + tx.send(()).unwrap(); + if ctx.index() % 2 == 0 { + panic!("Hello, world!"); + } + }); + drop(pool); // including panic_tx + assert_eq!(rx.into_iter().count(), 7); + assert_eq!(panic_rx.into_iter().count(), 4); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn broadcast_sleep_race() { + let test_duration = time::Duration::from_secs(1); + let pool = ThreadPoolBuilder::new().num_threads(7).build().unwrap(); + let start = time::Instant::now(); + while start.elapsed() < test_duration { + pool.broadcast(|ctx| { + // A slight spread of sleep duration increases the chance that one + // of the threads will race in the pool's idle sleep afterward. + thread::sleep(time::Duration::from_micros(ctx.index() as u64)); + }); + } +} + +#[test] +fn broadcast_after_spawn_broadcast() { + let (tx, rx) = channel(); + + // Queue a non-blocking spawn_broadcast. + crate::spawn_broadcast(move |ctx| tx.send(ctx.index()).unwrap()); + + // This blocking broadcast runs after all prior broadcasts. + crate::broadcast(|_| {}); + + // The spawn_broadcast **must** have run by now on all threads. + let mut v: Vec<_> = rx.try_iter().collect(); + v.sort_unstable(); + assert!(v.into_iter().eq(0..crate::current_num_threads())); +} + +#[test] +fn broadcast_after_spawn() { + let (tx, rx) = channel(); + + // Queue a regular spawn on a thread-local deque. + crate::registry::in_worker(move |_, _| { + crate::spawn(move || tx.send(22).unwrap()); + }); + + // Broadcast runs after the local deque is empty. + crate::broadcast(|_| {}); + + // The spawn **must** have run by now. + assert_eq!(22, rx.try_recv().unwrap()); +} diff --git a/anneal/v2/vendor/rayon-core/src/compile_fail/mod.rs b/anneal/v2/vendor/rayon-core/src/compile_fail/mod.rs new file mode 100644 index 0000000000..f2ec646a4d --- /dev/null +++ b/anneal/v2/vendor/rayon-core/src/compile_fail/mod.rs @@ -0,0 +1,7 @@ +// These modules contain `compile_fail` doc tests. +mod quicksort_race1; +mod quicksort_race2; +mod quicksort_race3; +mod rc_return; +mod rc_upvar; +mod scope_join_bad; diff --git a/anneal/v2/vendor/rayon-core/src/compile_fail/quicksort_race1.rs b/anneal/v2/vendor/rayon-core/src/compile_fail/quicksort_race1.rs new file mode 100644 index 0000000000..5615033895 --- /dev/null +++ b/anneal/v2/vendor/rayon-core/src/compile_fail/quicksort_race1.rs @@ -0,0 +1,28 @@ +/*! ```compile_fail,E0524 + +fn quick_sort(v: &mut [T]) { + if v.len() <= 1 { + return; + } + + let mid = partition(v); + let (lo, _hi) = v.split_at_mut(mid); + rayon_core::join(|| quick_sort(lo), || quick_sort(lo)); //~ ERROR +} + +fn partition(v: &mut [T]) -> usize { + let pivot = v.len() - 1; + let mut i = 0; + for j in 0..pivot { + if v[j] <= v[pivot] { + v.swap(i, j); + i += 1; + } + } + v.swap(i, pivot); + i +} + +fn main() { } + +``` */ diff --git a/anneal/v2/vendor/rayon-core/src/compile_fail/quicksort_race2.rs b/anneal/v2/vendor/rayon-core/src/compile_fail/quicksort_race2.rs new file mode 100644 index 0000000000..020589c29a --- /dev/null +++ b/anneal/v2/vendor/rayon-core/src/compile_fail/quicksort_race2.rs @@ -0,0 +1,28 @@ +/*! ```compile_fail,E0500 + +fn quick_sort(v: &mut [T]) { + if v.len() <= 1 { + return; + } + + let mid = partition(v); + let (lo, _hi) = v.split_at_mut(mid); + rayon_core::join(|| quick_sort(lo), || quick_sort(v)); //~ ERROR +} + +fn partition(v: &mut [T]) -> usize { + let pivot = v.len() - 1; + let mut i = 0; + for j in 0..pivot { + if v[j] <= v[pivot] { + v.swap(i, j); + i += 1; + } + } + v.swap(i, pivot); + i +} + +fn main() { } + +``` */ diff --git a/anneal/v2/vendor/rayon-core/src/compile_fail/quicksort_race3.rs b/anneal/v2/vendor/rayon-core/src/compile_fail/quicksort_race3.rs new file mode 100644 index 0000000000..16fbf3b824 --- /dev/null +++ b/anneal/v2/vendor/rayon-core/src/compile_fail/quicksort_race3.rs @@ -0,0 +1,28 @@ +/*! ```compile_fail,E0524 + +fn quick_sort(v: &mut [T]) { + if v.len() <= 1 { + return; + } + + let mid = partition(v); + let (_lo, hi) = v.split_at_mut(mid); + rayon_core::join(|| quick_sort(hi), || quick_sort(hi)); //~ ERROR +} + +fn partition(v: &mut [T]) -> usize { + let pivot = v.len() - 1; + let mut i = 0; + for j in 0..pivot { + if v[j] <= v[pivot] { + v.swap(i, j); + i += 1; + } + } + v.swap(i, pivot); + i +} + +fn main() { } + +``` */ diff --git a/anneal/v2/vendor/rayon-core/src/compile_fail/rc_return.rs b/anneal/v2/vendor/rayon-core/src/compile_fail/rc_return.rs new file mode 100644 index 0000000000..93e3a60384 --- /dev/null +++ b/anneal/v2/vendor/rayon-core/src/compile_fail/rc_return.rs @@ -0,0 +1,17 @@ +/** ```compile_fail,E0277 + +use std::rc::Rc; + +rayon_core::join(|| Rc::new(22), || ()); //~ ERROR + +``` */ +mod left {} + +/** ```compile_fail,E0277 + +use std::rc::Rc; + +rayon_core::join(|| (), || Rc::new(23)); //~ ERROR + +``` */ +mod right {} diff --git a/anneal/v2/vendor/rayon-core/src/compile_fail/rc_upvar.rs b/anneal/v2/vendor/rayon-core/src/compile_fail/rc_upvar.rs new file mode 100644 index 0000000000..d8aebcfcbf --- /dev/null +++ b/anneal/v2/vendor/rayon-core/src/compile_fail/rc_upvar.rs @@ -0,0 +1,9 @@ +/*! ```compile_fail,E0277 + +use std::rc::Rc; + +let r = Rc::new(22); +rayon_core::join(|| r.clone(), || r.clone()); +//~^ ERROR + +``` */ diff --git a/anneal/v2/vendor/rayon-core/src/compile_fail/scope_join_bad.rs b/anneal/v2/vendor/rayon-core/src/compile_fail/scope_join_bad.rs new file mode 100644 index 0000000000..75e4c5ca6c --- /dev/null +++ b/anneal/v2/vendor/rayon-core/src/compile_fail/scope_join_bad.rs @@ -0,0 +1,24 @@ +/*! ```compile_fail,E0373 + +fn bad_scope(f: F) + where F: FnOnce(&i32) + Send, +{ + rayon_core::scope(|s| { + let x = 22; + s.spawn(|_| f(&x)); //~ ERROR `x` does not live long enough + }); +} + +fn good_scope(f: F) + where F: FnOnce(&i32) + Send, +{ + let x = 22; + rayon_core::scope(|s| { + s.spawn(|_| f(&x)); + }); +} + +fn main() { +} + +``` */ diff --git a/anneal/v2/vendor/rayon-core/src/job.rs b/anneal/v2/vendor/rayon-core/src/job.rs new file mode 100644 index 0000000000..5664bb385f --- /dev/null +++ b/anneal/v2/vendor/rayon-core/src/job.rs @@ -0,0 +1,270 @@ +use crate::latch::Latch; +use crate::unwind; +use crossbeam_deque::{Injector, Steal}; +use std::any::Any; +use std::cell::UnsafeCell; +use std::mem; +use std::sync::Arc; + +pub(super) enum JobResult { + None, + Ok(T), + Panic(Box), +} + +/// A `Job` is used to advertise work for other threads that they may +/// want to steal. In accordance with time honored tradition, jobs are +/// arranged in a deque, so that thieves can take from the top of the +/// deque while the main worker manages the bottom of the deque. This +/// deque is managed by the `thread_pool` module. +pub(super) trait Job { + /// Unsafe: this may be called from a different thread than the one + /// which scheduled the job, so the implementer must ensure the + /// appropriate traits are met, whether `Send`, `Sync`, or both. + unsafe fn execute(this: *const ()); +} + +/// Effectively a Job trait object. Each JobRef **must** be executed +/// exactly once, or else data may leak. +/// +/// Internally, we store the job's data in a `*const ()` pointer. The +/// true type is something like `*const StackJob<...>`, but we hide +/// it. We also carry the "execute fn" from the `Job` trait. +pub(super) struct JobRef { + pointer: *const (), + execute_fn: unsafe fn(*const ()), +} + +unsafe impl Send for JobRef {} +unsafe impl Sync for JobRef {} + +impl JobRef { + /// Unsafe: caller asserts that `data` will remain valid until the + /// job is executed. + pub(super) unsafe fn new(data: *const T) -> JobRef + where + T: Job, + { + // erase types: + JobRef { + pointer: data as *const (), + execute_fn: ::execute, + } + } + + /// Returns an opaque handle that can be saved and compared, + /// without making `JobRef` itself `Copy + Eq`. + #[inline] + pub(super) fn id(&self) -> impl Eq { + (self.pointer, self.execute_fn) + } + + #[inline] + pub(super) unsafe fn execute(self) { + (self.execute_fn)(self.pointer) + } +} + +/// A job that will be owned by a stack slot. This means that when it +/// executes it need not free any heap data, the cleanup occurs when +/// the stack frame is later popped. The function parameter indicates +/// `true` if the job was stolen -- executed on a different thread. +pub(super) struct StackJob +where + L: Latch + Sync, + F: FnOnce(bool) -> R + Send, + R: Send, +{ + pub(super) latch: L, + func: UnsafeCell>, + result: UnsafeCell>, +} + +impl StackJob +where + L: Latch + Sync, + F: FnOnce(bool) -> R + Send, + R: Send, +{ + pub(super) fn new(func: F, latch: L) -> StackJob { + StackJob { + latch, + func: UnsafeCell::new(Some(func)), + result: UnsafeCell::new(JobResult::None), + } + } + + pub(super) unsafe fn as_job_ref(&self) -> JobRef { + JobRef::new(self) + } + + pub(super) unsafe fn run_inline(self, stolen: bool) -> R { + self.func.into_inner().unwrap()(stolen) + } + + pub(super) unsafe fn into_result(self) -> R { + self.result.into_inner().into_return_value() + } +} + +impl Job for StackJob +where + L: Latch + Sync, + F: FnOnce(bool) -> R + Send, + R: Send, +{ + unsafe fn execute(this: *const ()) { + let this = &*(this as *const Self); + let abort = unwind::AbortIfPanic; + let func = (*this.func.get()).take().unwrap(); + (*this.result.get()) = JobResult::call(func); + Latch::set(&this.latch); + mem::forget(abort); + } +} + +/// Represents a job stored in the heap. Used to implement +/// `scope`. Unlike `StackJob`, when executed, `HeapJob` simply +/// invokes a closure, which then triggers the appropriate logic to +/// signal that the job executed. +/// +/// (Probably `StackJob` should be refactored in a similar fashion.) +pub(super) struct HeapJob +where + BODY: FnOnce() + Send, +{ + job: BODY, +} + +impl HeapJob +where + BODY: FnOnce() + Send, +{ + pub(super) fn new(job: BODY) -> Box { + Box::new(HeapJob { job }) + } + + /// Creates a `JobRef` from this job -- note that this hides all + /// lifetimes, so it is up to you to ensure that this JobRef + /// doesn't outlive any data that it closes over. + pub(super) unsafe fn into_job_ref(self: Box) -> JobRef { + JobRef::new(Box::into_raw(self)) + } + + /// Creates a static `JobRef` from this job. + pub(super) fn into_static_job_ref(self: Box) -> JobRef + where + BODY: 'static, + { + unsafe { self.into_job_ref() } + } +} + +impl Job for HeapJob +where + BODY: FnOnce() + Send, +{ + unsafe fn execute(this: *const ()) { + let this = Box::from_raw(this as *mut Self); + (this.job)(); + } +} + +/// Represents a job stored in an `Arc` -- like `HeapJob`, but may +/// be turned into multiple `JobRef`s and called multiple times. +pub(super) struct ArcJob +where + BODY: Fn() + Send + Sync, +{ + job: BODY, +} + +impl ArcJob +where + BODY: Fn() + Send + Sync, +{ + pub(super) fn new(job: BODY) -> Arc { + Arc::new(ArcJob { job }) + } + + /// Creates a `JobRef` from this job -- note that this hides all + /// lifetimes, so it is up to you to ensure that this JobRef + /// doesn't outlive any data that it closes over. + pub(super) unsafe fn as_job_ref(this: &Arc) -> JobRef { + JobRef::new(Arc::into_raw(Arc::clone(this))) + } + + /// Creates a static `JobRef` from this job. + pub(super) fn as_static_job_ref(this: &Arc) -> JobRef + where + BODY: 'static, + { + unsafe { Self::as_job_ref(this) } + } +} + +impl Job for ArcJob +where + BODY: Fn() + Send + Sync, +{ + unsafe fn execute(this: *const ()) { + let this = Arc::from_raw(this as *mut Self); + (this.job)(); + } +} + +impl JobResult { + fn call(func: impl FnOnce(bool) -> T) -> Self { + match unwind::halt_unwinding(|| func(true)) { + Ok(x) => JobResult::Ok(x), + Err(x) => JobResult::Panic(x), + } + } + + /// Convert the `JobResult` for a job that has finished (and hence + /// its JobResult is populated) into its return value. + /// + /// NB. This will panic if the job panicked. + pub(super) fn into_return_value(self) -> T { + match self { + JobResult::None => unreachable!(), + JobResult::Ok(x) => x, + JobResult::Panic(x) => unwind::resume_unwinding(x), + } + } +} + +/// Indirect queue to provide FIFO job priority. +pub(super) struct JobFifo { + inner: Injector, +} + +impl JobFifo { + pub(super) fn new() -> Self { + JobFifo { + inner: Injector::new(), + } + } + + pub(super) unsafe fn push(&self, job_ref: JobRef) -> JobRef { + // A little indirection ensures that spawns are always prioritized in FIFO order. The + // jobs in a thread's deque may be popped from the back (LIFO) or stolen from the front + // (FIFO), but either way they will end up popping from the front of this queue. + self.inner.push(job_ref); + JobRef::new(self) + } +} + +impl Job for JobFifo { + unsafe fn execute(this: *const ()) { + // We "execute" a queue by executing its first job, FIFO. + let this = &*(this as *const Self); + loop { + match this.inner.steal() { + Steal::Success(job_ref) => break job_ref.execute(), + Steal::Empty => panic!("FIFO is empty"), + Steal::Retry => {} + } + } + } +} diff --git a/anneal/v2/vendor/rayon-core/src/join/mod.rs b/anneal/v2/vendor/rayon-core/src/join/mod.rs new file mode 100644 index 0000000000..5a2335822a --- /dev/null +++ b/anneal/v2/vendor/rayon-core/src/join/mod.rs @@ -0,0 +1,186 @@ +use crate::job::StackJob; +use crate::latch::SpinLatch; +use crate::registry::{self, WorkerThread}; +use crate::unwind; +use std::any::Any; + +use crate::FnContext; + +#[cfg(test)] +mod test; + +/// Takes two closures and *potentially* runs them in parallel. It +/// returns a pair of the results from those closures. +/// +/// Conceptually, calling `join()` is similar to spawning two threads, +/// one executing each of the two closures. However, the +/// implementation is quite different and incurs very low +/// overhead. The underlying technique is called "work stealing": the +/// Rayon runtime uses a fixed pool of worker threads and attempts to +/// only execute code in parallel when there are idle CPUs to handle +/// it. +/// +/// When `join` is called from outside the thread pool, the calling +/// thread will block while the closures execute in the pool. When +/// `join` is called within the pool, the calling thread still actively +/// participates in the thread pool. It will begin by executing closure +/// A (on the current thread). While it is doing that, it will advertise +/// closure B as being available for other threads to execute. Once closure A +/// has completed, the current thread will try to execute closure B; +/// if however closure B has been stolen, then it will look for other work +/// while waiting for the thief to fully execute closure B. (This is the +/// typical work-stealing strategy). +/// +/// # Examples +/// +/// This example uses join to perform a quick-sort (note this is not a +/// particularly optimized implementation: if you **actually** want to +/// sort for real, you should prefer [the `par_sort` method] offered +/// by Rayon). +/// +/// [the `par_sort` method]: ../rayon/slice/trait.ParallelSliceMut.html#method.par_sort +/// +/// ```rust +/// # use rayon_core as rayon; +/// let mut v = vec![5, 1, 8, 22, 0, 44]; +/// quick_sort(&mut v); +/// assert_eq!(v, vec![0, 1, 5, 8, 22, 44]); +/// +/// fn quick_sort(v: &mut [T]) { +/// if v.len() > 1 { +/// let mid = partition(v); +/// let (lo, hi) = v.split_at_mut(mid); +/// rayon::join(|| quick_sort(lo), +/// || quick_sort(hi)); +/// } +/// } +/// +/// // Partition rearranges all items `<=` to the pivot +/// // item (arbitrary selected to be the last item in the slice) +/// // to the first half of the slice. It then returns the +/// // "dividing point" where the pivot is placed. +/// fn partition(v: &mut [T]) -> usize { +/// let pivot = v.len() - 1; +/// let mut i = 0; +/// for j in 0..pivot { +/// if v[j] <= v[pivot] { +/// v.swap(i, j); +/// i += 1; +/// } +/// } +/// v.swap(i, pivot); +/// i +/// } +/// ``` +/// +/// # Warning about blocking I/O +/// +/// The assumption is that the closures given to `join()` are +/// CPU-bound tasks that do not perform I/O or other blocking +/// operations. If you do perform I/O, and that I/O should block +/// (e.g., waiting for a network request), the overall performance may +/// be poor. Moreover, if you cause one closure to be blocked waiting +/// on another (for example, using a channel), that could lead to a +/// deadlock. +/// +/// # Panics +/// +/// No matter what happens, both closures will always be executed. If +/// a single closure panics, whether it be the first or second +/// closure, that panic will be propagated and hence `join()` will +/// panic with the same panic value. If both closures panic, `join()` +/// will panic with the panic value from the first closure. +pub fn join(oper_a: A, oper_b: B) -> (RA, RB) +where + A: FnOnce() -> RA + Send, + B: FnOnce() -> RB + Send, + RA: Send, + RB: Send, +{ + #[inline] + fn call(f: impl FnOnce() -> R) -> impl FnOnce(FnContext) -> R { + move |_| f() + } + + join_context(call(oper_a), call(oper_b)) +} + +/// Identical to `join`, except that the closures have a parameter +/// that provides context for the way the closure has been called, +/// especially indicating whether they're executing on a different +/// thread than where `join_context` was called. This will occur if +/// the second job is stolen by a different thread, or if +/// `join_context` was called from outside the thread pool to begin +/// with. +pub fn join_context(oper_a: A, oper_b: B) -> (RA, RB) +where + A: FnOnce(FnContext) -> RA + Send, + B: FnOnce(FnContext) -> RB + Send, + RA: Send, + RB: Send, +{ + #[inline] + fn call_a(f: impl FnOnce(FnContext) -> R, injected: bool) -> impl FnOnce() -> R { + move || f(FnContext::new(injected)) + } + + #[inline] + fn call_b(f: impl FnOnce(FnContext) -> R) -> impl FnOnce(bool) -> R { + move |migrated| f(FnContext::new(migrated)) + } + + registry::in_worker(|worker_thread, injected| unsafe { + // Create virtual wrapper for task b; this all has to be + // done here so that the stack frame can keep it all live + // long enough. + let job_b = StackJob::new(call_b(oper_b), SpinLatch::new(worker_thread)); + let job_b_ref = job_b.as_job_ref(); + let job_b_id = job_b_ref.id(); + worker_thread.push(job_b_ref); + + // Execute task a; hopefully b gets stolen in the meantime. + let status_a = unwind::halt_unwinding(call_a(oper_a, injected)); + let result_a = match status_a { + Ok(v) => v, + Err(err) => join_recover_from_panic(worker_thread, &job_b.latch, err), + }; + + // Now that task A has finished, try to pop job B from the + // local stack. It may already have been popped by job A; it + // may also have been stolen. There may also be some tasks + // pushed on top of it in the stack, and we will have to pop + // those off to get to it. + while !job_b.latch.probe() { + let Some(job) = worker_thread.take_local_job() else { + // Local deque is empty. Time to steal from other + // threads. + worker_thread.wait_until(&job_b.latch); + debug_assert!(job_b.latch.probe()); + break; + }; + if job_b_id == job.id() { + // Found it! Let's run it. + // + // Note that this could panic, but it's ok if we unwind here. + let result_b = job_b.run_inline(injected); + return (result_a, result_b); + } + worker_thread.execute(job); + } + + (result_a, job_b.into_result()) + }) +} + +/// If job A panics, we still cannot return until we are sure that job +/// B is complete. This is because it may contain references into the +/// enclosing stack frame(s). +#[cold] // cold path +unsafe fn join_recover_from_panic( + worker_thread: &WorkerThread, + job_b_latch: &SpinLatch<'_>, + err: Box, +) -> ! { + worker_thread.wait_until(job_b_latch); + unwind::resume_unwinding(err) +} diff --git a/anneal/v2/vendor/rayon-core/src/join/test.rs b/anneal/v2/vendor/rayon-core/src/join/test.rs new file mode 100644 index 0000000000..03f4ab4478 --- /dev/null +++ b/anneal/v2/vendor/rayon-core/src/join/test.rs @@ -0,0 +1,150 @@ +//! Tests for the join code. + +use super::*; +use crate::ThreadPoolBuilder; +use rand::distr::StandardUniform; +use rand::{Rng, SeedableRng}; +use rand_xorshift::XorShiftRng; + +fn quick_sort(v: &mut [T]) { + if v.len() <= 1 { + return; + } + + let mid = partition(v); + let (lo, hi) = v.split_at_mut(mid); + join(|| quick_sort(lo), || quick_sort(hi)); +} + +fn partition(v: &mut [T]) -> usize { + let pivot = v.len() - 1; + let mut i = 0; + for j in 0..pivot { + if v[j] <= v[pivot] { + v.swap(i, j); + i += 1; + } + } + v.swap(i, pivot); + i +} + +fn seeded_rng() -> XorShiftRng { + let mut seed = ::Seed::default(); + (0..).zip(seed.as_mut()).for_each(|(i, x)| *x = i); + XorShiftRng::from_seed(seed) +} + +#[test] +fn sort() { + let rng = seeded_rng(); + let mut data: Vec = rng.sample_iter(&StandardUniform).take(6 * 1024).collect(); + let mut sorted_data = data.clone(); + sorted_data.sort(); + quick_sort(&mut data); + assert_eq!(data, sorted_data); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn sort_in_pool() { + let rng = seeded_rng(); + let mut data: Vec = rng.sample_iter(&StandardUniform).take(12 * 1024).collect(); + + let pool = ThreadPoolBuilder::new().build().unwrap(); + let mut sorted_data = data.clone(); + sorted_data.sort(); + pool.install(|| quick_sort(&mut data)); + assert_eq!(data, sorted_data); +} + +#[test] +#[should_panic(expected = "Hello, world!")] +fn panic_propagate_a() { + join(|| panic!("Hello, world!"), || ()); +} + +#[test] +#[should_panic(expected = "Hello, world!")] +fn panic_propagate_b() { + join(|| (), || panic!("Hello, world!")); +} + +#[test] +#[should_panic(expected = "Hello, world!")] +fn panic_propagate_both() { + join(|| panic!("Hello, world!"), || panic!("Goodbye, world!")); +} + +#[test] +#[cfg_attr(not(panic = "unwind"), ignore)] +fn panic_b_still_executes() { + let mut x = false; + match unwind::halt_unwinding(|| join(|| panic!("Hello, world!"), || x = true)) { + Ok(_) => panic!("failed to propagate panic from closure A,"), + Err(_) => assert!(x, "closure b failed to execute"), + } +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn join_context_both() { + // If we're not in a pool, both should be marked stolen as they're injected. + let (a_migrated, b_migrated) = join_context(|a| a.migrated(), |b| b.migrated()); + assert!(a_migrated); + assert!(b_migrated); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn join_context_neither() { + // If we're already in a 1-thread pool, neither job should be stolen. + let pool = ThreadPoolBuilder::new().num_threads(1).build().unwrap(); + let (a_migrated, b_migrated) = + pool.install(|| join_context(|a| a.migrated(), |b| b.migrated())); + assert!(!a_migrated); + assert!(!b_migrated); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn join_context_second() { + use std::sync::Barrier; + + // If we're already in a 2-thread pool, the second job should be stolen. + let barrier = Barrier::new(2); + let pool = ThreadPoolBuilder::new().num_threads(2).build().unwrap(); + let (a_migrated, b_migrated) = pool.install(|| { + join_context( + |a| { + barrier.wait(); + a.migrated() + }, + |b| { + barrier.wait(); + b.migrated() + }, + ) + }); + assert!(!a_migrated); + assert!(b_migrated); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn join_counter_overflow() { + const MAX: u32 = 500_000; + + let mut i = 0; + let mut j = 0; + let pool = ThreadPoolBuilder::new().num_threads(2).build().unwrap(); + + // Hammer on join a bunch of times -- used to hit overflow debug-assertions + // in JEC on 32-bit targets: https://github.com/rayon-rs/rayon/issues/797 + for _ in 0..MAX { + pool.join(|| i += 1, || j += 1); + } + + assert_eq!(i, MAX); + assert_eq!(j, MAX); +} diff --git a/anneal/v2/vendor/rayon-core/src/latch.rs b/anneal/v2/vendor/rayon-core/src/latch.rs new file mode 100644 index 0000000000..f4c96a3425 --- /dev/null +++ b/anneal/v2/vendor/rayon-core/src/latch.rs @@ -0,0 +1,457 @@ +use std::marker::PhantomData; +use std::ops::Deref; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +use crate::registry::{Registry, WorkerThread}; +use crate::sync::{Condvar, Mutex}; + +/// We define various kinds of latches, which are all a primitive signaling +/// mechanism. A latch starts as false. Eventually someone calls `set()` and +/// it becomes true. You can test if it has been set by calling `probe()`. +/// +/// Some kinds of latches, but not all, support a `wait()` operation +/// that will wait until the latch is set, blocking efficiently. That +/// is not part of the trait since it is not possibly to do with all +/// latches. +/// +/// The intention is that `set()` is called once, but `probe()` may be +/// called any number of times. Once `probe()` returns true, the memory +/// effects that occurred before `set()` become visible. +/// +/// It'd probably be better to refactor the API into two paired types, +/// but that's a bit of work, and this is not a public API. +/// +/// ## Memory ordering +/// +/// Latches need to guarantee two things: +/// +/// - Once `probe()` returns true, all memory effects from the `set()` +/// are visible (in other words, the set should synchronize-with +/// the probe). +/// - Once `set()` occurs, the next `probe()` *will* observe it. This +/// typically requires a seq-cst ordering. See [the "tickle-then-get-sleepy" scenario in the sleep +/// README](/src/sleep/README.md#tickle-then-get-sleepy) for details. +pub(super) trait Latch { + /// Set the latch, signalling others. + /// + /// # WARNING + /// + /// Setting a latch triggers other threads to wake up and (in some + /// cases) complete. This may, in turn, cause memory to be + /// deallocated and so forth. One must be very careful about this, + /// and it's typically better to read all the fields you will need + /// to access *before* a latch is set! + /// + /// This function operates on `*const Self` instead of `&self` to allow it + /// to become dangling during this call. The caller must ensure that the + /// pointer is valid upon entry, and not invalidated during the call by any + /// actions other than `set` itself. + unsafe fn set(this: *const Self); +} + +pub(super) trait AsCoreLatch { + fn as_core_latch(&self) -> &CoreLatch; +} + +/// Latch is not set, owning thread is awake +const UNSET: usize = 0; + +/// Latch is not set, owning thread is going to sleep on this latch +/// (but has not yet fallen asleep). +const SLEEPY: usize = 1; + +/// Latch is not set, owning thread is asleep on this latch and +/// must be awoken. +const SLEEPING: usize = 2; + +/// Latch is set. +const SET: usize = 3; + +/// Spin latches are the simplest, most efficient kind, but they do +/// not support a `wait()` operation. They just have a boolean flag +/// that becomes true when `set()` is called. +#[derive(Debug)] +pub(super) struct CoreLatch { + state: AtomicUsize, +} + +impl CoreLatch { + #[inline] + fn new() -> Self { + Self { + state: AtomicUsize::new(0), + } + } + + /// Invoked by owning thread as it prepares to sleep. Returns true + /// if the owning thread may proceed to fall asleep, false if the + /// latch was set in the meantime. + #[inline] + pub(super) fn get_sleepy(&self) -> bool { + self.state + .compare_exchange(UNSET, SLEEPY, Ordering::SeqCst, Ordering::Relaxed) + .is_ok() + } + + /// Invoked by owning thread as it falls asleep sleep. Returns + /// true if the owning thread should block, or false if the latch + /// was set in the meantime. + #[inline] + pub(super) fn fall_asleep(&self) -> bool { + self.state + .compare_exchange(SLEEPY, SLEEPING, Ordering::SeqCst, Ordering::Relaxed) + .is_ok() + } + + /// Invoked by owning thread as it falls asleep sleep. Returns + /// true if the owning thread should block, or false if the latch + /// was set in the meantime. + #[inline] + pub(super) fn wake_up(&self) { + if !self.probe() { + let _ = + self.state + .compare_exchange(SLEEPING, UNSET, Ordering::SeqCst, Ordering::Relaxed); + } + } + + /// Set the latch. If this returns true, the owning thread was sleeping + /// and must be awoken. + /// + /// This is private because, typically, setting a latch involves + /// doing some wakeups; those are encapsulated in the surrounding + /// latch code. + #[inline] + unsafe fn set(this: *const Self) -> bool { + let old_state = (*this).state.swap(SET, Ordering::AcqRel); + old_state == SLEEPING + } + + /// Test if this latch has been set. + #[inline] + pub(super) fn probe(&self) -> bool { + self.state.load(Ordering::Acquire) == SET + } +} + +impl AsCoreLatch for CoreLatch { + #[inline] + fn as_core_latch(&self) -> &CoreLatch { + self + } +} + +/// Spin latches are the simplest, most efficient kind, but they do +/// not support a `wait()` operation. They just have a boolean flag +/// that becomes true when `set()` is called. +pub(super) struct SpinLatch<'r> { + core_latch: CoreLatch, + registry: &'r Arc, + target_worker_index: usize, + cross: bool, +} + +impl<'r> SpinLatch<'r> { + /// Creates a new spin latch that is owned by `thread`. This means + /// that `thread` is the only thread that should be blocking on + /// this latch -- it also means that when the latch is set, we + /// will wake `thread` if it is sleeping. + #[inline] + pub(super) fn new(thread: &'r WorkerThread) -> SpinLatch<'r> { + SpinLatch { + core_latch: CoreLatch::new(), + registry: thread.registry(), + target_worker_index: thread.index(), + cross: false, + } + } + + /// Creates a new spin latch for cross-thread-pool blocking. Notably, we + /// need to make sure the registry is kept alive after setting, so we can + /// safely call the notification. + #[inline] + pub(super) fn cross(thread: &'r WorkerThread) -> SpinLatch<'r> { + SpinLatch { + cross: true, + ..SpinLatch::new(thread) + } + } + + #[inline] + pub(super) fn probe(&self) -> bool { + self.core_latch.probe() + } +} + +impl AsCoreLatch for SpinLatch<'_> { + #[inline] + fn as_core_latch(&self) -> &CoreLatch { + &self.core_latch + } +} + +impl Latch for SpinLatch<'_> { + #[inline] + unsafe fn set(this: *const Self) { + let registry: &Registry = if (*this).cross { + // Ensure the registry stays alive while we notify it. + // Otherwise, it would be possible that we set the spin + // latch and the other thread sees it and exits, causing + // the registry to be deallocated, all before we get a + // chance to invoke `registry.notify_worker_latch_is_set`. + &Arc::clone((*this).registry) + } else { + // If this is not a "cross-registry" spin-latch, then the + // thread which is performing `set` is itself ensuring + // that the registry stays alive. However, that doesn't + // include this *particular* `Arc` handle if the waiting + // thread then exits, so we must completely dereference it. + (*this).registry + }; + let target_worker_index = (*this).target_worker_index; + + // NOTE: Once we `set`, the target may proceed and invalidate `this`! + if CoreLatch::set(&(*this).core_latch) { + // Subtle: at this point, we can no longer read from + // `self`, because the thread owning this spin latch may + // have awoken and deallocated the latch. Therefore, we + // only use fields whose values we already read. + registry.notify_worker_latch_is_set(target_worker_index); + } + } +} + +/// A Latch starts as false and eventually becomes true. You can block +/// until it becomes true. +#[derive(Debug)] +pub(super) struct LockLatch { + m: Mutex, + v: Condvar, +} + +impl LockLatch { + #[inline] + pub(super) const fn new() -> LockLatch { + LockLatch { + m: Mutex::new(false), + v: Condvar::new(), + } + } + + /// Block until latch is set, then resets this lock latch so it can be reused again. + pub(super) fn wait_and_reset(&self) { + let mut guard = self.m.lock().unwrap(); + while !*guard { + guard = self.v.wait(guard).unwrap(); + } + *guard = false; + } + + /// Block until latch is set. + pub(super) fn wait(&self) { + let mut guard = self.m.lock().unwrap(); + while !*guard { + guard = self.v.wait(guard).unwrap(); + } + } +} + +impl Latch for LockLatch { + #[inline] + unsafe fn set(this: *const Self) { + let mut guard = (*this).m.lock().unwrap(); + *guard = true; + (*this).v.notify_all(); + } +} + +/// Once latches are used to implement one-time blocking, primarily +/// for the termination flag of the threads in the pool. +/// +/// Note: like a `SpinLatch`, once-latches are always associated with +/// some registry that is probing them, which must be tickled when +/// they are set. *Unlike* a `SpinLatch`, they don't themselves hold a +/// reference to that registry. This is because in some cases the +/// registry owns the once-latch, and that would create a cycle. So a +/// `OnceLatch` must be given a reference to its owning registry when +/// it is set. For this reason, it does not implement the `Latch` +/// trait (but it doesn't have to, as it is not used in those generic +/// contexts). +#[derive(Debug)] +pub(super) struct OnceLatch { + core_latch: CoreLatch, +} + +impl OnceLatch { + #[inline] + pub(super) fn new() -> OnceLatch { + Self { + core_latch: CoreLatch::new(), + } + } + + /// Set the latch, then tickle the specific worker thread, + /// which should be the one that owns this latch. + #[inline] + pub(super) unsafe fn set_and_tickle_one( + this: *const Self, + registry: &Registry, + target_worker_index: usize, + ) { + if CoreLatch::set(&(*this).core_latch) { + registry.notify_worker_latch_is_set(target_worker_index); + } + } +} + +impl AsCoreLatch for OnceLatch { + #[inline] + fn as_core_latch(&self) -> &CoreLatch { + &self.core_latch + } +} + +/// Counting latches are used to implement scopes. They track a +/// counter. Unlike other latches, calling `set()` does not +/// necessarily make the latch be considered `set()`; instead, it just +/// decrements the counter. The latch is only "set" (in the sense that +/// `probe()` returns true) once the counter reaches zero. +#[derive(Debug)] +pub(super) struct CountLatch { + counter: AtomicUsize, + kind: CountLatchKind, +} + +enum CountLatchKind { + /// A latch for scopes created on a rayon thread which will participate in work + /// stealing while it waits for completion. This thread is not necessarily part + /// of the same registry as the scope itself! + Stealing { + latch: CoreLatch, + /// If a worker thread in registry A calls `in_place_scope` on a ThreadPool + /// with registry B, when a job completes in a thread of registry B, we may + /// need to call `notify_worker_latch_is_set()` to wake the thread in registry A. + /// That means we need a reference to registry A (since at that point we will + /// only have a reference to registry B), so we stash it here. + registry: Arc, + /// The index of the worker to wake in `registry` + worker_index: usize, + }, + + /// A latch for scopes created on a non-rayon thread which will block to wait. + Blocking { latch: LockLatch }, +} + +impl std::fmt::Debug for CountLatchKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + CountLatchKind::Stealing { latch, .. } => { + f.debug_tuple("Stealing").field(latch).finish() + } + CountLatchKind::Blocking { latch, .. } => { + f.debug_tuple("Blocking").field(latch).finish() + } + } + } +} + +impl CountLatch { + pub(super) fn new(owner: Option<&WorkerThread>) -> Self { + Self::with_count(1, owner) + } + + pub(super) fn with_count(count: usize, owner: Option<&WorkerThread>) -> Self { + Self { + counter: AtomicUsize::new(count), + kind: match owner { + Some(owner) => CountLatchKind::Stealing { + latch: CoreLatch::new(), + registry: Arc::clone(owner.registry()), + worker_index: owner.index(), + }, + None => CountLatchKind::Blocking { + latch: LockLatch::new(), + }, + }, + } + } + + #[inline] + pub(super) fn increment(&self) { + let old_counter = self.counter.fetch_add(1, Ordering::Relaxed); + debug_assert!(old_counter != 0); + } + + pub(super) fn wait(&self, owner: Option<&WorkerThread>) { + match &self.kind { + CountLatchKind::Stealing { + latch, + registry, + worker_index, + } => unsafe { + let owner = owner.expect("owner thread"); + debug_assert_eq!(registry.id(), owner.registry().id()); + debug_assert_eq!(*worker_index, owner.index()); + owner.wait_until(latch); + }, + CountLatchKind::Blocking { latch } => latch.wait(), + } + } +} + +impl Latch for CountLatch { + #[inline] + unsafe fn set(this: *const Self) { + if (*this).counter.fetch_sub(1, Ordering::SeqCst) == 1 { + // NOTE: Once we call `set` on the internal `latch`, + // the target may proceed and invalidate `this`! + match (*this).kind { + CountLatchKind::Stealing { + ref latch, + ref registry, + worker_index, + } => { + let registry = Arc::clone(registry); + if CoreLatch::set(latch) { + registry.notify_worker_latch_is_set(worker_index); + } + } + CountLatchKind::Blocking { ref latch } => LockLatch::set(latch), + } + } + } +} + +/// `&L` without any implication of `dereferenceable` for `Latch::set` +pub(super) struct LatchRef<'a, L> { + inner: *const L, + marker: PhantomData<&'a L>, +} + +impl LatchRef<'_, L> { + pub(super) fn new(inner: &L) -> LatchRef<'_, L> { + LatchRef { + inner, + marker: PhantomData, + } + } +} + +unsafe impl Sync for LatchRef<'_, L> {} + +impl Deref for LatchRef<'_, L> { + type Target = L; + + fn deref(&self) -> &L { + // SAFETY: if we have &self, the inner latch is still alive + unsafe { &*self.inner } + } +} + +impl Latch for LatchRef<'_, L> { + #[inline] + unsafe fn set(this: *const Self) { + L::set((*this).inner); + } +} diff --git a/anneal/v2/vendor/rayon-core/src/lib.rs b/anneal/v2/vendor/rayon-core/src/lib.rs new file mode 100644 index 0000000000..3745076324 --- /dev/null +++ b/anneal/v2/vendor/rayon-core/src/lib.rs @@ -0,0 +1,864 @@ +//! Rayon-core houses the core stable APIs of Rayon. +//! +//! These APIs have been mirrored in the Rayon crate and it is recommended to use these from there. +//! +//! [`join()`] is used to take two closures and potentially run them in parallel. +//! - It will run in parallel if task B gets stolen before task A can finish. +//! - It will run sequentially if task A finishes before task B is stolen and can continue on task B. +//! +//! [`scope()`] creates a scope in which you can run any number of parallel tasks. +//! These tasks can spawn nested tasks and scopes, but given the nature of work stealing, the order of execution can not be guaranteed. +//! The scope will exist until all tasks spawned within the scope have been completed. +//! +//! [`spawn()`] add a task into the 'static' or 'global' scope, or a local scope created by the [`scope()`] function. +//! +//! [`ThreadPool`] can be used to create your own thread pools (using [`ThreadPoolBuilder`]) or to customize the global one. +//! Tasks spawned within the pool (using [`install()`][tpinstall], [`join()`][tpjoin], etc.) will be added to a deque, +//! where it becomes available for work stealing from other threads in the local thread pool. +//! +//! [tpinstall]: ThreadPool::install() +//! [tpjoin]: ThreadPool::join() +//! +//! # Global fallback when threading is unsupported +//! +//! Rayon uses `std` APIs for threading, but some targets have incomplete implementations that +//! always return `Unsupported` errors. The WebAssembly `wasm32-unknown-unknown` and `wasm32-wasi` +//! targets are notable examples of this. Rather than panicking on the unsupported error when +//! creating the implicit global thread pool, Rayon configures a fallback mode instead. +//! +//! This fallback mode mostly functions as if it were using a single-threaded "pool", like setting +//! `RAYON_NUM_THREADS=1`. For example, `join` will execute its two closures sequentially, since +//! there is no other thread to share the work. However, since the pool is not running independent +//! of the main thread, non-blocking calls like `spawn` may not execute at all, unless a lower- +//! priority call like `broadcast` gives them an opening. The fallback mode does not try to emulate +//! anything like thread preemption or `async` task switching, but `yield_now` or `yield_local` +//! can also volunteer execution time. +//! +//! Explicit `ThreadPoolBuilder` methods always report their error without any fallback. +//! +//! # Restricting multiple versions +//! +//! In order to ensure proper coordination between thread pools, and especially +//! to make sure there's only one global thread pool, `rayon-core` is actively +//! restricted from building multiple versions of itself into a single target. +//! You may see a build error like this in violation: +//! +//! ```text +//! error: native library `rayon-core` is being linked to by more +//! than one package, and can only be linked to by one package +//! ``` +//! +//! While we strive to keep `rayon-core` semver-compatible, it's still +//! possible to arrive at this situation if different crates have overly +//! restrictive tilde or inequality requirements for `rayon-core`. The +//! conflicting requirements will need to be resolved before the build will +//! succeed. + +#![deny(missing_debug_implementations)] +#![deny(missing_docs)] +#![deny(unreachable_pub)] +#![warn(rust_2018_idioms)] + +use std::any::Any; +use std::env; +use std::error::Error; +use std::fmt; +use std::io; +use std::marker::PhantomData; +use std::str::FromStr; +use std::thread; + +#[macro_use] +mod private; + +mod broadcast; +mod job; +mod join; +mod latch; +mod registry; +mod scope; +mod sleep; +mod spawn; +mod thread_pool; +mod unwind; + +mod compile_fail; +mod test; + +pub use self::broadcast::{broadcast, spawn_broadcast, BroadcastContext}; +pub use self::join::{join, join_context}; +pub use self::registry::ThreadBuilder; +pub use self::scope::{in_place_scope, scope, Scope}; +pub use self::scope::{in_place_scope_fifo, scope_fifo, ScopeFifo}; +pub use self::spawn::{spawn, spawn_fifo}; +pub use self::thread_pool::current_thread_has_pending_tasks; +pub use self::thread_pool::current_thread_index; +pub use self::thread_pool::ThreadPool; +pub use self::thread_pool::{yield_local, yield_now, Yield}; + +#[cfg(not(feature = "web_spin_lock"))] +use std::sync; + +#[cfg(feature = "web_spin_lock")] +use wasm_sync as sync; + +use self::registry::{CustomSpawn, DefaultSpawn, ThreadSpawn}; + +/// Returns the maximum number of threads that Rayon supports in a single thread pool. +/// +/// If a higher thread count is requested by calling `ThreadPoolBuilder::num_threads` or by setting +/// the `RAYON_NUM_THREADS` environment variable, then it will be reduced to this maximum. +/// +/// The value may vary between different targets, and is subject to change in new Rayon versions. +pub fn max_num_threads() -> usize { + // We are limited by the bits available in the sleep counter's `AtomicUsize`. + crate::sleep::THREADS_MAX +} + +/// Returns the number of threads in the current registry. If this +/// code is executing within a Rayon thread pool, then this will be +/// the number of threads for the thread pool of the current +/// thread. Otherwise, it will be the number of threads for the global +/// thread pool. +/// +/// This can be useful when trying to judge how many times to split +/// parallel work (the parallel iterator traits use this value +/// internally for this purpose). +/// +/// # Future compatibility note +/// +/// Note that unless this thread pool was created with a +/// builder that specifies the number of threads, then this +/// number may vary over time in future versions (see [the +/// `num_threads()` method for details][snt]). +/// +/// [snt]: ThreadPoolBuilder::num_threads +pub fn current_num_threads() -> usize { + crate::registry::Registry::current_num_threads() +} + +/// Error when initializing a thread pool. +#[derive(Debug)] +pub struct ThreadPoolBuildError { + kind: ErrorKind, +} + +#[derive(Debug)] +enum ErrorKind { + GlobalPoolAlreadyInitialized, + CurrentThreadAlreadyInPool, + IOError(io::Error), +} + +/// Used to create a new [`ThreadPool`] or to configure the global rayon thread pool. +/// ## Creating a ThreadPool +/// The following creates a thread pool with 22 threads. +/// +/// ```ignore-wasm +/// # use rayon_core as rayon; +/// let pool = rayon::ThreadPoolBuilder::new().num_threads(22).build().unwrap(); +/// ``` +/// +/// To instead configure the global thread pool, use [`build_global()`]: +/// +/// ```ignore-wasm +/// # use rayon_core as rayon; +/// rayon::ThreadPoolBuilder::new().num_threads(22).build_global().unwrap(); +/// ``` +/// +/// [`build_global()`]: Self::build_global() +pub struct ThreadPoolBuilder { + /// The number of threads in the rayon thread pool. + /// If zero will use the RAYON_NUM_THREADS environment variable. + /// If RAYON_NUM_THREADS is invalid or zero will use the default. + num_threads: usize, + + /// The thread we're building *from* will also be part of the pool. + use_current_thread: bool, + + /// Custom closure, if any, to handle a panic that we cannot propagate + /// anywhere else. + panic_handler: Option>, + + /// Closure to compute the name of a thread. + get_thread_name: Option String>>, + + /// The stack size for the created worker threads + stack_size: Option, + + /// Closure invoked on worker-thread start. + start_handler: Option>, + + /// Closure invoked on worker-thread exit. + exit_handler: Option>, + + /// Closure invoked to spawn threads. + spawn_handler: S, + + /// If false, worker threads will execute spawned jobs in a + /// "depth-first" fashion. If true, they will do a "breadth-first" + /// fashion. Depth-first is the default. + breadth_first: bool, +} + +/// Contains the rayon thread pool configuration. Use [`ThreadPoolBuilder`] instead. +#[deprecated(note = "Use `ThreadPoolBuilder`")] +#[derive(Default)] +pub struct Configuration { + builder: ThreadPoolBuilder, +} + +/// The type for a panic-handling closure. Note that this same closure +/// may be invoked multiple times in parallel. +type PanicHandler = dyn Fn(Box) + Send + Sync; + +/// The type for a closure that gets invoked when a thread starts. The +/// closure is passed the index of the thread on which it is invoked. +/// Note that this same closure may be invoked multiple times in parallel. +type StartHandler = dyn Fn(usize) + Send + Sync; + +/// The type for a closure that gets invoked when a thread exits. The +/// closure is passed the index of the thread on which it is invoked. +/// Note that this same closure may be invoked multiple times in parallel. +type ExitHandler = dyn Fn(usize) + Send + Sync; + +// NB: We can't `#[derive(Default)]` because `S` is left ambiguous. +impl Default for ThreadPoolBuilder { + fn default() -> Self { + ThreadPoolBuilder { + num_threads: 0, + use_current_thread: false, + panic_handler: None, + get_thread_name: None, + stack_size: None, + start_handler: None, + exit_handler: None, + spawn_handler: DefaultSpawn, + breadth_first: false, + } + } +} + +impl ThreadPoolBuilder { + /// Creates and returns a valid rayon thread pool builder, but does not initialize it. + pub fn new() -> Self { + Self::default() + } +} + +/// Note: the `S: ThreadSpawn` constraint is an internal implementation detail for the +/// default spawn and those set by [`spawn_handler`](#method.spawn_handler). +impl ThreadPoolBuilder +where + S: ThreadSpawn, +{ + /// Creates a new `ThreadPool` initialized using this configuration. + pub fn build(self) -> Result { + ThreadPool::build(self) + } + + /// Initializes the global thread pool. This initialization is + /// **optional**. If you do not call this function, the thread pool + /// will be automatically initialized with the default + /// configuration. Calling `build_global` is not recommended, except + /// in two scenarios: + /// + /// - You wish to change the default configuration. + /// - You are running a benchmark, in which case initializing may + /// yield slightly more consistent results, since the worker threads + /// will already be ready to go even in the first iteration. But + /// this cost is minimal. + /// + /// Initialization of the global thread pool happens exactly + /// once. Once started, the configuration cannot be + /// changed. Therefore, if you call `build_global` a second time, it + /// will return an error. An `Ok` result indicates that this + /// is the first initialization of the thread pool. + pub fn build_global(self) -> Result<(), ThreadPoolBuildError> { + let registry = registry::init_global_registry(self)?; + registry.wait_until_primed(); + Ok(()) + } +} + +impl ThreadPoolBuilder { + /// Creates a scoped `ThreadPool` initialized using this configuration. + /// + /// This is a convenience function for building a pool using [`std::thread::scope`] + /// to spawn threads in a [`spawn_handler`]. + /// The threads in this pool will start by calling `wrapper`, which should + /// do initialization and continue by calling `ThreadBuilder::run()`. + /// + /// [`spawn_handler`]: Self::spawn_handler() + /// + /// # Examples + /// + /// A scoped pool may be useful in combination with scoped thread-local variables. + /// + /// ```ignore-wasm + /// # use rayon_core as rayon; + /// + /// scoped_tls::scoped_thread_local!(static POOL_DATA: Vec); + /// + /// fn main() -> Result<(), rayon::ThreadPoolBuildError> { + /// let pool_data = vec![1, 2, 3]; + /// + /// // We haven't assigned any TLS data yet. + /// assert!(!POOL_DATA.is_set()); + /// + /// rayon::ThreadPoolBuilder::new() + /// .build_scoped( + /// // Borrow `pool_data` in TLS for each thread. + /// |thread| POOL_DATA.set(&pool_data, || thread.run()), + /// // Do some work that needs the TLS data. + /// |pool| pool.install(|| assert!(POOL_DATA.is_set())), + /// )?; + /// + /// // Once we've returned, `pool_data` is no longer borrowed. + /// drop(pool_data); + /// Ok(()) + /// } + /// ``` + pub fn build_scoped(self, wrapper: W, with_pool: F) -> Result + where + W: Fn(ThreadBuilder) + Sync, // expected to call `run()` + F: FnOnce(&ThreadPool) -> R, + { + std::thread::scope(|scope| { + let pool = self + .spawn_handler(|thread| { + let mut builder = std::thread::Builder::new(); + if let Some(name) = thread.name() { + builder = builder.name(name.to_string()); + } + if let Some(size) = thread.stack_size() { + builder = builder.stack_size(size); + } + builder.spawn_scoped(scope, || wrapper(thread))?; + Ok(()) + }) + .build()?; + Ok(with_pool(&pool)) + }) + } +} + +impl ThreadPoolBuilder { + /// Sets a custom function for spawning threads. + /// + /// Note that the threads will not exit until after the pool is dropped. It + /// is up to the caller to wait for thread termination if that is important + /// for any invariants. For instance, threads created in [`std::thread::scope`] + /// will be joined before that scope returns, and this will block indefinitely + /// if the pool is leaked. Furthermore, the global thread pool doesn't terminate + /// until the entire process exits! + /// + /// # Examples + /// + /// A minimal spawn handler just needs to call `run()` from an independent thread. + /// + /// ```ignore-wasm + /// # use rayon_core as rayon; + /// fn main() -> Result<(), rayon::ThreadPoolBuildError> { + /// let pool = rayon::ThreadPoolBuilder::new() + /// .spawn_handler(|thread| { + /// std::thread::spawn(|| thread.run()); + /// Ok(()) + /// }) + /// .build()?; + /// + /// pool.install(|| println!("Hello from my custom thread!")); + /// Ok(()) + /// } + /// ``` + /// + /// The default spawn handler sets the name and stack size if given, and propagates + /// any errors from the thread builder. + /// + /// ```ignore-wasm + /// # use rayon_core as rayon; + /// fn main() -> Result<(), rayon::ThreadPoolBuildError> { + /// let pool = rayon::ThreadPoolBuilder::new() + /// .spawn_handler(|thread| { + /// let mut b = std::thread::Builder::new(); + /// if let Some(name) = thread.name() { + /// b = b.name(name.to_owned()); + /// } + /// if let Some(stack_size) = thread.stack_size() { + /// b = b.stack_size(stack_size); + /// } + /// b.spawn(|| thread.run())?; + /// Ok(()) + /// }) + /// .build()?; + /// + /// pool.install(|| println!("Hello from my fully custom thread!")); + /// Ok(()) + /// } + /// ``` + /// + /// This can also be used for a pool of scoped threads like [`crossbeam::scope`], + /// or [`std::thread::scope`] introduced in Rust 1.63, which is encapsulated in + /// [`build_scoped`]. + /// + /// [`crossbeam::scope`]: https://docs.rs/crossbeam/0.8/crossbeam/fn.scope.html + /// [`build_scoped`]: Self::build_scoped() + /// + /// ```ignore-wasm + /// # use rayon_core as rayon; + /// fn main() -> Result<(), rayon::ThreadPoolBuildError> { + /// std::thread::scope(|scope| { + /// let pool = rayon::ThreadPoolBuilder::new() + /// .spawn_handler(|thread| { + /// let mut builder = std::thread::Builder::new(); + /// if let Some(name) = thread.name() { + /// builder = builder.name(name.to_string()); + /// } + /// if let Some(size) = thread.stack_size() { + /// builder = builder.stack_size(size); + /// } + /// builder.spawn_scoped(scope, || { + /// // Add any scoped initialization here, then run! + /// thread.run() + /// })?; + /// Ok(()) + /// }) + /// .build()?; + /// + /// pool.install(|| println!("Hello from my custom scoped thread!")); + /// Ok(()) + /// }) + /// } + /// ``` + pub fn spawn_handler(self, spawn: F) -> ThreadPoolBuilder> + where + F: FnMut(ThreadBuilder) -> io::Result<()>, + { + ThreadPoolBuilder { + spawn_handler: CustomSpawn::new(spawn), + // ..self + num_threads: self.num_threads, + use_current_thread: self.use_current_thread, + panic_handler: self.panic_handler, + get_thread_name: self.get_thread_name, + stack_size: self.stack_size, + start_handler: self.start_handler, + exit_handler: self.exit_handler, + breadth_first: self.breadth_first, + } + } + + /// Returns a reference to the current spawn handler. + fn get_spawn_handler(&mut self) -> &mut S { + &mut self.spawn_handler + } + + /// Get the number of threads that will be used for the thread + /// pool. See `num_threads()` for more information. + fn get_num_threads(&self) -> usize { + if self.num_threads > 0 { + self.num_threads + } else { + let default = || { + thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1) + }; + + match env::var("RAYON_NUM_THREADS") + .ok() + .and_then(|s| usize::from_str(&s).ok()) + { + Some(x @ 1..) => return x, + Some(0) => return default(), + _ => {} + } + + // Support for deprecated `RAYON_RS_NUM_CPUS`. + match env::var("RAYON_RS_NUM_CPUS") + .ok() + .and_then(|s| usize::from_str(&s).ok()) + { + Some(x @ 1..) => x, + _ => default(), + } + } + } + + /// Get the thread name for the thread with the given index. + fn get_thread_name(&mut self, index: usize) -> Option { + let f = self.get_thread_name.as_mut()?; + Some(f(index)) + } + + /// Sets a closure which takes a thread index and returns + /// the thread's name. + pub fn thread_name(mut self, closure: F) -> Self + where + F: FnMut(usize) -> String + 'static, + { + self.get_thread_name = Some(Box::new(closure)); + self + } + + /// Sets the number of threads to be used in the rayon thread pool. + /// + /// If you specify a non-zero number of threads using this + /// function, then the resulting thread pools are guaranteed to + /// start at most this number of threads. + /// + /// If `num_threads` is 0, or you do not call this function, then + /// the Rayon runtime will select the number of threads + /// automatically. At present, this is based on the + /// `RAYON_NUM_THREADS` environment variable (if set), + /// or the number of logical CPUs (otherwise). + /// In the future, however, the default behavior may + /// change to dynamically add or remove threads as needed. + /// + /// **Future compatibility warning:** Given the default behavior + /// may change in the future, if you wish to rely on a fixed + /// number of threads, you should use this function to specify + /// that number. To reproduce the current default behavior, you + /// may wish to use [`std::thread::available_parallelism`] + /// to query the number of CPUs dynamically. + /// + /// **Old environment variable:** `RAYON_NUM_THREADS` is a one-to-one + /// replacement of the now deprecated `RAYON_RS_NUM_CPUS` environment + /// variable. If both variables are specified, `RAYON_NUM_THREADS` will + /// be preferred. + pub fn num_threads(mut self, num_threads: usize) -> Self { + self.num_threads = num_threads; + self + } + + /// Use the current thread as one of the threads in the pool. + /// + /// The current thread is guaranteed to be at index 0, and since the thread is not managed by + /// rayon, the spawn and exit handlers do not run for that thread. + /// + /// Note that the current thread won't run the main work-stealing loop, so jobs spawned into + /// the thread pool will generally not be picked up automatically by this thread unless you + /// yield to rayon in some way, like via [`yield_now()`], [`yield_local()`], or [`scope()`]. + /// + /// # Local thread pools + /// + /// Using this in a local thread pool means the registry will be leaked. In future versions + /// there might be a way of cleaning up the current-thread state. + pub fn use_current_thread(mut self) -> Self { + self.use_current_thread = true; + self + } + + /// Returns a copy of the current panic handler. + fn take_panic_handler(&mut self) -> Option> { + self.panic_handler.take() + } + + /// Normally, whenever Rayon catches a panic, it tries to + /// propagate it to someplace sensible, to try and reflect the + /// semantics of sequential execution. But in some cases, + /// particularly with the `spawn()` APIs, there is no + /// obvious place where we should propagate the panic to. + /// In that case, this panic handler is invoked. + /// + /// If no panic handler is set, the default is to abort the + /// process, under the principle that panics should not go + /// unobserved. + /// + /// If the panic handler itself panics, this will abort the + /// process. To prevent this, wrap the body of your panic handler + /// in a call to `std::panic::catch_unwind()`. + pub fn panic_handler(mut self, panic_handler: H) -> Self + where + H: Fn(Box) + Send + Sync + 'static, + { + self.panic_handler = Some(Box::new(panic_handler)); + self + } + + /// Get the stack size of the worker threads + fn get_stack_size(&self) -> Option { + self.stack_size + } + + /// Sets the stack size of the worker threads + pub fn stack_size(mut self, stack_size: usize) -> Self { + self.stack_size = Some(stack_size); + self + } + + /// **(DEPRECATED)** Suggest to worker threads that they execute + /// spawned jobs in a "breadth-first" fashion. + /// + /// Typically, when a worker thread is idle or blocked, it will + /// attempt to execute the job from the *top* of its local deque of + /// work (i.e., the job most recently spawned). If this flag is set + /// to true, however, workers will prefer to execute in a + /// *breadth-first* fashion -- that is, they will search for jobs at + /// the *bottom* of their local deque. (At present, workers *always* + /// steal from the bottom of other workers' deques, regardless of + /// the setting of this flag.) + /// + /// If you think of the tasks as a tree, where a parent task + /// spawns its children in the tree, then this flag loosely + /// corresponds to doing a breadth-first traversal of the tree, + /// whereas the default would be to do a depth-first traversal. + /// + /// **Note that this is an "execution hint".** Rayon's task + /// execution is highly dynamic and the precise order in which + /// independent tasks are executed is not intended to be + /// guaranteed. + /// + /// This `breadth_first()` method is now deprecated per [RFC #1], + /// and in the future its effect may be removed. Consider using + /// [`scope_fifo()`] for a similar effect. + /// + /// [RFC #1]: https://github.com/rayon-rs/rfcs/blob/main/accepted/rfc0001-scope-scheduling.md + #[deprecated(note = "use `scope_fifo` and `spawn_fifo` for similar effect")] + pub fn breadth_first(mut self) -> Self { + self.breadth_first = true; + self + } + + fn get_breadth_first(&self) -> bool { + self.breadth_first + } + + /// Takes the current thread start callback, leaving `None`. + fn take_start_handler(&mut self) -> Option> { + self.start_handler.take() + } + + /// Sets a callback to be invoked on thread start. + /// + /// The closure is passed the index of the thread on which it is invoked. + /// Note that this same closure may be invoked multiple times in parallel. + /// If this closure panics, the panic will be passed to the panic handler. + /// If that handler returns, then startup will continue normally. + pub fn start_handler(mut self, start_handler: H) -> Self + where + H: Fn(usize) + Send + Sync + 'static, + { + self.start_handler = Some(Box::new(start_handler)); + self + } + + /// Returns a current thread exit callback, leaving `None`. + fn take_exit_handler(&mut self) -> Option> { + self.exit_handler.take() + } + + /// Sets a callback to be invoked on thread exit. + /// + /// The closure is passed the index of the thread on which it is invoked. + /// Note that this same closure may be invoked multiple times in parallel. + /// If this closure panics, the panic will be passed to the panic handler. + /// If that handler returns, then the thread will exit normally. + pub fn exit_handler(mut self, exit_handler: H) -> Self + where + H: Fn(usize) + Send + Sync + 'static, + { + self.exit_handler = Some(Box::new(exit_handler)); + self + } +} + +#[allow(deprecated)] +impl Configuration { + /// Creates and return a valid rayon thread pool configuration, but does not initialize it. + pub fn new() -> Configuration { + Configuration { + builder: ThreadPoolBuilder::new(), + } + } + + /// Deprecated in favor of `ThreadPoolBuilder::build`. + pub fn build(self) -> Result> { + self.builder.build().map_err(Box::from) + } + + /// Deprecated in favor of `ThreadPoolBuilder::thread_name`. + pub fn thread_name(mut self, closure: F) -> Self + where + F: FnMut(usize) -> String + 'static, + { + self.builder = self.builder.thread_name(closure); + self + } + + /// Deprecated in favor of `ThreadPoolBuilder::num_threads`. + pub fn num_threads(mut self, num_threads: usize) -> Configuration { + self.builder = self.builder.num_threads(num_threads); + self + } + + /// Deprecated in favor of `ThreadPoolBuilder::panic_handler`. + pub fn panic_handler(mut self, panic_handler: H) -> Configuration + where + H: Fn(Box) + Send + Sync + 'static, + { + self.builder = self.builder.panic_handler(panic_handler); + self + } + + /// Deprecated in favor of `ThreadPoolBuilder::stack_size`. + pub fn stack_size(mut self, stack_size: usize) -> Self { + self.builder = self.builder.stack_size(stack_size); + self + } + + /// Deprecated in favor of `ThreadPoolBuilder::breadth_first`. + pub fn breadth_first(mut self) -> Self { + self.builder = self.builder.breadth_first(); + self + } + + /// Deprecated in favor of `ThreadPoolBuilder::start_handler`. + pub fn start_handler(mut self, start_handler: H) -> Configuration + where + H: Fn(usize) + Send + Sync + 'static, + { + self.builder = self.builder.start_handler(start_handler); + self + } + + /// Deprecated in favor of `ThreadPoolBuilder::exit_handler`. + pub fn exit_handler(mut self, exit_handler: H) -> Configuration + where + H: Fn(usize) + Send + Sync + 'static, + { + self.builder = self.builder.exit_handler(exit_handler); + self + } + + /// Returns a ThreadPoolBuilder with identical parameters. + fn into_builder(self) -> ThreadPoolBuilder { + self.builder + } +} + +impl ThreadPoolBuildError { + fn new(kind: ErrorKind) -> ThreadPoolBuildError { + ThreadPoolBuildError { kind } + } + + fn is_unsupported(&self) -> bool { + matches!(&self.kind, ErrorKind::IOError(e) if e.kind() == io::ErrorKind::Unsupported) + } +} + +const GLOBAL_POOL_ALREADY_INITIALIZED: &str = + "The global thread pool has already been initialized."; + +const CURRENT_THREAD_ALREADY_IN_POOL: &str = + "The current thread is already part of another thread pool."; + +impl Error for ThreadPoolBuildError { + #[allow(deprecated)] + fn description(&self) -> &str { + match self.kind { + ErrorKind::GlobalPoolAlreadyInitialized => GLOBAL_POOL_ALREADY_INITIALIZED, + ErrorKind::CurrentThreadAlreadyInPool => CURRENT_THREAD_ALREADY_IN_POOL, + ErrorKind::IOError(ref e) => e.description(), + } + } + + fn source(&self) -> Option<&(dyn Error + 'static)> { + match &self.kind { + ErrorKind::GlobalPoolAlreadyInitialized | ErrorKind::CurrentThreadAlreadyInPool => None, + ErrorKind::IOError(e) => Some(e), + } + } +} + +impl fmt::Display for ThreadPoolBuildError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self.kind { + ErrorKind::CurrentThreadAlreadyInPool => CURRENT_THREAD_ALREADY_IN_POOL.fmt(f), + ErrorKind::GlobalPoolAlreadyInitialized => GLOBAL_POOL_ALREADY_INITIALIZED.fmt(f), + ErrorKind::IOError(e) => e.fmt(f), + } + } +} + +/// Deprecated in favor of `ThreadPoolBuilder::build_global`. +#[deprecated(note = "use `ThreadPoolBuilder::build_global`")] +#[allow(deprecated)] +pub fn initialize(config: Configuration) -> Result<(), Box> { + config.into_builder().build_global().map_err(Box::from) +} + +impl fmt::Debug for ThreadPoolBuilder { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let ThreadPoolBuilder { + ref num_threads, + ref use_current_thread, + ref get_thread_name, + ref panic_handler, + ref stack_size, + ref start_handler, + ref exit_handler, + spawn_handler: _, + ref breadth_first, + } = *self; + + // Just print `Some()` or `None` to the debug + // output. + struct ClosurePlaceholder; + impl fmt::Debug for ClosurePlaceholder { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("") + } + } + let get_thread_name = get_thread_name.as_ref().map(|_| ClosurePlaceholder); + let panic_handler = panic_handler.as_ref().map(|_| ClosurePlaceholder); + let start_handler = start_handler.as_ref().map(|_| ClosurePlaceholder); + let exit_handler = exit_handler.as_ref().map(|_| ClosurePlaceholder); + + f.debug_struct("ThreadPoolBuilder") + .field("num_threads", num_threads) + .field("use_current_thread", use_current_thread) + .field("get_thread_name", &get_thread_name) + .field("panic_handler", &panic_handler) + .field("stack_size", &stack_size) + .field("start_handler", &start_handler) + .field("exit_handler", &exit_handler) + .field("breadth_first", &breadth_first) + .finish() + } +} + +#[allow(deprecated)] +impl fmt::Debug for Configuration { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.builder.fmt(f) + } +} + +/// Provides the calling context to a closure called by `join_context`. +#[derive(Debug)] +pub struct FnContext { + migrated: bool, + + /// disable `Send` and `Sync`, just for a little future-proofing. + _marker: PhantomData<*mut ()>, +} + +impl FnContext { + #[inline] + fn new(migrated: bool) -> Self { + FnContext { + migrated, + _marker: PhantomData, + } + } +} + +impl FnContext { + /// Returns `true` if the closure was called from a different thread + /// than it was provided from. + #[inline] + pub fn migrated(&self) -> bool { + self.migrated + } +} diff --git a/anneal/v2/vendor/rayon-core/src/private.rs b/anneal/v2/vendor/rayon-core/src/private.rs new file mode 100644 index 0000000000..c85e77b9cb --- /dev/null +++ b/anneal/v2/vendor/rayon-core/src/private.rs @@ -0,0 +1,26 @@ +//! The public parts of this private module are used to create traits +//! that cannot be implemented outside of our own crate. This way we +//! can feel free to extend those traits without worrying about it +//! being a breaking change for other implementations. + +/// If this type is pub but not publicly reachable, third parties +/// can't name it and can't implement traits using it. +#[allow(missing_debug_implementations)] +pub struct PrivateMarker; + +macro_rules! private_decl { + () => { + /// This trait is private; this method exists to make it + /// impossible to implement outside the crate. + #[doc(hidden)] + fn __rayon_private__(&self) -> crate::private::PrivateMarker; + }; +} + +macro_rules! private_impl { + () => { + fn __rayon_private__(&self) -> crate::private::PrivateMarker { + crate::private::PrivateMarker + } + }; +} diff --git a/anneal/v2/vendor/rayon-core/src/registry.rs b/anneal/v2/vendor/rayon-core/src/registry.rs new file mode 100644 index 0000000000..809976a081 --- /dev/null +++ b/anneal/v2/vendor/rayon-core/src/registry.rs @@ -0,0 +1,1002 @@ +use crate::job::{JobFifo, JobRef, StackJob}; +use crate::latch::{AsCoreLatch, CoreLatch, Latch, LatchRef, LockLatch, OnceLatch, SpinLatch}; +use crate::sleep::Sleep; +use crate::sync::Mutex; +use crate::unwind; +use crate::{ + ErrorKind, ExitHandler, PanicHandler, StartHandler, ThreadPoolBuildError, ThreadPoolBuilder, + Yield, +}; +use crossbeam_deque::{Injector, Steal, Stealer, Worker}; +use std::cell::Cell; +use std::fmt; +use std::hash::{DefaultHasher, Hasher}; +use std::io; +use std::mem; +use std::ptr; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Once}; +use std::thread; + +/// Thread builder used for customization via [`ThreadPoolBuilder::spawn_handler()`]. +pub struct ThreadBuilder { + name: Option, + stack_size: Option, + worker: Worker, + stealer: Stealer, + registry: Arc, + index: usize, +} + +impl ThreadBuilder { + /// Gets the index of this thread in the pool, within `0..num_threads`. + pub fn index(&self) -> usize { + self.index + } + + /// Gets the string that was specified by `ThreadPoolBuilder::name()`. + pub fn name(&self) -> Option<&str> { + self.name.as_deref() + } + + /// Gets the value that was specified by `ThreadPoolBuilder::stack_size()`. + pub fn stack_size(&self) -> Option { + self.stack_size + } + + /// Executes the main loop for this thread. This will not return until the + /// thread pool is dropped. + pub fn run(self) { + unsafe { main_loop(self) } + } +} + +impl fmt::Debug for ThreadBuilder { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ThreadBuilder") + .field("pool", &self.registry.id()) + .field("index", &self.index) + .field("name", &self.name) + .field("stack_size", &self.stack_size) + .finish() + } +} + +/// Generalized trait for spawning a thread in the `Registry`. +/// +/// This trait is pub-in-private -- E0445 forces us to make it public, +/// but we don't actually want to expose these details in the API. +pub trait ThreadSpawn { + private_decl! {} + + /// Spawn a thread with the `ThreadBuilder` parameters, and then + /// call `ThreadBuilder::run()`. + fn spawn(&mut self, thread: ThreadBuilder) -> io::Result<()>; +} + +/// Spawns a thread in the "normal" way with `std::thread::Builder`. +/// +/// This type is pub-in-private -- E0445 forces us to make it public, +/// but we don't actually want to expose these details in the API. +#[derive(Debug, Default)] +pub struct DefaultSpawn; + +impl ThreadSpawn for DefaultSpawn { + private_impl! {} + + fn spawn(&mut self, thread: ThreadBuilder) -> io::Result<()> { + let mut b = thread::Builder::new(); + if let Some(name) = thread.name() { + b = b.name(name.to_owned()); + } + if let Some(stack_size) = thread.stack_size() { + b = b.stack_size(stack_size); + } + b.spawn(|| thread.run())?; + Ok(()) + } +} + +/// Spawns a thread with a user's custom callback. +/// +/// This type is pub-in-private -- E0445 forces us to make it public, +/// but we don't actually want to expose these details in the API. +#[derive(Debug)] +pub struct CustomSpawn(F); + +impl CustomSpawn +where + F: FnMut(ThreadBuilder) -> io::Result<()>, +{ + pub(super) fn new(spawn: F) -> Self { + CustomSpawn(spawn) + } +} + +impl ThreadSpawn for CustomSpawn +where + F: FnMut(ThreadBuilder) -> io::Result<()>, +{ + private_impl! {} + + #[inline] + fn spawn(&mut self, thread: ThreadBuilder) -> io::Result<()> { + (self.0)(thread) + } +} + +pub(super) struct Registry { + thread_infos: Vec, + sleep: Sleep, + injected_jobs: Injector, + broadcasts: Mutex>>, + panic_handler: Option>, + start_handler: Option>, + exit_handler: Option>, + + // When this latch reaches 0, it means that all work on this + // registry must be complete. This is ensured in the following ways: + // + // - if this is the global registry, there is a ref-count that never + // gets released. + // - if this is a user-created thread pool, then so long as the thread pool + // exists, it holds a reference. + // - when we inject a "blocking job" into the registry with `ThreadPool::install()`, + // no adjustment is needed; the `ThreadPool` holds the reference, and since we won't + // return until the blocking job is complete, that ref will continue to be held. + // - when `join()` or `scope()` is invoked, similarly, no adjustments are needed. + // These are always owned by some other job (e.g., one injected by `ThreadPool::install()`) + // and that job will keep the pool alive. + terminate_count: AtomicUsize, +} + +// //////////////////////////////////////////////////////////////////////// +// Initialization + +static mut THE_REGISTRY: Option> = None; +static THE_REGISTRY_SET: Once = Once::new(); + +/// Starts the worker threads (if that has not already happened). If +/// initialization has not already occurred, use the default +/// configuration. +pub(super) fn global_registry() -> &'static Arc { + set_global_registry(default_global_registry) + .or_else(|err| { + // SAFETY: we only create a shared reference to `THE_REGISTRY` after the `call_once` + // that initializes it, and there will be no more mutable accesses at all. + debug_assert!(THE_REGISTRY_SET.is_completed()); + let the_registry = unsafe { &*ptr::addr_of!(THE_REGISTRY) }; + the_registry.as_ref().ok_or(err) + }) + .expect("The global thread pool has not been initialized.") +} + +/// Starts the worker threads (if that has not already happened) with +/// the given builder. +pub(super) fn init_global_registry( + builder: ThreadPoolBuilder, +) -> Result<&'static Arc, ThreadPoolBuildError> +where + S: ThreadSpawn, +{ + set_global_registry(|| Registry::new(builder)) +} + +/// Starts the worker threads (if that has not already happened) +/// by creating a registry with the given callback. +fn set_global_registry(registry: F) -> Result<&'static Arc, ThreadPoolBuildError> +where + F: FnOnce() -> Result, ThreadPoolBuildError>, +{ + let mut result = Err(ThreadPoolBuildError::new( + ErrorKind::GlobalPoolAlreadyInitialized, + )); + + THE_REGISTRY_SET.call_once(|| { + result = registry().map(|registry: Arc| { + // SAFETY: this is the only mutable access to `THE_REGISTRY`, thanks to `Once`, and + // `global_registry()` only takes a shared reference **after** this `call_once`. + unsafe { + ptr::addr_of_mut!(THE_REGISTRY).write(Some(registry)); + (*ptr::addr_of!(THE_REGISTRY)).as_ref().unwrap_unchecked() + } + }) + }); + + result +} + +fn default_global_registry() -> Result, ThreadPoolBuildError> { + let result = Registry::new(ThreadPoolBuilder::new()); + + // If we're running in an environment that doesn't support threads at all, we can fall back to + // using the current thread alone. This is crude, and probably won't work for non-blocking + // calls like `spawn` or `broadcast_spawn`, but a lot of stuff does work fine. + // + // Notably, this allows current WebAssembly targets to work even though their threading support + // is stubbed out, and we won't have to change anything if they do add real threading. + let unsupported = matches!(&result, Err(e) if e.is_unsupported()); + if unsupported && WorkerThread::current().is_null() { + let builder = ThreadPoolBuilder::new().num_threads(1).use_current_thread(); + let fallback_result = Registry::new(builder); + if fallback_result.is_ok() { + return fallback_result; + } + } + + result +} + +struct Terminator<'a>(&'a Arc); + +impl<'a> Drop for Terminator<'a> { + fn drop(&mut self) { + self.0.terminate() + } +} + +impl Registry { + pub(super) fn new( + mut builder: ThreadPoolBuilder, + ) -> Result, ThreadPoolBuildError> + where + S: ThreadSpawn, + { + // Soft-limit the number of threads that we can actually support. + let n_threads = Ord::min(builder.get_num_threads(), crate::max_num_threads()); + + let breadth_first = builder.get_breadth_first(); + + let (workers, stealers): (Vec<_>, Vec<_>) = (0..n_threads) + .map(|_| { + let worker = if breadth_first { + Worker::new_fifo() + } else { + Worker::new_lifo() + }; + + let stealer = worker.stealer(); + (worker, stealer) + }) + .unzip(); + + let (broadcasts, broadcast_stealers): (Vec<_>, Vec<_>) = (0..n_threads) + .map(|_| { + let worker = Worker::new_fifo(); + let stealer = worker.stealer(); + (worker, stealer) + }) + .unzip(); + + let registry = Arc::new(Registry { + thread_infos: stealers.into_iter().map(ThreadInfo::new).collect(), + sleep: Sleep::new(n_threads), + injected_jobs: Injector::new(), + broadcasts: Mutex::new(broadcasts), + terminate_count: AtomicUsize::new(1), + panic_handler: builder.take_panic_handler(), + start_handler: builder.take_start_handler(), + exit_handler: builder.take_exit_handler(), + }); + + // If we return early or panic, make sure to terminate existing threads. + let t1000 = Terminator(®istry); + + for (index, (worker, stealer)) in workers.into_iter().zip(broadcast_stealers).enumerate() { + let thread = ThreadBuilder { + name: builder.get_thread_name(index), + stack_size: builder.get_stack_size(), + registry: Arc::clone(®istry), + worker, + stealer, + index, + }; + + if index == 0 && builder.use_current_thread { + if !WorkerThread::current().is_null() { + return Err(ThreadPoolBuildError::new( + ErrorKind::CurrentThreadAlreadyInPool, + )); + } + // Rather than starting a new thread, we're just taking over the current thread + // *without* running the main loop, so we can still return from here. + // The WorkerThread is leaked, but we never shutdown the global pool anyway. + let worker_thread = Box::into_raw(Box::new(WorkerThread::from(thread))); + + unsafe { + WorkerThread::set_current(worker_thread); + Latch::set(®istry.thread_infos[index].primed); + } + continue; + } + + if let Err(e) = builder.get_spawn_handler().spawn(thread) { + return Err(ThreadPoolBuildError::new(ErrorKind::IOError(e))); + } + } + + // Returning normally now, without termination. + mem::forget(t1000); + + Ok(registry) + } + + pub(super) fn current() -> Arc { + unsafe { + let worker_thread = WorkerThread::current(); + let registry = if worker_thread.is_null() { + global_registry() + } else { + &(*worker_thread).registry + }; + Arc::clone(registry) + } + } + + /// Returns the number of threads in the current registry. This + /// is better than `Registry::current().num_threads()` because it + /// avoids incrementing the `Arc`. + pub(super) fn current_num_threads() -> usize { + unsafe { + let worker_thread = WorkerThread::current(); + if worker_thread.is_null() { + global_registry().num_threads() + } else { + (*worker_thread).registry.num_threads() + } + } + } + + /// Returns the current `WorkerThread` if it's part of this `Registry`. + pub(super) fn current_thread(&self) -> Option<&WorkerThread> { + unsafe { + let worker = WorkerThread::current().as_ref()?; + if worker.registry().id() == self.id() { + Some(worker) + } else { + None + } + } + } + + /// Returns an opaque identifier for this registry. + pub(super) fn id(&self) -> RegistryId { + // We can rely on `self` not to change since we only ever create + // registries that are boxed up in an `Arc` (see `new()` above). + RegistryId { + addr: self as *const Self as usize, + } + } + + pub(super) fn num_threads(&self) -> usize { + self.thread_infos.len() + } + + pub(super) fn catch_unwind(&self, f: impl FnOnce()) { + if let Err(err) = unwind::halt_unwinding(f) { + // If there is no handler, or if that handler itself panics, then we abort. + let abort_guard = unwind::AbortIfPanic; + if let Some(ref handler) = self.panic_handler { + handler(err); + mem::forget(abort_guard); + } + } + } + + /// Waits for the worker threads to get up and running. This is + /// meant to be used for benchmarking purposes, primarily, so that + /// you can get more consistent numbers by having everything + /// "ready to go". + pub(super) fn wait_until_primed(&self) { + for info in &self.thread_infos { + info.primed.wait(); + } + } + + /// Waits for the worker threads to stop. This is used for testing + /// -- so we can check that termination actually works. + #[cfg(test)] + pub(super) fn wait_until_stopped(&self) { + for info in &self.thread_infos { + info.stopped.wait(); + } + } + + // //////////////////////////////////////////////////////////////////////// + // MAIN LOOP + // + // So long as all of the worker threads are hanging out in their + // top-level loop, there is no work to be done. + + /// Push a job into the given `registry`. If we are running on a + /// worker thread for the registry, this will push onto the + /// deque. Else, it will inject from the outside (which is slower). + pub(super) fn inject_or_push(&self, job_ref: JobRef) { + let worker_thread = WorkerThread::current(); + unsafe { + if !worker_thread.is_null() && (*worker_thread).registry().id() == self.id() { + (*worker_thread).push(job_ref); + } else { + self.inject(job_ref); + } + } + } + + /// Push a job into the "external jobs" queue; it will be taken by + /// whatever worker has nothing to do. Use this if you know that + /// you are not on a worker of this registry. + pub(super) fn inject(&self, injected_job: JobRef) { + // It should not be possible for `state.terminate` to be true + // here. It is only set to true when the user creates (and + // drops) a `ThreadPool`; and, in that case, they cannot be + // calling `inject()` later, since they dropped their + // `ThreadPool`. + debug_assert_ne!( + self.terminate_count.load(Ordering::Acquire), + 0, + "inject() sees state.terminate as true" + ); + + let queue_was_empty = self.injected_jobs.is_empty(); + + self.injected_jobs.push(injected_job); + self.sleep.new_injected_jobs(1, queue_was_empty); + } + + fn has_injected_job(&self) -> bool { + !self.injected_jobs.is_empty() + } + + fn pop_injected_job(&self) -> Option { + loop { + match self.injected_jobs.steal() { + Steal::Success(job) => return Some(job), + Steal::Empty => return None, + Steal::Retry => {} + } + } + } + + /// Push a job into each thread's own "external jobs" queue; it will be + /// executed only on that thread, when it has nothing else to do locally, + /// before it tries to steal other work. + /// + /// **Panics** if not given exactly as many jobs as there are threads. + pub(super) fn inject_broadcast(&self, injected_jobs: impl ExactSizeIterator) { + assert_eq!(self.num_threads(), injected_jobs.len()); + { + let broadcasts = self.broadcasts.lock().unwrap(); + + // It should not be possible for `state.terminate` to be true + // here. It is only set to true when the user creates (and + // drops) a `ThreadPool`; and, in that case, they cannot be + // calling `inject_broadcast()` later, since they dropped their + // `ThreadPool`. + debug_assert_ne!( + self.terminate_count.load(Ordering::Acquire), + 0, + "inject_broadcast() sees state.terminate as true" + ); + + assert_eq!(broadcasts.len(), injected_jobs.len()); + for (worker, job_ref) in broadcasts.iter().zip(injected_jobs) { + worker.push(job_ref); + } + } + for i in 0..self.num_threads() { + self.sleep.notify_worker_latch_is_set(i); + } + } + + /// If already in a worker-thread of this registry, just execute `op`. + /// Otherwise, inject `op` in this thread pool. Either way, block until `op` + /// completes and return its return value. If `op` panics, that panic will + /// be propagated as well. The second argument indicates `true` if injection + /// was performed, `false` if executed directly. + pub(super) fn in_worker(&self, op: OP) -> R + where + OP: FnOnce(&WorkerThread, bool) -> R + Send, + R: Send, + { + unsafe { + let worker_thread = WorkerThread::current(); + if worker_thread.is_null() { + self.in_worker_cold(op) + } else if (*worker_thread).registry().id() != self.id() { + self.in_worker_cross(&*worker_thread, op) + } else { + // Perfectly valid to give them a `&T`: this is the + // current thread, so we know the data structure won't be + // invalidated until we return. + op(&*worker_thread, false) + } + } + } + + #[cold] + unsafe fn in_worker_cold(&self, op: OP) -> R + where + OP: FnOnce(&WorkerThread, bool) -> R + Send, + R: Send, + { + thread_local!(static LOCK_LATCH: LockLatch = const { LockLatch::new() }); + + LOCK_LATCH.with(|l| { + // This thread isn't a member of *any* thread pool, so just block. + debug_assert!(WorkerThread::current().is_null()); + let job = StackJob::new( + |injected| { + let worker_thread = WorkerThread::current(); + assert!(injected && !worker_thread.is_null()); + op(&*worker_thread, true) + }, + LatchRef::new(l), + ); + self.inject(job.as_job_ref()); + job.latch.wait_and_reset(); // Make sure we can use the same latch again next time. + + job.into_result() + }) + } + + #[cold] + unsafe fn in_worker_cross(&self, current_thread: &WorkerThread, op: OP) -> R + where + OP: FnOnce(&WorkerThread, bool) -> R + Send, + R: Send, + { + // This thread is a member of a different pool, so let it process + // other work while waiting for this `op` to complete. + debug_assert!(current_thread.registry().id() != self.id()); + let latch = SpinLatch::cross(current_thread); + let job = StackJob::new( + |injected| { + let worker_thread = WorkerThread::current(); + assert!(injected && !worker_thread.is_null()); + op(&*worker_thread, true) + }, + latch, + ); + self.inject(job.as_job_ref()); + current_thread.wait_until(&job.latch); + job.into_result() + } + + /// Increments the terminate counter. This increment should be + /// balanced by a call to `terminate`, which will decrement. This + /// is used when spawning asynchronous work, which needs to + /// prevent the registry from terminating so long as it is active. + /// + /// Note that blocking functions such as `join` and `scope` do not + /// need to concern themselves with this fn; their context is + /// responsible for ensuring the current thread pool will not + /// terminate until they return. + /// + /// The global thread pool always has an outstanding reference + /// (the initial one). Custom thread pools have one outstanding + /// reference that is dropped when the `ThreadPool` is dropped: + /// since installing the thread pool blocks until any joins/scopes + /// complete, this ensures that joins/scopes are covered. + /// + /// The exception is `::spawn()`, which can create a job outside + /// of any blocking scope. In that case, the job itself holds a + /// terminate count and is responsible for invoking `terminate()` + /// when finished. + pub(super) fn increment_terminate_count(&self) { + let previous = self.terminate_count.fetch_add(1, Ordering::AcqRel); + debug_assert!(previous != 0, "registry ref count incremented from zero"); + assert!(previous != usize::MAX, "overflow in registry ref count"); + } + + /// Signals that the thread pool which owns this registry has been + /// dropped. The worker threads will gradually terminate, once any + /// extant work is completed. + pub(super) fn terminate(&self) { + if self.terminate_count.fetch_sub(1, Ordering::AcqRel) == 1 { + for (i, thread_info) in self.thread_infos.iter().enumerate() { + unsafe { OnceLatch::set_and_tickle_one(&thread_info.terminate, self, i) }; + } + } + } + + /// Notify the worker that the latch they are sleeping on has been "set". + pub(super) fn notify_worker_latch_is_set(&self, target_worker_index: usize) { + self.sleep.notify_worker_latch_is_set(target_worker_index); + } +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub(super) struct RegistryId { + addr: usize, +} + +struct ThreadInfo { + /// Latch set once thread has started and we are entering into the + /// main loop. Used to wait for worker threads to become primed, + /// primarily of interest for benchmarking. + primed: LockLatch, + + /// Latch is set once worker thread has completed. Used to wait + /// until workers have stopped; only used for tests. + stopped: LockLatch, + + /// The latch used to signal that terminated has been requested. + /// This latch is *set* by the `terminate` method on the + /// `Registry`, once the registry's main "terminate" counter + /// reaches zero. + terminate: OnceLatch, + + /// the "stealer" half of the worker's deque + stealer: Stealer, +} + +impl ThreadInfo { + fn new(stealer: Stealer) -> ThreadInfo { + ThreadInfo { + primed: LockLatch::new(), + stopped: LockLatch::new(), + terminate: OnceLatch::new(), + stealer, + } + } +} + +// //////////////////////////////////////////////////////////////////////// +// WorkerThread identifiers + +pub(super) struct WorkerThread { + /// the "worker" half of our local deque + worker: Worker, + + /// the "stealer" half of the worker's broadcast deque + stealer: Stealer, + + /// local queue used for `spawn_fifo` indirection + fifo: JobFifo, + + index: usize, + + /// A weak random number generator. + rng: XorShift64Star, + + registry: Arc, +} + +// This is a bit sketchy, but basically: the WorkerThread is +// allocated on the stack of the worker on entry and stored into this +// thread-local variable. So it will remain valid at least until the +// worker is fully unwound. Using an unsafe pointer avoids the need +// for a RefCell etc. +thread_local! { + static WORKER_THREAD_STATE: Cell<*const WorkerThread> = const { Cell::new(ptr::null()) }; +} + +impl From for WorkerThread { + fn from(thread: ThreadBuilder) -> Self { + Self { + worker: thread.worker, + stealer: thread.stealer, + fifo: JobFifo::new(), + index: thread.index, + rng: XorShift64Star::new(), + registry: thread.registry, + } + } +} + +impl Drop for WorkerThread { + fn drop(&mut self) { + // Undo `set_current` + WORKER_THREAD_STATE.with(|t| { + assert!(t.get().eq(&(self as *const _))); + t.set(ptr::null()); + }); + } +} + +impl WorkerThread { + /// Gets the `WorkerThread` index for the current thread; returns + /// NULL if this is not a worker thread. This pointer is valid + /// anywhere on the current thread. + #[inline] + pub(super) fn current() -> *const WorkerThread { + WORKER_THREAD_STATE.get() + } + + /// Sets `self` as the worker-thread index for the current thread. + /// This is done during worker-thread startup. + unsafe fn set_current(thread: *const WorkerThread) { + WORKER_THREAD_STATE.with(|t| { + assert!(t.get().is_null()); + t.set(thread); + }); + } + + /// Returns the registry that owns this worker thread. + #[inline] + pub(super) fn registry(&self) -> &Arc { + &self.registry + } + + /// Our index amongst the worker threads (ranges from `0..self.num_threads()`). + #[inline] + pub(super) fn index(&self) -> usize { + self.index + } + + #[inline] + pub(super) unsafe fn push(&self, job: JobRef) { + let queue_was_empty = self.worker.is_empty(); + self.worker.push(job); + self.registry.sleep.new_internal_jobs(1, queue_was_empty); + } + + #[inline] + pub(super) unsafe fn push_fifo(&self, job: JobRef) { + self.push(self.fifo.push(job)); + } + + #[inline] + pub(super) fn local_deque_is_empty(&self) -> bool { + self.worker.is_empty() + } + + /// Attempts to obtain a "local" job -- typically this means + /// popping from the top of the stack, though if we are configured + /// for breadth-first execution, it would mean dequeuing from the + /// bottom. + #[inline] + pub(super) fn take_local_job(&self) -> Option { + let popped_job = self.worker.pop(); + + if popped_job.is_some() { + return popped_job; + } + + loop { + match self.stealer.steal() { + Steal::Success(job) => return Some(job), + Steal::Empty => return None, + Steal::Retry => {} + } + } + } + + fn has_injected_job(&self) -> bool { + !self.stealer.is_empty() || self.registry.has_injected_job() + } + + /// Wait until the latch is set. Try to keep busy by popping and + /// stealing tasks as necessary. + #[inline] + pub(super) unsafe fn wait_until(&self, latch: &L) { + let latch = latch.as_core_latch(); + if !latch.probe() { + self.wait_until_cold(latch); + } + } + + #[cold] + unsafe fn wait_until_cold(&self, latch: &CoreLatch) { + // the code below should swallow all panics and hence never + // unwind; but if something does wrong, we want to abort, + // because otherwise other code in rayon may assume that the + // latch has been signaled, and that can lead to random memory + // accesses, which would be *very bad* + let abort_guard = unwind::AbortIfPanic; + + 'outer: while !latch.probe() { + // Check for local work *before* we start marking ourself idle, + // especially to avoid modifying shared sleep state. + if let Some(job) = self.take_local_job() { + self.execute(job); + continue; + } + + let mut idle_state = self.registry.sleep.start_looking(self.index); + while !latch.probe() { + if let Some(job) = self.find_work() { + self.registry.sleep.work_found(); + self.execute(job); + // The job might have injected local work, so go back to the outer loop. + continue 'outer; + } else { + self.registry + .sleep + .no_work_found(&mut idle_state, latch, || self.has_injected_job()) + } + } + + // If we were sleepy, we are not anymore. We "found work" -- + // whatever the surrounding thread was doing before it had to wait. + self.registry.sleep.work_found(); + break; + } + + mem::forget(abort_guard); // successful execution, do not abort + } + + unsafe fn wait_until_out_of_work(&self) { + debug_assert_eq!(self as *const _, WorkerThread::current()); + let registry = &*self.registry; + let index = self.index; + + self.wait_until(®istry.thread_infos[index].terminate); + + // Should not be any work left in our queue. + debug_assert!(self.take_local_job().is_none()); + + // Let registry know we are done + Latch::set(®istry.thread_infos[index].stopped); + } + + fn find_work(&self) -> Option { + // Try to find some work to do. We give preference first + // to things in our local deque, then in other workers + // deques, and finally to injected jobs from the + // outside. The idea is to finish what we started before + // we take on something new. + self.take_local_job() + .or_else(|| self.steal()) + .or_else(|| self.registry.pop_injected_job()) + } + + pub(super) fn yield_now(&self) -> Yield { + match self.find_work() { + Some(job) => unsafe { + self.execute(job); + Yield::Executed + }, + None => Yield::Idle, + } + } + + pub(super) fn yield_local(&self) -> Yield { + match self.take_local_job() { + Some(job) => unsafe { + self.execute(job); + Yield::Executed + }, + None => Yield::Idle, + } + } + + #[inline] + pub(super) unsafe fn execute(&self, job: JobRef) { + job.execute(); + } + + /// Try to steal a single job and return it. + /// + /// This should only be done as a last resort, when there is no + /// local work to do. + fn steal(&self) -> Option { + // we only steal when we don't have any work to do locally + debug_assert!(self.local_deque_is_empty()); + + // otherwise, try to steal + let thread_infos = &self.registry.thread_infos.as_slice(); + let num_threads = thread_infos.len(); + if num_threads <= 1 { + return None; + } + + loop { + let mut retry = false; + let start = self.rng.next_usize(num_threads); + let job = (start..num_threads) + .chain(0..start) + .filter(move |&i| i != self.index) + .find_map(|victim_index| { + let victim = &thread_infos[victim_index]; + match victim.stealer.steal() { + Steal::Success(job) => Some(job), + Steal::Empty => None, + Steal::Retry => { + retry = true; + None + } + } + }); + if job.is_some() || !retry { + return job; + } + } + } +} + +// //////////////////////////////////////////////////////////////////////// + +unsafe fn main_loop(thread: ThreadBuilder) { + let worker_thread = &WorkerThread::from(thread); + WorkerThread::set_current(worker_thread); + let registry = &*worker_thread.registry; + let index = worker_thread.index; + + // let registry know we are ready to do work + Latch::set(®istry.thread_infos[index].primed); + + // Worker threads should not panic. If they do, just abort, as the + // internal state of the thread pool is corrupted. Note that if + // **user code** panics, we should catch that and redirect. + let abort_guard = unwind::AbortIfPanic; + + // Inform a user callback that we started a thread. + if let Some(ref handler) = registry.start_handler { + registry.catch_unwind(|| handler(index)); + } + + worker_thread.wait_until_out_of_work(); + + // Normal termination, do not abort. + mem::forget(abort_guard); + + // Inform a user callback that we exited a thread. + if let Some(ref handler) = registry.exit_handler { + registry.catch_unwind(|| handler(index)); + // We're already exiting the thread, there's nothing else to do. + } +} + +/// If already in a worker-thread, just execute `op`. Otherwise, +/// execute `op` in the default thread pool. Either way, block until +/// `op` completes and return its return value. If `op` panics, that +/// panic will be propagated as well. The second argument indicates +/// `true` if injection was performed, `false` if executed directly. +pub(super) fn in_worker(op: OP) -> R +where + OP: FnOnce(&WorkerThread, bool) -> R + Send, + R: Send, +{ + unsafe { + let owner_thread = WorkerThread::current(); + if !owner_thread.is_null() { + // Perfectly valid to give them a `&T`: this is the + // current thread, so we know the data structure won't be + // invalidated until we return. + op(&*owner_thread, false) + } else { + global_registry().in_worker(op) + } + } +} + +/// [xorshift*] is a fast pseudorandom number generator which will +/// even tolerate weak seeding, as long as it's not zero. +/// +/// [xorshift*]: https://en.wikipedia.org/wiki/Xorshift#xorshift* +struct XorShift64Star { + state: Cell, +} + +impl XorShift64Star { + fn new() -> Self { + // Any non-zero seed will do -- this uses the hash of a global counter. + let mut seed = 0; + while seed == 0 { + let mut hasher = DefaultHasher::new(); + static COUNTER: AtomicUsize = AtomicUsize::new(0); + hasher.write_usize(COUNTER.fetch_add(1, Ordering::Relaxed)); + seed = hasher.finish(); + } + + XorShift64Star { + state: Cell::new(seed), + } + } + + fn next(&self) -> u64 { + let mut x = self.state.get(); + debug_assert_ne!(x, 0); + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + self.state.set(x); + x.wrapping_mul(0x2545_f491_4f6c_dd1d) + } + + /// Return a value from `0..n`. + fn next_usize(&self, n: usize) -> usize { + (self.next() % n as u64) as usize + } +} diff --git a/anneal/v2/vendor/rayon-core/src/scope/mod.rs b/anneal/v2/vendor/rayon-core/src/scope/mod.rs new file mode 100644 index 0000000000..586a8b7d92 --- /dev/null +++ b/anneal/v2/vendor/rayon-core/src/scope/mod.rs @@ -0,0 +1,773 @@ +//! Methods for custom fork-join scopes, created by the [`scope()`] +//! and [`in_place_scope()`] functions. These are a more flexible alternative to [`join()`]. +//! +//! [`join()`]: crate::join() + +use crate::broadcast::BroadcastContext; +use crate::job::{ArcJob, HeapJob, JobFifo, JobRef}; +use crate::latch::{CountLatch, Latch}; +use crate::registry::{global_registry, in_worker, Registry, WorkerThread}; +use crate::unwind; +use std::any::Any; +use std::fmt; +use std::marker::PhantomData; +use std::mem::ManuallyDrop; +use std::ptr; +use std::sync::atomic::{AtomicPtr, Ordering}; +use std::sync::Arc; + +#[cfg(test)] +mod test; + +/// Represents a fork-join scope which can be used to spawn any number of tasks. +/// See [`scope()`] for more information. +pub struct Scope<'scope> { + base: ScopeBase<'scope>, +} + +/// Represents a fork-join scope which can be used to spawn any number of tasks. +/// Those spawned from the same thread are prioritized in relative FIFO order. +/// See [`scope_fifo()`] for more information. +pub struct ScopeFifo<'scope> { + base: ScopeBase<'scope>, + fifos: Vec, +} + +struct ScopeBase<'scope> { + /// thread registry where `scope()` was executed or where `in_place_scope()` + /// should spawn jobs. + registry: Arc, + + /// if some job panicked, the error is stored here; it will be + /// propagated to the one who created the scope + panic: AtomicPtr>, + + /// latch to track job counts + job_completed_latch: CountLatch, + + /// You can think of a scope as containing a list of closures to execute, + /// all of which outlive `'scope`. They're not actually required to be + /// `Sync`, but it's still safe to let the `Scope` implement `Sync` because + /// the closures are only *moved* across threads to be executed. + #[allow(clippy::type_complexity)] + marker: PhantomData) + Send + Sync + 'scope>>, +} + +/// Creates a "fork-join" scope `s` and invokes the closure with a +/// reference to `s`. This closure can then spawn asynchronous tasks +/// into `s`. Those tasks may run asynchronously with respect to the +/// closure; they may themselves spawn additional tasks into `s`. When +/// the closure returns, it will block until all tasks that have been +/// spawned into `s` complete. +/// +/// `scope()` is a more flexible building block compared to `join()`, +/// since a loop can be used to spawn any number of tasks without +/// recursing. However, that flexibility comes at a performance price: +/// tasks spawned using `scope()` must be allocated onto the heap, +/// whereas `join()` can make exclusive use of the stack. **Prefer +/// `join()` (or, even better, parallel iterators) where possible.** +/// +/// # Example +/// +/// The Rayon `join()` function launches two closures and waits for them +/// to stop. One could implement `join()` using a scope like so, although +/// it would be less efficient than the real implementation: +/// +/// ```rust +/// # use rayon_core as rayon; +/// pub fn join(oper_a: A, oper_b: B) -> (RA, RB) +/// where A: FnOnce() -> RA + Send, +/// B: FnOnce() -> RB + Send, +/// RA: Send, +/// RB: Send, +/// { +/// let mut result_a: Option = None; +/// let mut result_b: Option = None; +/// rayon::scope(|s| { +/// s.spawn(|_| result_a = Some(oper_a())); +/// s.spawn(|_| result_b = Some(oper_b())); +/// }); +/// (result_a.unwrap(), result_b.unwrap()) +/// } +/// ``` +/// +/// # A note on threading +/// +/// The closure given to `scope()` executes in the Rayon thread pool, +/// as do those given to `spawn()`. This means that you can't access +/// thread-local variables (well, you can, but they may have +/// unexpected values). +/// +/// # Task execution +/// +/// Task execution potentially starts as soon as `spawn()` is called. +/// The task will end sometime before `scope()` returns. Note that the +/// *closure* given to scope may return much earlier. In general +/// the lifetime of a scope created like `scope(body)` goes something like this: +/// +/// - Scope begins when `scope(body)` is called +/// - Scope body `body()` is invoked +/// - Scope tasks may be spawned +/// - Scope body returns +/// - Scope tasks execute, possibly spawning more tasks +/// - Once all tasks are done, scope ends and `scope()` returns +/// +/// To see how and when tasks are joined, consider this example: +/// +/// ```rust +/// # use rayon_core as rayon; +/// // point start +/// rayon::scope(|s| { +/// s.spawn(|s| { // task s.1 +/// s.spawn(|s| { // task s.1.1 +/// rayon::scope(|t| { +/// t.spawn(|_| ()); // task t.1 +/// t.spawn(|_| ()); // task t.2 +/// }); +/// }); +/// }); +/// s.spawn(|s| { // task s.2 +/// }); +/// // point mid +/// }); +/// // point end +/// ``` +/// +/// The various tasks that are run will execute roughly like so: +/// +/// ```notrust +/// | (start) +/// | +/// | (scope `s` created) +/// +-----------------------------------------------+ (task s.2) +/// +-------+ (task s.1) | +/// | | | +/// | +---+ (task s.1.1) | +/// | | | | +/// | | | (scope `t` created) | +/// | | +----------------+ (task t.2) | +/// | | +---+ (task t.1) | | +/// | (mid) | | | | | +/// : | + <-+------------+ (scope `t` ends) | +/// : | | | +/// |<------+---+-----------------------------------+ (scope `s` ends) +/// | +/// | (end) +/// ``` +/// +/// The point here is that everything spawned into scope `s` will +/// terminate (at latest) at the same point -- right before the +/// original call to `rayon::scope` returns. This includes new +/// subtasks created by other subtasks (e.g., task `s.1.1`). If a new +/// scope is created (such as `t`), the things spawned into that scope +/// will be joined before that scope returns, which in turn occurs +/// before the creating task (task `s.1.1` in this case) finishes. +/// +/// There is no guaranteed order of execution for spawns in a scope, +/// given that other threads may steal tasks at any time. However, they +/// are generally prioritized in a LIFO order on the thread from which +/// they were spawned. So in this example, absent any stealing, we can +/// expect `s.2` to execute before `s.1`, and `t.2` before `t.1`. Other +/// threads always steal from the other end of the deque, like FIFO +/// order. The idea is that "recent" tasks are most likely to be fresh +/// in the local CPU's cache, while other threads can steal older +/// "stale" tasks. For an alternate approach, consider +/// [`scope_fifo()`] instead. +/// +/// # Accessing stack data +/// +/// In general, spawned tasks may access stack data in place that +/// outlives the scope itself. Other data must be fully owned by the +/// spawned task. +/// +/// ```rust +/// # use rayon_core as rayon; +/// let ok: Vec = vec![1, 2, 3]; +/// rayon::scope(|s| { +/// let bad: Vec = vec![4, 5, 6]; +/// s.spawn(|_| { +/// // We can access `ok` because outlives the scope `s`. +/// println!("ok: {:?}", ok); +/// +/// // If we just try to use `bad` here, the closure will borrow `bad` +/// // (because we are just printing it out, and that only requires a +/// // borrow), which will result in a compilation error. Read on +/// // for options. +/// // println!("bad: {:?}", bad); +/// }); +/// }); +/// ``` +/// +/// As the comments example above suggest, to reference `bad` we must +/// take ownership of it. One way to do this is to detach the closure +/// from the surrounding stack frame, using the `move` keyword. This +/// will cause it to take ownership of *all* the variables it touches, +/// in this case including both `ok` *and* `bad`: +/// +/// ```rust +/// # use rayon_core as rayon; +/// let ok: Vec = vec![1, 2, 3]; +/// rayon::scope(|s| { +/// let bad: Vec = vec![4, 5, 6]; +/// s.spawn(move |_| { +/// println!("ok: {:?}", ok); +/// println!("bad: {:?}", bad); +/// }); +/// +/// // That closure is fine, but now we can't use `ok` anywhere else, +/// // since it is owned by the previous task: +/// // s.spawn(|_| println!("ok: {:?}", ok)); +/// }); +/// ``` +/// +/// While this works, it could be a problem if we want to use `ok` elsewhere. +/// There are two choices. We can keep the closure as a `move` closure, but +/// instead of referencing the variable `ok`, we create a shadowed variable that +/// is a borrow of `ok` and capture *that*: +/// +/// ```rust +/// # use rayon_core as rayon; +/// let ok: Vec = vec![1, 2, 3]; +/// rayon::scope(|s| { +/// let bad: Vec = vec![4, 5, 6]; +/// let ok: &Vec = &ok; // shadow the original `ok` +/// s.spawn(move |_| { +/// println!("ok: {:?}", ok); // captures the shadowed version +/// println!("bad: {:?}", bad); +/// }); +/// +/// // Now we too can use the shadowed `ok`, since `&Vec` references +/// // can be shared freely. Note that we need a `move` closure here though, +/// // because otherwise we'd be trying to borrow the shadowed `ok`, +/// // and that doesn't outlive `scope`. +/// s.spawn(move |_| println!("ok: {:?}", ok)); +/// }); +/// ``` +/// +/// Another option is not to use the `move` keyword but instead to take ownership +/// of individual variables: +/// +/// ```rust +/// # use rayon_core as rayon; +/// let ok: Vec = vec![1, 2, 3]; +/// rayon::scope(|s| { +/// let bad: Vec = vec![4, 5, 6]; +/// s.spawn(|_| { +/// // Transfer ownership of `bad` into a local variable (also named `bad`). +/// // This will force the closure to take ownership of `bad` from the environment. +/// let bad = bad; +/// println!("ok: {:?}", ok); // `ok` is only borrowed. +/// println!("bad: {:?}", bad); // refers to our local variable, above. +/// }); +/// +/// s.spawn(|_| println!("ok: {:?}", ok)); // we too can borrow `ok` +/// }); +/// ``` +/// +/// # Panics +/// +/// If a panic occurs, either in the closure given to `scope()` or in +/// any of the spawned jobs, that panic will be propagated and the +/// call to `scope()` will panic. If multiple panics occurs, it is +/// non-deterministic which of their panic values will propagate. +/// Regardless, once a task is spawned using `scope.spawn()`, it will +/// execute, even if the spawning task should later panic. `scope()` +/// returns once all spawned jobs have completed, and any panics are +/// propagated at that point. +pub fn scope<'scope, OP, R>(op: OP) -> R +where + OP: FnOnce(&Scope<'scope>) -> R + Send, + R: Send, +{ + in_worker(|owner_thread, _| { + let scope = Scope::<'scope>::new(Some(owner_thread), None); + scope.base.complete(Some(owner_thread), || op(&scope)) + }) +} + +/// Creates a "fork-join" scope `s` with FIFO order, and invokes the +/// closure with a reference to `s`. This closure can then spawn +/// asynchronous tasks into `s`. Those tasks may run asynchronously with +/// respect to the closure; they may themselves spawn additional tasks +/// into `s`. When the closure returns, it will block until all tasks +/// that have been spawned into `s` complete. +/// +/// # Task execution +/// +/// Tasks in a `scope_fifo()` run similarly to [`scope()`], but there's a +/// difference in the order of execution. Consider a similar example: +/// +/// ```rust +/// # use rayon_core as rayon; +/// // point start +/// rayon::scope_fifo(|s| { +/// s.spawn_fifo(|s| { // task s.1 +/// s.spawn_fifo(|s| { // task s.1.1 +/// rayon::scope_fifo(|t| { +/// t.spawn_fifo(|_| ()); // task t.1 +/// t.spawn_fifo(|_| ()); // task t.2 +/// }); +/// }); +/// }); +/// s.spawn_fifo(|s| { // task s.2 +/// }); +/// // point mid +/// }); +/// // point end +/// ``` +/// +/// The various tasks that are run will execute roughly like so: +/// +/// ```notrust +/// | (start) +/// | +/// | (FIFO scope `s` created) +/// +--------------------+ (task s.1) +/// +-------+ (task s.2) | +/// | | +---+ (task s.1.1) +/// | | | | +/// | | | | (FIFO scope `t` created) +/// | | | +----------------+ (task t.1) +/// | | | +---+ (task t.2) | +/// | (mid) | | | | | +/// : | | + <-+------------+ (scope `t` ends) +/// : | | | +/// |<------+------------+---+ (scope `s` ends) +/// | +/// | (end) +/// ``` +/// +/// Under `scope_fifo()`, the spawns are prioritized in a FIFO order on +/// the thread from which they were spawned, as opposed to `scope()`'s +/// LIFO. So in this example, we can expect `s.1` to execute before +/// `s.2`, and `t.1` before `t.2`. Other threads also steal tasks in +/// FIFO order, as usual. Overall, this has roughly the same order as +/// the now-deprecated [`breadth_first`] option, except the effect is +/// isolated to a particular scope. If spawns are intermingled from any +/// combination of `scope()` and `scope_fifo()`, or from different +/// threads, their order is only specified with respect to spawns in the +/// same scope and thread. +/// +/// For more details on this design, see Rayon [RFC #1]. +/// +/// [`breadth_first`]: crate::ThreadPoolBuilder::breadth_first +/// [RFC #1]: https://github.com/rayon-rs/rfcs/blob/main/accepted/rfc0001-scope-scheduling.md +/// +/// # Panics +/// +/// If a panic occurs, either in the closure given to `scope_fifo()` or +/// in any of the spawned jobs, that panic will be propagated and the +/// call to `scope_fifo()` will panic. If multiple panics occurs, it is +/// non-deterministic which of their panic values will propagate. +/// Regardless, once a task is spawned using `scope.spawn_fifo()`, it +/// will execute, even if the spawning task should later panic. +/// `scope_fifo()` returns once all spawned jobs have completed, and any +/// panics are propagated at that point. +pub fn scope_fifo<'scope, OP, R>(op: OP) -> R +where + OP: FnOnce(&ScopeFifo<'scope>) -> R + Send, + R: Send, +{ + in_worker(|owner_thread, _| { + let scope = ScopeFifo::<'scope>::new(Some(owner_thread), None); + scope.base.complete(Some(owner_thread), || op(&scope)) + }) +} + +/// Creates a "fork-join" scope `s` and invokes the closure with a +/// reference to `s`. This closure can then spawn asynchronous tasks +/// into `s`. Those tasks may run asynchronously with respect to the +/// closure; they may themselves spawn additional tasks into `s`. When +/// the closure returns, it will block until all tasks that have been +/// spawned into `s` complete. +/// +/// This is just like `scope()` except the closure runs on the same thread +/// that calls `in_place_scope()`. Only work that it spawns runs in the +/// thread pool. +/// +/// # Panics +/// +/// If a panic occurs, either in the closure given to `in_place_scope()` or in +/// any of the spawned jobs, that panic will be propagated and the +/// call to `in_place_scope()` will panic. If multiple panics occurs, it is +/// non-deterministic which of their panic values will propagate. +/// Regardless, once a task is spawned using `scope.spawn()`, it will +/// execute, even if the spawning task should later panic. `in_place_scope()` +/// returns once all spawned jobs have completed, and any panics are +/// propagated at that point. +pub fn in_place_scope<'scope, OP, R>(op: OP) -> R +where + OP: FnOnce(&Scope<'scope>) -> R, +{ + do_in_place_scope(None, op) +} + +pub(crate) fn do_in_place_scope<'scope, OP, R>(registry: Option<&Arc>, op: OP) -> R +where + OP: FnOnce(&Scope<'scope>) -> R, +{ + let (thread, registry) = get_in_place_thread_registry(registry); + let scope = Scope::<'scope>::new(thread, registry); + scope.base.complete(thread, || op(&scope)) +} + +fn get_in_place_thread_registry( + registry: Option<&Arc>, +) -> (Option<&WorkerThread>, Option<&Arc>) { + let thread = unsafe { WorkerThread::current().as_ref() }; + if thread.is_none() && registry.is_none() { + // A new global registry may use the current thread, especially on WebAssembly, + // so we have to re-check our current status after it's built. + let global = global_registry(); + (global.current_thread(), Some(global)) + } else { + (thread, registry) + } +} + +/// Creates a "fork-join" scope `s` with FIFO order, and invokes the +/// closure with a reference to `s`. This closure can then spawn +/// asynchronous tasks into `s`. Those tasks may run asynchronously with +/// respect to the closure; they may themselves spawn additional tasks +/// into `s`. When the closure returns, it will block until all tasks +/// that have been spawned into `s` complete. +/// +/// This is just like `scope_fifo()` except the closure runs on the same thread +/// that calls `in_place_scope_fifo()`. Only work that it spawns runs in the +/// thread pool. +/// +/// # Panics +/// +/// If a panic occurs, either in the closure given to `in_place_scope_fifo()` or in +/// any of the spawned jobs, that panic will be propagated and the +/// call to `in_place_scope_fifo()` will panic. If multiple panics occurs, it is +/// non-deterministic which of their panic values will propagate. +/// Regardless, once a task is spawned using `scope.spawn_fifo()`, it will +/// execute, even if the spawning task should later panic. `in_place_scope_fifo()` +/// returns once all spawned jobs have completed, and any panics are +/// propagated at that point. +pub fn in_place_scope_fifo<'scope, OP, R>(op: OP) -> R +where + OP: FnOnce(&ScopeFifo<'scope>) -> R, +{ + do_in_place_scope_fifo(None, op) +} + +pub(crate) fn do_in_place_scope_fifo<'scope, OP, R>(registry: Option<&Arc>, op: OP) -> R +where + OP: FnOnce(&ScopeFifo<'scope>) -> R, +{ + let (thread, registry) = get_in_place_thread_registry(registry); + let scope = ScopeFifo::<'scope>::new(thread, registry); + scope.base.complete(thread, || op(&scope)) +} + +impl<'scope> Scope<'scope> { + fn new(owner: Option<&WorkerThread>, registry: Option<&Arc>) -> Self { + let base = ScopeBase::new(owner, registry); + Scope { base } + } + + /// Spawns a job into the fork-join scope `self`. This job will + /// execute sometime before the fork-join scope completes. The + /// job is specified as a closure, and this closure receives its + /// own reference to the scope `self` as argument. This can be + /// used to inject new jobs into `self`. + /// + /// # Returns + /// + /// Nothing. The spawned closures cannot pass back values to the + /// caller directly, though they can write to local variables on + /// the stack (if those variables outlive the scope) or + /// communicate through shared channels. + /// + /// (The intention is to eventually integrate with Rust futures to + /// support spawns of functions that compute a value.) + /// + /// # Examples + /// + /// ```rust + /// # use rayon_core as rayon; + /// let mut value_a = None; + /// let mut value_b = None; + /// let mut value_c = None; + /// rayon::scope(|s| { + /// s.spawn(|s1| { + /// // ^ this is the same scope as `s`; this handle `s1` + /// // is intended for use by the spawned task, + /// // since scope handles cannot cross thread boundaries. + /// + /// value_a = Some(22); + /// + /// // the scope `s` will not end until all these tasks are done + /// s1.spawn(|_| { + /// value_b = Some(44); + /// }); + /// }); + /// + /// s.spawn(|_| { + /// value_c = Some(66); + /// }); + /// }); + /// assert_eq!(value_a, Some(22)); + /// assert_eq!(value_b, Some(44)); + /// assert_eq!(value_c, Some(66)); + /// ``` + /// + /// # See also + /// + /// The [`scope` function] has more extensive documentation about + /// task spawning. + /// + /// [`scope` function]: scope() + pub fn spawn(&self, body: BODY) + where + BODY: FnOnce(&Scope<'scope>) + Send + 'scope, + { + let scope_ptr = ScopePtr(self); + let job = HeapJob::new(move || unsafe { + // SAFETY: this job will execute before the scope ends. + let scope = scope_ptr.as_ref(); + ScopeBase::execute_job(&scope.base, move || body(scope)) + }); + let job_ref = self.base.heap_job_ref(job); + + // Since `Scope` implements `Sync`, we can't be sure that we're still in a + // thread of this pool, so we can't just push to the local worker thread. + // Also, this might be an in-place scope. + self.base.registry.inject_or_push(job_ref); + } + + /// Spawns a job into every thread of the fork-join scope `self`. This job will + /// execute on each thread sometime before the fork-join scope completes. The + /// job is specified as a closure, and this closure receives its own reference + /// to the scope `self` as argument, as well as a `BroadcastContext`. + pub fn spawn_broadcast(&self, body: BODY) + where + BODY: Fn(&Scope<'scope>, BroadcastContext<'_>) + Send + Sync + 'scope, + { + let scope_ptr = ScopePtr(self); + let job = ArcJob::new(move || unsafe { + // SAFETY: this job will execute before the scope ends. + let scope = scope_ptr.as_ref(); + let body = &body; + let func = move || BroadcastContext::with(move |ctx| body(scope, ctx)); + ScopeBase::execute_job(&scope.base, func) + }); + self.base.inject_broadcast(job) + } +} + +impl<'scope> ScopeFifo<'scope> { + fn new(owner: Option<&WorkerThread>, registry: Option<&Arc>) -> Self { + let base = ScopeBase::new(owner, registry); + let num_threads = base.registry.num_threads(); + let fifos = (0..num_threads).map(|_| JobFifo::new()).collect(); + ScopeFifo { base, fifos } + } + + /// Spawns a job into the fork-join scope `self`. This job will + /// execute sometime before the fork-join scope completes. The + /// job is specified as a closure, and this closure receives its + /// own reference to the scope `self` as argument. This can be + /// used to inject new jobs into `self`. + /// + /// # See also + /// + /// This method is akin to [`Scope::spawn()`], but with a FIFO + /// priority. The [`scope_fifo` function] has more details about + /// this distinction. + /// + /// [`scope_fifo` function]: scope_fifo() + pub fn spawn_fifo(&self, body: BODY) + where + BODY: FnOnce(&ScopeFifo<'scope>) + Send + 'scope, + { + let scope_ptr = ScopePtr(self); + let job = HeapJob::new(move || unsafe { + // SAFETY: this job will execute before the scope ends. + let scope = scope_ptr.as_ref(); + ScopeBase::execute_job(&scope.base, move || body(scope)) + }); + let job_ref = self.base.heap_job_ref(job); + + // If we're in the pool, use our scope's private fifo for this thread to execute + // in a locally-FIFO order. Otherwise, just use the pool's global injector. + match self.base.registry.current_thread() { + Some(worker) => { + let fifo = &self.fifos[worker.index()]; + // SAFETY: this job will execute before the scope ends. + unsafe { worker.push(fifo.push(job_ref)) }; + } + None => self.base.registry.inject(job_ref), + } + } + + /// Spawns a job into every thread of the fork-join scope `self`. This job will + /// execute on each thread sometime before the fork-join scope completes. The + /// job is specified as a closure, and this closure receives its own reference + /// to the scope `self` as argument, as well as a `BroadcastContext`. + pub fn spawn_broadcast(&self, body: BODY) + where + BODY: Fn(&ScopeFifo<'scope>, BroadcastContext<'_>) + Send + Sync + 'scope, + { + let scope_ptr = ScopePtr(self); + let job = ArcJob::new(move || unsafe { + // SAFETY: this job will execute before the scope ends. + let scope = scope_ptr.as_ref(); + let body = &body; + let func = move || BroadcastContext::with(move |ctx| body(scope, ctx)); + ScopeBase::execute_job(&scope.base, func) + }); + self.base.inject_broadcast(job) + } +} + +impl<'scope> ScopeBase<'scope> { + /// Creates the base of a new scope for the given registry + fn new(owner: Option<&WorkerThread>, registry: Option<&Arc>) -> Self { + let registry = registry.unwrap_or_else(|| match owner { + Some(owner) => owner.registry(), + None => global_registry(), + }); + + ScopeBase { + registry: Arc::clone(registry), + panic: AtomicPtr::new(ptr::null_mut()), + job_completed_latch: CountLatch::new(owner), + marker: PhantomData, + } + } + + fn heap_job_ref(&self, job: Box>) -> JobRef + where + FUNC: FnOnce() + Send + 'scope, + { + unsafe { + self.job_completed_latch.increment(); + job.into_job_ref() + } + } + + fn inject_broadcast(&self, job: Arc>) + where + FUNC: Fn() + Send + Sync + 'scope, + { + let n_threads = self.registry.num_threads(); + let job_refs = (0..n_threads).map(|_| unsafe { + self.job_completed_latch.increment(); + ArcJob::as_job_ref(&job) + }); + + self.registry.inject_broadcast(job_refs); + } + + /// Executes `func` as a job, either aborting or executing as + /// appropriate. + fn complete(&self, owner: Option<&WorkerThread>, func: FUNC) -> R + where + FUNC: FnOnce() -> R, + { + let result = unsafe { Self::execute_job_closure(self, func) }; + self.job_completed_latch.wait(owner); + self.maybe_propagate_panic(); + result.unwrap() // only None if `op` panicked, and that would have been propagated + } + + /// Executes `func` as a job, either aborting or executing as + /// appropriate. + unsafe fn execute_job(this: *const Self, func: FUNC) + where + FUNC: FnOnce(), + { + let _: Option<()> = Self::execute_job_closure(this, func); + } + + /// Executes `func` as a job in scope. Adjusts the "job completed" + /// counters and also catches any panic and stores it into + /// `scope`. + unsafe fn execute_job_closure(this: *const Self, func: FUNC) -> Option + where + FUNC: FnOnce() -> R, + { + let result = match unwind::halt_unwinding(func) { + Ok(r) => Some(r), + Err(err) => { + (*this).job_panicked(err); + None + } + }; + Latch::set(&(*this).job_completed_latch); + result + } + + fn job_panicked(&self, err: Box) { + // capture the first error we see, free the rest + if self.panic.load(Ordering::Relaxed).is_null() { + let nil = ptr::null_mut(); + let mut err = ManuallyDrop::new(Box::new(err)); // box up the fat ptr + let err_ptr: *mut Box = &mut **err; + if self + .panic + .compare_exchange(nil, err_ptr, Ordering::Release, Ordering::Relaxed) + .is_ok() + { + // ownership now transferred into self.panic + } else { + // another panic raced in ahead of us, so drop ours + let _: Box> = ManuallyDrop::into_inner(err); + } + } + } + + fn maybe_propagate_panic(&self) { + // propagate panic, if any occurred; at this point, all + // outstanding jobs have completed, so we can use a relaxed + // ordering: + let panic = self.panic.swap(ptr::null_mut(), Ordering::Relaxed); + if !panic.is_null() { + let value = unsafe { Box::from_raw(panic) }; + unwind::resume_unwinding(*value); + } + } +} + +impl<'scope> fmt::Debug for Scope<'scope> { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.debug_struct("Scope") + .field("pool_id", &self.base.registry.id()) + .field("panic", &self.base.panic) + .field("job_completed_latch", &self.base.job_completed_latch) + .finish() + } +} + +impl<'scope> fmt::Debug for ScopeFifo<'scope> { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.debug_struct("ScopeFifo") + .field("num_fifos", &self.fifos.len()) + .field("pool_id", &self.base.registry.id()) + .field("panic", &self.base.panic) + .field("job_completed_latch", &self.base.job_completed_latch) + .finish() + } +} + +/// Used to capture a scope `&Self` pointer in jobs, without faking a lifetime. +/// +/// Unsafe code is still required to dereference the pointer, but that's fine in +/// scope jobs that are guaranteed to execute before the scope ends. +struct ScopePtr(*const T); + +// SAFETY: !Send for raw pointers is not for safety, just as a lint +unsafe impl Send for ScopePtr {} + +// SAFETY: !Sync for raw pointers is not for safety, just as a lint +unsafe impl Sync for ScopePtr {} + +impl ScopePtr { + // Helper to avoid disjoint captures of `scope_ptr.0` + unsafe fn as_ref(&self) -> &T { + &*self.0 + } +} diff --git a/anneal/v2/vendor/rayon-core/src/scope/test.rs b/anneal/v2/vendor/rayon-core/src/scope/test.rs new file mode 100644 index 0000000000..a563205899 --- /dev/null +++ b/anneal/v2/vendor/rayon-core/src/scope/test.rs @@ -0,0 +1,621 @@ +use crate::unwind; +use crate::ThreadPoolBuilder; +use crate::{scope, scope_fifo, Scope, ScopeFifo}; +use rand::{Rng, SeedableRng}; +use rand_xorshift::XorShiftRng; +use std::iter::once; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Barrier, Mutex}; +use std::vec; + +#[test] +fn scope_empty() { + scope(|_| {}); +} + +#[test] +fn scope_result() { + let x = scope(|_| 22); + assert_eq!(x, 22); +} + +#[test] +fn scope_two() { + let counter = &AtomicUsize::new(0); + scope(|s| { + s.spawn(move |_| { + counter.fetch_add(1, Ordering::SeqCst); + }); + s.spawn(move |_| { + counter.fetch_add(10, Ordering::SeqCst); + }); + }); + + let v = counter.load(Ordering::SeqCst); + assert_eq!(v, 11); +} + +#[test] +fn scope_divide_and_conquer() { + let counter_p = &AtomicUsize::new(0); + scope(|s| s.spawn(move |s| divide_and_conquer(s, counter_p, 1024))); + + let counter_s = &AtomicUsize::new(0); + divide_and_conquer_seq(counter_s, 1024); + + let p = counter_p.load(Ordering::SeqCst); + let s = counter_s.load(Ordering::SeqCst); + assert_eq!(p, s); +} + +fn divide_and_conquer<'scope>(scope: &Scope<'scope>, counter: &'scope AtomicUsize, size: usize) { + if size > 1 { + scope.spawn(move |scope| divide_and_conquer(scope, counter, size / 2)); + scope.spawn(move |scope| divide_and_conquer(scope, counter, size / 2)); + } else { + // count the leaves + counter.fetch_add(1, Ordering::SeqCst); + } +} + +fn divide_and_conquer_seq(counter: &AtomicUsize, size: usize) { + if size > 1 { + divide_and_conquer_seq(counter, size / 2); + divide_and_conquer_seq(counter, size / 2); + } else { + // count the leaves + counter.fetch_add(1, Ordering::SeqCst); + } +} + +struct Tree { + value: T, + children: Vec>, +} + +impl Tree { + fn iter(&self) -> vec::IntoIter<&T> { + once(&self.value) + .chain(self.children.iter().flat_map(Tree::iter)) + .collect::>() // seems like it shouldn't be needed... but prevents overflow + .into_iter() + } + + fn update(&mut self, op: OP) + where + OP: Fn(&mut T) + Sync, + T: Send, + { + scope(|s| self.update_in_scope(&op, s)); + } + + fn update_in_scope<'scope, OP>(&'scope mut self, op: &'scope OP, scope: &Scope<'scope>) + where + OP: Fn(&mut T) + Sync, + { + let Tree { + ref mut value, + ref mut children, + } = *self; + scope.spawn(move |scope| { + for child in children { + scope.spawn(move |scope| child.update_in_scope(op, scope)); + } + }); + + op(value); + } +} + +fn random_tree(depth: usize) -> Tree { + assert!(depth > 0); + let mut seed = ::Seed::default(); + (0..).zip(seed.as_mut()).for_each(|(i, x)| *x = i); + let mut rng = XorShiftRng::from_seed(seed); + random_tree1(depth, &mut rng) +} + +fn random_tree1(depth: usize, rng: &mut XorShiftRng) -> Tree { + let children = if depth == 0 { + vec![] + } else { + (0..rng.random_range(0..4)) // somewhere between 0 and 3 children at each level + .map(|_| random_tree1(depth - 1, rng)) + .collect() + }; + + Tree { + value: rng.random_range(0..1_000_000), + children, + } +} + +#[test] +fn update_tree() { + let mut tree: Tree = random_tree(10); + let values: Vec = tree.iter().cloned().collect(); + tree.update(|v| *v += 1); + let new_values: Vec = tree.iter().cloned().collect(); + assert_eq!(values.len(), new_values.len()); + for (&i, &j) in values.iter().zip(&new_values) { + assert_eq!(i + 1, j); + } +} + +/// Check that if you have a chain of scoped tasks where T0 spawns T1 +/// spawns T2 and so forth down to Tn, the stack space should not grow +/// linearly with N. We test this by some unsafe hackery and +/// permitting an approx 10% change with a 10x input change. +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn linear_stack_growth() { + let builder = ThreadPoolBuilder::new().num_threads(1); + let pool = builder.build().unwrap(); + pool.install(|| { + let mut max_diff = Mutex::new(0); + let bottom_of_stack = 0; + scope(|s| the_final_countdown(s, &bottom_of_stack, &max_diff, 5)); + let diff_when_5 = *max_diff.get_mut().unwrap() as f64; + + scope(|s| the_final_countdown(s, &bottom_of_stack, &max_diff, 500)); + let diff_when_500 = *max_diff.get_mut().unwrap() as f64; + + let ratio = diff_when_5 / diff_when_500; + assert!( + ratio > 0.9 && ratio < 1.1, + "stack usage ratio out of bounds: {ratio}" + ); + }); +} + +fn the_final_countdown<'scope>( + s: &Scope<'scope>, + bottom_of_stack: &'scope i32, + max: &'scope Mutex, + n: usize, +) { + let top_of_stack = 0; + let p = bottom_of_stack as *const i32 as usize; + let q = &top_of_stack as *const i32 as usize; + let diff = p.abs_diff(q); + + let mut data = max.lock().unwrap(); + *data = Ord::max(diff, *data); + + if n > 0 { + s.spawn(move |s| the_final_countdown(s, bottom_of_stack, max, n - 1)); + } +} + +#[test] +#[should_panic(expected = "Hello, world!")] +fn panic_propagate_scope() { + scope(|_| panic!("Hello, world!")); +} + +#[test] +#[should_panic(expected = "Hello, world!")] +fn panic_propagate_spawn() { + scope(|s| s.spawn(|_| panic!("Hello, world!"))); +} + +#[test] +#[should_panic(expected = "Hello, world!")] +fn panic_propagate_nested_spawn() { + scope(|s| s.spawn(|s| s.spawn(|s| s.spawn(|_| panic!("Hello, world!"))))); +} + +#[test] +#[should_panic(expected = "Hello, world!")] +fn panic_propagate_nested_scope_spawn() { + scope(|s| s.spawn(|_| scope(|s| s.spawn(|_| panic!("Hello, world!"))))); +} + +#[test] +#[cfg_attr(not(panic = "unwind"), ignore)] +fn panic_propagate_still_execute_1() { + let mut x = false; + let result = unwind::halt_unwinding(|| { + scope(|s| { + s.spawn(|_| panic!("Hello, world!")); // job A + s.spawn(|_| x = true); // job B, should still execute even though A panics + }); + }); + match result { + Ok(_) => panic!("failed to propagate panic"), + Err(_) => assert!(x, "job b failed to execute"), + } +} + +#[test] +#[cfg_attr(not(panic = "unwind"), ignore)] +fn panic_propagate_still_execute_2() { + let mut x = false; + let result = unwind::halt_unwinding(|| { + scope(|s| { + s.spawn(|_| x = true); // job B, should still execute even though A panics + s.spawn(|_| panic!("Hello, world!")); // job A + }); + }); + match result { + Ok(_) => panic!("failed to propagate panic"), + Err(_) => assert!(x, "job b failed to execute"), + } +} + +#[test] +#[cfg_attr(not(panic = "unwind"), ignore)] +fn panic_propagate_still_execute_3() { + let mut x = false; + let result = unwind::halt_unwinding(|| { + scope(|s| { + s.spawn(|_| x = true); // spawned job should still execute despite later panic + panic!("Hello, world!"); + }); + }); + match result { + Ok(_) => panic!("failed to propagate panic"), + Err(_) => assert!(x, "panic after spawn, spawn failed to execute"), + } +} + +#[test] +#[cfg_attr(not(panic = "unwind"), ignore)] +fn panic_propagate_still_execute_4() { + let mut x = false; + let result = unwind::halt_unwinding(|| { + scope(|s| { + s.spawn(|_| panic!("Hello, world!")); + x = true; + }); + }); + match result { + Ok(_) => panic!("failed to propagate panic"), + Err(_) => assert!(x, "panic in spawn tainted scope"), + } +} + +macro_rules! test_order { + ($scope:ident => $spawn:ident) => {{ + let builder = ThreadPoolBuilder::new().num_threads(1); + let pool = builder.build().unwrap(); + pool.install(|| { + let vec = Mutex::new(vec![]); + $scope(|scope| { + let vec = &vec; + for i in 0..10 { + scope.$spawn(move |scope| { + for j in 0..10 { + scope.$spawn(move |_| { + vec.lock().unwrap().push(i * 10 + j); + }); + } + }); + } + }); + vec.into_inner().unwrap() + }) + }}; +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn lifo_order() { + // In the absence of stealing, `scope()` runs its `spawn()` jobs in LIFO order. + let vec = test_order!(scope => spawn); + let expected: Vec = (0..100).rev().collect(); // LIFO -> reversed + assert_eq!(vec, expected); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn fifo_order() { + // In the absence of stealing, `scope_fifo()` runs its `spawn_fifo()` jobs in FIFO order. + let vec = test_order!(scope_fifo => spawn_fifo); + let expected: Vec = (0..100).collect(); // FIFO -> natural order + assert_eq!(vec, expected); +} + +macro_rules! test_nested_order { + ($outer_scope:ident => $outer_spawn:ident, + $inner_scope:ident => $inner_spawn:ident) => {{ + let builder = ThreadPoolBuilder::new().num_threads(1); + let pool = builder.build().unwrap(); + pool.install(|| { + let vec = Mutex::new(vec![]); + $outer_scope(|scope| { + let vec = &vec; + for i in 0..10 { + scope.$outer_spawn(move |_| { + $inner_scope(|scope| { + for j in 0..10 { + scope.$inner_spawn(move |_| { + vec.lock().unwrap().push(i * 10 + j); + }); + } + }); + }); + } + }); + vec.into_inner().unwrap() + }) + }}; +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn nested_lifo_order() { + // In the absence of stealing, `scope()` runs its `spawn()` jobs in LIFO order. + let vec = test_nested_order!(scope => spawn, scope => spawn); + let expected: Vec = (0..100).rev().collect(); // LIFO -> reversed + assert_eq!(vec, expected); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn nested_fifo_order() { + // In the absence of stealing, `scope_fifo()` runs its `spawn_fifo()` jobs in FIFO order. + let vec = test_nested_order!(scope_fifo => spawn_fifo, scope_fifo => spawn_fifo); + let expected: Vec = (0..100).collect(); // FIFO -> natural order + assert_eq!(vec, expected); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn nested_lifo_fifo_order() { + // LIFO on the outside, FIFO on the inside + let vec = test_nested_order!(scope => spawn, scope_fifo => spawn_fifo); + let expected: Vec = (0..10) + .rev() + .flat_map(|i| (0..10).map(move |j| i * 10 + j)) + .collect(); + assert_eq!(vec, expected); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn nested_fifo_lifo_order() { + // FIFO on the outside, LIFO on the inside + let vec = test_nested_order!(scope_fifo => spawn_fifo, scope => spawn); + let expected: Vec = (0..10) + .flat_map(|i| (0..10).rev().map(move |j| i * 10 + j)) + .collect(); + assert_eq!(vec, expected); +} + +macro_rules! spawn_push { + ($scope:ident . $spawn:ident, $vec:ident, $i:expr) => {{ + $scope.$spawn(move |_| $vec.lock().unwrap().push($i)); + }}; +} + +/// Test spawns pushing a series of numbers, interleaved +/// such that negative values are using an inner scope. +macro_rules! test_mixed_order { + ($outer_scope:ident => $outer_spawn:ident, + $inner_scope:ident => $inner_spawn:ident) => {{ + let builder = ThreadPoolBuilder::new().num_threads(1); + let pool = builder.build().unwrap(); + pool.install(|| { + let vec = Mutex::new(vec![]); + $outer_scope(|outer_scope| { + let vec = &vec; + spawn_push!(outer_scope.$outer_spawn, vec, 0); + $inner_scope(|inner_scope| { + spawn_push!(inner_scope.$inner_spawn, vec, -1); + spawn_push!(outer_scope.$outer_spawn, vec, 1); + spawn_push!(inner_scope.$inner_spawn, vec, -2); + spawn_push!(outer_scope.$outer_spawn, vec, 2); + spawn_push!(inner_scope.$inner_spawn, vec, -3); + }); + spawn_push!(outer_scope.$outer_spawn, vec, 3); + }); + vec.into_inner().unwrap() + }) + }}; +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn mixed_lifo_order() { + // NB: the end of the inner scope makes us execute some of the outer scope + // before they've all been spawned, so they're not perfectly LIFO. + let vec = test_mixed_order!(scope => spawn, scope => spawn); + let expected = vec![-3, 2, -2, 1, -1, 3, 0]; + assert_eq!(vec, expected); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn mixed_fifo_order() { + let vec = test_mixed_order!(scope_fifo => spawn_fifo, scope_fifo => spawn_fifo); + let expected = vec![-1, 0, -2, 1, -3, 2, 3]; + assert_eq!(vec, expected); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn mixed_lifo_fifo_order() { + // NB: the end of the inner scope makes us execute some of the outer scope + // before they've all been spawned, so they're not perfectly LIFO. + let vec = test_mixed_order!(scope => spawn, scope_fifo => spawn_fifo); + let expected = vec![-1, 2, -2, 1, -3, 3, 0]; + assert_eq!(vec, expected); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn mixed_fifo_lifo_order() { + let vec = test_mixed_order!(scope_fifo => spawn_fifo, scope => spawn); + let expected = vec![-3, 0, -2, 1, -1, 2, 3]; + assert_eq!(vec, expected); +} + +#[test] +fn static_scope() { + static COUNTER: AtomicUsize = AtomicUsize::new(0); + + let mut range = 0..100; + let sum = range.clone().sum(); + let iter = &mut range; + + COUNTER.store(0, Ordering::Relaxed); + scope(|s: &Scope<'static>| { + // While we're allowed the locally borrowed iterator, + // the spawns must be static. + for i in iter { + s.spawn(move |_| { + COUNTER.fetch_add(i, Ordering::Relaxed); + }); + } + }); + + assert_eq!(COUNTER.load(Ordering::Relaxed), sum); +} + +#[test] +fn static_scope_fifo() { + static COUNTER: AtomicUsize = AtomicUsize::new(0); + + let mut range = 0..100; + let sum = range.clone().sum(); + let iter = &mut range; + + COUNTER.store(0, Ordering::Relaxed); + scope_fifo(|s: &ScopeFifo<'static>| { + // While we're allowed the locally borrowed iterator, + // the spawns must be static. + for i in iter { + s.spawn_fifo(move |_| { + COUNTER.fetch_add(i, Ordering::Relaxed); + }); + } + }); + + assert_eq!(COUNTER.load(Ordering::Relaxed), sum); +} + +#[test] +fn mixed_lifetime_scope() { + fn increment<'slice, 'counter>(counters: &'slice [&'counter AtomicUsize]) { + scope(move |s: &Scope<'counter>| { + // We can borrow 'slice here, but the spawns can only borrow 'counter. + for &c in counters { + s.spawn(move |_| { + c.fetch_add(1, Ordering::Relaxed); + }); + } + }); + } + + let counter = AtomicUsize::new(0); + increment(&[&counter; 100]); + assert_eq!(counter.into_inner(), 100); +} + +#[test] +fn mixed_lifetime_scope_fifo() { + fn increment<'slice, 'counter>(counters: &'slice [&'counter AtomicUsize]) { + scope_fifo(move |s: &ScopeFifo<'counter>| { + // We can borrow 'slice here, but the spawns can only borrow 'counter. + for &c in counters { + s.spawn_fifo(move |_| { + c.fetch_add(1, Ordering::Relaxed); + }); + } + }); + } + + let counter = AtomicUsize::new(0); + increment(&[&counter; 100]); + assert_eq!(counter.into_inner(), 100); +} + +#[test] +fn scope_spawn_broadcast() { + let sum = AtomicUsize::new(0); + let n = scope(|s| { + s.spawn_broadcast(|_, ctx| { + sum.fetch_add(ctx.index(), Ordering::Relaxed); + }); + crate::current_num_threads() + }); + assert_eq!(sum.into_inner(), n * (n - 1) / 2); +} + +#[test] +fn scope_fifo_spawn_broadcast() { + let sum = AtomicUsize::new(0); + let n = scope_fifo(|s| { + s.spawn_broadcast(|_, ctx| { + sum.fetch_add(ctx.index(), Ordering::Relaxed); + }); + crate::current_num_threads() + }); + assert_eq!(sum.into_inner(), n * (n - 1) / 2); +} + +#[test] +fn scope_spawn_broadcast_nested() { + let sum = AtomicUsize::new(0); + let n = scope(|s| { + s.spawn_broadcast(|s, _| { + s.spawn_broadcast(|_, ctx| { + sum.fetch_add(ctx.index(), Ordering::Relaxed); + }); + }); + crate::current_num_threads() + }); + assert_eq!(sum.into_inner(), n * n * (n - 1) / 2); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn scope_spawn_broadcast_barrier() { + let barrier = Barrier::new(8); + let pool = ThreadPoolBuilder::new().num_threads(7).build().unwrap(); + pool.in_place_scope(|s| { + s.spawn_broadcast(|_, _| { + barrier.wait(); + }); + barrier.wait(); + }); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn scope_spawn_broadcast_panic_one() { + let count = AtomicUsize::new(0); + let pool = ThreadPoolBuilder::new().num_threads(7).build().unwrap(); + let result = crate::unwind::halt_unwinding(|| { + pool.scope(|s| { + s.spawn_broadcast(|_, ctx| { + count.fetch_add(1, Ordering::Relaxed); + if ctx.index() == 3 { + panic!("Hello, world!"); + } + }); + }); + }); + assert_eq!(count.into_inner(), 7); + assert!(result.is_err(), "broadcast panic should propagate!"); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn scope_spawn_broadcast_panic_many() { + let count = AtomicUsize::new(0); + let pool = ThreadPoolBuilder::new().num_threads(7).build().unwrap(); + let result = crate::unwind::halt_unwinding(|| { + pool.scope(|s| { + s.spawn_broadcast(|_, ctx| { + count.fetch_add(1, Ordering::Relaxed); + if ctx.index() % 2 == 0 { + panic!("Hello, world!"); + } + }); + }); + }); + assert_eq!(count.into_inner(), 7); + assert!(result.is_err(), "broadcast panic should propagate!"); +} diff --git a/anneal/v2/vendor/rayon-core/src/sleep/README.md b/anneal/v2/vendor/rayon-core/src/sleep/README.md new file mode 100644 index 0000000000..fd650f8ad1 --- /dev/null +++ b/anneal/v2/vendor/rayon-core/src/sleep/README.md @@ -0,0 +1,219 @@ +# Introduction: the sleep module + +The code in this module governs when worker threads should go to +sleep. The system used in this code was introduced in [Rayon RFC #5]. +There is also a [video walkthrough] available. Both of those may be +valuable resources to understanding the code, though naturally they +will also grow stale over time. The comments in this file are +extracted from the RFC and meant to be kept up to date. + +[Rayon RFC #5]: https://github.com/rayon-rs/rfcs/pull/5 +[video walkthrough]: https://youtu.be/HvmQsE5M4cY + +# The `Sleep` struct + +The `Sleep` struct is embedded into each registry. It performs several functions: + +* It tracks when workers are awake or asleep. +* It decides how long a worker should look for work before it goes to sleep, + via a callback that is invoked periodically from the worker's search loop. +* It is notified when latches are set, jobs are published, or other + events occur, and it will go and wake the appropriate threads if + they are sleeping. + +# Thread states + +There are three main thread states: + +* An **active** thread is one that is actively executing a job. +* An **idle** thread is one that is searching for work to do. It will be + trying to steal work or pop work from the global injector queue. +* A **sleeping** thread is one that is blocked on a condition variable, + waiting to be awoken. + +We sometimes refer to the final two states collectively as **inactive**. +Threads begin as idle but transition to idle and finally sleeping when +they're unable to find work to do. + +## Sleepy threads + +There is one other special state worth mentioning. During the idle state, +threads can get **sleepy**. A sleepy thread is still idle, in that it is still +searching for work, but it is *about* to go to sleep after it does one more +search (or some other number, potentially). When a thread enters the sleepy +state, it signals (via the **jobs event counter**, described below) that it is +about to go to sleep. If new work is published, this will lead to the counter +being adjusted. When the thread actually goes to sleep, it will (hopefully, but +not guaranteed) see that the counter has changed and elect not to sleep, but +instead to search again. See the section on the **jobs event counter** for more +details. + +# The counters + +One of the key structs in the sleep module is `AtomicCounters`, found in +`counters.rs`. It packs three counters into one atomically managed value: + +* Two **thread counters**, which track the number of threads in a particular state. +* The **jobs event counter**, which is used to signal when new work is available. + It (sort of) tracks the number of jobs posted, but not quite, and it can rollover. + +## Thread counters + +There are two thread counters, one that tracks **inactive** threads and one that +tracks **sleeping** threads. From this, one can deduce the number of threads +that are idle by subtracting sleeping threads from inactive threads. We track +the counters in this way because it permits simpler atomic operations. One can +increment the number of sleeping threads (and thus decrease the number of idle +threads) simply by doing one atomic increment, for example. Similarly, one can +decrease the number of sleeping threads (and increase the number of idle +threads) through one atomic decrement. + +These counters are adjusted as follows: + +* When a thread enters the idle state: increment the inactive thread counter. +* When a thread enters the sleeping state: increment the sleeping thread counter. +* When a thread awakens a sleeping thread: decrement the sleeping thread counter. + * Subtle point: the thread that *awakens* the sleeping thread decrements the + counter, not the thread that is *sleeping*. This is because there is a delay + between signaling a thread to wake and the thread actually waking: + decrementing the counter when awakening the thread means that other threads + that may be posting work will see the up-to-date value that much faster. +* When a thread finds work, exiting the idle state: decrement the inactive + thread counter. + +## Jobs event counter + +The final counter is the **jobs event counter**. The role of this counter is to +help sleepy threads detect when new work is posted in a lightweight fashion. In +its simplest form, we would simply have a counter that gets incremented each +time a new job is posted. This way, when a thread gets sleepy, it could read the +counter, and then compare to see if the value has changed before it actually +goes to sleep. But this [turns out to be too expensive] in practice, so we use a +somewhat more complex scheme. + +[turns out to be too expensive]: https://github.com/rayon-rs/rayon/pull/746#issuecomment-624802747 + +The idea is that the counter toggles between two states, depending on whether +its value is even or odd (or, equivalently, on the value of its low bit): + +* Even -- If the low bit is zero, then it means that there has been no new work + since the last thread got sleepy. +* Odd -- If the low bit is one, then it means that new work was posted since + the last thread got sleepy. + +### New work is posted + +When new work is posted, we check the value of the counter: if it is even, +then we increment it by one, so that it becomes odd. + +### Worker thread gets sleepy + +When a worker thread gets sleepy, it will read the value of the counter. If the +counter is odd, it will increment the counter so that it is even. Either way, it +remembers the final value of the counter. The final value will be used later, +when the thread is going to sleep. If at that time the counter has not changed, +then we can assume no new jobs have been posted (though note the remote +possibility of rollover, discussed in detail below). + +# Protocol for a worker thread to post work + +The full protocol for a thread to post work is as follows + +* If the work is posted into the injection queue, then execute a seq-cst fence (see below). +* Load the counters, incrementing the JEC if it is even so that it is odd. +* Check if there are idle threads available to handle this new job. If not, + and there are sleeping threads, then wake one or more threads. + +# Protocol for a worker thread to fall asleep + +The full protocol for a thread to fall asleep is as follows: + +* After completing all its jobs, the worker goes idle and begins to + search for work. As it searches, it counts "rounds". In each round, + it searches all other work threads' queues, plus the 'injector queue' for + work injected from the outside. If work is found in this search, the thread + becomes active again and hence restarts this protocol from the top. +* After a certain number of rounds, the thread "gets sleepy" and executes `get_sleepy` + above, remembering the `final_value` of the JEC. It does one more search for work. +* If no work is found, the thread atomically: + * Checks the JEC to see that it has not changed from `final_value`. + * If it has, then the thread goes back to searching for work. We reset to + just before we got sleepy, so that we will do one more search + before attempting to sleep again (rather than searching for many rounds). + * Increments the number of sleeping threads by 1. +* The thread then executes a seq-cst fence operation (see below). +* The thread then does one final check for injected jobs (see below). If any + are available, it returns to the 'pre-sleepy' state as if the JEC had changed. +* The thread waits to be signaled. Once signaled, it returns to the idle state. + +# The jobs event counter and deadlock + +As described in the section on the JEC, the main concern around going to sleep +is avoiding a race condition wherein: + +* Thread A looks for work, finds none. +* Thread B posts work but sees no sleeping threads. +* Thread A goes to sleep. + +The JEC protocol largely prevents this, but due to rollover, this prevention is +not complete. It is possible -- if unlikely -- that enough activity occurs for +Thread A to observe the same JEC value that it saw when getting sleepy. If the +new work being published came from *inside* the thread pool, then this race +condition isn't too harmful. It means that we have fewer workers processing the +work then we should, but we won't deadlock. This seems like an acceptable risk +given that this is unlikely in practice. + +However, if the work was posted as an *external* job, that is a problem. In that +case, it's possible that all of our workers could go to sleep, and the external +job would never get processed. To prevent that, the sleeping protocol includes +one final check to see if the injector queue is empty before fully falling +asleep. Note that this final check occurs **after** the number of sleeping +threads has been incremented. We are not concerned therefore with races against +injections that occur after that increment, only before. + +Unfortunately, there is one rather subtle point concerning this final check: +we wish to avoid the possibility that: + +* work is pushed into the injection queue by an outside thread X, +* the sleepy thread S sees the JEC but it has rolled over and is equal +* the sleepy thread S reads the injection queue but does not see the work posted by X. + +This is possible because the C++ memory model typically offers guarantees of the +form "if you see the access A, then you must see those other accesses" -- but it +doesn't guarantee that you will see the access A (i.e., if you think of +processors with independent caches, you may be operating on very out of date +cache state). + +## Using seq-cst fences to prevent deadlock + +To overcome this problem, we have inserted two sequentially consistent fence +operations into the protocols above: + +* One fence occurs after work is posted into the injection queue, but before the + counters are read (including the number of sleeping threads). + * Note that no fence is needed for work posted to internal queues, since it is ok + to overlook work in that case. +* One fence occurs after the number of sleeping threads is incremented, but + before the injection queue is read. + +### Proof sketch + +What follows is a "proof sketch" that the protocol is deadlock free. We model +two relevant bits of memory, the job injector queue J and the atomic counters C. + +Consider the actions of the injecting thread: + +* PushJob: Job is injected, which can be modeled as an atomic write to J with release semantics. +* PushFence: A sequentially consistent fence is executed. +* ReadSleepers: The counters C are read (they may also be incremented, but we just consider the read that comes first). + +Meanwhile, the sleepy thread does the following: + +* IncSleepers: The number of sleeping threads is incremented, which is atomic exchange to C. +* SleepFence: A sequentially consistent fence is executed. +* ReadJob: We look to see if the queue is empty, which is a read of J with acquire semantics. + +Either PushFence or SleepFence must come first: + +* If PushFence comes first, then PushJob must be visible to ReadJob. +* If SleepFence comes first, then IncSleepers is visible to ReadSleepers. diff --git a/anneal/v2/vendor/rayon-core/src/sleep/counters.rs b/anneal/v2/vendor/rayon-core/src/sleep/counters.rs new file mode 100644 index 0000000000..0100518be8 --- /dev/null +++ b/anneal/v2/vendor/rayon-core/src/sleep/counters.rs @@ -0,0 +1,273 @@ +use std::sync::atomic::{AtomicUsize, Ordering}; + +pub(super) struct AtomicCounters { + /// Packs together a number of counters. The counters are ordered as + /// follows, from least to most significant bits (here, we assuming + /// that [`THREADS_BITS`] is equal to 10): + /// + /// * Bits 0..10: Stores the number of **sleeping threads** + /// * Bits 10..20: Stores the number of **inactive threads** + /// * Bits 20..: Stores the **job event counter** (JEC) + /// + /// This uses 10 bits ([`THREADS_BITS`]) to encode the number of threads. Note + /// that the total number of bits (and hence the number of bits used for the + /// JEC) will depend on whether we are using a 32- or 64-bit architecture. + value: AtomicUsize, +} + +#[derive(Copy, Clone)] +pub(super) struct Counters { + word: usize, +} + +/// A value read from the **Jobs Event Counter**. +/// See the [`README.md`](README.md) for more +/// coverage of how the jobs event counter works. +#[derive(Copy, Clone, Debug, PartialEq, PartialOrd)] +pub(super) struct JobsEventCounter(usize); + +impl JobsEventCounter { + pub(super) const DUMMY: JobsEventCounter = JobsEventCounter(usize::MAX); + + #[inline] + pub(super) fn as_usize(self) -> usize { + self.0 + } + + /// The JEC "is sleepy" if the last thread to increment it was in the + /// process of becoming sleepy. This is indicated by its value being *even*. + /// When new jobs are posted, they check if the JEC is sleepy, and if so + /// they incremented it. + #[inline] + pub(super) fn is_sleepy(self) -> bool { + (self.as_usize() & 1) == 0 + } + + /// The JEC "is active" if the last thread to increment it was posting new + /// work. This is indicated by its value being *odd*. When threads get + /// sleepy, they will check if the JEC is active, and increment it. + #[inline] + pub(super) fn is_active(self) -> bool { + !self.is_sleepy() + } +} + +/// Number of bits used for the thread counters. +#[cfg(target_pointer_width = "64")] +const THREADS_BITS: usize = 16; + +#[cfg(target_pointer_width = "32")] +const THREADS_BITS: usize = 8; + +/// Bits to shift to select the sleeping threads +/// (used with `select_bits`). +#[allow(clippy::erasing_op)] +const SLEEPING_SHIFT: usize = 0 * THREADS_BITS; + +/// Bits to shift to select the inactive threads +/// (used with `select_bits`). +#[allow(clippy::identity_op)] +const INACTIVE_SHIFT: usize = 1 * THREADS_BITS; + +/// Bits to shift to select the JEC +/// (use JOBS_BITS). +const JEC_SHIFT: usize = 2 * THREADS_BITS; + +/// Max value for the thread counters. +pub(crate) const THREADS_MAX: usize = (1 << THREADS_BITS) - 1; + +/// Constant that can be added to add one sleeping thread. +const ONE_SLEEPING: usize = 1; + +/// Constant that can be added to add one inactive thread. +/// An inactive thread is either idle, sleepy, or sleeping. +const ONE_INACTIVE: usize = 1 << INACTIVE_SHIFT; + +/// Constant that can be added to add one to the JEC. +const ONE_JEC: usize = 1 << JEC_SHIFT; + +impl AtomicCounters { + #[inline] + pub(super) fn new() -> AtomicCounters { + AtomicCounters { + value: AtomicUsize::new(0), + } + } + + /// Load and return the current value of the various counters. + /// This value can then be given to other method which will + /// attempt to update the counters via compare-and-swap. + #[inline] + pub(super) fn load(&self, ordering: Ordering) -> Counters { + Counters::new(self.value.load(ordering)) + } + + #[inline] + fn try_exchange(&self, old_value: Counters, new_value: Counters, ordering: Ordering) -> bool { + self.value + .compare_exchange(old_value.word, new_value.word, ordering, Ordering::Relaxed) + .is_ok() + } + + /// Adds an inactive thread. This cannot fail. + /// + /// This should be invoked when a thread enters its idle loop looking + /// for work. It is decremented when work is found. Note that it is + /// not decremented if the thread transitions from idle to sleepy or sleeping; + /// so the number of inactive threads is always greater-than-or-equal + /// to the number of sleeping threads. + #[inline] + pub(super) fn add_inactive_thread(&self) { + self.value.fetch_add(ONE_INACTIVE, Ordering::SeqCst); + } + + /// Increments the jobs event counter if `increment_when`, when applied to + /// the current value, is true. Used to toggle the JEC from even (sleepy) to + /// odd (active) or vice versa. Returns the final value of the counters, for + /// which `increment_when` is guaranteed to return false. + pub(super) fn increment_jobs_event_counter_if( + &self, + increment_when: impl Fn(JobsEventCounter) -> bool, + ) -> Counters { + loop { + let old_value = self.load(Ordering::SeqCst); + if increment_when(old_value.jobs_counter()) { + let new_value = old_value.increment_jobs_counter(); + if self.try_exchange(old_value, new_value, Ordering::SeqCst) { + return new_value; + } + } else { + return old_value; + } + } + } + + /// Subtracts an inactive thread. This cannot fail. It is invoked + /// when a thread finds work and hence becomes active. It returns the + /// number of sleeping threads to wake up (if any). + /// + /// See `add_inactive_thread`. + #[inline] + pub(super) fn sub_inactive_thread(&self) -> usize { + let old_value = Counters::new(self.value.fetch_sub(ONE_INACTIVE, Ordering::SeqCst)); + debug_assert!( + old_value.inactive_threads() > 0, + "sub_inactive_thread: old_value {old_value:?} has no inactive threads", + ); + debug_assert!( + old_value.sleeping_threads() <= old_value.inactive_threads(), + "sub_inactive_thread: old_value {:?} had {} sleeping threads and {} inactive threads", + old_value, + old_value.sleeping_threads(), + old_value.inactive_threads(), + ); + + // Current heuristic: whenever an inactive thread goes away, if + // there are any sleeping threads, wake 'em up. + let sleeping_threads = old_value.sleeping_threads(); + Ord::min(sleeping_threads, 2) + } + + /// Subtracts a sleeping thread. This cannot fail, but it is only + /// safe to do if you you know the number of sleeping threads is + /// non-zero (i.e., because you have just awoken a sleeping + /// thread). + #[inline] + pub(super) fn sub_sleeping_thread(&self) { + let old_value = Counters::new(self.value.fetch_sub(ONE_SLEEPING, Ordering::SeqCst)); + debug_assert!( + old_value.sleeping_threads() > 0, + "sub_sleeping_thread: old_value {old_value:?} had no sleeping threads", + ); + debug_assert!( + old_value.sleeping_threads() <= old_value.inactive_threads(), + "sub_sleeping_thread: old_value {:?} had {} sleeping threads and {} inactive threads", + old_value, + old_value.sleeping_threads(), + old_value.inactive_threads(), + ); + } + + #[inline] + pub(super) fn try_add_sleeping_thread(&self, old_value: Counters) -> bool { + debug_assert!( + old_value.inactive_threads() > 0, + "try_add_sleeping_thread: old_value {old_value:?} has no inactive threads", + ); + debug_assert!( + old_value.sleeping_threads() < THREADS_MAX, + "try_add_sleeping_thread: old_value {old_value:?} has too many sleeping threads", + ); + + let mut new_value = old_value; + new_value.word += ONE_SLEEPING; + + self.try_exchange(old_value, new_value, Ordering::SeqCst) + } +} + +#[inline] +fn select_thread(word: usize, shift: usize) -> usize { + (word >> shift) & THREADS_MAX +} + +#[inline] +fn select_jec(word: usize) -> usize { + word >> JEC_SHIFT +} + +impl Counters { + #[inline] + fn new(word: usize) -> Counters { + Counters { word } + } + + #[inline] + fn increment_jobs_counter(self) -> Counters { + // We can freely add to JEC because it occupies the most significant bits. + // Thus it doesn't overflow into the other counters, just wraps itself. + Counters { + word: self.word.wrapping_add(ONE_JEC), + } + } + + #[inline] + pub(super) fn jobs_counter(self) -> JobsEventCounter { + JobsEventCounter(select_jec(self.word)) + } + + /// The number of threads that are not actively + /// executing work. They may be idle, sleepy, or asleep. + #[inline] + pub(super) fn inactive_threads(self) -> usize { + select_thread(self.word, INACTIVE_SHIFT) + } + + #[inline] + pub(super) fn awake_but_idle_threads(self) -> usize { + debug_assert!( + self.sleeping_threads() <= self.inactive_threads(), + "sleeping threads: {} > raw idle threads {}", + self.sleeping_threads(), + self.inactive_threads() + ); + self.inactive_threads() - self.sleeping_threads() + } + + #[inline] + pub(super) fn sleeping_threads(self) -> usize { + select_thread(self.word, SLEEPING_SHIFT) + } +} + +impl std::fmt::Debug for Counters { + fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let word = format!("{:016x}", self.word); + fmt.debug_struct("Counters") + .field("word", &word) + .field("jobs", &self.jobs_counter().0) + .field("inactive", &self.inactive_threads()) + .field("sleeping", &self.sleeping_threads()) + .finish() + } +} diff --git a/anneal/v2/vendor/rayon-core/src/sleep/mod.rs b/anneal/v2/vendor/rayon-core/src/sleep/mod.rs new file mode 100644 index 0000000000..9b02b39b56 --- /dev/null +++ b/anneal/v2/vendor/rayon-core/src/sleep/mod.rs @@ -0,0 +1,324 @@ +//! Code that decides when workers should go to sleep. See README.md +//! for an overview. + +use crate::latch::CoreLatch; +use crate::sync::{Condvar, Mutex}; +use crossbeam_utils::CachePadded; +use std::sync::atomic::Ordering; +use std::thread; + +mod counters; +pub(crate) use self::counters::THREADS_MAX; +use self::counters::{AtomicCounters, JobsEventCounter}; + +/// The `Sleep` struct is embedded into each registry. It governs the waking and sleeping +/// of workers. It has callbacks that are invoked periodically at significant events, +/// such as when workers are looping and looking for work, when latches are set, or when +/// jobs are published, and it either blocks threads or wakes them in response to these +/// events. See the [`README.md`] in this module for more details. +/// +/// [`README.md`] README.md +pub(super) struct Sleep { + /// One "sleep state" per worker. Used to track if a worker is sleeping and to have + /// them block. + worker_sleep_states: Vec>, + + counters: AtomicCounters, +} + +/// An instance of this struct is created when a thread becomes idle. +/// It is consumed when the thread finds work, and passed by `&mut` +/// reference for operations that preserve the idle state. (In other +/// words, producing one of these structs is evidence the thread is +/// idle.) It tracks state such as how long the thread has been idle. +pub(super) struct IdleState { + /// What is worker index of the idle thread? + worker_index: usize, + + /// How many rounds have we been circling without sleeping? + rounds: u32, + + /// Once we become sleepy, what was the sleepy counter value? + /// Set to `INVALID_SLEEPY_COUNTER` otherwise. + jobs_counter: JobsEventCounter, +} + +/// The "sleep state" for an individual worker. +#[derive(Default)] +struct WorkerSleepState { + /// Set to true when the worker goes to sleep; set to false when + /// the worker is notified or when it wakes. + is_blocked: Mutex, + + condvar: Condvar, +} + +const ROUNDS_UNTIL_SLEEPY: u32 = 32; +const ROUNDS_UNTIL_SLEEPING: u32 = ROUNDS_UNTIL_SLEEPY + 1; + +impl Sleep { + pub(super) fn new(n_threads: usize) -> Sleep { + assert!(n_threads <= THREADS_MAX); + Sleep { + worker_sleep_states: (0..n_threads).map(|_| Default::default()).collect(), + counters: AtomicCounters::new(), + } + } + + #[inline] + pub(super) fn start_looking(&self, worker_index: usize) -> IdleState { + self.counters.add_inactive_thread(); + + IdleState { + worker_index, + rounds: 0, + jobs_counter: JobsEventCounter::DUMMY, + } + } + + #[inline] + pub(super) fn work_found(&self) { + // If we were the last idle thread and other threads are still sleeping, + // then we should wake up another thread. + let threads_to_wake = self.counters.sub_inactive_thread(); + self.wake_any_threads(threads_to_wake as u32); + } + + #[inline] + pub(super) fn no_work_found( + &self, + idle_state: &mut IdleState, + latch: &CoreLatch, + has_injected_jobs: impl FnOnce() -> bool, + ) { + if idle_state.rounds < ROUNDS_UNTIL_SLEEPY { + thread::yield_now(); + idle_state.rounds += 1; + } else if idle_state.rounds == ROUNDS_UNTIL_SLEEPY { + idle_state.jobs_counter = self.announce_sleepy(); + idle_state.rounds += 1; + thread::yield_now(); + } else if idle_state.rounds < ROUNDS_UNTIL_SLEEPING { + idle_state.rounds += 1; + thread::yield_now(); + } else { + debug_assert_eq!(idle_state.rounds, ROUNDS_UNTIL_SLEEPING); + self.sleep(idle_state, latch, has_injected_jobs); + } + } + + #[cold] + fn announce_sleepy(&self) -> JobsEventCounter { + self.counters + .increment_jobs_event_counter_if(JobsEventCounter::is_active) + .jobs_counter() + } + + #[cold] + fn sleep( + &self, + idle_state: &mut IdleState, + latch: &CoreLatch, + has_injected_jobs: impl FnOnce() -> bool, + ) { + let worker_index = idle_state.worker_index; + + if !latch.get_sleepy() { + return; + } + + let sleep_state = &self.worker_sleep_states[worker_index]; + let mut is_blocked = sleep_state.is_blocked.lock().unwrap(); + debug_assert!(!*is_blocked); + + // Our latch was signalled. We should wake back up fully as we + // will have some stuff to do. + if !latch.fall_asleep() { + idle_state.wake_fully(); + return; + } + + loop { + let counters = self.counters.load(Ordering::SeqCst); + + // Check if the JEC has changed since we got sleepy. + debug_assert!(idle_state.jobs_counter.is_sleepy()); + if counters.jobs_counter() != idle_state.jobs_counter { + // JEC has changed, so a new job was posted, but for some reason + // we didn't see it. We should return to just before the SLEEPY + // state so we can do another search and (if we fail to find + // work) go back to sleep. + idle_state.wake_partly(); + latch.wake_up(); + return; + } + + // Otherwise, let's move from IDLE to SLEEPING. + if self.counters.try_add_sleeping_thread(counters) { + break; + } + } + + // Successfully registered as asleep. + + // We have one last check for injected jobs to do. This protects against + // deadlock in the very unlikely event that + // + // - an external job is being injected while we are sleepy + // - that job triggers the rollover over the JEC such that we don't see it + // - we are the last active worker thread + std::sync::atomic::fence(Ordering::SeqCst); + if has_injected_jobs() { + // If we see an externally injected job, then we have to 'wake + // ourselves up'. (Ordinarily, `sub_sleeping_thread` is invoked by + // the one that wakes us.) + self.counters.sub_sleeping_thread(); + } else { + // If we don't see an injected job (the normal case), then flag + // ourselves as asleep and wait till we are notified. + // + // (Note that `is_blocked` is held under a mutex and the mutex was + // acquired *before* we incremented the "sleepy counter". This means + // that whomever is coming to wake us will have to wait until we + // release the mutex in the call to `wait`, so they will see this + // boolean as true.) + *is_blocked = true; + while *is_blocked { + is_blocked = sleep_state.condvar.wait(is_blocked).unwrap(); + } + } + + // Update other state: + idle_state.wake_fully(); + latch.wake_up(); + } + + /// Notify the given thread that it should wake up (if it is + /// sleeping). When this method is invoked, we typically know the + /// thread is asleep, though in rare cases it could have been + /// awoken by (e.g.) new work having been posted. + pub(super) fn notify_worker_latch_is_set(&self, target_worker_index: usize) { + self.wake_specific_thread(target_worker_index); + } + + /// Signals that `num_jobs` new jobs were injected into the thread + /// pool from outside. This function will ensure that there are + /// threads available to process them, waking threads from sleep + /// if necessary. + /// + /// # Parameters + /// + /// - `num_jobs` -- lower bound on number of jobs available for stealing. + /// We'll try to get at least one thread per job. + #[inline] + pub(super) fn new_injected_jobs(&self, num_jobs: u32, queue_was_empty: bool) { + // This fence is needed to guarantee that threads + // as they are about to fall asleep, observe any + // new jobs that may have been injected. + std::sync::atomic::fence(Ordering::SeqCst); + + self.new_jobs(num_jobs, queue_was_empty) + } + + /// Signals that `num_jobs` new jobs were pushed onto a thread's + /// local deque. This function will try to ensure that there are + /// threads available to process them, waking threads from sleep + /// if necessary. However, this is not guaranteed: under certain + /// race conditions, the function may fail to wake any new + /// threads; in that case the existing thread should eventually + /// pop the job. + /// + /// # Parameters + /// + /// - `num_jobs` -- lower bound on number of jobs available for stealing. + /// We'll try to get at least one thread per job. + #[inline] + pub(super) fn new_internal_jobs(&self, num_jobs: u32, queue_was_empty: bool) { + self.new_jobs(num_jobs, queue_was_empty) + } + + /// Common helper for `new_injected_jobs` and `new_internal_jobs`. + #[inline] + fn new_jobs(&self, num_jobs: u32, queue_was_empty: bool) { + // Read the counters and -- if sleepy workers have announced themselves + // -- announce that there is now work available. The final value of `counters` + // with which we exit the loop thus corresponds to a state when + let counters = self + .counters + .increment_jobs_event_counter_if(JobsEventCounter::is_sleepy); + let num_awake_but_idle = counters.awake_but_idle_threads(); + let num_sleepers = counters.sleeping_threads(); + + if num_sleepers == 0 { + // nobody to wake + return; + } + + // Promote from u16 to u32 so we can interoperate with + // num_jobs more easily. + let num_awake_but_idle = num_awake_but_idle as u32; + let num_sleepers = num_sleepers as u32; + + // If the queue is non-empty, then we always wake up a worker + // -- clearly the existing idle jobs aren't enough. Otherwise, + // check to see if we have enough idle workers. + if !queue_was_empty { + let num_to_wake = Ord::min(num_jobs, num_sleepers); + self.wake_any_threads(num_to_wake); + } else if num_awake_but_idle < num_jobs { + let num_to_wake = Ord::min(num_jobs - num_awake_but_idle, num_sleepers); + self.wake_any_threads(num_to_wake); + } + } + + #[cold] + fn wake_any_threads(&self, mut num_to_wake: u32) { + if num_to_wake > 0 { + for i in 0..self.worker_sleep_states.len() { + if self.wake_specific_thread(i) { + num_to_wake -= 1; + if num_to_wake == 0 { + return; + } + } + } + } + } + + fn wake_specific_thread(&self, index: usize) -> bool { + let sleep_state = &self.worker_sleep_states[index]; + + let mut is_blocked = sleep_state.is_blocked.lock().unwrap(); + if *is_blocked { + *is_blocked = false; + sleep_state.condvar.notify_one(); + + // When the thread went to sleep, it will have incremented + // this value. When we wake it, its our job to decrement + // it. We could have the thread do it, but that would + // introduce a delay between when the thread was + // *notified* and when this counter was decremented. That + // might mislead people with new work into thinking that + // there are sleeping threads that they should try to + // wake, when in fact there is nothing left for them to + // do. + self.counters.sub_sleeping_thread(); + + true + } else { + false + } + } +} + +impl IdleState { + fn wake_fully(&mut self) { + self.rounds = 0; + self.jobs_counter = JobsEventCounter::DUMMY; + } + + fn wake_partly(&mut self) { + self.rounds = ROUNDS_UNTIL_SLEEPY; + self.jobs_counter = JobsEventCounter::DUMMY; + } +} diff --git a/anneal/v2/vendor/rayon-core/src/spawn/mod.rs b/anneal/v2/vendor/rayon-core/src/spawn/mod.rs new file mode 100644 index 0000000000..de3ef08fa0 --- /dev/null +++ b/anneal/v2/vendor/rayon-core/src/spawn/mod.rs @@ -0,0 +1,163 @@ +use crate::job::*; +use crate::registry::Registry; +use crate::unwind; +use std::mem; +use std::sync::Arc; + +/// Puts the task into the Rayon thread pool's job queue in the "static" +/// or "global" scope. Just like a standard thread, this task is not +/// tied to the current stack frame, and hence it cannot hold any +/// references other than those with `'static` lifetime. If you want +/// to spawn a task that references stack data, use [the `scope()` +/// function] to create a scope. +/// +/// [the `scope()` function]: crate::scope() +/// +/// Since tasks spawned with this function cannot hold references into +/// the enclosing stack frame, you almost certainly want to use a +/// `move` closure as their argument (otherwise, the closure will +/// typically hold references to any variables from the enclosing +/// function that you happen to use). +/// +/// This API assumes that the closure is executed purely for its +/// side-effects (i.e., it might send messages, modify data protected +/// by a mutex, or some such thing). +/// +/// There is no guaranteed order of execution for spawns, given that +/// other threads may steal tasks at any time. However, they are +/// generally prioritized in a LIFO order on the thread from which +/// they were spawned. Other threads always steal from the other end of +/// the deque, like FIFO order. The idea is that "recent" tasks are +/// most likely to be fresh in the local CPU's cache, while other +/// threads can steal older "stale" tasks. For an alternate approach, +/// consider [`spawn_fifo()`] instead. +/// +/// # Panic handling +/// +/// If this closure should panic, the resulting panic will be +/// propagated to the panic handler registered in the `ThreadPoolBuilder`, +/// if any. See [`ThreadPoolBuilder::panic_handler()`] for more +/// details. +/// +/// [`ThreadPoolBuilder::panic_handler()`]: crate::ThreadPoolBuilder::panic_handler() +/// +/// # Examples +/// +/// This code creates a Rayon task that increments a global counter. +/// +/// ```rust +/// # use rayon_core as rayon; +/// use std::sync::atomic::{AtomicUsize, Ordering, ATOMIC_USIZE_INIT}; +/// +/// static GLOBAL_COUNTER: AtomicUsize = ATOMIC_USIZE_INIT; +/// +/// rayon::spawn(move || { +/// GLOBAL_COUNTER.fetch_add(1, Ordering::SeqCst); +/// }); +/// ``` +pub fn spawn(func: F) +where + F: FnOnce() + Send + 'static, +{ + // We assert that current registry has not terminated. + unsafe { spawn_in(func, &Registry::current()) } +} + +/// Spawns an asynchronous job in `registry.` +/// +/// Unsafe because `registry` must not yet have terminated. +pub(super) unsafe fn spawn_in(func: F, registry: &Arc) +where + F: FnOnce() + Send + 'static, +{ + // We assert that this does not hold any references (we know + // this because of the `'static` bound in the interface); + // moreover, we assert that the code below is not supposed to + // be able to panic, and hence the data won't leak but will be + // enqueued into some deque for later execution. + let abort_guard = unwind::AbortIfPanic; // just in case we are wrong, and code CAN panic + let job_ref = spawn_job(func, registry); + registry.inject_or_push(job_ref); + mem::forget(abort_guard); +} + +unsafe fn spawn_job(func: F, registry: &Arc) -> JobRef +where + F: FnOnce() + Send + 'static, +{ + // Ensure that registry cannot terminate until this job has + // executed. This ref is decremented at the (*) below. + registry.increment_terminate_count(); + + HeapJob::new({ + let registry = Arc::clone(registry); + move || { + registry.catch_unwind(func); + registry.terminate(); // (*) permit registry to terminate now + } + }) + .into_static_job_ref() +} + +/// Fires off a task into the Rayon thread pool in the "static" or +/// "global" scope. Just like a standard thread, this task is not +/// tied to the current stack frame, and hence it cannot hold any +/// references other than those with `'static` lifetime. If you want +/// to spawn a task that references stack data, use [the `scope_fifo()` +/// function] to create a scope. +/// +/// The behavior is essentially the same as [the `spawn` +/// function], except that calls from the same thread +/// will be prioritized in FIFO order. This is similar to the now- +/// deprecated [`breadth_first`] option, except the effect is isolated +/// to relative `spawn_fifo` calls, not all thread-pool tasks. +/// +/// For more details on this design, see Rayon [RFC #1]. +/// +/// [the `scope_fifo()` function]: crate::scope_fifo() +/// [the `spawn` function]: crate::spawn() +/// [`breadth_first`]: crate::ThreadPoolBuilder::breadth_first +/// [RFC #1]: https://github.com/rayon-rs/rfcs/blob/main/accepted/rfc0001-scope-scheduling.md +/// +/// # Panic handling +/// +/// If this closure should panic, the resulting panic will be +/// propagated to the panic handler registered in the `ThreadPoolBuilder`, +/// if any. See [`ThreadPoolBuilder::panic_handler()`] for more +/// details. +/// +/// [`ThreadPoolBuilder::panic_handler()`]: crate::ThreadPoolBuilder::panic_handler +pub fn spawn_fifo(func: F) +where + F: FnOnce() + Send + 'static, +{ + // We assert that current registry has not terminated. + unsafe { spawn_fifo_in(func, &Registry::current()) } +} + +/// Spawns an asynchronous FIFO job in `registry.` +/// +/// Unsafe because `registry` must not yet have terminated. +pub(super) unsafe fn spawn_fifo_in(func: F, registry: &Arc) +where + F: FnOnce() + Send + 'static, +{ + // We assert that this does not hold any references (we know + // this because of the `'static` bound in the interface); + // moreover, we assert that the code below is not supposed to + // be able to panic, and hence the data won't leak but will be + // enqueued into some deque for later execution. + let abort_guard = unwind::AbortIfPanic; // just in case we are wrong, and code CAN panic + let job_ref = spawn_job(func, registry); + + // If we're in the pool, use our thread's private fifo for this thread to execute + // in a locally-FIFO order. Otherwise, just use the pool's global injector. + match registry.current_thread() { + Some(worker) => worker.push_fifo(job_ref), + None => registry.inject(job_ref), + } + mem::forget(abort_guard); +} + +#[cfg(test)] +mod test; diff --git a/anneal/v2/vendor/rayon-core/src/spawn/test.rs b/anneal/v2/vendor/rayon-core/src/spawn/test.rs new file mode 100644 index 0000000000..83e2b9c477 --- /dev/null +++ b/anneal/v2/vendor/rayon-core/src/spawn/test.rs @@ -0,0 +1,255 @@ +use crate::scope; +use std::any::Any; +use std::sync::mpsc::channel; +use std::sync::Mutex; + +use super::{spawn, spawn_fifo}; +use crate::ThreadPoolBuilder; + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn spawn_then_join_in_worker() { + let (tx, rx) = channel(); + scope(move |_| { + spawn(move || tx.send(22).unwrap()); + }); + assert_eq!(22, rx.recv().unwrap()); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn spawn_then_join_outside_worker() { + let (tx, rx) = channel(); + spawn(move || tx.send(22).unwrap()); + assert_eq!(22, rx.recv().unwrap()); +} + +#[test] +#[cfg_attr(not(panic = "unwind"), ignore)] +fn panic_fwd() { + let (tx, rx) = channel(); + + let tx = Mutex::new(tx); + let panic_handler = move |err: Box| { + let tx = tx.lock().unwrap(); + if let Some(&msg) = err.downcast_ref::<&str>() { + if msg == "Hello, world!" { + tx.send(1).unwrap(); + } else { + tx.send(2).unwrap(); + } + } else { + tx.send(3).unwrap(); + } + }; + + let builder = ThreadPoolBuilder::new().panic_handler(panic_handler); + + builder + .build() + .unwrap() + .spawn(move || panic!("Hello, world!")); + + assert_eq!(1, rx.recv().unwrap()); +} + +/// Test what happens when the thread pool is dropped but there are +/// still active asynchronous tasks. We expect the thread pool to stay +/// alive and executing until those threads are complete. +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn termination_while_things_are_executing() { + let (tx0, rx0) = channel(); + let (tx1, rx1) = channel(); + + // Create a thread pool and spawn some code in it, but then drop + // our reference to it. + { + let thread_pool = ThreadPoolBuilder::new().build().unwrap(); + thread_pool.spawn(move || { + let data = rx0.recv().unwrap(); + + // At this point, we know the "main" reference to the + // `ThreadPool` has been dropped, but there are still + // active threads. Launch one more. + spawn(move || { + tx1.send(data).unwrap(); + }); + }); + } + + tx0.send(22).unwrap(); + let v = rx1.recv().unwrap(); + assert_eq!(v, 22); +} + +#[test] +#[cfg_attr(not(panic = "unwind"), ignore)] +fn custom_panic_handler_and_spawn() { + let (tx, rx) = channel(); + + // Create a parallel closure that will send panics on the + // channel; since the closure is potentially executed in parallel + // with itself, we have to wrap `tx` in a mutex. + let tx = Mutex::new(tx); + let panic_handler = move |e: Box| { + tx.lock().unwrap().send(e).unwrap(); + }; + + // Execute an async that will panic. + let builder = ThreadPoolBuilder::new().panic_handler(panic_handler); + builder.build().unwrap().spawn(move || { + panic!("Hello, world!"); + }); + + // Check that we got back the panic we expected. + let error = rx.recv().unwrap(); + if let Some(&msg) = error.downcast_ref::<&str>() { + assert_eq!(msg, "Hello, world!"); + } else { + panic!("did not receive a string from panic handler"); + } +} + +#[test] +#[cfg_attr(not(panic = "unwind"), ignore)] +fn custom_panic_handler_and_nested_spawn() { + let (tx, rx) = channel(); + + // Create a parallel closure that will send panics on the + // channel; since the closure is potentially executed in parallel + // with itself, we have to wrap `tx` in a mutex. + let tx = Mutex::new(tx); + let panic_handler = move |e| { + tx.lock().unwrap().send(e).unwrap(); + }; + + // Execute an async that will (eventually) panic. + const PANICS: usize = 3; + let builder = ThreadPoolBuilder::new().panic_handler(panic_handler); + builder.build().unwrap().spawn(move || { + // launch 3 nested spawn-asyncs; these should be in the same + // thread pool and hence inherit the same panic handler + for _ in 0..PANICS { + spawn(move || { + panic!("Hello, world!"); + }); + } + }); + + // Check that we get back the panics we expected. + for _ in 0..PANICS { + let error = rx.recv().unwrap(); + if let Some(&msg) = error.downcast_ref::<&str>() { + assert_eq!(msg, "Hello, world!"); + } else { + panic!("did not receive a string from panic handler"); + } + } +} + +macro_rules! test_order { + ($outer_spawn:ident, $inner_spawn:ident) => {{ + let builder = ThreadPoolBuilder::new().num_threads(1); + let pool = builder.build().unwrap(); + let (tx, rx) = channel(); + pool.install(move || { + for i in 0..10 { + let tx = tx.clone(); + $outer_spawn(move || { + for j in 0..10 { + let tx = tx.clone(); + $inner_spawn(move || { + tx.send(i * 10 + j).unwrap(); + }); + } + }); + } + }); + rx.iter().collect::>() + }}; +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn lifo_order() { + // In the absence of stealing, `spawn()` jobs on a thread will run in LIFO order. + let vec = test_order!(spawn, spawn); + let expected: Vec = (0..100).rev().collect(); // LIFO -> reversed + assert_eq!(vec, expected); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn fifo_order() { + // In the absence of stealing, `spawn_fifo()` jobs on a thread will run in FIFO order. + let vec = test_order!(spawn_fifo, spawn_fifo); + let expected: Vec = (0..100).collect(); // FIFO -> natural order + assert_eq!(vec, expected); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn lifo_fifo_order() { + // LIFO on the outside, FIFO on the inside + let vec = test_order!(spawn, spawn_fifo); + let expected: Vec = (0..10) + .rev() + .flat_map(|i| (0..10).map(move |j| i * 10 + j)) + .collect(); + assert_eq!(vec, expected); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn fifo_lifo_order() { + // FIFO on the outside, LIFO on the inside + let vec = test_order!(spawn_fifo, spawn); + let expected: Vec = (0..10) + .flat_map(|i| (0..10).rev().map(move |j| i * 10 + j)) + .collect(); + assert_eq!(vec, expected); +} + +macro_rules! spawn_send { + ($spawn:ident, $tx:ident, $i:expr) => {{ + let tx = $tx.clone(); + $spawn(move || tx.send($i).unwrap()); + }}; +} + +/// Test mixed spawns pushing a series of numbers, interleaved such +/// such that negative values are using the second kind of spawn. +macro_rules! test_mixed_order { + ($pos_spawn:ident, $neg_spawn:ident) => {{ + let builder = ThreadPoolBuilder::new().num_threads(1); + let pool = builder.build().unwrap(); + let (tx, rx) = channel(); + pool.install(move || { + spawn_send!($pos_spawn, tx, 0); + spawn_send!($neg_spawn, tx, -1); + spawn_send!($pos_spawn, tx, 1); + spawn_send!($neg_spawn, tx, -2); + spawn_send!($pos_spawn, tx, 2); + spawn_send!($neg_spawn, tx, -3); + spawn_send!($pos_spawn, tx, 3); + }); + rx.iter().collect::>() + }}; +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn mixed_lifo_fifo_order() { + let vec = test_mixed_order!(spawn, spawn_fifo); + let expected = vec![3, -1, 2, -2, 1, -3, 0]; + assert_eq!(vec, expected); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn mixed_fifo_lifo_order() { + let vec = test_mixed_order!(spawn_fifo, spawn); + let expected = vec![0, -3, 1, -2, 2, -1, 3]; + assert_eq!(vec, expected); +} diff --git a/anneal/v2/vendor/rayon-core/src/test.rs b/anneal/v2/vendor/rayon-core/src/test.rs new file mode 100644 index 0000000000..e88b6f882d --- /dev/null +++ b/anneal/v2/vendor/rayon-core/src/test.rs @@ -0,0 +1,200 @@ +#![cfg(test)] + +use crate::{ThreadPoolBuildError, ThreadPoolBuilder}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Barrier}; + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn worker_thread_index() { + let pool = ThreadPoolBuilder::new().num_threads(22).build().unwrap(); + assert_eq!(pool.current_num_threads(), 22); + assert_eq!(pool.current_thread_index(), None); + let index = pool.install(|| pool.current_thread_index().unwrap()); + assert!(index < 22); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn start_callback_called() { + let n_threads = 16; + let n_called = Arc::new(AtomicUsize::new(0)); + // Wait for all the threads in the pool plus the one running tests. + let barrier = Arc::new(Barrier::new(n_threads + 1)); + + let b = Arc::clone(&barrier); + let nc = Arc::clone(&n_called); + let start_handler = move |_| { + nc.fetch_add(1, Ordering::SeqCst); + b.wait(); + }; + + let conf = ThreadPoolBuilder::new() + .num_threads(n_threads) + .start_handler(start_handler); + let _ = conf.build().unwrap(); + + // Wait for all the threads to have been scheduled to run. + barrier.wait(); + + // The handler must have been called on every started thread. + assert_eq!(n_called.load(Ordering::SeqCst), n_threads); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn exit_callback_called() { + let n_threads = 16; + let n_called = Arc::new(AtomicUsize::new(0)); + // Wait for all the threads in the pool plus the one running tests. + let barrier = Arc::new(Barrier::new(n_threads + 1)); + + let b = Arc::clone(&barrier); + let nc = Arc::clone(&n_called); + let exit_handler = move |_| { + nc.fetch_add(1, Ordering::SeqCst); + b.wait(); + }; + + let conf = ThreadPoolBuilder::new() + .num_threads(n_threads) + .exit_handler(exit_handler); + { + let _ = conf.build().unwrap(); + // Drop the pool so it stops the running threads. + } + + // Wait for all the threads to have been scheduled to run. + barrier.wait(); + + // The handler must have been called on every exiting thread. + assert_eq!(n_called.load(Ordering::SeqCst), n_threads); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn handler_panics_handled_correctly() { + let n_threads = 16; + let n_called = Arc::new(AtomicUsize::new(0)); + // Wait for all the threads in the pool plus the one running tests. + let start_barrier = Arc::new(Barrier::new(n_threads + 1)); + let exit_barrier = Arc::new(Barrier::new(n_threads + 1)); + + let start_handler = move |_| { + panic!("ensure panic handler is called when starting"); + }; + let exit_handler = move |_| { + panic!("ensure panic handler is called when exiting"); + }; + + let sb = Arc::clone(&start_barrier); + let eb = Arc::clone(&exit_barrier); + let nc = Arc::clone(&n_called); + let panic_handler = move |_| { + let val = nc.fetch_add(1, Ordering::SeqCst); + if val < n_threads { + sb.wait(); + } else { + eb.wait(); + } + }; + + let conf = ThreadPoolBuilder::new() + .num_threads(n_threads) + .start_handler(start_handler) + .exit_handler(exit_handler) + .panic_handler(panic_handler); + { + let _ = conf.build().unwrap(); + + // Wait for all the threads to start, panic in the start handler, + // and been taken care of by the panic handler. + start_barrier.wait(); + + // Drop the pool so it stops the running threads. + } + + // Wait for all the threads to exit, panic in the exit handler, + // and been taken care of by the panic handler. + exit_barrier.wait(); + + // The panic handler must have been called twice on every thread. + assert_eq!(n_called.load(Ordering::SeqCst), 2 * n_threads); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn check_config_build() { + let pool = ThreadPoolBuilder::new().num_threads(22).build().unwrap(); + assert_eq!(pool.current_num_threads(), 22); +} + +/// Helper used by check_error_send_sync to ensure ThreadPoolBuildError is Send + Sync +fn _send_sync() {} + +#[test] +fn check_error_send_sync() { + _send_sync::(); +} + +#[allow(deprecated)] +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn configuration() { + let start_handler = move |_| {}; + let exit_handler = move |_| {}; + let panic_handler = move |_| {}; + let thread_name = move |i| format!("thread_name_{i}"); + + // Ensure we can call all public methods on Configuration + crate::Configuration::new() + .thread_name(thread_name) + .num_threads(5) + .panic_handler(panic_handler) + .stack_size(4e6 as usize) + .breadth_first() + .start_handler(start_handler) + .exit_handler(exit_handler) + .build() + .unwrap(); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn default_pool() { + ThreadPoolBuilder::default().build().unwrap(); +} + +/// Test that custom spawned threads get their `WorkerThread` cleared once +/// the pool is done with them, allowing them to be used with rayon again +/// later. e.g. WebAssembly want to have their own pool of available threads. +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn cleared_current_thread() -> Result<(), ThreadPoolBuildError> { + let n_threads = 5; + let mut handles = vec![]; + let pool = ThreadPoolBuilder::new() + .num_threads(n_threads) + .spawn_handler(|thread| { + let handle = std::thread::spawn(move || { + thread.run(); + + // Afterward, the current thread shouldn't be set anymore. + assert_eq!(crate::current_thread_index(), None); + }); + handles.push(handle); + Ok(()) + }) + .build()?; + assert_eq!(handles.len(), n_threads); + + pool.install(|| assert!(crate::current_thread_index().is_some())); + drop(pool); + + // Wait for all threads to make their assertions and exit + for handle in handles { + handle.join().unwrap(); + } + + Ok(()) +} diff --git a/anneal/v2/vendor/rayon-core/src/thread_pool/mod.rs b/anneal/v2/vendor/rayon-core/src/thread_pool/mod.rs new file mode 100644 index 0000000000..58f2c9233d --- /dev/null +++ b/anneal/v2/vendor/rayon-core/src/thread_pool/mod.rs @@ -0,0 +1,502 @@ +//! Contains support for user-managed thread pools, represented by the +//! the [`ThreadPool`] type (see that struct for details). + +use crate::broadcast::{self, BroadcastContext}; +use crate::join; +use crate::registry::{Registry, ThreadSpawn, WorkerThread}; +use crate::scope::{do_in_place_scope, do_in_place_scope_fifo}; +use crate::spawn; +use crate::{scope, Scope}; +use crate::{scope_fifo, ScopeFifo}; +use crate::{ThreadPoolBuildError, ThreadPoolBuilder}; +use std::error::Error; +use std::fmt; +use std::sync::Arc; + +mod test; + +/// Represents a user-created [thread pool]. +/// +/// Use a [`ThreadPoolBuilder`] to specify the number and/or names of threads +/// in the pool. After calling [`ThreadPoolBuilder::build()`], you can then +/// execute functions explicitly within this [`ThreadPool`] using +/// [`ThreadPool::install()`]. By contrast, top-level rayon functions +/// (like `join()`) will execute implicitly within the current thread pool. +/// +/// +/// ## Creating a ThreadPool +/// +/// ```ignore-wasm +/// # use rayon_core as rayon; +/// let pool = rayon::ThreadPoolBuilder::new().num_threads(8).build().unwrap(); +/// ``` +/// +/// [`install()`][`ThreadPool::install()`] executes a closure in one of the `ThreadPool`'s +/// threads. In addition, any other rayon operations called inside of `install()` will also +/// execute in the context of the `ThreadPool`. +/// +/// When the `ThreadPool` is dropped, that's a signal for the threads it manages to terminate, +/// they will complete executing any remaining work that you have spawned, and automatically +/// terminate. +/// +/// +/// [thread pool]: https://en.wikipedia.org/wiki/Thread_pool +/// [`ThreadPoolBuilder::build()`]: ThreadPoolBuilder::build() +/// [`ThreadPool::install()`]: Self::install() +pub struct ThreadPool { + registry: Arc, +} + +impl ThreadPool { + #[deprecated(note = "Use `ThreadPoolBuilder::build`")] + #[allow(deprecated)] + /// Deprecated in favor of `ThreadPoolBuilder::build`. + pub fn new(configuration: crate::Configuration) -> Result> { + Self::build(configuration.into_builder()).map_err(Box::from) + } + + pub(super) fn build( + builder: ThreadPoolBuilder, + ) -> Result + where + S: ThreadSpawn, + { + let registry = Registry::new(builder)?; + Ok(ThreadPool { registry }) + } + + /// Executes `op` within the thread pool. Any attempts to use + /// `join`, `scope`, or parallel iterators will then operate + /// within that thread pool. + /// + /// # Warning: thread-local data + /// + /// Because `op` is executing within the Rayon thread pool, + /// thread-local data from the current thread will not be + /// accessible. + /// + /// # Warning: execution order + /// + /// If the current thread is part of a different thread pool, it will try to + /// keep busy while the `op` completes in its target pool, similar to + /// calling [`ThreadPool::yield_now()`] in a loop. Therefore, it may + /// potentially schedule other tasks to run on the current thread in the + /// meantime. For example + /// + /// ```ignore-wasm + /// # use rayon_core as rayon; + /// fn main() { + /// rayon::ThreadPoolBuilder::new().num_threads(1).build_global().unwrap(); + /// let pool = rayon_core::ThreadPoolBuilder::default().build().unwrap(); + /// let do_it = || { + /// print!("one "); + /// pool.install(||{}); + /// print!("two "); + /// }; + /// rayon::join(|| do_it(), || do_it()); + /// } + /// ``` + /// + /// Since we configured just one thread in the global pool, one might + /// expect `do_it()` to run sequentially, producing: + /// + /// ```ascii + /// one two one two + /// ``` + /// + /// However each call to `install()` yields implicitly, allowing rayon to + /// run multiple instances of `do_it()` concurrently on the single, global + /// thread. The following output would be equally valid: + /// + /// ```ascii + /// one one two two + /// ``` + /// + /// # Panics + /// + /// If `op` should panic, that panic will be propagated. + /// + /// ## Using `install()` + /// + /// ```ignore-wasm + /// # use rayon_core as rayon; + /// fn main() { + /// let pool = rayon::ThreadPoolBuilder::new().num_threads(8).build().unwrap(); + /// let n = pool.install(|| fib(20)); + /// println!("{}", n); + /// } + /// + /// fn fib(n: usize) -> usize { + /// if n == 0 || n == 1 { + /// return n; + /// } + /// let (a, b) = rayon::join(|| fib(n - 1), || fib(n - 2)); // runs inside of `pool` + /// return a + b; + /// } + /// ``` + pub fn install(&self, op: OP) -> R + where + OP: FnOnce() -> R + Send, + R: Send, + { + self.registry.in_worker(|_, _| op()) + } + + /// Executes `op` within every thread in the thread pool. Any attempts to use + /// `join`, `scope`, or parallel iterators will then operate within that + /// thread pool. + /// + /// Broadcasts are executed on each thread after they have exhausted their + /// local work queue, before they attempt work-stealing from other threads. + /// The goal of that strategy is to run everywhere in a timely manner + /// *without* being too disruptive to current work. There may be alternative + /// broadcast styles added in the future for more or less aggressive + /// injection, if the need arises. + /// + /// # Warning: thread-local data + /// + /// Because `op` is executing within the Rayon thread pool, + /// thread-local data from the current thread will not be + /// accessible. + /// + /// # Panics + /// + /// If `op` should panic on one or more threads, exactly one panic + /// will be propagated, only after all threads have completed + /// (or panicked) their own `op`. + /// + /// # Examples + /// + /// ```ignore-wasm + /// # use rayon_core as rayon; + /// use std::sync::atomic::{AtomicUsize, Ordering}; + /// + /// fn main() { + /// let pool = rayon::ThreadPoolBuilder::new().num_threads(5).build().unwrap(); + /// + /// // The argument gives context, including the index of each thread. + /// let v: Vec = pool.broadcast(|ctx| ctx.index() * ctx.index()); + /// assert_eq!(v, &[0, 1, 4, 9, 16]); + /// + /// // The closure can reference the local stack + /// let count = AtomicUsize::new(0); + /// pool.broadcast(|_| count.fetch_add(1, Ordering::Relaxed)); + /// assert_eq!(count.into_inner(), 5); + /// } + /// ``` + pub fn broadcast(&self, op: OP) -> Vec + where + OP: Fn(BroadcastContext<'_>) -> R + Sync, + R: Send, + { + // We assert that `self.registry` has not terminated. + unsafe { broadcast::broadcast_in(op, &self.registry) } + } + + /// Returns the (current) number of threads in the thread pool. + /// + /// # Future compatibility note + /// + /// Note that unless this thread pool was created with a + /// [`ThreadPoolBuilder`] that specifies the number of threads, + /// then this number may vary over time in future versions (see [the + /// `num_threads()` method for details][snt]). + /// + /// [snt]: ThreadPoolBuilder::num_threads() + #[inline] + pub fn current_num_threads(&self) -> usize { + self.registry.num_threads() + } + + /// If called from a Rayon worker thread in this thread pool, + /// returns the index of that thread; if not called from a Rayon + /// thread, or called from a Rayon thread that belongs to a + /// different thread pool, returns `None`. + /// + /// The index for a given thread will not change over the thread's + /// lifetime. However, multiple threads may share the same index if + /// they are in distinct thread pools. + /// + /// # Future compatibility note + /// + /// Currently, every thread pool (including the global + /// thread pool) has a fixed number of threads, but this may + /// change in future Rayon versions (see [the `num_threads()` method + /// for details][snt]). In that case, the index for a + /// thread would not change during its lifetime, but thread + /// indices may wind up being reused if threads are terminated and + /// restarted. + /// + /// [snt]: ThreadPoolBuilder::num_threads() + #[inline] + pub fn current_thread_index(&self) -> Option { + let curr = self.registry.current_thread()?; + Some(curr.index()) + } + + /// Returns true if the current worker thread currently has "local + /// tasks" pending. This can be useful as part of a heuristic for + /// deciding whether to spawn a new task or execute code on the + /// current thread, particularly in breadth-first + /// schedulers. However, keep in mind that this is an inherently + /// racy check, as other worker threads may be actively "stealing" + /// tasks from our local deque. + /// + /// **Background:** Rayon's uses a [work-stealing] scheduler. The + /// key idea is that each thread has its own [deque] of + /// tasks. Whenever a new task is spawned -- whether through + /// `join()`, `Scope::spawn()`, or some other means -- that new + /// task is pushed onto the thread's *local* deque. Worker threads + /// have a preference for executing their own tasks; if however + /// they run out of tasks, they will go try to "steal" tasks from + /// other threads. This function therefore has an inherent race + /// with other active worker threads, which may be removing items + /// from the local deque. + /// + /// [work-stealing]: https://en.wikipedia.org/wiki/Work_stealing + /// [deque]: https://en.wikipedia.org/wiki/Double-ended_queue + #[inline] + pub fn current_thread_has_pending_tasks(&self) -> Option { + let curr = self.registry.current_thread()?; + Some(!curr.local_deque_is_empty()) + } + + /// Execute `oper_a` and `oper_b` in the thread pool and return + /// the results. Equivalent to `self.install(|| join(oper_a, + /// oper_b))`. + pub fn join(&self, oper_a: A, oper_b: B) -> (RA, RB) + where + A: FnOnce() -> RA + Send, + B: FnOnce() -> RB + Send, + RA: Send, + RB: Send, + { + self.install(|| join(oper_a, oper_b)) + } + + /// Creates a scope that executes within this thread pool. + /// Equivalent to `self.install(|| scope(...))`. + /// + /// See also: [the `scope()` function]. + /// + /// [the `scope()` function]: crate::scope() + pub fn scope<'scope, OP, R>(&self, op: OP) -> R + where + OP: FnOnce(&Scope<'scope>) -> R + Send, + R: Send, + { + self.install(|| scope(op)) + } + + /// Creates a scope that executes within this thread pool. + /// Spawns from the same thread are prioritized in relative FIFO order. + /// Equivalent to `self.install(|| scope_fifo(...))`. + /// + /// See also: [the `scope_fifo()` function]. + /// + /// [the `scope_fifo()` function]: crate::scope_fifo() + pub fn scope_fifo<'scope, OP, R>(&self, op: OP) -> R + where + OP: FnOnce(&ScopeFifo<'scope>) -> R + Send, + R: Send, + { + self.install(|| scope_fifo(op)) + } + + /// Creates a scope that spawns work into this thread pool. + /// + /// See also: [the `in_place_scope()` function]. + /// + /// [the `in_place_scope()` function]: crate::in_place_scope() + pub fn in_place_scope<'scope, OP, R>(&self, op: OP) -> R + where + OP: FnOnce(&Scope<'scope>) -> R, + { + do_in_place_scope(Some(&self.registry), op) + } + + /// Creates a scope that spawns work into this thread pool in FIFO order. + /// + /// See also: [the `in_place_scope_fifo()` function]. + /// + /// [the `in_place_scope_fifo()` function]: crate::in_place_scope_fifo() + pub fn in_place_scope_fifo<'scope, OP, R>(&self, op: OP) -> R + where + OP: FnOnce(&ScopeFifo<'scope>) -> R, + { + do_in_place_scope_fifo(Some(&self.registry), op) + } + + /// Spawns an asynchronous task in this thread pool. This task will + /// run in the implicit, global scope, which means that it may outlast + /// the current stack frame -- therefore, it cannot capture any references + /// onto the stack (you will likely need a `move` closure). + /// + /// See also: [the `spawn()` function defined on scopes][spawn]. + /// + /// [spawn]: Scope::spawn() + pub fn spawn(&self, op: OP) + where + OP: FnOnce() + Send + 'static, + { + // We assert that `self.registry` has not terminated. + unsafe { spawn::spawn_in(op, &self.registry) } + } + + /// Spawns an asynchronous task in this thread pool. This task will + /// run in the implicit, global scope, which means that it may outlast + /// the current stack frame -- therefore, it cannot capture any references + /// onto the stack (you will likely need a `move` closure). + /// + /// See also: [the `spawn_fifo()` function defined on scopes][spawn_fifo]. + /// + /// [spawn_fifo]: ScopeFifo::spawn_fifo() + pub fn spawn_fifo(&self, op: OP) + where + OP: FnOnce() + Send + 'static, + { + // We assert that `self.registry` has not terminated. + unsafe { spawn::spawn_fifo_in(op, &self.registry) } + } + + /// Spawns an asynchronous task on every thread in this thread pool. This task + /// will run in the implicit, global scope, which means that it may outlast the + /// current stack frame -- therefore, it cannot capture any references onto the + /// stack (you will likely need a `move` closure). + pub fn spawn_broadcast(&self, op: OP) + where + OP: Fn(BroadcastContext<'_>) + Send + Sync + 'static, + { + // We assert that `self.registry` has not terminated. + unsafe { broadcast::spawn_broadcast_in(op, &self.registry) } + } + + /// Cooperatively yields execution to Rayon. + /// + /// This is similar to the general [`yield_now()`], but only if the current + /// thread is part of *this* thread pool. + /// + /// Returns `Some(Yield::Executed)` if anything was executed, `Some(Yield::Idle)` if + /// nothing was available, or `None` if the current thread is not part this pool. + pub fn yield_now(&self) -> Option { + let curr = self.registry.current_thread()?; + Some(curr.yield_now()) + } + + /// Cooperatively yields execution to local Rayon work. + /// + /// This is similar to the general [`yield_local()`], but only if the current + /// thread is part of *this* thread pool. + /// + /// Returns `Some(Yield::Executed)` if anything was executed, `Some(Yield::Idle)` if + /// nothing was available, or `None` if the current thread is not part this pool. + pub fn yield_local(&self) -> Option { + let curr = self.registry.current_thread()?; + Some(curr.yield_local()) + } +} + +impl Drop for ThreadPool { + fn drop(&mut self) { + self.registry.terminate(); + } +} + +impl fmt::Debug for ThreadPool { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.debug_struct("ThreadPool") + .field("num_threads", &self.current_num_threads()) + .field("id", &self.registry.id()) + .finish() + } +} + +/// If called from a Rayon worker thread, returns the index of that +/// thread within its current pool; if not called from a Rayon thread, +/// returns `None`. +/// +/// The index for a given thread will not change over the thread's +/// lifetime. However, multiple threads may share the same index if +/// they are in distinct thread pools. +/// +/// See also: [the `ThreadPool::current_thread_index()` method][m]. +/// +/// [m]: ThreadPool::current_thread_index() +/// +/// # Future compatibility note +/// +/// Currently, every thread pool (including the global +/// thread pool) has a fixed number of threads, but this may +/// change in future Rayon versions (see [the `num_threads()` method +/// for details][snt]). In that case, the index for a +/// thread would not change during its lifetime, but thread +/// indices may wind up being reused if threads are terminated and +/// restarted. +/// +/// [snt]: ThreadPoolBuilder::num_threads() +#[inline] +pub fn current_thread_index() -> Option { + unsafe { + let curr = WorkerThread::current().as_ref()?; + Some(curr.index()) + } +} + +/// If called from a Rayon worker thread, indicates whether that +/// thread's local deque still has pending tasks. Otherwise, returns +/// `None`. For more information, see [the +/// `ThreadPool::current_thread_has_pending_tasks()` method][m]. +/// +/// [m]: ThreadPool::current_thread_has_pending_tasks() +#[inline] +pub fn current_thread_has_pending_tasks() -> Option { + unsafe { + let curr = WorkerThread::current().as_ref()?; + Some(!curr.local_deque_is_empty()) + } +} + +/// Cooperatively yields execution to Rayon. +/// +/// If the current thread is part of a rayon thread pool, this looks for a +/// single unit of pending work in the pool, then executes it. Completion of +/// that work might include nested work or further work stealing. +/// +/// This is similar to [`std::thread::yield_now()`], but does not literally make +/// that call. If you are implementing a polling loop, you may want to also +/// yield to the OS scheduler yourself if no Rayon work was found. +/// +/// Returns `Some(Yield::Executed)` if anything was executed, `Some(Yield::Idle)` if +/// nothing was available, or `None` if this thread is not part of any pool at all. +pub fn yield_now() -> Option { + unsafe { + let thread = WorkerThread::current().as_ref()?; + Some(thread.yield_now()) + } +} + +/// Cooperatively yields execution to local Rayon work. +/// +/// If the current thread is part of a rayon thread pool, this looks for a +/// single unit of pending work in this thread's queue, then executes it. +/// Completion of that work might include nested work or further work stealing. +/// +/// This is similar to [`yield_now()`], but does not steal from other threads. +/// +/// Returns `Some(Yield::Executed)` if anything was executed, `Some(Yield::Idle)` if +/// nothing was available, or `None` if this thread is not part of any pool at all. +pub fn yield_local() -> Option { + unsafe { + let thread = WorkerThread::current().as_ref()?; + Some(thread.yield_local()) + } +} + +/// Result of [`yield_now()`] or [`yield_local()`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Yield { + /// Work was found and executed. + Executed, + /// No available work was found. + Idle, +} diff --git a/anneal/v2/vendor/rayon-core/src/thread_pool/test.rs b/anneal/v2/vendor/rayon-core/src/thread_pool/test.rs new file mode 100644 index 0000000000..b13c41a605 --- /dev/null +++ b/anneal/v2/vendor/rayon-core/src/thread_pool/test.rs @@ -0,0 +1,418 @@ +#![cfg(test)] + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::mpsc::channel; +use std::sync::{Arc, Mutex}; + +use crate::{join, Scope, ScopeFifo, ThreadPool, ThreadPoolBuilder}; + +#[test] +#[should_panic(expected = "Hello, world!")] +fn panic_propagate() { + let thread_pool = ThreadPoolBuilder::new().build().unwrap(); + thread_pool.install(|| { + panic!("Hello, world!"); + }); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn workers_stop() { + let registry; + + { + // once we exit this block, thread pool will be dropped + let thread_pool = ThreadPoolBuilder::new().num_threads(22).build().unwrap(); + registry = thread_pool.install(|| { + // do some work on these threads + join_a_lot(22); + + Arc::clone(&thread_pool.registry) + }); + assert_eq!(registry.num_threads(), 22); + } + + // once thread pool is dropped, registry should terminate, which + // should lead to worker threads stopping + registry.wait_until_stopped(); +} + +fn join_a_lot(n: usize) { + if n > 0 { + join(|| join_a_lot(n - 1), || join_a_lot(n - 1)); + } +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn sleeper_stop() { + use std::{thread, time}; + + let registry; + + { + // once we exit this block, thread pool will be dropped + let thread_pool = ThreadPoolBuilder::new().num_threads(22).build().unwrap(); + registry = Arc::clone(&thread_pool.registry); + + // Give time for at least some of the thread pool to fall asleep. + thread::sleep(time::Duration::from_secs(1)); + } + + // once thread pool is dropped, registry should terminate, which + // should lead to worker threads stopping + registry.wait_until_stopped(); +} + +/// Creates a start/exit handler that increments an atomic counter. +fn count_handler() -> (Arc, impl Fn(usize)) { + let count = Arc::new(AtomicUsize::new(0)); + (Arc::clone(&count), move |_| { + count.fetch_add(1, Ordering::SeqCst); + }) +} + +/// Wait until a counter is no longer shared, then return its value. +fn wait_for_counter(mut counter: Arc) -> usize { + use std::{thread, time}; + + for _ in 0..60 { + counter = match Arc::try_unwrap(counter) { + Ok(counter) => return counter.into_inner(), + Err(counter) => { + thread::sleep(time::Duration::from_secs(1)); + counter + } + }; + } + + // That's too long! + panic!("Counter is still shared!"); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn failed_thread_stack() { + // Note: we first tried to force failure with a `usize::MAX` stack, but + // macOS and Windows weren't fazed, or at least didn't fail the way we want. + // They work with `isize::MAX`, but 32-bit platforms may feasibly allocate a + // 2GB stack, so it might not fail until the second thread. + let stack_size = isize::MAX as usize; + + let (start_count, start_handler) = count_handler(); + let (exit_count, exit_handler) = count_handler(); + let builder = ThreadPoolBuilder::new() + .num_threads(10) + .stack_size(stack_size) + .start_handler(start_handler) + .exit_handler(exit_handler); + + let pool = builder.build(); + assert!(pool.is_err(), "thread stack should have failed!"); + + // With such a huge stack, 64-bit will probably fail on the first thread; + // 32-bit might manage the first 2GB, but certainly fail the second. + let start_count = wait_for_counter(start_count); + assert!(start_count <= 1); + assert_eq!(start_count, wait_for_counter(exit_count)); +} + +#[test] +#[cfg_attr(not(panic = "unwind"), ignore)] +fn panic_thread_name() { + let (start_count, start_handler) = count_handler(); + let (exit_count, exit_handler) = count_handler(); + let builder = ThreadPoolBuilder::new() + .num_threads(10) + .start_handler(start_handler) + .exit_handler(exit_handler) + .thread_name(|i| { + if i >= 5 { + panic!(); + } + format!("panic_thread_name#{i}") + }); + + let pool = crate::unwind::halt_unwinding(|| builder.build()); + assert!(pool.is_err(), "thread-name panic should propagate!"); + + // Assuming they're created in order, threads 0 through 4 should have + // been started already, and then terminated by the panic. + assert_eq!(5, wait_for_counter(start_count)); + assert_eq!(5, wait_for_counter(exit_count)); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn self_install() { + let pool = ThreadPoolBuilder::new().num_threads(1).build().unwrap(); + + // If the inner `install` blocks, then nothing will actually run it! + assert!(pool.install(|| pool.install(|| true))); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn mutual_install() { + let pool1 = ThreadPoolBuilder::new().num_threads(1).build().unwrap(); + let pool2 = ThreadPoolBuilder::new().num_threads(1).build().unwrap(); + + let ok = pool1.install(|| { + // This creates a dependency from `pool1` -> `pool2` + pool2.install(|| { + // This creates a dependency from `pool2` -> `pool1` + pool1.install(|| { + // If they blocked on inter-pool installs, there would be no + // threads left to run this! + true + }) + }) + }); + assert!(ok); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn mutual_install_sleepy() { + use std::{thread, time}; + + let pool1 = ThreadPoolBuilder::new().num_threads(1).build().unwrap(); + let pool2 = ThreadPoolBuilder::new().num_threads(1).build().unwrap(); + + let ok = pool1.install(|| { + // This creates a dependency from `pool1` -> `pool2` + pool2.install(|| { + // Give `pool1` time to fall asleep. + thread::sleep(time::Duration::from_secs(1)); + + // This creates a dependency from `pool2` -> `pool1` + pool1.install(|| { + // Give `pool2` time to fall asleep. + thread::sleep(time::Duration::from_secs(1)); + + // If they blocked on inter-pool installs, there would be no + // threads left to run this! + true + }) + }) + }); + assert!(ok); +} + +#[test] +#[allow(deprecated)] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn check_thread_pool_new() { + let pool = ThreadPool::new(crate::Configuration::new().num_threads(22)).unwrap(); + assert_eq!(pool.current_num_threads(), 22); +} + +macro_rules! test_scope_order { + ($scope:ident => $spawn:ident) => {{ + let builder = ThreadPoolBuilder::new().num_threads(1); + let pool = builder.build().unwrap(); + pool.install(|| { + let vec = Mutex::new(vec![]); + pool.$scope(|scope| { + let vec = &vec; + for i in 0..10 { + scope.$spawn(move |_| { + vec.lock().unwrap().push(i); + }); + } + }); + vec.into_inner().unwrap() + }) + }}; +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn scope_lifo_order() { + let vec = test_scope_order!(scope => spawn); + let expected: Vec = (0..10).rev().collect(); // LIFO -> reversed + assert_eq!(vec, expected); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn scope_fifo_order() { + let vec = test_scope_order!(scope_fifo => spawn_fifo); + let expected: Vec = (0..10).collect(); // FIFO -> natural order + assert_eq!(vec, expected); +} + +macro_rules! test_spawn_order { + ($spawn:ident) => {{ + let builder = ThreadPoolBuilder::new().num_threads(1); + let pool = &builder.build().unwrap(); + let (tx, rx) = channel(); + pool.install(move || { + for i in 0..10 { + let tx = tx.clone(); + pool.$spawn(move || { + tx.send(i).unwrap(); + }); + } + }); + rx.iter().collect::>() + }}; +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn spawn_lifo_order() { + let vec = test_spawn_order!(spawn); + let expected: Vec = (0..10).rev().collect(); // LIFO -> reversed + assert_eq!(vec, expected); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn spawn_fifo_order() { + let vec = test_spawn_order!(spawn_fifo); + let expected: Vec = (0..10).collect(); // FIFO -> natural order + assert_eq!(vec, expected); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn nested_scopes() { + // Create matching scopes for every thread pool. + fn nest<'scope, OP>(pools: &[ThreadPool], scopes: Vec<&Scope<'scope>>, op: OP) + where + OP: FnOnce(&[&Scope<'scope>]) + Send, + { + if let Some((pool, tail)) = pools.split_first() { + pool.scope(move |s| { + // This move reduces the reference lifetimes by variance to match s, + // but the actual scopes are still tied to the invariant 'scope. + let mut scopes = scopes; + scopes.push(s); + nest(tail, scopes, op) + }) + } else { + (op)(&scopes) + } + } + + let pools: Vec<_> = (0..10) + .map(|_| ThreadPoolBuilder::new().num_threads(1).build().unwrap()) + .collect(); + + let counter = AtomicUsize::new(0); + nest(&pools, vec![], |scopes| { + for &s in scopes { + s.spawn(|_| { + // Our 'scope lets us borrow the counter in every pool. + counter.fetch_add(1, Ordering::Relaxed); + }); + } + }); + assert_eq!(counter.into_inner(), pools.len()); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn nested_fifo_scopes() { + // Create matching fifo scopes for every thread pool. + fn nest<'scope, OP>(pools: &[ThreadPool], scopes: Vec<&ScopeFifo<'scope>>, op: OP) + where + OP: FnOnce(&[&ScopeFifo<'scope>]) + Send, + { + if let Some((pool, tail)) = pools.split_first() { + pool.scope_fifo(move |s| { + // This move reduces the reference lifetimes by variance to match s, + // but the actual scopes are still tied to the invariant 'scope. + let mut scopes = scopes; + scopes.push(s); + nest(tail, scopes, op) + }) + } else { + (op)(&scopes) + } + } + + let pools: Vec<_> = (0..10) + .map(|_| ThreadPoolBuilder::new().num_threads(1).build().unwrap()) + .collect(); + + let counter = AtomicUsize::new(0); + nest(&pools, vec![], |scopes| { + for &s in scopes { + s.spawn_fifo(|_| { + // Our 'scope lets us borrow the counter in every pool. + counter.fetch_add(1, Ordering::Relaxed); + }); + } + }); + assert_eq!(counter.into_inner(), pools.len()); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn in_place_scope_no_deadlock() { + let pool = ThreadPoolBuilder::new().num_threads(1).build().unwrap(); + let (tx, rx) = channel(); + let rx_ref = ℞ + pool.in_place_scope(move |s| { + // With regular scopes this closure would never run because this scope op + // itself would block the only worker thread. + s.spawn(move |_| { + tx.send(()).unwrap(); + }); + rx_ref.recv().unwrap(); + }); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn in_place_scope_fifo_no_deadlock() { + let pool = ThreadPoolBuilder::new().num_threads(1).build().unwrap(); + let (tx, rx) = channel(); + let rx_ref = ℞ + pool.in_place_scope_fifo(move |s| { + // With regular scopes this closure would never run because this scope op + // itself would block the only worker thread. + s.spawn_fifo(move |_| { + tx.send(()).unwrap(); + }); + rx_ref.recv().unwrap(); + }); +} + +#[test] +fn yield_now_to_spawn() { + let (tx, rx) = channel(); + + // Queue a regular spawn. + crate::spawn(move || tx.send(22).unwrap()); + + // The single-threaded fallback mode (for wasm etc.) won't + // get a chance to run the spawn if we never yield to it. + crate::registry::in_worker(move |_, _| { + crate::yield_now(); + }); + + // The spawn **must** have started by now, but we still might have to wait + // for it to finish if a different thread stole it first. + assert_eq!(22, rx.recv().unwrap()); +} + +#[test] +fn yield_local_to_spawn() { + let (tx, rx) = channel(); + + // Queue a regular spawn. + crate::spawn(move || tx.send(22).unwrap()); + + // The single-threaded fallback mode (for wasm etc.) won't + // get a chance to run the spawn if we never yield to it. + crate::registry::in_worker(move |_, _| { + crate::yield_local(); + }); + + // The spawn **must** have started by now, but we still might have to wait + // for it to finish if a different thread stole it first. + assert_eq!(22, rx.recv().unwrap()); +} diff --git a/anneal/v2/vendor/rayon-core/src/unwind.rs b/anneal/v2/vendor/rayon-core/src/unwind.rs new file mode 100644 index 0000000000..9671fa5782 --- /dev/null +++ b/anneal/v2/vendor/rayon-core/src/unwind.rs @@ -0,0 +1,31 @@ +//! Package up unwind recovery. Note that if you are in some sensitive +//! place, you can use the `AbortIfPanic` helper to protect against +//! accidental panics in the rayon code itself. + +use std::any::Any; +use std::panic::{self, AssertUnwindSafe}; +use std::thread; + +/// Executes `f` and captures any panic, translating that panic into a +/// `Err` result. The assumption is that any panic will be propagated +/// later with `resume_unwinding`, and hence `f` can be treated as +/// exception safe. +pub(super) fn halt_unwinding(func: F) -> thread::Result +where + F: FnOnce() -> R, +{ + panic::catch_unwind(AssertUnwindSafe(func)) +} + +pub(super) fn resume_unwinding(payload: Box) -> ! { + panic::resume_unwind(payload) +} + +pub(super) struct AbortIfPanic; + +impl Drop for AbortIfPanic { + fn drop(&mut self) { + eprintln!("Rayon: detected unexpected panic; aborting"); + ::std::process::abort(); + } +} diff --git a/anneal/v2/vendor/rayon-core/tests/double_init_fail.rs b/anneal/v2/vendor/rayon-core/tests/double_init_fail.rs new file mode 100644 index 0000000000..15915304dd --- /dev/null +++ b/anneal/v2/vendor/rayon-core/tests/double_init_fail.rs @@ -0,0 +1,15 @@ +use rayon_core::ThreadPoolBuilder; +use std::error::Error; + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn double_init_fail() { + let result1 = ThreadPoolBuilder::new().build_global(); + assert!(result1.is_ok()); + let err = ThreadPoolBuilder::new().build_global().unwrap_err(); + assert!(err.source().is_none()); + assert_eq!( + err.to_string(), + "The global thread pool has already been initialized.", + ); +} diff --git a/anneal/v2/vendor/rayon-core/tests/init_zero_threads.rs b/anneal/v2/vendor/rayon-core/tests/init_zero_threads.rs new file mode 100644 index 0000000000..3c1ad251c7 --- /dev/null +++ b/anneal/v2/vendor/rayon-core/tests/init_zero_threads.rs @@ -0,0 +1,10 @@ +use rayon_core::ThreadPoolBuilder; + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn init_zero_threads() { + ThreadPoolBuilder::new() + .num_threads(0) + .build_global() + .unwrap(); +} diff --git a/anneal/v2/vendor/rayon-core/tests/scope_join.rs b/anneal/v2/vendor/rayon-core/tests/scope_join.rs new file mode 100644 index 0000000000..9d88133bc5 --- /dev/null +++ b/anneal/v2/vendor/rayon-core/tests/scope_join.rs @@ -0,0 +1,45 @@ +/// Test that one can emulate join with `scope`: +fn pseudo_join(f: F, g: G) +where + F: FnOnce() + Send, + G: FnOnce() + Send, +{ + rayon_core::scope(|s| { + s.spawn(|_| g()); + f(); + }); +} + +fn quick_sort(v: &mut [T]) { + if v.len() <= 1 { + return; + } + + let mid = partition(v); + let (lo, hi) = v.split_at_mut(mid); + pseudo_join(|| quick_sort(lo), || quick_sort(hi)); +} + +fn partition(v: &mut [T]) -> usize { + let pivot = v.len() - 1; + let mut i = 0; + for j in 0..pivot { + if v[j] <= v[pivot] { + v.swap(i, j); + i += 1; + } + } + v.swap(i, pivot); + i +} + +fn is_sorted(v: &[T]) -> bool { + (1..v.len()).all(|i| v[i - 1] <= v[i]) +} + +#[test] +fn scope_join() { + let mut v: Vec = (0..256).rev().collect(); + quick_sort(&mut v); + assert!(is_sorted(&v)); +} diff --git a/anneal/v2/vendor/rayon-core/tests/scoped_threadpool.rs b/anneal/v2/vendor/rayon-core/tests/scoped_threadpool.rs new file mode 100644 index 0000000000..932147179f --- /dev/null +++ b/anneal/v2/vendor/rayon-core/tests/scoped_threadpool.rs @@ -0,0 +1,99 @@ +use crossbeam_utils::thread; +use rayon_core::ThreadPoolBuilder; + +#[derive(PartialEq, Eq, Debug)] +struct Local(i32); + +scoped_tls::scoped_thread_local!(static LOCAL: Local); + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn missing_scoped_tls() { + LOCAL.set(&Local(42), || { + let pool = ThreadPoolBuilder::new() + .build() + .expect("thread pool created"); + + // `LOCAL` is not set in the pool. + pool.install(|| { + assert!(!LOCAL.is_set()); + }); + }); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn spawn_scoped_tls_threadpool() { + LOCAL.set(&Local(42), || { + LOCAL.with(|x| { + thread::scope(|scope| { + let pool = ThreadPoolBuilder::new() + .spawn_handler(move |thread| { + scope + .builder() + .spawn(move |_| { + // Borrow the same local value in the thread pool. + LOCAL.set(x, || thread.run()) + }) + .map(|_| ()) + }) + .build() + .expect("thread pool created"); + + // The pool matches our local value. + pool.install(|| { + assert!(LOCAL.is_set()); + LOCAL.with(|y| { + assert_eq!(x, y); + }); + }); + + // If we change our local value, the pool is not affected. + LOCAL.set(&Local(-1), || { + pool.install(|| { + assert!(LOCAL.is_set()); + LOCAL.with(|y| { + assert_eq!(x, y); + }); + }); + }); + }) + .expect("scope threads ok"); + // `thread::scope` will wait for the threads to exit before returning. + }); + }); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn build_scoped_tls_threadpool() { + LOCAL.set(&Local(42), || { + LOCAL.with(|x| { + ThreadPoolBuilder::new() + .build_scoped( + move |thread| LOCAL.set(x, || thread.run()), + |pool| { + // The pool matches our local value. + pool.install(|| { + assert!(LOCAL.is_set()); + LOCAL.with(|y| { + assert_eq!(x, y); + }); + }); + + // If we change our local value, the pool is not affected. + LOCAL.set(&Local(-1), || { + pool.install(|| { + assert!(LOCAL.is_set()); + LOCAL.with(|y| { + assert_eq!(x, y); + }); + }); + }); + }, + ) + .expect("thread pool created"); + // Internally, `std::thread::scope` will wait for the threads to exit before returning. + }); + }); +} diff --git a/anneal/v2/vendor/rayon-core/tests/simple_panic.rs b/anneal/v2/vendor/rayon-core/tests/simple_panic.rs new file mode 100644 index 0000000000..2564482a47 --- /dev/null +++ b/anneal/v2/vendor/rayon-core/tests/simple_panic.rs @@ -0,0 +1,7 @@ +use rayon_core::join; + +#[test] +#[should_panic(expected = "should panic")] +fn simple_panic() { + join(|| {}, || panic!("should panic")); +} diff --git a/anneal/v2/vendor/rayon-core/tests/stack_overflow_crash.rs b/anneal/v2/vendor/rayon-core/tests/stack_overflow_crash.rs new file mode 100644 index 0000000000..7dcde43c42 --- /dev/null +++ b/anneal/v2/vendor/rayon-core/tests/stack_overflow_crash.rs @@ -0,0 +1,97 @@ +use rayon_core::ThreadPoolBuilder; + +use std::env; +use std::process::{Command, ExitStatus, Stdio}; + +#[cfg(target_os = "linux")] +use std::os::unix::process::ExitStatusExt; + +fn force_stack_overflow(depth: u32) { + let mut buffer = [0u8; 1024 * 1024]; + std::hint::black_box(&mut buffer); + if depth > 0 { + force_stack_overflow(depth - 1); + } +} + +#[cfg(unix)] +fn disable_core() { + unsafe { + libc::setrlimit( + libc::RLIMIT_CORE, + &libc::rlimit { + rlim_cur: 0, + rlim_max: 0, + }, + ); + } +} + +#[cfg(unix)] +fn overflow_code() -> Option { + None +} + +#[cfg(windows)] +fn overflow_code() -> Option { + use std::os::windows::process::ExitStatusExt; + + ExitStatus::from_raw(0xc00000fd /*STATUS_STACK_OVERFLOW*/).code() +} + +#[test] +#[cfg_attr(not(any(unix, windows)), ignore)] +fn stack_overflow_crash() { + // First check that the recursive call actually causes a stack overflow, + // and does not get optimized away. + let status = run_ignored("run_with_small_stack"); + assert!(!status.success()); + #[cfg(any(unix, windows))] + assert_eq!(status.code(), overflow_code()); + #[cfg(target_os = "linux")] + assert!(matches!( + status.signal(), + Some(libc::SIGABRT | libc::SIGSEGV) + )); + + // Now run with a larger stack and verify correct operation. + let status = run_ignored("run_with_large_stack"); + assert_eq!(status.code(), Some(0)); + #[cfg(target_os = "linux")] + assert_eq!(status.signal(), None); +} + +fn run_ignored(test: &str) -> ExitStatus { + Command::new(env::current_exe().unwrap()) + .arg("--ignored") + .arg("--exact") + .arg(test) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .unwrap() +} + +#[test] +#[ignore] +fn run_with_small_stack() { + run_with_stack(8); +} + +#[test] +#[ignore] +fn run_with_large_stack() { + run_with_stack(48); +} + +fn run_with_stack(stack_size_in_mb: usize) { + let pool = ThreadPoolBuilder::new() + .stack_size(stack_size_in_mb * 1024 * 1024) + .build() + .unwrap(); + pool.install(|| { + #[cfg(unix)] + disable_core(); + force_stack_overflow(32); + }); +} diff --git a/anneal/v2/vendor/rayon-core/tests/use_current_thread.rs b/anneal/v2/vendor/rayon-core/tests/use_current_thread.rs new file mode 100644 index 0000000000..c509c9bb30 --- /dev/null +++ b/anneal/v2/vendor/rayon-core/tests/use_current_thread.rs @@ -0,0 +1,57 @@ +use rayon_core::ThreadPoolBuilder; +use std::sync::{Arc, Condvar, Mutex}; +use std::thread::{self, JoinHandle}; + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn use_current_thread_basic() { + static JOIN_HANDLES: Mutex>> = Mutex::new(Vec::new()); + let pool = ThreadPoolBuilder::new() + .num_threads(2) + .use_current_thread() + .spawn_handler(|builder| { + let handle = thread::Builder::new().spawn(|| builder.run())?; + JOIN_HANDLES.lock().unwrap().push(handle); + Ok(()) + }) + .build() + .unwrap(); + assert_eq!(rayon_core::current_thread_index(), Some(0)); + assert_eq!( + JOIN_HANDLES.lock().unwrap().len(), + 1, + "Should only spawn one extra thread" + ); + + let another_pool = ThreadPoolBuilder::new() + .num_threads(2) + .use_current_thread() + .build(); + assert!( + another_pool.is_err(), + "Should error if the thread is already part of a pool" + ); + + let pair = Arc::new((Mutex::new(false), Condvar::new())); + let pair2 = Arc::clone(&pair); + pool.spawn(move || { + assert_ne!(rayon_core::current_thread_index(), Some(0)); + // This should execute even if the current thread is blocked, since we have two threads in + // the pool. + let (started, condvar) = &*pair2; + *started.lock().unwrap() = true; + condvar.notify_one(); + }); + + let _guard = pair + .1 + .wait_while(pair.0.lock().unwrap(), |ran| !*ran) + .unwrap(); + std::mem::drop(pool); // Drop the pool. + + // Wait until all threads have actually exited. This is not really needed, other than to + // reduce noise of leak-checking tools. + for handle in std::mem::take(&mut *JOIN_HANDLES.lock().unwrap()) { + let _ = handle.join(); + } +} diff --git a/anneal/v2/vendor/rayon/.cargo-checksum.json b/anneal/v2/vendor/rayon/.cargo-checksum.json new file mode 100644 index 0000000000..b94119d80c --- /dev/null +++ b/anneal/v2/vendor/rayon/.cargo-checksum.json @@ -0,0 +1 @@ +{"files":{".cargo_vcs_info.json":"42c572ec636a8257d2f1017b0d83d234c2835b5ba7ca40375eeddd311ae071f9","Cargo.lock":"ef6c4aa754fdb92783965ad5b80f35a1a06d57e5660adc83882cbcf29a417977","Cargo.toml":"bcc6eda50025991d8db6ccaaad2e26477072a359762451a4e55a7e700ad0e42d","Cargo.toml.orig":"730637d19a4ae811bae3f60000a6cceb5f052b1fd5e76946459ef5b31ecdf399","FAQ.md":"3b47ed781d832942f5f45db9acbf102df41b401111b6fd5303af911101746449","LICENSE-APACHE":"a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2","LICENSE-MIT":"0621878e61f0d0fda054bcbe02df75192c28bde1ecc8289cbd86aeba2dd72720","README.md":"9bd7b1a9b53023afc7220db62f6b7f580f56f2d2d98b521b07569749bfea6329","RELEASES.md":"3120f4b516120b1a6fd005916d914a639f21bbf3b66e524aa05339a4345a85e8","src/array.rs":"e3889df405f6cb773cebe0205fb53256775cc6dc1c99d409a940cd48b8cf3bfd","src/collections/binary_heap.rs":"dcfaa5b7c3d658ad8894c8c1269bb11b517ad2fe76b1c9771666f15cdba10b1a","src/collections/btree_map.rs":"9d723cdf69698c7def1e495ba179c62200166b9a3ba16ec849c893f99081eb55","src/collections/btree_set.rs":"ee251d95a8226dff2eb087965804dac388c502f730d48587f054ad45a81c1c06","src/collections/hash_map.rs":"19b55af98d8f93375148b2d66d249054d85e0db6ac0c33a273281140b0f61965","src/collections/hash_set.rs":"3b267e2bf9d2a10e0bea43d183729f41d1cc45c6691daf5850a40f0c67271e30","src/collections/linked_list.rs":"e6e450293666386fc140ced0ffc1717464e162394b2af8706ad9b76a0f0d43d3","src/collections/mod.rs":"07a4a9c3c627ab8c181cf30b334c78c633c752bd7984593ecc52e13168f58249","src/collections/vec_deque.rs":"29d7d9e9ee23c3e218eeaa6dddef32d0da1b5f7ce003209029b42151d8902d5a","src/compile_fail/cannot_collect_filtermap_data.rs":"f76e6727ae8bd0dc8fff5f4a79df3fc598daf5d7371bdf8b2e730fba4ba82719","src/compile_fail/cannot_zip_filtered_data.rs":"3588fcf72e804ea867ea525b6dfa7a3d45fe14e28088e849fa9ddb029acc8a7a","src/compile_fail/cell_par_iter.rs":"ebf8804993c103f9d8228aba0bb2a5f0aadb957de9f1eb59cf546dbace285201","src/compile_fail/mod.rs":"2c346e4b52941fe0ac2667eeb753c0c21a1f998a544bb9d9167f200caba0a7bb","src/compile_fail/must_use.rs":"39ab32af552c2b35e67593b54da146d8edd8e022dbadcf7a81983983a065bbc2","src/compile_fail/no_send_par_iter.rs":"b241446439c659f35060df12ba91590ea4267f373ddc688e4ffc203321db24b3","src/compile_fail/rc_par_iter.rs":"938776b08612e35c85b99962024d54af5a8c87d45a050a76375d16ef6fe4299f","src/delegate.rs":"aad2a11998240fb7dd96bd054b588cac469df3b716b8dffb05239102b6507d05","src/iter/blocks.rs":"0f0dce562c879f47eb5c2e24a41943c216c56268af68e8e0b22f74928ffe46e1","src/iter/chain.rs":"63bbeb40c7f5a472184efe15a42fdcb7bd08122bbb8533aced5acb926e894ce2","src/iter/chunks.rs":"03c70c4d71e69bc2721f9549c3451154593690f55baae76370bf8fd5b818e595","src/iter/cloned.rs":"474a33608dd1c4e65c1121837d546dfe2922328ad512d2ad3c56eb3c9016d022","src/iter/collect/consumer.rs":"8c47eb678385a245897b6bf24329a41cfc9d12c6db1875a25f1a0bbf19e868e1","src/iter/collect/mod.rs":"c0338cdcd3a7e45dc4476a208d8bd26fed9049e33e3995591059914b46971393","src/iter/collect/test.rs":"734e105f777de9d5e12cb103dac3a728d3d8e3a201f4f3f765c026d99d93376f","src/iter/copied.rs":"572fede95761180c4e42f377f05a3a67fd6a700b66488d563c7feb30e569e36a","src/iter/empty.rs":"5b39bb26be7735b320da72264ffe3678c2c9f813aa8efe0b25c3deae1722bfb0","src/iter/enumerate.rs":"41f517a1ad788dd82ac8dc75d90339a2203d647b8adb8b1b8b4d7c2527886e26","src/iter/extend.rs":"7f1d1f4e90f046ef2fd35736850aab0c1a92395cf83175f93f6bdac74c48f5b3","src/iter/filter.rs":"1440688f50004c466d1ba1f3c4bd6ffefbc9614a7fc53e7c4c28efdc61cbf29a","src/iter/filter_map.rs":"8048566363eb2b2b87244cef2a1505bc288d962a9d3e7918b44a1b03c038e5fe","src/iter/find.rs":"896ddb05b2fa7368462e0ff2a73982ced5f93266c0e0e8c27bb3fc4ec737af21","src/iter/find_first_last/mod.rs":"2dfd1e960f3c0e5aa003fea8a31433dbaef270117de91733b4a0cd096a9935f5","src/iter/find_first_last/test.rs":"890034e32485971e8a39d4004f1bf730ac6bbd4190ae3780d6542707a4ce4abc","src/iter/flat_map.rs":"8ce5f09aa573e42af379cf7e271c7f28f8e841b8c8db6934c7f247bb2ee7c2fb","src/iter/flat_map_iter.rs":"db0ba448895fae8f404083d2a00f867fe72b350f6a93f5140d49f76f5e9c5f8b","src/iter/flatten.rs":"945265a13b9f83e73b0ac188bf89fbe6ba851d6e1ec4dd3cb05c2bccdf22f28b","src/iter/flatten_iter.rs":"e7a5fd5a9d206d78b7579b5f8f390af079c70acbc1665cfe60df70d81c9067bb","src/iter/fold.rs":"109d175087df6b6f419d486ad2883c3c5571e5421184f667fee972cf6df2c980","src/iter/fold_chunks.rs":"567c7bc65ec4d2355ae7f65aca975bccc2f4c1b2aeaf8bbe7cf7876f62b3e211","src/iter/fold_chunks_with.rs":"49a1d2fd8a1864773b7c72ba572c573d9fcae03ff145203f9f7692d8d9db9be6","src/iter/for_each.rs":"7af0e21ed8479eec65831d1409f61a88d45a31764f2385ec759eda1a46d388b2","src/iter/from_par_iter.rs":"140df213a38d4fc4b61b21fcf7e0b1153da7e04eb6ed95f00cd47269691380bc","src/iter/inspect.rs":"9cd0304be9dfef02fbecb240cfea2c0549a0c45f5f7e18d3a16deefc88ecf038","src/iter/interleave.rs":"ad4b282a9793292a0166bac5568abaf5f62f75068a6a8710e837f885484fb2f3","src/iter/interleave_shortest.rs":"ecb777284b0f06771082079792379998106dc00ed4c02b2b9722e81b5decf980","src/iter/intersperse.rs":"2df834a436d307541f28e26c9f7aea83f072b08b4517b79c4e53bc5c27a47857","src/iter/len.rs":"a47a96f49be5569b54701e8911a4273edca1b665b60e68f20b6c6ea5d51f594e","src/iter/map.rs":"ef57c66e90421096a4cf971a014bb0c109a888b946ab4c3a24960a976339a1aa","src/iter/map_with.rs":"33ede9b43a30210c1e7e962f040a57625514ad0cfee2a41d967265b88e5bd49b","src/iter/mod.rs":"2d011a9262aef7231d8fb8ed1129b9e13a041ca26565f0a8cdc46fdcde91b50c","src/iter/multizip.rs":"330634cc61a2f1f3138d67608e27d853e5f7b43921094dbfe38a7fd807c39bbc","src/iter/noop.rs":"5be6332ddfbb8fdbae1ffdb00983950a8b37a295bcb58e9a265b33806ee504e6","src/iter/once.rs":"b6c4ac9654210a8c55ddf6e56080370154f6b9df3c06312ebafcb5e4d60fa34b","src/iter/panic_fuse.rs":"fceff8a02a67ea90bb182127e5ea99c372be629d0aabc5d8366e771c4a4b8c30","src/iter/par_bridge.rs":"ba4457370b5f1c53ce02c1512604d29849a82d741f02780eeccfe6c32ef7a18c","src/iter/plumbing/README.md":"da11d90143745b30efe99934bb5d8267d7b205c8096d0936ddb1bbba0b13ea17","src/iter/plumbing/mod.rs":"c03482b78af72212b795df90fa5d3af2c49fa0fd1a08addcaeb02a730db1918a","src/iter/positions.rs":"639a68213d2a3e48f802102fd3ba09d5dcc69dd96dbb7d0dd0fe09c74533ff87","src/iter/product.rs":"eaddbf216b068d33cb92f594cf57522f6ea1e2d7699887a086484843bdec3585","src/iter/reduce.rs":"2f5d6e07d7c0de2360505fa5d9198c31fd43ba7e58a6ec40f79edec19319e502","src/iter/repeat.rs":"ff800e82304ed0ab1101ac0f21177eba04b498cdb3f99117d28d29e702cb4c3e","src/iter/rev.rs":"954d71773e366d7139d71a2468c3079cf26a711f6c39dcf8c25518ff5773cabe","src/iter/skip.rs":"177755cf51a78ed8ea546fea8b064d1af0c2dd39ddc932351920b0dbc5b8fc74","src/iter/skip_any.rs":"0ff1ea2751da6f29af8a0de14464e9c1ceff07d22498f8320c99803f6000eed3","src/iter/skip_any_while.rs":"8f334f7e1774cb4a4cc591d6dcf80544c1088347df6bcd5f71b576e168aa16e6","src/iter/splitter.rs":"775f11fe4ddf7066426da90a41de90e35bf5686dfba4b2d9105f2e9ce2c6f41e","src/iter/step_by.rs":"ed8384ea24606d324ec630a381e50eeee1404a9d75d76c447328c2d059b6898b","src/iter/sum.rs":"073a886b8d6b0d7ca5621449a20e9980c90c6d39de56f074371fb075d9df2bed","src/iter/take.rs":"5579efee3552bb841b1d9e0577802b8921bf1d75889efa58665130628466f94a","src/iter/take_any.rs":"e6f3100c7d5712ebe81bbe17d6d1647881167100525c2f10038b2c727202747f","src/iter/take_any_while.rs":"4716f032b223c064ea27317b9f028b42f53d171f13e859c8177217636c94056d","src/iter/test.rs":"c825efce0b9374dab30474d59a9d2b710b22597e4d90e93daf2fd1491b60e86f","src/iter/try_fold.rs":"dcd0c27b0183c8046e7443afaf3a874c60f26bb5b56c5d637e0871ab44c37215","src/iter/try_reduce.rs":"12317a649f777e76e6ae610d205104d7946fbe45804fbf1caa0843118531baed","src/iter/try_reduce_with.rs":"9171563fc22110d7a359f19de7ca66a6823d8f712099d05d01560795781fdeec","src/iter/unzip.rs":"9534982854873517e66b7387160cbb66c78004dcbb3ed0e7904e464dbc7b6649","src/iter/update.rs":"4fbaed7dac25fb42ed6a3eb0d9521fb2435413532b243594ae89f87a7d174976","src/iter/walk_tree.rs":"aca6d5b34993e54fb247f3e22b44bf2f3c69b262ab1b5a5d572d71ae4ec7eeb1","src/iter/while_some.rs":"8f3276171ee6f21ceec84e9788300afad2f694a2957702f389470904485de32b","src/iter/zip.rs":"116f65704f553bfa570017ec6ce702d1d9932dc7a7334b4c2ad294ce4aaa3af7","src/iter/zip_eq.rs":"35e9fc4eb305e06dd45c1c38dd52cb2b750f69601c9263566674b682c6fe7246","src/lib.rs":"40b39bfb9a469f8a4d78d0833166e7ce2b89bbd9cf199e5dac86ad5e6301d16d","src/math.rs":"4462c9edacd456834fe3fd39fc4f53d9187107934069ce7c1e81622672c82763","src/option.rs":"2ec9b7dc5d94e81cbd4ca6611ac3a6242e47e760f8961680b5cc5ef5f4eff97b","src/par_either.rs":"afa4b04ba6ea1d37aed2d68eca44d7ba0d1d09ea985c9091540dd8d3c51974f1","src/prelude.rs":"b1af578abff57f9c8a285c39e3793125cf40d9901d0f2a4f0c5f1bb9252a08de","src/private.rs":"152f6d65ce4741616a1dec796b9442f78a018d38bb040f76c4cd85008333a3bb","src/range.rs":"e60c755faaff73ea5fa85e401a71efe60a12a036e33c105a9ac8e475744f2d58","src/range_inclusive.rs":"da0dc1191f554fa9ccd0a360660514ff2ce7d94f1ec25a7356bfedb12084641e","src/result.rs":"ab80ce9f0b19d0a375e0de91d1c79a926fa3cf43ad151d2ceebb7ecfbcf79e95","src/slice/chunk_by.rs":"b75999500ba13dee5d2b78ea026b66b564768874127397177681177fb52b5657","src/slice/chunks.rs":"0f20d0afa6f6fe2b6a65cb67d6460ee89b5b3bfc69ebbc5d91388e101b11201f","src/slice/mod.rs":"59a08f86184d01c276e921996943114d35eeacde68950e84ccadfd1957dae713","src/slice/rchunks.rs":"c3267e21a0a20593f61b7d381050a205aca22f3f78c1ff7483d057f7d2736121","src/slice/sort.rs":"780813f9f2b740e261b9a7032fc4ab661772284464d834f47c0abbb4fa29a422","src/slice/test.rs":"5f38a9552c08a6db0e2a523dfe018992e3590a17f95203722a07ffa6e5405e17","src/slice/windows.rs":"f2506ea36905e56f37afdf9bdce7b32565ba06827a59c4489553df747107e3a4","src/split_producer.rs":"b2633e62bf87ccdc81d0d738de8f184f5892129fc31e81bc891e978b7179e695","src/str.rs":"1e2df4342a535c278cc801028a7e384ba8c46911d2d733cf5849c9e4a0df6e86","src/string.rs":"6691dd31264bd93a0528fc584585415c12f56cfb4baebbfd31ea2f1b21550f77","src/vec.rs":"a482c3eb267921d515e3089ceb67a6357921b0d781a3e9a313ddb38c34783018","tests/chars.rs":"6e1aeda8aa604f15965aa3c735da07f2145bd06079abb684b70598c86d99a263","tests/clones.rs":"071a28cd27715691b2eb4e1ac2f7dc0e7b3ec6626be2493cfc1103fde7d7f622","tests/collect.rs":"6ca03103e97936bed1d7b844e6e792cc2bf845d442567c29e224b9635c6b895b","tests/cross-pool.rs":"d2ce3b9013f77c955aa02313b27436c5bf760ab0e6a1e7d768effae69b5bdbfe","tests/debug.rs":"6eaa5ac5c6e0bc713cffd9ef8756eba325d5afd418ce56d88a080f028215d854","tests/drain_vec.rs":"305ca40ba41ed3b459a334386cd742c66586d62aadfb624278aabdf10b872a12","tests/intersperse.rs":"bda4fb2179086e32c77c8293b9bb85d55509c282441837ba1849c2d3aa3186a7","tests/issue671-unzip.rs":"d5eb38d8a6d8f66fdf1c40293abbf58f3ac42b5acfc5dca62b02c7ace5bfc1a4","tests/issue671.rs":"52914cac517074deaedcb81bd76b867f0b99cc7b65c3e01cfe12dc9fe38c0266","tests/iter_panic.rs":"2ca857b37b740528b2e8395b479a409227383b46e0ed012286dcdd1330dacd70","tests/named-threads.rs":"f1df1f522e81c63ad3e02be7e4107daba1d8c1e8f4fd93e0b144910134ea330d","tests/octillion.rs":"756d3789300d6eb4d5f7690937e06f87714b6b0a62ac812f72b622bb6974ccd2","tests/par_bridge_recursion.rs":"6e7030de6d97d1db1c335f676dd2a6348cde2873520df290c4a16bf949c06f00","tests/producer_split_at.rs":"0a9b824d2c1012b978cffe58786a3df88c6a2deb76e1f99520237d7c90c1c1a3","tests/sort-panic-safe.rs":"27199d55ca4c6b57ac7af121713a1abbc0c93071722539202f359bbecf777a6c","tests/str.rs":"7155d4a3ae920eb37d40cc58d538052302cfb45517dbafec685b414305e2c98f"},"package":"fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"} \ No newline at end of file diff --git a/anneal/v2/vendor/rayon/.cargo_vcs_info.json b/anneal/v2/vendor/rayon/.cargo_vcs_info.json new file mode 100644 index 0000000000..0e880e9603 --- /dev/null +++ b/anneal/v2/vendor/rayon/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "c9ced185ae3508246a9eb70c8407a1199bb1b77f" + }, + "path_in_vcs": "" +} \ No newline at end of file diff --git a/anneal/v2/vendor/rayon/Cargo.lock b/anneal/v2/vendor/rayon/Cargo.lock new file mode 100644 index 0000000000..f763800def --- /dev/null +++ b/anneal/v2/vendor/rayon/Cargo.lock @@ -0,0 +1,297 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "js-sys" +version = "0.3.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.185" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rayon" +version = "1.12.0" +dependencies = [ + "either", + "rand", + "rand_xorshift", + "rayon-core", + "wasm_sync", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", + "wasm_sync", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm_sync" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cff360cade7fec41ff0e9d2cda57fe58258c5f16def0e21302394659e6bbb0ea" +dependencies = [ + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" + +[[package]] +name = "zerocopy" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/anneal/v2/vendor/rayon/Cargo.toml b/anneal/v2/vendor/rayon/Cargo.toml new file mode 100644 index 0000000000..0fc560d876 --- /dev/null +++ b/anneal/v2/vendor/rayon/Cargo.toml @@ -0,0 +1,130 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +rust-version = "1.80" +name = "rayon" +version = "1.12.0" +build = false +exclude = [ + "/ci/*", + "/.github/*", +] +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Simple work-stealing parallelism for Rust" +documentation = "https://docs.rs/rayon/" +readme = "README.md" +keywords = [ + "parallel", + "thread", + "concurrency", + "join", + "performance", +] +categories = ["concurrency"] +license = "MIT OR Apache-2.0" +repository = "https://github.com/rayon-rs/rayon" + +[features] +web_spin_lock = [ + "dep:wasm_sync", + "rayon-core/web_spin_lock", +] + +[lib] +name = "rayon" +path = "src/lib.rs" + +[[test]] +name = "chars" +path = "tests/chars.rs" + +[[test]] +name = "clones" +path = "tests/clones.rs" + +[[test]] +name = "collect" +path = "tests/collect.rs" + +[[test]] +name = "cross-pool" +path = "tests/cross-pool.rs" + +[[test]] +name = "debug" +path = "tests/debug.rs" + +[[test]] +name = "drain_vec" +path = "tests/drain_vec.rs" + +[[test]] +name = "intersperse" +path = "tests/intersperse.rs" + +[[test]] +name = "issue671" +path = "tests/issue671.rs" + +[[test]] +name = "issue671-unzip" +path = "tests/issue671-unzip.rs" + +[[test]] +name = "iter_panic" +path = "tests/iter_panic.rs" + +[[test]] +name = "named-threads" +path = "tests/named-threads.rs" + +[[test]] +name = "octillion" +path = "tests/octillion.rs" + +[[test]] +name = "par_bridge_recursion" +path = "tests/par_bridge_recursion.rs" + +[[test]] +name = "producer_split_at" +path = "tests/producer_split_at.rs" + +[[test]] +name = "sort-panic-safe" +path = "tests/sort-panic-safe.rs" + +[[test]] +name = "str" +path = "tests/str.rs" + +[dependencies.either] +version = "1" +default-features = false + +[dependencies.rayon-core] +version = "1.13.0" + +[dependencies.wasm_sync] +version = "0.1.0" +optional = true + +[dev-dependencies.rand] +version = "0.9" + +[dev-dependencies.rand_xorshift] +version = "0.4" diff --git a/anneal/v2/vendor/rayon/Cargo.toml.orig b/anneal/v2/vendor/rayon/Cargo.toml.orig new file mode 100644 index 0000000000..4fdfcffab2 --- /dev/null +++ b/anneal/v2/vendor/rayon/Cargo.toml.orig @@ -0,0 +1,57 @@ +[package] +name = "rayon" +version = "1.12.0" +description = "Simple work-stealing parallelism for Rust" +documentation = "https://docs.rs/rayon/" +exclude = ["/ci/*", "/.github/*"] + +categories.workspace = true +edition.workspace = true +keywords.workspace = true +license.workspace = true +readme.workspace = true +repository.workspace = true +rust-version.workspace = true + +[features] +# This feature switches to a spin-lock implementation on the browser's +# main thread to avoid the forbidden `atomics.wait`. +# +# Only useful on the `wasm32-unknown-unknown` target. +web_spin_lock = ["dep:wasm_sync", "rayon-core/web_spin_lock"] + +[dependencies] +# These are both public dependencies! +rayon-core = { version = "1.13.0", path = "rayon-core" } +either.workspace = true + +wasm_sync = { workspace = true, optional = true } + +[dev-dependencies] +rand.workspace = true +rand_xorshift.workspace = true + + +[workspace] +members = ["rayon-demo", "rayon-core"] +exclude = ["ci"] + +[workspace.package] +readme = "README.md" +rust-version = "1.80" +edition = "2021" +license = "MIT OR Apache-2.0" +repository = "https://github.com/rayon-rs/rayon" +keywords = ["parallel", "thread", "concurrency", "join", "performance"] +categories = ["concurrency"] + +# Some dependencies may not be their latest version, in order to support older rustc. +[workspace.dependencies] +crossbeam-deque = "0.8.1" +crossbeam-utils = "0.8.0" +either = { version = "1", default-features = false } +libc = "0.2" +rand = "0.9" +rand_xorshift = "0.4" +scoped-tls = "1.0" +wasm_sync = "0.1.0" diff --git a/anneal/v2/vendor/rayon/FAQ.md b/anneal/v2/vendor/rayon/FAQ.md new file mode 100644 index 0000000000..e66185ccac --- /dev/null +++ b/anneal/v2/vendor/rayon/FAQ.md @@ -0,0 +1,213 @@ +# Rayon FAQ + +This file is for general questions that don't fit into the README or crate docs. + +## How many threads will Rayon spawn? + +By default, Rayon uses the same number of threads as the number of CPUs +available. Note that on systems with hyperthreading enabled this equals the +number of logical cores and not the physical ones. + +If you want to alter the number of threads spawned, you can set the +environmental variable `RAYON_NUM_THREADS` to the desired number of threads or +use the +[`ThreadPoolBuilder::build_global` function](https://docs.rs/rayon/*/rayon/struct.ThreadPoolBuilder.html#method.build_global) +method. + +## How does Rayon balance work between threads? + +Behind the scenes, Rayon uses a technique called **work stealing** to try and +dynamically ascertain how much parallelism is available and exploit it. The idea +is very simple: we always have a pool of worker threads available, waiting for +some work to do. When you call `join` the first time, we shift over into that +pool of threads. But if you call `join(a, b)` from a worker thread W, then W +will place `b` into its work queue, advertising that this is work that other +worker threads might help out with. W will then start executing `a`. + +While W is busy with `a`, other threads might come along and take `b` from its +queue. That is called *stealing* `b`. Once `a` is done, W checks whether `b` was +stolen by another thread and, if not, executes `b` itself. If W runs out of jobs +in its own queue, it will look through the other threads' queues and try to +steal work from them. + +This technique is not new. It was first introduced by the [Cilk project][cilk], +done at MIT in the late nineties. The name Rayon is an homage to that work. + +[cilk]: http://supertech.csail.mit.edu/cilk/ + +## What should I do if I use `Rc`, `Cell`, `RefCell` or other non-Send-and-Sync types? + +There are a number of non-threadsafe types in the Rust standard library, and if +your code is using them, you will not be able to combine it with Rayon. +Similarly, even if you don't have such types, but you try to have multiple +closures mutating the same state, you will get compilation errors; for example, +this function won't work, because both closures access `slice`: + +```rust +/// Increment all values in slice. +fn increment_all(slice: &mut [i32]) { + rayon::join(|| process(slice), || process(slice)); +} +``` + +The correct way to resolve such errors will depend on the case. Some cases are +easy: for example, uses of [`Rc`] can typically be replaced with [`Arc`], which +is basically equivalent, but thread-safe. + +Code that uses `Cell` or `RefCell`, however, can be somewhat more complicated. +If you can refactor your code to avoid those types, that is often the best way +forward, but otherwise, you can try to replace those types with their threadsafe +equivalents: + +- `Cell` -- replacement: `AtomicUsize`, `AtomicBool`, etc +- `RefCell` -- replacement: `RwLock`, or perhaps `Mutex` + +However, you have to be wary! The parallel versions of these types have +different atomicity guarantees. For example, with a `Cell`, you can increment a +counter like so: + +```rust +let value = counter.get(); +counter.set(value + 1); +``` + +But when you use the equivalent `AtomicUsize` methods, you are actually +introducing a potential race condition (not a data race, technically, but it can +be an awfully fine distinction): + +```rust +let value = tscounter.load(Ordering::SeqCst); +tscounter.store(value + 1, Ordering::SeqCst); +``` + +You can already see that the `AtomicUsize` API is a bit more complex, as it +requires you to specify an +[ordering](https://doc.rust-lang.org/std/sync/atomic/enum.Ordering.html). (I +won't go into the details on ordering here, but suffice to say that if you don't +know what an ordering is, and probably even if you do, you should use +`Ordering::SeqCst`.) The danger in this parallel version of the counter is that +other threads might be running at the same time and they could cause our counter +to get out of sync. For example, if we have two threads, then they might both +execute the "load" before either has a chance to execute the "store": + +``` +Thread 1 Thread 2 +let value = tscounter.load(Ordering::SeqCst); +// value = X let value = tscounter.load(Ordering::SeqCst); + // value = X +tscounter.store(value+1); tscounter.store(value+1); +// tscounter = X+1 // tscounter = X+1 +``` + +Now even though we've had two increments, we'll only increase the counter by +one! Even though we've got no data race, this is still probably not the result +we wanted. The problem here is that the `Cell` API doesn't make clear the scope +of a "transaction" -- that is, the set of reads/writes that should occur +atomically. In this case, we probably wanted the get/set to occur together. + +In fact, when using the `Atomic` types, you very rarely want a plain `load` or +plain `store`. You probably want the more complex operations. A counter, for +example, would use `fetch_add` to atomically load and increment the value in one +step. Compare-and-swap is another popular building block. + +A similar problem can arise when converting `RefCell` to `RwLock`, but it is +somewhat less likely, because the `RefCell` API does in fact have a notion of a +transaction: the scope of the handle returned by `borrow` or `borrow_mut`. So if +you convert each call to `borrow` to `read` (and `borrow_mut` to `write`), +things will mostly work fine in a parallel setting, but there can still be +changes in behavior. Consider using a `handle: RefCell>` like: + +```rust +let len = handle.borrow().len(); +for i in 0 .. len { + let data = handle.borrow()[i]; + println!("{}", data); +} +``` + +In sequential code, we know that this loop is safe. But if we convert this to +parallel code with an `RwLock`, we do not: this is because another thread could +come along and do `handle.write().unwrap().pop()`, and thus change the length of +the vector. In fact, even in *sequential* code, using very small borrow sections +like this is an anti-pattern: you ought to be enclosing the entire transaction +together, like so: + +```rust +let vec = handle.borrow(); +let len = vec.len(); +for i in 0 .. len { + let data = vec[i]; + println!("{}", data); +} +``` + +Or, even better, using an iterator instead of indexing: + +```rust +let vec = handle.borrow(); +for data in vec { + println!("{}", data); +} +``` + +There are several reasons to prefer one borrow over many. The most obvious is +that it is more efficient, since each borrow has to perform some safety checks. +But it's also more reliable: suppose we modified the loop above to not just +print things out, but also call into a helper function: + +```rust +let vec = handle.borrow(); +for data in vec { + helper(...); +} +``` + +And now suppose, independently, this helper fn evolved and had to pop something +off of the vector: + +```rust +fn helper(...) { + handle.borrow_mut().pop(); +} +``` + +Under the old model, where we did lots of small borrows, this would yield +precisely the same error that we saw in parallel land using an `RwLock`: the +length would be out of sync and our indexing would fail (note that in neither +case would there be an actual *data race* and hence there would never be +undefined behavior). But now that we use a single borrow, we'll see a borrow +error instead, which is much easier to diagnose, since it occurs at the point of +the `borrow_mut`, rather than downstream. Similarly, if we move to an `RwLock`, +we'll find that the code either deadlocks (if the write is on the same thread as +the read) or, if the write is on another thread, works just fine. Both of these +are preferable to random failures in my experience. + +## But wait, isn't Rust supposed to free me from this kind of thinking? + +You might think that Rust is supposed to mean that you don't have to think about +atomicity at all. In fact, if you avoid interior mutability (`Cell` and +`RefCell` in a sequential setting, or `AtomicUsize`, `RwLock`, `Mutex`, et al. +in parallel code), then this is true: the type system will basically guarantee +that you don't have to think about atomicity at all. But often there are times +when you WANT threads to interleave in the ways I showed above. + +Consider for example when you are conducting a search in parallel, say to find +the shortest route. To avoid fruitless search, you might want to keep a cell +with the shortest route you've found thus far. This way, when you are searching +down some path that's already longer than this shortest route, you can just stop +and avoid wasted effort. In sequential land, you might model this "best result" +as a shared value like `Rc>` (here the `usize` represents the length +of best path found so far); in parallel land, you'd use a `Arc`. + +```rust +fn search(path: &Path, cost_so_far: usize, best_cost: &AtomicUsize) { + if cost_so_far >= best_cost.load(Ordering::SeqCst) { + return; + } + // Using `fetch_min` to avoid a race condition, in case it changed since `load`. + best_cost.fetch_min(..., Ordering::SeqCst); +} +``` + +Now in this case, we really WANT to see results from other threads interjected +into our execution! diff --git a/anneal/v2/vendor/rayon/LICENSE-APACHE b/anneal/v2/vendor/rayon/LICENSE-APACHE new file mode 100644 index 0000000000..16fe87b06e --- /dev/null +++ b/anneal/v2/vendor/rayon/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/anneal/v2/vendor/rayon/LICENSE-MIT b/anneal/v2/vendor/rayon/LICENSE-MIT new file mode 100644 index 0000000000..25597d5838 --- /dev/null +++ b/anneal/v2/vendor/rayon/LICENSE-MIT @@ -0,0 +1,25 @@ +Copyright (c) 2010 The Rust Project Developers + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/anneal/v2/vendor/rayon/README.md b/anneal/v2/vendor/rayon/README.md new file mode 100644 index 0000000000..f4b8086944 --- /dev/null +++ b/anneal/v2/vendor/rayon/README.md @@ -0,0 +1,150 @@ +# Rayon + +[![Rayon crate](https://img.shields.io/crates/v/rayon.svg)](https://crates.io/crates/rayon) +[![Rayon documentation](https://docs.rs/rayon/badge.svg)](https://docs.rs/rayon) +![minimum rustc 1.80](https://img.shields.io/badge/rustc-1.80+-red.svg) +[![build status](https://github.com/rayon-rs/rayon/workflows/main/badge.svg)](https://github.com/rayon-rs/rayon/actions) + +Rayon is a data-parallelism library for Rust. It is extremely +lightweight and makes it easy to convert a sequential computation into +a parallel one. It also guarantees data-race freedom. (You may also +enjoy [this blog post][blog] about Rayon, which gives more background +and details about how it works, or [this video][video], from the Rust +Belt Rust conference.) Rayon is +[available on crates.io](https://crates.io/crates/rayon), and +[API documentation is available on docs.rs](https://docs.rs/rayon). + +[blog]: https://smallcultfollowing.com/babysteps/blog/2015/12/18/rayon-data-parallelism-in-rust/ +[video]: https://www.youtube.com/watch?v=gof_OEv71Aw + +## Parallel iterators and more + +Rayon makes it drop-dead simple to convert sequential iterators into +parallel ones: usually, you just change your `foo.iter()` call into +`foo.par_iter()`, and Rayon does the rest: + +```rust +use rayon::prelude::*; +fn sum_of_squares(input: &[i32]) -> i32 { + input.par_iter() // <-- just change that! + .map(|&i| i * i) + .sum() +} +``` + +[Parallel iterators] take care of deciding how to divide your data +into tasks; it will dynamically adapt for maximum performance. If you +need more flexibility than that, Rayon also offers the [join] and +[scope] functions, which let you create parallel tasks on your own. +For even more control, you can create [custom thread pools] rather than +using Rayon's default, global thread pool. + +[Parallel iterators]: https://docs.rs/rayon/*/rayon/iter/index.html +[join]: https://docs.rs/rayon/*/rayon/fn.join.html +[scope]: https://docs.rs/rayon/*/rayon/fn.scope.html +[custom thread pools]: https://docs.rs/rayon/*/rayon/struct.ThreadPool.html + +## No data races + +You may have heard that parallel execution can produce all kinds of +crazy bugs. Well, rest easy. Rayon's APIs all guarantee **data-race +freedom**, which generally rules out most parallel bugs (though not +all). In other words, **if your code compiles**, it typically does the +same thing it did before. + +For the most, parallel iterators in particular are guaranteed to +produce the same results as their sequential counterparts. One caveat: +If your iterator has side effects (for example, sending methods to +other threads through a [Rust channel] or writing to disk), those side +effects may occur in a different order. Note also that, in some cases, +parallel iterators offer alternative versions of the sequential +iterator methods that can have higher performance. + +[Rust channel]: https://doc.rust-lang.org/std/sync/mpsc/fn.channel.html + +## Using Rayon + +[Rayon is available on crates.io](https://crates.io/crates/rayon). The +recommended way to use it is to add a line into your Cargo.toml such +as: + +```toml +[dependencies] +rayon = "1.12" +``` + +To use the parallel iterator APIs, a number of traits have to be in +scope. The easiest way to bring those things into scope is to use the +[Rayon prelude](https://docs.rs/rayon/*/rayon/prelude/index.html). In +each module where you would like to use the parallel iterator APIs, +just add: + +```rust +use rayon::prelude::*; +``` + +Rayon currently requires `rustc 1.80.0` or greater. + +### Usage with WebAssembly + +By default, when building to WebAssembly, Rayon will treat it as any +other platform without multithreading support and will fall back to +sequential iteration. This allows existing code to compile and run +successfully with no changes necessary, but it will run slower as it +will only use a single CPU core. + +You can build Rayon-based projects with proper multithreading support +for the Web, but you'll need an adapter and some project configuration +to account for differences between WebAssembly threads and threads on +the other platforms. + +Check out the +[wasm-bindgen-rayon](https://github.com/RReverser/wasm-bindgen-rayon) +docs for more details. + +## Contribution + +Rayon is an open source project! If you'd like to contribute to Rayon, +check out +[the list of "help wanted" issues](https://github.com/rayon-rs/rayon/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22). +These are all (or should be) issues that are suitable for getting +started, and they generally include a detailed set of instructions for +what to do. Please ask questions if anything is unclear! Also, check +out the +[Guide to Development](https://github.com/rayon-rs/rayon/wiki/Guide-to-Development) +page on the wiki. Note that all code submitted in PRs to Rayon is +assumed to +[be licensed under Rayon's dual MIT/Apache 2.0 licensing](https://github.com/rayon-rs/rayon/blob/main/README.md#license). + +## Quick demo + +To see Rayon in action, check out the `rayon-demo` directory, which +includes a number of demos of code using Rayon. For example, run this +command to get a visualization of an N-body simulation. To see the +effect of using Rayon, press `s` to run sequentially and `p` to run in +parallel. + +```text +> cd rayon-demo +> cargo run --release -- nbody visualize +``` + +For more information on demos, try: + +```text +> cd rayon-demo +> cargo run --release -- --help +``` + +## Other questions? + +See [the Rayon FAQ][faq]. + +[faq]: https://github.com/rayon-rs/rayon/blob/main/FAQ.md + +## License + +Rayon is distributed under the terms of both the MIT license and the +Apache License (Version 2.0). See [LICENSE-APACHE](LICENSE-APACHE) and +[LICENSE-MIT](LICENSE-MIT) for details. Opening a pull request is +assumed to signal agreement with these licensing terms. diff --git a/anneal/v2/vendor/rayon/RELEASES.md b/anneal/v2/vendor/rayon/RELEASES.md new file mode 100644 index 0000000000..7eab599165 --- /dev/null +++ b/anneal/v2/vendor/rayon/RELEASES.md @@ -0,0 +1,929 @@ +# Release rayon 1.12.0 (2026-04-13) + +- Fixed a bug in parallel `Range` when the end is 0xE000, just past the + surrogate boundary, which was unsafely producing invalid `char` values. +- The new method `ParallelSlice::par_array_windows` works like `par_windows` + but with a constant length, producing `&[T; N]` items. + +# Release rayon 1.11.0 / rayon-core 1.13.0 (2025-08-12) + +- The minimum supported `rustc` is now 1.80. +- `iter::repeatn` has been renamed to `iter::repeat_n` to match the name + stabilized in the standard library. The old name still exists as a deprecated + function for compatibility. +- Fixed a bug in `in_place_scope` when the default global registry uses the + current thread, like on WebAssembly without threading support. +- `binary_heap::Iter` no longer requires a temporary allocation. +- Relaxed trait bounds on many of the public structs. +- Implemented `IntoParallelIterator for Box<[T]>` and its references. +- Implemented `FromParallelIterator<_> for Box` via `String`. + +# Release rayon 1.10.0 (2024-03-23) + +- The new methods `ParallelSlice::par_chunk_by` and + `ParallelSliceMut::par_chunk_by_mut` work like the slice methods `chunk_by` + and `chunk_by_mut` added in Rust 1.77. + +# Release rayon 1.9.0 (2024-02-27) + +- The new methods `IndexedParallelIterator::by_exponential_blocks` and + `by_uniform_blocks` allow processing items in smaller groups at a time. +- The new `iter::walk_tree`, `walk_tree_prefix`, and `walk_tree_postfix` + functions enable custom parallel iteration over tree-like structures. +- The new method `ParallelIterator::collect_vec_list` returns items as a linked + list of vectors, which is an efficient mode of parallel collection used by + many of the internal implementations of `collect`. +- The new methods `ParallelSliceMut::par_split_inclusive_mut`, + `ParallelSlice::par_split_inclusive`, and + `ParallelString::par_split_inclusive` all work like a normal split but + keeping the separator as part of the left slice. +- The new `ParallelString::par_split_ascii_whitespace` splits only on ASCII + whitespace, which is faster than including Unicode multi-byte whitespace. +- `OsString` now implements `FromParallelIterator<_>` and `ParallelExtend<_>` + for a few item types similar to the standard `FromIterator` and `Extend`. +- The internal `Pattern` trait for string methods is now implemented for + `[char; N]` and `&[char; N]`, matching any of the given characters. + +# Release rayon 1.8.1 / rayon-core 1.12.1 (2024-01-17) + +- The new `"web_spin_lock"` crate feature makes mutexes spin on the main + browser thread in WebAssembly, rather than suffer an error about forbidden + `atomics.wait` if they were to block in that context. Thanks @RReverser! + +# Release rayon 1.8.0 / rayon-core 1.12.0 (2023-09-20) + +- The minimum supported `rustc` is now 1.63. +- Added `ThreadPoolBuilder::use_current_thread` to use the builder thread as + part of the new thread pool. That thread does not run the pool's main loop, + but it may participate in work-stealing if it yields to rayon in some way. +- Implemented `FromParallelIterator` for `Box<[T]>`, `Rc<[T]>`, and + `Arc<[T]>`, as well as `FromParallelIterator>` and + `ParallelExtend>` for `String`. +- `ThreadPoolBuilder::build_scoped` now uses `std::thread::scope`. +- The default number of threads is now determined using + `std::thread::available_parallelism` instead of the `num_cpus` crate. +- The internal logging facility has been removed, reducing bloat for all users. +- Many smaller performance tweaks and documentation updates. + +# Release rayon 1.7.0 / rayon-core 1.11.0 (2023-03-03) + +- The minimum supported `rustc` is now 1.59. +- Added a fallback when threading is unsupported. +- The new `ParallelIterator::take_any` and `skip_any` methods work like + unordered `IndexedParallelIterator::take` and `skip`, counting items in + whatever order they are visited in parallel. +- The new `ParallelIterator::take_any_while` and `skip_any_while` methods work + like unordered `Iterator::take_while` and `skip_while`, which previously had + no parallel equivalent. The "while" condition may be satisfied from anywhere + in the parallel iterator, affecting all future items regardless of position. +- The new `yield_now` and `yield_local` functions will cooperatively yield + execution to Rayon, either trying to execute pending work from the entire + pool or from just the local deques of the current thread, respectively. + +# Release rayon-core 1.10.2 (2023-01-22) + +- Fixed miri-reported UB for SharedReadOnly tags protected by a call. + +# Release rayon 1.6.1 (2022-12-09) + +- Simplified `par_bridge` to only pull one item at a time from the iterator, + without batching. Threads that are waiting for iterator items will now block + appropriately rather than spinning CPU. (Thanks @njaard!) +- Added protection against recursion in `par_bridge`, so iterators that also + invoke rayon will not cause mutex recursion deadlocks. + +# Release rayon-core 1.10.1 (2022-11-18) + +- Fixed a race condition with threads going to sleep while a broadcast starts. + +# Release rayon 1.6.0 / rayon-core 1.10.0 (2022-11-18) + +- The minimum supported `rustc` is now 1.56. +- The new `IndexedParallelIterator::fold_chunks` and `fold_chunks_with` methods + work like `ParallelIterator::fold` and `fold_with` with fixed-size chunks of + items. This may be useful for predictable batching performance, without the + allocation overhead of `IndexedParallelIterator::chunks`. +- New "broadcast" methods run a given function on all threads in the pool. + These run at a sort of reduced priority after each thread has exhausted their + local work queue, but before they attempt work-stealing from other threads. + - The global `broadcast` function and `ThreadPool::broadcast` method will + block until completion, returning a `Vec` of all return values. + - The global `spawn_broadcast` function and methods on `ThreadPool`, `Scope`, + and `ScopeFifo` will run detached, without blocking the current thread. +- Panicking methods now use `#[track_caller]` to report the caller's location. +- Fixed a truncated length in `vec::Drain` when given an empty range. + +## Contributors + +Thanks to all of the contributors for this release! + +- @cuviper +- @idanmuze +- @JoeyBF +- @JustForFun88 +- @kianmeng +- @kornelski +- @ritchie46 +- @ryanrussell +- @steffahn +- @TheIronBorn +- @willcrozi + +# Release rayon 1.5.3 (2022-05-13) + +- The new `ParallelSliceMut::par_sort_by_cached_key` is a stable sort that caches + the keys for each item -- a parallel version of `slice::sort_by_cached_key`. + +# Release rayon-core 1.9.3 (2022-05-13) + +- Fixed a use-after-free race in job notification. + +# Release rayon 1.5.2 / rayon-core 1.9.2 (2022-04-13) + +- The new `ParallelSlice::par_rchunks()` and `par_rchunks_exact()` iterate + slice chunks in reverse, aligned the against the end of the slice if the + length is not a perfect multiple of the chunk size. The new + `ParallelSliceMut::par_rchunks_mut()` and `par_rchunks_exact_mut()` are the + same for mutable slices. +- The `ParallelIterator::try_*` methods now support `std::ops::ControlFlow` and + `std::task::Poll` items, mirroring the unstable `Try` implementations in the + standard library. +- The `ParallelString` pattern-based methods now support `&[char]` patterns, + which match when any character in that slice is found in the string. +- A soft limit is now enforced on the number of threads allowed in a single + thread pool, respecting internal bit limits that already existed. The current + maximum is publicly available from the new function `max_num_threads()`. +- Fixed several Stacked Borrow and provenance issues found by `cargo miri`. + +## Contributors + +Thanks to all of the contributors for this release! + +- @atouchet +- @bluss +- @cuviper +- @fzyzcjy +- @nyanzebra +- @paolobarbolini +- @RReverser +- @saethlin + +# Release rayon 1.5.1 / rayon-core 1.9.1 (2021-05-18) + +- The new `in_place_scope` and `in_place_scope_fifo` are variations of `scope` + and `scope_fifo`, running the initial non-`Send` callback directly on the + current thread, rather than moving execution to the thread pool. +- With Rust 1.51 or later, arrays now implement `IntoParallelIterator`. +- New implementations of `FromParallelIterator` make it possible to `collect` + complicated nestings of items. + - `FromParallelIterator<(A, B)> for (FromA, FromB)` works like `unzip`. + - `FromParallelIterator> for (A, B)` works like `partition_map`. +- Type inference now works better with parallel `Range` and `RangeInclusive`. +- The implementation of `FromParallelIterator` and `ParallelExtend` for + `Vec` now uses `MaybeUninit` internally to avoid creating any + references to uninitialized data. +- `ParallelBridge` fixed a bug with threads missing available work. + +## Contributors + +Thanks to all of the contributors for this release! + +- @atouchet +- @cuviper +- @Hywan +- @iRaiko +- @Qwaz +- @rocallahan + +# Release rayon 1.5.0 / rayon-core 1.9.0 (2020-10-21) + +- Update crossbeam dependencies. +- The minimum supported `rustc` is now 1.36. + +## Contributors + +Thanks to all of the contributors for this release! + +- @cuviper +- @mbrubeck +- @mrksu + +# Release rayon 1.4.1 (2020-09-29) + +- The new `flat_map_iter` and `flatten_iter` methods can be used to flatten + sequential iterators, which may perform better in cases that don't need the + nested parallelism of `flat_map` and `flatten`. +- The new `par_drain` method is a parallel version of the standard `drain` for + collections, removing items while keeping the original capacity. Collections + that implement this through `ParallelDrainRange` support draining items from + arbitrary index ranges, while `ParallelDrainFull` always drains everything. +- The new `positions` method finds all items that match the given predicate and + returns their indices in a new iterator. + +# Release rayon-core 1.8.1 (2020-09-17) + +- Fixed an overflow panic on high-contention workloads, for a counter that was + meant to simply wrap. This panic only occurred with debug assertions enabled, + and was much more likely on 32-bit targets. + +# Release rayon 1.4.0 / rayon-core 1.8.0 (2020-08-24) + +- Implemented a new thread scheduler, [RFC 5], which uses targeted wakeups for + new work and for notifications of completed stolen work, reducing wasteful + CPU usage in idle threads. +- Implemented `IntoParallelIterator for Range` and `RangeInclusive` + with the same iteration semantics as Rust 1.45. +- Relaxed the lifetime requirements of the initial `scope` closure. + +[RFC 5]: https://github.com/rayon-rs/rfcs/pull/5 + +## Contributors + +Thanks to all of the contributors for this release! + +- @CAD97 +- @cuviper +- @kmaork +- @nikomatsakis +- @SuperFluffy + + +# Release rayon 1.3.1 / rayon-core 1.7.1 (2020-06-15) + +- Fixed a use-after-free race in calls blocked between two rayon thread pools. +- Collecting to an indexed `Vec` now drops any partial writes while unwinding, + rather than just leaking them. If dropping also panics, Rust will abort. + - Note: the old leaking behavior is considered _safe_, just not ideal. +- The new `IndexedParallelIterator::step_by()` adapts an iterator to step + through items by the given count, like `Iterator::step_by()`. +- The new `ParallelSlice::par_chunks_exact()` and mutable equivalent + `ParallelSliceMut::par_chunks_exact_mut()` ensure that the chunks always have + the exact length requested, leaving any remainder separate, like the slice + methods `chunks_exact()` and `chunks_exact_mut()`. + +## Contributors + +Thanks to all of the contributors for this release! + +- @adrian5 +- @bluss +- @cuviper +- @FlyingCanoe +- @GuillaumeGomez +- @matthiasbeyer +- @picoHz +- @zesterer + + +# Release rayon 1.3.0 / rayon-core 1.7.0 (2019-12-21) + +- Tuples up to length 12 now implement `IntoParallelIterator`, creating a + `MultiZip` iterator that produces items as similarly-shaped tuples. +- The `--cfg=rayon_unstable` supporting code for `rayon-futures` is removed. +- The minimum supported `rustc` is now 1.31. + +## Contributors + +Thanks to all of the contributors for this release! + +- @cuviper +- @c410-f3r +- @silwol + + +# Release rayon-futures 0.1.1 (2019-12-21) + +- `Send` bounds have been added for the `Item` and `Error` associated types on + all generic `F: Future` interfaces. While technically a breaking change, this + is a soundness fix, so we are not increasing the semantic version for this. +- This crate is now deprecated, and the `--cfg=rayon_unstable` supporting code + will be removed in `rayon-core 1.7.0`. This only supported the now-obsolete + `Future` from `futures 0.1`, while support for `std::future::Future` is + expected to come directly in `rayon-core` -- although that is not ready yet. + +## Contributors + +Thanks to all of the contributors for this release! + +- @cuviper +- @kornelski +- @jClaireCodesStuff +- @jwass +- @seanchen1991 + + +# Release rayon 1.2.1 / rayon-core 1.6.1 (2019-11-20) + +- Update crossbeam dependencies. +- Add top-level doc links for the iterator traits. +- Document that the iterator traits are not object safe. + +## Contributors + +Thanks to all of the contributors for this release! + +- @cuviper +- @dnaka91 +- @matklad +- @nikomatsakis +- @Qqwy +- @vorner + + +# Release rayon 1.2.0 / rayon-core 1.6.0 (2019-08-30) + +- The new `ParallelIterator::copied()` converts an iterator of references into + copied values, like `Iterator::copied()`. +- `ParallelExtend` is now implemented for the unit `()`. +- Internal updates were made to improve test determinism, reduce closure type + sizes, reduce task allocations, and update dependencies. +- The minimum supported `rustc` is now 1.28. + +## Contributors + +Thanks to all of the contributors for this release! + +- @Aaron1011 +- @cuviper +- @ralfbiedert + + +# Release rayon 1.1.0 / rayon-core 1.5.0 (2019-06-12) + +- FIFO spawns are now supported using the new `spawn_fifo()` and `scope_fifo()` + global functions, and their corresponding `ThreadPool` methods. + - Normally when tasks are queued on a thread, the most recent is processed + first (LIFO) while other threads will steal the oldest (FIFO). With FIFO + spawns, those tasks are processed locally in FIFO order too. + - Regular spawns and other tasks like `join` are not affected. + - The `breadth_first` configuration flag, which globally approximated this + effect, is now deprecated. + - For more design details, please see [RFC 1]. +- `ThreadPoolBuilder` can now take a custom `spawn_handler` to control how + threads will be created in the pool. + - `ThreadPoolBuilder::build_scoped()` uses this to create a scoped thread + pool, where the threads are able to use non-static data. + - This may also be used to support threading in exotic environments, like + WebAssembly, which don't support the normal `std::thread`. +- `ParallelIterator` has 3 new methods: `find_map_any()`, `find_map_first()`, + and `find_map_last()`, like `Iterator::find_map()` with ordering constraints. +- The new `ParallelIterator::panic_fuse()` makes a parallel iterator halt as soon + as possible if any of its threads panic. Otherwise, the panic state is not + usually noticed until the iterator joins its parallel tasks back together. +- `IntoParallelIterator` is now implemented for integral `RangeInclusive`. +- Several internal `Folder`s now have optimized `consume_iter` implementations. +- `rayon_core::current_thread_index()` is now re-exported in `rayon`. +- The minimum `rustc` is now 1.26, following the update policy defined in [RFC 3]. + +## Contributors + +Thanks to all of the contributors for this release! + +- @cuviper +- @didroe +- @GuillaumeGomez +- @huonw +- @janriemer +- @kornelski +- @nikomatsakis +- @seanchen1991 +- @yegeun542 + +[RFC 1]: https://github.com/rayon-rs/rfcs/blob/main/accepted/rfc0001-scope-scheduling.md +[RFC 3]: https://github.com/rayon-rs/rfcs/blob/main/accepted/rfc0003-minimum-rustc.md + + +# Release rayon 1.0.3 (2018-11-02) + +- `ParallelExtend` is now implemented for tuple pairs, enabling nested + `unzip()` and `partition_map()` operations. For instance, `(A, (B, C))` + items can be unzipped into `(Vec, (Vec, Vec))`. + - `ParallelExtend<(A, B)>` works like `unzip()`. + - `ParallelExtend>` works like `partition_map()`. +- `ParallelIterator` now has a method `map_init()` which calls an `init` + function for a value to pair with items, like `map_with()` but dynamically + constructed. That value type has no constraints, not even `Send` or `Sync`. + - The new `for_each_init()` is a variant of this for simple iteration. + - The new `try_for_each_init()` is a variant for fallible iteration. + +## Contributors + +Thanks to all of the contributors for this release! + +- @cuviper +- @dan-zheng +- @dholbert +- @ignatenkobrain +- @mdonoughe + + +# Release rayon 1.0.2 / rayon-core 1.4.1 (2018-07-17) + +- The `ParallelBridge` trait with method `par_bridge()` makes it possible to + use any `Send`able `Iterator` in parallel! + - This trait has been added to `rayon::prelude`. + - It automatically implements internal synchronization and queueing to + spread the `Item`s across the thread pool. Iteration order is not + preserved by this adaptor. + - "Native" Rayon iterators like `par_iter()` should still be preferred when + possible for better efficiency. +- `ParallelString` now has additional methods for parity with `std` string + iterators: `par_char_indices()`, `par_bytes()`, `par_encode_utf16()`, + `par_matches()`, and `par_match_indices()`. +- `ParallelIterator` now has fallible methods `try_fold()`, `try_reduce()`, + and `try_for_each`, plus `*_with()` variants of each, for automatically + short-circuiting iterators on `None` or `Err` values. These are inspired by + `Iterator::try_fold()` and `try_for_each()` that were stabilized in Rust 1.27. +- `Range` and `Range` are now supported with Rust 1.26 and later. +- Small improvements have been made to the documentation. +- `rayon-core` now only depends on `rand` for testing. +- Rayon tests now work on stable Rust. + +## Contributors + +Thanks to all of the contributors for this release! + +- @AndyGauge +- @cuviper +- @ignatenkobrain +- @LukasKalbertodt +- @MajorBreakfast +- @nikomatsakis +- @paulkernfeld +- @QuietMisdreavus + + +# Release rayon 1.0.1 (2018-03-16) + +- Added more documentation for `rayon::iter::split()`. +- Corrected links and typos in documentation. + +## Contributors + +Thanks to all of the contributors for this release! + +- @cuviper +- @HadrienG2 +- @matthiasbeyer +- @nikomatsakis + + +# Release rayon 1.0.0 / rayon-core 1.4.0 (2018-02-15) + +- `ParallelIterator` added the `update` method which applies a function to + mutable references, inspired by `itertools`. +- `IndexedParallelIterator` added the `chunks` method which yields vectors of + consecutive items from the base iterator, inspired by `itertools`. +- `String` now implements `FromParallelIterator>` and + `ParallelExtend>`, inspired by `std`. +- `()` now implements `FromParallelIterator<()>`, inspired by `std`. +- The new `ThreadPoolBuilder` replaces and deprecates `Configuration`. + - Errors from initialization now have the concrete `ThreadPoolBuildError` + type, rather than `Box`, and this type implements `Send` and `Sync`. + - `ThreadPool::new` is deprecated in favor of `ThreadPoolBuilder::build`. + - `initialize` is deprecated in favor of `ThreadPoolBuilder::build_global`. +- Examples have been added to most of the parallel iterator methods. +- A lot of the documentation has been reorganized and extended. + +## Breaking changes + +- Rayon now requires rustc 1.13 or greater. +- `IndexedParallelIterator::len` and `ParallelIterator::opt_len` now operate on + `&self` instead of `&mut self`. +- `IndexedParallelIterator::collect_into` is now `collect_into_vec`. +- `IndexedParallelIterator::unzip_into` is now `unzip_into_vecs`. +- Rayon no longer exports the deprecated `Configuration` and `initialize` from + rayon-core. + +## Contributors + +Thanks to all of the contributors for this release! + +- @Bilkow +- @cuviper +- @Enet4 +- @ignatenkobrain +- @iwillspeak +- @jeehoonkang +- @jwass +- @Kerollmops +- @KodrAus +- @kornelski +- @MaloJaffre +- @nikomatsakis +- @obv-mikhail +- @oddg +- @phimuemue +- @stjepang +- @tmccombs +- bors[bot] + + +# Release rayon 0.9.0 / rayon-core 1.3.0 / rayon-futures 0.1.0 (2017-11-09) + +- `Configuration` now has a `build` method. +- `ParallelIterator` added `flatten` and `intersperse`, both inspired by + itertools. +- `IndexedParallelIterator` added `interleave`, `interleave_shortest`, and + `zip_eq`, all inspired by itertools. +- The new functions `iter::empty` and `once` create parallel iterators of + exactly zero or one item, like their `std` counterparts. +- The new functions `iter::repeat` and `repeatn` create parallel iterators + repeating an item indefinitely or `n` times, respectively. +- The new function `join_context` works like `join`, with an added `FnContext` + parameter that indicates whether the job was stolen. +- `Either` (used by `ParallelIterator::partition_map`) is now re-exported from + the `either` crate, instead of defining our own type. + - `Either` also now implements `ParallelIterator`, `IndexedParallelIterator`, + and `ParallelExtend` when both of its `Left` and `Right` types do. +- All public types now implement `Debug`. +- Many of the parallel iterators now implement `Clone` where possible. +- Much of the documentation has been extended. (but still could use more help!) +- All rayon crates have improved metadata. +- Rayon was evaluated in the Libz Blitz, leading to many of these improvements. +- Rayon pull requests are now guarded by bors-ng. + +## Futures + +The `spawn_future()` method has been refactored into its own `rayon-futures` +crate, now through a `ScopeFutureExt` trait for `ThreadPool` and `Scope`. The +supporting `rayon-core` APIs are still gated by `--cfg rayon_unstable`. + +## Breaking changes + +- Two breaking changes have been made to `rayon-core`, but since they're fixing + soundness bugs, we are considering these _minor_ changes for semver. + - `Scope::spawn` now requires `Send` for the closure. + - `ThreadPool::install` now requires `Send` for the return value. +- The `iter::internal` module has been renamed to `iter::plumbing`, to hopefully + indicate that while these are low-level details, they're not really internal + or private to rayon. The contents of that module are needed for third-parties + to implement new parallel iterators, and we'll treat them with normal semver + stability guarantees. +- The function `rayon::iter::split` is no longer re-exported as `rayon::split`. + +## Contributors + +Thanks to all of the contributors for this release! + +- @AndyGauge +- @ChristopherDavenport +- @chrisvittal +- @cuviper +- @dns2utf8 +- @dtolnay +- @frewsxcv +- @gsquire +- @Hittherhod +- @jdr023 +- @laumann +- @leodasvacas +- @lvillani +- @MajorBreakfast +- @mamuleanu +- @marmistrz +- @mbrubeck +- @mgattozzi +- @nikomatsakis +- @smt923 +- @stjepang +- @tmccombs +- @vishalsodani +- bors[bot] + + +# Release rayon 0.8.2 (2017-06-28) + +- `ParallelSliceMut` now has six parallel sorting methods with the same + variations as the standard library. + - `par_sort`, `par_sort_by`, and `par_sort_by_key` perform stable sorts in + parallel, using the default order, a custom comparator, or a key extraction + function, respectively. + - `par_sort_unstable`, `par_sort_unstable_by`, and `par_sort_unstable_by_key` + perform unstable sorts with the same comparison options. + - Thanks to @stjepang! + + +# Release rayon 0.8.1 / rayon-core 1.2.0 (2017-06-14) + +- The following core APIs are being stabilized: + - `rayon::spawn()` -- spawns a task into the Rayon thread pool; as it + is contained in the global scope (rather than a user-created + scope), the task cannot capture anything from the current stack + frame. + - `ThreadPool::join()`, `ThreadPool::spawn()`, `ThreadPool::scope()` + -- convenience APIs for launching new work within a thread pool. +- The various iterator adapters are now tagged with `#[must_use]` +- Parallel iterators now offer a `for_each_with` adapter, similar to + `map_with`. +- We are adopting a new approach to handling the remaining unstable + APIs (which primarily pertain to futures integration). As awlays, + unstable APIs are intended for experimentation, but do not come with + any promise of compatibility (in other words, we might change them + in arbitrary ways in any release). Previously, we designated such + APIs using a Cargo feature "unstable". Now, we are using a regular + `#[cfg]` flag. This means that to see the unstable APIs, you must do + `RUSTFLAGS='--cfg rayon_unstable' cargo build`. This is + intentionally inconvenient; in particular, if you are a library, + then your clients must also modify their environment, signaling + their agreement to instability. + + +# Release rayon 0.8.0 / rayon-core 1.1.0 (2017-06-13) + +## Rayon 0.8.0 + +- Added the `map_with` and `fold_with` combinators, which help for + passing along state (like channels) that cannot be shared between + threads but which can be cloned on each thread split. +- Added the `while_some` combinator, which helps for writing short-circuiting iterators. +- Added support for "short-circuiting" collection: e.g., collecting + from an iterator producing `Option` or `Result` into a + `Option>` or `Result, E>`. +- Support `FromParallelIterator` for `Cow`. +- Removed the deprecated weight APIs. +- Simplified the parallel iterator trait hierarchy by removing the + `BoundedParallelIterator` and `ExactParallelIterator` traits, + which were not serving much purpose. +- Improved documentation. +- Added some missing `Send` impls. +- Fixed some small bugs. + +## Rayon-core 1.1.0 + +- We now have more documentation. +- Renamed the (unstable) methods `spawn_async` and + `spawn_future_async` -- which spawn tasks that cannot hold + references -- to simply `spawn` and `spawn_future`, respectively. +- We are now using the coco library for our deque. +- Individual thread pools can now be configured in "breadth-first" + mode, which causes them to execute spawned tasks in the reverse + order that they used to. In some specific scenarios, this can be a + win (though it is not generally the right choice). +- Added top-level functions: + - `current_thread_index`, for querying the index of the current worker thread within + its thread pool (previously available as `thread_pool.current_thread_index()`); + - `current_thread_has_pending_tasks`, for querying whether the + current worker that has an empty task deque or not. This can be + useful when deciding whether to spawn a task. +- The environment variables for controlling Rayon are now + `RAYON_NUM_THREADS` and `RAYON_LOG`. The older variables (e.g., + `RAYON_RS_NUM_CPUS` are still supported but deprecated). + +## Rayon-demo + +- Added a new game-of-life benchmark. + +## Contributors + +Thanks to the following contributors: + +- @ChristopherDavenport +- @SuperFluffy +- @antoinewdg +- @crazymykl +- @cuviper +- @glandium +- @julian-seward1 +- @leodasvacas +- @leshow +- @lilianmoraru +- @mschmo +- @nikomatsakis +- @stjepang + + +# Release rayon 0.7.1 / rayon-core 1.0.2 (2017-05-30) + +This release is a targeted performance fix for #343, an issue where +rayon threads could sometimes enter into a spin loop where they would +be unable to make progress until they are pre-empted. + + +# Release rayon 0.7 / rayon-core 1.0 (2017-04-06) + +This release marks the first step towards Rayon 1.0. **For best +performance, it is important that all Rayon users update to at least +Rayon 0.7.** This is because, as of Rayon 0.7, we have taken steps to +ensure that, no matter how many versions of rayon are actively in use, +there will only be a single global scheduler. This is achieved via the +`rayon-core` crate, which is being released at version 1.0, and which +encapsulates the core schedule APIs like `join()`. (Note: the +`rayon-core` crate is, to some degree, an implementation detail, and +not intended to be imported directly; it's entire API surface is +mirrored through the rayon crate.) + +We have also done a lot of work reorganizing the API for Rayon 0.7 in +preparation for 1.0. The names of iterator types have been changed and +reorganized (but few users are expected to be naming those types +explicitly anyhow). In addition, a number of parallel iterator methods +have been adjusted to match those in the standard iterator traits more +closely. See the "Breaking Changes" section below for +details. + +Finally, Rayon 0.7 includes a number of new features and new parallel +iterator methods. **As of this release, Rayon's parallel iterators +have officially reached parity with sequential iterators** -- that is, +every sequential iterator method that makes any sense in parallel is +supported in some capacity. + +### New features and methods + +- The internal `Producer` trait now features `fold_with`, which enables + better performance for some parallel iterators. +- Strings now support `par_split()` and `par_split_whitespace()`. +- The `Configuration` API is expanded and simplified: + - `num_threads(0)` no longer triggers an error + - you can now supply a closure to name the Rayon threads that get created + by using `Configuration::thread_name`. + - you can now inject code when Rayon threads start up and finish + - you can now set a custom panic handler to handle panics in various odd situations +- Threadpools are now able to more gracefully put threads to sleep when not needed. +- Parallel iterators now support `find_first()`, `find_last()`, `position_first()`, + and `position_last()`. +- Parallel iterators now support `rev()`, which primarily affects subsequent calls + to `enumerate()`. +- The `scope()` API is now considered stable (and part of `rayon-core`). +- There is now a useful `rayon::split` function for creating custom + Rayon parallel iterators. +- Parallel iterators now allow you to customize the min/max number of + items to be processed in a given thread. This mechanism replaces the + older `weight` mechanism, which is deprecated. +- `sum()` and friends now use the standard `Sum` traits + +### Breaking changes + +In the move towards 1.0, there have been a number of minor breaking changes: + +- Configuration setters like `Configuration::set_num_threads()` lost the `set_` prefix, + and hence become something like `Configuration::num_threads()`. +- `Configuration` getters are removed +- Iterator types have been shuffled around and exposed more consistently: + - combinator types live in `rayon::iter`, e.g. `rayon::iter::Filter` + - iterators over various types live in a module named after their type, + e.g. `rayon::slice::Windows` +- When doing a `sum()` or `product()`, type annotations are needed for the result + since it is now possible to have the resulting sum be of a type other than the value + you are iterating over (this mirrors sequential iterators). + +### Experimental features + +Experimental features require the use of the `unstable` feature. Their +APIs may change or disappear entirely in future releases (even minor +releases) and hence they should be avoided for production code. + +- We now have (unstable) support for futures integration. You can use + `Scope::spawn_future` or `rayon::spawn_future_async()`. +- There is now a `rayon::spawn_async()` function for using the Rayon + thread pool to run tasks that do not have references to the stack. + +### Contributors + +Thanks to the following people for their contributions to this release: + +- @Aaronepower +- @ChristopherDavenport +- @bluss +- @cuviper +- @froydnj +- @gaurikholkar +- @hniksic +- @leodasvacas +- @leshow +- @martinhath +- @mbrubeck +- @nikomatsakis +- @pegomes +- @schuster +- @torkleyy + + +# Release 0.6 (2016-12-21) + +This release includes a lot of progress towards the goal of parity +with the sequential iterator API, though there are still a few methods +that are not yet complete. If you'd like to help with that effort, +[check out the milestone](https://github.com/rayon-rs/rayon/issues?q=is%3Aopen+is%3Aissue+milestone%3A%22Parity+with+the+%60Iterator%60+trait%22) +to see the remaining issues. + +**Announcement:** @cuviper has been added as a collaborator to the +Rayon repository for all of his outstanding work on Rayon, which +includes both internal refactoring and helping to shape the public +API. Thanks @cuviper! Keep it up. + +- We now support `collect()` and not just `collect_with()`. + You can use `collect()` to build a number of collections, + including vectors, maps, and sets. Moreover, when building a vector + with `collect()`, you are no longer limited to exact parallel iterators. + Thanks @nikomatsakis, @cuviper! +- We now support `skip()` and `take()` on parallel iterators. + Thanks @martinhath! +- **Breaking change:** We now match the sequential APIs for `min()` and `max()`. + We also support `min_by_key()` and `max_by_key()`. Thanks @tapeinosyne! +- **Breaking change:** The `mul()` method is now renamed to `product()`, + to match sequential iterators. Thanks @jonathandturner! +- We now support parallel iterator over ranges on `u64` values. Thanks @cuviper! +- We now offer a `par_chars()` method on strings for iterating over characters + in parallel. Thanks @cuviper! +- We now have new demos: a traveling salesman problem solver as well as matrix + multiplication. Thanks @nikomatsakis, @edre! +- We are now documenting our minimum rustc requirement (currently + v1.12.0). We will attempt to maintain compatibility with rustc + stable v1.12.0 as long as it remains convenient, but if new features + are stabilized or added that would be helpful to Rayon, or there are + bug fixes that we need, we will bump to the most recent rustc. Thanks @cuviper! +- The `reduce()` functionality now has better inlining. + Thanks @bluss! +- The `join()` function now has some documentation. Thanks @gsquire! +- The project source has now been fully run through rustfmt. + Thanks @ChristopherDavenport! +- Exposed helper methods for accessing the current thread index. + Thanks @bholley! + + +# Release 0.5 (2016-11-04) + +- **Breaking change:** The `reduce` method has been vastly + simplified, and `reduce_with_identity` has been deprecated. +- **Breaking change:** The `fold` method has been changed. It used to + always reduce the values, but now instead it is a combinator that + returns a parallel iterator which can itself be reduced. See the + docs for more information. +- The following parallel iterator combinators are now available (thanks @cuviper!): + - `find_any()`: similar to `find` on a sequential iterator, + but doesn't necessarily return the *first* matching item + - `position_any()`: similar to `position` on a sequential iterator, + but doesn't necessarily return the index of *first* matching item + - `any()`, `all()`: just like their sequential counterparts +- The `count()` combinator is now available for parallel iterators. +- We now build with older versions of rustc again (thanks @durango!), + as we removed a stray semicolon from `thread_local!`. +- Various improvements to the (unstable) `scope()` API implementation. + + +# Release 0.4.3 (2016-10-25) + +- Parallel iterators now offer an adaptive weight scheme, + which means that explicit weights should no longer + be necessary in most cases! Thanks @cuviper! + - We are considering removing weights or changing the weight mechanism + before 1.0. Examples of scenarios where you still need weights even + with this adaptive mechanism would be great. Join the discussion + at . +- New (unstable) scoped threads API, see `rayon::scope` for details. + - You will need to supply the [cargo feature] `unstable`. +- The various demos and benchmarks have been consolidated into one + program, `rayon-demo`. +- Optimizations in Rayon's inner workings. Thanks @emilio! +- Update `num_cpus` to 1.0. Thanks @jamwt! +- Various internal cleanup in the implementation and typo fixes. + Thanks @cuviper, @Eh2406, and @spacejam! + +[cargo feature]: https://doc.rust-lang.org/cargo/reference/features.html#the-features-section + + +# Release 0.4.2 (2016-09-15) + +- Updated crates.io metadata. + + +# Release 0.4.1 (2016-09-14) + +- New `chain` combinator for parallel iterators. +- `Option`, `Result`, as well as many more collection types now have + parallel iterators. +- New mergesort demo. +- Misc fixes. + +Thanks to @cuviper, @edre, @jdanford, @frewsxcv for their contributions! + + +# Release 0.4 (2016-05-16) + +- Make use of latest versions of catch-panic and various fixes to panic propagation. +- Add new prime sieve demo. +- Add `cloned()` and `inspect()` combinators. +- Misc fixes for Rust RFC 1214. + +Thanks to @areilb1, @Amanieu, @SharplEr, and @cuviper for their contributions! + + +# Release 0.3 (2016-02-23) + +- Expanded `par_iter` APIs now available: + - `into_par_iter` is now supported on vectors (taking ownership of the elements) +- Panic handling is much improved: + - if you use the Nightly feature, experimental panic recovery is available + - otherwise, panics propagate out and poision the workpool +- New `Configuration` object to control number of threads and other details +- New demos and benchmarks + - try `cargo run --release -- visualize` in `demo/nbody` :) + - Note: a nightly compiler is required for this demo due to the + use of the `+=` syntax + +Thanks to @bjz, @cuviper, @Amanieu, and @willi-kappler for their contributions! + + +# Release 0.2 and earlier + +No release notes were being kept at this time. diff --git a/anneal/v2/vendor/rayon/src/array.rs b/anneal/v2/vendor/rayon/src/array.rs new file mode 100644 index 0000000000..d94ff8fdde --- /dev/null +++ b/anneal/v2/vendor/rayon/src/array.rs @@ -0,0 +1,85 @@ +//! Parallel iterator types for [arrays] (`[T; N]`) +//! +//! You will rarely need to interact with this module directly unless you need +//! to name one of the iterator types. +//! +//! [arrays]: primitive@array + +use crate::iter::plumbing::*; +use crate::iter::*; +use crate::slice::{Iter, IterMut}; +use crate::vec::DrainProducer; +use std::mem::ManuallyDrop; + +impl<'data, T: Sync + 'data, const N: usize> IntoParallelIterator for &'data [T; N] { + type Item = &'data T; + type Iter = Iter<'data, T>; + + fn into_par_iter(self) -> Self::Iter { + <&[T]>::into_par_iter(self) + } +} + +impl<'data, T: Send + 'data, const N: usize> IntoParallelIterator for &'data mut [T; N] { + type Item = &'data mut T; + type Iter = IterMut<'data, T>; + + fn into_par_iter(self) -> Self::Iter { + <&mut [T]>::into_par_iter(self) + } +} + +impl IntoParallelIterator for [T; N] { + type Item = T; + type Iter = IntoIter; + + fn into_par_iter(self) -> Self::Iter { + IntoIter { array: self } + } +} + +/// Parallel iterator that moves out of an array. +#[derive(Debug, Clone)] +pub struct IntoIter { + array: [T; N], +} + +impl ParallelIterator for IntoIter { + type Item = T; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + bridge(self, consumer) + } + + fn opt_len(&self) -> Option { + Some(N) + } +} + +impl IndexedParallelIterator for IntoIter { + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + bridge(self, consumer) + } + + fn len(&self) -> usize { + N + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + unsafe { + // Drain every item, and then the local array can just fall out of scope. + let mut array = ManuallyDrop::new(self.array); + let producer = DrainProducer::new(array.as_mut_slice()); + callback.callback(producer) + } + } +} diff --git a/anneal/v2/vendor/rayon/src/collections/binary_heap.rs b/anneal/v2/vendor/rayon/src/collections/binary_heap.rs new file mode 100644 index 0000000000..0eb5ae3dff --- /dev/null +++ b/anneal/v2/vendor/rayon/src/collections/binary_heap.rs @@ -0,0 +1,129 @@ +//! This module contains the parallel iterator types for heaps +//! (`BinaryHeap`). You will rarely need to interact with it directly +//! unless you have need to name one of the iterator types. + +use std::collections::BinaryHeap; + +use crate::iter::plumbing::*; +use crate::iter::*; + +use crate::slice; +use crate::vec; + +/// Parallel iterator over a binary heap +#[derive(Debug, Clone)] +pub struct IntoIter { + inner: vec::IntoIter, +} + +impl IntoParallelIterator for BinaryHeap { + type Item = T; + type Iter = IntoIter; + + fn into_par_iter(self) -> Self::Iter { + IntoIter { + inner: Vec::from(self).into_par_iter(), + } + } +} + +delegate_indexed_iterator! { + IntoIter => T, + impl +} + +/// Parallel iterator over an immutable reference to a binary heap +#[derive(Debug)] +pub struct Iter<'a, T> { + inner: slice::Iter<'a, T>, +} + +impl Clone for Iter<'_, T> { + fn clone(&self) -> Self { + Iter { + inner: self.inner.clone(), + } + } +} + +impl<'a, T: Sync> IntoParallelIterator for &'a BinaryHeap { + type Item = &'a T; + type Iter = Iter<'a, T>; + + fn into_par_iter(self) -> Self::Iter { + Iter { + inner: self.as_slice().into_par_iter(), + } + } +} + +delegate_indexed_iterator! { + Iter<'a, T> => &'a T, + impl<'a, T: Sync + 'a> +} + +// `BinaryHeap` doesn't have a mutable `Iterator` + +/// Draining parallel iterator that moves out of a binary heap, +/// but keeps the total capacity. +#[derive(Debug)] +pub struct Drain<'a, T> { + heap: &'a mut BinaryHeap, +} + +// NB: The only reason we require `T: Ord` is for `DrainGuard` to reconstruct +// the heap `From>` afterward, even though that will actually be empty. +impl<'a, T: Ord + Send> ParallelDrainFull for &'a mut BinaryHeap { + type Iter = Drain<'a, T>; + type Item = T; + + fn par_drain(self) -> Self::Iter { + Drain { heap: self } + } +} + +impl ParallelIterator for Drain<'_, T> { + type Item = T; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + bridge(self, consumer) + } + + fn opt_len(&self) -> Option { + Some(self.len()) + } +} + +impl IndexedParallelIterator for Drain<'_, T> { + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + bridge(self, consumer) + } + + fn len(&self) -> usize { + self.heap.len() + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + super::DrainGuard::new(self.heap) + .par_drain(..) + .with_producer(callback) + } +} + +impl Drop for Drain<'_, T> { + fn drop(&mut self) { + if !self.heap.is_empty() { + // We must not have produced, so just call a normal drain to remove the items. + self.heap.drain(); + } + } +} diff --git a/anneal/v2/vendor/rayon/src/collections/btree_map.rs b/anneal/v2/vendor/rayon/src/collections/btree_map.rs new file mode 100644 index 0000000000..56ed4b5544 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/collections/btree_map.rs @@ -0,0 +1,66 @@ +//! This module contains the parallel iterator types for B-Tree maps +//! (`BTreeMap`). You will rarely need to interact with it directly +//! unless you have need to name one of the iterator types. + +use std::collections::BTreeMap; + +use crate::iter::plumbing::*; +use crate::iter::*; + +use crate::vec; + +/// Parallel iterator over a B-Tree map +#[derive(Debug)] // std doesn't Clone +pub struct IntoIter { + inner: vec::IntoIter<(K, V)>, +} + +into_par_vec! { + BTreeMap => IntoIter, + impl +} + +delegate_iterator! { + IntoIter => (K, V), + impl +} + +/// Parallel iterator over an immutable reference to a B-Tree map +#[derive(Debug)] +pub struct Iter<'a, K, V> { + inner: vec::IntoIter<(&'a K, &'a V)>, +} + +impl Clone for Iter<'_, K, V> { + fn clone(&self) -> Self { + Iter { + inner: self.inner.clone(), + } + } +} + +into_par_vec! { + &'a BTreeMap => Iter<'a, K, V>, + impl<'a, K: Sync, V: Sync> +} + +delegate_iterator! { + Iter<'a, K, V> => (&'a K, &'a V), + impl<'a, K: Sync + 'a, V: Sync + 'a> +} + +/// Parallel iterator over a mutable reference to a B-Tree map +#[derive(Debug)] +pub struct IterMut<'a, K, V> { + inner: vec::IntoIter<(&'a K, &'a mut V)>, +} + +into_par_vec! { + &'a mut BTreeMap => IterMut<'a, K, V>, + impl<'a, K: Sync, V: Send> +} + +delegate_iterator! { + IterMut<'a, K, V> => (&'a K, &'a mut V), + impl<'a, K: Sync + 'a, V: Send + 'a> +} diff --git a/anneal/v2/vendor/rayon/src/collections/btree_set.rs b/anneal/v2/vendor/rayon/src/collections/btree_set.rs new file mode 100644 index 0000000000..35583625d6 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/collections/btree_set.rs @@ -0,0 +1,52 @@ +//! This module contains the parallel iterator types for B-Tree sets +//! (`BTreeSet`). You will rarely need to interact with it directly +//! unless you have need to name one of the iterator types. + +use std::collections::BTreeSet; + +use crate::iter::plumbing::*; +use crate::iter::*; + +use crate::vec; + +/// Parallel iterator over a B-Tree set +#[derive(Debug)] // std doesn't Clone +pub struct IntoIter { + inner: vec::IntoIter, +} + +into_par_vec! { + BTreeSet => IntoIter, + impl +} + +delegate_iterator! { + IntoIter => T, + impl +} + +/// Parallel iterator over an immutable reference to a B-Tree set +#[derive(Debug)] +pub struct Iter<'a, T> { + inner: vec::IntoIter<&'a T>, +} + +impl Clone for Iter<'_, T> { + fn clone(&self) -> Self { + Iter { + inner: self.inner.clone(), + } + } +} + +into_par_vec! { + &'a BTreeSet => Iter<'a, T>, + impl<'a, T: Sync> +} + +delegate_iterator! { + Iter<'a, T> => &'a T, + impl<'a, T: Sync + 'a> +} + +// `BTreeSet` doesn't have a mutable `Iterator` diff --git a/anneal/v2/vendor/rayon/src/collections/hash_map.rs b/anneal/v2/vendor/rayon/src/collections/hash_map.rs new file mode 100644 index 0000000000..a40b671c72 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/collections/hash_map.rs @@ -0,0 +1,93 @@ +//! This module contains the parallel iterator types for hash maps +//! (`HashMap`). You will rarely need to interact with it directly +//! unless you have need to name one of the iterator types. + +use std::collections::HashMap; +use std::marker::PhantomData; + +use crate::iter::plumbing::*; +use crate::iter::*; + +use crate::vec; + +/// Parallel iterator over a hash map +#[derive(Debug)] // std doesn't Clone +pub struct IntoIter { + inner: vec::IntoIter<(K, V)>, +} + +into_par_vec! { + HashMap => IntoIter, + impl +} + +delegate_iterator! { + IntoIter => (K, V), + impl +} + +/// Parallel iterator over an immutable reference to a hash map +#[derive(Debug)] +pub struct Iter<'a, K, V> { + inner: vec::IntoIter<(&'a K, &'a V)>, +} + +impl Clone for Iter<'_, K, V> { + fn clone(&self) -> Self { + Iter { + inner: self.inner.clone(), + } + } +} + +into_par_vec! { + &'a HashMap => Iter<'a, K, V>, + impl<'a, K: Sync, V: Sync, S> +} + +delegate_iterator! { + Iter<'a, K, V> => (&'a K, &'a V), + impl<'a, K: Sync, V: Sync> +} + +/// Parallel iterator over a mutable reference to a hash map +#[derive(Debug)] +pub struct IterMut<'a, K, V> { + inner: vec::IntoIter<(&'a K, &'a mut V)>, +} + +into_par_vec! { + &'a mut HashMap => IterMut<'a, K, V>, + impl<'a, K: Sync, V: Send, S> +} + +delegate_iterator! { + IterMut<'a, K, V> => (&'a K, &'a mut V), + impl<'a, K: Sync, V: Send> +} + +/// Draining parallel iterator that moves out of a hash map, +/// but keeps the total capacity. +#[derive(Debug)] +pub struct Drain<'a, K, V> { + inner: vec::IntoIter<(K, V)>, + marker: PhantomData<&'a mut HashMap>, +} + +impl<'a, K: Send, V: Send, S> ParallelDrainFull for &'a mut HashMap { + type Iter = Drain<'a, K, V>; + type Item = (K, V); + + fn par_drain(self) -> Self::Iter { + let vec: Vec<_> = self.drain().collect(); + Drain { + inner: vec.into_par_iter(), + marker: PhantomData, + } + } +} + +delegate_iterator! { + Drain<'_, K, V> => (K, V), + impl +} diff --git a/anneal/v2/vendor/rayon/src/collections/hash_set.rs b/anneal/v2/vendor/rayon/src/collections/hash_set.rs new file mode 100644 index 0000000000..8d08f35ba5 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/collections/hash_set.rs @@ -0,0 +1,79 @@ +//! This module contains the parallel iterator types for hash sets +//! (`HashSet`). You will rarely need to interact with it directly +//! unless you have need to name one of the iterator types. + +use std::collections::HashSet; +use std::marker::PhantomData; + +use crate::iter::plumbing::*; +use crate::iter::*; + +use crate::vec; + +/// Parallel iterator over a hash set +#[derive(Debug)] // std doesn't Clone +pub struct IntoIter { + inner: vec::IntoIter, +} + +into_par_vec! { + HashSet => IntoIter, + impl +} + +delegate_iterator! { + IntoIter => T, + impl +} + +/// Parallel iterator over an immutable reference to a hash set +#[derive(Debug)] +pub struct Iter<'a, T> { + inner: vec::IntoIter<&'a T>, +} + +impl Clone for Iter<'_, T> { + fn clone(&self) -> Self { + Iter { + inner: self.inner.clone(), + } + } +} + +into_par_vec! { + &'a HashSet => Iter<'a, T>, + impl<'a, T: Sync, S> +} + +delegate_iterator! { + Iter<'a, T> => &'a T, + impl<'a, T: Sync> +} + +// `HashSet` doesn't have a mutable `Iterator` + +/// Draining parallel iterator that moves out of a hash set, +/// but keeps the total capacity. +#[derive(Debug)] +pub struct Drain<'a, T> { + inner: vec::IntoIter, + marker: PhantomData<&'a mut HashSet>, +} + +impl<'a, T: Send, S> ParallelDrainFull for &'a mut HashSet { + type Iter = Drain<'a, T>; + type Item = T; + + fn par_drain(self) -> Self::Iter { + let vec: Vec<_> = self.drain().collect(); + Drain { + inner: vec.into_par_iter(), + marker: PhantomData, + } + } +} + +delegate_iterator! { + Drain<'_, T> => T, + impl +} diff --git a/anneal/v2/vendor/rayon/src/collections/linked_list.rs b/anneal/v2/vendor/rayon/src/collections/linked_list.rs new file mode 100644 index 0000000000..76f505aa26 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/collections/linked_list.rs @@ -0,0 +1,66 @@ +//! This module contains the parallel iterator types for linked lists +//! (`LinkedList`). You will rarely need to interact with it directly +//! unless you have need to name one of the iterator types. + +use std::collections::LinkedList; + +use crate::iter::plumbing::*; +use crate::iter::*; + +use crate::vec; + +/// Parallel iterator over a linked list +#[derive(Debug, Clone)] +pub struct IntoIter { + inner: vec::IntoIter, +} + +into_par_vec! { + LinkedList => IntoIter, + impl +} + +delegate_iterator! { + IntoIter => T, + impl +} + +/// Parallel iterator over an immutable reference to a linked list +#[derive(Debug)] +pub struct Iter<'a, T> { + inner: vec::IntoIter<&'a T>, +} + +impl Clone for Iter<'_, T> { + fn clone(&self) -> Self { + Iter { + inner: self.inner.clone(), + } + } +} + +into_par_vec! { + &'a LinkedList => Iter<'a, T>, + impl<'a, T: Sync> +} + +delegate_iterator! { + Iter<'a, T> => &'a T, + impl<'a, T: Sync> +} + +/// Parallel iterator over a mutable reference to a linked list +#[derive(Debug)] +pub struct IterMut<'a, T> { + inner: vec::IntoIter<&'a mut T>, +} + +into_par_vec! { + &'a mut LinkedList => IterMut<'a, T>, + impl<'a, T: Send> +} + +delegate_iterator! { + IterMut<'a, T> => &'a mut T, + impl<'a, T: Send> +} diff --git a/anneal/v2/vendor/rayon/src/collections/mod.rs b/anneal/v2/vendor/rayon/src/collections/mod.rs new file mode 100644 index 0000000000..e4675e49b3 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/collections/mod.rs @@ -0,0 +1,84 @@ +//! Parallel iterator types for [standard collections] +//! +//! You will rarely need to interact with this module directly unless you need +//! to name one of the iterator types. +//! +//! [standard collections]: std::collections + +/// Convert an iterable collection into a parallel iterator by first +/// collecting into a temporary `Vec`, then iterating that. +macro_rules! into_par_vec { + ($t:ty => $iter:ident<$($i:tt),*>, impl $($args:tt)*) => { + impl $($args)* IntoParallelIterator for $t { + type Item = <$t as IntoIterator>::Item; + type Iter = $iter<$($i),*>; + + fn into_par_iter(self) -> Self::Iter { + use std::iter::FromIterator; + $iter { inner: Vec::from_iter(self).into_par_iter() } + } + } + }; +} + +pub mod binary_heap; +pub mod btree_map; +pub mod btree_set; +pub mod hash_map; +pub mod hash_set; +pub mod linked_list; +pub mod vec_deque; + +use self::drain_guard::DrainGuard; + +mod drain_guard { + use crate::iter::ParallelDrainRange; + use std::mem; + use std::ops::RangeBounds; + + /// A proxy for draining a collection by converting to a `Vec` and back. + /// + /// This is used for draining `BinaryHeap` and `VecDeque`, which both have + /// zero-allocation conversions to/from `Vec`, though not zero-cost: + /// - `BinaryHeap` will heapify from `Vec`, but at least that will be empty. + /// - `VecDeque` has to shift items to offset 0 when converting to `Vec`. + #[allow(missing_debug_implementations)] + pub(super) struct DrainGuard<'a, T, C: From>> { + collection: &'a mut C, + vec: Vec, + } + + impl<'a, T, C> DrainGuard<'a, T, C> + where + C: Default + From>, + Vec: From, + { + pub(super) fn new(collection: &'a mut C) -> Self { + Self { + // Temporarily steal the inner `Vec` so we can drain in place. + vec: Vec::from(mem::take(collection)), + collection, + } + } + } + + impl<'a, T, C: From>> Drop for DrainGuard<'a, T, C> { + fn drop(&mut self) { + // Restore the collection from the `Vec` with its original capacity. + *self.collection = C::from(mem::take(&mut self.vec)); + } + } + + impl<'a, T, C> ParallelDrainRange for &'a mut DrainGuard<'_, T, C> + where + T: Send, + C: From>, + { + type Iter = crate::vec::Drain<'a, T>; + type Item = T; + + fn par_drain>(self, range: R) -> Self::Iter { + self.vec.par_drain(range) + } + } +} diff --git a/anneal/v2/vendor/rayon/src/collections/vec_deque.rs b/anneal/v2/vendor/rayon/src/collections/vec_deque.rs new file mode 100644 index 0000000000..22d00dec40 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/collections/vec_deque.rs @@ -0,0 +1,159 @@ +//! This module contains the parallel iterator types for double-ended queues +//! (`VecDeque`). You will rarely need to interact with it directly +//! unless you have need to name one of the iterator types. + +use std::collections::VecDeque; +use std::ops::{Range, RangeBounds}; + +use crate::iter::plumbing::*; +use crate::iter::*; +use crate::math::simplify_range; + +use crate::slice; +use crate::vec; + +/// Parallel iterator over a double-ended queue +#[derive(Debug, Clone)] +pub struct IntoIter { + inner: vec::IntoIter, +} + +impl IntoParallelIterator for VecDeque { + type Item = T; + type Iter = IntoIter; + + fn into_par_iter(self) -> Self::Iter { + // NOTE: requires data movement if the deque doesn't start at offset 0. + let inner = Vec::from(self).into_par_iter(); + IntoIter { inner } + } +} + +delegate_indexed_iterator! { + IntoIter => T, + impl +} + +/// Parallel iterator over an immutable reference to a double-ended queue +#[derive(Debug)] +pub struct Iter<'a, T> { + inner: Chain, slice::Iter<'a, T>>, +} + +impl Clone for Iter<'_, T> { + fn clone(&self) -> Self { + Iter { + inner: self.inner.clone(), + } + } +} + +impl<'a, T: Sync> IntoParallelIterator for &'a VecDeque { + type Item = &'a T; + type Iter = Iter<'a, T>; + + fn into_par_iter(self) -> Self::Iter { + let (a, b) = self.as_slices(); + Iter { + inner: a.into_par_iter().chain(b), + } + } +} + +delegate_indexed_iterator! { + Iter<'a, T> => &'a T, + impl<'a, T: Sync> +} + +/// Parallel iterator over a mutable reference to a double-ended queue +#[derive(Debug)] +pub struct IterMut<'a, T> { + inner: Chain, slice::IterMut<'a, T>>, +} + +impl<'a, T: Send> IntoParallelIterator for &'a mut VecDeque { + type Item = &'a mut T; + type Iter = IterMut<'a, T>; + + fn into_par_iter(self) -> Self::Iter { + let (a, b) = self.as_mut_slices(); + IterMut { + inner: a.into_par_iter().chain(b), + } + } +} + +delegate_indexed_iterator! { + IterMut<'a, T> => &'a mut T, + impl<'a, T: Send> +} + +/// Draining parallel iterator that moves a range out of a double-ended queue, +/// but keeps the total capacity. +#[derive(Debug)] +pub struct Drain<'a, T> { + deque: &'a mut VecDeque, + range: Range, + orig_len: usize, +} + +impl<'a, T: Send> ParallelDrainRange for &'a mut VecDeque { + type Iter = Drain<'a, T>; + type Item = T; + + fn par_drain>(self, range: R) -> Self::Iter { + Drain { + orig_len: self.len(), + range: simplify_range(range, self.len()), + deque: self, + } + } +} + +impl ParallelIterator for Drain<'_, T> { + type Item = T; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + bridge(self, consumer) + } + + fn opt_len(&self) -> Option { + Some(self.len()) + } +} + +impl IndexedParallelIterator for Drain<'_, T> { + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + bridge(self, consumer) + } + + fn len(&self) -> usize { + self.range.len() + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + // NOTE: requires data movement if the deque doesn't start at offset 0. + super::DrainGuard::new(self.deque) + .par_drain(self.range.clone()) + .with_producer(callback) + } +} + +impl Drop for Drain<'_, T> { + fn drop(&mut self) { + if self.deque.len() != self.orig_len - self.range.len() { + // We must not have produced, so just call a normal drain to remove the items. + assert_eq!(self.deque.len(), self.orig_len); + self.deque.drain(self.range.clone()); + } + } +} diff --git a/anneal/v2/vendor/rayon/src/compile_fail/cannot_collect_filtermap_data.rs b/anneal/v2/vendor/rayon/src/compile_fail/cannot_collect_filtermap_data.rs new file mode 100644 index 0000000000..bb62ce02f5 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/compile_fail/cannot_collect_filtermap_data.rs @@ -0,0 +1,14 @@ +/*! ```compile_fail,E0599 + +use rayon::prelude::*; + +// zip requires data of exact size, but filter yields only bounded +// size, so check that we cannot apply it. + +let a: Vec = (0..1024).collect(); +let mut v = vec![]; +a.par_iter() + .filter_map(|&x| Some(x as f32)) + .collect_into_vec(&mut v); //~ ERROR no method + +``` */ diff --git a/anneal/v2/vendor/rayon/src/compile_fail/cannot_zip_filtered_data.rs b/anneal/v2/vendor/rayon/src/compile_fail/cannot_zip_filtered_data.rs new file mode 100644 index 0000000000..54fd50de1d --- /dev/null +++ b/anneal/v2/vendor/rayon/src/compile_fail/cannot_zip_filtered_data.rs @@ -0,0 +1,14 @@ +/*! ```compile_fail,E0277 + +use rayon::prelude::*; + +// zip requires data of exact size, but filter yields only bounded +// size, so check that we cannot apply it. + +let mut a: Vec = (0..1024).rev().collect(); +let b: Vec = (0..1024).collect(); + +a.par_iter() + .zip(b.par_iter().filter(|&&x| x > 3)); //~ ERROR + +``` */ diff --git a/anneal/v2/vendor/rayon/src/compile_fail/cell_par_iter.rs b/anneal/v2/vendor/rayon/src/compile_fail/cell_par_iter.rs new file mode 100644 index 0000000000..98a1cf96ef --- /dev/null +++ b/anneal/v2/vendor/rayon/src/compile_fail/cell_par_iter.rs @@ -0,0 +1,13 @@ +/*! ```compile_fail,E0277 + +// Check that we can't use the par-iter API to access contents of a `Cell`. + +use rayon::prelude::*; +use std::cell::Cell; + +let c = Cell::new(42_i32); +(0_i32..1024).into_par_iter() + .map(|_| c.get()) //~ ERROR E0277 + .min(); + +``` */ diff --git a/anneal/v2/vendor/rayon/src/compile_fail/mod.rs b/anneal/v2/vendor/rayon/src/compile_fail/mod.rs new file mode 100644 index 0000000000..13209a40d0 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/compile_fail/mod.rs @@ -0,0 +1,7 @@ +// These modules contain `compile_fail` doc tests. +mod cannot_collect_filtermap_data; +mod cannot_zip_filtered_data; +mod cell_par_iter; +mod must_use; +mod no_send_par_iter; +mod rc_par_iter; diff --git a/anneal/v2/vendor/rayon/src/compile_fail/must_use.rs b/anneal/v2/vendor/rayon/src/compile_fail/must_use.rs new file mode 100644 index 0000000000..da03b8f61a --- /dev/null +++ b/anneal/v2/vendor/rayon/src/compile_fail/must_use.rs @@ -0,0 +1,71 @@ +// Check that we are flagged for ignoring `must_use` parallel adaptors. +// (unfortunately there's no error code for `unused_must_use`) + +macro_rules! must_use { + ($( $name:ident #[$expr:meta] )*) => {$( + /// First sanity check that the expression is OK. + /// + /// ``` + /// #![deny(unused_must_use)] + /// + /// use rayon::prelude::*; + /// + /// let v: Vec<_> = (0..100).map(Some).collect(); + /// let _ = + #[$expr] + /// ``` + /// + /// Now trigger the `must_use`. + /// + /// ```compile_fail + /// #![deny(unused_must_use)] + /// + /// use rayon::prelude::*; + /// + /// let v: Vec<_> = (0..100).map(Some).collect(); + #[$expr] + /// ``` + mod $name {} + )*} +} + +must_use! { + by_exponential_blocks /** v.par_iter().by_exponential_blocks(); */ + by_uniform_blocks /** v.par_iter().by_uniform_blocks(2); */ + step_by /** v.par_iter().step_by(2); */ + chain /** v.par_iter().chain(&v); */ + chunks /** v.par_iter().chunks(2); */ + fold_chunks /** v.par_iter().fold_chunks(2, || 0, |x, _| x); */ + fold_chunks_with /** v.par_iter().fold_chunks_with(2, 0, |x, _| x); */ + cloned /** v.par_iter().cloned(); */ + copied /** v.par_iter().copied(); */ + enumerate /** v.par_iter().enumerate(); */ + filter /** v.par_iter().filter(|_| true); */ + filter_map /** v.par_iter().filter_map(|x| *x); */ + flat_map /** v.par_iter().flat_map(|x| *x); */ + flat_map_iter /** v.par_iter().flat_map_iter(|x| *x); */ + flatten /** v.par_iter().flatten(); */ + flatten_iter /** v.par_iter().flatten_iter(); */ + fold /** v.par_iter().fold(|| 0, |x, _| x); */ + fold_with /** v.par_iter().fold_with(0, |x, _| x); */ + try_fold /** v.par_iter().try_fold(|| 0, |x, _| Some(x)); */ + try_fold_with /** v.par_iter().try_fold_with(0, |x, _| Some(x)); */ + inspect /** v.par_iter().inspect(|_| {}); */ + interleave /** v.par_iter().interleave(&v); */ + interleave_shortest /** v.par_iter().interleave_shortest(&v); */ + intersperse /** v.par_iter().intersperse(&None); */ + map /** v.par_iter().map(|x| x); */ + map_with /** v.par_iter().map_with(0, |_, x| x); */ + map_init /** v.par_iter().map_init(|| 0, |_, x| x); */ + panic_fuse /** v.par_iter().panic_fuse(); */ + positions /** v.par_iter().positions(|_| true); */ + rev /** v.par_iter().rev(); */ + skip /** v.par_iter().skip(1); */ + take /** v.par_iter().take(1); */ + update /** v.par_iter().update(|_| {}); */ + while_some /** v.par_iter().cloned().while_some(); */ + with_max_len /** v.par_iter().with_max_len(1); */ + with_min_len /** v.par_iter().with_min_len(1); */ + zip /** v.par_iter().zip(&v); */ + zip_eq /** v.par_iter().zip_eq(&v); */ +} diff --git a/anneal/v2/vendor/rayon/src/compile_fail/no_send_par_iter.rs b/anneal/v2/vendor/rayon/src/compile_fail/no_send_par_iter.rs new file mode 100644 index 0000000000..7573c0346f --- /dev/null +++ b/anneal/v2/vendor/rayon/src/compile_fail/no_send_par_iter.rs @@ -0,0 +1,58 @@ +// Check that `!Send` types fail early. + +/** ```compile_fail,E0277 + +use rayon::prelude::*; +use std::ptr::null; + +#[derive(Copy, Clone)] +struct NoSend(*const ()); + +unsafe impl Sync for NoSend {} + +let x = Some(NoSend(null())); + +x.par_iter() + .map(|&x| x) //~ ERROR + .count(); //~ ERROR + +``` */ +mod map {} + +/** ```compile_fail,E0277 + +use rayon::prelude::*; +use std::ptr::null; + +#[derive(Copy, Clone)] +struct NoSend(*const ()); + +unsafe impl Sync for NoSend {} + +let x = Some(NoSend(null())); + +x.par_iter() + .filter_map(|&x| Some(x)) //~ ERROR + .count(); //~ ERROR + +``` */ +mod filter_map {} + +/** ```compile_fail,E0277 + +use rayon::prelude::*; +use std::ptr::null; + +#[derive(Copy, Clone)] +struct NoSend(*const ()); + +unsafe impl Sync for NoSend {} + +let x = Some(NoSend(null())); + +x.par_iter() + .cloned() //~ ERROR + .count(); //~ ERROR + +``` */ +mod cloned {} diff --git a/anneal/v2/vendor/rayon/src/compile_fail/rc_par_iter.rs b/anneal/v2/vendor/rayon/src/compile_fail/rc_par_iter.rs new file mode 100644 index 0000000000..aca63e7ace --- /dev/null +++ b/anneal/v2/vendor/rayon/src/compile_fail/rc_par_iter.rs @@ -0,0 +1,15 @@ +/*! ```compile_fail,E0599 + +// Check that we can't use the par-iter API to access contents of an +// `Rc`. + +use rayon::prelude::*; +use std::rc::Rc; + +let x = vec![Rc::new(22), Rc::new(23)]; +let mut y = vec![]; +x.into_par_iter() //~ ERROR no method named `into_par_iter` + .map(|rc| *rc) + .collect_into_vec(&mut y); + +``` */ diff --git a/anneal/v2/vendor/rayon/src/delegate.rs b/anneal/v2/vendor/rayon/src/delegate.rs new file mode 100644 index 0000000000..2a6e331930 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/delegate.rs @@ -0,0 +1,109 @@ +//! Macros for delegating newtype iterators to inner types. + +// Note: these place `impl` bounds at the end, as token gobbling is the only way +// I know how to consume an arbitrary list of constraints, with `$($args:tt)*`. + +/// Creates a parallel iterator implementation which simply wraps an inner type +/// and delegates all methods inward. The actual struct must already be +/// declared with an `inner` field. +/// +/// The implementation of `IntoParallelIterator` should be added separately. +macro_rules! delegate_iterator { + ($iter:ty => $item:ty , + impl $( $args:tt )* + ) => { + impl $( $args )* ParallelIterator for $iter { + type Item = $item; + + fn drive_unindexed(self, consumer: C) -> C::Result + where C: UnindexedConsumer + { + self.inner.drive_unindexed(consumer) + } + + fn opt_len(&self) -> Option { + self.inner.opt_len() + } + } + } +} + +/// Creates an indexed parallel iterator implementation which simply wraps an +/// inner type and delegates all methods inward. The actual struct must already +/// be declared with an `inner` field. +macro_rules! delegate_indexed_iterator { + ($iter:ty => $item:ty , + impl $( $args:tt )* + ) => { + delegate_iterator!{ + $iter => $item , + impl $( $args )* + } + + impl $( $args )* IndexedParallelIterator for $iter { + fn drive(self, consumer: C) -> C::Result + where C: Consumer + { + self.inner.drive(consumer) + } + + fn len(&self) -> usize { + self.inner.len() + } + + fn with_producer(self, callback: CB) -> CB::Output + where CB: ProducerCallback + { + self.inner.with_producer(callback) + } + } + } +} + +#[test] +fn unindexed_example() { + use crate::collections::btree_map::IntoIter; + use crate::iter::plumbing::*; + use crate::prelude::*; + + use std::collections::BTreeMap; + + struct MyIntoIter { + inner: IntoIter, + } + + delegate_iterator! { + MyIntoIter => (T, U), + impl + } + + let map = BTreeMap::from([(1, 'a'), (2, 'b'), (3, 'c')]); + let iter = MyIntoIter { + inner: map.into_par_iter(), + }; + let vec: Vec<_> = iter.map(|(k, _)| k).collect(); + assert_eq!(vec, &[1, 2, 3]); +} + +#[test] +fn indexed_example() { + use crate::iter::plumbing::*; + use crate::prelude::*; + use crate::vec::IntoIter; + + struct MyIntoIter { + inner: IntoIter, + } + + delegate_indexed_iterator! { + MyIntoIter => T, + impl + } + + let iter = MyIntoIter { + inner: vec![1, 2, 3].into_par_iter(), + }; + let mut vec = vec![]; + iter.collect_into_vec(&mut vec); + assert_eq!(vec, &[1, 2, 3]); +} diff --git a/anneal/v2/vendor/rayon/src/iter/blocks.rs b/anneal/v2/vendor/rayon/src/iter/blocks.rs new file mode 100644 index 0000000000..1c07391d18 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/blocks.rs @@ -0,0 +1,129 @@ +use super::plumbing::*; +use super::*; + +struct BlocksCallback { + sizes: S, + consumer: C, + len: usize, +} + +impl ProducerCallback for BlocksCallback +where + C: UnindexedConsumer, + S: Iterator, +{ + type Output = C::Result; + + fn callback>(mut self, mut producer: P) -> Self::Output { + let mut remaining_len = self.len; + let mut consumer = self.consumer; + + // we need a local variable for the accumulated results + // we call the reducer's identity by splitting at 0 + let (left_consumer, right_consumer, _) = consumer.split_at(0); + let mut leftmost_res = left_consumer.into_folder().complete(); + consumer = right_consumer; + + // now we loop on each block size + while remaining_len > 0 && !consumer.full() { + // we compute the next block's size + let size = self.sizes.next().unwrap_or(usize::MAX); + let capped_size = remaining_len.min(size); + remaining_len -= capped_size; + + // split the producer + let (left_producer, right_producer) = producer.split_at(capped_size); + producer = right_producer; + + // split the consumer + let (left_consumer, right_consumer, _) = consumer.split_at(capped_size); + consumer = right_consumer; + + leftmost_res = consumer.to_reducer().reduce( + leftmost_res, + bridge_producer_consumer(capped_size, left_producer, left_consumer), + ); + } + leftmost_res + } +} + +/// `ExponentialBlocks` is a parallel iterator that consumes itself as a sequence +/// of parallel blocks of increasing sizes (exponentially). +/// +/// This struct is created by the [`by_exponential_blocks()`] method on [`IndexedParallelIterator`] +/// +/// [`by_exponential_blocks()`]: IndexedParallelIterator::by_exponential_blocks() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Debug, Clone)] +pub struct ExponentialBlocks { + base: I, +} + +impl ExponentialBlocks { + pub(super) fn new(base: I) -> Self { + Self { base } + } +} + +impl ParallelIterator for ExponentialBlocks +where + I: IndexedParallelIterator, +{ + type Item = I::Item; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let first = crate::current_num_threads(); + let callback = BlocksCallback { + consumer, + sizes: std::iter::successors(Some(first), exponential_size), + len: self.base.len(), + }; + self.base.with_producer(callback) + } +} + +fn exponential_size(size: &usize) -> Option { + Some(size.saturating_mul(2)) +} + +/// `UniformBlocks` is a parallel iterator that consumes itself as a sequence +/// of parallel blocks of constant sizes. +/// +/// This struct is created by the [`by_uniform_blocks()`] method on [`IndexedParallelIterator`] +/// +/// [`by_uniform_blocks()`]: IndexedParallelIterator::by_uniform_blocks() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Debug, Clone)] +pub struct UniformBlocks { + base: I, + block_size: usize, +} + +impl UniformBlocks { + pub(super) fn new(base: I, block_size: usize) -> Self { + Self { base, block_size } + } +} + +impl ParallelIterator for UniformBlocks +where + I: IndexedParallelIterator, +{ + type Item = I::Item; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let callback = BlocksCallback { + consumer, + sizes: std::iter::repeat(self.block_size), + len: self.base.len(), + }; + self.base.with_producer(callback) + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/chain.rs b/anneal/v2/vendor/rayon/src/iter/chain.rs new file mode 100644 index 0000000000..57f6c62092 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/chain.rs @@ -0,0 +1,258 @@ +use super::plumbing::*; +use super::*; +use rayon_core::join; +use std::iter; + +/// `Chain` is an iterator that joins `b` after `a` in one continuous iterator. +/// This struct is created by the [`chain()`] method on [`ParallelIterator`] +/// +/// [`chain()`]: ParallelIterator::chain() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Debug, Clone)] +pub struct Chain { + a: A, + b: B, +} + +impl Chain { + /// Creates a new `Chain` iterator. + pub(super) fn new(a: A, b: B) -> Self { + Chain { a, b } + } +} + +impl ParallelIterator for Chain +where + A: ParallelIterator, + B: ParallelIterator, +{ + type Item = A::Item; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let Chain { a, b } = self; + + // If we returned a value from our own `opt_len`, then the collect consumer in particular + // will balk at being treated like an actual `UnindexedConsumer`. But when we do know the + // length, we can use `Consumer::split_at` instead, and this is still harmless for other + // truly-unindexed consumers too. + let (left, right, reducer) = if let Some(len) = a.opt_len() { + consumer.split_at(len) + } else { + let reducer = consumer.to_reducer(); + (consumer.split_off_left(), consumer, reducer) + }; + + let (a, b) = join(|| a.drive_unindexed(left), || b.drive_unindexed(right)); + reducer.reduce(a, b) + } + + fn opt_len(&self) -> Option { + self.a.opt_len()?.checked_add(self.b.opt_len()?) + } +} + +impl IndexedParallelIterator for Chain +where + A: IndexedParallelIterator, + B: IndexedParallelIterator, +{ + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + let Chain { a, b } = self; + let (left, right, reducer) = consumer.split_at(a.len()); + let (a, b) = join(|| a.drive(left), || b.drive(right)); + reducer.reduce(a, b) + } + + fn len(&self) -> usize { + self.a.len().checked_add(self.b.len()).expect("overflow") + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + let a_len = self.a.len(); + return self.a.with_producer(CallbackA { + callback, + a_len, + b: self.b, + }); + + struct CallbackA { + callback: CB, + a_len: usize, + b: B, + } + + impl ProducerCallback for CallbackA + where + B: IndexedParallelIterator, + CB: ProducerCallback, + { + type Output = CB::Output; + + fn callback(self, a_producer: A) -> Self::Output + where + A: Producer, + { + self.b.with_producer(CallbackB { + callback: self.callback, + a_len: self.a_len, + a_producer, + }) + } + } + + struct CallbackB { + callback: CB, + a_len: usize, + a_producer: A, + } + + impl ProducerCallback for CallbackB + where + A: Producer, + CB: ProducerCallback, + { + type Output = CB::Output; + + fn callback(self, b_producer: B) -> Self::Output + where + B: Producer, + { + let producer = ChainProducer::new(self.a_len, self.a_producer, b_producer); + self.callback.callback(producer) + } + } + } +} + +// //////////////////////////////////////////////////////////////////////// + +struct ChainProducer +where + A: Producer, + B: Producer, +{ + a_len: usize, + a: A, + b: B, +} + +impl ChainProducer +where + A: Producer, + B: Producer, +{ + fn new(a_len: usize, a: A, b: B) -> Self { + ChainProducer { a_len, a, b } + } +} + +impl Producer for ChainProducer +where + A: Producer, + B: Producer, +{ + type Item = A::Item; + type IntoIter = ChainSeq; + + fn into_iter(self) -> Self::IntoIter { + ChainSeq::new(self.a.into_iter(), self.b.into_iter()) + } + + fn min_len(&self) -> usize { + Ord::max(self.a.min_len(), self.b.min_len()) + } + + fn max_len(&self) -> usize { + Ord::min(self.a.max_len(), self.b.max_len()) + } + + fn split_at(self, index: usize) -> (Self, Self) { + if index <= self.a_len { + let a_rem = self.a_len - index; + let (a_left, a_right) = self.a.split_at(index); + let (b_left, b_right) = self.b.split_at(0); + ( + ChainProducer::new(index, a_left, b_left), + ChainProducer::new(a_rem, a_right, b_right), + ) + } else { + let (a_left, a_right) = self.a.split_at(self.a_len); + let (b_left, b_right) = self.b.split_at(index - self.a_len); + ( + ChainProducer::new(self.a_len, a_left, b_left), + ChainProducer::new(0, a_right, b_right), + ) + } + } + + fn fold_with(self, mut folder: F) -> F + where + F: Folder, + { + folder = self.a.fold_with(folder); + if folder.full() { + folder + } else { + self.b.fold_with(folder) + } + } +} + +// //////////////////////////////////////////////////////////////////////// + +/// Wrapper for `Chain` to implement `ExactSizeIterator` +struct ChainSeq { + chain: iter::Chain, +} + +impl ChainSeq { + fn new(a: A, b: B) -> ChainSeq + where + A: ExactSizeIterator, + B: ExactSizeIterator, + { + ChainSeq { chain: a.chain(b) } + } +} + +impl Iterator for ChainSeq +where + A: Iterator, + B: Iterator, +{ + type Item = A::Item; + + fn next(&mut self) -> Option { + self.chain.next() + } + + fn size_hint(&self) -> (usize, Option) { + self.chain.size_hint() + } +} + +impl ExactSizeIterator for ChainSeq +where + A: ExactSizeIterator, + B: ExactSizeIterator, +{ +} + +impl DoubleEndedIterator for ChainSeq +where + A: DoubleEndedIterator, + B: DoubleEndedIterator, +{ + fn next_back(&mut self) -> Option { + self.chain.next_back() + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/chunks.rs b/anneal/v2/vendor/rayon/src/iter/chunks.rs new file mode 100644 index 0000000000..97ef081520 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/chunks.rs @@ -0,0 +1,216 @@ +use super::plumbing::*; +use super::*; + +/// `Chunks` is an iterator that groups elements of an underlying iterator. +/// +/// This struct is created by the [`chunks()`] method on [`IndexedParallelIterator`] +/// +/// [`chunks()`]: IndexedParallelIterator::chunks() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Debug, Clone)] +pub struct Chunks { + size: usize, + i: I, +} + +impl Chunks { + /// Creates a new `Chunks` iterator + pub(super) fn new(i: I, size: usize) -> Self { + Chunks { i, size } + } +} + +impl ParallelIterator for Chunks +where + I: IndexedParallelIterator, +{ + type Item = Vec; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: Consumer>, + { + bridge(self, consumer) + } + + fn opt_len(&self) -> Option { + Some(self.len()) + } +} + +impl IndexedParallelIterator for Chunks +where + I: IndexedParallelIterator, +{ + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + bridge(self, consumer) + } + + fn len(&self) -> usize { + self.i.len().div_ceil(self.size) + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + let len = self.i.len(); + return self.i.with_producer(Callback { + size: self.size, + len, + callback, + }); + + struct Callback { + size: usize, + len: usize, + callback: CB, + } + + impl ProducerCallback for Callback + where + CB: ProducerCallback>, + { + type Output = CB::Output; + + fn callback

(self, base: P) -> CB::Output + where + P: Producer, + { + let producer = ChunkProducer::new(self.size, self.len, base, Vec::from_iter); + self.callback.callback(producer) + } + } + } +} + +pub(super) struct ChunkProducer { + chunk_size: usize, + len: usize, + base: P, + map: F, +} + +impl ChunkProducer { + pub(super) fn new(chunk_size: usize, len: usize, base: P, map: F) -> Self { + Self { + chunk_size, + len, + base, + map, + } + } +} + +impl Producer for ChunkProducer +where + P: Producer, + F: Fn(P::IntoIter) -> T + Send + Clone, +{ + type Item = T; + type IntoIter = std::iter::Map, F>; + + fn into_iter(self) -> Self::IntoIter { + let chunks = ChunkSeq { + chunk_size: self.chunk_size, + len: self.len, + inner: if self.len > 0 { Some(self.base) } else { None }, + }; + chunks.map(self.map) + } + + fn split_at(self, index: usize) -> (Self, Self) { + let elem_index = Ord::min(index * self.chunk_size, self.len); + let (left, right) = self.base.split_at(elem_index); + ( + ChunkProducer { + chunk_size: self.chunk_size, + len: elem_index, + base: left, + map: self.map.clone(), + }, + ChunkProducer { + chunk_size: self.chunk_size, + len: self.len - elem_index, + base: right, + map: self.map, + }, + ) + } + + fn min_len(&self) -> usize { + self.base.min_len().div_ceil(self.chunk_size) + } + + fn max_len(&self) -> usize { + self.base.max_len() / self.chunk_size + } +} + +pub(super) struct ChunkSeq

{ + chunk_size: usize, + len: usize, + inner: Option

, +} + +impl

Iterator for ChunkSeq

+where + P: Producer, +{ + type Item = P::IntoIter; + + fn next(&mut self) -> Option { + let producer = self.inner.take()?; + if self.len > self.chunk_size { + let (left, right) = producer.split_at(self.chunk_size); + self.inner = Some(right); + self.len -= self.chunk_size; + Some(left.into_iter()) + } else { + debug_assert!(self.len > 0); + self.len = 0; + Some(producer.into_iter()) + } + } + + fn size_hint(&self) -> (usize, Option) { + let len = self.len(); + (len, Some(len)) + } +} + +impl

ExactSizeIterator for ChunkSeq

+where + P: Producer, +{ + #[inline] + fn len(&self) -> usize { + self.len.div_ceil(self.chunk_size) + } +} + +impl

DoubleEndedIterator for ChunkSeq

+where + P: Producer, +{ + fn next_back(&mut self) -> Option { + let producer = self.inner.take()?; + if self.len > self.chunk_size { + let mut size = self.len % self.chunk_size; + if size == 0 { + size = self.chunk_size; + } + let (left, right) = producer.split_at(self.len - size); + self.inner = Some(left); + self.len -= size; + Some(right.into_iter()) + } else { + debug_assert!(self.len > 0); + self.len = 0; + Some(producer.into_iter()) + } + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/cloned.rs b/anneal/v2/vendor/rayon/src/iter/cloned.rs new file mode 100644 index 0000000000..381b12f372 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/cloned.rs @@ -0,0 +1,219 @@ +use super::plumbing::*; +use super::*; + +use std::iter; + +/// `Cloned` is an iterator that clones the elements of an underlying iterator. +/// +/// This struct is created by the [`cloned()`] method on [`ParallelIterator`] +/// +/// [`cloned()`]: ParallelIterator::cloned() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Debug, Clone)] +pub struct Cloned { + base: I, +} + +impl Cloned { + /// Creates a new `Cloned` iterator. + pub(super) fn new(base: I) -> Self { + Cloned { base } + } +} + +impl<'a, T, I> ParallelIterator for Cloned +where + I: ParallelIterator, + T: 'a + Clone + Send + Sync, +{ + type Item = T; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let consumer1 = ClonedConsumer::new(consumer); + self.base.drive_unindexed(consumer1) + } + + fn opt_len(&self) -> Option { + self.base.opt_len() + } +} + +impl<'a, T, I> IndexedParallelIterator for Cloned +where + I: IndexedParallelIterator, + T: 'a + Clone + Send + Sync, +{ + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + let consumer1 = ClonedConsumer::new(consumer); + self.base.drive(consumer1) + } + + fn len(&self) -> usize { + self.base.len() + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + return self.base.with_producer(Callback { callback }); + + struct Callback { + callback: CB, + } + + impl<'a, T, CB> ProducerCallback<&'a T> for Callback + where + CB: ProducerCallback, + T: 'a + Clone + Send, + { + type Output = CB::Output; + + fn callback

(self, base: P) -> CB::Output + where + P: Producer, + { + let producer = ClonedProducer { base }; + self.callback.callback(producer) + } + } + } +} + +// //////////////////////////////////////////////////////////////////////// + +struct ClonedProducer

{ + base: P, +} + +impl<'a, T, P> Producer for ClonedProducer

+where + P: Producer, + T: 'a + Clone, +{ + type Item = T; + type IntoIter = iter::Cloned; + + fn into_iter(self) -> Self::IntoIter { + self.base.into_iter().cloned() + } + + fn min_len(&self) -> usize { + self.base.min_len() + } + + fn max_len(&self) -> usize { + self.base.max_len() + } + + fn split_at(self, index: usize) -> (Self, Self) { + let (left, right) = self.base.split_at(index); + ( + ClonedProducer { base: left }, + ClonedProducer { base: right }, + ) + } + + fn fold_with(self, folder: F) -> F + where + F: Folder, + { + self.base.fold_with(ClonedFolder { base: folder }).base + } +} + +// //////////////////////////////////////////////////////////////////////// +// Consumer implementation + +struct ClonedConsumer { + base: C, +} + +impl ClonedConsumer { + fn new(base: C) -> Self { + ClonedConsumer { base } + } +} + +impl<'a, T, C> Consumer<&'a T> for ClonedConsumer +where + C: Consumer, + T: 'a + Clone, +{ + type Folder = ClonedFolder; + type Reducer = C::Reducer; + type Result = C::Result; + + fn split_at(self, index: usize) -> (Self, Self, Self::Reducer) { + let (left, right, reducer) = self.base.split_at(index); + ( + ClonedConsumer::new(left), + ClonedConsumer::new(right), + reducer, + ) + } + + fn into_folder(self) -> Self::Folder { + ClonedFolder { + base: self.base.into_folder(), + } + } + + fn full(&self) -> bool { + self.base.full() + } +} + +impl<'a, T, C> UnindexedConsumer<&'a T> for ClonedConsumer +where + C: UnindexedConsumer, + T: 'a + Clone, +{ + fn split_off_left(&self) -> Self { + ClonedConsumer::new(self.base.split_off_left()) + } + + fn to_reducer(&self) -> Self::Reducer { + self.base.to_reducer() + } +} + +struct ClonedFolder { + base: F, +} + +impl<'a, T, F> Folder<&'a T> for ClonedFolder +where + F: Folder, + T: 'a + Clone, +{ + type Result = F::Result; + + fn consume(self, item: &'a T) -> Self { + ClonedFolder { + base: self.base.consume(item.clone()), + } + } + + fn consume_iter(mut self, iter: I) -> Self + where + I: IntoIterator, + { + self.base = self.base.consume_iter(iter.into_iter().cloned()); + self + } + + fn complete(self) -> F::Result { + self.base.complete() + } + + fn full(&self) -> bool { + self.base.full() + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/collect/consumer.rs b/anneal/v2/vendor/rayon/src/iter/collect/consumer.rs new file mode 100644 index 0000000000..329dce5c96 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/collect/consumer.rs @@ -0,0 +1,185 @@ +use super::super::plumbing::*; +use crate::SendPtr; +use std::marker::PhantomData; +use std::ptr; + +pub(super) struct CollectConsumer<'c, T: Send> { + /// See `CollectResult` for explanation of why this is not a slice + start: SendPtr, + len: usize, + marker: PhantomData<&'c mut T>, +} + +impl CollectConsumer<'_, T> { + /// Create a collector for `len` items in the unused capacity of the vector. + pub(super) fn appender(vec: &mut Vec, len: usize) -> CollectConsumer<'_, T> { + let start = vec.len(); + assert!(vec.capacity() - start >= len); + + // SAFETY: We already made sure to have the additional space allocated. + // The pointer is derived from `Vec` directly, not through a `Deref`, + // so it has provenance over the whole allocation. + unsafe { CollectConsumer::new(vec.as_mut_ptr().add(start), len) } + } +} + +impl<'c, T: Send + 'c> CollectConsumer<'c, T> { + /// The target memory is considered uninitialized, and will be + /// overwritten without reading or dropping existing values. + unsafe fn new(start: *mut T, len: usize) -> Self { + CollectConsumer { + start: SendPtr(start), + len, + marker: PhantomData, + } + } +} + +/// CollectResult represents an initialized part of the target slice. +/// +/// This is a proxy owner of the elements in the slice; when it drops, +/// the elements will be dropped, unless its ownership is released before then. +#[must_use] +pub(super) struct CollectResult<'c, T> { + /// This pointer and length has the same representation as a slice, + /// but retains the provenance of the entire array so that we can merge + /// these regions together in `CollectReducer`. + start: SendPtr, + total_len: usize, + /// The current initialized length after `start` + initialized_len: usize, + /// Lifetime invariance guarantees that the data flows from consumer to result, + /// especially for the `scope_fn` callback in `Collect::with_consumer`. + invariant_lifetime: PhantomData<&'c mut &'c mut [T]>, +} + +unsafe impl<'c, T> Send for CollectResult<'c, T> where T: Send {} + +impl<'c, T> CollectResult<'c, T> { + /// The current length of the collect result + pub(super) fn len(&self) -> usize { + self.initialized_len + } + + /// Release ownership of the slice of elements, and return the length + pub(super) fn release_ownership(mut self) -> usize { + let ret = self.initialized_len; + self.initialized_len = 0; + ret + } +} + +impl<'c, T> Drop for CollectResult<'c, T> { + fn drop(&mut self) { + // Drop the first `self.initialized_len` elements, which have been recorded + // to be initialized by the folder. + unsafe { + ptr::drop_in_place(ptr::slice_from_raw_parts_mut( + self.start.0, + self.initialized_len, + )); + } + } +} + +impl<'c, T: Send + 'c> Consumer for CollectConsumer<'c, T> { + type Folder = CollectResult<'c, T>; + type Reducer = CollectReducer; + type Result = CollectResult<'c, T>; + + fn split_at(self, index: usize) -> (Self, Self, CollectReducer) { + let CollectConsumer { start, len, .. } = self; + + // Produce new consumers. + // SAFETY: This assert checks that `index` is a valid offset for `start` + unsafe { + assert!(index <= len); + ( + CollectConsumer::new(start.0, index), + CollectConsumer::new(start.0.add(index), len - index), + CollectReducer, + ) + } + } + + fn into_folder(self) -> Self::Folder { + // Create a result/folder that consumes values and writes them + // into the region after start. The initial result has length 0. + CollectResult { + start: self.start, + total_len: self.len, + initialized_len: 0, + invariant_lifetime: PhantomData, + } + } + + fn full(&self) -> bool { + false + } +} + +impl<'c, T: Send + 'c> Folder for CollectResult<'c, T> { + type Result = Self; + + fn consume(mut self, item: T) -> Self { + assert!( + self.initialized_len < self.total_len, + "too many values pushed to consumer" + ); + + // SAFETY: The assert above is a bounds check for this write, and we + // avoid assignment here so we do not drop an uninitialized T. + unsafe { + // Write item and increase the initialized length + self.start.0.add(self.initialized_len).write(item); + self.initialized_len += 1; + } + + self + } + + fn complete(self) -> Self::Result { + // NB: We don't explicitly check that the local writes were complete, + // but Collect will assert the total result length in the end. + self + } + + fn full(&self) -> bool { + false + } +} + +/// Pretend to be unindexed for `special_collect_into_vec`, +/// but we should never actually get used that way... +impl<'c, T: Send + 'c> UnindexedConsumer for CollectConsumer<'c, T> { + fn split_off_left(&self) -> Self { + unreachable!("CollectConsumer must be indexed!") + } + fn to_reducer(&self) -> Self::Reducer { + CollectReducer + } +} + +/// CollectReducer combines adjacent chunks; the result must always +/// be contiguous so that it is one combined slice. +pub(super) struct CollectReducer; + +impl<'c, T> Reducer> for CollectReducer { + fn reduce( + self, + mut left: CollectResult<'c, T>, + right: CollectResult<'c, T>, + ) -> CollectResult<'c, T> { + // Merge if the CollectResults are adjacent and in left to right order + // else: drop the right piece now and total length will end up short in the end, + // when the correctness of the collected result is asserted. + unsafe { + let left_end = left.start.0.add(left.initialized_len); + if left_end == right.start.0 { + left.total_len += right.total_len; + left.initialized_len += right.release_ownership(); + } + left + } + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/collect/mod.rs b/anneal/v2/vendor/rayon/src/iter/collect/mod.rs new file mode 100644 index 0000000000..385c1320e8 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/collect/mod.rs @@ -0,0 +1,114 @@ +use super::{IndexedParallelIterator, ParallelIterator}; + +mod consumer; +use self::consumer::CollectConsumer; +use self::consumer::CollectResult; +use super::unzip::unzip_indexed; + +mod test; + +/// Collects the results of the exact iterator into the specified vector. +/// +/// This is called by `IndexedParallelIterator::collect_into_vec`. +pub(super) fn collect_into_vec(pi: I, v: &mut Vec) +where + I: IndexedParallelIterator, + T: Send, +{ + v.truncate(0); // clear any old data + let len = pi.len(); + collect_with_consumer(v, len, |consumer| pi.drive(consumer)); +} + +/// Collects the results of the iterator into the specified vector. +/// +/// Technically, this only works for `IndexedParallelIterator`, but we're faking a +/// bit of specialization here until Rust can do that natively. Callers are +/// using `opt_len` to find the length before calling this, and only exact +/// iterators will return anything but `None` there. +/// +/// Since the type system doesn't understand that contract, we have to allow +/// *any* `ParallelIterator` here, and `CollectConsumer` has to also implement +/// `UnindexedConsumer`. That implementation panics `unreachable!` in case +/// there's a bug where we actually do try to use this unindexed. +pub(super) fn special_extend(pi: I, len: usize, v: &mut Vec) +where + I: ParallelIterator, + T: Send, +{ + collect_with_consumer(v, len, |consumer| pi.drive_unindexed(consumer)); +} + +/// Unzips the results of the exact iterator into the specified vectors. +/// +/// This is called by `IndexedParallelIterator::unzip_into_vecs`. +pub(super) fn unzip_into_vecs(pi: I, left: &mut Vec, right: &mut Vec) +where + I: IndexedParallelIterator, + A: Send, + B: Send, +{ + // clear any old data + left.truncate(0); + right.truncate(0); + + let len = pi.len(); + collect_with_consumer(right, len, |right_consumer| { + let mut right_result = None; + collect_with_consumer(left, len, |left_consumer| { + let (left_r, right_r) = unzip_indexed(pi, left_consumer, right_consumer); + right_result = Some(right_r); + left_r + }); + right_result.unwrap() + }); +} + +/// Create a consumer on the slice of memory we are collecting into. +/// +/// The consumer needs to be used inside the scope function, and the +/// complete collect result passed back. +/// +/// This method will verify the collect result, and panic if the slice +/// was not fully written into. Otherwise, in the successful case, +/// the vector is complete with the collected result. +fn collect_with_consumer(vec: &mut Vec, len: usize, scope_fn: F) +where + T: Send, + F: FnOnce(CollectConsumer<'_, T>) -> CollectResult<'_, T>, +{ + // Reserve space for `len` more elements in the vector, + vec.reserve(len); + + // Create the consumer and run the callback for collection. + let result = scope_fn(CollectConsumer::appender(vec, len)); + + // The `CollectResult` represents a contiguous part of the slice, that has + // been written to. On unwind here, the `CollectResult` will be dropped. If + // some producers on the way did not produce enough elements, partial + // `CollectResult`s may have been dropped without being reduced to the final + // result, and we will see that as the length coming up short. + // + // Here, we assert that added length is fully initialized. This is checked + // by the following assert, which verifies if a complete `CollectResult` + // was produced; if the length is correct, it is necessarily covering the + // target slice. Since we know that the consumer cannot have escaped from + // `drive` (by parametricity, essentially), we know that any stores that + // will happen, have happened. Unless some code is buggy, that means we + // should have seen `len` total writes. + let actual_writes = result.len(); + assert!( + actual_writes == len, + "expected {len} total writes, but got {actual_writes}" + ); + + // Release the result's mutable borrow and "proxy ownership" + // of the elements, before the vector takes it over. + result.release_ownership(); + + let new_len = vec.len() + len; + + unsafe { + vec.set_len(new_len); + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/collect/test.rs b/anneal/v2/vendor/rayon/src/iter/collect/test.rs new file mode 100644 index 0000000000..f72594064b --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/collect/test.rs @@ -0,0 +1,368 @@ +#![cfg(test)] +#![allow(unused_assignments)] + +// These tests are primarily targeting "abusive" producers that will +// try to drive the "collect consumer" incorrectly. These should +// result in panics. + +use super::collect_with_consumer; +use crate::iter::plumbing::*; +use rayon_core::join; + +use std::fmt; +use std::panic; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::thread::Result as ThreadResult; + +/// Promises to produce 2 items, but then produces 3. Does not do any +/// splits at all. +#[test] +#[should_panic(expected = "too many values")] +fn produce_too_many_items() { + let mut v = vec![]; + collect_with_consumer(&mut v, 2, |consumer| { + let mut folder = consumer.into_folder(); + folder = folder.consume(22); + folder = folder.consume(23); + folder = folder.consume(24); + unreachable!("folder does not complete") + }); +} + +/// Produces fewer items than promised. Does not do any +/// splits at all. +#[test] +#[should_panic(expected = "expected 5 total writes, but got 2")] +fn produce_fewer_items() { + let mut v = vec![]; + collect_with_consumer(&mut v, 5, |consumer| { + let mut folder = consumer.into_folder(); + folder = folder.consume(22); + folder = folder.consume(23); + folder.complete() + }); +} + +// Complete is not called by the consumer. Hence,the collection vector is not fully initialized. +#[test] +#[should_panic(expected = "expected 4 total writes, but got 2")] +fn left_produces_items_with_no_complete() { + let mut v = vec![]; + collect_with_consumer(&mut v, 4, |consumer| { + let (left_consumer, right_consumer, _) = consumer.split_at(2); + let mut left_folder = left_consumer.into_folder(); + let mut right_folder = right_consumer.into_folder(); + left_folder = left_folder.consume(0).consume(1); + right_folder = right_folder.consume(2).consume(3); + right_folder.complete() + }); +} + +// Complete is not called by the right consumer. Hence,the +// collection vector is not fully initialized. +#[test] +#[should_panic(expected = "expected 4 total writes, but got 2")] +fn right_produces_items_with_no_complete() { + let mut v = vec![]; + collect_with_consumer(&mut v, 4, |consumer| { + let (left_consumer, right_consumer, _) = consumer.split_at(2); + let mut left_folder = left_consumer.into_folder(); + let mut right_folder = right_consumer.into_folder(); + left_folder = left_folder.consume(0).consume(1); + right_folder = right_folder.consume(2).consume(3); + left_folder.complete() + }); +} + +// Complete is not called by the consumer. Hence,the collection vector is not fully initialized. +#[test] +#[cfg_attr(not(panic = "unwind"), ignore)] +fn produces_items_with_no_complete() { + let counter = DropCounter::default(); + let mut v = vec![]; + let panic_result = panic::catch_unwind(panic::AssertUnwindSafe(|| { + collect_with_consumer(&mut v, 2, |consumer| { + let mut folder = consumer.into_folder(); + folder = folder.consume(counter.element()); + folder = folder.consume(counter.element()); + panic!("folder does not complete"); + }); + })); + assert!(v.is_empty()); + assert_is_panic_with_message(&panic_result, "folder does not complete"); + counter.assert_drop_count(); +} + +// The left consumer produces too many items while the right +// consumer produces correct number. +#[test] +#[should_panic(expected = "too many values")] +fn left_produces_too_many_items() { + let mut v = vec![]; + collect_with_consumer(&mut v, 4, |consumer| { + let (left_consumer, right_consumer, _) = consumer.split_at(2); + let mut left_folder = left_consumer.into_folder(); + let mut right_folder = right_consumer.into_folder(); + left_folder = left_folder.consume(0).consume(1).consume(2); + right_folder = right_folder.consume(2).consume(3); + let _ = right_folder.complete(); + unreachable!("folder does not complete"); + }); +} + +// The right consumer produces too many items while the left +// consumer produces correct number. +#[test] +#[should_panic(expected = "too many values")] +fn right_produces_too_many_items() { + let mut v = vec![]; + collect_with_consumer(&mut v, 4, |consumer| { + let (left_consumer, right_consumer, _) = consumer.split_at(2); + let mut left_folder = left_consumer.into_folder(); + let mut right_folder = right_consumer.into_folder(); + left_folder = left_folder.consume(0).consume(1); + right_folder = right_folder.consume(2).consume(3).consume(4); + let _ = left_folder.complete(); + unreachable!("folder does not complete"); + }); +} + +// The left consumer produces fewer items while the right +// consumer produces correct number. +#[test] +#[should_panic(expected = "expected 4 total writes, but got 1")] +fn left_produces_fewer_items() { + let mut v = vec![]; + collect_with_consumer(&mut v, 4, |consumer| { + let reducer = consumer.to_reducer(); + let (left_consumer, right_consumer, _) = consumer.split_at(2); + let mut left_folder = left_consumer.into_folder(); + let mut right_folder = right_consumer.into_folder(); + left_folder = left_folder.consume(0); + right_folder = right_folder.consume(2).consume(3); + let left_result = left_folder.complete(); + let right_result = right_folder.complete(); + reducer.reduce(left_result, right_result) + }); +} + +// The left and right consumer produce the correct number but +// only left result is returned +#[test] +#[should_panic(expected = "expected 4 total writes, but got 2")] +fn only_left_result() { + let mut v = vec![]; + collect_with_consumer(&mut v, 4, |consumer| { + let (left_consumer, right_consumer, _) = consumer.split_at(2); + let mut left_folder = left_consumer.into_folder(); + let mut right_folder = right_consumer.into_folder(); + left_folder = left_folder.consume(0).consume(1); + right_folder = right_folder.consume(2).consume(3); + let left_result = left_folder.complete(); + let _ = right_folder.complete(); + left_result + }); +} + +// The left and right consumer produce the correct number but +// only right result is returned +#[test] +#[should_panic(expected = "expected 4 total writes, but got 2")] +fn only_right_result() { + let mut v = vec![]; + collect_with_consumer(&mut v, 4, |consumer| { + let (left_consumer, right_consumer, _) = consumer.split_at(2); + let mut left_folder = left_consumer.into_folder(); + let mut right_folder = right_consumer.into_folder(); + left_folder = left_folder.consume(0).consume(1); + right_folder = right_folder.consume(2).consume(3); + let _ = left_folder.complete(); + right_folder.complete() + }); +} + +// The left and right consumer produce the correct number but reduce +// in the wrong order. +#[test] +#[should_panic(expected = "expected 4 total writes, but got 2")] +fn reducer_does_not_preserve_order() { + let mut v = vec![]; + collect_with_consumer(&mut v, 4, |consumer| { + let reducer = consumer.to_reducer(); + let (left_consumer, right_consumer, _) = consumer.split_at(2); + let mut left_folder = left_consumer.into_folder(); + let mut right_folder = right_consumer.into_folder(); + left_folder = left_folder.consume(0).consume(1); + right_folder = right_folder.consume(2).consume(3); + let left_result = left_folder.complete(); + let right_result = right_folder.complete(); + reducer.reduce(right_result, left_result) + }); +} + +// The right consumer produces fewer items while the left +// consumer produces correct number. +#[test] +#[should_panic(expected = "expected 4 total writes, but got 3")] +fn right_produces_fewer_items() { + let mut v = vec![]; + collect_with_consumer(&mut v, 4, |consumer| { + let reducer = consumer.to_reducer(); + let (left_consumer, right_consumer, _) = consumer.split_at(2); + let mut left_folder = left_consumer.into_folder(); + let mut right_folder = right_consumer.into_folder(); + left_folder = left_folder.consume(0).consume(1); + right_folder = right_folder.consume(2); + let left_result = left_folder.complete(); + let right_result = right_folder.complete(); + reducer.reduce(left_result, right_result) + }); +} + +// The left consumer panics and the right stops short, like `panic_fuse()`. +// We should get the left panic without finishing `collect_with_consumer`. +#[test] +#[should_panic(expected = "left consumer panic")] +fn left_panics() { + let mut v = vec![]; + collect_with_consumer(&mut v, 4, |consumer| { + let reducer = consumer.to_reducer(); + let (left_consumer, right_consumer, _) = consumer.split_at(2); + let (left_result, right_result) = join( + || { + let mut left_folder = left_consumer.into_folder(); + left_folder = left_folder.consume(0); + panic!("left consumer panic"); + }, + || { + let mut right_folder = right_consumer.into_folder(); + right_folder = right_folder.consume(2); + right_folder.complete() // early return + }, + ); + reducer.reduce(left_result, right_result) + }); + unreachable!(); +} + +// The right consumer panics and the left stops short, like `panic_fuse()`. +// We should get the right panic without finishing `collect_with_consumer`. +#[test] +#[should_panic(expected = "right consumer panic")] +fn right_panics() { + let mut v = vec![]; + collect_with_consumer(&mut v, 4, |consumer| { + let reducer = consumer.to_reducer(); + let (left_consumer, right_consumer, _) = consumer.split_at(2); + let (left_result, right_result) = join( + || { + let mut left_folder = left_consumer.into_folder(); + left_folder = left_folder.consume(0); + left_folder.complete() // early return + }, + || { + let mut right_folder = right_consumer.into_folder(); + right_folder = right_folder.consume(2); + panic!("right consumer panic"); + }, + ); + reducer.reduce(left_result, right_result) + }); + unreachable!(); +} + +// The left consumer produces fewer items while the right +// consumer produces correct number; check that created elements are dropped +#[test] +#[cfg_attr(not(panic = "unwind"), ignore)] +fn left_produces_fewer_items_drops() { + let counter = DropCounter::default(); + let mut v = vec![]; + let panic_result = panic::catch_unwind(panic::AssertUnwindSafe(|| { + collect_with_consumer(&mut v, 4, |consumer| { + let reducer = consumer.to_reducer(); + let (left_consumer, right_consumer, _) = consumer.split_at(2); + let mut left_folder = left_consumer.into_folder(); + let mut right_folder = right_consumer.into_folder(); + left_folder = left_folder.consume(counter.element()); + right_folder = right_folder + .consume(counter.element()) + .consume(counter.element()); + let left_result = left_folder.complete(); + let right_result = right_folder.complete(); + reducer.reduce(left_result, right_result) + }); + })); + assert!(v.is_empty()); + assert_is_panic_with_message(&panic_result, "expected 4 total writes, but got 1"); + counter.assert_drop_count(); +} + +/// This counter can create elements, and then count and verify +/// the number of which have actually been dropped again. +#[derive(Default)] +struct DropCounter { + created: AtomicUsize, + dropped: AtomicUsize, +} + +struct Element<'a>(&'a AtomicUsize); + +impl DropCounter { + fn created(&self) -> usize { + self.created.load(Ordering::SeqCst) + } + + fn dropped(&self) -> usize { + self.dropped.load(Ordering::SeqCst) + } + + fn element(&self) -> Element<'_> { + self.created.fetch_add(1, Ordering::SeqCst); + Element(&self.dropped) + } + + fn assert_drop_count(&self) { + assert_eq!( + self.created(), + self.dropped(), + "Expected {} dropped elements, but found {}", + self.created(), + self.dropped() + ); + } +} + +impl<'a> Drop for Element<'a> { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } +} + +/// Assert that the result from catch_unwind is a panic that contains expected message +fn assert_is_panic_with_message(result: &ThreadResult, expected: &str) +where + T: fmt::Debug, +{ + match result { + Ok(value) => { + panic!("assertion failure: Expected panic, got successful {value:?}"); + } + Err(error) => { + let message_str = error.downcast_ref::<&'static str>().cloned(); + let message_string = error.downcast_ref::().map(String::as_str); + if let Some(message) = message_str.or(message_string) { + if !message.contains(expected) { + panic!( + "assertion failure: Expected {expected:?}, but found panic with {message:?}" + ); + } + // assertion passes + } else { + panic!( + "assertion failure: Expected {expected:?}, but found panic with unknown value" + ); + } + } + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/copied.rs b/anneal/v2/vendor/rayon/src/iter/copied.rs new file mode 100644 index 0000000000..41a6547a3f --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/copied.rs @@ -0,0 +1,219 @@ +use super::plumbing::*; +use super::*; + +use std::iter; + +/// `Copied` is an iterator that copies the elements of an underlying iterator. +/// +/// This struct is created by the [`copied()`] method on [`ParallelIterator`] +/// +/// [`copied()`]: ParallelIterator::copied() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Debug, Clone)] +pub struct Copied { + base: I, +} + +impl Copied { + /// Creates a new `Copied` iterator. + pub(super) fn new(base: I) -> Self { + Copied { base } + } +} + +impl<'a, T, I> ParallelIterator for Copied +where + I: ParallelIterator, + T: 'a + Copy + Send + Sync, +{ + type Item = T; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let consumer1 = CopiedConsumer::new(consumer); + self.base.drive_unindexed(consumer1) + } + + fn opt_len(&self) -> Option { + self.base.opt_len() + } +} + +impl<'a, T, I> IndexedParallelIterator for Copied +where + I: IndexedParallelIterator, + T: 'a + Copy + Send + Sync, +{ + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + let consumer1 = CopiedConsumer::new(consumer); + self.base.drive(consumer1) + } + + fn len(&self) -> usize { + self.base.len() + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + return self.base.with_producer(Callback { callback }); + + struct Callback { + callback: CB, + } + + impl<'a, T, CB> ProducerCallback<&'a T> for Callback + where + CB: ProducerCallback, + T: 'a + Copy + Send, + { + type Output = CB::Output; + + fn callback

(self, base: P) -> CB::Output + where + P: Producer, + { + let producer = CopiedProducer { base }; + self.callback.callback(producer) + } + } + } +} + +// //////////////////////////////////////////////////////////////////////// + +struct CopiedProducer

{ + base: P, +} + +impl<'a, T, P> Producer for CopiedProducer

+where + P: Producer, + T: 'a + Copy, +{ + type Item = T; + type IntoIter = iter::Copied; + + fn into_iter(self) -> Self::IntoIter { + self.base.into_iter().copied() + } + + fn min_len(&self) -> usize { + self.base.min_len() + } + + fn max_len(&self) -> usize { + self.base.max_len() + } + + fn split_at(self, index: usize) -> (Self, Self) { + let (left, right) = self.base.split_at(index); + ( + CopiedProducer { base: left }, + CopiedProducer { base: right }, + ) + } + + fn fold_with(self, folder: F) -> F + where + F: Folder, + { + self.base.fold_with(CopiedFolder { base: folder }).base + } +} + +// //////////////////////////////////////////////////////////////////////// +// Consumer implementation + +struct CopiedConsumer { + base: C, +} + +impl CopiedConsumer { + fn new(base: C) -> Self { + CopiedConsumer { base } + } +} + +impl<'a, T, C> Consumer<&'a T> for CopiedConsumer +where + C: Consumer, + T: 'a + Copy, +{ + type Folder = CopiedFolder; + type Reducer = C::Reducer; + type Result = C::Result; + + fn split_at(self, index: usize) -> (Self, Self, Self::Reducer) { + let (left, right, reducer) = self.base.split_at(index); + ( + CopiedConsumer::new(left), + CopiedConsumer::new(right), + reducer, + ) + } + + fn into_folder(self) -> Self::Folder { + CopiedFolder { + base: self.base.into_folder(), + } + } + + fn full(&self) -> bool { + self.base.full() + } +} + +impl<'a, T, C> UnindexedConsumer<&'a T> for CopiedConsumer +where + C: UnindexedConsumer, + T: 'a + Copy, +{ + fn split_off_left(&self) -> Self { + CopiedConsumer::new(self.base.split_off_left()) + } + + fn to_reducer(&self) -> Self::Reducer { + self.base.to_reducer() + } +} + +struct CopiedFolder { + base: F, +} + +impl<'a, T, F> Folder<&'a T> for CopiedFolder +where + F: Folder, + T: 'a + Copy, +{ + type Result = F::Result; + + fn consume(self, &item: &'a T) -> Self { + CopiedFolder { + base: self.base.consume(item), + } + } + + fn consume_iter(mut self, iter: I) -> Self + where + I: IntoIterator, + { + self.base = self.base.consume_iter(iter.into_iter().copied()); + self + } + + fn complete(self) -> F::Result { + self.base.complete() + } + + fn full(&self) -> bool { + self.base.full() + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/empty.rs b/anneal/v2/vendor/rayon/src/iter/empty.rs new file mode 100644 index 0000000000..ed05342733 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/empty.rs @@ -0,0 +1,108 @@ +use crate::iter::plumbing::*; +use crate::iter::*; + +use std::fmt; +use std::marker::PhantomData; + +/// Creates a parallel iterator that produces nothing. +/// +/// This admits no parallelism on its own, but it could be used for code that +/// deals with generic parallel iterators. +/// +/// # Examples +/// +/// ``` +/// use rayon::prelude::*; +/// use rayon::iter::empty; +/// +/// let pi = (0..1234).into_par_iter() +/// .chain(empty()) +/// .chain(1234..10_000); +/// +/// assert_eq!(pi.count(), 10_000); +/// ``` +pub fn empty() -> Empty { + Empty { + marker: PhantomData, + } +} + +/// Iterator adaptor for [the `empty()` function]. +/// +/// [the `empty()` function]: empty() +pub struct Empty { + marker: PhantomData, +} + +impl Clone for Empty { + fn clone(&self) -> Self { + Empty { + marker: PhantomData, + } + } +} + +impl fmt::Debug for Empty { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad("Empty") + } +} + +impl ParallelIterator for Empty { + type Item = T; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + self.drive(consumer) + } + + fn opt_len(&self) -> Option { + Some(0) + } +} + +impl IndexedParallelIterator for Empty { + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + consumer.into_folder().complete() + } + + fn len(&self) -> usize { + 0 + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + callback.callback(EmptyProducer(PhantomData)) + } +} + +/// Private empty producer +struct EmptyProducer(PhantomData); + +impl Producer for EmptyProducer { + type Item = T; + type IntoIter = std::iter::Empty; + + fn into_iter(self) -> Self::IntoIter { + std::iter::empty() + } + + fn split_at(self, index: usize) -> (Self, Self) { + debug_assert_eq!(index, 0); + (self, EmptyProducer(PhantomData)) + } + + fn fold_with(self, folder: F) -> F + where + F: Folder, + { + folder + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/enumerate.rs b/anneal/v2/vendor/rayon/src/iter/enumerate.rs new file mode 100644 index 0000000000..65f0c12bba --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/enumerate.rs @@ -0,0 +1,128 @@ +use super::plumbing::*; +use super::*; +use std::iter; +use std::ops::Range; + +/// `Enumerate` is an iterator that returns the current count along with the element. +/// This struct is created by the [`enumerate()`] method on [`IndexedParallelIterator`] +/// +/// [`enumerate()`]: IndexedParallelIterator::enumerate() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Debug, Clone)] +pub struct Enumerate { + base: I, +} + +impl Enumerate { + /// Creates a new `Enumerate` iterator. + pub(super) fn new(base: I) -> Self { + Enumerate { base } + } +} + +impl ParallelIterator for Enumerate +where + I: IndexedParallelIterator, +{ + type Item = (usize, I::Item); + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + bridge(self, consumer) + } + + fn opt_len(&self) -> Option { + Some(self.len()) + } +} + +impl IndexedParallelIterator for Enumerate +where + I: IndexedParallelIterator, +{ + fn drive>(self, consumer: C) -> C::Result { + bridge(self, consumer) + } + + fn len(&self) -> usize { + self.base.len() + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + return self.base.with_producer(Callback { callback }); + + struct Callback { + callback: CB, + } + + impl ProducerCallback for Callback + where + CB: ProducerCallback<(usize, I)>, + { + type Output = CB::Output; + fn callback

(self, base: P) -> CB::Output + where + P: Producer, + { + let producer = EnumerateProducer { base, offset: 0 }; + self.callback.callback(producer) + } + } + } +} + +// //////////////////////////////////////////////////////////////////////// +// Producer implementation + +struct EnumerateProducer

{ + base: P, + offset: usize, +} + +impl

Producer for EnumerateProducer

+where + P: Producer, +{ + type Item = (usize, P::Item); + type IntoIter = iter::Zip, P::IntoIter>; + + fn into_iter(self) -> Self::IntoIter { + // Enumerate only works for IndexedParallelIterators. Since those + // have a max length of usize::MAX, their max index is + // usize::MAX - 1, so the range 0..usize::MAX includes all + // possible indices. + // + // However, we should to use a precise end to the range, otherwise + // reversing the iterator may have to walk back a long ways before + // `Zip::next_back` can produce anything. + let base = self.base.into_iter(); + let end = self.offset + base.len(); + (self.offset..end).zip(base) + } + + fn min_len(&self) -> usize { + self.base.min_len() + } + fn max_len(&self) -> usize { + self.base.max_len() + } + + fn split_at(self, index: usize) -> (Self, Self) { + let (left, right) = self.base.split_at(index); + ( + EnumerateProducer { + base: left, + offset: self.offset, + }, + EnumerateProducer { + base: right, + offset: self.offset + index, + }, + ) + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/extend.rs b/anneal/v2/vendor/rayon/src/iter/extend.rs new file mode 100644 index 0000000000..3d19d9d802 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/extend.rs @@ -0,0 +1,619 @@ +use super::noop::NoopConsumer; +use super::plumbing::{Consumer, Folder, Reducer, UnindexedConsumer}; +use super::{IntoParallelIterator, ParallelExtend, ParallelIterator}; + +use either::Either; +use std::borrow::Cow; +use std::collections::LinkedList; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::collections::{BinaryHeap, VecDeque}; +use std::ffi::{OsStr, OsString}; +use std::hash::{BuildHasher, Hash}; + +/// Performs a generic `par_extend` by collecting to a `LinkedList>` in +/// parallel, then extending the collection sequentially. +macro_rules! extend { + ($self:ident, $par_iter:ident) => { + extend!($self <- fast_collect($par_iter)) + }; + ($self:ident <- $vecs:expr) => { + match $vecs { + Either::Left(vec) => $self.extend(vec), + Either::Right(list) => { + for vec in list { + $self.extend(vec); + } + } + } + }; +} +macro_rules! extend_reserved { + ($self:ident, $par_iter:ident, $len:ident) => { + let vecs = fast_collect($par_iter); + $self.reserve($len(&vecs)); + extend!($self <- vecs) + }; + ($self:ident, $par_iter:ident) => { + extend_reserved!($self, $par_iter, len) + }; +} + +/// Computes the total length of a `fast_collect` result. +fn len(vecs: &Either, LinkedList>>) -> usize { + match vecs { + Either::Left(vec) => vec.len(), + Either::Right(list) => list.iter().map(Vec::len).sum(), + } +} + +/// Computes the total string length of a `fast_collect` result. +fn string_len>(vecs: &Either, LinkedList>>) -> usize { + let strs = match vecs { + Either::Left(vec) => Either::Left(vec.iter()), + Either::Right(list) => Either::Right(list.iter().flatten()), + }; + strs.map(AsRef::as_ref).map(str::len).sum() +} + +/// Computes the total OS-string length of a `fast_collect` result. +fn osstring_len>(vecs: &Either, LinkedList>>) -> usize { + let osstrs = match vecs { + Either::Left(vec) => Either::Left(vec.iter()), + Either::Right(list) => Either::Right(list.iter().flatten()), + }; + osstrs.map(AsRef::as_ref).map(OsStr::len).sum() +} + +pub(super) fn fast_collect(pi: I) -> Either, LinkedList>> +where + I: IntoParallelIterator, + T: Send, +{ + let par_iter = pi.into_par_iter(); + match par_iter.opt_len() { + Some(len) => { + // Pseudo-specialization. See impl of ParallelExtend for Vec for more details. + let mut vec = Vec::new(); + super::collect::special_extend(par_iter, len, &mut vec); + Either::Left(vec) + } + None => Either::Right(par_iter.drive_unindexed(ListVecConsumer)), + } +} + +struct ListVecConsumer; + +struct ListVecFolder { + vec: Vec, +} + +impl Consumer for ListVecConsumer { + type Folder = ListVecFolder; + type Reducer = ListReducer; + type Result = LinkedList>; + + fn split_at(self, _index: usize) -> (Self, Self, Self::Reducer) { + (Self, Self, ListReducer) + } + + fn into_folder(self) -> Self::Folder { + ListVecFolder { vec: Vec::new() } + } + + fn full(&self) -> bool { + false + } +} + +impl UnindexedConsumer for ListVecConsumer { + fn split_off_left(&self) -> Self { + Self + } + + fn to_reducer(&self) -> Self::Reducer { + ListReducer + } +} + +impl Folder for ListVecFolder { + type Result = LinkedList>; + + fn consume(mut self, item: T) -> Self { + self.vec.push(item); + self + } + + fn consume_iter(mut self, iter: I) -> Self + where + I: IntoIterator, + { + self.vec.extend(iter); + self + } + + fn complete(self) -> Self::Result { + let mut list = LinkedList::new(); + if !self.vec.is_empty() { + list.push_back(self.vec); + } + list + } + + fn full(&self) -> bool { + false + } +} + +/// Extends a binary heap with items from a parallel iterator. +impl ParallelExtend for BinaryHeap +where + T: Ord + Send, +{ + fn par_extend(&mut self, par_iter: I) + where + I: IntoParallelIterator, + { + extend_reserved!(self, par_iter); + } +} + +/// Extends a binary heap with copied items from a parallel iterator. +impl<'a, T> ParallelExtend<&'a T> for BinaryHeap +where + T: 'a + Copy + Ord + Send + Sync, +{ + fn par_extend(&mut self, par_iter: I) + where + I: IntoParallelIterator, + { + extend_reserved!(self, par_iter); + } +} + +/// Extends a B-tree map with items from a parallel iterator. +impl ParallelExtend<(K, V)> for BTreeMap +where + K: Ord + Send, + V: Send, +{ + fn par_extend(&mut self, par_iter: I) + where + I: IntoParallelIterator, + { + extend!(self, par_iter); + } +} + +/// Extends a B-tree map with copied items from a parallel iterator. +impl<'a, K: 'a, V: 'a> ParallelExtend<(&'a K, &'a V)> for BTreeMap +where + K: Copy + Ord + Send + Sync, + V: Copy + Send + Sync, +{ + fn par_extend(&mut self, par_iter: I) + where + I: IntoParallelIterator, + { + extend!(self, par_iter); + } +} + +/// Extends a B-tree set with items from a parallel iterator. +impl ParallelExtend for BTreeSet +where + T: Ord + Send, +{ + fn par_extend(&mut self, par_iter: I) + where + I: IntoParallelIterator, + { + extend!(self, par_iter); + } +} + +/// Extends a B-tree set with copied items from a parallel iterator. +impl<'a, T> ParallelExtend<&'a T> for BTreeSet +where + T: 'a + Copy + Ord + Send + Sync, +{ + fn par_extend(&mut self, par_iter: I) + where + I: IntoParallelIterator, + { + extend!(self, par_iter); + } +} + +/// Extends a hash map with items from a parallel iterator. +impl ParallelExtend<(K, V)> for HashMap +where + K: Eq + Hash + Send, + V: Send, + S: BuildHasher + Send, +{ + fn par_extend(&mut self, par_iter: I) + where + I: IntoParallelIterator, + { + // See the map_collect benchmarks in rayon-demo for different strategies. + extend_reserved!(self, par_iter); + } +} + +/// Extends a hash map with copied items from a parallel iterator. +impl<'a, K: 'a, V: 'a, S> ParallelExtend<(&'a K, &'a V)> for HashMap +where + K: Copy + Eq + Hash + Send + Sync, + V: Copy + Send + Sync, + S: BuildHasher + Send, +{ + fn par_extend(&mut self, par_iter: I) + where + I: IntoParallelIterator, + { + extend_reserved!(self, par_iter); + } +} + +/// Extends a hash set with items from a parallel iterator. +impl ParallelExtend for HashSet +where + T: Eq + Hash + Send, + S: BuildHasher + Send, +{ + fn par_extend(&mut self, par_iter: I) + where + I: IntoParallelIterator, + { + extend_reserved!(self, par_iter); + } +} + +/// Extends a hash set with copied items from a parallel iterator. +impl<'a, T, S> ParallelExtend<&'a T> for HashSet +where + T: 'a + Copy + Eq + Hash + Send + Sync, + S: BuildHasher + Send, +{ + fn par_extend(&mut self, par_iter: I) + where + I: IntoParallelIterator, + { + extend_reserved!(self, par_iter); + } +} + +/// Extends a linked list with items from a parallel iterator. +impl ParallelExtend for LinkedList +where + T: Send, +{ + fn par_extend(&mut self, par_iter: I) + where + I: IntoParallelIterator, + { + let mut list = par_iter.into_par_iter().drive_unindexed(ListConsumer); + self.append(&mut list); + } +} + +/// Extends a linked list with copied items from a parallel iterator. +impl<'a, T> ParallelExtend<&'a T> for LinkedList +where + T: 'a + Copy + Send + Sync, +{ + fn par_extend(&mut self, par_iter: I) + where + I: IntoParallelIterator, + { + self.par_extend(par_iter.into_par_iter().copied()) + } +} + +struct ListConsumer; + +struct ListFolder { + list: LinkedList, +} + +struct ListReducer; + +impl Consumer for ListConsumer { + type Folder = ListFolder; + type Reducer = ListReducer; + type Result = LinkedList; + + fn split_at(self, _index: usize) -> (Self, Self, Self::Reducer) { + (Self, Self, ListReducer) + } + + fn into_folder(self) -> Self::Folder { + ListFolder { + list: LinkedList::new(), + } + } + + fn full(&self) -> bool { + false + } +} + +impl UnindexedConsumer for ListConsumer { + fn split_off_left(&self) -> Self { + Self + } + + fn to_reducer(&self) -> Self::Reducer { + ListReducer + } +} + +impl Folder for ListFolder { + type Result = LinkedList; + + fn consume(mut self, item: T) -> Self { + self.list.push_back(item); + self + } + + fn consume_iter(mut self, iter: I) -> Self + where + I: IntoIterator, + { + self.list.extend(iter); + self + } + + fn complete(self) -> Self::Result { + self.list + } + + fn full(&self) -> bool { + false + } +} + +impl Reducer> for ListReducer { + fn reduce(self, mut left: LinkedList, mut right: LinkedList) -> LinkedList { + left.append(&mut right); + left + } +} + +/// Extends an OS-string with string slices from a parallel iterator. +impl<'a> ParallelExtend<&'a OsStr> for OsString { + fn par_extend(&mut self, par_iter: I) + where + I: IntoParallelIterator, + { + extend_reserved!(self, par_iter, osstring_len); + } +} + +/// Extends an OS-string with strings from a parallel iterator. +impl ParallelExtend for OsString { + fn par_extend(&mut self, par_iter: I) + where + I: IntoParallelIterator, + { + extend_reserved!(self, par_iter, osstring_len); + } +} + +/// Extends an OS-string with string slices from a parallel iterator. +impl<'a> ParallelExtend> for OsString { + fn par_extend(&mut self, par_iter: I) + where + I: IntoParallelIterator>, + { + extend_reserved!(self, par_iter, osstring_len); + } +} + +/// Extends a string with characters from a parallel iterator. +impl ParallelExtend for String { + fn par_extend(&mut self, par_iter: I) + where + I: IntoParallelIterator, + { + // This is like `extend`, but `Vec` is less efficient to deal + // with than `String`, so instead collect to `LinkedList`. + let list = par_iter.into_par_iter().drive_unindexed(ListStringConsumer); + self.reserve(list.iter().map(String::len).sum()); + self.extend(list); + } +} + +/// Extends a string with copied characters from a parallel iterator. +impl<'a> ParallelExtend<&'a char> for String { + fn par_extend(&mut self, par_iter: I) + where + I: IntoParallelIterator, + { + self.par_extend(par_iter.into_par_iter().copied()) + } +} + +struct ListStringConsumer; + +struct ListStringFolder { + string: String, +} + +impl Consumer for ListStringConsumer { + type Folder = ListStringFolder; + type Reducer = ListReducer; + type Result = LinkedList; + + fn split_at(self, _index: usize) -> (Self, Self, Self::Reducer) { + (Self, Self, ListReducer) + } + + fn into_folder(self) -> Self::Folder { + ListStringFolder { + string: String::new(), + } + } + + fn full(&self) -> bool { + false + } +} + +impl UnindexedConsumer for ListStringConsumer { + fn split_off_left(&self) -> Self { + Self + } + + fn to_reducer(&self) -> Self::Reducer { + ListReducer + } +} + +impl Folder for ListStringFolder { + type Result = LinkedList; + + fn consume(mut self, item: char) -> Self { + self.string.push(item); + self + } + + fn consume_iter(mut self, iter: I) -> Self + where + I: IntoIterator, + { + self.string.extend(iter); + self + } + + fn complete(self) -> Self::Result { + let mut list = LinkedList::new(); + if !self.string.is_empty() { + list.push_back(self.string); + } + list + } + + fn full(&self) -> bool { + false + } +} + +/// Extends a string with string slices from a parallel iterator. +impl<'a> ParallelExtend<&'a str> for String { + fn par_extend(&mut self, par_iter: I) + where + I: IntoParallelIterator, + { + extend_reserved!(self, par_iter, string_len); + } +} + +/// Extends a string with strings from a parallel iterator. +impl ParallelExtend for String { + fn par_extend(&mut self, par_iter: I) + where + I: IntoParallelIterator, + { + extend_reserved!(self, par_iter, string_len); + } +} + +/// Extends a string with boxed strings from a parallel iterator. +impl ParallelExtend> for String { + fn par_extend(&mut self, par_iter: I) + where + I: IntoParallelIterator>, + { + extend_reserved!(self, par_iter, string_len); + } +} + +/// Extends a string with string slices from a parallel iterator. +impl<'a> ParallelExtend> for String { + fn par_extend(&mut self, par_iter: I) + where + I: IntoParallelIterator>, + { + extend_reserved!(self, par_iter, string_len); + } +} + +/// Extends a deque with items from a parallel iterator. +impl ParallelExtend for VecDeque +where + T: Send, +{ + fn par_extend(&mut self, par_iter: I) + where + I: IntoParallelIterator, + { + extend_reserved!(self, par_iter); + } +} + +/// Extends a deque with copied items from a parallel iterator. +impl<'a, T> ParallelExtend<&'a T> for VecDeque +where + T: 'a + Copy + Send + Sync, +{ + fn par_extend(&mut self, par_iter: I) + where + I: IntoParallelIterator, + { + extend_reserved!(self, par_iter); + } +} + +/// Extends a vector with items from a parallel iterator. +impl ParallelExtend for Vec +where + T: Send, +{ + fn par_extend(&mut self, par_iter: I) + where + I: IntoParallelIterator, + { + // See the vec_collect benchmarks in rayon-demo for different strategies. + let par_iter = par_iter.into_par_iter(); + match par_iter.opt_len() { + Some(len) => { + // When Rust gets specialization, we can get here for indexed iterators + // without relying on `opt_len`. Until then, `special_extend()` fakes + // an unindexed mode on the promise that `opt_len()` is accurate. + super::collect::special_extend(par_iter, len, self); + } + None => { + // This works like `extend`, but `Vec::append` is more efficient. + let list = par_iter.drive_unindexed(ListVecConsumer); + self.reserve(list.iter().map(Vec::len).sum()); + for mut other in list { + self.append(&mut other); + } + } + } + } +} + +/// Extends a vector with copied items from a parallel iterator. +impl<'a, T> ParallelExtend<&'a T> for Vec +where + T: 'a + Copy + Send + Sync, +{ + fn par_extend(&mut self, par_iter: I) + where + I: IntoParallelIterator, + { + self.par_extend(par_iter.into_par_iter().copied()) + } +} + +/// Collapses all unit items from a parallel iterator into one. +impl ParallelExtend<()> for () { + fn par_extend(&mut self, par_iter: I) + where + I: IntoParallelIterator, + { + par_iter.into_par_iter().drive_unindexed(NoopConsumer) + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/filter.rs b/anneal/v2/vendor/rayon/src/iter/filter.rs new file mode 100644 index 0000000000..f97c8708d1 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/filter.rs @@ -0,0 +1,137 @@ +use super::plumbing::*; +use super::*; + +use std::fmt::{self, Debug}; + +/// `Filter` takes a predicate `filter_op` and filters out elements that match. +/// This struct is created by the [`filter()`] method on [`ParallelIterator`] +/// +/// [`filter()`]: ParallelIterator::filter() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Clone)] +pub struct Filter { + base: I, + filter_op: P, +} + +impl Debug for Filter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Filter").field("base", &self.base).finish() + } +} + +impl Filter { + /// Creates a new `Filter` iterator. + pub(super) fn new(base: I, filter_op: P) -> Self { + Filter { base, filter_op } + } +} + +impl ParallelIterator for Filter +where + I: ParallelIterator, + P: Fn(&I::Item) -> bool + Sync + Send, +{ + type Item = I::Item; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let consumer1 = FilterConsumer::new(consumer, &self.filter_op); + self.base.drive_unindexed(consumer1) + } +} + +// //////////////////////////////////////////////////////////////////////// +// Consumer implementation + +struct FilterConsumer<'p, C, P> { + base: C, + filter_op: &'p P, +} + +impl<'p, C, P> FilterConsumer<'p, C, P> { + fn new(base: C, filter_op: &'p P) -> Self { + FilterConsumer { base, filter_op } + } +} + +impl<'p, T, C, P: 'p> Consumer for FilterConsumer<'p, C, P> +where + C: Consumer, + P: Fn(&T) -> bool + Sync, +{ + type Folder = FilterFolder<'p, C::Folder, P>; + type Reducer = C::Reducer; + type Result = C::Result; + + fn split_at(self, index: usize) -> (Self, Self, C::Reducer) { + let (left, right, reducer) = self.base.split_at(index); + ( + FilterConsumer::new(left, self.filter_op), + FilterConsumer::new(right, self.filter_op), + reducer, + ) + } + + fn into_folder(self) -> Self::Folder { + FilterFolder { + base: self.base.into_folder(), + filter_op: self.filter_op, + } + } + + fn full(&self) -> bool { + self.base.full() + } +} + +impl<'p, T, C, P: 'p> UnindexedConsumer for FilterConsumer<'p, C, P> +where + C: UnindexedConsumer, + P: Fn(&T) -> bool + Sync, +{ + fn split_off_left(&self) -> Self { + FilterConsumer::new(self.base.split_off_left(), self.filter_op) + } + + fn to_reducer(&self) -> Self::Reducer { + self.base.to_reducer() + } +} + +struct FilterFolder<'p, C, P> { + base: C, + filter_op: &'p P, +} + +impl<'p, C, P, T> Folder for FilterFolder<'p, C, P> +where + C: Folder, + P: Fn(&T) -> bool + 'p, +{ + type Result = C::Result; + + fn consume(self, item: T) -> Self { + let filter_op = self.filter_op; + if filter_op(&item) { + let base = self.base.consume(item); + FilterFolder { base, filter_op } + } else { + self + } + } + + // This cannot easily specialize `consume_iter` to be better than + // the default, because that requires checking `self.base.full()` + // during a call to `self.base.consume_iter()`. (#632) + + fn complete(self) -> Self::Result { + self.base.complete() + } + + fn full(&self) -> bool { + self.base.full() + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/filter_map.rs b/anneal/v2/vendor/rayon/src/iter/filter_map.rs new file mode 100644 index 0000000000..4642c9f141 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/filter_map.rs @@ -0,0 +1,141 @@ +use super::plumbing::*; +use super::*; + +use std::fmt::{self, Debug}; + +/// `FilterMap` creates an iterator that uses `filter_op` to both filter and map elements. +/// This struct is created by the [`filter_map()`] method on [`ParallelIterator`]. +/// +/// [`filter_map()`]: ParallelIterator::filter_map() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Clone)] +pub struct FilterMap { + base: I, + filter_op: P, +} + +impl Debug for FilterMap { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("FilterMap") + .field("base", &self.base) + .finish() + } +} + +impl FilterMap { + /// Creates a new `FilterMap` iterator. + pub(super) fn new(base: I, filter_op: P) -> Self { + FilterMap { base, filter_op } + } +} + +impl ParallelIterator for FilterMap +where + I: ParallelIterator, + P: Fn(I::Item) -> Option + Sync + Send, + R: Send, +{ + type Item = R; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let consumer = FilterMapConsumer::new(consumer, &self.filter_op); + self.base.drive_unindexed(consumer) + } +} + +// //////////////////////////////////////////////////////////////////////// +// Consumer implementation + +struct FilterMapConsumer<'p, C, P> { + base: C, + filter_op: &'p P, +} + +impl<'p, C, P: 'p> FilterMapConsumer<'p, C, P> { + fn new(base: C, filter_op: &'p P) -> Self { + FilterMapConsumer { base, filter_op } + } +} + +impl<'p, T, U, C, P> Consumer for FilterMapConsumer<'p, C, P> +where + C: Consumer, + P: Fn(T) -> Option + Sync + 'p, +{ + type Folder = FilterMapFolder<'p, C::Folder, P>; + type Reducer = C::Reducer; + type Result = C::Result; + + fn split_at(self, index: usize) -> (Self, Self, Self::Reducer) { + let (left, right, reducer) = self.base.split_at(index); + ( + FilterMapConsumer::new(left, self.filter_op), + FilterMapConsumer::new(right, self.filter_op), + reducer, + ) + } + + fn into_folder(self) -> Self::Folder { + let base = self.base.into_folder(); + FilterMapFolder { + base, + filter_op: self.filter_op, + } + } + + fn full(&self) -> bool { + self.base.full() + } +} + +impl<'p, T, U, C, P> UnindexedConsumer for FilterMapConsumer<'p, C, P> +where + C: UnindexedConsumer, + P: Fn(T) -> Option + Sync + 'p, +{ + fn split_off_left(&self) -> Self { + FilterMapConsumer::new(self.base.split_off_left(), self.filter_op) + } + + fn to_reducer(&self) -> Self::Reducer { + self.base.to_reducer() + } +} + +struct FilterMapFolder<'p, C, P> { + base: C, + filter_op: &'p P, +} + +impl<'p, T, U, C, P> Folder for FilterMapFolder<'p, C, P> +where + C: Folder, + P: Fn(T) -> Option + Sync + 'p, +{ + type Result = C::Result; + + fn consume(self, item: T) -> Self { + let filter_op = self.filter_op; + if let Some(mapped_item) = filter_op(item) { + let base = self.base.consume(mapped_item); + FilterMapFolder { base, filter_op } + } else { + self + } + } + + // This cannot easily specialize `consume_iter` to be better than + // the default, because that requires checking `self.base.full()` + // during a call to `self.base.consume_iter()`. (#632) + + fn complete(self) -> C::Result { + self.base.complete() + } + + fn full(&self) -> bool { + self.base.full() + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/find.rs b/anneal/v2/vendor/rayon/src/iter/find.rs new file mode 100644 index 0000000000..b16ee84466 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/find.rs @@ -0,0 +1,120 @@ +use super::plumbing::*; +use super::*; +use std::sync::atomic::{AtomicBool, Ordering}; + +pub(super) fn find(pi: I, find_op: P) -> Option +where + I: ParallelIterator, + P: Fn(&I::Item) -> bool + Sync, +{ + let found = AtomicBool::new(false); + let consumer = FindConsumer::new(&find_op, &found); + pi.drive_unindexed(consumer) +} + +struct FindConsumer<'p, P> { + find_op: &'p P, + found: &'p AtomicBool, +} + +impl<'p, P> FindConsumer<'p, P> { + fn new(find_op: &'p P, found: &'p AtomicBool) -> Self { + FindConsumer { find_op, found } + } +} + +impl<'p, T, P: 'p> Consumer for FindConsumer<'p, P> +where + T: Send, + P: Fn(&T) -> bool + Sync, +{ + type Folder = FindFolder<'p, T, P>; + type Reducer = FindReducer; + type Result = Option; + + fn split_at(self, _index: usize) -> (Self, Self, Self::Reducer) { + (self.split_off_left(), self, FindReducer) + } + + fn into_folder(self) -> Self::Folder { + FindFolder { + find_op: self.find_op, + found: self.found, + item: None, + } + } + + fn full(&self) -> bool { + self.found.load(Ordering::Relaxed) + } +} + +impl<'p, T, P: 'p> UnindexedConsumer for FindConsumer<'p, P> +where + T: Send, + P: Fn(&T) -> bool + Sync, +{ + fn split_off_left(&self) -> Self { + FindConsumer::new(self.find_op, self.found) + } + + fn to_reducer(&self) -> Self::Reducer { + FindReducer + } +} + +struct FindFolder<'p, T, P> { + find_op: &'p P, + found: &'p AtomicBool, + item: Option, +} + +impl<'p, T, P> Folder for FindFolder<'p, T, P> +where + P: Fn(&T) -> bool + 'p, +{ + type Result = Option; + + fn consume(mut self, item: T) -> Self { + if (self.find_op)(&item) { + self.found.store(true, Ordering::Relaxed); + self.item = Some(item); + } + self + } + + fn consume_iter(mut self, iter: I) -> Self + where + I: IntoIterator, + { + fn not_full(found: &AtomicBool) -> impl Fn(&T) -> bool + '_ { + move |_| !found.load(Ordering::Relaxed) + } + + self.item = iter + .into_iter() + // stop iterating if another thread has found something + .take_while(not_full(self.found)) + .find(self.find_op); + if self.item.is_some() { + self.found.store(true, Ordering::Relaxed) + } + self + } + + fn complete(self) -> Self::Result { + self.item + } + + fn full(&self) -> bool { + self.found.load(Ordering::Relaxed) + } +} + +struct FindReducer; + +impl Reducer> for FindReducer { + fn reduce(self, left: Option, right: Option) -> Option { + left.or(right) + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/find_first_last/mod.rs b/anneal/v2/vendor/rayon/src/iter/find_first_last/mod.rs new file mode 100644 index 0000000000..fd2c6fd317 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/find_first_last/mod.rs @@ -0,0 +1,230 @@ +use super::plumbing::*; +use super::*; +use std::cell::Cell; +use std::sync::atomic::{AtomicUsize, Ordering}; + +#[cfg(test)] +mod test; + +// The key optimization for find_first is that a consumer can stop its search if +// some consumer to its left already found a match (and similarly for consumers +// to the right for find_last). To make this work, all consumers need some +// notion of their position in the data relative to other consumers, including +// unindexed consumers that have no built-in notion of position. +// +// To solve this, we assign each consumer a lower and upper bound for an +// imaginary "range" of data that it consumes. The initial consumer starts with +// the range 0..usize::MAX. The split divides this range in half so that +// one resulting consumer has the range 0..(usize::MAX / 2), and the +// other has (usize::MAX / 2)..usize::max_value(). Every subsequent +// split divides the range in half again until it cannot be split anymore +// (i.e. its length is 1), in which case the split returns two consumers with +// the same range. In that case both consumers will continue to consume all +// their data regardless of whether a better match is found, but the reducer +// will still return the correct answer. + +#[derive(Copy, Clone)] +enum MatchPosition { + Leftmost, + Rightmost, +} + +/// Returns true if pos1 is a better match than pos2 according to MatchPosition +#[inline] +fn better_position(pos1: usize, pos2: usize, mp: MatchPosition) -> bool { + match mp { + MatchPosition::Leftmost => pos1 < pos2, + MatchPosition::Rightmost => pos1 > pos2, + } +} + +pub(super) fn find_first(pi: I, find_op: P) -> Option +where + I: ParallelIterator, + P: Fn(&I::Item) -> bool + Sync, +{ + let best_found = AtomicUsize::new(usize::MAX); + let consumer = FindConsumer::new(&find_op, MatchPosition::Leftmost, &best_found); + pi.drive_unindexed(consumer) +} + +pub(super) fn find_last(pi: I, find_op: P) -> Option +where + I: ParallelIterator, + P: Fn(&I::Item) -> bool + Sync, +{ + let best_found = AtomicUsize::new(0); + let consumer = FindConsumer::new(&find_op, MatchPosition::Rightmost, &best_found); + pi.drive_unindexed(consumer) +} + +struct FindConsumer<'p, P> { + find_op: &'p P, + lower_bound: Cell, + upper_bound: usize, + match_position: MatchPosition, + best_found: &'p AtomicUsize, +} + +impl<'p, P> FindConsumer<'p, P> { + fn new(find_op: &'p P, match_position: MatchPosition, best_found: &'p AtomicUsize) -> Self { + FindConsumer { + find_op, + lower_bound: Cell::new(0), + upper_bound: usize::MAX, + match_position, + best_found, + } + } + + fn current_index(&self) -> usize { + match self.match_position { + MatchPosition::Leftmost => self.lower_bound.get(), + MatchPosition::Rightmost => self.upper_bound, + } + } +} + +impl<'p, T, P> Consumer for FindConsumer<'p, P> +where + T: Send, + P: Fn(&T) -> bool + Sync, +{ + type Folder = FindFolder<'p, T, P>; + type Reducer = FindReducer; + type Result = Option; + + fn split_at(self, _index: usize) -> (Self, Self, Self::Reducer) { + let dir = self.match_position; + ( + self.split_off_left(), + self, + FindReducer { + match_position: dir, + }, + ) + } + + fn into_folder(self) -> Self::Folder { + FindFolder { + find_op: self.find_op, + boundary: self.current_index(), + match_position: self.match_position, + best_found: self.best_found, + item: None, + } + } + + fn full(&self) -> bool { + // can stop consuming if the best found index so far is *strictly* + // better than anything this consumer will find + better_position( + self.best_found.load(Ordering::Relaxed), + self.current_index(), + self.match_position, + ) + } +} + +impl<'p, T, P> UnindexedConsumer for FindConsumer<'p, P> +where + T: Send, + P: Fn(&T) -> bool + Sync, +{ + fn split_off_left(&self) -> Self { + // Upper bound for one consumer will be lower bound for the other. This + // overlap is okay, because only one of the bounds will be used for + // comparing against best_found; the other is kept only to be able to + // divide the range in half. + // + // When the resolution of usize has been exhausted (i.e. when + // upper_bound = lower_bound), both results of this split will have the + // same range. When that happens, we lose the ability to tell one + // consumer to stop working when the other finds a better match, but the + // reducer ensures that the best answer is still returned (see the test + // above). + let old_lower_bound = self.lower_bound.get(); + let median = old_lower_bound + ((self.upper_bound - old_lower_bound) / 2); + self.lower_bound.set(median); + + FindConsumer { + find_op: self.find_op, + lower_bound: Cell::new(old_lower_bound), + upper_bound: median, + match_position: self.match_position, + best_found: self.best_found, + } + } + + fn to_reducer(&self) -> Self::Reducer { + FindReducer { + match_position: self.match_position, + } + } +} + +struct FindFolder<'p, T, P> { + find_op: &'p P, + boundary: usize, + match_position: MatchPosition, + best_found: &'p AtomicUsize, + item: Option, +} + +impl<'p, P: 'p + Fn(&T) -> bool, T> Folder for FindFolder<'p, T, P> { + type Result = Option; + + fn consume(mut self, item: T) -> Self { + let found_best_in_range = match self.match_position { + MatchPosition::Leftmost => self.item.is_some(), + MatchPosition::Rightmost => false, + }; + + if !found_best_in_range && (self.find_op)(&item) { + // Update the best found index if ours is better. + let update = + self.best_found + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| { + better_position(self.boundary, current, self.match_position) + .then_some(self.boundary) + }); + + // Save this item if our index was better or equal. + if update.is_ok() || update == Err(self.boundary) { + self.item = Some(item); + } + } + self + } + + fn complete(self) -> Self::Result { + self.item + } + + fn full(&self) -> bool { + let found_best_in_range = match self.match_position { + MatchPosition::Leftmost => self.item.is_some(), + MatchPosition::Rightmost => false, + }; + + found_best_in_range + || better_position( + self.best_found.load(Ordering::Relaxed), + self.boundary, + self.match_position, + ) + } +} + +struct FindReducer { + match_position: MatchPosition, +} + +impl Reducer> for FindReducer { + fn reduce(self, left: Option, right: Option) -> Option { + match self.match_position { + MatchPosition::Leftmost => left.or(right), + MatchPosition::Rightmost => right.or(left), + } + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/find_first_last/test.rs b/anneal/v2/vendor/rayon/src/iter/find_first_last/test.rs new file mode 100644 index 0000000000..401d352b7d --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/find_first_last/test.rs @@ -0,0 +1,102 @@ +use super::*; + +#[test] +fn same_range_first_consumers_return_correct_answer() { + let find_op = |x: &i32| x % 2 == 0; + let first_found = AtomicUsize::new(usize::MAX); + let far_right_consumer = FindConsumer::new(&find_op, MatchPosition::Leftmost, &first_found); + + // We save a consumer that will be far to the right of the main consumer (and therefore not + // sharing an index range with that consumer) for fullness testing + let consumer = far_right_consumer.split_off_left(); + + // split until we have an indivisible range + for _ in 0..usize::BITS { + consumer.split_off_left(); + } + + let reducer = consumer.to_reducer(); + // the left and right folders should now have the same range, having + // exhausted the resolution of usize + let left_folder = consumer.split_off_left().into_folder(); + let right_folder = consumer.into_folder(); + + let left_folder = left_folder.consume(0).consume(1); + assert_eq!(left_folder.boundary, right_folder.boundary); + // expect not full even though a better match has been found because the + // ranges are the same + assert!(!right_folder.full()); + assert!(far_right_consumer.full()); + let right_folder = right_folder.consume(2).consume(3); + assert_eq!( + reducer.reduce(left_folder.complete(), right_folder.complete()), + Some(0) + ); +} + +#[test] +fn same_range_last_consumers_return_correct_answer() { + let find_op = |x: &i32| x % 2 == 0; + let last_found = AtomicUsize::new(0); + let consumer = FindConsumer::new(&find_op, MatchPosition::Rightmost, &last_found); + + // We save a consumer that will be far to the left of the main consumer (and therefore not + // sharing an index range with that consumer) for fullness testing + let far_left_consumer = consumer.split_off_left(); + + // split until we have an indivisible range + for _ in 0..usize::BITS { + consumer.split_off_left(); + } + + let reducer = consumer.to_reducer(); + // due to the exact calculation in split_off_left, the very last consumer has a + // range of width 2, so we use the second-to-last consumer instead to get + // the same boundary on both folders + let consumer = consumer.split_off_left(); + let left_folder = consumer.split_off_left().into_folder(); + let right_folder = consumer.into_folder(); + let right_folder = right_folder.consume(2).consume(3); + assert_eq!(left_folder.boundary, right_folder.boundary); + // expect not full even though a better match has been found because the + // ranges are the same + assert!(!left_folder.full()); + assert!(far_left_consumer.full()); + let left_folder = left_folder.consume(0).consume(1); + assert_eq!( + reducer.reduce(left_folder.complete(), right_folder.complete()), + Some(2) + ); +} + +// These tests requires that a folder be assigned to an iterator with more than +// one element. We can't necessarily determine when that will happen for a given +// input to find_first/find_last, so we test the folder directly here instead. +#[test] +fn find_first_folder_does_not_clobber_first_found() { + let best_found = AtomicUsize::new(usize::MAX); + let f = FindFolder { + find_op: &(|&_: &i32| -> bool { true }), + boundary: 0, + match_position: MatchPosition::Leftmost, + best_found: &best_found, + item: None, + }; + let f = f.consume(0_i32).consume(1_i32).consume(2_i32); + assert!(f.full()); + assert_eq!(f.complete(), Some(0_i32)); +} + +#[test] +fn find_last_folder_yields_last_match() { + let best_found = AtomicUsize::new(0); + let f = FindFolder { + find_op: &(|&_: &i32| -> bool { true }), + boundary: 0, + match_position: MatchPosition::Rightmost, + best_found: &best_found, + item: None, + }; + let f = f.consume(0_i32).consume(1_i32).consume(2_i32); + assert_eq!(f.complete(), Some(2_i32)); +} diff --git a/anneal/v2/vendor/rayon/src/iter/flat_map.rs b/anneal/v2/vendor/rayon/src/iter/flat_map.rs new file mode 100644 index 0000000000..3e44dc5cdd --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/flat_map.rs @@ -0,0 +1,153 @@ +use super::plumbing::*; +use super::*; + +use std::fmt::{self, Debug}; + +/// `FlatMap` maps each element to a parallel iterator, then flattens these iterators together. +/// This struct is created by the [`flat_map()`] method on [`ParallelIterator`] +/// +/// [`flat_map()`]: ParallelIterator::flat_map() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Clone)] +pub struct FlatMap { + base: I, + map_op: F, +} + +impl Debug for FlatMap { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("FlatMap").field("base", &self.base).finish() + } +} + +impl FlatMap { + /// Creates a new `FlatMap` iterator. + pub(super) fn new(base: I, map_op: F) -> Self { + FlatMap { base, map_op } + } +} + +impl ParallelIterator for FlatMap +where + I: ParallelIterator, + F: Fn(I::Item) -> PI + Sync + Send, + PI: IntoParallelIterator, +{ + type Item = PI::Item; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let consumer = FlatMapConsumer::new(consumer, &self.map_op); + self.base.drive_unindexed(consumer) + } +} + +// //////////////////////////////////////////////////////////////////////// +// Consumer implementation + +struct FlatMapConsumer<'f, C, F> { + base: C, + map_op: &'f F, +} + +impl<'f, C, F> FlatMapConsumer<'f, C, F> { + fn new(base: C, map_op: &'f F) -> Self { + FlatMapConsumer { base, map_op } + } +} + +impl<'f, T, U, C, F> Consumer for FlatMapConsumer<'f, C, F> +where + C: UnindexedConsumer, + F: Fn(T) -> U + Sync, + U: IntoParallelIterator, +{ + type Folder = FlatMapFolder<'f, C, F, C::Result>; + type Reducer = C::Reducer; + type Result = C::Result; + + fn split_at(self, index: usize) -> (Self, Self, C::Reducer) { + let (left, right, reducer) = self.base.split_at(index); + ( + FlatMapConsumer::new(left, self.map_op), + FlatMapConsumer::new(right, self.map_op), + reducer, + ) + } + + fn into_folder(self) -> Self::Folder { + FlatMapFolder { + base: self.base, + map_op: self.map_op, + previous: None, + } + } + + fn full(&self) -> bool { + self.base.full() + } +} + +impl<'f, T, U, C, F> UnindexedConsumer for FlatMapConsumer<'f, C, F> +where + C: UnindexedConsumer, + F: Fn(T) -> U + Sync, + U: IntoParallelIterator, +{ + fn split_off_left(&self) -> Self { + FlatMapConsumer::new(self.base.split_off_left(), self.map_op) + } + + fn to_reducer(&self) -> Self::Reducer { + self.base.to_reducer() + } +} + +struct FlatMapFolder<'f, C, F, R> { + base: C, + map_op: &'f F, + previous: Option, +} + +impl<'f, T, U, C, F> Folder for FlatMapFolder<'f, C, F, C::Result> +where + C: UnindexedConsumer, + F: Fn(T) -> U + Sync, + U: IntoParallelIterator, +{ + type Result = C::Result; + + fn consume(self, item: T) -> Self { + let map_op = self.map_op; + let par_iter = map_op(item).into_par_iter(); + let consumer = self.base.split_off_left(); + let result = par_iter.drive_unindexed(consumer); + + let previous = match self.previous { + None => Some(result), + Some(previous) => { + let reducer = self.base.to_reducer(); + Some(reducer.reduce(previous, result)) + } + }; + + FlatMapFolder { + base: self.base, + map_op, + previous, + } + } + + fn complete(self) -> Self::Result { + match self.previous { + Some(previous) => previous, + None => self.base.into_folder().complete(), + } + } + + fn full(&self) -> bool { + self.base.full() + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/flat_map_iter.rs b/anneal/v2/vendor/rayon/src/iter/flat_map_iter.rs new file mode 100644 index 0000000000..9a99e5bdb8 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/flat_map_iter.rs @@ -0,0 +1,145 @@ +use super::plumbing::*; +use super::*; + +use std::fmt::{self, Debug}; + +/// `FlatMapIter` maps each element to a serial iterator, then flattens these iterators together. +/// This struct is created by the [`flat_map_iter()`] method on [`ParallelIterator`] +/// +/// [`flat_map_iter()`]: ParallelIterator::flat_map_iter() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Clone)] +pub struct FlatMapIter { + base: I, + map_op: F, +} + +impl Debug for FlatMapIter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("FlatMapIter") + .field("base", &self.base) + .finish() + } +} + +impl FlatMapIter { + /// Creates a new `FlatMapIter` iterator. + pub(super) fn new(base: I, map_op: F) -> Self { + FlatMapIter { base, map_op } + } +} + +impl ParallelIterator for FlatMapIter +where + I: ParallelIterator, + F: Fn(I::Item) -> SI + Sync + Send, + SI: IntoIterator, +{ + type Item = SI::Item; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let consumer = FlatMapIterConsumer::new(consumer, &self.map_op); + self.base.drive_unindexed(consumer) + } +} + +// //////////////////////////////////////////////////////////////////////// +// Consumer implementation + +struct FlatMapIterConsumer<'f, C, F> { + base: C, + map_op: &'f F, +} + +impl<'f, C, F> FlatMapIterConsumer<'f, C, F> { + fn new(base: C, map_op: &'f F) -> Self { + FlatMapIterConsumer { base, map_op } + } +} + +impl<'f, T, U, C, F> Consumer for FlatMapIterConsumer<'f, C, F> +where + C: UnindexedConsumer, + F: Fn(T) -> U + Sync, + U: IntoIterator, +{ + type Folder = FlatMapIterFolder<'f, C::Folder, F>; + type Reducer = C::Reducer; + type Result = C::Result; + + fn split_at(self, index: usize) -> (Self, Self, C::Reducer) { + let (left, right, reducer) = self.base.split_at(index); + ( + FlatMapIterConsumer::new(left, self.map_op), + FlatMapIterConsumer::new(right, self.map_op), + reducer, + ) + } + + fn into_folder(self) -> Self::Folder { + FlatMapIterFolder { + base: self.base.into_folder(), + map_op: self.map_op, + } + } + + fn full(&self) -> bool { + self.base.full() + } +} + +impl<'f, T, U, C, F> UnindexedConsumer for FlatMapIterConsumer<'f, C, F> +where + C: UnindexedConsumer, + F: Fn(T) -> U + Sync, + U: IntoIterator, +{ + fn split_off_left(&self) -> Self { + FlatMapIterConsumer::new(self.base.split_off_left(), self.map_op) + } + + fn to_reducer(&self) -> Self::Reducer { + self.base.to_reducer() + } +} + +struct FlatMapIterFolder<'f, C, F> { + base: C, + map_op: &'f F, +} + +impl<'f, T, U, C, F> Folder for FlatMapIterFolder<'f, C, F> +where + C: Folder, + F: Fn(T) -> U, + U: IntoIterator, +{ + type Result = C::Result; + + fn consume(self, item: T) -> Self { + let map_op = self.map_op; + let base = self.base.consume_iter(map_op(item)); + FlatMapIterFolder { base, map_op } + } + + fn consume_iter(self, iter: I) -> Self + where + I: IntoIterator, + { + let map_op = self.map_op; + let iter = iter.into_iter().flat_map(map_op); + let base = self.base.consume_iter(iter); + FlatMapIterFolder { base, map_op } + } + + fn complete(self) -> Self::Result { + self.base.complete() + } + + fn full(&self) -> bool { + self.base.full() + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/flatten.rs b/anneal/v2/vendor/rayon/src/iter/flatten.rs new file mode 100644 index 0000000000..e3dfe51d35 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/flatten.rs @@ -0,0 +1,134 @@ +use super::plumbing::*; +use super::*; + +/// `Flatten` turns each element to a parallel iterator, then flattens these iterators +/// together. This struct is created by the [`flatten()`] method on [`ParallelIterator`]. +/// +/// [`flatten()`]: ParallelIterator::flatten() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Debug, Clone)] +pub struct Flatten { + base: I, +} + +impl Flatten { + /// Creates a new `Flatten` iterator. + pub(super) fn new(base: I) -> Self { + Flatten { base } + } +} + +impl ParallelIterator for Flatten +where + I: ParallelIterator, +{ + type Item = ::Item; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let consumer = FlattenConsumer::new(consumer); + self.base.drive_unindexed(consumer) + } +} + +// //////////////////////////////////////////////////////////////////////// +// Consumer implementation + +struct FlattenConsumer { + base: C, +} + +impl FlattenConsumer { + fn new(base: C) -> Self { + FlattenConsumer { base } + } +} + +impl Consumer for FlattenConsumer +where + C: UnindexedConsumer, + T: IntoParallelIterator, +{ + type Folder = FlattenFolder; + type Reducer = C::Reducer; + type Result = C::Result; + + fn split_at(self, index: usize) -> (Self, Self, C::Reducer) { + let (left, right, reducer) = self.base.split_at(index); + ( + FlattenConsumer::new(left), + FlattenConsumer::new(right), + reducer, + ) + } + + fn into_folder(self) -> Self::Folder { + FlattenFolder { + base: self.base, + previous: None, + } + } + + fn full(&self) -> bool { + self.base.full() + } +} + +impl UnindexedConsumer for FlattenConsumer +where + C: UnindexedConsumer, + T: IntoParallelIterator, +{ + fn split_off_left(&self) -> Self { + FlattenConsumer::new(self.base.split_off_left()) + } + + fn to_reducer(&self) -> Self::Reducer { + self.base.to_reducer() + } +} + +struct FlattenFolder { + base: C, + previous: Option, +} + +impl Folder for FlattenFolder +where + C: UnindexedConsumer, + T: IntoParallelIterator, +{ + type Result = C::Result; + + fn consume(self, item: T) -> Self { + let par_iter = item.into_par_iter(); + let consumer = self.base.split_off_left(); + let result = par_iter.drive_unindexed(consumer); + + let previous = match self.previous { + None => Some(result), + Some(previous) => { + let reducer = self.base.to_reducer(); + Some(reducer.reduce(previous, result)) + } + }; + + FlattenFolder { + base: self.base, + previous, + } + } + + fn complete(self) -> Self::Result { + match self.previous { + Some(previous) => previous, + None => self.base.into_folder().complete(), + } + } + + fn full(&self) -> bool { + self.base.full() + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/flatten_iter.rs b/anneal/v2/vendor/rayon/src/iter/flatten_iter.rs new file mode 100644 index 0000000000..d6f2e64bdf --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/flatten_iter.rs @@ -0,0 +1,124 @@ +use super::plumbing::*; +use super::*; + +/// `FlattenIter` turns each element to a serial iterator, then flattens these iterators +/// together. This struct is created by the [`flatten_iter()`] method on [`ParallelIterator`]. +/// +/// [`flatten_iter()`]: ParallelIterator::flatten_iter() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Debug, Clone)] +pub struct FlattenIter { + base: I, +} + +impl FlattenIter { + /// Creates a new `FlattenIter` iterator. + pub(super) fn new(base: I) -> Self { + FlattenIter { base } + } +} + +impl ParallelIterator for FlattenIter +where + I: ParallelIterator>, +{ + type Item = ::Item; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let consumer = FlattenIterConsumer::new(consumer); + self.base.drive_unindexed(consumer) + } +} + +// //////////////////////////////////////////////////////////////////////// +// Consumer implementation + +struct FlattenIterConsumer { + base: C, +} + +impl FlattenIterConsumer { + fn new(base: C) -> Self { + FlattenIterConsumer { base } + } +} + +impl Consumer for FlattenIterConsumer +where + C: UnindexedConsumer, + T: IntoIterator, +{ + type Folder = FlattenIterFolder; + type Reducer = C::Reducer; + type Result = C::Result; + + fn split_at(self, index: usize) -> (Self, Self, C::Reducer) { + let (left, right, reducer) = self.base.split_at(index); + ( + FlattenIterConsumer::new(left), + FlattenIterConsumer::new(right), + reducer, + ) + } + + fn into_folder(self) -> Self::Folder { + FlattenIterFolder { + base: self.base.into_folder(), + } + } + + fn full(&self) -> bool { + self.base.full() + } +} + +impl UnindexedConsumer for FlattenIterConsumer +where + C: UnindexedConsumer, + T: IntoIterator, +{ + fn split_off_left(&self) -> Self { + FlattenIterConsumer::new(self.base.split_off_left()) + } + + fn to_reducer(&self) -> Self::Reducer { + self.base.to_reducer() + } +} + +struct FlattenIterFolder { + base: C, +} + +impl Folder for FlattenIterFolder +where + C: Folder, + T: IntoIterator, +{ + type Result = C::Result; + + fn consume(self, item: T) -> Self { + let base = self.base.consume_iter(item); + FlattenIterFolder { base } + } + + fn consume_iter(self, iter: I) -> Self + where + I: IntoIterator, + { + let iter = iter.into_iter().flatten(); + let base = self.base.consume_iter(iter); + FlattenIterFolder { base } + } + + fn complete(self) -> Self::Result { + self.base.complete() + } + + fn full(&self) -> bool { + self.base.full() + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/fold.rs b/anneal/v2/vendor/rayon/src/iter/fold.rs new file mode 100644 index 0000000000..f637059dc6 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/fold.rs @@ -0,0 +1,289 @@ +use super::plumbing::*; +use super::*; + +use std::fmt::{self, Debug}; + +impl Fold { + pub(super) fn new(base: I, identity: ID, fold_op: F) -> Self { + Fold { + base, + identity, + fold_op, + } + } +} + +/// `Fold` is an iterator that applies a function over an iterator producing a single value. +/// This struct is created by the [`fold()`] method on [`ParallelIterator`] +/// +/// [`fold()`]: ParallelIterator::fold() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Clone)] +pub struct Fold { + base: I, + identity: ID, + fold_op: F, +} + +impl Debug for Fold { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Fold").field("base", &self.base).finish() + } +} + +impl ParallelIterator for Fold +where + I: ParallelIterator, + F: Fn(U, I::Item) -> U + Sync + Send, + ID: Fn() -> U + Sync + Send, + U: Send, +{ + type Item = U; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let consumer1 = FoldConsumer { + base: consumer, + fold_op: &self.fold_op, + identity: &self.identity, + }; + self.base.drive_unindexed(consumer1) + } +} + +struct FoldConsumer<'c, C, ID, F> { + base: C, + fold_op: &'c F, + identity: &'c ID, +} + +impl<'r, U, T, C, ID, F> Consumer for FoldConsumer<'r, C, ID, F> +where + C: Consumer, + F: Fn(U, T) -> U + Sync, + ID: Fn() -> U + Sync, + U: Send, +{ + type Folder = FoldFolder<'r, C::Folder, U, F>; + type Reducer = C::Reducer; + type Result = C::Result; + + fn split_at(self, index: usize) -> (Self, Self, Self::Reducer) { + let (left, right, reducer) = self.base.split_at(index); + ( + FoldConsumer { base: left, ..self }, + FoldConsumer { + base: right, + ..self + }, + reducer, + ) + } + + fn into_folder(self) -> Self::Folder { + FoldFolder { + base: self.base.into_folder(), + item: (self.identity)(), + fold_op: self.fold_op, + } + } + + fn full(&self) -> bool { + self.base.full() + } +} + +impl<'r, U, T, C, ID, F> UnindexedConsumer for FoldConsumer<'r, C, ID, F> +where + C: UnindexedConsumer, + F: Fn(U, T) -> U + Sync, + ID: Fn() -> U + Sync, + U: Send, +{ + fn split_off_left(&self) -> Self { + FoldConsumer { + base: self.base.split_off_left(), + ..*self + } + } + + fn to_reducer(&self) -> Self::Reducer { + self.base.to_reducer() + } +} + +struct FoldFolder<'r, C, ID, F> { + base: C, + fold_op: &'r F, + item: ID, +} + +impl<'r, C, ID, F, T> Folder for FoldFolder<'r, C, ID, F> +where + C: Folder, + F: Fn(ID, T) -> ID + Sync, +{ + type Result = C::Result; + + fn consume(self, item: T) -> Self { + let item = (self.fold_op)(self.item, item); + FoldFolder { + base: self.base, + fold_op: self.fold_op, + item, + } + } + + fn consume_iter(self, iter: I) -> Self + where + I: IntoIterator, + { + fn not_full(base: &C) -> impl Fn(&T) -> bool + '_ + where + C: Folder, + { + move |_| !base.full() + } + + let base = self.base; + let item = iter + .into_iter() + // stop iterating if another thread has finished + .take_while(not_full(&base)) + .fold(self.item, self.fold_op); + + FoldFolder { + base, + item, + fold_op: self.fold_op, + } + } + + fn complete(self) -> C::Result { + self.base.consume(self.item).complete() + } + + fn full(&self) -> bool { + self.base.full() + } +} + +// /////////////////////////////////////////////////////////////////////////// + +impl FoldWith { + pub(super) fn new(base: I, item: U, fold_op: F) -> Self { + FoldWith { + base, + item, + fold_op, + } + } +} + +/// `FoldWith` is an iterator that applies a function over an iterator producing a single value. +/// This struct is created by the [`fold_with()`] method on [`ParallelIterator`] +/// +/// [`fold_with()`]: ParallelIterator::fold_with() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Clone)] +pub struct FoldWith { + base: I, + item: U, + fold_op: F, +} + +impl Debug for FoldWith { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("FoldWith") + .field("base", &self.base) + .field("item", &self.item) + .finish() + } +} + +impl ParallelIterator for FoldWith +where + I: ParallelIterator, + F: Fn(U, I::Item) -> U + Sync + Send, + U: Send + Clone, +{ + type Item = U; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let consumer1 = FoldWithConsumer { + base: consumer, + item: self.item, + fold_op: &self.fold_op, + }; + self.base.drive_unindexed(consumer1) + } +} + +struct FoldWithConsumer<'c, C, U, F> { + base: C, + item: U, + fold_op: &'c F, +} + +impl<'r, U, T, C, F> Consumer for FoldWithConsumer<'r, C, U, F> +where + C: Consumer, + F: Fn(U, T) -> U + Sync, + U: Send + Clone, +{ + type Folder = FoldFolder<'r, C::Folder, U, F>; + type Reducer = C::Reducer; + type Result = C::Result; + + fn split_at(self, index: usize) -> (Self, Self, Self::Reducer) { + let (left, right, reducer) = self.base.split_at(index); + ( + FoldWithConsumer { + base: left, + item: self.item.clone(), + ..self + }, + FoldWithConsumer { + base: right, + ..self + }, + reducer, + ) + } + + fn into_folder(self) -> Self::Folder { + FoldFolder { + base: self.base.into_folder(), + item: self.item, + fold_op: self.fold_op, + } + } + + fn full(&self) -> bool { + self.base.full() + } +} + +impl<'r, U, T, C, F> UnindexedConsumer for FoldWithConsumer<'r, C, U, F> +where + C: UnindexedConsumer, + F: Fn(U, T) -> U + Sync, + U: Send + Clone, +{ + fn split_off_left(&self) -> Self { + FoldWithConsumer { + base: self.base.split_off_left(), + item: self.item.clone(), + ..*self + } + } + + fn to_reducer(&self) -> Self::Reducer { + self.base.to_reducer() + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/fold_chunks.rs b/anneal/v2/vendor/rayon/src/iter/fold_chunks.rs new file mode 100644 index 0000000000..a8bc394229 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/fold_chunks.rs @@ -0,0 +1,224 @@ +use std::fmt::{self, Debug}; + +use super::chunks::ChunkProducer; +use super::plumbing::*; +use super::*; + +/// `FoldChunks` is an iterator that groups elements of an underlying iterator and applies a +/// function over them, producing a single value for each group. +/// +/// This struct is created by the [`fold_chunks()`] method on [`IndexedParallelIterator`] +/// +/// [`fold_chunks()`]: IndexedParallelIterator::fold_chunks() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Clone)] +pub struct FoldChunks { + base: I, + chunk_size: usize, + fold_op: F, + identity: ID, +} + +impl Debug for FoldChunks { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Fold") + .field("base", &self.base) + .field("chunk_size", &self.chunk_size) + .finish() + } +} + +impl FoldChunks { + /// Creates a new `FoldChunks` iterator + pub(super) fn new(base: I, chunk_size: usize, identity: ID, fold_op: F) -> Self { + FoldChunks { + base, + chunk_size, + identity, + fold_op, + } + } +} + +impl ParallelIterator for FoldChunks +where + I: IndexedParallelIterator, + ID: Fn() -> U + Send + Sync, + F: Fn(U, I::Item) -> U + Send + Sync, + U: Send, +{ + type Item = U; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: Consumer, + { + bridge(self, consumer) + } + + fn opt_len(&self) -> Option { + Some(self.len()) + } +} + +impl IndexedParallelIterator for FoldChunks +where + I: IndexedParallelIterator, + ID: Fn() -> U + Send + Sync, + F: Fn(U, I::Item) -> U + Send + Sync, + U: Send, +{ + fn len(&self) -> usize { + self.base.len().div_ceil(self.chunk_size) + } + + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + bridge(self, consumer) + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + let len = self.base.len(); + return self.base.with_producer(Callback { + chunk_size: self.chunk_size, + len, + identity: self.identity, + fold_op: self.fold_op, + callback, + }); + + struct Callback { + chunk_size: usize, + len: usize, + identity: ID, + fold_op: F, + callback: CB, + } + + impl ProducerCallback for Callback + where + CB: ProducerCallback, + ID: Fn() -> U + Send + Sync, + F: Fn(U, T) -> U + Send + Sync, + { + type Output = CB::Output; + + fn callback

(self, base: P) -> CB::Output + where + P: Producer, + { + let identity = &self.identity; + let fold_op = &self.fold_op; + let fold_iter = move |iter: P::IntoIter| iter.fold(identity(), fold_op); + let producer = ChunkProducer::new(self.chunk_size, self.len, base, fold_iter); + self.callback.callback(producer) + } + } + } +} + +#[cfg(test)] +mod test { + use super::*; + use std::ops::Add; + + #[test] + fn check_fold_chunks() { + let words = "bishbashbosh!" + .chars() + .collect::>() + .into_par_iter() + .fold_chunks(4, String::new, |mut s, c| { + s.push(c); + s + }) + .collect::>(); + + assert_eq!(words, vec!["bish", "bash", "bosh", "!"]); + } + + // 'closure' values for tests below + fn id() -> i32 { + 0 + } + fn sum(x: T, y: U) -> T + where + T: Add, + { + x + y + } + + #[test] + #[should_panic(expected = "chunk_size must not be zero")] + fn check_fold_chunks_zero_size() { + let _: Vec = vec![1, 2, 3] + .into_par_iter() + .fold_chunks(0, id, sum) + .collect(); + } + + #[test] + fn check_fold_chunks_even_size() { + assert_eq!( + vec![1 + 2 + 3, 4 + 5 + 6, 7 + 8 + 9], + (1..10) + .into_par_iter() + .fold_chunks(3, id, sum) + .collect::>() + ); + } + + #[test] + fn check_fold_chunks_empty() { + let v: Vec = vec![]; + let expected: Vec = vec![]; + assert_eq!( + expected, + v.into_par_iter() + .fold_chunks(2, id, sum) + .collect::>() + ); + } + + #[test] + fn check_fold_chunks_len() { + assert_eq!(4, (0..8).into_par_iter().fold_chunks(2, id, sum).len()); + assert_eq!(3, (0..9).into_par_iter().fold_chunks(3, id, sum).len()); + assert_eq!(3, (0..8).into_par_iter().fold_chunks(3, id, sum).len()); + assert_eq!(1, [1].par_iter().fold_chunks(3, id, sum).len()); + assert_eq!(0, (0..0).into_par_iter().fold_chunks(3, id, sum).len()); + } + + #[test] + fn check_fold_chunks_uneven() { + let cases: Vec<(Vec, usize, Vec)> = vec![ + ((0..5).collect(), 3, vec![1 + 2, 3 + 4]), + (vec![1], 5, vec![1]), + ((0..4).collect(), 3, vec![1 + 2, 3]), + ]; + + for (i, (v, n, expected)) in cases.into_iter().enumerate() { + let mut res: Vec = vec![]; + v.par_iter() + .fold_chunks(n, || 0, sum) + .collect_into_vec(&mut res); + assert_eq!(expected, res, "Case {i} failed"); + + res.truncate(0); + v.into_par_iter() + .fold_chunks(n, || 0, sum) + .rev() + .collect_into_vec(&mut res); + assert_eq!( + expected.into_iter().rev().collect::>(), + res, + "Case {i} reversed failed" + ); + } + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/fold_chunks_with.rs b/anneal/v2/vendor/rayon/src/iter/fold_chunks_with.rs new file mode 100644 index 0000000000..5b3ab2e25c --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/fold_chunks_with.rs @@ -0,0 +1,220 @@ +use std::fmt::{self, Debug}; + +use super::chunks::ChunkProducer; +use super::plumbing::*; +use super::*; + +/// `FoldChunksWith` is an iterator that groups elements of an underlying iterator and applies a +/// function over them, producing a single value for each group. +/// +/// This struct is created by the [`fold_chunks_with()`] method on [`IndexedParallelIterator`] +/// +/// [`fold_chunks_with()`]: IndexedParallelIterator::fold_chunks() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Clone)] +pub struct FoldChunksWith { + base: I, + chunk_size: usize, + item: U, + fold_op: F, +} + +impl Debug for FoldChunksWith { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Fold") + .field("base", &self.base) + .field("chunk_size", &self.chunk_size) + .field("item", &self.item) + .finish() + } +} + +impl FoldChunksWith { + /// Creates a new `FoldChunksWith` iterator + pub(super) fn new(base: I, chunk_size: usize, item: U, fold_op: F) -> Self { + FoldChunksWith { + base, + chunk_size, + item, + fold_op, + } + } +} + +impl ParallelIterator for FoldChunksWith +where + I: IndexedParallelIterator, + U: Send + Clone, + F: Fn(U, I::Item) -> U + Send + Sync, +{ + type Item = U; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: Consumer, + { + bridge(self, consumer) + } + + fn opt_len(&self) -> Option { + Some(self.len()) + } +} + +impl IndexedParallelIterator for FoldChunksWith +where + I: IndexedParallelIterator, + U: Send + Clone, + F: Fn(U, I::Item) -> U + Send + Sync, +{ + fn len(&self) -> usize { + self.base.len().div_ceil(self.chunk_size) + } + + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + bridge(self, consumer) + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + let len = self.base.len(); + return self.base.with_producer(Callback { + chunk_size: self.chunk_size, + len, + item: self.item, + fold_op: self.fold_op, + callback, + }); + + struct Callback { + chunk_size: usize, + len: usize, + item: T, + fold_op: F, + callback: CB, + } + + impl ProducerCallback for Callback + where + CB: ProducerCallback, + U: Send + Clone, + F: Fn(U, T) -> U + Send + Sync, + { + type Output = CB::Output; + + fn callback

(self, base: P) -> CB::Output + where + P: Producer, + { + let item = self.item; + let fold_op = &self.fold_op; + let fold_iter = move |iter: P::IntoIter| iter.fold(item.clone(), fold_op); + let producer = ChunkProducer::new(self.chunk_size, self.len, base, fold_iter); + self.callback.callback(producer) + } + } + } +} + +#[cfg(test)] +mod test { + use super::*; + use std::ops::Add; + + #[test] + fn check_fold_chunks_with() { + let words = "bishbashbosh!" + .chars() + .collect::>() + .into_par_iter() + .fold_chunks_with(4, String::new(), |mut s, c| { + s.push(c); + s + }) + .collect::>(); + + assert_eq!(words, vec!["bish", "bash", "bosh", "!"]); + } + + // 'closure' value for tests below + fn sum(x: T, y: U) -> T + where + T: Add, + { + x + y + } + + #[test] + #[should_panic(expected = "chunk_size must not be zero")] + fn check_fold_chunks_zero_size() { + let _: Vec = vec![1, 2, 3] + .into_par_iter() + .fold_chunks_with(0, 0, sum) + .collect(); + } + + #[test] + fn check_fold_chunks_even_size() { + assert_eq!( + vec![1 + 2 + 3, 4 + 5 + 6, 7 + 8 + 9], + (1..10) + .into_par_iter() + .fold_chunks_with(3, 0, sum) + .collect::>() + ); + } + + #[test] + fn check_fold_chunks_with_empty() { + let v: Vec = vec![]; + let expected: Vec = vec![]; + assert_eq!( + expected, + v.into_par_iter() + .fold_chunks_with(2, 0, sum) + .collect::>() + ); + } + + #[test] + fn check_fold_chunks_len() { + assert_eq!(4, (0..8).into_par_iter().fold_chunks_with(2, 0, sum).len()); + assert_eq!(3, (0..9).into_par_iter().fold_chunks_with(3, 0, sum).len()); + assert_eq!(3, (0..8).into_par_iter().fold_chunks_with(3, 0, sum).len()); + assert_eq!(1, [1].par_iter().fold_chunks_with(3, 0, sum).len()); + assert_eq!(0, (0..0).into_par_iter().fold_chunks_with(3, 0, sum).len()); + } + + #[test] + fn check_fold_chunks_uneven() { + let cases: Vec<(Vec, usize, Vec)> = vec![ + ((0..5).collect(), 3, vec![1 + 2, 3 + 4]), + (vec![1], 5, vec![1]), + ((0..4).collect(), 3, vec![1 + 2, 3]), + ]; + + for (i, (v, n, expected)) in cases.into_iter().enumerate() { + let mut res: Vec = vec![]; + v.par_iter() + .fold_chunks_with(n, 0, sum) + .collect_into_vec(&mut res); + assert_eq!(expected, res, "Case {i} failed"); + + res.truncate(0); + v.into_par_iter() + .fold_chunks_with(n, 0, sum) + .rev() + .collect_into_vec(&mut res); + assert_eq!( + expected.into_iter().rev().collect::>(), + res, + "Case {i} reversed failed" + ); + } + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/for_each.rs b/anneal/v2/vendor/rayon/src/iter/for_each.rs new file mode 100644 index 0000000000..3b77bebe1b --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/for_each.rs @@ -0,0 +1,77 @@ +use super::noop::*; +use super::plumbing::*; +use super::ParallelIterator; + +pub(super) fn for_each(pi: I, op: &F) +where + I: ParallelIterator, + F: Fn(T) + Sync, + T: Send, +{ + let consumer = ForEachConsumer { op }; + pi.drive_unindexed(consumer) +} + +struct ForEachConsumer<'f, F> { + op: &'f F, +} + +impl<'f, F, T> Consumer for ForEachConsumer<'f, F> +where + F: Fn(T) + Sync, +{ + type Folder = ForEachConsumer<'f, F>; + type Reducer = NoopReducer; + type Result = (); + + fn split_at(self, _index: usize) -> (Self, Self, NoopReducer) { + (self.split_off_left(), self, NoopReducer) + } + + fn into_folder(self) -> Self { + self + } + + fn full(&self) -> bool { + false + } +} + +impl<'f, F, T> Folder for ForEachConsumer<'f, F> +where + F: Fn(T) + Sync, +{ + type Result = (); + + fn consume(self, item: T) -> Self { + (self.op)(item); + self + } + + fn consume_iter(self, iter: I) -> Self + where + I: IntoIterator, + { + iter.into_iter().for_each(self.op); + self + } + + fn complete(self) {} + + fn full(&self) -> bool { + false + } +} + +impl<'f, F, T> UnindexedConsumer for ForEachConsumer<'f, F> +where + F: Fn(T) + Sync, +{ + fn split_off_left(&self) -> Self { + ForEachConsumer { op: self.op } + } + + fn to_reducer(&self) -> NoopReducer { + NoopReducer + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/from_par_iter.rs b/anneal/v2/vendor/rayon/src/iter/from_par_iter.rs new file mode 100644 index 0000000000..448a4b1022 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/from_par_iter.rs @@ -0,0 +1,280 @@ +use super::noop::NoopConsumer; +use super::{FromParallelIterator, IntoParallelIterator, ParallelExtend, ParallelIterator}; + +use std::borrow::Cow; +use std::collections::LinkedList; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::collections::{BinaryHeap, VecDeque}; +use std::ffi::{OsStr, OsString}; +use std::hash::{BuildHasher, Hash}; +use std::rc::Rc; +use std::sync::Arc; + +/// Creates an empty default collection and extends it. +fn collect_extended(par_iter: I) -> C +where + I: IntoParallelIterator, + C: ParallelExtend + Default, +{ + let mut collection = C::default(); + collection.par_extend(par_iter); + collection +} + +/// Collects items from a parallel iterator into a vector. +impl FromParallelIterator for Vec +where + T: Send, +{ + fn from_par_iter(par_iter: I) -> Self + where + I: IntoParallelIterator, + { + collect_extended(par_iter) + } +} + +/// Collects items from a parallel iterator into a boxed slice. +impl FromParallelIterator for Box<[T]> +where + T: Send, +{ + fn from_par_iter(par_iter: I) -> Self + where + I: IntoParallelIterator, + { + Vec::from_par_iter(par_iter).into() + } +} + +/// Collects items from a parallel iterator into a reference-counted slice. +impl FromParallelIterator for Rc<[T]> +where + T: Send, +{ + fn from_par_iter(par_iter: I) -> Self + where + I: IntoParallelIterator, + { + Vec::from_par_iter(par_iter).into() + } +} + +/// Collects items from a parallel iterator into an atomically-reference-counted slice. +impl FromParallelIterator for Arc<[T]> +where + T: Send, +{ + fn from_par_iter(par_iter: I) -> Self + where + I: IntoParallelIterator, + { + Vec::from_par_iter(par_iter).into() + } +} + +/// Collects items from a parallel iterator into a vecdeque. +impl FromParallelIterator for VecDeque +where + T: Send, +{ + fn from_par_iter(par_iter: I) -> Self + where + I: IntoParallelIterator, + { + Vec::from_par_iter(par_iter).into() + } +} + +/// Collects items from a parallel iterator into a binaryheap. +/// The heap-ordering is calculated serially after all items are collected. +impl FromParallelIterator for BinaryHeap +where + T: Ord + Send, +{ + fn from_par_iter(par_iter: I) -> Self + where + I: IntoParallelIterator, + { + Vec::from_par_iter(par_iter).into() + } +} + +/// Collects items from a parallel iterator into a freshly allocated +/// linked list. +impl FromParallelIterator for LinkedList +where + T: Send, +{ + fn from_par_iter(par_iter: I) -> Self + where + I: IntoParallelIterator, + { + collect_extended(par_iter) + } +} + +/// Collects (key, value) pairs from a parallel iterator into a +/// hashmap. If multiple pairs correspond to the same key, then the +/// ones produced earlier in the parallel iterator will be +/// overwritten, just as with a sequential iterator. +impl FromParallelIterator<(K, V)> for HashMap +where + K: Eq + Hash + Send, + V: Send, + S: BuildHasher + Default + Send, +{ + fn from_par_iter(par_iter: I) -> Self + where + I: IntoParallelIterator, + { + collect_extended(par_iter) + } +} + +/// Collects (key, value) pairs from a parallel iterator into a +/// btreemap. If multiple pairs correspond to the same key, then the +/// ones produced earlier in the parallel iterator will be +/// overwritten, just as with a sequential iterator. +impl FromParallelIterator<(K, V)> for BTreeMap +where + K: Ord + Send, + V: Send, +{ + fn from_par_iter(par_iter: I) -> Self + where + I: IntoParallelIterator, + { + collect_extended(par_iter) + } +} + +/// Collects values from a parallel iterator into a hashset. +impl FromParallelIterator for HashSet +where + V: Eq + Hash + Send, + S: BuildHasher + Default + Send, +{ + fn from_par_iter(par_iter: I) -> Self + where + I: IntoParallelIterator, + { + collect_extended(par_iter) + } +} + +/// Collects values from a parallel iterator into a btreeset. +impl FromParallelIterator for BTreeSet +where + V: Send + Ord, +{ + fn from_par_iter(par_iter: I) -> Self + where + I: IntoParallelIterator, + { + collect_extended(par_iter) + } +} + +macro_rules! collect_string { + ($desc:literal, $item:ty $(, $a:lifetime)?) => { + #[doc = concat!("Collects ", $desc, " from a parallel iterator into a string.")] + impl$(<$a>)? FromParallelIterator<$item> for String { + fn from_par_iter(par_iter: I) -> Self + where + I: IntoParallelIterator, + { + collect_extended(par_iter) + } + } + + #[doc = concat!("Collects ", $desc, " from a parallel iterator into a boxed string.")] + impl$(<$a>)? FromParallelIterator<$item> for Box { + fn from_par_iter(par_iter: I) -> Self + where + I: IntoParallelIterator, + { + String::from_par_iter(par_iter).into_boxed_str() + } + } + } +} + +collect_string!("characters", char); +collect_string!("characters", &'a char, 'a); +collect_string!("string slices", &'a str, 'a); +collect_string!("string slices", Cow<'a, str>, 'a); +collect_string!("boxed strings", Box); +collect_string!("strings", String); + +/// Collects OS-string slices from a parallel iterator into an OS-string. +impl<'a> FromParallelIterator<&'a OsStr> for OsString { + fn from_par_iter(par_iter: I) -> Self + where + I: IntoParallelIterator, + { + collect_extended(par_iter) + } +} + +/// Collects OS-strings from a parallel iterator into one large OS-string. +impl FromParallelIterator for OsString { + fn from_par_iter(par_iter: I) -> Self + where + I: IntoParallelIterator, + { + collect_extended(par_iter) + } +} + +/// Collects OS-string slices from a parallel iterator into an OS-string. +impl<'a> FromParallelIterator> for OsString { + fn from_par_iter(par_iter: I) -> Self + where + I: IntoParallelIterator>, + { + collect_extended(par_iter) + } +} + +/// Collects an arbitrary `Cow` collection. +/// +/// Note, the standard library only has `FromIterator` for `Cow<'a, str>` and +/// `Cow<'a, [T]>`, because no one thought to add a blanket implementation +/// before it was stabilized. +impl<'a, C, T> FromParallelIterator for Cow<'a, C> +where + C: ToOwned> + ?Sized, + T: Send, +{ + fn from_par_iter(par_iter: I) -> Self + where + I: IntoParallelIterator, + { + Cow::Owned(C::Owned::from_par_iter(par_iter)) + } +} + +/// Collapses all unit items from a parallel iterator into one. +/// +/// This is more useful when combined with higher-level abstractions, like +/// collecting to a `Result<(), E>` where you only care about errors: +/// +/// ``` +/// use std::io::*; +/// use rayon::prelude::*; +/// +/// let data = vec![1, 2, 3, 4, 5]; +/// let res: Result<()> = data.par_iter() +/// .map(|x| writeln!(stdout(), "{}", x)) +/// .collect(); +/// assert!(res.is_ok()); +/// ``` +impl FromParallelIterator<()> for () { + fn from_par_iter(par_iter: I) -> Self + where + I: IntoParallelIterator, + { + par_iter.into_par_iter().drive_unindexed(NoopConsumer) + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/inspect.rs b/anneal/v2/vendor/rayon/src/iter/inspect.rs new file mode 100644 index 0000000000..3870e457f6 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/inspect.rs @@ -0,0 +1,253 @@ +use super::plumbing::*; +use super::*; + +use std::fmt::{self, Debug}; +use std::iter; + +/// `Inspect` is an iterator that calls a function with a reference to each +/// element before yielding it. +/// +/// This struct is created by the [`inspect()`] method on [`ParallelIterator`] +/// +/// [`inspect()`]: ParallelIterator::inspect() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Clone)] +pub struct Inspect { + base: I, + inspect_op: F, +} + +impl Debug for Inspect { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Inspect").field("base", &self.base).finish() + } +} + +impl Inspect { + /// Creates a new `Inspect` iterator. + pub(super) fn new(base: I, inspect_op: F) -> Self { + Inspect { base, inspect_op } + } +} + +impl ParallelIterator for Inspect +where + I: ParallelIterator, + F: Fn(&I::Item) + Sync + Send, +{ + type Item = I::Item; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let consumer1 = InspectConsumer::new(consumer, &self.inspect_op); + self.base.drive_unindexed(consumer1) + } + + fn opt_len(&self) -> Option { + self.base.opt_len() + } +} + +impl IndexedParallelIterator for Inspect +where + I: IndexedParallelIterator, + F: Fn(&I::Item) + Sync + Send, +{ + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + let consumer1 = InspectConsumer::new(consumer, &self.inspect_op); + self.base.drive(consumer1) + } + + fn len(&self) -> usize { + self.base.len() + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + return self.base.with_producer(Callback { + callback, + inspect_op: self.inspect_op, + }); + + struct Callback { + callback: CB, + inspect_op: F, + } + + impl ProducerCallback for Callback + where + CB: ProducerCallback, + F: Fn(&T) + Sync, + { + type Output = CB::Output; + + fn callback

(self, base: P) -> CB::Output + where + P: Producer, + { + let producer = InspectProducer { + base, + inspect_op: &self.inspect_op, + }; + self.callback.callback(producer) + } + } + } +} + +// //////////////////////////////////////////////////////////////////////// + +struct InspectProducer<'f, P, F> { + base: P, + inspect_op: &'f F, +} + +impl<'f, P, F> Producer for InspectProducer<'f, P, F> +where + P: Producer, + F: Fn(&P::Item) + Sync, +{ + type Item = P::Item; + type IntoIter = iter::Inspect; + + fn into_iter(self) -> Self::IntoIter { + self.base.into_iter().inspect(self.inspect_op) + } + + fn min_len(&self) -> usize { + self.base.min_len() + } + + fn max_len(&self) -> usize { + self.base.max_len() + } + + fn split_at(self, index: usize) -> (Self, Self) { + let (left, right) = self.base.split_at(index); + ( + InspectProducer { + base: left, + inspect_op: self.inspect_op, + }, + InspectProducer { + base: right, + inspect_op: self.inspect_op, + }, + ) + } + + fn fold_with(self, folder: G) -> G + where + G: Folder, + { + let folder1 = InspectFolder { + base: folder, + inspect_op: self.inspect_op, + }; + self.base.fold_with(folder1).base + } +} + +// //////////////////////////////////////////////////////////////////////// +// Consumer implementation + +struct InspectConsumer<'f, C, F> { + base: C, + inspect_op: &'f F, +} + +impl<'f, C, F> InspectConsumer<'f, C, F> { + fn new(base: C, inspect_op: &'f F) -> Self { + InspectConsumer { base, inspect_op } + } +} + +impl<'f, T, C, F> Consumer for InspectConsumer<'f, C, F> +where + C: Consumer, + F: Fn(&T) + Sync, +{ + type Folder = InspectFolder<'f, C::Folder, F>; + type Reducer = C::Reducer; + type Result = C::Result; + + fn split_at(self, index: usize) -> (Self, Self, Self::Reducer) { + let (left, right, reducer) = self.base.split_at(index); + ( + InspectConsumer::new(left, self.inspect_op), + InspectConsumer::new(right, self.inspect_op), + reducer, + ) + } + + fn into_folder(self) -> Self::Folder { + InspectFolder { + base: self.base.into_folder(), + inspect_op: self.inspect_op, + } + } + + fn full(&self) -> bool { + self.base.full() + } +} + +impl<'f, T, C, F> UnindexedConsumer for InspectConsumer<'f, C, F> +where + C: UnindexedConsumer, + F: Fn(&T) + Sync, +{ + fn split_off_left(&self) -> Self { + InspectConsumer::new(self.base.split_off_left(), self.inspect_op) + } + + fn to_reducer(&self) -> Self::Reducer { + self.base.to_reducer() + } +} + +struct InspectFolder<'f, C, F> { + base: C, + inspect_op: &'f F, +} + +impl<'f, T, C, F> Folder for InspectFolder<'f, C, F> +where + C: Folder, + F: Fn(&T), +{ + type Result = C::Result; + + fn consume(self, item: T) -> Self { + (self.inspect_op)(&item); + InspectFolder { + base: self.base.consume(item), + inspect_op: self.inspect_op, + } + } + + fn consume_iter(mut self, iter: I) -> Self + where + I: IntoIterator, + { + self.base = self + .base + .consume_iter(iter.into_iter().inspect(self.inspect_op)); + self + } + + fn complete(self) -> C::Result { + self.base.complete() + } + + fn full(&self) -> bool { + self.base.full() + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/interleave.rs b/anneal/v2/vendor/rayon/src/iter/interleave.rs new file mode 100644 index 0000000000..466003da28 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/interleave.rs @@ -0,0 +1,326 @@ +use super::plumbing::*; +use super::*; +use std::iter::Fuse; + +/// `Interleave` is an iterator that interleaves elements of iterators +/// `i` and `j` in one continuous iterator. This struct is created by +/// the [`interleave()`] method on [`IndexedParallelIterator`] +/// +/// [`interleave()`]: IndexedParallelIterator::interleave() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Debug, Clone)] +pub struct Interleave { + i: I, + j: J, +} + +impl Interleave { + /// Creates a new `Interleave` iterator + pub(super) fn new(i: I, j: J) -> Self { + Interleave { i, j } + } +} + +impl ParallelIterator for Interleave +where + I: IndexedParallelIterator, + J: IndexedParallelIterator, +{ + type Item = I::Item; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: Consumer, + { + bridge(self, consumer) + } + + fn opt_len(&self) -> Option { + Some(self.len()) + } +} + +impl IndexedParallelIterator for Interleave +where + I: IndexedParallelIterator, + J: IndexedParallelIterator, +{ + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + bridge(self, consumer) + } + + fn len(&self) -> usize { + self.i.len().checked_add(self.j.len()).expect("overflow") + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + let (i_len, j_len) = (self.i.len(), self.j.len()); + return self.i.with_producer(CallbackI { + callback, + i_len, + j_len, + i_next: false, + j: self.j, + }); + + struct CallbackI { + callback: CB, + i_len: usize, + j_len: usize, + i_next: bool, + j: J, + } + + impl ProducerCallback for CallbackI + where + J: IndexedParallelIterator, + CB: ProducerCallback, + { + type Output = CB::Output; + + fn callback(self, i_producer: I) -> Self::Output + where + I: Producer, + { + self.j.with_producer(CallbackJ { + i_producer, + i_len: self.i_len, + j_len: self.j_len, + i_next: self.i_next, + callback: self.callback, + }) + } + } + + struct CallbackJ { + callback: CB, + i_len: usize, + j_len: usize, + i_next: bool, + i_producer: I, + } + + impl ProducerCallback for CallbackJ + where + I: Producer, + CB: ProducerCallback, + { + type Output = CB::Output; + + fn callback(self, j_producer: J) -> Self::Output + where + J: Producer, + { + let producer = InterleaveProducer::new( + self.i_producer, + j_producer, + self.i_len, + self.j_len, + self.i_next, + ); + self.callback.callback(producer) + } + } + } +} + +struct InterleaveProducer +where + I: Producer, + J: Producer, +{ + i: I, + j: J, + i_len: usize, + j_len: usize, + i_next: bool, +} + +impl InterleaveProducer +where + I: Producer, + J: Producer, +{ + fn new(i: I, j: J, i_len: usize, j_len: usize, i_next: bool) -> InterleaveProducer { + InterleaveProducer { + i, + j, + i_len, + j_len, + i_next, + } + } +} + +impl Producer for InterleaveProducer +where + I: Producer, + J: Producer, +{ + type Item = I::Item; + type IntoIter = InterleaveSeq; + + fn into_iter(self) -> Self::IntoIter { + InterleaveSeq { + i: self.i.into_iter().fuse(), + j: self.j.into_iter().fuse(), + i_next: self.i_next, + } + } + + fn min_len(&self) -> usize { + Ord::max(self.i.min_len(), self.j.min_len()) + } + + fn max_len(&self) -> usize { + Ord::min(self.i.max_len(), self.j.max_len()) + } + + /// We know 0 < index <= self.i_len + self.j_len + /// + /// Find a, b satisfying: + /// + /// (1) 0 < a <= self.i_len + /// (2) 0 < b <= self.j_len + /// (3) a + b == index + /// + /// For even splits, set a = b = index/2. + /// For odd splits, set a = (index/2)+1, b = index/2, if `i` + /// should yield the next element, otherwise, if `j` should yield + /// the next element, set a = index/2 and b = (index/2)+1 + fn split_at(self, index: usize) -> (Self, Self) { + #[inline] + fn odd_offset(flag: bool) -> usize { + (!flag) as usize + } + + let even = index % 2 == 0; + let idx = index >> 1; + + // desired split + let (i_idx, j_idx) = ( + idx + odd_offset(even || self.i_next), + idx + odd_offset(even || !self.i_next), + ); + + let (i_split, j_split) = if self.i_len >= i_idx && self.j_len >= j_idx { + (i_idx, j_idx) + } else if self.i_len >= i_idx { + // j too short + (index - self.j_len, self.j_len) + } else { + // i too short + (self.i_len, index - self.i_len) + }; + + let trailing_i_next = even == self.i_next; + let (i_left, i_right) = self.i.split_at(i_split); + let (j_left, j_right) = self.j.split_at(j_split); + + ( + InterleaveProducer::new(i_left, j_left, i_split, j_split, self.i_next), + InterleaveProducer::new( + i_right, + j_right, + self.i_len - i_split, + self.j_len - j_split, + trailing_i_next, + ), + ) + } +} + +/// Wrapper for Interleave to implement DoubleEndedIterator and +/// ExactSizeIterator. +/// +/// This iterator is fused. +struct InterleaveSeq { + i: Fuse, + j: Fuse, + + /// Flag to control which iterator should provide the next element. When + /// `false` then `i` produces the next element, otherwise `j` produces the + /// next element. + i_next: bool, +} + +/// Iterator implementation for InterleaveSeq. This implementation is +/// taken more or less verbatim from itertools. It is replicated here +/// (instead of calling itertools directly), because we also need to +/// implement `DoubledEndedIterator` and `ExactSizeIterator`. +impl Iterator for InterleaveSeq +where + I: Iterator, + J: Iterator, +{ + type Item = I::Item; + + #[inline] + fn next(&mut self) -> Option { + self.i_next = !self.i_next; + if self.i_next { + match self.i.next() { + None => self.j.next(), + r => r, + } + } else { + match self.j.next() { + None => self.i.next(), + r => r, + } + } + } + + fn size_hint(&self) -> (usize, Option) { + let (ih, jh) = (self.i.size_hint(), self.j.size_hint()); + let min = ih.0.saturating_add(jh.0); + let max = match (ih.1, jh.1) { + (Some(x), Some(y)) => x.checked_add(y), + _ => None, + }; + (min, max) + } +} + +// The implementation for DoubleEndedIterator requires +// ExactSizeIterator to provide `next_back()`. The last element will +// come from the iterator that runs out last (ie has the most elements +// in it). If the iterators have the same number of elements, then the +// last iterator will provide the last element. +impl DoubleEndedIterator for InterleaveSeq +where + I: DoubleEndedIterator + ExactSizeIterator, + J: DoubleEndedIterator + ExactSizeIterator, +{ + #[inline] + fn next_back(&mut self) -> Option { + match self.i.len().cmp(&self.j.len()) { + Ordering::Less => self.j.next_back(), + Ordering::Equal => { + if self.i_next { + self.i.next_back() + } else { + self.j.next_back() + } + } + Ordering::Greater => self.i.next_back(), + } + } +} + +impl ExactSizeIterator for InterleaveSeq +where + I: ExactSizeIterator, + J: ExactSizeIterator, +{ + #[inline] + fn len(&self) -> usize { + self.i.len() + self.j.len() + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/interleave_shortest.rs b/anneal/v2/vendor/rayon/src/iter/interleave_shortest.rs new file mode 100644 index 0000000000..40dd142e58 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/interleave_shortest.rs @@ -0,0 +1,80 @@ +use super::plumbing::*; +use super::*; + +/// `InterleaveShortest` is an iterator that works similarly to +/// `Interleave`, but this version stops returning elements once one +/// of the iterators run out. +/// +/// This struct is created by the [`interleave_shortest()`] method on +/// [`IndexedParallelIterator`]. +/// +/// [`interleave_shortest()`]: IndexedParallelIterator::interleave_shortest() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Debug, Clone)] +pub struct InterleaveShortest { + interleave: Interleave, Take>, +} + +impl InterleaveShortest +where + I: IndexedParallelIterator, + J: IndexedParallelIterator, +{ + /// Creates a new `InterleaveShortest` iterator + pub(super) fn new(i: I, j: J) -> Self { + InterleaveShortest { + interleave: if i.len() <= j.len() { + // take equal lengths from both iterators + let n = i.len(); + i.take(n).interleave(j.take(n)) + } else { + // take one extra item from the first iterator + let n = j.len(); + i.take(n + 1).interleave(j.take(n)) + }, + } + } +} + +impl ParallelIterator for InterleaveShortest +where + I: IndexedParallelIterator, + J: IndexedParallelIterator, +{ + type Item = I::Item; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: Consumer, + { + bridge(self, consumer) + } + + fn opt_len(&self) -> Option { + Some(self.len()) + } +} + +impl IndexedParallelIterator for InterleaveShortest +where + I: IndexedParallelIterator, + J: IndexedParallelIterator, +{ + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + bridge(self, consumer) + } + + fn len(&self) -> usize { + self.interleave.len() + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + self.interleave.with_producer(callback) + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/intersperse.rs b/anneal/v2/vendor/rayon/src/iter/intersperse.rs new file mode 100644 index 0000000000..84fdcd4999 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/intersperse.rs @@ -0,0 +1,401 @@ +use super::plumbing::*; +use super::*; +use std::cell::Cell; +use std::iter::{self, Fuse}; + +/// `Intersperse` is an iterator that inserts a particular item between each +/// item of the adapted iterator. This struct is created by the +/// [`intersperse()`] method on [`ParallelIterator`] +/// +/// [`intersperse()`]: ParallelIterator::intersperse() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Clone, Debug)] +pub struct Intersperse +where + I: ParallelIterator, +{ + base: I, + item: I::Item, +} + +impl Intersperse +where + I: ParallelIterator, +{ + /// Creates a new `Intersperse` iterator + pub(super) fn new(base: I, item: I::Item) -> Self { + Intersperse { base, item } + } +} + +impl ParallelIterator for Intersperse +where + I: ParallelIterator, +{ + type Item = I::Item; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let consumer1 = IntersperseConsumer::new(consumer, self.item); + self.base.drive_unindexed(consumer1) + } + + fn opt_len(&self) -> Option { + match self.base.opt_len()? { + 0 => Some(0), + len => len.checked_add(len - 1), + } + } +} + +impl IndexedParallelIterator for Intersperse +where + I: IndexedParallelIterator, +{ + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + let consumer1 = IntersperseConsumer::new(consumer, self.item); + self.base.drive(consumer1) + } + + fn len(&self) -> usize { + let len = self.base.len(); + if len > 0 { + len.checked_add(len - 1).expect("overflow") + } else { + 0 + } + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + let len = self.len(); + return self.base.with_producer(Callback { + callback, + item: self.item, + len, + }); + + struct Callback { + callback: CB, + item: T, + len: usize, + } + + impl ProducerCallback for Callback + where + CB: ProducerCallback, + T: Clone + Send, + { + type Output = CB::Output; + + fn callback

(self, base: P) -> CB::Output + where + P: Producer, + { + let producer = IntersperseProducer::new(base, self.item, self.len); + self.callback.callback(producer) + } + } + } +} + +struct IntersperseProducer

+where + P: Producer, +{ + base: P, + item: P::Item, + len: usize, + clone_first: bool, +} + +impl

IntersperseProducer

+where + P: Producer, +{ + fn new(base: P, item: P::Item, len: usize) -> Self { + IntersperseProducer { + base, + item, + len, + clone_first: false, + } + } +} + +impl

Producer for IntersperseProducer

+where + P: Producer, +{ + type Item = P::Item; + type IntoIter = IntersperseIter; + + fn into_iter(self) -> Self::IntoIter { + IntersperseIter { + base: self.base.into_iter().fuse(), + item: self.item, + clone_first: self.len > 0 && self.clone_first, + + // If there's more than one item, then even lengths end the opposite + // of how they started with respect to interspersed clones. + clone_last: self.len > 1 && ((self.len & 1 == 0) ^ self.clone_first), + } + } + + fn min_len(&self) -> usize { + self.base.min_len() + } + fn max_len(&self) -> usize { + self.base.max_len() + } + + fn split_at(self, index: usize) -> (Self, Self) { + debug_assert!(index <= self.len); + + // The left needs half of the items from the base producer, and the + // other half will be our interspersed item. If we're not leading with + // a cloned item, then we need to round up the base number of items, + // otherwise round down. + let base_index = (index + !self.clone_first as usize) / 2; + let (left_base, right_base) = self.base.split_at(base_index); + + let left = IntersperseProducer { + base: left_base, + item: self.item.clone(), + len: index, + clone_first: self.clone_first, + }; + + let right = IntersperseProducer { + base: right_base, + item: self.item, + len: self.len - index, + + // If the index is odd, the right side toggles `clone_first`. + clone_first: (index & 1 == 1) ^ self.clone_first, + }; + + (left, right) + } + + fn fold_with(self, folder: F) -> F + where + F: Folder, + { + let folder1 = IntersperseFolder { + base: folder, + item: self.item, + clone_first: self.clone_first, + }; + self.base.fold_with(folder1).base + } +} + +struct IntersperseIter +where + I: Iterator, +{ + base: Fuse, + item: I::Item, + clone_first: bool, + clone_last: bool, +} + +impl Iterator for IntersperseIter +where + I: DoubleEndedIterator + ExactSizeIterator, +{ + type Item = I::Item; + + fn next(&mut self) -> Option { + if self.clone_first { + self.clone_first = false; + Some(self.item.clone()) + } else if let next @ Some(_) = self.base.next() { + // If there are any items left, we'll need another clone in front. + self.clone_first = self.base.len() != 0; + next + } else if self.clone_last { + self.clone_last = false; + Some(self.item.clone()) + } else { + None + } + } + + fn size_hint(&self) -> (usize, Option) { + let len = self.len(); + (len, Some(len)) + } +} + +impl DoubleEndedIterator for IntersperseIter +where + I: DoubleEndedIterator + ExactSizeIterator, +{ + fn next_back(&mut self) -> Option { + if self.clone_last { + self.clone_last = false; + Some(self.item.clone()) + } else if let next_back @ Some(_) = self.base.next_back() { + // If there are any items left, we'll need another clone in back. + self.clone_last = self.base.len() != 0; + next_back + } else if self.clone_first { + self.clone_first = false; + Some(self.item.clone()) + } else { + None + } + } +} + +impl ExactSizeIterator for IntersperseIter +where + I: DoubleEndedIterator + ExactSizeIterator, +{ + fn len(&self) -> usize { + let len = self.base.len(); + len + len.saturating_sub(1) + self.clone_first as usize + self.clone_last as usize + } +} + +struct IntersperseConsumer { + base: C, + item: T, + clone_first: Cell, +} + +impl IntersperseConsumer +where + C: Consumer, +{ + fn new(base: C, item: T) -> Self { + IntersperseConsumer { + base, + item, + clone_first: false.into(), + } + } +} + +impl Consumer for IntersperseConsumer +where + C: Consumer, + T: Clone + Send, +{ + type Folder = IntersperseFolder; + type Reducer = C::Reducer; + type Result = C::Result; + + fn split_at(mut self, index: usize) -> (Self, Self, Self::Reducer) { + // We'll feed twice as many items to the base consumer, except if we're + // not currently leading with a cloned item, then it's one less. + let base_index = index + index.saturating_sub(!self.clone_first.get() as usize); + let (left, right, reducer) = self.base.split_at(base_index); + + let right = IntersperseConsumer { + base: right, + item: self.item.clone(), + clone_first: true.into(), + }; + self.base = left; + (self, right, reducer) + } + + fn into_folder(self) -> Self::Folder { + IntersperseFolder { + base: self.base.into_folder(), + item: self.item, + clone_first: self.clone_first.get(), + } + } + + fn full(&self) -> bool { + self.base.full() + } +} + +impl UnindexedConsumer for IntersperseConsumer +where + C: UnindexedConsumer, + T: Clone + Send, +{ + fn split_off_left(&self) -> Self { + let left = IntersperseConsumer { + base: self.base.split_off_left(), + item: self.item.clone(), + clone_first: self.clone_first.clone(), + }; + self.clone_first.set(true); + left + } + + fn to_reducer(&self) -> Self::Reducer { + self.base.to_reducer() + } +} + +struct IntersperseFolder { + base: C, + item: T, + clone_first: bool, +} + +impl Folder for IntersperseFolder +where + C: Folder, + T: Clone, +{ + type Result = C::Result; + + fn consume(mut self, item: T) -> Self { + if self.clone_first { + self.base = self.base.consume(self.item.clone()); + if self.base.full() { + return self; + } + } else { + self.clone_first = true; + } + self.base = self.base.consume(item); + self + } + + fn consume_iter(self, iter: I) -> Self + where + I: IntoIterator, + { + let mut clone_first = self.clone_first; + let between_item = self.item; + let base = self.base.consume_iter(iter.into_iter().flat_map(|item| { + let first = if clone_first { + Some(between_item.clone()) + } else { + clone_first = true; + None + }; + first.into_iter().chain(iter::once(item)) + })); + IntersperseFolder { + base, + item: between_item, + clone_first, + } + } + + fn complete(self) -> C::Result { + self.base.complete() + } + + fn full(&self) -> bool { + self.base.full() + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/len.rs b/anneal/v2/vendor/rayon/src/iter/len.rs new file mode 100644 index 0000000000..2cea2e7bb8 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/len.rs @@ -0,0 +1,262 @@ +use super::plumbing::*; +use super::*; + +/// `MinLen` is an iterator that imposes a minimum length on iterator splits. +/// This struct is created by the [`with_min_len()`] method on [`IndexedParallelIterator`] +/// +/// [`with_min_len()`]: IndexedParallelIterator::with_min_len() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Debug, Clone)] +pub struct MinLen { + base: I, + min: usize, +} + +impl MinLen { + /// Creates a new `MinLen` iterator. + pub(super) fn new(base: I, min: usize) -> Self { + MinLen { base, min } + } +} + +impl ParallelIterator for MinLen +where + I: IndexedParallelIterator, +{ + type Item = I::Item; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + bridge(self, consumer) + } + + fn opt_len(&self) -> Option { + Some(self.len()) + } +} + +impl IndexedParallelIterator for MinLen +where + I: IndexedParallelIterator, +{ + fn drive>(self, consumer: C) -> C::Result { + bridge(self, consumer) + } + + fn len(&self) -> usize { + self.base.len() + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + return self.base.with_producer(Callback { + callback, + min: self.min, + }); + + struct Callback { + callback: CB, + min: usize, + } + + impl ProducerCallback for Callback + where + CB: ProducerCallback, + { + type Output = CB::Output; + fn callback

(self, base: P) -> CB::Output + where + P: Producer, + { + let producer = MinLenProducer { + base, + min: self.min, + }; + self.callback.callback(producer) + } + } + } +} + +// //////////////////////////////////////////////////////////////////////// +// `MinLenProducer` implementation + +struct MinLenProducer

{ + base: P, + min: usize, +} + +impl

Producer for MinLenProducer

+where + P: Producer, +{ + type Item = P::Item; + type IntoIter = P::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.base.into_iter() + } + + fn min_len(&self) -> usize { + Ord::max(self.min, self.base.min_len()) + } + + fn max_len(&self) -> usize { + self.base.max_len() + } + + fn split_at(self, index: usize) -> (Self, Self) { + let (left, right) = self.base.split_at(index); + ( + MinLenProducer { + base: left, + min: self.min, + }, + MinLenProducer { + base: right, + min: self.min, + }, + ) + } + + fn fold_with(self, folder: F) -> F + where + F: Folder, + { + self.base.fold_with(folder) + } +} + +/// `MaxLen` is an iterator that imposes a maximum length on iterator splits. +/// This struct is created by the [`with_max_len()`] method on [`IndexedParallelIterator`] +/// +/// [`with_max_len()`]: IndexedParallelIterator::with_max_len() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Debug, Clone)] +pub struct MaxLen { + base: I, + max: usize, +} + +impl MaxLen { + /// Creates a new `MaxLen` iterator. + pub(super) fn new(base: I, max: usize) -> Self { + MaxLen { base, max } + } +} + +impl ParallelIterator for MaxLen +where + I: IndexedParallelIterator, +{ + type Item = I::Item; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + bridge(self, consumer) + } + + fn opt_len(&self) -> Option { + Some(self.len()) + } +} + +impl IndexedParallelIterator for MaxLen +where + I: IndexedParallelIterator, +{ + fn drive>(self, consumer: C) -> C::Result { + bridge(self, consumer) + } + + fn len(&self) -> usize { + self.base.len() + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + return self.base.with_producer(Callback { + callback, + max: self.max, + }); + + struct Callback { + callback: CB, + max: usize, + } + + impl ProducerCallback for Callback + where + CB: ProducerCallback, + { + type Output = CB::Output; + fn callback

(self, base: P) -> CB::Output + where + P: Producer, + { + let producer = MaxLenProducer { + base, + max: self.max, + }; + self.callback.callback(producer) + } + } + } +} + +// //////////////////////////////////////////////////////////////////////// +// `MaxLenProducer` implementation + +struct MaxLenProducer

{ + base: P, + max: usize, +} + +impl

Producer for MaxLenProducer

+where + P: Producer, +{ + type Item = P::Item; + type IntoIter = P::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.base.into_iter() + } + + fn min_len(&self) -> usize { + self.base.min_len() + } + + fn max_len(&self) -> usize { + Ord::min(self.max, self.base.max_len()) + } + + fn split_at(self, index: usize) -> (Self, Self) { + let (left, right) = self.base.split_at(index); + ( + MaxLenProducer { + base: left, + max: self.max, + }, + MaxLenProducer { + base: right, + max: self.max, + }, + ) + } + + fn fold_with(self, folder: F) -> F + where + F: Folder, + { + self.base.fold_with(folder) + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/map.rs b/anneal/v2/vendor/rayon/src/iter/map.rs new file mode 100644 index 0000000000..ea5bc1f652 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/map.rs @@ -0,0 +1,255 @@ +use super::plumbing::*; +use super::*; + +use std::fmt::{self, Debug}; +use std::iter; + +/// `Map` is an iterator that transforms the elements of an underlying iterator. +/// +/// This struct is created by the [`map()`] method on [`ParallelIterator`] +/// +/// [`map()`]: ParallelIterator::map() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Clone)] +pub struct Map { + base: I, + map_op: F, +} + +impl Debug for Map { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Map").field("base", &self.base).finish() + } +} + +impl Map { + /// Creates a new `Map` iterator. + pub(super) fn new(base: I, map_op: F) -> Self { + Map { base, map_op } + } +} + +impl ParallelIterator for Map +where + I: ParallelIterator, + F: Fn(I::Item) -> R + Sync + Send, + R: Send, +{ + type Item = F::Output; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let consumer1 = MapConsumer::new(consumer, &self.map_op); + self.base.drive_unindexed(consumer1) + } + + fn opt_len(&self) -> Option { + self.base.opt_len() + } +} + +impl IndexedParallelIterator for Map +where + I: IndexedParallelIterator, + F: Fn(I::Item) -> R + Sync + Send, + R: Send, +{ + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + let consumer1 = MapConsumer::new(consumer, &self.map_op); + self.base.drive(consumer1) + } + + fn len(&self) -> usize { + self.base.len() + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + return self.base.with_producer(Callback { + callback, + map_op: self.map_op, + }); + + struct Callback { + callback: CB, + map_op: F, + } + + impl ProducerCallback for Callback + where + CB: ProducerCallback, + F: Fn(T) -> R + Sync, + R: Send, + { + type Output = CB::Output; + + fn callback

(self, base: P) -> CB::Output + where + P: Producer, + { + let producer = MapProducer { + base, + map_op: &self.map_op, + }; + self.callback.callback(producer) + } + } + } +} + +// //////////////////////////////////////////////////////////////////////// + +struct MapProducer<'f, P, F> { + base: P, + map_op: &'f F, +} + +impl<'f, P, F, R> Producer for MapProducer<'f, P, F> +where + P: Producer, + F: Fn(P::Item) -> R + Sync, + R: Send, +{ + type Item = F::Output; + type IntoIter = iter::Map; + + fn into_iter(self) -> Self::IntoIter { + self.base.into_iter().map(self.map_op) + } + + fn min_len(&self) -> usize { + self.base.min_len() + } + fn max_len(&self) -> usize { + self.base.max_len() + } + + fn split_at(self, index: usize) -> (Self, Self) { + let (left, right) = self.base.split_at(index); + ( + MapProducer { + base: left, + map_op: self.map_op, + }, + MapProducer { + base: right, + map_op: self.map_op, + }, + ) + } + + fn fold_with(self, folder: G) -> G + where + G: Folder, + { + let folder1 = MapFolder { + base: folder, + map_op: self.map_op, + }; + self.base.fold_with(folder1).base + } +} + +// //////////////////////////////////////////////////////////////////////// +// Consumer implementation + +struct MapConsumer<'f, C, F> { + base: C, + map_op: &'f F, +} + +impl<'f, C, F> MapConsumer<'f, C, F> { + fn new(base: C, map_op: &'f F) -> Self { + MapConsumer { base, map_op } + } +} + +impl<'f, T, R, C, F> Consumer for MapConsumer<'f, C, F> +where + C: Consumer, + F: Fn(T) -> R + Sync, + R: Send, +{ + type Folder = MapFolder<'f, C::Folder, F>; + type Reducer = C::Reducer; + type Result = C::Result; + + fn split_at(self, index: usize) -> (Self, Self, Self::Reducer) { + let (left, right, reducer) = self.base.split_at(index); + ( + MapConsumer::new(left, self.map_op), + MapConsumer::new(right, self.map_op), + reducer, + ) + } + + fn into_folder(self) -> Self::Folder { + MapFolder { + base: self.base.into_folder(), + map_op: self.map_op, + } + } + + fn full(&self) -> bool { + self.base.full() + } +} + +impl<'f, T, R, C, F> UnindexedConsumer for MapConsumer<'f, C, F> +where + C: UnindexedConsumer, + F: Fn(T) -> R + Sync, + R: Send, +{ + fn split_off_left(&self) -> Self { + MapConsumer::new(self.base.split_off_left(), self.map_op) + } + + fn to_reducer(&self) -> Self::Reducer { + self.base.to_reducer() + } +} + +struct MapFolder<'f, C, F> { + base: C, + map_op: &'f F, +} + +impl<'f, T, R, C, F> Folder for MapFolder<'f, C, F> +where + C: Folder, + F: Fn(T) -> R, +{ + type Result = C::Result; + + fn consume(self, item: T) -> Self { + let mapped_item = (self.map_op)(item); + MapFolder { + base: self.base.consume(mapped_item), + map_op: self.map_op, + } + } + + fn consume_iter(mut self, iter: I) -> Self + where + I: IntoIterator, + { + self.base = self.base.consume_iter(iter.into_iter().map(self.map_op)); + self + } + + fn complete(self) -> C::Result { + self.base.complete() + } + + fn full(&self) -> bool { + self.base.full() + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/map_with.rs b/anneal/v2/vendor/rayon/src/iter/map_with.rs new file mode 100644 index 0000000000..9aef862a85 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/map_with.rs @@ -0,0 +1,565 @@ +use super::plumbing::*; +use super::*; + +use std::fmt::{self, Debug}; + +/// `MapWith` is an iterator that transforms the elements of an underlying iterator. +/// +/// This struct is created by the [`map_with()`] method on [`ParallelIterator`] +/// +/// [`map_with()`]: ParallelIterator::map_with() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Clone)] +pub struct MapWith { + base: I, + item: T, + map_op: F, +} + +impl Debug for MapWith { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("MapWith") + .field("base", &self.base) + .field("item", &self.item) + .finish() + } +} + +impl MapWith { + /// Creates a new `MapWith` iterator. + pub(super) fn new(base: I, item: T, map_op: F) -> Self { + MapWith { base, item, map_op } + } +} + +impl ParallelIterator for MapWith +where + I: ParallelIterator, + T: Send + Clone, + F: Fn(&mut T, I::Item) -> R + Sync + Send, + R: Send, +{ + type Item = R; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let consumer1 = MapWithConsumer::new(consumer, self.item, &self.map_op); + self.base.drive_unindexed(consumer1) + } + + fn opt_len(&self) -> Option { + self.base.opt_len() + } +} + +impl IndexedParallelIterator for MapWith +where + I: IndexedParallelIterator, + T: Send + Clone, + F: Fn(&mut T, I::Item) -> R + Sync + Send, + R: Send, +{ + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + let consumer1 = MapWithConsumer::new(consumer, self.item, &self.map_op); + self.base.drive(consumer1) + } + + fn len(&self) -> usize { + self.base.len() + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + return self.base.with_producer(Callback { + callback, + item: self.item, + map_op: self.map_op, + }); + + struct Callback { + callback: CB, + item: U, + map_op: F, + } + + impl ProducerCallback for Callback + where + CB: ProducerCallback, + U: Send + Clone, + F: Fn(&mut U, T) -> R + Sync, + R: Send, + { + type Output = CB::Output; + + fn callback

(self, base: P) -> CB::Output + where + P: Producer, + { + let producer = MapWithProducer { + base, + item: self.item, + map_op: &self.map_op, + }; + self.callback.callback(producer) + } + } + } +} + +// //////////////////////////////////////////////////////////////////////// + +struct MapWithProducer<'f, P, U, F> { + base: P, + item: U, + map_op: &'f F, +} + +impl<'f, P, U, F, R> Producer for MapWithProducer<'f, P, U, F> +where + P: Producer, + U: Send + Clone, + F: Fn(&mut U, P::Item) -> R + Sync, + R: Send, +{ + type Item = R; + type IntoIter = MapWithIter<'f, P::IntoIter, U, F>; + + fn into_iter(self) -> Self::IntoIter { + MapWithIter { + base: self.base.into_iter(), + item: self.item, + map_op: self.map_op, + } + } + + fn min_len(&self) -> usize { + self.base.min_len() + } + fn max_len(&self) -> usize { + self.base.max_len() + } + + fn split_at(self, index: usize) -> (Self, Self) { + let (left, right) = self.base.split_at(index); + ( + MapWithProducer { + base: left, + item: self.item.clone(), + map_op: self.map_op, + }, + MapWithProducer { + base: right, + item: self.item, + map_op: self.map_op, + }, + ) + } + + fn fold_with(self, folder: G) -> G + where + G: Folder, + { + let folder1 = MapWithFolder { + base: folder, + item: self.item, + map_op: self.map_op, + }; + self.base.fold_with(folder1).base + } +} + +struct MapWithIter<'f, I, U, F> { + base: I, + item: U, + map_op: &'f F, +} + +impl<'f, I, U, F, R> Iterator for MapWithIter<'f, I, U, F> +where + I: Iterator, + F: Fn(&mut U, I::Item) -> R + Sync, + R: Send, +{ + type Item = R; + + fn next(&mut self) -> Option { + let item = self.base.next()?; + Some((self.map_op)(&mut self.item, item)) + } + + fn size_hint(&self) -> (usize, Option) { + self.base.size_hint() + } +} + +impl<'f, I, U, F, R> DoubleEndedIterator for MapWithIter<'f, I, U, F> +where + I: DoubleEndedIterator, + F: Fn(&mut U, I::Item) -> R + Sync, + R: Send, +{ + fn next_back(&mut self) -> Option { + let item = self.base.next_back()?; + Some((self.map_op)(&mut self.item, item)) + } +} + +impl<'f, I, U, F, R> ExactSizeIterator for MapWithIter<'f, I, U, F> +where + I: ExactSizeIterator, + F: Fn(&mut U, I::Item) -> R + Sync, + R: Send, +{ +} + +// //////////////////////////////////////////////////////////////////////// +// Consumer implementation + +struct MapWithConsumer<'f, C, U, F> { + base: C, + item: U, + map_op: &'f F, +} + +impl<'f, C, U, F> MapWithConsumer<'f, C, U, F> { + fn new(base: C, item: U, map_op: &'f F) -> Self { + MapWithConsumer { base, item, map_op } + } +} + +impl<'f, T, U, R, C, F> Consumer for MapWithConsumer<'f, C, U, F> +where + C: Consumer, + U: Send + Clone, + F: Fn(&mut U, T) -> R + Sync, + R: Send, +{ + type Folder = MapWithFolder<'f, C::Folder, U, F>; + type Reducer = C::Reducer; + type Result = C::Result; + + fn split_at(self, index: usize) -> (Self, Self, Self::Reducer) { + let (left, right, reducer) = self.base.split_at(index); + ( + MapWithConsumer::new(left, self.item.clone(), self.map_op), + MapWithConsumer::new(right, self.item, self.map_op), + reducer, + ) + } + + fn into_folder(self) -> Self::Folder { + MapWithFolder { + base: self.base.into_folder(), + item: self.item, + map_op: self.map_op, + } + } + + fn full(&self) -> bool { + self.base.full() + } +} + +impl<'f, T, U, R, C, F> UnindexedConsumer for MapWithConsumer<'f, C, U, F> +where + C: UnindexedConsumer, + U: Send + Clone, + F: Fn(&mut U, T) -> R + Sync, + R: Send, +{ + fn split_off_left(&self) -> Self { + MapWithConsumer::new(self.base.split_off_left(), self.item.clone(), self.map_op) + } + + fn to_reducer(&self) -> Self::Reducer { + self.base.to_reducer() + } +} + +struct MapWithFolder<'f, C, U, F> { + base: C, + item: U, + map_op: &'f F, +} + +impl<'f, T, U, R, C, F> Folder for MapWithFolder<'f, C, U, F> +where + C: Folder, + F: Fn(&mut U, T) -> R, +{ + type Result = C::Result; + + fn consume(mut self, item: T) -> Self { + let mapped_item = (self.map_op)(&mut self.item, item); + self.base = self.base.consume(mapped_item); + self + } + + fn consume_iter(mut self, iter: I) -> Self + where + I: IntoIterator, + { + fn with<'f, T, U, R>( + item: &'f mut U, + map_op: impl Fn(&mut U, T) -> R + 'f, + ) -> impl FnMut(T) -> R + 'f { + move |x| map_op(item, x) + } + + { + let mapped_iter = iter.into_iter().map(with(&mut self.item, self.map_op)); + self.base = self.base.consume_iter(mapped_iter); + } + self + } + + fn complete(self) -> C::Result { + self.base.complete() + } + + fn full(&self) -> bool { + self.base.full() + } +} + +// ------------------------------------------------------------------------------------------------ + +/// `MapInit` is an iterator that transforms the elements of an underlying iterator. +/// +/// This struct is created by the [`map_init()`] method on [`ParallelIterator`] +/// +/// [`map_init()`]: ParallelIterator::map_init() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Clone)] +pub struct MapInit { + base: I, + init: INIT, + map_op: F, +} + +impl Debug for MapInit { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("MapInit").field("base", &self.base).finish() + } +} + +impl MapInit { + /// Creates a new `MapInit` iterator. + pub(super) fn new(base: I, init: INIT, map_op: F) -> Self { + MapInit { base, init, map_op } + } +} + +impl ParallelIterator for MapInit +where + I: ParallelIterator, + INIT: Fn() -> T + Sync + Send, + F: Fn(&mut T, I::Item) -> R + Sync + Send, + R: Send, +{ + type Item = R; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let consumer1 = MapInitConsumer::new(consumer, &self.init, &self.map_op); + self.base.drive_unindexed(consumer1) + } + + fn opt_len(&self) -> Option { + self.base.opt_len() + } +} + +impl IndexedParallelIterator for MapInit +where + I: IndexedParallelIterator, + INIT: Fn() -> T + Sync + Send, + F: Fn(&mut T, I::Item) -> R + Sync + Send, + R: Send, +{ + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + let consumer1 = MapInitConsumer::new(consumer, &self.init, &self.map_op); + self.base.drive(consumer1) + } + + fn len(&self) -> usize { + self.base.len() + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + return self.base.with_producer(Callback { + callback, + init: self.init, + map_op: self.map_op, + }); + + struct Callback { + callback: CB, + init: INIT, + map_op: F, + } + + impl ProducerCallback for Callback + where + CB: ProducerCallback, + INIT: Fn() -> U + Sync, + F: Fn(&mut U, T) -> R + Sync, + R: Send, + { + type Output = CB::Output; + + fn callback

(self, base: P) -> CB::Output + where + P: Producer, + { + let producer = MapInitProducer { + base, + init: &self.init, + map_op: &self.map_op, + }; + self.callback.callback(producer) + } + } + } +} + +// //////////////////////////////////////////////////////////////////////// + +struct MapInitProducer<'f, P, INIT, F> { + base: P, + init: &'f INIT, + map_op: &'f F, +} + +impl<'f, P, INIT, U, F, R> Producer for MapInitProducer<'f, P, INIT, F> +where + P: Producer, + INIT: Fn() -> U + Sync, + F: Fn(&mut U, P::Item) -> R + Sync, + R: Send, +{ + type Item = R; + type IntoIter = MapWithIter<'f, P::IntoIter, U, F>; + + fn into_iter(self) -> Self::IntoIter { + MapWithIter { + base: self.base.into_iter(), + item: (self.init)(), + map_op: self.map_op, + } + } + + fn min_len(&self) -> usize { + self.base.min_len() + } + fn max_len(&self) -> usize { + self.base.max_len() + } + + fn split_at(self, index: usize) -> (Self, Self) { + let (left, right) = self.base.split_at(index); + ( + MapInitProducer { + base: left, + init: self.init, + map_op: self.map_op, + }, + MapInitProducer { + base: right, + init: self.init, + map_op: self.map_op, + }, + ) + } + + fn fold_with(self, folder: G) -> G + where + G: Folder, + { + let folder1 = MapWithFolder { + base: folder, + item: (self.init)(), + map_op: self.map_op, + }; + self.base.fold_with(folder1).base + } +} + +// //////////////////////////////////////////////////////////////////////// +// Consumer implementation + +struct MapInitConsumer<'f, C, INIT, F> { + base: C, + init: &'f INIT, + map_op: &'f F, +} + +impl<'f, C, INIT, F> MapInitConsumer<'f, C, INIT, F> { + fn new(base: C, init: &'f INIT, map_op: &'f F) -> Self { + MapInitConsumer { base, init, map_op } + } +} + +impl<'f, T, INIT, U, R, C, F> Consumer for MapInitConsumer<'f, C, INIT, F> +where + C: Consumer, + INIT: Fn() -> U + Sync, + F: Fn(&mut U, T) -> R + Sync, + R: Send, +{ + type Folder = MapWithFolder<'f, C::Folder, U, F>; + type Reducer = C::Reducer; + type Result = C::Result; + + fn split_at(self, index: usize) -> (Self, Self, Self::Reducer) { + let (left, right, reducer) = self.base.split_at(index); + ( + MapInitConsumer::new(left, self.init, self.map_op), + MapInitConsumer::new(right, self.init, self.map_op), + reducer, + ) + } + + fn into_folder(self) -> Self::Folder { + MapWithFolder { + base: self.base.into_folder(), + item: (self.init)(), + map_op: self.map_op, + } + } + + fn full(&self) -> bool { + self.base.full() + } +} + +impl<'f, T, INIT, U, R, C, F> UnindexedConsumer for MapInitConsumer<'f, C, INIT, F> +where + C: UnindexedConsumer, + INIT: Fn() -> U + Sync, + F: Fn(&mut U, T) -> R + Sync, + R: Send, +{ + fn split_off_left(&self) -> Self { + MapInitConsumer::new(self.base.split_off_left(), self.init, self.map_op) + } + + fn to_reducer(&self) -> Self::Reducer { + self.base.to_reducer() + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/mod.rs b/anneal/v2/vendor/rayon/src/iter/mod.rs new file mode 100644 index 0000000000..016b2c92a5 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/mod.rs @@ -0,0 +1,3628 @@ +//! Traits for writing parallel programs using an iterator-style interface +//! +//! You will rarely need to interact with this module directly unless you have +//! need to name one of the iterator types. +//! +//! Parallel iterators make it easy to write iterator-like chains that +//! execute in parallel: typically all you have to do is convert the +//! first `.iter()` (or `iter_mut()`, `into_iter()`, etc) method into +//! `par_iter()` (or `par_iter_mut()`, `into_par_iter()`, etc). For +//! example, to compute the sum of the squares of a sequence of +//! integers, one might write: +//! +//! ```rust +//! use rayon::prelude::*; +//! fn sum_of_squares(input: &[i32]) -> i32 { +//! input.par_iter() +//! .map(|i| i * i) +//! .sum() +//! } +//! ``` +//! +//! Or, to increment all the integers in a slice, you could write: +//! +//! ```rust +//! use rayon::prelude::*; +//! fn increment_all(input: &mut [i32]) { +//! input.par_iter_mut() +//! .for_each(|p| *p += 1); +//! } +//! ``` +//! +//! To use parallel iterators, first import the traits by adding +//! something like `use rayon::prelude::*` to your module. You can +//! then call `par_iter`, `par_iter_mut`, or `into_par_iter` to get a +//! parallel iterator. Like a [regular iterator][], parallel +//! iterators work by first constructing a computation and then +//! executing it. +//! +//! In addition to `par_iter()` and friends, some types offer other +//! ways to create (or consume) parallel iterators: +//! +//! - Slices (`&[T]`, `&mut [T]`) offer methods like `par_split` and +//! `par_windows`, as well as various parallel sorting +//! operations. See [the `ParallelSlice` trait] for the full list. +//! - Strings (`&str`) offer methods like `par_split` and `par_lines`. +//! See [the `ParallelString` trait] for the full list. +//! - Various collections offer [`par_extend`], which grows a +//! collection given a parallel iterator. (If you don't have a +//! collection to extend, you can use [`collect()`] to create a new +//! one from scratch.) +//! +//! [the `ParallelSlice` trait]: crate::slice::ParallelSlice +//! [the `ParallelString` trait]: crate::str::ParallelString +//! [`par_extend`]: ParallelExtend +//! [`collect()`]: ParallelIterator::collect() +//! +//! To see the full range of methods available on parallel iterators, +//! check out the [`ParallelIterator`] and [`IndexedParallelIterator`] +//! traits. +//! +//! If you'd like to build a custom parallel iterator, or to write your own +//! combinator, then check out the [split] function and the [plumbing] module. +//! +//! [regular iterator]: Iterator +//! [split]: split() +//! [plumbing]: plumbing +//! +//! Note: Several of the `ParallelIterator` methods rely on a `Try` trait which +//! has been deliberately obscured from the public API. This trait is intended +//! to mirror the unstable `std::ops::Try` with implementations for `Option` and +//! `Result`, where `Some`/`Ok` values will let those iterators continue, but +//! `None`/`Err` values will exit early. +//! +//! A note about +//! [dyn compatiblity](https://doc.rust-lang.org/reference/items/traits.html#dyn-compatibility): +//! It is currently _not_ possible to wrap a `ParallelIterator` (or any trait +//! that depends on it) using a `Box` or other kind of +//! dynamic allocation, because `ParallelIterator` is **not dyn-compatible**. +//! (This keeps the implementation simpler and allows extra optimizations.) + +use self::plumbing::*; +use self::private::Try; +pub use either::Either; +use std::cmp::Ordering; +use std::collections::LinkedList; +use std::iter::{Product, Sum}; +use std::ops::{Fn, RangeBounds}; + +pub mod plumbing; + +#[cfg(test)] +mod test; + +// There is a method to the madness here: +// +// - These modules are private but expose certain types to the end-user +// (e.g., `enumerate::Enumerate`) -- specifically, the types that appear in the +// public API surface of the `ParallelIterator` traits. +// - In **this** module, those public types are always used unprefixed, which forces +// us to add a `pub use` and helps identify if we missed anything. +// - In contrast, items that appear **only** in the body of a method, +// e.g. `find::find()`, are always used **prefixed**, so that they +// can be readily distinguished. + +mod blocks; +mod chain; +mod chunks; +mod cloned; +mod collect; +mod copied; +mod empty; +mod enumerate; +mod extend; +mod filter; +mod filter_map; +mod find; +mod find_first_last; +mod flat_map; +mod flat_map_iter; +mod flatten; +mod flatten_iter; +mod fold; +mod fold_chunks; +mod fold_chunks_with; +mod for_each; +mod from_par_iter; +mod inspect; +mod interleave; +mod interleave_shortest; +mod intersperse; +mod len; +mod map; +mod map_with; +mod multizip; +mod noop; +mod once; +mod panic_fuse; +mod par_bridge; +mod positions; +mod product; +mod reduce; +mod repeat; +mod rev; +mod skip; +mod skip_any; +mod skip_any_while; +mod splitter; +mod step_by; +mod sum; +mod take; +mod take_any; +mod take_any_while; +mod try_fold; +mod try_reduce; +mod try_reduce_with; +mod unzip; +mod update; +mod walk_tree; +mod while_some; +mod zip; +mod zip_eq; + +pub use self::{ + blocks::{ExponentialBlocks, UniformBlocks}, + chain::Chain, + chunks::Chunks, + cloned::Cloned, + copied::Copied, + empty::{empty, Empty}, + enumerate::Enumerate, + filter::Filter, + filter_map::FilterMap, + flat_map::FlatMap, + flat_map_iter::FlatMapIter, + flatten::Flatten, + flatten_iter::FlattenIter, + fold::{Fold, FoldWith}, + fold_chunks::FoldChunks, + fold_chunks_with::FoldChunksWith, + inspect::Inspect, + interleave::Interleave, + interleave_shortest::InterleaveShortest, + intersperse::Intersperse, + len::{MaxLen, MinLen}, + map::Map, + map_with::{MapInit, MapWith}, + multizip::MultiZip, + once::{once, Once}, + panic_fuse::PanicFuse, + par_bridge::{IterBridge, ParallelBridge}, + positions::Positions, + repeat::{repeat, repeat_n, Repeat, RepeatN}, + rev::Rev, + skip::Skip, + skip_any::SkipAny, + skip_any_while::SkipAnyWhile, + splitter::{split, Split}, + step_by::StepBy, + take::Take, + take_any::TakeAny, + take_any_while::TakeAnyWhile, + try_fold::{TryFold, TryFoldWith}, + update::Update, + walk_tree::{ + walk_tree, walk_tree_postfix, walk_tree_prefix, WalkTree, WalkTreePostfix, WalkTreePrefix, + }, + while_some::WhileSome, + zip::Zip, + zip_eq::ZipEq, +}; + +#[allow(deprecated)] +pub use repeat::repeatn; + +/// `IntoParallelIterator` implements the conversion to a [`ParallelIterator`]. +/// +/// By implementing `IntoParallelIterator` for a type, you define how it will +/// transformed into an iterator. This is a parallel version of the standard +/// library's [`std::iter::IntoIterator`] trait. +pub trait IntoParallelIterator { + /// The parallel iterator type that will be created. + type Iter: ParallelIterator; + + /// The type of item that the parallel iterator will produce. + type Item: Send; + + /// Converts `self` into a parallel iterator. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// println!("counting in parallel:"); + /// (0..100).into_par_iter() + /// .for_each(|i| println!("{}", i)); + /// ``` + /// + /// This conversion is often implicit for arguments to methods like [`zip`]. + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let v: Vec<_> = (0..5).into_par_iter().zip(5..10).collect(); + /// assert_eq!(v, [(0, 5), (1, 6), (2, 7), (3, 8), (4, 9)]); + /// ``` + /// + /// [`zip`]: IndexedParallelIterator::zip() + fn into_par_iter(self) -> Self::Iter; +} + +/// `IntoParallelRefIterator` implements the conversion to a +/// [`ParallelIterator`], providing shared references to the data. +/// +/// This is a parallel version of the `iter()` method +/// defined by various collections. +/// +/// This trait is automatically implemented +/// `for I where &I: IntoParallelIterator`. In most cases, users +/// will want to implement [`IntoParallelIterator`] rather than implement +/// this trait directly. +pub trait IntoParallelRefIterator<'data> { + /// The type of the parallel iterator that will be returned. + type Iter: ParallelIterator; + + /// The type of item that the parallel iterator will produce. + /// This will typically be an `&'data T` reference type. + type Item: Send + 'data; + + /// Converts `self` into a parallel iterator. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let v: Vec<_> = (0..100).collect(); + /// assert_eq!(v.par_iter().sum::(), 100 * 99 / 2); + /// + /// // `v.par_iter()` is shorthand for `(&v).into_par_iter()`, + /// // producing the exact same references. + /// assert!(v.par_iter().zip(&v) + /// .all(|(a, b)| std::ptr::eq(a, b))); + /// ``` + fn par_iter(&'data self) -> Self::Iter; +} + +impl<'data, I: 'data + ?Sized> IntoParallelRefIterator<'data> for I +where + &'data I: IntoParallelIterator, +{ + type Iter = <&'data I as IntoParallelIterator>::Iter; + type Item = <&'data I as IntoParallelIterator>::Item; + + fn par_iter(&'data self) -> Self::Iter { + self.into_par_iter() + } +} + +/// `IntoParallelRefMutIterator` implements the conversion to a +/// [`ParallelIterator`], providing mutable references to the data. +/// +/// This is a parallel version of the `iter_mut()` method +/// defined by various collections. +/// +/// This trait is automatically implemented +/// `for I where &mut I: IntoParallelIterator`. In most cases, users +/// will want to implement [`IntoParallelIterator`] rather than implement +/// this trait directly. +pub trait IntoParallelRefMutIterator<'data> { + /// The type of iterator that will be created. + type Iter: ParallelIterator; + + /// The type of item that will be produced; this is typically an + /// `&'data mut T` reference. + type Item: Send + 'data; + + /// Creates the parallel iterator from `self`. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let mut v = vec![0usize; 5]; + /// v.par_iter_mut().enumerate().for_each(|(i, x)| *x = i); + /// assert_eq!(v, [0, 1, 2, 3, 4]); + /// ``` + fn par_iter_mut(&'data mut self) -> Self::Iter; +} + +impl<'data, I: 'data + ?Sized> IntoParallelRefMutIterator<'data> for I +where + &'data mut I: IntoParallelIterator, +{ + type Iter = <&'data mut I as IntoParallelIterator>::Iter; + type Item = <&'data mut I as IntoParallelIterator>::Item; + + fn par_iter_mut(&'data mut self) -> Self::Iter { + self.into_par_iter() + } +} + +/// Parallel version of the standard iterator trait. +/// +/// The combinators on this trait are available on **all** parallel +/// iterators. Additional methods can be found on the +/// [`IndexedParallelIterator`] trait: those methods are only +/// available for parallel iterators where the number of items is +/// known in advance (so, e.g., after invoking `filter`, those methods +/// become unavailable). +/// +/// For examples of using parallel iterators, see [the docs on the +/// `iter` module][iter]. +/// +/// [iter]: self +pub trait ParallelIterator: Sized + Send { + /// The type of item that this parallel iterator produces. + /// For example, if you use the [`for_each`] method, this is the type of + /// item that your closure will be invoked with. + /// + /// [`for_each`]: #method.for_each + type Item: Send; + + /// Executes `OP` on each item produced by the iterator, in parallel. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// (0..100).into_par_iter().for_each(|x| println!("{:?}", x)); + /// ``` + fn for_each(self, op: OP) + where + OP: Fn(Self::Item) + Sync + Send, + { + for_each::for_each(self, &op) + } + + /// Executes `OP` on the given `init` value with each item produced by + /// the iterator, in parallel. + /// + /// The `init` value will be cloned only as needed to be paired with + /// the group of items in each rayon job. It does not require the type + /// to be `Sync`. + /// + /// # Examples + /// + /// ``` + /// use std::sync::mpsc::channel; + /// use rayon::prelude::*; + /// + /// let (sender, receiver) = channel(); + /// + /// (0..5).into_par_iter().for_each_with(sender, |s, x| s.send(x).unwrap()); + /// + /// let mut res: Vec<_> = receiver.iter().collect(); + /// + /// res.sort(); + /// + /// assert_eq!(&res[..], &[0, 1, 2, 3, 4]) + /// ``` + fn for_each_with(self, init: T, op: OP) + where + OP: Fn(&mut T, Self::Item) + Sync + Send, + T: Send + Clone, + { + self.map_with(init, op).collect() + } + + /// Executes `OP` on a value returned by `init` with each item produced by + /// the iterator, in parallel. + /// + /// The `init` function will be called only as needed for a value to be + /// paired with the group of items in each rayon job. There is no + /// constraint on that returned type at all! + /// + /// # Examples + /// + /// ``` + /// use rand::Rng; + /// use rayon::prelude::*; + /// + /// let mut v = vec![0u8; 1_000_000]; + /// + /// v.par_chunks_mut(1000) + /// .for_each_init( + /// || rand::rng(), + /// |rng, chunk| rng.fill(chunk), + /// ); + /// + /// // There's a remote chance that this will fail... + /// for i in 0u8..=255 { + /// assert!(v.contains(&i)); + /// } + /// ``` + fn for_each_init(self, init: INIT, op: OP) + where + OP: Fn(&mut T, Self::Item) + Sync + Send, + INIT: Fn() -> T + Sync + Send, + { + self.map_init(init, op).collect() + } + + /// Executes a fallible `OP` on each item produced by the iterator, in parallel. + /// + /// If the `OP` returns `Result::Err` or `Option::None`, we will attempt to + /// stop processing the rest of the items in the iterator as soon as + /// possible, and we will return that terminating value. Otherwise, we will + /// return an empty `Result::Ok(())` or `Option::Some(())`. If there are + /// multiple errors in parallel, it is not specified which will be returned. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// use std::io::{self, Write}; + /// + /// // This will stop iteration early if there's any write error, like + /// // having piped output get closed on the other end. + /// (0..100).into_par_iter() + /// .try_for_each(|x| writeln!(io::stdout(), "{:?}", x)) + /// .expect("expected no write errors"); + /// ``` + fn try_for_each(self, op: OP) -> R + where + OP: Fn(Self::Item) -> R + Sync + Send, + R: Try + Send, + { + fn ok>(_: (), _: ()) -> R { + R::from_output(()) + } + + self.map(op).try_reduce(<()>::default, ok) + } + + /// Executes a fallible `OP` on the given `init` value with each item + /// produced by the iterator, in parallel. + /// + /// This combines the `init` semantics of [`for_each_with()`] and the + /// failure semantics of [`try_for_each()`]. + /// + /// [`for_each_with()`]: #method.for_each_with + /// [`try_for_each()`]: #method.try_for_each + /// + /// # Examples + /// + /// ``` + /// use std::sync::mpsc::channel; + /// use rayon::prelude::*; + /// + /// let (sender, receiver) = channel(); + /// + /// (0..5).into_par_iter() + /// .try_for_each_with(sender, |s, x| s.send(x)) + /// .expect("expected no send errors"); + /// + /// let mut res: Vec<_> = receiver.iter().collect(); + /// + /// res.sort(); + /// + /// assert_eq!(&res[..], &[0, 1, 2, 3, 4]) + /// ``` + fn try_for_each_with(self, init: T, op: OP) -> R + where + OP: Fn(&mut T, Self::Item) -> R + Sync + Send, + T: Send + Clone, + R: Try + Send, + { + fn ok>(_: (), _: ()) -> R { + R::from_output(()) + } + + self.map_with(init, op).try_reduce(<()>::default, ok) + } + + /// Executes a fallible `OP` on a value returned by `init` with each item + /// produced by the iterator, in parallel. + /// + /// This combines the `init` semantics of [`for_each_init()`] and the + /// failure semantics of [`try_for_each()`]. + /// + /// [`for_each_init()`]: #method.for_each_init + /// [`try_for_each()`]: #method.try_for_each + /// + /// # Examples + /// + /// ``` + /// use rand::{Rng, TryRngCore}; + /// use rayon::prelude::*; + /// + /// let mut v = vec![0u8; 1_000_000]; + /// + /// v.par_chunks_mut(1000) + /// .try_for_each_init( + /// || rand::rng(), + /// |rng, chunk| rng.try_fill_bytes(chunk), + /// ) + /// .expect("expected no rand errors"); + /// + /// // There's a remote chance that this will fail... + /// for i in 0u8..=255 { + /// assert!(v.contains(&i)); + /// } + /// ``` + fn try_for_each_init(self, init: INIT, op: OP) -> R + where + OP: Fn(&mut T, Self::Item) -> R + Sync + Send, + INIT: Fn() -> T + Sync + Send, + R: Try + Send, + { + fn ok>(_: (), _: ()) -> R { + R::from_output(()) + } + + self.map_init(init, op).try_reduce(<()>::default, ok) + } + + /// Counts the number of items in this parallel iterator. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let count = (0..100).into_par_iter().count(); + /// + /// assert_eq!(count, 100); + /// ``` + fn count(self) -> usize { + fn one(_: T) -> usize { + 1 + } + + self.map(one).sum() + } + + /// Applies `map_op` to each item of this iterator, producing a new + /// iterator with the results. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let mut par_iter = (0..5).into_par_iter().map(|x| x * 2); + /// + /// let doubles: Vec<_> = par_iter.collect(); + /// + /// assert_eq!(&doubles[..], &[0, 2, 4, 6, 8]); + /// ``` + fn map(self, map_op: F) -> Map + where + F: Fn(Self::Item) -> R + Sync + Send, + R: Send, + { + Map::new(self, map_op) + } + + /// Applies `map_op` to the given `init` value with each item of this + /// iterator, producing a new iterator with the results. + /// + /// The `init` value will be cloned only as needed to be paired with + /// the group of items in each rayon job. It does not require the type + /// to be `Sync`. + /// + /// # Examples + /// + /// ``` + /// use std::sync::mpsc::channel; + /// use rayon::prelude::*; + /// + /// let (sender, receiver) = channel(); + /// + /// let a: Vec<_> = (0..5) + /// .into_par_iter() // iterating over i32 + /// .map_with(sender, |s, x| { + /// s.send(x).unwrap(); // sending i32 values through the channel + /// x // returning i32 + /// }) + /// .collect(); // collecting the returned values into a vector + /// + /// let mut b: Vec<_> = receiver.iter() // iterating over the values in the channel + /// .collect(); // and collecting them + /// b.sort(); + /// + /// assert_eq!(a, b); + /// ``` + fn map_with(self, init: T, map_op: F) -> MapWith + where + F: Fn(&mut T, Self::Item) -> R + Sync + Send, + T: Send + Clone, + R: Send, + { + MapWith::new(self, init, map_op) + } + + /// Applies `map_op` to a value returned by `init` with each item of this + /// iterator, producing a new iterator with the results. + /// + /// The `init` function will be called only as needed for a value to be + /// paired with the group of items in each rayon job. There is no + /// constraint on that returned type at all! + /// + /// # Examples + /// + /// ``` + /// use rand::Rng; + /// use rayon::prelude::*; + /// + /// let a: Vec<_> = (1i32..1_000_000) + /// .into_par_iter() + /// .map_init( + /// || rand::rng(), // get the thread-local RNG + /// |rng, x| if rng.random() { // randomly negate items + /// -x + /// } else { + /// x + /// }, + /// ).collect(); + /// + /// // There's a remote chance that this will fail... + /// assert!(a.iter().any(|&x| x < 0)); + /// assert!(a.iter().any(|&x| x > 0)); + /// ``` + fn map_init(self, init: INIT, map_op: F) -> MapInit + where + F: Fn(&mut T, Self::Item) -> R + Sync + Send, + INIT: Fn() -> T + Sync + Send, + R: Send, + { + MapInit::new(self, init, map_op) + } + + /// Creates an iterator which clones all of its elements. This may be + /// useful when you have an iterator over `&T`, but you need `T`, and + /// that type implements `Clone`. See also [`copied()`]. + /// + /// [`copied()`]: #method.copied + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let a = [1, 2, 3]; + /// + /// let v_cloned: Vec<_> = a.par_iter().cloned().collect(); + /// + /// // cloned is the same as .map(|&x| x), for integers + /// let v_map: Vec<_> = a.par_iter().map(|&x| x).collect(); + /// + /// assert_eq!(v_cloned, vec![1, 2, 3]); + /// assert_eq!(v_map, vec![1, 2, 3]); + /// ``` + fn cloned<'a, T>(self) -> Cloned + where + T: 'a + Clone + Send, + Self: ParallelIterator, + { + Cloned::new(self) + } + + /// Creates an iterator which copies all of its elements. This may be + /// useful when you have an iterator over `&T`, but you need `T`, and + /// that type implements `Copy`. See also [`cloned()`]. + /// + /// [`cloned()`]: #method.cloned + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let a = [1, 2, 3]; + /// + /// let v_copied: Vec<_> = a.par_iter().copied().collect(); + /// + /// // copied is the same as .map(|&x| x), for integers + /// let v_map: Vec<_> = a.par_iter().map(|&x| x).collect(); + /// + /// assert_eq!(v_copied, vec![1, 2, 3]); + /// assert_eq!(v_map, vec![1, 2, 3]); + /// ``` + fn copied<'a, T>(self) -> Copied + where + T: 'a + Copy + Send, + Self: ParallelIterator, + { + Copied::new(self) + } + + /// Applies `inspect_op` to a reference to each item of this iterator, + /// producing a new iterator passing through the original items. This is + /// often useful for debugging to see what's happening in iterator stages. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let a = [1, 4, 2, 3]; + /// + /// // this iterator sequence is complex. + /// let sum = a.par_iter() + /// .cloned() + /// .filter(|&x| x % 2 == 0) + /// .reduce(|| 0, |sum, i| sum + i); + /// + /// println!("{}", sum); + /// + /// // let's add some inspect() calls to investigate what's happening + /// let sum = a.par_iter() + /// .cloned() + /// .inspect(|x| println!("about to filter: {}", x)) + /// .filter(|&x| x % 2 == 0) + /// .inspect(|x| println!("made it through filter: {}", x)) + /// .reduce(|| 0, |sum, i| sum + i); + /// + /// println!("{}", sum); + /// ``` + fn inspect(self, inspect_op: OP) -> Inspect + where + OP: Fn(&Self::Item) + Sync + Send, + { + Inspect::new(self, inspect_op) + } + + /// Mutates each item of this iterator before yielding it. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let par_iter = (0..5).into_par_iter().update(|x| {*x *= 2;}); + /// + /// let doubles: Vec<_> = par_iter.collect(); + /// + /// assert_eq!(&doubles[..], &[0, 2, 4, 6, 8]); + /// ``` + fn update(self, update_op: F) -> Update + where + F: Fn(&mut Self::Item) + Sync + Send, + { + Update::new(self, update_op) + } + + /// Applies `filter_op` to each item of this iterator, producing a new + /// iterator with only the items that gave `true` results. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let mut par_iter = (0..10).into_par_iter().filter(|x| x % 2 == 0); + /// + /// let even_numbers: Vec<_> = par_iter.collect(); + /// + /// assert_eq!(&even_numbers[..], &[0, 2, 4, 6, 8]); + /// ``` + fn filter

(self, filter_op: P) -> Filter + where + P: Fn(&Self::Item) -> bool + Sync + Send, + { + Filter::new(self, filter_op) + } + + /// Applies `filter_op` to each item of this iterator to get an `Option`, + /// producing a new iterator with only the items from `Some` results. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let mut par_iter = (0..10).into_par_iter() + /// .filter_map(|x| { + /// if x % 2 == 0 { Some(x * 3) } + /// else { None } + /// }); + /// + /// let even_numbers: Vec<_> = par_iter.collect(); + /// + /// assert_eq!(&even_numbers[..], &[0, 6, 12, 18, 24]); + /// ``` + fn filter_map(self, filter_op: P) -> FilterMap + where + P: Fn(Self::Item) -> Option + Sync + Send, + R: Send, + { + FilterMap::new(self, filter_op) + } + + /// Applies `map_op` to each item of this iterator to get nested parallel iterators, + /// producing a new parallel iterator that flattens these back into one. + /// + /// See also [`flat_map_iter`](#method.flat_map_iter). + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let a = [[1, 2], [3, 4], [5, 6], [7, 8]]; + /// + /// let par_iter = a.par_iter().cloned().flat_map(|a| a.to_vec()); + /// + /// let vec: Vec<_> = par_iter.collect(); + /// + /// assert_eq!(&vec[..], &[1, 2, 3, 4, 5, 6, 7, 8]); + /// ``` + fn flat_map(self, map_op: F) -> FlatMap + where + F: Fn(Self::Item) -> PI + Sync + Send, + PI: IntoParallelIterator, + { + FlatMap::new(self, map_op) + } + + /// Applies `map_op` to each item of this iterator to get nested serial iterators, + /// producing a new parallel iterator that flattens these back into one. + /// + /// # `flat_map_iter` versus `flat_map` + /// + /// These two methods are similar but behave slightly differently. With [`flat_map`], + /// each of the nested iterators must be a parallel iterator, and they will be further + /// split up with nested parallelism. With `flat_map_iter`, each nested iterator is a + /// sequential `Iterator`, and we only parallelize _between_ them, while the items + /// produced by each nested iterator are processed sequentially. + /// + /// When choosing between these methods, consider whether nested parallelism suits the + /// potential iterators at hand. If there's little computation involved, or its length + /// is much less than the outer parallel iterator, then it may perform better to avoid + /// the overhead of parallelism, just flattening sequentially with `flat_map_iter`. + /// If there is a lot of computation, potentially outweighing the outer parallel + /// iterator, then the nested parallelism of `flat_map` may be worthwhile. + /// + /// [`flat_map`]: #method.flat_map + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// use std::cell::RefCell; + /// + /// let a = [[1, 2], [3, 4], [5, 6], [7, 8]]; + /// + /// let par_iter = a.par_iter().flat_map_iter(|a| { + /// // The serial iterator doesn't have to be thread-safe, just its items. + /// let cell_iter = RefCell::new(a.iter().cloned()); + /// std::iter::from_fn(move || cell_iter.borrow_mut().next()) + /// }); + /// + /// let vec: Vec<_> = par_iter.collect(); + /// + /// assert_eq!(&vec[..], &[1, 2, 3, 4, 5, 6, 7, 8]); + /// ``` + fn flat_map_iter(self, map_op: F) -> FlatMapIter + where + F: Fn(Self::Item) -> SI + Sync + Send, + SI: IntoIterator, + { + FlatMapIter::new(self, map_op) + } + + /// An adaptor that flattens parallel-iterable `Item`s into one large iterator. + /// + /// See also [`flatten_iter`](#method.flatten_iter). + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let x: Vec> = vec![vec![1, 2], vec![3, 4]]; + /// let y: Vec<_> = x.into_par_iter().flatten().collect(); + /// + /// assert_eq!(y, vec![1, 2, 3, 4]); + /// ``` + fn flatten(self) -> Flatten + where + Self::Item: IntoParallelIterator, + { + Flatten::new(self) + } + + /// An adaptor that flattens serial-iterable `Item`s into one large iterator. + /// + /// See also [`flatten`](#method.flatten) and the analogous comparison of + /// [`flat_map_iter` versus `flat_map`](#flat_map_iter-versus-flat_map). + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let x: Vec> = vec![vec![1, 2], vec![3, 4]]; + /// let iters: Vec<_> = x.into_iter().map(Vec::into_iter).collect(); + /// let y: Vec<_> = iters.into_par_iter().flatten_iter().collect(); + /// + /// assert_eq!(y, vec![1, 2, 3, 4]); + /// ``` + fn flatten_iter(self) -> FlattenIter + where + Self::Item: IntoIterator, + { + FlattenIter::new(self) + } + + /// Reduces the items in the iterator into one item using `op`. + /// The argument `identity` should be a closure that can produce + /// "identity" value which may be inserted into the sequence as + /// needed to create opportunities for parallel execution. So, for + /// example, if you are doing a summation, then `identity()` ought + /// to produce something that represents the zero for your type + /// (but consider just calling `sum()` in that case). + /// + /// # Examples + /// + /// ``` + /// // Iterate over a sequence of pairs `(x0, y0), ..., (xN, yN)` + /// // and use reduce to compute one pair `(x0 + ... + xN, y0 + ... + yN)` + /// // where the first/second elements are summed separately. + /// use rayon::prelude::*; + /// let sums = [(0, 1), (5, 6), (16, 2), (8, 9)] + /// .par_iter() // iterating over &(i32, i32) + /// .cloned() // iterating over (i32, i32) + /// .reduce(|| (0, 0), // the "identity" is 0 in both columns + /// |a, b| (a.0 + b.0, a.1 + b.1)); + /// assert_eq!(sums, (0 + 5 + 16 + 8, 1 + 6 + 2 + 9)); + /// ``` + /// + /// **Note:** unlike a sequential `fold` operation, the order in + /// which `op` will be applied to reduce the result is not fully + /// specified. So `op` should be [associative] or else the results + /// will be non-deterministic. And of course `identity()` should + /// produce a true identity. + /// + /// [associative]: https://en.wikipedia.org/wiki/Associative_property + fn reduce(self, identity: ID, op: OP) -> Self::Item + where + OP: Fn(Self::Item, Self::Item) -> Self::Item + Sync + Send, + ID: Fn() -> Self::Item + Sync + Send, + { + reduce::reduce(self, identity, op) + } + + /// Reduces the items in the iterator into one item using `op`. + /// If the iterator is empty, `None` is returned; otherwise, + /// `Some` is returned. + /// + /// This version of `reduce` is simple but somewhat less + /// efficient. If possible, it is better to call `reduce()`, which + /// requires an identity element. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// let sums = [(0, 1), (5, 6), (16, 2), (8, 9)] + /// .par_iter() // iterating over &(i32, i32) + /// .cloned() // iterating over (i32, i32) + /// .reduce_with(|a, b| (a.0 + b.0, a.1 + b.1)) + /// .unwrap(); + /// assert_eq!(sums, (0 + 5 + 16 + 8, 1 + 6 + 2 + 9)); + /// ``` + /// + /// **Note:** unlike a sequential `fold` operation, the order in + /// which `op` will be applied to reduce the result is not fully + /// specified. So `op` should be [associative] or else the results + /// will be non-deterministic. + /// + /// [associative]: https://en.wikipedia.org/wiki/Associative_property + fn reduce_with(self, op: OP) -> Option + where + OP: Fn(Self::Item, Self::Item) -> Self::Item + Sync + Send, + { + fn opt_fold(op: impl Fn(T, T) -> T) -> impl Fn(Option, T) -> Option { + move |opt_a, b| match opt_a { + Some(a) => Some(op(a, b)), + None => Some(b), + } + } + + fn opt_reduce(op: impl Fn(T, T) -> T) -> impl Fn(Option, Option) -> Option { + move |opt_a, opt_b| match (opt_a, opt_b) { + (Some(a), Some(b)) => Some(op(a, b)), + (Some(v), None) | (None, Some(v)) => Some(v), + (None, None) => None, + } + } + + self.fold(<_>::default, opt_fold(&op)) + .reduce(<_>::default, opt_reduce(&op)) + } + + /// Reduces the items in the iterator into one item using a fallible `op`. + /// The `identity` argument is used the same way as in [`reduce()`]. + /// + /// [`reduce()`]: #method.reduce + /// + /// If a `Result::Err` or `Option::None` item is found, or if `op` reduces + /// to one, we will attempt to stop processing the rest of the items in the + /// iterator as soon as possible, and we will return that terminating value. + /// Otherwise, we will return the final reduced `Result::Ok(T)` or + /// `Option::Some(T)`. If there are multiple errors in parallel, it is not + /// specified which will be returned. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// // Compute the sum of squares, being careful about overflow. + /// fn sum_squares>(iter: I) -> Option { + /// iter.into_par_iter() + /// .map(|i| i.checked_mul(i)) // square each item, + /// .try_reduce(|| 0, i32::checked_add) // and add them up! + /// } + /// assert_eq!(sum_squares(0..5), Some(0 + 1 + 4 + 9 + 16)); + /// + /// // The sum might overflow + /// assert_eq!(sum_squares(0..10_000), None); + /// + /// // Or the squares might overflow before it even reaches `try_reduce` + /// assert_eq!(sum_squares(1_000_000..1_000_001), None); + /// ``` + fn try_reduce(self, identity: ID, op: OP) -> Self::Item + where + OP: Fn(T, T) -> Self::Item + Sync + Send, + ID: Fn() -> T + Sync + Send, + Self::Item: Try, + { + try_reduce::try_reduce(self, identity, op) + } + + /// Reduces the items in the iterator into one item using a fallible `op`. + /// + /// Like [`reduce_with()`], if the iterator is empty, `None` is returned; + /// otherwise, `Some` is returned. Beyond that, it behaves like + /// [`try_reduce()`] for handling `Err`/`None`. + /// + /// [`reduce_with()`]: #method.reduce_with + /// [`try_reduce()`]: #method.try_reduce + /// + /// For instance, with `Option` items, the return value may be: + /// - `None`, the iterator was empty + /// - `Some(None)`, we stopped after encountering `None`. + /// - `Some(Some(x))`, the entire iterator reduced to `x`. + /// + /// With `Result` items, the nesting is more obvious: + /// - `None`, the iterator was empty + /// - `Some(Err(e))`, we stopped after encountering an error `e`. + /// - `Some(Ok(x))`, the entire iterator reduced to `x`. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let files = ["/dev/null", "/does/not/exist"]; + /// + /// // Find the biggest file + /// files.into_par_iter() + /// .map(|path| std::fs::metadata(path).map(|m| (path, m.len()))) + /// .try_reduce_with(|a, b| { + /// Ok(if a.1 >= b.1 { a } else { b }) + /// }) + /// .expect("Some value, since the iterator is not empty") + /// .expect_err("not found"); + /// ``` + fn try_reduce_with(self, op: OP) -> Option + where + OP: Fn(T, T) -> Self::Item + Sync + Send, + Self::Item: Try, + { + try_reduce_with::try_reduce_with(self, op) + } + + /// Parallel fold is similar to sequential fold except that the + /// sequence of items may be subdivided before it is + /// folded. Consider a list of numbers like `22 3 77 89 46`. If + /// you used sequential fold to add them (`fold(0, |a,b| a+b)`, + /// you would wind up first adding 0 + 22, then 22 + 3, then 25 + + /// 77, and so forth. The **parallel fold** works similarly except + /// that it first breaks up your list into sublists, and hence + /// instead of yielding up a single sum at the end, it yields up + /// multiple sums. The number of results is nondeterministic, as + /// is the point where the breaks occur. + /// + /// So if we did the same parallel fold (`fold(0, |a,b| a+b)`) on + /// our example list, we might wind up with a sequence of two numbers, + /// like so: + /// + /// ```notrust + /// 22 3 77 89 46 + /// | | + /// 102 135 + /// ``` + /// + /// Or perhaps these three numbers: + /// + /// ```notrust + /// 22 3 77 89 46 + /// | | | + /// 102 89 46 + /// ``` + /// + /// In general, Rayon will attempt to find good breaking points + /// that keep all of your cores busy. + /// + /// ### Fold versus reduce + /// + /// The `fold()` and `reduce()` methods each take an identity element + /// and a combining function, but they operate rather differently. + /// + /// `reduce()` requires that the identity function has the same + /// type as the things you are iterating over, and it fully + /// reduces the list of items into a single item. So, for example, + /// imagine we are iterating over a list of bytes `bytes: [128_u8, + /// 64_u8, 64_u8]`. If we used `bytes.reduce(|| 0_u8, |a: u8, b: + /// u8| a + b)`, we would get an overflow. This is because `0`, + /// `a`, and `b` here are all bytes, just like the numbers in the + /// list (I wrote the types explicitly above, but those are the + /// only types you can use). To avoid the overflow, we would need + /// to do something like `bytes.map(|b| b as u32).reduce(|| 0, |a, + /// b| a + b)`, in which case our result would be `256`. + /// + /// In contrast, with `fold()`, the identity function does not + /// have to have the same type as the things you are iterating + /// over, and you potentially get back many results. So, if we + /// continue with the `bytes` example from the previous paragraph, + /// we could do `bytes.fold(|| 0_u32, |a, b| a + (b as u32))` to + /// convert our bytes into `u32`. And of course we might not get + /// back a single sum. + /// + /// There is a more subtle distinction as well, though it's + /// actually implied by the above points. When you use `reduce()`, + /// your reduction function is sometimes called with values that + /// were never part of your original parallel iterator (for + /// example, both the left and right might be a partial sum). With + /// `fold()`, in contrast, the left value in the fold function is + /// always the accumulator, and the right value is always from + /// your original sequence. + /// + /// ### Fold vs Map/Reduce + /// + /// Fold makes sense if you have some operation where it is + /// cheaper to create groups of elements at a time. For example, + /// imagine collecting characters into a string. If you were going + /// to use map/reduce, you might try this: + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let s = + /// ['a', 'b', 'c', 'd', 'e'] + /// .par_iter() + /// .map(|c: &char| format!("{}", c)) + /// .reduce(|| String::new(), + /// |mut a: String, b: String| { a.push_str(&b); a }); + /// + /// assert_eq!(s, "abcde"); + /// ``` + /// + /// Because reduce produces the same type of element as its input, + /// you have to first map each character into a string, and then + /// you can reduce them. This means we create one string per + /// element in our iterator -- not so great. Using `fold`, we can + /// do this instead: + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let s = + /// ['a', 'b', 'c', 'd', 'e'] + /// .par_iter() + /// .fold(|| String::new(), + /// |mut s: String, c: &char| { s.push(*c); s }) + /// .reduce(|| String::new(), + /// |mut a: String, b: String| { a.push_str(&b); a }); + /// + /// assert_eq!(s, "abcde"); + /// ``` + /// + /// Now `fold` will process groups of our characters at a time, + /// and we only make one string per group. We should wind up with + /// some small-ish number of strings roughly proportional to the + /// number of CPUs you have (it will ultimately depend on how busy + /// your processors are). Note that we still need to do a reduce + /// afterwards to combine those groups of strings into a single + /// string. + /// + /// You could use a similar trick to save partial results (e.g., a + /// cache) or something similar. + /// + /// ### Combining fold with other operations + /// + /// You can combine `fold` with `reduce` if you want to produce a + /// single value. This is then roughly equivalent to a map/reduce + /// combination in effect: + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let bytes = 0..22_u8; + /// let sum = bytes.into_par_iter() + /// .fold(|| 0_u32, |a: u32, b: u8| a + (b as u32)) + /// .sum::(); + /// + /// assert_eq!(sum, (0..22).sum()); // compare to sequential + /// ``` + fn fold(self, identity: ID, fold_op: F) -> Fold + where + F: Fn(T, Self::Item) -> T + Sync + Send, + ID: Fn() -> T + Sync + Send, + T: Send, + { + Fold::new(self, identity, fold_op) + } + + /// Applies `fold_op` to the given `init` value with each item of this + /// iterator, finally producing the value for further use. + /// + /// This works essentially like `fold(|| init.clone(), fold_op)`, except + /// it doesn't require the `init` type to be `Sync`, nor any other form + /// of added synchronization. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let bytes = 0..22_u8; + /// let sum = bytes.into_par_iter() + /// .fold_with(0_u32, |a: u32, b: u8| a + (b as u32)) + /// .sum::(); + /// + /// assert_eq!(sum, (0..22).sum()); // compare to sequential + /// ``` + fn fold_with(self, init: T, fold_op: F) -> FoldWith + where + F: Fn(T, Self::Item) -> T + Sync + Send, + T: Send + Clone, + { + FoldWith::new(self, init, fold_op) + } + + /// Performs a fallible parallel fold. + /// + /// This is a variation of [`fold()`] for operations which can fail with + /// `Option::None` or `Result::Err`. The first such failure stops + /// processing the local set of items, without affecting other folds in the + /// iterator's subdivisions. + /// + /// Often, `try_fold()` will be followed by [`try_reduce()`] + /// for a final reduction and global short-circuiting effect. + /// + /// [`fold()`]: #method.fold + /// [`try_reduce()`]: #method.try_reduce + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let bytes = 0..22_u8; + /// let sum = bytes.into_par_iter() + /// .try_fold(|| 0_u32, |a: u32, b: u8| a.checked_add(b as u32)) + /// .try_reduce(|| 0, u32::checked_add); + /// + /// assert_eq!(sum, Some((0..22).sum())); // compare to sequential + /// ``` + fn try_fold(self, identity: ID, fold_op: F) -> TryFold + where + F: Fn(T, Self::Item) -> R + Sync + Send, + ID: Fn() -> T + Sync + Send, + R: Try + Send, + { + TryFold::new(self, identity, fold_op) + } + + /// Performs a fallible parallel fold with a cloneable `init` value. + /// + /// This combines the `init` semantics of [`fold_with()`] and the failure + /// semantics of [`try_fold()`]. + /// + /// [`fold_with()`]: #method.fold_with + /// [`try_fold()`]: #method.try_fold + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let bytes = 0..22_u8; + /// let sum = bytes.into_par_iter() + /// .try_fold_with(0_u32, |a: u32, b: u8| a.checked_add(b as u32)) + /// .try_reduce(|| 0, u32::checked_add); + /// + /// assert_eq!(sum, Some((0..22).sum())); // compare to sequential + /// ``` + fn try_fold_with(self, init: T, fold_op: F) -> TryFoldWith + where + F: Fn(T, Self::Item) -> R + Sync + Send, + R: Try + Send, + T: Clone + Send, + { + TryFoldWith::new(self, init, fold_op) + } + + /// Sums up the items in the iterator. + /// + /// Note that the order in items will be reduced is not specified, + /// so if the `+` operator is not truly [associative] \(as is the + /// case for floating point numbers), then the results are not + /// fully deterministic. + /// + /// [associative]: https://en.wikipedia.org/wiki/Associative_property + /// + /// Basically equivalent to `self.reduce(|| 0, |a, b| a + b)`, + /// except that the type of `0` and the `+` operation may vary + /// depending on the type of value being produced. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let a = [1, 5, 7]; + /// + /// let sum: i32 = a.par_iter().sum(); + /// + /// assert_eq!(sum, 13); + /// ``` + fn sum(self) -> S + where + S: Send + Sum + Sum, + { + sum::sum(self) + } + + /// Multiplies all the items in the iterator. + /// + /// Note that the order in items will be reduced is not specified, + /// so if the `*` operator is not truly [associative] \(as is the + /// case for floating point numbers), then the results are not + /// fully deterministic. + /// + /// [associative]: https://en.wikipedia.org/wiki/Associative_property + /// + /// Basically equivalent to `self.reduce(|| 1, |a, b| a * b)`, + /// except that the type of `1` and the `*` operation may vary + /// depending on the type of value being produced. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// fn factorial(n: u32) -> u32 { + /// (1..n+1).into_par_iter().product() + /// } + /// + /// assert_eq!(factorial(0), 1); + /// assert_eq!(factorial(1), 1); + /// assert_eq!(factorial(5), 120); + /// ``` + fn product

(self) -> P + where + P: Send + Product + Product

, + { + product::product(self) + } + + /// Computes the minimum of all the items in the iterator. If the + /// iterator is empty, `None` is returned; otherwise, `Some(min)` + /// is returned. + /// + /// Note that the order in which the items will be reduced is not + /// specified, so if the `Ord` impl is not truly associative, then + /// the results are not deterministic. + /// + /// Basically equivalent to `self.reduce_with(|a, b| Ord::min(a, b))`. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let a = [45, 74, 32]; + /// + /// assert_eq!(a.par_iter().min(), Some(&32)); + /// + /// let b: [i32; 0] = []; + /// + /// assert_eq!(b.par_iter().min(), None); + /// ``` + fn min(self) -> Option + where + Self::Item: Ord, + { + self.reduce_with(Ord::min) + } + + /// Computes the minimum of all the items in the iterator with respect to + /// the given comparison function. If the iterator is empty, `None` is + /// returned; otherwise, `Some(min)` is returned. + /// + /// Note that the order in which the items will be reduced is not + /// specified, so if the comparison function is not associative, then + /// the results are not deterministic. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let a = [-3_i32, 77, 53, 240, -1]; + /// + /// assert_eq!(a.par_iter().min_by(|x, y| x.cmp(y)), Some(&-3)); + /// ``` + fn min_by(self, f: F) -> Option + where + F: Sync + Send + Fn(&Self::Item, &Self::Item) -> Ordering, + { + fn min(f: impl Fn(&T, &T) -> Ordering) -> impl Fn(T, T) -> T { + move |a, b| match f(&a, &b) { + Ordering::Greater => b, + _ => a, + } + } + + self.reduce_with(min(f)) + } + + /// Computes the item that yields the minimum value for the given + /// function. If the iterator is empty, `None` is returned; + /// otherwise, `Some(item)` is returned. + /// + /// Note that the order in which the items will be reduced is not + /// specified, so if the `Ord` impl is not truly associative, then + /// the results are not deterministic. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let a = [-3_i32, 34, 2, 5, -10, -3, -23]; + /// + /// assert_eq!(a.par_iter().min_by_key(|x| x.abs()), Some(&2)); + /// ``` + fn min_by_key(self, f: F) -> Option + where + K: Ord + Send, + F: Sync + Send + Fn(&Self::Item) -> K, + { + fn key(f: impl Fn(&T) -> K) -> impl Fn(T) -> (K, T) { + move |x| (f(&x), x) + } + + fn min_key(a: (K, T), b: (K, T)) -> (K, T) { + match (a.0).cmp(&b.0) { + Ordering::Greater => b, + _ => a, + } + } + + let (_, x) = self.map(key(f)).reduce_with(min_key)?; + Some(x) + } + + /// Computes the maximum of all the items in the iterator. If the + /// iterator is empty, `None` is returned; otherwise, `Some(max)` + /// is returned. + /// + /// Note that the order in which the items will be reduced is not + /// specified, so if the `Ord` impl is not truly associative, then + /// the results are not deterministic. + /// + /// Basically equivalent to `self.reduce_with(|a, b| Ord::max(a, b))`. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let a = [45, 74, 32]; + /// + /// assert_eq!(a.par_iter().max(), Some(&74)); + /// + /// let b: [i32; 0] = []; + /// + /// assert_eq!(b.par_iter().max(), None); + /// ``` + fn max(self) -> Option + where + Self::Item: Ord, + { + self.reduce_with(Ord::max) + } + + /// Computes the maximum of all the items in the iterator with respect to + /// the given comparison function. If the iterator is empty, `None` is + /// returned; otherwise, `Some(max)` is returned. + /// + /// Note that the order in which the items will be reduced is not + /// specified, so if the comparison function is not associative, then + /// the results are not deterministic. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let a = [-3_i32, 77, 53, 240, -1]; + /// + /// assert_eq!(a.par_iter().max_by(|x, y| x.abs().cmp(&y.abs())), Some(&240)); + /// ``` + fn max_by(self, f: F) -> Option + where + F: Sync + Send + Fn(&Self::Item, &Self::Item) -> Ordering, + { + fn max(f: impl Fn(&T, &T) -> Ordering) -> impl Fn(T, T) -> T { + move |a, b| match f(&a, &b) { + Ordering::Greater => a, + _ => b, + } + } + + self.reduce_with(max(f)) + } + + /// Computes the item that yields the maximum value for the given + /// function. If the iterator is empty, `None` is returned; + /// otherwise, `Some(item)` is returned. + /// + /// Note that the order in which the items will be reduced is not + /// specified, so if the `Ord` impl is not truly associative, then + /// the results are not deterministic. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let a = [-3_i32, 34, 2, 5, -10, -3, -23]; + /// + /// assert_eq!(a.par_iter().max_by_key(|x| x.abs()), Some(&34)); + /// ``` + fn max_by_key(self, f: F) -> Option + where + K: Ord + Send, + F: Sync + Send + Fn(&Self::Item) -> K, + { + fn key(f: impl Fn(&T) -> K) -> impl Fn(T) -> (K, T) { + move |x| (f(&x), x) + } + + fn max_key(a: (K, T), b: (K, T)) -> (K, T) { + match (a.0).cmp(&b.0) { + Ordering::Greater => a, + _ => b, + } + } + + let (_, x) = self.map(key(f)).reduce_with(max_key)?; + Some(x) + } + + /// Takes two iterators and creates a new iterator over both. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let a = [0, 1, 2]; + /// let b = [9, 8, 7]; + /// + /// let par_iter = a.par_iter().chain(b.par_iter()); + /// + /// let chained: Vec<_> = par_iter.cloned().collect(); + /// + /// assert_eq!(&chained[..], &[0, 1, 2, 9, 8, 7]); + /// ``` + fn chain(self, chain: C) -> Chain + where + C: IntoParallelIterator, + { + Chain::new(self, chain.into_par_iter()) + } + + /// Searches for **some** item in the parallel iterator that + /// matches the given predicate and returns it. This operation + /// is similar to [`find` on sequential iterators][find] but + /// the item returned may not be the **first** one in the parallel + /// sequence which matches, since we search the entire sequence in parallel. + /// + /// Once a match is found, we will attempt to stop processing + /// the rest of the items in the iterator as soon as possible + /// (just as `find` stops iterating once a match is found). + /// + /// [find]: Iterator::find() + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let a = [1, 2, 3, 3]; + /// + /// assert_eq!(a.par_iter().find_any(|&&x| x == 3), Some(&3)); + /// + /// assert_eq!(a.par_iter().find_any(|&&x| x == 100), None); + /// ``` + fn find_any

(self, predicate: P) -> Option + where + P: Fn(&Self::Item) -> bool + Sync + Send, + { + find::find(self, predicate) + } + + /// Searches for the sequentially **first** item in the parallel iterator + /// that matches the given predicate and returns it. + /// + /// Once a match is found, all attempts to the right of the match + /// will be stopped, while attempts to the left must continue in case + /// an earlier match is found. + /// + /// For added performance, you might consider using `find_first` in conjunction with + /// [`by_exponential_blocks()`][IndexedParallelIterator::by_exponential_blocks]. + /// + /// Note that not all parallel iterators have a useful order, much like + /// sequential `HashMap` iteration, so "first" may be nebulous. If you + /// just want the first match that discovered anywhere in the iterator, + /// `find_any` is a better choice. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let a = [1, 2, 3, 3]; + /// + /// assert_eq!(a.par_iter().find_first(|&&x| x == 3), Some(&3)); + /// + /// assert_eq!(a.par_iter().find_first(|&&x| x == 100), None); + /// ``` + fn find_first

(self, predicate: P) -> Option + where + P: Fn(&Self::Item) -> bool + Sync + Send, + { + find_first_last::find_first(self, predicate) + } + + /// Searches for the sequentially **last** item in the parallel iterator + /// that matches the given predicate and returns it. + /// + /// Once a match is found, all attempts to the left of the match + /// will be stopped, while attempts to the right must continue in case + /// a later match is found. + /// + /// Note that not all parallel iterators have a useful order, much like + /// sequential `HashMap` iteration, so "last" may be nebulous. When the + /// order doesn't actually matter to you, `find_any` is a better choice. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let a = [1, 2, 3, 3]; + /// + /// assert_eq!(a.par_iter().find_last(|&&x| x == 3), Some(&3)); + /// + /// assert_eq!(a.par_iter().find_last(|&&x| x == 100), None); + /// ``` + fn find_last

(self, predicate: P) -> Option + where + P: Fn(&Self::Item) -> bool + Sync + Send, + { + find_first_last::find_last(self, predicate) + } + + /// Applies the given predicate to the items in the parallel iterator + /// and returns **any** non-None result of the map operation. + /// + /// Once a non-None value is produced from the map operation, we will + /// attempt to stop processing the rest of the items in the iterator + /// as soon as possible. + /// + /// Note that this method only returns **some** item in the parallel + /// iterator that is not None from the map predicate. The item returned + /// may not be the **first** non-None value produced in the parallel + /// sequence, since the entire sequence is mapped over in parallel. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let c = ["lol", "NaN", "5", "5"]; + /// + /// let found_number = c.par_iter().find_map_any(|s| s.parse().ok()); + /// + /// assert_eq!(found_number, Some(5)); + /// ``` + fn find_map_any(self, predicate: P) -> Option + where + P: Fn(Self::Item) -> Option + Sync + Send, + R: Send, + { + fn yes(_: &T) -> bool { + true + } + self.filter_map(predicate).find_any(yes) + } + + /// Applies the given predicate to the items in the parallel iterator and + /// returns the sequentially **first** non-None result of the map operation. + /// + /// Once a non-None value is produced from the map operation, all attempts + /// to the right of the match will be stopped, while attempts to the left + /// must continue in case an earlier match is found. + /// + /// Note that not all parallel iterators have a useful order, much like + /// sequential `HashMap` iteration, so "first" may be nebulous. If you + /// just want the first non-None value discovered anywhere in the iterator, + /// `find_map_any` is a better choice. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let c = ["lol", "NaN", "2", "5"]; + /// + /// let first_number = c.par_iter().find_map_first(|s| s.parse().ok()); + /// + /// assert_eq!(first_number, Some(2)); + /// ``` + fn find_map_first(self, predicate: P) -> Option + where + P: Fn(Self::Item) -> Option + Sync + Send, + R: Send, + { + fn yes(_: &T) -> bool { + true + } + self.filter_map(predicate).find_first(yes) + } + + /// Applies the given predicate to the items in the parallel iterator and + /// returns the sequentially **last** non-None result of the map operation. + /// + /// Once a non-None value is produced from the map operation, all attempts + /// to the left of the match will be stopped, while attempts to the right + /// must continue in case a later match is found. + /// + /// Note that not all parallel iterators have a useful order, much like + /// sequential `HashMap` iteration, so "first" may be nebulous. If you + /// just want the first non-None value discovered anywhere in the iterator, + /// `find_map_any` is a better choice. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let c = ["lol", "NaN", "2", "5"]; + /// + /// let last_number = c.par_iter().find_map_last(|s| s.parse().ok()); + /// + /// assert_eq!(last_number, Some(5)); + /// ``` + fn find_map_last(self, predicate: P) -> Option + where + P: Fn(Self::Item) -> Option + Sync + Send, + R: Send, + { + fn yes(_: &T) -> bool { + true + } + self.filter_map(predicate).find_last(yes) + } + + #[doc(hidden)] + #[deprecated(note = "parallel `find` does not search in order -- use `find_any`, \\ + `find_first`, or `find_last`")] + fn find

(self, predicate: P) -> Option + where + P: Fn(&Self::Item) -> bool + Sync + Send, + { + self.find_any(predicate) + } + + /// Searches for **some** item in the parallel iterator that + /// matches the given predicate, and if so returns true. Once + /// a match is found, we'll attempt to stop process the rest + /// of the items. Proving that there's no match, returning false, + /// does require visiting every item. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let a = [0, 12, 3, 4, 0, 23, 0]; + /// + /// let is_valid = a.par_iter().any(|&x| x > 10); + /// + /// assert!(is_valid); + /// ``` + fn any

(self, predicate: P) -> bool + where + P: Fn(Self::Item) -> bool + Sync + Send, + { + self.map(predicate).find_any(bool::clone).is_some() + } + + /// Tests that every item in the parallel iterator matches the given + /// predicate, and if so returns true. If a counter-example is found, + /// we'll attempt to stop processing more items, then return false. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let a = [0, 12, 3, 4, 0, 23, 0]; + /// + /// let is_valid = a.par_iter().all(|&x| x > 10); + /// + /// assert!(!is_valid); + /// ``` + fn all

(self, predicate: P) -> bool + where + P: Fn(Self::Item) -> bool + Sync + Send, + { + #[inline] + fn is_false(x: &bool) -> bool { + !x + } + + self.map(predicate).find_any(is_false).is_none() + } + + /// Creates an iterator over the `Some` items of this iterator, halting + /// as soon as any `None` is found. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// use std::sync::atomic::{AtomicUsize, Ordering}; + /// + /// let counter = AtomicUsize::new(0); + /// let value = (0_i32..2048) + /// .into_par_iter() + /// .map(|x| { + /// counter.fetch_add(1, Ordering::SeqCst); + /// if x < 1024 { Some(x) } else { None } + /// }) + /// .while_some() + /// .max(); + /// + /// assert!(value < Some(1024)); + /// assert!(counter.load(Ordering::SeqCst) < 2048); // should not have visited every single one + /// ``` + fn while_some(self) -> WhileSome + where + Self: ParallelIterator>, + T: Send, + { + WhileSome::new(self) + } + + /// Wraps an iterator with a fuse in case of panics, to halt all threads + /// as soon as possible. + /// + /// Panics within parallel iterators are always propagated to the caller, + /// but they don't always halt the rest of the iterator right away, due to + /// the internal semantics of [`join`]. This adaptor makes a greater effort + /// to stop processing other items sooner, with the cost of additional + /// synchronization overhead, which may also inhibit some optimizations. + /// + /// [`join`]: crate::join()#panics + /// + /// # Examples + /// + /// If this code didn't use `panic_fuse()`, it would continue processing + /// many more items in other threads (with long sleep delays) before the + /// panic is finally propagated. + /// + /// ```should_panic + /// use rayon::prelude::*; + /// use std::{thread, time}; + /// + /// (0..1_000_000) + /// .into_par_iter() + /// .panic_fuse() + /// .for_each(|i| { + /// // simulate some work + /// thread::sleep(time::Duration::from_secs(1)); + /// assert!(i > 0); // oops! + /// }); + /// ``` + fn panic_fuse(self) -> PanicFuse { + PanicFuse::new(self) + } + + /// Creates a fresh collection containing all the elements produced + /// by this parallel iterator. + /// + /// You may prefer [`collect_into_vec()`] implemented on + /// [`IndexedParallelIterator`], if your underlying iterator also implements + /// it. [`collect_into_vec()`] allocates efficiently with precise knowledge + /// of how many elements the iterator contains, and even allows you to reuse + /// an existing vector's backing store rather than allocating a fresh vector. + /// + /// See also [`collect_vec_list()`] for collecting into a + /// `LinkedList>`. + /// + /// [`collect_into_vec()`]: IndexedParallelIterator::collect_into_vec() + /// [`collect_vec_list()`]: Self::collect_vec_list() + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let sync_vec: Vec<_> = (0..100).into_iter().collect(); + /// + /// let async_vec: Vec<_> = (0..100).into_par_iter().collect(); + /// + /// assert_eq!(sync_vec, async_vec); + /// ``` + /// + /// You can collect a pair of collections like [`unzip`](#method.unzip) + /// for paired items: + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let a = [(0, 1), (1, 2), (2, 3), (3, 4)]; + /// let (first, second): (Vec<_>, Vec<_>) = a.into_par_iter().collect(); + /// + /// assert_eq!(first, [0, 1, 2, 3]); + /// assert_eq!(second, [1, 2, 3, 4]); + /// ``` + /// + /// Or like [`partition_map`](#method.partition_map) for `Either` items: + /// + /// ``` + /// use rayon::prelude::*; + /// use rayon::iter::Either; + /// + /// let (left, right): (Vec<_>, Vec<_>) = (0..8).into_par_iter().map(|x| { + /// if x % 2 == 0 { + /// Either::Left(x * 4) + /// } else { + /// Either::Right(x * 3) + /// } + /// }).collect(); + /// + /// assert_eq!(left, [0, 8, 16, 24]); + /// assert_eq!(right, [3, 9, 15, 21]); + /// ``` + /// + /// You can even collect an arbitrarily-nested combination of pairs and `Either`: + /// + /// ``` + /// use rayon::prelude::*; + /// use rayon::iter::Either; + /// + /// let (first, (left, right)): (Vec<_>, (Vec<_>, Vec<_>)) + /// = (0..8).into_par_iter().map(|x| { + /// if x % 2 == 0 { + /// (x, Either::Left(x * 4)) + /// } else { + /// (-x, Either::Right(x * 3)) + /// } + /// }).collect(); + /// + /// assert_eq!(first, [0, -1, 2, -3, 4, -5, 6, -7]); + /// assert_eq!(left, [0, 8, 16, 24]); + /// assert_eq!(right, [3, 9, 15, 21]); + /// ``` + /// + /// All of that can _also_ be combined with short-circuiting collection of + /// `Result` or `Option` types: + /// + /// ``` + /// use rayon::prelude::*; + /// use rayon::iter::Either; + /// + /// let result: Result<(Vec<_>, (Vec<_>, Vec<_>)), _> + /// = (0..8).into_par_iter().map(|x| { + /// if x > 5 { + /// Err(x) + /// } else if x % 2 == 0 { + /// Ok((x, Either::Left(x * 4))) + /// } else { + /// Ok((-x, Either::Right(x * 3))) + /// } + /// }).collect(); + /// + /// let error = result.unwrap_err(); + /// assert!(error == 6 || error == 7); + /// ``` + fn collect(self) -> C + where + C: FromParallelIterator, + { + C::from_par_iter(self) + } + + /// Unzips the items of a parallel iterator into a pair of arbitrary + /// `ParallelExtend` containers. + /// + /// You may prefer to use `unzip_into_vecs()`, which allocates more + /// efficiently with precise knowledge of how many elements the + /// iterator contains, and even allows you to reuse existing + /// vectors' backing stores rather than allocating fresh vectors. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let a = [(0, 1), (1, 2), (2, 3), (3, 4)]; + /// + /// let (left, right): (Vec<_>, Vec<_>) = a.par_iter().cloned().unzip(); + /// + /// assert_eq!(left, [0, 1, 2, 3]); + /// assert_eq!(right, [1, 2, 3, 4]); + /// ``` + /// + /// Nested pairs can be unzipped too. + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let (values, (squares, cubes)): (Vec<_>, (Vec<_>, Vec<_>)) = (0..4).into_par_iter() + /// .map(|i| (i, (i * i, i * i * i))) + /// .unzip(); + /// + /// assert_eq!(values, [0, 1, 2, 3]); + /// assert_eq!(squares, [0, 1, 4, 9]); + /// assert_eq!(cubes, [0, 1, 8, 27]); + /// ``` + fn unzip(self) -> (FromA, FromB) + where + Self: ParallelIterator, + FromA: Default + Send + ParallelExtend, + FromB: Default + Send + ParallelExtend, + A: Send, + B: Send, + { + unzip::unzip(self) + } + + /// Partitions the items of a parallel iterator into a pair of arbitrary + /// `ParallelExtend` containers. Items for which the `predicate` returns + /// true go into the first container, and the rest go into the second. + /// + /// Note: unlike the standard `Iterator::partition`, this allows distinct + /// collection types for the left and right items. This is more flexible, + /// but may require new type annotations when converting sequential code + /// that used type inference assuming the two were the same. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let (left, right): (Vec<_>, Vec<_>) = (0..8).into_par_iter().partition(|x| x % 2 == 0); + /// + /// assert_eq!(left, [0, 2, 4, 6]); + /// assert_eq!(right, [1, 3, 5, 7]); + /// ``` + fn partition(self, predicate: P) -> (A, B) + where + A: Default + Send + ParallelExtend, + B: Default + Send + ParallelExtend, + P: Fn(&Self::Item) -> bool + Sync + Send, + { + unzip::partition(self, predicate) + } + + /// Partitions and maps the items of a parallel iterator into a pair of + /// arbitrary `ParallelExtend` containers. `Either::Left` items go into + /// the first container, and `Either::Right` items go into the second. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// use rayon::iter::Either; + /// + /// let (left, right): (Vec<_>, Vec<_>) = (0..8).into_par_iter() + /// .partition_map(|x| { + /// if x % 2 == 0 { + /// Either::Left(x * 4) + /// } else { + /// Either::Right(x * 3) + /// } + /// }); + /// + /// assert_eq!(left, [0, 8, 16, 24]); + /// assert_eq!(right, [3, 9, 15, 21]); + /// ``` + /// + /// Nested `Either` enums can be split as well. + /// + /// ``` + /// use rayon::prelude::*; + /// use rayon::iter::Either::*; + /// + /// let ((fizzbuzz, fizz), (buzz, other)): ((Vec<_>, Vec<_>), (Vec<_>, Vec<_>)) = (1..20) + /// .into_par_iter() + /// .partition_map(|x| match (x % 3, x % 5) { + /// (0, 0) => Left(Left(x)), + /// (0, _) => Left(Right(x)), + /// (_, 0) => Right(Left(x)), + /// (_, _) => Right(Right(x)), + /// }); + /// + /// assert_eq!(fizzbuzz, [15]); + /// assert_eq!(fizz, [3, 6, 9, 12, 18]); + /// assert_eq!(buzz, [5, 10]); + /// assert_eq!(other, [1, 2, 4, 7, 8, 11, 13, 14, 16, 17, 19]); + /// ``` + fn partition_map(self, predicate: P) -> (A, B) + where + A: Default + Send + ParallelExtend, + B: Default + Send + ParallelExtend, + P: Fn(Self::Item) -> Either + Sync + Send, + L: Send, + R: Send, + { + unzip::partition_map(self, predicate) + } + + /// Intersperses clones of an element between items of this iterator. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let x = vec![1, 2, 3]; + /// let r: Vec<_> = x.into_par_iter().intersperse(-1).collect(); + /// + /// assert_eq!(r, vec![1, -1, 2, -1, 3]); + /// ``` + fn intersperse(self, element: Self::Item) -> Intersperse + where + Self::Item: Clone, + { + Intersperse::new(self, element) + } + + /// Creates an iterator that yields `n` elements from *anywhere* in the original iterator. + /// + /// This is similar to [`IndexedParallelIterator::take`] without being + /// constrained to the "first" `n` of the original iterator order. The + /// taken items will still maintain their relative order where that is + /// visible in `collect`, `reduce`, and similar outputs. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let result: Vec<_> = (0..100) + /// .into_par_iter() + /// .filter(|&x| x % 2 == 0) + /// .take_any(5) + /// .collect(); + /// + /// assert_eq!(result.len(), 5); + /// assert!(result.windows(2).all(|w| w[0] < w[1])); + /// ``` + fn take_any(self, n: usize) -> TakeAny { + TakeAny::new(self, n) + } + + /// Creates an iterator that skips `n` elements from *anywhere* in the original iterator. + /// + /// This is similar to [`IndexedParallelIterator::skip`] without being + /// constrained to the "first" `n` of the original iterator order. The + /// remaining items will still maintain their relative order where that is + /// visible in `collect`, `reduce`, and similar outputs. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let result: Vec<_> = (0..100) + /// .into_par_iter() + /// .filter(|&x| x % 2 == 0) + /// .skip_any(5) + /// .collect(); + /// + /// assert_eq!(result.len(), 45); + /// assert!(result.windows(2).all(|w| w[0] < w[1])); + /// ``` + fn skip_any(self, n: usize) -> SkipAny { + SkipAny::new(self, n) + } + + /// Creates an iterator that takes elements from *anywhere* in the original iterator + /// until the given `predicate` returns `false`. + /// + /// The `predicate` may be anything -- e.g. it could be checking a fact about the item, a + /// global condition unrelated to the item itself, or some combination thereof. + /// + /// If parallel calls to the `predicate` race and give different results, then the + /// `true` results will still take those particular items, while respecting the `false` + /// result from elsewhere to skip any further items. + /// + /// This is similar to [`Iterator::take_while`] without being constrained to the original + /// iterator order. The taken items will still maintain their relative order where that is + /// visible in `collect`, `reduce`, and similar outputs. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let result: Vec<_> = (0..100) + /// .into_par_iter() + /// .take_any_while(|x| *x < 50) + /// .collect(); + /// + /// assert!(result.len() <= 50); + /// assert!(result.windows(2).all(|w| w[0] < w[1])); + /// ``` + /// + /// ``` + /// use rayon::prelude::*; + /// use std::sync::atomic::AtomicUsize; + /// use std::sync::atomic::Ordering::Relaxed; + /// + /// // Collect any group of items that sum <= 1000 + /// let quota = AtomicUsize::new(1000); + /// let result: Vec<_> = (0_usize..100) + /// .into_par_iter() + /// .take_any_while(|&x| { + /// quota.fetch_update(Relaxed, Relaxed, |q| q.checked_sub(x)) + /// .is_ok() + /// }) + /// .collect(); + /// + /// let sum = result.iter().sum::(); + /// assert!(matches!(sum, 902..=1000)); + /// ``` + fn take_any_while

(self, predicate: P) -> TakeAnyWhile + where + P: Fn(&Self::Item) -> bool + Sync + Send, + { + TakeAnyWhile::new(self, predicate) + } + + /// Creates an iterator that skips elements from *anywhere* in the original iterator + /// until the given `predicate` returns `false`. + /// + /// The `predicate` may be anything -- e.g. it could be checking a fact about the item, a + /// global condition unrelated to the item itself, or some combination thereof. + /// + /// If parallel calls to the `predicate` race and give different results, then the + /// `true` results will still skip those particular items, while respecting the `false` + /// result from elsewhere to skip any further items. + /// + /// This is similar to [`Iterator::skip_while`] without being constrained to the original + /// iterator order. The remaining items will still maintain their relative order where that is + /// visible in `collect`, `reduce`, and similar outputs. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let result: Vec<_> = (0..100) + /// .into_par_iter() + /// .skip_any_while(|x| *x < 50) + /// .collect(); + /// + /// assert!(result.len() >= 50); + /// assert!(result.windows(2).all(|w| w[0] < w[1])); + /// ``` + fn skip_any_while

(self, predicate: P) -> SkipAnyWhile + where + P: Fn(&Self::Item) -> bool + Sync + Send, + { + SkipAnyWhile::new(self, predicate) + } + + /// Collects this iterator into a linked list of vectors. + /// + /// This is useful when you need to condense a parallel iterator into a collection, + /// but have no specific requirements for what that collection should be. If you + /// plan to store the collection longer-term, `Vec` is, as always, likely the + /// best default choice, despite the overhead that comes from concatenating each + /// vector. Or, if this is an `IndexedParallelIterator`, you should also prefer to + /// just collect to a `Vec`. + /// + /// Internally, most [`FromParallelIterator`]/[`ParallelExtend`] implementations + /// use this strategy; each job collecting their chunk of the iterator to a `Vec` + /// and those chunks getting merged into a `LinkedList`, before then extending the + /// collection with each vector. This is a very efficient way to collect an + /// unindexed parallel iterator, without much intermediate data movement. + /// + /// # Examples + /// + /// ``` + /// # use std::collections::LinkedList; + /// use rayon::prelude::*; + /// + /// let result: LinkedList> = (0..=100) + /// .into_par_iter() + /// .filter(|x| x % 2 == 0) + /// .flat_map(|x| 0..x) + /// .collect_vec_list(); + /// + /// // `par_iter.collect_vec_list().into_iter().flatten()` turns + /// // a parallel iterator into a serial one + /// let total_len = result.into_iter().flatten().count(); + /// assert_eq!(total_len, 2550); + /// ``` + fn collect_vec_list(self) -> LinkedList> { + match extend::fast_collect(self) { + Either::Left(vec) => { + let mut list = LinkedList::new(); + if !vec.is_empty() { + list.push_back(vec); + } + list + } + Either::Right(list) => list, + } + } + + /// Internal method used to define the behavior of this parallel + /// iterator. You should not need to call this directly. + /// + /// This method causes the iterator `self` to start producing + /// items and to feed them to the consumer `consumer` one by one. + /// It may split the consumer before doing so to create the + /// opportunity to produce in parallel. + /// + /// See the [README] for more details on the internals of parallel + /// iterators. + /// + /// [README]: https://github.com/rayon-rs/rayon/blob/main/src/iter/plumbing/README.md + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer; + + /// Internal method used to define the behavior of this parallel + /// iterator. You should not need to call this directly. + /// + /// Returns the number of items produced by this iterator, if known + /// statically. This can be used by consumers to trigger special fast + /// paths. Therefore, if `Some(_)` is returned, this iterator must only + /// use the (indexed) `Consumer` methods when driving a consumer, such + /// as `split_at()`. Calling `UnindexedConsumer::split_off_left()` or + /// other `UnindexedConsumer` methods -- or returning an inaccurate + /// value -- may result in panics. + /// + /// This method is currently used to optimize `collect` for want + /// of true Rust specialization; it may be removed when + /// specialization is stable. + fn opt_len(&self) -> Option { + None + } +} + +impl IntoParallelIterator for T { + type Iter = T; + type Item = T::Item; + + fn into_par_iter(self) -> T { + self + } +} + +/// An iterator that supports "random access" to its data, meaning +/// that you can split it at arbitrary indices and draw data from +/// those points. +/// +/// **Note:** Not implemented for `u64`, `i64`, `u128`, or `i128` ranges +// Waiting for `ExactSizeIterator::is_empty` to be stabilized. See rust-lang/rust#35428 +#[allow(clippy::len_without_is_empty)] +pub trait IndexedParallelIterator: ParallelIterator { + /// Divides an iterator into sequential blocks of exponentially-increasing size. + /// + /// Normally, parallel iterators are recursively divided into tasks in parallel. + /// This adaptor changes the default behavior by splitting the iterator into a **sequence** + /// of parallel iterators of increasing sizes. + /// Sizes grow exponentially in order to avoid creating + /// too many blocks. This also allows to balance the current block with all previous ones. + /// + /// This can have many applications but the most notable ones are: + /// - better performance with [`find_first()`][ParallelIterator::find_first] + /// - more predictable performance with [`find_any()`][ParallelIterator::find_any] + /// or any interruptible computation + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// assert_eq!((0..10_000).into_par_iter() + /// .by_exponential_blocks() + /// .find_first(|&e| e==4_999), Some(4_999)) + /// ``` + /// + /// In this example, without blocks, rayon will split the initial range into two but all work + /// on the right hand side (from 5,000 onwards) is **useless** since the sequential algorithm + /// never goes there. This means that if two threads are used there will be **no** speedup **at + /// all**. + /// + /// `by_exponential_blocks` on the other hand will start with the leftmost range from 0 + /// to `p` (threads number), continue with p to 3p, the 3p to 7p... + /// + /// Each subrange is treated in parallel, while all subranges are treated sequentially. + /// We therefore ensure a logarithmic number of blocks (and overhead) while guaranteeing + /// we stop at the first block containing the searched data. + fn by_exponential_blocks(self) -> ExponentialBlocks { + ExponentialBlocks::new(self) + } + + /// Divides an iterator into sequential blocks of the given size. + /// + /// Normally, parallel iterators are recursively divided into tasks in parallel. + /// This adaptor changes the default behavior by splitting the iterator into a **sequence** + /// of parallel iterators of given `block_size`. + /// The main application is to obtain better + /// memory locality (especially if the reduce operation re-use folded data). + /// + /// **Panics** if `block_size` is 0. + /// + /// # Example + /// ``` + /// use rayon::prelude::*; + /// // during most reductions v1 and v2 fit the cache + /// let v = (0u32..10_000_000) + /// .into_par_iter() + /// .by_uniform_blocks(1_000_000) + /// .fold(Vec::new, |mut v, e| { v.push(e); v}) + /// .reduce(Vec::new, |mut v1, mut v2| { v1.append(&mut v2); v1}); + /// assert_eq!(v, (0u32..10_000_000).collect::>()); + /// ``` + #[track_caller] + fn by_uniform_blocks(self, block_size: usize) -> UniformBlocks { + assert!(block_size != 0, "block_size must not be zero"); + UniformBlocks::new(self, block_size) + } + + /// Collects the results of the iterator into the specified + /// vector. The vector is always cleared before execution + /// begins. If possible, reusing the vector across calls can lead + /// to better performance since it reuses the same backing buffer. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// // any prior data will be cleared + /// let mut vec = vec![-1, -2, -3]; + /// + /// (0..5).into_par_iter() + /// .collect_into_vec(&mut vec); + /// + /// assert_eq!(vec, [0, 1, 2, 3, 4]); + /// ``` + fn collect_into_vec(self, target: &mut Vec) { + collect::collect_into_vec(self, target); + } + + /// Unzips the results of the iterator into the specified + /// vectors. The vectors are always cleared before execution + /// begins. If possible, reusing the vectors across calls can lead + /// to better performance since they reuse the same backing buffer. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// // any prior data will be cleared + /// let mut left = vec![42; 10]; + /// let mut right = vec![-1; 10]; + /// + /// (10..15).into_par_iter() + /// .enumerate() + /// .unzip_into_vecs(&mut left, &mut right); + /// + /// assert_eq!(left, [0, 1, 2, 3, 4]); + /// assert_eq!(right, [10, 11, 12, 13, 14]); + /// ``` + fn unzip_into_vecs(self, left: &mut Vec, right: &mut Vec) + where + Self: IndexedParallelIterator, + A: Send, + B: Send, + { + collect::unzip_into_vecs(self, left, right); + } + + /// Iterates over tuples `(A, B)`, where the items `A` are from + /// this iterator and `B` are from the iterator given as argument. + /// Like the `zip` method on ordinary iterators, if the two + /// iterators are of unequal length, you only get the items they + /// have in common. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let result: Vec<_> = (1..4) + /// .into_par_iter() + /// .zip(vec!['a', 'b', 'c']) + /// .collect(); + /// + /// assert_eq!(result, [(1, 'a'), (2, 'b'), (3, 'c')]); + /// ``` + fn zip(self, zip_op: Z) -> Zip + where + Z: IntoParallelIterator, + { + Zip::new(self, zip_op.into_par_iter()) + } + + /// The same as `Zip`, but requires that both iterators have the same length. + /// + /// # Panics + /// Will panic if `self` and `zip_op` are not the same length. + /// + /// ```should_panic + /// use rayon::prelude::*; + /// + /// let one = [1u8]; + /// let two = [2u8, 2]; + /// let one_iter = one.par_iter(); + /// let two_iter = two.par_iter(); + /// + /// // this will panic + /// let zipped: Vec<(&u8, &u8)> = one_iter.zip_eq(two_iter).collect(); + /// + /// // we should never get here + /// assert_eq!(1, zipped.len()); + /// ``` + #[track_caller] + fn zip_eq(self, zip_op: Z) -> ZipEq + where + Z: IntoParallelIterator, + { + let zip_op_iter = zip_op.into_par_iter(); + assert_eq!( + self.len(), + zip_op_iter.len(), + "iterators must have the same length" + ); + ZipEq::new(self, zip_op_iter) + } + + /// Interleaves elements of this iterator and the other given + /// iterator. Alternately yields elements from this iterator and + /// the given iterator, until both are exhausted. If one iterator + /// is exhausted before the other, the last elements are provided + /// from the other. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// let (x, y) = (vec![1, 2], vec![3, 4, 5, 6]); + /// let r: Vec = x.into_par_iter().interleave(y).collect(); + /// assert_eq!(r, vec![1, 3, 2, 4, 5, 6]); + /// ``` + fn interleave(self, other: I) -> Interleave + where + I: IntoParallelIterator, + { + Interleave::new(self, other.into_par_iter()) + } + + /// Interleaves elements of this iterator and the other given + /// iterator, until one is exhausted. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// let (x, y) = (vec![1, 2, 3, 4], vec![5, 6]); + /// let r: Vec = x.into_par_iter().interleave_shortest(y).collect(); + /// assert_eq!(r, vec![1, 5, 2, 6, 3]); + /// ``` + fn interleave_shortest(self, other: I) -> InterleaveShortest + where + I: IntoParallelIterator, + { + InterleaveShortest::new(self, other.into_par_iter()) + } + + /// Splits an iterator up into fixed-size chunks. + /// + /// Returns an iterator that returns `Vec`s of the given number of elements. + /// If the number of elements in the iterator is not divisible by `chunk_size`, + /// the last chunk may be shorter than `chunk_size`. + /// + /// See also [`par_chunks()`] and [`par_chunks_mut()`] for similar behavior on + /// slices, without having to allocate intermediate `Vec`s for the chunks. + /// + /// [`par_chunks()`]: crate::slice::ParallelSlice::par_chunks() + /// [`par_chunks_mut()`]: crate::slice::ParallelSliceMut::par_chunks_mut() + /// + /// **Panics** if `chunk_size` is 0. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// let a = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + /// let r: Vec> = a.into_par_iter().chunks(3).collect(); + /// assert_eq!(r, vec![vec![1,2,3], vec![4,5,6], vec![7,8,9], vec![10]]); + /// ``` + #[track_caller] + fn chunks(self, chunk_size: usize) -> Chunks { + assert!(chunk_size != 0, "chunk_size must not be zero"); + Chunks::new(self, chunk_size) + } + + /// Splits an iterator into fixed-size chunks, performing a sequential [`fold()`] on + /// each chunk. + /// + /// Returns an iterator that produces a folded result for each chunk of items + /// produced by this iterator. + /// + /// This works essentially like: + /// + /// ```text + /// iter.chunks(chunk_size) + /// .map(|chunk| + /// chunk.into_iter() + /// .fold(identity, fold_op) + /// ) + /// ``` + /// + /// except there is no per-chunk allocation overhead. + /// + /// [`fold()`]: std::iter::Iterator#method.fold + /// + /// **Panics** if `chunk_size` is 0. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// let nums = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + /// let chunk_sums = nums.into_par_iter().fold_chunks(2, || 0, |a, n| a + n).collect::>(); + /// assert_eq!(chunk_sums, vec![3, 7, 11, 15, 19]); + /// ``` + #[track_caller] + fn fold_chunks( + self, + chunk_size: usize, + identity: ID, + fold_op: F, + ) -> FoldChunks + where + ID: Fn() -> T + Send + Sync, + F: Fn(T, Self::Item) -> T + Send + Sync, + T: Send, + { + assert!(chunk_size != 0, "chunk_size must not be zero"); + FoldChunks::new(self, chunk_size, identity, fold_op) + } + + /// Splits an iterator into fixed-size chunks, performing a sequential [`fold()`] on + /// each chunk. + /// + /// Returns an iterator that produces a folded result for each chunk of items + /// produced by this iterator. + /// + /// This works essentially like `fold_chunks(chunk_size, || init.clone(), fold_op)`, + /// except it doesn't require the `init` type to be `Sync`, nor any other form of + /// added synchronization. + /// + /// [`fold()`]: std::iter::Iterator#method.fold + /// + /// **Panics** if `chunk_size` is 0. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// let nums = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + /// let chunk_sums = nums.into_par_iter().fold_chunks_with(2, 0, |a, n| a + n).collect::>(); + /// assert_eq!(chunk_sums, vec![3, 7, 11, 15, 19]); + /// ``` + #[track_caller] + fn fold_chunks_with( + self, + chunk_size: usize, + init: T, + fold_op: F, + ) -> FoldChunksWith + where + T: Send + Clone, + F: Fn(T, Self::Item) -> T + Send + Sync, + { + assert!(chunk_size != 0, "chunk_size must not be zero"); + FoldChunksWith::new(self, chunk_size, init, fold_op) + } + + /// Lexicographically compares the elements of this `ParallelIterator` with those of + /// another. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// use std::cmp::Ordering::*; + /// + /// let x = vec![1, 2, 3]; + /// assert_eq!(x.par_iter().cmp(&vec![1, 3, 0]), Less); + /// assert_eq!(x.par_iter().cmp(&vec![1, 2, 3]), Equal); + /// assert_eq!(x.par_iter().cmp(&vec![1, 2]), Greater); + /// ``` + fn cmp(self, other: I) -> Ordering + where + I: IntoParallelIterator, + Self::Item: Ord, + { + #[inline] + fn ordering((x, y): (T, T)) -> Ordering { + Ord::cmp(&x, &y) + } + + #[inline] + fn inequal(&ord: &Ordering) -> bool { + ord != Ordering::Equal + } + + let other = other.into_par_iter(); + let ord_len = self.len().cmp(&other.len()); + self.zip(other) + .map(ordering) + .find_first(inequal) + .unwrap_or(ord_len) + } + + /// Lexicographically compares the elements of this `ParallelIterator` with those of + /// another. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// use std::cmp::Ordering::*; + /// + /// let x = vec![1.0, 2.0, 3.0]; + /// assert_eq!(x.par_iter().partial_cmp(&vec![1.0, 3.0, 0.0]), Some(Less)); + /// assert_eq!(x.par_iter().partial_cmp(&vec![1.0, 2.0, 3.0]), Some(Equal)); + /// assert_eq!(x.par_iter().partial_cmp(&vec![1.0, 2.0]), Some(Greater)); + /// assert_eq!(x.par_iter().partial_cmp(&vec![1.0, f64::NAN]), None); + /// ``` + fn partial_cmp(self, other: I) -> Option + where + I: IntoParallelIterator, + Self::Item: PartialOrd, + { + #[inline] + fn ordering, U>((x, y): (T, U)) -> Option { + PartialOrd::partial_cmp(&x, &y) + } + + #[inline] + fn inequal(&ord: &Option) -> bool { + ord != Some(Ordering::Equal) + } + + let other = other.into_par_iter(); + let ord_len = self.len().cmp(&other.len()); + self.zip(other) + .map(ordering) + .find_first(inequal) + .unwrap_or(Some(ord_len)) + } + + /// Determines if the elements of this `ParallelIterator` + /// are equal to those of another + fn eq(self, other: I) -> bool + where + I: IntoParallelIterator, + Self::Item: PartialEq, + { + #[inline] + fn eq, U>((x, y): (T, U)) -> bool { + PartialEq::eq(&x, &y) + } + + let other = other.into_par_iter(); + self.len() == other.len() && self.zip(other).all(eq) + } + + /// Determines if the elements of this `ParallelIterator` + /// are unequal to those of another + fn ne(self, other: I) -> bool + where + I: IntoParallelIterator, + Self::Item: PartialEq, + { + !self.eq(other) + } + + /// Determines if the elements of this `ParallelIterator` + /// are lexicographically less than those of another. + fn lt(self, other: I) -> bool + where + I: IntoParallelIterator, + Self::Item: PartialOrd, + { + self.partial_cmp(other) == Some(Ordering::Less) + } + + /// Determines if the elements of this `ParallelIterator` + /// are less than or equal to those of another. + fn le(self, other: I) -> bool + where + I: IntoParallelIterator, + Self::Item: PartialOrd, + { + let ord = self.partial_cmp(other); + ord == Some(Ordering::Equal) || ord == Some(Ordering::Less) + } + + /// Determines if the elements of this `ParallelIterator` + /// are lexicographically greater than those of another. + fn gt(self, other: I) -> bool + where + I: IntoParallelIterator, + Self::Item: PartialOrd, + { + self.partial_cmp(other) == Some(Ordering::Greater) + } + + /// Determines if the elements of this `ParallelIterator` + /// are greater than or equal to those of another. + fn ge(self, other: I) -> bool + where + I: IntoParallelIterator, + Self::Item: PartialOrd, + { + let ord = self.partial_cmp(other); + ord == Some(Ordering::Equal) || ord == Some(Ordering::Greater) + } + + /// Yields an index along with each item. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let chars = vec!['a', 'b', 'c']; + /// let result: Vec<_> = chars + /// .into_par_iter() + /// .enumerate() + /// .collect(); + /// + /// assert_eq!(result, [(0, 'a'), (1, 'b'), (2, 'c')]); + /// ``` + fn enumerate(self) -> Enumerate { + Enumerate::new(self) + } + + /// Creates an iterator that steps by the given amount + /// + /// # Examples + /// + /// ``` + ///use rayon::prelude::*; + /// + /// let range = (3..10); + /// let result: Vec = range + /// .into_par_iter() + /// .step_by(3) + /// .collect(); + /// + /// assert_eq!(result, [3, 6, 9]) + /// ``` + fn step_by(self, step: usize) -> StepBy { + StepBy::new(self, step) + } + + /// Creates an iterator that skips the first `n` elements. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let result: Vec<_> = (0..100) + /// .into_par_iter() + /// .skip(95) + /// .collect(); + /// + /// assert_eq!(result, [95, 96, 97, 98, 99]); + /// ``` + fn skip(self, n: usize) -> Skip { + Skip::new(self, n) + } + + /// Creates an iterator that yields the first `n` elements. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let result: Vec<_> = (0..100) + /// .into_par_iter() + /// .take(5) + /// .collect(); + /// + /// assert_eq!(result, [0, 1, 2, 3, 4]); + /// ``` + fn take(self, n: usize) -> Take { + Take::new(self, n) + } + + /// Searches for **some** item in the parallel iterator that + /// matches the given predicate, and returns its index. Like + /// `ParallelIterator::find_any`, the parallel search will not + /// necessarily find the **first** match, and once a match is + /// found we'll attempt to stop processing any more. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let a = [1, 2, 3, 3]; + /// + /// let i = a.par_iter().position_any(|&x| x == 3).expect("found"); + /// assert!(i == 2 || i == 3); + /// + /// assert_eq!(a.par_iter().position_any(|&x| x == 100), None); + /// ``` + fn position_any

(self, predicate: P) -> Option + where + P: Fn(Self::Item) -> bool + Sync + Send, + { + #[inline] + fn check(&(_, p): &(usize, bool)) -> bool { + p + } + + let (i, _) = self.map(predicate).enumerate().find_any(check)?; + Some(i) + } + + /// Searches for the sequentially **first** item in the parallel iterator + /// that matches the given predicate, and returns its index. + /// + /// Like `ParallelIterator::find_first`, once a match is found, + /// all attempts to the right of the match will be stopped, while + /// attempts to the left must continue in case an earlier match + /// is found. + /// + /// Note that not all parallel iterators have a useful order, much like + /// sequential `HashMap` iteration, so "first" may be nebulous. If you + /// just want the first match that discovered anywhere in the iterator, + /// `position_any` is a better choice. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let a = [1, 2, 3, 3]; + /// + /// assert_eq!(a.par_iter().position_first(|&x| x == 3), Some(2)); + /// + /// assert_eq!(a.par_iter().position_first(|&x| x == 100), None); + /// ``` + fn position_first

(self, predicate: P) -> Option + where + P: Fn(Self::Item) -> bool + Sync + Send, + { + #[inline] + fn check(&(_, p): &(usize, bool)) -> bool { + p + } + + let (i, _) = self.map(predicate).enumerate().find_first(check)?; + Some(i) + } + + /// Searches for the sequentially **last** item in the parallel iterator + /// that matches the given predicate, and returns its index. + /// + /// Like `ParallelIterator::find_last`, once a match is found, + /// all attempts to the left of the match will be stopped, while + /// attempts to the right must continue in case a later match + /// is found. + /// + /// Note that not all parallel iterators have a useful order, much like + /// sequential `HashMap` iteration, so "last" may be nebulous. When the + /// order doesn't actually matter to you, `position_any` is a better + /// choice. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let a = [1, 2, 3, 3]; + /// + /// assert_eq!(a.par_iter().position_last(|&x| x == 3), Some(3)); + /// + /// assert_eq!(a.par_iter().position_last(|&x| x == 100), None); + /// ``` + fn position_last

(self, predicate: P) -> Option + where + P: Fn(Self::Item) -> bool + Sync + Send, + { + #[inline] + fn check(&(_, p): &(usize, bool)) -> bool { + p + } + + let (i, _) = self.map(predicate).enumerate().find_last(check)?; + Some(i) + } + + #[doc(hidden)] + #[deprecated( + note = "parallel `position` does not search in order -- use `position_any`, \\ + `position_first`, or `position_last`" + )] + fn position

(self, predicate: P) -> Option + where + P: Fn(Self::Item) -> bool + Sync + Send, + { + self.position_any(predicate) + } + + /// Searches for items in the parallel iterator that match the given + /// predicate, and returns their indices. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let primes = vec![2, 3, 5, 7, 11, 13, 17, 19, 23, 29]; + /// + /// // Find the positions of primes congruent to 1 modulo 6 + /// let p1mod6: Vec<_> = primes.par_iter().positions(|&p| p % 6 == 1).collect(); + /// assert_eq!(p1mod6, [3, 5, 7]); // primes 7, 13, and 19 + /// + /// // Find the positions of primes congruent to 5 modulo 6 + /// let p5mod6: Vec<_> = primes.par_iter().positions(|&p| p % 6 == 5).collect(); + /// assert_eq!(p5mod6, [2, 4, 6, 8, 9]); // primes 5, 11, 17, 23, and 29 + /// ``` + fn positions

(self, predicate: P) -> Positions + where + P: Fn(Self::Item) -> bool + Sync + Send, + { + Positions::new(self, predicate) + } + + /// Produces a new iterator with the elements of this iterator in + /// reverse order. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let result: Vec<_> = (0..5) + /// .into_par_iter() + /// .rev() + /// .collect(); + /// + /// assert_eq!(result, [4, 3, 2, 1, 0]); + /// ``` + fn rev(self) -> Rev { + Rev::new(self) + } + + /// Sets the minimum length of iterators desired to process in each + /// rayon job. Rayon will not split any smaller than this length, but + /// of course an iterator could already be smaller to begin with. + /// + /// Producers like `zip` and `interleave` will use greater of the two + /// minimums. + /// Chained iterators and iterators inside `flat_map` may each use + /// their own minimum length. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let min = (0..1_000_000) + /// .into_par_iter() + /// .with_min_len(1234) + /// .fold(|| 0, |acc, _| acc + 1) // count how many are in this segment + /// .min().unwrap(); + /// + /// assert!(min >= 1234); + /// ``` + fn with_min_len(self, min: usize) -> MinLen { + MinLen::new(self, min) + } + + /// Sets the maximum length of iterators desired to process in each + /// rayon job. Rayon will try to split at least below this length, + /// unless that would put it below the length from `with_min_len()`. + /// For example, given min=10 and max=15, a length of 16 will not be + /// split any further. + /// + /// Producers like `zip` and `interleave` will use lesser of the two + /// maximums. + /// Chained iterators and iterators inside `flat_map` may each use + /// their own maximum length. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let max = (0..1_000_000) + /// .into_par_iter() + /// .with_max_len(1234) + /// .fold(|| 0, |acc, _| acc + 1) // count how many are in this segment + /// .max().unwrap(); + /// + /// assert!(max <= 1234); + /// ``` + fn with_max_len(self, max: usize) -> MaxLen { + MaxLen::new(self, max) + } + + /// Produces an exact count of how many items this iterator will + /// produce, presuming no panic occurs. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let par_iter = (0..100).into_par_iter().zip(vec![0; 10]); + /// assert_eq!(par_iter.len(), 10); + /// + /// let vec: Vec<_> = par_iter.collect(); + /// assert_eq!(vec.len(), 10); + /// ``` + fn len(&self) -> usize; + + /// Internal method used to define the behavior of this parallel + /// iterator. You should not need to call this directly. + /// + /// This method causes the iterator `self` to start producing + /// items and to feed them to the consumer `consumer` one by one. + /// It may split the consumer before doing so to create the + /// opportunity to produce in parallel. If a split does happen, it + /// will inform the consumer of the index where the split should + /// occur (unlike `ParallelIterator::drive_unindexed()`). + /// + /// See the [README] for more details on the internals of parallel + /// iterators. + /// + /// [README]: https://github.com/rayon-rs/rayon/blob/main/src/iter/plumbing/README.md + fn drive>(self, consumer: C) -> C::Result; + + /// Internal method used to define the behavior of this parallel + /// iterator. You should not need to call this directly. + /// + /// This method converts the iterator into a producer P and then + /// invokes `callback.callback()` with P. Note that the type of + /// this producer is not defined as part of the API, since + /// `callback` must be defined generically for all producers. This + /// allows the producer type to contain references; it also means + /// that parallel iterators can adjust that type without causing a + /// breaking change. + /// + /// See the [README] for more details on the internals of parallel + /// iterators. + /// + /// [README]: https://github.com/rayon-rs/rayon/blob/main/src/iter/plumbing/README.md + fn with_producer>(self, callback: CB) -> CB::Output; +} + +/// `FromParallelIterator` implements the creation of a collection +/// from a [`ParallelIterator`]. By implementing +/// `FromParallelIterator` for a given type, you define how it will be +/// created from an iterator. +/// +/// `FromParallelIterator` is used through [`ParallelIterator`]'s [`collect()`] method. +/// +/// [`collect()`]: ParallelIterator::collect() +/// +/// # Examples +/// +/// Implementing `FromParallelIterator` for your type: +/// +/// ``` +/// use rayon::prelude::*; +/// +/// struct BlackHole { +/// mass: usize, +/// } +/// +/// impl FromParallelIterator for BlackHole { +/// fn from_par_iter(par_iter: I) -> Self +/// where I: IntoParallelIterator +/// { +/// let par_iter = par_iter.into_par_iter(); +/// BlackHole { +/// mass: par_iter.count() * size_of::(), +/// } +/// } +/// } +/// +/// let bh: BlackHole = (0i32..1000).into_par_iter().collect(); +/// assert_eq!(bh.mass, 4000); +/// ``` +pub trait FromParallelIterator +where + T: Send, +{ + /// Creates an instance of the collection from the parallel iterator `par_iter`. + /// + /// If your collection is not naturally parallel, the easiest (and + /// fastest) way to do this is often to collect `par_iter` into a + /// [`LinkedList`] (via [`collect_vec_list`]) or another intermediate + /// data structure and then sequentially extend your collection. However, + /// a more 'native' technique is to use the [`par_iter.fold`] or + /// [`par_iter.fold_with`] methods to create the collection. + /// Alternatively, if your collection is 'natively' parallel, you + /// can use [`par_iter.for_each`] to process each element in turn. + /// + /// [`LinkedList`]: std::collections::LinkedList + /// [`collect_vec_list`]: ParallelIterator::collect_vec_list + /// [`par_iter.fold`]: ParallelIterator::fold() + /// [`par_iter.fold_with`]: ParallelIterator::fold_with() + /// [`par_iter.for_each`]: ParallelIterator::for_each() + fn from_par_iter(par_iter: I) -> Self + where + I: IntoParallelIterator; +} + +/// `ParallelExtend` extends an existing collection with items from a [`ParallelIterator`]. +/// +/// # Examples +/// +/// Implementing `ParallelExtend` for your type: +/// +/// ``` +/// use rayon::prelude::*; +/// +/// struct BlackHole { +/// mass: usize, +/// } +/// +/// impl ParallelExtend for BlackHole { +/// fn par_extend(&mut self, par_iter: I) +/// where I: IntoParallelIterator +/// { +/// let par_iter = par_iter.into_par_iter(); +/// self.mass += par_iter.count() * size_of::(); +/// } +/// } +/// +/// let mut bh = BlackHole { mass: 0 }; +/// bh.par_extend(0i32..1000); +/// assert_eq!(bh.mass, 4000); +/// bh.par_extend(0i64..10); +/// assert_eq!(bh.mass, 4080); +/// ``` +pub trait ParallelExtend +where + T: Send, +{ + /// Extends an instance of the collection with the elements drawn + /// from the parallel iterator `par_iter`. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let mut vec = vec![]; + /// vec.par_extend(0..5); + /// vec.par_extend((0..5).into_par_iter().map(|i| i * i)); + /// assert_eq!(vec, [0, 1, 2, 3, 4, 0, 1, 4, 9, 16]); + /// ``` + fn par_extend(&mut self, par_iter: I) + where + I: IntoParallelIterator; +} + +/// `ParallelDrainFull` creates a parallel iterator that moves all items +/// from a collection while retaining the original capacity. +/// +/// Types which are indexable typically implement [`ParallelDrainRange`] +/// instead, where you can drain fully with `par_drain(..)`. +pub trait ParallelDrainFull { + /// The draining parallel iterator type that will be created. + type Iter: ParallelIterator; + + /// The type of item that the parallel iterator will produce. + /// This is usually the same as `IntoParallelIterator::Item`. + type Item: Send; + + /// Returns a draining parallel iterator over an entire collection. + /// + /// When the iterator is dropped, all items are removed, even if the + /// iterator was not fully consumed. If the iterator is leaked, for example + /// using `std::mem::forget`, it is unspecified how many items are removed. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// use std::collections::{BinaryHeap, HashSet}; + /// + /// let squares: HashSet = (0..10).map(|x| x * x).collect(); + /// + /// let mut heap: BinaryHeap<_> = squares.iter().copied().collect(); + /// assert_eq!( + /// // heaps are drained in arbitrary order + /// heap.par_drain() + /// .inspect(|x| assert!(squares.contains(x))) + /// .count(), + /// squares.len(), + /// ); + /// assert!(heap.is_empty()); + /// assert!(heap.capacity() >= squares.len()); + /// ``` + fn par_drain(self) -> Self::Iter; +} + +/// `ParallelDrainRange` creates a parallel iterator that moves a range of items +/// from a collection while retaining the original capacity. +/// +/// Types which are not indexable may implement [`ParallelDrainFull`] instead. +pub trait ParallelDrainRange { + /// The draining parallel iterator type that will be created. + type Iter: ParallelIterator; + + /// The type of item that the parallel iterator will produce. + /// This is usually the same as `IntoParallelIterator::Item`. + type Item: Send; + + /// Returns a draining parallel iterator over a range of the collection. + /// + /// When the iterator is dropped, all items in the range are removed, even + /// if the iterator was not fully consumed. If the iterator is leaked, for + /// example using `std::mem::forget`, it is unspecified how many items are + /// removed. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let squares: Vec = (0..10).map(|x| x * x).collect(); + /// + /// println!("RangeFull"); + /// let mut vec = squares.clone(); + /// assert!(vec.par_drain(..) + /// .eq(squares.par_iter().copied())); + /// assert!(vec.is_empty()); + /// assert!(vec.capacity() >= squares.len()); + /// + /// println!("RangeFrom"); + /// let mut vec = squares.clone(); + /// assert!(vec.par_drain(5..) + /// .eq(squares[5..].par_iter().copied())); + /// assert_eq!(&vec[..], &squares[..5]); + /// assert!(vec.capacity() >= squares.len()); + /// + /// println!("RangeTo"); + /// let mut vec = squares.clone(); + /// assert!(vec.par_drain(..5) + /// .eq(squares[..5].par_iter().copied())); + /// assert_eq!(&vec[..], &squares[5..]); + /// assert!(vec.capacity() >= squares.len()); + /// + /// println!("RangeToInclusive"); + /// let mut vec = squares.clone(); + /// assert!(vec.par_drain(..=5) + /// .eq(squares[..=5].par_iter().copied())); + /// assert_eq!(&vec[..], &squares[6..]); + /// assert!(vec.capacity() >= squares.len()); + /// + /// println!("Range"); + /// let mut vec = squares.clone(); + /// assert!(vec.par_drain(3..7) + /// .eq(squares[3..7].par_iter().copied())); + /// assert_eq!(&vec[..3], &squares[..3]); + /// assert_eq!(&vec[3..], &squares[7..]); + /// assert!(vec.capacity() >= squares.len()); + /// + /// println!("RangeInclusive"); + /// let mut vec = squares.clone(); + /// assert!(vec.par_drain(3..=7) + /// .eq(squares[3..=7].par_iter().copied())); + /// assert_eq!(&vec[..3], &squares[..3]); + /// assert_eq!(&vec[3..], &squares[8..]); + /// assert!(vec.capacity() >= squares.len()); + /// ``` + fn par_drain>(self, range: R) -> Self::Iter; +} + +/// We hide the `Try` trait in a private module, as it's only meant to be a +/// stable clone of the standard library's `Try` trait, as yet unstable. +mod private { + use std::convert::Infallible; + use std::ops::ControlFlow::{self, Break, Continue}; + use std::task::Poll; + + /// Clone of `std::ops::Try`. + /// + /// Implementing this trait is not permitted outside of `rayon`. + pub trait Try { + private_decl! {} + + type Output; + type Residual; + + fn from_output(output: Self::Output) -> Self; + + fn from_residual(residual: Self::Residual) -> Self; + + fn branch(self) -> ControlFlow; + } + + impl Try for ControlFlow { + private_impl! {} + + type Output = C; + type Residual = ControlFlow; + + fn from_output(output: Self::Output) -> Self { + Continue(output) + } + + fn from_residual(residual: Self::Residual) -> Self { + match residual { + Break(b) => Break(b), + #[allow(unreachable_patterns)] + Continue(_) => unreachable!(), + } + } + + fn branch(self) -> ControlFlow { + match self { + Continue(c) => Continue(c), + Break(b) => Break(Break(b)), + } + } + } + + impl Try for Option { + private_impl! {} + + type Output = T; + type Residual = Option; + + fn from_output(output: Self::Output) -> Self { + Some(output) + } + + fn from_residual(residual: Self::Residual) -> Self { + match residual { + None => None, + #[allow(unreachable_patterns)] + Some(_) => unreachable!(), + } + } + + fn branch(self) -> ControlFlow { + match self { + Some(c) => Continue(c), + None => Break(None), + } + } + } + + impl Try for Result { + private_impl! {} + + type Output = T; + type Residual = Result; + + fn from_output(output: Self::Output) -> Self { + Ok(output) + } + + fn from_residual(residual: Self::Residual) -> Self { + match residual { + Err(e) => Err(e), + #[allow(unreachable_patterns)] + Ok(_) => unreachable!(), + } + } + + fn branch(self) -> ControlFlow { + match self { + Ok(c) => Continue(c), + Err(e) => Break(Err(e)), + } + } + } + + impl Try for Poll> { + private_impl! {} + + type Output = Poll; + type Residual = Result; + + fn from_output(output: Self::Output) -> Self { + output.map(Ok) + } + + fn from_residual(residual: Self::Residual) -> Self { + match residual { + Err(e) => Poll::Ready(Err(e)), + #[allow(unreachable_patterns)] + Ok(_) => unreachable!(), + } + } + + fn branch(self) -> ControlFlow { + match self { + Poll::Pending => Continue(Poll::Pending), + Poll::Ready(Ok(c)) => Continue(Poll::Ready(c)), + Poll::Ready(Err(e)) => Break(Err(e)), + } + } + } + + impl Try for Poll>> { + private_impl! {} + + type Output = Poll>; + type Residual = Result; + + fn from_output(output: Self::Output) -> Self { + match output { + Poll::Ready(o) => Poll::Ready(o.map(Ok)), + Poll::Pending => Poll::Pending, + } + } + + fn from_residual(residual: Self::Residual) -> Self { + match residual { + Err(e) => Poll::Ready(Some(Err(e))), + #[allow(unreachable_patterns)] + Ok(_) => unreachable!(), + } + } + + fn branch(self) -> ControlFlow { + match self { + Poll::Pending => Continue(Poll::Pending), + Poll::Ready(None) => Continue(Poll::Ready(None)), + Poll::Ready(Some(Ok(c))) => Continue(Poll::Ready(Some(c))), + Poll::Ready(Some(Err(e))) => Break(Err(e)), + } + } + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/multizip.rs b/anneal/v2/vendor/rayon/src/iter/multizip.rs new file mode 100644 index 0000000000..51d38421bb --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/multizip.rs @@ -0,0 +1,335 @@ +use super::plumbing::*; +use super::*; + +/// `MultiZip` is an iterator that zips up a tuple of parallel iterators to +/// produce tuples of their items. +/// +/// It is created by calling `into_par_iter()` on a tuple of types that +/// implement `IntoParallelIterator`, or `par_iter()`/`par_iter_mut()` with +/// types that are iterable by reference. +/// +/// The implementation currently support tuples up to length 12. +/// +/// # Examples +/// +/// ``` +/// use rayon::prelude::*; +/// +/// // This will iterate `r` by mutable reference, like `par_iter_mut()`, while +/// // ranges are all iterated by value like `into_par_iter()`. +/// // Note that the zipped iterator is only as long as the shortest input. +/// let mut r = vec![0; 3]; +/// (&mut r, 1..10, 10..100, 100..1000).into_par_iter() +/// .for_each(|(r, x, y, z)| *r = x * y + z); +/// +/// assert_eq!(&r, &[1 * 10 + 100, 2 * 11 + 101, 3 * 12 + 102]); +/// ``` +/// +/// For a group that should all be iterated by reference, you can use a tuple reference. +/// +/// ``` +/// use rayon::prelude::*; +/// +/// let xs: Vec<_> = (1..10).collect(); +/// let ys: Vec<_> = (10..100).collect(); +/// let zs: Vec<_> = (100..1000).collect(); +/// +/// // Reference each input separately with `IntoParallelIterator`: +/// let r1: Vec<_> = (&xs, &ys, &zs).into_par_iter() +/// .map(|(x, y, z)| x * y + z) +/// .collect(); +/// +/// // Reference them all together with `IntoParallelRefIterator`: +/// let r2: Vec<_> = (xs, ys, zs).par_iter() +/// .map(|(x, y, z)| x * y + z) +/// .collect(); +/// +/// assert_eq!(r1, r2); +/// ``` +/// +/// Mutable references to a tuple will work similarly. +/// +/// ``` +/// use rayon::prelude::*; +/// +/// let mut xs: Vec<_> = (1..4).collect(); +/// let mut ys: Vec<_> = (-4..-1).collect(); +/// let mut zs = vec![0; 3]; +/// +/// // Mutably reference each input separately with `IntoParallelIterator`: +/// (&mut xs, &mut ys, &mut zs).into_par_iter().for_each(|(x, y, z)| { +/// *z += *x + *y; +/// std::mem::swap(x, y); +/// }); +/// +/// assert_eq!(xs, (vec![-4, -3, -2])); +/// assert_eq!(ys, (vec![1, 2, 3])); +/// assert_eq!(zs, (vec![-3, -1, 1])); +/// +/// // Mutably reference them all together with `IntoParallelRefMutIterator`: +/// let mut tuple = (xs, ys, zs); +/// tuple.par_iter_mut().for_each(|(x, y, z)| { +/// *z += *x + *y; +/// std::mem::swap(x, y); +/// }); +/// +/// assert_eq!(tuple, (vec![1, 2, 3], vec![-4, -3, -2], vec![-6, -2, 2])); +/// ``` +#[derive(Debug, Clone)] +pub struct MultiZip { + tuple: T, +} + +// These macros greedily consume 4 or 2 items first to achieve log2 nesting depth. +// For example, 5 => 4,1 => (2,2),1. +// +// The tuples go up to 12, so we might want to greedily consume 8 too, but +// the depth works out the same if we let that expand on the right: +// 9 => 4,5 => (2,2),(4,1) => (2,2),((2,2),1) +// 12 => 4,8 => (2,2),(4,4) => (2,2),((2,2),(2,2)) +// +// But if we ever increase to 13, we would want to split 8,5 rather than 4,9. + +macro_rules! reduce { + ($a:expr, $b:expr, $c:expr, $d:expr, $( $x:expr ),+ => $fn:path) => { + reduce!(reduce!($a, $b, $c, $d => $fn), + reduce!($( $x ),+ => $fn) + => $fn) + }; + ($a:expr, $b:expr, $( $x:expr ),+ => $fn:path) => { + reduce!(reduce!($a, $b => $fn), + reduce!($( $x ),+ => $fn) + => $fn) + }; + ($a:expr, $b:expr => $fn:path) => { $fn($a, $b) }; + ($a:expr => $fn:path) => { $a }; +} + +macro_rules! nest { + ($A:tt, $B:tt, $C:tt, $D:tt, $( $X:tt ),+) => { + (nest!($A, $B, $C, $D), nest!($( $X ),+)) + }; + ($A:tt, $B:tt, $( $X:tt ),+) => { + (($A, $B), nest!($( $X ),+)) + }; + ($A:tt, $B:tt) => { ($A, $B) }; + ($A:tt) => { $A }; +} + +macro_rules! flatten { + ($( $T:ident ),+) => {{ + #[allow(non_snake_case)] + fn flatten<$( $T ),+>(nest!($( $T ),+) : nest!($( $T ),+)) -> ($( $T, )+) { + ($( $T, )+) + } + flatten + }}; +} + +macro_rules! multizip_impls { + ($( + $Tuple:ident { + $(($idx:tt) -> $T:ident)+ + } + )+) => { + $( + impl<$( $T, )+> IntoParallelIterator for ($( $T, )+) + where + $( + $T: IntoParallelIterator, + )+ + { + type Item = ($( $T::Item, )+); + type Iter = MultiZip<($( $T::Iter, )+)>; + + fn into_par_iter(self) -> Self::Iter { + MultiZip { + tuple: ( $( self.$idx.into_par_iter(), )+ ), + } + } + } + + impl<'a, $( $T, )+> IntoParallelIterator for &'a ($( $T, )+) + where + $( + $T: IntoParallelRefIterator<'a, Iter: IndexedParallelIterator>, + )+ + { + type Item = ($( $T::Item, )+); + type Iter = MultiZip<($( $T::Iter, )+)>; + + fn into_par_iter(self) -> Self::Iter { + MultiZip { + tuple: ( $( self.$idx.par_iter(), )+ ), + } + } + } + + impl<'a, $( $T, )+> IntoParallelIterator for &'a mut ($( $T, )+) + where + $( + $T: IntoParallelRefMutIterator<'a, Iter: IndexedParallelIterator>, + )+ + { + type Item = ($( $T::Item, )+); + type Iter = MultiZip<($( $T::Iter, )+)>; + + fn into_par_iter(self) -> Self::Iter { + MultiZip { + tuple: ( $( self.$idx.par_iter_mut(), )+ ), + } + } + } + + impl<$( $T, )+> ParallelIterator for MultiZip<($( $T, )+)> + where + $( $T: IndexedParallelIterator, )+ + { + type Item = ($( $T::Item, )+); + + fn drive_unindexed(self, consumer: CONSUMER) -> CONSUMER::Result + where + CONSUMER: UnindexedConsumer, + { + self.drive(consumer) + } + + fn opt_len(&self) -> Option { + Some(self.len()) + } + } + + impl<$( $T, )+> IndexedParallelIterator for MultiZip<($( $T, )+)> + where + $( $T: IndexedParallelIterator, )+ + { + fn drive(self, consumer: CONSUMER) -> CONSUMER::Result + where + CONSUMER: Consumer, + { + reduce!($( self.tuple.$idx ),+ => IndexedParallelIterator::zip) + .map(flatten!($( $T ),+)) + .drive(consumer) + } + + fn len(&self) -> usize { + reduce!($( self.tuple.$idx.len() ),+ => Ord::min) + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + reduce!($( self.tuple.$idx ),+ => IndexedParallelIterator::zip) + .map(flatten!($( $T ),+)) + .with_producer(callback) + } + } + )+ + } +} + +multizip_impls! { + Tuple1 { + (0) -> A + } + Tuple2 { + (0) -> A + (1) -> B + } + Tuple3 { + (0) -> A + (1) -> B + (2) -> C + } + Tuple4 { + (0) -> A + (1) -> B + (2) -> C + (3) -> D + } + Tuple5 { + (0) -> A + (1) -> B + (2) -> C + (3) -> D + (4) -> E + } + Tuple6 { + (0) -> A + (1) -> B + (2) -> C + (3) -> D + (4) -> E + (5) -> F + } + Tuple7 { + (0) -> A + (1) -> B + (2) -> C + (3) -> D + (4) -> E + (5) -> F + (6) -> G + } + Tuple8 { + (0) -> A + (1) -> B + (2) -> C + (3) -> D + (4) -> E + (5) -> F + (6) -> G + (7) -> H + } + Tuple9 { + (0) -> A + (1) -> B + (2) -> C + (3) -> D + (4) -> E + (5) -> F + (6) -> G + (7) -> H + (8) -> I + } + Tuple10 { + (0) -> A + (1) -> B + (2) -> C + (3) -> D + (4) -> E + (5) -> F + (6) -> G + (7) -> H + (8) -> I + (9) -> J + } + Tuple11 { + (0) -> A + (1) -> B + (2) -> C + (3) -> D + (4) -> E + (5) -> F + (6) -> G + (7) -> H + (8) -> I + (9) -> J + (10) -> K + } + Tuple12 { + (0) -> A + (1) -> B + (2) -> C + (3) -> D + (4) -> E + (5) -> F + (6) -> G + (7) -> H + (8) -> I + (9) -> J + (10) -> K + (11) -> L + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/noop.rs b/anneal/v2/vendor/rayon/src/iter/noop.rs new file mode 100644 index 0000000000..1e55ecb20e --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/noop.rs @@ -0,0 +1,59 @@ +use super::plumbing::*; + +pub(super) struct NoopConsumer; + +impl Consumer for NoopConsumer { + type Folder = NoopConsumer; + type Reducer = NoopReducer; + type Result = (); + + fn split_at(self, _index: usize) -> (Self, Self, NoopReducer) { + (NoopConsumer, NoopConsumer, NoopReducer) + } + + fn into_folder(self) -> Self { + self + } + + fn full(&self) -> bool { + false + } +} + +impl Folder for NoopConsumer { + type Result = (); + + fn consume(self, _item: T) -> Self { + self + } + + fn consume_iter(self, iter: I) -> Self + where + I: IntoIterator, + { + iter.into_iter().for_each(drop); + self + } + + fn complete(self) {} + + fn full(&self) -> bool { + false + } +} + +impl UnindexedConsumer for NoopConsumer { + fn split_off_left(&self) -> Self { + NoopConsumer + } + + fn to_reducer(&self) -> NoopReducer { + NoopReducer + } +} + +pub(super) struct NoopReducer; + +impl Reducer<()> for NoopReducer { + fn reduce(self, _left: (), _right: ()) {} +} diff --git a/anneal/v2/vendor/rayon/src/iter/once.rs b/anneal/v2/vendor/rayon/src/iter/once.rs new file mode 100644 index 0000000000..4664d3d5f3 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/once.rs @@ -0,0 +1,70 @@ +use crate::iter::plumbing::*; +use crate::iter::*; + +/// Creates a parallel iterator that produces an element exactly once. +/// +/// This admits no parallelism on its own, but it could be chained to existing +/// parallel iterators to extend their contents, or otherwise used for any code +/// that deals with generic parallel iterators. +/// +/// # Examples +/// +/// ``` +/// use rayon::prelude::*; +/// use rayon::iter::once; +/// +/// let pi = (0..1234).into_par_iter() +/// .chain(once(-1)) +/// .chain(1234..10_000); +/// +/// assert_eq!(pi.clone().count(), 10_001); +/// assert_eq!(pi.clone().filter(|&x| x < 0).count(), 1); +/// assert_eq!(pi.position_any(|x| x < 0), Some(1234)); +/// ``` +pub fn once(item: T) -> Once { + Once { item } +} + +/// Iterator adaptor for [the `once()` function]. +/// +/// [the `once()` function]: once() +#[derive(Clone, Debug)] +pub struct Once { + item: T, +} + +impl ParallelIterator for Once { + type Item = T; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + self.drive(consumer) + } + + fn opt_len(&self) -> Option { + Some(1) + } +} + +impl IndexedParallelIterator for Once { + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + consumer.into_folder().consume(self.item).complete() + } + + fn len(&self) -> usize { + 1 + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + // Let `OptionProducer` handle it. + Some(self.item).into_par_iter().with_producer(callback) + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/panic_fuse.rs b/anneal/v2/vendor/rayon/src/iter/panic_fuse.rs new file mode 100644 index 0000000000..7616f672cb --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/panic_fuse.rs @@ -0,0 +1,338 @@ +use super::plumbing::*; +use super::*; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::thread; + +/// `PanicFuse` is an adaptor that wraps an iterator with a fuse in case +/// of panics, to halt all threads as soon as possible. +/// +/// This struct is created by the [`panic_fuse()`] method on [`ParallelIterator`] +/// +/// [`panic_fuse()`]: ParallelIterator::panic_fuse() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Debug, Clone)] +pub struct PanicFuse { + base: I, +} + +/// Helper that sets a bool to `true` if dropped while unwinding. +#[derive(Clone)] +struct Fuse<'a>(&'a AtomicBool); + +impl<'a> Drop for Fuse<'a> { + #[inline] + fn drop(&mut self) { + if thread::panicking() { + self.0.store(true, Ordering::Relaxed); + } + } +} + +impl<'a> Fuse<'a> { + #[inline] + fn panicked(&self) -> bool { + self.0.load(Ordering::Relaxed) + } +} + +impl PanicFuse { + /// Creates a new `PanicFuse` iterator. + pub(super) fn new(base: I) -> PanicFuse { + PanicFuse { base } + } +} + +impl ParallelIterator for PanicFuse +where + I: ParallelIterator, +{ + type Item = I::Item; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let panicked = AtomicBool::new(false); + let consumer1 = PanicFuseConsumer { + base: consumer, + fuse: Fuse(&panicked), + }; + self.base.drive_unindexed(consumer1) + } + + fn opt_len(&self) -> Option { + self.base.opt_len() + } +} + +impl IndexedParallelIterator for PanicFuse +where + I: IndexedParallelIterator, +{ + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + let panicked = AtomicBool::new(false); + let consumer1 = PanicFuseConsumer { + base: consumer, + fuse: Fuse(&panicked), + }; + self.base.drive(consumer1) + } + + fn len(&self) -> usize { + self.base.len() + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + return self.base.with_producer(Callback { callback }); + + struct Callback { + callback: CB, + } + + impl ProducerCallback for Callback + where + CB: ProducerCallback, + { + type Output = CB::Output; + + fn callback

(self, base: P) -> CB::Output + where + P: Producer, + { + let panicked = AtomicBool::new(false); + let producer = PanicFuseProducer { + base, + fuse: Fuse(&panicked), + }; + self.callback.callback(producer) + } + } + } +} + +// //////////////////////////////////////////////////////////////////////// +// Producer implementation + +struct PanicFuseProducer<'a, P> { + base: P, + fuse: Fuse<'a>, +} + +impl<'a, P> Producer for PanicFuseProducer<'a, P> +where + P: Producer, +{ + type Item = P::Item; + type IntoIter = PanicFuseIter<'a, P::IntoIter>; + + fn into_iter(self) -> Self::IntoIter { + PanicFuseIter { + base: self.base.into_iter(), + fuse: self.fuse, + } + } + + fn min_len(&self) -> usize { + self.base.min_len() + } + fn max_len(&self) -> usize { + self.base.max_len() + } + + fn split_at(self, index: usize) -> (Self, Self) { + let (left, right) = self.base.split_at(index); + ( + PanicFuseProducer { + base: left, + fuse: self.fuse.clone(), + }, + PanicFuseProducer { + base: right, + fuse: self.fuse, + }, + ) + } + + fn fold_with(self, folder: G) -> G + where + G: Folder, + { + let folder1 = PanicFuseFolder { + base: folder, + fuse: self.fuse, + }; + self.base.fold_with(folder1).base + } +} + +struct PanicFuseIter<'a, I> { + base: I, + fuse: Fuse<'a>, +} + +impl<'a, I> Iterator for PanicFuseIter<'a, I> +where + I: Iterator, +{ + type Item = I::Item; + + fn next(&mut self) -> Option { + if self.fuse.panicked() { + None + } else { + self.base.next() + } + } + + fn size_hint(&self) -> (usize, Option) { + self.base.size_hint() + } +} + +impl<'a, I> DoubleEndedIterator for PanicFuseIter<'a, I> +where + I: DoubleEndedIterator, +{ + fn next_back(&mut self) -> Option { + if self.fuse.panicked() { + None + } else { + self.base.next_back() + } + } +} + +impl<'a, I> ExactSizeIterator for PanicFuseIter<'a, I> +where + I: ExactSizeIterator, +{ + fn len(&self) -> usize { + self.base.len() + } +} + +// //////////////////////////////////////////////////////////////////////// +// Consumer implementation + +struct PanicFuseConsumer<'a, C> { + base: C, + fuse: Fuse<'a>, +} + +impl<'a, T, C> Consumer for PanicFuseConsumer<'a, C> +where + C: Consumer, +{ + type Folder = PanicFuseFolder<'a, C::Folder>; + type Reducer = PanicFuseReducer<'a, C::Reducer>; + type Result = C::Result; + + fn split_at(self, index: usize) -> (Self, Self, Self::Reducer) { + let (left, right, reducer) = self.base.split_at(index); + ( + PanicFuseConsumer { + base: left, + fuse: self.fuse.clone(), + }, + PanicFuseConsumer { + base: right, + fuse: self.fuse.clone(), + }, + PanicFuseReducer { + base: reducer, + _fuse: self.fuse, + }, + ) + } + + fn into_folder(self) -> Self::Folder { + PanicFuseFolder { + base: self.base.into_folder(), + fuse: self.fuse, + } + } + + fn full(&self) -> bool { + self.fuse.panicked() || self.base.full() + } +} + +impl<'a, T, C> UnindexedConsumer for PanicFuseConsumer<'a, C> +where + C: UnindexedConsumer, +{ + fn split_off_left(&self) -> Self { + PanicFuseConsumer { + base: self.base.split_off_left(), + fuse: self.fuse.clone(), + } + } + + fn to_reducer(&self) -> Self::Reducer { + PanicFuseReducer { + base: self.base.to_reducer(), + _fuse: self.fuse.clone(), + } + } +} + +struct PanicFuseFolder<'a, C> { + base: C, + fuse: Fuse<'a>, +} + +impl<'a, T, C> Folder for PanicFuseFolder<'a, C> +where + C: Folder, +{ + type Result = C::Result; + + fn consume(mut self, item: T) -> Self { + self.base = self.base.consume(item); + self + } + + fn consume_iter(mut self, iter: I) -> Self + where + I: IntoIterator, + { + fn cool<'a, T>(fuse: &'a Fuse<'_>) -> impl Fn(&T) -> bool + 'a { + move |_| !fuse.panicked() + } + + self.base = { + let fuse = &self.fuse; + let iter = iter.into_iter().take_while(cool(fuse)); + self.base.consume_iter(iter) + }; + self + } + + fn complete(self) -> C::Result { + self.base.complete() + } + + fn full(&self) -> bool { + self.fuse.panicked() || self.base.full() + } +} + +struct PanicFuseReducer<'a, C> { + base: C, + _fuse: Fuse<'a>, +} + +impl<'a, T, C> Reducer for PanicFuseReducer<'a, C> +where + C: Reducer, +{ + fn reduce(self, left: T, right: T) -> T { + self.base.reduce(left, right) + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/par_bridge.rs b/anneal/v2/vendor/rayon/src/iter/par_bridge.rs new file mode 100644 index 0000000000..6bc1343d86 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/par_bridge.rs @@ -0,0 +1,157 @@ +#[cfg(not(feature = "web_spin_lock"))] +use std::sync::Mutex; + +#[cfg(feature = "web_spin_lock")] +use wasm_sync::Mutex; + +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + +use crate::iter::plumbing::{bridge_unindexed, Folder, UnindexedConsumer, UnindexedProducer}; +use crate::iter::ParallelIterator; +use crate::{current_num_threads, current_thread_index}; + +/// Conversion trait to convert an `Iterator` to a `ParallelIterator`. +/// +/// This creates a "bridge" from a sequential iterator to a parallel one, by distributing its items +/// across the Rayon thread pool. This has the advantage of being able to parallelize just about +/// anything, but the resulting `ParallelIterator` can be less efficient than if you started with +/// `par_iter` instead. However, it can still be useful for iterators that are difficult to +/// parallelize by other means, like channels or file or network I/O. +/// +/// Iterator items are pulled by `next()` one at a time, synchronized from each thread that is +/// ready for work, so this may become a bottleneck if the serial iterator can't keep up with the +/// parallel demand. The items are not buffered by `IterBridge`, so it's fine to use this with +/// large or even unbounded iterators. +/// +/// The resulting iterator is not guaranteed to keep the order of the original iterator. +/// +/// # Examples +/// +/// To use this trait, take an existing `Iterator` and call `par_bridge` on it. After that, you can +/// use any of the `ParallelIterator` methods: +/// +/// ``` +/// use rayon::iter::ParallelBridge; +/// use rayon::prelude::ParallelIterator; +/// use std::sync::mpsc::channel; +/// +/// let rx = { +/// let (tx, rx) = channel(); +/// +/// tx.send("one!"); +/// tx.send("two!"); +/// tx.send("three!"); +/// +/// rx +/// }; +/// +/// let mut output: Vec<&'static str> = rx.into_iter().par_bridge().collect(); +/// output.sort_unstable(); +/// +/// assert_eq!(&*output, &["one!", "three!", "two!"]); +/// ``` +pub trait ParallelBridge: Sized { + /// Creates a bridge from this type to a `ParallelIterator`. + fn par_bridge(self) -> IterBridge; +} + +impl ParallelBridge for T +where + T: Iterator + Send, +{ + fn par_bridge(self) -> IterBridge { + IterBridge { iter: self } + } +} + +/// `IterBridge` is a parallel iterator that wraps a sequential iterator. +/// +/// This type is created when using the `par_bridge` method on `ParallelBridge`. See the +/// [`ParallelBridge`] documentation for details. +#[derive(Debug, Clone)] +pub struct IterBridge { + iter: Iter, +} + +impl ParallelIterator for IterBridge +where + Iter: Iterator + Send, +{ + type Item = Iter::Item; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let num_threads = current_num_threads(); + let threads_started: Vec<_> = (0..num_threads).map(|_| AtomicBool::new(false)).collect(); + + bridge_unindexed( + &IterParallelProducer { + split_count: AtomicUsize::new(num_threads), + iter: Mutex::new(self.iter.fuse()), + threads_started: &threads_started, + }, + consumer, + ) + } +} + +struct IterParallelProducer<'a, Iter> { + split_count: AtomicUsize, + iter: Mutex>, + threads_started: &'a [AtomicBool], +} + +impl UnindexedProducer for &IterParallelProducer<'_, Iter> { + type Item = Iter::Item; + + fn split(self) -> (Self, Option) { + // Check if the iterator is exhausted + let update = self + .split_count + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |c| c.checked_sub(1)); + (self, update.is_ok().then_some(self)) + } + + fn fold_with(self, mut folder: F) -> F + where + F: Folder, + { + // Guard against work-stealing-induced recursion, in case `Iter::next()` + // calls rayon internally, so we don't deadlock our mutex. We might also + // be recursing via `folder` methods, which doesn't present a mutex hazard, + // but it's lower overhead for us to just check this once, rather than + // updating additional shared state on every mutex lock/unlock. + // (If this isn't a rayon thread, then there's no work-stealing anyway...) + if let Some(i) = current_thread_index() { + // Note: If the number of threads in the pool ever grows dynamically, then + // we'll end up sharing flags and may falsely detect recursion -- that's + // still fine for overall correctness, just not optimal for parallelism. + let thread_started = &self.threads_started[i % self.threads_started.len()]; + if thread_started.swap(true, Ordering::Relaxed) { + // We can't make progress with a nested mutex, so just return and let + // the outermost loop continue with the rest of the iterator items. + return folder; + } + } + + loop { + if let Ok(mut iter) = self.iter.lock() { + if let Some(it) = iter.next() { + drop(iter); + folder = folder.consume(it); + if folder.full() { + return folder; + } + } else { + return folder; + } + } else { + // any panics from other threads will have been caught by the pool, + // and will be re-thrown when joined - just exit + return folder; + } + } + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/plumbing/README.md b/anneal/v2/vendor/rayon/src/iter/plumbing/README.md new file mode 100644 index 0000000000..42d22effe9 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/plumbing/README.md @@ -0,0 +1,315 @@ +# Parallel Iterators + +These are some notes on the design of the parallel iterator traits. +This file does not describe how to **use** parallel iterators. + +## The challenge + +Parallel iterators are more complicated than sequential iterators. +The reason is that they have to be able to split themselves up and +operate in parallel across the two halves. + +The current design for parallel iterators has two distinct modes in +which they can be used; as we will see, not all iterators support both +modes (which is why there are two): + +- **Pull mode** (the `Producer` and `UnindexedProducer` traits): in this mode, + the iterator is asked to produce the next item using a call to `next`. This + is basically like a normal iterator, but with a twist: you can split the + iterator in half to produce disjoint items in separate threads. + - in the `Producer` trait, splitting is done with `split_at`, which accepts + an index where the split should be performed. Only indexed iterators can + work in this mode, as they know exactly how much data they will produce, + and how to locate the requested index. + - in the `UnindexedProducer` trait, splitting is done with `split`, which + simply requests that the producer divide itself *approximately* in half. + This is useful when the exact length and/or layout is unknown, as with + `String` characters, or when the length might exceed `usize`, as with + `Range` on 32-bit platforms. + - In theory, any `Producer` could act unindexed, but we don't currently + use that possibility. When you know the exact length, a `split` can + simply be implemented as `split_at(length/2)`. +- **Push mode** (the `Consumer` and `UnindexedConsumer` traits): in + this mode, the iterator instead is *given* each item in turn, which + is then processed. This is the opposite of a normal iterator. It's + more like a `for_each` call: each time a new item is produced, the + `consume` method is called with that item. (The traits themselves are + a bit more complex, as they support state that can be threaded + through and ultimately reduced.) Like producers, there are two + variants of consumers which differ in how the split is performed: + - in the `Consumer` trait, splitting is done with `split_at`, which + accepts an index where the split should be performed. All + iterators can work in this mode. The resulting halves thus have an + idea about how much data they expect to consume. + - in the `UnindexedConsumer` trait, splitting is done with + `split_off_left`. There is no index: the resulting halves must be + prepared to process any amount of data, and they don't know where that + data falls in the overall stream. + - Not all consumers can operate in this mode. It works for + `for_each` and `reduce`, for example, but it does not work for + `collect_into_vec`, since in that case the position of each item is + important for knowing where it ends up in the target collection. + +## How iterator execution proceeds + +We'll walk through this example iterator chain to start. This chain +demonstrates more-or-less the full complexity of what can happen. + +```rust +vec1.par_iter() + .zip(vec2.par_iter()) + .flat_map(some_function) + .for_each(some_other_function) +``` + +To handle an iterator chain, we start by creating consumers. This +works from the end. So in this case, the call to `for_each` is the +final step, so it will create a `ForEachConsumer` that, given an item, +just calls `some_other_function` with that item. (`ForEachConsumer` is +a very simple consumer because it doesn't need to thread any state +between items at all.) + +Now, the `for_each` call will pass this consumer to the base iterator, +which is the `flat_map`. It will do this by calling the `drive_unindexed` +method on the `ParallelIterator` trait. `drive_unindexed` basically +says "produce items for this iterator and feed them to this consumer"; +it only works for unindexed consumers. + +(As an aside, it is interesting that only some consumers can work in +unindexed mode, but all producers can *drive* an unindexed consumer. +In contrast, only some producers can drive an *indexed* consumer, but +all consumers can be supplied indexes. Isn't variance neat.) + +As it happens, `FlatMap` only works with unindexed consumers anyway. +This is because flat-map basically has no idea how many items it will +produce. If you ask flat-map to produce the 22nd item, it can't do it, +at least not without some intermediate state. It doesn't know whether +processing the first item will create 1 item, 3 items, or 100; +therefore, to produce an arbitrary item, it would basically just have +to start at the beginning and execute sequentially, which is not what +we want. But for unindexed consumers, this doesn't matter, since they +don't need to know how much data they will get. + +Therefore, `FlatMap` can wrap the `ForEachConsumer` with a +`FlatMapConsumer` that feeds to it. This `FlatMapConsumer` will be +given one item. It will then invoke `some_function` to get a parallel +iterator out. It will then ask this new parallel iterator to drive the +`ForEachConsumer`. The `drive_unindexed` method on `flat_map` can then +pass the `FlatMapConsumer` up the chain to the previous item, which is +`zip`. At this point, something interesting happens. + +## Switching from push to pull mode + +If you think about `zip`, it can't really be implemented as a +consumer, at least not without an intermediate thread and some +channels or something (or maybe coroutines). The problem is that it +has to walk two iterators *in lockstep*. Basically, it can't call two +`drive` methods simultaneously, it can only call one at a time. So at +this point, the `zip` iterator needs to switch from *push mode* into +*pull mode*. + +You'll note that `Zip` is only usable if its inputs implement +`IndexedParallelIterator`, meaning that they can produce data starting +at random points in the stream. This need to switch to push mode is +exactly why. If we want to split a zip iterator at position 22, we +need to be able to start zipping items from index 22 right away, +without having to start from index 0. + +Anyway, so at this point, the `drive_unindexed` method for `Zip` stops +creating consumers. Instead, it creates a *producer*, a `ZipProducer`, +to be exact, and calls the `bridge` function in the `internals` +module. Creating a `ZipProducer` will in turn create producers for +the two iterators being zipped. This is possible because they both +implement `IndexedParallelIterator`. + +The `bridge` function will then connect the consumer, which is +handling the `flat_map` and `for_each`, with the producer, which is +handling the `zip` and its predecessors. It will split down until the +chunks seem reasonably small, then pull items from the producer and +feed them to the consumer. + +## The base case + +The other time that `bridge` gets used is when we bottom out in an +indexed producer, such as a slice or range. There is also a +`bridge_unindexed` equivalent for - you guessed it - unindexed producers, +such as string characters. + + + +## What on earth is `ProducerCallback`? + +We saw that when you call a parallel action method like +`par_iter.reduce()`, that will create a "reducing" consumer and then +invoke `par_iter.drive_unindexed()` (or `par_iter.drive()`) as +appropriate. This may create yet more consumers as we proceed up the +parallel iterator chain. But at some point we're going to get to the +start of the chain, or to a parallel iterator (like `zip()`) that has +to coordinate multiple inputs. At that point, we need to start +converting parallel iterators into producers. + +The way we do this is by invoking the method `with_producer()`, defined on +`IndexedParallelIterator`. This is a callback scheme. In an ideal world, +it would work like this: + +```rust +base_iter.with_producer(|base_producer| { + // here, `base_producer` is the producer for `base_iter` +}); +``` + +In that case, we could implement a combinator like `map()` by getting +the producer for the base iterator, wrapping it to make our own +`MapProducer`, and then passing that to the callback. Something like +this: + +```rust +struct MapProducer<'f, P, F: 'f> { + base: P, + map_op: &'f F, +} + +impl IndexedParallelIterator for Map + where I: IndexedParallelIterator, + F: MapOp, +{ + fn with_producer(self, callback: CB) -> CB::Output { + let map_op = &self.map_op; + self.base_iter.with_producer(|base_producer| { + // Here `producer` is the producer for `self.base_iter`. + // Wrap that to make a `MapProducer` + let map_producer = MapProducer { + base: base_producer, + map_op: map_op + }; + + // invoke the callback with the wrapped version + callback(map_producer) + }); + } +}); +``` + +This example demonstrates some of the power of the callback scheme. +It winds up being a very flexible setup. For one thing, it means we +can take ownership of `par_iter`; we can then in turn give ownership +away of its bits and pieces into the producer (this is very useful if +the iterator owns an `&mut` slice, for example), or create shared +references and put *those* in the producer. In the case of map, for +example, the parallel iterator owns the `map_op`, and we borrow +references to it which we then put into the `MapProducer` (this means +the `MapProducer` can easily split itself and share those references). +The `with_producer` method can also create resources that are needed +during the parallel execution, since the producer does not have to be +returned. + +Unfortunately there is a catch. We can't actually use closures the way +I showed you. To see why, think about the type that `map_producer` +would have to have. If we were going to write the `with_producer` +method using a closure, it would have to look something like this: + +```rust +pub trait IndexedParallelIterator: ParallelIterator { + type Producer; + fn with_producer(self, callback: CB) -> R + where CB: FnOnce(Self::Producer) -> R; + ... +} +``` + +Note that we had to add this associated type `Producer` so that +we could specify the argument of the callback to be `Self::Producer`. +Now, imagine trying to write that `MapProducer` impl using this style: + +```rust +impl IndexedParallelIterator for Map + where I: IndexedParallelIterator, + F: MapOp, +{ + type MapProducer = MapProducer<'f, P::Producer, F>; + // ^^ wait, what is this `'f`? + + fn with_producer(self, callback: CB) -> R + where CB: FnOnce(Self::Producer) -> R + { + let map_op = &self.map_op; + // ^^^^^^ `'f` is (conceptually) the lifetime of this reference, + // so it will be different for each call to `with_producer`! + } +} +``` + +This may look familiar to you: it's the same problem that we have +trying to define an `Iterable` trait. Basically, the producer type +needs to include a lifetime (here, `'f`) that refers to the body of +`with_producer` and hence is not in scope at the impl level. + +If we had [associated type constructors][1598], we could solve this +problem that way. But there is another solution. We can use a +dedicated callback trait like `ProducerCallback`, instead of `FnOnce`: + +[1598]: https://github.com/rust-lang/rfcs/pull/1598 + +```rust +pub trait ProducerCallback { + type Output; + fn callback

(self, producer: P) -> Self::Output + where P: Producer; +} +``` + +Using this trait, the signature of `with_producer()` looks like this: + +```rust +fn with_producer>(self, callback: CB) -> CB::Output; +``` + +Notice that this signature **never has to name the producer type** -- +there is no associated type `Producer` anymore. This is because the +`callback()` method is generically over **all** producers `P`. + +The problem is that now the `||` sugar doesn't work anymore. So we +have to manually create the callback struct, which is a mite tedious. +So our `MapProducer` code looks like this: + +```rust +impl IndexedParallelIterator for Map + where I: IndexedParallelIterator, + F: MapOp, +{ + fn with_producer(self, callback: CB) -> CB::Output + where CB: ProducerCallback + { + return self.base.with_producer(Callback { callback: callback, map_op: self.map_op }); + // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + // Manual version of the closure sugar: create an instance + // of a struct that implements `ProducerCallback`. + + // The struct declaration. Each field is something that need to capture from the + // creating scope. + struct Callback { + callback: CB, + map_op: F, + } + + // Implement the `ProducerCallback` trait. This is pure boilerplate. + impl ProducerCallback for Callback + where F: MapOp, + CB: ProducerCallback + { + type Output = CB::Output; + + fn callback

(self, base: P) -> CB::Output + where P: Producer + { + // The body of the closure is here: + let producer = MapProducer { base: base, + map_op: &self.map_op }; + self.callback.callback(producer) + } + } + } +} +``` + +OK, a bit tedious, but it works! diff --git a/anneal/v2/vendor/rayon/src/iter/plumbing/mod.rs b/anneal/v2/vendor/rayon/src/iter/plumbing/mod.rs new file mode 100644 index 0000000000..25489a2ef6 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/plumbing/mod.rs @@ -0,0 +1,476 @@ +//! Traits and functions used to implement parallel iteration. These are +//! low-level details -- users of parallel iterators should not need to +//! interact with them directly. See [the `plumbing` README][r] for a general overview. +//! +//! [r]: https://github.com/rayon-rs/rayon/blob/main/src/iter/plumbing/README.md + +use crate::join_context; + +use super::IndexedParallelIterator; + +/// The `ProducerCallback` trait is a kind of generic closure, +/// [analogous to `FnOnce`][FnOnce]. See [the corresponding section in +/// the plumbing README][r] for more details. +/// +/// [r]: https://github.com/rayon-rs/rayon/blob/main/src/iter/plumbing/README.md#producer-callback +/// [FnOnce]: std::ops::FnOnce +pub trait ProducerCallback { + /// The type of value returned by this callback. Analogous to + /// [`Output` from the `FnOnce` trait][Output]. + /// + /// [Output]: std::ops::FnOnce::Output + type Output; + + /// Invokes the callback with the given producer as argument. The + /// key point of this trait is that this method is generic over + /// `P`, and hence implementors must be defined for any producer. + fn callback

(self, producer: P) -> Self::Output + where + P: Producer; +} + +/// A `Producer` is effectively a "splittable `IntoIterator`". That +/// is, a producer is a value which can be converted into an iterator +/// at any time: at that point, it simply produces items on demand, +/// like any iterator. But what makes a `Producer` special is that, +/// *before* we convert to an iterator, we can also **split** it at a +/// particular point using the `split_at` method. This will yield up +/// two producers, one producing the items before that point, and one +/// producing the items after that point (these two producers can then +/// independently be split further, or be converted into iterators). +/// In Rayon, this splitting is used to divide between threads. +/// See [the `plumbing` README][r] for further details. +/// +/// Note that each producer will always produce a fixed number of +/// items N. However, this number N is not queryable through the API; +/// the consumer is expected to track it. +/// +/// NB. You might expect `Producer` to extend the `IntoIterator` +/// trait. However, [rust-lang/rust#20671][20671] prevents us from +/// declaring the DoubleEndedIterator and ExactSizeIterator +/// constraints on a required IntoIterator trait, so we inline +/// IntoIterator here until that issue is fixed. +/// +/// [r]: https://github.com/rayon-rs/rayon/blob/main/src/iter/plumbing/README.md +/// [20671]: https://github.com/rust-lang/rust/issues/20671 +pub trait Producer: Send + Sized { + /// The type of item that will be produced by this producer once + /// it is converted into an iterator. + type Item; + + /// The type of iterator we will become. + type IntoIter: Iterator + DoubleEndedIterator + ExactSizeIterator; + + /// Convert `self` into an iterator; at this point, no more parallel splits + /// are possible. + fn into_iter(self) -> Self::IntoIter; + + /// The minimum number of items that we will process + /// sequentially. Defaults to 1, which means that we will split + /// all the way down to a single item. This can be raised higher + /// using the [`with_min_len`] method, which will force us to + /// create sequential tasks at a larger granularity. Note that + /// Rayon automatically normally attempts to adjust the size of + /// parallel splits to reduce overhead, so this should not be + /// needed. + /// + /// [`with_min_len`]: super::IndexedParallelIterator::with_min_len() + fn min_len(&self) -> usize { + 1 + } + + /// The maximum number of items that we will process + /// sequentially. Defaults to MAX, which means that we can choose + /// not to split at all. This can be lowered using the + /// [`with_max_len`] method, which will force us to create more + /// parallel tasks. Note that Rayon automatically normally + /// attempts to adjust the size of parallel splits to reduce + /// overhead, so this should not be needed. + /// + /// [`with_max_len`]: super::IndexedParallelIterator::with_max_len() + fn max_len(&self) -> usize { + usize::MAX + } + + /// Split into two producers; one produces items `0..index`, the + /// other `index..N`. Index must be less than or equal to `N`. + fn split_at(self, index: usize) -> (Self, Self); + + /// Iterate the producer, feeding each element to `folder`, and + /// stop when the folder is full (or all elements have been consumed). + /// + /// The provided implementation is sufficient for most iterables. + fn fold_with(self, folder: F) -> F + where + F: Folder, + { + folder.consume_iter(self.into_iter()) + } +} + +/// A consumer is effectively a [generalized "fold" operation][fold], +/// and in fact each consumer will eventually be converted into a +/// [`Folder`]. What makes a consumer special is that, like a +/// [`Producer`], it can be **split** into multiple consumers using +/// the `split_at` method. When a consumer is split, it produces two +/// consumers, as well as a **reducer**. The two consumers can be fed +/// items independently, and when they are done the reducer is used to +/// combine their two results into one. See [the `plumbing` +/// README][r] for further details. +/// +/// [r]: https://github.com/rayon-rs/rayon/blob/main/src/iter/plumbing/README.md +/// [fold]: Iterator::fold() +pub trait Consumer: Send + Sized { + /// The type of folder that this consumer can be converted into. + type Folder: Folder; + + /// The type of reducer that is produced if this consumer is split. + type Reducer: Reducer; + + /// The type of result that this consumer will ultimately produce. + type Result: Send; + + /// Divide the consumer into two consumers, one processing items + /// `0..index` and one processing items from `index..`. Also + /// produces a reducer that can be used to reduce the results at + /// the end. + fn split_at(self, index: usize) -> (Self, Self, Self::Reducer); + + /// Convert the consumer into a folder that can consume items + /// sequentially, eventually producing a final result. + fn into_folder(self) -> Self::Folder; + + /// Hint whether this `Consumer` would like to stop processing + /// further items, e.g. if a search has been completed. + fn full(&self) -> bool; +} + +/// The `Folder` trait encapsulates [the standard fold +/// operation][fold]. It can be fed many items using the `consume` +/// method. At the end, once all items have been consumed, it can then +/// be converted (using `complete`) into a final value. +/// +/// [fold]: Iterator::fold() +pub trait Folder: Sized { + /// The type of result that will ultimately be produced by the folder. + type Result; + + /// Consume next item and return new sequential state. + fn consume(self, item: Item) -> Self; + + /// Consume items from the iterator until full, and return new sequential state. + /// + /// This method is **optional**. The default simply iterates over + /// `iter`, invoking `consume` and checking after each iteration + /// whether `full` returns false. + /// + /// The main reason to override it is if you can provide a more + /// specialized, efficient implementation. + fn consume_iter(mut self, iter: I) -> Self + where + I: IntoIterator, + { + for item in iter { + self = self.consume(item); + if self.full() { + break; + } + } + self + } + + /// Finish consuming items, produce final result. + fn complete(self) -> Self::Result; + + /// Hint whether this `Folder` would like to stop processing + /// further items, e.g. if a search has been completed. + fn full(&self) -> bool; +} + +/// The reducer is the final step of a `Consumer` -- after a consumer +/// has been split into two parts, and each of those parts has been +/// fully processed, we are left with two results. The reducer is then +/// used to combine those two results into one. See [the `plumbing` +/// README][r] for further details. +/// +/// [r]: https://github.com/rayon-rs/rayon/blob/main/src/iter/plumbing/README.md +pub trait Reducer { + /// Reduce two final results into one; this is executed after a + /// split. + fn reduce(self, left: Result, right: Result) -> Result; +} + +/// A stateless consumer can be freely copied. These consumers can be +/// used like regular consumers, but they also support a +/// `split_off_left` method that does not take an index to split, but +/// simply splits at some arbitrary point (`for_each`, for example, +/// produces an unindexed consumer). +pub trait UnindexedConsumer: Consumer { + /// Splits off a "left" consumer and returns it. The `self` + /// consumer should then be used to consume the "right" portion of + /// the data. (The ordering matters for methods like find_first -- + /// values produced by the returned value are given precedence + /// over values produced by `self`.) Once the left and right + /// halves have been fully consumed, you should reduce the results + /// with the result of `to_reducer`. + fn split_off_left(&self) -> Self; + + /// Creates a reducer that can be used to combine the results from + /// a split consumer. + fn to_reducer(&self) -> Self::Reducer; +} + +/// A variant on `Producer` which does not know its exact length or +/// cannot represent it in a `usize`. These producers act like +/// ordinary producers except that they cannot be told to split at a +/// particular point. Instead, you just ask them to split 'somewhere'. +/// +/// (In principle, `Producer` could extend this trait; however, it +/// does not because to do so would require producers to carry their +/// own length with them.) +pub trait UnindexedProducer: Send + Sized { + /// The type of item returned by this producer. + type Item; + + /// Split midway into a new producer if possible, otherwise return `None`. + fn split(self) -> (Self, Option); + + /// Iterate the producer, feeding each element to `folder`, and + /// stop when the folder is full (or all elements have been consumed). + fn fold_with(self, folder: F) -> F + where + F: Folder; +} + +/// A splitter controls the policy for splitting into smaller work items. +/// +/// Thief-splitting is an adaptive policy that starts by splitting into +/// enough jobs for every worker thread, and then resets itself whenever a +/// job is actually stolen into a different thread. +#[derive(Clone, Copy)] +struct Splitter { + /// The `splits` tell us approximately how many remaining times we'd + /// like to split this job. We always just divide it by two though, so + /// the effective number of pieces will be `next_power_of_two()`. + splits: usize, +} + +impl Splitter { + #[inline] + fn new() -> Splitter { + Splitter { + splits: crate::current_num_threads(), + } + } + + #[inline] + fn try_split(&mut self, stolen: bool) -> bool { + let Splitter { splits } = *self; + + if stolen { + // This job was stolen! Reset the number of desired splits to the + // thread count, if that's more than we had remaining anyway. + self.splits = Ord::max(crate::current_num_threads(), self.splits / 2); + true + } else if splits > 0 { + // We have splits remaining, make it so. + self.splits /= 2; + true + } else { + // Not stolen, and no more splits -- we're done! + false + } + } +} + +/// The length splitter is built on thief-splitting, but additionally takes +/// into account the remaining length of the iterator. +#[derive(Clone, Copy)] +struct LengthSplitter { + inner: Splitter, + + /// The smallest we're willing to divide into. Usually this is just 1, + /// but you can choose a larger working size with `with_min_len()`. + min: usize, +} + +impl LengthSplitter { + /// Creates a new splitter based on lengths. + /// + /// The `min` is a hard lower bound. We'll never split below that, but + /// of course an iterator might start out smaller already. + /// + /// The `max` is an upper bound on the working size, used to determine + /// the minimum number of times we need to split to get under that limit. + /// The adaptive algorithm may very well split even further, but never + /// smaller than the `min`. + #[inline] + fn new(min: usize, max: usize, len: usize) -> LengthSplitter { + let mut splitter = LengthSplitter { + inner: Splitter::new(), + min: Ord::max(min, 1), + }; + + // Divide the given length by the max working length to get the minimum + // number of splits we need to get under that max. This rounds down, + // but the splitter actually gives `next_power_of_two()` pieces anyway. + // e.g. len 12345 / max 100 = 123 min_splits -> 128 pieces. + let min_splits = len / Ord::max(max, 1); + + // Only update the value if it's not splitting enough already. + if min_splits > splitter.inner.splits { + splitter.inner.splits = min_splits; + } + + splitter + } + + #[inline] + fn try_split(&mut self, len: usize, stolen: bool) -> bool { + // If splitting wouldn't make us too small, try the inner splitter. + len / 2 >= self.min && self.inner.try_split(stolen) + } +} + +/// This helper function is used to "connect" a parallel iterator to a +/// consumer. It will convert the `par_iter` into a producer P and +/// then pull items from P and feed them to `consumer`, splitting and +/// creating parallel threads as needed. +/// +/// This is useful when you are implementing your own parallel +/// iterators: it is often used as the definition of the +/// [`drive_unindexed`] or [`drive`] methods. +/// +/// [`drive_unindexed`]: super::ParallelIterator::drive_unindexed() +/// [`drive`]: super::IndexedParallelIterator::drive() +pub fn bridge(par_iter: I, consumer: C) -> C::Result +where + I: IndexedParallelIterator, + C: Consumer, +{ + let len = par_iter.len(); + return par_iter.with_producer(Callback { len, consumer }); + + struct Callback { + len: usize, + consumer: C, + } + + impl ProducerCallback for Callback + where + C: Consumer, + { + type Output = C::Result; + fn callback

(self, producer: P) -> C::Result + where + P: Producer, + { + bridge_producer_consumer(self.len, producer, self.consumer) + } + } +} + +/// This helper function is used to "connect" a producer and a +/// consumer. You may prefer to call [`bridge()`], which wraps this +/// function. This function will draw items from `producer` and feed +/// them to `consumer`, splitting and creating parallel tasks when +/// needed. +/// +/// This is useful when you are implementing your own parallel +/// iterators: it is often used as the definition of the +/// [`drive_unindexed`] or [`drive`] methods. +/// +/// [`drive_unindexed`]: super::ParallelIterator::drive_unindexed() +/// [`drive`]: super::IndexedParallelIterator::drive() +pub fn bridge_producer_consumer(len: usize, producer: P, consumer: C) -> C::Result +where + P: Producer, + C: Consumer, +{ + let splitter = LengthSplitter::new(producer.min_len(), producer.max_len(), len); + return helper(len, false, splitter, producer, consumer); + + fn helper( + len: usize, + migrated: bool, + mut splitter: LengthSplitter, + producer: P, + consumer: C, + ) -> C::Result + where + P: Producer, + C: Consumer, + { + if consumer.full() { + consumer.into_folder().complete() + } else if splitter.try_split(len, migrated) { + let mid = len / 2; + let (left_producer, right_producer) = producer.split_at(mid); + let (left_consumer, right_consumer, reducer) = consumer.split_at(mid); + let (left_result, right_result) = join_context( + |context| { + helper( + mid, + context.migrated(), + splitter, + left_producer, + left_consumer, + ) + }, + |context| { + helper( + len - mid, + context.migrated(), + splitter, + right_producer, + right_consumer, + ) + }, + ); + reducer.reduce(left_result, right_result) + } else { + producer.fold_with(consumer.into_folder()).complete() + } + } +} + +/// A variant of [`bridge_producer_consumer()`] where the producer is an unindexed producer. +pub fn bridge_unindexed(producer: P, consumer: C) -> C::Result +where + P: UnindexedProducer, + C: UnindexedConsumer, +{ + let splitter = Splitter::new(); + bridge_unindexed_producer_consumer(false, splitter, producer, consumer) +} + +fn bridge_unindexed_producer_consumer( + migrated: bool, + mut splitter: Splitter, + producer: P, + consumer: C, +) -> C::Result +where + P: UnindexedProducer, + C: UnindexedConsumer, +{ + if consumer.full() { + consumer.into_folder().complete() + } else if splitter.try_split(migrated) { + match producer.split() { + (left_producer, Some(right_producer)) => { + let (reducer, left_consumer, right_consumer) = + (consumer.to_reducer(), consumer.split_off_left(), consumer); + let bridge = bridge_unindexed_producer_consumer; + let (left_result, right_result) = join_context( + |context| bridge(context.migrated(), splitter, left_producer, left_consumer), + |context| bridge(context.migrated(), splitter, right_producer, right_consumer), + ); + reducer.reduce(left_result, right_result) + } + (producer, None) => producer.fold_with(consumer.into_folder()).complete(), + } + } else { + producer.fold_with(consumer.into_folder()).complete() + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/positions.rs b/anneal/v2/vendor/rayon/src/iter/positions.rs new file mode 100644 index 0000000000..ea81fc6b51 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/positions.rs @@ -0,0 +1,133 @@ +use super::plumbing::*; +use super::*; + +use std::fmt::{self, Debug}; + +/// `Positions` takes a predicate `predicate` and filters out elements that match, +/// yielding their indices. +/// +/// This struct is created by the [`positions()`] method on [`IndexedParallelIterator`] +/// +/// [`positions()`]: IndexedParallelIterator::positions() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Clone)] +pub struct Positions { + base: I, + predicate: P, +} + +impl Debug for Positions { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Positions") + .field("base", &self.base) + .finish() + } +} + +impl Positions { + /// Create a new `Positions` iterator. + pub(super) fn new(base: I, predicate: P) -> Self { + Positions { base, predicate } + } +} + +impl ParallelIterator for Positions +where + I: IndexedParallelIterator, + P: Fn(I::Item) -> bool + Sync + Send, +{ + type Item = usize; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let consumer1 = PositionsConsumer::new(consumer, &self.predicate, 0); + self.base.drive(consumer1) + } +} + +// //////////////////////////////////////////////////////////////////////// +// Consumer implementation + +struct PositionsConsumer<'p, C, P> { + base: C, + predicate: &'p P, + offset: usize, +} + +impl<'p, C, P> PositionsConsumer<'p, C, P> { + fn new(base: C, predicate: &'p P, offset: usize) -> Self { + PositionsConsumer { + base, + predicate, + offset, + } + } +} + +impl<'p, T, C, P> Consumer for PositionsConsumer<'p, C, P> +where + C: Consumer, + P: Fn(T) -> bool + Sync, +{ + type Folder = PositionsFolder<'p, C::Folder, P>; + type Reducer = C::Reducer; + type Result = C::Result; + + fn split_at(self, index: usize) -> (Self, Self, C::Reducer) { + let (left, right, reducer) = self.base.split_at(index); + ( + PositionsConsumer::new(left, self.predicate, self.offset), + PositionsConsumer::new(right, self.predicate, self.offset + index), + reducer, + ) + } + + fn into_folder(self) -> Self::Folder { + PositionsFolder { + base: self.base.into_folder(), + predicate: self.predicate, + offset: self.offset, + } + } + + fn full(&self) -> bool { + self.base.full() + } +} + +struct PositionsFolder<'p, F, P> { + base: F, + predicate: &'p P, + offset: usize, +} + +impl Folder for PositionsFolder<'_, F, P> +where + F: Folder, + P: Fn(T) -> bool, +{ + type Result = F::Result; + + fn consume(mut self, item: T) -> Self { + let index = self.offset; + self.offset += 1; + if (self.predicate)(item) { + self.base = self.base.consume(index); + } + self + } + + // This cannot easily specialize `consume_iter` to be better than + // the default, because that requires checking `self.base.full()` + // during a call to `self.base.consume_iter()`. (#632) + + fn complete(self) -> Self::Result { + self.base.complete() + } + + fn full(&self) -> bool { + self.base.full() + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/product.rs b/anneal/v2/vendor/rayon/src/iter/product.rs new file mode 100644 index 0000000000..e081be06bd --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/product.rs @@ -0,0 +1,114 @@ +use super::plumbing::*; +use super::ParallelIterator; + +use std::iter::{self, Product}; +use std::marker::PhantomData; + +pub(super) fn product(pi: PI) -> P +where + PI: ParallelIterator, + P: Send + Product + Product, +{ + pi.drive_unindexed(ProductConsumer::new()) +} + +fn mul(left: T, right: T) -> T { + [left, right].into_iter().product() +} + +struct ProductConsumer { + _marker: PhantomData<*const P>, +} + +unsafe impl Send for ProductConsumer

{} + +impl ProductConsumer

{ + fn new() -> ProductConsumer

{ + ProductConsumer { + _marker: PhantomData, + } + } +} + +impl Consumer for ProductConsumer

+where + P: Send + Product + Product, +{ + type Folder = ProductFolder

; + type Reducer = Self; + type Result = P; + + fn split_at(self, _index: usize) -> (Self, Self, Self) { + ( + ProductConsumer::new(), + ProductConsumer::new(), + ProductConsumer::new(), + ) + } + + fn into_folder(self) -> Self::Folder { + ProductFolder { + product: iter::empty::().product(), + } + } + + fn full(&self) -> bool { + false + } +} + +impl UnindexedConsumer for ProductConsumer

+where + P: Send + Product + Product, +{ + fn split_off_left(&self) -> Self { + ProductConsumer::new() + } + + fn to_reducer(&self) -> Self::Reducer { + ProductConsumer::new() + } +} + +impl

Reducer

for ProductConsumer

+where + P: Send + Product, +{ + fn reduce(self, left: P, right: P) -> P { + mul(left, right) + } +} + +struct ProductFolder

{ + product: P, +} + +impl Folder for ProductFolder

+where + P: Product + Product, +{ + type Result = P; + + fn consume(self, item: T) -> Self { + ProductFolder { + product: mul(self.product, iter::once(item).product()), + } + } + + fn consume_iter(self, iter: I) -> Self + where + I: IntoIterator, + { + ProductFolder { + product: mul(self.product, iter.into_iter().product()), + } + } + + fn complete(self) -> P { + self.product + } + + fn full(&self) -> bool { + false + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/reduce.rs b/anneal/v2/vendor/rayon/src/iter/reduce.rs new file mode 100644 index 0000000000..321b5dd864 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/reduce.rs @@ -0,0 +1,116 @@ +use super::plumbing::*; +use super::ParallelIterator; + +pub(super) fn reduce(pi: PI, identity: ID, reduce_op: R) -> T +where + PI: ParallelIterator, + R: Fn(T, T) -> T + Sync, + ID: Fn() -> T + Sync, + T: Send, +{ + let consumer = ReduceConsumer { + identity: &identity, + reduce_op: &reduce_op, + }; + pi.drive_unindexed(consumer) +} + +struct ReduceConsumer<'r, R, ID> { + identity: &'r ID, + reduce_op: &'r R, +} + +impl<'r, R, ID> Copy for ReduceConsumer<'r, R, ID> {} + +impl<'r, R, ID> Clone for ReduceConsumer<'r, R, ID> { + fn clone(&self) -> Self { + *self + } +} + +impl<'r, R, ID, T> Consumer for ReduceConsumer<'r, R, ID> +where + R: Fn(T, T) -> T + Sync, + ID: Fn() -> T + Sync, + T: Send, +{ + type Folder = ReduceFolder<'r, R, T>; + type Reducer = Self; + type Result = T; + + fn split_at(self, _index: usize) -> (Self, Self, Self) { + (self, self, self) + } + + fn into_folder(self) -> Self::Folder { + ReduceFolder { + reduce_op: self.reduce_op, + item: (self.identity)(), + } + } + + fn full(&self) -> bool { + false + } +} + +impl<'r, R, ID, T> UnindexedConsumer for ReduceConsumer<'r, R, ID> +where + R: Fn(T, T) -> T + Sync, + ID: Fn() -> T + Sync, + T: Send, +{ + fn split_off_left(&self) -> Self { + *self + } + + fn to_reducer(&self) -> Self::Reducer { + *self + } +} + +impl<'r, R, ID, T> Reducer for ReduceConsumer<'r, R, ID> +where + R: Fn(T, T) -> T + Sync, +{ + fn reduce(self, left: T, right: T) -> T { + (self.reduce_op)(left, right) + } +} + +struct ReduceFolder<'r, R, T> { + reduce_op: &'r R, + item: T, +} + +impl<'r, R, T> Folder for ReduceFolder<'r, R, T> +where + R: Fn(T, T) -> T, +{ + type Result = T; + + fn consume(self, item: T) -> Self { + ReduceFolder { + reduce_op: self.reduce_op, + item: (self.reduce_op)(self.item, item), + } + } + + fn consume_iter(self, iter: I) -> Self + where + I: IntoIterator, + { + ReduceFolder { + reduce_op: self.reduce_op, + item: iter.into_iter().fold(self.item, self.reduce_op), + } + } + + fn complete(self) -> T { + self.item + } + + fn full(&self) -> bool { + false + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/repeat.rs b/anneal/v2/vendor/rayon/src/iter/repeat.rs new file mode 100644 index 0000000000..d58e59b84d --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/repeat.rs @@ -0,0 +1,295 @@ +use super::plumbing::*; +use super::*; +use std::num::NonZeroUsize; +use std::{fmt, iter, mem}; + +/// Iterator adaptor for [the `repeat()` function]. +/// +/// [the `repeat()` function]: repeat() +#[derive(Debug, Clone)] +pub struct Repeat { + element: T, +} + +/// Creates a parallel iterator that endlessly repeats `element` (by +/// cloning it). Note that this iterator has "infinite" length, so +/// typically you would want to use `zip` or `take` or some other +/// means to shorten it, or consider using +/// [the `repeat_n()` function] instead. +/// +/// [the `repeat_n()` function]: repeat_n() +/// +/// # Examples +/// +/// ``` +/// use rayon::prelude::*; +/// use rayon::iter::repeat; +/// let x: Vec<(i32, i32)> = repeat(22).zip(0..3).collect(); +/// assert_eq!(x, vec![(22, 0), (22, 1), (22, 2)]); +/// ``` +pub fn repeat(element: T) -> Repeat { + Repeat { element } +} + +impl Repeat +where + T: Clone + Send, +{ + /// Takes only `n` repeats of the element, similar to the general + /// [`take()`]. + /// + /// The resulting `RepeatN` is an `IndexedParallelIterator`, allowing + /// more functionality than `Repeat` alone. + /// + /// [`take()`]: IndexedParallelIterator::take() + pub fn take(self, n: usize) -> RepeatN { + repeat_n(self.element, n) + } + + /// Iterates tuples, repeating the element with items from another + /// iterator, similar to the general [`zip()`]. + /// + /// [`zip()`]: IndexedParallelIterator::zip() + pub fn zip(self, zip_op: Z) -> Zip, Z::Iter> + where + Z: IntoParallelIterator, + { + let z = zip_op.into_par_iter(); + let n = z.len(); + self.take(n).zip(z) + } +} + +impl ParallelIterator for Repeat +where + T: Clone + Send, +{ + type Item = T; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let producer = RepeatProducer { + element: self.element, + }; + bridge_unindexed(producer, consumer) + } +} + +/// Unindexed producer for `Repeat`. +struct RepeatProducer { + element: T, +} + +impl UnindexedProducer for RepeatProducer { + type Item = T; + + fn split(self) -> (Self, Option) { + ( + RepeatProducer { + element: self.element.clone(), + }, + Some(RepeatProducer { + element: self.element, + }), + ) + } + + fn fold_with(self, folder: F) -> F + where + F: Folder, + { + folder.consume_iter(iter::repeat(self.element)) + } +} + +/// Iterator adaptor for [the `repeat_n()` function]. +/// +/// [the `repeat_n()` function]: repeat_n() +#[derive(Clone)] +pub struct RepeatN { + inner: RepeatNProducer, +} + +/// Creates a parallel iterator that produces `n` repeats of `element` +/// (by cloning it). +/// +/// # Examples +/// +/// ``` +/// use rayon::prelude::*; +/// use rayon::iter::repeat_n; +/// let x: Vec<(i32, i32)> = repeat_n(22, 3).zip(0..3).collect(); +/// assert_eq!(x, vec![(22, 0), (22, 1), (22, 2)]); +/// ``` +pub fn repeat_n(element: T, n: usize) -> RepeatN { + let inner = match NonZeroUsize::new(n) { + Some(count) => RepeatNProducer::Repeats(element, count), + None => RepeatNProducer::Empty, + }; + RepeatN { inner } +} + +/// Creates a parallel iterator that produces `n` repeats of `element` +/// (by cloning it). +/// +/// Deprecated in favor of [`repeat_n`] for consistency with the standard library. +#[deprecated(note = "use `repeat_n`")] +pub fn repeatn(element: T, n: usize) -> RepeatN { + repeat_n(element, n) +} + +impl fmt::Debug for RepeatN { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut dbg = f.debug_struct("RepeatN"); + if let RepeatNProducer::Repeats(element, count) = &self.inner { + dbg.field("count", &count.get()) + .field("element", element) + .finish() + } else { + dbg.field("count", &0usize).finish_non_exhaustive() + } + } +} + +impl ParallelIterator for RepeatN +where + T: Clone + Send, +{ + type Item = T; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + bridge(self, consumer) + } + + fn opt_len(&self) -> Option { + Some(self.inner.len()) + } +} + +impl IndexedParallelIterator for RepeatN +where + T: Clone + Send, +{ + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + bridge(self, consumer) + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + callback.callback(self.inner) + } + + fn len(&self) -> usize { + self.inner.len() + } +} + +/// Producer for `RepeatN`. +#[derive(Clone)] +enum RepeatNProducer { + Repeats(T, NonZeroUsize), + Empty, +} + +impl Producer for RepeatNProducer { + type Item = T; + type IntoIter = Self; + + fn into_iter(self) -> Self::IntoIter { + // We could potentially use `std::iter::RepeatN` with MSRV 1.82, but we have no way to + // create an empty instance without a value in hand, like `repeat_n(value, 0)`. + self + } + + fn split_at(self, index: usize) -> (Self, Self) { + if let Self::Repeats(element, count) = self { + assert!(index <= count.get()); + match ( + NonZeroUsize::new(index), + NonZeroUsize::new(count.get() - index), + ) { + (Some(left), Some(right)) => ( + Self::Repeats(element.clone(), left), + Self::Repeats(element, right), + ), + (Some(left), None) => (Self::Repeats(element, left), Self::Empty), + (None, Some(right)) => (Self::Empty, Self::Repeats(element, right)), + (None, None) => unreachable!(), + } + } else { + assert!(index == 0); + (Self::Empty, Self::Empty) + } + } +} + +impl Iterator for RepeatNProducer { + type Item = T; + + #[inline] + fn next(&mut self) -> Option { + if let Self::Repeats(element, count) = self { + if let Some(rem) = NonZeroUsize::new(count.get() - 1) { + *count = rem; + Some(element.clone()) + } else { + match mem::replace(self, Self::Empty) { + Self::Repeats(element, _) => Some(element), + Self::Empty => unreachable!(), + } + } + } else { + None + } + } + + #[inline] + fn nth(&mut self, n: usize) -> Option { + if let Self::Repeats(_, count) = self { + if let Some(rem) = NonZeroUsize::new(count.get().saturating_sub(n)) { + *count = rem; + return self.next(); + } + *self = Self::Empty; + } + None + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let len = self.len(); + (len, Some(len)) + } +} + +impl DoubleEndedIterator for RepeatNProducer { + #[inline] + fn next_back(&mut self) -> Option { + self.next() + } + + #[inline] + fn nth_back(&mut self, n: usize) -> Option { + self.nth(n) + } +} + +impl ExactSizeIterator for RepeatNProducer { + #[inline] + fn len(&self) -> usize { + match self { + Self::Repeats(_, count) => count.get(), + Self::Empty => 0, + } + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/rev.rs b/anneal/v2/vendor/rayon/src/iter/rev.rs new file mode 100644 index 0000000000..b8b60a207c --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/rev.rs @@ -0,0 +1,119 @@ +use super::plumbing::*; +use super::*; +use std::iter; + +/// `Rev` is an iterator that produces elements in reverse order. This struct +/// is created by the [`rev()`] method on [`IndexedParallelIterator`] +/// +/// [`rev()`]: IndexedParallelIterator::rev() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Debug, Clone)] +pub struct Rev { + base: I, +} + +impl Rev { + /// Creates a new `Rev` iterator. + pub(super) fn new(base: I) -> Self { + Rev { base } + } +} + +impl ParallelIterator for Rev +where + I: IndexedParallelIterator, +{ + type Item = I::Item; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + bridge(self, consumer) + } + + fn opt_len(&self) -> Option { + Some(self.len()) + } +} + +impl IndexedParallelIterator for Rev +where + I: IndexedParallelIterator, +{ + fn drive>(self, consumer: C) -> C::Result { + bridge(self, consumer) + } + + fn len(&self) -> usize { + self.base.len() + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + let len = self.base.len(); + return self.base.with_producer(Callback { callback, len }); + + struct Callback { + callback: CB, + len: usize, + } + + impl ProducerCallback for Callback + where + CB: ProducerCallback, + { + type Output = CB::Output; + fn callback

(self, base: P) -> CB::Output + where + P: Producer, + { + let producer = RevProducer { + base, + len: self.len, + }; + self.callback.callback(producer) + } + } + } +} + +struct RevProducer

{ + base: P, + len: usize, +} + +impl

Producer for RevProducer

+where + P: Producer, +{ + type Item = P::Item; + type IntoIter = iter::Rev; + + fn into_iter(self) -> Self::IntoIter { + self.base.into_iter().rev() + } + + fn min_len(&self) -> usize { + self.base.min_len() + } + fn max_len(&self) -> usize { + self.base.max_len() + } + + fn split_at(self, index: usize) -> (Self, Self) { + let (left, right) = self.base.split_at(self.len - index); + ( + RevProducer { + base: right, + len: index, + }, + RevProducer { + base: left, + len: self.len - index, + }, + ) + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/skip.rs b/anneal/v2/vendor/rayon/src/iter/skip.rs new file mode 100644 index 0000000000..8b66c837b5 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/skip.rs @@ -0,0 +1,93 @@ +use super::noop::NoopConsumer; +use super::plumbing::*; +use super::*; + +/// `Skip` is an iterator that skips over the first `n` elements. +/// This struct is created by the [`skip()`] method on [`IndexedParallelIterator`] +/// +/// [`skip()`]: IndexedParallelIterator::skip() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Debug, Clone)] +pub struct Skip { + base: I, + n: usize, +} + +impl Skip +where + I: IndexedParallelIterator, +{ + /// Creates a new `Skip` iterator. + pub(super) fn new(base: I, n: usize) -> Self { + let n = Ord::min(base.len(), n); + Skip { base, n } + } +} + +impl ParallelIterator for Skip +where + I: IndexedParallelIterator, +{ + type Item = I::Item; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + bridge(self, consumer) + } + + fn opt_len(&self) -> Option { + Some(self.len()) + } +} + +impl IndexedParallelIterator for Skip +where + I: IndexedParallelIterator, +{ + fn len(&self) -> usize { + self.base.len() - self.n + } + + fn drive>(self, consumer: C) -> C::Result { + bridge(self, consumer) + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + return self.base.with_producer(Callback { + callback, + n: self.n, + }); + + struct Callback { + callback: CB, + n: usize, + } + + impl ProducerCallback for Callback + where + CB: ProducerCallback, + { + type Output = CB::Output; + fn callback

(self, base: P) -> CB::Output + where + P: Producer, + { + crate::in_place_scope(|scope| { + let Self { callback, n } = self; + let (before_skip, after_skip) = base.split_at(n); + + // Run the skipped part separately for side effects. + // We'll still get any panics propagated back by the scope. + scope.spawn(move |_| bridge_producer_consumer(n, before_skip, NoopConsumer)); + + callback.callback(after_skip) + }) + } + } + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/skip_any.rs b/anneal/v2/vendor/rayon/src/iter/skip_any.rs new file mode 100644 index 0000000000..f8de69433f --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/skip_any.rs @@ -0,0 +1,140 @@ +use super::plumbing::*; +use super::*; +use std::sync::atomic::{AtomicUsize, Ordering}; + +/// `SkipAny` is an iterator that skips over `n` elements from anywhere in `I`. +/// This struct is created by the [`skip_any()`] method on [`ParallelIterator`] +/// +/// [`skip_any()`]: ParallelIterator::skip_any() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Clone, Debug)] +pub struct SkipAny { + base: I, + count: usize, +} + +impl SkipAny { + /// Creates a new `SkipAny` iterator. + pub(super) fn new(base: I, count: usize) -> Self { + SkipAny { base, count } + } +} + +impl ParallelIterator for SkipAny +where + I: ParallelIterator, +{ + type Item = I::Item; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let consumer1 = SkipAnyConsumer { + base: consumer, + count: &AtomicUsize::new(self.count), + }; + self.base.drive_unindexed(consumer1) + } +} + +// //////////////////////////////////////////////////////////////////////// +// Consumer implementation + +struct SkipAnyConsumer<'f, C> { + base: C, + count: &'f AtomicUsize, +} + +impl<'f, T, C> Consumer for SkipAnyConsumer<'f, C> +where + C: Consumer, + T: Send, +{ + type Folder = SkipAnyFolder<'f, C::Folder>; + type Reducer = C::Reducer; + type Result = C::Result; + + fn split_at(self, index: usize) -> (Self, Self, Self::Reducer) { + let (left, right, reducer) = self.base.split_at(index); + ( + SkipAnyConsumer { base: left, ..self }, + SkipAnyConsumer { + base: right, + ..self + }, + reducer, + ) + } + + fn into_folder(self) -> Self::Folder { + SkipAnyFolder { + base: self.base.into_folder(), + count: self.count, + } + } + + fn full(&self) -> bool { + self.base.full() + } +} + +impl<'f, T, C> UnindexedConsumer for SkipAnyConsumer<'f, C> +where + C: UnindexedConsumer, + T: Send, +{ + fn split_off_left(&self) -> Self { + SkipAnyConsumer { + base: self.base.split_off_left(), + ..*self + } + } + + fn to_reducer(&self) -> Self::Reducer { + self.base.to_reducer() + } +} + +struct SkipAnyFolder<'f, C> { + base: C, + count: &'f AtomicUsize, +} + +fn checked_decrement(u: &AtomicUsize) -> bool { + u.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |u| u.checked_sub(1)) + .is_ok() +} + +impl<'f, T, C> Folder for SkipAnyFolder<'f, C> +where + C: Folder, +{ + type Result = C::Result; + + fn consume(mut self, item: T) -> Self { + if !checked_decrement(self.count) { + self.base = self.base.consume(item); + } + self + } + + fn consume_iter(mut self, iter: I) -> Self + where + I: IntoIterator, + { + self.base = self.base.consume_iter( + iter.into_iter() + .skip_while(move |_| checked_decrement(self.count)), + ); + self + } + + fn complete(self) -> C::Result { + self.base.complete() + } + + fn full(&self) -> bool { + self.base.full() + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/skip_any_while.rs b/anneal/v2/vendor/rayon/src/iter/skip_any_while.rs new file mode 100644 index 0000000000..6b3ff200d7 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/skip_any_while.rs @@ -0,0 +1,162 @@ +use super::plumbing::*; +use super::*; +use std::fmt; +use std::sync::atomic::{AtomicBool, Ordering}; + +/// `SkipAnyWhile` is an iterator that skips over elements from anywhere in `I` +/// until the callback returns `false`. +/// This struct is created by the [`skip_any_while()`] method on [`ParallelIterator`] +/// +/// [`skip_any_while()`]: ParallelIterator::skip_any_while() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Clone)] +pub struct SkipAnyWhile { + base: I, + predicate: P, +} + +impl fmt::Debug for SkipAnyWhile { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SkipAnyWhile") + .field("base", &self.base) + .finish() + } +} + +impl SkipAnyWhile { + /// Creates a new `SkipAnyWhile` iterator. + pub(super) fn new(base: I, predicate: P) -> Self { + SkipAnyWhile { base, predicate } + } +} + +impl ParallelIterator for SkipAnyWhile +where + I: ParallelIterator, + P: Fn(&I::Item) -> bool + Sync + Send, +{ + type Item = I::Item; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let consumer1 = SkipAnyWhileConsumer { + base: consumer, + predicate: &self.predicate, + skipping: &AtomicBool::new(true), + }; + self.base.drive_unindexed(consumer1) + } +} + +// //////////////////////////////////////////////////////////////////////// +// Consumer implementation + +struct SkipAnyWhileConsumer<'p, C, P> { + base: C, + predicate: &'p P, + skipping: &'p AtomicBool, +} + +impl<'p, T, C, P> Consumer for SkipAnyWhileConsumer<'p, C, P> +where + C: Consumer, + P: Fn(&T) -> bool + Sync, +{ + type Folder = SkipAnyWhileFolder<'p, C::Folder, P>; + type Reducer = C::Reducer; + type Result = C::Result; + + fn split_at(self, index: usize) -> (Self, Self, Self::Reducer) { + let (left, right, reducer) = self.base.split_at(index); + ( + SkipAnyWhileConsumer { base: left, ..self }, + SkipAnyWhileConsumer { + base: right, + ..self + }, + reducer, + ) + } + + fn into_folder(self) -> Self::Folder { + SkipAnyWhileFolder { + base: self.base.into_folder(), + predicate: self.predicate, + skipping: self.skipping, + } + } + + fn full(&self) -> bool { + self.base.full() + } +} + +impl<'p, T, C, P> UnindexedConsumer for SkipAnyWhileConsumer<'p, C, P> +where + C: UnindexedConsumer, + P: Fn(&T) -> bool + Sync, +{ + fn split_off_left(&self) -> Self { + SkipAnyWhileConsumer { + base: self.base.split_off_left(), + ..*self + } + } + + fn to_reducer(&self) -> Self::Reducer { + self.base.to_reducer() + } +} + +struct SkipAnyWhileFolder<'p, C, P> { + base: C, + predicate: &'p P, + skipping: &'p AtomicBool, +} + +fn skip(item: &T, skipping: &AtomicBool, predicate: &impl Fn(&T) -> bool) -> bool { + if !skipping.load(Ordering::Relaxed) { + return false; + } + if predicate(item) { + return true; + } + skipping.store(false, Ordering::Relaxed); + false +} + +impl<'p, T, C, P> Folder for SkipAnyWhileFolder<'p, C, P> +where + C: Folder, + P: Fn(&T) -> bool + 'p, +{ + type Result = C::Result; + + fn consume(mut self, item: T) -> Self { + if !skip(&item, self.skipping, self.predicate) { + self.base = self.base.consume(item); + } + self + } + + fn consume_iter(mut self, iter: I) -> Self + where + I: IntoIterator, + { + self.base = self.base.consume_iter( + iter.into_iter() + .skip_while(move |x| skip(x, self.skipping, self.predicate)), + ); + self + } + + fn complete(self) -> C::Result { + self.base.complete() + } + + fn full(&self) -> bool { + self.base.full() + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/splitter.rs b/anneal/v2/vendor/rayon/src/iter/splitter.rs new file mode 100644 index 0000000000..fffb625acb --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/splitter.rs @@ -0,0 +1,172 @@ +use super::plumbing::*; +use super::*; + +use std::fmt::{self, Debug}; + +/// The `split` function takes arbitrary data and a closure that knows how to +/// split it, and turns this into a `ParallelIterator`. +/// +/// # Examples +/// +/// As a simple example, Rayon can recursively split ranges of indices +/// +/// ``` +/// use rayon::iter; +/// use rayon::prelude::*; +/// use std::ops::Range; +/// +/// +/// // We define a range of indices as follows +/// type Range1D = Range; +/// +/// // Splitting it in two can be done like this +/// fn split_range1(r: Range1D) -> (Range1D, Option) { +/// // We are mathematically unable to split the range if there is only +/// // one point inside of it, but we could stop splitting before that. +/// if r.end - r.start <= 1 { return (r, None); } +/// +/// // Here, our range is considered large enough to be splittable +/// let midpoint = r.start + (r.end - r.start) / 2; +/// (r.start..midpoint, Some(midpoint..r.end)) +/// } +/// +/// // By using iter::split, Rayon will split the range until it has enough work +/// // to feed the CPU cores, then give us the resulting sub-ranges +/// iter::split(0..4096, split_range1).for_each(|sub_range| { +/// // As our initial range had a power-of-two size, the final sub-ranges +/// // should have power-of-two sizes too +/// assert!((sub_range.end - sub_range.start).is_power_of_two()); +/// }); +/// ``` +/// +/// This recursive splitting can be extended to two or three dimensions, +/// to reproduce a classic "block-wise" parallelization scheme of graphics and +/// numerical simulations: +/// +/// ``` +/// # use rayon::iter; +/// # use rayon::prelude::*; +/// # use std::ops::Range; +/// # type Range1D = Range; +/// # fn split_range1(r: Range1D) -> (Range1D, Option) { +/// # if r.end - r.start <= 1 { return (r, None); } +/// # let midpoint = r.start + (r.end - r.start) / 2; +/// # (r.start..midpoint, Some(midpoint..r.end)) +/// # } +/// # +/// // A two-dimensional range of indices can be built out of two 1D ones +/// struct Range2D { +/// // Range of horizontal indices +/// pub rx: Range1D, +/// +/// // Range of vertical indices +/// pub ry: Range1D, +/// } +/// +/// // We want to recursively split them by the largest dimension until we have +/// // enough sub-ranges to feed our mighty multi-core CPU. This function +/// // carries out one such split. +/// fn split_range2(r2: Range2D) -> (Range2D, Option) { +/// // Decide on which axis (horizontal/vertical) the range should be split +/// let width = r2.rx.end - r2.rx.start; +/// let height = r2.ry.end - r2.ry.start; +/// if width >= height { +/// // This is a wide range, split it on the horizontal axis +/// let (split_rx, ry) = (split_range1(r2.rx), r2.ry); +/// let out1 = Range2D { +/// rx: split_rx.0, +/// ry: ry.clone(), +/// }; +/// let out2 = split_rx.1.map(|rx| Range2D { rx, ry }); +/// (out1, out2) +/// } else { +/// // This is a tall range, split it on the vertical axis +/// let (rx, split_ry) = (r2.rx, split_range1(r2.ry)); +/// let out1 = Range2D { +/// rx: rx.clone(), +/// ry: split_ry.0, +/// }; +/// let out2 = split_ry.1.map(|ry| Range2D { rx, ry, }); +/// (out1, out2) +/// } +/// } +/// +/// // Again, rayon can handle the recursive splitting for us +/// let range = Range2D { rx: 0..800, ry: 0..600 }; +/// iter::split(range, split_range2).for_each(|sub_range| { +/// // If the sub-ranges were indeed split by the largest dimension, then +/// // if no dimension was twice larger than the other initially, this +/// // property will remain true in the final sub-ranges. +/// let width = sub_range.rx.end - sub_range.rx.start; +/// let height = sub_range.ry.end - sub_range.ry.start; +/// assert!((width / 2 <= height) && (height / 2 <= width)); +/// }); +/// ``` +/// +pub fn split(data: D, splitter: S) -> Split +where + D: Send, + S: Fn(D) -> (D, Option) + Sync, +{ + Split { data, splitter } +} + +/// `Split` is a parallel iterator using arbitrary data and a splitting function. +/// This struct is created by the [`split()`] function. +#[derive(Clone)] +pub struct Split { + data: D, + splitter: S, +} + +impl Debug for Split { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Split").field("data", &self.data).finish() + } +} + +impl ParallelIterator for Split +where + D: Send, + S: Fn(D) -> (D, Option) + Sync + Send, +{ + type Item = D; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let producer = SplitProducer { + data: self.data, + splitter: &self.splitter, + }; + bridge_unindexed(producer, consumer) + } +} + +struct SplitProducer<'a, D, S> { + data: D, + splitter: &'a S, +} + +impl<'a, D, S> UnindexedProducer for SplitProducer<'a, D, S> +where + D: Send, + S: Fn(D) -> (D, Option) + Sync, +{ + type Item = D; + + fn split(mut self) -> (Self, Option) { + let splitter = self.splitter; + let (left, right) = splitter(self.data); + self.data = left; + (self, right.map(|data| SplitProducer { data, splitter })) + } + + fn fold_with(self, folder: F) -> F + where + F: Folder, + { + folder.consume(self.data) + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/step_by.rs b/anneal/v2/vendor/rayon/src/iter/step_by.rs new file mode 100644 index 0000000000..3dce0645dc --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/step_by.rs @@ -0,0 +1,135 @@ +use super::plumbing::*; +use super::*; +use std::iter; + +/// `StepBy` is an iterator that skips `n` elements between each yield, where `n` is the given step. +/// This struct is created by the [`step_by()`] method on [`IndexedParallelIterator`] +/// +/// [`step_by()`]: IndexedParallelIterator::step_by() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Debug, Clone)] +pub struct StepBy { + base: I, + step: usize, +} + +impl StepBy { + /// Creates a new `StepBy` iterator. + pub(super) fn new(base: I, step: usize) -> Self { + StepBy { base, step } + } +} + +impl ParallelIterator for StepBy +where + I: IndexedParallelIterator, +{ + type Item = I::Item; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + bridge(self, consumer) + } + + fn opt_len(&self) -> Option { + Some(self.len()) + } +} + +impl IndexedParallelIterator for StepBy +where + I: IndexedParallelIterator, +{ + fn drive>(self, consumer: C) -> C::Result { + bridge(self, consumer) + } + + fn len(&self) -> usize { + self.base.len().div_ceil(self.step) + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + let len = self.base.len(); + return self.base.with_producer(Callback { + callback, + step: self.step, + len, + }); + + struct Callback { + callback: CB, + step: usize, + len: usize, + } + + impl ProducerCallback for Callback + where + CB: ProducerCallback, + { + type Output = CB::Output; + fn callback

(self, base: P) -> CB::Output + where + P: Producer, + { + let producer = StepByProducer { + base, + step: self.step, + len: self.len, + }; + self.callback.callback(producer) + } + } + } +} + +// //////////////////////////////////////////////////////////////////////// +// Producer implementation + +struct StepByProducer

{ + base: P, + step: usize, + len: usize, +} + +impl

Producer for StepByProducer

+where + P: Producer, +{ + type Item = P::Item; + type IntoIter = iter::StepBy; + + fn into_iter(self) -> Self::IntoIter { + self.base.into_iter().step_by(self.step) + } + + fn split_at(self, index: usize) -> (Self, Self) { + let elem_index = Ord::min(index * self.step, self.len); + + let (left, right) = self.base.split_at(elem_index); + ( + StepByProducer { + base: left, + step: self.step, + len: elem_index, + }, + StepByProducer { + base: right, + step: self.step, + len: self.len - elem_index, + }, + ) + } + + fn min_len(&self) -> usize { + self.base.min_len().div_ceil(self.step) + } + + fn max_len(&self) -> usize { + self.base.max_len() / self.step + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/sum.rs b/anneal/v2/vendor/rayon/src/iter/sum.rs new file mode 100644 index 0000000000..ddae810f95 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/sum.rs @@ -0,0 +1,110 @@ +use super::plumbing::*; +use super::ParallelIterator; + +use std::iter::{self, Sum}; +use std::marker::PhantomData; + +pub(super) fn sum(pi: PI) -> S +where + PI: ParallelIterator, + S: Send + Sum + Sum, +{ + pi.drive_unindexed(SumConsumer::new()) +} + +fn add(left: T, right: T) -> T { + [left, right].into_iter().sum() +} + +struct SumConsumer { + _marker: PhantomData<*const S>, +} + +unsafe impl Send for SumConsumer {} + +impl SumConsumer { + fn new() -> SumConsumer { + SumConsumer { + _marker: PhantomData, + } + } +} + +impl Consumer for SumConsumer +where + S: Send + Sum + Sum, +{ + type Folder = SumFolder; + type Reducer = Self; + type Result = S; + + fn split_at(self, _index: usize) -> (Self, Self, Self) { + (SumConsumer::new(), SumConsumer::new(), SumConsumer::new()) + } + + fn into_folder(self) -> Self::Folder { + SumFolder { + sum: iter::empty::().sum(), + } + } + + fn full(&self) -> bool { + false + } +} + +impl UnindexedConsumer for SumConsumer +where + S: Send + Sum + Sum, +{ + fn split_off_left(&self) -> Self { + SumConsumer::new() + } + + fn to_reducer(&self) -> Self::Reducer { + SumConsumer::new() + } +} + +impl Reducer for SumConsumer +where + S: Send + Sum, +{ + fn reduce(self, left: S, right: S) -> S { + add(left, right) + } +} + +struct SumFolder { + sum: S, +} + +impl Folder for SumFolder +where + S: Sum + Sum, +{ + type Result = S; + + fn consume(self, item: T) -> Self { + SumFolder { + sum: add(self.sum, iter::once(item).sum()), + } + } + + fn consume_iter(self, iter: I) -> Self + where + I: IntoIterator, + { + SumFolder { + sum: add(self.sum, iter.into_iter().sum()), + } + } + + fn complete(self) -> S { + self.sum + } + + fn full(&self) -> bool { + false + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/take.rs b/anneal/v2/vendor/rayon/src/iter/take.rs new file mode 100644 index 0000000000..717d4ead73 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/take.rs @@ -0,0 +1,84 @@ +use super::plumbing::*; +use super::*; + +/// `Take` is an iterator that iterates over the first `n` elements. +/// This struct is created by the [`take()`] method on [`IndexedParallelIterator`] +/// +/// [`take()`]: IndexedParallelIterator::take() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Debug, Clone)] +pub struct Take { + base: I, + n: usize, +} + +impl Take +where + I: IndexedParallelIterator, +{ + /// Creates a new `Take` iterator. + pub(super) fn new(base: I, n: usize) -> Self { + let n = Ord::min(base.len(), n); + Take { base, n } + } +} + +impl ParallelIterator for Take +where + I: IndexedParallelIterator, +{ + type Item = I::Item; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + bridge(self, consumer) + } + + fn opt_len(&self) -> Option { + Some(self.len()) + } +} + +impl IndexedParallelIterator for Take +where + I: IndexedParallelIterator, +{ + fn len(&self) -> usize { + self.n + } + + fn drive>(self, consumer: C) -> C::Result { + bridge(self, consumer) + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + return self.base.with_producer(Callback { + callback, + n: self.n, + }); + + struct Callback { + callback: CB, + n: usize, + } + + impl ProducerCallback for Callback + where + CB: ProducerCallback, + { + type Output = CB::Output; + fn callback

(self, base: P) -> CB::Output + where + P: Producer, + { + let (producer, _) = base.split_at(self.n); + self.callback.callback(producer) + } + } + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/take_any.rs b/anneal/v2/vendor/rayon/src/iter/take_any.rs new file mode 100644 index 0000000000..17434d72b2 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/take_any.rs @@ -0,0 +1,140 @@ +use super::plumbing::*; +use super::*; +use std::sync::atomic::{AtomicUsize, Ordering}; + +/// `TakeAny` is an iterator that iterates over `n` elements from anywhere in `I`. +/// This struct is created by the [`take_any()`] method on [`ParallelIterator`] +/// +/// [`take_any()`]: ParallelIterator::take_any() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Clone, Debug)] +pub struct TakeAny { + base: I, + count: usize, +} + +impl TakeAny { + /// Creates a new `TakeAny` iterator. + pub(super) fn new(base: I, count: usize) -> Self { + TakeAny { base, count } + } +} + +impl ParallelIterator for TakeAny +where + I: ParallelIterator, +{ + type Item = I::Item; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let consumer1 = TakeAnyConsumer { + base: consumer, + count: &AtomicUsize::new(self.count), + }; + self.base.drive_unindexed(consumer1) + } +} + +// //////////////////////////////////////////////////////////////////////// +// Consumer implementation + +struct TakeAnyConsumer<'f, C> { + base: C, + count: &'f AtomicUsize, +} + +impl<'f, T, C> Consumer for TakeAnyConsumer<'f, C> +where + C: Consumer, + T: Send, +{ + type Folder = TakeAnyFolder<'f, C::Folder>; + type Reducer = C::Reducer; + type Result = C::Result; + + fn split_at(self, index: usize) -> (Self, Self, Self::Reducer) { + let (left, right, reducer) = self.base.split_at(index); + ( + TakeAnyConsumer { base: left, ..self }, + TakeAnyConsumer { + base: right, + ..self + }, + reducer, + ) + } + + fn into_folder(self) -> Self::Folder { + TakeAnyFolder { + base: self.base.into_folder(), + count: self.count, + } + } + + fn full(&self) -> bool { + self.count.load(Ordering::Relaxed) == 0 || self.base.full() + } +} + +impl<'f, T, C> UnindexedConsumer for TakeAnyConsumer<'f, C> +where + C: UnindexedConsumer, + T: Send, +{ + fn split_off_left(&self) -> Self { + TakeAnyConsumer { + base: self.base.split_off_left(), + ..*self + } + } + + fn to_reducer(&self) -> Self::Reducer { + self.base.to_reducer() + } +} + +struct TakeAnyFolder<'f, C> { + base: C, + count: &'f AtomicUsize, +} + +fn checked_decrement(u: &AtomicUsize) -> bool { + u.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |u| u.checked_sub(1)) + .is_ok() +} + +impl<'f, T, C> Folder for TakeAnyFolder<'f, C> +where + C: Folder, +{ + type Result = C::Result; + + fn consume(mut self, item: T) -> Self { + if checked_decrement(self.count) { + self.base = self.base.consume(item); + } + self + } + + fn consume_iter(mut self, iter: I) -> Self + where + I: IntoIterator, + { + self.base = self.base.consume_iter( + iter.into_iter() + .take_while(move |_| checked_decrement(self.count)), + ); + self + } + + fn complete(self) -> C::Result { + self.base.complete() + } + + fn full(&self) -> bool { + self.count.load(Ordering::Relaxed) == 0 || self.base.full() + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/take_any_while.rs b/anneal/v2/vendor/rayon/src/iter/take_any_while.rs new file mode 100644 index 0000000000..3dde3071e0 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/take_any_while.rs @@ -0,0 +1,162 @@ +use super::plumbing::*; +use super::*; +use std::fmt; +use std::sync::atomic::{AtomicBool, Ordering}; + +/// `TakeAnyWhile` is an iterator that iterates over elements from anywhere in `I` +/// until the callback returns `false`. +/// This struct is created by the [`take_any_while()`] method on [`ParallelIterator`] +/// +/// [`take_any_while()`]: ParallelIterator::take_any_while() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Clone)] +pub struct TakeAnyWhile { + base: I, + predicate: P, +} + +impl fmt::Debug for TakeAnyWhile { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("TakeAnyWhile") + .field("base", &self.base) + .finish() + } +} + +impl TakeAnyWhile { + /// Creates a new `TakeAnyWhile` iterator. + pub(super) fn new(base: I, predicate: P) -> Self { + TakeAnyWhile { base, predicate } + } +} + +impl ParallelIterator for TakeAnyWhile +where + I: ParallelIterator, + P: Fn(&I::Item) -> bool + Sync + Send, +{ + type Item = I::Item; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let consumer1 = TakeAnyWhileConsumer { + base: consumer, + predicate: &self.predicate, + taking: &AtomicBool::new(true), + }; + self.base.drive_unindexed(consumer1) + } +} + +// //////////////////////////////////////////////////////////////////////// +// Consumer implementation + +struct TakeAnyWhileConsumer<'p, C, P> { + base: C, + predicate: &'p P, + taking: &'p AtomicBool, +} + +impl<'p, T, C, P> Consumer for TakeAnyWhileConsumer<'p, C, P> +where + C: Consumer, + P: Fn(&T) -> bool + Sync, +{ + type Folder = TakeAnyWhileFolder<'p, C::Folder, P>; + type Reducer = C::Reducer; + type Result = C::Result; + + fn split_at(self, index: usize) -> (Self, Self, Self::Reducer) { + let (left, right, reducer) = self.base.split_at(index); + ( + TakeAnyWhileConsumer { base: left, ..self }, + TakeAnyWhileConsumer { + base: right, + ..self + }, + reducer, + ) + } + + fn into_folder(self) -> Self::Folder { + TakeAnyWhileFolder { + base: self.base.into_folder(), + predicate: self.predicate, + taking: self.taking, + } + } + + fn full(&self) -> bool { + !self.taking.load(Ordering::Relaxed) || self.base.full() + } +} + +impl<'p, T, C, P> UnindexedConsumer for TakeAnyWhileConsumer<'p, C, P> +where + C: UnindexedConsumer, + P: Fn(&T) -> bool + Sync, +{ + fn split_off_left(&self) -> Self { + TakeAnyWhileConsumer { + base: self.base.split_off_left(), + ..*self + } + } + + fn to_reducer(&self) -> Self::Reducer { + self.base.to_reducer() + } +} + +struct TakeAnyWhileFolder<'p, C, P> { + base: C, + predicate: &'p P, + taking: &'p AtomicBool, +} + +fn take(item: &T, taking: &AtomicBool, predicate: &impl Fn(&T) -> bool) -> bool { + if !taking.load(Ordering::Relaxed) { + return false; + } + if predicate(item) { + return true; + } + taking.store(false, Ordering::Relaxed); + false +} + +impl<'p, T, C, P> Folder for TakeAnyWhileFolder<'p, C, P> +where + C: Folder, + P: Fn(&T) -> bool + 'p, +{ + type Result = C::Result; + + fn consume(mut self, item: T) -> Self { + if take(&item, self.taking, self.predicate) { + self.base = self.base.consume(item); + } + self + } + + fn consume_iter(mut self, iter: I) -> Self + where + I: IntoIterator, + { + self.base = self.base.consume_iter( + iter.into_iter() + .take_while(move |x| take(x, self.taking, self.predicate)), + ); + self + } + + fn complete(self) -> C::Result { + self.base.complete() + } + + fn full(&self) -> bool { + !self.taking.load(Ordering::Relaxed) || self.base.full() + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/test.rs b/anneal/v2/vendor/rayon/src/iter/test.rs new file mode 100644 index 0000000000..a35e98303c --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/test.rs @@ -0,0 +1,2392 @@ +use std::sync::atomic::{AtomicUsize, Ordering}; + +use super::*; +use crate::prelude::*; +use rayon_core::*; + +use rand::distr::StandardUniform; +use rand::{Rng, SeedableRng}; +use rand_xorshift::XorShiftRng; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::collections::{BinaryHeap, VecDeque}; +use std::ffi::OsStr; +use std::fmt::Debug; +use std::sync::mpsc; + +fn is_indexed(_: T) {} + +fn seeded_rng() -> XorShiftRng { + let mut seed = ::Seed::default(); + (0..).zip(seed.as_mut()).for_each(|(i, x)| *x = i); + XorShiftRng::from_seed(seed) +} + +#[test] +fn execute() { + let a: Vec = (0..1024).collect(); + let mut b = vec![]; + a.par_iter().map(|&i| i + 1).collect_into_vec(&mut b); + let c: Vec = (0..1024).map(|i| i + 1).collect(); + assert_eq!(b, c); +} + +#[test] +fn execute_cloned() { + let a: Vec = (0..1024).collect(); + let mut b: Vec = vec![]; + a.par_iter().cloned().collect_into_vec(&mut b); + let c: Vec = (0..1024).collect(); + assert_eq!(b, c); +} + +#[test] +fn execute_range() { + let a = 0i32..1024; + let mut b = vec![]; + a.into_par_iter().map(|i| i + 1).collect_into_vec(&mut b); + let c: Vec = (0..1024).map(|i| i + 1).collect(); + assert_eq!(b, c); +} + +#[test] +fn execute_unindexed_range() { + let a = 0i64..1024; + let b: LinkedList = a.into_par_iter().map(|i| i + 1).collect(); + let c: LinkedList = (0..1024).map(|i| i + 1).collect(); + assert_eq!(b, c); +} + +#[test] +fn execute_pseudo_indexed_range() { + let range = i128::MAX - 1024..i128::MAX; + + // Given `Some` length, collecting `Vec` will try to act indexed. + let a = range.clone().into_par_iter(); + assert_eq!(a.opt_len(), Some(1024)); + + let b: Vec = a.map(|i| i + 1).collect(); + let c: Vec = range.map(|i| i + 1).collect(); + assert_eq!(b, c); +} + +#[test] +fn check_map_indexed() { + let a = [1, 2, 3]; + is_indexed(a.par_iter().map(|x| x)); +} + +#[test] +fn map_sum() { + let a: Vec = (0..1024).collect(); + let r1: i32 = a.par_iter().map(|&i| i + 1).sum(); + let r2 = a.iter().map(|&i| i + 1).sum(); + assert_eq!(r1, r2); +} + +#[test] +fn map_reduce() { + let a: Vec = (0..1024).collect(); + let r1 = a.par_iter().map(|&i| i + 1).reduce(|| 0, |i, j| i + j); + let r2 = a.iter().map(|&i| i + 1).sum(); + assert_eq!(r1, r2); +} + +#[test] +fn map_reduce_with() { + let a: Vec = (0..1024).collect(); + let r1 = a.par_iter().map(|&i| i + 1).reduce_with(|i, j| i + j); + let r2 = a.iter().map(|&i| i + 1).sum(); + assert_eq!(r1, Some(r2)); +} + +#[test] +fn fold_map_reduce() { + // Kind of a weird test, but it demonstrates various + // transformations that are taking place. Relies on + // `with_max_len(1).fold()` being equivalent to `map()`. + // + // Take each number from 0 to 32 and fold them by appending to a + // vector. Because of `with_max_len(1)`, this will produce 32 vectors, + // each with one item. We then collect all of these into an + // individual vector by mapping each into their own vector (so we + // have Vec>) and then reducing those into a single + // vector. + let r1 = (0_i32..32) + .into_par_iter() + .with_max_len(1) + .fold(Vec::new, |mut v, e| { + v.push(e); + v + }) + .map(|v| vec![v]) + .reduce_with(|mut v_a, v_b| { + v_a.extend(v_b); + v_a + }); + assert_eq!( + r1, + Some(vec![ + vec![0], + vec![1], + vec![2], + vec![3], + vec![4], + vec![5], + vec![6], + vec![7], + vec![8], + vec![9], + vec![10], + vec![11], + vec![12], + vec![13], + vec![14], + vec![15], + vec![16], + vec![17], + vec![18], + vec![19], + vec![20], + vec![21], + vec![22], + vec![23], + vec![24], + vec![25], + vec![26], + vec![27], + vec![28], + vec![29], + vec![30], + vec![31] + ]) + ); +} + +#[test] +fn fold_is_full() { + let counter = AtomicUsize::new(0); + let a = (0_i32..2048) + .into_par_iter() + .inspect(|_| { + counter.fetch_add(1, Ordering::SeqCst); + }) + .fold(|| 0, |a, b| a + b) + .find_any(|_| true); + assert!(a.is_some()); + assert!(counter.load(Ordering::SeqCst) < 2048); // should not have visited every single one +} + +#[test] +fn check_step_by() { + let a: Vec = (0..1024).step_by(2).collect(); + let b: Vec = (0..1024).into_par_iter().step_by(2).collect(); + + assert_eq!(a, b); +} + +#[test] +fn check_step_by_unaligned() { + let a: Vec = (0..1029).step_by(10).collect(); + let b: Vec = (0..1029).into_par_iter().step_by(10).collect(); + + assert_eq!(a, b) +} + +#[test] +fn check_step_by_rev() { + let a: Vec = (0..1024).step_by(2).rev().collect(); + let b: Vec = (0..1024).into_par_iter().step_by(2).rev().collect(); + + assert_eq!(a, b); +} + +#[test] +fn check_enumerate() { + let a: Vec = (0..1024).rev().collect(); + + let mut b = vec![]; + a.par_iter() + .enumerate() + .map(|(i, &x)| i + x) + .collect_into_vec(&mut b); + assert!(b.iter().all(|&x| x == a.len() - 1)); +} + +#[test] +fn check_enumerate_rev() { + let a: Vec = (0..1024).rev().collect(); + + let mut b = vec![]; + a.par_iter() + .enumerate() + .rev() + .map(|(i, &x)| i + x) + .collect_into_vec(&mut b); + assert!(b.iter().all(|&x| x == a.len() - 1)); +} + +#[test] +fn check_indices_after_enumerate_split() { + let a: Vec = (0..1024).collect(); + a.par_iter().enumerate().with_producer(WithProducer); + + struct WithProducer; + impl<'a> ProducerCallback<(usize, &'a i32)> for WithProducer { + type Output = (); + fn callback

(self, producer: P) + where + P: Producer, + { + let (a, b) = producer.split_at(512); + for ((index, value), trusted_index) in a.into_iter().zip(0..) { + assert_eq!(index, trusted_index); + assert_eq!(index, *value as usize); + } + for ((index, value), trusted_index) in b.into_iter().zip(512..) { + assert_eq!(index, trusted_index); + assert_eq!(index, *value as usize); + } + } + } +} + +#[test] +fn check_increment() { + let mut a: Vec = (0..1024).rev().collect(); + + a.par_iter_mut().enumerate().for_each(|(i, v)| *v += i); + + assert!(a.iter().all(|&x| x == a.len() - 1)); +} + +#[test] +fn check_skip() { + let a: Vec = (0..1024).collect(); + + let mut v1 = Vec::new(); + a.par_iter().skip(16).collect_into_vec(&mut v1); + let v2 = a.iter().skip(16).collect::>(); + assert_eq!(v1, v2); + + let mut v1 = Vec::new(); + a.par_iter().skip(2048).collect_into_vec(&mut v1); + let v2 = a.iter().skip(2048).collect::>(); + assert_eq!(v1, v2); + + let mut v1 = Vec::new(); + a.par_iter().skip(0).collect_into_vec(&mut v1); + #[allow(clippy::iter_skip_zero)] + let v2 = a.iter().skip(0).collect::>(); + assert_eq!(v1, v2); + + // Check that the skipped elements side effects are executed + use std::sync::atomic::{AtomicUsize, Ordering}; + let num = AtomicUsize::new(0); + a.par_iter() + .map(|&n| num.fetch_add(n, Ordering::Relaxed)) + .skip(512) + .count(); + assert_eq!(num.load(Ordering::Relaxed), a.iter().sum::()); +} + +#[test] +fn check_take() { + let a: Vec = (0..1024).collect(); + + let mut v1 = Vec::new(); + a.par_iter().take(16).collect_into_vec(&mut v1); + let v2 = a.iter().take(16).collect::>(); + assert_eq!(v1, v2); + + let mut v1 = Vec::new(); + a.par_iter().take(2048).collect_into_vec(&mut v1); + let v2 = a.iter().take(2048).collect::>(); + assert_eq!(v1, v2); + + let mut v1 = Vec::new(); + a.par_iter().take(0).collect_into_vec(&mut v1); + let v2 = a.iter().take(0).collect::>(); + assert_eq!(v1, v2); +} + +#[test] +fn check_inspect() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + let a = AtomicUsize::new(0); + let b: usize = (0_usize..1024) + .into_par_iter() + .inspect(|&i| { + a.fetch_add(i, Ordering::Relaxed); + }) + .sum(); + + assert_eq!(a.load(Ordering::Relaxed), b); +} + +#[test] +fn check_move() { + let a = vec![vec![1, 2, 3]]; + let ptr = a[0].as_ptr(); + + let mut b = vec![]; + a.into_par_iter().collect_into_vec(&mut b); + + // a simple move means the inner vec will be completely unchanged + assert_eq!(ptr, b[0].as_ptr()); +} + +#[test] +fn check_drops() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + let c = AtomicUsize::new(0); + let a = vec![DropCounter(&c); 10]; + + let mut b = vec![]; + a.clone().into_par_iter().collect_into_vec(&mut b); + assert_eq!(c.load(Ordering::Relaxed), 0); + + b.into_par_iter(); + assert_eq!(c.load(Ordering::Relaxed), 10); + + a.into_par_iter().with_producer(Partial); + assert_eq!(c.load(Ordering::Relaxed), 20); + + #[derive(Clone)] + struct DropCounter<'a>(&'a AtomicUsize); + impl<'a> Drop for DropCounter<'a> { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::Relaxed); + } + } + + struct Partial; + impl<'a> ProducerCallback> for Partial { + type Output = (); + fn callback

(self, producer: P) + where + P: Producer>, + { + let (a, _) = producer.split_at(5); + a.into_iter().next(); + } + } +} + +#[test] +fn check_slice_indexed() { + let a = vec![1, 2, 3]; + is_indexed(a.par_iter()); +} + +#[test] +fn check_slice_mut_indexed() { + let mut a = vec![1, 2, 3]; + is_indexed(a.par_iter_mut()); +} + +#[test] +fn check_vec_indexed() { + let a = vec![1, 2, 3]; + is_indexed(a.into_par_iter()); +} + +#[test] +fn check_range_indexed() { + is_indexed((1..5).into_par_iter()); +} + +#[test] +fn check_cmp_direct() { + let a = (0..1024).into_par_iter(); + let b = (0..1024).into_par_iter(); + + let result = a.cmp(b); + + assert!(result == ::std::cmp::Ordering::Equal); +} + +#[test] +fn check_cmp_to_seq() { + assert_eq!( + (0..1024).into_par_iter().cmp(0..1024), + (0..1024).cmp(0..1024) + ); +} + +#[test] +fn check_cmp_rng_to_seq() { + let mut rng = seeded_rng(); + let rng = &mut rng; + let a: Vec = rng.sample_iter(&StandardUniform).take(1024).collect(); + let b: Vec = rng.sample_iter(&StandardUniform).take(1024).collect(); + for i in 0..a.len() { + let par_result = a[i..].par_iter().cmp(b[i..].par_iter()); + let seq_result = a[i..].iter().cmp(b[i..].iter()); + + assert_eq!(par_result, seq_result); + } +} + +#[test] +fn check_cmp_lt_direct() { + let a = (0..1024).into_par_iter(); + let b = (1..1024).into_par_iter(); + + let result = a.cmp(b); + + assert!(result == ::std::cmp::Ordering::Less); +} + +#[test] +fn check_cmp_lt_to_seq() { + assert_eq!( + (0..1024).into_par_iter().cmp(1..1024), + (0..1024).cmp(1..1024) + ) +} + +#[test] +fn check_cmp_gt_direct() { + let a = (1..1024).into_par_iter(); + let b = (0..1024).into_par_iter(); + + let result = a.cmp(b); + + assert!(result == ::std::cmp::Ordering::Greater); +} + +#[test] +fn check_cmp_gt_to_seq() { + assert_eq!( + (1..1024).into_par_iter().cmp(0..1024), + (1..1024).cmp(0..1024) + ) +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn check_cmp_short_circuit() { + // We only use a single thread in order to make the short-circuit behavior deterministic. + let pool = ThreadPoolBuilder::new().num_threads(1).build().unwrap(); + + let a = vec![0; 1024]; + let mut b = a.clone(); + b[42] = 1; + + pool.install(|| { + let expected = ::std::cmp::Ordering::Less; + assert_eq!(a.par_iter().cmp(&b), expected); + + for len in 1..10 { + let counter = AtomicUsize::new(0); + let result = a + .par_iter() + .with_max_len(len) + .inspect(|_| { + counter.fetch_add(1, Ordering::SeqCst); + }) + .cmp(&b); + assert_eq!(result, expected); + // should not have visited every single one + assert!(counter.into_inner() < a.len()); + } + }); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn check_partial_cmp_short_circuit() { + // We only use a single thread to make the short-circuit behavior deterministic. + let pool = ThreadPoolBuilder::new().num_threads(1).build().unwrap(); + + let a = vec![0; 1024]; + let mut b = a.clone(); + b[42] = 1; + + pool.install(|| { + let expected = Some(::std::cmp::Ordering::Less); + assert_eq!(a.par_iter().partial_cmp(&b), expected); + + for len in 1..10 { + let counter = AtomicUsize::new(0); + let result = a + .par_iter() + .with_max_len(len) + .inspect(|_| { + counter.fetch_add(1, Ordering::SeqCst); + }) + .partial_cmp(&b); + assert_eq!(result, expected); + // should not have visited every single one + assert!(counter.into_inner() < a.len()); + } + }); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn check_partial_cmp_nan_short_circuit() { + // We only use a single thread to make the short-circuit behavior deterministic. + let pool = ThreadPoolBuilder::new().num_threads(1).build().unwrap(); + + let a = vec![0.0; 1024]; + let mut b = a.clone(); + b[42] = f64::NAN; + + pool.install(|| { + let expected = None; + assert_eq!(a.par_iter().partial_cmp(&b), expected); + + for len in 1..10 { + let counter = AtomicUsize::new(0); + let result = a + .par_iter() + .with_max_len(len) + .inspect(|_| { + counter.fetch_add(1, Ordering::SeqCst); + }) + .partial_cmp(&b); + assert_eq!(result, expected); + // should not have visited every single one + assert!(counter.into_inner() < a.len()); + } + }); +} + +#[test] +fn check_partial_cmp_direct() { + let a = (0..1024).into_par_iter(); + let b = (0..1024).into_par_iter(); + + let result = a.partial_cmp(b); + + assert!(result == Some(::std::cmp::Ordering::Equal)); +} + +#[test] +fn check_partial_cmp_to_seq() { + let par_result = (0..1024).into_par_iter().partial_cmp(0..1024); + let seq_result = (0..1024).partial_cmp(0..1024); + assert_eq!(par_result, seq_result); +} + +#[test] +fn check_partial_cmp_rng_to_seq() { + let mut rng = seeded_rng(); + let rng = &mut rng; + let a: Vec = rng.sample_iter(&StandardUniform).take(1024).collect(); + let b: Vec = rng.sample_iter(&StandardUniform).take(1024).collect(); + for i in 0..a.len() { + let par_result = a[i..].par_iter().partial_cmp(b[i..].par_iter()); + let seq_result = a[i..].iter().partial_cmp(b[i..].iter()); + + assert_eq!(par_result, seq_result); + } +} + +#[test] +fn check_partial_cmp_lt_direct() { + let a = (0..1024).into_par_iter(); + let b = (1..1024).into_par_iter(); + + let result = a.partial_cmp(b); + + assert!(result == Some(::std::cmp::Ordering::Less)); +} + +#[test] +fn check_partial_cmp_lt_to_seq() { + let par_result = (0..1024).into_par_iter().partial_cmp(1..1024); + let seq_result = (0..1024).partial_cmp(1..1024); + assert_eq!(par_result, seq_result); +} + +#[test] +fn check_partial_cmp_gt_direct() { + let a = (1..1024).into_par_iter(); + let b = (0..1024).into_par_iter(); + + let result = a.partial_cmp(b); + + assert!(result == Some(::std::cmp::Ordering::Greater)); +} + +#[test] +fn check_partial_cmp_gt_to_seq() { + let par_result = (1..1024).into_par_iter().partial_cmp(0..1024); + let seq_result = (1..1024).partial_cmp(0..1024); + assert_eq!(par_result, seq_result); +} + +#[test] +fn check_partial_cmp_none_direct() { + let a = vec![f64::NAN, 0.0]; + let b = vec![0.0, 1.0]; + + let result = a.par_iter().partial_cmp(b.par_iter()); + + assert!(result.is_none()); +} + +#[test] +fn check_partial_cmp_none_to_seq() { + let a = vec![f64::NAN, 0.0]; + let b = vec![0.0, 1.0]; + + let par_result = a.par_iter().partial_cmp(b.par_iter()); + let seq_result = a.iter().partial_cmp(b.iter()); + + assert_eq!(par_result, seq_result); +} + +#[test] +fn check_partial_cmp_late_nan_direct() { + let a = vec![0.0, f64::NAN]; + let b = vec![1.0, 1.0]; + + let result = a.par_iter().partial_cmp(b.par_iter()); + + assert!(result == Some(::std::cmp::Ordering::Less)); +} + +#[test] +fn check_partial_cmp_late_nan_to_seq() { + let a = vec![0.0, f64::NAN]; + let b = vec![1.0, 1.0]; + + let par_result = a.par_iter().partial_cmp(b.par_iter()); + let seq_result = a.iter().partial_cmp(b.iter()); + + assert_eq!(par_result, seq_result); +} + +#[test] +fn check_cmp_lengths() { + // comparisons should consider length if they are otherwise equal + let a = vec![0; 1024]; + let b = vec![0; 1025]; + + assert_eq!(a.par_iter().cmp(&b), a.iter().cmp(&b)); + assert_eq!(a.par_iter().partial_cmp(&b), a.iter().partial_cmp(&b)); +} + +#[test] +fn check_eq_direct() { + let a = (0..1024).into_par_iter(); + let b = (0..1024).into_par_iter(); + + let result = a.eq(b); + + assert!(result); +} + +#[test] +fn check_eq_to_seq() { + let par_result = (0..1024).into_par_iter().eq((0..1024).into_par_iter()); + let seq_result = (0..1024).eq(0..1024); + + assert_eq!(par_result, seq_result); +} + +#[test] +fn check_ne_direct() { + let a = (0..1024).into_par_iter(); + let b = (1..1024).into_par_iter(); + + let result = a.ne(b); + + assert!(result); +} + +#[test] +fn check_ne_to_seq() { + let par_result = (0..1024).into_par_iter().ne((1..1025).into_par_iter()); + let seq_result = (0..1024).ne(1..1025); + + assert_eq!(par_result, seq_result); +} + +#[test] +fn check_ne_lengths() { + // equality should consider length too + let a = vec![0; 1024]; + let b = vec![0; 1025]; + + assert_eq!(a.par_iter().eq(&b), a.iter().eq(&b)); + assert_eq!(a.par_iter().ne(&b), a.iter().ne(&b)); +} + +#[test] +fn check_lt_direct() { + assert!((0..1024).into_par_iter().lt(1..1024)); + assert!(!(1..1024).into_par_iter().lt(0..1024)); +} + +#[test] +fn check_lt_to_seq() { + let par_result = (0..1024).into_par_iter().lt((1..1024).into_par_iter()); + let seq_result = (0..1024).lt(1..1024); + + assert_eq!(par_result, seq_result); +} + +#[test] +fn check_le_equal_direct() { + assert!((0..1024).into_par_iter().le((0..1024).into_par_iter())); +} + +#[test] +fn check_le_equal_to_seq() { + let par_result = (0..1024).into_par_iter().le((0..1024).into_par_iter()); + let seq_result = (0..1024).le(0..1024); + + assert_eq!(par_result, seq_result); +} + +#[test] +fn check_le_less_direct() { + assert!((0..1024).into_par_iter().le((1..1024).into_par_iter())); +} + +#[test] +fn check_le_less_to_seq() { + let par_result = (0..1024).into_par_iter().le((1..1024).into_par_iter()); + let seq_result = (0..1024).le(1..1024); + + assert_eq!(par_result, seq_result); +} + +#[test] +fn check_gt_direct() { + assert!((1..1024).into_par_iter().gt((0..1024).into_par_iter())); +} + +#[test] +fn check_gt_to_seq() { + let par_result = (1..1024).into_par_iter().gt((0..1024).into_par_iter()); + let seq_result = (1..1024).gt(0..1024); + + assert_eq!(par_result, seq_result); +} + +#[test] +fn check_ge_equal_direct() { + assert!((0..1024).into_par_iter().ge((0..1024).into_par_iter())); +} + +#[test] +fn check_ge_equal_to_seq() { + let par_result = (0..1024).into_par_iter().ge((0..1024).into_par_iter()); + let seq_result = (0..1024).ge(0..1024); + + assert_eq!(par_result, seq_result); +} + +#[test] +fn check_ge_greater_direct() { + assert!((1..1024).into_par_iter().ge((0..1024).into_par_iter())); +} + +#[test] +fn check_ge_greater_to_seq() { + let par_result = (1..1024).into_par_iter().ge((0..1024).into_par_iter()); + let seq_result = (1..1024).ge(0..1024); + + assert_eq!(par_result, seq_result); +} + +#[test] +fn check_zip() { + let mut a: Vec = (0..1024).rev().collect(); + let b: Vec = (0..1024).collect(); + + a.par_iter_mut().zip(&b[..]).for_each(|(a, &b)| *a += b); + + assert!(a.iter().all(|&x| x == a.len() - 1)); +} + +#[test] +fn check_zip_into_par_iter() { + let mut a: Vec = (0..1024).rev().collect(); + let b: Vec = (0..1024).collect(); + + a.par_iter_mut() + .zip(&b) // here we rely on &b iterating over &usize + .for_each(|(a, &b)| *a += b); + + assert!(a.iter().all(|&x| x == a.len() - 1)); +} + +#[test] +fn check_zip_into_mut_par_iter() { + let a: Vec = (0..1024).rev().collect(); + let mut b: Vec = (0..1024).collect(); + + a.par_iter().zip(&mut b).for_each(|(&a, b)| *b += a); + + assert!(b.iter().all(|&x| x == b.len() - 1)); +} + +#[test] +fn check_zip_range() { + let mut a: Vec = (0..1024).rev().collect(); + + a.par_iter_mut() + .zip(0usize..1024) + .for_each(|(a, b)| *a += b); + + assert!(a.iter().all(|&x| x == a.len() - 1)); +} + +#[test] +fn check_zip_eq() { + let mut a: Vec = (0..1024).rev().collect(); + let b: Vec = (0..1024).collect(); + + a.par_iter_mut().zip_eq(&b[..]).for_each(|(a, &b)| *a += b); + + assert!(a.iter().all(|&x| x == a.len() - 1)); +} + +#[test] +fn check_zip_eq_into_par_iter() { + let mut a: Vec = (0..1024).rev().collect(); + let b: Vec = (0..1024).collect(); + + a.par_iter_mut() + .zip_eq(&b) // here we rely on &b iterating over &usize + .for_each(|(a, &b)| *a += b); + + assert!(a.iter().all(|&x| x == a.len() - 1)); +} + +#[test] +fn check_zip_eq_into_mut_par_iter() { + let a: Vec = (0..1024).rev().collect(); + let mut b: Vec = (0..1024).collect(); + + a.par_iter().zip_eq(&mut b).for_each(|(&a, b)| *b += a); + + assert!(b.iter().all(|&x| x == b.len() - 1)); +} + +#[test] +fn check_zip_eq_range() { + let mut a: Vec = (0..1024).rev().collect(); + + a.par_iter_mut() + .zip_eq(0usize..1024) + .for_each(|(a, b)| *a += b); + + assert!(a.iter().all(|&x| x == a.len() - 1)); +} + +#[test] +fn check_sum_filtered_ints() { + let a: Vec = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + let par_sum_evens: i32 = a.par_iter().filter(|&x| (x & 1) == 0).sum(); + let seq_sum_evens = a.iter().filter(|&x| (x & 1) == 0).sum(); + assert_eq!(par_sum_evens, seq_sum_evens); +} + +#[test] +fn check_sum_filtermap_ints() { + let a: Vec = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + let par_sum_evens: u32 = a + .par_iter() + .filter_map(|&x| if (x & 1) == 0 { Some(x as u32) } else { None }) + .sum(); + let seq_sum_evens = a + .iter() + .filter_map(|&x| if (x & 1) == 0 { Some(x as u32) } else { None }) + .sum(); + assert_eq!(par_sum_evens, seq_sum_evens); +} + +#[test] +fn check_flat_map_nested_ranges() { + // FIXME -- why are precise type hints required on the integers here? + + let v: i32 = (0_i32..10) + .into_par_iter() + .flat_map(|i| (0_i32..10).into_par_iter().map(move |j| (i, j))) + .map(|(i, j)| i * j) + .sum(); + + let w = (0_i32..10) + .flat_map(|i| (0_i32..10).map(move |j| (i, j))) + .map(|(i, j)| i * j) + .sum(); + + assert_eq!(v, w); +} + +#[test] +fn check_empty_flat_map_sum() { + let a: Vec = (0..1024).collect(); + let empty = &a[..0]; + + // empty on the inside + let b: i32 = a.par_iter().flat_map(|_| empty).sum(); + assert_eq!(b, 0); + + // empty on the outside + let c: i32 = empty.par_iter().flat_map(|_| a.par_iter()).sum(); + assert_eq!(c, 0); +} + +#[test] +fn check_flatten_vec() { + let a: Vec = (0..1024).collect(); + let b: Vec> = vec![a.clone(), a.clone(), a.clone(), a.clone()]; + let c: Vec = b.par_iter().flatten().cloned().collect(); + let mut d = a.clone(); + d.extend(&a); + d.extend(&a); + d.extend(&a); + + assert_eq!(d, c); +} + +#[test] +fn check_flatten_vec_empty() { + let a: Vec> = vec![vec![]]; + let b: Vec = a.par_iter().flatten().cloned().collect(); + + assert_eq!(vec![] as Vec, b); +} + +#[test] +fn check_slice_split() { + let v: Vec<_> = (0..1000).collect(); + for m in 1..100 { + let a: Vec<_> = v.split(|x| x % m == 0).collect(); + let b: Vec<_> = v.par_split(|x| x % m == 0).collect(); + assert_eq!(a, b); + } + + // same as std::slice::split() examples + let slice = [10, 40, 33, 20]; + let v: Vec<_> = slice.par_split(|num| num % 3 == 0).collect(); + assert_eq!(v, &[&slice[..2], &slice[3..]]); + + let slice = [10, 40, 33]; + let v: Vec<_> = slice.par_split(|num| num % 3 == 0).collect(); + assert_eq!(v, &[&slice[..2], &slice[..0]]); + + let slice = [10, 6, 33, 20]; + let v: Vec<_> = slice.par_split(|num| num % 3 == 0).collect(); + assert_eq!(v, &[&slice[..1], &slice[..0], &slice[3..]]); +} + +#[test] +fn check_slice_split_inclusive() { + let v: Vec<_> = (0..1000).collect(); + for m in 1..100 { + let a: Vec<_> = v.split_inclusive(|x| x % m == 0).collect(); + let b: Vec<_> = v.par_split_inclusive(|x| x % m == 0).collect(); + assert_eq!(a, b); + } + + // same as std::slice::split_inclusive() examples + let slice = [10, 40, 33, 20]; + let v: Vec<_> = slice.par_split_inclusive(|num| num % 3 == 0).collect(); + assert_eq!(v, &[&slice[..3], &slice[3..]]); + + let slice = [3, 10, 40, 33]; + let v: Vec<_> = slice.par_split_inclusive(|num| num % 3 == 0).collect(); + assert_eq!(v, &[&slice[..1], &slice[1..]]); +} + +#[test] +fn check_slice_split_mut() { + let mut v1: Vec<_> = (0..1000).collect(); + let mut v2 = v1.clone(); + for m in 1..100 { + let a: Vec<_> = v1.split_mut(|x| x % m == 0).collect(); + let b: Vec<_> = v2.par_split_mut(|x| x % m == 0).collect(); + assert_eq!(a, b); + } + + // same as std::slice::split_mut() example + let mut v = [10, 40, 30, 20, 60, 50]; + v.par_split_mut(|num| num % 3 == 0).for_each(|group| { + group[0] = 1; + }); + assert_eq!(v, [1, 40, 30, 1, 60, 1]); +} + +#[test] +fn check_slice_split_inclusive_mut() { + let mut v1: Vec<_> = (0..1000).collect(); + let mut v2 = v1.clone(); + for m in 1..100 { + let a: Vec<_> = v1.split_inclusive_mut(|x| x % m == 0).collect(); + let b: Vec<_> = v2.par_split_inclusive_mut(|x| x % m == 0).collect(); + assert_eq!(a, b); + } + + // same as std::slice::split_inclusive_mut() example + let mut v = [10, 40, 30, 20, 60, 50]; + v.par_split_inclusive_mut(|num| num % 3 == 0) + .for_each(|group| { + let terminator_idx = group.len() - 1; + group[terminator_idx] = 1; + }); + assert_eq!(v, [10, 40, 1, 20, 1, 1]); +} + +#[test] +fn check_chunks() { + let a: Vec = vec![1, 5, 10, 4, 100, 3, 1000, 2, 10000, 1]; + let par_sum_product_pairs: i32 = a.par_chunks(2).map(|c| c.iter().product::()).sum(); + let seq_sum_product_pairs = a.chunks(2).map(|c| c.iter().product::()).sum(); + assert_eq!(par_sum_product_pairs, 12345); + assert_eq!(par_sum_product_pairs, seq_sum_product_pairs); + + let par_sum_product_triples: i32 = a.par_chunks(3).map(|c| c.iter().product::()).sum(); + let seq_sum_product_triples = a.chunks(3).map(|c| c.iter().product::()).sum(); + assert_eq!(par_sum_product_triples, 5_0 + 12_00 + 20_000_000 + 1); + assert_eq!(par_sum_product_triples, seq_sum_product_triples); +} + +#[test] +fn check_chunks_mut() { + let mut a: Vec = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + let mut b: Vec = a.clone(); + a.par_chunks_mut(2).for_each(|c| c[0] = c.iter().sum()); + b.chunks_mut(2).for_each(|c| c[0] = c.iter().sum()); + assert_eq!(a, &[3, 2, 7, 4, 11, 6, 15, 8, 19, 10]); + assert_eq!(a, b); + + let mut a: Vec = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + let mut b: Vec = a.clone(); + a.par_chunks_mut(3).for_each(|c| c[0] = c.iter().sum()); + b.chunks_mut(3).for_each(|c| c[0] = c.iter().sum()); + assert_eq!(a, &[6, 2, 3, 15, 5, 6, 24, 8, 9, 10]); + assert_eq!(a, b); +} + +#[test] +fn check_windows() { + let a: Vec = (0..1024).collect(); + let par: Vec<_> = a.par_windows(2).collect(); + let seq: Vec<_> = a.windows(2).collect(); + assert_eq!(par, seq); + + let par: Vec<_> = a.par_windows(100).collect(); + let seq: Vec<_> = a.windows(100).collect(); + assert_eq!(par, seq); + + let par: Vec<_> = a.par_windows(1_000_000).collect(); + let seq: Vec<_> = a.windows(1_000_000).collect(); + assert_eq!(par, seq); + + let par: Vec<_> = a + .par_windows(2) + .chain(a.par_windows(1_000_000)) + .zip(a.par_windows(2)) + .collect(); + let seq: Vec<_> = a + .windows(2) + .chain(a.windows(1_000_000)) + .zip(a.windows(2)) + .collect(); + assert_eq!(par, seq); +} + +#[test] +fn check_options() { + let mut a = vec![None, Some(1), None, None, Some(2), Some(4)]; + + assert_eq!(7, a.par_iter().flat_map(|opt| opt).sum::()); + assert_eq!(7, a.par_iter().flat_map(|opt| opt).sum::()); + + a.par_iter_mut() + .flat_map(|opt| opt) + .for_each(|x| *x = *x * *x); + + assert_eq!(21, a.into_par_iter().flat_map(|opt| opt).sum::()); +} + +#[test] +fn check_results() { + let mut a = vec![Err(()), Ok(1i32), Err(()), Err(()), Ok(2), Ok(4)]; + + assert_eq!(7, a.par_iter().flat_map(|res| res).sum::()); + + assert_eq!(Err::(()), a.par_iter().cloned().sum()); + assert_eq!(Ok(7), a.par_iter().cloned().filter(Result::is_ok).sum()); + + assert_eq!(Err::(()), a.par_iter().cloned().product()); + assert_eq!(Ok(8), a.par_iter().cloned().filter(Result::is_ok).product()); + + a.par_iter_mut() + .flat_map(|res| res) + .for_each(|x| *x = *x * *x); + + assert_eq!(21, a.into_par_iter().flat_map(|res| res).sum::()); +} + +#[test] +fn check_binary_heap() { + use std::collections::BinaryHeap; + + let a: BinaryHeap = (0..10).collect(); + + assert_eq!(45, a.par_iter().sum::()); + assert_eq!(45, a.into_par_iter().sum::()); +} + +#[test] +fn check_btree_map() { + use std::collections::BTreeMap; + + let mut a: BTreeMap = (0..10).map(|i| (i, -i)).collect(); + + assert_eq!(45, a.par_iter().map(|(&k, _)| k).sum::()); + assert_eq!(-45, a.par_iter().map(|(_, &v)| v).sum::()); + + a.par_iter_mut().for_each(|(k, v)| *v += *k); + + assert_eq!(0, a.into_par_iter().map(|(_, v)| v).sum::()); +} + +#[test] +fn check_btree_set() { + use std::collections::BTreeSet; + + let a: BTreeSet = (0..10).collect(); + + assert_eq!(45, a.par_iter().sum::()); + assert_eq!(45, a.into_par_iter().sum::()); +} + +#[test] +fn check_hash_map() { + use std::collections::HashMap; + + let mut a: HashMap = (0..10).map(|i| (i, -i)).collect(); + + assert_eq!(45, a.par_iter().map(|(&k, _)| k).sum::()); + assert_eq!(-45, a.par_iter().map(|(_, &v)| v).sum::()); + + a.par_iter_mut().for_each(|(k, v)| *v += *k); + + assert_eq!(0, a.into_par_iter().map(|(_, v)| v).sum::()); +} + +#[test] +fn check_hash_set() { + use std::collections::HashSet; + + let a: HashSet = (0..10).collect(); + + assert_eq!(45, a.par_iter().sum::()); + assert_eq!(45, a.into_par_iter().sum::()); +} + +#[test] +fn check_linked_list() { + use std::collections::LinkedList; + + let mut a: LinkedList = (0..10).collect(); + + assert_eq!(45, a.par_iter().sum::()); + + a.par_iter_mut().for_each(|x| *x = -*x); + + assert_eq!(-45, a.into_par_iter().sum::()); +} + +#[test] +fn check_vec_deque() { + use std::collections::VecDeque; + + let mut a: VecDeque = (0..10).collect(); + + // try to get it to wrap around + a.drain(..5); + a.extend(0..5); + + assert_eq!(45, a.par_iter().sum::()); + + a.par_iter_mut().for_each(|x| *x = -*x); + + assert_eq!(-45, a.into_par_iter().sum::()); +} + +#[test] +fn check_chain() { + let mut res = vec![]; + + // stays indexed in the face of madness + Some(0) + .into_par_iter() + .chain(Ok::<_, ()>(1)) + .chain(1..4) + .chain(Err("huh?")) + .chain(None) + .chain(vec![5, 8, 13]) + .map(|x| (x as u8 + b'a') as char) + .chain(vec!['x', 'y', 'z']) + .zip((0i32..1000).into_par_iter().map(|x| -x)) + .enumerate() + .map(|(a, (b, c))| (a, b, c)) + .chain(None) + .collect_into_vec(&mut res); + + assert_eq!( + res, + vec![ + (0, 'a', 0), + (1, 'b', -1), + (2, 'b', -2), + (3, 'c', -3), + (4, 'd', -4), + (5, 'f', -5), + (6, 'i', -6), + (7, 'n', -7), + (8, 'x', -8), + (9, 'y', -9), + (10, 'z', -10) + ] + ); + + // unindexed is ok too + let res: Vec = Some(1i32) + .into_par_iter() + .chain( + (2i32..4) + .into_par_iter() + .chain(vec![5, 6, 7, 8, 9]) + .chain(Some((10, 100)).into_par_iter().flat_map(|(a, b)| a..b)) + .filter(|x| x & 1 == 1), + ) + .collect(); + let other: Vec = (0..100).filter(|x| x & 1 == 1).collect(); + assert_eq!(res, other); + + // chain collect is ok with the "fake" specialization + let res: Vec = Some(1i32).into_par_iter().chain(None).collect(); + assert_eq!(res, &[1]); +} + +#[test] +fn check_count() { + let c0 = (0_u32..24 * 1024).filter(|i| i % 2 == 0).count(); + let c1 = (0_u32..24 * 1024) + .into_par_iter() + .filter(|i| i % 2 == 0) + .count(); + assert_eq!(c0, c1); +} + +#[test] +fn find_any() { + let a: Vec = (0..1024).collect(); + + assert!(a.par_iter().find_any(|&&x| x % 42 == 41).is_some()); + assert_eq!( + a.par_iter().find_any(|&&x| x % 19 == 1 && x % 53 == 0), + Some(&742_i32) + ); + assert_eq!(a.par_iter().find_any(|&&x| x < 0), None); + + assert!(a.par_iter().position_any(|&x| x % 42 == 41).is_some()); + assert_eq!( + a.par_iter().position_any(|&x| x % 19 == 1 && x % 53 == 0), + Some(742_usize) + ); + assert_eq!(a.par_iter().position_any(|&x| x < 0), None); + + assert!(a.par_iter().any(|&x| x > 1000)); + assert!(!a.par_iter().any(|&x| x < 0)); + + assert!(!a.par_iter().all(|&x| x > 1000)); + assert!(a.par_iter().all(|&x| x >= 0)); +} + +#[test] +fn find_first_or_last() { + let a: Vec = (0..1024).collect(); + + assert_eq!(a.par_iter().find_first(|&&x| x % 42 == 41), Some(&41_i32)); + assert_eq!( + a.par_iter().find_first(|&&x| x % 19 == 1 && x % 53 == 0), + Some(&742_i32) + ); + assert_eq!(a.par_iter().find_first(|&&x| x < 0), None); + + assert_eq!( + a.par_iter().position_first(|&x| x % 42 == 41), + Some(41_usize) + ); + assert_eq!( + a.par_iter().position_first(|&x| x % 19 == 1 && x % 53 == 0), + Some(742_usize) + ); + assert_eq!(a.par_iter().position_first(|&x| x < 0), None); + + assert_eq!(a.par_iter().find_last(|&&x| x % 42 == 41), Some(&1007_i32)); + assert_eq!( + a.par_iter().find_last(|&&x| x % 19 == 1 && x % 53 == 0), + Some(&742_i32) + ); + assert_eq!(a.par_iter().find_last(|&&x| x < 0), None); + + assert_eq!( + a.par_iter().position_last(|&x| x % 42 == 41), + Some(1007_usize) + ); + assert_eq!( + a.par_iter().position_last(|&x| x % 19 == 1 && x % 53 == 0), + Some(742_usize) + ); + assert_eq!(a.par_iter().position_last(|&x| x < 0), None); +} + +#[test] +fn find_map_first_or_last_or_any() { + let mut a: Vec = vec![]; + + assert!(a.par_iter().find_map_any(half_if_positive).is_none()); + assert!(a.par_iter().find_map_first(half_if_positive).is_none()); + assert!(a.par_iter().find_map_last(half_if_positive).is_none()); + + a = (-1024..-3).collect(); + + assert!(a.par_iter().find_map_any(half_if_positive).is_none()); + assert!(a.par_iter().find_map_first(half_if_positive).is_none()); + assert!(a.par_iter().find_map_last(half_if_positive).is_none()); + + assert!(a.par_iter().find_map_any(half_if_negative).is_some()); + assert_eq!( + a.par_iter().find_map_first(half_if_negative), + Some(-512_i32) + ); + assert_eq!(a.par_iter().find_map_last(half_if_negative), Some(-2_i32)); + + a.append(&mut (2..1025).collect()); + + assert!(a.par_iter().find_map_any(half_if_positive).is_some()); + assert_eq!(a.par_iter().find_map_first(half_if_positive), Some(1_i32)); + assert_eq!(a.par_iter().find_map_last(half_if_positive), Some(512_i32)); + + fn half_if_positive(x: &i32) -> Option { + if *x > 0 { + Some(x / 2) + } else { + None + } + } + + fn half_if_negative(x: &i32) -> Option { + if *x < 0 { + Some(x / 2) + } else { + None + } + } +} + +#[test] +fn check_find_not_present() { + let counter = AtomicUsize::new(0); + let value: Option = (0_i32..2048).into_par_iter().find_any(|&p| { + counter.fetch_add(1, Ordering::SeqCst); + p >= 2048 + }); + assert!(value.is_none()); + assert!(counter.load(Ordering::SeqCst) == 2048); // should have visited every single one +} + +#[test] +fn check_find_is_present() { + let counter = AtomicUsize::new(0); + let value: Option = (0_i32..2048).into_par_iter().find_any(|&p| { + counter.fetch_add(1, Ordering::SeqCst); + (1024..1096).contains(&p) + }); + let q = value.unwrap(); + assert!((1024..1096).contains(&q)); + assert!(counter.load(Ordering::SeqCst) < 2048); // should not have visited every single one +} + +#[test] +fn check_while_some() { + let value = (0_i32..2048).into_par_iter().map(Some).while_some().max(); + assert_eq!(value, Some(2047)); + + let counter = AtomicUsize::new(0); + let value = (0_i32..2048) + .into_par_iter() + .map(|x| { + counter.fetch_add(1, Ordering::SeqCst); + if x < 1024 { + Some(x) + } else { + None + } + }) + .while_some() + .max(); + assert!(value < Some(1024)); + assert!(counter.load(Ordering::SeqCst) < 2048); // should not have visited every single one +} + +#[test] +fn par_iter_collect_option() { + let a: Option> = (0_i32..2048).map(Some).collect(); + let b: Option> = (0_i32..2048).into_par_iter().map(Some).collect(); + assert_eq!(a, b); + + let c: Option> = (0_i32..2048) + .into_par_iter() + .map(|x| if x == 1234 { None } else { Some(x) }) + .collect(); + assert_eq!(c, None); +} + +#[test] +fn par_iter_collect_result() { + let a: Result, ()> = (0_i32..2048).map(Ok).collect(); + let b: Result, ()> = (0_i32..2048).into_par_iter().map(Ok).collect(); + assert_eq!(a, b); + + let c: Result, _> = (0_i32..2048) + .into_par_iter() + .map(|x| if x == 1234 { Err(x) } else { Ok(x) }) + .collect(); + assert_eq!(c, Err(1234)); + + let d: Result, _> = (0_i32..2048) + .into_par_iter() + .map(|x| if x % 100 == 99 { Err(x) } else { Ok(x) }) + .collect(); + assert_eq!(d.map_err(|x| x % 100), Err(99)); +} + +#[test] +fn par_iter_collect() { + let a: Vec = (0..1024).collect(); + let b: Vec = a.par_iter().map(|&i| i + 1).collect(); + let c: Vec = (0..1024).map(|i| i + 1).collect(); + assert_eq!(b, c); +} + +#[test] +fn par_iter_collect_vecdeque() { + let a: Vec = (0..1024).collect(); + let b: VecDeque = a.par_iter().cloned().collect(); + let c: VecDeque = a.iter().cloned().collect(); + assert_eq!(b, c); +} + +#[test] +fn par_iter_collect_binaryheap() { + let a: Vec = (0..1024).collect(); + let mut b: BinaryHeap = a.par_iter().cloned().collect(); + assert_eq!(b.peek(), Some(&1023)); + assert_eq!(b.len(), 1024); + for n in (0..1024).rev() { + assert_eq!(b.pop(), Some(n)); + assert_eq!(b.len() as i32, n); + } +} + +#[test] +fn par_iter_collect_hashmap() { + let a: Vec = (0..1024).collect(); + let b: HashMap = a.par_iter().map(|&i| (i, format!("{i}"))).collect(); + assert_eq!(&b[&3], "3"); + assert_eq!(b.len(), 1024); +} + +#[test] +fn par_iter_collect_hashset() { + let a: Vec = (0..1024).collect(); + let b: HashSet = a.par_iter().cloned().collect(); + assert_eq!(b.len(), 1024); +} + +#[test] +fn par_iter_collect_btreemap() { + let a: Vec = (0..1024).collect(); + let b: BTreeMap = a.par_iter().map(|&i| (i, format!("{i}"))).collect(); + assert_eq!(&b[&3], "3"); + assert_eq!(b.len(), 1024); +} + +#[test] +fn par_iter_collect_btreeset() { + let a: Vec = (0..1024).collect(); + let b: BTreeSet = a.par_iter().cloned().collect(); + assert_eq!(b.len(), 1024); +} + +#[test] +fn par_iter_collect_linked_list() { + let a: Vec = (0..1024).collect(); + let b: LinkedList<_> = a.par_iter().map(|&i| (i, format!("{i}"))).collect(); + let c: LinkedList<_> = a.iter().map(|&i| (i, format!("{i}"))).collect(); + assert_eq!(b, c); +} + +#[test] +fn par_iter_collect_linked_list_flat_map_filter() { + let b: LinkedList = (0_i32..1024) + .into_par_iter() + .flat_map(|i| 0..i) + .filter(|&i| i % 2 == 0) + .collect(); + let c: LinkedList = (0_i32..1024) + .flat_map(|i| 0..i) + .filter(|&i| i % 2 == 0) + .collect(); + assert_eq!(b, c); +} + +#[test] +fn par_iter_collect_cows() { + use std::borrow::Cow; + + let s = "Fearless Concurrency with Rust"; + + // Collects `i32` into a `Vec` + let a: Cow<'_, [i32]> = (0..1024).collect(); + let b: Cow<'_, [i32]> = a.par_iter().cloned().collect(); + assert_eq!(a, b); + + // Collects `char` into a `String` + let a: Cow<'_, str> = s.chars().collect(); + let b: Cow<'_, str> = s.par_chars().collect(); + assert_eq!(a, b); + + // Collects `str` into a `String` + let sw = s.split_whitespace(); + let psw = s.par_split_whitespace(); + let a: Cow<'_, str> = sw.clone().collect(); + let b: Cow<'_, str> = psw.clone().collect(); + assert_eq!(a, b); + + // Collects `String` into a `String` + let a: Cow<'_, str> = sw.map(str::to_owned).collect(); + let b: Cow<'_, str> = psw.map(str::to_owned).collect(); + assert_eq!(a, b); + + // Collects `OsStr` into a `OsString` + let sw = s.split_whitespace().map(OsStr::new); + let psw = s.par_split_whitespace().map(OsStr::new); + let a: Cow<'_, OsStr> = Cow::Owned(sw.clone().collect()); + let b: Cow<'_, OsStr> = psw.clone().collect(); + assert_eq!(a, b); + + // Collects `OsString` into a `OsString` + let a: Cow<'_, OsStr> = Cow::Owned(sw.map(OsStr::to_owned).collect()); + let b: Cow<'_, OsStr> = psw.map(OsStr::to_owned).collect(); + assert_eq!(a, b); +} + +#[test] +fn par_iter_unindexed_flat_map() { + let b: Vec = (0_i64..1024).into_par_iter().flat_map(Some).collect(); + let c: Vec = (0_i64..1024).flat_map(Some).collect(); + assert_eq!(b, c); +} + +#[test] +fn min_max() { + let rng = seeded_rng(); + let a: Vec = rng.sample_iter(&StandardUniform).take(1024).collect(); + for i in 0..=a.len() { + let slice = &a[..i]; + assert_eq!(slice.par_iter().min(), slice.iter().min()); + assert_eq!(slice.par_iter().max(), slice.iter().max()); + } +} + +#[test] +fn min_max_by() { + let rng = seeded_rng(); + // Make sure there are duplicate keys, for testing sort stability + let r: Vec = rng.sample_iter(&StandardUniform).take(512).collect(); + let a: Vec<(i32, u16)> = r.iter().chain(&r).cloned().zip(0..).collect(); + for i in 0..=a.len() { + let slice = &a[..i]; + assert_eq!( + slice.par_iter().min_by(|x, y| x.0.cmp(&y.0)), + slice.iter().min_by(|x, y| x.0.cmp(&y.0)) + ); + assert_eq!( + slice.par_iter().max_by(|x, y| x.0.cmp(&y.0)), + slice.iter().max_by(|x, y| x.0.cmp(&y.0)) + ); + } +} + +#[test] +fn min_max_by_key() { + let rng = seeded_rng(); + // Make sure there are duplicate keys, for testing sort stability + let r: Vec = rng.sample_iter(&StandardUniform).take(512).collect(); + let a: Vec<(i32, u16)> = r.iter().chain(&r).cloned().zip(0..).collect(); + for i in 0..=a.len() { + let slice = &a[..i]; + assert_eq!( + slice.par_iter().min_by_key(|x| x.0), + slice.iter().min_by_key(|x| x.0) + ); + assert_eq!( + slice.par_iter().max_by_key(|x| x.0), + slice.iter().max_by_key(|x| x.0) + ); + } +} + +#[test] +fn check_rev() { + let a: Vec = (0..1024).rev().collect(); + let b: Vec = (0..1024).collect(); + + assert!(a.par_iter().rev().zip(b).all(|(&a, b)| a == b)); +} + +#[test] +fn scope_mix() { + let counter_p = &AtomicUsize::new(0); + scope(|s| { + s.spawn(move |s| { + divide_and_conquer(s, counter_p, 1024); + }); + s.spawn(move |_| { + let a: Vec = (0..1024).collect(); + let r1 = a.par_iter().map(|&i| i + 1).reduce_with(|i, j| i + j); + let r2 = a.iter().map(|&i| i + 1).sum(); + assert_eq!(r1.unwrap(), r2); + }); + }); +} + +fn divide_and_conquer<'scope>(scope: &Scope<'scope>, counter: &'scope AtomicUsize, size: usize) { + if size > 1 { + scope.spawn(move |scope| divide_and_conquer(scope, counter, size / 2)); + scope.spawn(move |scope| divide_and_conquer(scope, counter, size / 2)); + } else { + // count the leaves + counter.fetch_add(1, Ordering::SeqCst); + } +} + +#[test] +fn check_split() { + use std::ops::Range; + + let a = (0..1024).into_par_iter(); + + let b = split(0..1024, |Range { start, end }| { + let mid = (end - start) / 2; + if mid > start { + (start..mid, Some(mid..end)) + } else { + (start..end, None) + } + }) + .flat_map(|range| range); + + assert_eq!(a.collect::>(), b.collect::>()); +} + +#[test] +fn check_lengths() { + fn check(min: usize, max: usize) { + let range = 0..1024 * 1024; + + // Check against normalized values. + let min_check = Ord::min(Ord::max(min, 1), range.len()); + let max_check = Ord::max(max, min_check.saturating_add(min_check - 1)); + + assert!( + range + .into_par_iter() + .with_min_len(min) + .with_max_len(max) + .fold(|| 0, |count, _| count + 1) + .all(|c| c >= min_check && c <= max_check), + "check_lengths failed {:?} -> {:?} ", + (min, max), + (min_check, max_check) + ); + } + + let lengths = [0, 1, 10, 100, 1_000, 10_000, 100_000, 1_000_000, usize::MAX]; + for &min in &lengths { + for &max in &lengths { + check(min, max); + } + } +} + +#[test] +fn check_map_with() { + let (sender, receiver) = mpsc::channel(); + let a: HashSet<_> = (0..1024).collect(); + + a.par_iter() + .cloned() + .map_with(sender, |s, i| s.send(i).unwrap()) + .count(); + + let b: HashSet<_> = receiver.iter().collect(); + assert_eq!(a, b); +} + +#[test] +fn check_fold_with() { + let (sender, receiver) = mpsc::channel(); + let a: HashSet<_> = (0..1024).collect(); + + a.par_iter() + .cloned() + .fold_with(sender, |s, i| { + s.send(i).unwrap(); + s + }) + .count(); + + let b: HashSet<_> = receiver.iter().collect(); + assert_eq!(a, b); +} + +#[test] +fn check_for_each_with() { + let (sender, receiver) = mpsc::channel(); + let a: HashSet<_> = (0..1024).collect(); + + a.par_iter() + .cloned() + .for_each_with(sender, |s, i| s.send(i).unwrap()); + + let b: HashSet<_> = receiver.iter().collect(); + assert_eq!(a, b); +} + +#[test] +fn check_extend_items() { + fn check() + where + C: Default + + Eq + + Debug + + Extend + + for<'a> Extend<&'a i32> + + ParallelExtend + + for<'a> ParallelExtend<&'a i32>, + { + let mut serial = C::default(); + let mut parallel = C::default(); + + // extend with references + let v: Vec<_> = (0..128).collect(); + serial.extend(&v); + parallel.par_extend(&v); + assert_eq!(serial, parallel); + + // extend with values + serial.extend(-128..0); + parallel.par_extend(-128..0); + assert_eq!(serial, parallel); + } + + check::>(); + check::>(); + check::>(); + check::>(); + check::>(); +} + +#[test] +fn check_extend_heap() { + let mut serial: BinaryHeap<_> = Default::default(); + let mut parallel: BinaryHeap<_> = Default::default(); + + // extend with references + let v: Vec<_> = (0..128).collect(); + serial.extend(&v); + parallel.par_extend(&v); + assert_eq!( + serial.clone().into_sorted_vec(), + parallel.clone().into_sorted_vec() + ); + + // extend with values + serial.extend(-128..0); + parallel.par_extend(-128..0); + assert_eq!(serial.into_sorted_vec(), parallel.into_sorted_vec()); +} + +#[test] +fn check_extend_pairs() { + fn check() + where + C: Default + + Eq + + Debug + + Extend<(usize, i32)> + + for<'a> Extend<(&'a usize, &'a i32)> + + ParallelExtend<(usize, i32)> + + for<'a> ParallelExtend<(&'a usize, &'a i32)>, + { + let mut serial = C::default(); + let mut parallel = C::default(); + + // extend with references + let m: HashMap<_, _> = (0..128).enumerate().collect(); + serial.extend(&m); + parallel.par_extend(&m); + assert_eq!(serial, parallel); + + // extend with values + let v: Vec<(_, _)> = (-128..0).enumerate().collect(); + serial.extend(v.clone()); + parallel.par_extend(v); + assert_eq!(serial, parallel); + } + + check::>(); + check::>(); +} + +#[test] +fn check_unzip_into_vecs() { + let mut a = vec![]; + let mut b = vec![]; + (0..1024) + .into_par_iter() + .map(|i| i * i) + .enumerate() + .unzip_into_vecs(&mut a, &mut b); + + let (c, d): (Vec<_>, Vec<_>) = (0..1024).map(|i| i * i).enumerate().unzip(); + assert_eq!(a, c); + assert_eq!(b, d); +} + +#[test] +fn check_unzip() { + // indexed, unindexed + let (a, b): (Vec<_>, HashSet<_>) = (0..1024).into_par_iter().map(|i| i * i).enumerate().unzip(); + let (c, d): (Vec<_>, HashSet<_>) = (0..1024).map(|i| i * i).enumerate().unzip(); + assert_eq!(a, c); + assert_eq!(b, d); + + // unindexed, indexed + let (a, b): (HashSet<_>, Vec<_>) = (0..1024).into_par_iter().map(|i| i * i).enumerate().unzip(); + let (c, d): (HashSet<_>, Vec<_>) = (0..1024).map(|i| i * i).enumerate().unzip(); + assert_eq!(a, c); + assert_eq!(b, d); + + // indexed, indexed + let (a, b): (Vec<_>, Vec<_>) = (0..1024).into_par_iter().map(|i| i * i).enumerate().unzip(); + let (c, d): (Vec<_>, Vec<_>) = (0..1024).map(|i| i * i).enumerate().unzip(); + assert_eq!(a, c); + assert_eq!(b, d); + + // unindexed producer + let (a, b): (Vec<_>, Vec<_>) = (0..1024) + .into_par_iter() + .filter_map(|i| Some((i, i * i))) + .unzip(); + let (c, d): (Vec<_>, Vec<_>) = (0..1024).map(|i| (i, i * i)).unzip(); + assert_eq!(a, c); + assert_eq!(b, d); +} + +#[test] +fn check_partition() { + let (a, b): (Vec<_>, Vec<_>) = (0..1024).into_par_iter().partition(|&i| i % 3 == 0); + let (c, d): (Vec<_>, Vec<_>) = (0..1024).partition(|&i| i % 3 == 0); + assert_eq!(a, c); + assert_eq!(b, d); +} + +#[test] +fn check_partition_map() { + let input = "a b c 1 2 3 x y z"; + let (a, b): (Vec<_>, String) = + input + .par_split_whitespace() + .partition_map(|s| match s.parse::() { + Ok(n) => Either::Left(n), + Err(_) => Either::Right(s), + }); + assert_eq!(a, vec![1, 2, 3]); + assert_eq!(b, "abcxyz"); +} + +#[test] +fn check_either() { + type I = crate::vec::IntoIter; + type E = Either; + + let v: Vec = (0..1024).collect(); + + // try iterating the left side + let left: E = Either::Left(v.clone().into_par_iter()); + assert!(left.eq(v.clone())); + + // try iterating the right side + let right: E = Either::Right(v.clone().into_par_iter()); + assert!(right.eq(v.clone())); + + // try an indexed iterator + let left: E = Either::Left(v.clone().into_par_iter()); + assert!(left.enumerate().eq(v.into_par_iter().enumerate())); +} + +#[test] +fn check_either_extend() { + type E = Either, HashSet>; + + let v: Vec = (0..1024).collect(); + + // try extending the left side + let mut left: E = Either::Left(vec![]); + left.par_extend(v.clone()); + assert_eq!(left.as_ref(), Either::Left(&v)); + + // try extending the right side + let mut right: E = Either::Right(HashSet::default()); + right.par_extend(v.clone()); + assert_eq!(right, Either::Right(v.iter().cloned().collect())); +} + +#[test] +fn check_interleave_eq() { + let xs: Vec = (0..10).collect(); + let ys: Vec = (10..20).collect(); + + let mut actual = vec![]; + xs.par_iter() + .interleave(&ys) + .map(|&i| i) + .collect_into_vec(&mut actual); + + let expected: Vec = (0..10) + .zip(10..20) + .flat_map(|(i, j)| vec![i, j].into_iter()) + .collect(); + assert_eq!(expected, actual); +} + +#[test] +fn check_interleave_uneven() { + let cases: Vec<(Vec, Vec, Vec)> = vec![ + ( + (0..9).collect(), + vec![10], + vec![0, 10, 1, 2, 3, 4, 5, 6, 7, 8], + ), + ( + vec![10], + (0..9).collect(), + vec![10, 0, 1, 2, 3, 4, 5, 6, 7, 8], + ), + ( + (0..5).collect(), + (5..10).collect(), + (0..5) + .zip(5..10) + .flat_map(|(i, j)| vec![i, j].into_iter()) + .collect(), + ), + (vec![], (0..9).collect(), (0..9).collect()), + ((0..9).collect(), vec![], (0..9).collect()), + ( + (0..50).collect(), + (50..100).collect(), + (0..50) + .zip(50..100) + .flat_map(|(i, j)| vec![i, j].into_iter()) + .collect(), + ), + ]; + + for (i, (xs, ys, expected)) in cases.into_iter().enumerate() { + let mut res = vec![]; + xs.par_iter() + .interleave(&ys) + .map(|&i| i) + .collect_into_vec(&mut res); + assert_eq!(expected, res, "Case {i} failed"); + + res.truncate(0); + xs.par_iter() + .interleave(&ys) + .rev() + .map(|&i| i) + .collect_into_vec(&mut res); + assert_eq!( + expected.into_iter().rev().collect::>(), + res, + "Case {i} reversed failed" + ); + } +} + +#[test] +fn check_interleave_shortest() { + let cases: Vec<(Vec, Vec, Vec)> = vec![ + ((0..9).collect(), vec![10], vec![0, 10, 1]), + (vec![10], (0..9).collect(), vec![10, 0]), + ( + (0..5).collect(), + (5..10).collect(), + (0..5) + .zip(5..10) + .flat_map(|(i, j)| vec![i, j].into_iter()) + .collect(), + ), + (vec![], (0..9).collect(), vec![]), + ((0..9).collect(), vec![], vec![0]), + ( + (0..50).collect(), + (50..100).collect(), + (0..50) + .zip(50..100) + .flat_map(|(i, j)| vec![i, j].into_iter()) + .collect(), + ), + ]; + + for (i, (xs, ys, expected)) in cases.into_iter().enumerate() { + let mut res = vec![]; + xs.par_iter() + .interleave_shortest(&ys) + .map(|&i| i) + .collect_into_vec(&mut res); + assert_eq!(expected, res, "Case {i} failed"); + + res.truncate(0); + xs.par_iter() + .interleave_shortest(&ys) + .rev() + .map(|&i| i) + .collect_into_vec(&mut res); + assert_eq!( + expected.into_iter().rev().collect::>(), + res, + "Case {i} reversed failed" + ); + } +} + +#[test] +#[should_panic(expected = "chunk_size must not be zero")] +fn check_chunks_zero_size() { + let _: Vec> = vec![1, 2, 3].into_par_iter().chunks(0).collect(); +} + +#[test] +fn check_chunks_even_size() { + assert_eq!( + vec![vec![1, 2, 3], vec![4, 5, 6], vec![7, 8, 9]], + (1..10).into_par_iter().chunks(3).collect::>>() + ); +} + +#[test] +fn check_chunks_empty() { + let v: Vec = vec![]; + let expected: Vec> = vec![]; + assert_eq!( + expected, + v.into_par_iter().chunks(2).collect::>>() + ); +} + +#[test] +fn check_chunks_len() { + assert_eq!(4, (0..8).into_par_iter().chunks(2).len()); + assert_eq!(3, (0..9).into_par_iter().chunks(3).len()); + assert_eq!(3, (0..8).into_par_iter().chunks(3).len()); + assert_eq!(1, [1].par_iter().chunks(3).len()); + assert_eq!(0, (0..0).into_par_iter().chunks(3).len()); +} + +#[test] +fn check_chunks_uneven() { + let cases: Vec<(Vec, usize, Vec>)> = vec![ + ((0..5).collect(), 3, vec![vec![0, 1, 2], vec![3, 4]]), + (vec![1], 5, vec![vec![1]]), + ((0..4).collect(), 3, vec![vec![0, 1, 2], vec![3]]), + ]; + + for (i, (v, n, expected)) in cases.into_iter().enumerate() { + let mut res: Vec> = vec![]; + v.par_iter() + .chunks(n) + .map(|v| v.into_iter().cloned().collect()) + .collect_into_vec(&mut res); + assert_eq!(expected, res, "Case {i} failed"); + + res.truncate(0); + v.into_par_iter().chunks(n).rev().collect_into_vec(&mut res); + assert_eq!( + expected.into_iter().rev().collect::>>(), + res, + "Case {i} reversed failed" + ); + } +} + +#[test] +#[ignore] // it's quick enough on optimized 32-bit platforms, but otherwise... ... ... +#[should_panic(expected = "overflow")] +#[cfg(debug_assertions)] +fn check_repeat_unbounded() { + // use just one thread, so we don't get infinite adaptive splitting + // (forever stealing and re-splitting jobs that will panic on overflow) + let pool = ThreadPoolBuilder::new().num_threads(1).build().unwrap(); + pool.install(|| { + println!("counted {} repeats", repeat(()).count()); + }); +} + +#[test] +fn check_repeat_find_any() { + let even = repeat(4).find_any(|&x| x % 2 == 0); + assert_eq!(even, Some(4)); +} + +#[test] +fn check_repeat_take() { + let v: Vec<_> = repeat(4).take(4).collect(); + assert_eq!(v, [4, 4, 4, 4]); +} + +#[test] +fn check_repeat_zip() { + let v = vec![4, 4, 4, 4]; + let mut fours: Vec<_> = repeat(4).zip(v).collect(); + assert_eq!(fours.len(), 4); + while let Some(item) = fours.pop() { + assert_eq!(item, (4, 4)); + } +} + +#[test] +fn check_repeat_n_zip_left() { + let v = vec![4, 4, 4, 4]; + let mut fours: Vec<_> = repeat_n(4, usize::MAX).zip(v).collect(); + assert_eq!(fours.len(), 4); + while let Some(item) = fours.pop() { + assert_eq!(item, (4, 4)); + } +} + +#[test] +fn check_repeat_n_zip_right() { + let v = vec![4, 4, 4, 4]; + let mut fours: Vec<_> = v.into_par_iter().zip(repeat_n(4, usize::MAX)).collect(); + assert_eq!(fours.len(), 4); + while let Some(item) = fours.pop() { + assert_eq!(item, (4, 4)); + } +} + +#[test] +fn count_repeat_n_clones() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + static CLONES: AtomicUsize = AtomicUsize::new(0); + static DROPS: AtomicUsize = AtomicUsize::new(0); + + struct Counter; + + impl Clone for Counter { + fn clone(&self) -> Self { + CLONES.fetch_add(1, Ordering::Relaxed); + Counter + } + } + + impl Drop for Counter { + fn drop(&mut self) { + DROPS.fetch_add(1, Ordering::Relaxed); + } + } + + #[track_caller] + fn check(clones: usize, drops: usize) { + assert_eq!(CLONES.swap(0, Ordering::Relaxed), clones, "clones"); + assert_eq!(DROPS.swap(0, Ordering::Relaxed), drops, "drops"); + } + + drop(repeat_n(Counter, 100)); + check(0, 1); + + let empty = repeat_n(Counter, 0); + check(0, 1); + let empty2 = empty.clone(); + check(0, 0); + assert_eq!(empty.count(), 0); + assert_eq!(empty2.count(), 0); + check(0, 0); + + let par_iter = repeat_n(Counter, 100); + let par_iter2 = par_iter.clone(); + check(1, 0); + assert_eq!(par_iter.count(), 100); + check(99, 100); + assert_eq!(par_iter2.map(std::mem::forget).count(), 100); + check(99, 0); + + // Clone once in `split_at` and again for the first item, leaving its unused tail. + // The other split doesn't have a tail, so it can avoid a clone. + let step99 = repeat_n(Counter, 100).step_by(99); + assert_eq!(step99.count(), 2); + check(2, 3); + + // Same without any parallel splitting + let step99 = repeat_n(Counter, 100).step_by(99).with_min_len(2); + assert_eq!(step99.count(), 2); + check(1, 2); + + // Clone once in `split_at` and again for both items, leaving both unused tails. + let step50 = repeat_n(Counter, 100).step_by(50); + assert_eq!(step50.count(), 2); + check(3, 4); +} + +#[test] +fn check_empty() { + // drive_unindexed + let mut v: Vec = empty().filter(|_| unreachable!()).collect(); + assert!(v.is_empty()); + + // drive (indexed) + empty().collect_into_vec(&mut v); + assert!(v.is_empty()); + + // with_producer + let v: Vec<(i32, i32)> = empty().zip(1..10).collect(); + assert!(v.is_empty()); +} + +#[test] +fn check_once() { + // drive_unindexed + let mut v: Vec = once(42).filter(|_| true).collect(); + assert_eq!(v, &[42]); + + // drive (indexed) + once(42).collect_into_vec(&mut v); + assert_eq!(v, &[42]); + + // with_producer + let v: Vec<(i32, i32)> = once(42).zip(1..10).collect(); + assert_eq!(v, &[(42, 1)]); +} + +#[test] +fn check_update() { + let mut v: Vec> = vec![vec![1], vec![3, 2, 1]]; + v.par_iter_mut().update(|v| v.push(0)).for_each(|_| ()); + + assert_eq!(v, vec![vec![1, 0], vec![3, 2, 1, 0]]); +} + +#[test] +fn walk_tree_prefix() { + let v: Vec = crate::iter::walk_tree_prefix(0u32..100, |r| { + // root is smallest + let mid = (r.start + 1 + r.end) / 2; + // small indices to the left, large to the right + std::iter::once((r.start + 1)..mid) + .chain(std::iter::once(mid..r.end)) + .filter(|r| !r.is_empty()) + }) + .map(|r| r.start) + .collect(); + assert!(v.into_iter().eq(0..100)); +} + +#[test] +fn walk_tree_postfix() { + let v: Vec<_> = crate::iter::walk_tree_postfix(0u64..100, |r| { + // root is largest + let mid = (r.start + r.end - 1) / 2; + // small indices to the left, large to the right + std::iter::once(r.start..mid) + .chain(std::iter::once(mid..(r.end - 1))) + .filter(|r| !r.is_empty()) + }) + .map(|r| r.end - 1) + .collect(); + assert!(v.into_iter().eq(0..100)); +} + +#[test] +fn walk_flat_tree_prefix() { + let v: Vec<_> = + crate::iter::walk_tree_prefix(0, |&e| if e < 99 { Some(e + 1) } else { None }).collect(); + assert!(v.into_iter().eq(0..100)); +} + +#[test] +fn walk_flat_tree_postfix() { + let v: Vec<_> = + crate::iter::walk_tree_postfix(99, |&e| if e > 0 { Some(e - 1) } else { None }).collect(); + assert!(v.into_iter().eq(0..100)); +} + +#[test] +fn walk_tree_prefix_degree5() { + let depth = 5; + let nodes_number = (1 - 5i32.pow(depth)) / (1 - 5); + let nodes = (0..nodes_number).collect::>(); + let v: Vec = crate::iter::walk_tree_prefix(nodes.as_slice(), |&r| { + r.split_first() + .into_iter() + .filter_map(|(_, r)| if r.is_empty() { None } else { Some(r) }) + .flat_map(|r| r.chunks(r.len() / 5)) + }) + .filter_map(|r| r.first().copied()) + .collect(); + assert_eq!(v, nodes); +} + +#[test] +fn walk_tree_postfix_degree5() { + let depth = 5; + let nodes_number = (1 - 5i32.pow(depth)) / (1 - 5); + let nodes = (0..nodes_number).collect::>(); + let v: Vec = crate::iter::walk_tree_postfix(nodes.as_slice(), |&r| { + r.split_last() + .into_iter() + .filter_map(|(_, r)| if r.is_empty() { None } else { Some(r) }) + .flat_map(|r| r.chunks(r.len() / 5)) + }) + .filter_map(|r| r.last().copied()) + .collect(); + assert_eq!(v, nodes) +} + +#[test] +fn blocks() { + let count = AtomicUsize::new(0); + let v: Vec = (0..1000) + .into_par_iter() + .map(|_| count.fetch_add(1, Ordering::Relaxed)) + .by_uniform_blocks(100) + .collect(); + let m = v + .chunks(100) + .map(|c| c.iter().max().copied().unwrap()) + .collect::>(); + assert!(m.windows(2).all(|w| w[0].lt(&w[1]))); + assert_eq!(v.len(), 1000); +} diff --git a/anneal/v2/vendor/rayon/src/iter/try_fold.rs b/anneal/v2/vendor/rayon/src/iter/try_fold.rs new file mode 100644 index 0000000000..0367512670 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/try_fold.rs @@ -0,0 +1,282 @@ +use super::plumbing::*; +use super::ParallelIterator; +use super::Try; + +use std::fmt::{self, Debug}; +use std::marker::PhantomData; +use std::ops::ControlFlow::{self, Break, Continue}; + +impl TryFold { + pub(super) fn new(base: I, identity: ID, fold_op: F) -> Self { + TryFold { + base, + identity, + fold_op, + marker: PhantomData, + } + } +} + +/// `TryFold` is an iterator that applies a function over an iterator producing a single value. +/// This struct is created by the [`try_fold()`] method on [`ParallelIterator`] +/// +/// [`try_fold()`]: ParallelIterator::try_fold() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Clone)] +pub struct TryFold { + base: I, + identity: ID, + fold_op: F, + marker: PhantomData, +} + +impl Debug for TryFold { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("TryFold").field("base", &self.base).finish() + } +} + +impl ParallelIterator for TryFold +where + I: ParallelIterator, + F: Fn(U::Output, I::Item) -> U + Sync + Send, + ID: Fn() -> U::Output + Sync + Send, + U: Try + Send, +{ + type Item = U; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let consumer1 = TryFoldConsumer { + base: consumer, + identity: &self.identity, + fold_op: &self.fold_op, + marker: PhantomData, + }; + self.base.drive_unindexed(consumer1) + } +} + +struct TryFoldConsumer<'c, U, C, ID, F> { + base: C, + identity: &'c ID, + fold_op: &'c F, + marker: PhantomData, +} + +impl<'r, U, T, C, ID, F> Consumer for TryFoldConsumer<'r, U, C, ID, F> +where + C: Consumer, + F: Fn(U::Output, T) -> U + Sync, + ID: Fn() -> U::Output + Sync, + U: Try + Send, +{ + type Folder = TryFoldFolder<'r, C::Folder, U, F>; + type Reducer = C::Reducer; + type Result = C::Result; + + fn split_at(self, index: usize) -> (Self, Self, Self::Reducer) { + let (left, right, reducer) = self.base.split_at(index); + ( + TryFoldConsumer { base: left, ..self }, + TryFoldConsumer { + base: right, + ..self + }, + reducer, + ) + } + + fn into_folder(self) -> Self::Folder { + TryFoldFolder { + base: self.base.into_folder(), + control: Continue((self.identity)()), + fold_op: self.fold_op, + } + } + + fn full(&self) -> bool { + self.base.full() + } +} + +impl<'r, U, T, C, ID, F> UnindexedConsumer for TryFoldConsumer<'r, U, C, ID, F> +where + C: UnindexedConsumer, + F: Fn(U::Output, T) -> U + Sync, + ID: Fn() -> U::Output + Sync, + U: Try + Send, +{ + fn split_off_left(&self) -> Self { + TryFoldConsumer { + base: self.base.split_off_left(), + ..*self + } + } + + fn to_reducer(&self) -> Self::Reducer { + self.base.to_reducer() + } +} + +struct TryFoldFolder<'r, C, U: Try, F> { + base: C, + fold_op: &'r F, + control: ControlFlow, +} + +impl<'r, C, U, F, T> Folder for TryFoldFolder<'r, C, U, F> +where + C: Folder, + F: Fn(U::Output, T) -> U + Sync, + U: Try, +{ + type Result = C::Result; + + fn consume(mut self, item: T) -> Self { + let fold_op = self.fold_op; + if let Continue(acc) = self.control { + self.control = fold_op(acc, item).branch(); + } + self + } + + fn complete(self) -> C::Result { + let item = match self.control { + Continue(c) => U::from_output(c), + Break(r) => U::from_residual(r), + }; + self.base.consume(item).complete() + } + + fn full(&self) -> bool { + match self.control { + Break(_) => true, + _ => self.base.full(), + } + } +} + +// /////////////////////////////////////////////////////////////////////////// + +impl TryFoldWith { + pub(super) fn new(base: I, item: U::Output, fold_op: F) -> Self { + TryFoldWith { + base, + item, + fold_op, + } + } +} + +/// `TryFoldWith` is an iterator that applies a function over an iterator producing a single value. +/// This struct is created by the [`try_fold_with()`] method on [`ParallelIterator`] +/// +/// [`try_fold_with()`]: ParallelIterator::try_fold_with() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Clone)] +pub struct TryFoldWith { + base: I, + item: U::Output, + fold_op: F, +} + +impl Debug for TryFoldWith +where + I: Debug, + U: Try, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("TryFoldWith") + .field("base", &self.base) + .field("item", &self.item) + .finish() + } +} + +impl ParallelIterator for TryFoldWith +where + I: ParallelIterator, + F: Fn(U::Output, I::Item) -> U + Sync + Send, + U: Try + Send, +{ + type Item = U; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let consumer1 = TryFoldWithConsumer { + base: consumer, + item: self.item, + fold_op: &self.fold_op, + }; + self.base.drive_unindexed(consumer1) + } +} + +struct TryFoldWithConsumer<'c, C, U: Try, F> { + base: C, + item: U::Output, + fold_op: &'c F, +} + +impl<'r, U, T, C, F> Consumer for TryFoldWithConsumer<'r, C, U, F> +where + C: Consumer, + F: Fn(U::Output, T) -> U + Sync, + U: Try + Send, +{ + type Folder = TryFoldFolder<'r, C::Folder, U, F>; + type Reducer = C::Reducer; + type Result = C::Result; + + fn split_at(self, index: usize) -> (Self, Self, Self::Reducer) { + let (left, right, reducer) = self.base.split_at(index); + ( + TryFoldWithConsumer { + base: left, + item: self.item.clone(), + ..self + }, + TryFoldWithConsumer { + base: right, + ..self + }, + reducer, + ) + } + + fn into_folder(self) -> Self::Folder { + TryFoldFolder { + base: self.base.into_folder(), + control: Continue(self.item), + fold_op: self.fold_op, + } + } + + fn full(&self) -> bool { + self.base.full() + } +} + +impl<'r, U, T, C, F> UnindexedConsumer for TryFoldWithConsumer<'r, C, U, F> +where + C: UnindexedConsumer, + F: Fn(U::Output, T) -> U + Sync, + U: Try + Send, +{ + fn split_off_left(&self) -> Self { + TryFoldWithConsumer { + base: self.base.split_off_left(), + item: self.item.clone(), + ..*self + } + } + + fn to_reducer(&self) -> Self::Reducer { + self.base.to_reducer() + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/try_reduce.rs b/anneal/v2/vendor/rayon/src/iter/try_reduce.rs new file mode 100644 index 0000000000..35a724c94f --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/try_reduce.rs @@ -0,0 +1,131 @@ +use super::plumbing::*; +use super::ParallelIterator; +use super::Try; + +use std::ops::ControlFlow::{self, Break, Continue}; +use std::sync::atomic::{AtomicBool, Ordering}; + +pub(super) fn try_reduce(pi: PI, identity: ID, reduce_op: R) -> T +where + PI: ParallelIterator, + R: Fn(T::Output, T::Output) -> T + Sync, + ID: Fn() -> T::Output + Sync, + T: Try + Send, +{ + let full = AtomicBool::new(false); + let consumer = TryReduceConsumer { + identity: &identity, + reduce_op: &reduce_op, + full: &full, + }; + pi.drive_unindexed(consumer) +} + +struct TryReduceConsumer<'r, R, ID> { + identity: &'r ID, + reduce_op: &'r R, + full: &'r AtomicBool, +} + +impl<'r, R, ID> Copy for TryReduceConsumer<'r, R, ID> {} + +impl<'r, R, ID> Clone for TryReduceConsumer<'r, R, ID> { + fn clone(&self) -> Self { + *self + } +} + +impl<'r, R, ID, T> Consumer for TryReduceConsumer<'r, R, ID> +where + R: Fn(T::Output, T::Output) -> T + Sync, + ID: Fn() -> T::Output + Sync, + T: Try + Send, +{ + type Folder = TryReduceFolder<'r, R, T>; + type Reducer = Self; + type Result = T; + + fn split_at(self, _index: usize) -> (Self, Self, Self) { + (self, self, self) + } + + fn into_folder(self) -> Self::Folder { + TryReduceFolder { + reduce_op: self.reduce_op, + control: Continue((self.identity)()), + full: self.full, + } + } + + fn full(&self) -> bool { + self.full.load(Ordering::Relaxed) + } +} + +impl<'r, R, ID, T> UnindexedConsumer for TryReduceConsumer<'r, R, ID> +where + R: Fn(T::Output, T::Output) -> T + Sync, + ID: Fn() -> T::Output + Sync, + T: Try + Send, +{ + fn split_off_left(&self) -> Self { + *self + } + + fn to_reducer(&self) -> Self::Reducer { + *self + } +} + +impl<'r, R, ID, T> Reducer for TryReduceConsumer<'r, R, ID> +where + R: Fn(T::Output, T::Output) -> T + Sync, + T: Try, +{ + fn reduce(self, left: T, right: T) -> T { + match (left.branch(), right.branch()) { + (Continue(left), Continue(right)) => (self.reduce_op)(left, right), + (Break(r), _) | (_, Break(r)) => T::from_residual(r), + } + } +} + +struct TryReduceFolder<'r, R, T: Try> { + reduce_op: &'r R, + control: ControlFlow, + full: &'r AtomicBool, +} + +impl<'r, R, T> Folder for TryReduceFolder<'r, R, T> +where + R: Fn(T::Output, T::Output) -> T, + T: Try, +{ + type Result = T; + + fn consume(mut self, item: T) -> Self { + let reduce_op = self.reduce_op; + self.control = match (self.control, item.branch()) { + (Continue(left), Continue(right)) => reduce_op(left, right).branch(), + (control @ Break(_), _) | (_, control @ Break(_)) => control, + }; + if let Break(_) = self.control { + self.full.store(true, Ordering::Relaxed); + } + self + } + + fn complete(self) -> T { + match self.control { + Continue(c) => T::from_output(c), + Break(r) => T::from_residual(r), + } + } + + fn full(&self) -> bool { + match self.control { + Break(_) => true, + _ => self.full.load(Ordering::Relaxed), + } + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/try_reduce_with.rs b/anneal/v2/vendor/rayon/src/iter/try_reduce_with.rs new file mode 100644 index 0000000000..cd7c83e272 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/try_reduce_with.rs @@ -0,0 +1,132 @@ +use super::plumbing::*; +use super::ParallelIterator; +use super::Try; + +use std::ops::ControlFlow::{self, Break, Continue}; +use std::sync::atomic::{AtomicBool, Ordering}; + +pub(super) fn try_reduce_with(pi: PI, reduce_op: R) -> Option +where + PI: ParallelIterator, + R: Fn(T::Output, T::Output) -> T + Sync, + T: Try + Send, +{ + let full = AtomicBool::new(false); + let consumer = TryReduceWithConsumer { + reduce_op: &reduce_op, + full: &full, + }; + pi.drive_unindexed(consumer) +} + +struct TryReduceWithConsumer<'r, R> { + reduce_op: &'r R, + full: &'r AtomicBool, +} + +impl<'r, R> Copy for TryReduceWithConsumer<'r, R> {} + +impl<'r, R> Clone for TryReduceWithConsumer<'r, R> { + fn clone(&self) -> Self { + *self + } +} + +impl<'r, R, T> Consumer for TryReduceWithConsumer<'r, R> +where + R: Fn(T::Output, T::Output) -> T + Sync, + T: Try + Send, +{ + type Folder = TryReduceWithFolder<'r, R, T>; + type Reducer = Self; + type Result = Option; + + fn split_at(self, _index: usize) -> (Self, Self, Self) { + (self, self, self) + } + + fn into_folder(self) -> Self::Folder { + TryReduceWithFolder { + reduce_op: self.reduce_op, + opt_control: None, + full: self.full, + } + } + + fn full(&self) -> bool { + self.full.load(Ordering::Relaxed) + } +} + +impl<'r, R, T> UnindexedConsumer for TryReduceWithConsumer<'r, R> +where + R: Fn(T::Output, T::Output) -> T + Sync, + T: Try + Send, +{ + fn split_off_left(&self) -> Self { + *self + } + + fn to_reducer(&self) -> Self::Reducer { + *self + } +} + +impl<'r, R, T> Reducer> for TryReduceWithConsumer<'r, R> +where + R: Fn(T::Output, T::Output) -> T + Sync, + T: Try, +{ + fn reduce(self, left: Option, right: Option) -> Option { + let reduce_op = self.reduce_op; + match (left, right) { + (Some(left), Some(right)) => match (left.branch(), right.branch()) { + (Continue(left), Continue(right)) => Some(reduce_op(left, right)), + (Break(r), _) | (_, Break(r)) => Some(T::from_residual(r)), + }, + (None, x) | (x, None) => x, + } + } +} + +struct TryReduceWithFolder<'r, R, T: Try> { + reduce_op: &'r R, + opt_control: Option>, + full: &'r AtomicBool, +} + +impl<'r, R, T> Folder for TryReduceWithFolder<'r, R, T> +where + R: Fn(T::Output, T::Output) -> T, + T: Try, +{ + type Result = Option; + + fn consume(mut self, item: T) -> Self { + let reduce_op = self.reduce_op; + let control = match (self.opt_control, item.branch()) { + (Some(Continue(left)), Continue(right)) => reduce_op(left, right).branch(), + (Some(control @ Break(_)), _) | (_, control) => control, + }; + if let Break(_) = control { + self.full.store(true, Ordering::Relaxed) + } + self.opt_control = Some(control); + self + } + + fn complete(self) -> Option { + match self.opt_control { + Some(Continue(c)) => Some(T::from_output(c)), + Some(Break(r)) => Some(T::from_residual(r)), + None => None, + } + } + + fn full(&self) -> bool { + match self.opt_control { + Some(Break(_)) => true, + _ => self.full.load(Ordering::Relaxed), + } + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/unzip.rs b/anneal/v2/vendor/rayon/src/iter/unzip.rs new file mode 100644 index 0000000000..a016f8d942 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/unzip.rs @@ -0,0 +1,524 @@ +use super::plumbing::*; +use super::*; + +/// This trait abstracts the different ways we can "unzip" one parallel +/// iterator into two distinct consumers, which we can handle almost +/// identically apart from how to process the individual items. +trait UnzipOp: Sync + Send { + /// The type of item expected by the left consumer. + type Left: Send; + + /// The type of item expected by the right consumer. + type Right: Send; + + /// Consumes one item and feeds it to one or both of the underlying folders. + fn consume(&self, item: T, left: FA, right: FB) -> (FA, FB) + where + FA: Folder, + FB: Folder; + + /// Reports whether this op may support indexed consumers. + /// - e.g. true for `unzip` where the item count passed through directly. + /// - e.g. false for `partition` where the sorting is not yet known. + fn indexable() -> bool { + false + } +} + +/// Runs an unzip-like operation into default `ParallelExtend` collections. +fn execute(pi: I, op: OP) -> (FromA, FromB) +where + I: ParallelIterator, + OP: UnzipOp, + FromA: Default + Send + ParallelExtend, + FromB: Default + Send + ParallelExtend, +{ + let mut a = FromA::default(); + let mut b = FromB::default(); + execute_into(&mut a, &mut b, pi, op); + (a, b) +} + +/// Runs an unzip-like operation into `ParallelExtend` collections. +fn execute_into(a: &mut FromA, b: &mut FromB, pi: I, op: OP) +where + I: ParallelIterator, + OP: UnzipOp, + FromA: Send + ParallelExtend, + FromB: Send + ParallelExtend, +{ + // We have no idea what the consumers will look like for these + // collections' `par_extend`, but we can intercept them in our own + // `drive_unindexed`. Start with the left side, type `A`: + let iter = UnzipA { base: pi, op, b }; + a.par_extend(iter); +} + +/// Unzips the items of a parallel iterator into a pair of arbitrary +/// `ParallelExtend` containers. +/// +/// This is called by `ParallelIterator::unzip`. +pub(super) fn unzip(pi: I) -> (FromA, FromB) +where + I: ParallelIterator, + FromA: Default + Send + ParallelExtend, + FromB: Default + Send + ParallelExtend, + A: Send, + B: Send, +{ + execute(pi, Unzip) +} + +/// Unzips an `IndexedParallelIterator` into two arbitrary `Consumer`s. +/// +/// This is called by `super::collect::unzip_into_vecs`. +pub(super) fn unzip_indexed(pi: I, left: CA, right: CB) -> (CA::Result, CB::Result) +where + I: IndexedParallelIterator, + CA: Consumer, + CB: Consumer, + A: Send, + B: Send, +{ + let consumer = UnzipConsumer { + op: &Unzip, + left, + right, + }; + pi.drive(consumer) +} + +/// An `UnzipOp` that splits a tuple directly into the two consumers. +struct Unzip; + +impl UnzipOp<(A, B)> for Unzip { + type Left = A; + type Right = B; + + fn consume(&self, item: (A, B), left: FA, right: FB) -> (FA, FB) + where + FA: Folder, + FB: Folder, + { + (left.consume(item.0), right.consume(item.1)) + } + + fn indexable() -> bool { + true + } +} + +/// Partitions the items of a parallel iterator into a pair of arbitrary +/// `ParallelExtend` containers. +/// +/// This is called by `ParallelIterator::partition`. +pub(super) fn partition(pi: I, predicate: P) -> (A, B) +where + I: ParallelIterator, + A: Default + Send + ParallelExtend, + B: Default + Send + ParallelExtend, + P: Fn(&I::Item) -> bool + Sync + Send, +{ + execute(pi, Partition { predicate }) +} + +/// An `UnzipOp` that routes items depending on a predicate function. +struct Partition

{ + predicate: P, +} + +impl UnzipOp for Partition

+where + P: Fn(&T) -> bool + Sync + Send, + T: Send, +{ + type Left = T; + type Right = T; + + fn consume(&self, item: T, left: FA, right: FB) -> (FA, FB) + where + FA: Folder, + FB: Folder, + { + if (self.predicate)(&item) { + (left.consume(item), right) + } else { + (left, right.consume(item)) + } + } +} + +/// Partitions and maps the items of a parallel iterator into a pair of +/// arbitrary `ParallelExtend` containers. +/// +/// This called by `ParallelIterator::partition_map`. +pub(super) fn partition_map(pi: I, predicate: P) -> (A, B) +where + I: ParallelIterator, + A: Default + Send + ParallelExtend, + B: Default + Send + ParallelExtend, + P: Fn(I::Item) -> Either + Sync + Send, + L: Send, + R: Send, +{ + execute(pi, PartitionMap { predicate }) +} + +/// An `UnzipOp` that routes items depending on how they are mapped `Either`. +struct PartitionMap

{ + predicate: P, +} + +impl UnzipOp for PartitionMap

+where + P: Fn(T) -> Either + Sync + Send, + L: Send, + R: Send, +{ + type Left = L; + type Right = R; + + fn consume(&self, item: T, left: FA, right: FB) -> (FA, FB) + where + FA: Folder, + FB: Folder, + { + match (self.predicate)(item) { + Either::Left(item) => (left.consume(item), right), + Either::Right(item) => (left, right.consume(item)), + } + } +} + +/// A fake iterator to intercept the `Consumer` for type `A`. +struct UnzipA<'b, I, OP, FromB> { + base: I, + op: OP, + b: &'b mut FromB, +} + +impl<'b, I, OP, FromB> ParallelIterator for UnzipA<'b, I, OP, FromB> +where + I: ParallelIterator, + OP: UnzipOp, + FromB: Send + ParallelExtend, +{ + type Item = OP::Left; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let mut result = None; + { + // Now it's time to find the consumer for type `B` + let iter = UnzipB { + base: self.base, + op: self.op, + left_consumer: consumer, + left_result: &mut result, + }; + self.b.par_extend(iter); + } + // NB: If for some reason `b.par_extend` doesn't actually drive the + // iterator, then we won't have a result for the left side to return + // at all. We can't fake an arbitrary consumer's result, so panic. + result.expect("unzip consumers didn't execute!") + } + + fn opt_len(&self) -> Option { + if OP::indexable() { + self.base.opt_len() + } else { + None + } + } +} + +/// A fake iterator to intercept the `Consumer` for type `B`. +struct UnzipB<'r, I, OP, CA> +where + I: ParallelIterator, + OP: UnzipOp, + CA: UnindexedConsumer, +{ + base: I, + op: OP, + left_consumer: CA, + left_result: &'r mut Option, +} + +impl<'r, I, OP, CA> ParallelIterator for UnzipB<'r, I, OP, CA> +where + I: ParallelIterator, + OP: UnzipOp, + CA: UnindexedConsumer, +{ + type Item = OP::Right; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + // Now that we have two consumers, we can unzip the real iterator. + let consumer = UnzipConsumer { + op: &self.op, + left: self.left_consumer, + right: consumer, + }; + + let result = self.base.drive_unindexed(consumer); + *self.left_result = Some(result.0); + result.1 + } + + fn opt_len(&self) -> Option { + if OP::indexable() { + self.base.opt_len() + } else { + None + } + } +} + +/// `Consumer` that unzips into two other `Consumer`s +struct UnzipConsumer<'a, OP, CA, CB> { + op: &'a OP, + left: CA, + right: CB, +} + +impl<'a, T, OP, CA, CB> Consumer for UnzipConsumer<'a, OP, CA, CB> +where + OP: UnzipOp, + CA: Consumer, + CB: Consumer, +{ + type Folder = UnzipFolder<'a, OP, CA::Folder, CB::Folder>; + type Reducer = UnzipReducer; + type Result = (CA::Result, CB::Result); + + fn split_at(self, index: usize) -> (Self, Self, Self::Reducer) { + let (left1, left2, left_reducer) = self.left.split_at(index); + let (right1, right2, right_reducer) = self.right.split_at(index); + + ( + UnzipConsumer { + op: self.op, + left: left1, + right: right1, + }, + UnzipConsumer { + op: self.op, + left: left2, + right: right2, + }, + UnzipReducer { + left: left_reducer, + right: right_reducer, + }, + ) + } + + fn into_folder(self) -> Self::Folder { + UnzipFolder { + op: self.op, + left: self.left.into_folder(), + right: self.right.into_folder(), + } + } + + fn full(&self) -> bool { + // don't stop until everyone is full + self.left.full() && self.right.full() + } +} + +impl<'a, T, OP, CA, CB> UnindexedConsumer for UnzipConsumer<'a, OP, CA, CB> +where + OP: UnzipOp, + CA: UnindexedConsumer, + CB: UnindexedConsumer, +{ + fn split_off_left(&self) -> Self { + UnzipConsumer { + op: self.op, + left: self.left.split_off_left(), + right: self.right.split_off_left(), + } + } + + fn to_reducer(&self) -> Self::Reducer { + UnzipReducer { + left: self.left.to_reducer(), + right: self.right.to_reducer(), + } + } +} + +/// `Folder` that unzips into two other `Folder`s +struct UnzipFolder<'a, OP, FA, FB> { + op: &'a OP, + left: FA, + right: FB, +} + +impl<'a, T, OP, FA, FB> Folder for UnzipFolder<'a, OP, FA, FB> +where + OP: UnzipOp, + FA: Folder, + FB: Folder, +{ + type Result = (FA::Result, FB::Result); + + fn consume(self, item: T) -> Self { + let (left, right) = self.op.consume(item, self.left, self.right); + UnzipFolder { + op: self.op, + left, + right, + } + } + + fn complete(self) -> Self::Result { + (self.left.complete(), self.right.complete()) + } + + fn full(&self) -> bool { + // don't stop until everyone is full + self.left.full() && self.right.full() + } +} + +/// `Reducer` that unzips into two other `Reducer`s +struct UnzipReducer { + left: RA, + right: RB, +} + +impl Reducer<(A, B)> for UnzipReducer +where + RA: Reducer, + RB: Reducer, +{ + fn reduce(self, left: (A, B), right: (A, B)) -> (A, B) { + ( + self.left.reduce(left.0, right.0), + self.right.reduce(left.1, right.1), + ) + } +} + +impl ParallelExtend<(A, B)> for (FromA, FromB) +where + A: Send, + B: Send, + FromA: Send + ParallelExtend, + FromB: Send + ParallelExtend, +{ + fn par_extend(&mut self, pi: I) + where + I: IntoParallelIterator, + { + execute_into(&mut self.0, &mut self.1, pi.into_par_iter(), Unzip); + } +} + +impl ParallelExtend> for (A, B) +where + L: Send, + R: Send, + A: Send + ParallelExtend, + B: Send + ParallelExtend, +{ + fn par_extend(&mut self, pi: I) + where + I: IntoParallelIterator>, + { + execute_into(&mut self.0, &mut self.1, pi.into_par_iter(), UnEither); + } +} + +/// An `UnzipOp` that routes items depending on their `Either` variant. +struct UnEither; + +impl UnzipOp> for UnEither +where + L: Send, + R: Send, +{ + type Left = L; + type Right = R; + + fn consume(&self, item: Either, left: FL, right: FR) -> (FL, FR) + where + FL: Folder, + FR: Folder, + { + match item { + Either::Left(item) => (left.consume(item), right), + Either::Right(item) => (left, right.consume(item)), + } + } +} + +impl FromParallelIterator<(A, B)> for (FromA, FromB) +where + A: Send, + B: Send, + FromA: Send + FromParallelIterator, + FromB: Send + FromParallelIterator, +{ + fn from_par_iter(pi: I) -> Self + where + I: IntoParallelIterator, + { + let (a, b): (Collector, Collector) = pi.into_par_iter().unzip(); + (a.result.unwrap(), b.result.unwrap()) + } +} + +impl FromParallelIterator> for (A, B) +where + L: Send, + R: Send, + A: Send + FromParallelIterator, + B: Send + FromParallelIterator, +{ + fn from_par_iter(pi: I) -> Self + where + I: IntoParallelIterator>, + { + fn identity(x: T) -> T { + x + } + + let (a, b): (Collector, Collector) = pi.into_par_iter().partition_map(identity); + (a.result.unwrap(), b.result.unwrap()) + } +} + +/// Shim to implement a one-time `ParallelExtend` using `FromParallelIterator`. +struct Collector { + result: Option, +} + +impl Default for Collector { + fn default() -> Self { + Collector { result: None } + } +} + +impl ParallelExtend for Collector +where + T: Send, + FromT: Send + FromParallelIterator, +{ + fn par_extend(&mut self, pi: I) + where + I: IntoParallelIterator, + { + debug_assert!(self.result.is_none()); + self.result = Some(pi.into_par_iter().collect()); + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/update.rs b/anneal/v2/vendor/rayon/src/iter/update.rs new file mode 100644 index 0000000000..b382783068 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/update.rs @@ -0,0 +1,323 @@ +use super::plumbing::*; +use super::*; + +use std::fmt::{self, Debug}; + +/// `Update` is an iterator that mutates the elements of an +/// underlying iterator before they are yielded. +/// +/// This struct is created by the [`update()`] method on [`ParallelIterator`] +/// +/// [`update()`]: ParallelIterator::update() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Clone)] +pub struct Update { + base: I, + update_op: F, +} + +impl Debug for Update { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Update").field("base", &self.base).finish() + } +} + +impl Update { + /// Creates a new `Update` iterator. + pub(super) fn new(base: I, update_op: F) -> Self { + Update { base, update_op } + } +} + +impl ParallelIterator for Update +where + I: ParallelIterator, + F: Fn(&mut I::Item) + Send + Sync, +{ + type Item = I::Item; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let consumer1 = UpdateConsumer::new(consumer, &self.update_op); + self.base.drive_unindexed(consumer1) + } + + fn opt_len(&self) -> Option { + self.base.opt_len() + } +} + +impl IndexedParallelIterator for Update +where + I: IndexedParallelIterator, + F: Fn(&mut I::Item) + Send + Sync, +{ + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + let consumer1 = UpdateConsumer::new(consumer, &self.update_op); + self.base.drive(consumer1) + } + + fn len(&self) -> usize { + self.base.len() + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + return self.base.with_producer(Callback { + callback, + update_op: self.update_op, + }); + + struct Callback { + callback: CB, + update_op: F, + } + + impl ProducerCallback for Callback + where + CB: ProducerCallback, + F: Fn(&mut T) + Send + Sync, + { + type Output = CB::Output; + + fn callback

(self, base: P) -> CB::Output + where + P: Producer, + { + let producer = UpdateProducer { + base, + update_op: &self.update_op, + }; + self.callback.callback(producer) + } + } + } +} + +// //////////////////////////////////////////////////////////////////////// + +struct UpdateProducer<'f, P, F> { + base: P, + update_op: &'f F, +} + +impl<'f, P, F> Producer for UpdateProducer<'f, P, F> +where + P: Producer, + F: Fn(&mut P::Item) + Send + Sync, +{ + type Item = P::Item; + type IntoIter = UpdateSeq; + + fn into_iter(self) -> Self::IntoIter { + UpdateSeq { + base: self.base.into_iter(), + update_op: self.update_op, + } + } + + fn min_len(&self) -> usize { + self.base.min_len() + } + fn max_len(&self) -> usize { + self.base.max_len() + } + + fn split_at(self, index: usize) -> (Self, Self) { + let (left, right) = self.base.split_at(index); + ( + UpdateProducer { + base: left, + update_op: self.update_op, + }, + UpdateProducer { + base: right, + update_op: self.update_op, + }, + ) + } + + fn fold_with(self, folder: G) -> G + where + G: Folder, + { + let folder1 = UpdateFolder { + base: folder, + update_op: self.update_op, + }; + self.base.fold_with(folder1).base + } +} + +// //////////////////////////////////////////////////////////////////////// +// Consumer implementation + +struct UpdateConsumer<'f, C, F> { + base: C, + update_op: &'f F, +} + +impl<'f, C, F> UpdateConsumer<'f, C, F> { + fn new(base: C, update_op: &'f F) -> Self { + UpdateConsumer { base, update_op } + } +} + +impl<'f, T, C, F> Consumer for UpdateConsumer<'f, C, F> +where + C: Consumer, + F: Fn(&mut T) + Send + Sync, +{ + type Folder = UpdateFolder<'f, C::Folder, F>; + type Reducer = C::Reducer; + type Result = C::Result; + + fn split_at(self, index: usize) -> (Self, Self, Self::Reducer) { + let (left, right, reducer) = self.base.split_at(index); + ( + UpdateConsumer::new(left, self.update_op), + UpdateConsumer::new(right, self.update_op), + reducer, + ) + } + + fn into_folder(self) -> Self::Folder { + UpdateFolder { + base: self.base.into_folder(), + update_op: self.update_op, + } + } + + fn full(&self) -> bool { + self.base.full() + } +} + +impl<'f, T, C, F> UnindexedConsumer for UpdateConsumer<'f, C, F> +where + C: UnindexedConsumer, + F: Fn(&mut T) + Send + Sync, +{ + fn split_off_left(&self) -> Self { + UpdateConsumer::new(self.base.split_off_left(), self.update_op) + } + + fn to_reducer(&self) -> Self::Reducer { + self.base.to_reducer() + } +} + +struct UpdateFolder<'f, C, F> { + base: C, + update_op: &'f F, +} + +fn apply(update_op: impl Fn(&mut T)) -> impl Fn(T) -> T { + move |mut item| { + update_op(&mut item); + item + } +} + +impl<'f, T, C, F> Folder for UpdateFolder<'f, C, F> +where + C: Folder, + F: Fn(&mut T), +{ + type Result = C::Result; + + fn consume(self, mut item: T) -> Self { + (self.update_op)(&mut item); + + UpdateFolder { + base: self.base.consume(item), + update_op: self.update_op, + } + } + + fn consume_iter(mut self, iter: I) -> Self + where + I: IntoIterator, + { + let update_op = self.update_op; + self.base = self + .base + .consume_iter(iter.into_iter().map(apply(update_op))); + self + } + + fn complete(self) -> C::Result { + self.base.complete() + } + + fn full(&self) -> bool { + self.base.full() + } +} + +/// Standard Update adaptor, based on `itertools::adaptors::Update` +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Debug, Clone)] +struct UpdateSeq { + base: I, + update_op: F, +} + +impl Iterator for UpdateSeq +where + I: Iterator, + F: Fn(&mut I::Item), +{ + type Item = I::Item; + + fn next(&mut self) -> Option { + let mut v = self.base.next()?; + (self.update_op)(&mut v); + Some(v) + } + + fn size_hint(&self) -> (usize, Option) { + self.base.size_hint() + } + + fn fold(self, init: Acc, g: G) -> Acc + where + G: FnMut(Acc, Self::Item) -> Acc, + { + self.base.map(apply(self.update_op)).fold(init, g) + } + + // if possible, re-use inner iterator specializations in collect + fn collect(self) -> C + where + C: ::std::iter::FromIterator, + { + self.base.map(apply(self.update_op)).collect() + } +} + +impl ExactSizeIterator for UpdateSeq +where + I: ExactSizeIterator, + F: Fn(&mut I::Item), +{ +} + +impl DoubleEndedIterator for UpdateSeq +where + I: DoubleEndedIterator, + F: Fn(&mut I::Item), +{ + fn next_back(&mut self) -> Option { + let mut v = self.base.next_back()?; + (self.update_op)(&mut v); + Some(v) + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/walk_tree.rs b/anneal/v2/vendor/rayon/src/iter/walk_tree.rs new file mode 100644 index 0000000000..f1fc2654f7 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/walk_tree.rs @@ -0,0 +1,524 @@ +use crate::iter::plumbing::{bridge_unindexed, Folder, UnindexedConsumer, UnindexedProducer}; +use crate::prelude::*; +use std::iter::once; + +#[derive(Debug)] +struct WalkTreePrefixProducer<'b, S, B> { + to_explore: Vec, // nodes (and subtrees) we have to process + seen: Vec, // nodes which have already been explored + children_of: &'b B, // function generating children +} + +impl UnindexedProducer for WalkTreePrefixProducer<'_, S, B> +where + S: Send, + B: Fn(&S) -> I + Send + Sync, + I: IntoIterator, +{ + type Item = S; + + fn split(mut self) -> (Self, Option) { + // explore while front is of size one. + while self.to_explore.len() == 1 { + let front_node = self.to_explore.pop().unwrap(); + self.to_explore + .extend((self.children_of)(&front_node).into_iter().rev()); + self.seen.push(front_node); + } + // now take half of the front. + let right_children = split_vec(&mut self.to_explore); + let right = right_children + .map(|mut c| { + std::mem::swap(&mut c, &mut self.to_explore); + WalkTreePrefixProducer { + to_explore: c, + seen: Vec::new(), + children_of: self.children_of, + } + }) + .or_else(|| { + // we can still try to divide 'seen' + let right_seen = split_vec(&mut self.seen); + right_seen.map(|s| WalkTreePrefixProducer { + to_explore: Default::default(), + seen: s, + children_of: self.children_of, + }) + }); + (self, right) + } + + fn fold_with(mut self, mut folder: F) -> F + where + F: Folder, + { + // start by consuming everything seen + folder = folder.consume_iter(self.seen); + if folder.full() { + return folder; + } + // now do all remaining explorations + while let Some(e) = self.to_explore.pop() { + self.to_explore + .extend((self.children_of)(&e).into_iter().rev()); + folder = folder.consume(e); + if folder.full() { + return folder; + } + } + folder + } +} + +/// ParallelIterator for arbitrary tree-shaped patterns. +/// Returned by the [`walk_tree_prefix()`] function. +#[derive(Debug)] +pub struct WalkTreePrefix { + initial_state: S, + children_of: B, +} + +impl ParallelIterator for WalkTreePrefix +where + S: Send, + B: Fn(&S) -> I + Send + Sync, + I: IntoIterator, +{ + type Item = S; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let producer = WalkTreePrefixProducer { + to_explore: once(self.initial_state).collect(), + seen: Vec::new(), + children_of: &self.children_of, + }; + bridge_unindexed(producer, consumer) + } +} + +/// Create a tree-like prefix parallel iterator from an initial root node. +/// The `children_of` function should take a node and return an iterator over its child nodes. +/// The best parallelization is obtained when the tree is balanced +/// but we should also be able to handle harder cases. +/// +/// # Ordering +/// +/// This function guarantees a prefix ordering. See also [`walk_tree_postfix`], +/// which guarantees a postfix order. +/// If you don't care about ordering, you should use [`walk_tree`], +/// which will use whatever is believed to be fastest. +/// For example a perfect binary tree of 7 nodes will reduced in the following order: +/// +/// ```text +/// a +/// / \ +/// / \ +/// b c +/// / \ / \ +/// d e f g +/// +/// reduced as a,b,d,e,c,f,g +/// +/// ``` +/// +/// # Example +/// +/// ```text +/// 4 +/// / \ +/// / \ +/// 2 3 +/// / \ +/// 1 2 +/// ``` +/// +/// ``` +/// use rayon::iter::walk_tree_prefix; +/// use rayon::prelude::*; +/// +/// let par_iter = walk_tree_prefix(4, |&e| { +/// if e <= 2 { +/// Vec::new() +/// } else { +/// vec![e / 2, e / 2 + 1] +/// } +/// }); +/// assert_eq!(par_iter.sum::(), 12); +/// ``` +/// +/// # Example +/// +/// ``` +/// use rayon::prelude::*; +/// use rayon::iter::walk_tree_prefix; +/// +/// struct Node { +/// content: u32, +/// left: Option>, +/// right: Option>, +/// } +/// +/// // Here we loop on the following tree: +/// // +/// // 10 +/// // / \ +/// // / \ +/// // 3 14 +/// // \ +/// // \ +/// // 18 +/// +/// let root = Node { +/// content: 10, +/// left: Some(Box::new(Node { +/// content: 3, +/// left: None, +/// right: None, +/// })), +/// right: Some(Box::new(Node { +/// content: 14, +/// left: None, +/// right: Some(Box::new(Node { +/// content: 18, +/// left: None, +/// right: None, +/// })), +/// })), +/// }; +/// +/// let mut v: Vec = walk_tree_prefix(&root, |r| { +/// r.left +/// .as_ref() +/// .into_iter() +/// .chain(r.right.as_ref()) +/// .map(|n| &**n) +/// }) +/// .map(|node| node.content) +/// .collect(); +/// assert_eq!(v, vec![10, 3, 14, 18]); +/// ``` +/// +pub fn walk_tree_prefix(root: S, children_of: B) -> WalkTreePrefix +where + S: Send, + B: Fn(&S) -> I + Send + Sync, + I: IntoIterator, +{ + WalkTreePrefix { + initial_state: root, + children_of, + } +} + +// post fix + +#[derive(Debug)] +struct WalkTreePostfixProducer<'b, S, B> { + to_explore: Vec, // nodes (and subtrees) we have to process + seen: Vec, // nodes which have already been explored + children_of: &'b B, // function generating children +} + +impl UnindexedProducer for WalkTreePostfixProducer<'_, S, B> +where + S: Send, + B: Fn(&S) -> I + Send + Sync, + I: IntoIterator, +{ + type Item = S; + + fn split(mut self) -> (Self, Option) { + // explore while front is of size one. + while self.to_explore.len() == 1 { + let front_node = self.to_explore.pop().unwrap(); + self.to_explore + .extend((self.children_of)(&front_node).into_iter()); + self.seen.push(front_node); + } + // now take half of the front. + let right_children = split_vec(&mut self.to_explore); + let right = right_children + .map(|c| { + let right_seen = std::mem::take(&mut self.seen); // postfix -> upper nodes are processed last + WalkTreePostfixProducer { + to_explore: c, + seen: right_seen, + children_of: self.children_of, + } + }) + .or_else(|| { + // we can still try to divide 'seen' + let right_seen = split_vec(&mut self.seen); + right_seen.map(|mut s| { + std::mem::swap(&mut self.seen, &mut s); + WalkTreePostfixProducer { + to_explore: Default::default(), + seen: s, + children_of: self.children_of, + } + }) + }); + (self, right) + } + + fn fold_with(self, mut folder: F) -> F + where + F: Folder, + { + // now do all remaining explorations + for e in self.to_explore { + folder = consume_rec_postfix(&self.children_of, e, folder); + if folder.full() { + return folder; + } + } + // end by consuming everything seen + folder.consume_iter(self.seen.into_iter().rev()) + } +} + +fn consume_rec_postfix(children_of: &B, s: S, mut folder: F) -> F +where + F: Folder, + B: Fn(&S) -> I, + I: IntoIterator, +{ + let children = (children_of)(&s).into_iter(); + for child in children { + folder = consume_rec_postfix(children_of, child, folder); + if folder.full() { + return folder; + } + } + folder.consume(s) +} + +/// ParallelIterator for arbitrary tree-shaped patterns. +/// Returned by the [`walk_tree_postfix()`] function. +#[derive(Debug)] +pub struct WalkTreePostfix { + initial_state: S, + children_of: B, +} + +impl ParallelIterator for WalkTreePostfix +where + S: Send, + B: Fn(&S) -> I + Send + Sync, + I: IntoIterator, +{ + type Item = S; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let producer = WalkTreePostfixProducer { + to_explore: once(self.initial_state).collect(), + seen: Vec::new(), + children_of: &self.children_of, + }; + bridge_unindexed(producer, consumer) + } +} + +/// Divide given vector in two equally sized vectors. +/// Return `None` if initial size is <=1. +/// We return the first half and keep the last half in `v`. +fn split_vec(v: &mut Vec) -> Option> { + if v.len() <= 1 { + None + } else { + let n = v.len() / 2; + Some(v.split_off(n)) + } +} + +/// Create a tree like postfix parallel iterator from an initial root node. +/// The `children_of` function should take a node and iterate on all of its child nodes. +/// The best parallelization is obtained when the tree is balanced +/// but we should also be able to handle harder cases. +/// +/// # Ordering +/// +/// This function guarantees a postfix ordering. See also [`walk_tree_prefix`] which guarantees a +/// prefix order. If you don't care about ordering, you should use [`walk_tree`], which will use +/// whatever is believed to be fastest. +/// +/// Between siblings, children are reduced in order -- that is first children are reduced first. +/// +/// For example a perfect binary tree of 7 nodes will reduced in the following order: +/// +/// ```text +/// a +/// / \ +/// / \ +/// b c +/// / \ / \ +/// d e f g +/// +/// reduced as d,e,b,f,g,c,a +/// +/// ``` +/// +/// # Example +/// +/// ```text +/// 4 +/// / \ +/// / \ +/// 2 3 +/// / \ +/// 1 2 +/// ``` +/// +/// ``` +/// use rayon::iter::walk_tree_postfix; +/// use rayon::prelude::*; +/// +/// let par_iter = walk_tree_postfix(4, |&e| { +/// if e <= 2 { +/// Vec::new() +/// } else { +/// vec![e / 2, e / 2 + 1] +/// } +/// }); +/// assert_eq!(par_iter.sum::(), 12); +/// ``` +/// +/// # Example +/// +/// ``` +/// use rayon::prelude::*; +/// use rayon::iter::walk_tree_postfix; +/// +/// struct Node { +/// content: u32, +/// left: Option>, +/// right: Option>, +/// } +/// +/// // Here we loop on the following tree: +/// // +/// // 10 +/// // / \ +/// // / \ +/// // 3 14 +/// // \ +/// // \ +/// // 18 +/// +/// let root = Node { +/// content: 10, +/// left: Some(Box::new(Node { +/// content: 3, +/// left: None, +/// right: None, +/// })), +/// right: Some(Box::new(Node { +/// content: 14, +/// left: None, +/// right: Some(Box::new(Node { +/// content: 18, +/// left: None, +/// right: None, +/// })), +/// })), +/// }; +/// +/// let mut v: Vec = walk_tree_postfix(&root, |r| { +/// r.left +/// .as_ref() +/// .into_iter() +/// .chain(r.right.as_ref()) +/// .map(|n| &**n) +/// }) +/// .map(|node| node.content) +/// .collect(); +/// assert_eq!(v, vec![3, 18, 14, 10]); +/// ``` +/// +pub fn walk_tree_postfix(root: S, children_of: B) -> WalkTreePostfix +where + S: Send, + B: Fn(&S) -> I + Send + Sync, + I: IntoIterator, +{ + WalkTreePostfix { + initial_state: root, + children_of, + } +} + +/// ParallelIterator for arbitrary tree-shaped patterns. +/// Returned by the [`walk_tree()`] function. +#[derive(Debug)] +pub struct WalkTree(WalkTreePostfix); + +/// Create a tree like parallel iterator from an initial root node. +/// The `children_of` function should take a node and iterate on all of its child nodes. +/// The best parallelization is obtained when the tree is balanced +/// but we should also be able to handle harder cases. +/// +/// # Ordering +/// +/// This function does not guarantee any ordering but will +/// use whatever algorithm is thought to achieve the fastest traversal. +/// See also [`walk_tree_prefix`] which guarantees a +/// prefix order and [`walk_tree_postfix`] which guarantees a postfix order. +/// +/// # Example +/// +/// ```text +/// 4 +/// / \ +/// / \ +/// 2 3 +/// / \ +/// 1 2 +/// ``` +/// +/// ``` +/// use rayon::iter::walk_tree; +/// use rayon::prelude::*; +/// +/// let par_iter = walk_tree(4, |&e| { +/// if e <= 2 { +/// Vec::new() +/// } else { +/// vec![e / 2, e / 2 + 1] +/// } +/// }); +/// assert_eq!(par_iter.sum::(), 12); +/// ``` +pub fn walk_tree(root: S, children_of: B) -> WalkTree +where + S: Send, + B: Fn(&S) -> I + Send + Sync, + I: IntoIterator, +{ + let walker = WalkTreePostfix { + initial_state: root, + children_of, + }; + WalkTree(walker) +} + +impl ParallelIterator for WalkTree +where + S: Send, + B: Fn(&S) -> I + Send + Sync, + I: IntoIterator + Send, +{ + type Item = S; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + self.0.drive_unindexed(consumer) + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/while_some.rs b/anneal/v2/vendor/rayon/src/iter/while_some.rs new file mode 100644 index 0000000000..49f8b9fd82 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/while_some.rs @@ -0,0 +1,150 @@ +use super::plumbing::*; +use super::*; +use std::sync::atomic::{AtomicBool, Ordering}; + +/// `WhileSome` is an iterator that yields the `Some` elements of an iterator, +/// halting as soon as any `None` is produced. +/// +/// This struct is created by the [`while_some()`] method on [`ParallelIterator`] +/// +/// [`while_some()`]: ParallelIterator::while_some() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Debug, Clone)] +pub struct WhileSome { + base: I, +} + +impl WhileSome { + /// Creates a new `WhileSome` iterator. + pub(super) fn new(base: I) -> Self { + WhileSome { base } + } +} + +impl ParallelIterator for WhileSome +where + I: ParallelIterator>, + T: Send, +{ + type Item = T; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let full = AtomicBool::new(false); + let consumer1 = WhileSomeConsumer { + base: consumer, + full: &full, + }; + self.base.drive_unindexed(consumer1) + } +} + +// //////////////////////////////////////////////////////////////////////// +// Consumer implementation + +struct WhileSomeConsumer<'f, C> { + base: C, + full: &'f AtomicBool, +} + +impl<'f, T, C> Consumer> for WhileSomeConsumer<'f, C> +where + C: Consumer, + T: Send, +{ + type Folder = WhileSomeFolder<'f, C::Folder>; + type Reducer = C::Reducer; + type Result = C::Result; + + fn split_at(self, index: usize) -> (Self, Self, Self::Reducer) { + let (left, right, reducer) = self.base.split_at(index); + ( + WhileSomeConsumer { base: left, ..self }, + WhileSomeConsumer { + base: right, + ..self + }, + reducer, + ) + } + + fn into_folder(self) -> Self::Folder { + WhileSomeFolder { + base: self.base.into_folder(), + full: self.full, + } + } + + fn full(&self) -> bool { + self.full.load(Ordering::Relaxed) || self.base.full() + } +} + +impl<'f, T, C> UnindexedConsumer> for WhileSomeConsumer<'f, C> +where + C: UnindexedConsumer, + T: Send, +{ + fn split_off_left(&self) -> Self { + WhileSomeConsumer { + base: self.base.split_off_left(), + ..*self + } + } + + fn to_reducer(&self) -> Self::Reducer { + self.base.to_reducer() + } +} + +struct WhileSomeFolder<'f, C> { + base: C, + full: &'f AtomicBool, +} + +impl<'f, T, C> Folder> for WhileSomeFolder<'f, C> +where + C: Folder, +{ + type Result = C::Result; + + fn consume(mut self, item: Option) -> Self { + match item { + Some(item) => self.base = self.base.consume(item), + None => self.full.store(true, Ordering::Relaxed), + } + self + } + + fn consume_iter(mut self, iter: I) -> Self + where + I: IntoIterator>, + { + fn some(full: &AtomicBool) -> impl Fn(&Option) -> bool + '_ { + move |x| match *x { + Some(_) => !full.load(Ordering::Relaxed), + None => { + full.store(true, Ordering::Relaxed); + false + } + } + } + + self.base = self.base.consume_iter( + iter.into_iter() + .take_while(some(self.full)) + .map(Option::unwrap), + ); + self + } + + fn complete(self) -> C::Result { + self.base.complete() + } + + fn full(&self) -> bool { + self.full.load(Ordering::Relaxed) || self.base.full() + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/zip.rs b/anneal/v2/vendor/rayon/src/iter/zip.rs new file mode 100644 index 0000000000..1479b67155 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/zip.rs @@ -0,0 +1,153 @@ +use super::plumbing::*; +use super::*; +use std::iter; + +/// `Zip` is an iterator that zips up `a` and `b` into a single iterator +/// of pairs. This struct is created by the [`zip()`] method on +/// [`IndexedParallelIterator`] +/// +/// [`zip()`]: IndexedParallelIterator::zip() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Debug, Clone)] +pub struct Zip { + a: A, + b: B, +} + +impl Zip { + /// Creates a new `Zip` iterator. + pub(super) fn new(a: A, b: B) -> Self { + Zip { a, b } + } +} + +impl ParallelIterator for Zip +where + A: IndexedParallelIterator, + B: IndexedParallelIterator, +{ + type Item = (A::Item, B::Item); + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + bridge(self, consumer) + } + + fn opt_len(&self) -> Option { + Some(self.len()) + } +} + +impl IndexedParallelIterator for Zip +where + A: IndexedParallelIterator, + B: IndexedParallelIterator, +{ + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + bridge(self, consumer) + } + + fn len(&self) -> usize { + Ord::min(self.a.len(), self.b.len()) + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + return self.a.with_producer(CallbackA { + callback, + b: self.b, + }); + + struct CallbackA { + callback: CB, + b: B, + } + + impl ProducerCallback for CallbackA + where + B: IndexedParallelIterator, + CB: ProducerCallback<(ITEM, B::Item)>, + { + type Output = CB::Output; + + fn callback(self, a_producer: A) -> Self::Output + where + A: Producer, + { + self.b.with_producer(CallbackB { + a_producer, + callback: self.callback, + }) + } + } + + struct CallbackB { + a_producer: A, + callback: CB, + } + + impl ProducerCallback for CallbackB + where + A: Producer, + CB: ProducerCallback<(A::Item, ITEM)>, + { + type Output = CB::Output; + + fn callback(self, b_producer: B) -> Self::Output + where + B: Producer, + { + self.callback.callback(ZipProducer { + a: self.a_producer, + b: b_producer, + }) + } + } + } +} + +// //////////////////////////////////////////////////////////////////////// + +struct ZipProducer { + a: A, + b: B, +} + +impl Producer for ZipProducer { + type Item = (A::Item, B::Item); + type IntoIter = iter::Zip; + + fn into_iter(self) -> Self::IntoIter { + self.a.into_iter().zip(self.b.into_iter()) + } + + fn min_len(&self) -> usize { + Ord::max(self.a.min_len(), self.b.min_len()) + } + + fn max_len(&self) -> usize { + Ord::min(self.a.max_len(), self.b.max_len()) + } + + fn split_at(self, index: usize) -> (Self, Self) { + let (a_left, a_right) = self.a.split_at(index); + let (b_left, b_right) = self.b.split_at(index); + ( + ZipProducer { + a: a_left, + b: b_left, + }, + ZipProducer { + a: a_right, + b: b_right, + }, + ) + } +} diff --git a/anneal/v2/vendor/rayon/src/iter/zip_eq.rs b/anneal/v2/vendor/rayon/src/iter/zip_eq.rs new file mode 100644 index 0000000000..298093094c --- /dev/null +++ b/anneal/v2/vendor/rayon/src/iter/zip_eq.rs @@ -0,0 +1,67 @@ +use super::plumbing::*; +use super::*; + +/// An [`IndexedParallelIterator`] that iterates over two parallel iterators of equal +/// length simultaneously. +/// +/// This struct is created by the [`zip_eq`] method on [`IndexedParallelIterator`], +/// see its documentation for more information. +/// +/// [`zip_eq`]: IndexedParallelIterator::zip_eq() +#[must_use = "iterator adaptors are lazy and do nothing unless consumed"] +#[derive(Debug, Clone)] +pub struct ZipEq { + zip: Zip, +} + +impl ZipEq { + /// Creates a new `ZipEq` iterator. + pub(super) fn new(a: A, b: B) -> Self { + ZipEq { + zip: super::Zip::new(a, b), + } + } +} + +impl ParallelIterator for ZipEq +where + A: IndexedParallelIterator, + B: IndexedParallelIterator, +{ + type Item = (A::Item, B::Item); + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + bridge(self.zip, consumer) + } + + fn opt_len(&self) -> Option { + Some(self.zip.len()) + } +} + +impl IndexedParallelIterator for ZipEq +where + A: IndexedParallelIterator, + B: IndexedParallelIterator, +{ + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + bridge(self.zip, consumer) + } + + fn len(&self) -> usize { + self.zip.len() + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + self.zip.with_producer(callback) + } +} diff --git a/anneal/v2/vendor/rayon/src/lib.rs b/anneal/v2/vendor/rayon/src/lib.rs new file mode 100644 index 0000000000..52fc01fe1d --- /dev/null +++ b/anneal/v2/vendor/rayon/src/lib.rs @@ -0,0 +1,156 @@ +//! Rayon is a data-parallelism library that makes it easy to convert sequential +//! computations into parallel. +//! +//! It is lightweight and convenient for introducing parallelism into existing +//! code. It guarantees data-race free executions and takes advantage of +//! parallelism when sensible, based on work-load at runtime. +//! +//! # How to use Rayon +//! +//! There are two ways to use Rayon: +//! +//! - **High-level parallel constructs** are the simplest way to use Rayon and also +//! typically the most efficient. +//! - [Parallel iterators] make it easy to convert a sequential iterator to +//! execute in parallel. +//! - The [`ParallelIterator`] trait defines general methods for all parallel iterators. +//! - The [`IndexedParallelIterator`] trait adds methods for iterators that support random +//! access. +//! - The [`par_sort`] method sorts `&mut [T]` slices (or vectors) in parallel. +//! - [`par_extend`] can be used to efficiently grow collections with items produced +//! by a parallel iterator. +//! - **Custom tasks** let you divide your work into parallel tasks yourself. +//! - [`join`] is used to subdivide a task into two pieces. +//! - [`scope`] creates a scope within which you can create any number of parallel tasks. +//! - [`ThreadPoolBuilder`] can be used to create your own thread pools or customize +//! the global one. +//! +//! [Parallel iterators]: iter +//! [`par_sort`]: slice::ParallelSliceMut::par_sort +//! [`par_extend`]: iter::ParallelExtend::par_extend +//! [`ParallelIterator`]: iter::ParallelIterator +//! [`IndexedParallelIterator`]: iter::IndexedParallelIterator +//! +//! # Basic usage and the Rayon prelude +//! +//! First, you will need to add `rayon` to your `Cargo.toml`. +//! +//! Next, to use parallel iterators or the other high-level methods, +//! you need to import several traits. Those traits are bundled into +//! the module [`rayon::prelude`]. It is recommended that you import +//! all of these traits at once by adding `use rayon::prelude::*` at +//! the top of each module that uses Rayon methods. +//! +//! These traits give you access to the `par_iter` method which provides +//! parallel implementations of many iterative functions such as [`map`], +//! [`for_each`], [`filter`], [`fold`], and [more]. +//! +//! [`rayon::prelude`]: prelude +//! [`map`]: iter::ParallelIterator::map +//! [`for_each`]: iter::ParallelIterator::for_each +//! [`filter`]: iter::ParallelIterator::filter +//! [`fold`]: iter::ParallelIterator::fold +//! [more]: iter::ParallelIterator#provided-methods +//! +//! # Crate Layout +//! +//! Rayon extends many of the types found in the standard library with +//! parallel iterator implementations. The modules in the `rayon` +//! crate mirror [`std`] itself: so, e.g., the `option` module in +//! Rayon contains parallel iterators for the `Option` type, which is +//! found in [the `option` module of `std`]. Similarly, the +//! `collections` module in Rayon offers parallel iterator types for +//! [the `collections` from `std`]. You will rarely need to access +//! these submodules unless you need to name iterator types +//! explicitly. +//! +//! [the `option` module of `std`]: std::option +//! [the `collections` from `std`]: std::collections +//! +//! # Targets without threading +//! +//! Rayon has limited support for targets without `std` threading implementations. +//! See the [`rayon_core`] documentation for more information about its global fallback. +//! +//! # Other questions? +//! +//! See [the Rayon FAQ][faq]. +//! +//! [faq]: https://github.com/rayon-rs/rayon/blob/main/FAQ.md + +#![deny(missing_debug_implementations)] +#![deny(missing_docs)] +#![deny(unreachable_pub)] +#![warn(rust_2018_idioms)] + +#[macro_use] +mod delegate; + +#[macro_use] +mod private; + +mod split_producer; + +pub mod array; +pub mod collections; +pub mod iter; +pub mod option; +pub mod prelude; +pub mod range; +pub mod range_inclusive; +pub mod result; +pub mod slice; +pub mod str; +pub mod string; +pub mod vec; + +mod math; +mod par_either; + +mod compile_fail; + +pub use rayon_core::FnContext; +pub use rayon_core::ThreadBuilder; +pub use rayon_core::ThreadPool; +pub use rayon_core::ThreadPoolBuildError; +pub use rayon_core::ThreadPoolBuilder; +pub use rayon_core::{broadcast, spawn_broadcast, BroadcastContext}; +pub use rayon_core::{current_num_threads, current_thread_index, max_num_threads}; +pub use rayon_core::{in_place_scope, scope, Scope}; +pub use rayon_core::{in_place_scope_fifo, scope_fifo, ScopeFifo}; +pub use rayon_core::{join, join_context}; +pub use rayon_core::{spawn, spawn_fifo}; +pub use rayon_core::{yield_local, yield_now, Yield}; + +/// We need to transmit raw pointers across threads. It is possible to do this +/// without any unsafe code by converting pointers to usize or to AtomicPtr +/// then back to a raw pointer for use. We prefer this approach because code +/// that uses this type is more explicit. +/// +/// Unsafe code is still required to dereference the pointer, so this type is +/// not unsound on its own, although it does partly lift the unconditional +/// !Send and !Sync on raw pointers. As always, dereference with care. +struct SendPtr(*mut T); + +// SAFETY: !Send for raw pointers is not for safety, just as a lint +unsafe impl Send for SendPtr {} + +// SAFETY: !Sync for raw pointers is not for safety, just as a lint +unsafe impl Sync for SendPtr {} + +impl SendPtr { + // Helper to avoid disjoint captures of `send_ptr.0` + fn get(self) -> *mut T { + self.0 + } +} + +// Implement Clone without the T: Clone bound from the derive +impl Clone for SendPtr { + fn clone(&self) -> Self { + *self + } +} + +// Implement Copy without the T: Copy bound from the derive +impl Copy for SendPtr {} diff --git a/anneal/v2/vendor/rayon/src/math.rs b/anneal/v2/vendor/rayon/src/math.rs new file mode 100644 index 0000000000..f259bc99c2 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/math.rs @@ -0,0 +1,25 @@ +use std::ops::{Bound, Range, RangeBounds}; + +/// Normalize arbitrary `RangeBounds` to a `Range` +pub(super) fn simplify_range(range: impl RangeBounds, len: usize) -> Range { + let start = match range.start_bound() { + Bound::Unbounded => 0, + Bound::Included(&i) if i <= len => i, + Bound::Excluded(&i) if i < len => i + 1, + bound => panic!("range start {bound:?} should be <= length {len}"), + }; + let end = match range.end_bound() { + Bound::Unbounded => len, + Bound::Excluded(&i) if i <= len => i, + Bound::Included(&i) if i < len => i + 1, + bound => panic!("range end {bound:?} should be <= length {len}"), + }; + if start > end { + panic!( + "range start {:?} should be <= range end {:?}", + range.start_bound(), + range.end_bound() + ); + } + start..end +} diff --git a/anneal/v2/vendor/rayon/src/option.rs b/anneal/v2/vendor/rayon/src/option.rs new file mode 100644 index 0000000000..ff89622d81 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/option.rs @@ -0,0 +1,197 @@ +//! Parallel iterator types for [options] +//! +//! You will rarely need to interact with this module directly unless you need +//! to name one of the iterator types. +//! +//! [options]: std::option + +use crate::iter::plumbing::*; +use crate::iter::*; +use std::sync::atomic::{AtomicBool, Ordering}; + +/// A parallel iterator over the value in [`Some`] variant of an [`Option`]. +/// +/// The iterator yields one value if the [`Option`] is a [`Some`], otherwise none. +/// +/// This `struct` is created by the [`into_par_iter`] function. +/// +/// [`into_par_iter`]: IntoParallelIterator::into_par_iter() +#[derive(Debug, Clone)] +pub struct IntoIter { + opt: Option, +} + +impl IntoParallelIterator for Option { + type Item = T; + type Iter = IntoIter; + + fn into_par_iter(self) -> Self::Iter { + IntoIter { opt: self } + } +} + +impl ParallelIterator for IntoIter { + type Item = T; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + self.drive(consumer) + } + + fn opt_len(&self) -> Option { + Some(self.len()) + } +} + +impl IndexedParallelIterator for IntoIter { + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + let mut folder = consumer.into_folder(); + if let Some(item) = self.opt { + folder = folder.consume(item); + } + folder.complete() + } + + fn len(&self) -> usize { + match self.opt { + Some(_) => 1, + None => 0, + } + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + callback.callback(OptionProducer { opt: self.opt }) + } +} + +/// A parallel iterator over a reference to the [`Some`] variant of an [`Option`]. +/// +/// The iterator yields one value if the [`Option`] is a [`Some`], otherwise none. +/// +/// This `struct` is created by the [`par_iter`] function. +/// +/// [`par_iter`]: IntoParallelRefIterator::par_iter() +#[derive(Debug)] +pub struct Iter<'a, T> { + inner: IntoIter<&'a T>, +} + +impl Clone for Iter<'_, T> { + fn clone(&self) -> Self { + Iter { + inner: self.inner.clone(), + } + } +} + +impl<'a, T: Sync> IntoParallelIterator for &'a Option { + type Item = &'a T; + type Iter = Iter<'a, T>; + + fn into_par_iter(self) -> Self::Iter { + Iter { + inner: self.as_ref().into_par_iter(), + } + } +} + +delegate_indexed_iterator! { + Iter<'a, T> => &'a T, + impl<'a, T: Sync> +} + +/// A parallel iterator over a mutable reference to the [`Some`] variant of an [`Option`]. +/// +/// The iterator yields one value if the [`Option`] is a [`Some`], otherwise none. +/// +/// This `struct` is created by the [`par_iter_mut`] function. +/// +/// [`par_iter_mut`]: IntoParallelRefMutIterator::par_iter_mut() +#[derive(Debug)] +pub struct IterMut<'a, T> { + inner: IntoIter<&'a mut T>, +} + +impl<'a, T: Send> IntoParallelIterator for &'a mut Option { + type Item = &'a mut T; + type Iter = IterMut<'a, T>; + + fn into_par_iter(self) -> Self::Iter { + IterMut { + inner: self.as_mut().into_par_iter(), + } + } +} + +delegate_indexed_iterator! { + IterMut<'a, T> => &'a mut T, + impl<'a, T: Send> +} + +/// Private producer for an option +struct OptionProducer { + opt: Option, +} + +impl Producer for OptionProducer { + type Item = T; + type IntoIter = std::option::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.opt.into_iter() + } + + fn split_at(self, index: usize) -> (Self, Self) { + debug_assert!(index <= 1); + let none = OptionProducer { opt: None }; + if index == 0 { + (none, self) + } else { + (self, none) + } + } +} + +/// Collect an arbitrary `Option`-wrapped collection. +/// +/// If any item is `None`, then all previous items collected are discarded, +/// and it returns only `None`. +impl FromParallelIterator> for Option +where + C: FromParallelIterator, + T: Send, +{ + fn from_par_iter(par_iter: I) -> Self + where + I: IntoParallelIterator>, + { + fn check(found_none: &AtomicBool) -> impl Fn(&Option) + '_ { + move |item| { + if item.is_none() { + found_none.store(true, Ordering::Relaxed); + } + } + } + + let found_none = AtomicBool::new(false); + let collection = par_iter + .into_par_iter() + .inspect(check(&found_none)) + .while_some() + .collect(); + + if found_none.load(Ordering::Relaxed) { + None + } else { + Some(collection) + } + } +} diff --git a/anneal/v2/vendor/rayon/src/par_either.rs b/anneal/v2/vendor/rayon/src/par_either.rs new file mode 100644 index 0000000000..a19ce53998 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/par_either.rs @@ -0,0 +1,74 @@ +use crate::iter::plumbing::*; +use crate::iter::Either::{Left, Right}; +use crate::iter::*; + +/// `Either` is a parallel iterator if both `L` and `R` are parallel iterators. +impl ParallelIterator for Either +where + L: ParallelIterator, + R: ParallelIterator, +{ + type Item = L::Item; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + match self { + Left(iter) => iter.drive_unindexed(consumer), + Right(iter) => iter.drive_unindexed(consumer), + } + } + + fn opt_len(&self) -> Option { + self.as_ref().either(L::opt_len, R::opt_len) + } +} + +impl IndexedParallelIterator for Either +where + L: IndexedParallelIterator, + R: IndexedParallelIterator, +{ + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + match self { + Left(iter) => iter.drive(consumer), + Right(iter) => iter.drive(consumer), + } + } + + fn len(&self) -> usize { + self.as_ref().either(L::len, R::len) + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + match self { + Left(iter) => iter.with_producer(callback), + Right(iter) => iter.with_producer(callback), + } + } +} + +/// `Either` can be extended if both `L` and `R` are parallel extendable. +impl ParallelExtend for Either +where + L: ParallelExtend, + R: ParallelExtend, + T: Send, +{ + fn par_extend(&mut self, par_iter: I) + where + I: IntoParallelIterator, + { + match self.as_mut() { + Left(collection) => collection.par_extend(par_iter), + Right(collection) => collection.par_extend(par_iter), + } + } +} diff --git a/anneal/v2/vendor/rayon/src/prelude.rs b/anneal/v2/vendor/rayon/src/prelude.rs new file mode 100644 index 0000000000..6eaca06c1b --- /dev/null +++ b/anneal/v2/vendor/rayon/src/prelude.rs @@ -0,0 +1,17 @@ +//! The rayon prelude imports the various `ParallelIterator` traits. +//! The intention is that one can include `use rayon::prelude::*` and +//! have easy access to the various traits and methods you will need. + +pub use crate::iter::FromParallelIterator; +pub use crate::iter::IndexedParallelIterator; +pub use crate::iter::IntoParallelIterator; +pub use crate::iter::IntoParallelRefIterator; +pub use crate::iter::IntoParallelRefMutIterator; +pub use crate::iter::ParallelBridge; +pub use crate::iter::ParallelDrainFull; +pub use crate::iter::ParallelDrainRange; +pub use crate::iter::ParallelExtend; +pub use crate::iter::ParallelIterator; +pub use crate::slice::ParallelSlice; +pub use crate::slice::ParallelSliceMut; +pub use crate::str::ParallelString; diff --git a/anneal/v2/vendor/rayon/src/private.rs b/anneal/v2/vendor/rayon/src/private.rs new file mode 100644 index 0000000000..c85e77b9cb --- /dev/null +++ b/anneal/v2/vendor/rayon/src/private.rs @@ -0,0 +1,26 @@ +//! The public parts of this private module are used to create traits +//! that cannot be implemented outside of our own crate. This way we +//! can feel free to extend those traits without worrying about it +//! being a breaking change for other implementations. + +/// If this type is pub but not publicly reachable, third parties +/// can't name it and can't implement traits using it. +#[allow(missing_debug_implementations)] +pub struct PrivateMarker; + +macro_rules! private_decl { + () => { + /// This trait is private; this method exists to make it + /// impossible to implement outside the crate. + #[doc(hidden)] + fn __rayon_private__(&self) -> crate::private::PrivateMarker; + }; +} + +macro_rules! private_impl { + () => { + fn __rayon_private__(&self) -> crate::private::PrivateMarker { + crate::private::PrivateMarker + } + }; +} diff --git a/anneal/v2/vendor/rayon/src/range.rs b/anneal/v2/vendor/rayon/src/range.rs new file mode 100644 index 0000000000..828093353c --- /dev/null +++ b/anneal/v2/vendor/rayon/src/range.rs @@ -0,0 +1,474 @@ +//! Parallel iterator types for [ranges], +//! the type for values created by `a..b` expressions +//! +//! You will rarely need to interact with this module directly unless you have +//! need to name one of the iterator types. +//! +//! ``` +//! use rayon::prelude::*; +//! +//! let r = (0..100u64).into_par_iter() +//! .sum(); +//! +//! // compare result with sequential calculation +//! assert_eq!((0..100).sum::(), r); +//! ``` +//! +//! [ranges]: std::ops::Range + +use crate::iter::plumbing::*; +use crate::iter::*; +use std::ops::Range; + +/// Parallel iterator over a range, implemented for all integer types and `char`. +/// +/// **Note:** The `zip` operation requires `IndexedParallelIterator` +/// which is not implemented for `u64`, `i64`, `u128`, or `i128`. +/// +/// ``` +/// use rayon::prelude::*; +/// +/// let p = (0..25usize).into_par_iter() +/// .zip(0..25usize) +/// .filter(|&(x, y)| x % 5 == 0 || y % 5 == 0) +/// .map(|(x, y)| x * y) +/// .sum::(); +/// +/// let s = (0..25usize).zip(0..25) +/// .filter(|&(x, y)| x % 5 == 0 || y % 5 == 0) +/// .map(|(x, y)| x * y) +/// .sum(); +/// +/// assert_eq!(p, s); +/// ``` +#[derive(Debug, Clone)] +pub struct Iter { + range: Range, +} + +/// Implemented for ranges of all primitive integer types and `char`. +impl IntoParallelIterator for Range +where + Iter: ParallelIterator, +{ + type Item = as ParallelIterator>::Item; + type Iter = Iter; + + fn into_par_iter(self) -> Self::Iter { + Iter { range: self } + } +} + +struct IterProducer { + range: Range, +} + +impl IntoIterator for IterProducer +where + Range: Iterator, +{ + type Item = as Iterator>::Item; + type IntoIter = Range; + + fn into_iter(self) -> Self::IntoIter { + self.range + } +} + +/// These traits help drive integer type inference. Without them, an unknown `{integer}` type only +/// has constraints on `Iter<{integer}>`, which will probably give up and use `i32`. By adding +/// these traits on the item type, the compiler can see a more direct constraint to infer like +/// `{integer}: RangeInteger`, which works better. See `test_issue_833` for an example. +/// +/// They have to be `pub` since they're seen in the public `impl ParallelIterator` constraints, but +/// we put them in a private modules so they're not actually reachable in our public API. +mod private { + use super::*; + + /// Implementation details of `ParallelIterator for Iter` + pub trait RangeInteger: Sized + Send { + private_decl! {} + + fn drive_unindexed(iter: Iter, consumer: C) -> C::Result + where + C: UnindexedConsumer; + + fn opt_len(iter: &Iter) -> Option; + } + + /// Implementation details of `IndexedParallelIterator for Iter` + pub trait IndexedRangeInteger: RangeInteger { + private_decl! {} + + fn drive(iter: Iter, consumer: C) -> C::Result + where + C: Consumer; + + fn len(iter: &Iter) -> usize; + + fn with_producer(iter: Iter, callback: CB) -> CB::Output + where + CB: ProducerCallback; + } +} +use private::{IndexedRangeInteger, RangeInteger}; + +impl ParallelIterator for Iter { + type Item = T; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + T::drive_unindexed(self, consumer) + } + + #[inline] + fn opt_len(&self) -> Option { + T::opt_len(self) + } +} + +impl IndexedParallelIterator for Iter { + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + T::drive(self, consumer) + } + + #[inline] + fn len(&self) -> usize { + T::len(self) + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + T::with_producer(self, callback) + } +} + +macro_rules! indexed_range_impl { + ( $t:ty ) => { + impl RangeInteger for $t { + private_impl! {} + + fn drive_unindexed(iter: Iter<$t>, consumer: C) -> C::Result + where + C: UnindexedConsumer<$t>, + { + bridge(iter, consumer) + } + + fn opt_len(iter: &Iter<$t>) -> Option { + Some(iter.range.len()) + } + } + + impl IndexedRangeInteger for $t { + private_impl! {} + + fn drive(iter: Iter<$t>, consumer: C) -> C::Result + where + C: Consumer<$t>, + { + bridge(iter, consumer) + } + + fn len(iter: &Iter<$t>) -> usize { + iter.range.len() + } + + fn with_producer(iter: Iter<$t>, callback: CB) -> CB::Output + where + CB: ProducerCallback<$t>, + { + callback.callback(IterProducer { range: iter.range }) + } + } + + impl Producer for IterProducer<$t> { + type Item = as Iterator>::Item; + type IntoIter = Range<$t>; + fn into_iter(self) -> Self::IntoIter { + self.range + } + + fn split_at(self, index: usize) -> (Self, Self) { + assert!(index <= self.range.len()); + // For signed $t, the length and requested index could be greater than $t::MAX, and + // then `index as $t` could wrap to negative, so wrapping_add is necessary. + let mid = self.range.start.wrapping_add(index as $t); + let left = self.range.start..mid; + let right = mid..self.range.end; + (IterProducer { range: left }, IterProducer { range: right }) + } + } + }; +} + +trait UnindexedRangeLen { + fn unindexed_len(&self) -> L; +} + +macro_rules! unindexed_range_impl { + ( $t:ty, $len_t:ty ) => { + impl UnindexedRangeLen<$len_t> for Range<$t> { + fn unindexed_len(&self) -> $len_t { + let &Range { start, end } = self; + if end > start { + end.wrapping_sub(start) as $len_t + } else { + 0 + } + } + } + + impl RangeInteger for $t { + private_impl! {} + + fn drive_unindexed(iter: Iter<$t>, consumer: C) -> C::Result + where + C: UnindexedConsumer<$t>, + { + #[inline] + fn offset(start: $t) -> impl Fn(usize) -> $t { + move |i| start.wrapping_add(i as $t) + } + + if let Some(len) = iter.opt_len() { + // Drive this in indexed mode for better `collect`. + (0..len) + .into_par_iter() + .map(offset(iter.range.start)) + .drive(consumer) + } else { + bridge_unindexed(IterProducer { range: iter.range }, consumer) + } + } + + fn opt_len(iter: &Iter<$t>) -> Option { + usize::try_from(iter.range.unindexed_len()).ok() + } + } + + impl UnindexedProducer for IterProducer<$t> { + type Item = $t; + + fn split(mut self) -> (Self, Option) { + let index = self.range.unindexed_len() / 2; + if index > 0 { + let mid = self.range.start.wrapping_add(index as $t); + let right = mid..self.range.end; + self.range.end = mid; + (self, Some(IterProducer { range: right })) + } else { + (self, None) + } + } + + fn fold_with(self, folder: F) -> F + where + F: Folder, + { + folder.consume_iter(self) + } + } + }; +} + +// all Range with ExactSizeIterator +indexed_range_impl! {u8} +indexed_range_impl! {u16} +indexed_range_impl! {u32} +indexed_range_impl! {usize} +indexed_range_impl! {i8} +indexed_range_impl! {i16} +indexed_range_impl! {i32} +indexed_range_impl! {isize} + +// other Range with just Iterator +unindexed_range_impl! {u64, u64} +unindexed_range_impl! {i64, u64} +unindexed_range_impl! {u128, u128} +unindexed_range_impl! {i128, u128} + +// char is special because of the surrogate range hole +macro_rules! convert_char { + ( $self:ident . $method:ident ( $( $arg:expr ),* ) ) => {{ + let start = $self.range.start as u32; + let end = $self.range.end as u32; + if start < 0xD800 && 0xE000 <= end { + // chain the before and after surrogate range fragments + (start..0xD800) + .into_par_iter() + .chain(0xE000..end) + .map(|codepoint| unsafe { char::from_u32_unchecked(codepoint) }) + .$method($( $arg ),*) + } else { + // no surrogate range to worry about + (start..end) + .into_par_iter() + .map(|codepoint| unsafe { char::from_u32_unchecked(codepoint) }) + .$method($( $arg ),*) + } + }}; +} + +impl ParallelIterator for Iter { + type Item = char; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + convert_char!(self.drive(consumer)) + } + + fn opt_len(&self) -> Option { + Some(self.len()) + } +} + +impl IndexedParallelIterator for Iter { + // Split at the surrogate range first if we're allowed to + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + convert_char!(self.drive(consumer)) + } + + fn len(&self) -> usize { + // Taken from ::steps_between + let start = self.range.start as u32; + let end = self.range.end as u32; + if start < end { + let mut count = end - start; + if start < 0xD800 && 0xE000 <= end { + count -= 0x800 + } + count as usize + } else { + 0 + } + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + convert_char!(self.with_producer(callback)) + } +} + +#[test] +fn test_char_range_at_surrogate_boundary() { + use crate::prelude::*; + + // Ranges ending exactly at '\u{E000}' must not produce surrogate codepoints. + let chars: Vec = ('\u{D7FE}'..'\u{E000}').into_par_iter().collect(); + assert_eq!(chars, vec!['\u{D7FE}', '\u{D7FF}']); + + // Ranges starting at '\u{E000}' are entirely above surrogates. + let chars: Vec = ('\u{E000}'..'\u{E002}').into_par_iter().collect(); + assert_eq!(chars, vec!['\u{E000}', '\u{E001}']); + + // A range that spans the surrogate gap should skip it. + let chars: Vec = ('\u{D7FE}'..'\u{E002}').into_par_iter().collect(); + assert_eq!(chars, vec!['\u{D7FE}', '\u{D7FF}', '\u{E000}', '\u{E001}']); +} + +#[test] +fn check_range_split_at_overflow() { + // Note, this split index overflows i8! + let producer = IterProducer { range: -100i8..100 }; + let (left, right) = producer.split_at(150); + let r1: i32 = left.range.map(i32::from).sum(); + let r2: i32 = right.range.map(i32::from).sum(); + assert_eq!(r1 + r2, -100); +} + +#[test] +fn test_i128_len_doesnt_overflow() { + // Using parse because some versions of rust don't allow long literals + let octillion: i128 = "1000000000000000000000000000".parse().unwrap(); + let producer = IterProducer { + range: 0..octillion, + }; + + assert_eq!(octillion as u128, producer.range.unindexed_len()); + assert_eq!(octillion as u128, (0..octillion).unindexed_len()); + assert_eq!( + 2 * octillion as u128, + (-octillion..octillion).unindexed_len() + ); + + assert_eq!(u128::MAX, (i128::MIN..i128::MAX).unindexed_len()); +} + +#[test] +fn test_u64_opt_len() { + assert_eq!(Some(100), (0..100u64).into_par_iter().opt_len()); + assert_eq!( + Some(usize::MAX), + (0..usize::MAX as u64).into_par_iter().opt_len() + ); + if (usize::MAX as u64) < u64::MAX { + assert_eq!( + None, + (0..(usize::MAX as u64).wrapping_add(1)) + .into_par_iter() + .opt_len() + ); + assert_eq!(None, (0..u64::MAX).into_par_iter().opt_len()); + } +} + +#[test] +fn test_u128_opt_len() { + assert_eq!(Some(100), (0..100u128).into_par_iter().opt_len()); + assert_eq!( + Some(usize::MAX), + (0..usize::MAX as u128).into_par_iter().opt_len() + ); + assert_eq!(None, (0..1 + usize::MAX as u128).into_par_iter().opt_len()); + assert_eq!(None, (0..u128::MAX).into_par_iter().opt_len()); +} + +// `usize as i64` can overflow, so make sure to wrap it appropriately +// when using the `opt_len` "indexed" mode. +#[test] +#[cfg(target_pointer_width = "64")] +fn test_usize_i64_overflow() { + use crate::ThreadPoolBuilder; + + let iter = (-2..i64::MAX).into_par_iter(); + assert_eq!(iter.opt_len(), Some(i64::MAX as usize + 2)); + + // always run with multiple threads to split into, or this will take forever... + let pool = ThreadPoolBuilder::new().num_threads(8).build().unwrap(); + pool.install(|| assert_eq!(iter.find_last(|_| true), Some(i64::MAX - 1))); +} + +#[test] +fn test_issue_833() { + fn is_even(n: i64) -> bool { + n % 2 == 0 + } + + // The integer type should be inferred from `is_even` + let v: Vec<_> = (1..100).into_par_iter().filter(|&x| is_even(x)).collect(); + assert!(v.into_iter().eq((2..100).step_by(2))); + + // Try examples with indexed iterators too + let pos = (0..100).into_par_iter().position_any(|x| x == 50i16); + assert_eq!(pos, Some(50usize)); + + assert!((0..100) + .into_par_iter() + .zip(0..100) + .all(|(a, b)| i16::eq(&a, &b))); +} diff --git a/anneal/v2/vendor/rayon/src/range_inclusive.rs b/anneal/v2/vendor/rayon/src/range_inclusive.rs new file mode 100644 index 0000000000..e12af985ba --- /dev/null +++ b/anneal/v2/vendor/rayon/src/range_inclusive.rs @@ -0,0 +1,398 @@ +//! Parallel iterator types for [inclusive ranges], +//! the type for values created by `a..=b` expressions +//! +//! You will rarely need to interact with this module directly unless you have +//! need to name one of the iterator types. +//! +//! ``` +//! use rayon::prelude::*; +//! +//! let r = (0..=100u64).into_par_iter() +//! .sum(); +//! +//! // compare result with sequential calculation +//! assert_eq!((0..=100).sum::(), r); +//! ``` +//! +//! [inclusive ranges]: std::ops::RangeInclusive + +use crate::iter::plumbing::*; +use crate::iter::*; +use std::ops::RangeInclusive; + +/// Parallel iterator over an inclusive range, implemented for all integer types and `char`. +/// +/// **Note:** The `zip` operation requires `IndexedParallelIterator` +/// which is only implemented for `u8`, `i8`, `u16`, `i16`, and `char`. +/// +/// ``` +/// use rayon::prelude::*; +/// +/// let p = (0..=25u16).into_par_iter() +/// .zip(0..=25u16) +/// .filter(|&(x, y)| x % 5 == 0 || y % 5 == 0) +/// .map(|(x, y)| x * y) +/// .sum::(); +/// +/// let s = (0..=25u16).zip(0..=25u16) +/// .filter(|&(x, y)| x % 5 == 0 || y % 5 == 0) +/// .map(|(x, y)| x * y) +/// .sum(); +/// +/// assert_eq!(p, s); +/// ``` +#[derive(Debug, Clone)] +pub struct Iter { + range: RangeInclusive, +} + +impl Iter +where + RangeInclusive: Eq, + T: Ord + Copy, +{ + /// Returns `Some((start, end))` for `start..=end`, or `None` if it is exhausted. + /// + /// Note that `RangeInclusive` does not specify the bounds of an exhausted iterator, + /// so this is a way for us to figure out what we've got. Thankfully, all of the + /// integer types we care about can be trivially cloned. + fn bounds(&self) -> Option<(T, T)> { + let start = *self.range.start(); + let end = *self.range.end(); + if start <= end && self.range == (start..=end) { + // If the range is still nonempty, this is obviously true + // If the range is exhausted, either start > end or + // the range does not equal start..=end. + Some((start, end)) + } else { + None + } + } +} + +/// Implemented for ranges of all primitive integer types and `char`. +impl IntoParallelIterator for RangeInclusive +where + Iter: ParallelIterator, +{ + type Item = as ParallelIterator>::Item; + type Iter = Iter; + + fn into_par_iter(self) -> Self::Iter { + Iter { range: self } + } +} + +/// These traits help drive integer type inference. Without them, an unknown `{integer}` type only +/// has constraints on `Iter<{integer}>`, which will probably give up and use `i32`. By adding +/// these traits on the item type, the compiler can see a more direct constraint to infer like +/// `{integer}: RangeInteger`, which works better. See `test_issue_833` for an example. +/// +/// They have to be `pub` since they're seen in the public `impl ParallelIterator` constraints, but +/// we put them in a private modules so they're not actually reachable in our public API. +mod private { + use super::*; + + /// Implementation details of `ParallelIterator for Iter` + pub trait RangeInteger: Sized + Send { + private_decl! {} + + fn drive_unindexed(iter: Iter, consumer: C) -> C::Result + where + C: UnindexedConsumer; + + fn opt_len(iter: &Iter) -> Option; + } + + /// Implementation details of `IndexedParallelIterator for Iter` + pub trait IndexedRangeInteger: RangeInteger { + private_decl! {} + + fn drive(iter: Iter, consumer: C) -> C::Result + where + C: Consumer; + + fn len(iter: &Iter) -> usize; + + fn with_producer(iter: Iter, callback: CB) -> CB::Output + where + CB: ProducerCallback; + } +} +use private::{IndexedRangeInteger, RangeInteger}; + +impl ParallelIterator for Iter { + type Item = T; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + T::drive_unindexed(self, consumer) + } + + #[inline] + fn opt_len(&self) -> Option { + T::opt_len(self) + } +} + +impl IndexedParallelIterator for Iter { + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + T::drive(self, consumer) + } + + #[inline] + fn len(&self) -> usize { + T::len(self) + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + T::with_producer(self, callback) + } +} + +macro_rules! convert { + ( $iter:ident . $method:ident ( $( $arg:expr ),* ) ) => { + if let Some((start, end)) = $iter.bounds() { + if let Some(end) = end.checked_add(1) { + (start..end).into_par_iter().$method($( $arg ),*) + } else { + (start..end).into_par_iter().chain(once(end)).$method($( $arg ),*) + } + } else { + empty::().$method($( $arg ),*) + } + }; +} + +macro_rules! parallel_range_impl { + ( $t:ty ) => { + impl RangeInteger for $t { + private_impl! {} + + fn drive_unindexed(iter: Iter<$t>, consumer: C) -> C::Result + where + C: UnindexedConsumer<$t>, + { + convert!(iter.drive_unindexed(consumer)) + } + + fn opt_len(iter: &Iter<$t>) -> Option { + convert!(iter.opt_len()) + } + } + }; +} + +macro_rules! indexed_range_impl { + ( $t:ty ) => { + parallel_range_impl! { $t } + + impl IndexedRangeInteger for $t { + private_impl! {} + + fn drive(iter: Iter<$t>, consumer: C) -> C::Result + where + C: Consumer<$t>, + { + convert!(iter.drive(consumer)) + } + + fn len(iter: &Iter<$t>) -> usize { + iter.range.len() + } + + fn with_producer(iter: Iter<$t>, callback: CB) -> CB::Output + where + CB: ProducerCallback<$t>, + { + convert!(iter.with_producer(callback)) + } + } + }; +} + +// all RangeInclusive with ExactSizeIterator +indexed_range_impl! {u8} +indexed_range_impl! {u16} +indexed_range_impl! {i8} +indexed_range_impl! {i16} + +// other RangeInclusive with just Iterator +parallel_range_impl! {usize} +parallel_range_impl! {isize} +parallel_range_impl! {u32} +parallel_range_impl! {i32} +parallel_range_impl! {u64} +parallel_range_impl! {i64} +parallel_range_impl! {u128} +parallel_range_impl! {i128} + +// char is special +macro_rules! convert_char { + ( $self:ident . $method:ident ( $( $arg:expr ),* ) ) => { + if let Some((start, end)) = $self.bounds() { + let start = start as u32; + let end = end as u32; + if start < 0xD800 && 0xE000 <= end { + // chain the before and after surrogate range fragments + (start..0xD800) + .into_par_iter() + .chain(0xE000..end + 1) // cannot use RangeInclusive, so add one to end + .map(|codepoint| unsafe { char::from_u32_unchecked(codepoint) }) + .$method($( $arg ),*) + } else { + // no surrogate range to worry about + (start..end + 1) // cannot use RangeInclusive, so add one to end + .into_par_iter() + .map(|codepoint| unsafe { char::from_u32_unchecked(codepoint) }) + .$method($( $arg ),*) + } + } else { + empty::().$method($( $arg ),*) + } + }; +} + +impl ParallelIterator for Iter { + type Item = char; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + convert_char!(self.drive(consumer)) + } + + fn opt_len(&self) -> Option { + Some(self.len()) + } +} + +// Range is broken on 16 bit platforms, may as well benefit from it +impl IndexedParallelIterator for Iter { + // Split at the surrogate range first if we're allowed to + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + convert_char!(self.drive(consumer)) + } + + fn len(&self) -> usize { + if let Some((start, end)) = self.bounds() { + // Taken from ::steps_between + let start = start as u32; + let end = end as u32; + let mut count = end - start; + if start < 0xD800 && 0xE000 <= end { + count -= 0x800 + } + (count + 1) as usize // add one for inclusive + } else { + 0 + } + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + convert_char!(self.with_producer(callback)) + } +} + +#[test] +fn test_char_range_inclusive_at_surrogate_boundary() { + use crate::prelude::*; + + // Ranges ending exactly at '\u{E000}' must not produce surrogate codepoints. + let chars: Vec = ('\u{D7FE}'..='\u{E000}').into_par_iter().collect(); + assert_eq!(chars, vec!['\u{D7FE}', '\u{D7FF}', '\u{E000}']); + + // Ranges starting at '\u{E000}' are entirely above surrogates. + let chars: Vec = ('\u{E000}'..='\u{E001}').into_par_iter().collect(); + assert_eq!(chars, vec!['\u{E000}', '\u{E001}']); + + // A range that spans the surrogate gap should skip it. + let chars: Vec = ('\u{D7FE}'..='\u{E001}').into_par_iter().collect(); + assert_eq!(chars, vec!['\u{D7FE}', '\u{D7FF}', '\u{E000}', '\u{E001}']); +} + +#[test] +#[cfg(target_pointer_width = "64")] +fn test_u32_opt_len() { + assert_eq!(Some(101), (0..=100u32).into_par_iter().opt_len()); + assert_eq!( + Some(u32::MAX as usize), + (0..=u32::MAX - 1).into_par_iter().opt_len() + ); + assert_eq!( + Some(u32::MAX as usize + 1), + (0..=u32::MAX).into_par_iter().opt_len() + ); +} + +#[test] +fn test_u64_opt_len() { + assert_eq!(Some(101), (0..=100u64).into_par_iter().opt_len()); + assert_eq!( + Some(usize::MAX), + (0..=usize::MAX as u64 - 1).into_par_iter().opt_len() + ); + assert_eq!(None, (0..=usize::MAX as u64).into_par_iter().opt_len()); + assert_eq!(None, (0..=u64::MAX).into_par_iter().opt_len()); +} + +#[test] +fn test_u128_opt_len() { + assert_eq!(Some(101), (0..=100u128).into_par_iter().opt_len()); + assert_eq!( + Some(usize::MAX), + (0..=usize::MAX as u128 - 1).into_par_iter().opt_len() + ); + assert_eq!(None, (0..=usize::MAX as u128).into_par_iter().opt_len()); + assert_eq!(None, (0..=u128::MAX).into_par_iter().opt_len()); +} + +// `usize as i64` can overflow, so make sure to wrap it appropriately +// when using the `opt_len` "indexed" mode. +#[test] +#[cfg(target_pointer_width = "64")] +fn test_usize_i64_overflow() { + use crate::ThreadPoolBuilder; + + let iter = (-2..=i64::MAX).into_par_iter(); + assert_eq!(iter.opt_len(), Some(i64::MAX as usize + 3)); + + // always run with multiple threads to split into, or this will take forever... + let pool = ThreadPoolBuilder::new().num_threads(8).build().unwrap(); + pool.install(|| assert_eq!(iter.find_last(|_| true), Some(i64::MAX))); +} + +#[test] +fn test_issue_833() { + fn is_even(n: i64) -> bool { + n % 2 == 0 + } + + // The integer type should be inferred from `is_even` + let v: Vec<_> = (1..=100).into_par_iter().filter(|&x| is_even(x)).collect(); + assert!(v.into_iter().eq((2..=100).step_by(2))); + + // Try examples with indexed iterators too + let pos = (0..=100).into_par_iter().position_any(|x| x == 50i16); + assert_eq!(pos, Some(50usize)); + + assert!((0..=100) + .into_par_iter() + .zip(0..=100) + .all(|(a, b)| i16::eq(&a, &b))); +} diff --git a/anneal/v2/vendor/rayon/src/result.rs b/anneal/v2/vendor/rayon/src/result.rs new file mode 100644 index 0000000000..61eb8be665 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/result.rs @@ -0,0 +1,132 @@ +//! Parallel iterator types for [results] +//! +//! You will rarely need to interact with this module directly unless you need +//! to name one of the iterator types. +//! +//! [results]: std::result + +use crate::iter::plumbing::*; +use crate::iter::*; +use std::sync::Mutex; + +use crate::option; + +/// Parallel iterator over a result +#[derive(Debug, Clone)] +pub struct IntoIter { + inner: option::IntoIter, +} + +impl IntoParallelIterator for Result { + type Item = T; + type Iter = IntoIter; + + fn into_par_iter(self) -> Self::Iter { + IntoIter { + inner: self.ok().into_par_iter(), + } + } +} + +delegate_indexed_iterator! { + IntoIter => T, + impl +} + +/// Parallel iterator over an immutable reference to a result +#[derive(Debug)] +pub struct Iter<'a, T> { + inner: option::IntoIter<&'a T>, +} + +impl Clone for Iter<'_, T> { + fn clone(&self) -> Self { + Iter { + inner: self.inner.clone(), + } + } +} + +impl<'a, T: Sync, E> IntoParallelIterator for &'a Result { + type Item = &'a T; + type Iter = Iter<'a, T>; + + fn into_par_iter(self) -> Self::Iter { + Iter { + inner: self.as_ref().ok().into_par_iter(), + } + } +} + +delegate_indexed_iterator! { + Iter<'a, T> => &'a T, + impl<'a, T: Sync> +} + +/// Parallel iterator over a mutable reference to a result +#[derive(Debug)] +pub struct IterMut<'a, T> { + inner: option::IntoIter<&'a mut T>, +} + +impl<'a, T: Send, E> IntoParallelIterator for &'a mut Result { + type Item = &'a mut T; + type Iter = IterMut<'a, T>; + + fn into_par_iter(self) -> Self::Iter { + IterMut { + inner: self.as_mut().ok().into_par_iter(), + } + } +} + +delegate_indexed_iterator! { + IterMut<'a, T> => &'a mut T, + impl<'a, T: Send> +} + +/// Collect an arbitrary `Result`-wrapped collection. +/// +/// If any item is `Err`, then all previous `Ok` items collected are +/// discarded, and it returns that error. If there are multiple errors, the +/// one returned is not deterministic. +impl FromParallelIterator> for Result +where + C: FromParallelIterator, + T: Send, + E: Send, +{ + fn from_par_iter(par_iter: I) -> Self + where + I: IntoParallelIterator>, + { + fn ok(saved: &Mutex>) -> impl Fn(Result) -> Option + '_ { + move |item| match item { + Ok(item) => Some(item), + Err(error) => { + // We don't need a blocking `lock()`, as anybody + // else holding the lock will also be writing + // `Some(error)`, and then ours is irrelevant. + if let Ok(mut guard) = saved.try_lock() { + if guard.is_none() { + *guard = Some(error); + } + } + None + } + } + } + + let saved_error = Mutex::new(None); + let collection = par_iter + .into_par_iter() + .map(ok(&saved_error)) + .while_some() + .collect(); + + match saved_error.into_inner().unwrap() { + Some(error) => Err(error), + None => Ok(collection), + } + } +} diff --git a/anneal/v2/vendor/rayon/src/slice/chunk_by.rs b/anneal/v2/vendor/rayon/src/slice/chunk_by.rs new file mode 100644 index 0000000000..d01a3e7b12 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/slice/chunk_by.rs @@ -0,0 +1,239 @@ +use crate::iter::plumbing::*; +use crate::iter::*; +use std::fmt; +use std::marker::PhantomData; + +trait ChunkBySlice: AsRef<[T]> + Default + Send { + fn split(self, index: usize) -> (Self, Self); + fn chunk_by(self, pred: &impl Fn(&T, &T) -> bool) -> impl Iterator; + + fn find(&self, pred: &impl Fn(&T, &T) -> bool, start: usize, end: usize) -> Option { + self.as_ref()[start..end] + .windows(2) + .position(move |w| !pred(&w[0], &w[1])) + .map(|i| i + 1) + } + + fn rfind(&self, pred: &impl Fn(&T, &T) -> bool, end: usize) -> Option { + self.as_ref()[..end] + .windows(2) + .rposition(move |w| !pred(&w[0], &w[1])) + .map(|i| i + 1) + } +} + +impl ChunkBySlice for &[T] { + fn split(self, index: usize) -> (Self, Self) { + self.split_at(index) + } + + fn chunk_by(self, pred: &impl Fn(&T, &T) -> bool) -> impl Iterator { + self.chunk_by(pred) + } +} + +impl ChunkBySlice for &mut [T] { + fn split(self, index: usize) -> (Self, Self) { + self.split_at_mut(index) + } + + fn chunk_by(self, pred: &impl Fn(&T, &T) -> bool) -> impl Iterator { + self.chunk_by_mut(pred) + } +} + +struct ChunkByProducer<'p, T, Slice, Pred> { + slice: Slice, + pred: &'p Pred, + tail: usize, + marker: PhantomData, +} + +// Note: this implementation is very similar to `SplitProducer`. +impl UnindexedProducer for ChunkByProducer<'_, T, Slice, Pred> +where + Slice: ChunkBySlice, + Pred: Fn(&T, &T) -> bool + Send + Sync, +{ + type Item = Slice; + + fn split(self) -> (Self, Option) { + if self.tail < 2 { + return (Self { tail: 0, ..self }, None); + } + + // Look forward for the separator, and failing that look backward. + let mid = self.tail / 2; + let index = match self.slice.find(self.pred, mid, self.tail) { + Some(i) => Some(mid + i), + None => self.slice.rfind(self.pred, mid + 1), + }; + + if let Some(index) = index { + let (left, right) = self.slice.split(index); + + let (left_tail, right_tail) = if index <= mid { + // If we scanned backwards to find the separator, everything in + // the right side is exhausted, with no separators left to find. + (index, 0) + } else { + (mid + 1, self.tail - index) + }; + + // Create the left split before the separator. + let left = Self { + slice: left, + tail: left_tail, + ..self + }; + + // Create the right split following the separator. + let right = Self { + slice: right, + tail: right_tail, + ..self + }; + + (left, Some(right)) + } else { + // The search is exhausted, no more separators... + (Self { tail: 0, ..self }, None) + } + } + + fn fold_with(self, mut folder: F) -> F + where + F: Folder, + { + let Self { + slice, pred, tail, .. + } = self; + + let (slice, tail) = if tail == slice.as_ref().len() { + // No tail section, so just let `consume_iter` do it all. + (Some(slice), None) + } else if let Some(index) = slice.rfind(pred, tail) { + // We found the last separator to complete the tail, so + // end with that slice after `consume_iter` finds the rest. + let (left, right) = slice.split(index); + (Some(left), Some(right)) + } else { + // We know there are no separators at all, so it's all "tail". + (None, Some(slice)) + }; + + if let Some(slice) = slice { + folder = folder.consume_iter(slice.chunk_by(pred)); + } + + if let Some(tail) = tail { + folder = folder.consume(tail); + } + + folder + } +} + +/// Parallel iterator over slice in (non-overlapping) chunks separated by a predicate. +/// +/// This struct is created by the [`par_chunk_by`] method on `&[T]`. +/// +/// [`par_chunk_by`]: super::ParallelSlice::par_chunk_by() +pub struct ChunkBy<'data, T, P> { + pred: P, + slice: &'data [T], +} + +impl Clone for ChunkBy<'_, T, P> { + fn clone(&self) -> Self { + ChunkBy { + pred: self.pred.clone(), + slice: self.slice, + } + } +} + +impl fmt::Debug for ChunkBy<'_, T, P> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ChunkBy") + .field("slice", &self.slice) + .finish() + } +} + +impl<'data, T, P> ChunkBy<'data, T, P> { + pub(super) fn new(slice: &'data [T], pred: P) -> Self { + Self { pred, slice } + } +} + +impl<'data, T, P> ParallelIterator for ChunkBy<'data, T, P> +where + T: Sync, + P: Fn(&T, &T) -> bool + Send + Sync, +{ + type Item = &'data [T]; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + bridge_unindexed( + ChunkByProducer { + tail: self.slice.len(), + slice: self.slice, + pred: &self.pred, + marker: PhantomData, + }, + consumer, + ) + } +} + +/// Parallel iterator over slice in (non-overlapping) mutable chunks +/// separated by a predicate. +/// +/// This struct is created by the [`par_chunk_by_mut`] method on `&mut [T]`. +/// +/// [`par_chunk_by_mut`]: super::ParallelSliceMut::par_chunk_by_mut() +pub struct ChunkByMut<'data, T, P> { + pred: P, + slice: &'data mut [T], +} + +impl fmt::Debug for ChunkByMut<'_, T, P> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ChunkByMut") + .field("slice", &self.slice) + .finish() + } +} + +impl<'data, T, P> ChunkByMut<'data, T, P> { + pub(super) fn new(slice: &'data mut [T], pred: P) -> Self { + Self { pred, slice } + } +} + +impl<'data, T, P> ParallelIterator for ChunkByMut<'data, T, P> +where + T: Send, + P: Fn(&T, &T) -> bool + Send + Sync, +{ + type Item = &'data mut [T]; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + bridge_unindexed( + ChunkByProducer { + tail: self.slice.len(), + slice: self.slice, + pred: &self.pred, + marker: PhantomData, + }, + consumer, + ) + } +} diff --git a/anneal/v2/vendor/rayon/src/slice/chunks.rs b/anneal/v2/vendor/rayon/src/slice/chunks.rs new file mode 100644 index 0000000000..ba678f056c --- /dev/null +++ b/anneal/v2/vendor/rayon/src/slice/chunks.rs @@ -0,0 +1,387 @@ +use crate::iter::plumbing::*; +use crate::iter::*; + +/// Parallel iterator over immutable non-overlapping chunks of a slice +#[derive(Debug)] +pub struct Chunks<'data, T> { + chunk_size: usize, + slice: &'data [T], +} + +impl<'data, T> Chunks<'data, T> { + pub(super) fn new(chunk_size: usize, slice: &'data [T]) -> Self { + Self { chunk_size, slice } + } +} + +impl Clone for Chunks<'_, T> { + fn clone(&self) -> Self { + Chunks { ..*self } + } +} + +impl<'data, T: Sync> ParallelIterator for Chunks<'data, T> { + type Item = &'data [T]; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + bridge(self, consumer) + } + + fn opt_len(&self) -> Option { + Some(self.len()) + } +} + +impl IndexedParallelIterator for Chunks<'_, T> { + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + bridge(self, consumer) + } + + fn len(&self) -> usize { + self.slice.len().div_ceil(self.chunk_size) + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + callback.callback(ChunksProducer { + chunk_size: self.chunk_size, + slice: self.slice, + }) + } +} + +struct ChunksProducer<'data, T: Sync> { + chunk_size: usize, + slice: &'data [T], +} + +impl<'data, T: 'data + Sync> Producer for ChunksProducer<'data, T> { + type Item = &'data [T]; + type IntoIter = ::std::slice::Chunks<'data, T>; + + fn into_iter(self) -> Self::IntoIter { + self.slice.chunks(self.chunk_size) + } + + fn split_at(self, index: usize) -> (Self, Self) { + let elem_index = Ord::min(index * self.chunk_size, self.slice.len()); + let (left, right) = self.slice.split_at(elem_index); + ( + ChunksProducer { + chunk_size: self.chunk_size, + slice: left, + }, + ChunksProducer { + chunk_size: self.chunk_size, + slice: right, + }, + ) + } +} + +/// Parallel iterator over immutable non-overlapping chunks of a slice +#[derive(Debug)] +pub struct ChunksExact<'data, T> { + chunk_size: usize, + slice: &'data [T], + rem: &'data [T], +} + +impl<'data, T> ChunksExact<'data, T> { + pub(super) fn new(chunk_size: usize, slice: &'data [T]) -> Self { + let rem_len = slice.len() % chunk_size; + let len = slice.len() - rem_len; + let (slice, rem) = slice.split_at(len); + Self { + chunk_size, + slice, + rem, + } + } + + /// Return the remainder of the original slice that is not going to be + /// returned by the iterator. The returned slice has at most `chunk_size-1` + /// elements. + pub fn remainder(&self) -> &'data [T] { + self.rem + } +} + +impl Clone for ChunksExact<'_, T> { + fn clone(&self) -> Self { + ChunksExact { ..*self } + } +} + +impl<'data, T: Sync> ParallelIterator for ChunksExact<'data, T> { + type Item = &'data [T]; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + bridge(self, consumer) + } + + fn opt_len(&self) -> Option { + Some(self.len()) + } +} + +impl IndexedParallelIterator for ChunksExact<'_, T> { + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + bridge(self, consumer) + } + + fn len(&self) -> usize { + self.slice.len() / self.chunk_size + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + callback.callback(ChunksExactProducer { + chunk_size: self.chunk_size, + slice: self.slice, + }) + } +} + +struct ChunksExactProducer<'data, T: Sync> { + chunk_size: usize, + slice: &'data [T], +} + +impl<'data, T: 'data + Sync> Producer for ChunksExactProducer<'data, T> { + type Item = &'data [T]; + type IntoIter = ::std::slice::ChunksExact<'data, T>; + + fn into_iter(self) -> Self::IntoIter { + self.slice.chunks_exact(self.chunk_size) + } + + fn split_at(self, index: usize) -> (Self, Self) { + let elem_index = index * self.chunk_size; + let (left, right) = self.slice.split_at(elem_index); + ( + ChunksExactProducer { + chunk_size: self.chunk_size, + slice: left, + }, + ChunksExactProducer { + chunk_size: self.chunk_size, + slice: right, + }, + ) + } +} + +/// Parallel iterator over mutable non-overlapping chunks of a slice +#[derive(Debug)] +pub struct ChunksMut<'data, T> { + chunk_size: usize, + slice: &'data mut [T], +} + +impl<'data, T> ChunksMut<'data, T> { + pub(super) fn new(chunk_size: usize, slice: &'data mut [T]) -> Self { + Self { chunk_size, slice } + } +} + +impl<'data, T: Send> ParallelIterator for ChunksMut<'data, T> { + type Item = &'data mut [T]; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + bridge(self, consumer) + } + + fn opt_len(&self) -> Option { + Some(self.len()) + } +} + +impl IndexedParallelIterator for ChunksMut<'_, T> { + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + bridge(self, consumer) + } + + fn len(&self) -> usize { + self.slice.len().div_ceil(self.chunk_size) + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + callback.callback(ChunksMutProducer { + chunk_size: self.chunk_size, + slice: self.slice, + }) + } +} + +struct ChunksMutProducer<'data, T: Send> { + chunk_size: usize, + slice: &'data mut [T], +} + +impl<'data, T: 'data + Send> Producer for ChunksMutProducer<'data, T> { + type Item = &'data mut [T]; + type IntoIter = ::std::slice::ChunksMut<'data, T>; + + fn into_iter(self) -> Self::IntoIter { + self.slice.chunks_mut(self.chunk_size) + } + + fn split_at(self, index: usize) -> (Self, Self) { + let elem_index = Ord::min(index * self.chunk_size, self.slice.len()); + let (left, right) = self.slice.split_at_mut(elem_index); + ( + ChunksMutProducer { + chunk_size: self.chunk_size, + slice: left, + }, + ChunksMutProducer { + chunk_size: self.chunk_size, + slice: right, + }, + ) + } +} + +/// Parallel iterator over mutable non-overlapping chunks of a slice +#[derive(Debug)] +pub struct ChunksExactMut<'data, T> { + chunk_size: usize, + slice: &'data mut [T], + rem: &'data mut [T], +} + +impl<'data, T> ChunksExactMut<'data, T> { + pub(super) fn new(chunk_size: usize, slice: &'data mut [T]) -> Self { + let rem_len = slice.len() % chunk_size; + let len = slice.len() - rem_len; + let (slice, rem) = slice.split_at_mut(len); + Self { + chunk_size, + slice, + rem, + } + } + + /// Return the remainder of the original slice that is not going to be + /// returned by the iterator. The returned slice has at most `chunk_size-1` + /// elements. + /// + /// Note that this has to consume `self` to return the original lifetime of + /// the data, which prevents this from actually being used as a parallel + /// iterator since that also consumes. This method is provided for parity + /// with `std::iter::ChunksExactMut`, but consider calling `remainder()` or + /// `take_remainder()` as alternatives. + pub fn into_remainder(self) -> &'data mut [T] { + self.rem + } + + /// Return the remainder of the original slice that is not going to be + /// returned by the iterator. The returned slice has at most `chunk_size-1` + /// elements. + /// + /// Consider `take_remainder()` if you need access to the data with its + /// original lifetime, rather than borrowing through `&mut self` here. + pub fn remainder(&mut self) -> &mut [T] { + self.rem + } + + /// Return the remainder of the original slice that is not going to be + /// returned by the iterator. The returned slice has at most `chunk_size-1` + /// elements. Subsequent calls will return an empty slice. + pub fn take_remainder(&mut self) -> &'data mut [T] { + std::mem::take(&mut self.rem) + } +} + +impl<'data, T: Send> ParallelIterator for ChunksExactMut<'data, T> { + type Item = &'data mut [T]; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + bridge(self, consumer) + } + + fn opt_len(&self) -> Option { + Some(self.len()) + } +} + +impl IndexedParallelIterator for ChunksExactMut<'_, T> { + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + bridge(self, consumer) + } + + fn len(&self) -> usize { + self.slice.len() / self.chunk_size + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + callback.callback(ChunksExactMutProducer { + chunk_size: self.chunk_size, + slice: self.slice, + }) + } +} + +struct ChunksExactMutProducer<'data, T: Send> { + chunk_size: usize, + slice: &'data mut [T], +} + +impl<'data, T: 'data + Send> Producer for ChunksExactMutProducer<'data, T> { + type Item = &'data mut [T]; + type IntoIter = ::std::slice::ChunksExactMut<'data, T>; + + fn into_iter(self) -> Self::IntoIter { + self.slice.chunks_exact_mut(self.chunk_size) + } + + fn split_at(self, index: usize) -> (Self, Self) { + let elem_index = index * self.chunk_size; + let (left, right) = self.slice.split_at_mut(elem_index); + ( + ChunksExactMutProducer { + chunk_size: self.chunk_size, + slice: left, + }, + ChunksExactMutProducer { + chunk_size: self.chunk_size, + slice: right, + }, + ) + } +} diff --git a/anneal/v2/vendor/rayon/src/slice/mod.rs b/anneal/v2/vendor/rayon/src/slice/mod.rs new file mode 100644 index 0000000000..b53defad40 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/slice/mod.rs @@ -0,0 +1,1173 @@ +//! Parallel iterator types for [slices] +//! +//! You will rarely need to interact with this module directly unless you need +//! to name one of the iterator types. +//! +//! [slices]: std::slice + +mod chunk_by; +mod chunks; +mod rchunks; +mod sort; +mod windows; + +mod test; + +use self::sort::par_mergesort; +use self::sort::par_quicksort; +use crate::iter::plumbing::*; +use crate::iter::*; +use crate::split_producer::*; + +use std::cmp::Ordering; +use std::fmt::{self, Debug}; + +pub use self::chunk_by::{ChunkBy, ChunkByMut}; +pub use self::chunks::{Chunks, ChunksExact, ChunksExactMut, ChunksMut}; +pub use self::rchunks::{RChunks, RChunksExact, RChunksExactMut, RChunksMut}; +pub use self::windows::{ArrayWindows, Windows}; + +/// Parallel extensions for slices. +pub trait ParallelSlice { + /// Returns a plain slice, which is used to implement the rest of the + /// parallel methods. + fn as_parallel_slice(&self) -> &[T]; + + /// Returns a parallel iterator over subslices separated by elements that + /// match the separator. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// let products: Vec<_> = [1, 2, 3, 0, 2, 4, 8, 0, 3, 6, 9] + /// .par_split(|i| *i == 0) + /// .map(|numbers| numbers.iter().product::()) + /// .collect(); + /// assert_eq!(products, [6, 64, 162]); + /// ``` + fn par_split

(&self, separator: P) -> Split<'_, T, P> + where + P: Fn(&T) -> bool + Sync + Send, + { + Split { + slice: self.as_parallel_slice(), + separator, + } + } + + /// Returns a parallel iterator over subslices separated by elements that + /// match the separator, including the matched part as a terminator. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// let lengths: Vec<_> = [1, 2, 3, 0, 2, 4, 8, 0, 3, 6, 9] + /// .par_split_inclusive(|i| *i == 0) + /// .map(|numbers| numbers.len()) + /// .collect(); + /// assert_eq!(lengths, [4, 4, 3]); + /// ``` + fn par_split_inclusive

(&self, separator: P) -> SplitInclusive<'_, T, P> + where + P: Fn(&T) -> bool + Sync + Send, + { + SplitInclusive { + slice: self.as_parallel_slice(), + separator, + } + } + + /// Returns a parallel iterator over all contiguous windows of length + /// `window_size`. The windows overlap. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// let windows: Vec<_> = [1, 2, 3].par_windows(2).collect(); + /// assert_eq!(vec![[1, 2], [2, 3]], windows); + /// ``` + fn par_windows(&self, window_size: usize) -> Windows<'_, T> { + Windows::new(window_size, self.as_parallel_slice()) + } + + /// Returns a parallel iterator over all contiguous array windows of + /// length `N`. The windows overlap. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// let windows: Vec<_> = [1, 2, 3].par_array_windows().collect(); + /// assert_eq!(vec![&[1, 2], &[2, 3]], windows); + /// ``` + fn par_array_windows(&self) -> ArrayWindows<'_, T, N> { + ArrayWindows::new(self.as_parallel_slice()) + } + + /// Returns a parallel iterator over at most `chunk_size` elements of + /// `self` at a time. The chunks do not overlap. + /// + /// If the number of elements in the iterator is not divisible by + /// `chunk_size`, the last chunk may be shorter than `chunk_size`. All + /// other chunks will have that exact length. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// let chunks: Vec<_> = [1, 2, 3, 4, 5].par_chunks(2).collect(); + /// assert_eq!(chunks, vec![&[1, 2][..], &[3, 4], &[5]]); + /// ``` + #[track_caller] + fn par_chunks(&self, chunk_size: usize) -> Chunks<'_, T> { + assert!(chunk_size != 0, "chunk_size must not be zero"); + Chunks::new(chunk_size, self.as_parallel_slice()) + } + + /// Returns a parallel iterator over `chunk_size` elements of + /// `self` at a time. The chunks do not overlap. + /// + /// If `chunk_size` does not divide the length of the slice, then the + /// last up to `chunk_size-1` elements will be omitted and can be + /// retrieved from the remainder function of the iterator. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// let chunks: Vec<_> = [1, 2, 3, 4, 5].par_chunks_exact(2).collect(); + /// assert_eq!(chunks, vec![&[1, 2][..], &[3, 4]]); + /// ``` + #[track_caller] + fn par_chunks_exact(&self, chunk_size: usize) -> ChunksExact<'_, T> { + assert!(chunk_size != 0, "chunk_size must not be zero"); + ChunksExact::new(chunk_size, self.as_parallel_slice()) + } + + /// Returns a parallel iterator over at most `chunk_size` elements of `self` at a time, + /// starting at the end. The chunks do not overlap. + /// + /// If the number of elements in the iterator is not divisible by + /// `chunk_size`, the last chunk may be shorter than `chunk_size`. All + /// other chunks will have that exact length. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// let chunks: Vec<_> = [1, 2, 3, 4, 5].par_rchunks(2).collect(); + /// assert_eq!(chunks, vec![&[4, 5][..], &[2, 3], &[1]]); + /// ``` + #[track_caller] + fn par_rchunks(&self, chunk_size: usize) -> RChunks<'_, T> { + assert!(chunk_size != 0, "chunk_size must not be zero"); + RChunks::new(chunk_size, self.as_parallel_slice()) + } + + /// Returns a parallel iterator over `chunk_size` elements of `self` at a time, + /// starting at the end. The chunks do not overlap. + /// + /// If `chunk_size` does not divide the length of the slice, then the + /// last up to `chunk_size-1` elements will be omitted and can be + /// retrieved from the remainder function of the iterator. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// let chunks: Vec<_> = [1, 2, 3, 4, 5].par_rchunks_exact(2).collect(); + /// assert_eq!(chunks, vec![&[4, 5][..], &[2, 3]]); + /// ``` + #[track_caller] + fn par_rchunks_exact(&self, chunk_size: usize) -> RChunksExact<'_, T> { + assert!(chunk_size != 0, "chunk_size must not be zero"); + RChunksExact::new(chunk_size, self.as_parallel_slice()) + } + + /// Returns a parallel iterator over the slice producing non-overlapping runs + /// of elements using the predicate to separate them. + /// + /// The predicate is called on two elements following themselves, + /// it means the predicate is called on `slice[0]` and `slice[1]` + /// then on `slice[1]` and `slice[2]` and so on. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// let chunks: Vec<_> = [1, 2, 2, 3, 3, 3].par_chunk_by(|&x, &y| x == y).collect(); + /// assert_eq!(chunks[0], &[1]); + /// assert_eq!(chunks[1], &[2, 2]); + /// assert_eq!(chunks[2], &[3, 3, 3]); + /// ``` + fn par_chunk_by(&self, pred: F) -> ChunkBy<'_, T, F> + where + F: Fn(&T, &T) -> bool + Send + Sync, + { + ChunkBy::new(self.as_parallel_slice(), pred) + } +} + +impl ParallelSlice for [T] { + #[inline] + fn as_parallel_slice(&self) -> &[T] { + self + } +} + +/// Parallel extensions for mutable slices. +pub trait ParallelSliceMut { + /// Returns a plain mutable slice, which is used to implement the rest of + /// the parallel methods. + fn as_parallel_slice_mut(&mut self) -> &mut [T]; + + /// Returns a parallel iterator over mutable subslices separated by + /// elements that match the separator. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// let mut array = [1, 2, 3, 0, 2, 4, 8, 0, 3, 6, 9]; + /// array.par_split_mut(|i| *i == 0) + /// .for_each(|slice| slice.reverse()); + /// assert_eq!(array, [3, 2, 1, 0, 8, 4, 2, 0, 9, 6, 3]); + /// ``` + fn par_split_mut

(&mut self, separator: P) -> SplitMut<'_, T, P> + where + P: Fn(&T) -> bool + Sync + Send, + { + SplitMut { + slice: self.as_parallel_slice_mut(), + separator, + } + } + + /// Returns a parallel iterator over mutable subslices separated by elements + /// that match the separator, including the matched part as a terminator. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// let mut array = [1, 2, 3, 0, 2, 4, 8, 0, 3, 6, 9]; + /// array.par_split_inclusive_mut(|i| *i == 0) + /// .for_each(|slice| slice.reverse()); + /// assert_eq!(array, [0, 3, 2, 1, 0, 8, 4, 2, 9, 6, 3]); + /// ``` + fn par_split_inclusive_mut

(&mut self, separator: P) -> SplitInclusiveMut<'_, T, P> + where + P: Fn(&T) -> bool + Sync + Send, + { + SplitInclusiveMut { + slice: self.as_parallel_slice_mut(), + separator, + } + } + + /// Returns a parallel iterator over at most `chunk_size` elements of + /// `self` at a time. The chunks are mutable and do not overlap. + /// + /// If the number of elements in the iterator is not divisible by + /// `chunk_size`, the last chunk may be shorter than `chunk_size`. All + /// other chunks will have that exact length. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// let mut array = [1, 2, 3, 4, 5]; + /// array.par_chunks_mut(2) + /// .for_each(|slice| slice.reverse()); + /// assert_eq!(array, [2, 1, 4, 3, 5]); + /// ``` + #[track_caller] + fn par_chunks_mut(&mut self, chunk_size: usize) -> ChunksMut<'_, T> { + assert!(chunk_size != 0, "chunk_size must not be zero"); + ChunksMut::new(chunk_size, self.as_parallel_slice_mut()) + } + + /// Returns a parallel iterator over `chunk_size` elements of + /// `self` at a time. The chunks are mutable and do not overlap. + /// + /// If `chunk_size` does not divide the length of the slice, then the + /// last up to `chunk_size-1` elements will be omitted and can be + /// retrieved from the remainder function of the iterator. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// let mut array = [1, 2, 3, 4, 5]; + /// array.par_chunks_exact_mut(3) + /// .for_each(|slice| slice.reverse()); + /// assert_eq!(array, [3, 2, 1, 4, 5]); + /// ``` + #[track_caller] + fn par_chunks_exact_mut(&mut self, chunk_size: usize) -> ChunksExactMut<'_, T> { + assert!(chunk_size != 0, "chunk_size must not be zero"); + ChunksExactMut::new(chunk_size, self.as_parallel_slice_mut()) + } + + /// Returns a parallel iterator over at most `chunk_size` elements of `self` at a time, + /// starting at the end. The chunks are mutable and do not overlap. + /// + /// If the number of elements in the iterator is not divisible by + /// `chunk_size`, the last chunk may be shorter than `chunk_size`. All + /// other chunks will have that exact length. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// let mut array = [1, 2, 3, 4, 5]; + /// array.par_rchunks_mut(2) + /// .for_each(|slice| slice.reverse()); + /// assert_eq!(array, [1, 3, 2, 5, 4]); + /// ``` + #[track_caller] + fn par_rchunks_mut(&mut self, chunk_size: usize) -> RChunksMut<'_, T> { + assert!(chunk_size != 0, "chunk_size must not be zero"); + RChunksMut::new(chunk_size, self.as_parallel_slice_mut()) + } + + /// Returns a parallel iterator over `chunk_size` elements of `self` at a time, + /// starting at the end. The chunks are mutable and do not overlap. + /// + /// If `chunk_size` does not divide the length of the slice, then the + /// last up to `chunk_size-1` elements will be omitted and can be + /// retrieved from the remainder function of the iterator. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// let mut array = [1, 2, 3, 4, 5]; + /// array.par_rchunks_exact_mut(3) + /// .for_each(|slice| slice.reverse()); + /// assert_eq!(array, [1, 2, 5, 4, 3]); + /// ``` + #[track_caller] + fn par_rchunks_exact_mut(&mut self, chunk_size: usize) -> RChunksExactMut<'_, T> { + assert!(chunk_size != 0, "chunk_size must not be zero"); + RChunksExactMut::new(chunk_size, self.as_parallel_slice_mut()) + } + + /// Sorts the slice in parallel. + /// + /// This sort is stable (i.e., does not reorder equal elements) and *O*(*n* \* log(*n*)) worst-case. + /// + /// When applicable, unstable sorting is preferred because it is generally faster than stable + /// sorting and it doesn't allocate auxiliary memory. + /// See [`par_sort_unstable`](#method.par_sort_unstable). + /// + /// # Current implementation + /// + /// The current algorithm is an adaptive merge sort inspired by + /// [timsort](https://en.wikipedia.org/wiki/Timsort). + /// It is designed to be very fast in cases where the slice is nearly sorted, or consists of + /// two or more sorted sequences concatenated one after another. + /// + /// Also, it allocates temporary storage the same size as `self`, but for very short slices a + /// non-allocating insertion sort is used instead. + /// + /// In order to sort the slice in parallel, the slice is first divided into smaller chunks and + /// all chunks are sorted in parallel. Then, adjacent chunks that together form non-descending + /// or descending runs are concatenated. Finally, the remaining chunks are merged together using + /// parallel subdivision of chunks and parallel merge operation. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let mut v = [-5, 4, 1, -3, 2]; + /// + /// v.par_sort(); + /// assert_eq!(v, [-5, -3, 1, 2, 4]); + /// ``` + fn par_sort(&mut self) + where + T: Ord, + { + par_mergesort(self.as_parallel_slice_mut(), T::lt); + } + + /// Sorts the slice in parallel with a comparator function. + /// + /// This sort is stable (i.e., does not reorder equal elements) and *O*(*n* \* log(*n*)) worst-case. + /// + /// The comparator function must define a total ordering for the elements in the slice. If + /// the ordering is not total, the order of the elements is unspecified. An order is a + /// total order if it is (for all `a`, `b` and `c`): + /// + /// * total and antisymmetric: exactly one of `a < b`, `a == b` or `a > b` is true, and + /// * transitive, `a < b` and `b < c` implies `a < c`. The same must hold for both `==` and `>`. + /// + /// For example, while [`f64`] doesn't implement [`Ord`] because `NaN != NaN`, we can use + /// `partial_cmp` as our sort function when we know the slice doesn't contain a `NaN`. + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let mut floats = [5f64, 4.0, 1.0, 3.0, 2.0]; + /// floats.par_sort_by(|a, b| a.partial_cmp(b).unwrap()); + /// assert_eq!(floats, [1.0, 2.0, 3.0, 4.0, 5.0]); + /// ``` + /// + /// When applicable, unstable sorting is preferred because it is generally faster than stable + /// sorting and it doesn't allocate auxiliary memory. + /// See [`par_sort_unstable_by`](#method.par_sort_unstable_by). + /// + /// # Current implementation + /// + /// The current algorithm is an adaptive merge sort inspired by + /// [timsort](https://en.wikipedia.org/wiki/Timsort). + /// It is designed to be very fast in cases where the slice is nearly sorted, or consists of + /// two or more sorted sequences concatenated one after another. + /// + /// Also, it allocates temporary storage the same size as `self`, but for very short slices a + /// non-allocating insertion sort is used instead. + /// + /// In order to sort the slice in parallel, the slice is first divided into smaller chunks and + /// all chunks are sorted in parallel. Then, adjacent chunks that together form non-descending + /// or descending runs are concatenated. Finally, the remaining chunks are merged together using + /// parallel subdivision of chunks and parallel merge operation. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let mut v = [5, 4, 1, 3, 2]; + /// v.par_sort_by(|a, b| a.cmp(b)); + /// assert_eq!(v, [1, 2, 3, 4, 5]); + /// + /// // reverse sorting + /// v.par_sort_by(|a, b| b.cmp(a)); + /// assert_eq!(v, [5, 4, 3, 2, 1]); + /// ``` + fn par_sort_by(&mut self, compare: F) + where + F: Fn(&T, &T) -> Ordering + Sync, + { + par_mergesort(self.as_parallel_slice_mut(), |a, b| { + compare(a, b) == Ordering::Less + }); + } + + /// Sorts the slice in parallel with a key extraction function. + /// + /// This sort is stable (i.e., does not reorder equal elements) and *O*(*m* \* *n* \* log(*n*)) + /// worst-case, where the key function is *O*(*m*). + /// + /// For expensive key functions (e.g. functions that are not simple property accesses or + /// basic operations), [`par_sort_by_cached_key`](#method.par_sort_by_cached_key) is likely to + /// be significantly faster, as it does not recompute element keys. + /// + /// When applicable, unstable sorting is preferred because it is generally faster than stable + /// sorting and it doesn't allocate auxiliary memory. + /// See [`par_sort_unstable_by_key`](#method.par_sort_unstable_by_key). + /// + /// # Current implementation + /// + /// The current algorithm is an adaptive merge sort inspired by + /// [timsort](https://en.wikipedia.org/wiki/Timsort). + /// It is designed to be very fast in cases where the slice is nearly sorted, or consists of + /// two or more sorted sequences concatenated one after another. + /// + /// Also, it allocates temporary storage the same size as `self`, but for very short slices a + /// non-allocating insertion sort is used instead. + /// + /// In order to sort the slice in parallel, the slice is first divided into smaller chunks and + /// all chunks are sorted in parallel. Then, adjacent chunks that together form non-descending + /// or descending runs are concatenated. Finally, the remaining chunks are merged together using + /// parallel subdivision of chunks and parallel merge operation. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let mut v = [-5i32, 4, 1, -3, 2]; + /// + /// v.par_sort_by_key(|k| k.abs()); + /// assert_eq!(v, [1, 2, -3, 4, -5]); + /// ``` + fn par_sort_by_key(&mut self, f: F) + where + K: Ord, + F: Fn(&T) -> K + Sync, + { + par_mergesort(self.as_parallel_slice_mut(), |a, b| f(a).lt(&f(b))); + } + + /// Sorts the slice in parallel with a key extraction function. + /// + /// During sorting, the key function is called at most once per element, by using + /// temporary storage to remember the results of key evaluation. + /// The key function is called in parallel, so the order of calls is completely unspecified. + /// + /// This sort is stable (i.e., does not reorder equal elements) and *O*(*m* \* *n* + *n* \* log(*n*)) + /// worst-case, where the key function is *O*(*m*). + /// + /// For simple key functions (e.g., functions that are property accesses or + /// basic operations), [`par_sort_by_key`](#method.par_sort_by_key) is likely to be + /// faster. + /// + /// # Current implementation + /// + /// The current algorithm is based on [pattern-defeating quicksort][pdqsort] by Orson Peters, + /// which combines the fast average case of randomized quicksort with the fast worst case of + /// heapsort, while achieving linear time on slices with certain patterns. It uses some + /// randomization to avoid degenerate cases, but with a fixed seed to always provide + /// deterministic behavior. + /// + /// In the worst case, the algorithm allocates temporary storage in a `Vec<(K, usize)>` the + /// length of the slice. + /// + /// All quicksorts work in two stages: partitioning into two halves followed by recursive + /// calls. The partitioning phase is sequential, but the two recursive calls are performed in + /// parallel. Finally, after sorting the cached keys, the item positions are updated sequentially. + /// + /// [pdqsort]: https://github.com/orlp/pdqsort + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let mut v = [-5i32, 4, 32, -3, 2]; + /// + /// v.par_sort_by_cached_key(|k| k.to_string()); + /// assert!(v == [-3, -5, 2, 32, 4]); + /// ``` + fn par_sort_by_cached_key(&mut self, f: F) + where + F: Fn(&T) -> K + Sync, + K: Ord + Send, + { + let slice = self.as_parallel_slice_mut(); + let len = slice.len(); + if len < 2 { + return; + } + + // Helper macro for indexing our vector by the smallest possible type, to reduce allocation. + macro_rules! sort_by_key { + ($t:ty) => {{ + let mut indices: Vec<_> = slice + .par_iter_mut() + .enumerate() + .map(|(i, x)| (f(&*x), i as $t)) + .collect(); + // The elements of `indices` are unique, as they are indexed, so any sort will be + // stable with respect to the original slice. We use `sort_unstable` here because + // it requires less memory allocation. + indices.par_sort_unstable(); + for i in 0..len { + let mut index = indices[i].1; + while (index as usize) < i { + index = indices[index as usize].1; + } + indices[i].1 = index; + slice.swap(i, index as usize); + } + }}; + } + + let sz_u8 = size_of::<(K, u8)>(); + let sz_u16 = size_of::<(K, u16)>(); + let sz_u32 = size_of::<(K, u32)>(); + let sz_usize = size_of::<(K, usize)>(); + + if sz_u8 < sz_u16 && len <= (u8::MAX as usize) { + return sort_by_key!(u8); + } + if sz_u16 < sz_u32 && len <= (u16::MAX as usize) { + return sort_by_key!(u16); + } + if sz_u32 < sz_usize && len <= (u32::MAX as usize) { + return sort_by_key!(u32); + } + sort_by_key!(usize) + } + + /// Sorts the slice in parallel, but might not preserve the order of equal elements. + /// + /// This sort is unstable (i.e., may reorder equal elements), in-place + /// (i.e., does not allocate), and *O*(*n* \* log(*n*)) worst-case. + /// + /// # Current implementation + /// + /// The current algorithm is based on [pattern-defeating quicksort][pdqsort] by Orson Peters, + /// which combines the fast average case of randomized quicksort with the fast worst case of + /// heapsort, while achieving linear time on slices with certain patterns. It uses some + /// randomization to avoid degenerate cases, but with a fixed seed to always provide + /// deterministic behavior. + /// + /// It is typically faster than stable sorting, except in a few special cases, e.g., when the + /// slice consists of several concatenated sorted sequences. + /// + /// All quicksorts work in two stages: partitioning into two halves followed by recursive + /// calls. The partitioning phase is sequential, but the two recursive calls are performed in + /// parallel. + /// + /// [pdqsort]: https://github.com/orlp/pdqsort + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let mut v = [-5, 4, 1, -3, 2]; + /// + /// v.par_sort_unstable(); + /// assert_eq!(v, [-5, -3, 1, 2, 4]); + /// ``` + fn par_sort_unstable(&mut self) + where + T: Ord, + { + par_quicksort(self.as_parallel_slice_mut(), T::lt); + } + + /// Sorts the slice in parallel with a comparator function, but might not preserve the order of + /// equal elements. + /// + /// This sort is unstable (i.e., may reorder equal elements), in-place + /// (i.e., does not allocate), and *O*(*n* \* log(*n*)) worst-case. + /// + /// The comparator function must define a total ordering for the elements in the slice. If + /// the ordering is not total, the order of the elements is unspecified. An order is a + /// total order if it is (for all `a`, `b` and `c`): + /// + /// * total and antisymmetric: exactly one of `a < b`, `a == b` or `a > b` is true, and + /// * transitive, `a < b` and `b < c` implies `a < c`. The same must hold for both `==` and `>`. + /// + /// For example, while [`f64`] doesn't implement [`Ord`] because `NaN != NaN`, we can use + /// `partial_cmp` as our sort function when we know the slice doesn't contain a `NaN`. + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let mut floats = [5f64, 4.0, 1.0, 3.0, 2.0]; + /// floats.par_sort_unstable_by(|a, b| a.partial_cmp(b).unwrap()); + /// assert_eq!(floats, [1.0, 2.0, 3.0, 4.0, 5.0]); + /// ``` + /// + /// # Current implementation + /// + /// The current algorithm is based on [pattern-defeating quicksort][pdqsort] by Orson Peters, + /// which combines the fast average case of randomized quicksort with the fast worst case of + /// heapsort, while achieving linear time on slices with certain patterns. It uses some + /// randomization to avoid degenerate cases, but with a fixed seed to always provide + /// deterministic behavior. + /// + /// It is typically faster than stable sorting, except in a few special cases, e.g., when the + /// slice consists of several concatenated sorted sequences. + /// + /// All quicksorts work in two stages: partitioning into two halves followed by recursive + /// calls. The partitioning phase is sequential, but the two recursive calls are performed in + /// parallel. + /// + /// [pdqsort]: https://github.com/orlp/pdqsort + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let mut v = [5, 4, 1, 3, 2]; + /// v.par_sort_unstable_by(|a, b| a.cmp(b)); + /// assert_eq!(v, [1, 2, 3, 4, 5]); + /// + /// // reverse sorting + /// v.par_sort_unstable_by(|a, b| b.cmp(a)); + /// assert_eq!(v, [5, 4, 3, 2, 1]); + /// ``` + fn par_sort_unstable_by(&mut self, compare: F) + where + F: Fn(&T, &T) -> Ordering + Sync, + { + par_quicksort(self.as_parallel_slice_mut(), |a, b| { + compare(a, b) == Ordering::Less + }); + } + + /// Sorts the slice in parallel with a key extraction function, but might not preserve the order + /// of equal elements. + /// + /// This sort is unstable (i.e., may reorder equal elements), in-place + /// (i.e., does not allocate), and *O*(m \* *n* \* log(*n*)) worst-case, + /// where the key function is *O*(*m*). + /// + /// # Current implementation + /// + /// The current algorithm is based on [pattern-defeating quicksort][pdqsort] by Orson Peters, + /// which combines the fast average case of randomized quicksort with the fast worst case of + /// heapsort, while achieving linear time on slices with certain patterns. It uses some + /// randomization to avoid degenerate cases, but with a fixed seed to always provide + /// deterministic behavior. + /// + /// Due to its key calling strategy, `par_sort_unstable_by_key` is likely to be slower than + /// [`par_sort_by_cached_key`](#method.par_sort_by_cached_key) in cases where the key function + /// is expensive. + /// + /// All quicksorts work in two stages: partitioning into two halves followed by recursive + /// calls. The partitioning phase is sequential, but the two recursive calls are performed in + /// parallel. + /// + /// [pdqsort]: https://github.com/orlp/pdqsort + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let mut v = [-5i32, 4, 1, -3, 2]; + /// + /// v.par_sort_unstable_by_key(|k| k.abs()); + /// assert_eq!(v, [1, 2, -3, 4, -5]); + /// ``` + fn par_sort_unstable_by_key(&mut self, f: F) + where + K: Ord, + F: Fn(&T) -> K + Sync, + { + par_quicksort(self.as_parallel_slice_mut(), |a, b| f(a).lt(&f(b))); + } + + /// Returns a parallel iterator over the slice producing non-overlapping mutable + /// runs of elements using the predicate to separate them. + /// + /// The predicate is called on two elements following themselves, + /// it means the predicate is called on `slice[0]` and `slice[1]` + /// then on `slice[1]` and `slice[2]` and so on. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// let mut xs = [1, 2, 2, 3, 3, 3]; + /// let chunks: Vec<_> = xs.par_chunk_by_mut(|&x, &y| x == y).collect(); + /// assert_eq!(chunks[0], &mut [1]); + /// assert_eq!(chunks[1], &mut [2, 2]); + /// assert_eq!(chunks[2], &mut [3, 3, 3]); + /// ``` + fn par_chunk_by_mut(&mut self, pred: F) -> ChunkByMut<'_, T, F> + where + F: Fn(&T, &T) -> bool + Send + Sync, + { + ChunkByMut::new(self.as_parallel_slice_mut(), pred) + } +} + +impl ParallelSliceMut for [T] { + #[inline] + fn as_parallel_slice_mut(&mut self) -> &mut [T] { + self + } +} + +impl<'data, T: Sync> IntoParallelIterator for &'data [T] { + type Item = &'data T; + type Iter = Iter<'data, T>; + + fn into_par_iter(self) -> Self::Iter { + Iter { slice: self } + } +} + +impl<'data, T: Sync> IntoParallelIterator for &'data Box<[T]> { + type Item = &'data T; + type Iter = Iter<'data, T>; + + fn into_par_iter(self) -> Self::Iter { + Iter { slice: self } + } +} + +impl<'data, T: Send> IntoParallelIterator for &'data mut [T] { + type Item = &'data mut T; + type Iter = IterMut<'data, T>; + + fn into_par_iter(self) -> Self::Iter { + IterMut { slice: self } + } +} + +impl<'data, T: Send> IntoParallelIterator for &'data mut Box<[T]> { + type Item = &'data mut T; + type Iter = IterMut<'data, T>; + + fn into_par_iter(self) -> Self::Iter { + IterMut { slice: self } + } +} + +/// Parallel iterator over immutable items in a slice +#[derive(Debug)] +pub struct Iter<'data, T> { + slice: &'data [T], +} + +impl Clone for Iter<'_, T> { + fn clone(&self) -> Self { + Iter { ..*self } + } +} + +impl<'data, T: Sync> ParallelIterator for Iter<'data, T> { + type Item = &'data T; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + bridge(self, consumer) + } + + fn opt_len(&self) -> Option { + Some(self.len()) + } +} + +impl IndexedParallelIterator for Iter<'_, T> { + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + bridge(self, consumer) + } + + fn len(&self) -> usize { + self.slice.len() + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + callback.callback(IterProducer { slice: self.slice }) + } +} + +struct IterProducer<'data, T: Sync> { + slice: &'data [T], +} + +impl<'data, T: 'data + Sync> Producer for IterProducer<'data, T> { + type Item = &'data T; + type IntoIter = ::std::slice::Iter<'data, T>; + + fn into_iter(self) -> Self::IntoIter { + self.slice.iter() + } + + fn split_at(self, index: usize) -> (Self, Self) { + let (left, right) = self.slice.split_at(index); + (IterProducer { slice: left }, IterProducer { slice: right }) + } +} + +/// Parallel iterator over mutable items in a slice +#[derive(Debug)] +pub struct IterMut<'data, T> { + slice: &'data mut [T], +} + +impl<'data, T: Send> ParallelIterator for IterMut<'data, T> { + type Item = &'data mut T; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + bridge(self, consumer) + } + + fn opt_len(&self) -> Option { + Some(self.len()) + } +} + +impl IndexedParallelIterator for IterMut<'_, T> { + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + bridge(self, consumer) + } + + fn len(&self) -> usize { + self.slice.len() + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + callback.callback(IterMutProducer { slice: self.slice }) + } +} + +struct IterMutProducer<'data, T: Send> { + slice: &'data mut [T], +} + +impl<'data, T: 'data + Send> Producer for IterMutProducer<'data, T> { + type Item = &'data mut T; + type IntoIter = ::std::slice::IterMut<'data, T>; + + fn into_iter(self) -> Self::IntoIter { + self.slice.iter_mut() + } + + fn split_at(self, index: usize) -> (Self, Self) { + let (left, right) = self.slice.split_at_mut(index); + ( + IterMutProducer { slice: left }, + IterMutProducer { slice: right }, + ) + } +} + +/// Parallel iterator over slices separated by a predicate +pub struct Split<'data, T, P> { + slice: &'data [T], + separator: P, +} + +impl Clone for Split<'_, T, P> { + fn clone(&self) -> Self { + Split { + separator: self.separator.clone(), + ..*self + } + } +} + +impl Debug for Split<'_, T, P> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Split").field("slice", &self.slice).finish() + } +} + +impl<'data, T, P> ParallelIterator for Split<'data, T, P> +where + P: Fn(&T) -> bool + Sync + Send, + T: Sync, +{ + type Item = &'data [T]; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let producer = SplitProducer::new(self.slice, &self.separator); + bridge_unindexed(producer, consumer) + } +} + +/// Parallel iterator over slices separated by a predicate, +/// including the matched part as a terminator. +pub struct SplitInclusive<'data, T, P> { + slice: &'data [T], + separator: P, +} + +impl Clone for SplitInclusive<'_, T, P> { + fn clone(&self) -> Self { + SplitInclusive { + separator: self.separator.clone(), + ..*self + } + } +} + +impl Debug for SplitInclusive<'_, T, P> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SplitInclusive") + .field("slice", &self.slice) + .finish() + } +} + +impl<'data, T, P> ParallelIterator for SplitInclusive<'data, T, P> +where + P: Fn(&T) -> bool + Sync + Send, + T: Sync, +{ + type Item = &'data [T]; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let producer = SplitInclusiveProducer::new_incl(self.slice, &self.separator); + bridge_unindexed(producer, consumer) + } +} + +/// Implement support for `SplitProducer`. +impl Fissile

for &[T] +where + P: Fn(&T) -> bool, +{ + fn length(&self) -> usize { + self.len() + } + + fn midpoint(&self, end: usize) -> usize { + end / 2 + } + + fn find(&self, separator: &P, start: usize, end: usize) -> Option { + self[start..end].iter().position(separator) + } + + fn rfind(&self, separator: &P, end: usize) -> Option { + self[..end].iter().rposition(separator) + } + + fn split_once(self, index: usize) -> (Self, Self) { + if INCL { + // include the separator in the left side + self.split_at(index + 1) + } else { + let (left, right) = self.split_at(index); + (left, &right[1..]) // skip the separator + } + } + + fn fold_splits(self, separator: &P, folder: F, skip_last: bool) -> F + where + F: Folder, + Self: Send, + { + if INCL { + debug_assert!(!skip_last); + folder.consume_iter(self.split_inclusive(separator)) + } else { + let mut split = self.split(separator); + if skip_last { + split.next_back(); + } + folder.consume_iter(split) + } + } +} + +/// Parallel iterator over mutable slices separated by a predicate +pub struct SplitMut<'data, T, P> { + slice: &'data mut [T], + separator: P, +} + +impl Debug for SplitMut<'_, T, P> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SplitMut") + .field("slice", &self.slice) + .finish() + } +} + +impl<'data, T, P> ParallelIterator for SplitMut<'data, T, P> +where + P: Fn(&T) -> bool + Sync + Send, + T: Send, +{ + type Item = &'data mut [T]; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let producer = SplitProducer::new(self.slice, &self.separator); + bridge_unindexed(producer, consumer) + } +} + +/// Parallel iterator over mutable slices separated by a predicate, +/// including the matched part as a terminator. +pub struct SplitInclusiveMut<'data, T, P> { + slice: &'data mut [T], + separator: P, +} + +impl Debug for SplitInclusiveMut<'_, T, P> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SplitInclusiveMut") + .field("slice", &self.slice) + .finish() + } +} + +impl<'data, T, P> ParallelIterator for SplitInclusiveMut<'data, T, P> +where + P: Fn(&T) -> bool + Sync + Send, + T: Send, +{ + type Item = &'data mut [T]; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let producer = SplitInclusiveProducer::new_incl(self.slice, &self.separator); + bridge_unindexed(producer, consumer) + } +} + +/// Implement support for `SplitProducer`. +impl Fissile

for &mut [T] +where + P: Fn(&T) -> bool, +{ + fn length(&self) -> usize { + self.len() + } + + fn midpoint(&self, end: usize) -> usize { + end / 2 + } + + fn find(&self, separator: &P, start: usize, end: usize) -> Option { + self[start..end].iter().position(separator) + } + + fn rfind(&self, separator: &P, end: usize) -> Option { + self[..end].iter().rposition(separator) + } + + fn split_once(self, index: usize) -> (Self, Self) { + if INCL { + // include the separator in the left side + self.split_at_mut(index + 1) + } else { + let (left, right) = self.split_at_mut(index); + (left, &mut right[1..]) // skip the separator + } + } + + fn fold_splits(self, separator: &P, folder: F, skip_last: bool) -> F + where + F: Folder, + Self: Send, + { + if INCL { + debug_assert!(!skip_last); + folder.consume_iter(self.split_inclusive_mut(separator)) + } else { + let mut split = self.split_mut(separator); + if skip_last { + split.next_back(); + } + folder.consume_iter(split) + } + } +} diff --git a/anneal/v2/vendor/rayon/src/slice/rchunks.rs b/anneal/v2/vendor/rayon/src/slice/rchunks.rs new file mode 100644 index 0000000000..c942e9ffe6 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/slice/rchunks.rs @@ -0,0 +1,385 @@ +use crate::iter::plumbing::*; +use crate::iter::*; + +/// Parallel iterator over immutable non-overlapping chunks of a slice, starting at the end. +#[derive(Debug)] +pub struct RChunks<'data, T> { + chunk_size: usize, + slice: &'data [T], +} + +impl<'data, T> RChunks<'data, T> { + pub(super) fn new(chunk_size: usize, slice: &'data [T]) -> Self { + Self { chunk_size, slice } + } +} + +impl Clone for RChunks<'_, T> { + fn clone(&self) -> Self { + RChunks { ..*self } + } +} + +impl<'data, T: Sync> ParallelIterator for RChunks<'data, T> { + type Item = &'data [T]; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + bridge(self, consumer) + } + + fn opt_len(&self) -> Option { + Some(self.len()) + } +} + +impl IndexedParallelIterator for RChunks<'_, T> { + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + bridge(self, consumer) + } + + fn len(&self) -> usize { + self.slice.len().div_ceil(self.chunk_size) + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + callback.callback(RChunksProducer { + chunk_size: self.chunk_size, + slice: self.slice, + }) + } +} + +struct RChunksProducer<'data, T: Sync> { + chunk_size: usize, + slice: &'data [T], +} + +impl<'data, T: 'data + Sync> Producer for RChunksProducer<'data, T> { + type Item = &'data [T]; + type IntoIter = ::std::slice::RChunks<'data, T>; + + fn into_iter(self) -> Self::IntoIter { + self.slice.rchunks(self.chunk_size) + } + + fn split_at(self, index: usize) -> (Self, Self) { + let elem_index = self.slice.len().saturating_sub(index * self.chunk_size); + let (left, right) = self.slice.split_at(elem_index); + ( + RChunksProducer { + chunk_size: self.chunk_size, + slice: right, + }, + RChunksProducer { + chunk_size: self.chunk_size, + slice: left, + }, + ) + } +} + +/// Parallel iterator over immutable non-overlapping chunks of a slice, starting at the end. +#[derive(Debug)] +pub struct RChunksExact<'data, T> { + chunk_size: usize, + slice: &'data [T], + rem: &'data [T], +} + +impl<'data, T> RChunksExact<'data, T> { + pub(super) fn new(chunk_size: usize, slice: &'data [T]) -> Self { + let rem_len = slice.len() % chunk_size; + let (rem, slice) = slice.split_at(rem_len); + Self { + chunk_size, + slice, + rem, + } + } + + /// Return the remainder of the original slice that is not going to be + /// returned by the iterator. The returned slice has at most `chunk_size-1` + /// elements. + pub fn remainder(&self) -> &'data [T] { + self.rem + } +} + +impl Clone for RChunksExact<'_, T> { + fn clone(&self) -> Self { + RChunksExact { ..*self } + } +} + +impl<'data, T: Sync> ParallelIterator for RChunksExact<'data, T> { + type Item = &'data [T]; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + bridge(self, consumer) + } + + fn opt_len(&self) -> Option { + Some(self.len()) + } +} + +impl IndexedParallelIterator for RChunksExact<'_, T> { + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + bridge(self, consumer) + } + + fn len(&self) -> usize { + self.slice.len() / self.chunk_size + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + callback.callback(RChunksExactProducer { + chunk_size: self.chunk_size, + slice: self.slice, + }) + } +} + +struct RChunksExactProducer<'data, T: Sync> { + chunk_size: usize, + slice: &'data [T], +} + +impl<'data, T: 'data + Sync> Producer for RChunksExactProducer<'data, T> { + type Item = &'data [T]; + type IntoIter = ::std::slice::RChunksExact<'data, T>; + + fn into_iter(self) -> Self::IntoIter { + self.slice.rchunks_exact(self.chunk_size) + } + + fn split_at(self, index: usize) -> (Self, Self) { + let elem_index = self.slice.len() - index * self.chunk_size; + let (left, right) = self.slice.split_at(elem_index); + ( + RChunksExactProducer { + chunk_size: self.chunk_size, + slice: right, + }, + RChunksExactProducer { + chunk_size: self.chunk_size, + slice: left, + }, + ) + } +} + +/// Parallel iterator over mutable non-overlapping chunks of a slice, starting at the end. +#[derive(Debug)] +pub struct RChunksMut<'data, T> { + chunk_size: usize, + slice: &'data mut [T], +} + +impl<'data, T> RChunksMut<'data, T> { + pub(super) fn new(chunk_size: usize, slice: &'data mut [T]) -> Self { + Self { chunk_size, slice } + } +} + +impl<'data, T: Send> ParallelIterator for RChunksMut<'data, T> { + type Item = &'data mut [T]; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + bridge(self, consumer) + } + + fn opt_len(&self) -> Option { + Some(self.len()) + } +} + +impl IndexedParallelIterator for RChunksMut<'_, T> { + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + bridge(self, consumer) + } + + fn len(&self) -> usize { + self.slice.len().div_ceil(self.chunk_size) + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + callback.callback(RChunksMutProducer { + chunk_size: self.chunk_size, + slice: self.slice, + }) + } +} + +struct RChunksMutProducer<'data, T: Send> { + chunk_size: usize, + slice: &'data mut [T], +} + +impl<'data, T: 'data + Send> Producer for RChunksMutProducer<'data, T> { + type Item = &'data mut [T]; + type IntoIter = ::std::slice::RChunksMut<'data, T>; + + fn into_iter(self) -> Self::IntoIter { + self.slice.rchunks_mut(self.chunk_size) + } + + fn split_at(self, index: usize) -> (Self, Self) { + let elem_index = self.slice.len().saturating_sub(index * self.chunk_size); + let (left, right) = self.slice.split_at_mut(elem_index); + ( + RChunksMutProducer { + chunk_size: self.chunk_size, + slice: right, + }, + RChunksMutProducer { + chunk_size: self.chunk_size, + slice: left, + }, + ) + } +} + +/// Parallel iterator over mutable non-overlapping chunks of a slice, starting at the end. +#[derive(Debug)] +pub struct RChunksExactMut<'data, T: Send> { + chunk_size: usize, + slice: &'data mut [T], + rem: &'data mut [T], +} + +impl<'data, T: Send> RChunksExactMut<'data, T> { + pub(super) fn new(chunk_size: usize, slice: &'data mut [T]) -> Self { + let rem_len = slice.len() % chunk_size; + let (rem, slice) = slice.split_at_mut(rem_len); + Self { + chunk_size, + slice, + rem, + } + } + + /// Return the remainder of the original slice that is not going to be + /// returned by the iterator. The returned slice has at most `chunk_size-1` + /// elements. + /// + /// Note that this has to consume `self` to return the original lifetime of + /// the data, which prevents this from actually being used as a parallel + /// iterator since that also consumes. This method is provided for parity + /// with `std::iter::RChunksExactMut`, but consider calling `remainder()` or + /// `take_remainder()` as alternatives. + pub fn into_remainder(self) -> &'data mut [T] { + self.rem + } + + /// Return the remainder of the original slice that is not going to be + /// returned by the iterator. The returned slice has at most `chunk_size-1` + /// elements. + /// + /// Consider `take_remainder()` if you need access to the data with its + /// original lifetime, rather than borrowing through `&mut self` here. + pub fn remainder(&mut self) -> &mut [T] { + self.rem + } + + /// Return the remainder of the original slice that is not going to be + /// returned by the iterator. The returned slice has at most `chunk_size-1` + /// elements. Subsequent calls will return an empty slice. + pub fn take_remainder(&mut self) -> &'data mut [T] { + std::mem::take(&mut self.rem) + } +} + +impl<'data, T: Send + 'data> ParallelIterator for RChunksExactMut<'data, T> { + type Item = &'data mut [T]; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + bridge(self, consumer) + } + + fn opt_len(&self) -> Option { + Some(self.len()) + } +} + +impl<'data, T: Send + 'data> IndexedParallelIterator for RChunksExactMut<'data, T> { + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + bridge(self, consumer) + } + + fn len(&self) -> usize { + self.slice.len() / self.chunk_size + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + callback.callback(RChunksExactMutProducer { + chunk_size: self.chunk_size, + slice: self.slice, + }) + } +} + +struct RChunksExactMutProducer<'data, T: Send> { + chunk_size: usize, + slice: &'data mut [T], +} + +impl<'data, T: 'data + Send> Producer for RChunksExactMutProducer<'data, T> { + type Item = &'data mut [T]; + type IntoIter = ::std::slice::RChunksExactMut<'data, T>; + + fn into_iter(self) -> Self::IntoIter { + self.slice.rchunks_exact_mut(self.chunk_size) + } + + fn split_at(self, index: usize) -> (Self, Self) { + let elem_index = self.slice.len() - index * self.chunk_size; + let (left, right) = self.slice.split_at_mut(elem_index); + ( + RChunksExactMutProducer { + chunk_size: self.chunk_size, + slice: right, + }, + RChunksExactMutProducer { + chunk_size: self.chunk_size, + slice: left, + }, + ) + } +} diff --git a/anneal/v2/vendor/rayon/src/slice/sort.rs b/anneal/v2/vendor/rayon/src/slice/sort.rs new file mode 100644 index 0000000000..ff3a3fde35 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/slice/sort.rs @@ -0,0 +1,1686 @@ +//! **Parallel** Slice sorting +//! +//! This implementation is mostly copied from the `core::slice::sort` module, with minimal changes +//! to support stable Rust and parallel `is_less` (e.g. `Fn` rather than `FnMut`). +//! +//! --- +//! +//! This module contains a sorting algorithm based on Orson Peters' pattern-defeating quicksort, +//! published at: +//! +//! Unstable sorting is compatible with core because it doesn't allocate memory, unlike our +//! stable sorting implementation. +//! +//! In addition it also contains the core logic of the stable sort used by `slice::sort` based on +//! TimSort. + +use core::cmp; +use core::mem::{self, MaybeUninit}; +use core::ptr; +use core::slice; + +use crate::iter::{IndexedParallelIterator, ParallelIterator}; +use crate::slice::ParallelSliceMut; +use crate::SendPtr; + +// When dropped, copies from `src` into `dest`. +struct InsertionHole { + src: *const T, + dest: *mut T, +} + +impl Drop for InsertionHole { + fn drop(&mut self) { + // SAFETY: This is a helper class. Please refer to its usage for correctness. Namely, one + // must be sure that `src` and `dst` does not overlap as required by + // `ptr::copy_nonoverlapping` and are both valid for writes. + unsafe { + ptr::copy_nonoverlapping(self.src, self.dest, 1); + } + } +} + +/// Inserts `v[v.len() - 1]` into pre-sorted sequence `v[..v.len() - 1]` so that whole `v[..]` +/// becomes sorted. +unsafe fn insert_tail(v: &mut [T], is_less: &F) +where + F: Fn(&T, &T) -> bool, +{ + debug_assert!(v.len() >= 2); + + let arr_ptr = v.as_mut_ptr(); + let i = v.len() - 1; + + // SAFETY: caller must ensure v is at least len 2. + unsafe { + // See insert_head which talks about why this approach is beneficial. + let i_ptr = arr_ptr.add(i); + + // It's important that we use i_ptr here. If this check is positive and we continue, + // We want to make sure that no other copy of the value was seen by is_less. + // Otherwise we would have to copy it back. + if is_less(&*i_ptr, &*i_ptr.sub(1)) { + // It's important, that we use tmp for comparison from now on. As it is the value that + // will be copied back. And notionally we could have created a divergence if we copy + // back the wrong value. + let tmp = mem::ManuallyDrop::new(ptr::read(i_ptr)); + // Intermediate state of the insertion process is always tracked by `hole`, which + // serves two purposes: + // 1. Protects integrity of `v` from panics in `is_less`. + // 2. Fills the remaining hole in `v` in the end. + // + // Panic safety: + // + // If `is_less` panics at any point during the process, `hole` will get dropped and + // fill the hole in `v` with `tmp`, thus ensuring that `v` still holds every object it + // initially held exactly once. + let mut hole = InsertionHole { + src: &*tmp, + dest: i_ptr.sub(1), + }; + ptr::copy_nonoverlapping(hole.dest, i_ptr, 1); + + // SAFETY: We know i is at least 1. + for j in (0..(i - 1)).rev() { + let j_ptr = arr_ptr.add(j); + if !is_less(&*tmp, &*j_ptr) { + break; + } + + ptr::copy_nonoverlapping(j_ptr, hole.dest, 1); + hole.dest = j_ptr; + } + // `hole` gets dropped and thus copies `tmp` into the remaining hole in `v`. + } + } +} + +/// Inserts `v[0]` into pre-sorted sequence `v[1..]` so that whole `v[..]` becomes sorted. +/// +/// This is the integral subroutine of insertion sort. +unsafe fn insert_head(v: &mut [T], is_less: &F) +where + F: Fn(&T, &T) -> bool, +{ + debug_assert!(v.len() >= 2); + + // SAFETY: caller must ensure v is at least len 2. + unsafe { + if is_less(v.get_unchecked(1), v.get_unchecked(0)) { + let arr_ptr = v.as_mut_ptr(); + + // There are three ways to implement insertion here: + // + // 1. Swap adjacent elements until the first one gets to its final destination. + // However, this way we copy data around more than is necessary. If elements are big + // structures (costly to copy), this method will be slow. + // + // 2. Iterate until the right place for the first element is found. Then shift the + // elements succeeding it to make room for it and finally place it into the + // remaining hole. This is a good method. + // + // 3. Copy the first element into a temporary variable. Iterate until the right place + // for it is found. As we go along, copy every traversed element into the slot + // preceding it. Finally, copy data from the temporary variable into the remaining + // hole. This method is very good. Benchmarks demonstrated slightly better + // performance than with the 2nd method. + // + // All methods were benchmarked, and the 3rd showed best results. So we chose that one. + let tmp = mem::ManuallyDrop::new(ptr::read(arr_ptr)); + + // Intermediate state of the insertion process is always tracked by `hole`, which + // serves two purposes: + // 1. Protects integrity of `v` from panics in `is_less`. + // 2. Fills the remaining hole in `v` in the end. + // + // Panic safety: + // + // If `is_less` panics at any point during the process, `hole` will get dropped and + // fill the hole in `v` with `tmp`, thus ensuring that `v` still holds every object it + // initially held exactly once. + let mut hole = InsertionHole { + src: &*tmp, + dest: arr_ptr.add(1), + }; + ptr::copy_nonoverlapping(arr_ptr.add(1), arr_ptr.add(0), 1); + + for i in 2..v.len() { + if !is_less(v.get_unchecked(i), &*tmp) { + break; + } + ptr::copy_nonoverlapping(arr_ptr.add(i), arr_ptr.add(i - 1), 1); + hole.dest = arr_ptr.add(i); + } + // `hole` gets dropped and thus copies `tmp` into the remaining hole in `v`. + } + } +} + +/// Sort `v` assuming `v[..offset]` is already sorted. +/// +/// Never inline this function to avoid code bloat. It still optimizes nicely and has practically no +/// performance impact. Even improving performance in some cases. +#[inline(never)] +fn insertion_sort_shift_left(v: &mut [T], offset: usize, is_less: &F) +where + F: Fn(&T, &T) -> bool, +{ + let len = v.len(); + + // Using assert here improves performance. + assert!(offset != 0 && offset <= len); + + // Shift each element of the unsorted region v[i..] as far left as is needed to make v sorted. + for i in offset..len { + // SAFETY: we tested that `offset` must be at least 1, so this loop is only entered if len + // >= 2. The range is exclusive and we know `i` must be at least 1 so this slice has at + // >least len 2. + unsafe { + insert_tail(&mut v[..=i], is_less); + } + } +} + +/// Sort `v` assuming `v[offset..]` is already sorted. +/// +/// Never inline this function to avoid code bloat. It still optimizes nicely and has practically no +/// performance impact. Even improving performance in some cases. +#[inline(never)] +fn insertion_sort_shift_right(v: &mut [T], offset: usize, is_less: &F) +where + F: Fn(&T, &T) -> bool, +{ + let len = v.len(); + + // Using assert here improves performance. + assert!(offset != 0 && offset <= len && len >= 2); + + // Shift each element of the unsorted region v[..i] as far left as is needed to make v sorted. + for i in (0..offset).rev() { + // SAFETY: we tested that `offset` must be at least 1, so this loop is only entered if len + // >= 2.We ensured that the slice length is always at least 2 long. We know that start_found + // will be at least one less than end, and the range is exclusive. Which gives us i always + // <= (end - 2). + unsafe { + insert_head(&mut v[i..len], is_less); + } + } +} + +/// Partially sorts a slice by shifting several out-of-order elements around. +/// +/// Returns `true` if the slice is sorted at the end. This function is *O*(*n*) worst-case. +#[cold] +fn partial_insertion_sort(v: &mut [T], is_less: &F) -> bool +where + F: Fn(&T, &T) -> bool, +{ + // Maximum number of adjacent out-of-order pairs that will get shifted. + const MAX_STEPS: usize = 5; + // If the slice is shorter than this, don't shift any elements. + const SHORTEST_SHIFTING: usize = 50; + + let len = v.len(); + let mut i = 1; + + for _ in 0..MAX_STEPS { + // SAFETY: We already explicitly did the bound checking with `i < len`. + // All our subsequent indexing is only in the range `0 <= index < len` + unsafe { + // Find the next pair of adjacent out-of-order elements. + while i < len && !is_less(v.get_unchecked(i), v.get_unchecked(i - 1)) { + i += 1; + } + } + + // Are we done? + if i == len { + return true; + } + + // Don't shift elements on short arrays, that has a performance cost. + if len < SHORTEST_SHIFTING { + return false; + } + + // Swap the found pair of elements. This puts them in correct order. + v.swap(i - 1, i); + + if i >= 2 { + // Shift the smaller element to the left. + insertion_sort_shift_left(&mut v[..i], i - 1, is_less); + + // Shift the greater element to the right. + insertion_sort_shift_right(&mut v[..i], 1, is_less); + } + } + + // Didn't manage to sort the slice in the limited number of steps. + false +} + +/// Sorts `v` using heapsort, which guarantees *O*(*n* \* log(*n*)) worst-case. +#[cold] +fn heapsort(v: &mut [T], is_less: F) +where + F: Fn(&T, &T) -> bool, +{ + // This binary heap respects the invariant `parent >= child`. + let sift_down = |v: &mut [T], mut node| { + loop { + // Children of `node`. + let mut child = 2 * node + 1; + if child >= v.len() { + break; + } + + // Choose the greater child. + if child + 1 < v.len() { + // We need a branch to be sure not to out-of-bounds index, + // but it's highly predictable. The comparison, however, + // is better done branchless, especially for primitives. + child += is_less(&v[child], &v[child + 1]) as usize; + } + + // Stop if the invariant holds at `node`. + if !is_less(&v[node], &v[child]) { + break; + } + + // Swap `node` with the greater child, move one step down, and continue sifting. + v.swap(node, child); + node = child; + } + }; + + // Build the heap in linear time. + for i in (0..v.len() / 2).rev() { + sift_down(v, i); + } + + // Pop maximal elements from the heap. + for i in (1..v.len()).rev() { + v.swap(0, i); + sift_down(&mut v[..i], 0); + } +} + +/// Partitions `v` into elements smaller than `pivot`, followed by elements greater than or equal +/// to `pivot`. +/// +/// Returns the number of elements smaller than `pivot`. +/// +/// Partitioning is performed block-by-block in order to minimize the cost of branching operations. +/// This idea is presented in the [BlockQuicksort][pdf] paper. +/// +/// [pdf]: https://drops.dagstuhl.de/opus/volltexte/2016/6389/pdf/LIPIcs-ESA-2016-38.pdf +fn partition_in_blocks(v: &mut [T], pivot: &T, is_less: &F) -> usize +where + F: Fn(&T, &T) -> bool, +{ + // Number of elements in a typical block. + const BLOCK: usize = 128; + + // The partitioning algorithm repeats the following steps until completion: + // + // 1. Trace a block from the left side to identify elements greater than or equal to the pivot. + // 2. Trace a block from the right side to identify elements smaller than the pivot. + // 3. Exchange the identified elements between the left and right side. + // + // We keep the following variables for a block of elements: + // + // 1. `block` - Number of elements in the block. + // 2. `start` - Start pointer into the `offsets` array. + // 3. `end` - End pointer into the `offsets` array. + // 4. `offsets` - Indices of out-of-order elements within the block. + + // The current block on the left side (from `l` to `l.add(block_l)`). + let mut l = v.as_mut_ptr(); + let mut block_l = BLOCK; + let mut start_l = ptr::null_mut(); + let mut end_l = ptr::null_mut(); + let mut offsets_l = [MaybeUninit::::uninit(); BLOCK]; + + // The current block on the right side (from `r.sub(block_r)` to `r`). + // SAFETY: The documentation for .add() specifically mention that `vec.as_ptr().add(vec.len())` is always safe + let mut r = unsafe { l.add(v.len()) }; + let mut block_r = BLOCK; + let mut start_r = ptr::null_mut(); + let mut end_r = ptr::null_mut(); + let mut offsets_r = [MaybeUninit::::uninit(); BLOCK]; + + // FIXME: When we get VLAs, try creating one array of length `min(v.len(), 2 * BLOCK)` rather + // than two fixed-size arrays of length `BLOCK`. VLAs might be more cache-efficient. + + // Returns the number of elements between pointers `l` (inclusive) and `r` (exclusive). + fn width(l: *mut T, r: *mut T) -> usize { + assert!(size_of::() > 0); + // FIXME: this should *likely* use `offset_from`, but more + // investigation is needed (including running tests in miri). + (r as usize - l as usize) / size_of::() + } + + loop { + // We are done with partitioning block-by-block when `l` and `r` get very close. Then we do + // some patch-up work in order to partition the remaining elements in between. + let is_done = width(l, r) <= 2 * BLOCK; + + if is_done { + // Number of remaining elements (still not compared to the pivot). + let mut rem = width(l, r); + if start_l < end_l || start_r < end_r { + rem -= BLOCK; + } + + // Adjust block sizes so that the left and right block don't overlap, but get perfectly + // aligned to cover the whole remaining gap. + if start_l < end_l { + block_r = rem; + } else if start_r < end_r { + block_l = rem; + } else { + // There were the same number of elements to switch on both blocks during the last + // iteration, so there are no remaining elements on either block. Cover the remaining + // items with roughly equally-sized blocks. + block_l = rem / 2; + block_r = rem - block_l; + } + debug_assert!(block_l <= BLOCK && block_r <= BLOCK); + debug_assert!(width(l, r) == block_l + block_r); + } + + if start_l == end_l { + // Trace `block_l` elements from the left side. + start_l = offsets_l.as_mut_ptr() as *mut u8; + end_l = start_l; + let mut elem = l; + + for i in 0..block_l { + // SAFETY: The unsafety operations below involve the usage of the `offset`. + // According to the conditions required by the function, we satisfy them because: + // 1. `offsets_l` is stack-allocated, and thus considered separate allocated object. + // 2. The function `is_less` returns a `bool`. + // Casting a `bool` will never overflow `isize`. + // 3. We have guaranteed that `block_l` will be `<= BLOCK`. + // Plus, `end_l` was initially set to the begin pointer of `offsets_` which was declared on the stack. + // Thus, we know that even in the worst case (all invocations of `is_less` returns false) we will only be at most 1 byte pass the end. + // Another unsafety operation here is dereferencing `elem`. + // However, `elem` was initially the begin pointer to the slice which is always valid. + unsafe { + // Branchless comparison. + *end_l = i as u8; + end_l = end_l.add(!is_less(&*elem, pivot) as usize); + elem = elem.add(1); + } + } + } + + if start_r == end_r { + // Trace `block_r` elements from the right side. + start_r = offsets_r.as_mut_ptr() as *mut u8; + end_r = start_r; + let mut elem = r; + + for i in 0..block_r { + // SAFETY: The unsafety operations below involve the usage of the `offset`. + // According to the conditions required by the function, we satisfy them because: + // 1. `offsets_r` is stack-allocated, and thus considered separate allocated object. + // 2. The function `is_less` returns a `bool`. + // Casting a `bool` will never overflow `isize`. + // 3. We have guaranteed that `block_r` will be `<= BLOCK`. + // Plus, `end_r` was initially set to the begin pointer of `offsets_` which was declared on the stack. + // Thus, we know that even in the worst case (all invocations of `is_less` returns true) we will only be at most 1 byte pass the end. + // Another unsafety operation here is dereferencing `elem`. + // However, `elem` was initially `1 * sizeof(T)` past the end and we decrement it by `1 * sizeof(T)` before accessing it. + // Plus, `block_r` was asserted to be less than `BLOCK` and `elem` will therefore at most be pointing to the beginning of the slice. + unsafe { + // Branchless comparison. + elem = elem.sub(1); + *end_r = i as u8; + end_r = end_r.add(is_less(&*elem, pivot) as usize); + } + } + } + + // Number of out-of-order elements to swap between the left and right side. + let count = cmp::min(width(start_l, end_l), width(start_r, end_r)); + + if count > 0 { + macro_rules! left { + () => { + l.add(usize::from(*start_l)) + }; + } + macro_rules! right { + () => { + r.sub(usize::from(*start_r) + 1) + }; + } + + // Instead of swapping one pair at the time, it is more efficient to perform a cyclic + // permutation. This is not strictly equivalent to swapping, but produces a similar + // result using fewer memory operations. + + // SAFETY: The use of `ptr::read` is valid because there is at least one element in + // both `offsets_l` and `offsets_r`, so `left!` is a valid pointer to read from. + // + // The uses of `left!` involve calls to `offset` on `l`, which points to the + // beginning of `v`. All the offsets pointed-to by `start_l` are at most `block_l`, so + // these `offset` calls are safe as all reads are within the block. The same argument + // applies for the uses of `right!`. + // + // The calls to `start_l.offset` are valid because there are at most `count-1` of them, + // plus the final one at the end of the unsafe block, where `count` is the minimum number + // of collected offsets in `offsets_l` and `offsets_r`, so there is no risk of there not + // being enough elements. The same reasoning applies to the calls to `start_r.offset`. + // + // The calls to `copy_nonoverlapping` are safe because `left!` and `right!` are guaranteed + // not to overlap, and are valid because of the reasoning above. + unsafe { + let tmp = ptr::read(left!()); + ptr::copy_nonoverlapping(right!(), left!(), 1); + + for _ in 1..count { + start_l = start_l.add(1); + ptr::copy_nonoverlapping(left!(), right!(), 1); + start_r = start_r.add(1); + ptr::copy_nonoverlapping(right!(), left!(), 1); + } + + ptr::copy_nonoverlapping(&tmp, right!(), 1); + mem::forget(tmp); + start_l = start_l.add(1); + start_r = start_r.add(1); + } + } + + if start_l == end_l { + // All out-of-order elements in the left block were moved. Move to the next block. + + // block-width-guarantee + // SAFETY: if `!is_done` then the slice width is guaranteed to be at least `2*BLOCK` wide. There + // are at most `BLOCK` elements in `offsets_l` because of its size, so the `offset` operation is + // safe. Otherwise, the debug assertions in the `is_done` case guarantee that + // `width(l, r) == block_l + block_r`, namely, that the block sizes have been adjusted to account + // for the smaller number of remaining elements. + l = unsafe { l.add(block_l) }; + } + + if start_r == end_r { + // All out-of-order elements in the right block were moved. Move to the previous block. + + // SAFETY: Same argument as [block-width-guarantee]. Either this is a full block `2*BLOCK`-wide, + // or `block_r` has been adjusted for the last handful of elements. + r = unsafe { r.sub(block_r) }; + } + + if is_done { + break; + } + } + + // All that remains now is at most one block (either the left or the right) with out-of-order + // elements that need to be moved. Such remaining elements can be simply shifted to the end + // within their block. + + if start_l < end_l { + // The left block remains. + // Move its remaining out-of-order elements to the far right. + debug_assert_eq!(width(l, r), block_l); + while start_l < end_l { + // remaining-elements-safety + // SAFETY: while the loop condition holds there are still elements in `offsets_l`, so it + // is safe to point `end_l` to the previous element. + // + // The `ptr::swap` is safe if both its arguments are valid for reads and writes: + // - Per the debug assert above, the distance between `l` and `r` is `block_l` + // elements, so there can be at most `block_l` remaining offsets between `start_l` + // and `end_l`. This means `r` will be moved at most `block_l` steps back, which + // makes the `r.offset` calls valid (at that point `l == r`). + // - `offsets_l` contains valid offsets into `v` collected during the partitioning of + // the last block, so the `l.offset` calls are valid. + unsafe { + end_l = end_l.sub(1); + ptr::swap(l.add(usize::from(*end_l)), r.sub(1)); + r = r.sub(1); + } + } + width(v.as_mut_ptr(), r) + } else if start_r < end_r { + // The right block remains. + // Move its remaining out-of-order elements to the far left. + debug_assert_eq!(width(l, r), block_r); + while start_r < end_r { + // SAFETY: See the reasoning in [remaining-elements-safety]. + unsafe { + end_r = end_r.sub(1); + ptr::swap(l, r.sub(usize::from(*end_r) + 1)); + l = l.add(1); + } + } + width(v.as_mut_ptr(), l) + } else { + // Nothing else to do, we're done. + width(v.as_mut_ptr(), l) + } +} + +/// Partitions `v` into elements smaller than `v[pivot]`, followed by elements greater than or +/// equal to `v[pivot]`. +/// +/// Returns a tuple of: +/// +/// 1. Number of elements smaller than `v[pivot]`. +/// 2. True if `v` was already partitioned. +fn partition(v: &mut [T], pivot: usize, is_less: &F) -> (usize, bool) +where + F: Fn(&T, &T) -> bool, +{ + let (mid, was_partitioned) = { + // Place the pivot at the beginning of slice. + v.swap(0, pivot); + let (pivot, v) = v.split_at_mut(1); + let pivot = &mut pivot[0]; + + // Read the pivot into a stack-allocated variable for efficiency. If a following comparison + // operation panics, the pivot will be automatically written back into the slice. + + // SAFETY: `pivot` is a reference to the first element of `v`, so `ptr::read` is safe. + let tmp = mem::ManuallyDrop::new(unsafe { ptr::read(pivot) }); + let _pivot_guard = InsertionHole { + src: &*tmp, + dest: pivot, + }; + let pivot = &*tmp; + + // Find the first pair of out-of-order elements. + let mut l = 0; + let mut r = v.len(); + + // SAFETY: The unsafety below involves indexing an array. + // For the first one: We already do the bounds checking here with `l < r`. + // For the second one: We initially have `l == 0` and `r == v.len()` and we checked that `l < r` at every indexing operation. + // From here we know that `r` must be at least `r == l` which was shown to be valid from the first one. + unsafe { + // Find the first element greater than or equal to the pivot. + while l < r && is_less(v.get_unchecked(l), pivot) { + l += 1; + } + + // Find the last element smaller that the pivot. + while l < r && !is_less(v.get_unchecked(r - 1), pivot) { + r -= 1; + } + } + + ( + l + partition_in_blocks(&mut v[l..r], pivot, is_less), + l >= r, + ) + + // `_pivot_guard` goes out of scope and writes the pivot (which is a stack-allocated + // variable) back into the slice where it originally was. This step is critical in ensuring + // safety! + }; + + // Place the pivot between the two partitions. + v.swap(0, mid); + + (mid, was_partitioned) +} + +/// Partitions `v` into elements equal to `v[pivot]` followed by elements greater than `v[pivot]`. +/// +/// Returns the number of elements equal to the pivot. It is assumed that `v` does not contain +/// elements smaller than the pivot. +fn partition_equal(v: &mut [T], pivot: usize, is_less: &F) -> usize +where + F: Fn(&T, &T) -> bool, +{ + // Place the pivot at the beginning of slice. + v.swap(0, pivot); + let (pivot, v) = v.split_at_mut(1); + let pivot = &mut pivot[0]; + + // Read the pivot into a stack-allocated variable for efficiency. If a following comparison + // operation panics, the pivot will be automatically written back into the slice. + // SAFETY: The pointer here is valid because it is obtained from a reference to a slice. + let tmp = mem::ManuallyDrop::new(unsafe { ptr::read(pivot) }); + let _pivot_guard = InsertionHole { + src: &*tmp, + dest: pivot, + }; + let pivot = &*tmp; + + let len = v.len(); + if len == 0 { + return 0; + } + + // Now partition the slice. + let mut l = 0; + let mut r = len; + loop { + // SAFETY: The unsafety below involves indexing an array. + // For the first one: We already do the bounds checking here with `l < r`. + // For the second one: We initially have `l == 0` and `r == v.len()` and we checked that `l < r` at every indexing operation. + // From here we know that `r` must be at least `r == l` which was shown to be valid from the first one. + unsafe { + // Find the first element greater than the pivot. + while l < r && !is_less(pivot, v.get_unchecked(l)) { + l += 1; + } + + // Find the last element equal to the pivot. + loop { + r -= 1; + if l >= r || !is_less(pivot, v.get_unchecked(r)) { + break; + } + } + + // Are we done? + if l >= r { + break; + } + + // Swap the found pair of out-of-order elements. + let ptr = v.as_mut_ptr(); + ptr::swap(ptr.add(l), ptr.add(r)); + l += 1; + } + } + + // We found `l` elements equal to the pivot. Add 1 to account for the pivot itself. + l + 1 + + // `_pivot_guard` goes out of scope and writes the pivot (which is a stack-allocated variable) + // back into the slice where it originally was. This step is critical in ensuring safety! +} + +/// Scatters some elements around in an attempt to break patterns that might cause imbalanced +/// partitions in quicksort. +#[cold] +fn break_patterns(v: &mut [T]) { + let len = v.len(); + if len >= 8 { + let mut seed = len; + let mut gen_usize = || { + // Pseudorandom number generator from the "Xorshift RNGs" paper by George Marsaglia. + if usize::BITS <= 32 { + let mut r = seed as u32; + r ^= r << 13; + r ^= r >> 17; + r ^= r << 5; + seed = r as usize; + seed + } else { + let mut r = seed as u64; + r ^= r << 13; + r ^= r >> 7; + r ^= r << 17; + seed = r as usize; + seed + } + }; + + // Take random numbers modulo this number. + // The number fits into `usize` because `len` is not greater than `isize::MAX`. + let modulus = len.next_power_of_two(); + + // Some pivot candidates will be in the nearby of this index. Let's randomize them. + let pos = len / 4 * 2; + + for i in 0..3 { + // Generate a random number modulo `len`. However, in order to avoid costly operations + // we first take it modulo a power of two, and then decrease by `len` until it fits + // into the range `[0, len - 1]`. + let mut other = gen_usize() & (modulus - 1); + + // `other` is guaranteed to be less than `2 * len`. + if other >= len { + other -= len; + } + + v.swap(pos - 1 + i, other); + } + } +} + +/// Chooses a pivot in `v` and returns the index and `true` if the slice is likely already sorted. +/// +/// Elements in `v` might be reordered in the process. +fn choose_pivot(v: &mut [T], is_less: &F) -> (usize, bool) +where + F: Fn(&T, &T) -> bool, +{ + // Minimum length to choose the median-of-medians method. + // Shorter slices use the simple median-of-three method. + const SHORTEST_MEDIAN_OF_MEDIANS: usize = 50; + // Maximum number of swaps that can be performed in this function. + const MAX_SWAPS: usize = 4 * 3; + + let len = v.len(); + + // Three indices near which we are going to choose a pivot. + #[allow(clippy::identity_op)] + let mut a = len / 4 * 1; + let mut b = len / 4 * 2; + let mut c = len / 4 * 3; + + // Counts the total number of swaps we are about to perform while sorting indices. + let mut swaps = 0; + + if len >= 8 { + // Swaps indices so that `v[a] <= v[b]`. + // SAFETY: `len >= 8` so there are at least two elements in the neighborhoods of + // `a`, `b` and `c`. This means the three calls to `sort_adjacent` result in + // corresponding calls to `sort3` with valid 3-item neighborhoods around each + // pointer, which in turn means the calls to `sort2` are done with valid + // references. Thus the `v.get_unchecked` calls are safe, as is the `ptr::swap` + // call. + let mut sort2 = |a: &mut usize, b: &mut usize| unsafe { + if is_less(v.get_unchecked(*b), v.get_unchecked(*a)) { + ptr::swap(a, b); + swaps += 1; + } + }; + + // Swaps indices so that `v[a] <= v[b] <= v[c]`. + let mut sort3 = |a: &mut usize, b: &mut usize, c: &mut usize| { + sort2(a, b); + sort2(b, c); + sort2(a, b); + }; + + if len >= SHORTEST_MEDIAN_OF_MEDIANS { + // Finds the median of `v[a - 1], v[a], v[a + 1]` and stores the index into `a`. + let mut sort_adjacent = |a: &mut usize| { + let tmp = *a; + sort3(&mut (tmp - 1), a, &mut (tmp + 1)); + }; + + // Find medians in the neighborhoods of `a`, `b`, and `c`. + sort_adjacent(&mut a); + sort_adjacent(&mut b); + sort_adjacent(&mut c); + } + + // Find the median among `a`, `b`, and `c`. + sort3(&mut a, &mut b, &mut c); + } + + if swaps < MAX_SWAPS { + (b, swaps == 0) + } else { + // The maximum number of swaps was performed. Chances are the slice is descending or mostly + // descending, so reversing will probably help sort it faster. + v.reverse(); + (len - 1 - b, true) + } +} + +/// Sorts `v` recursively. +/// +/// If the slice had a predecessor in the original array, it is specified as `pred`. +/// +/// `limit` is the number of allowed imbalanced partitions before switching to `heapsort`. If zero, +/// this function will immediately switch to heapsort. +fn recurse<'a, T, F>(mut v: &'a mut [T], is_less: &F, mut pred: Option<&'a mut T>, mut limit: u32) +where + T: Send, + F: Fn(&T, &T) -> bool + Sync, +{ + // Slices of up to this length get sorted using insertion sort. + const MAX_INSERTION: usize = 20; + + // If both partitions are up to this length, we continue sequentially. This number is as small + // as possible but so that the overhead of Rayon's task scheduling is still negligible. + const MAX_SEQUENTIAL: usize = 2000; + + // True if the last partitioning was reasonably balanced. + let mut was_balanced = true; + // True if the last partitioning didn't shuffle elements (the slice was already partitioned). + let mut was_partitioned = true; + + loop { + let len = v.len(); + + // Very short slices get sorted using insertion sort. + if len <= MAX_INSERTION { + if len >= 2 { + insertion_sort_shift_left(v, 1, is_less); + } + return; + } + + // If too many bad pivot choices were made, simply fall back to heapsort in order to + // guarantee `O(n * log(n))` worst-case. + if limit == 0 { + heapsort(v, is_less); + return; + } + + // If the last partitioning was imbalanced, try breaking patterns in the slice by shuffling + // some elements around. Hopefully we'll choose a better pivot this time. + if !was_balanced { + break_patterns(v); + limit -= 1; + } + + // Choose a pivot and try guessing whether the slice is already sorted. + let (pivot, likely_sorted) = choose_pivot(v, is_less); + + // If the last partitioning was decently balanced and didn't shuffle elements, and if pivot + // selection predicts the slice is likely already sorted... + if was_balanced && was_partitioned && likely_sorted { + // Try identifying several out-of-order elements and shifting them to correct + // positions. If the slice ends up being completely sorted, we're done. + if partial_insertion_sort(v, is_less) { + return; + } + } + + // If the chosen pivot is equal to the predecessor, then it's the smallest element in the + // slice. Partition the slice into elements equal to and elements greater than the pivot. + // This case is usually hit when the slice contains many duplicate elements. + if let Some(&mut ref p) = pred { + if !is_less(p, &v[pivot]) { + let mid = partition_equal(v, pivot, is_less); + + // Continue sorting elements greater than the pivot. + v = &mut v[mid..]; + continue; + } + } + + // Partition the slice. + let (mid, was_p) = partition(v, pivot, is_less); + was_balanced = cmp::min(mid, len - mid) >= len / 8; + was_partitioned = was_p; + + // Split the slice into `left`, `pivot`, and `right`. + let (left, right) = v.split_at_mut(mid); + let (pivot, right) = right.split_at_mut(1); + let pivot = &mut pivot[0]; + + if Ord::max(left.len(), right.len()) <= MAX_SEQUENTIAL { + // Recurse into the shorter side only in order to minimize the total number of recursive + // calls and consume less stack space. Then just continue with the longer side (this is + // akin to tail recursion). + if left.len() < right.len() { + recurse(left, is_less, pred, limit); + v = right; + pred = Some(pivot); + } else { + recurse(right, is_less, Some(pivot), limit); + v = left; + } + } else { + // Sort the left and right half in parallel. + rayon_core::join( + || recurse(left, is_less, pred, limit), + || recurse(right, is_less, Some(pivot), limit), + ); + break; + } + } +} + +/// Sorts `v` using pattern-defeating quicksort in parallel. +/// +/// The algorithm is unstable, in-place, and *O*(*n* \* log(*n*)) worst-case. +pub(super) fn par_quicksort(v: &mut [T], is_less: F) +where + T: Send, + F: Fn(&T, &T) -> bool + Sync, +{ + // Sorting has no meaningful behavior on zero-sized types. + if size_of::() == 0 { + return; + } + + // Limit the number of imbalanced partitions to `floor(log2(len)) + 1`. + let limit = usize::BITS - v.len().leading_zeros(); + + recurse(v, &is_less, None, limit); +} + +/// Merges non-decreasing runs `v[..mid]` and `v[mid..]` using `buf` as temporary storage, and +/// stores the result into `v[..]`. +/// +/// # Safety +/// +/// The two slices must be non-empty and `mid` must be in bounds. Buffer `buf` must be long enough +/// to hold a copy of the shorter slice. Also, `T` must not be a zero-sized type. +unsafe fn merge(v: &mut [T], mid: usize, buf: *mut T, is_less: &F) +where + F: Fn(&T, &T) -> bool, +{ + let len = v.len(); + let v = v.as_mut_ptr(); + + // SAFETY: mid and len must be in-bounds of v. + let (v_mid, v_end) = unsafe { (v.add(mid), v.add(len)) }; + + // The merge process first copies the shorter run into `buf`. Then it traces the newly copied + // run and the longer run forwards (or backwards), comparing their next unconsumed elements and + // copying the lesser (or greater) one into `v`. + // + // As soon as the shorter run is fully consumed, the process is done. If the longer run gets + // consumed first, then we must copy whatever is left of the shorter run into the remaining + // hole in `v`. + // + // Intermediate state of the process is always tracked by `hole`, which serves two purposes: + // 1. Protects integrity of `v` from panics in `is_less`. + // 2. Fills the remaining hole in `v` if the longer run gets consumed first. + // + // Panic safety: + // + // If `is_less` panics at any point during the process, `hole` will get dropped and fill the + // hole in `v` with the unconsumed range in `buf`, thus ensuring that `v` still holds every + // object it initially held exactly once. + let mut hole; + + if mid <= len - mid { + // The left run is shorter. + + // SAFETY: buf must have enough capacity for `v[..mid]`. + unsafe { + ptr::copy_nonoverlapping(v, buf, mid); + hole = MergeHole { + start: buf, + end: buf.add(mid), + dest: v, + }; + } + + // Initially, these pointers point to the beginnings of their arrays. + let left = &mut hole.start; + let mut right = v_mid; + let out = &mut hole.dest; + + while *left < hole.end && right < v_end { + // Consume the lesser side. + // If equal, prefer the left run to maintain stability. + + // SAFETY: left and right must be valid and part of v same for out. + unsafe { + let is_l = is_less(&*right, &**left); + let to_copy = if is_l { right } else { *left }; + ptr::copy_nonoverlapping(to_copy, *out, 1); + *out = out.add(1); + right = right.add(is_l as usize); + *left = left.add(!is_l as usize); + } + } + } else { + // The right run is shorter. + + // SAFETY: buf must have enough capacity for `v[mid..]`. + unsafe { + ptr::copy_nonoverlapping(v_mid, buf, len - mid); + hole = MergeHole { + start: buf, + end: buf.add(len - mid), + dest: v_mid, + }; + } + + // Initially, these pointers point past the ends of their arrays. + let left = &mut hole.dest; + let right = &mut hole.end; + let mut out = v_end; + + while v < *left && buf < *right { + // Consume the greater side. + // If equal, prefer the right run to maintain stability. + + // SAFETY: left and right must be valid and part of v same for out. + unsafe { + let is_l = is_less(&*right.sub(1), &*left.sub(1)); + *left = left.sub(is_l as usize); + *right = right.sub(!is_l as usize); + let to_copy = if is_l { *left } else { *right }; + out = out.sub(1); + ptr::copy_nonoverlapping(to_copy, out, 1); + } + } + } + // Finally, `hole` gets dropped. If the shorter run was not fully consumed, whatever remains of + // it will now be copied into the hole in `v`. +} + +// When dropped, copies the range `start..end` into `dest..`. +struct MergeHole { + start: *mut T, + end: *mut T, + dest: *mut T, +} + +impl Drop for MergeHole { + fn drop(&mut self) { + // SAFETY: `T` is not a zero-sized type, and these are pointers into a slice's elements. + unsafe { + let len = self.end.offset_from(self.start) as usize; + ptr::copy_nonoverlapping(self.start, self.dest, len); + } + } +} + +/// The result of merge sort. +#[must_use] +#[derive(Clone, Copy, PartialEq, Eq)] +enum MergeSortResult { + /// The slice has already been sorted. + NonDescending, + /// The slice has been descending and therefore it was left intact. + Descending, + /// The slice was sorted. + Sorted, +} + +/// This merge sort borrows some (but not all) ideas from TimSort, which used to be described in +/// detail [here](https://github.com/python/cpython/blob/main/Objects/listsort.txt). However Python +/// has switched to a Powersort based implementation. +/// +/// The algorithm identifies strictly descending and non-descending subsequences, which are called +/// natural runs. There is a stack of pending runs yet to be merged. Each newly found run is pushed +/// onto the stack, and then some pairs of adjacent runs are merged until these two invariants are +/// satisfied: +/// +/// 1. for every `i` in `1..runs.len()`: `runs[i - 1].len > runs[i].len` +/// 2. for every `i` in `2..runs.len()`: `runs[i - 2].len > runs[i - 1].len + runs[i].len` +/// +/// The invariants ensure that the total running time is *O*(*n* \* log(*n*)) worst-case. +/// +/// # Safety +/// +/// The argument `buf` is used as a temporary buffer and must hold at least `v.len() / 2`. +unsafe fn merge_sort(v: &mut [T], buf_ptr: *mut T, is_less: &CmpF) -> MergeSortResult +where + CmpF: Fn(&T, &T) -> bool, +{ + // The caller should have already checked that. + debug_assert_ne!(size_of::(), 0); + + let len = v.len(); + + let mut runs = Vec::new(); + + let mut end = 0; + let mut start = 0; + + // Scan forward. Memory pre-fetching prefers forward scanning vs backwards scanning, and the + // code-gen is usually better. For the most sensitive types such as integers, these are merged + // bidirectionally at once. So there is no benefit in scanning backwards. + while end < len { + let (streak_end, was_reversed) = find_streak(&v[start..], is_less); + end += streak_end; + if start == 0 && end == len { + return if was_reversed { + MergeSortResult::Descending + } else { + MergeSortResult::NonDescending + }; + } + if was_reversed { + v[start..end].reverse(); + } + + // Insert some more elements into the run if it's too short. Insertion sort is faster than + // merge sort on short sequences, so this significantly improves performance. + end = provide_sorted_batch(v, start, end, is_less); + + // Push this run onto the stack. + runs.push(TimSortRun { + start, + len: end - start, + }); + start = end; + + // Merge some pairs of adjacent runs to satisfy the invariants. + while let Some(r) = collapse(runs.as_slice(), len) { + let left = runs[r]; + let right = runs[r + 1]; + let merge_slice = &mut v[left.start..right.start + right.len]; + // SAFETY: `buf_ptr` must hold enough capacity for the shorter of the two sides, and + // neither side may be on length 0. + unsafe { + merge(merge_slice, left.len, buf_ptr, is_less); + } + runs[r + 1] = TimSortRun { + start: left.start, + len: left.len + right.len, + }; + runs.remove(r); + } + } + + // Finally, exactly one run must remain in the stack. + debug_assert!(runs.len() == 1 && runs[0].start == 0 && runs[0].len == len); + + // The original order of the slice was neither non-descending nor descending. + return MergeSortResult::Sorted; + + // Examines the stack of runs and identifies the next pair of runs to merge. More specifically, + // if `Some(r)` is returned, that means `runs[r]` and `runs[r + 1]` must be merged next. If the + // algorithm should continue building a new run instead, `None` is returned. + // + // TimSort is infamous for its buggy implementations, as described here: + // http://envisage-project.eu/timsort-specification-and-verification/ + // + // The gist of the story is: we must enforce the invariants on the top four runs on the stack. + // Enforcing them on just top three is not sufficient to ensure that the invariants will still + // hold for *all* runs in the stack. + // + // This function correctly checks invariants for the top four runs. Additionally, if the top + // run starts at index 0, it will always demand a merge operation until the stack is fully + // collapsed, in order to complete the sort. + #[inline] + fn collapse(runs: &[TimSortRun], stop: usize) -> Option { + let n = runs.len(); + if n >= 2 + && (runs[n - 1].start + runs[n - 1].len == stop + || runs[n - 2].len <= runs[n - 1].len + || (n >= 3 && runs[n - 3].len <= runs[n - 2].len + runs[n - 1].len) + || (n >= 4 && runs[n - 4].len <= runs[n - 3].len + runs[n - 2].len)) + { + if n >= 3 && runs[n - 3].len < runs[n - 1].len { + Some(n - 3) + } else { + Some(n - 2) + } + } else { + None + } + } +} + +/// Internal type used by merge_sort. +#[derive(Clone, Copy, Debug)] +struct TimSortRun { + len: usize, + start: usize, +} + +/// Takes a range as denoted by start and end, that is already sorted and extends it to the right if +/// necessary with sorts optimized for smaller ranges such as insertion sort. +fn provide_sorted_batch(v: &mut [T], start: usize, mut end: usize, is_less: &F) -> usize +where + F: Fn(&T, &T) -> bool, +{ + let len = v.len(); + assert!(end >= start && end <= len); + + // This value is a balance between least comparisons and best performance, as + // influenced by for example cache locality. + const MIN_INSERTION_RUN: usize = 10; + + // Insert some more elements into the run if it's too short. Insertion sort is faster than + // merge sort on short sequences, so this significantly improves performance. + let start_end_diff = end - start; + + if start_end_diff < MIN_INSERTION_RUN && end < len { + // v[start_found..end] are elements that are already sorted in the input. We want to extend + // the sorted region to the left, so we push up MIN_INSERTION_RUN - 1 to the right. Which is + // more efficient that trying to push those already sorted elements to the left. + end = cmp::min(start + MIN_INSERTION_RUN, len); + let presorted_start = cmp::max(start_end_diff, 1); + + insertion_sort_shift_left(&mut v[start..end], presorted_start, is_less); + } + + end +} + +/// Finds a streak of presorted elements starting at the beginning of the slice. Returns the first +/// value that is not part of said streak, and a bool denoting whether the streak was reversed. +/// Streaks can be increasing or decreasing. +fn find_streak(v: &[T], is_less: &F) -> (usize, bool) +where + F: Fn(&T, &T) -> bool, +{ + let len = v.len(); + + if len < 2 { + return (len, false); + } + + let mut end = 2; + + // SAFETY: See below specific. + unsafe { + // SAFETY: We checked that len >= 2, so 0 and 1 are valid indices. + let assume_reverse = is_less(v.get_unchecked(1), v.get_unchecked(0)); + + // SAFETY: We know end >= 2 and check end < len. + // From that follows that accessing v at end and end - 1 is safe. + if assume_reverse { + while end < len && is_less(v.get_unchecked(end), v.get_unchecked(end - 1)) { + end += 1; + } + + (end, true) + } else { + while end < len && !is_less(v.get_unchecked(end), v.get_unchecked(end - 1)) { + end += 1; + } + (end, false) + } + } +} + +//////////////////////////////////////////////////////////////////////////// +// Everything above this line is copied from `core::slice::sort` (with very minor tweaks). +// Everything below this line is custom parallelization for rayon. +//////////////////////////////////////////////////////////////////////////// + +/// Splits two sorted slices so that they can be merged in parallel. +/// +/// Returns two indices `(a, b)` so that slices `left[..a]` and `right[..b]` come before +/// `left[a..]` and `right[b..]`. +fn split_for_merge(left: &[T], right: &[T], is_less: &F) -> (usize, usize) +where + F: Fn(&T, &T) -> bool, +{ + let left_len = left.len(); + let right_len = right.len(); + + if left_len >= right_len { + let left_mid = left_len / 2; + + // Find the first element in `right` that is greater than or equal to `left[left_mid]`. + let mut a = 0; + let mut b = right_len; + while a < b { + let m = a + (b - a) / 2; + if is_less(&right[m], &left[left_mid]) { + a = m + 1; + } else { + b = m; + } + } + + (left_mid, a) + } else { + let right_mid = right_len / 2; + + // Find the first element in `left` that is greater than `right[right_mid]`. + let mut a = 0; + let mut b = left_len; + while a < b { + let m = a + (b - a) / 2; + if is_less(&right[right_mid], &left[m]) { + b = m; + } else { + a = m + 1; + } + } + + (a, right_mid) + } +} + +/// Merges slices `left` and `right` in parallel and stores the result into `dest`. +/// +/// # Safety +/// +/// The `dest` pointer must have enough space to store the result. +/// +/// Even if `is_less` panics at any point during the merge process, this function will fully copy +/// all elements from `left` and `right` into `dest` (not necessarily in sorted order). +unsafe fn par_merge(left: &mut [T], right: &mut [T], dest: *mut T, is_less: &F) +where + T: Send, + F: Fn(&T, &T) -> bool + Sync, +{ + // Slices whose lengths sum up to this value are merged sequentially. This number is slightly + // larger than `CHUNK_LENGTH`, and the reason is that merging is faster than merge sorting, so + // merging needs a bit coarser granularity in order to hide the overhead of Rayon's task + // scheduling. + const MAX_SEQUENTIAL: usize = 5000; + + let left_len = left.len(); + let right_len = right.len(); + + // Intermediate state of the merge process, which serves two purposes: + // 1. Protects integrity of `dest` from panics in `is_less`. + // 2. Copies the remaining elements as soon as one of the two sides is exhausted. + // + // Panic safety: + // + // If `is_less` panics at any point during the merge process, `s` will get dropped and copy the + // remaining parts of `left` and `right` into `dest`. + let mut s = State { + left_start: left.as_mut_ptr(), + left_end: left.as_mut_ptr().add(left_len), + right_start: right.as_mut_ptr(), + right_end: right.as_mut_ptr().add(right_len), + dest, + }; + + if left_len == 0 || right_len == 0 || left_len + right_len < MAX_SEQUENTIAL { + while s.left_start < s.left_end && s.right_start < s.right_end { + // Consume the lesser side. + // If equal, prefer the left run to maintain stability. + let is_l = is_less(&*s.right_start, &*s.left_start); + let to_copy = if is_l { s.right_start } else { s.left_start }; + ptr::copy_nonoverlapping(to_copy, s.dest, 1); + s.dest = s.dest.add(1); + s.right_start = s.right_start.add(is_l as usize); + s.left_start = s.left_start.add(!is_l as usize); + } + } else { + // Function `split_for_merge` might panic. If that happens, `s` will get destructed and copy + // the whole `left` and `right` into `dest`. + let (left_mid, right_mid) = split_for_merge(left, right, is_less); + let (left_l, left_r) = left.split_at_mut(left_mid); + let (right_l, right_r) = right.split_at_mut(right_mid); + + // Prevent the destructor of `s` from running. Rayon will ensure that both calls to + // `par_merge` happen. If one of the two calls panics, they will ensure that elements still + // get copied into `dest_left` and `dest_right``. + mem::forget(s); + + // Wrap pointers in SendPtr so that they can be sent to another thread + // See the documentation of SendPtr for a full explanation + let dest_l = SendPtr(dest); + let dest_r = SendPtr(dest.add(left_l.len() + right_l.len())); + rayon_core::join( + move || par_merge(left_l, right_l, dest_l.get(), is_less), + move || par_merge(left_r, right_r, dest_r.get(), is_less), + ); + } + // Finally, `s` gets dropped if we used sequential merge, thus copying the remaining elements + // all at once. + + // When dropped, copies arrays `left_start..left_end` and `right_start..right_end` into `dest`, + // in that order. + struct State { + left_start: *mut T, + left_end: *mut T, + right_start: *mut T, + right_end: *mut T, + dest: *mut T, + } + + impl Drop for State { + fn drop(&mut self) { + // Copy array `left`, followed by `right`. + unsafe { + let left_len = self.left_end.offset_from(self.left_start) as usize; + ptr::copy_nonoverlapping(self.left_start, self.dest, left_len); + self.dest = self.dest.add(left_len); + + let right_len = self.right_end.offset_from(self.right_start) as usize; + ptr::copy_nonoverlapping(self.right_start, self.dest, right_len); + } + } + } +} + +/// Recursively merges pre-sorted chunks inside `v`. +/// +/// Chunks of `v` are stored in `chunks` as intervals (inclusive left and exclusive right bound). +/// Argument `buf` is an auxiliary buffer that will be used during the procedure. +/// If `into_buf` is true, the result will be stored into `buf`, otherwise it will be in `v`. +/// +/// # Safety +/// +/// The number of chunks must be positive and they must be adjacent: the right bound of each chunk +/// must equal the left bound of the following chunk. +/// +/// The buffer must be at least as long as `v`. +unsafe fn merge_recurse( + v: *mut T, + buf: *mut T, + chunks: &[(usize, usize)], + into_buf: bool, + is_less: &F, +) where + T: Send, + F: Fn(&T, &T) -> bool + Sync, +{ + let len = chunks.len(); + debug_assert!(len > 0); + + // Base case of the algorithm. + // If only one chunk is remaining, there's no more work to split and merge. + if len == 1 { + if into_buf { + // Copy the chunk from `v` into `buf`. + let (start, end) = chunks[0]; + let src = v.add(start); + let dest = buf.add(start); + ptr::copy_nonoverlapping(src, dest, end - start); + } + return; + } + + // Split the chunks into two halves. + let (start, _) = chunks[0]; + let (mid, _) = chunks[len / 2]; + let (_, end) = chunks[len - 1]; + let (left, right) = chunks.split_at(len / 2); + + // After recursive calls finish we'll have to merge chunks `(start, mid)` and `(mid, end)` from + // `src` into `dest`. If the current invocation has to store the result into `buf`, we'll + // merge chunks from `v` into `buf`, and vice versa. + // + // Recursive calls flip `into_buf` at each level of recursion. More concretely, `par_merge` + // merges chunks from `buf` into `v` at the first level, from `v` into `buf` at the second + // level etc. + let (src, dest) = if into_buf { (v, buf) } else { (buf, v) }; + + // Panic safety: + // + // If `is_less` panics at any point during the recursive calls, the destructor of `guard` will + // be executed, thus copying everything from `src` into `dest`. This way we ensure that all + // chunks are in fact copied into `dest`, even if the merge process doesn't finish. + let guard = MergeHole { + start: src.add(start), + end: src.add(end), + dest: dest.add(start), + }; + + // Wrap pointers in SendPtr so that they can be sent to another thread + // See the documentation of SendPtr for a full explanation + let v = SendPtr(v); + let buf = SendPtr(buf); + rayon_core::join( + move || merge_recurse(v.get(), buf.get(), left, !into_buf, is_less), + move || merge_recurse(v.get(), buf.get(), right, !into_buf, is_less), + ); + + // Everything went all right - recursive calls didn't panic. + // Forget the guard in order to prevent its destructor from running. + mem::forget(guard); + + // Merge chunks `(start, mid)` and `(mid, end)` from `src` into `dest`. + let src_left = slice::from_raw_parts_mut(src.add(start), mid - start); + let src_right = slice::from_raw_parts_mut(src.add(mid), end - mid); + par_merge(src_left, src_right, dest.add(start), is_less); +} + +/// Sorts `v` using merge sort in parallel. +/// +/// The algorithm is stable, allocates memory, and `O(n log n)` worst-case. +/// The allocated temporary buffer is of the same length as is `v`. +pub(super) fn par_mergesort(v: &mut [T], is_less: F) +where + T: Send, + F: Fn(&T, &T) -> bool + Sync, +{ + // Slices of up to this length get sorted using insertion sort in order to avoid the cost of + // buffer allocation. + const MAX_INSERTION: usize = 20; + + // The length of initial chunks. This number is as small as possible but so that the overhead + // of Rayon's task scheduling is still negligible. + const CHUNK_LENGTH: usize = 2000; + + // Sorting has no meaningful behavior on zero-sized types. + if size_of::() == 0 { + return; + } + + let len = v.len(); + + // Short slices get sorted in-place via insertion sort to avoid allocations. + if len <= MAX_INSERTION { + if len >= 2 { + insertion_sort_shift_left(v, 1, &is_less); + } + return; + } + + // Allocate a buffer to use as scratch memory. We keep the length 0 so we can keep in it + // shallow copies of the contents of `v` without risking the dtors running on copies if + // `is_less` panics. + let mut buf = Vec::::with_capacity(len); + let buf = buf.as_mut_ptr(); + + // If the slice is not longer than one chunk would be, do sequential merge sort and return. + if len <= CHUNK_LENGTH { + let res = unsafe { merge_sort(v, buf, &is_less) }; + if res == MergeSortResult::Descending { + v.reverse(); + } + return; + } + + // Split the slice into chunks and merge sort them in parallel. + // However, descending chunks will not be sorted - they will be simply left intact. + let mut iter = { + // Wrap pointer in SendPtr so that it can be sent to another thread + // See the documentation of SendPtr for a full explanation + let buf = SendPtr(buf); + let is_less = &is_less; + + v.par_chunks_mut(CHUNK_LENGTH) + .with_max_len(1) + .enumerate() + .map(move |(i, chunk)| { + let l = CHUNK_LENGTH * i; + let r = l + chunk.len(); + unsafe { + let buf = buf.get().add(l); + (l, r, merge_sort(chunk, buf, is_less)) + } + }) + .collect::>() + .into_iter() + .peekable() + }; + + // Now attempt to concatenate adjacent chunks that were left intact. + let mut chunks = Vec::with_capacity(iter.len()); + + while let Some((a, mut b, res)) = iter.next() { + // If this chunk was not modified by the sort procedure... + if res != MergeSortResult::Sorted { + while let Some(&(x, y, r)) = iter.peek() { + // If the following chunk is of the same type and can be concatenated... + if r == res && (r == MergeSortResult::Descending) == is_less(&v[x], &v[x - 1]) { + // Concatenate them. + b = y; + iter.next(); + } else { + break; + } + } + } + + // Descending chunks must be reversed. + if res == MergeSortResult::Descending { + v[a..b].reverse(); + } + + chunks.push((a, b)); + } + + // All chunks are properly sorted. + // Now we just have to merge them together. + unsafe { + merge_recurse(v.as_mut_ptr(), buf, &chunks, false, &is_less); + } +} + +#[cfg(test)] +mod tests { + use super::heapsort; + use super::split_for_merge; + use rand::distr::Uniform; + use rand::{rng, Rng}; + + #[test] + fn test_heapsort() { + let rng = &mut rng(); + + for len in (0..25).chain(500..501) { + for &modulus in &[5, 10, 100] { + let dist = Uniform::new(0, modulus).unwrap(); + for _ in 0..100 { + let v: Vec = rng.sample_iter(&dist).take(len).collect(); + + // Test heapsort using `<` operator. + let mut tmp = v.clone(); + heapsort(&mut tmp, |a, b| a < b); + assert!(tmp.windows(2).all(|w| w[0] <= w[1])); + + // Test heapsort using `>` operator. + let mut tmp = v.clone(); + heapsort(&mut tmp, |a, b| a > b); + assert!(tmp.windows(2).all(|w| w[0] >= w[1])); + } + } + } + + // Sort using a completely random comparison function. + // This will reorder the elements *somehow*, but won't panic. + let mut v: Vec<_> = (0..100).collect(); + heapsort(&mut v, |_, _| rand::rng().random()); + heapsort(&mut v, |a, b| a < b); + + for (i, &entry) in v.iter().enumerate() { + assert_eq!(entry, i); + } + } + + #[test] + fn test_split_for_merge() { + fn check(left: &[u32], right: &[u32]) { + let (l, r) = split_for_merge(left, right, &|&a, &b| a < b); + assert!(left[..l] + .iter() + .all(|&x| right[r..].iter().all(|&y| x <= y))); + assert!(right[..r].iter().all(|&x| left[l..].iter().all(|&y| x < y))); + } + + check(&[1, 2, 2, 2, 2, 3], &[1, 2, 2, 2, 2, 3]); + check(&[1, 2, 2, 2, 2, 3], &[]); + check(&[], &[1, 2, 2, 2, 2, 3]); + + let rng = &mut rng(); + + for _ in 0..100 { + let limit: u32 = rng.random_range(1..21); + let left_len: usize = rng.random_range(0..20); + let right_len: usize = rng.random_range(0..20); + + let mut left = rng + .sample_iter(&Uniform::new(0, limit).unwrap()) + .take(left_len) + .collect::>(); + let mut right = rng + .sample_iter(&Uniform::new(0, limit).unwrap()) + .take(right_len) + .collect::>(); + + left.sort(); + right.sort(); + check(&left, &right); + } + } +} diff --git a/anneal/v2/vendor/rayon/src/slice/test.rs b/anneal/v2/vendor/rayon/src/slice/test.rs new file mode 100644 index 0000000000..627df8f96b --- /dev/null +++ b/anneal/v2/vendor/rayon/src/slice/test.rs @@ -0,0 +1,216 @@ +#![cfg(test)] + +use crate::prelude::*; +use rand::distr::Uniform; +use rand::seq::IndexedRandom; +use rand::{rng, Rng}; +use std::cmp::Ordering::{Equal, Greater, Less}; +use std::sync::atomic::{AtomicUsize, Ordering::Relaxed}; + +macro_rules! sort { + ($f:ident, $name:ident) => { + #[test] + fn $name() { + let rng = &mut rng(); + + for len in (0..25).chain(500..501) { + for &modulus in &[5, 10, 100] { + let dist = Uniform::new(0, modulus).unwrap(); + for _ in 0..100 { + let v: Vec = rng.sample_iter(&dist).take(len).collect(); + + // Test sort using `<` operator. + let mut tmp = v.clone(); + tmp.$f(|a, b| a.cmp(b)); + assert!(tmp.windows(2).all(|w| w[0] <= w[1])); + + // Test sort using `>` operator. + let mut tmp = v.clone(); + tmp.$f(|a, b| b.cmp(a)); + assert!(tmp.windows(2).all(|w| w[0] >= w[1])); + } + } + } + + // Test sort with many duplicates. + for &len in &[1_000, 10_000, 100_000] { + for &modulus in &[5, 10, 100, 10_000] { + let dist = Uniform::new(0, modulus).unwrap(); + let mut v: Vec = rng.sample_iter(&dist).take(len).collect(); + + v.$f(|a, b| a.cmp(b)); + assert!(v.windows(2).all(|w| w[0] <= w[1])); + } + } + + // Test sort with many pre-sorted runs. + for &len in &[1_000, 10_000, 100_000] { + let len_dist = Uniform::new(0, len).unwrap(); + for &modulus in &[5, 10, 1000, 50_000] { + let dist = Uniform::new(0, modulus).unwrap(); + let mut v: Vec = rng.sample_iter(&dist).take(len).collect(); + + v.sort(); + v.reverse(); + + for _ in 0..5 { + let a = rng.sample(&len_dist); + let b = rng.sample(&len_dist); + if a < b { + v[a..b].reverse(); + } else { + v.swap(a, b); + } + } + + v.$f(|a, b| a.cmp(b)); + assert!(v.windows(2).all(|w| w[0] <= w[1])); + } + } + + // Sort using a completely random comparison function. + // This will reorder the elements *somehow*, but won't panic. + let mut v: Vec<_> = (0..100).collect(); + v.$f(|_, _| *[Less, Equal, Greater].choose(&mut rand::rng()).unwrap()); + v.$f(|a, b| a.cmp(b)); + for i in 0..v.len() { + assert_eq!(v[i], i); + } + + // Should not panic. + [0i32; 0].$f(|a, b| a.cmp(b)); + [(); 10].$f(|a, b| a.cmp(b)); + [(); 100].$f(|a, b| a.cmp(b)); + + let mut v = [0xDEAD_BEEFu64]; + v.$f(|a, b| a.cmp(b)); + assert!(v == [0xDEAD_BEEF]); + } + }; +} + +sort!(par_sort_by, test_par_sort); +sort!(par_sort_unstable_by, test_par_sort_unstable); + +#[test] +fn test_par_sort_stability() { + for len in (2..25).chain(500..510).chain(50_000..50_010) { + for _ in 0..10 { + let mut counts = [0; 10]; + + // Create a vector like [(6, 1), (5, 1), (6, 2), ...], + // where the first item of each tuple is random, but + // the second item represents which occurrence of that + // number this element is, i.e. the second elements + // will occur in sorted order. + let mut rng = rng(); + let mut v: Vec<_> = (0..len) + .map(|_| { + let n: usize = rng.random_range(0..10); + counts[n] += 1; + (n, counts[n]) + }) + .collect(); + + // Only sort on the first element, so an unstable sort + // may mix up the counts. + v.par_sort_by(|&(a, _), &(b, _)| a.cmp(&b)); + + // This comparison includes the count (the second item + // of the tuple), so elements with equal first items + // will need to be ordered with increasing + // counts... i.e. exactly asserting that this sort is + // stable. + assert!(v.windows(2).all(|w| w[0] <= w[1])); + } + } +} + +#[test] +fn test_par_chunks_exact_remainder() { + let v: &[i32] = &[0, 1, 2, 3, 4]; + let c = v.par_chunks_exact(2); + assert_eq!(c.remainder(), &[4]); + assert_eq!(c.len(), 2); +} + +#[test] +fn test_par_chunks_exact_mut_remainder() { + let v: &mut [i32] = &mut [0, 1, 2, 3, 4]; + let mut c = v.par_chunks_exact_mut(2); + assert_eq!(c.remainder(), &[4]); + assert_eq!(c.len(), 2); + assert_eq!(c.into_remainder(), &[4]); + + let mut c = v.par_chunks_exact_mut(2); + assert_eq!(c.take_remainder(), &[4]); + assert_eq!(c.take_remainder(), &[]); + assert_eq!(c.len(), 2); +} + +#[test] +fn test_par_rchunks_exact_remainder() { + let v: &[i32] = &[0, 1, 2, 3, 4]; + let c = v.par_rchunks_exact(2); + assert_eq!(c.remainder(), &[0]); + assert_eq!(c.len(), 2); +} + +#[test] +fn test_par_rchunks_exact_mut_remainder() { + let v: &mut [i32] = &mut [0, 1, 2, 3, 4]; + let mut c = v.par_rchunks_exact_mut(2); + assert_eq!(c.remainder(), &[0]); + assert_eq!(c.len(), 2); + assert_eq!(c.into_remainder(), &[0]); + + let mut c = v.par_rchunks_exact_mut(2); + assert_eq!(c.take_remainder(), &[0]); + assert_eq!(c.take_remainder(), &[]); + assert_eq!(c.len(), 2); +} + +#[test] +fn slice_chunk_by() { + let v: Vec<_> = (0..1000).collect(); + assert_eq!(v[..0].par_chunk_by(|_, _| todo!()).count(), 0); + assert_eq!(v[..1].par_chunk_by(|_, _| todo!()).count(), 1); + assert_eq!(v[..2].par_chunk_by(|_, _| true).count(), 1); + assert_eq!(v[..2].par_chunk_by(|_, _| false).count(), 2); + + let count = AtomicUsize::new(0); + let par: Vec<_> = v + .par_chunk_by(|x, y| { + count.fetch_add(1, Relaxed); + (x % 10 < 3) == (y % 10 < 3) + }) + .collect(); + assert_eq!(count.into_inner(), v.len() - 1); + + let seq: Vec<_> = v.chunk_by(|x, y| (x % 10 < 3) == (y % 10 < 3)).collect(); + assert_eq!(par, seq); +} + +#[test] +fn slice_chunk_by_mut() { + let mut v: Vec<_> = (0..1000).collect(); + assert_eq!(v[..0].par_chunk_by_mut(|_, _| todo!()).count(), 0); + assert_eq!(v[..1].par_chunk_by_mut(|_, _| todo!()).count(), 1); + assert_eq!(v[..2].par_chunk_by_mut(|_, _| true).count(), 1); + assert_eq!(v[..2].par_chunk_by_mut(|_, _| false).count(), 2); + + let mut v2 = v.clone(); + let count = AtomicUsize::new(0); + let par: Vec<_> = v + .par_chunk_by_mut(|x, y| { + count.fetch_add(1, Relaxed); + (x % 10 < 3) == (y % 10 < 3) + }) + .collect(); + assert_eq!(count.into_inner(), v2.len() - 1); + + let seq: Vec<_> = v2 + .chunk_by_mut(|x, y| (x % 10 < 3) == (y % 10 < 3)) + .collect(); + assert_eq!(par, seq); +} diff --git a/anneal/v2/vendor/rayon/src/slice/windows.rs b/anneal/v2/vendor/rayon/src/slice/windows.rs new file mode 100644 index 0000000000..ff2633f632 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/slice/windows.rs @@ -0,0 +1,147 @@ +use crate::iter::plumbing::*; +use crate::iter::*; + +/// Parallel iterator over immutable overlapping windows of a slice +#[derive(Debug)] +pub struct Windows<'data, T> { + window_size: usize, + slice: &'data [T], +} + +impl<'data, T> Windows<'data, T> { + pub(super) fn new(window_size: usize, slice: &'data [T]) -> Self { + Self { window_size, slice } + } +} + +impl Clone for Windows<'_, T> { + fn clone(&self) -> Self { + Windows { ..*self } + } +} + +impl<'data, T: Sync> ParallelIterator for Windows<'data, T> { + type Item = &'data [T]; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + bridge(self, consumer) + } + + fn opt_len(&self) -> Option { + Some(self.len()) + } +} + +impl IndexedParallelIterator for Windows<'_, T> { + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + bridge(self, consumer) + } + + fn len(&self) -> usize { + assert!(self.window_size >= 1); + self.slice.len().saturating_sub(self.window_size - 1) + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + callback.callback(WindowsProducer { + window_size: self.window_size, + slice: self.slice, + }) + } +} + +struct WindowsProducer<'data, T: Sync> { + window_size: usize, + slice: &'data [T], +} + +impl<'data, T: Sync> Producer for WindowsProducer<'data, T> { + type Item = &'data [T]; + type IntoIter = ::std::slice::Windows<'data, T>; + + fn into_iter(self) -> Self::IntoIter { + self.slice.windows(self.window_size) + } + + fn split_at(self, index: usize) -> (Self, Self) { + let left_index = Ord::min(self.slice.len(), index + (self.window_size - 1)); + let left = &self.slice[..left_index]; + let right = &self.slice[index..]; + ( + WindowsProducer { + window_size: self.window_size, + slice: left, + }, + WindowsProducer { + window_size: self.window_size, + slice: right, + }, + ) + } +} + +/// Parallel iterator over immutable overlapping windows of a slice +#[derive(Debug)] +pub struct ArrayWindows<'data, T: Sync, const N: usize> { + slice: &'data [T], +} + +impl<'data, T: Sync, const N: usize> ArrayWindows<'data, T, N> { + pub(super) fn new(slice: &'data [T]) -> Self { + ArrayWindows { slice } + } +} + +impl Clone for ArrayWindows<'_, T, N> { + fn clone(&self) -> Self { + ArrayWindows { ..*self } + } +} + +impl<'data, T: Sync, const N: usize> ParallelIterator for ArrayWindows<'data, T, N> { + type Item = &'data [T; N]; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + bridge(self, consumer) + } + + fn opt_len(&self) -> Option { + Some(self.len()) + } +} + +impl IndexedParallelIterator for ArrayWindows<'_, T, N> { + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + bridge(self, consumer) + } + + fn len(&self) -> usize { + assert!(N >= 1); + self.slice.len().saturating_sub(const { N - 1 }) + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + // TODO (MSRV 1.94): use our own producer and the standard `array_windows` + Windows::new(N, self.slice) + .map(|slice| slice.try_into().unwrap()) + .with_producer(callback) + } +} diff --git a/anneal/v2/vendor/rayon/src/split_producer.rs b/anneal/v2/vendor/rayon/src/split_producer.rs new file mode 100644 index 0000000000..6e15b792d2 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/split_producer.rs @@ -0,0 +1,152 @@ +//! Common splitter for strings and slices +//! +//! This module is private, so these items are effectively `pub(super)` + +use crate::iter::plumbing::{Folder, UnindexedProducer}; + +/// Common producer for splitting on a predicate. +pub(super) struct SplitProducer<'p, P, V, const INCL: bool = false> { + data: V, + separator: &'p P, + + /// Marks the endpoint beyond which we've already found no separators. + tail: usize, +} + +pub(super) type SplitInclusiveProducer<'p, P, V> = SplitProducer<'p, P, V, true>; + +/// Helper trait so `&str`, `&[T]`, and `&mut [T]` can share `SplitProducer`. +pub(super) trait Fissile

: Sized { + fn length(&self) -> usize; + fn midpoint(&self, end: usize) -> usize; + fn find(&self, separator: &P, start: usize, end: usize) -> Option; + fn rfind(&self, separator: &P, end: usize) -> Option; + fn split_once(self, index: usize) -> (Self, Self); + fn fold_splits(self, separator: &P, folder: F, skip_last: bool) -> F + where + F: Folder, + Self: Send; +} + +impl<'p, P, V> SplitProducer<'p, P, V> +where + V: Fissile

+ Send, +{ + pub(super) fn new(data: V, separator: &'p P) -> Self { + SplitProducer { + tail: data.length(), + data, + separator, + } + } +} + +impl<'p, P, V> SplitInclusiveProducer<'p, P, V> +where + V: Fissile

+ Send, +{ + pub(super) fn new_incl(data: V, separator: &'p P) -> Self { + SplitProducer { + tail: data.length(), + data, + separator, + } + } +} + +impl<'p, P, V, const INCL: bool> SplitProducer<'p, P, V, INCL> +where + V: Fissile

+ Send, +{ + /// Common `fold_with` implementation, integrating `SplitTerminator`'s + /// need to sometimes skip its final empty item. + pub(super) fn fold_with(self, folder: F, skip_last: bool) -> F + where + F: Folder, + { + let SplitProducer { + data, + separator, + tail, + } = self; + + if tail == data.length() { + // No tail section, so just let `fold_splits` handle it. + data.fold_splits::(separator, folder, skip_last) + } else if let Some(index) = data.rfind(separator, tail) { + // We found the last separator to complete the tail, so + // end with that slice after `fold_splits` finds the rest. + let (left, right) = data.split_once::(index); + let folder = left.fold_splits::(separator, folder, false); + if skip_last || folder.full() { + folder + } else { + folder.consume(right) + } + } else { + // We know there are no separators at all. Return our whole data. + if skip_last { + folder + } else { + folder.consume(data) + } + } + } +} + +impl<'p, P, V, const INCL: bool> UnindexedProducer for SplitProducer<'p, P, V, INCL> +where + V: Fissile

+ Send, + P: Sync, +{ + type Item = V; + + fn split(self) -> (Self, Option) { + // Look forward for the separator, and failing that look backward. + let mid = self.data.midpoint(self.tail); + let index = match self.data.find(self.separator, mid, self.tail) { + Some(i) => Some(mid + i), + None => self.data.rfind(self.separator, mid), + }; + + if let Some(index) = index { + let len = self.data.length(); + let (left, right) = self.data.split_once::(index); + + let (left_tail, right_tail) = if index < mid { + // If we scanned backwards to find the separator, everything in + // the right side is exhausted, with no separators left to find. + (index, 0) + } else { + let right_index = len - right.length(); + (mid, self.tail - right_index) + }; + + // Create the left split before the separator. + let left = SplitProducer { + data: left, + tail: left_tail, + ..self + }; + + // Create the right split following the separator. + let right = SplitProducer { + data: right, + tail: right_tail, + ..self + }; + + (left, Some(right)) + } else { + // The search is exhausted, no more separators... + (SplitProducer { tail: 0, ..self }, None) + } + } + + fn fold_with(self, folder: F) -> F + where + F: Folder, + { + self.fold_with(folder, false) + } +} diff --git a/anneal/v2/vendor/rayon/src/str.rs b/anneal/v2/vendor/rayon/src/str.rs new file mode 100644 index 0000000000..20548ff96e --- /dev/null +++ b/anneal/v2/vendor/rayon/src/str.rs @@ -0,0 +1,1006 @@ +//! Parallel iterator types for [strings] +//! +//! You will rarely need to interact with this module directly unless you need +//! to name one of the iterator types. +//! +//! Note: [`ParallelString::par_split()`] and [`par_split_terminator()`] +//! reference a `Pattern` trait which is not visible outside this crate. +//! This trait is intentionally kept private, for use only by Rayon itself. +//! It is implemented for `char`, `&[char]`, `[char; N]`, `&[char; N]`, +//! and any function or closure `F: Fn(char) -> bool + Sync + Send`. +//! +//! [`par_split_terminator()`]: ParallelString::par_split_terminator() +//! [strings]: std::str + +use crate::iter::plumbing::*; +use crate::iter::*; +use crate::split_producer::*; + +/// Test if a byte is the start of a UTF-8 character. +/// (extracted from `str::is_char_boundary`) +#[inline] +fn is_char_boundary(b: u8) -> bool { + // This is bit magic equivalent to: b < 128 || b >= 192 + (b as i8) >= -0x40 +} + +/// Find the index of a character boundary near the midpoint. +#[inline] +fn find_char_midpoint(chars: &str) -> usize { + let mid = chars.len() / 2; + + // We want to split near the midpoint, but we need to find an actual + // character boundary. So we look at the raw bytes, first scanning + // forward from the midpoint for a boundary, then trying backward. + // TODO (MSRV 1.91): use `str::ceil_char_boundary`, else `floor_...`. + let (left, right) = chars.as_bytes().split_at(mid); + match right.iter().copied().position(is_char_boundary) { + Some(i) => mid + i, + None => left + .iter() + .copied() + .rposition(is_char_boundary) + .unwrap_or(0), + } +} + +/// Try to split a string near the midpoint. +#[inline] +fn split(chars: &str) -> Option<(&str, &str)> { + let index = find_char_midpoint(chars); + if index > 0 { + Some(chars.split_at(index)) + } else { + None + } +} + +/// Parallel extensions for strings. +pub trait ParallelString { + /// Returns a plain string slice, which is used to implement the rest of + /// the parallel methods. + fn as_parallel_string(&self) -> &str; + + /// Returns a parallel iterator over the characters of a string. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// let max = "hello".par_chars().max_by_key(|c| *c as i32); + /// assert_eq!(Some('o'), max); + /// ``` + fn par_chars(&self) -> Chars<'_> { + Chars { + chars: self.as_parallel_string(), + } + } + + /// Returns a parallel iterator over the characters of a string, with their positions. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// let min = "hello".par_char_indices().min_by_key(|&(_i, c)| c as i32); + /// assert_eq!(Some((1, 'e')), min); + /// ``` + fn par_char_indices(&self) -> CharIndices<'_> { + CharIndices { + chars: self.as_parallel_string(), + } + } + + /// Returns a parallel iterator over the bytes of a string. + /// + /// Note that multi-byte sequences (for code points greater than `U+007F`) + /// are produced as separate items, but will not be split across threads. + /// If you would prefer an indexed iterator without that guarantee, consider + /// `string.as_bytes().par_iter().copied()` instead. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// let max = "hello".par_bytes().max(); + /// assert_eq!(Some(b'o'), max); + /// ``` + fn par_bytes(&self) -> Bytes<'_> { + Bytes { + chars: self.as_parallel_string(), + } + } + + /// Returns a parallel iterator over a string encoded as UTF-16. + /// + /// Note that surrogate pairs (for code points greater than `U+FFFF`) are + /// produced as separate items, but will not be split across threads. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// + /// let max = "hello".par_encode_utf16().max(); + /// assert_eq!(Some(b'o' as u16), max); + /// + /// let text = "Zażółć gęślą jaźń"; + /// let utf8_len = text.len(); + /// let utf16_len = text.par_encode_utf16().count(); + /// assert!(utf16_len <= utf8_len); + /// ``` + fn par_encode_utf16(&self) -> EncodeUtf16<'_> { + EncodeUtf16 { + chars: self.as_parallel_string(), + } + } + + /// Returns a parallel iterator over substrings separated by a + /// given character or predicate, similar to `str::split`. + /// + /// Note: the `Pattern` trait is private, for use only by Rayon itself. + /// It is implemented for `char`, `&[char]`, `[char; N]`, `&[char; N]`, + /// and any function or closure `F: Fn(char) -> bool + Sync + Send`. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// let total = "1, 2, buckle, 3, 4, door" + /// .par_split(',') + /// .filter_map(|s| s.trim().parse::().ok()) + /// .sum(); + /// assert_eq!(10, total); + /// ``` + fn par_split(&self, separator: P) -> Split<'_, P> { + Split::new(self.as_parallel_string(), separator) + } + + /// Returns a parallel iterator over substrings separated by a + /// given character or predicate, keeping the matched part as a terminator + /// of the substring similar to `str::split_inclusive`. + /// + /// Note: the `Pattern` trait is private, for use only by Rayon itself. + /// It is implemented for `char`, `&[char]`, `[char; N]`, `&[char; N]`, + /// and any function or closure `F: Fn(char) -> bool + Sync + Send`. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// let lines: Vec<_> = "Mary had a little lamb\nlittle lamb\nlittle lamb." + /// .par_split_inclusive('\n') + /// .collect(); + /// assert_eq!(lines, ["Mary had a little lamb\n", "little lamb\n", "little lamb."]); + /// ``` + fn par_split_inclusive(&self, separator: P) -> SplitInclusive<'_, P> { + SplitInclusive::new(self.as_parallel_string(), separator) + } + + /// Returns a parallel iterator over substrings terminated by a + /// given character or predicate, similar to `str::split_terminator`. + /// It's equivalent to `par_split`, except it doesn't produce an empty + /// substring after a trailing terminator. + /// + /// Note: the `Pattern` trait is private, for use only by Rayon itself. + /// It is implemented for `char`, `&[char]`, `[char; N]`, `&[char; N]`, + /// and any function or closure `F: Fn(char) -> bool + Sync + Send`. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// let parts: Vec<_> = "((1 + 3) * 2)" + /// .par_split_terminator(|c| c == '(' || c == ')') + /// .collect(); + /// assert_eq!(vec!["", "", "1 + 3", " * 2"], parts); + /// ``` + fn par_split_terminator(&self, terminator: P) -> SplitTerminator<'_, P> { + SplitTerminator::new(self.as_parallel_string(), terminator) + } + + /// Returns a parallel iterator over the lines of a string, ending with an + /// optional carriage return and with a newline (`\r\n` or just `\n`). + /// The final line ending is optional, and line endings are not included in + /// the output strings. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// let lengths: Vec<_> = "hello world\nfizbuzz" + /// .par_lines() + /// .map(|l| l.len()) + /// .collect(); + /// assert_eq!(vec![11, 7], lengths); + /// ``` + fn par_lines(&self) -> Lines<'_> { + Lines(self.as_parallel_string()) + } + + /// Returns a parallel iterator over the sub-slices of a string that are + /// separated by any amount of whitespace. + /// + /// As with `str::split_whitespace`, 'whitespace' is defined according to + /// the terms of the Unicode Derived Core Property `White_Space`. + /// If you only want to split on ASCII whitespace instead, use + /// [`par_split_ascii_whitespace`][`ParallelString::par_split_ascii_whitespace`]. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// let longest = "which is the longest word?" + /// .par_split_whitespace() + /// .max_by_key(|word| word.len()); + /// assert_eq!(Some("longest"), longest); + /// ``` + /// + /// All kinds of whitespace are considered: + /// + /// ``` + /// use rayon::prelude::*; + /// let words: Vec<&str> = " Mary had\ta\u{2009}little \n\t lamb" + /// .par_split_whitespace() + /// .collect(); + /// assert_eq!(words, ["Mary", "had", "a", "little", "lamb"]); + /// ``` + /// + /// If the string is empty or all whitespace, the iterator yields no string slices: + /// + /// ``` + /// use rayon::prelude::*; + /// assert_eq!("".par_split_whitespace().count(), 0); + /// assert_eq!(" ".par_split_whitespace().count(), 0); + /// ``` + fn par_split_whitespace(&self) -> SplitWhitespace<'_> { + SplitWhitespace(self.as_parallel_string()) + } + + /// Returns a parallel iterator over the sub-slices of a string that are + /// separated by any amount of ASCII whitespace. + /// + /// To split by Unicode `White_Space` instead, use + /// [`par_split_whitespace`][`ParallelString::par_split_whitespace`]. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// let longest = "which is the longest word?" + /// .par_split_ascii_whitespace() + /// .max_by_key(|word| word.len()); + /// assert_eq!(Some("longest"), longest); + /// ``` + /// + /// All kinds of ASCII whitespace are considered, but not Unicode `White_Space`: + /// + /// ``` + /// use rayon::prelude::*; + /// let words: Vec<&str> = " Mary had\ta\u{2009}little \n\t lamb" + /// .par_split_ascii_whitespace() + /// .collect(); + /// assert_eq!(words, ["Mary", "had", "a\u{2009}little", "lamb"]); + /// ``` + /// + /// If the string is empty or all ASCII whitespace, the iterator yields no string slices: + /// + /// ``` + /// use rayon::prelude::*; + /// assert_eq!("".par_split_whitespace().count(), 0); + /// assert_eq!(" ".par_split_whitespace().count(), 0); + /// ``` + fn par_split_ascii_whitespace(&self) -> SplitAsciiWhitespace<'_> { + SplitAsciiWhitespace(self.as_parallel_string()) + } + + /// Returns a parallel iterator over substrings that match a + /// given character or predicate, similar to `str::matches`. + /// + /// Note: the `Pattern` trait is private, for use only by Rayon itself. + /// It is implemented for `char`, `&[char]`, `[char; N]`, `&[char; N]`, + /// and any function or closure `F: Fn(char) -> bool + Sync + Send`. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// let total = "1, 2, buckle, 3, 4, door" + /// .par_matches(char::is_numeric) + /// .map(|s| s.parse::().expect("digit")) + /// .sum(); + /// assert_eq!(10, total); + /// ``` + fn par_matches(&self, pattern: P) -> Matches<'_, P> { + Matches { + chars: self.as_parallel_string(), + pattern, + } + } + + /// Returns a parallel iterator over substrings that match a given character + /// or predicate, with their positions, similar to `str::match_indices`. + /// + /// Note: the `Pattern` trait is private, for use only by Rayon itself. + /// It is implemented for `char`, `&[char]`, `[char; N]`, `&[char; N]`, + /// and any function or closure `F: Fn(char) -> bool + Sync + Send`. + /// + /// # Examples + /// + /// ``` + /// use rayon::prelude::*; + /// let digits: Vec<_> = "1, 2, buckle, 3, 4, door" + /// .par_match_indices(char::is_numeric) + /// .collect(); + /// assert_eq!(digits, vec![(0, "1"), (3, "2"), (14, "3"), (17, "4")]); + /// ``` + fn par_match_indices(&self, pattern: P) -> MatchIndices<'_, P> { + MatchIndices { + chars: self.as_parallel_string(), + pattern, + } + } +} + +impl ParallelString for str { + #[inline] + fn as_parallel_string(&self) -> &str { + self + } +} + +// ///////////////////////////////////////////////////////////////////////// + +/// We hide the `Pattern` trait in a private module, as its API is not meant +/// for general consumption. If we could have privacy on trait items, then it +/// would be nicer to have its basic existence and implementors public while +/// keeping all of the methods private. +mod private { + use crate::iter::plumbing::Folder; + + /// Pattern-matching trait for `ParallelString`, somewhat like a mix of + /// `std::str::pattern::{Pattern, Searcher}`. + /// + /// Implementing this trait is not permitted outside of `rayon`. + pub trait Pattern: Sized + Sync + Send { + private_decl! {} + fn find_in(&self, haystack: &str) -> Option; + fn rfind_in(&self, haystack: &str) -> Option; + fn is_suffix_of(&self, haystack: &str) -> bool; + fn fold_splits<'ch, F>(&self, haystack: &'ch str, folder: F, skip_last: bool) -> F + where + F: Folder<&'ch str>; + fn fold_inclusive_splits<'ch, F>(&self, haystack: &'ch str, folder: F) -> F + where + F: Folder<&'ch str>; + fn fold_matches<'ch, F>(&self, haystack: &'ch str, folder: F) -> F + where + F: Folder<&'ch str>; + fn fold_match_indices<'ch, F>(&self, haystack: &'ch str, folder: F, base: usize) -> F + where + F: Folder<(usize, &'ch str)>; + } +} +use self::private::Pattern; + +#[inline] +fn offset(base: usize) -> impl Fn((usize, T)) -> (usize, T) { + move |(i, x)| (base + i, x) +} + +macro_rules! impl_pattern { + (&$self:ident => $pattern:expr) => { + private_impl! {} + + #[inline] + fn find_in(&$self, chars: &str) -> Option { + chars.find($pattern) + } + + #[inline] + fn rfind_in(&$self, chars: &str) -> Option { + chars.rfind($pattern) + } + + #[inline] + fn is_suffix_of(&$self, chars: &str) -> bool { + chars.ends_with($pattern) + } + + fn fold_splits<'ch, F>(&$self, chars: &'ch str, folder: F, skip_last: bool) -> F + where + F: Folder<&'ch str>, + { + let mut split = chars.split($pattern); + if skip_last { + split.next_back(); + } + folder.consume_iter(split) + } + + fn fold_inclusive_splits<'ch, F>(&$self, chars: &'ch str, folder: F) -> F + where + F: Folder<&'ch str>, + { + folder.consume_iter(chars.split_inclusive($pattern)) + } + + fn fold_matches<'ch, F>(&$self, chars: &'ch str, folder: F) -> F + where + F: Folder<&'ch str>, + { + folder.consume_iter(chars.matches($pattern)) + } + + fn fold_match_indices<'ch, F>(&$self, chars: &'ch str, folder: F, base: usize) -> F + where + F: Folder<(usize, &'ch str)>, + { + folder.consume_iter(chars.match_indices($pattern).map(offset(base))) + } + } +} + +impl Pattern for char { + impl_pattern!(&self => *self); +} + +impl Pattern for &[char] { + impl_pattern!(&self => *self); +} + +impl Pattern for [char; N] { + impl_pattern!(&self => *self); +} + +impl Pattern for &[char; N] { + impl_pattern!(&self => *self); +} + +impl bool> Pattern for FN { + impl_pattern!(&self => self); +} + +// ///////////////////////////////////////////////////////////////////////// + +/// Parallel iterator over the characters of a string +#[derive(Debug, Clone)] +pub struct Chars<'ch> { + chars: &'ch str, +} + +struct CharsProducer<'ch> { + chars: &'ch str, +} + +impl<'ch> ParallelIterator for Chars<'ch> { + type Item = char; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + bridge_unindexed(CharsProducer { chars: self.chars }, consumer) + } +} + +impl<'ch> UnindexedProducer for CharsProducer<'ch> { + type Item = char; + + fn split(self) -> (Self, Option) { + match split(self.chars) { + Some((left, right)) => ( + CharsProducer { chars: left }, + Some(CharsProducer { chars: right }), + ), + None => (self, None), + } + } + + fn fold_with(self, folder: F) -> F + where + F: Folder, + { + folder.consume_iter(self.chars.chars()) + } +} + +// ///////////////////////////////////////////////////////////////////////// + +/// Parallel iterator over the characters of a string, with their positions +#[derive(Debug, Clone)] +pub struct CharIndices<'ch> { + chars: &'ch str, +} + +struct CharIndicesProducer<'ch> { + index: usize, + chars: &'ch str, +} + +impl<'ch> ParallelIterator for CharIndices<'ch> { + type Item = (usize, char); + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let producer = CharIndicesProducer { + index: 0, + chars: self.chars, + }; + bridge_unindexed(producer, consumer) + } +} + +impl<'ch> UnindexedProducer for CharIndicesProducer<'ch> { + type Item = (usize, char); + + fn split(self) -> (Self, Option) { + match split(self.chars) { + Some((left, right)) => ( + CharIndicesProducer { + chars: left, + ..self + }, + Some(CharIndicesProducer { + chars: right, + index: self.index + left.len(), + }), + ), + None => (self, None), + } + } + + fn fold_with(self, folder: F) -> F + where + F: Folder, + { + let base = self.index; + folder.consume_iter(self.chars.char_indices().map(offset(base))) + } +} + +// ///////////////////////////////////////////////////////////////////////// + +/// Parallel iterator over the bytes of a string +#[derive(Debug, Clone)] +pub struct Bytes<'ch> { + chars: &'ch str, +} + +struct BytesProducer<'ch> { + chars: &'ch str, +} + +impl<'ch> ParallelIterator for Bytes<'ch> { + type Item = u8; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + bridge_unindexed(BytesProducer { chars: self.chars }, consumer) + } +} + +impl<'ch> UnindexedProducer for BytesProducer<'ch> { + type Item = u8; + + fn split(self) -> (Self, Option) { + match split(self.chars) { + Some((left, right)) => ( + BytesProducer { chars: left }, + Some(BytesProducer { chars: right }), + ), + None => (self, None), + } + } + + fn fold_with(self, folder: F) -> F + where + F: Folder, + { + folder.consume_iter(self.chars.bytes()) + } +} + +// ///////////////////////////////////////////////////////////////////////// + +/// Parallel iterator over a string encoded as UTF-16 +#[derive(Debug, Clone)] +pub struct EncodeUtf16<'ch> { + chars: &'ch str, +} + +struct EncodeUtf16Producer<'ch> { + chars: &'ch str, +} + +impl<'ch> ParallelIterator for EncodeUtf16<'ch> { + type Item = u16; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + bridge_unindexed(EncodeUtf16Producer { chars: self.chars }, consumer) + } +} + +impl<'ch> UnindexedProducer for EncodeUtf16Producer<'ch> { + type Item = u16; + + fn split(self) -> (Self, Option) { + match split(self.chars) { + Some((left, right)) => ( + EncodeUtf16Producer { chars: left }, + Some(EncodeUtf16Producer { chars: right }), + ), + None => (self, None), + } + } + + fn fold_with(self, folder: F) -> F + where + F: Folder, + { + folder.consume_iter(self.chars.encode_utf16()) + } +} + +// ///////////////////////////////////////////////////////////////////////// + +/// Parallel iterator over substrings separated by a pattern +#[derive(Debug, Clone)] +pub struct Split<'ch, P: Pattern> { + chars: &'ch str, + separator: P, +} + +impl<'ch, P: Pattern> Split<'ch, P> { + fn new(chars: &'ch str, separator: P) -> Self { + Split { chars, separator } + } +} + +impl<'ch, P: Pattern> ParallelIterator for Split<'ch, P> { + type Item = &'ch str; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let producer = SplitProducer::new(self.chars, &self.separator); + bridge_unindexed(producer, consumer) + } +} + +/// Implement support for `SplitProducer`. +impl Fissile

for &str { + fn length(&self) -> usize { + self.len() + } + + fn midpoint(&self, end: usize) -> usize { + // First find a suitable UTF-8 boundary. + find_char_midpoint(&self[..end]) + } + + fn find(&self, separator: &P, start: usize, end: usize) -> Option { + separator.find_in(&self[start..end]) + } + + fn rfind(&self, separator: &P, end: usize) -> Option { + separator.rfind_in(&self[..end]) + } + + fn split_once(self, index: usize) -> (Self, Self) { + if INCL { + // include the separator in the left side + let separator = self[index..].chars().next().unwrap(); + self.split_at(index + separator.len_utf8()) + } else { + let (left, right) = self.split_at(index); + let mut right_iter = right.chars(); + right_iter.next(); // skip the separator + (left, right_iter.as_str()) + } + } + + fn fold_splits(self, separator: &P, folder: F, skip_last: bool) -> F + where + F: Folder, + { + if INCL { + debug_assert!(!skip_last); + separator.fold_inclusive_splits(self, folder) + } else { + separator.fold_splits(self, folder, skip_last) + } + } +} + +// ///////////////////////////////////////////////////////////////////////// + +/// Parallel iterator over substrings separated by a pattern +#[derive(Debug, Clone)] +pub struct SplitInclusive<'ch, P: Pattern> { + chars: &'ch str, + separator: P, +} + +impl<'ch, P: Pattern> SplitInclusive<'ch, P> { + fn new(chars: &'ch str, separator: P) -> Self { + SplitInclusive { chars, separator } + } +} + +impl<'ch, P: Pattern> ParallelIterator for SplitInclusive<'ch, P> { + type Item = &'ch str; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let producer = SplitInclusiveProducer::new_incl(self.chars, &self.separator); + bridge_unindexed(producer, consumer) + } +} + +// ///////////////////////////////////////////////////////////////////////// + +/// Parallel iterator over substrings separated by a terminator pattern +#[derive(Debug, Clone)] +pub struct SplitTerminator<'ch, P: Pattern> { + chars: &'ch str, + terminator: P, +} + +struct SplitTerminatorProducer<'ch, 'sep, P: Pattern> { + splitter: SplitProducer<'sep, P, &'ch str>, + skip_last: bool, +} + +impl<'ch, P: Pattern> SplitTerminator<'ch, P> { + fn new(chars: &'ch str, terminator: P) -> Self { + SplitTerminator { chars, terminator } + } +} + +impl<'ch, 'sep, P: Pattern + 'sep> SplitTerminatorProducer<'ch, 'sep, P> { + fn new(chars: &'ch str, terminator: &'sep P) -> Self { + SplitTerminatorProducer { + splitter: SplitProducer::new(chars, terminator), + skip_last: chars.is_empty() || terminator.is_suffix_of(chars), + } + } +} + +impl<'ch, P: Pattern> ParallelIterator for SplitTerminator<'ch, P> { + type Item = &'ch str; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let producer = SplitTerminatorProducer::new(self.chars, &self.terminator); + bridge_unindexed(producer, consumer) + } +} + +impl<'ch, 'sep, P: Pattern + 'sep> UnindexedProducer for SplitTerminatorProducer<'ch, 'sep, P> { + type Item = &'ch str; + + fn split(mut self) -> (Self, Option) { + let (left, right) = self.splitter.split(); + self.splitter = left; + let right = right.map(|right| { + let skip_last = self.skip_last; + self.skip_last = false; + SplitTerminatorProducer { + splitter: right, + skip_last, + } + }); + (self, right) + } + + fn fold_with(self, folder: F) -> F + where + F: Folder, + { + self.splitter.fold_with(folder, self.skip_last) + } +} + +// ///////////////////////////////////////////////////////////////////////// + +/// Parallel iterator over lines in a string +#[derive(Debug, Clone)] +pub struct Lines<'ch>(&'ch str); + +#[inline] +fn no_carriage_return(line: &str) -> &str { + line.strip_suffix('\r').unwrap_or(line) +} + +impl<'ch> ParallelIterator for Lines<'ch> { + type Item = &'ch str; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + self.0 + .par_split_terminator('\n') + .map(no_carriage_return) + .drive_unindexed(consumer) + } +} + +// ///////////////////////////////////////////////////////////////////////// + +/// Parallel iterator over substrings separated by whitespace +#[derive(Debug, Clone)] +pub struct SplitWhitespace<'ch>(&'ch str); + +#[inline] +fn not_empty(s: &&str) -> bool { + !s.is_empty() +} + +impl<'ch> ParallelIterator for SplitWhitespace<'ch> { + type Item = &'ch str; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + self.0 + .par_split(char::is_whitespace) + .filter(not_empty) + .drive_unindexed(consumer) + } +} + +// ///////////////////////////////////////////////////////////////////////// + +/// Parallel iterator over substrings separated by ASCII whitespace +#[derive(Debug, Clone)] +pub struct SplitAsciiWhitespace<'ch>(&'ch str); + +#[inline] +fn is_ascii_whitespace(c: char) -> bool { + c.is_ascii_whitespace() +} + +impl<'ch> ParallelIterator for SplitAsciiWhitespace<'ch> { + type Item = &'ch str; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + self.0 + .par_split(is_ascii_whitespace) + .filter(not_empty) + .drive_unindexed(consumer) + } +} + +// ///////////////////////////////////////////////////////////////////////// + +/// Parallel iterator over substrings that match a pattern +#[derive(Debug, Clone)] +pub struct Matches<'ch, P: Pattern> { + chars: &'ch str, + pattern: P, +} + +struct MatchesProducer<'ch, 'pat, P: Pattern> { + chars: &'ch str, + pattern: &'pat P, +} + +impl<'ch, P: Pattern> ParallelIterator for Matches<'ch, P> { + type Item = &'ch str; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let producer = MatchesProducer { + chars: self.chars, + pattern: &self.pattern, + }; + bridge_unindexed(producer, consumer) + } +} + +impl<'ch, 'pat, P: Pattern> UnindexedProducer for MatchesProducer<'ch, 'pat, P> { + type Item = &'ch str; + + fn split(self) -> (Self, Option) { + match split(self.chars) { + Some((left, right)) => ( + MatchesProducer { + chars: left, + ..self + }, + Some(MatchesProducer { + chars: right, + ..self + }), + ), + None => (self, None), + } + } + + fn fold_with(self, folder: F) -> F + where + F: Folder, + { + self.pattern.fold_matches(self.chars, folder) + } +} + +// ///////////////////////////////////////////////////////////////////////// + +/// Parallel iterator over substrings that match a pattern, with their positions +#[derive(Debug, Clone)] +pub struct MatchIndices<'ch, P: Pattern> { + chars: &'ch str, + pattern: P, +} + +struct MatchIndicesProducer<'ch, 'pat, P: Pattern> { + index: usize, + chars: &'ch str, + pattern: &'pat P, +} + +impl<'ch, P: Pattern> ParallelIterator for MatchIndices<'ch, P> { + type Item = (usize, &'ch str); + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + let producer = MatchIndicesProducer { + index: 0, + chars: self.chars, + pattern: &self.pattern, + }; + bridge_unindexed(producer, consumer) + } +} + +impl<'ch, 'pat, P: Pattern> UnindexedProducer for MatchIndicesProducer<'ch, 'pat, P> { + type Item = (usize, &'ch str); + + fn split(self) -> (Self, Option) { + match split(self.chars) { + Some((left, right)) => ( + MatchIndicesProducer { + chars: left, + ..self + }, + Some(MatchIndicesProducer { + chars: right, + index: self.index + left.len(), + ..self + }), + ), + None => (self, None), + } + } + + fn fold_with(self, folder: F) -> F + where + F: Folder, + { + self.pattern + .fold_match_indices(self.chars, folder, self.index) + } +} diff --git a/anneal/v2/vendor/rayon/src/string.rs b/anneal/v2/vendor/rayon/src/string.rs new file mode 100644 index 0000000000..91e69f9a3c --- /dev/null +++ b/anneal/v2/vendor/rayon/src/string.rs @@ -0,0 +1,48 @@ +//! This module contains the parallel iterator types for owned strings +//! (`String`). You will rarely need to interact with it directly +//! unless you have need to name one of the iterator types. + +use crate::iter::plumbing::*; +use crate::math::simplify_range; +use crate::prelude::*; +use std::ops::{Range, RangeBounds}; + +impl<'a> ParallelDrainRange for &'a mut String { + type Iter = Drain<'a>; + type Item = char; + + fn par_drain>(self, range: R) -> Self::Iter { + Drain { + range: simplify_range(range, self.len()), + string: self, + } + } +} + +/// Draining parallel iterator that moves a range of characters out of a string, +/// but keeps the total capacity. +#[derive(Debug)] +pub struct Drain<'a> { + string: &'a mut String, + range: Range, +} + +impl<'a> ParallelIterator for Drain<'a> { + type Item = char; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + self.string[self.range.clone()] + .par_chars() + .drive_unindexed(consumer) + } +} + +impl<'a> Drop for Drain<'a> { + fn drop(&mut self) { + // Remove the drained range. + self.string.drain(self.range.clone()); + } +} diff --git a/anneal/v2/vendor/rayon/src/vec.rs b/anneal/v2/vendor/rayon/src/vec.rs new file mode 100644 index 0000000000..ad011894b7 --- /dev/null +++ b/anneal/v2/vendor/rayon/src/vec.rs @@ -0,0 +1,292 @@ +//! Parallel iterator types for [vectors] (`Vec`) +//! +//! You will rarely need to interact with this module directly unless you need +//! to name one of the iterator types. +//! +//! [vectors]: mod@std::vec + +use crate::iter::plumbing::*; +use crate::iter::*; +use crate::math::simplify_range; +use crate::slice::{Iter, IterMut}; +use std::iter; +use std::mem; +use std::ops::{Range, RangeBounds}; +use std::ptr; +use std::slice; + +impl<'data, T: Sync + 'data> IntoParallelIterator for &'data Vec { + type Item = &'data T; + type Iter = Iter<'data, T>; + + fn into_par_iter(self) -> Self::Iter { + <&[T]>::into_par_iter(self) + } +} + +impl<'data, T: Send + 'data> IntoParallelIterator for &'data mut Vec { + type Item = &'data mut T; + type Iter = IterMut<'data, T>; + + fn into_par_iter(self) -> Self::Iter { + <&mut [T]>::into_par_iter(self) + } +} + +/// Parallel iterator that moves out of a vector. +#[derive(Debug, Clone)] +pub struct IntoIter { + vec: Vec, +} + +impl IntoParallelIterator for Vec { + type Item = T; + type Iter = IntoIter; + + fn into_par_iter(self) -> Self::Iter { + IntoIter { vec: self } + } +} + +impl IntoParallelIterator for Box<[T]> { + type Item = T; + type Iter = IntoIter; + + fn into_par_iter(self) -> Self::Iter { + IntoIter { vec: self.into() } + } +} + +impl ParallelIterator for IntoIter { + type Item = T; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + bridge(self, consumer) + } + + fn opt_len(&self) -> Option { + Some(self.len()) + } +} + +impl IndexedParallelIterator for IntoIter { + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + bridge(self, consumer) + } + + fn len(&self) -> usize { + self.vec.len() + } + + fn with_producer(mut self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + // Drain every item, and then the vector only needs to free its buffer. + self.vec.par_drain(..).with_producer(callback) + } +} + +impl<'data, T: Send> ParallelDrainRange for &'data mut Vec { + type Iter = Drain<'data, T>; + type Item = T; + + fn par_drain>(self, range: R) -> Self::Iter { + Drain { + orig_len: self.len(), + range: simplify_range(range, self.len()), + vec: self, + } + } +} + +/// Draining parallel iterator that moves a range out of a vector, but keeps the total capacity. +#[derive(Debug)] +pub struct Drain<'data, T: Send> { + vec: &'data mut Vec, + range: Range, + orig_len: usize, +} + +impl<'data, T: Send> ParallelIterator for Drain<'data, T> { + type Item = T; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + bridge(self, consumer) + } + + fn opt_len(&self) -> Option { + Some(self.len()) + } +} + +impl<'data, T: Send> IndexedParallelIterator for Drain<'data, T> { + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + bridge(self, consumer) + } + + fn len(&self) -> usize { + self.range.len() + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + unsafe { + // Make the vector forget about the drained items, and temporarily the tail too. + self.vec.set_len(self.range.start); + + // Create the producer as the exclusive "owner" of the slice. + let producer = DrainProducer::from_vec(self.vec, self.range.len()); + + // The producer will move or drop each item from the drained range. + callback.callback(producer) + } + } +} + +impl<'data, T: Send> Drop for Drain<'data, T> { + fn drop(&mut self) { + let Range { start, end } = self.range; + if self.vec.len() == self.orig_len { + // We must not have produced, so just call a normal drain to remove the items. + self.vec.drain(start..end); + } else if start == end { + // Empty range, so just restore the length to its original state + unsafe { + self.vec.set_len(self.orig_len); + } + } else if end < self.orig_len { + // The producer was responsible for consuming the drained items. + // Move the tail items to their new place, then set the length to include them. + unsafe { + let ptr = self.vec.as_mut_ptr().add(start); + let tail_ptr = self.vec.as_ptr().add(end); + let tail_len = self.orig_len - end; + ptr::copy(tail_ptr, ptr, tail_len); + self.vec.set_len(start + tail_len); + } + } + } +} + +// //////////////////////////////////////////////////////////////////////// + +pub(crate) struct DrainProducer<'data, T: Send> { + slice: &'data mut [T], +} + +impl DrainProducer<'_, T> { + /// Creates a draining producer, which *moves* items from the slice. + /// + /// Unsafe because `!Copy` data must not be read after the borrow is released. + pub(crate) unsafe fn new(slice: &mut [T]) -> DrainProducer<'_, T> { + DrainProducer { slice } + } + + /// Creates a draining producer, which *moves* items from the tail of the vector. + /// + /// Unsafe because we're moving from beyond `vec.len()`, so the caller must ensure + /// that data is initialized and not read after the borrow is released. + unsafe fn from_vec(vec: &mut Vec, len: usize) -> DrainProducer<'_, T> { + let start = vec.len(); + assert!(vec.capacity() - start >= len); + + // The pointer is derived from `Vec` directly, not through a `Deref`, + // so it has provenance over the whole allocation. + let ptr = vec.as_mut_ptr().add(start); + DrainProducer::new(slice::from_raw_parts_mut(ptr, len)) + } +} + +impl<'data, T: 'data + Send> Producer for DrainProducer<'data, T> { + type Item = T; + type IntoIter = SliceDrain<'data, T>; + + fn into_iter(mut self) -> Self::IntoIter { + // replace the slice so we don't drop it twice + let slice = mem::take(&mut self.slice); + SliceDrain { + iter: slice.iter_mut(), + } + } + + fn split_at(mut self, index: usize) -> (Self, Self) { + // replace the slice so we don't drop it twice + let slice = mem::take(&mut self.slice); + let (left, right) = slice.split_at_mut(index); + unsafe { (DrainProducer::new(left), DrainProducer::new(right)) } + } +} + +impl<'data, T: 'data + Send> Drop for DrainProducer<'data, T> { + fn drop(&mut self) { + // extract the slice so we can use `Drop for [T]` + let slice_ptr: *mut [T] = mem::take::<&'data mut [T]>(&mut self.slice); + unsafe { ptr::drop_in_place::<[T]>(slice_ptr) }; + } +} + +// //////////////////////////////////////////////////////////////////////// + +// like std::vec::Drain, without updating a source Vec +pub(crate) struct SliceDrain<'data, T> { + iter: slice::IterMut<'data, T>, +} + +impl<'data, T: 'data> Iterator for SliceDrain<'data, T> { + type Item = T; + + fn next(&mut self) -> Option { + // Coerce the pointer early, so we don't keep the + // reference that's about to be invalidated. + let ptr: *const T = self.iter.next()?; + Some(unsafe { ptr::read(ptr) }) + } + + fn size_hint(&self) -> (usize, Option) { + self.iter.size_hint() + } + + fn count(self) -> usize { + self.iter.len() + } +} + +impl<'data, T: 'data> DoubleEndedIterator for SliceDrain<'data, T> { + fn next_back(&mut self) -> Option { + // Coerce the pointer early, so we don't keep the + // reference that's about to be invalidated. + let ptr: *const T = self.iter.next_back()?; + Some(unsafe { ptr::read(ptr) }) + } +} + +impl<'data, T: 'data> ExactSizeIterator for SliceDrain<'data, T> { + fn len(&self) -> usize { + self.iter.len() + } +} + +impl<'data, T: 'data> iter::FusedIterator for SliceDrain<'data, T> {} + +impl<'data, T: 'data> Drop for SliceDrain<'data, T> { + fn drop(&mut self) { + // extract the iterator so we can use `Drop for [T]` + let slice_ptr: *mut [T] = mem::replace(&mut self.iter, [].iter_mut()).into_slice(); + unsafe { ptr::drop_in_place::<[T]>(slice_ptr) }; + } +} diff --git a/anneal/v2/vendor/rayon/tests/chars.rs b/anneal/v2/vendor/rayon/tests/chars.rs new file mode 100644 index 0000000000..6e2804f40c --- /dev/null +++ b/anneal/v2/vendor/rayon/tests/chars.rs @@ -0,0 +1,38 @@ +use rayon::prelude::*; + +#[test] +fn half_open_correctness() { + let low = char::from_u32(0xD800 - 0x7).unwrap(); + let high = char::from_u32(0xE000 + 0x7).unwrap(); + + let range = low..high; + let mut chars: Vec = range.into_par_iter().collect(); + chars.sort(); + + assert_eq!( + chars, + vec![ + '\u{D7F9}', '\u{D7FA}', '\u{D7FB}', '\u{D7FC}', '\u{D7FD}', '\u{D7FE}', '\u{D7FF}', + '\u{E000}', '\u{E001}', '\u{E002}', '\u{E003}', '\u{E004}', '\u{E005}', '\u{E006}', + ] + ); +} + +#[test] +fn closed_correctness() { + let low = char::from_u32(0xD800 - 0x7).unwrap(); + let high = char::from_u32(0xE000 + 0x7).unwrap(); + + let range = low..=high; + let mut chars: Vec = range.into_par_iter().collect(); + chars.sort(); + + assert_eq!( + chars, + vec![ + '\u{D7F9}', '\u{D7FA}', '\u{D7FB}', '\u{D7FC}', '\u{D7FD}', '\u{D7FE}', '\u{D7FF}', + '\u{E000}', '\u{E001}', '\u{E002}', '\u{E003}', '\u{E004}', '\u{E005}', '\u{E006}', + '\u{E007}', + ] + ); +} diff --git a/anneal/v2/vendor/rayon/tests/clones.rs b/anneal/v2/vendor/rayon/tests/clones.rs new file mode 100644 index 0000000000..127e47e79a --- /dev/null +++ b/anneal/v2/vendor/rayon/tests/clones.rs @@ -0,0 +1,223 @@ +use rayon::prelude::*; +use std::fmt::Debug; + +fn check(iter: I) +where + I: ParallelIterator + Clone, +{ + let a: Vec<_> = iter.clone().collect(); + let b: Vec<_> = iter.collect(); + assert_eq!(a, b); +} + +fn check_count(iter: I) +where + I: ParallelIterator + Clone, +{ + assert_eq!(iter.clone().count(), iter.count()); +} + +#[test] +fn clone_binary_heap() { + use std::collections::BinaryHeap; + let heap: BinaryHeap<_> = (0..1000).collect(); + check(heap.par_iter()); + check(heap.into_par_iter()); +} + +#[test] +fn clone_btree_map() { + use std::collections::BTreeMap; + let map: BTreeMap<_, _> = (0..1000).enumerate().collect(); + check(map.par_iter()); +} + +#[test] +fn clone_btree_set() { + use std::collections::BTreeSet; + let set: BTreeSet<_> = (0..1000).collect(); + check(set.par_iter()); +} + +#[test] +fn clone_hash_map() { + use std::collections::HashMap; + let map: HashMap<_, _> = (0..1000).enumerate().collect(); + check(map.par_iter()); +} + +#[test] +fn clone_hash_set() { + use std::collections::HashSet; + let set: HashSet<_> = (0..1000).collect(); + check(set.par_iter()); +} + +#[test] +fn clone_linked_list() { + use std::collections::LinkedList; + let list: LinkedList<_> = (0..1000).collect(); + check(list.par_iter()); + check(list.into_par_iter()); +} + +#[test] +fn clone_vec_deque() { + use std::collections::VecDeque; + let deque: VecDeque<_> = (0..1000).collect(); + check(deque.par_iter()); + check(deque.into_par_iter()); +} + +#[test] +fn clone_option() { + let option = Some(0); + check(option.par_iter()); + check(option.into_par_iter()); +} + +#[test] +fn clone_result() { + let result = Ok::<_, ()>(0); + check(result.par_iter()); + check(result.into_par_iter()); +} + +#[test] +fn clone_range() { + check((0..1000).into_par_iter()); +} + +#[test] +fn clone_range_inclusive() { + check((0..=1000).into_par_iter()); +} + +#[test] +fn clone_str() { + let s = include_str!("clones.rs"); + check(s.par_chars()); + check(s.par_lines()); + check(s.par_split('\n')); + check(s.par_split_inclusive('\n')); + check(s.par_split_terminator('\n')); + check(s.par_split_whitespace()); + check(s.par_split_ascii_whitespace()); +} + +#[test] +fn clone_vec() { + let v: Vec<_> = (0..1000).collect(); + check(v.par_iter()); + check(v.par_chunk_by(i32::eq)); + check(v.par_chunks(42)); + check(v.par_chunks_exact(42)); + check(v.par_rchunks(42)); + check(v.par_rchunks_exact(42)); + check(v.par_windows(42)); + check(v.par_array_windows::<42>()); + check(v.par_split(|x| x % 3 == 0)); + check(v.par_split_inclusive(|x| x % 3 == 0)); + check(v.into_par_iter()); +} + +#[test] +fn clone_array() { + let a = [0i32; 100]; + check(a.into_par_iter()); +} + +#[test] +fn clone_adaptors() { + let v: Vec<_> = (0..1000).map(Some).collect(); + check(v.par_iter().by_exponential_blocks()); + check(v.par_iter().by_uniform_blocks(100)); + check(v.par_iter().chain(&v)); + check(v.par_iter().cloned()); + check(v.par_iter().copied()); + check(v.par_iter().enumerate()); + check(v.par_iter().filter(|_| true)); + check(v.par_iter().filter_map(|x| *x)); + check(v.par_iter().flat_map(|x| *x)); + check(v.par_iter().flat_map_iter(|x| *x)); + check(v.par_iter().flatten()); + check(v.par_iter().flatten_iter()); + check(v.par_iter().with_max_len(1).fold(|| 0, |x, _| x)); + check(v.par_iter().with_max_len(1).fold_with(0, |x, _| x)); + check(v.par_iter().with_max_len(1).fold_chunks(1, || 0, |x, _| x)); + check( + v.par_iter() + .with_max_len(1) + .fold_chunks_with(1, 0, |x, _| x), + ); + check(v.par_iter().with_max_len(1).try_fold(|| 0, |_, &x| x)); + check(v.par_iter().with_max_len(1).try_fold_with(0, |_, &x| x)); + check(v.par_iter().inspect(|_| ())); + check(v.par_iter().update(|_| ())); + check(v.par_iter().interleave(&v)); + check(v.par_iter().interleave_shortest(&v)); + check(v.par_iter().intersperse(&None)); + check(v.par_iter().chunks(3)); + check(v.par_iter().map(|x| x)); + check(v.par_iter().map_with(0, |_, x| x)); + check(v.par_iter().map_init(|| 0, |_, x| x)); + check(v.par_iter().panic_fuse()); + check(v.par_iter().positions(|_| true)); + check(v.par_iter().rev()); + check(v.par_iter().skip(42)); + check(v.par_iter().skip_any_while(|_| false)); + check(v.par_iter().take(42)); + check(v.par_iter().take_any_while(|_| true)); + check(v.par_iter().cloned().while_some()); + check(v.par_iter().with_max_len(1)); + check(v.par_iter().with_min_len(1)); + check(v.par_iter().zip(&v)); + check(v.par_iter().zip_eq(&v)); + check(v.par_iter().step_by(2)); +} + +#[test] +fn clone_counted_adaptors() { + let v: Vec<_> = (0..1000).collect(); + check_count(v.par_iter().skip_any(42)); + check_count(v.par_iter().take_any(42)); +} + +#[test] +fn clone_empty() { + check(rayon::iter::empty::()); +} + +#[test] +fn clone_once() { + check(rayon::iter::once(10)); +} + +#[test] +fn clone_repeat() { + let x: Option = None; + check(rayon::iter::repeat(x).while_some()); + check(rayon::iter::repeat_n(x, 1000)); +} + +#[test] +fn clone_splitter() { + check(rayon::iter::split(0..1000, |x| (x, None))); +} + +#[test] +fn clone_multizip() { + let v: &Vec<_> = &(0..1000).collect(); + check((v,).into_par_iter()); + check((v, v).into_par_iter()); + check((v, v, v).into_par_iter()); + check((v, v, v, v).into_par_iter()); + check((v, v, v, v, v).into_par_iter()); + check((v, v, v, v, v, v).into_par_iter()); + check((v, v, v, v, v, v, v).into_par_iter()); + check((v, v, v, v, v, v, v, v).into_par_iter()); + check((v, v, v, v, v, v, v, v, v).into_par_iter()); + check((v, v, v, v, v, v, v, v, v, v).into_par_iter()); + check((v, v, v, v, v, v, v, v, v, v, v).into_par_iter()); + check((v, v, v, v, v, v, v, v, v, v, v, v).into_par_iter()); +} diff --git a/anneal/v2/vendor/rayon/tests/collect.rs b/anneal/v2/vendor/rayon/tests/collect.rs new file mode 100644 index 0000000000..431e829705 --- /dev/null +++ b/anneal/v2/vendor/rayon/tests/collect.rs @@ -0,0 +1,113 @@ +use rayon::prelude::*; + +use std::panic; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; +use std::sync::Mutex; + +#[test] +#[cfg_attr(not(panic = "unwind"), ignore)] +fn collect_drop_on_unwind() { + struct Recorddrop<'a>(i64, &'a Mutex>); + + impl<'a> Drop for Recorddrop<'a> { + fn drop(&mut self) { + self.1.lock().unwrap().push(self.0); + } + } + + let test_collect_panic = |will_panic: bool| { + let test_vec_len = 1024; + let panic_point = 740; + + let mut inserts = Mutex::new(Vec::new()); + let mut drops = Mutex::new(Vec::new()); + + let mut a = (0..test_vec_len).collect::>(); + let b = (0..test_vec_len).collect::>(); + + let _result = panic::catch_unwind(panic::AssertUnwindSafe(|| { + let mut result = Vec::new(); + a.par_iter_mut() + .zip(&b) + .map(|(&mut a, &b)| { + if a > panic_point && will_panic { + panic!("unwinding for test"); + } + let elt = a + b; + inserts.lock().unwrap().push(elt); + Recorddrop(elt, &drops) + }) + .collect_into_vec(&mut result); + + // If we reach this point, this must pass + assert_eq!(a.len(), result.len()); + })); + + let inserts = inserts.get_mut().unwrap(); + let drops = drops.get_mut().unwrap(); + println!("{inserts:?}"); + println!("{drops:?}"); + + assert_eq!(inserts.len(), drops.len(), "Incorrect number of drops"); + // sort to normalize order + inserts.sort(); + drops.sort(); + assert_eq!(inserts, drops, "Incorrect elements were dropped"); + }; + + for &should_panic in &[true, false] { + test_collect_panic(should_panic); + } +} + +#[test] +#[cfg_attr(not(panic = "unwind"), ignore)] +fn collect_drop_on_unwind_zst() { + static INSERTS: AtomicUsize = AtomicUsize::new(0); + static DROPS: AtomicUsize = AtomicUsize::new(0); + + struct RecorddropZst; + + impl Drop for RecorddropZst { + fn drop(&mut self) { + DROPS.fetch_add(1, Ordering::SeqCst); + } + } + + let test_collect_panic = |will_panic: bool| { + INSERTS.store(0, Ordering::SeqCst); + DROPS.store(0, Ordering::SeqCst); + + let test_vec_len = 1024; + let panic_point = 740; + + let a = (0..test_vec_len).collect::>(); + + let _result = panic::catch_unwind(panic::AssertUnwindSafe(|| { + let mut result = Vec::new(); + a.par_iter() + .map(|&a| { + if a > panic_point && will_panic { + panic!("unwinding for test"); + } + INSERTS.fetch_add(1, Ordering::SeqCst); + RecorddropZst + }) + .collect_into_vec(&mut result); + + // If we reach this point, this must pass + assert_eq!(a.len(), result.len()); + })); + + let inserts = INSERTS.load(Ordering::SeqCst); + let drops = DROPS.load(Ordering::SeqCst); + + assert_eq!(inserts, drops, "Incorrect number of drops"); + assert!(will_panic || drops == test_vec_len) + }; + + for &should_panic in &[true, false] { + test_collect_panic(should_panic); + } +} diff --git a/anneal/v2/vendor/rayon/tests/cross-pool.rs b/anneal/v2/vendor/rayon/tests/cross-pool.rs new file mode 100644 index 0000000000..835e2e27f8 --- /dev/null +++ b/anneal/v2/vendor/rayon/tests/cross-pool.rs @@ -0,0 +1,22 @@ +use rayon::prelude::*; +use rayon::ThreadPoolBuilder; + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn cross_pool_busy() { + let pool1 = ThreadPoolBuilder::new().num_threads(1).build().unwrap(); + let pool2 = ThreadPoolBuilder::new().num_threads(1).build().unwrap(); + + let n: i32 = 100; + let sum: i32 = pool1.install(move || { + // Each item will block on pool2, but pool1 can continue processing other work from the + // parallel iterator in the meantime. There's a chance that pool1 will still be awake to + // see the latch set without being tickled, and then it will drop that stack job. The latch + // internals must not assume that the job will still be alive after it's set! + (1..=n) + .into_par_iter() + .map(|i| pool2.install(move || i)) + .sum() + }); + assert_eq!(sum, n * (n + 1) / 2); +} diff --git a/anneal/v2/vendor/rayon/tests/debug.rs b/anneal/v2/vendor/rayon/tests/debug.rs new file mode 100644 index 0000000000..21bafd4bcb --- /dev/null +++ b/anneal/v2/vendor/rayon/tests/debug.rs @@ -0,0 +1,234 @@ +use rayon::prelude::*; +use std::fmt::Debug; + +fn check(iter: I) +where + I: ParallelIterator + Debug, +{ + println!("{iter:?}"); +} + +#[test] +fn debug_binary_heap() { + use std::collections::BinaryHeap; + let mut heap: BinaryHeap<_> = (0..10).collect(); + check(heap.par_iter()); + check(heap.par_drain()); + check(heap.into_par_iter()); +} + +#[test] +fn debug_btree_map() { + use std::collections::BTreeMap; + let mut map: BTreeMap<_, _> = (0..10).enumerate().collect(); + check(map.par_iter()); + check(map.par_iter_mut()); + check(map.into_par_iter()); +} + +#[test] +fn debug_btree_set() { + use std::collections::BTreeSet; + let set: BTreeSet<_> = (0..10).collect(); + check(set.par_iter()); + check(set.into_par_iter()); +} + +#[test] +fn debug_hash_map() { + use std::collections::HashMap; + let mut map: HashMap<_, _> = (0..10).enumerate().collect(); + check(map.par_iter()); + check(map.par_iter_mut()); + check(map.par_drain()); + check(map.into_par_iter()); +} + +#[test] +fn debug_hash_set() { + use std::collections::HashSet; + let mut set: HashSet<_> = (0..10).collect(); + check(set.par_iter()); + check(set.par_drain()); + check(set.into_par_iter()); +} + +#[test] +fn debug_linked_list() { + use std::collections::LinkedList; + let mut list: LinkedList<_> = (0..10).collect(); + check(list.par_iter()); + check(list.par_iter_mut()); + check(list.into_par_iter()); +} + +#[test] +fn debug_vec_deque() { + use std::collections::VecDeque; + let mut deque: VecDeque<_> = (0..10).collect(); + check(deque.par_iter()); + check(deque.par_iter_mut()); + check(deque.par_drain(..)); + check(deque.into_par_iter()); +} + +#[test] +fn debug_option() { + let mut option = Some(0); + check(option.par_iter()); + check(option.par_iter_mut()); + check(option.into_par_iter()); +} + +#[test] +fn debug_result() { + let mut result = Ok::<_, ()>(0); + check(result.par_iter()); + check(result.par_iter_mut()); + check(result.into_par_iter()); +} + +#[test] +fn debug_range() { + check((0..10).into_par_iter()); +} + +#[test] +fn debug_range_inclusive() { + check((0..=10).into_par_iter()); +} + +#[test] +fn debug_str() { + let s = "a b c d\ne f g"; + check(s.par_chars()); + check(s.par_lines()); + check(s.par_split('\n')); + check(s.par_split_inclusive('\n')); + check(s.par_split_terminator('\n')); + check(s.par_split_whitespace()); + check(s.par_split_ascii_whitespace()); +} + +#[test] +fn debug_string() { + let mut s = "a b c d\ne f g".to_string(); + s.par_drain(..); +} + +#[test] +fn debug_vec() { + let mut v: Vec<_> = (0..10).collect(); + check(v.par_iter()); + check(v.par_iter_mut()); + check(v.par_chunk_by(i32::eq)); + check(v.par_chunk_by_mut(i32::eq)); + check(v.par_chunks(42)); + check(v.par_chunks_exact(42)); + check(v.par_chunks_mut(42)); + check(v.par_chunks_exact_mut(42)); + check(v.par_rchunks(42)); + check(v.par_rchunks_exact(42)); + check(v.par_rchunks_mut(42)); + check(v.par_rchunks_exact_mut(42)); + check(v.par_windows(42)); + check(v.par_array_windows::<42>()); + check(v.par_split(|x| x % 3 == 0)); + check(v.par_split_inclusive(|x| x % 3 == 0)); + check(v.par_split_mut(|x| x % 3 == 0)); + check(v.par_split_inclusive_mut(|x| x % 3 == 0)); + check(v.par_drain(..)); + check(v.into_par_iter()); +} + +#[test] +fn debug_array() { + let a = [0i32; 10]; + check(a.into_par_iter()); +} + +#[test] +fn debug_adaptors() { + let v: Vec<_> = (0..10).collect(); + check(v.par_iter().by_exponential_blocks()); + check(v.par_iter().by_uniform_blocks(5)); + check(v.par_iter().chain(&v)); + check(v.par_iter().cloned()); + check(v.par_iter().copied()); + check(v.par_iter().enumerate()); + check(v.par_iter().filter(|_| true)); + check(v.par_iter().filter_map(Some)); + check(v.par_iter().flat_map(Some)); + check(v.par_iter().flat_map_iter(Some)); + check(v.par_iter().map(Some).flatten()); + check(v.par_iter().map(Some).flatten_iter()); + check(v.par_iter().fold(|| 0, |x, _| x)); + check(v.par_iter().fold_with(0, |x, _| x)); + check(v.par_iter().fold_chunks(3, || 0, |x, _| x)); + check(v.par_iter().fold_chunks_with(3, 0, |x, _| x)); + check(v.par_iter().try_fold(|| 0, |x, _| Some(x))); + check(v.par_iter().try_fold_with(0, |x, _| Some(x))); + check(v.par_iter().inspect(|_| ())); + check(v.par_iter().update(|_| ())); + check(v.par_iter().interleave(&v)); + check(v.par_iter().interleave_shortest(&v)); + check(v.par_iter().intersperse(&-1)); + check(v.par_iter().chunks(3)); + check(v.par_iter().map(|x| x)); + check(v.par_iter().map_with(0, |_, x| x)); + check(v.par_iter().map_init(|| 0, |_, x| x)); + check(v.par_iter().panic_fuse()); + check(v.par_iter().positions(|_| true)); + check(v.par_iter().rev()); + check(v.par_iter().skip(1)); + check(v.par_iter().skip_any(1)); + check(v.par_iter().skip_any_while(|_| false)); + check(v.par_iter().take(1)); + check(v.par_iter().take_any(1)); + check(v.par_iter().take_any_while(|_| true)); + check(v.par_iter().map(Some).while_some()); + check(v.par_iter().with_max_len(1)); + check(v.par_iter().with_min_len(1)); + check(v.par_iter().zip(&v)); + check(v.par_iter().zip_eq(&v)); + check(v.par_iter().step_by(2)); +} + +#[test] +fn debug_empty() { + check(rayon::iter::empty::()); +} + +#[test] +fn debug_once() { + check(rayon::iter::once(10)); +} + +#[test] +fn debug_repeat() { + let x: Option = None; + check(rayon::iter::repeat(x)); + check(rayon::iter::repeat_n(x, 10)); +} + +#[test] +fn debug_splitter() { + check(rayon::iter::split(0..10, |x| (x, None))); +} + +#[test] +fn debug_multizip() { + let v: &Vec<_> = &(0..10).collect(); + check((v,).into_par_iter()); + check((v, v).into_par_iter()); + check((v, v, v).into_par_iter()); + check((v, v, v, v).into_par_iter()); + check((v, v, v, v, v).into_par_iter()); + check((v, v, v, v, v, v).into_par_iter()); + check((v, v, v, v, v, v, v).into_par_iter()); + check((v, v, v, v, v, v, v, v).into_par_iter()); + check((v, v, v, v, v, v, v, v, v).into_par_iter()); + check((v, v, v, v, v, v, v, v, v, v).into_par_iter()); + check((v, v, v, v, v, v, v, v, v, v, v).into_par_iter()); + check((v, v, v, v, v, v, v, v, v, v, v, v).into_par_iter()); +} diff --git a/anneal/v2/vendor/rayon/tests/drain_vec.rs b/anneal/v2/vendor/rayon/tests/drain_vec.rs new file mode 100644 index 0000000000..08f1120b79 --- /dev/null +++ b/anneal/v2/vendor/rayon/tests/drain_vec.rs @@ -0,0 +1,41 @@ +use rayon::prelude::*; + +#[test] +fn drain_vec_yielded() { + let mut vec_org = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]; + + let yielded = vec_org.par_drain(0..5).collect::>(); + + assert_eq!(&yielded, &[0, 1, 2, 3, 4]); + assert_eq!(&vec_org, &[5, 6, 7, 8, 9]); +} + +#[test] +fn drain_vec_dropped() { + let mut vec_org = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]; + + let yielded = vec_org.par_drain(0..5); + + drop(yielded); + assert_eq!(&vec_org, &[5, 6, 7, 8, 9]); +} + +#[test] +fn drain_vec_empty_range_yielded() { + let mut vec_org = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]; + + let yielded = vec_org.par_drain(5..5).collect::>(); + + assert_eq!(&yielded, &[]); + assert_eq!(&vec_org, &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); +} + +#[test] +fn drain_vec_empty_range_dropped() { + let mut vec_org = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]; + + let yielded = vec_org.par_drain(5..5); + + drop(yielded); + assert_eq!(&vec_org, &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); +} diff --git a/anneal/v2/vendor/rayon/tests/intersperse.rs b/anneal/v2/vendor/rayon/tests/intersperse.rs new file mode 100644 index 0000000000..aaa5b6530d --- /dev/null +++ b/anneal/v2/vendor/rayon/tests/intersperse.rs @@ -0,0 +1,60 @@ +use rayon::prelude::*; + +#[test] +fn check_intersperse() { + let v: Vec<_> = (0..1000).into_par_iter().intersperse(-1).collect(); + assert_eq!(v.len(), 1999); + for (i, x) in v.into_iter().enumerate() { + assert_eq!(x, if i % 2 == 0 { i as i32 / 2 } else { -1 }); + } +} + +#[test] +fn check_intersperse_again() { + let v: Vec<_> = (0..1000) + .into_par_iter() + .intersperse(-1) + .intersperse(-2) + .collect(); + assert_eq!(v.len(), 3997); + for (i, x) in v.into_iter().enumerate() { + let y = match i % 4 { + 0 => i as i32 / 4, + 2 => -1, + _ => -2, + }; + assert_eq!(x, y); + } +} + +#[test] +fn check_intersperse_unindexed() { + let v: Vec<_> = (0..1000).map(|i| i.to_string()).collect(); + let s = v.join(","); + let s2 = v.join(";"); + let par: String = s.par_split(',').intersperse(";").collect(); + assert_eq!(par, s2); +} + +#[test] +fn check_intersperse_producer() { + (0..1000) + .into_par_iter() + .intersperse(-1) + .zip_eq(0..1999) + .for_each(|(x, i)| { + assert_eq!(x, if i % 2 == 0 { i / 2 } else { -1 }); + }); +} + +#[test] +fn check_intersperse_rev() { + (0..1000) + .into_par_iter() + .intersperse(-1) + .zip_eq(0..1999) + .rev() + .for_each(|(x, i)| { + assert_eq!(x, if i % 2 == 0 { i / 2 } else { -1 }); + }); +} diff --git a/anneal/v2/vendor/rayon/tests/issue671-unzip.rs b/anneal/v2/vendor/rayon/tests/issue671-unzip.rs new file mode 100644 index 0000000000..c9af7e6e3a --- /dev/null +++ b/anneal/v2/vendor/rayon/tests/issue671-unzip.rs @@ -0,0 +1,17 @@ +#![type_length_limit = "10000"] + +use rayon::prelude::*; + +#[test] +fn type_length_limit() { + let input = vec![1, 2, 3, 4, 5]; + let (indexes, (squares, cubes)): (Vec<_>, (Vec<_>, Vec<_>)) = input + .par_iter() + .map(|x| (x * x, x * x * x)) + .enumerate() + .unzip(); + + drop(indexes); + drop(squares); + drop(cubes); +} diff --git a/anneal/v2/vendor/rayon/tests/issue671.rs b/anneal/v2/vendor/rayon/tests/issue671.rs new file mode 100644 index 0000000000..9d0953fe6a --- /dev/null +++ b/anneal/v2/vendor/rayon/tests/issue671.rs @@ -0,0 +1,16 @@ +#![type_length_limit = "500000"] + +use rayon::prelude::*; + +#[test] +fn type_length_limit() { + let _ = Vec::>::new() + .into_par_iter() + .map(|x| x) + .map(|x| x) + .map(|x| x) + .map(|x| x) + .map(|x| x) + .map(|x| x) + .collect::>(); +} diff --git a/anneal/v2/vendor/rayon/tests/iter_panic.rs b/anneal/v2/vendor/rayon/tests/iter_panic.rs new file mode 100644 index 0000000000..c62a8dbdaf --- /dev/null +++ b/anneal/v2/vendor/rayon/tests/iter_panic.rs @@ -0,0 +1,53 @@ +use rayon::prelude::*; +use rayon::ThreadPoolBuilder; +use std::ops::Range; +use std::panic::{self, UnwindSafe}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +const ITER: Range = 0..0x1_0000; +const PANIC: i32 = 0xC000; + +fn check(&i: &i32) { + if i == PANIC { + panic!("boom") + } +} + +#[test] +#[should_panic(expected = "boom")] +fn iter_panic() { + ITER.into_par_iter().for_each(|i| check(&i)); +} + +#[test] +#[cfg_attr(not(panic = "unwind"), ignore)] +fn iter_panic_fuse() { + // We only use a single thread in order to make the behavior + // of 'panic_fuse' deterministic + let pool = ThreadPoolBuilder::new().num_threads(1).build().unwrap(); + + pool.install(|| { + fn count(iter: impl ParallelIterator + UnwindSafe) -> usize { + let count = AtomicUsize::new(0); + let result = panic::catch_unwind(|| { + iter.for_each(|_| { + count.fetch_add(1, Ordering::Relaxed); + }); + }); + assert!(result.is_err()); + count.into_inner() + } + + // Without `panic_fuse()`, we'll reach every item except the panicking one. + let expected = ITER.len() - 1; + let iter = ITER.into_par_iter().with_max_len(1); + assert_eq!(count(iter.clone().inspect(check)), expected); + + // With `panic_fuse()` anywhere in the chain, we'll reach fewer items. + assert!(count(iter.clone().inspect(check).panic_fuse()) < expected); + assert!(count(iter.clone().panic_fuse().inspect(check)) < expected); + + // Try in reverse to be sure we hit the producer case. + assert!(count(iter.panic_fuse().inspect(check).rev()) < expected); + }); +} diff --git a/anneal/v2/vendor/rayon/tests/named-threads.rs b/anneal/v2/vendor/rayon/tests/named-threads.rs new file mode 100644 index 0000000000..7eb4b96ddd --- /dev/null +++ b/anneal/v2/vendor/rayon/tests/named-threads.rs @@ -0,0 +1,25 @@ +use std::collections::HashSet; + +use rayon::prelude::*; +use rayon::*; + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn named_threads() { + ThreadPoolBuilder::new() + .thread_name(|i| format!("hello-name-test-{i}")) + .build_global() + .unwrap(); + + const N: usize = 10000; + + let thread_names = (0..N) + .into_par_iter() + .flat_map(|_| ::std::thread::current().name().map(str::to_owned)) + .collect::>(); + + let all_contains_name = thread_names + .iter() + .all(|name| name.starts_with("hello-name-test-")); + assert!(all_contains_name); +} diff --git a/anneal/v2/vendor/rayon/tests/octillion.rs b/anneal/v2/vendor/rayon/tests/octillion.rs new file mode 100644 index 0000000000..1af9ad8bad --- /dev/null +++ b/anneal/v2/vendor/rayon/tests/octillion.rs @@ -0,0 +1,156 @@ +use rayon::prelude::*; + +const OCTILLION: u128 = 1_000_000_000_000_000_000_000_000_000; + +/// Produce a parallel iterator for 0u128..10²⁷ +fn octillion() -> rayon::range::Iter { + (0..OCTILLION).into_par_iter() +} + +/// Produce a parallel iterator for 0u128..=10²⁷ +fn octillion_inclusive() -> rayon::range_inclusive::Iter { + (0..=OCTILLION).into_par_iter() +} + +/// Produce a parallel iterator for 0u128..10²⁷ using `flat_map` +fn octillion_flat() -> impl ParallelIterator { + (0u32..1_000_000_000) + .into_par_iter() + .with_max_len(1_000) + .map(|i| u64::from(i) * 1_000_000_000) + .flat_map(|i| { + (0u32..1_000_000_000) + .into_par_iter() + .with_max_len(1_000) + .map(move |j| i + u64::from(j)) + }) + .map(|i| u128::from(i) * 1_000_000_000) + .flat_map(|i| { + (0u32..1_000_000_000) + .into_par_iter() + .with_max_len(1_000) + .map(move |j| i + u128::from(j)) + }) +} + +// NOTE: `find_first` and `find_last` currently take too long on 32-bit targets, +// because the `AtomicUsize` match position has much too limited resolution. + +#[test] +#[cfg_attr(not(target_pointer_width = "64"), ignore)] +fn find_first_octillion() { + let x = octillion().find_first(|_| true); + assert_eq!(x, Some(0)); +} + +#[test] +#[cfg_attr(not(target_pointer_width = "64"), ignore)] +fn find_first_octillion_inclusive() { + let x = octillion_inclusive().find_first(|_| true); + assert_eq!(x, Some(0)); +} + +#[test] +#[cfg_attr(not(target_pointer_width = "64"), ignore)] +fn find_first_octillion_flat() { + let x = octillion_flat().find_first(|_| true); + assert_eq!(x, Some(0)); +} + +fn two_threads R, R: Send>(f: F) -> R { + // FIXME: If we don't use at least two threads, then we end up walking + // through the entire iterator sequentially, without the benefit of any + // short-circuiting. We probably don't want testing to wait that long. ;) + let builder = rayon::ThreadPoolBuilder::new().num_threads(2); + let pool = builder.build().unwrap(); + + pool.install(f) +} + +#[test] +#[cfg_attr( + any( + not(target_pointer_width = "64"), + target_os = "emscripten", + target_family = "wasm" + ), + ignore +)] +fn find_last_octillion() { + // It would be nice if `find_last` could prioritize the later splits, + // basically flipping the `join` args, without needing indexed `rev`. + // (or could we have an unindexed `rev`?) + let x = two_threads(|| octillion().find_last(|_| true)); + assert_eq!(x, Some(OCTILLION - 1)); +} + +#[test] +#[cfg_attr( + any( + not(target_pointer_width = "64"), + target_os = "emscripten", + target_family = "wasm" + ), + ignore +)] +fn find_last_octillion_inclusive() { + let x = two_threads(|| octillion_inclusive().find_last(|_| true)); + assert_eq!(x, Some(OCTILLION)); +} + +#[test] +#[cfg_attr( + any( + not(target_pointer_width = "64"), + target_os = "emscripten", + target_family = "wasm" + ), + ignore +)] +fn find_last_octillion_flat() { + let x = two_threads(|| octillion_flat().find_last(|_| true)); + assert_eq!(x, Some(OCTILLION - 1)); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn find_any_octillion() { + let x = two_threads(|| octillion().find_any(|x| *x > OCTILLION / 2)); + assert!(x.is_some()); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn find_any_octillion_flat() { + let x = two_threads(|| octillion_flat().find_any(|x| *x > OCTILLION / 2)); + assert!(x.is_some()); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn filter_find_any_octillion() { + let x = two_threads(|| { + octillion() + .filter(|x| *x > OCTILLION / 2) + .find_any(|_| true) + }); + assert!(x.is_some()); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn filter_find_any_octillion_flat() { + let x = two_threads(|| { + octillion_flat() + .filter(|x| *x > OCTILLION / 2) + .find_any(|_| true) + }); + assert!(x.is_some()); +} + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn fold_find_any_octillion_flat() { + let x = two_threads(|| octillion_flat().fold(|| (), |_, _| ()).find_any(|_| true)); + assert!(x.is_some()); +} diff --git a/anneal/v2/vendor/rayon/tests/par_bridge_recursion.rs b/anneal/v2/vendor/rayon/tests/par_bridge_recursion.rs new file mode 100644 index 0000000000..3a48ef143d --- /dev/null +++ b/anneal/v2/vendor/rayon/tests/par_bridge_recursion.rs @@ -0,0 +1,31 @@ +use rayon::prelude::*; +use std::iter::once_with; + +const N: usize = 100_000; + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn par_bridge_recursion() { + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(10) + .build() + .unwrap(); + + let seq: Vec<_> = (0..N).map(|i| (i, i.to_string())).collect(); + + pool.broadcast(|_| { + let mut par: Vec<_> = (0..N) + .into_par_iter() + .flat_map(|i| { + once_with(move || { + // Using rayon within the serial iterator creates an opportunity for + // work-stealing to make par_bridge's mutex accidentally recursive. + rayon::join(move || i, move || i.to_string()) + }) + .par_bridge() + }) + .collect(); + par.par_sort_unstable(); + assert_eq!(seq, par); + }); +} diff --git a/anneal/v2/vendor/rayon/tests/producer_split_at.rs b/anneal/v2/vendor/rayon/tests/producer_split_at.rs new file mode 100644 index 0000000000..4571a6a3bd --- /dev/null +++ b/anneal/v2/vendor/rayon/tests/producer_split_at.rs @@ -0,0 +1,393 @@ +use rayon::iter::plumbing::*; +use rayon::prelude::*; +use std::fmt::Debug; + +/// Stress-test indexes for `Producer::split_at`. +fn check(expected: &[I::Item], mut f: F) +where + F: FnMut() -> I, + I: IntoParallelIterator, +{ + map_triples(expected.len() + 1, |i, j, k| { + Split::forward(f(), i, j, k, expected); + Split::reverse(f(), i, j, k, expected); + }); +} + +fn map_triples(end: usize, mut f: F) +where + F: FnMut(usize, usize, usize), +{ + for i in 0..end { + for j in i..end { + for k in j..end { + f(i, j, k); + } + } + } +} + +#[derive(Debug)] +struct Split { + i: usize, + j: usize, + k: usize, + reverse: bool, +} + +impl Split { + fn forward(iter: I, i: usize, j: usize, k: usize, expected: &[I::Item]) + where + I: IntoParallelIterator, + { + let result = iter.into_par_iter().with_producer(Split { + i, + j, + k, + reverse: false, + }); + assert_eq!(result, expected); + } + + fn reverse(iter: I, i: usize, j: usize, k: usize, expected: &[I::Item]) + where + I: IntoParallelIterator, + { + let result = iter.into_par_iter().with_producer(Split { + i, + j, + k, + reverse: true, + }); + assert!(result.iter().eq(expected.iter().rev())); + } +} + +impl ProducerCallback for Split { + type Output = Vec; + + fn callback

(self, producer: P) -> Self::Output + where + P: Producer, + { + println!("{self:?}"); + + // Splitting the outer indexes first gets us an arbitrary mid section, + // which we then split further to get full test coverage. + let (left, d) = producer.split_at(self.k); + let (a, mid) = left.split_at(self.i); + let (b, c) = mid.split_at(self.j - self.i); + + let a = a.into_iter(); + let b = b.into_iter(); + let c = c.into_iter(); + let d = d.into_iter(); + + check_len(&a, self.i); + check_len(&b, self.j - self.i); + check_len(&c, self.k - self.j); + + let chain = a.chain(b).chain(c).chain(d); + if self.reverse { + chain.rev().collect() + } else { + chain.collect() + } + } +} + +fn check_len(iter: &I, len: usize) { + assert_eq!(iter.size_hint(), (len, Some(len))); + assert_eq!(iter.len(), len); +} + +// **** Base Producers **** + +#[test] +fn array() { + let a = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]; + check(&a, || a); +} + +#[test] +fn empty() { + check(&[], rayon::iter::empty::); +} + +#[test] +fn once() { + check(&[42], || rayon::iter::once(42)); +} + +#[test] +fn option() { + check(&[42], || Some(42)); +} + +#[test] +fn range() { + let v: Vec<_> = (0..10).collect(); + check(&v, || 0..10); +} + +#[test] +fn range_inclusive() { + let v: Vec<_> = (0u16..=10).collect(); + check(&v, || 0u16..=10); +} + +#[test] +fn repeat_n() { + let v: Vec<_> = std::iter::repeat(1).take(5).collect(); + check(&v, || rayon::iter::repeat_n(1, 5)); +} + +#[test] +fn slice_iter() { + let s: Vec<_> = (0..10).collect(); + let v: Vec<_> = s.iter().collect(); + check(&v, || &s); +} + +#[test] +fn slice_iter_mut() { + let mut s: Vec<_> = (0..10).collect(); + let mut v: Vec<_> = s.clone(); + let expected: Vec<_> = v.iter_mut().collect(); + + map_triples(expected.len() + 1, |i, j, k| { + Split::forward(s.par_iter_mut(), i, j, k, &expected); + Split::reverse(s.par_iter_mut(), i, j, k, &expected); + }); +} + +#[test] +fn slice_chunks() { + let s: Vec<_> = (0..10).collect(); + for len in 1..s.len() + 2 { + let v: Vec<_> = s.chunks(len).collect(); + check(&v, || s.par_chunks(len)); + } +} + +#[test] +fn slice_chunks_exact() { + let s: Vec<_> = (0..10).collect(); + for len in 1..s.len() + 2 { + let v: Vec<_> = s.chunks_exact(len).collect(); + check(&v, || s.par_chunks_exact(len)); + } +} + +#[test] +fn slice_chunks_mut() { + let mut s: Vec<_> = (0..10).collect(); + let mut v: Vec<_> = s.clone(); + for len in 1..s.len() + 2 { + let expected: Vec<_> = v.chunks_mut(len).collect(); + map_triples(expected.len() + 1, |i, j, k| { + Split::forward(s.par_chunks_mut(len), i, j, k, &expected); + Split::reverse(s.par_chunks_mut(len), i, j, k, &expected); + }); + } +} + +#[test] +fn slice_chunks_exact_mut() { + let mut s: Vec<_> = (0..10).collect(); + let mut v: Vec<_> = s.clone(); + for len in 1..s.len() + 2 { + let expected: Vec<_> = v.chunks_exact_mut(len).collect(); + map_triples(expected.len() + 1, |i, j, k| { + Split::forward(s.par_chunks_exact_mut(len), i, j, k, &expected); + Split::reverse(s.par_chunks_exact_mut(len), i, j, k, &expected); + }); + } +} + +#[test] +fn slice_rchunks() { + let s: Vec<_> = (0..10).collect(); + for len in 1..s.len() + 2 { + let v: Vec<_> = s.rchunks(len).collect(); + check(&v, || s.par_rchunks(len)); + } +} + +#[test] +fn slice_rchunks_exact() { + let s: Vec<_> = (0..10).collect(); + for len in 1..s.len() + 2 { + let v: Vec<_> = s.rchunks_exact(len).collect(); + check(&v, || s.par_rchunks_exact(len)); + } +} + +#[test] +fn slice_rchunks_mut() { + let mut s: Vec<_> = (0..10).collect(); + let mut v: Vec<_> = s.clone(); + for len in 1..s.len() + 2 { + let expected: Vec<_> = v.rchunks_mut(len).collect(); + map_triples(expected.len() + 1, |i, j, k| { + Split::forward(s.par_rchunks_mut(len), i, j, k, &expected); + Split::reverse(s.par_rchunks_mut(len), i, j, k, &expected); + }); + } +} + +#[test] +fn slice_rchunks_exact_mut() { + let mut s: Vec<_> = (0..10).collect(); + let mut v: Vec<_> = s.clone(); + for len in 1..s.len() + 2 { + let expected: Vec<_> = v.rchunks_exact_mut(len).collect(); + map_triples(expected.len() + 1, |i, j, k| { + Split::forward(s.par_rchunks_exact_mut(len), i, j, k, &expected); + Split::reverse(s.par_rchunks_exact_mut(len), i, j, k, &expected); + }); + } +} + +#[test] +fn slice_windows() { + let s: Vec<_> = (0..10).collect(); + let v: Vec<_> = s.windows(2).collect(); + check(&v, || s.par_windows(2)); +} + +#[test] +fn slice_array_windows() { + let s: Vec<_> = (0..10).collect(); + let v: Vec<&[_; 2]> = s.array_windows().collect(); + check(&v, || s.par_array_windows::<2>()); +} + +#[test] +fn vec() { + let v: Vec<_> = (0..10).collect(); + check(&v, || v.clone()); +} + +// **** Adaptors **** + +#[test] +fn chain() { + let v: Vec<_> = (0..10).collect(); + check(&v, || (0..5).into_par_iter().chain(5..10)); +} + +#[test] +fn cloned() { + let v: Vec<_> = (0..10).collect(); + check(&v, || v.par_iter().cloned()); +} + +#[test] +fn copied() { + let v: Vec<_> = (0..10).collect(); + check(&v, || v.par_iter().copied()); +} + +#[test] +fn enumerate() { + let v: Vec<_> = (0..10).enumerate().collect(); + check(&v, || (0..10).into_par_iter().enumerate()); +} + +#[test] +fn step_by() { + let v: Vec<_> = (0..10).step_by(2).collect(); + check(&v, || (0..10).into_par_iter().step_by(2)) +} + +#[test] +fn step_by_unaligned() { + let v: Vec<_> = (0..10).step_by(3).collect(); + check(&v, || (0..10).into_par_iter().step_by(3)) +} + +#[test] +fn inspect() { + let v: Vec<_> = (0..10).collect(); + check(&v, || (0..10).into_par_iter().inspect(|_| ())); +} + +#[test] +fn update() { + let v: Vec<_> = (0..10).collect(); + check(&v, || (0..10).into_par_iter().update(|_| ())); +} + +#[test] +fn interleave() { + let v = [0, 10, 1, 11, 2, 12, 3, 4]; + check(&v, || (0..5).into_par_iter().interleave(10..13)); + check(&v[..6], || (0..3).into_par_iter().interleave(10..13)); + + let v = [0, 10, 1, 11, 2, 12, 13, 14]; + check(&v, || (0..3).into_par_iter().interleave(10..15)); +} + +#[test] +fn intersperse() { + let v = [0, -1, 1, -1, 2, -1, 3, -1, 4]; + check(&v, || (0..5).into_par_iter().intersperse(-1)); +} + +#[test] +fn chunks() { + let s: Vec<_> = (0..10).collect(); + let v: Vec<_> = s.chunks(2).map(|c| c.to_vec()).collect(); + check(&v, || s.par_iter().cloned().chunks(2)); +} + +#[test] +fn map() { + let v: Vec<_> = (0..10).collect(); + check(&v, || v.par_iter().map(Clone::clone)); +} + +#[test] +fn map_with() { + let v: Vec<_> = (0..10).collect(); + check(&v, || v.par_iter().map_with(vec![0], |_, &x| x)); +} + +#[test] +fn map_init() { + let v: Vec<_> = (0..10).collect(); + check(&v, || v.par_iter().map_init(|| vec![0], |_, &x| x)); +} + +#[test] +fn panic_fuse() { + let v: Vec<_> = (0..10).collect(); + check(&v, || (0..10).into_par_iter().panic_fuse()); +} + +#[test] +fn rev() { + let v: Vec<_> = (0..10).rev().collect(); + check(&v, || (0..10).into_par_iter().rev()); +} + +#[test] +fn with_max_len() { + let v: Vec<_> = (0..10).collect(); + check(&v, || (0..10).into_par_iter().with_max_len(1)); +} + +#[test] +fn with_min_len() { + let v: Vec<_> = (0..10).collect(); + check(&v, || (0..10).into_par_iter().with_min_len(1)); +} + +#[test] +fn zip() { + let v: Vec<_> = (0..10).zip(10..20).collect(); + check(&v, || (0..10).into_par_iter().zip(10..20)); + check(&v[..5], || (0..5).into_par_iter().zip(10..20)); + check(&v[..5], || (0..10).into_par_iter().zip(10..15)); +} diff --git a/anneal/v2/vendor/rayon/tests/sort-panic-safe.rs b/anneal/v2/vendor/rayon/tests/sort-panic-safe.rs new file mode 100644 index 0000000000..eb9a417e37 --- /dev/null +++ b/anneal/v2/vendor/rayon/tests/sort-panic-safe.rs @@ -0,0 +1,164 @@ +use rand::distr::Uniform; +use rand::{rng, Rng}; +use rayon::prelude::*; +use std::cell::Cell; +use std::cmp::Ordering; +use std::panic; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering::Relaxed; +use std::thread; + +const LEN: usize = 20_000; + +static VERSIONS: AtomicUsize = AtomicUsize::new(0); + +static DROP_COUNTS: [AtomicUsize; LEN] = [const { AtomicUsize::new(0) }; LEN]; + +#[derive(Clone, Eq)] +struct DropCounter { + x: u32, + id: usize, + version: Cell, +} + +impl PartialEq for DropCounter { + fn eq(&self, other: &Self) -> bool { + self.partial_cmp(other) == Some(Ordering::Equal) + } +} + +#[allow(clippy::non_canonical_partial_ord_impl)] +impl PartialOrd for DropCounter { + fn partial_cmp(&self, other: &Self) -> Option { + self.version.set(self.version.get() + 1); + other.version.set(other.version.get() + 1); + VERSIONS.fetch_add(2, Relaxed); + self.x.partial_cmp(&other.x) + } +} + +impl Ord for DropCounter { + fn cmp(&self, other: &Self) -> Ordering { + self.partial_cmp(other).unwrap() + } +} + +impl Drop for DropCounter { + fn drop(&mut self) { + DROP_COUNTS[self.id].fetch_add(1, Relaxed); + VERSIONS.fetch_sub(self.version.get(), Relaxed); + } +} + +macro_rules! test { + ($input:ident, $func:ident) => { + let len = $input.len(); + + // Work out the total number of comparisons required to sort + // this array... + let count = AtomicUsize::new(0); + $input.to_owned().$func(|a, b| { + count.fetch_add(1, Relaxed); + a.cmp(b) + }); + + let mut panic_countdown = count.load(Relaxed); + let step = if len <= 100 { + 1 + } else { + Ord::max(1, panic_countdown / 10) + }; + + // ... and then panic after each `step` comparisons. + loop { + // Refresh the counters. + VERSIONS.store(0, Relaxed); + for i in 0..len { + DROP_COUNTS[i].store(0, Relaxed); + } + + let v = $input.to_owned(); + let _ = thread::spawn(move || { + let mut v = v; + let panic_countdown = AtomicUsize::new(panic_countdown); + v.$func(|a, b| { + if panic_countdown.fetch_sub(1, Relaxed) == 1 { + SILENCE_PANIC.set(true); + panic!(); + } + a.cmp(b) + }) + }) + .join(); + + // Check that the number of things dropped is exactly + // what we expect (i.e. the contents of `v`). + for (i, c) in DROP_COUNTS.iter().enumerate().take(len) { + let count = c.load(Relaxed); + assert!( + count == 1, + "found drop count == {} for i == {}, len == {}", + count, + i, + len + ); + } + + // Check that the most recent versions of values were dropped. + assert_eq!(VERSIONS.load(Relaxed), 0); + + if panic_countdown < step { + break; + } + panic_countdown -= step; + } + }; +} + +thread_local!(static SILENCE_PANIC: Cell = const { Cell::new(false) }); + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_family = "wasm"), ignore)] +fn sort_panic_safe() { + let prev = panic::take_hook(); + panic::set_hook(Box::new(move |info| { + if !SILENCE_PANIC.get() { + prev(info); + } + })); + + for &len in &[1, 2, 3, 4, 5, 10, 20, 100, 500, 5_000, 20_000] { + let len_dist = Uniform::new(0, len).unwrap(); + for &modulus in &[5, 30, 1_000, 20_000] { + for &has_runs in &[false, true] { + let mut rng = rng(); + let mut input = (0..len) + .map(|id| DropCounter { + x: rng.random_range(0..modulus), + id, + version: Cell::new(0), + }) + .collect::>(); + + if has_runs { + for c in &mut input { + c.x = c.id as u32; + } + + for _ in 0..5 { + let a = rng.sample(len_dist); + let b = rng.sample(len_dist); + if a < b { + input[a..b].reverse(); + } else { + input.swap(a, b); + } + } + } + + test!(input, par_sort_by); + test!(input, par_sort_unstable_by); + } + } + } +} diff --git a/anneal/v2/vendor/rayon/tests/str.rs b/anneal/v2/vendor/rayon/tests/str.rs new file mode 100644 index 0000000000..aef4ea1156 --- /dev/null +++ b/anneal/v2/vendor/rayon/tests/str.rs @@ -0,0 +1,122 @@ +use rand::distr::StandardUniform; +use rand::{Rng, SeedableRng}; +use rand_xorshift::XorShiftRng; +use rayon::prelude::*; + +fn seeded_rng() -> XorShiftRng { + let mut seed = ::Seed::default(); + (0..).zip(seed.as_mut()).for_each(|(i, x)| *x = i); + XorShiftRng::from_seed(seed) +} + +#[test] +pub fn execute_strings() { + let rng = seeded_rng(); + let s: String = rng + .sample_iter::(&StandardUniform) + .take(1024) + .collect(); + + let par_chars: String = s.par_chars().collect(); + assert_eq!(s, par_chars); + + let par_even: String = s.par_chars().filter(|&c| (c as u32) & 1 == 0).collect(); + let ser_even: String = s.chars().filter(|&c| (c as u32) & 1 == 0).collect(); + assert_eq!(par_even, ser_even); + + // test `FromParallelIterator<&char> for String` + let vchars: Vec = s.par_chars().collect(); + let par_chars: String = vchars.par_iter().collect(); + assert_eq!(s, par_chars); + + let par_bytes: Vec = s.par_bytes().collect(); + assert_eq!(s.as_bytes(), &*par_bytes); + + let par_utf16: Vec = s.par_encode_utf16().collect(); + let ser_utf16: Vec = s.encode_utf16().collect(); + assert_eq!(par_utf16, ser_utf16); + + let par_charind: Vec<_> = s.par_char_indices().collect(); + let ser_charind: Vec<_> = s.char_indices().collect(); + assert_eq!(par_charind, ser_charind); +} + +#[test] +pub fn execute_strings_split() { + // char testcases from examples in `str::split` etc., + // plus a large self-test for good measure. + let tests = vec![ + ("Mary had a little lamb", ' '), + ("", 'X'), + ("lionXXtigerXleopard", 'X'), + ("||||a||b|c", '|'), + ("(///)", '/'), + ("010", '0'), + (" a b c", ' '), + ("A.B.", '.'), + ("A..B..", '.'), + ("foo\r\nbar\n\nbaz\n", '\n'), + ("foo\nbar\n\r\nbaz", '\n'), + ("A few words", ' '), + (" Mary had\ta\u{2009}little \n\t lamb", ' '), + ("Mary had a little lamb\nlittle lamb\nlittle lamb.", '\n'), + ("Mary had a little lamb\nlittle lamb\nlittle lamb.\n", '\n'), + (include_str!("str.rs"), ' '), + ]; + + macro_rules! check_separators { + ($split:ident, $par_split:ident) => { + for &(string, separator) in &tests { + let serial: Vec<_> = string.$split(separator).collect(); + let parallel: Vec<_> = string.$par_split(separator).collect(); + assert_eq!(serial, parallel); + + let array = ['\u{0}', separator, '\u{1F980}']; + let array_ref = &array; + let slice: &[char] = array_ref; + + let serial: Vec<_> = string.$split(slice).collect(); + let parallel: Vec<_> = string.$par_split(slice).collect(); + assert_eq!(serial, parallel); + + let serial: Vec<_> = string.$split(array).collect(); + let parallel: Vec<_> = string.$par_split(array).collect(); + assert_eq!(serial, parallel); + + let serial: Vec<_> = string.$split(array_ref).collect(); + let parallel: Vec<_> = string.$par_split(array_ref).collect(); + assert_eq!(serial, parallel); + + let serial_fn: Vec<_> = string.$split(|c| c == separator).collect(); + let parallel_fn: Vec<_> = string.$par_split(|c| c == separator).collect(); + assert_eq!(serial_fn, parallel_fn); + } + }; + } + + check_separators!(split, par_split); + check_separators!(split_inclusive, par_split_inclusive); + check_separators!(split_terminator, par_split_terminator); + + for &(string, _) in &tests { + let serial: Vec<_> = string.lines().collect(); + let parallel: Vec<_> = string.par_lines().collect(); + assert_eq!(serial, parallel); + } + + for &(string, _) in &tests { + let serial: Vec<_> = string.split_whitespace().collect(); + let parallel: Vec<_> = string.par_split_whitespace().collect(); + assert_eq!(serial, parallel); + } + + for &(string, _) in &tests { + let serial: Vec<_> = string.split_ascii_whitespace().collect(); + let parallel: Vec<_> = string.par_split_ascii_whitespace().collect(); + assert_eq!(serial, parallel); + } + + // try matching separators too! + check_separators!(matches, par_matches); + check_separators!(match_indices, par_match_indices); +}