diff --git a/Cargo.lock b/Cargo.lock index c49e7b75..402cd83b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -134,6 +134,7 @@ version = "0.11.0" dependencies = [ "cfg-if", "cipher", + "cpufeatures", "hex-literal", ] diff --git a/benches/Cargo.toml b/benches/Cargo.toml index 47e3700c..2d9f25a8 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -12,6 +12,7 @@ publish = false [dependencies] criterion = "0.5" chacha20 = { path = "../chacha20/", features = ["rng", "zeroize"] } +salsa20 = { path = "../salsa20/" } [target.'cfg(any(target_arch = "x86_64", target_arch = "x86", all(target_arch = "aarch64", target_os = "linux")))'.dependencies] criterion-cycles-per-byte = "0.6.0" @@ -19,4 +20,9 @@ criterion-cycles-per-byte = "0.6.0" [[bench]] name = "chacha20" path = "src/chacha20.rs" +harness = false + +[[bench]] +name = "salsa20" +path = "src/salsa20.rs" harness = false \ No newline at end of file diff --git a/benches/src/salsa20.rs b/benches/src/salsa20.rs new file mode 100644 index 00000000..2b8717bb --- /dev/null +++ b/benches/src/salsa20.rs @@ -0,0 +1,32 @@ +//! Salsa20 benchmark +use benches::{criterion_group_bench, Benchmarker}; +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; + +use salsa20::{ + cipher::{KeyIvInit, StreamCipher}, + Salsa20, +}; + +const KB: usize = 1024; +fn bench(c: &mut Benchmarker) { + let mut group = c.benchmark_group("stream-cipher"); + + for size in &[KB, 2 * KB, 4 * KB, 8 * KB, 16 * KB] { + let mut buf = vec![0u8; *size]; + + group.throughput(Throughput::Bytes(*size as u64)); + + group.bench_function(BenchmarkId::new("apply_keystream", size), |b| { + let key = Default::default(); + let nonce = Default::default(); + let mut cipher = Salsa20::new(&key, &nonce); + b.iter(|| cipher.apply_keystream(&mut buf)); + }); + } + + group.finish(); +} + +criterion_group_bench!(benches, bench); + +criterion_main!(benches); diff --git a/salsa20/CHANGELOG.md b/salsa20/CHANGELOG.md index efc51c61..6ec3c5f2 100644 --- a/salsa20/CHANGELOG.md +++ b/salsa20/CHANGELOG.md @@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased +### Added +- AVX2 intrinsics backend with runtime detection via `cpufeatures` ([#576]) + ## 0.11.0 (2026-03-30) ### Added - SSE2 backend ([#333]) @@ -20,6 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [#333]: https://github.com/RustCrypto/stream-ciphers/pull/333 [#338]: https://github.com/RustCrypto/stream-ciphers/pull/338 [#397]: https://github.com/RustCrypto/stream-ciphers/pull/397 +[#576]: https://github.com/RustCrypto/stream-ciphers/pull/576 ## 0.10.2 (2022-02-17) ### Added diff --git a/salsa20/Cargo.toml b/salsa20/Cargo.toml index 3f3e549a..75d8af20 100644 --- a/salsa20/Cargo.toml +++ b/salsa20/Cargo.toml @@ -16,6 +16,9 @@ description = "Pure Rust implementation of the Salsa20 stream cipher" cfg-if = "1" cipher = { version = "0.5", features = ["stream-wrapper"] } +[target.'cfg(any(target_arch = "x86_64", target_arch = "x86"))'.dependencies] +cpufeatures = "0.3" + [dev-dependencies] cipher = { version = "0.5", features = ["dev"] } hex-literal = "1" diff --git a/salsa20/README.md b/salsa20/README.md index 9b7b38be..22f7c91b 100644 --- a/salsa20/README.md +++ b/salsa20/README.md @@ -11,6 +11,9 @@ Implementation of the [Salsa] family of stream ciphers, including the [XSalsa] variants with an extended 192-bit (24-byte) nonce. +SIMD-accelerated on x86/x86\_64 with an AVX2 backend, selected +automatically at runtime via [cpufeatures]. + ## ⚠️ Security Warning: [Hazmat!][hazmat-link] This crate does not ensure ciphertexts are authentic (i.e. by using a MAC to @@ -96,3 +99,4 @@ dual licensed as above, without any additional terms or conditions. [Salsa]: https://en.wikipedia.org/wiki/Salsa20 [XSalsa]: https://cr.yp.to/snuffle/xsalsa-20081128.pdf +[cpufeatures]: https://crates.io/crates/cpufeatures diff --git a/salsa20/src/backends.rs b/salsa20/src/backends.rs index fbc9393c..f0ffda7c 100644 --- a/salsa20/src/backends.rs +++ b/salsa20/src/backends.rs @@ -1 +1,7 @@ pub(crate) mod soft; + +cfg_if::cfg_if! { + if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { + pub(crate) mod avx2; + } +} diff --git a/salsa20/src/backends/avx2.rs b/salsa20/src/backends/avx2.rs new file mode 100644 index 00000000..5ede5393 --- /dev/null +++ b/salsa20/src/backends/avx2.rs @@ -0,0 +1,228 @@ +#![allow(unsafe_op_in_unsafe_fn)] +#![allow(clippy::cast_possible_wrap)] + +#[cfg(target_arch = "x86")] +use core::arch::x86::*; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::*; + +use crate::{STATE_WORDS, Unsigned}; +use cipher::{ + BlockSizeUser, ParBlocksSizeUser, StreamCipherBackend, StreamCipherClosure, + consts::{U4, U64}, +}; +use core::marker::PhantomData; + +/// Number of blocks processed in parallel. +const PAR_BLOCKS: usize = 4; +/// Number of `__m256i` sets (each holds 2 blocks). +const N: usize = PAR_BLOCKS / 2; + +/// Diagonal indices for the Salsa20 state matrix. +const IDX: [[usize; 4]; 4] = [[0, 5, 10, 15], [4, 9, 14, 3], [8, 13, 2, 7], [12, 1, 6, 11]]; + +macro_rules! rotl_epi32 { + ($v:expr, $n:literal) => {{ + let v = $v; + _mm256_or_si256(_mm256_slli_epi32(v, $n), _mm256_srli_epi32(v, 32 - $n)) + }}; +} + +#[inline] +#[target_feature(enable = "avx2")] +pub(crate) unsafe fn inner(state: &mut [u32; STATE_WORDS], f: F) +where + R: Unsigned, + F: StreamCipherClosure, +{ + f.call(&mut Backend:: { + state, + _pd: PhantomData, + }); +} + +struct Backend<'a, R: Unsigned> { + state: &'a mut [u32; STATE_WORDS], + _pd: PhantomData, +} + +impl BlockSizeUser for Backend<'_, R> { + type BlockSize = U64; +} + +impl ParBlocksSizeUser for Backend<'_, R> { + type ParBlocksSize = U4; +} + +impl StreamCipherBackend for Backend<'_, R> { + #[inline(always)] + fn gen_ks_block(&mut self, block: &mut cipher::Block) { + unsafe { + let res = rounds::(self.state); + let lo = extract_lo(&res[0]); + store_block(&lo, block); + let counter = get_counter(self.state); + set_counter(self.state, counter + 1); + } + } + + #[inline(always)] + fn gen_par_ks_blocks(&mut self, blocks: &mut cipher::ParBlocks) { + unsafe { + let res = rounds::(self.state); + for i in 0..N { + let lo = extract_lo(&res[i]); + let hi = extract_hi(&res[i]); + store_block(&lo, &mut blocks[2 * i]); + store_block(&hi, &mut blocks[2 * i + 1]); + } + let counter = get_counter(self.state); + set_counter(self.state, counter + PAR_BLOCKS as u64); + } + } +} + +/// Load two blocks' diagonal values into `__m256i` registers. +/// +/// Each `__m256i` holds two blocks: the lower 128 bits contain block 0's +/// diagonal and the upper 128 bits contain block 1's diagonal. +#[inline(always)] +unsafe fn load_diag_pair(state: &[u32; STATE_WORDS], ctr0: u64, ctr1: u64) -> [__m256i; 4] { + let s = state; + [ + // a: [s0, s5, s10, s15] (same for both blocks) + _mm256_set_epi32( + s[15] as i32, + s[10] as i32, + s[5] as i32, + s[0] as i32, + s[15] as i32, + s[10] as i32, + s[5] as i32, + s[0] as i32, + ), + // b: [s4, ctr_hi, s14, s3] (ctr_hi differs) + _mm256_set_epi32( + s[3] as i32, + s[14] as i32, + (ctr1 >> 32) as i32, + s[4] as i32, + s[3] as i32, + s[14] as i32, + (ctr0 >> 32) as i32, + s[4] as i32, + ), + // c: [ctr_lo, s13, s2, s7] (ctr_lo differs) + _mm256_set_epi32( + s[7] as i32, + s[2] as i32, + s[13] as i32, + ctr1 as i32, + s[7] as i32, + s[2] as i32, + s[13] as i32, + ctr0 as i32, + ), + // d: [s12, s1, s6, s11] (same for both blocks) + _mm256_set_epi32( + s[11] as i32, + s[6] as i32, + s[1] as i32, + s[12] as i32, + s[11] as i32, + s[6] as i32, + s[1] as i32, + s[12] as i32, + ), + ] +} + +#[inline] +#[target_feature(enable = "avx2")] +unsafe fn rounds(state: &[u32; STATE_WORDS]) -> [[__m256i; 4]; N] { + let counter = get_counter(state); + let mut res = [[_mm256_setzero_si256(); 4]; N]; + for (i, block) in res.iter_mut().enumerate() { + let base = counter + (i * 2) as u64; + *block = load_diag_pair(state, base, base + 1); + } + let initial = res; + + for _ in 0..R::USIZE * 2 { + for block in res.iter_mut() { + quarter_round(block); + shuffle(block); + } + } + + for i in 0..N { + for j in 0..4 { + res[i][j] = _mm256_add_epi32(res[i][j], initial[i][j]); + } + } + + res +} + +#[inline(always)] +unsafe fn quarter_round(block: &mut [__m256i; 4]) { + let [a, b, c, d] = block; + *b = _mm256_xor_si256(*b, rotl_epi32!(_mm256_add_epi32(*a, *d), 7)); + *c = _mm256_xor_si256(*c, rotl_epi32!(_mm256_add_epi32(*b, *a), 9)); + *d = _mm256_xor_si256(*d, rotl_epi32!(_mm256_add_epi32(*c, *b), 13)); + *a = _mm256_xor_si256(*a, rotl_epi32!(_mm256_add_epi32(*d, *c), 18)); +} + +#[inline(always)] +unsafe fn shuffle(block: &mut [__m256i; 4]) { + let [_, b, c, d] = block; + let tmp = *b; + *b = _mm256_shuffle_epi32(*d, 0b_00_11_10_01); + *c = _mm256_shuffle_epi32(*c, 0b_01_00_11_10); + *d = _mm256_shuffle_epi32(tmp, 0b_10_01_00_11); +} + +#[inline(always)] +unsafe fn extract_lo(set: &[__m256i; 4]) -> [__m128i; 4] { + [ + _mm256_castsi256_si128(set[0]), + _mm256_castsi256_si128(set[1]), + _mm256_castsi256_si128(set[2]), + _mm256_castsi256_si128(set[3]), + ] +} + +#[inline(always)] +unsafe fn extract_hi(set: &[__m256i; 4]) -> [__m128i; 4] { + [ + _mm256_extracti128_si256(set[0], 1), + _mm256_extracti128_si256(set[1], 1), + _mm256_extracti128_si256(set[2], 1), + _mm256_extracti128_si256(set[3], 1), + ] +} + +/// Convert diagonal SIMD layout back to sequential little-endian bytes. +#[inline(always)] +unsafe fn store_block(diag: &[__m128i; 4], block: &mut [u8]) { + let mut vals = [[0u32; 4]; 4]; + for (i, v) in diag.iter().enumerate() { + _mm_storeu_si128(vals[i].as_mut_ptr().cast(), *v); + } + for (reg, idxs) in vals.iter().zip(IDX.iter()) { + for (&val, &idx) in reg.iter().zip(idxs.iter()) { + block[idx * 4..idx * 4 + 4].copy_from_slice(&val.to_le_bytes()); + } + } +} + +#[inline(always)] +fn get_counter(state: &[u32; STATE_WORDS]) -> u64 { + (state[8] as u64) | ((state[9] as u64) << 32) +} + +#[inline(always)] +fn set_counter(state: &mut [u32; STATE_WORDS], pos: u64) { + state[8] = pos as u32; + state[9] = (pos >> 32) as u32; +} diff --git a/salsa20/src/backends/soft.rs b/salsa20/src/backends/soft.rs index caf2693f..66b52b36 100644 --- a/salsa20/src/backends/soft.rs +++ b/salsa20/src/backends/soft.rs @@ -46,7 +46,7 @@ pub(crate) fn quarter_round( } #[inline(always)] -fn run_rounds(state: &[u32; STATE_WORDS]) -> [u32; STATE_WORDS] { +pub(crate) fn run_rounds(state: &[u32; STATE_WORDS]) -> [u32; STATE_WORDS] { let mut res = *state; for _ in 0..R::USIZE { diff --git a/salsa20/src/lib.rs b/salsa20/src/lib.rs index e794b593..3bf1b556 100644 --- a/salsa20/src/lib.rs +++ b/salsa20/src/lib.rs @@ -23,6 +23,21 @@ use cipher::zeroize::{Zeroize, ZeroizeOnDrop}; mod backends; mod xsalsa; +cfg_if::cfg_if! { + if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { + cpufeatures::new!(avx2_cpuid, "avx2"); + type Tokens = avx2_cpuid::InitToken; + + fn init_tokens() -> Tokens { + avx2_cpuid::init() + } + } else { + type Tokens = (); + + fn init_tokens() -> Tokens {} + } +} + pub use xsalsa::{XSalsa8, XSalsa12, XSalsa20, XSalsaCore, hsalsa}; /// Salsa20/8 stream cipher @@ -56,6 +71,12 @@ const CONSTANTS: [u32; 4] = [0x6170_7865, 0x3320_646e, 0x7962_2d32, 0x6b20_6574] pub struct SalsaCore { /// Internal state of the core function state: [u32; STATE_WORDS], + /// CPU target feature tokens + #[allow( + dead_code, + reason = "tokens are held for their type-level proof, used by SIMD backends" + )] + tokens: Tokens, /// Number of rounds to perform rounds: PhantomData, } @@ -68,6 +89,7 @@ impl SalsaCore { pub fn from_raw_state(state: [u32; STATE_WORDS]) -> Self { Self { state, + tokens: init_tokens(), rounds: PhantomData, } } @@ -112,6 +134,7 @@ impl KeyIvInit for SalsaCore { Self { state, + tokens: init_tokens(), rounds: PhantomData, } } @@ -124,7 +147,19 @@ impl StreamCipherCore for SalsaCore { rem.try_into().ok() } fn process_with_backend(&mut self, f: impl StreamCipherClosure) { - f.call(&mut backends::soft::Backend(self)); + cfg_if::cfg_if! { + if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { + if self.tokens.get() { + unsafe { + backends::avx2::inner::(&mut self.state, f); + } + } else { + f.call(&mut backends::soft::Backend(self)); + } + } else { + f.call(&mut backends::soft::Backend(self)); + } + } } } diff --git a/salsa20/src/xsalsa.rs b/salsa20/src/xsalsa.rs index 9f84cef5..ae1482fc 100644 --- a/salsa20/src/xsalsa.rs +++ b/salsa20/src/xsalsa.rs @@ -8,8 +8,6 @@ use cipher::{ consts::{U4, U6, U10, U16, U24, U32, U64}, }; -use crate::backends::soft::quarter_round; - #[cfg(feature = "zeroize")] use cipher::zeroize::ZeroizeOnDrop; @@ -113,27 +111,24 @@ pub fn hsalsa(key: &Key, input: &Array) -> Array .for_each(|(v, chunk)| *v = to_u32(chunk)); state[15] = CONSTANTS[3]; - // 20 rounds consisting of 10 column rounds and 10 diagonal rounds + use crate::backends::soft::quarter_round; for _ in 0..R::USIZE { - // column rounds quarter_round(0, 4, 8, 12, &mut state); quarter_round(5, 9, 13, 1, &mut state); quarter_round(10, 14, 2, 6, &mut state); quarter_round(15, 3, 7, 11, &mut state); - - // diagonal rounds quarter_round(0, 1, 2, 3, &mut state); quarter_round(5, 6, 7, 4, &mut state); quarter_round(10, 11, 8, 9, &mut state); quarter_round(15, 12, 13, 14, &mut state); } - let mut output = Array::default(); let key_idx: [usize; 8] = [0, 5, 10, 15, 6, 7, 8, 9]; + let words: [u32; 8] = core::array::from_fn(|i| state[key_idx[i]]); + let mut output = Array::default(); for (i, chunk) in output.chunks_exact_mut(4).enumerate() { - chunk.copy_from_slice(&state[key_idx[i]].to_le_bytes()); + chunk.copy_from_slice(&words[i].to_le_bytes()); } - output }