From ff34ee13d11f491d2f15ee6b2938365b1cadcb4e Mon Sep 17 00:00:00 2001 From: Henry Date: Mon, 17 Aug 2026 00:00:21 +0200 Subject: [PATCH 1/3] chore: reduce instruction enum size this is a fairly big commit since I've been just playing around with this for a while now. the main goal is to reduce instruction size by moving some operands into two separate vecs. memory operations still seem like the biggest issue right now, so as a follow op I'll probably remove the public linear memory trait again but let's see how this goes first. performance wise this is about the same as before, but serialization/deserialization is about 20% faster now. Signed-off-by: Henry --- CHANGELOG.md | 1 + crates/parser/src/conversion.rs | 33 +- crates/parser/src/lib.rs | 16 + crates/parser/src/macros.rs | 155 +- crates/parser/src/module.rs | 42 +- crates/parser/src/optimize.rs | 1201 -------------- crates/parser/src/optimize/mod.rs | 4 + crates/parser/src/optimize/rewrite.rs | 1400 +++++++++++++++++ crates/parser/src/optimize/targets.rs | 290 ++++ crates/parser/src/parallel.rs | 12 +- crates/parser/src/visit.rs | 281 +++- crates/tinywasm/src/interpreter/executor.rs | 542 ++++--- .../src/interpreter/stack/value_stack.rs | 11 + crates/tinywasm/src/interpreter/values.rs | 6 + .../wasm-custom/conditional-rewrite.wast | 35 + .../wasm-custom/sign-extension-rewrite.wast | 59 + .../tests/wasm-custom/simd-stack-local.wast | 53 + crates/types/src/archive.rs | 26 + crates/types/src/instructions.rs | 800 +++++++--- crates/types/src/lib.rs | 13 +- crates/types/src/operands.rs | 218 +++ crates/types/src/reference.rs | 12 + examples/rust/src/print.twasm | Bin 154 -> 0 bytes 23 files changed, 3265 insertions(+), 1945 deletions(-) delete mode 100644 crates/parser/src/optimize.rs create mode 100644 crates/parser/src/optimize/mod.rs create mode 100644 crates/parser/src/optimize/rewrite.rs create mode 100644 crates/parser/src/optimize/targets.rs create mode 100644 crates/tinywasm/tests/wasm-custom/conditional-rewrite.wast create mode 100644 crates/tinywasm/tests/wasm-custom/sign-extension-rewrite.wast create mode 100644 crates/tinywasm/tests/wasm-custom/simd-stack-local.wast create mode 100644 crates/types/src/operands.rs delete mode 100644 examples/rust/src/print.twasm diff --git a/CHANGELOG.md b/CHANGELOG.md index 3427a80..701fc84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `WasmValue::ty` and `WasmValue::matches_type` - Added `ValueLane` for mapping WebAssembly value types to their physical 32-bit, 64-bit, or 128-bit storage lane. - Added a `validate` feature to `tinywasm` and `tinywasm-parser` (enabled by default) to optionally skip wasmparser validation for faster parsing of trusted modules. +- Added optional parse-time operand deduplication to reduce precompiled module and `.twasm` archive size. ### Changed diff --git a/crates/parser/src/conversion.rs b/crates/parser/src/conversion.rs index 480a8d0..b94ebe6 100644 --- a/crates/parser/src/conversion.rs +++ b/crates/parser/src/conversion.rs @@ -1,7 +1,7 @@ use crate::validation::{FuncValidator, FuncValidatorAllocations, ValidatorResources}; #[cfg(feature = "validate")] use crate::visit::process_operators_and_validate; -use crate::{Result, module::FunctionCode, visit::process_operators}; +use crate::{ParserOptions, Result, module::FunctionCode, visit::process_operators}; use alloc::{boxed::Box, format, vec::Vec}; use tinywasm_types::*; use wasmparser::{CompositeInnerType, OperatorsReader, OperatorsReaderAllocations, UnpackedIndex}; @@ -149,6 +149,7 @@ pub(crate) fn convert_module_code( reader_allocs: OperatorsReaderAllocations, metadata: &crate::visit::ModuleMetadata, ty_idx: u32, + options: &ParserOptions, ) -> Result<(FunctionCode, Option, OperatorsReaderAllocations)> { let locals_reader = func.get_locals_reader()?; #[cfg(feature = "validate")] @@ -194,32 +195,40 @@ pub(crate) fn convert_module_code( let (body, data, validator_allocs, reader_allocs) = process_operators_and_validate( validator, func, - local_types, - local_addr_map, + (local_types, local_addr_map), metadata, ty_idx, reader_allocs, + options.deduplicate_operands(), )?; (body, data, Some(validator_allocs), reader_allocs) } None => { - let (body, data, reader_allocs) = - process_operators(func, local_types, local_addr_map, metadata, ty_idx, reader_allocs)?; + let (body, data, reader_allocs) = process_operators( + func, + (local_types, local_addr_map), + metadata, + ty_idx, + reader_allocs, + options.deduplicate_operands(), + )?; (body, data, None, reader_allocs) } }; #[cfg(not(feature = "validate"))] let (body, data, validator_allocs, reader_allocs) = { let _ = validator; - let (body, data, reader_allocs) = - process_operators(func, local_types, local_addr_map, metadata, ty_idx, reader_allocs)?; + let (body, data, reader_allocs) = process_operators( + func, + (local_types, local_addr_map), + metadata, + ty_idx, + reader_allocs, + options.deduplicate_operands(), + )?; (body, data, None, reader_allocs) }; - Ok(( - FunctionCode { instructions: body, data, locals: local_counts, uses_local_memory: false }, - validator_allocs, - reader_allocs, - )) + Ok((FunctionCode { instructions: body, data, locals: local_counts }, validator_allocs, reader_allocs)) } pub(crate) fn convert_rec_group(ty: wasmparser::RecGroup, group_start: u32, types: &mut Vec) -> Result { diff --git a/crates/parser/src/lib.rs b/crates/parser/src/lib.rs index 8db7920..90e239b 100644 --- a/crates/parser/src/lib.rs +++ b/crates/parser/src/lib.rs @@ -65,6 +65,10 @@ pub struct ParserOptions { pub optimize_local_memory_allocation: bool, /// Whether to run the peephole rewrite optimizer. pub optimize_rewrite: bool, + /// Whether to deduplicate immutable function operands while parsing. + /// + /// This uses more parse CPU to reduce precompiled module and archive size. + pub deduplicate_operands: bool, #[cfg(parallel_parser)] /// Number of threads to use for parallel parsing. @@ -83,6 +87,7 @@ impl Default for ParserOptions { validation: cfg!(feature = "validate"), optimize_local_memory_allocation: true, optimize_rewrite: true, + deduplicate_operands: false, #[cfg(parallel_parser)] parser_threads: None, } @@ -134,6 +139,17 @@ impl ParserOptions { self.optimize_rewrite } + /// Enable or disable parse-time deduplication of immutable function operands. + pub const fn with_operand_deduplication(mut self, enabled: bool) -> Self { + self.deduplicate_operands = enabled; + self + } + + /// Returns whether immutable function operands are deduplicated while parsing. + pub const fn deduplicate_operands(&self) -> bool { + self.deduplicate_operands + } + #[cfg(parallel_parser)] /// Set the number of threads for parallel parsing. /// diff --git a/crates/parser/src/macros.rs b/crates/parser/src/macros.rs index 9fa7a01..5e675e6 100644 --- a/crates/parser/src/macros.rs +++ b/crates/parser/src/macros.rs @@ -72,10 +72,17 @@ pub(crate) mod visit { ) => { fn $visit(&mut self, memarg: wasmparser::MemArg $(, $lane: $ty)?) -> Self::Output { let address = self.metadata.memory_size(memarg.memory)?; + let memory_arg_idx = self.push_operand(MemoryArg::new(memarg.offset, memarg.memory))?; lowering_ops!(@emit self address(address) [$($input),*] => [$($output),*] - Instruction::$instr(MemoryArg::new(memarg.offset, memarg.memory) $(, $lane)?).into()) + lowering_ops!(@memory_instruction $instr memory_arg_idx $(, $lane)?)) } }; + (@memory_instruction $instr:ident $memory_arg_idx:ident) => { + Instruction::$instr($memory_arg_idx) + }; + (@memory_instruction $instr:ident $memory_arg_idx:ident, $lane:ident) => { + Instruction::$instr(tinywasm_types::MemoryLaneArg { memory_arg_idx: $memory_arg_idx, lane: $lane }) + }; (@global $inputs:tt => $outputs:tt $($operator:tt)*) => { lowering_ops!(@resolved global_size $inputs => $outputs $($operator)*); }; @@ -191,6 +198,20 @@ pub(crate) mod optimize { ($instructions:ident, $read:ident, $consumed:expr => $out:expr) => { replace!($instructions, $read, $consumed => [$out]); }; + ($instructions:ident, *$read:ident, $consumed:expr => [$($out:expr),+ $(,)?]) => {{ + const { + assert!($consumed >= 1 && $consumed <= 3); + assert!([$(stringify!($out)),+].len() <= $consumed + 1); + } + let replacements = [$($out),+]; + let start = *$read - $consumed; + $instructions[start..start + replacements.len()].copy_from_slice(&replacements); + $instructions.truncate(start + replacements.len()); + *$read = $instructions.len() - 1; + }}; + ($instructions:ident, *$read:ident, $consumed:expr => $out:expr) => { + replace!($instructions, *$read, $consumed => [$out]) + }; } macro_rules! rewrite { @@ -201,16 +222,13 @@ pub(crate) mod optimize { }; ($instructions:ident, $read:ident, [$($pattern:pat),+] $(if ($($guard:tt)+))? => $body:block $(,)?) => {{ const CONSUMED: usize = [$(stringify!($pattern)),+].len(); - if !$instructions.tail_rewritten - && $read < $instructions.len() - && $read >= $instructions.block_start + CONSUMED - { + if $read >= $instructions.block_start + CONSUMED { let previous: [Instruction; CONSUMED] = $instructions[$read - CONSUMED..$read].try_into().unwrap(); if let [$($pattern),+] = previous $( && $($guard)+ )? { - $instructions.tail_rewritten = true; $body + continue; } } }}; @@ -221,128 +239,5 @@ pub(crate) mod optimize { }; } - macro_rules! define_local_source_resolver { - ( - $name:ident, - get = $get:ident, - tee = $tee:ident, - set = $set:ident, - binop_local_local_tee = $lltee:ident, - binop_local_local_set = $llset:ident, - binop_local_const_tee = $lctee:ident, - binop_local_const_set = $lcset:ident - $(, load_local_tee = $loadtee:ident, load_local_set = $loadset:ident)? - ) => { - fn $name(instr: Instruction) -> Option<(Option, u16)> { - Some(match instr { - Instruction::$get(local) => (None, local), - Instruction::$tee(local) => (Some(Instruction::$set(local)), local), - Instruction::$lltee(op, a, b, local) => (Some(Instruction::$llset(op, a, b, local)), local), - Instruction::$lctee(op, src, c, local) => (Some(Instruction::$lcset(op, src, c, local)), local), - $(Instruction::$loadtee(memarg, addr, local) => (Some(Instruction::$loadset(memarg, addr, local)), local.into()),)? - _ => return None, - }) - } - }; - } - - macro_rules! fold_local_binop { - ( - $instrs:ident, $read:expr, $dst:expr, - source = $source:ident, - op = $op:ident, - const = $const:ident, - local_local = $local_local:ident, - local_const = $local_const:expr - ) => {{ - if !$instrs.tail_rewritten - && $read < $instrs.len() - && $read >= $instrs.block_start + 3 - && let [lhs_src, rhs_src, raw_op] = [$instrs[$read - 3], $instrs[$read - 2], $instrs[$read - 1]] - && let Some((lhs_instr, lhs)) = $source(lhs_src) - && let Some(op) = $op(raw_op) - { - if let Some((rhs_instr, rhs)) = $source(rhs_src) { - if rhs_instr.is_none() || rhs != lhs { - $instrs.tail_rewritten = true; - $instrs.truncate($read - 3); - $instrs.extend(lhs_instr); - $instrs.extend(rhs_instr); - $instrs.push(Instruction::$local_local(op, lhs, rhs, $dst)); - $read = $instrs.len() - 1; - } - } else if let Some(imm) = $const(rhs_src, raw_op) { - $instrs.tail_rewritten = true; - $instrs.truncate($read - 3); - $instrs.extend(lhs_instr); - $instrs.push($local_const($dst, lhs, op, imm)); - $read = $instrs.len() - 1; - } - } - }}; - } - - macro_rules! rewrite_local_set_direct { - ( - $instrs:ident, $read:ident, $dst:expr, - get = $get:ident, - copy = $copy:ident, - binop_local_local = $ll:ident, - binop_local_local_set = $llset:ident, - binop_local_const = $lc:ident, - binop_local_const_set = $lcset:expr - $(, const_instr = $const_instr:ident, set_local_const = $set_local_const:ident)? - ) => {{ - rewrite!($instrs, $read, [$get(src)] if (src != $dst) => Instruction::$copy(src, $dst)); - if !$instrs.tail_rewritten - && $read < $instrs.len() - && $read > $instrs.block_start - && let Instruction::$get(src) = $instrs[$read - 1] - && src == $dst - { - $instrs.tail_rewritten = true; - $instrs.truncate($read - 1); - $read = $instrs.len(); - } - $(rewrite!($instrs, $read, [$const_instr(c)] => Instruction::$set_local_const($dst, c));)? - rewrite!($instrs, $read, [$ll(op, a, b)] => Instruction::$llset(op, a, b, $dst)); - rewrite!($instrs, $read, [$lc(op, src, c)] => { replace!($instrs, $read, 1 => $lcset($dst, src, op, c)); }); - }}; - } - - macro_rules! rewrite_local_tee_direct { - ( - $instrs:ident, $read:ident, $dst:expr, - get = $get:ident, - binop_local_local = $ll:ident, - binop_local_local_tee = $lltee:ident, - binop_local_const = $lc:ident, - binop_local_const_tee = $lctee:ident - ) => {{ - rewrite!($instrs, $read, [$get(src)] if (src == $dst) => Instruction::$get(src)); - rewrite!($instrs, $read, [$ll(op, a, b)] => Instruction::$lltee(op, a, b, $dst)); - rewrite!($instrs, $read, [$lc(op, src, c)] => Instruction::$lctee(op, src, c, $dst)); - }}; - } - - macro_rules! rewrite_drop_tee_direct { - ( - $instrs:ident, $read:ident, - tee = $tee:ident, - set = $set:ident, - binop_local_local_tee = $lltee:ident, - binop_local_local_set = $llset:ident, - binop_local_const_tee = $lctee:ident, - binop_local_const_set = $lcset:ident - ) => {{ - rewrite!($instrs, $read, [$tee(local)] => Instruction::$set(local)); - rewrite!($instrs, $read, [$lltee(op, a, b, dst)] => Instruction::$llset(op, a, b, dst)); - rewrite!($instrs, $read, [$lctee(op, src, c, dst)] => Instruction::$lcset(op, src, c, dst)); - }}; - } - - pub(crate) use { - define_local_source_resolver, fold_local_binop, replace, rewrite, rewrite_drop_tee_direct, - rewrite_local_set_direct, rewrite_local_tee_direct, - }; + pub(crate) use {replace, rewrite}; } diff --git a/crates/parser/src/module.rs b/crates/parser/src/module.rs index 3d38ad6..8138666 100644 --- a/crates/parser/src/module.rs +++ b/crates/parser/src/module.rs @@ -10,6 +10,12 @@ use tinywasm_types::*; use wasmparser::{OperatorsReaderAllocations, Payload}; pub(crate) struct FunctionCode { + pub instructions: Vec, + pub data: crate::visit::FunctionDataBuilder, + pub locals: ValueCounts, +} + +pub(crate) struct OptimizedFunctionCode { pub instructions: Vec, pub data: WasmFunctionData, pub locals: ValueCounts, @@ -22,19 +28,15 @@ pub(crate) fn optimize_function_code( function_results: ValueCounts, self_func_addr: u32, imported_memory_count: u32, -) -> Result { - let optimized = optimize::optimize_instructions( - code.instructions, - &mut code.data, - options, - function_results, - self_func_addr, - imported_memory_count, - )?; - - code.instructions = optimized.instructions; - code.uses_local_memory = optimized.uses_local_memory; - Ok(code) +) -> Result { + let optimized = + optimize::optimize_instructions(code.instructions, &mut code.data, options, function_results, self_func_addr)?; + let data = code.data.finish(); + let uses_local_memory = optimized + .instructions + .iter() + .any(|instruction| instruction.memory_addr(&data).is_some_and(|memory| memory >= imported_memory_count)); + Ok(OptimizedFunctionCode { instructions: optimized.instructions, data, locals: code.locals, uses_local_memory }) } #[derive(Default)] @@ -53,7 +55,7 @@ pub(crate) struct ModuleReader<'a> { pub(crate) code_type_addrs: Box<[u32]>, code_results: Box<[ValueCounts]>, pub(crate) exports: Arc<[Export]>, - pub(crate) code: Vec, + pub(crate) code: Vec, pub(crate) globals: Box<[Global]>, pub(crate) tables: Box<[TableDefinition]>, pub(crate) memory_types: Box<[MemoryType]>, @@ -402,7 +404,7 @@ impl<'a> ModuleReader<'a> { let metadata = self.translation_metadata(); let (code, func_validator_allocs, operators_reader_allocs) = - convert_module_code(function, func_validator, operators_reader_allocs, metadata, ty_idx)?; + convert_module_code(function, func_validator, operators_reader_allocs, metadata, ty_idx, options)?; self.code.push(optimize_function_code( code, @@ -565,7 +567,7 @@ impl<'a> ModuleReader<'a> { .into_iter() .zip(self.code_type_addrs) .zip(self.code_results) - .map(|((code, ty_idx), results)| { + .map(|((code, ty_idx), results)| -> Result<_> { let ty = self.types.get(ty_idx).and_then(SubType::as_func).expect("function type was checked while parsing"); let params = ValueCounts::from_iter(ty.params()); @@ -573,15 +575,15 @@ impl<'a> ModuleReader<'a> { local_memory_allocation = LocalMemoryAllocation::Eager; } - Arc::new(WasmFunction { - instructions: code.instructions.into(), + Ok(Arc::new(WasmFunction { + instructions: code.instructions.into_boxed_slice(), data: code.data, locals: code.locals, params, results, - }) + })) }) - .collect(); + .collect::>()?; Ok(ModuleInner { funcs, diff --git a/crates/parser/src/optimize.rs b/crates/parser/src/optimize.rs deleted file mode 100644 index 969d686..0000000 --- a/crates/parser/src/optimize.rs +++ /dev/null @@ -1,1201 +0,0 @@ -use crate::macros::optimize::*; -use crate::{ParseError, ParserOptions, Result}; -use alloc::vec::Vec; -use core::ops::{Deref, DerefMut}; -use tinywasm_types::{BinOp, BinOp128, CmpOp, ConstIdx, Instruction, ValueCounts, WasmFunctionData}; - -pub(crate) struct OptimizeResult { - pub(crate) instructions: Vec, - pub(crate) uses_local_memory: bool, -} - -struct CompactOutput { - instructions: Vec, - block_start: usize, - tail_rewritten: bool, -} - -impl Deref for CompactOutput { - type Target = Vec; - - fn deref(&self) -> &Self::Target { - &self.instructions - } -} - -impl DerefMut for CompactOutput { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.instructions - } -} - -pub(crate) fn optimize_instructions( - instructions: Vec, - function_data: &mut WasmFunctionData, - options: &ParserOptions, - function_results: ValueCounts, - self_func_addr: u32, - imported_memory_count: u32, -) -> Result { - let (mut instructions, old_to_new) = if options.optimize_rewrite() { - let boundaries = target_boundaries(&instructions, function_data)?; - let (instructions, old_to_new) = rewrite(instructions, &boundaries, function_results, self_func_addr); - (instructions, Some(old_to_new)) - } else { - (instructions, None) - }; - let uses_local_memory = finalize(&mut instructions, function_data, old_to_new.as_deref(), imported_memory_count)?; - Ok(OptimizeResult { instructions, uses_local_memory }) -} - -fn rewrite( - source: Vec, - boundaries: &[bool], - function_results: ValueCounts, - self_func_addr: u32, -) -> (Vec, Vec) { - use Instruction::*; - let mut instrs = - CompactOutput { instructions: Vec::with_capacity(source.len()), block_start: 0, tail_rewritten: false }; - let mut old_to_new = alloc::vec![0; source.len() + 1]; - let mut after_terminator = false; - let return_instr = match function_results { - ValueCounts { c32: 0, c64: 0, c128: 0 } => Some(ReturnVoid), - ValueCounts { c32: 1, c64: 0, c128: 0 } => Some(Return32), - ValueCounts { c32: 0, c64: 1, c128: 0 } => Some(Return64), - ValueCounts { c32: 0, c64: 0, c128: 1 } => Some(Return128), - _ => None, - }; - - for (old_idx, instr) in source.iter().copied().enumerate() { - if boundaries[old_idx] || after_terminator { - instrs.block_start = instrs.len(); - } - old_to_new[old_idx] = instrs.len() as u32; - instrs.tail_rewritten = false; - instrs.push(instr); - let mut i = instrs.len() - 1; - match instrs[i] { - LocalCopy32(a, b) if a == b => { - instrs.pop(); - } - LocalCopy64(a, b) if a == b => { - instrs.pop(); - } - LocalCopy128(a, b) if a == b => { - instrs.pop(); - } - Call(addr) if addr == self_func_addr => instrs[i] = CallSelf, - ReturnCall(addr) if addr == self_func_addr => instrs[i] = ReturnCallSelf, - Return if let Some(return_instr) = return_instr => instrs[i] = return_instr, - instr @ (I32Add | I32Mul | I32And | I32Or | I32Xor) => { - let Some(op) = int_bin_op(instr) else { unreachable!() }; - rewrite!(instrs, i, [LocalGet32(a), LocalGet32(b)] => BinOpLocalLocal32(op, a, b)); - rewrite!(instrs, i, [LocalGet32(local), Const32(c)] => BinOpLocalConst32(op, local, c)); - rewrite!(instrs, i, [GlobalGet32(global), Const32(c)] => BinOpGlobalConst32(op, global, c)); - rewrite!(instrs, i, [Const32(c), LocalGet32(local)] => BinOpLocalConst32(op, local, c)); - rewrite!(instrs, i, [Const32(c), GlobalGet32(global)] => BinOpGlobalConst32(op, global, c)); - rewrite!(instrs, i, [GlobalGet32(global)] => BinOpStackGlobal32(op, global)); - rewrite!(instrs, i, [LocalGet32(local)] => BinOpStackLocal32(op, local)); - if matches!(op, BinOp::IAdd) { - rewrite!(instrs, i, [Const32(c)] => AddConst32(c)); - rewrite!(instrs, i, [I32Add] => I32Add3); - rewrite!(instrs, i, - [BinOpStackLocal32(BinOp::IAdd, local)] => - [LocalGet32(local), I32Add3] - ); - } - } - instr @ (I32Sub | I32Shl | I32ShrS | I32ShrU | I32Rotl | I32Rotr) => { - let Some(op) = int_bin_op(instr) else { unreachable!() }; - rewrite!(instrs, i, [LocalGet32(a), LocalGet32(b)] => BinOpLocalLocal32(op, a, b)); - rewrite!(instrs, i, [LocalGet32(local), Const32(c)] => BinOpLocalConst32(op, local, c)); - rewrite!(instrs, i, [GlobalGet32(global), Const32(c)] => BinOpGlobalConst32(op, global, c)); - rewrite!(instrs, i, [GlobalGet32(global)] => BinOpStackGlobal32(op, global)); - rewrite!(instrs, i, [LocalGet32(local)] => BinOpStackLocal32(op, local)); - if matches!(op, BinOp::IShrS) { - rewrite!(instrs, i, [BinOpLocalConst32(BinOp::IShl, local, 8), Const32(8)] => [LocalGet32(local), I32Extend8S]); - rewrite!(instrs, i, [BinOpLocalConst32(BinOp::IShl, local, 16), Const32(16)] => [LocalGet32(local), I32Extend16S]); - } - } - instr @ (I64Add | I64Mul | I64And | I64Or | I64Xor) => { - let Some(op) = int_bin_op(instr) else { unreachable!() }; - rewrite!(instrs, i, [LocalGet64(a), LocalGet64(b)] => BinOpLocalLocal64(op, a, b)); - rewrite!(instrs, i, [LocalGet64(local), Const64(c)] => BinOpLocalConst64(op, local, c)); - rewrite!(instrs, i, [GlobalGet64(global), Const64(c)] => BinOpGlobalConst64(op, global, c)); - rewrite!(instrs, i, [Const64(c), LocalGet64(local)] => BinOpLocalConst64(op, local, c)); - rewrite!(instrs, i, [Const64(c), GlobalGet64(global)] => BinOpGlobalConst64(op, global, c)); - rewrite!(instrs, i, [GlobalGet64(global)] => BinOpStackGlobal64(op, global)); - if matches!(op, BinOp::IAdd) { - rewrite!(instrs, i, [Const64(c)] => AddConst64(c)); - rewrite!(instrs, i, [I64Add] => I64Add3); - } - } - instr @ (I64Sub | I64Shl | I64ShrS | I64ShrU | I64Rotl | I64Rotr) => { - let Some(op) = int_bin_op(instr) else { unreachable!() }; - rewrite!(instrs, i, [LocalGet64(a), LocalGet64(b)] => BinOpLocalLocal64(op, a, b)); - rewrite!(instrs, i, [LocalGet64(local), Const64(c)] => BinOpLocalConst64(op, local, c)); - rewrite!(instrs, i, [GlobalGet64(global), Const64(c)] => BinOpGlobalConst64(op, global, c)); - rewrite!(instrs, i, [GlobalGet64(global)] => BinOpStackGlobal64(op, global)); - if matches!(op, BinOp::IShrS) { - rewrite!(instrs, i, [BinOpLocalConst64(BinOp::IShl, local, 8), Const64(8)] => [LocalGet64(local), I64Extend8S]); - rewrite!(instrs, i, [BinOpLocalConst64(BinOp::IShl, local, 16), Const64(16)] => [LocalGet64(local), I64Extend16S]); - rewrite!(instrs, i, [BinOpLocalConst64(BinOp::IShl, local, 32), Const64(32)] => [LocalGet64(local), I64Extend32S]); - } - } - instr @ (I32Eq | I32Ne | I32LtS | I32LtU | I32GtS | I32GtU | I32LeS | I32LeU | I32GeS | I32GeU) => { - let Some(op) = cmp_op(instr) else { unreachable!() }; - rewrite!(instrs, i, [LocalGet32(left), LocalGet32(right)] => CmpLocalLocal32(op, left, right)); - } - instr @ (I64Eq | I64Ne | I64LtS | I64LtU | I64GtS | I64GtU | I64LeS | I64LeU | I64GeS | I64GeU) => { - let Some(op) = cmp_op(instr) else { unreachable!() }; - rewrite!(instrs, i, [LocalGet64(left), LocalGet64(right)] => CmpLocalLocal64(op, left, right)); - } - instr @ (F32Add | F32Mul | F32Min | F32Max) => { - let Some(op) = float_bin_op(instr) else { unreachable!() }; - rewrite!(instrs, i, [LocalGet32(a), LocalGet32(b)] => BinOpLocalLocal32(op, a, b)); - rewrite!(instrs, i, [LocalGet32(local), Const32(c)] => BinOpLocalConst32(op, local, c)); - rewrite!(instrs, i, [GlobalGet32(global), Const32(c)] => BinOpGlobalConst32(op, global, c)); - rewrite!(instrs, i, [Const32(c), LocalGet32(local)] => BinOpLocalConst32(op, local, c)); - rewrite!(instrs, i, [Const32(c), GlobalGet32(global)] => BinOpGlobalConst32(op, global, c)); - rewrite!(instrs, i, [LocalGet32(local)] => BinOpStackLocal32(op, local)); - } - instr @ (F32Sub | F32Div | F32Copysign) => { - let Some(op) = float_bin_op(instr) else { unreachable!() }; - rewrite!(instrs, i, [LocalGet32(a), LocalGet32(b)] => BinOpLocalLocal32(op, a, b)); - rewrite!(instrs, i, [LocalGet32(local), Const32(c)] => BinOpLocalConst32(op, local, c)); - rewrite!(instrs, i, [GlobalGet32(global), Const32(c)] => BinOpGlobalConst32(op, global, c)); - rewrite!(instrs, i, [LocalGet32(local)] => BinOpStackLocal32(op, local)); - } - instr @ (F64Add | F64Mul | F64Min | F64Max) => { - let Some(op) = float_bin_op(instr) else { unreachable!() }; - rewrite!(instrs, i, [LocalGet64(a), LocalGet64(b)] => BinOpLocalLocal64(op, a, b)); - rewrite!(instrs, i, [LocalGet64(local), Const64(c)] => BinOpLocalConst64(op, local, c)); - rewrite!(instrs, i, [GlobalGet64(global), Const64(c)] => BinOpGlobalConst64(op, global, c)); - rewrite!(instrs, i, [Const64(c), LocalGet64(local)] => BinOpLocalConst64(op, local, c)); - rewrite!(instrs, i, [Const64(c), GlobalGet64(global)] => BinOpGlobalConst64(op, global, c)); - } - instr @ (F64Sub | F64Div | F64Copysign) => { - let Some(op) = float_bin_op(instr) else { unreachable!() }; - rewrite!(instrs, i, [LocalGet64(a), LocalGet64(b)] => BinOpLocalLocal64(op, a, b)); - rewrite!(instrs, i, [LocalGet64(local), Const64(c)] => BinOpLocalConst64(op, local, c)); - rewrite!(instrs, i, [GlobalGet64(global), Const64(c)] => BinOpGlobalConst64(op, global, c)); - } - instr @ (V128And | V128Or | V128Xor | I64x2Add | I64x2Mul) => { - let Some(op) = bin_op_128(instr) else { unreachable!() }; - rewrite!(instrs, i, [LocalGet128(a), LocalGet128(b)] => BinOpLocalLocal128(op, a, b)); - rewrite!(instrs, i, [LocalGet128(local), Const128(c)] => BinOpLocalConst128(op, local, c)); - rewrite!(instrs, i, [GlobalGet128(global), Const128(c)] => BinOpGlobalConst128(op, global, c)); - rewrite!(instrs, i, [Const128(c), LocalGet128(local)] => BinOpLocalConst128(op, local, c)); - rewrite!(instrs, i, [Const128(c), GlobalGet128(global)] => BinOpGlobalConst128(op, global, c)); - } - V128AndNot => { - rewrite!(instrs, i, [LocalGet128(a), LocalGet128(b)] => BinOpLocalLocal128(BinOp128::AndNot, a, b)); - rewrite!(instrs, i, [LocalGet128(local), Const128(c)] => BinOpLocalConst128(BinOp128::AndNot, local, c)); - rewrite!(instrs, i, [GlobalGet128(global), Const128(c)] => BinOpGlobalConst128(BinOp128::AndNot, global, c)); - } - I32Store(memarg) | F32Store(memarg) => { - rewrite!(instrs, i, - [LocalGet32(addr), LoadLocal32(load_memarg, load_addr), AddConst32(1)] if - (load_memarg == memarg && addr == u16::from(load_addr)) => - IncMemoryLocal32(memarg, load_addr) - ); - rewrite!(instrs, i, - [LocalGet64(addr), LoadLocal32(load_memarg, load_addr), AddConst32(1)] if - (load_memarg == memarg && addr == u16::from(load_addr)) => - IncMemoryLocal32(memarg, load_addr) - ); - rewrite!(instrs, i, [F32Mul, F32Add] => FMaStoreF32(memarg)); - rewrite!(instrs, i, - [BinOpStackLocal32(BinOp::FMul, local), F32Add] => - [LocalGet32(local), FMaStoreF32(memarg)] - ); - rewrite!(instrs, i, - [LocalGet32(addr_local), LocalGet32(value_local)] if - (let (Ok(addr_local), Ok(value_local)) = (u8::try_from(addr_local), u8::try_from(value_local))) => - StoreLocalLocal32(memarg, addr_local, value_local) - ); - } - I64Store(memarg) | F64Store(memarg) => { - rewrite!(instrs, i, - [LocalGet32(addr), LoadLocal64(load_memarg, load_addr), AddConst64(1)] if - (load_memarg == memarg && addr == u16::from(load_addr)) => - IncMemoryLocal64(memarg, load_addr) - ); - rewrite!(instrs, i, - [LocalGet64(addr), LoadLocal64(load_memarg, load_addr), AddConst64(1)] if - (load_memarg == memarg && addr == u16::from(load_addr)) => - IncMemoryLocal64(memarg, load_addr) - ); - rewrite!(instrs, i, [F64Mul, F64Add] => FMaStoreF64(memarg)); - rewrite!(instrs, i, - [LocalGet32(addr_local), LocalGet64(value_local)] if - (let (Ok(addr_local), Ok(value_local)) = (u8::try_from(addr_local), u8::try_from(value_local))) => - StoreLocalLocal64(memarg, addr_local, value_local) - ); - } - V128Store(memarg) => { - rewrite!(instrs, i, - [LocalGet32(addr_local), LocalGet128(value_local)] if - (let (Ok(addr_local), Ok(value_local)) = (u8::try_from(addr_local), u8::try_from(value_local))) => - StoreLocalLocal128(memarg, addr_local, value_local) - ); - } - I32Load(memarg) | F32Load(memarg) => { - rewrite!(instrs, i, - [LocalGet32(addr_local)] if (let Ok(addr_local) = u8::try_from(addr_local)) => - LoadLocal32(memarg, addr_local) - ); - rewrite!(instrs, i, - [LocalGet64(addr_local)] if (let Ok(addr_local) = u8::try_from(addr_local)) => - LoadLocal32(memarg, addr_local) - ); - } - I64Load(memarg) | F64Load(memarg) => { - rewrite!(instrs, i, - [LocalGet32(addr_local)] if (let Ok(addr_local) = u8::try_from(addr_local)) => - LoadLocal64(memarg, addr_local) - ); - rewrite!(instrs, i, - [LocalGet64(addr_local)] if (let Ok(addr_local) = u8::try_from(addr_local)) => - LoadLocal64(memarg, addr_local) - ); - } - I32Load8S(memarg) => { - rewrite!(instrs, i, - [LocalGet32(addr_local)] if (let Ok(addr_local) = u8::try_from(addr_local)) => - LoadLocal8S32(memarg, addr_local) - ); - rewrite!(instrs, i, - [LocalGet64(addr_local)] if (let Ok(addr_local) = u8::try_from(addr_local)) => - LoadLocal8S32(memarg, addr_local) - ); - } - I32Load8U(memarg) => { - rewrite!(instrs, i, - [LocalGet32(addr_local)] if (let Ok(addr_local) = u8::try_from(addr_local)) => - LoadLocal8U32(memarg, addr_local) - ); - rewrite!(instrs, i, - [LocalGet64(addr_local)] if (let Ok(addr_local) = u8::try_from(addr_local)) => - LoadLocal8U32(memarg, addr_local) - ); - } - I32Load16S(memarg) => { - rewrite!(instrs, i, - [LocalGet32(addr_local)] if (let Ok(addr_local) = u8::try_from(addr_local)) => - LoadLocal16S32(memarg, addr_local) - ); - rewrite!(instrs, i, - [LocalGet64(addr_local)] if (let Ok(addr_local) = u8::try_from(addr_local)) => - LoadLocal16S32(memarg, addr_local) - ); - } - I32Load16U(memarg) => { - rewrite!(instrs, i, - [LocalGet32(addr_local)] if (let Ok(addr_local) = u8::try_from(addr_local)) => - LoadLocal16U32(memarg, addr_local) - ); - rewrite!(instrs, i, - [LocalGet64(addr_local)] if (let Ok(addr_local) = u8::try_from(addr_local)) => - LoadLocal16U32(memarg, addr_local) - ); - } - MemoryFill(mem) => { - rewrite!(instrs, i, [Const32(val), Const32(size)] => MemoryFillImm(mem, val as u8, size)) - } - GlobalGet32(dst) => rewrite!(instrs, i, [GlobalSet32(src)] if (src == dst) => GlobalTee32(src)), - GlobalGet64(dst) => rewrite!(instrs, i, [GlobalSet64(src)] if (src == dst) => GlobalTee64(src)), - GlobalGet128(dst) => rewrite!(instrs, i, [GlobalSet128(src)] if (src == dst) => GlobalTee128(src)), - LocalGet32(dst) => rewrite!(instrs, i, [LocalSet32(src)] if (src == dst) => LocalTee32(src)), - LocalGet64(dst) => rewrite!(instrs, i, [LocalSet64(src)] if (src == dst) => LocalTee64(src)), - LocalGet128(dst) => rewrite!(instrs, i, [LocalSet128(src)] if (src == dst) => LocalTee128(src)), - LocalSet32(dst) => { - fold_local_binop!( - instrs, i, dst, - source = resolve_local_source_32, - op = scalar_bin_op, - const = scalar_const_32, - local_local = BinOpLocalLocalSet32, - local_const = |dst, lhs, op, imm| match (dst == lhs, op) { - (true, BinOp::IAdd) => Instruction::IncLocal32(dst, imm), - (true, BinOp::ISub) => Instruction::IncLocal32(dst, imm.wrapping_neg()), - _ => Instruction::BinOpLocalConstSet32(op, lhs, imm, dst), - } - ); - rewrite!(instrs, i, [I32Mul, LocalGet32(acc), I32Add] if (acc == dst) => MulAccLocal32(dst)); - rewrite!(instrs, i, [F32Mul, LocalGet32(acc), F32Add] if (acc == dst) => FMulAccLocal32(dst)); - rewrite!(instrs, i, - [I32Mul, BinOpStackLocal32(BinOp::IAdd, acc)] if (acc == dst) => - MulAccLocal32(dst) - ); - rewrite!(instrs, i, - [F32Mul, BinOpStackLocal32(BinOp::FAdd, acc)] if (acc == dst) => - FMulAccLocal32(dst) - ); - rewrite!(instrs, i, [BinOpStackLocal32(op, local)] => BinOpStackLocalSet32(op, local, dst)); - rewrite_local_set_direct!( - instrs, - i, - dst, - get = LocalGet32, - copy = LocalCopy32, - binop_local_local = BinOpLocalLocal32, - binop_local_local_set = BinOpLocalLocalSet32, - binop_local_const = BinOpLocalConst32, - binop_local_const_set = |dst, src, op, c| match (dst == src, op) { - (true, BinOp::IAdd) => IncLocal32(dst, c), - (true, BinOp::ISub) => IncLocal32(dst, c.wrapping_neg()), - _ => BinOpLocalConstSet32(op, src, c, dst), - }, - const_instr = Const32, - set_local_const = SetLocalConst32 - ); - rewrite!(instrs, i, [LoadLocal32(memarg, addr)] if (let Ok(dst) = u8::try_from(dst)) => LoadLocalSet32(memarg, addr, dst)); - rewrite!(instrs, i, [LoadLocal8S32(memarg, addr)] if (let Ok(dst) = u8::try_from(dst)) => LoadLocalSet8S32(memarg, addr, dst)); - rewrite!(instrs, i, [LoadLocal8U32(memarg, addr)] if (let Ok(dst) = u8::try_from(dst)) => LoadLocalSet8U32(memarg, addr, dst)); - rewrite!(instrs, i, [LoadLocal16S32(memarg, addr)] if (let Ok(dst) = u8::try_from(dst)) => LoadLocalSet16S32(memarg, addr, dst)); - rewrite!(instrs, i, [LoadLocal16U32(memarg, addr)] if (let Ok(dst) = u8::try_from(dst)) => LoadLocalSet16U32(memarg, addr, dst)); - rewrite!(instrs, i, - [LocalGet32(addr), I32Load(memarg)] if - (let (Ok(addr), Ok(dst)) = (u8::try_from(addr), u8::try_from(dst))) => - LoadLocalSet32(memarg, addr, dst) - ); - } - LocalSet64(dst) => { - fold_local_binop!( - instrs, i, dst, - source = resolve_local_source_64, - op = scalar_bin_op, - const = scalar_const_64, - local_local = BinOpLocalLocalSet64, - local_const = |dst, lhs, op, imm| match (dst == lhs, op) { - (true, BinOp::IAdd) => Instruction::IncLocal64(dst, imm), - (true, BinOp::ISub) => Instruction::IncLocal64(dst, imm.wrapping_neg()), - _ => Instruction::BinOpLocalConstSet64(op, lhs, imm, dst), - } - ); - rewrite!(instrs, i, [I64Mul, LocalGet64(acc), I64Add] if (acc == dst) => MulAccLocal64(dst)); - rewrite!(instrs, i, [F64Mul, LocalGet64(acc), F64Add] if (acc == dst) => FMulAccLocal64(dst)); - rewrite_local_set_direct!( - instrs, - i, - dst, - get = LocalGet64, - copy = LocalCopy64, - binop_local_local = BinOpLocalLocal64, - binop_local_local_set = BinOpLocalLocalSet64, - binop_local_const = BinOpLocalConst64, - binop_local_const_set = |dst, src, op, c| match (dst == src, op) { - (true, BinOp::IAdd) => IncLocal64(dst, c), - (true, BinOp::ISub) => IncLocal64(dst, c.wrapping_neg()), - _ => BinOpLocalConstSet64(op, src, c, dst), - }, - const_instr = Const64, - set_local_const = SetLocalConst64 - ); - } - LocalSet128(dst) => { - fold_local_binop!( - instrs, i, dst, - source = resolve_local_source_128, - op = bin_op_128, - const = const_128, - local_local = BinOpLocalLocalSet128, - local_const = |dst, lhs, op, imm| Instruction::BinOpLocalConstSet128(op, lhs, imm, dst) - ); - rewrite_local_set_direct!( - instrs, - i, - dst, - get = LocalGet128, - copy = LocalCopy128, - binop_local_local = BinOpLocalLocal128, - binop_local_local_set = BinOpLocalLocalSet128, - binop_local_const = BinOpLocalConst128, - binop_local_const_set = |dst, src, op, c| BinOpLocalConstSet128(op, src, c, dst), - const_instr = Const128, - set_local_const = SetLocalConst128 - ); - rewrite!(instrs, i, - [LocalGet32(addr), V128Load(memarg)] if - (let (Ok(addr), Ok(dst)) = (u8::try_from(addr), u8::try_from(dst))) => - LoadLocalSet128(memarg, addr, dst) - ); - } - LocalTee32(dst) => { - rewrite!(instrs, i, [BinOpStackLocal32(op, local)] => BinOpStackLocalTee32(op, local, dst)); - fold_local_binop!( - instrs, i, dst, - source = resolve_local_source_32, - op = scalar_bin_op, - const = scalar_const_32, - local_local = BinOpLocalLocalTee32, - local_const = |dst, lhs, op, imm| Instruction::BinOpLocalConstTee32(op, lhs, imm, dst) - ); - rewrite_local_tee_direct!( - instrs, - i, - dst, - get = LocalGet32, - binop_local_local = BinOpLocalLocal32, - binop_local_local_tee = BinOpLocalLocalTee32, - binop_local_const = BinOpLocalConst32, - binop_local_const_tee = BinOpLocalConstTee32 - ); - rewrite!(instrs, i, [Const32(c), I32And] => AndConstTee32(c, dst)); - rewrite!(instrs, i, [Const32(c), I32Sub] => SubConstTee32(c, dst)); - rewrite!(instrs, i, - [LocalGet32(addr), I32Load(memarg)] if - (let (Ok(addr), Ok(dst)) = (u8::try_from(addr), u8::try_from(dst))) => - LoadLocalTee32(memarg, addr, dst) - ); - rewrite!(instrs, i, - [LocalGet32(addr), F32Load(memarg)] if - (let (Ok(addr), Ok(dst)) = (u8::try_from(addr), u8::try_from(dst))) => - LoadLocalTee32(memarg, addr, dst) - ); - rewrite!(instrs, i, - [LoadLocal32(memarg, addr)] if (let Ok(dst) = u8::try_from(dst)) => - LoadLocalTee32(memarg, addr, dst) - ); - rewrite!(instrs, i, - [LoadLocal8S32(memarg, addr)] if (let Ok(dst) = u8::try_from(dst)) => - LoadLocalTee8S32(memarg, addr, dst) - ); - rewrite!(instrs, i, - [LoadLocal8U32(memarg, addr)] if (let Ok(dst) = u8::try_from(dst)) => - LoadLocalTee8U32(memarg, addr, dst) - ); - rewrite!(instrs, i, - [LoadLocal16S32(memarg, addr)] if (let Ok(dst) = u8::try_from(dst)) => - LoadLocalTee16S32(memarg, addr, dst) - ); - rewrite!(instrs, i, - [LoadLocal16U32(memarg, addr)] if (let Ok(dst) = u8::try_from(dst)) => - LoadLocalTee16U32(memarg, addr, dst) - ); - } - LocalTee64(dst) => { - fold_local_binop!( - instrs, i, dst, - source = resolve_local_source_64, - op = scalar_bin_op, - const = scalar_const_64, - local_local = BinOpLocalLocalTee64, - local_const = |dst, lhs, op, imm| Instruction::BinOpLocalConstTee64(op, lhs, imm, dst) - ); - rewrite_local_tee_direct!( - instrs, - i, - dst, - get = LocalGet64, - binop_local_local = BinOpLocalLocal64, - binop_local_local_tee = BinOpLocalLocalTee64, - binop_local_const = BinOpLocalConst64, - binop_local_const_tee = BinOpLocalConstTee64 - ); - rewrite!(instrs, i, [Const64(c), I64And] => AndConstTee64(c, dst)); - rewrite!(instrs, i, [Const64(c), I64Sub] => SubConstTee64(c, dst)); - } - LocalTee128(dst) => { - fold_local_binop!( - instrs, i, dst, - source = resolve_local_source_128, - op = bin_op_128, - const = const_128, - local_local = BinOpLocalLocalTee128, - local_const = |dst, lhs, op, imm| Instruction::BinOpLocalConstTee128(op, lhs, imm, dst) - ); - rewrite_local_tee_direct!( - instrs, - i, - dst, - get = LocalGet128, - binop_local_local = BinOpLocalLocal128, - binop_local_local_tee = BinOpLocalLocalTee128, - binop_local_const = BinOpLocalConst128, - binop_local_const_tee = BinOpLocalConstTee128 - ); - rewrite!(instrs, i, - [LocalGet32(addr), V128Load(memarg)] if - (let (Ok(addr), Ok(dst)) = (u8::try_from(addr), u8::try_from(dst))) => - LoadLocalTee128(memarg, addr, dst) - ); - } - Drop32 => { - rewrite!(instrs, i, - [BinOpStackLocalTee32(op, local, dst)] => - BinOpStackLocalSet32(op, local, dst) - ); - rewrite_drop_tee_direct!( - instrs, - i, - tee = LocalTee32, - set = LocalSet32, - binop_local_local_tee = BinOpLocalLocalTee32, - binop_local_local_set = BinOpLocalLocalSet32, - binop_local_const_tee = BinOpLocalConstTee32, - binop_local_const_set = BinOpLocalConstSet32 - ); - } - Drop64 => rewrite_drop_tee_direct!( - instrs, - i, - tee = LocalTee64, - set = LocalSet64, - binop_local_local_tee = BinOpLocalLocalTee64, - binop_local_local_set = BinOpLocalLocalSet64, - binop_local_const_tee = BinOpLocalConstTee64, - binop_local_const_set = BinOpLocalConstSet64 - ), - Drop128 => rewrite_drop_tee_direct!( - instrs, - i, - tee = LocalTee128, - set = LocalSet128, - binop_local_local_tee = BinOpLocalLocalTee128, - binop_local_local_set = BinOpLocalLocalSet128, - binop_local_const_tee = BinOpLocalConstTee128, - binop_local_const_set = BinOpLocalConstSet128 - ), - Jump(ip) => { - let target = resolve_jump_target(&source, ip); - let exit = old_idx as u32 + 1; - let body = target + 1; - - match source.get(target as usize).copied() { - Some(JumpCmpLocalLocal32 { target_ip, left, right, op }) - if resolve_jump_target(&source, target_ip) == exit && body > target => - { - instrs[i] = JumpCmpLocalLocal32 { target_ip: body, left, right, op: inverse_cmp_op(op) }; - } - Some(JumpCmpLocalLocal64 { target_ip, left, right, op }) - if resolve_jump_target(&source, target_ip) == exit && body > target => - { - instrs[i] = JumpCmpLocalLocal64 { target_ip: body, left, right, op: inverse_cmp_op(op) }; - } - _ => canonicalize_jump_like_with_target(&mut instrs, i, target, exit), - } - } - JumpIfZero32(ip) => { - let target = resolve_jump_target(&source, ip); - rewrite!(instrs, i, [LocalGet32(local), I32Eqz] => { - replace!(instrs, i, 2 => JumpIfLocalNonZero32 { target_ip: target, local }); - continue; - }); - rewrite!(instrs, i, [I32Eqz] => { - replace!(instrs, i, 1 => JumpIfNonZero32(target)); - continue; - }); - rewrite!(instrs, i, [LocalGet64(local), I64Eqz] => { - replace!(instrs, i, 2 => JumpIfLocalNonZero64 { target_ip: target, local }); - continue; - }); - rewrite!(instrs, i, [I64Eqz] => { - replace!(instrs, i, 1 => JumpIfNonZero64(target)); - continue; - }); - rewrite!(instrs, i, - [BinOpLocalConstTee32(op, local, imm, dst)] if (local == dst) => - match inc_delta(op, imm) { - Some(delta) => IncLocalJump32 { target_ip: target, local, delta, on_zero: true }, - None => BinOpLocalConstJump32 { target_ip: target, local, imm, op, on_zero: true }, - } - ); - rewrite!(instrs, i, - [AddConst32(imm), LocalTee32(local), LocalGet32(cond)] if (local == cond) => - IncStackTeeLocalJump32 { target_ip: target, local, delta: imm, on_zero: true } - ); - rewrite!(instrs, i, - [AndConstTee32(imm, local), LocalGet32(cond)] if (local == cond) => - BinOpStackConstTeeLocalJump32 { target_ip: target, local, imm, op: BinOp::IAnd, on_zero: true } - ); - rewrite!(instrs, i, - [SubConstTee32(imm, local), LocalGet32(cond)] if (local == cond) => - IncStackTeeLocalJump32 { target_ip: target, local, delta: imm.wrapping_neg(), on_zero: true } - ); - rewrite!(instrs, i, - [BinOpGlobalConst32(op, global, imm), GlobalTee32(dst)] if (global == dst) => - match inc_delta(op, imm) { - Some(delta) => IncGlobalJump32 { target_ip: target, global, delta, on_zero: true }, - None => BinOpGlobalConstJump32 { target_ip: target, global, imm, op, on_zero: true }, - } - ); - rewrite!(instrs, i, - [CmpLocalLocal32(op, left, right)] => - JumpCmpLocalLocal32 { target_ip: target, left, right, op: inverse_cmp_op(op) } - ); - rewrite!(instrs, i, - [CmpLocalLocal64(op, left, right)] => - JumpCmpLocalLocal64 { target_ip: target, left, right, op: inverse_cmp_op(op) } - ); - rewrite!(instrs, i, - [BinOpLocalConstTee32(binop, local, imm, dst), LocalGet32(right), cmp] if - (local == dst && let Some(cmp) = cmp_op(cmp)) => - match inc_delta(binop, imm) { - Some(delta) => IncLocalJumpCmpLocal32 { target_ip: target, local, delta, right, op: inverse_cmp_op(cmp) }, - None => BinOpLocalConstJumpCmpLocal32 { target_ip: target, local, imm, binop, right, cmp: inverse_cmp_op(cmp) }, - } - ); - rewrite!(instrs, i, - [LocalGet32(local), cmp] if (let Some(op) = cmp_op(cmp)) => - JumpCmpStackLocal32 { target_ip: target, local, op: inverse_cmp_op(op) } - ); - rewrite!(instrs, i, - [LocalGet64(local), cmp] if (let Some(op) = cmp_op(cmp)) => - JumpCmpStackLocal64 { target_ip: target, local, op: inverse_cmp_op(op) } - ); - rewrite!(instrs, i, - [LocalGet32(local), Const32(imm), cmp] if (let Some(op) = cmp_op(cmp)) => - match (imm, inverse_cmp_op(op)) { - (0, CmpOp::Eq) => JumpIfLocalZero32 { target_ip: target, local }, - (0, CmpOp::Ne) => JumpIfLocalNonZero32 { target_ip: target, local }, - (imm, op) => JumpCmpLocalConst32 { target_ip: target, local, imm, op }, - } - ); - rewrite!(instrs, i, - [LocalGet64(local), Const64(imm), cmp] if - (let Some(op) = cmp_op(cmp) && let Ok(imm) = i32::try_from(imm)) => - match (imm, inverse_cmp_op(op)) { - (0, CmpOp::Eq) => JumpIfLocalZero64 { target_ip: target, local }, - (0, CmpOp::Ne) => JumpIfLocalNonZero64 { target_ip: target, local }, - (imm, op) => JumpCmpLocalConst64 { target_ip: target, local, imm, op }, - } - ); - rewrite!(instrs, i, - [LocalGet32(left), LocalGet32(right), cmp] if (let Some(op) = cmp_op(cmp)) => - JumpCmpLocalLocal32 { target_ip: target, left, right, op: inverse_cmp_op(op) } - ); - rewrite!(instrs, i, - [LocalGet64(left), LocalGet64(right), cmp] if (let Some(op) = cmp_op(cmp)) => - JumpCmpLocalLocal64 { target_ip: target, left, right, op: inverse_cmp_op(op) } - ); - rewrite!(instrs, i, [Const32(imm), cmp] if (let Some(op) = cmp_op(cmp)) => match (imm, inverse_cmp_op(op)) { - (0, CmpOp::Eq) => JumpIfZero32(target), - (0, CmpOp::Ne) => JumpIfNonZero32(target), - (imm, op) => JumpCmpStackConst32 { target_ip: target, imm, op }, - }); - rewrite!(instrs, i, [Const64(imm), cmp] if (let Some(op) = cmp_op(cmp)) => match (imm, inverse_cmp_op(op)) { - (0, CmpOp::Eq) => JumpIfZero64(target), - (0, CmpOp::Ne) => JumpIfNonZero64(target), - (imm, op) => JumpCmpStackConst64 { target_ip: target, imm, op }, - }); - rewrite!(instrs, i, [LocalGet32(local)] => JumpIfLocalZero32 { target_ip: target, local }); - rewrite!(instrs, i, [LocalGet64(local)] => JumpIfLocalZero64 { target_ip: target, local }); - canonicalize_jump_like_with_target(&mut instrs, i, target, old_idx as u32 + 1); - } - JumpIfNonZero32(ip) => { - let target = resolve_jump_target(&source, ip); - rewrite!(instrs, i, [LocalGet32(local), I32Eqz] => { - replace!(instrs, i, 2 => JumpIfLocalZero32 { target_ip: target, local }); - continue; - }); - rewrite!(instrs, i, [I32Eqz] => { - replace!(instrs, i, 1 => JumpIfZero32(target)); - continue; - }); - rewrite!(instrs, i, [LocalGet64(local), I64Eqz] => { - replace!(instrs, i, 2 => JumpIfLocalZero64 { target_ip: target, local }); - continue; - }); - rewrite!(instrs, i, [I64Eqz] => { - replace!(instrs, i, 1 => JumpIfZero64(target)); - continue; - }); - rewrite!(instrs, i, - [BinOpLocalConstTee32(op, local, imm, dst)] if (local == dst) => - match inc_delta(op, imm) { - Some(delta) => IncLocalJump32 { target_ip: target, local, delta, on_zero: false }, - None => BinOpLocalConstJump32 { target_ip: target, local, imm, op, on_zero: false }, - } - ); - rewrite!(instrs, i, - [AddConst32(imm), LocalTee32(local), LocalGet32(cond)] if (local == cond) => - IncStackTeeLocalJump32 { target_ip: target, local, delta: imm, on_zero: false } - ); - rewrite!(instrs, i, - [AndConstTee32(imm, local), LocalGet32(cond)] if (local == cond) => - BinOpStackConstTeeLocalJump32 { target_ip: target, local, imm, op: BinOp::IAnd, on_zero: false } - ); - rewrite!(instrs, i, - [SubConstTee32(imm, local), LocalGet32(cond)] if (local == cond) => - IncStackTeeLocalJump32 { target_ip: target, local, delta: imm.wrapping_neg(), on_zero: false } - ); - rewrite!(instrs, i, - [BinOpGlobalConst32(op, global, imm), GlobalTee32(dst)] if (global == dst) => - match inc_delta(op, imm) { - Some(delta) => IncGlobalJump32 { target_ip: target, global, delta, on_zero: false }, - None => BinOpGlobalConstJump32 { target_ip: target, global, imm, op, on_zero: false }, - } - ); - rewrite!(instrs, i, - [CmpLocalLocal32(op, left, right)] => - JumpCmpLocalLocal32 { target_ip: target, left, right, op } - ); - rewrite!(instrs, i, - [CmpLocalLocal64(op, left, right)] => - JumpCmpLocalLocal64 { target_ip: target, left, right, op } - ); - rewrite!(instrs, i, - [BinOpLocalConstTee32(binop, local, imm, dst), LocalGet32(right), cmp] if - (local == dst && let Some(cmp) = cmp_op(cmp)) => - match inc_delta(binop, imm) { - Some(delta) => IncLocalJumpCmpLocal32 { target_ip: target, local, delta, right, op: cmp }, - None => BinOpLocalConstJumpCmpLocal32 { target_ip: target, local, imm, binop, right, cmp }, - } - ); - rewrite!(instrs, i, - [LocalGet32(local), cmp] if (let Some(op) = cmp_op(cmp)) => - JumpCmpStackLocal32 { target_ip: target, local, op } - ); - rewrite!(instrs, i, - [LocalGet64(local), cmp] if (let Some(op) = cmp_op(cmp)) => - JumpCmpStackLocal64 { target_ip: target, local, op } - ); - rewrite!(instrs, i, - [LocalGet32(local), Const32(imm), cmp] if (let Some(op) = cmp_op(cmp)) => - match (imm, op) { - (0, CmpOp::Eq) => JumpIfLocalZero32 { target_ip: target, local }, - (0, CmpOp::Ne) => JumpIfLocalNonZero32 { target_ip: target, local }, - (imm, op) => JumpCmpLocalConst32 { target_ip: target, local, imm, op }, - } - ); - rewrite!(instrs, i, - [LocalGet64(local), Const64(imm), cmp] if - (let Some(op) = cmp_op(cmp) && let Ok(imm) = i32::try_from(imm)) => - match (imm, op) { - (0, CmpOp::Eq) => JumpIfLocalZero64 { target_ip: target, local }, - (0, CmpOp::Ne) => JumpIfLocalNonZero64 { target_ip: target, local }, - (imm, op) => JumpCmpLocalConst64 { target_ip: target, local, imm, op }, - } - ); - rewrite!(instrs, i, - [LocalGet32(left), LocalGet32(right), cmp] if (let Some(op) = cmp_op(cmp)) => - JumpCmpLocalLocal32 { target_ip: target, left, right, op } - ); - rewrite!(instrs, i, - [LocalGet64(left), LocalGet64(right), cmp] if (let Some(op) = cmp_op(cmp)) => - JumpCmpLocalLocal64 { target_ip: target, left, right, op } - ); - rewrite!(instrs, i, [Const32(imm), cmp] if (let Some(op) = cmp_op(cmp)) => match (imm, op) { - (0, CmpOp::Eq) => JumpIfZero32(target), - (0, CmpOp::Ne) => JumpIfNonZero32(target), - (imm, op) => JumpCmpStackConst32 { target_ip: target, imm, op }, - }); - rewrite!(instrs, i, [Const64(imm), cmp] if (let Some(op) = cmp_op(cmp)) => match (imm, op) { - (0, CmpOp::Eq) => JumpIfZero64(target), - (0, CmpOp::Ne) => JumpIfNonZero64(target), - (imm, op) => JumpCmpStackConst64 { target_ip: target, imm, op }, - }); - rewrite!(instrs, i, [LocalGet32(local)] => JumpIfLocalNonZero32 { target_ip: target, local }); - rewrite!(instrs, i, [LocalGet64(local)] => JumpIfLocalNonZero64 { target_ip: target, local }); - canonicalize_jump_like_with_target(&mut instrs, i, target, old_idx as u32 + 1); - } - JumpIfZero64(ip) => { - let target = resolve_jump_target(&source, ip); - rewrite!(instrs, i, [LocalGet64(local)] => JumpIfLocalZero64 { target_ip: target, local }); - canonicalize_jump_like_with_target(&mut instrs, i, target, old_idx as u32 + 1); - } - JumpIfNonZero64(ip) => { - let target = resolve_jump_target(&source, ip); - rewrite!(instrs, i, [LocalGet64(local)] => JumpIfLocalNonZero64 { target_ip: target, local }); - canonicalize_jump_like_with_target(&mut instrs, i, target, old_idx as u32 + 1); - } - JumpCmpStackConst32 { target_ip, imm: 0, op } => { - match op { - CmpOp::Eq => instrs[i] = JumpIfZero32(target_ip), - CmpOp::Ne => instrs[i] = JumpIfNonZero32(target_ip), - _ => {} - } - canonicalize_jump_like(&source, &mut instrs, i, old_idx as u32 + 1); - } - JumpCmpStackConst64 { target_ip, imm: 0, op } => { - match op { - CmpOp::Eq => instrs[i] = JumpIfZero64(target_ip), - CmpOp::Ne => instrs[i] = JumpIfNonZero64(target_ip), - _ => {} - } - canonicalize_jump_like(&source, &mut instrs, i, old_idx as u32 + 1); - } - JumpCmpLocalConst32 { target_ip, local, imm: 0, op } => { - match op { - CmpOp::Eq => instrs[i] = JumpIfLocalZero32 { target_ip, local }, - CmpOp::Ne => instrs[i] = JumpIfLocalNonZero32 { target_ip, local }, - _ => {} - } - canonicalize_jump_like(&source, &mut instrs, i, old_idx as u32 + 1); - } - JumpCmpLocalConst64 { target_ip, local, imm: 0, op } => { - match op { - CmpOp::Eq => instrs[i] = JumpIfLocalZero64 { target_ip, local }, - CmpOp::Ne => instrs[i] = JumpIfLocalNonZero64 { target_ip, local }, - _ => {} - } - canonicalize_jump_like(&source, &mut instrs, i, old_idx as u32 + 1); - } - JumpCmpStackConst32 { .. } - | JumpCmpStackConst64 { .. } - | JumpCmpLocalConst32 { .. } - | JumpCmpLocalConst64 { .. } - | JumpCmpLocalLocal32 { .. } - | JumpCmpLocalLocal64 { .. } - | JumpIfLocalZero32 { .. } - | JumpIfLocalNonZero32 { .. } - | JumpIfLocalZero64 { .. } - | JumpIfLocalNonZero64 { .. } => { - canonicalize_jump_like(&source, &mut instrs, i, old_idx as u32 + 1); - } - _ => {} - } - - after_terminator = is_unconditional_terminator(instr); - } - - old_to_new[source.len()] = instrs.len() as u32; - (instrs.instructions, old_to_new) -} - -fn cmp_op(instr: Instruction) -> Option { - Some(match instr { - Instruction::I32Eq | Instruction::I64Eq => CmpOp::Eq, - Instruction::I32Ne | Instruction::I64Ne => CmpOp::Ne, - Instruction::I32LtS | Instruction::I64LtS => CmpOp::LtS, - Instruction::I32LtU | Instruction::I64LtU => CmpOp::LtU, - Instruction::I32GtS | Instruction::I64GtS => CmpOp::GtS, - Instruction::I32GtU | Instruction::I64GtU => CmpOp::GtU, - Instruction::I32LeS | Instruction::I64LeS => CmpOp::LeS, - Instruction::I32LeU | Instruction::I64LeU => CmpOp::LeU, - Instruction::I32GeS | Instruction::I64GeS => CmpOp::GeS, - Instruction::I32GeU | Instruction::I64GeU => CmpOp::GeU, - _ => return None, - }) -} - -fn inc_delta(op: BinOp, imm: i32) -> Option { - match op { - BinOp::IAdd => Some(imm), - BinOp::ISub => Some(imm.wrapping_neg()), - _ => None, - } -} - -fn int_bin_op(instr: Instruction) -> Option { - Some(match instr { - Instruction::I32Add | Instruction::I64Add => BinOp::IAdd, - Instruction::I32Sub | Instruction::I64Sub => BinOp::ISub, - Instruction::I32Mul | Instruction::I64Mul => BinOp::IMul, - Instruction::I32And | Instruction::I64And => BinOp::IAnd, - Instruction::I32Or | Instruction::I64Or => BinOp::IOr, - Instruction::I32Xor | Instruction::I64Xor => BinOp::IXor, - Instruction::I32Shl | Instruction::I64Shl => BinOp::IShl, - Instruction::I32ShrS | Instruction::I64ShrS => BinOp::IShrS, - Instruction::I32ShrU | Instruction::I64ShrU => BinOp::IShrU, - Instruction::I32Rotl | Instruction::I64Rotl => BinOp::IRotl, - Instruction::I32Rotr | Instruction::I64Rotr => BinOp::IRotr, - _ => return None, - }) -} - -fn float_bin_op(instr: Instruction) -> Option { - Some(match instr { - Instruction::F32Add | Instruction::F64Add => BinOp::FAdd, - Instruction::F32Sub | Instruction::F64Sub => BinOp::FSub, - Instruction::F32Mul | Instruction::F64Mul => BinOp::FMul, - Instruction::F32Div | Instruction::F64Div => BinOp::FDiv, - Instruction::F32Min | Instruction::F64Min => BinOp::FMin, - Instruction::F32Max | Instruction::F64Max => BinOp::FMax, - Instruction::F32Copysign | Instruction::F64Copysign => BinOp::FCopysign, - _ => return None, - }) -} - -fn scalar_bin_op(instr: Instruction) -> Option { - int_bin_op(instr).or_else(|| float_bin_op(instr)) -} - -fn scalar_const_32(instr: Instruction, op_instr: Instruction) -> Option { - match instr { - Instruction::Const32(c) if scalar_bin_op(op_instr).is_some() => Some(c), - _ => None, - } -} - -fn scalar_const_64(instr: Instruction, op_instr: Instruction) -> Option { - match instr { - Instruction::Const64(c) if scalar_bin_op(op_instr).is_some() => Some(c), - _ => None, - } -} - -fn const_128(instr: Instruction, op_instr: Instruction) -> Option { - match instr { - Instruction::Const128(c) if bin_op_128(op_instr).is_some() => Some(c), - _ => None, - } -} - -define_local_source_resolver!( - resolve_local_source_32, - get = LocalGet32, - tee = LocalTee32, - set = LocalSet32, - binop_local_local_tee = BinOpLocalLocalTee32, - binop_local_local_set = BinOpLocalLocalSet32, - binop_local_const_tee = BinOpLocalConstTee32, - binop_local_const_set = BinOpLocalConstSet32, - load_local_tee = LoadLocalTee32, - load_local_set = LoadLocalSet32 -); - -define_local_source_resolver!( - resolve_local_source_64, - get = LocalGet64, - tee = LocalTee64, - set = LocalSet64, - binop_local_local_tee = BinOpLocalLocalTee64, - binop_local_local_set = BinOpLocalLocalSet64, - binop_local_const_tee = BinOpLocalConstTee64, - binop_local_const_set = BinOpLocalConstSet64 -); - -define_local_source_resolver!( - resolve_local_source_128, - get = LocalGet128, - tee = LocalTee128, - set = LocalSet128, - binop_local_local_tee = BinOpLocalLocalTee128, - binop_local_local_set = BinOpLocalLocalSet128, - binop_local_const_tee = BinOpLocalConstTee128, - binop_local_const_set = BinOpLocalConstSet128, - load_local_tee = LoadLocalTee128, - load_local_set = LoadLocalSet128 -); - -fn bin_op_128(instr: Instruction) -> Option { - Some(match instr { - Instruction::V128And => BinOp128::And, - Instruction::V128AndNot => BinOp128::AndNot, - Instruction::V128Or => BinOp128::Or, - Instruction::V128Xor => BinOp128::Xor, - Instruction::I64x2Add => BinOp128::I64x2Add, - Instruction::I64x2Mul => BinOp128::I64x2Mul, - _ => return None, - }) -} - -fn inverse_cmp_op(op: CmpOp) -> CmpOp { - match op { - CmpOp::Eq => CmpOp::Ne, - CmpOp::Ne => CmpOp::Eq, - CmpOp::LtS => CmpOp::GeS, - CmpOp::LtU => CmpOp::GeU, - CmpOp::GtS => CmpOp::LeS, - CmpOp::GtU => CmpOp::LeU, - CmpOp::LeS => CmpOp::GtS, - CmpOp::LeU => CmpOp::GtU, - CmpOp::GeS => CmpOp::LtS, - CmpOp::GeU => CmpOp::LtU, - } -} - -fn resolve_jump_target(instrs: &[Instruction], target: u32) -> u32 { - let mut idx = target as usize; - let mut steps = 0usize; - - while let Some(Instruction::Jump(next)) = instrs.get(idx) - && steps < instrs.len() - { - idx = *next as usize; - steps += 1; - } - - idx as u32 -} - -fn instruction_target_mut(instr: &mut Instruction) -> Option<&mut u32> { - Some(match instr { - Instruction::Jump(ip) - | Instruction::JumpIfZero32(ip) - | Instruction::JumpIfNonZero32(ip) - | Instruction::JumpIfZero64(ip) - | Instruction::JumpIfNonZero64(ip) - | Instruction::JumpIfRefNull(ip) - | Instruction::JumpIfRefNonNull(ip) - | Instruction::BrOnCast(ip, _, _) - | Instruction::JumpCmpStackConst32 { target_ip: ip, .. } - | Instruction::JumpCmpStackConst64 { target_ip: ip, .. } - | Instruction::JumpCmpStackLocal32 { target_ip: ip, .. } - | Instruction::JumpCmpStackLocal64 { target_ip: ip, .. } - | Instruction::BinOpLocalConstJump32 { target_ip: ip, .. } - | Instruction::BinOpLocalConstJumpCmpLocal32 { target_ip: ip, .. } - | Instruction::BinOpStackConstTeeLocalJump32 { target_ip: ip, .. } - | Instruction::BinOpGlobalConstJump32 { target_ip: ip, .. } - | Instruction::IncLocalJump32 { target_ip: ip, .. } - | Instruction::IncStackTeeLocalJump32 { target_ip: ip, .. } - | Instruction::IncGlobalJump32 { target_ip: ip, .. } - | Instruction::IncLocalJumpCmpLocal32 { target_ip: ip, .. } - | Instruction::JumpIfLocalZero32 { target_ip: ip, .. } - | Instruction::JumpIfLocalNonZero32 { target_ip: ip, .. } - | Instruction::JumpIfLocalZero64 { target_ip: ip, .. } - | Instruction::JumpIfLocalNonZero64 { target_ip: ip, .. } - | Instruction::JumpCmpLocalConst32 { target_ip: ip, .. } - | Instruction::JumpCmpLocalConst64 { target_ip: ip, .. } - | Instruction::JumpCmpLocalLocal32 { target_ip: ip, .. } - | Instruction::JumpCmpLocalLocal64 { target_ip: ip, .. } => ip, - Instruction::BranchTable(ip, _, _) => ip, - _ => return None, - }) -} - -fn instruction_target(instr: &Instruction) -> Option { - let mut instr = *instr; - instruction_target_mut(&mut instr).copied() -} - -fn canonicalize_jump_like(source: &[Instruction], instrs: &mut Vec, idx: usize, fallthrough: u32) { - let Some(target) = instruction_target(&instrs[idx]) else { - return; - }; - - canonicalize_jump_like_with_target(instrs, idx, resolve_jump_target(source, target), fallthrough); -} - -fn canonicalize_jump_like_with_target(instrs: &mut Vec, idx: usize, target: u32, fallthrough: u32) { - if matches!(instrs[idx], Instruction::Jump(_)) && target == fallthrough { - instrs.truncate(idx); - } else if let Some(ip) = instruction_target_mut(&mut instrs[idx]) { - *ip = target; - } -} - -fn is_unconditional_terminator(instr: Instruction) -> bool { - matches!( - instr, - Instruction::Unreachable - | Instruction::Jump(_) - | Instruction::BranchTable(..) - | Instruction::Return - | Instruction::ReturnVoid - | Instruction::Return32 - | Instruction::Return64 - | Instruction::Return128 - | Instruction::ReturnCall(_) - | Instruction::ReturnCallSelf - | Instruction::ReturnCallIndirect(..) - | Instruction::ReturnCallRef(_) - | Instruction::Throw(_) - | Instruction::ThrowRef - ) -} - -fn target_boundaries(instructions: &[Instruction], function_data: &WasmFunctionData) -> Result> { - let mut boundaries = alloc::vec![false; instructions.len() + 1]; - for handler in &function_data.exception_handlers { - for target in [handler.start_ip, handler.end_ip] { - let boundary = boundaries.get_mut(target as usize).ok_or_else(|| { - ParseError::Other(alloc::format!("exception handler boundary out of bounds: {target}")) - })?; - *boundary = true; - } - for catch in &handler.catches { - let target = catch.landing_pad(); - let boundary = boundaries - .get_mut(target as usize) - .ok_or_else(|| ParseError::Other(alloc::format!("exception landing pad out of bounds: {target}")))?; - *boundary = true; - } - } - for instr in instructions { - if let Some(target) = instruction_target(instr) { - let boundary = boundaries - .get_mut(target as usize) - .ok_or_else(|| ParseError::Other(alloc::format!("instruction target out of bounds: {target}")))?; - *boundary = true; - } - if let Instruction::BranchTable(_, start, count) = *instr { - let end = - start.checked_add(count).ok_or_else(|| ParseError::Other("branch table range overflow".into()))?; - let targets = function_data - .branch_table_targets - .get(start as usize..end as usize) - .ok_or_else(|| ParseError::Other("branch table range out of bounds".into()))?; - for &target in targets { - let boundary = boundaries - .get_mut(target as usize) - .ok_or_else(|| ParseError::Other(alloc::format!("branch table target out of bounds: {target}")))?; - *boundary = true; - } - } - } - Ok(boundaries) -} - -/// Remaps rewritten targets, then validates targets and ranges while detecting local memory use. -fn finalize( - instructions: &mut [Instruction], - function_data: &mut WasmFunctionData, - old_to_new: Option<&[u32]>, - imported_memory_count: u32, -) -> Result { - let len = instructions.len() as u32; - for handler in &mut function_data.exception_handlers { - if let Some(old_to_new) = old_to_new { - handler.start_ip = *old_to_new.get(handler.start_ip as usize).ok_or_else(|| { - ParseError::Other(alloc::format!("exception handler boundary out of bounds: {}", handler.start_ip)) - })?; - handler.end_ip = *old_to_new.get(handler.end_ip as usize).ok_or_else(|| { - ParseError::Other(alloc::format!("exception handler boundary out of bounds: {}", handler.end_ip)) - })?; - for catch in &mut handler.catches { - let landing_pad = match catch { - tinywasm_types::ExceptionCatch::Tag { landing_pad, .. } - | tinywasm_types::ExceptionCatch::All { landing_pad, .. } => landing_pad, - }; - *landing_pad = *old_to_new.get(*landing_pad as usize).ok_or_else(|| { - ParseError::Other(alloc::format!("exception landing pad out of bounds: {landing_pad}")) - })?; - } - } - if handler.start_ip > handler.end_ip || handler.end_ip > len { - return Err(ParseError::Other("exception handler range out of bounds".into())); - } - if handler.catches.iter().any(|catch| catch.landing_pad() >= len) { - return Err(ParseError::Other("exception landing pad out of bounds".into())); - } - } - for target in &mut function_data.branch_table_targets { - if let Some(old_to_new) = old_to_new { - *target = *old_to_new - .get(*target as usize) - .ok_or_else(|| ParseError::Other(alloc::format!("instruction target out of bounds: {target}")))?; - } - if *target >= len { - return Err(ParseError::Other(alloc::format!("branch table target out of bounds: {target}"))); - } - } - - let mut uses_local_memory = false; - for instr in instructions { - if let Some(target) = instruction_target_mut(instr) { - if let Some(old_to_new) = old_to_new { - *target = *old_to_new - .get(*target as usize) - .ok_or_else(|| ParseError::Other(alloc::format!("instruction target out of bounds: {target}")))?; - } - if *target >= len { - return Err(ParseError::Other(alloc::format!("instruction target out of bounds: {target}"))); - } - } - if let Instruction::BranchTable(_, start, count) = *instr { - let end = - start.checked_add(count).ok_or_else(|| ParseError::Other("branch table range overflow".into()))?; - function_data - .branch_table_targets - .get(start as usize..end as usize) - .ok_or_else(|| ParseError::Other("branch table range out of bounds".into()))?; - } - uses_local_memory |= instr.memory_addr().is_some_and(|mem| mem >= imported_memory_count); - } - Ok(uses_local_memory) -} diff --git a/crates/parser/src/optimize/mod.rs b/crates/parser/src/optimize/mod.rs new file mode 100644 index 0000000..09501f9 --- /dev/null +++ b/crates/parser/src/optimize/mod.rs @@ -0,0 +1,4 @@ +mod rewrite; +mod targets; + +pub(crate) use rewrite::optimize_instructions; diff --git a/crates/parser/src/optimize/rewrite.rs b/crates/parser/src/optimize/rewrite.rs new file mode 100644 index 0000000..8e55fe9 --- /dev/null +++ b/crates/parser/src/optimize/rewrite.rs @@ -0,0 +1,1400 @@ +use super::targets::{finalize, resolve_jump_target, set_rewrite_target, target_boundaries}; +use crate::macros::optimize::{replace, rewrite}; +use crate::visit::FunctionDataBuilder as WasmFunctionData; +use crate::{ParserOptions, Result}; +use alloc::vec::Vec; +use core::ops::{Deref, DerefMut}; +use tinywasm_types::{ + BinOp, BinOp128, CmpOp, CompactMemoryArg, GlobalConst32, GlobalConst64, GlobalUpdate, GlobalV128, I32LocalArg, + I64Local, Instruction, LocalConst64, LocalConstCmp, LocalConstSet32, LocalConstSet64, LocalConstSetV128, + LocalLocalCmp, LocalTripleArg, LocalU32, LocalUpdate, LocalUpdateCmp, LocalV128, MemoryArg, MemoryFillConstOp, + MemoryLocalArg, OperandIdx, PackedOp, StackConst32, StackConst64, TargetLocal, TargetLocalArg, V128LocalArg, + V128Operand, ValueCounts, +}; + +pub(crate) struct OptimizeResult { + pub(crate) instructions: Vec, +} + +struct CompactOutput { + instructions: Vec, + block_start: usize, +} + +macro_rules! operand { + ($data:expr, $value:expr) => { + $data.push_operand($value) + }; +} + +macro_rules! target_operand { + ($data:expr, $value:expr) => { + $data.push_target_operand($value) + }; +} + +impl Deref for CompactOutput { + type Target = Vec; + + fn deref(&self) -> &Self::Target { + &self.instructions + } +} + +impl DerefMut for CompactOutput { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.instructions + } +} + +pub(crate) fn optimize_instructions( + mut instructions: Vec, + function_data: &mut WasmFunctionData, + options: &ParserOptions, + function_results: ValueCounts, + self_func_addr: u32, +) -> Result { + let boundaries = target_boundaries(&instructions, function_data)?; + let (rewritten, old_to_new) = if options.optimize_rewrite() { + let (instructions, map) = rewrite(instructions, function_data, &boundaries, function_results, self_func_addr)?; + (instructions, Some(map)) + } else { + (instructions, None) + }; + instructions = rewritten; + finalize(&mut instructions, function_data, old_to_new.as_deref())?; + Ok(OptimizeResult { instructions }) +} + +fn rewrite( + source: Vec, + data: &mut WasmFunctionData, + boundaries: &[bool], + function_results: ValueCounts, + self_func_addr: u32, +) -> Result<(Vec, Vec)> { + #![allow(unused_assignments)] + use Instruction::*; + + let return_instruction = match function_results { + ValueCounts { c32: 0, c64: 0, c128: 0 } => Some(ReturnVoid), + ValueCounts { c32: 1, c64: 0, c128: 0 } => Some(Return32), + ValueCounts { c32: 0, c64: 1, c128: 0 } => Some(Return64), + ValueCounts { c32: 0, c64: 0, c128: 1 } => Some(Return128), + _ => None, + }; + let mut output = CompactOutput { instructions: Vec::with_capacity(source.len()), block_start: 0 }; + let mut old_to_new = alloc::vec![0; source.len() + 1]; + let mut after_terminator = false; + + for (old_index, instruction) in source.iter().copied().enumerate() { + if boundaries[old_index] || after_terminator { + output.block_start = output.len(); + } + after_terminator = is_unconditional_terminator(instruction); + old_to_new[old_index] = output.len() as u32; + output.push(instruction); + let mut read = output.len() - 1; + + match output[read] { + LocalCopy32(a, b) | LocalCopy64(a, b) | LocalCopy128(a, b) if a == b => { + output.pop(); + } + Call(address) if address == self_func_addr => output[read] = CallSelf, + ReturnCall(address) if address == self_func_addr => output[read] = ReturnCallSelf, + Return if let Some(specialized) = return_instruction => output[read] = specialized, + raw @ (I32Add | I32Mul | I32And | I32Or | I32Xor) => { + let op = int_bin_op(raw).unwrap(); + rewrite!(output, read, [LocalGet32(a), LocalGet32(b)] => BinOpLocalLocal32(op, a, b)); + rewrite!(output, read, [GlobalGet32(global)] => BinOpStackGlobal32(op, global)); + rewrite!(output, read, [LocalGet32(local)] => BinOpStackLocal32(op, local)); + if rewrite_scalar_const32(&mut output, &mut read, data, op, true)? { + continue; + } + if op == BinOp::IAdd { + rewrite!(output, read, [Const32(value)] => AddConst32(value)); + rewrite!(output, read, [I32Add] => I32Add3); + if read > output.block_start + && let BinOpStackLocal32(BinOp::IAdd, local) = output[read - 1] + { + output.truncate(read - 1); + output.extend([LocalGet32(local), I32Add3]); + read = output.len() - 1; + } + } + } + raw @ (I32Sub | I32Shl | I32ShrS | I32ShrU | I32Rotl | I32Rotr) => { + let op = int_bin_op(raw).unwrap(); + rewrite!(output, read, [LocalGet32(a), LocalGet32(b)] => BinOpLocalLocal32(op, a, b)); + rewrite!(output, read, [GlobalGet32(global)] => BinOpStackGlobal32(op, global)); + rewrite!(output, read, [LocalGet32(local)] => BinOpStackLocal32(op, local)); + if rewrite_scalar_const32(&mut output, &mut read, data, op, false)? { + continue; + } + rewrite_sign_extend32(&mut output, &mut read, data, op); + } + raw @ (I64Add | I64Mul | I64And | I64Or | I64Xor) => { + let op = int_bin_op(raw).unwrap(); + rewrite!(output, read, [LocalGet64(a), LocalGet64(b)] => BinOpLocalLocal64(op, a, b)); + rewrite!(output, read, [GlobalGet64(global)] => BinOpStackGlobal64(op, global)); + if rewrite_scalar_const64(&mut output, &mut read, data, op, true)? { + continue; + } + if op == BinOp::IAdd { + rewrite!(output, read, [Const64(index)] => AddConst64(index)); + rewrite!(output, read, [I64Add] => I64Add3); + } + } + raw @ (I64Sub | I64Shl | I64ShrS | I64ShrU | I64Rotl | I64Rotr) => { + let op = int_bin_op(raw).unwrap(); + rewrite!(output, read, [LocalGet64(a), LocalGet64(b)] => BinOpLocalLocal64(op, a, b)); + rewrite!(output, read, [GlobalGet64(global)] => BinOpStackGlobal64(op, global)); + if rewrite_scalar_const64(&mut output, &mut read, data, op, false)? { + continue; + } + rewrite_sign_extend64(&mut output, &mut read, data, op); + } + raw if cmp_op(raw).is_some() => { + let op = cmp_op(raw).unwrap(); + rewrite!(output, read, [LocalGet32(a), LocalGet32(b)] => CmpLocalLocal32(op, a, b)); + rewrite!(output, read, [LocalGet64(a), LocalGet64(b)] => CmpLocalLocal64(op, a, b)); + } + raw @ (F32Add | F32Mul | F32Min | F32Max) => { + let op = float_bin_op(raw).unwrap(); + rewrite!(output, read, [LocalGet32(a), LocalGet32(b)] => BinOpLocalLocal32(op, a, b)); + rewrite!(output, read, [LocalGet32(local)] => BinOpStackLocal32(op, local)); + rewrite_scalar_const32(&mut output, &mut read, data, op, true)?; + } + raw @ (F32Sub | F32Div | F32Copysign) => { + let op = float_bin_op(raw).unwrap(); + rewrite!(output, read, [LocalGet32(a), LocalGet32(b)] => BinOpLocalLocal32(op, a, b)); + rewrite!(output, read, [LocalGet32(local)] => BinOpStackLocal32(op, local)); + rewrite_scalar_const32(&mut output, &mut read, data, op, false)?; + } + raw @ (F64Add | F64Mul | F64Min | F64Max) => { + let op = float_bin_op(raw).unwrap(); + rewrite!(output, read, [LocalGet64(a), LocalGet64(b)] => BinOpLocalLocal64(op, a, b)); + rewrite_scalar_const64(&mut output, &mut read, data, op, true)?; + } + raw @ (F64Sub | F64Div | F64Copysign) => { + let op = float_bin_op(raw).unwrap(); + rewrite!(output, read, [LocalGet64(a), LocalGet64(b)] => BinOpLocalLocal64(op, a, b)); + rewrite_scalar_const64(&mut output, &mut read, data, op, false)?; + } + raw @ (V128And | V128Or | V128Xor | I64x2Add | I64x2Mul | V128AndNot) => { + let op = bin_op_128(raw).unwrap(); + rewrite!(output, read, [LocalGet128(local)] => + BinOpStackLocal128(op, local) + ); + rewrite_vector_binop(&mut output, &mut read, data, op, raw != V128AndNot)?; + } + I32Store(index) | F32Store(index) => rewrite_store32(&mut output, &mut read, data, index)?, + I64Store(index) | F64Store(index) => rewrite_store64(&mut output, &mut read, data, index)?, + V128Store(index) => { + if let Some(memory_arg_idx) = compact_memory_arg(data, index)? { + rewrite!(output, read, + [LocalGet32(addr), LocalGet128(value)] if + (let (Ok(addr), Ok(value)) = (u8::try_from(addr), u8::try_from(value))) => + StoreLocalLocal128(MemoryLocalArg { memory_arg_idx, local1: addr, local2: value }) + ); + } + } + I32Load(index) | F32Load(index) => { + if let Some(memory_arg_idx) = compact_memory_arg(data, index)? { + rewrite!(output, read, [LocalGet32(local) | LocalGet64(local)] if + (let Ok(local) = u8::try_from(local)) => + LoadLocal32(MemoryLocalArg { memory_arg_idx, local1: local, local2: 0 }) + ); + } + } + I64Load(index) | F64Load(index) => { + if let Some(memory_arg_idx) = compact_memory_arg(data, index)? { + rewrite!(output, read, [LocalGet32(local) | LocalGet64(local)] if + (let Ok(local) = u8::try_from(local)) => + LoadLocal64(MemoryLocalArg { memory_arg_idx, local1: local, local2: 0 }) + ); + } + } + raw @ (I32Load8S(index) | I32Load8U(index) | I32Load16S(index) | I32Load16U(index)) => { + if let Some(memory_arg_idx) = compact_memory_arg(data, index)? { + rewrite!(output, read, [LocalGet32(local) | LocalGet64(local)] if + (let Ok(local) = u8::try_from(local)) => match raw { + I32Load8S(_) => LoadLocal8S32(MemoryLocalArg { memory_arg_idx, local1: local, local2: 0 }), + I32Load8U(_) => LoadLocal8U32(MemoryLocalArg { memory_arg_idx, local1: local, local2: 0 }), + I32Load16S(_) => LoadLocal16S32(MemoryLocalArg { memory_arg_idx, local1: local, local2: 0 }), + I32Load16U(_) => LoadLocal16U32(MemoryLocalArg { memory_arg_idx, local1: local, local2: 0 }), + _ => unreachable!(), + } + ); + } + } + MemoryFill(memory) => rewrite!(output, read, [Const32(value), Const32(size)] => { + let index = operand!(data, MemoryFillConstOp { memory, byte: value as u8, value: size })?; + replace!(output, read, 2 => MemoryFillConst(index)); + }), + GlobalGet32(dst) => rewrite!(output, read, [GlobalSet32(src)] if (src == dst) => GlobalTee32(src)), + GlobalGet64(dst) => rewrite!(output, read, [GlobalSet64(src)] if (src == dst) => GlobalTee64(src)), + GlobalGet128(dst) => rewrite!(output, read, [GlobalSet128(src)] if (src == dst) => GlobalTee128(src)), + LocalGet32(dst) => rewrite!(output, read, [LocalSet32(src)] if (src == dst) => LocalTee32(src)), + LocalGet64(dst) => rewrite!(output, read, [LocalSet64(src)] if (src == dst) => LocalTee64(src)), + LocalGet128(dst) => rewrite!(output, read, [LocalSet128(src)] if (src == dst) => LocalTee128(src)), + LocalSet32(dst) => rewrite_local_set32(&mut output, &mut read, data, dst)?, + LocalSet64(dst) => rewrite_local_set64(&mut output, &mut read, data, dst)?, + LocalSet128(dst) => rewrite_local_set128(&mut output, &mut read, data, dst)?, + LocalTee32(dst) => rewrite_local_tee32(&mut output, &mut read, data, dst)?, + LocalTee64(dst) => rewrite_local_tee64(&mut output, &mut read, data, dst)?, + LocalTee128(dst) => rewrite_local_tee128(&mut output, &mut read, data, dst)?, + Drop32 => { + rewrite!(output, read, [LocalTee32(local)] => LocalSet32(local)); + rewrite!(output, read, [BinOpStackLocalTee32(op, local, dst)] => BinOpStackLocalSet32(op, local, dst)); + rewrite!(output, read, [AddLocalLocalTee32(arg)] => AddLocalLocalSet32(arg)); + rewrite!(output, read, [BinOpLocalLocalTee32(packed)] => BinOpLocalLocalSet32(packed)); + rewrite!(output, read, [BinOpLocalConstTee32(packed)] => BinOpLocalConstSet32(packed)); + } + Drop64 => { + rewrite!(output, read, [LocalTee64(local)] => LocalSet64(local)); + rewrite!(output, read, [BinOpLocalLocalTee64(packed)] => BinOpLocalLocalSet64(packed)); + rewrite!(output, read, [BinOpLocalConstTee64(packed)] => BinOpLocalConstSet64(packed)); + } + Drop128 => { + rewrite!(output, read, [LocalTee128(local)] => LocalSet128(local)); + rewrite!(output, read, [BinOpLocalLocalTee128(packed)] => BinOpLocalLocalSet128(packed)); + rewrite!(output, read, [BinOpLocalConstTee128(packed)] => BinOpLocalConstSet128(packed)); + } + Jump(target) => rewrite_jump(&source, &mut output, read, data, old_index as u32, target)?, + JumpIfZero32(target) => rewrite_conditional(&source, &mut output, &mut read, data, target, true)?, + JumpIfNonZero32(target) => rewrite_conditional(&source, &mut output, &mut read, data, target, false)?, + JumpIfZero64(target) => { + rewrite!(output, read, [LocalGet64(local)] => + JumpIfLocalZero64(TargetLocalArg { target_ip: target, local }) + ); + } + JumpIfNonZero64(target) => { + rewrite!(output, read, [LocalGet64(local)] => + JumpIfLocalNonZero64(TargetLocalArg { target_ip: target, local }) + ); + } + JumpCmpStackConst32(_) + | JumpCmpStackConst64(_) + | JumpCmpLocalConst32(_) + | JumpCmpLocalConst64(_) + | JumpCmpLocalLocal32(_) + | JumpCmpLocalLocal64(_) + | JumpIfLocalZero32(_) + | JumpIfLocalNonZero32(_) + | JumpIfLocalZero64(_) + | JumpIfLocalNonZero64(_) => {} + _ => {} + } + } + for instruction in &mut output.instructions { + match *instruction { + Instruction::Const64(index) => { + let value = data.operand(index).value; + if value == i64::from(value as i32) { + *instruction = Instruction::Const64Imm(value as i32); + } + } + Instruction::Const128(index) => { + let value = u128::from_le_bytes(data.operand(index).value); + if let Ok(value) = u32::try_from(value) { + *instruction = Instruction::Const128Imm(value); + } + } + _ => {} + } + } + let end = old_to_new.len() - 1; + old_to_new[end] = output.len() as u32; + Ok((output.instructions, old_to_new)) +} + +fn local_const32(data: &WasmFunctionData, instruction: Instruction) -> Option<(BinOp, u16, i32)> { + match instruction { + Instruction::AddLocalConst32(arg) => Some((BinOp::IAdd, arg.local, arg.value)), + Instruction::SubLocalConst32(arg) => Some((BinOp::ISub, arg.local, arg.value)), + Instruction::MulLocalConst32(arg) => Some((BinOp::IMul, arg.local, arg.value)), + Instruction::BinOpLocalConst32(packed) => { + let value = data.operand(packed.index); + Some((packed.op, value.local, value.value as i32)) + } + _ => None, + } +} + +fn local_const64(data: &WasmFunctionData, instruction: Instruction) -> Option<(BinOp, u16, i64)> { + let Instruction::BinOpLocalConst64(packed) = instruction else { return None }; + let value = data.operand(packed.index); + Some((packed.op, value.local, value.value as i64)) +} + +fn local_const128( + data: &WasmFunctionData, + instruction: Instruction, +) -> Option<(BinOp128, u16, OperandIdx)> { + let Instruction::BinOpLocalConst128(packed) = instruction else { return None }; + let value = data.operand(packed.index); + Some((packed.op, value.local, value.value)) +} + +fn compact_memory_arg( + data: &mut WasmFunctionData, + index: OperandIdx, +) -> Result>> { + let Ok(arg) = CompactMemoryArg::try_from(data.operand(index)) else { return Ok(None) }; + Ok(Some(operand!(data, arg)?)) +} + +fn rewrite_scalar_const32( + output: &mut CompactOutput, + read: &mut usize, + data: &mut WasmFunctionData, + op: BinOp, + commutative: bool, +) -> Result { + if *read < output.block_start + 2 { + return Ok(false); + } + let previous = &output[*read - 2..*read]; + let replacement = match previous { + [Instruction::LocalGet32(local), Instruction::Const32(value)] => { + let arg = I32LocalArg { value: *value, local: *local }; + match op { + BinOp::IAdd => Instruction::AddLocalConst32(arg), + BinOp::ISub => Instruction::SubLocalConst32(arg), + BinOp::IMul => Instruction::MulLocalConst32(arg), + _ => Instruction::BinOpLocalConst32(PackedOp::new( + op, + operand!(data, LocalU32 { local: *local, value: *value as u32 })?, + )), + } + } + [Instruction::GlobalGet32(global), Instruction::Const32(value)] => Instruction::BinOpGlobalConst32( + PackedOp::new(op, operand!(data, GlobalConst32 { global: *global, value: *value as u32 })?), + ), + [Instruction::Const32(value), Instruction::LocalGet32(local)] if commutative => { + let arg = I32LocalArg { value: *value, local: *local }; + match op { + BinOp::IAdd => Instruction::AddLocalConst32(arg), + BinOp::IMul => Instruction::MulLocalConst32(arg), + _ => Instruction::BinOpLocalConst32(PackedOp::new( + op, + operand!(data, LocalU32 { local: *local, value: *value as u32 })?, + )), + } + } + [Instruction::Const32(value), Instruction::GlobalGet32(global)] if commutative => { + Instruction::BinOpGlobalConst32(PackedOp::new( + op, + operand!(data, GlobalConst32 { global: *global, value: *value as u32 })?, + )) + } + _ => return Ok(false), + }; + replace!(output, *read, 2 => replacement); + Ok(true) +} + +fn rewrite_scalar_const64( + output: &mut CompactOutput, + read: &mut usize, + data: &mut WasmFunctionData, + op: BinOp, + commutative: bool, +) -> Result { + if *read < output.block_start + 2 { + return Ok(false); + } + let previous = [output[*read - 2], output[*read - 1]]; + let replacement = match previous { + [Instruction::LocalGet64(local), Instruction::Const64(index)] => Instruction::BinOpLocalConst64(PackedOp::new( + op, + operand!(data, LocalConst64 { local, value: data.operand(index).value as u64 })?, + )), + [Instruction::GlobalGet64(global), Instruction::Const64(index)] => Instruction::BinOpGlobalConst64( + PackedOp::new(op, operand!(data, GlobalConst64 { global, value: data.operand(index).value as u64 })?), + ), + [Instruction::Const64(index), Instruction::LocalGet64(local)] if commutative => Instruction::BinOpLocalConst64( + PackedOp::new(op, operand!(data, LocalConst64 { local, value: data.operand(index).value as u64 })?), + ), + [Instruction::Const64(index), Instruction::GlobalGet64(global)] if commutative => { + Instruction::BinOpGlobalConst64(PackedOp::new( + op, + operand!(data, GlobalConst64 { global, value: data.operand(index).value as u64 })?, + )) + } + _ => return Ok(false), + }; + replace!(output, *read, 2 => replacement); + Ok(true) +} + +fn rewrite_sign_extend32(output: &mut CompactOutput, read: &mut usize, data: &WasmFunctionData, op: BinOp) -> bool { + if op != BinOp::IShrS || *read < output.block_start + 2 { + return false; + } + if let Some((BinOp::IShl, local, shift)) = local_const32(data, output[*read - 2]) + && let Instruction::Const32(right) = output[*read - 1] + && shift == right + && matches!(shift, 16 | 24) + { + output.truncate(*read - 2); + output.extend([ + Instruction::LocalGet32(local), + if shift == 24 { Instruction::I32Extend8S } else { Instruction::I32Extend16S }, + ]); + *read = output.len() - 1; + return true; + } + false +} + +fn rewrite_sign_extend64(output: &mut CompactOutput, read: &mut usize, data: &WasmFunctionData, op: BinOp) -> bool { + if op != BinOp::IShrS || *read < output.block_start + 2 { + return false; + } + if let Some((BinOp::IShl, local, shift)) = local_const64(data, output[*read - 2]) + && let Instruction::Const64(index) = output[*read - 1] + && shift == data.operand(index).value + { + let instruction = match shift { + 56 => Instruction::I64Extend8S, + 48 => Instruction::I64Extend16S, + 32 => Instruction::I64Extend32S, + _ => return false, + }; + output.truncate(*read - 2); + output.extend([Instruction::LocalGet64(local), instruction]); + *read = output.len() - 1; + return true; + } + false +} + +fn rewrite_vector_binop( + output: &mut CompactOutput, + read: &mut usize, + data: &mut WasmFunctionData, + op: BinOp128, + commutative: bool, +) -> Result<()> { + if *read >= output.block_start + 2 { + let previous = [output[*read - 2], output[*read - 1]]; + let replacement = match previous { + [Instruction::LocalGet128(a), Instruction::LocalGet128(b)] => Instruction::BinOpLocalLocal128(op, a, b), + [Instruction::LocalGet128(local), Instruction::Const128(value)] => { + Instruction::BinOpLocalConst128(PackedOp::new(op, operand!(data, LocalV128 { local, value })?)) + } + [Instruction::GlobalGet128(global), Instruction::Const128(value)] => { + Instruction::BinOpGlobalConst128(PackedOp::new(op, operand!(data, GlobalV128 { global, value })?)) + } + [Instruction::Const128(value), Instruction::LocalGet128(local)] if commutative => { + Instruction::BinOpLocalConst128(PackedOp::new(op, operand!(data, LocalV128 { local, value })?)) + } + [Instruction::Const128(value), Instruction::GlobalGet128(global)] if commutative => { + Instruction::BinOpGlobalConst128(PackedOp::new(op, operand!(data, GlobalV128 { global, value })?)) + } + _ => return Ok(()), + }; + replace!(output, *read, 2 => replacement); + } + Ok(()) +} + +fn rewrite_store32( + output: &mut CompactOutput, + read: &mut usize, + data: &mut WasmFunctionData, + index: OperandIdx, +) -> Result<()> { + let compact_arg = CompactMemoryArg::try_from(data.operand(index)).ok(); + if *read >= output.block_start + 3 { + let previous = [output[*read - 3], output[*read - 2], output[*read - 1]]; + if let [ + Instruction::LocalGet32(addr) | Instruction::LocalGet64(addr), + Instruction::LoadLocal32(arg), + Instruction::AddConst32(1), + ] = previous + && { compact_arg == Some(data.operand(arg.memory_arg_idx)) } + && addr == u16::from(arg.local1) + { + replace!(output, *read, 3 => Instruction::IncMemoryLocal32(arg)); + return Ok(()); + } + } + if *read >= output.block_start + 2 { + let previous = [output[*read - 2], output[*read - 1]]; + match previous { + [Instruction::F32Mul, Instruction::F32Add] + if let Ok(arg) = CompactMemoryArg::try_from(data.operand(index)) => + { + replace!(output, *read, 2 => Instruction::FMaStoreF32(arg)); + } + [Instruction::BinOpStackLocal32(BinOp::FMul, local), Instruction::F32Add] + if let Ok(arg) = CompactMemoryArg::try_from(data.operand(index)) => + { + replace!(output, *read, 2 => [Instruction::LocalGet32(local), Instruction::FMaStoreF32(arg)]); + } + [Instruction::LocalGet32(addr), Instruction::LocalGet32(value)] + if let (Ok(addr), Ok(value), Some(memory_arg)) = + (u8::try_from(addr), u8::try_from(value), compact_arg) => + { + let memory_arg_idx = operand!(data, memory_arg)?; + replace!(output, *read, 2 => Instruction::StoreLocalLocal32(MemoryLocalArg { memory_arg_idx, local1: addr, local2: value })); + } + _ => {} + } + } + Ok(()) +} + +fn rewrite_store64( + output: &mut CompactOutput, + read: &mut usize, + data: &mut WasmFunctionData, + index: OperandIdx, +) -> Result<()> { + let compact_arg = CompactMemoryArg::try_from(data.operand(index)).ok(); + if *read >= output.block_start + 3 { + let previous = [output[*read - 3], output[*read - 2], output[*read - 1]]; + if let [ + Instruction::LocalGet32(addr) | Instruction::LocalGet64(addr), + Instruction::LoadLocal64(arg), + Instruction::Const64(one), + ] = previous + && { compact_arg == Some(data.operand(arg.memory_arg_idx)) } + && addr == u16::from(arg.local1) + && data.operand(one).value == 1 + { + replace!(output, *read, 3 => Instruction::IncMemoryLocal64(arg)); + return Ok(()); + } + } + if *read >= output.block_start + 2 { + match [output[*read - 2], output[*read - 1]] { + [Instruction::F64Mul, Instruction::F64Add] + if let Ok(arg) = CompactMemoryArg::try_from(data.operand(index)) => + { + replace!(output, *read, 2 => Instruction::FMaStoreF64(arg)); + } + [Instruction::LocalGet32(addr), Instruction::LocalGet64(value)] + if let (Ok(addr), Ok(value), Some(memory_arg)) = + (u8::try_from(addr), u8::try_from(value), compact_arg) => + { + let memory_arg_idx = operand!(data, memory_arg)?; + replace!(output, *read, 2 => Instruction::StoreLocalLocal64(MemoryLocalArg { memory_arg_idx, local1: addr, local2: value })); + } + _ => {} + } + } + Ok(()) +} + +macro_rules! local_const_set { + ($data:expr, 32, $op:expr, $src:expr, $dst:expr, $value:expr, $tee:expr) => {{ + let index = operand!($data, LocalConstSet32 { local: $src, dst: $dst, value: $value as u32 })?; + if $tee { + Instruction::BinOpLocalConstTee32(PackedOp::new($op, index)) + } else { + Instruction::BinOpLocalConstSet32(PackedOp::new($op, index)) + } + }}; + ($data:expr, 64, $op:expr, $src:expr, $dst:expr, $value:expr, $tee:expr) => {{ + let index = operand!($data, LocalConstSet64 { local: $src, dst: $dst, value: $value as u64 })?; + if $tee { + Instruction::BinOpLocalConstTee64(PackedOp::new($op, index)) + } else { + Instruction::BinOpLocalConstSet64(PackedOp::new($op, index)) + } + }}; + ($data:expr, 128, $op:expr, $src:expr, $dst:expr, $value:expr, $tee:expr) => {{ + let index = operand!($data, LocalConstSetV128 { local: $src, dst: $dst, value: $value })?; + if $tee { + Instruction::BinOpLocalConstTee128(PackedOp::new($op, index)) + } else { + Instruction::BinOpLocalConstSet128(PackedOp::new($op, index)) + } + }}; +} + +fn rewrite_local_set32( + output: &mut CompactOutput, + read: &mut usize, + data: &mut WasmFunctionData, + dst: u16, +) -> Result<()> { + if *read >= output.block_start + 2 { + match [output[*read - 2], output[*read - 1]] { + [Instruction::I32Mul, Instruction::BinOpStackLocal32(BinOp::IAdd, acc)] if acc == dst => { + replace!(output, *read, 2 => Instruction::MulAccLocal32(dst)); + return Ok(()); + } + [Instruction::F32Mul, Instruction::BinOpStackLocal32(BinOp::FAdd, acc)] if acc == dst => { + replace!(output, *read, 2 => Instruction::FMulAccLocal32(dst)); + return Ok(()); + } + _ => {} + } + } + if *read > output.block_start { + match output[*read - 1] { + Instruction::LocalGet32(src) if src == dst => { + output.truncate(*read - 1); + *read = output.len(); + return Ok(()); + } + Instruction::LocalGet32(src) => replace!(output, *read, 1 => Instruction::LocalCopy32(src, dst)), + Instruction::Const32(value) => { + replace!(output, *read, 1 => Instruction::SetLocalConst32(I32LocalArg { value, local: dst })) + } + Instruction::BinOpLocalLocal32(op, left, right) => { + let replacement = if op == BinOp::IAdd { + Instruction::AddLocalLocalSet32(LocalTripleArg { left, right, dst }) + } else { + Instruction::BinOpLocalLocalSet32(PackedOp::new( + op, + operand!(data, LocalTripleArg { left, right, dst })?, + )) + }; + replace!(output, *read, 1 => replacement); + } + instruction if let Some((op, src, value)) = local_const32(data, instruction) => { + if src == dst + && let Some(delta) = op.inc_delta(value) + { + replace!(output, *read, 1 => Instruction::IncLocal32(I32LocalArg { value: delta, local: dst })); + } else { + let replacement = local_const_set!(data, 32, op, src, dst, value, false); + replace!(output, *read, 1 => replacement); + } + } + Instruction::BinOpStackLocal32(op, local) => { + replace!(output, *read, 1 => Instruction::BinOpStackLocalSet32(op, local, dst)); + } + Instruction::LoadLocal32(mut arg) if let Ok(dst) = u8::try_from(dst) => { + arg.local2 = dst; + replace!(output, *read, 1 => Instruction::LoadLocalSet32(arg)); + } + Instruction::LoadLocal8S32(mut arg) if let Ok(dst) = u8::try_from(dst) => { + arg.local2 = dst; + replace!(output, *read, 1 => Instruction::LoadLocalSet8S32(arg)); + } + Instruction::LoadLocal8U32(mut arg) if let Ok(dst) = u8::try_from(dst) => { + arg.local2 = dst; + replace!(output, *read, 1 => Instruction::LoadLocalSet8U32(arg)); + } + Instruction::LoadLocal16S32(mut arg) if let Ok(dst) = u8::try_from(dst) => { + arg.local2 = dst; + replace!(output, *read, 1 => Instruction::LoadLocalSet16S32(arg)); + } + Instruction::LoadLocal16U32(mut arg) if let Ok(dst) = u8::try_from(dst) => { + arg.local2 = dst; + replace!(output, *read, 1 => Instruction::LoadLocalSet16U32(arg)); + } + _ => {} + } + } + Ok(()) +} + +fn rewrite_local_set64( + output: &mut CompactOutput, + read: &mut usize, + data: &mut WasmFunctionData, + dst: u16, +) -> Result<()> { + if *read >= output.block_start + 3 { + match [output[*read - 3], output[*read - 2], output[*read - 1]] { + [Instruction::I64Mul, Instruction::LocalGet64(acc), Instruction::I64Add] if acc == dst => { + replace!(output, *read, 3 => Instruction::MulAccLocal64(dst)); + return Ok(()); + } + [Instruction::F64Mul, Instruction::LocalGet64(acc), Instruction::F64Add] if acc == dst => { + replace!(output, *read, 3 => Instruction::FMulAccLocal64(dst)); + return Ok(()); + } + _ => {} + } + } + if *read > output.block_start { + match output[*read - 1] { + Instruction::LocalGet64(src) if src == dst => { + output.truncate(*read - 1); + *read = output.len(); + return Ok(()); + } + Instruction::LocalGet64(src) => replace!(output, *read, 1 => Instruction::LocalCopy64(src, dst)), + Instruction::Const64(index) => { + let index = operand!(data, I64Local { value: data.operand(index).value, local: dst })?; + replace!(output, *read, 1 => Instruction::SetLocalConst64(index)); + } + Instruction::BinOpLocalLocal64(op, left, right) => { + let index = operand!(data, LocalTripleArg { left, right, dst })?; + replace!(output, *read, 1 => Instruction::BinOpLocalLocalSet64(PackedOp::new(op, index))); + } + instruction if let Some((op, src, value)) = local_const64(data, instruction) => { + if src == dst && matches!(op, BinOp::IAdd | BinOp::ISub) { + let delta = if op == BinOp::IAdd { value } else { value.wrapping_neg() }; + let index = operand!(data, I64Local { value: delta, local: dst })?; + replace!(output, *read, 1 => Instruction::IncLocal64(index)); + } else { + let replacement = local_const_set!(data, 64, op, src, dst, value, false); + replace!(output, *read, 1 => replacement); + } + } + _ => {} + } + } + Ok(()) +} + +fn rewrite_local_set128( + output: &mut CompactOutput, + read: &mut usize, + data: &mut WasmFunctionData, + dst: u16, +) -> Result<()> { + if *read >= output.block_start + 2 + && let [Instruction::LocalGet32(local), Instruction::V128Load(index)] = [output[*read - 2], output[*read - 1]] + && let (Ok(local), Ok(dst), Ok(memory_arg)) = + (u8::try_from(local), u8::try_from(dst), CompactMemoryArg::try_from(data.operand(index))) + { + let memory_arg_idx = operand!(data, memory_arg)?; + replace!(output, *read, 2 => Instruction::LoadLocalSet128(MemoryLocalArg { + memory_arg_idx, + local1: local, + local2: dst, + })); + return Ok(()); + } + if *read > output.block_start { + match output[*read - 1] { + Instruction::LocalGet128(src) if src == dst => { + output.truncate(*read - 1); + *read = output.len(); + return Ok(()); + } + Instruction::LocalGet128(src) => replace!(output, *read, 1 => Instruction::LocalCopy128(src, dst)), + Instruction::Const128(value) => { + replace!(output, *read, 1 => Instruction::SetLocalConst128(V128LocalArg { value, local: dst })) + } + Instruction::BinOpLocalLocal128(op, left, right) => { + let index = operand!(data, LocalTripleArg { left, right, dst })?; + replace!(output, *read, 1 => Instruction::BinOpLocalLocalSet128(PackedOp::new(op, index))); + } + instruction if let Some((op, src, value)) = local_const128(data, instruction) => { + let replacement = local_const_set!(data, 128, op, src, dst, value, false); + replace!(output, *read, 1 => replacement); + } + _ => {} + } + } + Ok(()) +} + +fn rewrite_local_tee32( + output: &mut CompactOutput, + read: &mut usize, + data: &mut WasmFunctionData, + dst: u16, +) -> Result<()> { + if *read >= output.block_start + 2 { + match [output[*read - 2], output[*read - 1]] { + [Instruction::Const32(value), Instruction::I32And] => { + replace!(output, *read, 2 => Instruction::AndConstTee32(I32LocalArg { value, local: dst })); + return Ok(()); + } + [Instruction::Const32(value), Instruction::I32Sub] => { + replace!(output, *read, 2 => Instruction::SubConstTee32(I32LocalArg { value, local: dst })); + return Ok(()); + } + _ => {} + } + } + if *read > output.block_start { + match output[*read - 1] { + Instruction::LocalGet32(src) if src == dst => replace!(output, *read, 1 => Instruction::LocalGet32(src)), + Instruction::BinOpLocalLocal32(op, left, right) => { + let replacement = if op == BinOp::IAdd { + Instruction::AddLocalLocalTee32(LocalTripleArg { left, right, dst }) + } else { + Instruction::BinOpLocalLocalTee32(PackedOp::new( + op, + operand!(data, LocalTripleArg { left, right, dst })?, + )) + }; + replace!(output, *read, 1 => replacement); + } + instruction if let Some((op, src, value)) = local_const32(data, instruction) => { + let replacement = local_const_set!(data, 32, op, src, dst, value, true); + replace!(output, *read, 1 => replacement); + } + Instruction::BinOpStackLocal32(op, local) => { + replace!(output, *read, 1 => Instruction::BinOpStackLocalTee32(op, local, dst)); + } + Instruction::LoadLocal32(mut arg) if let Ok(dst) = u8::try_from(dst) => { + arg.local2 = dst; + replace!(output, *read, 1 => Instruction::LoadLocalTee32(arg)); + } + Instruction::LoadLocal8S32(mut arg) if let Ok(dst) = u8::try_from(dst) => { + arg.local2 = dst; + replace!(output, *read, 1 => Instruction::LoadLocalTee8S32(arg)); + } + Instruction::LoadLocal8U32(mut arg) if let Ok(dst) = u8::try_from(dst) => { + arg.local2 = dst; + replace!(output, *read, 1 => Instruction::LoadLocalTee8U32(arg)); + } + Instruction::LoadLocal16S32(mut arg) if let Ok(dst) = u8::try_from(dst) => { + arg.local2 = dst; + replace!(output, *read, 1 => Instruction::LoadLocalTee16S32(arg)); + } + Instruction::LoadLocal16U32(mut arg) if let Ok(dst) = u8::try_from(dst) => { + arg.local2 = dst; + replace!(output, *read, 1 => Instruction::LoadLocalTee16U32(arg)); + } + _ => {} + } + } + Ok(()) +} + +fn rewrite_local_tee64( + output: &mut CompactOutput, + read: &mut usize, + data: &mut WasmFunctionData, + dst: u16, +) -> Result<()> { + if *read >= output.block_start + 2 { + match [output[*read - 2], output[*read - 1]] { + [Instruction::Const64(value), Instruction::I64And] => { + let index = operand!(data, I64Local { value: data.operand(value).value, local: dst })?; + replace!(output, *read, 2 => Instruction::AndConstTee64(index)); + return Ok(()); + } + [Instruction::Const64(value), Instruction::I64Sub] => { + let index = operand!(data, I64Local { value: data.operand(value).value, local: dst })?; + replace!(output, *read, 2 => Instruction::SubConstTee64(index)); + return Ok(()); + } + _ => {} + } + } + if *read > output.block_start { + match output[*read - 1] { + Instruction::LocalGet64(src) if src == dst => replace!(output, *read, 1 => Instruction::LocalGet64(src)), + Instruction::BinOpLocalLocal64(op, left, right) => { + let index = operand!(data, LocalTripleArg { left, right, dst })?; + replace!(output, *read, 1 => Instruction::BinOpLocalLocalTee64(PackedOp::new(op, index))); + } + instruction if let Some((op, src, value)) = local_const64(data, instruction) => { + let replacement = local_const_set!(data, 64, op, src, dst, value, true); + replace!(output, *read, 1 => replacement); + } + _ => {} + } + } + Ok(()) +} + +fn rewrite_local_tee128( + output: &mut CompactOutput, + read: &mut usize, + data: &mut WasmFunctionData, + dst: u16, +) -> Result<()> { + if *read >= output.block_start + 2 + && let [Instruction::LocalGet32(local), Instruction::V128Load(index)] = [output[*read - 2], output[*read - 1]] + && let (Ok(local), Ok(dst), Ok(memory_arg)) = + (u8::try_from(local), u8::try_from(dst), CompactMemoryArg::try_from(data.operand(index))) + { + let memory_arg_idx = operand!(data, memory_arg)?; + replace!(output, *read, 2 => Instruction::LoadLocalTee128(MemoryLocalArg { + memory_arg_idx, + local1: local, + local2: dst, + })); + return Ok(()); + } + if *read > output.block_start { + match output[*read - 1] { + Instruction::LocalGet128(src) if src == dst => replace!(output, *read, 1 => Instruction::LocalGet128(src)), + Instruction::BinOpLocalLocal128(op, left, right) => { + let index = operand!(data, LocalTripleArg { left, right, dst })?; + replace!(output, *read, 1 => Instruction::BinOpLocalLocalTee128(PackedOp::new(op, index))); + } + instruction if let Some((op, src, value)) = local_const128(data, instruction) => { + let replacement = local_const_set!(data, 128, op, src, dst, value, true); + replace!(output, *read, 1 => replacement); + } + _ => {} + } + } + Ok(()) +} + +fn int_bin_op(instruction: Instruction) -> Option { + Some(match instruction { + Instruction::I32Add | Instruction::I64Add => BinOp::IAdd, + Instruction::I32Sub | Instruction::I64Sub => BinOp::ISub, + Instruction::I32Mul | Instruction::I64Mul => BinOp::IMul, + Instruction::I32And | Instruction::I64And => BinOp::IAnd, + Instruction::I32Or | Instruction::I64Or => BinOp::IOr, + Instruction::I32Xor | Instruction::I64Xor => BinOp::IXor, + Instruction::I32Shl | Instruction::I64Shl => BinOp::IShl, + Instruction::I32ShrS | Instruction::I64ShrS => BinOp::IShrS, + Instruction::I32ShrU | Instruction::I64ShrU => BinOp::IShrU, + Instruction::I32Rotl | Instruction::I64Rotl => BinOp::IRotl, + Instruction::I32Rotr | Instruction::I64Rotr => BinOp::IRotr, + _ => return None, + }) +} + +fn float_bin_op(instruction: Instruction) -> Option { + Some(match instruction { + Instruction::F32Add | Instruction::F64Add => BinOp::FAdd, + Instruction::F32Sub | Instruction::F64Sub => BinOp::FSub, + Instruction::F32Mul | Instruction::F64Mul => BinOp::FMul, + Instruction::F32Div | Instruction::F64Div => BinOp::FDiv, + Instruction::F32Min | Instruction::F64Min => BinOp::FMin, + Instruction::F32Max | Instruction::F64Max => BinOp::FMax, + Instruction::F32Copysign | Instruction::F64Copysign => BinOp::FCopysign, + _ => return None, + }) +} + +fn cmp_op(instruction: Instruction) -> Option { + Some(match instruction { + Instruction::I32Eq | Instruction::I64Eq => CmpOp::Eq, + Instruction::I32Ne | Instruction::I64Ne => CmpOp::Ne, + Instruction::I32LtS | Instruction::I64LtS => CmpOp::LtS, + Instruction::I32LtU | Instruction::I64LtU => CmpOp::LtU, + Instruction::I32GtS | Instruction::I64GtS => CmpOp::GtS, + Instruction::I32GtU | Instruction::I64GtU => CmpOp::GtU, + Instruction::I32LeS | Instruction::I64LeS => CmpOp::LeS, + Instruction::I32LeU | Instruction::I64LeU => CmpOp::LeU, + Instruction::I32GeS | Instruction::I64GeS => CmpOp::GeS, + Instruction::I32GeU | Instruction::I64GeU => CmpOp::GeU, + _ => return None, + }) +} + +fn bin_op_128(instruction: Instruction) -> Option { + Some(match instruction { + Instruction::V128And => BinOp128::And, + Instruction::V128AndNot => BinOp128::AndNot, + Instruction::V128Or => BinOp128::Or, + Instruction::V128Xor => BinOp128::Xor, + Instruction::I64x2Add => BinOp128::I64x2Add, + Instruction::I64x2Mul => BinOp128::I64x2Mul, + _ => return None, + }) +} + +fn jump_cmp_local_local( + data: &mut WasmFunctionData, + target: u32, + left: u16, + right: u16, + op: CmpOp, + width64: bool, +) -> Result { + let index = target_operand!(data, LocalLocalCmp { target, left, right })?; + Ok(if width64 { + Instruction::JumpCmpLocalLocal64(PackedOp::new(op, index)) + } else { + Instruction::JumpCmpLocalLocal32(PackedOp::new(op, index)) + }) +} + +fn rewrite_jump( + source: &[Instruction], + output: &mut CompactOutput, + index: usize, + data: &mut WasmFunctionData, + old_index: u32, + target: u32, +) -> Result<()> { + let target = resolve_jump_target(source, data, target); + let exit = old_index + 1; + let body = target + 1; + if let Some(instruction) = source.get(target as usize).copied() { + match instruction { + Instruction::JumpCmpLocalLocal32(side) + if resolve_jump_target(source, data, data.operand(side.index).target) == exit && body > target => + { + let value = data.operand(side.index); + output[index] = jump_cmp_local_local(data, body, value.left, value.right, side.op.inverse(), false)?; + return Ok(()); + } + Instruction::JumpCmpLocalLocal64(side) + if resolve_jump_target(source, data, data.operand(side.index).target) == exit && body > target => + { + let value = data.operand(side.index); + output[index] = jump_cmp_local_local(data, body, value.left, value.right, side.op.inverse(), true)?; + return Ok(()); + } + _ => {} + } + } + if matches!(output[index], Instruction::Jump(_)) && target == exit { + output.truncate(index); + } else { + set_rewrite_target(&mut output[index], data, target)?; + } + Ok(()) +} + +fn jump_cmp_stack_local( + data: &mut WasmFunctionData, + target: u32, + local: u16, + op: CmpOp, + width64: bool, +) -> Result { + let index = target_operand!(data, TargetLocal { target, local })?; + Ok(if width64 { + Instruction::JumpCmpStackLocal64(PackedOp::new(op, index)) + } else { + Instruction::JumpCmpStackLocal32(PackedOp::new(op, index)) + }) +} + +fn jump_cmp_local_const32( + data: &mut WasmFunctionData, + target: u32, + local: u16, + value: i32, + op: CmpOp, +) -> Result { + if value == 0 { + return Ok(match op { + CmpOp::Eq => Instruction::JumpIfLocalZero32(TargetLocalArg { target_ip: target, local }), + CmpOp::Ne => Instruction::JumpIfLocalNonZero32(TargetLocalArg { target_ip: target, local }), + _ => Instruction::JumpCmpLocalConst32(PackedOp::new( + op, + target_operand!(data, LocalConstCmp { target, value, local })?, + )), + }); + } + Ok(Instruction::JumpCmpLocalConst32(PackedOp::new( + op, + target_operand!(data, LocalConstCmp { target, value, local })?, + ))) +} + +fn jump_cmp_local_const64( + data: &mut WasmFunctionData, + target: u32, + local: u16, + value: i32, + op: CmpOp, +) -> Result { + if value == 0 { + match op { + CmpOp::Eq => return Ok(Instruction::JumpIfLocalZero64(TargetLocalArg { target_ip: target, local })), + CmpOp::Ne => return Ok(Instruction::JumpIfLocalNonZero64(TargetLocalArg { target_ip: target, local })), + _ => {} + } + } + Ok(Instruction::JumpCmpLocalConst64(PackedOp::new( + op, + target_operand!(data, LocalConstCmp { target, value, local })?, + ))) +} + +fn jump_cmp_stack_const32(data: &mut WasmFunctionData, target: u32, value: i32, op: CmpOp) -> Result { + if value == 0 { + match op { + CmpOp::Eq => return Ok(Instruction::JumpIfZero32(target)), + CmpOp::Ne => return Ok(Instruction::JumpIfNonZero32(target)), + _ => {} + } + } + Ok(Instruction::JumpCmpStackConst32(PackedOp::new(op, target_operand!(data, StackConst32 { target, value })?))) +} + +fn jump_cmp_stack_const64(data: &mut WasmFunctionData, target: u32, value: i64, op: CmpOp) -> Result { + if value == 0 { + match op { + CmpOp::Eq => return Ok(Instruction::JumpIfZero64(target)), + CmpOp::Ne => return Ok(Instruction::JumpIfNonZero64(target)), + _ => {} + } + } + Ok(Instruction::JumpCmpStackConst64(PackedOp::new(op, target_operand!(data, StackConst64 { target, value })?))) +} + +fn update_jump( + data: &mut WasmFunctionData, + target: u32, + immediate: i32, + address: u32, + op: BinOp, + on_zero: bool, + global: bool, +) -> Result { + if let Some(delta) = op.inc_delta(immediate) { + Ok(if global { + Instruction::IncGlobalJump32(target_operand!( + data, + GlobalUpdate { target, value: delta, global: address, on_zero: u8::from(on_zero) } + )?) + } else { + Instruction::IncLocalJump32(target_operand!( + data, + LocalUpdate { target, value: delta, local: address as u16, on_zero: u8::from(on_zero) } + )?) + }) + } else { + Ok(if global { + Instruction::BinOpGlobalConstJump32(PackedOp::new( + op, + target_operand!( + data, + GlobalUpdate { target, value: immediate, global: address, on_zero: u8::from(on_zero) } + )?, + )) + } else { + Instruction::BinOpLocalConstJump32(PackedOp::new( + op, + target_operand!( + data, + LocalUpdate { target, value: immediate, local: address as u16, on_zero: u8::from(on_zero) } + )?, + )) + }) + } +} + +fn rewrite_conditional( + source: &[Instruction], + output: &mut CompactOutput, + read: &mut usize, + data: &mut WasmFunctionData, + target: u32, + on_zero: bool, +) -> Result<()> { + let target = resolve_jump_target(source, data, target); + if *read > output.block_start + && let Instruction::BinOpLocalConstTee32(packed) = output[*read - 1] + { + let value = data.operand(packed.index); + if value.local == value.dst { + let replacement = + update_jump(data, target, value.value as i32, u32::from(value.local), packed.op, on_zero, false)?; + replace!(output, *read, 1 => replacement); + return Ok(()); + } + } + if *read >= output.block_start + 2 + && let [Instruction::BinOpGlobalConst32(packed), Instruction::GlobalTee32(dst)] = + [output[*read - 2], output[*read - 1]] + { + let value = data.operand(packed.index); + if value.global == dst { + let replacement = update_jump(data, target, value.value as i32, dst, packed.op, on_zero, true)?; + replace!(output, *read, 2 => replacement); + return Ok(()); + } + } + if *read >= output.block_start + 3 + && let [Instruction::AddConst32(value), Instruction::LocalTee32(local), Instruction::LocalGet32(cond)] = + [output[*read - 3], output[*read - 2], output[*read - 1]] + && local == cond + { + let replacement = Instruction::IncStackTeeLocalJump32(target_operand!( + data, + LocalUpdate { target, value, local, on_zero: u8::from(on_zero) } + )?); + replace!(output, *read, 3 => replacement); + return Ok(()); + } + if *read >= output.block_start + 2 { + let update = match [output[*read - 2], output[*read - 1]] { + [Instruction::AndConstTee32(arg), Instruction::LocalGet32(cond)] if arg.local == cond => { + Some((arg.local, arg.value, Some(BinOp::IAnd))) + } + [Instruction::SubConstTee32(arg), Instruction::LocalGet32(cond)] if arg.local == cond => { + Some((arg.local, arg.value.wrapping_neg(), None)) + } + _ => None, + }; + if let Some((local, value, op)) = update { + let replacement = if let Some(op) = op { + Instruction::BinOpStackConstTeeLocalJump32(PackedOp::new( + op, + target_operand!(data, LocalUpdate { target, value, local, on_zero: u8::from(on_zero) })?, + )) + } else { + Instruction::IncStackTeeLocalJump32(target_operand!( + data, + LocalUpdate { target, value, local, on_zero: u8::from(on_zero) } + )?) + }; + replace!(output, *read, 2 => replacement); + return Ok(()); + } + } + if *read >= output.block_start + 3 + && let [Instruction::BinOpLocalConstTee32(packed), Instruction::LocalGet32(right), raw_cmp] = + [output[*read - 3], output[*read - 2], output[*read - 1]] + && let Some(mut cmp) = cmp_op(raw_cmp) + { + let value = data.operand(packed.index); + if value.local == value.dst { + if on_zero { + cmp = cmp.inverse(); + } + let replacement = if let Some(delta) = packed.op.inc_delta(value.value as i32) { + Instruction::IncLocalJumpCmpLocal32(PackedOp::new( + cmp, + target_operand!(data, LocalUpdateCmp { target, value: delta, local: value.local, right })?, + )) + } else { + Instruction::BinOpLocalConstJumpCmpLocal32(PackedOp::new( + (packed.op, cmp), + target_operand!( + data, + LocalUpdateCmp { target, value: value.value as i32, local: value.local, right } + )?, + )) + }; + replace!(output, *read, 3 => replacement); + return Ok(()); + } + } + if *read >= output.block_start + 2 { + match [output[*read - 2], output[*read - 1]] { + [Instruction::LocalGet32(local), Instruction::I32Eqz] => { + replace!(output, *read, 2 => + if on_zero { + Instruction::JumpIfLocalNonZero32(TargetLocalArg { target_ip: target, local }) + } else { + Instruction::JumpIfLocalZero32(TargetLocalArg { target_ip: target, local }) + } + ); + return Ok(()); + } + [Instruction::LocalGet64(local), Instruction::I64Eqz] => { + replace!(output, *read, 2 => + if on_zero { + Instruction::JumpIfLocalNonZero64(TargetLocalArg { target_ip: target, local }) + } else { + Instruction::JumpIfLocalZero64(TargetLocalArg { target_ip: target, local }) + } + ); + return Ok(()); + } + [Instruction::CmpLocalLocal32(op, left, right), Instruction::I32Eqz] => { + let op = if on_zero { op } else { op.inverse() }; + let replacement = jump_cmp_local_local(data, target, left, right, op, false)?; + replace!(output, *read, 2 => replacement); + return Ok(()); + } + _ => {} + } + } + if *read > output.block_start { + let previous = output[*read - 1]; + let replacement = match previous { + Instruction::I32Eqz => { + Some(if on_zero { Instruction::JumpIfNonZero32(target) } else { Instruction::JumpIfZero32(target) }) + } + Instruction::I64Eqz => { + Some(if on_zero { Instruction::JumpIfNonZero64(target) } else { Instruction::JumpIfZero64(target) }) + } + Instruction::CmpLocalLocal32(op, left, right) => { + Some(jump_cmp_local_local(data, target, left, right, if !on_zero { op } else { op.inverse() }, false)?) + } + Instruction::CmpLocalLocal64(op, left, right) => { + Some(jump_cmp_local_local(data, target, left, right, if !on_zero { op } else { op.inverse() }, true)?) + } + Instruction::LocalGet32(local) => Some(if on_zero { + Instruction::JumpIfLocalZero32(TargetLocalArg { target_ip: target, local }) + } else { + Instruction::JumpIfLocalNonZero32(TargetLocalArg { target_ip: target, local }) + }), + Instruction::LocalGet64(local) => Some(if on_zero { + Instruction::JumpIfLocalZero64(TargetLocalArg { target_ip: target, local }) + } else { + Instruction::JumpIfLocalNonZero64(TargetLocalArg { target_ip: target, local }) + }), + _ => None, + }; + if let Some(replacement) = replacement { + replace!(output, *read, 1 => replacement); + return Ok(()); + } + } + if *read >= output.block_start + 3 { + let raw_cmp = output[*read - 1]; + if let Some(mut op) = cmp_op(raw_cmp) { + if on_zero { + op = op.inverse(); + } + let replacement = match [output[*read - 3], output[*read - 2]] { + [Instruction::LocalGet32(local), Instruction::Const32(value)] => { + Some(jump_cmp_local_const32(data, target, local, value, op)?) + } + [Instruction::LocalGet64(local), Instruction::Const64(index)] + if let Ok(value) = i32::try_from(data.operand(index).value) => + { + Some(jump_cmp_local_const64(data, target, local, value, op)?) + } + [Instruction::LocalGet32(left), Instruction::LocalGet32(right)] => { + Some(jump_cmp_local_local(data, target, left, right, op, false)?) + } + [Instruction::LocalGet64(left), Instruction::LocalGet64(right)] => { + Some(jump_cmp_local_local(data, target, left, right, op, true)?) + } + _ => None, + }; + if let Some(replacement) = replacement { + replace!(output, *read, 3 => replacement); + return Ok(()); + } + } + } + if *read >= output.block_start + 2 { + let raw_cmp = output[*read - 1]; + if let Some(mut op) = cmp_op(raw_cmp) { + if on_zero { + op = op.inverse(); + } + let replacement = match output[*read - 2] { + Instruction::LocalGet32(local) => Some(jump_cmp_stack_local(data, target, local, op, false)?), + Instruction::LocalGet64(local) => Some(jump_cmp_stack_local(data, target, local, op, true)?), + Instruction::Const32(value) => Some(jump_cmp_stack_const32(data, target, value, op)?), + Instruction::Const64(index) => { + Some(jump_cmp_stack_const64(data, target, data.operand(index).value, op)?) + } + _ => None, + }; + if let Some(replacement) = replacement { + replace!(output, *read, 2 => replacement); + return Ok(()); + } + } + } + Ok(()) +} + +fn is_unconditional_terminator(instruction: Instruction) -> bool { + matches!( + instruction, + Instruction::Unreachable + | Instruction::Jump(_) + | Instruction::BranchTable(_) + | Instruction::Return + | Instruction::ReturnVoid + | Instruction::Return32 + | Instruction::Return64 + | Instruction::Return128 + | Instruction::ReturnCall(_) + | Instruction::ReturnCallSelf + | Instruction::ReturnCallIndirect(_) + | Instruction::ReturnCallRef(_) + | Instruction::Throw(_) + | Instruction::ThrowRef + ) +} diff --git a/crates/parser/src/optimize/targets.rs b/crates/parser/src/optimize/targets.rs new file mode 100644 index 0000000..88d5fbe --- /dev/null +++ b/crates/parser/src/optimize/targets.rs @@ -0,0 +1,290 @@ +use crate::visit::{BuilderRawOperand, FunctionDataBuilder}; +use crate::{ParseError, Result}; +use alloc::vec::Vec; +use tinywasm_types::{ + BranchTableArg, CastBranch, GlobalUpdate, Instruction, LocalConstCmp, LocalLocalCmp, LocalUpdate, LocalUpdateCmp, + OperandIdx, OperandType, PackedOp, StackConst32, StackConst64, TargetLocal, +}; + +pub(super) trait TargetOperand: OperandType { + fn set_target(&mut self, target: u32); +} + +macro_rules! target_operands { + ($($name:ty),+ $(,)?) => {$( + impl TargetOperand for $name { + fn set_target(&mut self, target: u32) { self.target = target; } + } + )+}; +} + +target_operands!( + TargetLocal, + CastBranch, + StackConst32, + StackConst64, + LocalUpdate, + GlobalUpdate, + LocalUpdateCmp, + LocalConstCmp, + LocalLocalCmp, + BranchTableArg, +); + +pub(super) fn resolve_jump_target(instructions: &[Instruction], data: &FunctionDataBuilder, target: u32) -> u32 { + let mut index = target as usize; + let mut steps = 0; + while let Some(instruction) = instructions.get(index) + && let Some(next) = instruction_target(data, *instruction) + && matches!(instruction, Instruction::Jump(_)) + && steps < instructions.len() + { + index = next as usize; + steps += 1; + } + index as u32 +} + +pub(super) fn set_rewrite_target( + instruction: &mut Instruction, + data: &mut FunctionDataBuilder, + target: u32, +) -> Result<()> { + use Instruction::*; + match instruction { + JumpCmpStackLocal32(packed) | JumpCmpStackLocal64(packed) => { + *packed = push_packed_target_copy(data, *packed, target)? + } + BrOnCast(index) => *index = push_target_copy(data, *index, target)?, + JumpCmpStackConst32(packed) => *packed = push_packed_target_copy(data, *packed, target)?, + JumpCmpStackConst64(packed) => *packed = push_packed_target_copy(data, *packed, target)?, + BinOpLocalConstJump32(packed) | BinOpStackConstTeeLocalJump32(packed) => { + *packed = push_packed_target_copy(data, *packed, target)? + } + BinOpLocalConstJumpCmpLocal32(packed) => *packed = push_packed_target_copy(data, *packed, target)?, + BinOpGlobalConstJump32(packed) => *packed = push_packed_target_copy(data, *packed, target)?, + IncLocalJump32(index) | IncStackTeeLocalJump32(index) => *index = push_target_copy(data, *index, target)?, + IncGlobalJump32(index) => *index = push_target_copy(data, *index, target)?, + IncLocalJumpCmpLocal32(packed) => *packed = push_packed_target_copy(data, *packed, target)?, + JumpCmpLocalConst32(packed) | JumpCmpLocalConst64(packed) => { + *packed = push_packed_target_copy(data, *packed, target)? + } + JumpCmpLocalLocal32(packed) | JumpCmpLocalLocal64(packed) => { + *packed = push_packed_target_copy(data, *packed, target)? + } + BranchTable(index) => *index = push_target_copy(data, *index, target)?, + _ => set_target(instruction, data, target), + } + Ok(()) +} + +fn push_target_copy( + data: &mut FunctionDataBuilder, + index: OperandIdx, + target: u32, +) -> Result> +where + T::Raw: BuilderRawOperand, +{ + let mut value = data.operand(index); + value.set_target(target); + data.push_target_operand(value) +} + +fn push_packed_target_copy( + data: &mut FunctionDataBuilder, + packed: PackedOp, + target: u32, +) -> Result> +where + T::Raw: BuilderRawOperand, +{ + let mut value = data.operand(packed.index); + value.set_target(target); + Ok(PackedOp::new(packed.op, data.push_target_operand(value)?)) +} + +fn instruction_target(data: &FunctionDataBuilder, instruction: Instruction) -> Option { + use Instruction::*; + Some(match instruction { + Jump(target) + | JumpIfZero32(target) + | JumpIfNonZero32(target) + | JumpIfZero64(target) + | JumpIfNonZero64(target) + | JumpIfRefNull(target) + | JumpIfRefNonNull(target) => target, + JumpIfLocalZero32(arg) | JumpIfLocalNonZero32(arg) | JumpIfLocalZero64(arg) | JumpIfLocalNonZero64(arg) => { + arg.target_ip + } + JumpCmpStackLocal32(packed) | JumpCmpStackLocal64(packed) => data.operand(packed.index).target, + BrOnCast(index) => data.operand(index).target, + JumpCmpStackConst32(packed) => data.operand(packed.index).target, + JumpCmpStackConst64(packed) => data.operand(packed.index).target, + BinOpLocalConstJump32(packed) | BinOpStackConstTeeLocalJump32(packed) => data.operand(packed.index).target, + BinOpLocalConstJumpCmpLocal32(packed) => data.operand(packed.index).target, + BinOpGlobalConstJump32(packed) => data.operand(packed.index).target, + IncLocalJump32(index) | IncStackTeeLocalJump32(index) => data.operand(index).target, + IncGlobalJump32(index) => data.operand(index).target, + IncLocalJumpCmpLocal32(packed) => data.operand(packed.index).target, + JumpCmpLocalConst32(packed) | JumpCmpLocalConst64(packed) => data.operand(packed.index).target, + JumpCmpLocalLocal32(packed) | JumpCmpLocalLocal64(packed) => data.operand(packed.index).target, + BranchTable(index) => data.operand(index).target, + _ => return None, + }) +} + +fn set_target(instruction: &mut Instruction, data: &mut FunctionDataBuilder, target: u32) { + use Instruction::*; + match instruction { + Jump(value) + | JumpIfZero32(value) + | JumpIfNonZero32(value) + | JumpIfZero64(value) + | JumpIfNonZero64(value) + | JumpIfRefNull(value) + | JumpIfRefNonNull(value) => *value = target, + JumpIfLocalZero32(arg) | JumpIfLocalNonZero32(arg) | JumpIfLocalZero64(arg) | JumpIfLocalNonZero64(arg) => { + arg.target_ip = target + } + JumpCmpStackLocal32(packed) | JumpCmpStackLocal64(packed) => set_packed_operand_target(data, *packed, target), + BrOnCast(index) => set_operand_target(data, *index, target), + JumpCmpStackConst32(packed) => set_packed_operand_target(data, *packed, target), + JumpCmpStackConst64(packed) => set_packed_operand_target(data, *packed, target), + BinOpLocalConstJump32(packed) | BinOpStackConstTeeLocalJump32(packed) => { + set_packed_operand_target(data, *packed, target) + } + BinOpLocalConstJumpCmpLocal32(packed) => set_packed_operand_target(data, *packed, target), + BinOpGlobalConstJump32(packed) => set_packed_operand_target(data, *packed, target), + IncLocalJump32(index) | IncStackTeeLocalJump32(index) => set_operand_target(data, *index, target), + IncGlobalJump32(index) => set_operand_target(data, *index, target), + IncLocalJumpCmpLocal32(packed) => set_packed_operand_target(data, *packed, target), + JumpCmpLocalConst32(packed) | JumpCmpLocalConst64(packed) => set_packed_operand_target(data, *packed, target), + JumpCmpLocalLocal32(packed) | JumpCmpLocalLocal64(packed) => set_packed_operand_target(data, *packed, target), + BranchTable(index) => set_operand_target(data, *index, target), + _ => {} + } +} + +fn set_operand_target(data: &mut FunctionDataBuilder, index: OperandIdx, target: u32) +where + T::Raw: BuilderRawOperand, +{ + let mut value = data.operand(index); + value.set_target(target); + data.set_operand(index, value); +} + +fn set_packed_operand_target( + data: &mut FunctionDataBuilder, + packed: PackedOp, + target: u32, +) where + T::Raw: BuilderRawOperand, +{ + let mut value = data.operand(packed.index); + value.set_target(target); + data.set_operand(packed.index, value); +} + +fn branch_table_range(data: &FunctionDataBuilder, instruction: Instruction) -> Option<(u32, u32)> { + let Instruction::BranchTable(index) = instruction else { return None }; + let operand = data.operand(index); + Some((operand.start, operand.len)) +} + +pub(super) fn target_boundaries(instructions: &[Instruction], data: &FunctionDataBuilder) -> Result> { + let mut boundaries = alloc::vec![false; instructions.len() + 1]; + for handler in &data.exception_handlers { + for target in [handler.start_ip, handler.end_ip] { + *boundaries.get_mut(target as usize).ok_or_else(|| { + ParseError::Other(alloc::format!("exception handler boundary out of bounds: {target}")) + })? = true; + } + for catch in &handler.catches { + let target = catch.landing_pad(); + *boundaries + .get_mut(target as usize) + .ok_or_else(|| ParseError::Other(alloc::format!("exception landing pad out of bounds: {target}")))? = + true; + } + } + for &instruction in instructions { + if let Some(target) = instruction_target(data, instruction) { + *boundaries + .get_mut(target as usize) + .ok_or_else(|| ParseError::Other(alloc::format!("instruction target out of bounds: {target}")))? = true; + } + if let Some((start, count)) = branch_table_range(data, instruction) { + let end = + start.checked_add(count).ok_or_else(|| ParseError::Other("branch table range overflow".into()))?; + let targets = data + .branch_table_targets + .get(start as usize..end as usize) + .ok_or_else(|| ParseError::Other("branch table range out of bounds".into()))?; + for &target in targets { + *boundaries.get_mut(target as usize).ok_or_else(|| { + ParseError::Other(alloc::format!("branch table target out of bounds: {target}")) + })? = true; + } + } + } + Ok(boundaries) +} + +fn remap_target(target: u32, old_to_new: Option<&[u32]>, len: u32) -> Result { + let target = if let Some(map) = old_to_new { + *map.get(target as usize) + .ok_or_else(|| ParseError::Other(alloc::format!("instruction target out of bounds: {target}")))? + } else { + target + }; + if target >= len { + return Err(ParseError::Other(alloc::format!("instruction target out of bounds: {target}"))); + } + Ok(target) +} + +pub(super) fn finalize( + instructions: &mut [Instruction], + data: &mut FunctionDataBuilder, + old_to_new: Option<&[u32]>, +) -> Result<()> { + let len = instructions.len() as u32; + for handler in &mut data.exception_handlers { + if let Some(map) = old_to_new { + handler.start_ip = *map + .get(handler.start_ip as usize) + .ok_or_else(|| ParseError::Other("exception handler boundary out of bounds".into()))?; + handler.end_ip = *map + .get(handler.end_ip as usize) + .ok_or_else(|| ParseError::Other("exception handler boundary out of bounds".into()))?; + for catch in &mut handler.catches { + let landing_pad = match catch { + tinywasm_types::ExceptionCatch::Tag { landing_pad, .. } + | tinywasm_types::ExceptionCatch::All { landing_pad, .. } => landing_pad, + }; + *landing_pad = remap_target(*landing_pad, Some(map), len)?; + } + } + if handler.start_ip > handler.end_ip || handler.end_ip > len { + return Err(ParseError::Other("exception handler range out of bounds".into())); + } + } + for target in &mut data.branch_table_targets { + *target = remap_target(*target, old_to_new, len)?; + } + for instruction in instructions { + if let Some(target) = instruction_target(data, *instruction) { + set_target(instruction, data, remap_target(target, old_to_new, len)?); + } + if let Some((start, count)) = branch_table_range(data, *instruction) { + let end = + start.checked_add(count).ok_or_else(|| ParseError::Other("branch table range overflow".into()))?; + data.branch_table_targets + .get(start as usize..end as usize) + .ok_or_else(|| ParseError::Other("branch table range out of bounds".into()))?; + } + } + Ok(()) +} diff --git a/crates/parser/src/parallel.rs b/crates/parser/src/parallel.rs index e0e3858..e29b38f 100644 --- a/crates/parser/src/parallel.rs +++ b/crates/parser/src/parallel.rs @@ -1,4 +1,4 @@ -use crate::module::{FunctionCode, optimize_function_code}; +use crate::module::{OptimizedFunctionCode, optimize_function_code}; use crate::validation::{FuncToValidate, FuncValidatorAllocations, ValidatorResources}; use crate::{ParseError, ParserOptions, Result, conversion}; use alloc::sync::Arc; @@ -64,7 +64,7 @@ fn process_function_job( imported_memory_count: u32, validator_allocs: Option, reader_allocs: OperatorsReaderAllocations, -) -> Result<(FunctionCode, Option, OperatorsReaderAllocations)> { +) -> Result<(OptimizedFunctionCode, Option, OperatorsReaderAllocations)> { #[cfg(feature = "validate")] let validator = job.func_to_validate.map(|func| func.into_validator(validator_allocs.unwrap_or_default())); #[cfg(not(feature = "validate"))] @@ -74,12 +74,12 @@ fn process_function_job( }; let (code, validator_allocs, reader_allocs) = match job.body { FunctionBodyInput::Borrowed(func) => { - conversion::convert_module_code(func, validator, reader_allocs, metadata, job.ty_idx)? + conversion::convert_module_code(func, validator, reader_allocs, metadata, job.ty_idx, options)? } FunctionBodyInput::Owned(body) => { let reader = wasmparser::BinaryReader::new(&body.section_bytes[body.body_range], body.body_offset); let func = wasmparser::FunctionBody::new(reader); - conversion::convert_module_code(func, validator, reader_allocs, metadata, job.ty_idx)? + conversion::convert_module_code(func, validator, reader_allocs, metadata, job.ty_idx, options)? } }; @@ -100,7 +100,7 @@ fn process_chunk<'a>( options: &ParserOptions, imported_func_count: usize, imported_memory_count: u32, -) -> Result> { +) -> Result> { let mut validator_allocs = None; let mut reader_allocs = OperatorsReaderAllocations::default(); let jobs = jobs.into_iter(); @@ -130,7 +130,7 @@ pub(crate) fn process_pending( options: &ParserOptions, imported_func_count: usize, imported_memory_count: u32, -) -> Result> { +) -> Result> { let num_workers = worker_count(options, pending.len()); if num_workers == 1 { return process_chunk(pending, metadata, options, imported_func_count, imported_memory_count); diff --git a/crates/parser/src/visit.rs b/crates/parser/src/visit.rs index cecbb8c..a5e7c2b 100644 --- a/crates/parser/src/visit.rs +++ b/crates/parser/src/visit.rs @@ -3,10 +3,11 @@ use crate::{ conversion::{convert_heap_type, value_lane}, macros::visit::*, }; -use alloc::{boxed::Box, string::ToString, vec::Vec}; +use alloc::{boxed::Box, collections::BTreeMap, string::ToString, vec::Vec}; use tinywasm_types::{ - Global, Import, ImportKind, Instruction, MemoryArg, MemoryType, StorageType, TableDefinition, TagType, TypeSection, - ValueCounts, ValueLane, WasmFunctionData, + BranchTableArg, CastBranch, ExceptionHandler, Global, I64Operand, Import, ImportKind, Instruction, MemoryArg, + MemoryType, Operand64, Operand128, OperandIdx, OperandType, StorageType, TableDefinition, TagType, TwoU32, + TypeSection, V128Operand, ValueCounts, ValueLane, WasmFunctionData, }; use wasmparser::{FunctionBody, OperatorsReader, OperatorsReaderAllocations, VisitSimdOperator}; @@ -52,19 +53,107 @@ pub(crate) struct ModuleMetadata { aggregate_fields: Vec, } +pub(crate) struct FunctionDataBuilder { + pub(crate) operands64: Vec, + pub(crate) operands128: Vec, + pub(crate) branch_table_targets: Vec, + pub(crate) exception_handlers: Vec, + deduplicate64: Option>, + deduplicate128: Option>, +} + +pub(crate) trait BuilderRawOperand: tinywasm_types::RawOperand + Ord { + fn push(builder: &mut FunctionDataBuilder, raw: Self, deduplicate: bool) -> Result; + fn get(builder: &FunctionDataBuilder, index: u32) -> Self; + fn set(builder: &mut FunctionDataBuilder, index: u32, raw: Self); +} + +macro_rules! builder_raw_operand { + ($raw:ty, $lane:ident, $map:ident) => { + impl BuilderRawOperand for $raw { + fn push(builder: &mut FunctionDataBuilder, raw: Self, deduplicate: bool) -> Result { + if deduplicate && let Some(index) = builder.$map.as_ref().and_then(|map| map.get(&raw)) { + return Ok(*index); + } + let index = u32::try_from(builder.$lane.len()) + .map_err(|_| crate::ParseError::Other("instruction operand index overflow".into()))?; + builder.$lane.push(raw); + if deduplicate && let Some(map) = &mut builder.$map { + map.insert(raw, index); + } + Ok(index) + } + + fn get(builder: &FunctionDataBuilder, index: u32) -> Self { + *builder.$lane.get(index as usize).unwrap_or_else(|| unreachable!("invalid operand index")) + } + + fn set(builder: &mut FunctionDataBuilder, index: u32, raw: Self) { + *builder.$lane.get_mut(index as usize).unwrap_or_else(|| unreachable!("invalid operand index")) = raw; + } + } + }; +} + +builder_raw_operand!(Operand64, operands64, deduplicate64); +builder_raw_operand!(Operand128, operands128, deduplicate128); + +impl FunctionDataBuilder { + pub(crate) fn new(deduplicate_operands: bool) -> Self { + Self { + operands64: Vec::new(), + operands128: Vec::new(), + branch_table_targets: Vec::new(), + exception_handlers: Vec::new(), + deduplicate64: deduplicate_operands.then(BTreeMap::new), + deduplicate128: deduplicate_operands.then(BTreeMap::new), + } + } + + pub(crate) fn push_operand(&mut self, operand: T) -> Result> + where + T::Raw: BuilderRawOperand, + { + ::push(self, operand.encode(), true).map(OperandIdx::new) + } + + pub(crate) fn push_target_operand(&mut self, operand: T) -> Result> + where + T::Raw: BuilderRawOperand, + { + ::push(self, operand.encode(), false).map(OperandIdx::new) + } + + pub(crate) fn operand(&self, index: OperandIdx) -> T + where + T::Raw: BuilderRawOperand, + { + T::decode(::get(self, index.index())) + } + + pub(crate) fn set_operand(&mut self, index: OperandIdx, operand: T) + where + T::Raw: BuilderRawOperand, + { + ::set(self, index.index(), operand.encode()) + } + + pub(crate) fn finish(self) -> WasmFunctionData { + WasmFunctionData { + operands64: self.operands64.into_boxed_slice(), + operands128: self.operands128.into_boxed_slice(), + branch_table_targets: self.branch_table_targets.into_boxed_slice(), + exception_handlers: self.exception_handlers.into_boxed_slice(), + } + } +} + enum AggregateFields { Other, Struct(Box<[ValueLane]>), Array(ValueLane), } -#[derive(Default)] -struct FunctionDataBuilder { - v128_constants: Vec<[u8; 16]>, - branch_table_targets: Vec, - exception_handlers: Vec, -} - pub(crate) struct FunctionBuilder<'a> { instructions: Vec, data: FunctionDataBuilder, @@ -83,13 +172,14 @@ impl<'a> FunctionBuilder<'a> { local_types: Vec, local_addr_map: Vec, body_size: usize, + deduplicate_operands: bool, ) -> Self { Self { local_types, local_addr_map, metadata, instructions: Vec::with_capacity(body_size.min(1024)), - data: FunctionDataBuilder::default(), + data: FunctionDataBuilder::new(deduplicate_operands), control_stack: alloc::vec![ControlFrame { kind: BlockKind::Function, has_else: false, @@ -112,10 +202,18 @@ impl<'a> FunctionBuilder<'a> { &mut self, type_index: u32, field_index: u32, - instruction: fn(u32, u32) -> Instruction, + instruction: fn(OperandIdx) -> Instruction, ) -> Result<()> { let size = self.metadata.struct_field(type_index, field_index)?; - self.emit(&[ValueLane::S32], &[size], instruction(type_index, field_index)) + let operand = self.push_operand(TwoU32 { first: type_index, second: field_index })?; + self.emit(&[ValueLane::S32], &[size], instruction(operand)) + } + + fn push_operand(&mut self, operand: T) -> Result> + where + T::Raw: BuilderRawOperand, + { + self.data.push_operand(operand) } } @@ -294,17 +392,19 @@ impl VisitSimdOperator<'_> for ValidateThenVisit<'_, '_> { pub(crate) fn process_operators( body: FunctionBody<'_>, - local_types: Vec, - local_addr_map: Vec, + locals: (Vec, Vec), metadata: &ModuleMetadata, ty_idx: u32, allocs: OperatorsReaderAllocations, -) -> Result<(Vec, WasmFunctionData, OperatorsReaderAllocations)> { + deduplicate_operands: bool, +) -> Result<(Vec, FunctionDataBuilder, OperatorsReaderAllocations)> { + let (local_types, local_addr_map) = locals; let body_size = body.as_bytes().len(); let reader = body.get_binary_reader_for_operators()?; let mut reader = OperatorsReader::new_with_allocs(reader, allocs); let signature = metadata.signature(ty_idx)?.clone(); - let mut builder = FunctionBuilder::new(metadata, signature, local_types, local_addr_map, body_size); + let mut builder = + FunctionBuilder::new(metadata, signature, local_types, local_addr_map, body_size, deduplicate_operands); while !reader.eof() { let position = reader.original_position(); @@ -319,29 +419,26 @@ pub(crate) fn process_operators( } reader.finish()?; - let data = WasmFunctionData { - v128_constants: builder.data.v128_constants.into_boxed_slice(), - branch_table_targets: builder.data.branch_table_targets.into_boxed_slice(), - exception_handlers: builder.data.exception_handlers.into_boxed_slice(), - }; - Ok((builder.instructions, data, reader.into_allocations())) + Ok((builder.instructions, builder.data, reader.into_allocations())) } #[cfg(feature = "validate")] pub(crate) fn process_operators_and_validate( mut validator: FuncValidator, body: FunctionBody<'_>, - local_types: Vec, - local_addr_map: Vec, + locals: (Vec, Vec), metadata: &ModuleMetadata, ty_idx: u32, allocs: OperatorsReaderAllocations, -) -> Result<(Vec, WasmFunctionData, FuncValidatorAllocations, OperatorsReaderAllocations)> { + deduplicate_operands: bool, +) -> Result<(Vec, FunctionDataBuilder, FuncValidatorAllocations, OperatorsReaderAllocations)> { + let (local_types, local_addr_map) = locals; let body_size = body.as_bytes().len(); let reader = body.get_binary_reader_for_operators()?; let mut reader = OperatorsReader::new_with_allocs(reader, allocs); let signature = metadata.signature(ty_idx)?.clone(); - let mut builder = FunctionBuilder::new(metadata, signature, local_types, local_addr_map, body_size); + let mut builder = + FunctionBuilder::new(metadata, signature, local_types, local_addr_map, body_size, deduplicate_operands); while !reader.eof() { let position = reader.original_position(); @@ -356,12 +453,7 @@ pub(crate) fn process_operators_and_validate( } reader.finish()?; - let data = WasmFunctionData { - v128_constants: builder.data.v128_constants.into_boxed_slice(), - branch_table_targets: builder.data.branch_table_targets.into_boxed_slice(), - exception_handlers: builder.data.exception_handlers.into_boxed_slice(), - }; - Ok((builder.instructions, data, validator.into_allocations(), reader.into_allocations())) + Ok((builder.instructions, builder.data, validator.into_allocations(), reader.into_allocations())) } impl<'a> wasmparser::VisitOperator<'a> for FunctionBuilder<'_> { @@ -395,7 +487,6 @@ impl<'a> wasmparser::VisitOperator<'a> for FunctionBuilder<'_> { } fixed [] => [] { visit_data_drop(segment: u32) => DataDrop, visit_elem_drop(segment: u32) => ElemDrop } fixed [] => [S32] { visit_i32_const(value: i32) => Const32, visit_ref_func(function: u32) => RefFunc } - fixed [] => [S64] { visit_i64_const(value: i64) => Const64 } heap false [] => [S32] { visit_ref_null => RefNull } heap false [S32] => [S32] { visit_ref_test_non_null => RefTest, visit_ref_cast_non_null => RefCast, @@ -483,22 +574,10 @@ impl<'a> wasmparser::VisitOperator<'a> for FunctionBuilder<'_> { table [] => [Addr] { visit_table_size(table: u32) => TableSize } table [S32, Addr] => [Addr] { visit_table_grow(table: u32) => TableGrow } table [Addr, S32, Addr] => [] { visit_table_fill(table: u32) => TableFill } - table [Addr, S32, S32] => [] { visit_table_init(elem_index: u32, table: u32) => TableInit } fixed [] => [S32] { visit_struct_new_default(type_index: u32) => StructNewDefault } fixed [S32] => [S32] { visit_array_new_default(type_index: u32) => ArrayNewDefault, visit_array_len => ArrayLen, } - fixed [S32, S32] => [S32] { - visit_array_new_data(type_index: u32, data_index: u32) => ArrayNewData, - visit_array_new_elem(type_index: u32, elem_index: u32) => ArrayNewElem, - } - fixed [S32, S32, S32, S32] => [] { - visit_array_init_data(type_index: u32, data_index: u32) => ArrayInitData, - visit_array_init_elem(type_index: u32, elem_index: u32) => ArrayInitElem, - } - fixed [S32, S32, S32, S32, S32] => [] { - visit_array_copy(type_index_dst: u32, type_index_src: u32) => ArrayCopy, - } array_field [Field, S32] => [S32] { visit_array_new(type_index: u32) => ArrayNew } array_field [S32, S32] => [Field] { visit_array_get(type_index: u32) => ArrayGet, visit_array_get_s(type_index: u32) => ArrayGetS, @@ -533,7 +612,8 @@ impl<'a> wasmparser::VisitOperator<'a> for FunctionBuilder<'_> { fn visit_struct_set(&mut self, type_index: u32, field_index: u32) -> Self::Output { let size = self.metadata.struct_field(type_index, field_index)?; - self.emit(&[ValueLane::S32, size], &[], Instruction::StructSet(type_index, field_index)) + let operand = self.push_operand(TwoU32 { first: type_index, second: field_index })?; + self.emit(&[ValueLane::S32, size], &[], Instruction::StructSet(operand)) } fn visit_array_new_fixed(&mut self, type_index: u32, array_size: u32) -> Self::Output { @@ -542,7 +622,8 @@ impl<'a> wasmparser::VisitOperator<'a> for FunctionBuilder<'_> { self.pop_expect(size)?; } self.push_sizes(&[ValueLane::S32])?; - self.instructions.push(Instruction::ArrayNewFixed(type_index, array_size)); + let operand = self.push_operand(TwoU32 { first: type_index, second: array_size })?; + self.instructions.push(Instruction::ArrayNewFixed(operand)); Ok(()) } @@ -555,7 +636,8 @@ impl<'a> wasmparser::VisitOperator<'a> for FunctionBuilder<'_> { let signature = self.metadata.signature(type_index)?.clone(); let mut inputs = signature.params; inputs.push(self.metadata.table_size(table_index)?); - self.emit(&inputs, &signature.results, Instruction::CallIndirect(type_index, table_index)) + let operand = self.push_operand(TwoU32 { first: type_index, second: table_index })?; + self.emit(&inputs, &signature.results, Instruction::CallIndirect(operand)) } fn visit_call_ref(&mut self, type_index: u32) -> Self::Output { @@ -579,7 +661,8 @@ impl<'a> wasmparser::VisitOperator<'a> for FunctionBuilder<'_> { inputs.push(self.metadata.table_size(table_index)?); self.apply_effect(&inputs, &[])?; self.mark_unreachable(); - self.instructions.push(Instruction::ReturnCallIndirect(type_index, table_index)); + let operand = self.push_operand(TwoU32 { first: type_index, second: table_index })?; + self.instructions.push(Instruction::ReturnCallIndirect(operand)); Ok(()) } @@ -851,7 +934,9 @@ impl<'a> wasmparser::VisitOperator<'a> for FunctionBuilder<'_> { let header_ip = self.instructions.len(); let branch_table_start = self.data.branch_table_targets.len() as u32; - self.instructions.push(Instruction::BranchTable(0, branch_table_start, len)); + let branch_operand = + self.data.push_target_operand(BranchTableArg { target: 0, start: branch_table_start, len })?; + self.instructions.push(Instruction::BranchTable(branch_operand)); struct PadInfo { depth: u32, @@ -894,8 +979,10 @@ impl<'a> wasmparser::VisitOperator<'a> for FunctionBuilder<'_> { .iter() .find(|pad| pad.depth == default_depth) .ok_or_else(|| crate::ParseError::Other("missing default branch table target".into()))?; - if let Instruction::BranchTable(default_ip, _, _) = &mut self.instructions[header_ip] { - *default_ip = default_pad.pad_start as u32; + if let Instruction::BranchTable(index) = self.instructions[header_ip] { + let mut operand = self.data.operand(index); + operand.target = default_pad.pad_start as u32; + self.data.set_operand(index, operand); } for pad in &pads { @@ -921,26 +1008,66 @@ impl<'a> wasmparser::VisitOperator<'a> for FunctionBuilder<'_> { } fn visit_f64_const(&mut self, val: wasmparser::Ieee64) -> Self::Output { - self.emit(&[], &[ValueLane::S64], Instruction::Const64(val.bits() as i64)) + let operand = self.push_operand(I64Operand { value: val.bits() as i64 })?; + self.emit(&[], &[ValueLane::S64], Instruction::Const64(operand)) + } + + fn visit_i64_const(&mut self, value: i64) -> Self::Output { + let operand = self.push_operand(I64Operand { value })?; + self.emit(&[], &[ValueLane::S64], Instruction::Const64(operand)) } fn visit_table_copy(&mut self, dst_table: u32, src_table: u32) -> Self::Output { let dst = self.metadata.table_size(dst_table)?; let src = self.metadata.table_size(src_table)?; let len = if dst == ValueLane::S32 || src == ValueLane::S32 { ValueLane::S32 } else { ValueLane::S64 }; - self.emit(&[dst, src, len], &[], Instruction::TableCopy { dst_table, src_table }) + let operand = self.push_operand(TwoU32 { first: dst_table, second: src_table })?; + self.emit(&[dst, src, len], &[], Instruction::TableCopy(operand)) } fn visit_memory_copy(&mut self, dst_mem: u32, src_mem: u32) -> Self::Output { let dst = self.metadata.memory_size(dst_mem)?; let src = self.metadata.memory_size(src_mem)?; let len = if dst == ValueLane::S32 || src == ValueLane::S32 { ValueLane::S32 } else { ValueLane::S64 }; - self.emit(&[dst, src, len], &[], Instruction::MemoryCopy { dst_mem, src_mem }) + let operand = self.push_operand(TwoU32 { first: dst_mem, second: src_mem })?; + self.emit(&[dst, src, len], &[], Instruction::MemoryCopy(operand)) } fn visit_memory_init(&mut self, data_index: u32, memory: u32) -> Self::Output { let dst = self.metadata.memory_size(memory)?; - self.emit(&[dst, ValueLane::S32, ValueLane::S32], &[], Instruction::MemoryInit(data_index, memory)) + let operand = self.push_operand(TwoU32 { first: data_index, second: memory })?; + self.emit(&[dst, ValueLane::S32, ValueLane::S32], &[], Instruction::MemoryInit(operand)) + } + + fn visit_table_init(&mut self, elem_index: u32, table: u32) -> Self::Output { + let address = self.metadata.table_size(table)?; + let operand = self.push_operand(TwoU32 { first: elem_index, second: table })?; + self.emit(&[address, ValueLane::S32, ValueLane::S32], &[], Instruction::TableInit(operand)) + } + + fn visit_array_new_data(&mut self, type_index: u32, data_index: u32) -> Self::Output { + let operand = self.push_operand(TwoU32 { first: type_index, second: data_index })?; + self.emit(&[ValueLane::S32, ValueLane::S32], &[ValueLane::S32], Instruction::ArrayNewData(operand)) + } + + fn visit_array_new_elem(&mut self, type_index: u32, elem_index: u32) -> Self::Output { + let operand = self.push_operand(TwoU32 { first: type_index, second: elem_index })?; + self.emit(&[ValueLane::S32, ValueLane::S32], &[ValueLane::S32], Instruction::ArrayNewElem(operand)) + } + + fn visit_array_init_data(&mut self, type_index: u32, data_index: u32) -> Self::Output { + let operand = self.push_operand(TwoU32 { first: type_index, second: data_index })?; + self.emit(&[ValueLane::S32; 4], &[], Instruction::ArrayInitData(operand)) + } + + fn visit_array_init_elem(&mut self, type_index: u32, elem_index: u32) -> Self::Output { + let operand = self.push_operand(TwoU32 { first: type_index, second: elem_index })?; + self.emit(&[ValueLane::S32; 4], &[], Instruction::ArrayInitElem(operand)) + } + + fn visit_array_copy(&mut self, type_index_dst: u32, type_index_src: u32) -> Self::Output { + let operand = self.push_operand(TwoU32 { first: type_index_dst, second: type_index_src })?; + self.emit(&[ValueLane::S32; 5], &[], Instruction::ArrayCopy(operand)) } fn visit_br_on_cast( @@ -1187,19 +1314,13 @@ impl wasmparser::VisitSimdOperator<'_> for FunctionBuilder<'_> { } fn visit_i8x16_shuffle(&mut self, lanes: [u8; 16]) -> Self::Output { - self.emit( - &[ValueLane::S128, ValueLane::S128], - &[ValueLane::S128], - Instruction::I8x16Shuffle(self.data.v128_constants.len() as u32), - )?; - self.data.v128_constants.push(lanes); - Ok(()) + let index = self.push_operand(V128Operand { value: lanes })?; + self.emit(&[ValueLane::S128, ValueLane::S128], &[ValueLane::S128], Instruction::I8x16Shuffle(index)) } fn visit_v128_const(&mut self, value: wasmparser::V128) -> Self::Output { - self.emit(&[], &[ValueLane::S128], Instruction::Const128(self.data.v128_constants.len() as u32))?; - self.data.v128_constants.push(*value.bytes()); - Ok(()) + let index = self.push_operand(V128Operand { value: *value.bytes() })?; + self.emit(&[], &[ValueLane::S128], Instruction::Const128(index)) } } @@ -1213,7 +1334,12 @@ impl FunctionBuilder<'_> { self.pop_expect(ValueLane::S32)?; let target = convert_heap_type(target.heap_type(), target.is_nullable())?; let conditional_ip = self.instructions.len(); - self.instructions.push(Instruction::BrOnCast(0, target, branch_on_fail)); + let operand = self.data.push_target_operand(CastBranch { + target: 0, + ref_type_bits: target.to_bits(), + branch_on_fail: u8::from(branch_on_fail), + })?; + self.instructions.push(Instruction::BrOnCast(operand)); self.push_sizes(&[ValueLane::S32])?; self.emit_dropkeep_to_label(relative_depth)?; self.emit_branch_jump_or_return(relative_depth)?; @@ -1357,10 +1483,15 @@ impl FunctionBuilder<'_> { /// Emits the stack-shaping instruction required by a branch. fn emit_dropkeep(&mut self, base: ValueCounts, keep: ValueCounts) { let target = ValueCounts { c32: base.c32 + keep.c32, c64: base.c64 + keep.c64, c128: base.c128 + keep.c128 }; - if self.lane_counts == target { - return; + if self.lane_counts.c32 != target.c32 { + self.instructions.push(Instruction::DropKeep32 { base: base.c32, keep: keep.c32 }); + } + if self.lane_counts.c64 != target.c64 { + self.instructions.push(Instruction::DropKeep64 { base: base.c64, keep: keep.c64 }); + } + if self.lane_counts.c128 != target.c128 { + self.instructions.push(Instruction::DropKeep128 { base: base.c128, keep: keep.c128 }); } - self.instructions.push(Instruction::DropKeep((base, keep).into())); } fn patch_jump(&mut self, jump_ip: usize, target: usize) { @@ -1369,10 +1500,14 @@ impl FunctionBuilder<'_> { | Instruction::JumpIfZero32(ip) | Instruction::JumpIfNonZero32(ip) | Instruction::JumpIfRefNull(ip) - | Instruction::JumpIfRefNonNull(ip) - | Instruction::BrOnCast(ip, _, _) => { + | Instruction::JumpIfRefNonNull(ip) => { *ip = target as u32; } + Instruction::BrOnCast(index) => { + let mut operand = self.data.operand(*index); + operand.target = target as u32; + self.data.set_operand(*index, operand); + } _ => {} } } diff --git a/crates/tinywasm/src/interpreter/executor.rs b/crates/tinywasm/src/interpreter/executor.rs index 0bdb474..c512d82 100644 --- a/crates/tinywasm/src/interpreter/executor.rs +++ b/crates/tinywasm/src/interpreter/executor.rs @@ -205,11 +205,11 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { SelectMulti(counts) => self.store.value_stack.select_multi(*counts), Call(v) => { self.exec_call_direct(*v)?; return Ok(None); } CallSelf => { self.exec_call_self()?; return Ok(None); } - CallIndirect(ty, table) => { self.exec_call_indirect::(*ty, *table)?; return Ok(None); } + CallIndirect(idx) => { self.exec_call_indirect::(*idx)?; return Ok(None); } CallRef(ty) => { self.exec_call_ref::(*ty)?; return Ok(None); } ReturnCall(v) => { if self.exec_return_call_direct(*v)? { return Ok(Some(())); } return Ok(None); } ReturnCallSelf => { self.exec_return_call_self()?; return Ok(None); } - ReturnCallIndirect(ty, table) => { if self.exec_call_indirect::(*ty, *table)? { return Ok(Some(())); } return Ok(None); } + ReturnCallIndirect(idx) => { if self.exec_call_indirect::(*idx)? { return Ok(Some(())); } return Ok(None); } ReturnCallRef(ty) => { if self.exec_call_ref::(*ty)? { return Ok(Some(())); } return Ok(None); } Throw(tag) => { self.exec_throw(*tag)?; return Ok(None); } ThrowRef => { self.exec_throw_ref()?; return Ok(None); } @@ -228,79 +228,55 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { if !is_non_null { ValueRef::stack_pop(&mut self.store.value_stack); } is_non_null }) { return Ok(None) }, - BrOnCast(ip, ty, on_fail) => if self.exec_ref_matches(*ty) == *on_fail { self.cf.instr_ptr = *ip as usize; return Ok(None); }, - JumpIfLocalZero32 { target_ip, local } => if Self::exec_jump_if(&mut self.cf, *target_ip, |cf| Value32::local_get(&self.store.value_stack, cf, *local) == 0) { return Ok(None) }, - JumpIfLocalNonZero32 { target_ip, local } => if Self::exec_jump_if(&mut self.cf, *target_ip, |cf| Value32::local_get(&self.store.value_stack, cf, *local) != 0) { return Ok(None) }, - JumpIfLocalZero64 { target_ip, local } => if Self::exec_jump_if(&mut self.cf, *target_ip, |cf| Value64::local_get(&self.store.value_stack, cf, *local) == 0) { return Ok(None) }, - JumpIfLocalNonZero64 { target_ip, local } => if Self::exec_jump_if(&mut self.cf, *target_ip, |cf| Value64::local_get(&self.store.value_stack, cf, *local) != 0) { return Ok(None) }, - JumpCmpStackConst32 { target_ip, imm, op } => if Self::exec_jump_if(&mut self.cf, *target_ip, |_| cmp_i32(i32::stack_pop(&mut self.store.value_stack), *imm, *op)) { return Ok(None) }, - JumpCmpStackConst64 { target_ip, imm, op } => if Self::exec_jump_if(&mut self.cf, *target_ip, |_| cmp_i64(i64::stack_pop(&mut self.store.value_stack), *imm, *op)) { return Ok(None) }, - JumpCmpStackLocal32 { target_ip, local, op } => if Self::exec_jump_if(&mut self.cf, *target_ip, |cf| { + BrOnCast(idx) => { let v = idx.get(&self.func.data); if self.exec_ref_matches(v.ref_type()) == (v.branch_on_fail != 0) { self.cf.instr_ptr = v.target as usize; return Ok(None); } }, + JumpIfLocalZero32(arg) => if Self::exec_jump_if(&mut self.cf, arg.target_ip, |cf| Value32::local_get(&self.store.value_stack, cf, arg.local) == 0) { return Ok(None) }, + JumpIfLocalNonZero32(arg) => if Self::exec_jump_if(&mut self.cf, arg.target_ip, |cf| Value32::local_get(&self.store.value_stack, cf, arg.local) != 0) { return Ok(None) }, + JumpIfLocalZero64(arg) => if Self::exec_jump_if(&mut self.cf, arg.target_ip, |cf| Value64::local_get(&self.store.value_stack, cf, arg.local) == 0) { return Ok(None) }, + JumpIfLocalNonZero64(arg) => if Self::exec_jump_if(&mut self.cf, arg.target_ip, |cf| Value64::local_get(&self.store.value_stack, cf, arg.local) != 0) { return Ok(None) }, + JumpCmpStackConst32(packed) => { let v = packed.index.get(&self.func.data); if Self::exec_jump_if(&mut self.cf, v.target, |_| cmp_i32(i32::stack_pop(&mut self.store.value_stack), v.value, packed.op)) { return Ok(None) } }, + JumpCmpStackConst64(packed) => { let v = packed.index.get(&self.func.data); if Self::exec_jump_if(&mut self.cf, v.target, |_| cmp_i64(i64::stack_pop(&mut self.store.value_stack), v.value, packed.op)) { return Ok(None) } }, + JumpCmpStackLocal32(packed) => { let v = packed.index.get(&self.func.data); if Self::exec_jump_if(&mut self.cf, v.target, |cf| { let lhs = i32::stack_pop(&mut self.store.value_stack); - cmp_i32(lhs, i32::local_get(&self.store.value_stack, cf, *local), *op) - }) { return Ok(None) }, - JumpCmpStackLocal64 { target_ip, local, op } => if Self::exec_jump_if(&mut self.cf, *target_ip, |cf| { + cmp_i32(lhs, i32::local_get(&self.store.value_stack, cf, v.local), packed.op) + }) { return Ok(None) } }, + JumpCmpStackLocal64(packed) => { let v = packed.index.get(&self.func.data); if Self::exec_jump_if(&mut self.cf, v.target, |cf| { let lhs = i64::stack_pop(&mut self.store.value_stack); - cmp_i64(lhs, i64::local_get(&self.store.value_stack, cf, *local), *op) - }) { return Ok(None) }, - BinOpLocalConstJump32 { target_ip, local, imm, op, on_zero } => if Self::exec_jump_if(&mut self.cf, *target_ip, |cf| { - let value = exec_binop_32(*op, i32::local_get(&self.store.value_stack, cf, *local) as u32, *imm as u32) as i32; - i32::local_set(&mut self.store.value_stack, cf, *local, value); - (value == 0) == *on_zero - }) { return Ok(None) }, - BinOpLocalConstJumpCmpLocal32 { target_ip, local, imm, binop, right, cmp } => if Self::exec_jump_if(&mut self.cf, *target_ip, |cf| { - let lhs = exec_binop_32(*binop, i32::local_get(&self.store.value_stack, cf, *local) as u32, *imm as u32) as i32; - i32::local_set(&mut self.store.value_stack, cf, *local, lhs); - cmp_i32(lhs, i32::local_get(&self.store.value_stack, cf, *right), *cmp) - }) { return Ok(None) }, - BinOpStackConstTeeLocalJump32 { target_ip, local, imm, op, on_zero } => { - let value = exec_binop_32(*op, i32::stack_pop(&mut self.store.value_stack) as u32, *imm as u32) as i32; - i32::local_set(&mut self.store.value_stack, &self.cf, *local, value); - i32::stack_push(&mut self.store.value_stack, value)?; - if Self::exec_jump_if(&mut self.cf, *target_ip, |_| (value == 0) == *on_zero) { return Ok(None) } - }, - BinOpGlobalConstJump32 { target_ip, global, imm, op, on_zero } => if Self::exec_jump_if(&mut self.cf, *target_ip, |_| { - let global = self.module.resolve_global_addr(*global); - let value = exec_binop_32(*op, i32::global_get(&self.store.state.globals, global) as u32, *imm as u32) as i32; + cmp_i64(lhs, i64::local_get(&self.store.value_stack, cf, v.local), packed.op) + }) { return Ok(None) } }, + BinOpLocalConstJump32(packed) => { let v = packed.index.get(&self.func.data); let op = packed.op; if Self::exec_jump_if(&mut self.cf, v.target, |cf| { let value = exec_binop_32(op, i32::local_get(&self.store.value_stack, cf, v.local) as u32, v.value as u32) as i32; i32::local_set(&mut self.store.value_stack, cf, v.local, value); (value == 0) == (v.on_zero != 0) }) { return Ok(None) } }, + BinOpLocalConstJumpCmpLocal32(packed) => { let v = packed.index.get(&self.func.data); let (binop, cmp) = packed.op; if Self::exec_jump_if(&mut self.cf, v.target, |cf| { let lhs = exec_binop_32(binop, i32::local_get(&self.store.value_stack, cf, v.local) as u32, v.value as u32) as i32; i32::local_set(&mut self.store.value_stack, cf, v.local, lhs); cmp_i32(lhs, i32::local_get(&self.store.value_stack, cf, v.right), cmp) }) { return Ok(None) } }, + BinOpStackConstTeeLocalJump32(packed) => { let v = packed.index.get(&self.func.data); let op = packed.op; let value = exec_binop_32(op, i32::stack_pop(&mut self.store.value_stack) as u32, v.value as u32) as i32; i32::local_set(&mut self.store.value_stack, &self.cf, v.local, value); i32::stack_push(&mut self.store.value_stack, value)?; if Self::exec_jump_if(&mut self.cf, v.target, |_| (value == 0) == (v.on_zero != 0)) { return Ok(None) } }, + BinOpGlobalConstJump32(packed) => { let v = packed.index.get(&self.func.data); let op = packed.op; if Self::exec_jump_if(&mut self.cf, v.target, |_| { + let global = self.module.resolve_global_addr(v.global); + let value = exec_binop_32(op, i32::global_get(&self.store.state.globals, global) as u32, v.value as u32) as i32; i32::global_set(&mut self.store.state.globals, global, value); - (value == 0) == *on_zero - }) { return Ok(None) }, - IncLocalJump32 { target_ip, local, delta, on_zero } => if Self::exec_jump_if(&mut self.cf, *target_ip, |cf| { - let value = i32::local_get(&self.store.value_stack, cf, *local).wrapping_add(*delta); - i32::local_set(&mut self.store.value_stack, cf, *local, value); - (value == 0) == *on_zero - }) { return Ok(None) }, - IncStackTeeLocalJump32 { target_ip, local, delta, on_zero } => { - let value = i32::stack_pop(&mut self.store.value_stack).wrapping_add(*delta); - i32::local_set(&mut self.store.value_stack, &self.cf, *local, value); - i32::stack_push(&mut self.store.value_stack, value)?; - if Self::exec_jump_if(&mut self.cf, *target_ip, |_| (value == 0) == *on_zero) { return Ok(None) } - }, - IncGlobalJump32 { target_ip, global, delta, on_zero } => if Self::exec_jump_if(&mut self.cf, *target_ip, |_| { - let global = self.module.resolve_global_addr(*global); - let value = i32::global_get(&self.store.state.globals, global).wrapping_add(*delta); + (value == 0) == (v.on_zero != 0) + }) { return Ok(None) } }, + IncLocalJump32(idx) => { let v = idx.get(&self.func.data); if Self::exec_jump_if(&mut self.cf, v.target, |cf| { let value = i32::local_get(&self.store.value_stack, cf, v.local).wrapping_add(v.value); i32::local_set(&mut self.store.value_stack, cf, v.local, value); (value == 0) == (v.on_zero != 0) }) { return Ok(None) } }, + IncStackTeeLocalJump32(idx) => { let v = idx.get(&self.func.data); let value = i32::stack_pop(&mut self.store.value_stack).wrapping_add(v.value); i32::local_set(&mut self.store.value_stack, &self.cf, v.local, value); i32::stack_push(&mut self.store.value_stack, value)?; if Self::exec_jump_if(&mut self.cf, v.target, |_| (value == 0) == (v.on_zero != 0)) { return Ok(None) } }, + IncGlobalJump32(idx) => { let v = idx.get(&self.func.data); if Self::exec_jump_if(&mut self.cf, v.target, |_| { + let global = self.module.resolve_global_addr(v.global); + let value = i32::global_get(&self.store.state.globals, global).wrapping_add(v.value); i32::global_set(&mut self.store.state.globals, global, value); - (value == 0) == *on_zero - }) { return Ok(None) }, - IncLocalJumpCmpLocal32 { target_ip, local, delta, right, op } => if Self::exec_jump_if(&mut self.cf, *target_ip, |cf| { - let lhs = i32::local_get(&self.store.value_stack, cf, *local).wrapping_add(*delta); - i32::local_set(&mut self.store.value_stack, cf, *local, lhs); - cmp_i32(lhs, i32::local_get(&self.store.value_stack, cf, *right), *op) - }) { return Ok(None) }, - JumpCmpLocalConst32 { target_ip, local, imm, op } => if Self::exec_jump_if(&mut self.cf, *target_ip, |cf| cmp_i32(i32::local_get(&self.store.value_stack, cf, *local), *imm, *op)) { return Ok(None) }, - JumpCmpLocalConst64 { target_ip, local, imm, op } => if Self::exec_jump_if(&mut self.cf, *target_ip, |cf| cmp_i64(i64::local_get(&self.store.value_stack, cf, *local), i64::from(*imm), *op)) { return Ok(None) }, - JumpCmpLocalLocal32 { target_ip, left, right, op } => if Self::exec_jump_if(&mut self.cf, *target_ip, |cf| cmp_i32(i32::local_get(&self.store.value_stack, cf, *left), i32::local_get(&self.store.value_stack, cf, *right), *op)) { return Ok(None) }, - JumpCmpLocalLocal64 { target_ip, left, right, op } => if Self::exec_jump_if(&mut self.cf, *target_ip, |cf| cmp_i64(i64::local_get(&self.store.value_stack, cf, *left), i64::local_get(&self.store.value_stack, cf, *right), *op)) { return Ok(None) }, - DropKeep(drop_keep) => self.exec_drop_keep(*drop_keep), - BranchTable(default_ip, start, len) => { self.exec_branch_table(*default_ip, *start, *len); return Ok(None); } + (value == 0) == (v.on_zero != 0) + }) { return Ok(None) } }, + IncLocalJumpCmpLocal32(packed) => { let v = packed.index.get(&self.func.data); let op = packed.op; if Self::exec_jump_if(&mut self.cf, v.target, |cf| { let lhs = i32::local_get(&self.store.value_stack, cf, v.local).wrapping_add(v.value); i32::local_set(&mut self.store.value_stack, cf, v.local, lhs); cmp_i32(lhs, i32::local_get(&self.store.value_stack, cf, v.right), op) }) { return Ok(None) } }, + JumpCmpLocalConst32(packed) => { let v = packed.index.get(&self.func.data); let op = packed.op; if Self::exec_jump_if(&mut self.cf, v.target, |cf| cmp_i32(i32::local_get(&self.store.value_stack, cf, v.local), v.value, op)) { return Ok(None) } }, + JumpCmpLocalConst64(packed) => { let v = packed.index.get(&self.func.data); let op = packed.op; if Self::exec_jump_if(&mut self.cf, v.target, |cf| cmp_i64(i64::local_get(&self.store.value_stack, cf, v.local), i64::from(v.value), op)) { return Ok(None) } }, + JumpCmpLocalLocal32(packed) => { let v = packed.index.get(&self.func.data); let op = packed.op; if Self::exec_jump_if(&mut self.cf, v.target, |cf| cmp_i32(i32::local_get(&self.store.value_stack, cf, v.left), i32::local_get(&self.store.value_stack, cf, v.right), op)) { return Ok(None) } }, + JumpCmpLocalLocal64(packed) => { let v = packed.index.get(&self.func.data); let op = packed.op; if Self::exec_jump_if(&mut self.cf, v.target, |cf| cmp_i64(i64::local_get(&self.store.value_stack, cf, v.left), i64::local_get(&self.store.value_stack, cf, v.right), op)) { return Ok(None) } }, + DropKeep32 { base, keep } => self.store.value_stack.stack_32.truncate_keep((self.cf.stack_base().s32 + u32::from(*base)) as usize, *keep as usize), + DropKeep64 { base, keep } => self.store.value_stack.stack_64.truncate_keep((self.cf.stack_base().s64 + u32::from(*base)) as usize, *keep as usize), + DropKeep128 { base, keep } => self.store.value_stack.stack_128.truncate_keep((self.cf.stack_base().s128 + u32::from(*base)) as usize, *keep as usize), + BranchTable(idx) => { self.exec_branch_table(*idx); return Ok(None); } Return => { if self.exec_return() { return Ok(Some(())); } return Ok(None); } ReturnVoid => { if self.exec_return_void() { return Ok(Some(())); } return Ok(None); } Return32 => { if self.exec_return_32() { return Ok(Some(())); } return Ok(None); } Return64 => { if self.exec_return_64() { return Ok(Some(())); } return Ok(None); } Return128 => { if self.exec_return_128() { return Ok(Some(())); } return Ok(None); } - LocalGet32(local_index) => self.store.value_stack.push(Value32::local_get(&self.store.value_stack, &self.cf, *local_index))?, - LocalGet64(local_index) => self.store.value_stack.push(Value64::local_get(&self.store.value_stack, &self.cf, *local_index))?, - LocalGet128(local_index) => self.store.value_stack.push(Value128::local_get(&self.store.value_stack, &self.cf, *local_index))?, + LocalGet32(local_index) => Value32::local_push(&mut self.store.value_stack, &self.cf, *local_index)?, + LocalGet64(local_index) => Value64::local_push(&mut self.store.value_stack, &self.cf, *local_index)?, + LocalGet128(local_index) => Value128::local_push(&mut self.store.value_stack, &self.cf, *local_index)?, LocalSet32(local_index) => exec_op!(local_set_pop Value32, local_index), LocalSet64(local_index) => exec_op!(local_set_pop Value64, local_index), LocalSet128(local_index) => exec_op!(local_set_pop Value128, local_index), @@ -308,9 +284,9 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { LocalCopy64(from, to) => Value64::local_copy(&mut self.store.value_stack, &self.cf, *from, *to), LocalCopy128(from, to) => Value128::local_copy(&mut self.store.value_stack, &self.cf, *from, *to), AddConst32(c) => exec_op!(unary i32 => i32, |v| v.wrapping_add(*c)), - AddConst64(c) => exec_op!(unary i64 => i64, |v| v.wrapping_add(*c)), - IncLocal32(local_index, delta) => i32::local_update(&mut self.store.value_stack, &self.cf, *local_index, |v| v.wrapping_add(*delta)), - IncLocal64(local_index, delta) => i64::local_update(&mut self.store.value_stack, &self.cf, *local_index, |v| v.wrapping_add(*delta )), + AddConst64(idx) => { let c = idx.get(&self.func.data).value; exec_op!(unary i64 => i64, |v| v.wrapping_add(c)); }, + IncLocal32(arg) => i32::local_update(&mut self.store.value_stack, &self.cf, arg.local, |v| v.wrapping_add(arg.value)), + IncLocal64(idx) => { let v = idx.get(&self.func.data); i64::local_update(&mut self.store.value_stack, &self.cf, v.local, |n| n.wrapping_add(v.value)); }, I32Add3 => exec_op!(ternary i32 => i32, |a, b, c| a.wrapping_add(b).wrapping_add(c)), I64Add3 => exec_op!(ternary i64 => i64, |a, b, c| a.wrapping_add(b).wrapping_add(c)), MulAccLocal32(acc) => exec_op!(binop_acc_local i32, acc, |a: i32, b| a.wrapping_mul(b), |a: i32, b| a.wrapping_add(b)), @@ -322,59 +298,65 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { BinOpLocalLocal128(op, a, b) => exec_op!(binop_local_local Value128, exec_binop_128, op, a, b), CmpLocalLocal32(op, a, b) => exec_op!(cmp_local_local i32, cmp_i32, op, a, b), CmpLocalLocal64(op, a, b) => exec_op!(cmp_local_local i64, cmp_i64, op, a, b), - BinOpLocalLocalSet32(op, a, b, dst) => exec_op!(binop_local_local_set Value32, exec_binop_32, op, a, b, dst), - BinOpLocalLocalSet64(op, a, b, dst) => exec_op!(binop_local_local_set Value64, exec_binop_64, op, a, b, dst), - BinOpLocalLocalSet128(op, a, b, dst) => exec_op!(binop_local_local_set Value128, exec_binop_128, op, a, b, dst), - BinOpLocalLocalTee32(op, a, b, dst) => exec_op!(binop_local_local_tee Value32, exec_binop_32, op, a, b, dst), - BinOpLocalLocalTee64(op, a, b, dst) => exec_op!(binop_local_local_tee Value64, exec_binop_64, op, a, b, dst), - BinOpLocalLocalTee128(op, a, b, dst) => exec_op!(binop_local_local_tee Value128, exec_binop_128, op, a, b, dst), - BinOpLocalConst32(op, local_index, c) => exec_op!(binop_local_const Value32, exec_binop_32, op, local_index, *c as u32), - BinOpLocalConst64(op, local_index, c) => exec_op!(binop_local_const Value64, exec_binop_64, op, local_index, *c as u64), - BinOpLocalConst128(op, local_index, c) => exec_op!(binop_local_const Value128, exec_binop_128, op, local_index, Value128(self.func.data.v128_const(*c))), - BinOpGlobalConst32(op, global_index, c) => exec_op!(binop_global_const Value32, exec_binop_32, op, global_index, *c as u32), - BinOpGlobalConst64(op, global_index, c) => exec_op!(binop_global_const Value64, exec_binop_64, op, global_index, *c as u64), - BinOpGlobalConst128(op, global_index, c) => exec_op!(binop_global_const Value128, exec_binop_128, op, global_index, Value128(self.func.data.v128_const(*c))), - BinOpLocalConstSet32(op, local_index, c, dst) => exec_op!(binop_local_const_set Value32, exec_binop_32, op, local_index, *c as u32, dst), - BinOpLocalConstSet64(op, local_index, c, dst) => exec_op!(binop_local_const_set Value64, exec_binop_64, op, local_index, *c as u64, dst), - BinOpLocalConstSet128(op, local_index, c, dst) => exec_op!(binop_local_const_set Value128, exec_binop_128, op, local_index, Value128(self.func.data.v128_const(*c)), dst), - BinOpLocalConstTee32(op, local_index, c, dst) => exec_op!(binop_local_const_tee Value32, exec_binop_32, op, local_index, *c as u32, dst), - BinOpLocalConstTee64(op, local_index, c, dst) => exec_op!(binop_local_const_tee Value64, exec_binop_64, op, local_index, *c as u64, dst), - BinOpLocalConstTee128(op, local_index, c, dst) => exec_op!(binop_local_const_tee Value128, exec_binop_128, op, local_index, Value128(self.func.data.v128_const(*c)), dst), + AddLocalLocalSet32(arg) => { let value = i32::local_get(&self.store.value_stack, &self.cf, arg.left).wrapping_add(i32::local_get(&self.store.value_stack, &self.cf, arg.right)); i32::local_set(&mut self.store.value_stack, &self.cf, arg.dst, value); }, + AddLocalLocalTee32(arg) => { let value = i32::local_get(&self.store.value_stack, &self.cf, arg.left).wrapping_add(i32::local_get(&self.store.value_stack, &self.cf, arg.right)); i32::local_set(&mut self.store.value_stack, &self.cf, arg.dst, value); i32::stack_push(&mut self.store.value_stack, value)?; }, + BinOpLocalLocalSet32(packed) => { let v = packed.index.get(&self.func.data); let value = exec_binop_32(packed.op, Value32::local_get(&self.store.value_stack, &self.cf, v.left), Value32::local_get(&self.store.value_stack, &self.cf, v.right)); Value32::local_set(&mut self.store.value_stack, &self.cf, v.dst, value); }, + BinOpLocalLocalSet64(packed) => { let v = packed.index.get(&self.func.data); let value = exec_binop_64(packed.op, Value64::local_get(&self.store.value_stack, &self.cf, v.left), Value64::local_get(&self.store.value_stack, &self.cf, v.right)); Value64::local_set(&mut self.store.value_stack, &self.cf, v.dst, value); }, + BinOpLocalLocalSet128(packed) => { let v = packed.index.get(&self.func.data); let value = exec_binop_128(packed.op, Value128::local_get(&self.store.value_stack, &self.cf, v.left), Value128::local_get(&self.store.value_stack, &self.cf, v.right)); Value128::local_set(&mut self.store.value_stack, &self.cf, v.dst, value); }, + BinOpLocalLocalTee32(packed) => { let v = packed.index.get(&self.func.data); let value = exec_binop_32(packed.op, Value32::local_get(&self.store.value_stack, &self.cf, v.left), Value32::local_get(&self.store.value_stack, &self.cf, v.right)); Value32::local_set(&mut self.store.value_stack, &self.cf, v.dst, value); Value32::stack_push(&mut self.store.value_stack, value)?; }, + BinOpLocalLocalTee64(packed) => { let v = packed.index.get(&self.func.data); let value = exec_binop_64(packed.op, Value64::local_get(&self.store.value_stack, &self.cf, v.left), Value64::local_get(&self.store.value_stack, &self.cf, v.right)); Value64::local_set(&mut self.store.value_stack, &self.cf, v.dst, value); Value64::stack_push(&mut self.store.value_stack, value)?; }, + BinOpLocalLocalTee128(packed) => { let v = packed.index.get(&self.func.data); let value = exec_binop_128(packed.op, Value128::local_get(&self.store.value_stack, &self.cf, v.left), Value128::local_get(&self.store.value_stack, &self.cf, v.right)); Value128::local_set(&mut self.store.value_stack, &self.cf, v.dst, value); Value128::stack_push(&mut self.store.value_stack, value)?; }, + AddLocalConst32(arg) => { let value = i32::local_get(&self.store.value_stack, &self.cf, arg.local).wrapping_add(arg.value); i32::stack_push(&mut self.store.value_stack, value)?; }, + SubLocalConst32(arg) => { let value = i32::local_get(&self.store.value_stack, &self.cf, arg.local).wrapping_sub(arg.value); i32::stack_push(&mut self.store.value_stack, value)?; }, + MulLocalConst32(arg) => { let value = i32::local_get(&self.store.value_stack, &self.cf, arg.local).wrapping_mul(arg.value); i32::stack_push(&mut self.store.value_stack, value)?; }, + BinOpLocalConst32(packed) => { let v = packed.index.get(&self.func.data); let lhs = Value32::local_get(&self.store.value_stack, &self.cf, v.local); Value32::stack_push(&mut self.store.value_stack, exec_binop_32(packed.op, lhs, v.value))?; }, + BinOpLocalConst64(packed) => { let v = packed.index.get(&self.func.data); let lhs = Value64::local_get(&self.store.value_stack, &self.cf, v.local); Value64::stack_push(&mut self.store.value_stack, exec_binop_64(packed.op, lhs, v.value))?; }, + BinOpLocalConst128(packed) => { let v = packed.index.get(&self.func.data); let value = exec_binop_128(packed.op, Value128::local_get(&self.store.value_stack, &self.cf, v.local), Value128(v.value.get(&self.func.data).value)); Value128::stack_push(&mut self.store.value_stack, value)?; }, + BinOpGlobalConst32(packed) => { let v = packed.index.get(&self.func.data); Value32::stack_push(&mut self.store.value_stack, exec_binop_32(packed.op, Value32::global_get(&self.store.state.globals, self.module.resolve_global_addr(v.global)), v.value))?; }, + BinOpGlobalConst64(packed) => { let v = packed.index.get(&self.func.data); Value64::stack_push(&mut self.store.value_stack, exec_binop_64(packed.op, Value64::global_get(&self.store.state.globals, self.module.resolve_global_addr(v.global)), v.value))?; }, + BinOpGlobalConst128(packed) => { let v = packed.index.get(&self.func.data); Value128::stack_push(&mut self.store.value_stack, exec_binop_128(packed.op, Value128::global_get(&self.store.state.globals, self.module.resolve_global_addr(v.global)), Value128(v.value.get(&self.func.data).value)))?; }, + BinOpLocalConstSet32(packed) => { let v = packed.index.get(&self.func.data); let value = exec_binop_32(packed.op, Value32::local_get(&self.store.value_stack, &self.cf, v.local), v.value); Value32::local_set(&mut self.store.value_stack, &self.cf, v.dst, value); }, + BinOpLocalConstTee32(packed) => { let v = packed.index.get(&self.func.data); let value = exec_binop_32(packed.op, Value32::local_get(&self.store.value_stack, &self.cf, v.local), v.value); Value32::local_set(&mut self.store.value_stack, &self.cf, v.dst, value); Value32::stack_push(&mut self.store.value_stack, value)?; }, + BinOpLocalConstSet64(packed) => { let v = packed.index.get(&self.func.data); let value = exec_binop_64(packed.op, Value64::local_get(&self.store.value_stack, &self.cf, v.local), v.value); Value64::local_set(&mut self.store.value_stack, &self.cf, v.dst, value); }, + BinOpLocalConstTee64(packed) => { let v = packed.index.get(&self.func.data); let value = exec_binop_64(packed.op, Value64::local_get(&self.store.value_stack, &self.cf, v.local), v.value); Value64::local_set(&mut self.store.value_stack, &self.cf, v.dst, value); Value64::stack_push(&mut self.store.value_stack, value)?; }, + BinOpLocalConstSet128(packed) => { let v = packed.index.get(&self.func.data); let value = exec_binop_128(packed.op, Value128::local_get(&self.store.value_stack, &self.cf, v.local), Value128(v.value.get(&self.func.data).value)); Value128::local_set(&mut self.store.value_stack, &self.cf, v.dst, value); }, + BinOpLocalConstTee128(packed) => { let v = packed.index.get(&self.func.data); let value = exec_binop_128(packed.op, Value128::local_get(&self.store.value_stack, &self.cf, v.local), Value128(v.value.get(&self.func.data).value)); Value128::local_set(&mut self.store.value_stack, &self.cf, v.dst, value); Value128::stack_push(&mut self.store.value_stack, value)?; }, BinOpStackGlobal32(op, global_index) => exec_op!(binop_stack_global Value32, exec_binop_32, op, global_index), BinOpStackGlobal64(op, global_index) => exec_op!(binop_stack_global Value64, exec_binop_64, op, global_index), BinOpStackLocal32(op, local) => exec_op!(binop_stack_local Value32, exec_binop_32, op, local), BinOpStackLocalSet32(op, local, dst) => exec_op!(binop_stack_local_set Value32, exec_binop_32, op, local, dst), BinOpStackLocalTee32(op, local, dst) => exec_op!(binop_stack_local_tee Value32, exec_binop_32, op, local, dst), - SetLocalConst32(local_index, c) => i32::local_set(&mut self.store.value_stack, &self.cf, *local_index, *c), - SetLocalConst64(local_index, c) => i64::local_set(&mut self.store.value_stack, &self.cf, *local_index, *c), - SetLocalConst128(local_index, c) => Value128::local_set(&mut self.store.value_stack, &self.cf, *local_index, Value128(self.func.data.v128_const(*c))), - IncMemoryLocal32(m, addr_local) => self.exec_inc_memory_local::(*m, *addr_local, |v| v.wrapping_add(1))?, - IncMemoryLocal64(m, addr_local) => self.exec_inc_memory_local::(*m, *addr_local, |v| v.wrapping_add(1))?, - StoreLocalLocal32(m, addr_local, value_local) => self.exec_store_local_local::(*m, *addr_local, *value_local)?, - StoreLocalLocal64(m, addr_local, value_local) => self.exec_store_local_local::(*m, *addr_local, *value_local)?, - StoreLocalLocal128(m, addr_local, value_local) => self.exec_store_local_local::(*m, *addr_local, *value_local)?, - LoadLocal32(m, addr_local) => self.exec_load_local::(*m, *addr_local, 0, |v| v)?, - LoadLocal64(m, addr_local) => self.exec_load_local::(*m, *addr_local, 0, |v| v)?, - LoadLocal8S32(m, addr_local) => self.exec_load_local::(*m, *addr_local, 0, i32::from)?, - LoadLocal8U32(m, addr_local) => self.exec_load_local::(*m, *addr_local, 0, i32::from)?, - LoadLocal16S32(m, addr_local) => self.exec_load_local::(*m, *addr_local, 0, i32::from)?, - LoadLocal16U32(m, addr_local) => self.exec_load_local::(*m, *addr_local, 0, i32::from)?, - LoadLocalTee32(m, addr_local, dst_local) => self.exec_load_local::(*m, *addr_local, *dst_local, |v| v)?, - LoadLocalSet32(m, addr_local, dst_local) => self.exec_load_local::(*m, *addr_local, *dst_local, |v| v)?, - LoadLocalTee8S32(m, addr_local, dst_local) => self.exec_load_local::(*m, *addr_local, *dst_local, i32::from)?, - LoadLocalTee8U32(m, addr_local, dst_local) => self.exec_load_local::(*m, *addr_local, *dst_local, i32::from)?, - LoadLocalTee16S32(m, addr_local, dst_local) => self.exec_load_local::(*m, *addr_local, *dst_local, i32::from)?, - LoadLocalTee16U32(m, addr_local, dst_local) => self.exec_load_local::(*m, *addr_local, *dst_local, i32::from)?, - LoadLocalSet8S32(m, addr_local, dst_local) => self.exec_load_local::(*m, *addr_local, *dst_local, i32::from)?, - LoadLocalSet8U32(m, addr_local, dst_local) => self.exec_load_local::(*m, *addr_local, *dst_local, i32::from)?, - LoadLocalSet16S32(m, addr_local, dst_local) => self.exec_load_local::(*m, *addr_local, *dst_local, i32::from)?, - LoadLocalSet16U32(m, addr_local, dst_local) => self.exec_load_local::(*m, *addr_local, *dst_local, i32::from)?, - LoadLocalTee128(m, addr_local, dst_local) => self.exec_load_local::(*m, *addr_local, *dst_local, |v| v)?, - LoadLocalSet128(m, addr_local, dst_local) => self.exec_load_local::(*m, *addr_local, *dst_local, |v| v)?, - AndConstTee32(c, local_index) => { exec_op!(unary i32 => i32, |v| v & *c); exec_op!(local_tee i32, local_index); } - SubConstTee32(c, local_index) => { exec_op!(unary i32 => i32, |v| v.wrapping_sub(*c)); exec_op!(local_tee i32, local_index); } - AndConstTee64(c, local_index) => { exec_op!(unary i64 => i64, |v| v & *c); exec_op!(local_tee i64, local_index); } - SubConstTee64(c, local_index) => { exec_op!(unary i64 => i64, |v| v.wrapping_sub(*c)); exec_op!(local_tee i64, local_index); } + BinOpStackLocal128(op, local) => exec_op!(binop_stack_local Value128, exec_binop_128, op, local), + SetLocalConst32(arg) => i32::local_set(&mut self.store.value_stack, &self.cf, arg.local, arg.value), + SetLocalConst64(idx) => { let v = idx.get(&self.func.data); i64::local_set(&mut self.store.value_stack, &self.cf, v.local, v.value); }, + SetLocalConst128(arg) => Value128::local_set(&mut self.store.value_stack, &self.cf, arg.local, Value128(arg.value.get(&self.func.data).value)), + IncMemoryLocal32(arg) => self.exec_inc_memory_local::(arg.memory_arg_idx.get(&self.func.data), arg.local1, |v| v.wrapping_add(1))?, + IncMemoryLocal64(arg) => self.exec_inc_memory_local::(arg.memory_arg_idx.get(&self.func.data), arg.local1, |v| v.wrapping_add(1))?, + StoreLocalLocal32(arg) => self.exec_store_local_local::(arg.memory_arg_idx.get(&self.func.data), arg.local1, arg.local2)?, + StoreLocalLocal64(arg) => self.exec_store_local_local::(arg.memory_arg_idx.get(&self.func.data), arg.local1, arg.local2)?, + StoreLocalLocal128(arg) => self.exec_store_local_local::(arg.memory_arg_idx.get(&self.func.data), arg.local1, arg.local2)?, + LoadLocal32(arg) => self.exec_load_local::(arg.memory_arg_idx.get(&self.func.data), arg.local1, 0, |v| v)?, + LoadLocal64(arg) => self.exec_load_local::(arg.memory_arg_idx.get(&self.func.data), arg.local1, 0, |v| v)?, + LoadLocal8S32(arg) => self.exec_load_local::(arg.memory_arg_idx.get(&self.func.data), arg.local1, 0, i32::from)?, + LoadLocal8U32(arg) => self.exec_load_local::(arg.memory_arg_idx.get(&self.func.data), arg.local1, 0, i32::from)?, + LoadLocal16S32(arg) => self.exec_load_local::(arg.memory_arg_idx.get(&self.func.data), arg.local1, 0, i32::from)?, + LoadLocal16U32(arg) => self.exec_load_local::(arg.memory_arg_idx.get(&self.func.data), arg.local1, 0, i32::from)?, + LoadLocalTee32(arg) => self.exec_load_local::(arg.memory_arg_idx.get(&self.func.data), arg.local1, arg.local2, |v| v)?, + LoadLocalSet32(arg) => self.exec_load_local::(arg.memory_arg_idx.get(&self.func.data), arg.local1, arg.local2, |v| v)?, + LoadLocalTee8S32(arg) => self.exec_load_local::(arg.memory_arg_idx.get(&self.func.data), arg.local1, arg.local2, i32::from)?, + LoadLocalTee8U32(arg) => self.exec_load_local::(arg.memory_arg_idx.get(&self.func.data), arg.local1, arg.local2, i32::from)?, + LoadLocalTee16S32(arg) => self.exec_load_local::(arg.memory_arg_idx.get(&self.func.data), arg.local1, arg.local2, i32::from)?, + LoadLocalTee16U32(arg) => self.exec_load_local::(arg.memory_arg_idx.get(&self.func.data), arg.local1, arg.local2, i32::from)?, + LoadLocalSet8S32(arg) => self.exec_load_local::(arg.memory_arg_idx.get(&self.func.data), arg.local1, arg.local2, i32::from)?, + LoadLocalSet8U32(arg) => self.exec_load_local::(arg.memory_arg_idx.get(&self.func.data), arg.local1, arg.local2, i32::from)?, + LoadLocalSet16S32(arg) => self.exec_load_local::(arg.memory_arg_idx.get(&self.func.data), arg.local1, arg.local2, i32::from)?, + LoadLocalSet16U32(arg) => self.exec_load_local::(arg.memory_arg_idx.get(&self.func.data), arg.local1, arg.local2, i32::from)?, + LoadLocalTee128(arg) => self.exec_load_local::(arg.memory_arg_idx.get(&self.func.data), arg.local1, arg.local2, |v| v)?, + LoadLocalSet128(arg) => self.exec_load_local::(arg.memory_arg_idx.get(&self.func.data), arg.local1, arg.local2, |v| v)?, + AndConstTee32(arg) => { exec_op!(unary i32 => i32, |v| v & arg.value); let value = i32::stack_peek(&self.store.value_stack); i32::local_set(&mut self.store.value_stack, &self.cf, arg.local, value); }, + SubConstTee32(arg) => { exec_op!(unary i32 => i32, |v| v.wrapping_sub(arg.value)); let value = i32::stack_peek(&self.store.value_stack); i32::local_set(&mut self.store.value_stack, &self.cf, arg.local, value); }, + AndConstTee64(idx) => { let v = idx.get(&self.func.data); exec_op!(unary i64 => i64, |n| n & v.value); let value = i64::stack_peek(&self.store.value_stack); i64::local_set(&mut self.store.value_stack, &self.cf, v.local, value); }, + SubConstTee64(idx) => { let v = idx.get(&self.func.data); exec_op!(unary i64 => i64, |n| n.wrapping_sub(v.value)); let value = i64::stack_peek(&self.store.value_stack); i64::local_set(&mut self.store.value_stack, &self.cf, v.local, value); }, LocalTee32(local_index) => exec_op!(local_tee Value32, local_index), LocalTee64(local_index) => exec_op!(local_tee Value64, local_index), LocalTee128(local_index) => exec_op!(local_tee Value128, local_index), @@ -388,7 +370,9 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { GlobalTee64(global_index) => exec_op!(global_tee Value64, global_index), GlobalTee128(global_index) => exec_op!(global_tee Value128, global_index), Const32(val) => i32::stack_push(&mut self.store.value_stack, *val)?, - Const64(val) => i64::stack_push(&mut self.store.value_stack, *val)?, + Const64Imm(val) => i64::stack_push(&mut self.store.value_stack, i64::from(*val))?, + Const64(idx) => i64::stack_push(&mut self.store.value_stack, idx.get(&self.func.data).value)?, + Const128Imm(val) => Value128::stack_push(&mut self.store.value_stack, Value128(u128::from(*val).to_le_bytes()))?, I64Eqz => exec_op!(unary i64 => i32, |v| i32::from(v == 0)), I32Eqz => exec_op!(unary i32 => i32, |v| i32::from(v == 0)), I32Eq => exec_op!(binary i32 => i32, |a, b| i32::from(a == b)), @@ -502,32 +486,32 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { // GC objects StructNew(ty) => self.exec_struct_new(*ty, false)?, StructNewDefault(ty) => self.exec_struct_new(*ty, true)?, - StructGet(ty, field) => self.exec_struct_get(*ty, *field, None)?, - StructGetS(ty, field) => self.exec_struct_get(*ty, *field, Some(true))?, - StructGetU(ty, field) => self.exec_struct_get(*ty, *field, Some(false))?, - StructSet(ty, field) => self.exec_struct_set(*ty, *field)?, + StructGet(idx) => self.exec_struct_get(*idx, None)?, + StructGetS(idx) => self.exec_struct_get(*idx, Some(true))?, + StructGetU(idx) => self.exec_struct_get(*idx, Some(false))?, + StructSet(idx) => self.exec_struct_set(*idx)?, ArrayNew(ty) => self.exec_array_new(*ty, false)?, ArrayNewDefault(ty) => self.exec_array_new(*ty, true)?, - ArrayNewFixed(ty, len) => self.exec_array_new_fixed(*ty, *len)?, - ArrayNewData(ty, data) => self.exec_array_new_data(*ty, *data)?, - ArrayNewElem(ty, elem) => self.exec_array_new_elem(*ty, *elem)?, + ArrayNewFixed(idx) => self.exec_array_new_fixed(*idx)?, + ArrayNewData(idx) => self.exec_array_new_data(*idx)?, + ArrayNewElem(idx) => self.exec_array_new_elem(*idx)?, ArrayGet(ty) => self.exec_array_get(*ty, None)?, ArrayGetS(ty) => self.exec_array_get(*ty, Some(true))?, ArrayGetU(ty) => self.exec_array_get(*ty, Some(false))?, ArraySet(ty) => self.exec_array_set(*ty)?, ArrayLen => self.exec_array_len()?, ArrayFill(ty) => self.exec_array_fill(*ty)?, - ArrayCopy(dst, src) => self.exec_array_copy(*dst, *src)?, - ArrayInitData(ty, data) => self.exec_array_init_data(*ty, *data)?, - ArrayInitElem(ty, elem) => self.exec_array_init_elem(*ty, *elem)?, + ArrayCopy(idx) => self.exec_array_copy(*idx)?, + ArrayInitData(idx) => self.exec_array_init_data(*idx)?, + ArrayInitElem(idx) => self.exec_array_init_elem(*idx)?, MemorySize(addr) => self.exec_memory_size(*addr)?, MemoryGrow(addr) => self.exec_memory_grow(*addr)?, // Bulk memory operations - MemoryCopy { dst_mem, src_mem } => self.exec_memory_copy(*dst_mem, *src_mem)?, + MemoryCopy(idx) => self.exec_memory_copy(*idx)?, MemoryFill(addr) => self.exec_memory_fill(*addr)?, - MemoryFillImm(addr, val, size) => self.exec_memory_fill_imm(*addr, *val, *size)?, - MemoryInit(data_idx, mem_idx) => self.exec_memory_init(*data_idx, *mem_idx)?, + MemoryFillConst(idx) => self.exec_memory_fill_const(*idx)?, + MemoryInit(idx) => self.exec_memory_init(*idx)?, DataDrop(data_index) => self.store.state.get_data_mut(self.module.resolve_data_addr(*data_index)).drop(), ElemDrop(elem_index) => self.store.state.get_elem_mut(self.module.resolve_elem_addr(*elem_index)).drop(), @@ -535,37 +519,37 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { TableGet(table_idx) => self.exec_table_get(*table_idx)?, TableSet(table_idx) => self.exec_table_set(*table_idx)?, TableSize(table_idx) => self.exec_table_size(*table_idx)?, - TableInit(elem_idx, table_idx) => self.exec_table_init(*elem_idx, *table_idx)?, + TableInit(idx) => self.exec_table_init(*idx)?, TableGrow(table_idx) => self.exec_table_grow(*table_idx)?, TableFill(table_idx) => self.exec_table_fill(*table_idx)?, - TableCopy { dst_table, src_table } => self.exec_table_copy(*dst_table, *src_table)?, + TableCopy(idx) => self.exec_table_copy(*idx)?, // Core memory load/store operations - I32Store(m) => self.exec_mem_store::(m.mem_addr(), m.offset(), |v| v)?, - I64Store(m) => self.exec_mem_store::(m.mem_addr(), m.offset(), |v| v)?, - F32Store(m) => self.exec_mem_store::(m.mem_addr(), m.offset(), |v| v)?, - F64Store(m) => self.exec_mem_store::(m.mem_addr(), m.offset(), |v| v)?, + I32Store(idx) => self.exec_mem_store::(*idx, |v| v)?, + I64Store(idx) => self.exec_mem_store::(*idx, |v| v)?, + F32Store(idx) => self.exec_mem_store::(*idx, |v| v)?, + F64Store(idx) => self.exec_mem_store::(*idx, |v| v)?, FMaStoreF32(m) => self.exec_fma_store::(*m)?, FMaStoreF64(m) => self.exec_fma_store::(*m)?, - I32Store8(m) => self.exec_mem_store::(m.mem_addr(), m.offset(), |v| v as i8)?, - I32Store16(m) => self.exec_mem_store::(m.mem_addr(), m.offset(), |v| v as i16)?, - I64Store8(m) => self.exec_mem_store::(m.mem_addr(), m.offset(), |v| v as i8)?, - I64Store16(m) => self.exec_mem_store::(m.mem_addr(), m.offset(), |v| v as i16)?, - I64Store32(m) => self.exec_mem_store::(m.mem_addr(), m.offset(), |v| v as i32)?, - I32Load(m) => self.exec_mem_load::(m.mem_addr(), m.offset(), |v| v)?, - I64Load(m) => self.exec_mem_load::(m.mem_addr(), m.offset(), |v| v)?, - F32Load(m) => self.exec_mem_load::(m.mem_addr(), m.offset(), |v| v)?, - F64Load(m) => self.exec_mem_load::(m.mem_addr(), m.offset(), |v| v)?, - I32Load8S(m) => self.exec_mem_load::(m.mem_addr(), m.offset(), i32::from)?, - I32Load8U(m) => self.exec_mem_load::(m.mem_addr(), m.offset(), i32::from)?, - I32Load16S(m) => self.exec_mem_load::(m.mem_addr(), m.offset(), i32::from)?, - I32Load16U(m) => self.exec_mem_load::(m.mem_addr(), m.offset(), i32::from)?, - I64Load8S(m) => self.exec_mem_load::(m.mem_addr(), m.offset(), i64::from)?, - I64Load8U(m) => self.exec_mem_load::(m.mem_addr(), m.offset(), i64::from)?, - I64Load16S(m) => self.exec_mem_load::(m.mem_addr(), m.offset(), i64::from)?, - I64Load16U(m) => self.exec_mem_load::(m.mem_addr(), m.offset(), i64::from)?, - I64Load32S(m) => self.exec_mem_load::(m.mem_addr(), m.offset(), i64::from)?, - I64Load32U(m) => self.exec_mem_load::(m.mem_addr(), m.offset(), i64::from)?, + I32Store8(idx) => self.exec_mem_store::(*idx, |v| v as i8)?, + I32Store16(idx) => self.exec_mem_store::(*idx, |v| v as i16)?, + I64Store8(idx) => self.exec_mem_store::(*idx, |v| v as i8)?, + I64Store16(idx) => self.exec_mem_store::(*idx, |v| v as i16)?, + I64Store32(idx) => self.exec_mem_store::(*idx, |v| v as i32)?, + I32Load(idx) => self.exec_mem_load::(*idx, |v| v)?, + I64Load(idx) => self.exec_mem_load::(*idx, |v| v)?, + F32Load(idx) => self.exec_mem_load::(*idx, |v| v)?, + F64Load(idx) => self.exec_mem_load::(*idx, |v| v)?, + I32Load8S(idx) => self.exec_mem_load::(*idx, i32::from)?, + I32Load8U(idx) => self.exec_mem_load::(*idx, i32::from)?, + I32Load16S(idx) => self.exec_mem_load::(*idx, i32::from)?, + I32Load16U(idx) => self.exec_mem_load::(*idx, i32::from)?, + I64Load8S(idx) => self.exec_mem_load::(*idx, i64::from)?, + I64Load8U(idx) => self.exec_mem_load::(*idx, i64::from)?, + I64Load16S(idx) => self.exec_mem_load::(*idx, i64::from)?, + I64Load16U(idx) => self.exec_mem_load::(*idx, i64::from)?, + I64Load32S(idx) => self.exec_mem_load::(*idx, i64::from)?, + I64Load32U(idx) => self.exec_mem_load::(*idx, i64::from)?, // Numeric conversion operations F32ConvertI32S => exec_op!(unary i32 => f32, |v| v as f32), @@ -637,25 +621,25 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { V128AnyTrue => exec_op!(unary Value128 => i32, |v| v.v128_any_true() as i32), I8x16Swizzle => exec_op!(binary Value128 => Value128, |a, s| a.i8x16_swizzle(s)), I8x16RelaxedSwizzle => exec_op!(binary Value128 => Value128, |a, s| a.i8x16_relaxed_swizzle(s)), - V128Load(arg) => self.exec_mem_load::(arg.mem_addr(), arg.offset(), |v| v)?, - V128Load8x8S(arg) => self.exec_mem_load::(arg.mem_addr(), arg.offset(), |v| Value128::v128_load8x8_s(v.to_le_bytes()))?, - V128Load8x8U(arg) => self.exec_mem_load::(arg.mem_addr(), arg.offset(), |v| Value128::v128_load8x8_u(v.to_le_bytes()))?, - V128Load16x4S(arg) => self.exec_mem_load::(arg.mem_addr(), arg.offset(), |v| Value128::v128_load16x4_s(v.to_le_bytes()))?, - V128Load16x4U(arg) => self.exec_mem_load::(arg.mem_addr(), arg.offset(), |v| Value128::v128_load16x4_u(v.to_le_bytes()))?, - V128Load32x2S(arg) => self.exec_mem_load::(arg.mem_addr(), arg.offset(), |v| Value128::v128_load32x2_s(v.to_le_bytes()))?, - V128Load32x2U(arg) => self.exec_mem_load::(arg.mem_addr(), arg.offset(), |v| Value128::v128_load32x2_u(v.to_le_bytes()))?, - V128Load8Splat(arg) => self.exec_mem_load::(arg.mem_addr(), arg.offset(), Value128::splat_i8)?, - V128Load16Splat(arg) => self.exec_mem_load::(arg.mem_addr(), arg.offset(), Value128::splat_i16)?, - V128Load32Splat(arg) => self.exec_mem_load::(arg.mem_addr(), arg.offset(), Value128::splat_i32)?, - V128Load64Splat(arg) => self.exec_mem_load::(arg.mem_addr(), arg.offset(), Value128::splat_i64)?, - V128Store(arg) => self.exec_mem_store::(arg.mem_addr(), arg.offset(), |v| v)?, - V128Store8Lane(arg, lane) => self.exec_mem_store_lane::(arg.mem_addr(), arg.offset(), *lane)?, - V128Store16Lane(arg, lane) => self.exec_mem_store_lane::(arg.mem_addr(), arg.offset(), *lane)?, - V128Store32Lane(arg, lane) => self.exec_mem_store_lane::(arg.mem_addr(), arg.offset(), *lane)?, - V128Store64Lane(arg, lane) => self.exec_mem_store_lane::(arg.mem_addr(), arg.offset(), *lane)?, - V128Load32Zero(arg) => self.exec_mem_load::(arg.mem_addr(), arg.offset(), |v| Value128::from_i32x4([v, 0, 0, 0]))?, - V128Load64Zero(arg) => self.exec_mem_load::(arg.mem_addr(), arg.offset(), |v| Value128::from_i64x2([v, 0]))?, - Const128(arg) => Value128::stack_push(&mut self.store.value_stack, Value128(self.func.data.v128_const(*arg)))?, + V128Load(idx) => self.exec_mem_load::(*idx, |v| v)?, + V128Load8x8S(idx) => self.exec_mem_load::(*idx, |v| Value128::v128_load8x8_s(v.to_le_bytes()))?, + V128Load8x8U(idx) => self.exec_mem_load::(*idx, |v| Value128::v128_load8x8_u(v.to_le_bytes()))?, + V128Load16x4S(idx) => self.exec_mem_load::(*idx, |v| Value128::v128_load16x4_s(v.to_le_bytes()))?, + V128Load16x4U(idx) => self.exec_mem_load::(*idx, |v| Value128::v128_load16x4_u(v.to_le_bytes()))?, + V128Load32x2S(idx) => self.exec_mem_load::(*idx, |v| Value128::v128_load32x2_s(v.to_le_bytes()))?, + V128Load32x2U(idx) => self.exec_mem_load::(*idx, |v| Value128::v128_load32x2_u(v.to_le_bytes()))?, + V128Load8Splat(idx) => self.exec_mem_load::(*idx, Value128::splat_i8)?, + V128Load16Splat(idx) => self.exec_mem_load::(*idx, Value128::splat_i16)?, + V128Load32Splat(idx) => self.exec_mem_load::(*idx, Value128::splat_i32)?, + V128Load64Splat(idx) => self.exec_mem_load::(*idx, Value128::splat_i64)?, + V128Store(idx) => self.exec_mem_store::(*idx, |v| v)?, + V128Store8Lane(arg) => self.exec_mem_store_lane::(*arg)?, + V128Store16Lane(arg) => self.exec_mem_store_lane::(*arg)?, + V128Store32Lane(arg) => self.exec_mem_store_lane::(*arg)?, + V128Store64Lane(arg) => self.exec_mem_store_lane::(*arg)?, + V128Load32Zero(idx) => self.exec_mem_load::(*idx, |v| Value128::from_i32x4([v, 0, 0, 0]))?, + V128Load64Zero(idx) => self.exec_mem_load::(*idx, |v| Value128::from_i64x2([v, 0]))?, + Const128(arg) => Value128::stack_push(&mut self.store.value_stack, Value128(arg.get(&self.func.data).value))?, I8x16ExtractLaneS(lane) => exec_op!(unary Value128 => i32, |v| v.extract_lane_i8(*lane) as i32), I8x16ExtractLaneU(lane) => exec_op!(unary Value128 => i32, |v| v.extract_lane_u8(*lane) as i32), I16x8ExtractLaneS(lane) => exec_op!(unary Value128 => i32, |v| v.extract_lane_i16(*lane) as i32), @@ -664,10 +648,10 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { I64x2ExtractLane(lane) => exec_op!(unary Value128 => i64, |v| v.extract_lane_i64(*lane)), F32x4ExtractLane(lane) => exec_op!(unary Value128 => f32, |v| v.extract_lane_f32(*lane)), F64x2ExtractLane(lane) => exec_op!(unary Value128 => f64, |v| v.extract_lane_f64(*lane)), - V128Load8Lane(arg, lane) => self.exec_mem_load_lane::(arg.mem_addr(), arg.offset(), *lane)?, - V128Load16Lane(arg, lane) => self.exec_mem_load_lane::(arg.mem_addr(), arg.offset(), *lane)?, - V128Load32Lane(arg, lane) => self.exec_mem_load_lane::(arg.mem_addr(), arg.offset(), *lane)?, - V128Load64Lane(arg, lane) => self.exec_mem_load_lane::(arg.mem_addr(), arg.offset(), *lane)?, + V128Load8Lane(arg) => self.exec_mem_load_lane::(*arg)?, + V128Load16Lane(arg) => self.exec_mem_load_lane::(*arg)?, + V128Load32Lane(arg) => self.exec_mem_load_lane::(*arg)?, + V128Load64Lane(arg) => self.exec_mem_load_lane::(*arg)?, I8x16ReplaceLane(lane) => exec_op!(binary_mixed i32, Value128 => Value128, |value, vec| vec.i8x16_replace_lane(*lane, value as i8)), I16x8ReplaceLane(lane) => exec_op!(binary_mixed i32, Value128 => Value128, |value, vec| vec.i16x8_replace_lane(*lane, value as i16)), I32x4ReplaceLane(lane) => exec_op!(binary_mixed i32, Value128 => Value128, |value, vec| vec.i32x4_replace_lane(*lane, value)), @@ -822,7 +806,7 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { I64x2ExtendHighI32x4S => exec_op!(unary Value128 => Value128, |a| a.i64x2_extend_high_i32x4_s()), I64x2ExtendHighI32x4U => exec_op!(unary Value128 => Value128, |a| a.i64x2_extend_high_i32x4_u()), I8x16Popcnt => exec_op!(unary Value128 => Value128, |v| v.i8x16_popcnt()), - I8x16Shuffle(idx) => exec_op!(binary Value128 => Value128, |a, b| Value128::i8x16_shuffle(a, b, Value128(self.func.data.v128_const(*idx)))), + I8x16Shuffle(idx) => exec_op!(binary Value128 => Value128, |a, b| Value128::i8x16_shuffle(a, b, Value128(idx.get(&self.func.data).value))), I16x8Q15MulrSatS => exec_op!(binary Value128 => Value128, |a, b| a.i16x8_q15mulr_sat_s(b)), I32x4DotI16x8S => exec_op!(binary Value128 => Value128, |a, b| a.i32x4_dot_i16x8_s(b)), I8x16RelaxedLaneselect => exec_op!(ternary Value128 => Value128, |a, b, c| Value128::i8x16_relaxed_laneselect(a, b, c)), @@ -900,25 +884,18 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { condition } - fn exec_branch_table(&mut self, default_ip: u32, start: u32, len: u32) { + fn exec_branch_table(&mut self, index: OperandIdx) { + let v = index.get(&self.func.data); let idx = ::stack_pop(&mut self.store.value_stack); - let target_ip = if idx >= 0 && (idx as u32) < len { - self.func.data.branch_table_targets.get((start + idx as u32) as usize).copied().unwrap_or(default_ip) + let target_ip = if idx >= 0 && (idx as u32) < v.len { + self.func.data.branch_table_targets.get((v.start + idx as u32) as usize).copied().unwrap_or(v.target) } else { - default_ip + v.target }; self.cf.instr_ptr = target_ip as usize; } - fn exec_drop_keep(&mut self, drop_keep: DropKeep) { - let mut base = self.cf.stack_base(); - base.s32 += drop_keep.base.c32 as u32; - base.s64 += drop_keep.base.c64 as u32; - base.s128 += drop_keep.base.c128 as u32; - self.store.value_stack.truncate_keep_counts(base, drop_keep.keep); - } - fn create_exception(&mut self, tag_index: TagAddr) -> Result { let tag_addr = self.module.resolve_tag_addr(tag_index); let type_addr = self.store.state.get_tag(tag_addr).type_addr; @@ -1043,22 +1020,26 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { } } - fn exec_call(&mut self, wasm_func: WasmFunctionInstance, func_addr: FuncAddr) -> Result<(), Trap> { - if !Arc::ptr_eq(&self.func, &wasm_func.func) { - self.func = wasm_func.func.clone(); - } - - let Ok(locals_base) = self.store.value_stack.enter_locals(&wasm_func.func.params, &wasm_func.func.locals) - else { + fn exec_call( + &mut self, + wasm_func: Arc, + owner: ModuleInstanceId, + func_addr: FuncAddr, + ) -> Result<(), Trap> { + let Ok(locals_base) = self.store.value_stack.enter_locals(&wasm_func.params, &wasm_func.locals) else { return cold!(Err(Trap::CallStackOverflow)); }; self.store.call_stack.push(self.cf)?; - self.cf = CallFrame::new(func_addr, locals_base, wasm_func.func.locals); - if wasm_func.owner != self.module.id() { + let locals = wasm_func.locals; + if !Arc::ptr_eq(&self.func, &wasm_func) { + self.func = wasm_func; + } + self.cf = CallFrame::new(func_addr, locals_base, locals); + if owner != self.module.id() { self.module = self .store - .get_module_instance(wasm_func.owner) + .get_module_instance(owner) .unwrap_or_else(|| unreachable!("invalid module instance")) .clone(); } @@ -1066,21 +1047,25 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { Ok(()) } - fn exec_return_call(&mut self, wasm_func: WasmFunctionInstance, func_addr: FuncAddr) -> Result<(), Trap> { - if !Arc::ptr_eq(&self.func, &wasm_func.func) { - self.func = wasm_func.func.clone(); - } - - self.store.value_stack.truncate_keep_counts(self.cf.locals_base, wasm_func.func.params); - let Ok(locals_base) = self.store.value_stack.enter_locals(&wasm_func.func.params, &wasm_func.func.locals) - else { + fn exec_return_call( + &mut self, + wasm_func: Arc, + owner: ModuleInstanceId, + func_addr: FuncAddr, + ) -> Result<(), Trap> { + self.store.value_stack.truncate_keep_counts(self.cf.locals_base, wasm_func.params); + let Ok(locals_base) = self.store.value_stack.enter_locals(&wasm_func.params, &wasm_func.locals) else { return cold!(Err(Trap::CallStackOverflow)); }; - self.cf = CallFrame::new(func_addr, locals_base, wasm_func.func.locals); - if wasm_func.owner != self.module.id() { + let locals = wasm_func.locals; + if !Arc::ptr_eq(&self.func, &wasm_func) { + self.func = wasm_func; + } + self.cf = CallFrame::new(func_addr, locals_base, locals); + if owner != self.module.id() { self.module = self .store - .get_module_instance(wasm_func.owner) + .get_module_instance(owner) .unwrap_or_else(|| unreachable!("invalid module instance")) .clone(); } @@ -1132,11 +1117,13 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { fn exec_call_direct(&mut self, v: u32) -> Result<(), Trap> { self.charge_call_fuel(FUEL_COST_CALL_TOTAL); let addr = self.module.resolve_func_addr(v); - let func = self.store.state.get_func(addr).clone(); - match func.kind { - crate::store::FunctionKind::Wasm(wasm_func) => self.exec_call(wasm_func, addr), + let func = self.store.state.get_func(addr); + match &func.kind { + crate::store::FunctionKind::Wasm(wasm_func) => { + self.exec_call(wasm_func.func.clone(), wasm_func.owner, addr) + } crate::store::FunctionKind::Host(host_func) => { - self.exec_call_host::(host_func, func.type_addr, func.gc.params)?; + self.exec_call_host::(host_func.clone(), func.type_addr, func.gc.params)?; Ok(()) } } @@ -1145,14 +1132,14 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { fn exec_return_call_direct(&mut self, v: u32) -> Result { self.charge_call_fuel(FUEL_COST_CALL_TOTAL); let addr = self.module.resolve_func_addr(v); - let func = self.store.state.get_func(addr).clone(); - match func.kind { + let func = self.store.state.get_func(addr); + match &func.kind { crate::store::FunctionKind::Wasm(wasm_func) => { - self.exec_return_call(wasm_func, addr)?; + self.exec_return_call(wasm_func.func.clone(), wasm_func.owner, addr)?; Ok(false) } crate::store::FunctionKind::Host(host_func) => { - self.exec_call_host::(host_func, func.type_addr, func.gc.params) + self.exec_call_host::(host_func.clone(), func.type_addr, func.gc.params) } } } @@ -1179,11 +1166,8 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { Ok(()) } - fn exec_call_indirect( - &mut self, - type_addr: u32, - table_addr: u32, - ) -> Result { + fn exec_call_indirect(&mut self, index: OperandIdx) -> Result { + let TwoU32 { first: type_addr, second: table_addr } = index.get(&self.func.data); self.charge_call_fuel(FUEL_COST_CALL_TOTAL); // verify that the table is of the right type, this should be validated by the parser already @@ -1207,20 +1191,20 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { func_addr: FuncAddr, expected_type_addr: TypeAddr, ) -> Result { - let func = self.store.state.get_func(func_addr).clone(); + let func = self.store.state.get_func(func_addr); if !self.store.state.type_addr_is_subtype(func.type_addr, expected_type_addr) { return cold!(Err(Trap::IndirectCallTypeMismatch { actual: Box::new(self.store.state.get_canonical_func_type(func.type_addr).clone()), expected: Box::new(self.store.state.get_canonical_func_type(expected_type_addr).clone()), })); } - match func.kind { + match &func.kind { crate::store::FunctionKind::Wasm(wasm_func) => match IS_RETURN_CALL { - true => self.exec_return_call(wasm_func, func_addr), - false => self.exec_call(wasm_func, func_addr), + true => self.exec_return_call(wasm_func.func.clone(), wasm_func.owner, func_addr), + false => self.exec_call(wasm_func.func.clone(), wasm_func.owner, func_addr), }, crate::store::FunctionKind::Host(host_func) => { - return self.exec_call_host::(host_func, func.type_addr, func.gc.params); + return self.exec_call_host::(host_func.clone(), func.type_addr, func.gc.params); } }?; Ok(false) @@ -1287,7 +1271,7 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { #[inline(always)] fn exec_store_local_local, const N: usize>( &mut self, - memarg: MemoryArg, + memarg: CompactMemoryArg, addr_local: u8, value_local: u8, ) -> Result<(), Trap> { @@ -1313,7 +1297,7 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { #[inline(always)] fn exec_inc_memory_local, const N: usize>( &mut self, - memarg: MemoryArg, + memarg: CompactMemoryArg, addr_local: u8, increment: impl FnOnce(T) -> T, ) -> Result<(), Trap> { @@ -1333,7 +1317,7 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { }; let mem = self.store.state.get_mem_mut(mem_addr); - let value = T::load_at(&*mem.inner, addr)?; + let value = cold_err!(T::load_at(&*mem.inner, addr))?; increment(value).store_at(&mut *mem.inner, addr) } @@ -1343,7 +1327,7 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { const N: usize, >( &mut self, - m: MemoryArg, + m: CompactMemoryArg, ) -> Result<(), Trap> { let rhs = T::stack_pop(&mut self.store.value_stack); let lhs = T::stack_pop(&mut self.store.value_stack); @@ -1361,7 +1345,7 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { #[inline(always)] fn exec_load_local_value, const N: usize>( &self, - memarg: MemoryArg, + memarg: CompactMemoryArg, addr_local: u8, ) -> Result { let mem = self.store.state.get_mem(self.module.resolve_mem_addr(memarg.mem_addr())); @@ -1377,7 +1361,7 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { let base = u32::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)); mem.effective_addr::(base as usize, memarg.offset())? }; - T::load_at(&*mem.inner, addr) + cold_err!(T::load_at(&*mem.inner, addr)) } #[inline(always)] @@ -1389,7 +1373,7 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { const TEE: bool, >( &mut self, - memarg: MemoryArg, + memarg: CompactMemoryArg, addr_local: u8, dst_local: u8, cast: impl Fn(LOAD) -> TARGET, @@ -1474,7 +1458,8 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { self.push_gc_object(type_addr, values) } - fn exec_struct_get(&mut self, type_index: TypeAddr, field_index: u32, signed: Option) -> Result<(), Trap> { + fn exec_struct_get(&mut self, index: OperandIdx, signed: Option) -> Result<(), Trap> { + let TwoU32 { first: type_index, second: field_index } = index.get(&self.func.data); let reference = ValueRef::stack_pop(&mut self.store.value_stack); let type_addr = self.module.resolve_type_addr(type_index); let storage = self.store.state.get_type(type_addr).as_struct().expect("validated struct.get type").fields @@ -1485,7 +1470,8 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { push_value(&mut self.store.value_stack, value, storage, signed) } - fn exec_struct_set(&mut self, type_index: TypeAddr, field_index: u32) -> Result<(), Trap> { + fn exec_struct_set(&mut self, index: OperandIdx) -> Result<(), Trap> { + let TwoU32 { first: type_index, second: field_index } = index.get(&self.func.data); let type_addr = self.module.resolve_type_addr(type_index); let storage = self.store.state.get_type(type_addr).as_struct().expect("validated struct.set type").fields [field_index as usize] @@ -1508,7 +1494,8 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { self.push_gc_object(type_addr, values) } - fn exec_array_new_fixed(&mut self, type_index: TypeAddr, len: u32) -> Result<(), Trap> { + fn exec_array_new_fixed(&mut self, index: OperandIdx) -> Result<(), Trap> { + let TwoU32 { first: type_index, second: len } = index.get(&self.func.data); let type_addr = self.module.resolve_type_addr(type_index); let storage = self.store.state.get_type(type_addr).as_array().expect("validated array.new_fixed type").field.storage; @@ -1567,7 +1554,8 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { Ok(()) } - fn exec_array_copy(&mut self, dst_type: TypeAddr, src_type: TypeAddr) -> Result<(), Trap> { + fn exec_array_copy(&mut self, index: OperandIdx) -> Result<(), Trap> { + let TwoU32 { first: dst_type, second: src_type } = index.get(&self.func.data); let len = u32::stack_pop(&mut self.store.value_stack) as usize; let src_index = u32::stack_pop(&mut self.store.value_stack) as usize; let src = ValueRef::stack_pop(&mut self.store.value_stack); @@ -1588,7 +1576,8 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { Ok(()) } - fn exec_array_new_data(&mut self, type_index: TypeAddr, data_index: DataAddr) -> Result<(), Trap> { + fn exec_array_new_data(&mut self, index: OperandIdx) -> Result<(), Trap> { + let TwoU32 { first: type_index, second: data_index } = index.get(&self.func.data); let len = u32::stack_pop(&mut self.store.value_stack) as usize; let src = u32::stack_pop(&mut self.store.value_stack) as usize; let type_addr = self.module.resolve_type_addr(type_index); @@ -1599,7 +1588,8 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { self.push_gc_object(type_addr, values) } - fn exec_array_new_elem(&mut self, type_index: TypeAddr, elem_index: ElemAddr) -> Result<(), Trap> { + fn exec_array_new_elem(&mut self, index: OperandIdx) -> Result<(), Trap> { + let TwoU32 { first: type_index, second: elem_index } = index.get(&self.func.data); let len = u32::stack_pop(&mut self.store.value_stack) as usize; let src = u32::stack_pop(&mut self.store.value_stack) as usize; let elem_addr = self.module.resolve_elem_addr(elem_index); @@ -1611,7 +1601,8 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { self.push_gc_object(type_addr, values) } - fn exec_array_init_data(&mut self, type_index: TypeAddr, data_index: DataAddr) -> Result<(), Trap> { + fn exec_array_init_data(&mut self, index: OperandIdx) -> Result<(), Trap> { + let TwoU32 { first: type_index, second: data_index } = index.get(&self.func.data); let len = u32::stack_pop(&mut self.store.value_stack) as usize; let src = u32::stack_pop(&mut self.store.value_stack) as usize; let dst = u32::stack_pop(&mut self.store.value_stack) as usize; @@ -1627,7 +1618,8 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { Ok(()) } - fn exec_array_init_elem(&mut self, type_index: TypeAddr, elem_index: ElemAddr) -> Result<(), Trap> { + fn exec_array_init_elem(&mut self, index: OperandIdx) -> Result<(), Trap> { + let TwoU32 { first: type_index, second: elem_index } = index.get(&self.func.data); let len = u32::stack_pop(&mut self.store.value_stack) as usize; let src = u32::stack_pop(&mut self.store.value_stack) as usize; let dst = u32::stack_pop(&mut self.store.value_stack) as usize; @@ -1669,7 +1661,8 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { Ok(()) } - fn exec_memory_copy(&mut self, dst_mem: u32, src_mem: u32) -> Result<(), Trap> { + fn exec_memory_copy(&mut self, index: OperandIdx) -> Result<(), Trap> { + let TwoU32 { first: dst_mem, second: src_mem } = index.get(&self.func.data); let dst_mem_addr = self.module.resolve_mem_addr(dst_mem); let src_mem_addr = self.module.resolve_mem_addr(src_mem); let dst_arch = self.store.state.get_mem(dst_mem_addr).kind.arch(); @@ -1701,7 +1694,8 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { self.exec_memory_fill_impl(mem_addr, dst, val as u8, size) } - fn exec_memory_fill_imm(&mut self, addr: u32, val: u8, size: i32) -> Result<(), Trap> { + fn exec_memory_fill_const(&mut self, index: OperandIdx) -> Result<(), Trap> { + let MemoryFillConstOp { memory: addr, byte: val, value: size } = index.get(&self.func.data); let mem_addr = self.module.resolve_mem_addr(addr); let arch = self.store.state.get_mem(mem_addr).kind.arch(); let dst = self.store.value_stack.pop_memory_operand(arch)?; @@ -1717,7 +1711,8 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { Ok(()) } - fn exec_memory_init(&mut self, data_index: u32, mem_index: u32) -> Result<(), Trap> { + fn exec_memory_init(&mut self, index: OperandIdx) -> Result<(), Trap> { + let TwoU32 { first: data_index, second: mem_index } = index.get(&self.func.data); let size = u32::stack_pop(&mut self.store.value_stack) as usize; let offset = u32::stack_pop(&mut self.store.value_stack) as usize; let mem_addr = self.module.resolve_mem_addr(mem_index); @@ -1749,7 +1744,8 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { Ok(()) } - fn exec_table_copy(&mut self, dst_table: u32, src_table: u32) -> Result<(), Trap> { + fn exec_table_copy(&mut self, index: OperandIdx) -> Result<(), Trap> { + let TwoU32 { first: dst_table, second: src_table } = index.get(&self.func.data); let dst_table_addr = self.module.resolve_table_addr(dst_table); let src_table_addr = self.module.resolve_table_addr(src_table); let dst_arch = self.store.state.get_table(dst_table_addr).kind.arch(); @@ -1772,15 +1768,14 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { fn exec_mem_load_lane, const LOAD_SIZE: usize>( &mut self, - mem_addr: tinywasm_types::MemAddr, - offset: u64, - lane: u8, + arg: MemoryLaneArg, ) -> Result<(), Trap> { - let mem = self.store.state.get_mem(self.module.resolve_mem_addr(mem_addr)); + let m = arg.memory_arg_idx.get(&self.func.data); + let mem = self.store.state.get_mem(self.module.resolve_mem_addr(m.mem_addr())); let base = self.store.value_stack.pop_memory_operand(mem.kind.arch())?; - let addr = mem.effective_addr::(base, offset)?; - let val = LOAD::load_at(&*mem.inner, addr)?; - let offset = lane as usize * LOAD_SIZE; + let addr = mem.effective_addr::(base, m.offset())?; + let val = cold_err!(LOAD::load_at(&*mem.inner, addr))?; + let offset = arg.lane as usize * LOAD_SIZE; let mut imm = ::stack_pop(&mut self.store.value_stack).to_mem_bytes(); imm[offset..offset + LOAD_SIZE].copy_from_slice(&val.to_mem_bytes()); self.store.value_stack.push(Value128(imm))?; @@ -1790,32 +1785,28 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { #[inline(always)] fn exec_mem_load, const LOAD_SIZE: usize, TARGET: InternalValue>( &mut self, - mem_addr: tinywasm_types::MemAddr, - offset: u64, + index: OperandIdx, cast: impl Fn(LOAD) -> TARGET, ) -> Result<(), Trap> { - let mem = self.store.state.get_mem(self.module.resolve_mem_addr(mem_addr)); + let m = index.get(&self.func.data); + let mem = self.store.state.get_mem(self.module.resolve_mem_addr(m.mem_addr())); let base = self.store.value_stack.pop_memory_operand(mem.kind.arch())?; - let addr = mem.effective_addr::(base, offset)?; + let addr = mem.effective_addr::(base, m.offset())?; let value = cold_err!(LOAD::load_at(&*mem.inner, addr))?; self.store.value_stack.push(cast(value)) } - fn exec_mem_store_lane + Copy, const N: usize>( - &mut self, - mem_addr: tinywasm_types::MemAddr, - offset: u64, - lane: u8, - ) -> Result<(), Trap> { + fn exec_mem_store_lane + Copy, const N: usize>(&mut self, arg: MemoryLaneArg) -> Result<(), Trap> { let bytes = ::stack_pop(&mut self.store.value_stack).to_mem_bytes(); - let lane_offset = lane as usize * N; + let lane_offset = arg.lane as usize * N; let mut val_bytes = [0u8; N]; val_bytes.copy_from_slice(&bytes[lane_offset..lane_offset + N]); let val = U::from_mem_bytes(val_bytes); - let mem_addr = self.module.resolve_mem_addr(mem_addr); + let m = arg.memory_arg_idx.get(&self.func.data); + let mem_addr = self.module.resolve_mem_addr(m.mem_addr()); let mem = self.store.state.get_mem(mem_addr); let base = self.store.value_stack.pop_memory_operand(mem.kind.arch())?; - let addr = mem.effective_addr::(base, offset)?; + let addr = mem.effective_addr::(base, m.offset())?; let mem = self.store.state.get_mem_mut(mem_addr); cold_err!(val.store_at(&mut *mem.inner, addr))?; Ok(()) @@ -1823,17 +1814,17 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { fn exec_mem_store, const N: usize>( &mut self, - mem_addr: tinywasm_types::MemAddr, - offset: u64, + index: OperandIdx, cast: impl Fn(T) -> U, ) -> Result<(), Trap> { let val = ::stack_pop(&mut self.store.value_stack); let val = cast(val); - let mem_addr = self.module.resolve_mem_addr(mem_addr); + let m = index.get(&self.func.data); + let mem_addr = self.module.resolve_mem_addr(m.mem_addr()); let mem = self.store.state.get_mem(mem_addr); let base = self.store.value_stack.pop_memory_operand(mem.kind.arch())?; - let addr = mem.effective_addr::(base, offset)?; + let addr = mem.effective_addr::(base, m.offset())?; let mem = self.store.state.get_mem_mut(mem_addr); cold_err!(val.store_at(&mut *mem.inner, addr))?; Ok(()) @@ -1862,7 +1853,8 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { } } - fn exec_table_init(&mut self, elem_index: u32, table_index: u32) -> Result<(), Trap> { + fn exec_table_init(&mut self, index: OperandIdx) -> Result<(), Trap> { + let TwoU32 { first: elem_index, second: table_index } = index.get(&self.func.data); let size = self.pop_table_operand(MemoryArch::I32)?; // n let offset = self.pop_table_operand(MemoryArch::I32)?; // s let table_addr = self.module.resolve_table_addr(table_index); diff --git a/crates/tinywasm/src/interpreter/stack/value_stack.rs b/crates/tinywasm/src/interpreter/stack/value_stack.rs index 43c214a..f2c0c28 100644 --- a/crates/tinywasm/src/interpreter/stack/value_stack.rs +++ b/crates/tinywasm/src/interpreter/stack/value_stack.rs @@ -109,6 +109,17 @@ impl Stack { Ok(()) } + #[inline(always)] + pub(crate) fn push_copy(&mut self, index: usize) -> Result<(), Trap> { + if !self.ensure_capacity_for(self.data.len() + 1) { + return cold!(Err(Trap::ValueStackOverflow)); + } + + let value = self.data[index]; + self.data.push(value); + Ok(()) + } + #[inline(always)] pub(crate) fn pop(&mut self) -> T { self.data.pop().unwrap_or_else(|| unreachable!("ValueStack underflow, this is a bug")) diff --git a/crates/tinywasm/src/interpreter/values.rs b/crates/tinywasm/src/interpreter/values.rs index e2cbef0..4fb81c9 100644 --- a/crates/tinywasm/src/interpreter/values.rs +++ b/crates/tinywasm/src/interpreter/values.rs @@ -137,6 +137,7 @@ pub(crate) trait InternalValue: sealed::Sealed + Copy + Default { fn stack_peek(stack: &ValueStack) -> Self; fn stack_select(stack: &mut ValueStack); fn local_get(stack: &ValueStack, frame: &CallFrame, index: LocalAddr) -> Self; + fn local_push(stack: &mut ValueStack, frame: &CallFrame, index: LocalAddr) -> Result<(), crate::Trap>; fn local_set(stack: &mut ValueStack, frame: &CallFrame, index: LocalAddr, value: Self); fn local_update(stack: &mut ValueStack, frame: &CallFrame, index: LocalAddr, f: impl FnOnce(Self) -> Self); fn local_copy(stack: &mut ValueStack, frame: &CallFrame, from: LocalAddr, to: LocalAddr); @@ -178,6 +179,11 @@ macro_rules! impl_internalvalue { $from_stack } + #[inline(always)] + fn local_push(stack: &mut ValueStack, frame: &CallFrame, index: LocalAddr) -> Result<(), crate::Trap> { + stack.$stack.push_copy(frame.locals_base.$stack_base as usize + index as usize) + } + #[inline(always)] fn local_set(stack: &mut ValueStack, frame: &CallFrame, index: LocalAddr, value: Self) { let $to_stack_v = value; diff --git a/crates/tinywasm/tests/wasm-custom/conditional-rewrite.wast b/crates/tinywasm/tests/wasm-custom/conditional-rewrite.wast new file mode 100644 index 0000000..bc92b30 --- /dev/null +++ b/crates/tinywasm/tests/wasm-custom/conditional-rewrite.wast @@ -0,0 +1,35 @@ +(module + (func (export "retained-value-before-update-branch") (param i32 i32) (result i32) + (block (result i32) + local.get 0 + i32.eqz + local.get 1 + i32.const 1 + i32.add + local.tee 1 + br_if 0 + ) + drop + local.get 1 + ) + + (func (export "compare-eqz-br-if") (param i32 i32) (result i32) + block + i32.const 1 + local.get 0 + local.get 1 + i32.lt_u + i32.eqz + br_if 0 + drop + i32.const 2 + return + end + i32.const 3 + ) +) + +(assert_return (invoke "retained-value-before-update-branch" (i32.const 0) (i32.const 0)) (i32.const 1)) +(assert_return (invoke "retained-value-before-update-branch" (i32.const 1) (i32.const -1)) (i32.const 0)) +(assert_return (invoke "compare-eqz-br-if" (i32.const 1) (i32.const 2)) (i32.const 2)) +(assert_return (invoke "compare-eqz-br-if" (i32.const 2) (i32.const 1)) (i32.const 3)) diff --git a/crates/tinywasm/tests/wasm-custom/sign-extension-rewrite.wast b/crates/tinywasm/tests/wasm-custom/sign-extension-rewrite.wast new file mode 100644 index 0000000..147858a --- /dev/null +++ b/crates/tinywasm/tests/wasm-custom/sign-extension-rewrite.wast @@ -0,0 +1,59 @@ +(module + (func (export "i32-shift-8") (param i32) (result i32) + local.get 0 + i32.const 8 + i32.shl + i32.const 8 + i32.shr_s + ) + (func (export "i32-extend-8") (param i32) (result i32) + local.get 0 + i32.const 24 + i32.shl + i32.const 24 + i32.shr_s + ) + (func (export "i32-extend-16") (param i32) (result i32) + local.get 0 + i32.const 16 + i32.shl + i32.const 16 + i32.shr_s + ) + (func (export "i64-shift-16") (param i64) (result i64) + local.get 0 + i64.const 16 + i64.shl + i64.const 16 + i64.shr_s + ) + (func (export "i64-extend-8") (param i64) (result i64) + local.get 0 + i64.const 56 + i64.shl + i64.const 56 + i64.shr_s + ) + (func (export "i64-extend-16") (param i64) (result i64) + local.get 0 + i64.const 48 + i64.shl + i64.const 48 + i64.shr_s + ) + (func (export "i64-extend-32") (param i64) (result i64) + local.get 0 + i64.const 32 + i64.shl + i64.const 32 + i64.shr_s + ) +) + +(assert_return (invoke "i32-shift-8" (i32.const 0x80)) (i32.const 0x80)) +(assert_return (invoke "i32-extend-8" (i32.const 0x80)) (i32.const -128)) +(assert_return (invoke "i32-extend-16" (i32.const 0x8000)) (i32.const -32768)) +(assert_return (invoke "i64-shift-16" (i64.const 0x80)) (i64.const 0x80)) +(assert_return (invoke "i64-extend-8" (i64.const 0x80)) (i64.const -128)) +(assert_return (invoke "i64-extend-16" (i64.const 0x8000)) (i64.const -32768)) +(assert_return (invoke "i64-extend-32" (i64.const 0x80000000)) (i64.const -2147483648)) diff --git a/crates/tinywasm/tests/wasm-custom/simd-stack-local.wast b/crates/tinywasm/tests/wasm-custom/simd-stack-local.wast new file mode 100644 index 0000000..0dabf4e --- /dev/null +++ b/crates/tinywasm/tests/wasm-custom/simd-stack-local.wast @@ -0,0 +1,53 @@ +(module + (func (export "and") (param v128 v128) (result v128) + local.get 0 + local.get 1 + v128.and + ) + (func (export "or") (param v128 v128) (result v128) + local.get 0 + local.get 1 + v128.or + ) + (func (export "add") (param v128 v128) (result v128) + local.get 0 + local.get 1 + i64x2.add + ) + (func (export "mul") (param v128 v128) (result v128) + local.get 0 + local.get 1 + i64x2.mul + ) + (func (export "deep-xor") (param v128 v128 v128) (result v128 v128) + local.get 0 + local.get 1 + local.get 2 + v128.xor + ) +) + +(assert_return + (invoke "and" (v128.const i32x4 -1 0 -1 0) (v128.const i32x4 1 2 3 4)) + (v128.const i32x4 1 0 3 0) +) +(assert_return + (invoke "or" (v128.const i32x4 1 0 3 0) (v128.const i32x4 0 2 0 4)) + (v128.const i32x4 1 2 3 4) +) +(assert_return + (invoke "add" (v128.const i64x2 1 2) (v128.const i64x2 3 4)) + (v128.const i64x2 4 6) +) +(assert_return + (invoke "mul" (v128.const i64x2 2 3) (v128.const i64x2 4 5)) + (v128.const i64x2 8 15) +) +(assert_return + (invoke "deep-xor" + (v128.const i32x4 1 2 3 4) + (v128.const i32x4 8 4 2 1) + (v128.const i32x4 1 1 1 1)) + (v128.const i32x4 1 2 3 4) + (v128.const i32x4 9 5 3 0) +) diff --git a/crates/types/src/archive.rs b/crates/types/src/archive.rs index 9904a7d..db40263 100644 --- a/crates/types/src/archive.rs +++ b/crates/types/src/archive.rs @@ -64,6 +64,9 @@ impl Module { #[cfg(test)] mod tests { use super::*; + use crate::{Instruction, ModuleInner, V128Operand, WasmFunction}; + use crate::{OperandIdx, OperandType}; + use alloc::{boxed::Box, sync::Arc, vec}; #[test] fn test_invalid_magic() { @@ -80,4 +83,27 @@ mod tests { twasm[4] = 0; assert!(matches!(Module::try_from_twasm(&twasm), Err(TwasmError::InvalidVersion))); } + + #[test] + fn v128_operands_round_trip_archive() { + let bytes = [0x00, 0x01, 0x02, 0x03, 0x7f, 0x80, 0xfe, 0xff, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x90]; + let mut function = WasmFunction::default(); + let constant = OperandIdx::new(0); + function.data.operands128 = vec![V128Operand { value: bytes }.encode()].into_boxed_slice(); + function.instructions = vec![Instruction::Const128(constant), Instruction::I8x16Shuffle(constant)].into(); + let module = Module::from(ModuleInner { funcs: Box::new([Arc::new(function)]), ..ModuleInner::default() }); + + let archive = module.serialize_twasm().expect("serialize archive"); + assert_eq!(&archive[..6], b"TWAS05"); + let decoded = Module::try_from_twasm(&archive).expect("deserialize archive"); + let function = &decoded.funcs[0]; + + for instruction in function.instructions.iter() { + let index = match instruction { + Instruction::Const128(index) | Instruction::I8x16Shuffle(index) => *index, + _ => panic!("unexpected instruction"), + }; + assert_eq!(index.get(&function.data).value, bytes); + } + } } diff --git a/crates/types/src/instructions.rs b/crates/types/src/instructions.rs index 10ab028..0773d3e 100644 --- a/crates/types/src/instructions.rs +++ b/crates/types/src/instructions.rs @@ -1,7 +1,8 @@ use alloc::boxed::Box; use super::{FuncAddr, GlobalAddr, LocalAddr, TableAddr, TagAddr, TypeAddr, ValueCounts}; -use crate::{ConstIdx, DataAddr, ElemAddr, MemAddr, RefType, RefValue}; +use crate::operands::sealed; +use crate::{DataAddr, ElemAddr, MemAddr, Operand64, Operand128, OperandIdx, OperandType, RefType, RefValue}; /// Represents a memory immediate in a WebAssembly memory instruction. #[derive(Copy, Clone, PartialEq, Eq)] @@ -13,13 +14,285 @@ pub struct MemoryArg { mem_addr: MemAddr, } -/// Stack lanes discarded and retained when branching to a control frame. -#[derive(Clone, Copy, PartialEq, Eq)] +/// A compact memory immediate used by optimized instructions. +/// +/// Optimized instructions use this representation when the offset and +/// module-local memory index fit its narrower fields. +#[derive(Copy, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "debug", derive(Debug))] +#[cfg_attr(feature = "archive", derive(serde::Serialize, serde::Deserialize))] +#[repr(Rust, packed)] +pub struct CompactMemoryArg { + offset: u32, + mem_addr: u16, +} + +/// An indexed memory argument and up to two local operands. +#[derive(Copy, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "debug", derive(Debug))] +#[cfg_attr(feature = "archive", derive(serde::Serialize, serde::Deserialize))] +#[repr(Rust, packed)] +pub struct MemoryLocalArg { + pub memory_arg_idx: OperandIdx, + pub local1: u8, + pub local2: u8, +} + +/// An indexed memory argument and SIMD lane. +#[derive(Copy, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "debug", derive(Debug))] +#[cfg_attr(feature = "archive", derive(serde::Serialize, serde::Deserialize))] +#[repr(Rust, packed)] +pub struct MemoryLaneArg { + pub memory_arg_idx: OperandIdx, + pub lane: u8, +} + +/// A 32-bit immediate and local operand that fit in an instruction payload. +#[derive(Copy, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "debug", derive(Debug))] +#[cfg_attr(feature = "archive", derive(serde::Serialize, serde::Deserialize))] +#[repr(Rust, packed)] +pub struct I32LocalArg { + pub value: i32, + pub local: LocalAddr, +} + +/// Three local operands that fit in an instruction payload. +#[derive(Copy, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "debug", derive(Debug))] +#[cfg_attr(feature = "archive", derive(serde::Serialize, serde::Deserialize))] +pub struct LocalTripleArg { + pub left: LocalAddr, + pub right: LocalAddr, + pub dst: LocalAddr, +} + +impl sealed::Sealed for LocalTripleArg {} + +impl OperandType for LocalTripleArg { + type Raw = Operand64; + + #[inline(always)] + fn decode(raw: Self::Raw) -> Self { + Self { left: raw.u16(0), right: raw.u16(2), dst: raw.u16(4) } + } + + #[inline] + fn encode(self) -> Self::Raw { + Operand64::default().with_u16(0, self.left).with_u16(2, self.right).with_u16(4, self.dst) + } +} + +/// An indexed SIMD value and local operand that fit in an instruction payload. +#[derive(Copy, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "debug", derive(Debug))] +#[cfg_attr(feature = "archive", derive(serde::Serialize, serde::Deserialize))] +#[repr(Rust, packed)] +pub struct V128LocalArg { + pub value: OperandIdx, + pub local: LocalAddr, +} + +/// A branch target and local operand that fit in an instruction payload. +#[derive(Copy, Clone, PartialEq, Eq)] #[cfg_attr(feature = "debug", derive(Debug))] #[cfg_attr(feature = "archive", derive(serde::Serialize, serde::Deserialize))] -pub struct DropKeep { - pub base: ValueCounts, - pub keep: ValueCounts, +#[repr(Rust, packed)] +pub struct TargetLocalArg { + pub target_ip: u32, + pub local: LocalAddr, +} + +macro_rules! operand_view { + ($name:ident, $lane:ident, $raw:ty, { $($field:ident : $ty:ty = $read:ident / $write:ident ($offset:expr)),+ $(,)? }) => { + #[derive(Copy, Clone, PartialEq, Eq)] + #[cfg_attr(feature = "debug", derive(Debug))] + pub struct $name { + $(pub $field: $ty),+ + } + + impl sealed::Sealed for $name {} + + impl OperandType for $name { + type Raw = $raw; + + #[inline(always)] + fn decode(raw: Self::Raw) -> Self { + Self { $($field: raw.$read($offset) as $ty),+ } + } + + #[inline] + fn encode(self) -> Self::Raw { + <$raw>::default()$(.$write($offset, self.$field as _))+ + } + } + }; +} + +operand_view!(I64Operand, operands64, Operand64, { value: i64 = i64 / with_u64(0) }); +operand_view!(TwoU32, operands64, Operand64, { + first: u32 = u32 / with_u32(0), second: u32 = u32 / with_u32(4) +}); +operand_view!(LocalU32, operands64, Operand64, { + local: u16 = u16 / with_u16(0), value: u32 = u32 / with_u32(2) +}); +operand_view!(TargetLocal, operands64, Operand64, { + target: u32 = u32 / with_u32(0), local: u16 = u16 / with_u16(4) +}); + +operand_view!(I64Local, operands128, Operand128, { + value: i64 = i64 / with_u64(0), local: u16 = u16 / with_u16(8) +}); +operand_view!(LocalConst64, operands128, Operand128, { + local: u16 = u16 / with_u16(0), value: u64 = u64 / with_u64(2) +}); +operand_view!(GlobalConst32, operands128, Operand128, { + global: u32 = u32 / with_u32(0), value: u32 = u32 / with_u32(4) +}); +operand_view!(GlobalConst64, operands128, Operand128, { + global: u32 = u32 / with_u32(0), value: u64 = u64 / with_u64(4) +}); +operand_view!(LocalConstSet32, operands128, Operand128, { + local: u16 = u16 / with_u16(0), dst: u16 = u16 / with_u16(2), value: u32 = u32 / with_u32(4) +}); +operand_view!(LocalConstSet64, operands128, Operand128, { + local: u16 = u16 / with_u16(0), dst: u16 = u16 / with_u16(2), value: u64 = u64 / with_u64(4) +}); +operand_view!(MemoryFillConstOp, operands128, Operand128, { + memory: u32 = u32 / with_u32(0), byte: u8 = u8 / with_u8(4), value: i32 = i32 / with_u32(5) +}); +operand_view!(CastBranch, operands128, Operand128, { + target: u32 = u32 / with_u32(0), ref_type_bits: u32 = u32 / with_u32(4), branch_on_fail: u8 = u8 / with_u8(8) +}); +operand_view!(BranchTableArg, operands128, Operand128, { + target: u32 = u32 / with_u32(0), start: u32 = u32 / with_u32(4), len: u32 = u32 / with_u32(8) +}); +operand_view!(StackConst32, operands128, Operand128, { + target: u32 = u32 / with_u32(0), value: i32 = i32 / with_u32(4) +}); +operand_view!(StackConst64, operands128, Operand128, { + target: u32 = u32 / with_u32(0), value: i64 = i64 / with_u64(4) +}); +operand_view!(LocalConstCmp, operands128, Operand128, { + target: u32 = u32 / with_u32(0), value: i32 = i32 / with_u32(4), local: u16 = u16 / with_u16(8) +}); +operand_view!(LocalLocalCmp, operands128, Operand128, { + target: u32 = u32 / with_u32(0), left: u16 = u16 / with_u16(4), right: u16 = u16 / with_u16(6) +}); +operand_view!(LocalUpdate, operands128, Operand128, { + target: u32 = u32 / with_u32(0), value: i32 = i32 / with_u32(4), local: u16 = u16 / with_u16(8), on_zero: u8 = u8 / with_u8(10) +}); +operand_view!(GlobalUpdate, operands128, Operand128, { + target: u32 = u32 / with_u32(0), value: i32 = i32 / with_u32(4), global: u32 = u32 / with_u32(8), on_zero: u8 = u8 / with_u8(12) +}); +operand_view!(LocalUpdateCmp, operands128, Operand128, { + target: u32 = u32 / with_u32(0), value: i32 = i32 / with_u32(4), local: u16 = u16 / with_u16(8), right: u16 = u16 / with_u16(10) +}); + +/// An operation packed inline with an indexed operand. +#[derive(Copy, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "debug", derive(Debug))] +#[cfg_attr(feature = "archive", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "archive", + serde(bound(serialize = "Op: Copy + serde::Serialize", deserialize = "Op: Copy + serde::de::DeserializeOwned",)) +)] +#[repr(Rust, packed)] +pub struct PackedOp { + pub op: Op, + pub index: OperandIdx, +} + +impl PackedOp { + /// Creates a packed operation from an operator and operand index. + #[inline] + pub const fn new(op: Op, index: OperandIdx) -> Self { + Self { op, index } + } +} + +/// A SIMD value stored in the 128-bit operand lane. +#[derive(Copy, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "debug", derive(Debug))] +pub struct V128Operand { + pub value: [u8; 16], +} + +/// A local SIMD operation with an indexed constant. +#[derive(Copy, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "debug", derive(Debug))] +pub struct LocalV128 { + pub local: u16, + pub value: OperandIdx, +} + +/// A global SIMD operation with an indexed constant. +#[derive(Copy, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "debug", derive(Debug))] +pub struct GlobalV128 { + pub global: u32, + pub value: OperandIdx, +} + +/// A local SIMD operation, indexed constant, and destination local. +#[derive(Copy, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "debug", derive(Debug))] +pub struct LocalConstSetV128 { + pub local: u16, + pub dst: u16, + pub value: OperandIdx, +} + +macro_rules! operand128_view { + ($name:ident, $decode:expr, $encode:expr) => { + impl sealed::Sealed for $name {} + + impl OperandType for $name { + type Raw = Operand128; + + #[inline(always)] + fn decode(raw: Self::Raw) -> Self { + $decode(raw) + } + + #[inline] + fn encode(self) -> Self::Raw { + $encode(self) + } + } + }; +} + +operand128_view!(V128Operand, |raw: Operand128| Self { value: raw.to_le_bytes() }, |value: Self| { + Operand128::from_le_bytes(value.value) +}); +operand128_view!( + LocalV128, + |raw: Operand128| Self { local: raw.u16(0), value: OperandIdx::new(raw.u32(2)) }, + |value: Self| { Operand128::default().with_u16(0, value.local).with_u32(2, value.value.index()) } +); +operand128_view!( + GlobalV128, + |raw: Operand128| Self { global: raw.u32(0), value: OperandIdx::new(raw.u32(4)) }, + |value: Self| { Operand128::default().with_u32(0, value.global).with_u32(4, value.value.index()) } +); +operand128_view!( + LocalConstSetV128, + |raw: Operand128| Self { local: raw.u16(0), dst: raw.u16(2), value: OperandIdx::new(raw.u32(4)) }, + |value: Self| { + Operand128::default().with_u16(0, value.local).with_u16(2, value.dst).with_u32(4, value.value.index()) + } +); + +impl CastBranch { + #[inline] + pub const fn ref_type(self) -> RefType { + match RefType::from_bits(self.ref_type_bits) { + Some(value) => value, + None => unreachable!(), + } + } } /// A catch clause attached to a lowered `try_table` instruction. @@ -62,12 +335,6 @@ pub struct ExceptionHandler { pub catches: Box<[ExceptionCatch]>, } -impl From<(ValueCounts, ValueCounts)> for DropKeep { - fn from((base, keep): (ValueCounts, ValueCounts)) -> Self { - Self { base, keep } - } -} - impl MemoryArg { #[inline] pub const fn new(offset: u64, mem_addr: MemAddr) -> Self { @@ -85,6 +352,81 @@ impl MemoryArg { } } +impl sealed::Sealed for MemoryArg {} + +impl OperandType for MemoryArg { + type Raw = Operand128; + + #[inline(always)] + fn decode(raw: Self::Raw) -> Self { + Self::new(raw.u64(0), raw.u32(8)) + } + + #[inline] + fn encode(self) -> Self::Raw { + Operand128::default().with_u64(0, self.offset).with_u32(8, self.mem_addr) + } +} + +impl CompactMemoryArg { + /// Returns the static byte offset. + #[inline] + pub const fn offset(self) -> u64 { + self.offset as u64 + } + + /// Returns the module-local memory index. + #[inline] + pub const fn mem_addr(self) -> MemAddr { + self.mem_addr as MemAddr + } +} + +impl sealed::Sealed for CompactMemoryArg {} + +impl OperandType for CompactMemoryArg { + type Raw = Operand64; + + #[inline(always)] + fn decode(raw: Self::Raw) -> Self { + Self { offset: raw.u32(0), mem_addr: raw.u16(4) } + } + + #[inline] + fn encode(self) -> Self::Raw { + Operand64::default().with_u32(0, self.offset).with_u16(4, self.mem_addr) + } +} + +impl TryFrom for CompactMemoryArg { + type Error = (); + + #[inline] + fn try_from(arg: MemoryArg) -> Result { + Ok(Self { + offset: u32::try_from(arg.offset()).map_err(|_| ())?, + mem_addr: u16::try_from(arg.mem_addr()).map_err(|_| ())?, + }) + } +} + +impl From for MemoryArg { + #[inline] + fn from(arg: CompactMemoryArg) -> Self { + Self::new(arg.offset(), arg.mem_addr()) + } +} + +const _: () = { + assert!(core::mem::size_of::>() == 4); + assert!(core::mem::size_of::() == 8); + assert!(core::mem::size_of::() == 16); + assert!(core::mem::size_of::() == 6); + assert!(core::mem::size_of::() == 6); + assert!(core::mem::size_of::() == 6); + assert!(core::mem::size_of::() == 6); +}; + #[derive(Clone, Copy, PartialEq)] #[cfg_attr(feature = "debug", derive(Debug))] #[cfg_attr(feature = "archive", derive(serde::Serialize, serde::Deserialize))] @@ -132,6 +474,25 @@ pub enum CmpOp { GeU, } +impl CmpOp { + /// Returns the comparison that is true exactly when `self` is false. + #[inline] + pub const fn inverse(self) -> Self { + match self { + Self::Eq => Self::Ne, + Self::Ne => Self::Eq, + Self::LtS => Self::GeS, + Self::LtU => Self::GeU, + Self::GtS => Self::LeS, + Self::GtU => Self::LeU, + Self::LeS => Self::GtS, + Self::LeU => Self::GtU, + Self::GeS => Self::LtS, + Self::GeU => Self::LtU, + } + } +} + #[derive(Clone, Copy, PartialEq, Eq)] #[cfg_attr(feature = "debug", derive(Debug))] #[cfg_attr(feature = "archive", derive(serde::Serialize, serde::Deserialize))] @@ -156,6 +517,18 @@ pub enum BinOp { FCopysign, } +impl BinOp { + /// Returns the update delta for `local OP immediate`, when it is a simple increment. + #[inline] + pub const fn inc_delta(self, immediate: i32) -> Option { + match self { + Self::IAdd => Some(immediate), + Self::ISub => Some(immediate.wrapping_neg()), + _ => None, + } + } +} + #[derive(Clone, Copy, PartialEq, Eq)] #[cfg_attr(feature = "debug", derive(Debug))] #[cfg_attr(feature = "archive", derive(serde::Serialize, serde::Deserialize))] @@ -181,69 +554,47 @@ pub enum BinOp128 { #[cfg_attr(feature = "archive", derive(serde::Serialize, serde::Deserialize))] pub enum Instruction { LocalCopy32(LocalAddr, LocalAddr), LocalCopy64(LocalAddr, LocalAddr), LocalCopy128(LocalAddr, LocalAddr), - AddConst32(i32), AddConst64(i64), - IncLocal32(LocalAddr, i32), IncLocal64(LocalAddr, i64), + AddConst32(i32), AddConst64(OperandIdx), + IncLocal32(I32LocalArg), IncLocal64(OperandIdx), // The 32/64 suffix describes the operand width. Future compare-style ops may still yield i32 results. BinOpLocalLocal32(BinOp, LocalAddr, LocalAddr), BinOpLocalLocal64(BinOp, LocalAddr, LocalAddr), BinOpLocalLocal128(BinOp128, LocalAddr, LocalAddr), CmpLocalLocal32(CmpOp, LocalAddr, LocalAddr), CmpLocalLocal64(CmpOp, LocalAddr, LocalAddr), - BinOpLocalLocalSet32(BinOp, LocalAddr, LocalAddr, LocalAddr), - BinOpLocalLocalSet64(BinOp, LocalAddr, LocalAddr, LocalAddr), - BinOpLocalLocalSet128(BinOp128, LocalAddr, LocalAddr, LocalAddr), - BinOpLocalLocalTee32(BinOp, LocalAddr, LocalAddr, LocalAddr), - BinOpLocalLocalTee64(BinOp, LocalAddr, LocalAddr, LocalAddr), - BinOpLocalLocalTee128(BinOp128, LocalAddr, LocalAddr, LocalAddr), - BinOpLocalConst32(BinOp, LocalAddr, i32), BinOpLocalConst64(BinOp, LocalAddr, i64), - BinOpGlobalConst32(BinOp, GlobalAddr, i32), BinOpGlobalConst64(BinOp, GlobalAddr, i64), - BinOpGlobalConst128(BinOp128, GlobalAddr, ConstIdx), - BinOpLocalConst128(BinOp128, LocalAddr, ConstIdx), - BinOpLocalConstSet32(BinOp, LocalAddr, i32, LocalAddr), - BinOpLocalConstSet64(BinOp, LocalAddr, i64, LocalAddr), - BinOpLocalConstSet128(BinOp128, LocalAddr, ConstIdx, LocalAddr), - BinOpLocalConstTee32(BinOp, LocalAddr, i32, LocalAddr), - BinOpLocalConstTee64(BinOp, LocalAddr, i64, LocalAddr), - BinOpLocalConstTee128(BinOp128, LocalAddr, ConstIdx, LocalAddr), + AddLocalLocalSet32(LocalTripleArg), AddLocalLocalTee32(LocalTripleArg), + BinOpLocalLocalSet32(PackedOp), BinOpLocalLocalSet64(PackedOp), BinOpLocalLocalSet128(PackedOp), + BinOpLocalLocalTee32(PackedOp), BinOpLocalLocalTee64(PackedOp), BinOpLocalLocalTee128(PackedOp), + AddLocalConst32(I32LocalArg), SubLocalConst32(I32LocalArg), MulLocalConst32(I32LocalArg), + BinOpLocalConst32(PackedOp), BinOpLocalConst64(PackedOp), + BinOpGlobalConst32(PackedOp), BinOpGlobalConst64(PackedOp), + BinOpGlobalConst128(PackedOp), BinOpLocalConst128(PackedOp), + BinOpLocalConstSet32(PackedOp), BinOpLocalConstSet64(PackedOp), BinOpLocalConstSet128(PackedOp), + BinOpLocalConstTee32(PackedOp), BinOpLocalConstTee64(PackedOp), BinOpLocalConstTee128(PackedOp), BinOpStackLocal32(BinOp, LocalAddr), BinOpStackLocalSet32(BinOp, LocalAddr, LocalAddr), BinOpStackLocalTee32(BinOp, LocalAddr, LocalAddr), + BinOpStackLocal128(BinOp128, LocalAddr), BinOpStackGlobal32(BinOp, u32), BinOpStackGlobal64(BinOp, u32), - SetLocalConst32(LocalAddr, i32), SetLocalConst64(LocalAddr, i64), SetLocalConst128(LocalAddr, ConstIdx), - IncMemoryLocal32(MemoryArg, u8), - IncMemoryLocal64(MemoryArg, u8), - StoreLocalLocal32(MemoryArg, u8, u8), - StoreLocalLocal64(MemoryArg, u8, u8), - StoreLocalLocal128(MemoryArg, u8, u8), - LoadLocal32(MemoryArg, u8), - LoadLocal64(MemoryArg, u8), - LoadLocal8S32(MemoryArg, u8), - LoadLocal8U32(MemoryArg, u8), - LoadLocal16S32(MemoryArg, u8), - LoadLocal16U32(MemoryArg, u8), - LoadLocalTee32(MemoryArg, u8, u8), - LoadLocalSet32(MemoryArg, u8, u8), - LoadLocalTee8S32(MemoryArg, u8, u8), - LoadLocalTee8U32(MemoryArg, u8, u8), - LoadLocalTee16S32(MemoryArg, u8, u8), - LoadLocalTee16U32(MemoryArg, u8, u8), - LoadLocalSet8S32(MemoryArg, u8, u8), - LoadLocalSet8U32(MemoryArg, u8, u8), - LoadLocalSet16S32(MemoryArg, u8, u8), - LoadLocalSet16U32(MemoryArg, u8, u8), - LoadLocalTee128(MemoryArg, u8, u8), - LoadLocalSet128(MemoryArg, u8, u8), - AndConstTee32(i32, LocalAddr), - SubConstTee32(i32, LocalAddr), - AndConstTee64(i64, LocalAddr), - SubConstTee64(i64, LocalAddr), - MulAccLocal32(LocalAddr), - MulAccLocal64(LocalAddr), - FMulAccLocal32(LocalAddr), - FMulAccLocal64(LocalAddr), + SetLocalConst32(I32LocalArg), SetLocalConst64(OperandIdx), SetLocalConst128(V128LocalArg), + IncMemoryLocal32(MemoryLocalArg), IncMemoryLocal64(MemoryLocalArg), + StoreLocalLocal32(MemoryLocalArg), StoreLocalLocal64(MemoryLocalArg), StoreLocalLocal128(MemoryLocalArg), + LoadLocal32(MemoryLocalArg), LoadLocal64(MemoryLocalArg), + LoadLocal8S32(MemoryLocalArg), LoadLocal8U32(MemoryLocalArg), + LoadLocal16S32(MemoryLocalArg), LoadLocal16U32(MemoryLocalArg), + LoadLocalTee32(MemoryLocalArg), LoadLocalSet32(MemoryLocalArg), + LoadLocalTee8S32(MemoryLocalArg), LoadLocalTee8U32(MemoryLocalArg), + LoadLocalTee16S32(MemoryLocalArg), LoadLocalTee16U32(MemoryLocalArg), + LoadLocalSet8S32(MemoryLocalArg), LoadLocalSet8U32(MemoryLocalArg), + LoadLocalSet16S32(MemoryLocalArg), LoadLocalSet16U32(MemoryLocalArg), + LoadLocalTee128(MemoryLocalArg), LoadLocalSet128(MemoryLocalArg), + AndConstTee32(I32LocalArg), SubConstTee32(I32LocalArg), + AndConstTee64(OperandIdx), SubConstTee64(OperandIdx), + MulAccLocal32(LocalAddr), MulAccLocal64(LocalAddr), + FMulAccLocal32(LocalAddr), FMulAccLocal64(LocalAddr), I32Add3, I64Add3, - FMaStoreF32(MemoryArg), - FMaStoreF64(MemoryArg), + FMaStoreF32(CompactMemoryArg), + FMaStoreF64(CompactMemoryArg), // > Control Instructions (jump-oriented, lowered from structured control during parsing) // See @@ -255,28 +606,16 @@ pub enum Instruction { JumpIfNonZero64(u32), JumpIfRefNull(u32), JumpIfRefNonNull(u32), - JumpIfLocalZero32 { target_ip: u32, local: LocalAddr }, - JumpIfLocalNonZero32 { target_ip: u32, local: LocalAddr }, - JumpIfLocalZero64 { target_ip: u32, local: LocalAddr }, - JumpIfLocalNonZero64 { target_ip: u32, local: LocalAddr }, - JumpCmpStackConst32 { target_ip: u32, imm: i32, op: CmpOp }, - JumpCmpStackConst64 { target_ip: u32, imm: i64, op: CmpOp }, - JumpCmpStackLocal32 { target_ip: u32, local: LocalAddr, op: CmpOp }, - JumpCmpStackLocal64 { target_ip: u32, local: LocalAddr, op: CmpOp }, - BinOpLocalConstJump32 { target_ip: u32, local: LocalAddr, imm: i32, op: BinOp, on_zero: bool }, - BinOpLocalConstJumpCmpLocal32 { target_ip: u32, local: LocalAddr, imm: i32, binop: BinOp, right: LocalAddr, cmp: CmpOp }, - BinOpStackConstTeeLocalJump32 { target_ip: u32, local: LocalAddr, imm: i32, op: BinOp, on_zero: bool }, - BinOpGlobalConstJump32 { target_ip: u32, global: GlobalAddr, imm: i32, op: BinOp, on_zero: bool }, - IncLocalJump32 { target_ip: u32, local: LocalAddr, delta: i32, on_zero: bool }, - IncStackTeeLocalJump32 { target_ip: u32, local: LocalAddr, delta: i32, on_zero: bool }, - IncGlobalJump32 { target_ip: u32, global: GlobalAddr, delta: i32, on_zero: bool }, - IncLocalJumpCmpLocal32 { target_ip: u32, local: LocalAddr, delta: i32, right: LocalAddr, op: CmpOp }, - JumpCmpLocalConst32 { target_ip: u32, local: LocalAddr, imm: i32, op: CmpOp }, - JumpCmpLocalConst64 { target_ip: u32, local: LocalAddr, imm: i32, op: CmpOp }, - JumpCmpLocalLocal32 { target_ip: u32, left: LocalAddr, right: LocalAddr, op: CmpOp }, - JumpCmpLocalLocal64 { target_ip: u32, left: LocalAddr, right: LocalAddr, op: CmpOp }, - DropKeep(DropKeep), - BranchTable(u32, u32, u32), // (default_landing_pad_ip, branch_table_start, target_count) + JumpIfLocalZero32(TargetLocalArg), JumpIfLocalNonZero32(TargetLocalArg), + JumpIfLocalZero64(TargetLocalArg), JumpIfLocalNonZero64(TargetLocalArg), + JumpCmpStackConst32(PackedOp), JumpCmpStackConst64(PackedOp), + JumpCmpStackLocal32(PackedOp), JumpCmpStackLocal64(PackedOp), + BinOpLocalConstJump32(PackedOp), BinOpLocalConstJumpCmpLocal32(PackedOp<(BinOp, CmpOp), LocalUpdateCmp>), + BinOpStackConstTeeLocalJump32(PackedOp), BinOpGlobalConstJump32(PackedOp), + IncLocalJump32(OperandIdx), IncStackTeeLocalJump32(OperandIdx), IncGlobalJump32(OperandIdx), + IncLocalJumpCmpLocal32(PackedOp), JumpCmpLocalConst32(PackedOp), JumpCmpLocalConst64(PackedOp), + JumpCmpLocalLocal32(PackedOp), JumpCmpLocalLocal64(PackedOp), + DropKeep32 { base: u16, keep: u16 }, DropKeep64 { base: u16, keep: u16 }, DropKeep128 { base: u16, keep: u16 }, BranchTable(OperandIdx), Return, ReturnVoid, Return32, @@ -284,11 +623,11 @@ pub enum Instruction { Return128, Call(FuncAddr), CallSelf, - CallIndirect(TypeAddr, TableAddr), + CallIndirect(OperandIdx), CallRef(TypeAddr), ReturnCall(FuncAddr), ReturnCallSelf, - ReturnCallIndirect(TypeAddr, TableAddr), + ReturnCallIndirect(OperandIdx), ReturnCallRef(TypeAddr), Throw(TagAddr), ThrowRef, @@ -307,35 +646,18 @@ pub enum Instruction { GlobalGet128(GlobalAddr), GlobalSet128(GlobalAddr), GlobalTee128(GlobalAddr), LocalGet128(LocalAddr), LocalSet128(LocalAddr), LocalTee128(LocalAddr), // > Memory Instructions - I32Load(MemoryArg), - I64Load(MemoryArg), - F32Load(MemoryArg), - F64Load(MemoryArg), - I32Load8S(MemoryArg), - I32Load8U(MemoryArg), - I32Load16S(MemoryArg), - I32Load16U(MemoryArg), - I64Load8S(MemoryArg), - I64Load8U(MemoryArg), - I64Load16S(MemoryArg), - I64Load16U(MemoryArg), - I64Load32S(MemoryArg), - I64Load32U(MemoryArg), - I32Store(MemoryArg), - I64Store(MemoryArg), - F32Store(MemoryArg), - F64Store(MemoryArg), - I32Store8(MemoryArg), - I32Store16(MemoryArg), - I64Store8(MemoryArg), - I64Store16(MemoryArg), - I64Store32(MemoryArg), + I32Load(OperandIdx), I64Load(OperandIdx), F32Load(OperandIdx), F64Load(OperandIdx), + I32Load8S(OperandIdx), I32Load8U(OperandIdx), I32Load16S(OperandIdx), I32Load16U(OperandIdx), + I64Load8S(OperandIdx), I64Load8U(OperandIdx), I64Load16S(OperandIdx), I64Load16U(OperandIdx), I64Load32S(OperandIdx), I64Load32U(OperandIdx), + I32Store(OperandIdx), I64Store(OperandIdx), F32Store(OperandIdx), F64Store(OperandIdx), + I32Store8(OperandIdx), I32Store16(OperandIdx), I64Store8(OperandIdx), I64Store16(OperandIdx), I64Store32(OperandIdx), MemorySize(MemAddr), MemoryGrow(MemAddr), // > Constants Const32(i32), - Const64(i64), + Const64Imm(i32), + Const64(OperandIdx), // > Reference Types RefNull(RefType), @@ -348,29 +670,23 @@ pub enum Instruction { RefEq, RefTest(RefType), RefCast(RefType), - BrOnCast(u32, RefType, bool), + BrOnCast(OperandIdx), // > GC Objects StructNew(TypeAddr), StructNewDefault(TypeAddr), - StructGet(TypeAddr, u32), - StructGetS(TypeAddr, u32), - StructGetU(TypeAddr, u32), - StructSet(TypeAddr, u32), + StructGet(OperandIdx), StructGetS(OperandIdx), StructGetU(OperandIdx), StructSet(OperandIdx), ArrayNew(TypeAddr), ArrayNewDefault(TypeAddr), - ArrayNewFixed(TypeAddr, u32), - ArrayNewData(TypeAddr, DataAddr), - ArrayNewElem(TypeAddr, ElemAddr), + ArrayNewFixed(OperandIdx), + ArrayNewData(OperandIdx), ArrayNewElem(OperandIdx), ArrayGet(TypeAddr), ArrayGetS(TypeAddr), ArrayGetU(TypeAddr), ArraySet(TypeAddr), ArrayLen, ArrayFill(TypeAddr), - ArrayCopy(TypeAddr, TypeAddr), - ArrayInitData(TypeAddr, DataAddr), - ArrayInitElem(TypeAddr, ElemAddr), + ArrayCopy(OperandIdx), ArrayInitData(OperandIdx), ArrayInitElem(OperandIdx), // > Numeric Instructions // See @@ -400,19 +716,18 @@ pub enum Instruction { I64TruncSatF32S, I64TruncSatF32U, I64TruncSatF64S, I64TruncSatF64U, // > Table Instructions - TableInit(ElemAddr, TableAddr), + TableInit(OperandIdx), TableGet(TableAddr), TableSet(TableAddr), - TableCopy { dst_table: TableAddr, src_table: TableAddr }, + TableCopy(OperandIdx), TableGrow(TableAddr), TableSize(TableAddr), TableFill(TableAddr), // > Bulk Memory Instructions - MemoryInit(DataAddr, MemAddr), - MemoryCopy { dst_mem: MemAddr, src_mem: MemAddr }, + MemoryInit(OperandIdx), MemoryCopy(OperandIdx), MemoryFill(MemAddr), - MemoryFillImm(MemAddr, u8, i32), + MemoryFillConst(OperandIdx), DataDrop(DataAddr), ElemDrop(ElemAddr), @@ -420,20 +735,18 @@ pub enum Instruction { I64Add128, I64Sub128, I64MulWideS, I64MulWideU, // > SIMD - V128Load(MemoryArg), - V128Load8x8S(MemoryArg), V128Load8x8U(MemoryArg), - V128Load16x4S(MemoryArg), V128Load16x4U(MemoryArg), - V128Load32x2S(MemoryArg), V128Load32x2U(MemoryArg), + V128Load(OperandIdx), V128Load8x8S(OperandIdx), V128Load8x8U(OperandIdx), + V128Load16x4S(OperandIdx), V128Load16x4U(OperandIdx), V128Load32x2S(OperandIdx), V128Load32x2U(OperandIdx), - V128Load8Splat(MemoryArg), V128Load16Splat(MemoryArg), V128Load32Splat(MemoryArg), V128Load64Splat(MemoryArg), - V128Load8Lane(MemoryArg, u8), V128Load16Lane(MemoryArg, u8), V128Load32Lane(MemoryArg, u8), V128Load64Lane(MemoryArg, u8), + V128Load8Splat(OperandIdx), V128Load16Splat(OperandIdx), V128Load32Splat(OperandIdx), V128Load64Splat(OperandIdx), + V128Load8Lane(MemoryLaneArg), V128Load16Lane(MemoryLaneArg), V128Load32Lane(MemoryLaneArg), V128Load64Lane(MemoryLaneArg), - V128Load32Zero(MemoryArg), V128Load64Zero(MemoryArg), + V128Load32Zero(OperandIdx), V128Load64Zero(OperandIdx), - V128Store(MemoryArg), V128Store8Lane(MemoryArg, u8), V128Store16Lane(MemoryArg, u8), V128Store32Lane(MemoryArg, u8), V128Store64Lane(MemoryArg, u8), + V128Store(OperandIdx), V128Store8Lane(MemoryLaneArg), V128Store16Lane(MemoryLaneArg), V128Store32Lane(MemoryLaneArg), V128Store64Lane(MemoryLaneArg), - I8x16Shuffle(ConstIdx), - Const128(ConstIdx), + I8x16Shuffle(OperandIdx), + Const128Imm(u32), Const128(OperandIdx), I8x16ExtractLaneS(u8), I8x16ExtractLaneU(u8), I8x16ReplaceLane(u8), I16x8ExtractLaneS(u8), I16x8ExtractLaneU(u8), I16x8ReplaceLane(u8), @@ -485,10 +798,8 @@ pub enum Instruction { I32x4RelaxedTruncF64x2SZero, I32x4RelaxedTruncF64x2UZero, F32x4RelaxedMadd, F32x4RelaxedNmadd, F64x2RelaxedMadd, F64x2RelaxedNmadd, - I8x16RelaxedLaneselect, - I16x8RelaxedLaneselect, - I32x4RelaxedLaneselect, - I64x2RelaxedLaneselect, + I8x16RelaxedLaneselect, I16x8RelaxedLaneselect, + I32x4RelaxedLaneselect, I64x2RelaxedLaneselect, F32x4RelaxedMin, F32x4RelaxedMax, F64x2RelaxedMin, F64x2RelaxedMax, I16x8RelaxedQ15mulrS, @@ -496,88 +807,139 @@ pub enum Instruction { I32x4RelaxedDotI8x16I7x16AddS, } -const _: () = assert!(core::mem::size_of::() <= 16); +const _: () = assert!(core::mem::size_of::() == 8); impl Instruction { + /// Returns the largest module-local memory index used by this instruction. #[inline] - pub const fn memory_addr(&self) -> Option { - match self { - Self::IncMemoryLocal32(arg, ..) - | Self::IncMemoryLocal64(arg, ..) - | Self::StoreLocalLocal32(arg, ..) - | Self::StoreLocalLocal64(arg, ..) - | Self::StoreLocalLocal128(arg, ..) - | Self::LoadLocal32(arg, ..) - | Self::LoadLocal64(arg, ..) - | Self::LoadLocal8S32(arg, ..) - | Self::LoadLocal8U32(arg, ..) - | Self::LoadLocal16S32(arg, ..) - | Self::LoadLocal16U32(arg, ..) - | Self::LoadLocalTee32(arg, ..) - | Self::LoadLocalSet32(arg, ..) - | Self::LoadLocalTee8S32(arg, ..) - | Self::LoadLocalTee8U32(arg, ..) - | Self::LoadLocalTee16S32(arg, ..) - | Self::LoadLocalTee16U32(arg, ..) - | Self::LoadLocalSet8S32(arg, ..) - | Self::LoadLocalSet8U32(arg, ..) - | Self::LoadLocalSet16S32(arg, ..) - | Self::LoadLocalSet16U32(arg, ..) - | Self::LoadLocalTee128(arg, ..) - | Self::LoadLocalSet128(arg, ..) - | Self::I32Load(arg) - | Self::I64Load(arg) - | Self::F32Load(arg) - | Self::F64Load(arg) - | Self::I32Load8S(arg) - | Self::I32Load8U(arg) - | Self::I32Load16S(arg) - | Self::I32Load16U(arg) - | Self::I64Load8S(arg) - | Self::I64Load8U(arg) - | Self::I64Load16S(arg) - | Self::I64Load16U(arg) - | Self::I64Load32S(arg) - | Self::I64Load32U(arg) - | Self::I32Store(arg) - | Self::I64Store(arg) - | Self::F32Store(arg) - | Self::F64Store(arg) - | Self::FMaStoreF32(arg) - | Self::FMaStoreF64(arg) - | Self::I32Store8(arg) - | Self::I32Store16(arg) - | Self::I64Store8(arg) - | Self::I64Store16(arg) - | Self::I64Store32(arg) - | Self::V128Load(arg) - | Self::V128Load8x8S(arg) - | Self::V128Load8x8U(arg) - | Self::V128Load16x4S(arg) - | Self::V128Load16x4U(arg) - | Self::V128Load32x2S(arg) - | Self::V128Load32x2U(arg) - | Self::V128Load8Splat(arg) - | Self::V128Load16Splat(arg) - | Self::V128Load32Splat(arg) - | Self::V128Load64Splat(arg) - | Self::V128Load8Lane(arg, ..) - | Self::V128Load16Lane(arg, ..) - | Self::V128Load32Lane(arg, ..) - | Self::V128Load64Lane(arg, ..) - | Self::V128Load32Zero(arg) - | Self::V128Load64Zero(arg) - | Self::V128Store(arg) - | Self::V128Store8Lane(arg, ..) - | Self::V128Store16Lane(arg, ..) - | Self::V128Store32Lane(arg, ..) - | Self::V128Store64Lane(arg, ..) => Some(arg.mem_addr()), - Self::MemorySize(mem) | Self::MemoryGrow(mem) | Self::MemoryFill(mem) | Self::MemoryFillImm(mem, ..) => { - Some(*mem) + pub fn memory_addr(&self, data: &super::WasmFunctionData) -> Option { + use Instruction::*; + match *self { + IncMemoryLocal32(arg) + | IncMemoryLocal64(arg) + | StoreLocalLocal32(arg) + | StoreLocalLocal64(arg) + | StoreLocalLocal128(arg) + | LoadLocal32(arg) + | LoadLocal64(arg) + | LoadLocal8S32(arg) + | LoadLocal8U32(arg) + | LoadLocal16S32(arg) + | LoadLocal16U32(arg) + | LoadLocalTee32(arg) + | LoadLocalSet32(arg) + | LoadLocalTee8S32(arg) + | LoadLocalTee8U32(arg) + | LoadLocalTee16S32(arg) + | LoadLocalTee16U32(arg) + | LoadLocalSet8S32(arg) + | LoadLocalSet8U32(arg) + | LoadLocalSet16S32(arg) + | LoadLocalSet16U32(arg) + | LoadLocalTee128(arg) + | LoadLocalSet128(arg) => Some(arg.memory_arg_idx.get(data).mem_addr()), + I32Load(index) + | I64Load(index) + | F32Load(index) + | F64Load(index) + | I32Load8S(index) + | I32Load8U(index) + | I32Load16S(index) + | I32Load16U(index) + | I64Load8S(index) + | I64Load8U(index) + | I64Load16S(index) + | I64Load16U(index) + | I64Load32S(index) + | I64Load32U(index) + | I32Store(index) + | I64Store(index) + | F32Store(index) + | F64Store(index) + | I32Store8(index) + | I32Store16(index) + | I64Store8(index) + | I64Store16(index) + | I64Store32(index) + | V128Load(index) + | V128Load8x8S(index) + | V128Load8x8U(index) + | V128Load16x4S(index) + | V128Load16x4U(index) + | V128Load32x2S(index) + | V128Load32x2U(index) + | V128Load8Splat(index) + | V128Load16Splat(index) + | V128Load32Splat(index) + | V128Load64Splat(index) + | V128Load32Zero(index) + | V128Load64Zero(index) + | V128Store(index) => Some(index.get(data).mem_addr()), + V128Load8Lane(arg) | V128Load16Lane(arg) | V128Load32Lane(arg) | V128Load64Lane(arg) + | V128Store8Lane(arg) | V128Store16Lane(arg) | V128Store32Lane(arg) | V128Store64Lane(arg) => { + Some(arg.memory_arg_idx.get(data).mem_addr()) + } + FMaStoreF32(arg) | FMaStoreF64(arg) => Some(arg.mem_addr()), + MemorySize(memory) | MemoryGrow(memory) | MemoryFill(memory) => Some(memory), + MemoryFillConst(index) => Some(index.get(data).memory), + MemoryInit(index) => Some(index.get(data).second), + MemoryCopy(index) => { + let value = index.get(data); + Some(value.first.max(value.second)) } - Self::MemoryInit(_, mem) => Some(*mem), - Self::MemoryCopy { dst_mem, src_mem } => Some(if *dst_mem >= *src_mem { *dst_mem } else { *src_mem }), _ => None, } } } + +#[cfg(test)] +mod tests { + use alloc::vec; + + use super::*; + + #[test] + fn compact_instruction_layout_is_exact() { + assert_eq!(core::mem::size_of::(), 8); + assert_eq!(core::mem::size_of::(), 6); + assert_eq!(core::mem::size_of::(), 6); + assert_eq!(core::mem::size_of::(), 6); + assert_eq!(core::mem::size_of::(), 6); + } + + #[test] + fn operand_lanes_round_trip_max_values() { + let memory = MemoryArg::new(u64::MAX, u32::MAX); + assert_eq!(MemoryArg::decode(memory.encode()), memory); + + let value = + Operand128::default().with_u8(0, u8::MAX).with_u16(1, u16::MAX).with_u32(3, u32::MAX).with_u64(7, u64::MAX); + assert_eq!((value.u8(0), value.u16(1), value.u32(3), value.u64(7)), (u8::MAX, u16::MAX, u32::MAX, u64::MAX)); + + let bytes = [u8::MAX; 16]; + assert_eq!(Operand128::from_le_bytes(bytes).to_le_bytes(), bytes); + } + + #[test] + fn v128_operand_views_round_trip_max_bytes() { + let value = OperandIdx::::new(0); + let local = OperandIdx::::new(1); + let global = OperandIdx::::new(2); + let set = OperandIdx::::new(3); + let data = super::super::WasmFunctionData { + operands128: vec![ + V128Operand { value: [u8::MAX; 16] }.encode(), + LocalV128 { local: u16::MAX, value }.encode(), + GlobalV128 { global: u32::MAX, value }.encode(), + LocalConstSetV128 { local: u16::MAX, dst: u16::MAX, value }.encode(), + ] + .into_boxed_slice(), + ..Default::default() + }; + + assert_eq!(value.get(&data).value, [u8::MAX; 16]); + assert_eq!(local.get(&data), LocalV128 { local: u16::MAX, value }); + assert_eq!(global.get(&data), GlobalV128 { global: u32::MAX, value }); + assert_eq!(set.get(&data), LocalConstSetV128 { local: u16::MAX, dst: u16::MAX, value }); + } +} diff --git a/crates/types/src/lib.rs b/crates/types/src/lib.rs index 9a7071c..bd92753 100644 --- a/crates/types/src/lib.rs +++ b/crates/types/src/lib.rs @@ -22,10 +22,12 @@ const fn max_page_count(arch: MemoryArch, page_size: u64) -> u64 { } mod instructions; +mod operands; mod reference; mod types; mod value; pub use instructions::*; +pub use operands::*; pub use reference::*; pub use types::*; pub use value::*; @@ -472,19 +474,12 @@ pub struct WasmFunction { #[cfg_attr(feature = "debug", derive(Debug))] #[cfg_attr(feature = "archive", derive(serde::Serialize, serde::Deserialize))] pub struct WasmFunctionData { - pub v128_constants: Box<[[u8; 16]]>, + pub operands64: Box<[Operand64]>, + pub operands128: Box<[Operand128]>, pub branch_table_targets: Box<[u32]>, pub exception_handlers: Box<[ExceptionHandler]>, } -impl WasmFunctionData { - /// Panics if `idx` is out of bounds. - #[inline(always)] - pub fn v128_const(&self, idx: ConstIdx) -> [u8; 16] { - *self.v128_constants.get(idx as usize).unwrap_or_else(|| unreachable!("invalid v128 constant index: {idx}")) - } -} - /// A WebAssembly Module Export #[derive(Clone, PartialEq, Eq)] #[cfg_attr(feature = "debug", derive(Debug))] diff --git a/crates/types/src/operands.rs b/crates/types/src/operands.rs new file mode 100644 index 0000000..dd7e3a1 --- /dev/null +++ b/crates/types/src/operands.rs @@ -0,0 +1,218 @@ +use core::marker::PhantomData; + +use crate::WasmFunctionData; + +/// An index into the operand lane used by `T`. +#[repr(transparent)] +pub struct OperandIdx { + index: u32, + marker: PhantomData, +} + +impl OperandIdx { + #[inline] + #[doc(hidden)] + pub const fn new(index: u32) -> Self { + Self { index, marker: PhantomData } + } + + /// Returns the underlying lane index. + #[inline] + pub const fn index(self) -> u32 { + self.index + } +} + +impl OperandIdx { + /// Returns the typed operand at this index. + /// + /// Panics if the index is outside its statically selected operand lane. + #[inline(always)] + pub fn get(self, data: &WasmFunctionData) -> T { + T::decode(T::Raw::get(data, self.index)) + } +} + +impl Copy for OperandIdx {} + +impl Clone for OperandIdx { + fn clone(&self) -> Self { + *self + } +} + +impl PartialEq for OperandIdx { + fn eq(&self, other: &Self) -> bool { + self.index == other.index + } +} + +impl Eq for OperandIdx {} + +#[cfg(feature = "debug")] +impl core::fmt::Debug for OperandIdx { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + self.index.fmt(formatter) + } +} + +#[cfg(feature = "archive")] +impl serde::Serialize for OperandIdx { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serde::Serialize::serialize(&self.index, serializer) + } +} + +#[cfg(feature = "archive")] +impl<'de, T> serde::Deserialize<'de> for OperandIdx { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + Ok(Self::new(serde::Deserialize::deserialize(deserializer)?)) + } +} + +/// Raw fields stored in a per-function 64-bit operand lane. +#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Default)] +#[cfg_attr(feature = "debug", derive(Debug))] +#[cfg_attr(feature = "archive", derive(serde::Serialize, serde::Deserialize))] +#[repr(transparent)] +pub struct Operand64(u64); + +/// Raw fields stored in a per-function 128-bit operand lane. +#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Default)] +#[cfg_attr(feature = "debug", derive(Debug))] +#[cfg_attr(feature = "archive", derive(serde::Serialize, serde::Deserialize))] +#[repr(transparent)] +pub struct Operand128(u128); + +pub(crate) mod sealed { + pub trait Sealed {} + pub trait RawSealed {} +} + +/// A typed view stored in one of the per-function operand lanes. +pub trait OperandType: sealed::Sealed + Copy { + #[doc(hidden)] + type Raw: RawOperand; + + #[doc(hidden)] + fn decode(raw: Self::Raw) -> Self; + #[doc(hidden)] + fn encode(self) -> Self::Raw; +} + +#[doc(hidden)] +pub trait RawOperand: sealed::RawSealed + Copy { + fn get(data: &WasmFunctionData, index: u32) -> Self; +} + +impl sealed::RawSealed for Operand64 {} +impl RawOperand for Operand64 { + #[inline(always)] + fn get(data: &WasmFunctionData, index: u32) -> Self { + *data.operands64.get(index as usize).unwrap_or_else(|| unreachable!("invalid operand index")) + } +} + +impl sealed::RawSealed for Operand128 {} +impl RawOperand for Operand128 { + #[inline(always)] + fn get(data: &WasmFunctionData, index: u32) -> Self { + *data.operands128.get(index as usize).unwrap_or_else(|| unreachable!("invalid operand index")) + } +} + +macro_rules! operand_fields { + ($ty:ty) => { + #[inline] + pub(crate) const fn u16(self, offset: u32) -> u16 { + (self.0 >> (offset * 8)) as u16 + } + #[inline] + pub(crate) const fn u32(self, offset: u32) -> u32 { + (self.0 >> (offset * 8)) as u32 + } + #[inline] + pub(crate) const fn u64(self, offset: u32) -> u64 { + (self.0 >> (offset * 8)) as u64 + } + #[inline] + pub(crate) const fn i64(self, offset: u32) -> i64 { + self.u64(offset) as i64 + } + }; +} + +impl Operand64 { + operand_fields!(u64); + + #[inline] + pub(crate) const fn with_u16(self, offset: u32, value: u16) -> Self { + let shift = offset * 8; + Self((self.0 & !((u16::MAX as u64) << shift)) | ((value as u64) << shift)) + } + + #[inline] + pub(crate) const fn with_u32(self, offset: u32, value: u32) -> Self { + let shift = offset * 8; + Self((self.0 & !((u32::MAX as u64) << shift)) | ((value as u64) << shift)) + } + + #[inline] + pub(crate) const fn with_u64(self, _offset: u32, value: u64) -> Self { + Self(value) + } +} + +impl Operand128 { + operand_fields!(u128); + + #[inline] + pub(crate) const fn u8(self, offset: u32) -> u8 { + (self.0 >> (offset * 8)) as u8 + } + + #[inline] + pub(crate) const fn i32(self, offset: u32) -> i32 { + self.u32(offset) as i32 + } + + #[inline] + pub(crate) const fn from_le_bytes(value: [u8; 16]) -> Self { + Self(u128::from_le_bytes(value)) + } + + #[inline] + pub(crate) const fn to_le_bytes(self) -> [u8; 16] { + self.0.to_le_bytes() + } + + #[inline] + pub(crate) const fn with_u8(self, offset: u32, value: u8) -> Self { + let shift = offset * 8; + Self((self.0 & !((u8::MAX as u128) << shift)) | ((value as u128) << shift)) + } + + #[inline] + pub(crate) const fn with_u16(self, offset: u32, value: u16) -> Self { + let shift = offset * 8; + Self((self.0 & !((u16::MAX as u128) << shift)) | ((value as u128) << shift)) + } + + #[inline] + pub(crate) const fn with_u32(self, offset: u32, value: u32) -> Self { + let shift = offset * 8; + Self((self.0 & !((u32::MAX as u128) << shift)) | ((value as u128) << shift)) + } + + #[inline] + pub(crate) const fn with_u64(self, offset: u32, value: u64) -> Self { + let shift = offset * 8; + Self((self.0 & !((u64::MAX as u128) << shift)) | ((value as u128) << shift)) + } +} diff --git a/crates/types/src/reference.rs b/crates/types/src/reference.rs index 9ed5df1..ed59ae5 100644 --- a/crates/types/src/reference.rs +++ b/crates/types/src/reference.rs @@ -67,6 +67,18 @@ impl RefType { pub const EXTERNREF: Self = Self::new_abstract(true, AbstractHeapType::Extern); pub const EXNREF: Self = Self::new_abstract(true, AbstractHeapType::Exn); + #[inline] + #[doc(hidden)] + pub const fn to_bits(self) -> u32 { + self.0 + } + + #[inline] + pub(crate) const fn from_bits(bits: u32) -> Option { + let ty = Self(bits); + if ty.is_concrete() || ty.abstract_heap_type().is_some() { Some(ty) } else { None } + } + #[inline] pub const fn new_abstract(nullable: bool, heap_type: AbstractHeapType) -> Self { Self((nullable as u32) << 31 | heap_type as u32) diff --git a/examples/rust/src/print.twasm b/examples/rust/src/print.twasm deleted file mode 100644 index f0fa9a87c3b90554afabdd92675a76a7a92f0820..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 154 zcmYL@!3qK~3`A#IQBedxMHD~4Z&2_e#7GM6Ve7Ikg13IUX)lJ52@htH%k^~L9q}*5 z1;9IQjza6Y&>85nqCf_4Ht$pa8ovP>H51Jy!`S5sT#1-YDk)pUTI88@kcYlgbf4#x WQd(Y9V(SMOQ~oTQzdP*fH~ay3HX1|# From 2fb5c4a59f9fdc9cbd1d4d16ba050c7e43354fa4 Mon Sep 17 00:00:00 2001 From: Henry Date: Mon, 17 Aug 2026 18:54:01 +0200 Subject: [PATCH 2/3] chore: simplify operand structs Signed-off-by: Henry --- Cargo.lock | 8 +- crates/parser/src/optimize/rewrite.rs | 21 +- crates/parser/src/optimize/targets.rs | 6 +- crates/parser/src/visit.rs | 14 +- crates/tinywasm/src/interpreter/executor.rs | 13 +- crates/types/src/instructions.rs | 117 +++++----- crates/types/src/operands.rs | 227 ++++++++------------ 7 files changed, 179 insertions(+), 227 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b06f70f..3c13501 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -134,9 +134,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.4.2" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" +checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d" dependencies = [ "find-msvc-tools", "shlex", @@ -365,9 +365,9 @@ checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "find-msvc-tools" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" [[package]] name = "float-cmp" diff --git a/crates/parser/src/optimize/rewrite.rs b/crates/parser/src/optimize/rewrite.rs index 8e55fe9..1146da7 100644 --- a/crates/parser/src/optimize/rewrite.rs +++ b/crates/parser/src/optimize/rewrite.rs @@ -6,10 +6,10 @@ use alloc::vec::Vec; use core::ops::{Deref, DerefMut}; use tinywasm_types::{ BinOp, BinOp128, CmpOp, CompactMemoryArg, GlobalConst32, GlobalConst64, GlobalUpdate, GlobalV128, I32LocalArg, - I64Local, Instruction, LocalConst64, LocalConstCmp, LocalConstSet32, LocalConstSet64, LocalConstSetV128, + I64Operand, Instruction, LocalConst64, LocalConstCmp, LocalConstSet32, LocalConstSet64, LocalConstSetV128, LocalLocalCmp, LocalTripleArg, LocalU32, LocalUpdate, LocalUpdateCmp, LocalV128, MemoryArg, MemoryFillConstOp, - MemoryLocalArg, OperandIdx, PackedOp, StackConst32, StackConst64, TargetLocal, TargetLocalArg, V128LocalArg, - V128Operand, ValueCounts, + MemoryLocalArg, OperandIdx, PackedOp, StackConst32, StackConst64, TargetLocal, TargetLocalArg, V128Operand, + ValueCounts, }; pub(crate) struct OptimizeResult { @@ -725,8 +725,7 @@ fn rewrite_local_set64( } Instruction::LocalGet64(src) => replace!(output, *read, 1 => Instruction::LocalCopy64(src, dst)), Instruction::Const64(index) => { - let index = operand!(data, I64Local { value: data.operand(index).value, local: dst })?; - replace!(output, *read, 1 => Instruction::SetLocalConst64(index)); + replace!(output, *read, 1 => Instruction::SetLocalConst64(PackedOp::new(dst, index))); } Instruction::BinOpLocalLocal64(op, left, right) => { let index = operand!(data, LocalTripleArg { left, right, dst })?; @@ -735,8 +734,8 @@ fn rewrite_local_set64( instruction if let Some((op, src, value)) = local_const64(data, instruction) => { if src == dst && matches!(op, BinOp::IAdd | BinOp::ISub) { let delta = if op == BinOp::IAdd { value } else { value.wrapping_neg() }; - let index = operand!(data, I64Local { value: delta, local: dst })?; - replace!(output, *read, 1 => Instruction::IncLocal64(index)); + let index = operand!(data, I64Operand { value: delta })?; + replace!(output, *read, 1 => Instruction::IncLocal64(PackedOp::new(dst, index))); } else { let replacement = local_const_set!(data, 64, op, src, dst, value, false); replace!(output, *read, 1 => replacement); @@ -776,7 +775,7 @@ fn rewrite_local_set128( } Instruction::LocalGet128(src) => replace!(output, *read, 1 => Instruction::LocalCopy128(src, dst)), Instruction::Const128(value) => { - replace!(output, *read, 1 => Instruction::SetLocalConst128(V128LocalArg { value, local: dst })) + replace!(output, *read, 1 => Instruction::SetLocalConst128(PackedOp::new(dst, value))) } Instruction::BinOpLocalLocal128(op, left, right) => { let index = operand!(data, LocalTripleArg { left, right, dst })?; @@ -867,13 +866,11 @@ fn rewrite_local_tee64( if *read >= output.block_start + 2 { match [output[*read - 2], output[*read - 1]] { [Instruction::Const64(value), Instruction::I64And] => { - let index = operand!(data, I64Local { value: data.operand(value).value, local: dst })?; - replace!(output, *read, 2 => Instruction::AndConstTee64(index)); + replace!(output, *read, 2 => Instruction::AndConstTee64(PackedOp::new(dst, value))); return Ok(()); } [Instruction::Const64(value), Instruction::I64Sub] => { - let index = operand!(data, I64Local { value: data.operand(value).value, local: dst })?; - replace!(output, *read, 2 => Instruction::SubConstTee64(index)); + replace!(output, *read, 2 => Instruction::SubConstTee64(PackedOp::new(dst, value))); return Ok(()); } _ => {} diff --git a/crates/parser/src/optimize/targets.rs b/crates/parser/src/optimize/targets.rs index 88d5fbe..36bdf2f 100644 --- a/crates/parser/src/optimize/targets.rs +++ b/crates/parser/src/optimize/targets.rs @@ -55,7 +55,7 @@ pub(super) fn set_rewrite_target( JumpCmpStackLocal32(packed) | JumpCmpStackLocal64(packed) => { *packed = push_packed_target_copy(data, *packed, target)? } - BrOnCast(index) => *index = push_target_copy(data, *index, target)?, + BrOnCast(index) | BrOnCastFail(index) => *index = push_target_copy(data, *index, target)?, JumpCmpStackConst32(packed) => *packed = push_packed_target_copy(data, *packed, target)?, JumpCmpStackConst64(packed) => *packed = push_packed_target_copy(data, *packed, target)?, BinOpLocalConstJump32(packed) | BinOpStackConstTeeLocalJump32(packed) => { @@ -118,7 +118,7 @@ fn instruction_target(data: &FunctionDataBuilder, instruction: Instruction) -> O arg.target_ip } JumpCmpStackLocal32(packed) | JumpCmpStackLocal64(packed) => data.operand(packed.index).target, - BrOnCast(index) => data.operand(index).target, + BrOnCast(index) | BrOnCastFail(index) => data.operand(index).target, JumpCmpStackConst32(packed) => data.operand(packed.index).target, JumpCmpStackConst64(packed) => data.operand(packed.index).target, BinOpLocalConstJump32(packed) | BinOpStackConstTeeLocalJump32(packed) => data.operand(packed.index).target, @@ -148,7 +148,7 @@ fn set_target(instruction: &mut Instruction, data: &mut FunctionDataBuilder, tar arg.target_ip = target } JumpCmpStackLocal32(packed) | JumpCmpStackLocal64(packed) => set_packed_operand_target(data, *packed, target), - BrOnCast(index) => set_operand_target(data, *index, target), + BrOnCast(index) | BrOnCastFail(index) => set_operand_target(data, *index, target), JumpCmpStackConst32(packed) => set_packed_operand_target(data, *packed, target), JumpCmpStackConst64(packed) => set_packed_operand_target(data, *packed, target), BinOpLocalConstJump32(packed) | BinOpStackConstTeeLocalJump32(packed) => { diff --git a/crates/parser/src/visit.rs b/crates/parser/src/visit.rs index a5e7c2b..8315fb5 100644 --- a/crates/parser/src/visit.rs +++ b/crates/parser/src/visit.rs @@ -1334,12 +1334,12 @@ impl FunctionBuilder<'_> { self.pop_expect(ValueLane::S32)?; let target = convert_heap_type(target.heap_type(), target.is_nullable())?; let conditional_ip = self.instructions.len(); - let operand = self.data.push_target_operand(CastBranch { - target: 0, - ref_type_bits: target.to_bits(), - branch_on_fail: u8::from(branch_on_fail), - })?; - self.instructions.push(Instruction::BrOnCast(operand)); + let operand = self.data.push_target_operand(CastBranch { target: 0, ref_type_bits: target.to_bits() })?; + self.instructions.push(if branch_on_fail { + Instruction::BrOnCastFail(operand) + } else { + Instruction::BrOnCast(operand) + }); self.push_sizes(&[ValueLane::S32])?; self.emit_dropkeep_to_label(relative_depth)?; self.emit_branch_jump_or_return(relative_depth)?; @@ -1503,7 +1503,7 @@ impl FunctionBuilder<'_> { | Instruction::JumpIfRefNonNull(ip) => { *ip = target as u32; } - Instruction::BrOnCast(index) => { + Instruction::BrOnCast(index) | Instruction::BrOnCastFail(index) => { let mut operand = self.data.operand(*index); operand.target = target as u32; self.data.set_operand(*index, operand); diff --git a/crates/tinywasm/src/interpreter/executor.rs b/crates/tinywasm/src/interpreter/executor.rs index c512d82..9da4a7a 100644 --- a/crates/tinywasm/src/interpreter/executor.rs +++ b/crates/tinywasm/src/interpreter/executor.rs @@ -228,7 +228,8 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { if !is_non_null { ValueRef::stack_pop(&mut self.store.value_stack); } is_non_null }) { return Ok(None) }, - BrOnCast(idx) => { let v = idx.get(&self.func.data); if self.exec_ref_matches(v.ref_type()) == (v.branch_on_fail != 0) { self.cf.instr_ptr = v.target as usize; return Ok(None); } }, + BrOnCast(idx) => { let v = idx.get(&self.func.data); if !self.exec_ref_matches(v.ref_type()) { self.cf.instr_ptr = v.target as usize; return Ok(None); } }, + BrOnCastFail(idx) => { let v = idx.get(&self.func.data); if self.exec_ref_matches(v.ref_type()) { self.cf.instr_ptr = v.target as usize; return Ok(None); } }, JumpIfLocalZero32(arg) => if Self::exec_jump_if(&mut self.cf, arg.target_ip, |cf| Value32::local_get(&self.store.value_stack, cf, arg.local) == 0) { return Ok(None) }, JumpIfLocalNonZero32(arg) => if Self::exec_jump_if(&mut self.cf, arg.target_ip, |cf| Value32::local_get(&self.store.value_stack, cf, arg.local) != 0) { return Ok(None) }, JumpIfLocalZero64(arg) => if Self::exec_jump_if(&mut self.cf, arg.target_ip, |cf| Value64::local_get(&self.store.value_stack, cf, arg.local) == 0) { return Ok(None) }, @@ -286,7 +287,7 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { AddConst32(c) => exec_op!(unary i32 => i32, |v| v.wrapping_add(*c)), AddConst64(idx) => { let c = idx.get(&self.func.data).value; exec_op!(unary i64 => i64, |v| v.wrapping_add(c)); }, IncLocal32(arg) => i32::local_update(&mut self.store.value_stack, &self.cf, arg.local, |v| v.wrapping_add(arg.value)), - IncLocal64(idx) => { let v = idx.get(&self.func.data); i64::local_update(&mut self.store.value_stack, &self.cf, v.local, |n| n.wrapping_add(v.value)); }, + IncLocal64(packed) => { let v = packed.index.get(&self.func.data).value; i64::local_update(&mut self.store.value_stack, &self.cf, packed.op, |n| n.wrapping_add(v)); }, I32Add3 => exec_op!(ternary i32 => i32, |a, b, c| a.wrapping_add(b).wrapping_add(c)), I64Add3 => exec_op!(ternary i64 => i64, |a, b, c| a.wrapping_add(b).wrapping_add(c)), MulAccLocal32(acc) => exec_op!(binop_acc_local i32, acc, |a: i32, b| a.wrapping_mul(b), |a: i32, b| a.wrapping_add(b)), @@ -328,8 +329,8 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { BinOpStackLocalTee32(op, local, dst) => exec_op!(binop_stack_local_tee Value32, exec_binop_32, op, local, dst), BinOpStackLocal128(op, local) => exec_op!(binop_stack_local Value128, exec_binop_128, op, local), SetLocalConst32(arg) => i32::local_set(&mut self.store.value_stack, &self.cf, arg.local, arg.value), - SetLocalConst64(idx) => { let v = idx.get(&self.func.data); i64::local_set(&mut self.store.value_stack, &self.cf, v.local, v.value); }, - SetLocalConst128(arg) => Value128::local_set(&mut self.store.value_stack, &self.cf, arg.local, Value128(arg.value.get(&self.func.data).value)), + SetLocalConst64(packed) => { let v = packed.index.get(&self.func.data).value; i64::local_set(&mut self.store.value_stack, &self.cf, packed.op, v); }, + SetLocalConst128(packed) => Value128::local_set(&mut self.store.value_stack, &self.cf, packed.op, Value128(packed.index.get(&self.func.data).value)), IncMemoryLocal32(arg) => self.exec_inc_memory_local::(arg.memory_arg_idx.get(&self.func.data), arg.local1, |v| v.wrapping_add(1))?, IncMemoryLocal64(arg) => self.exec_inc_memory_local::(arg.memory_arg_idx.get(&self.func.data), arg.local1, |v| v.wrapping_add(1))?, StoreLocalLocal32(arg) => self.exec_store_local_local::(arg.memory_arg_idx.get(&self.func.data), arg.local1, arg.local2)?, @@ -355,8 +356,8 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { LoadLocalSet128(arg) => self.exec_load_local::(arg.memory_arg_idx.get(&self.func.data), arg.local1, arg.local2, |v| v)?, AndConstTee32(arg) => { exec_op!(unary i32 => i32, |v| v & arg.value); let value = i32::stack_peek(&self.store.value_stack); i32::local_set(&mut self.store.value_stack, &self.cf, arg.local, value); }, SubConstTee32(arg) => { exec_op!(unary i32 => i32, |v| v.wrapping_sub(arg.value)); let value = i32::stack_peek(&self.store.value_stack); i32::local_set(&mut self.store.value_stack, &self.cf, arg.local, value); }, - AndConstTee64(idx) => { let v = idx.get(&self.func.data); exec_op!(unary i64 => i64, |n| n & v.value); let value = i64::stack_peek(&self.store.value_stack); i64::local_set(&mut self.store.value_stack, &self.cf, v.local, value); }, - SubConstTee64(idx) => { let v = idx.get(&self.func.data); exec_op!(unary i64 => i64, |n| n.wrapping_sub(v.value)); let value = i64::stack_peek(&self.store.value_stack); i64::local_set(&mut self.store.value_stack, &self.cf, v.local, value); }, + AndConstTee64(packed) => { let v = packed.index.get(&self.func.data).value; exec_op!(unary i64 => i64, |n| n & v); let value = i64::stack_peek(&self.store.value_stack); i64::local_set(&mut self.store.value_stack, &self.cf, packed.op, value); }, + SubConstTee64(packed) => { let v = packed.index.get(&self.func.data).value; exec_op!(unary i64 => i64, |n| n.wrapping_sub(v)); let value = i64::stack_peek(&self.store.value_stack); i64::local_set(&mut self.store.value_stack, &self.cf, packed.op, value); }, LocalTee32(local_index) => exec_op!(local_tee Value32, local_index), LocalTee64(local_index) => exec_op!(local_tee Value64, local_index), LocalTee128(local_index) => exec_op!(local_tee Value128, local_index), diff --git a/crates/types/src/instructions.rs b/crates/types/src/instructions.rs index 0773d3e..797f703 100644 --- a/crates/types/src/instructions.rs +++ b/crates/types/src/instructions.rs @@ -75,25 +75,15 @@ impl OperandType for LocalTripleArg { #[inline(always)] fn decode(raw: Self::Raw) -> Self { - Self { left: raw.u16(0), right: raw.u16(2), dst: raw.u16(4) } + Self { left: raw.u16::<0>(), right: raw.u16::<2>(), dst: raw.u16::<4>() } } #[inline] fn encode(self) -> Self::Raw { - Operand64::default().with_u16(0, self.left).with_u16(2, self.right).with_u16(4, self.dst) + Operand64::default().with_u16::<0>(self.left).with_u16::<2>(self.right).with_u16::<4>(self.dst) } } -/// An indexed SIMD value and local operand that fit in an instruction payload. -#[derive(Copy, Clone, PartialEq, Eq)] -#[cfg_attr(feature = "debug", derive(Debug))] -#[cfg_attr(feature = "archive", derive(serde::Serialize, serde::Deserialize))] -#[repr(Rust, packed)] -pub struct V128LocalArg { - pub value: OperandIdx, - pub local: LocalAddr, -} - /// A branch target and local operand that fit in an instruction payload. #[derive(Copy, Clone, PartialEq, Eq)] #[cfg_attr(feature = "debug", derive(Debug))] @@ -119,18 +109,18 @@ macro_rules! operand_view { #[inline(always)] fn decode(raw: Self::Raw) -> Self { - Self { $($field: raw.$read($offset) as $ty),+ } + Self { $($field: raw.$read::<$offset>() as $ty),+ } } #[inline] fn encode(self) -> Self::Raw { - <$raw>::default()$(.$write($offset, self.$field as _))+ + <$raw>::default()$(.$write::<$offset>(self.$field as _))+ } } }; } -operand_view!(I64Operand, operands64, Operand64, { value: i64 = i64 / with_u64(0) }); +operand_view!(I64Operand, operands64, Operand64, { value: i64 = u64 / with_u64(0) }); operand_view!(TwoU32, operands64, Operand64, { first: u32 = u32 / with_u32(0), second: u32 = u32 / with_u32(4) }); @@ -141,53 +131,50 @@ operand_view!(TargetLocal, operands64, Operand64, { target: u32 = u32 / with_u32(0), local: u16 = u16 / with_u16(4) }); -operand_view!(I64Local, operands128, Operand128, { - value: i64 = i64 / with_u64(0), local: u16 = u16 / with_u16(8) -}); operand_view!(LocalConst64, operands128, Operand128, { local: u16 = u16 / with_u16(0), value: u64 = u64 / with_u64(2) }); -operand_view!(GlobalConst32, operands128, Operand128, { +operand_view!(GlobalConst32, operands64, Operand64, { global: u32 = u32 / with_u32(0), value: u32 = u32 / with_u32(4) }); operand_view!(GlobalConst64, operands128, Operand128, { global: u32 = u32 / with_u32(0), value: u64 = u64 / with_u64(4) }); -operand_view!(LocalConstSet32, operands128, Operand128, { +operand_view!(LocalConstSet32, operands64, Operand64, { local: u16 = u16 / with_u16(0), dst: u16 = u16 / with_u16(2), value: u32 = u32 / with_u32(4) }); operand_view!(LocalConstSet64, operands128, Operand128, { local: u16 = u16 / with_u16(0), dst: u16 = u16 / with_u16(2), value: u64 = u64 / with_u64(4) }); operand_view!(MemoryFillConstOp, operands128, Operand128, { - memory: u32 = u32 / with_u32(0), byte: u8 = u8 / with_u8(4), value: i32 = i32 / with_u32(5) + memory: u32 = u32 / with_u32(0), byte: u8 = u8 / with_u8(4), value: i32 = u32 / with_u32(5) }); -operand_view!(CastBranch, operands128, Operand128, { - target: u32 = u32 / with_u32(0), ref_type_bits: u32 = u32 / with_u32(4), branch_on_fail: u8 = u8 / with_u8(8) +operand_view!(CastBranch, operands64, Operand64, { + target: u32 = u32 / with_u32(0), ref_type_bits: u32 = u32 / with_u32(4) }); operand_view!(BranchTableArg, operands128, Operand128, { target: u32 = u32 / with_u32(0), start: u32 = u32 / with_u32(4), len: u32 = u32 / with_u32(8) }); -operand_view!(StackConst32, operands128, Operand128, { - target: u32 = u32 / with_u32(0), value: i32 = i32 / with_u32(4) +operand_view!(StackConst32, operands64, Operand64, { + target: u32 = u32 / with_u32(0), value: i32 = u32 / with_u32(4) }); operand_view!(StackConst64, operands128, Operand128, { - target: u32 = u32 / with_u32(0), value: i64 = i64 / with_u64(4) + target: u32 = u32 / with_u32(0), value: i64 = u64 / with_u64(4) }); operand_view!(LocalConstCmp, operands128, Operand128, { - target: u32 = u32 / with_u32(0), value: i32 = i32 / with_u32(4), local: u16 = u16 / with_u16(8) + target: u32 = u32 / with_u32(0), value: i32 = u32 / with_u32(4), local: u16 = u16 / with_u16(8) }); -operand_view!(LocalLocalCmp, operands128, Operand128, { +operand_view!(LocalLocalCmp, operands64, Operand64, { target: u32 = u32 / with_u32(0), left: u16 = u16 / with_u16(4), right: u16 = u16 / with_u16(6) }); operand_view!(LocalUpdate, operands128, Operand128, { - target: u32 = u32 / with_u32(0), value: i32 = i32 / with_u32(4), local: u16 = u16 / with_u16(8), on_zero: u8 = u8 / with_u8(10) + target: u32 = u32 / with_u32(0), value: i32 = u32 / with_u32(4), local: u16 = u16 / with_u16(8), on_zero: u8 = u8 / with_u8(10) }); operand_view!(GlobalUpdate, operands128, Operand128, { - target: u32 = u32 / with_u32(0), value: i32 = i32 / with_u32(4), global: u32 = u32 / with_u32(8), on_zero: u8 = u8 / with_u8(12) + target: u32 = u32 / with_u32(0), value: i32 = u32 / with_u32(4), global: u32 = u32 / with_u32(8), on_zero: u8 = u8 / with_u8(12) }); operand_view!(LocalUpdateCmp, operands128, Operand128, { - target: u32 = u32 / with_u32(0), value: i32 = i32 / with_u32(4), local: u16 = u16 / with_u16(8), right: u16 = u16 / with_u16(10) + target: u32 = u32 / with_u32(0), value: i32 = u32 / with_u32(4), local: u16 = u16 / with_u16(8), right: u16 = u16 / with_u16(10) }); /// An operation packed inline with an indexed operand. @@ -244,12 +231,12 @@ pub struct LocalConstSetV128 { pub value: OperandIdx, } -macro_rules! operand128_view { - ($name:ident, $decode:expr, $encode:expr) => { +macro_rules! operand_raw_view { + ($name:ident, $raw:ty, $decode:expr, $encode:expr) => { impl sealed::Sealed for $name {} impl OperandType for $name { - type Raw = Operand128; + type Raw = $raw; #[inline(always)] fn decode(raw: Self::Raw) -> Self { @@ -264,24 +251,27 @@ macro_rules! operand128_view { }; } -operand128_view!(V128Operand, |raw: Operand128| Self { value: raw.to_le_bytes() }, |value: Self| { +operand_raw_view!(V128Operand, Operand128, |raw: Operand128| Self { value: raw.to_le_bytes() }, |value: Self| { Operand128::from_le_bytes(value.value) }); -operand128_view!( +operand_raw_view!( LocalV128, - |raw: Operand128| Self { local: raw.u16(0), value: OperandIdx::new(raw.u32(2)) }, - |value: Self| { Operand128::default().with_u16(0, value.local).with_u32(2, value.value.index()) } + Operand64, + |raw: Operand64| Self { local: raw.u16::<0>(), value: OperandIdx::new(raw.u32::<2>()) }, + |value: Self| { Operand64::default().with_u16::<0>(value.local).with_u32::<2>(value.value.index()) } ); -operand128_view!( +operand_raw_view!( GlobalV128, - |raw: Operand128| Self { global: raw.u32(0), value: OperandIdx::new(raw.u32(4)) }, - |value: Self| { Operand128::default().with_u32(0, value.global).with_u32(4, value.value.index()) } + Operand64, + |raw: Operand64| Self { global: raw.u32::<0>(), value: OperandIdx::new(raw.u32::<4>()) }, + |value: Self| { Operand64::default().with_u32::<0>(value.global).with_u32::<4>(value.value.index()) } ); -operand128_view!( +operand_raw_view!( LocalConstSetV128, - |raw: Operand128| Self { local: raw.u16(0), dst: raw.u16(2), value: OperandIdx::new(raw.u32(4)) }, + Operand64, + |raw: Operand64| Self { local: raw.u16::<0>(), dst: raw.u16::<2>(), value: OperandIdx::new(raw.u32::<4>()) }, |value: Self| { - Operand128::default().with_u16(0, value.local).with_u16(2, value.dst).with_u32(4, value.value.index()) + Operand64::default().with_u16::<0>(value.local).with_u16::<2>(value.dst).with_u32::<4>(value.value.index()) } ); @@ -359,12 +349,12 @@ impl OperandType for MemoryArg { #[inline(always)] fn decode(raw: Self::Raw) -> Self { - Self::new(raw.u64(0), raw.u32(8)) + Self::new(raw.u64::<0>(), raw.u32::<8>()) } #[inline] fn encode(self) -> Self::Raw { - Operand128::default().with_u64(0, self.offset).with_u32(8, self.mem_addr) + Operand128::default().with_u64::<0>(self.offset).with_u32::<8>(self.mem_addr) } } @@ -389,12 +379,12 @@ impl OperandType for CompactMemoryArg { #[inline(always)] fn decode(raw: Self::Raw) -> Self { - Self { offset: raw.u32(0), mem_addr: raw.u16(4) } + Self { offset: raw.u32::<0>(), mem_addr: raw.u16::<4>() } } #[inline] fn encode(self) -> Self::Raw { - Operand64::default().with_u32(0, self.offset).with_u16(4, self.mem_addr) + Operand64::default().with_u32::<0>(self.offset).with_u16::<4>(self.mem_addr) } } @@ -424,7 +414,6 @@ const _: () = { assert!(core::mem::size_of::() == 6); assert!(core::mem::size_of::() == 6); assert!(core::mem::size_of::() == 6); - assert!(core::mem::size_of::() == 6); }; #[derive(Clone, Copy, PartialEq)] @@ -555,7 +544,7 @@ pub enum BinOp128 { pub enum Instruction { LocalCopy32(LocalAddr, LocalAddr), LocalCopy64(LocalAddr, LocalAddr), LocalCopy128(LocalAddr, LocalAddr), AddConst32(i32), AddConst64(OperandIdx), - IncLocal32(I32LocalArg), IncLocal64(OperandIdx), + IncLocal32(I32LocalArg), IncLocal64(PackedOp), // The 32/64 suffix describes the operand width. Future compare-style ops may still yield i32 results. BinOpLocalLocal32(BinOp, LocalAddr, LocalAddr), BinOpLocalLocal64(BinOp, LocalAddr, LocalAddr), BinOpLocalLocal128(BinOp128, LocalAddr, LocalAddr), @@ -575,7 +564,7 @@ pub enum Instruction { BinOpStackLocal128(BinOp128, LocalAddr), BinOpStackGlobal32(BinOp, u32), BinOpStackGlobal64(BinOp, u32), - SetLocalConst32(I32LocalArg), SetLocalConst64(OperandIdx), SetLocalConst128(V128LocalArg), + SetLocalConst32(I32LocalArg), SetLocalConst64(PackedOp), SetLocalConst128(PackedOp), IncMemoryLocal32(MemoryLocalArg), IncMemoryLocal64(MemoryLocalArg), StoreLocalLocal32(MemoryLocalArg), StoreLocalLocal64(MemoryLocalArg), StoreLocalLocal128(MemoryLocalArg), LoadLocal32(MemoryLocalArg), LoadLocal64(MemoryLocalArg), @@ -588,7 +577,7 @@ pub enum Instruction { LoadLocalSet16S32(MemoryLocalArg), LoadLocalSet16U32(MemoryLocalArg), LoadLocalTee128(MemoryLocalArg), LoadLocalSet128(MemoryLocalArg), AndConstTee32(I32LocalArg), SubConstTee32(I32LocalArg), - AndConstTee64(OperandIdx), SubConstTee64(OperandIdx), + AndConstTee64(PackedOp), SubConstTee64(PackedOp), MulAccLocal32(LocalAddr), MulAccLocal64(LocalAddr), FMulAccLocal32(LocalAddr), FMulAccLocal64(LocalAddr), I32Add3, @@ -671,6 +660,7 @@ pub enum Instruction { RefTest(RefType), RefCast(RefType), BrOnCast(OperandIdx), + BrOnCastFail(OperandIdx), // > GC Objects StructNew(TypeAddr), @@ -904,7 +894,6 @@ mod tests { assert_eq!(core::mem::size_of::(), 6); assert_eq!(core::mem::size_of::(), 6); assert_eq!(core::mem::size_of::(), 6); - assert_eq!(core::mem::size_of::(), 6); } #[test] @@ -912,9 +901,15 @@ mod tests { let memory = MemoryArg::new(u64::MAX, u32::MAX); assert_eq!(MemoryArg::decode(memory.encode()), memory); - let value = - Operand128::default().with_u8(0, u8::MAX).with_u16(1, u16::MAX).with_u32(3, u32::MAX).with_u64(7, u64::MAX); - assert_eq!((value.u8(0), value.u16(1), value.u32(3), value.u64(7)), (u8::MAX, u16::MAX, u32::MAX, u64::MAX)); + let value = Operand128::default() + .with_u8::<0>(u8::MAX) + .with_u16::<1>(u16::MAX) + .with_u32::<3>(u32::MAX) + .with_u64::<7>(u64::MAX); + assert_eq!( + (value.u8::<0>(), value.u16::<1>(), value.u32::<3>(), value.u64::<7>()), + (u8::MAX, u16::MAX, u32::MAX, u64::MAX) + ); let bytes = [u8::MAX; 16]; assert_eq!(Operand128::from_le_bytes(bytes).to_le_bytes(), bytes); @@ -923,12 +918,12 @@ mod tests { #[test] fn v128_operand_views_round_trip_max_bytes() { let value = OperandIdx::::new(0); - let local = OperandIdx::::new(1); - let global = OperandIdx::::new(2); - let set = OperandIdx::::new(3); + let local = OperandIdx::::new(0); + let global = OperandIdx::::new(1); + let set = OperandIdx::::new(2); let data = super::super::WasmFunctionData { - operands128: vec![ - V128Operand { value: [u8::MAX; 16] }.encode(), + operands128: vec![V128Operand { value: [u8::MAX; 16] }.encode()].into_boxed_slice(), + operands64: vec![ LocalV128 { local: u16::MAX, value }.encode(), GlobalV128 { global: u32::MAX, value }.encode(), LocalConstSetV128 { local: u16::MAX, dst: u16::MAX, value }.encode(), diff --git a/crates/types/src/operands.rs b/crates/types/src/operands.rs index dd7e3a1..3cc311c 100644 --- a/crates/types/src/operands.rs +++ b/crates/types/src/operands.rs @@ -3,9 +3,12 @@ use core::marker::PhantomData; use crate::WasmFunctionData; /// An index into the operand lane used by `T`. +#[cfg_attr(feature = "archive", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "archive", serde(transparent))] #[repr(transparent)] pub struct OperandIdx { index: u32, + #[cfg_attr(feature = "archive", serde(skip))] marker: PhantomData, } @@ -56,43 +59,8 @@ impl core::fmt::Debug for OperandIdx { } } -#[cfg(feature = "archive")] -impl serde::Serialize for OperandIdx { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - serde::Serialize::serialize(&self.index, serializer) - } -} - -#[cfg(feature = "archive")] -impl<'de, T> serde::Deserialize<'de> for OperandIdx { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - Ok(Self::new(serde::Deserialize::deserialize(deserializer)?)) - } -} - -/// Raw fields stored in a per-function 64-bit operand lane. -#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Default)] -#[cfg_attr(feature = "debug", derive(Debug))] -#[cfg_attr(feature = "archive", derive(serde::Serialize, serde::Deserialize))] -#[repr(transparent)] -pub struct Operand64(u64); - -/// Raw fields stored in a per-function 128-bit operand lane. -#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Default)] -#[cfg_attr(feature = "debug", derive(Debug))] -#[cfg_attr(feature = "archive", derive(serde::Serialize, serde::Deserialize))] -#[repr(transparent)] -pub struct Operand128(u128); - pub(crate) mod sealed { pub trait Sealed {} - pub trait RawSealed {} } /// A typed view stored in one of the per-function operand lanes. @@ -107,112 +75,103 @@ pub trait OperandType: sealed::Sealed + Copy { } #[doc(hidden)] -pub trait RawOperand: sealed::RawSealed + Copy { +pub trait RawOperand: sealed::Sealed + Copy { fn get(data: &WasmFunctionData, index: u32) -> Self; } -impl sealed::RawSealed for Operand64 {} -impl RawOperand for Operand64 { - #[inline(always)] - fn get(data: &WasmFunctionData, index: u32) -> Self { - *data.operands64.get(index as usize).unwrap_or_else(|| unreachable!("invalid operand index")) - } -} +macro_rules! define_operand { + ($name:ident, $len:literal, $lane:ident) => { + #[doc = concat!("Raw fields stored in a per-function ", $len, "-byte operand lane.")] + #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Default)] + #[cfg_attr(feature = "debug", derive(Debug))] + #[cfg_attr(feature = "archive", derive(serde::Serialize, serde::Deserialize))] + #[repr(transparent)] + pub struct $name([u8; $len]); -impl sealed::RawSealed for Operand128 {} -impl RawOperand for Operand128 { - #[inline(always)] - fn get(data: &WasmFunctionData, index: u32) -> Self { - *data.operands128.get(index as usize).unwrap_or_else(|| unreachable!("invalid operand index")) - } -} + impl sealed::Sealed for $name {} -macro_rules! operand_fields { - ($ty:ty) => { - #[inline] - pub(crate) const fn u16(self, offset: u32) -> u16 { - (self.0 >> (offset * 8)) as u16 + impl RawOperand for $name { + #[inline(always)] + fn get(data: &WasmFunctionData, index: u32) -> Self { + *data.$lane.get(index as usize).unwrap_or_else(|| unreachable!("invalid operand index")) + } } - #[inline] - pub(crate) const fn u32(self, offset: u32) -> u32 { - (self.0 >> (offset * 8)) as u32 - } - #[inline] - pub(crate) const fn u64(self, offset: u32) -> u64 { - (self.0 >> (offset * 8)) as u64 - } - #[inline] - pub(crate) const fn i64(self, offset: u32) -> i64 { - self.u64(offset) as i64 + + impl $name { + /// Returns the byte at `OFFSET`. + #[inline] + pub fn u8(self) -> u8 { + const { assert!(OFFSET < $len) }; + self.0[OFFSET] + } + + /// Returns the little-endian `u16` at `OFFSET`. + #[inline] + pub fn u16(self) -> u16 { + const { assert!(OFFSET + 2 <= $len) }; + u16::from_le_bytes(self.0[OFFSET..OFFSET + 2].try_into().unwrap_or_else(|_| unreachable!())) + } + + /// Returns the little-endian `u32` at `OFFSET`. + #[inline] + pub fn u32(self) -> u32 { + const { assert!(OFFSET + 4 <= $len) }; + u32::from_le_bytes(self.0[OFFSET..OFFSET + 4].try_into().unwrap_or_else(|_| unreachable!())) + } + + /// Returns the little-endian `u64` at `OFFSET`. + #[inline] + pub fn u64(self) -> u64 { + const { assert!(OFFSET + 8 <= $len) }; + u64::from_le_bytes(self.0[OFFSET..OFFSET + 8].try_into().unwrap_or_else(|_| unreachable!())) + } + + /// Builds an operand from its raw little-endian bytes. + #[inline] + pub fn from_le_bytes(value: [u8; $len]) -> Self { + Self(value) + } + + /// Returns the raw little-endian bytes. + #[inline] + pub fn to_le_bytes(self) -> [u8; $len] { + self.0 + } + + /// Writes `value` at `OFFSET` and returns the updated operand. + #[inline] + pub fn with_u8(mut self, value: u8) -> Self { + const { assert!(OFFSET < $len) }; + self.0[OFFSET] = value; + self + } + + /// Writes `value` at `OFFSET` and returns the updated operand. + #[inline] + pub fn with_u16(mut self, value: u16) -> Self { + const { assert!(OFFSET + 2 <= $len) }; + self.0[OFFSET..OFFSET + 2].copy_from_slice(&value.to_le_bytes()); + self + } + + /// Writes `value` at `OFFSET` and returns the updated operand. + #[inline] + pub fn with_u32(mut self, value: u32) -> Self { + const { assert!(OFFSET + 4 <= $len) }; + self.0[OFFSET..OFFSET + 4].copy_from_slice(&value.to_le_bytes()); + self + } + + /// Writes `value` at `OFFSET` and returns the updated operand. + #[inline] + pub fn with_u64(mut self, value: u64) -> Self { + const { assert!(OFFSET + 8 <= $len) }; + self.0[OFFSET..OFFSET + 8].copy_from_slice(&value.to_le_bytes()); + self + } } }; } -impl Operand64 { - operand_fields!(u64); - - #[inline] - pub(crate) const fn with_u16(self, offset: u32, value: u16) -> Self { - let shift = offset * 8; - Self((self.0 & !((u16::MAX as u64) << shift)) | ((value as u64) << shift)) - } - - #[inline] - pub(crate) const fn with_u32(self, offset: u32, value: u32) -> Self { - let shift = offset * 8; - Self((self.0 & !((u32::MAX as u64) << shift)) | ((value as u64) << shift)) - } - - #[inline] - pub(crate) const fn with_u64(self, _offset: u32, value: u64) -> Self { - Self(value) - } -} - -impl Operand128 { - operand_fields!(u128); - - #[inline] - pub(crate) const fn u8(self, offset: u32) -> u8 { - (self.0 >> (offset * 8)) as u8 - } - - #[inline] - pub(crate) const fn i32(self, offset: u32) -> i32 { - self.u32(offset) as i32 - } - - #[inline] - pub(crate) const fn from_le_bytes(value: [u8; 16]) -> Self { - Self(u128::from_le_bytes(value)) - } - - #[inline] - pub(crate) const fn to_le_bytes(self) -> [u8; 16] { - self.0.to_le_bytes() - } - - #[inline] - pub(crate) const fn with_u8(self, offset: u32, value: u8) -> Self { - let shift = offset * 8; - Self((self.0 & !((u8::MAX as u128) << shift)) | ((value as u128) << shift)) - } - - #[inline] - pub(crate) const fn with_u16(self, offset: u32, value: u16) -> Self { - let shift = offset * 8; - Self((self.0 & !((u16::MAX as u128) << shift)) | ((value as u128) << shift)) - } - - #[inline] - pub(crate) const fn with_u32(self, offset: u32, value: u32) -> Self { - let shift = offset * 8; - Self((self.0 & !((u32::MAX as u128) << shift)) | ((value as u128) << shift)) - } - - #[inline] - pub(crate) const fn with_u64(self, offset: u32, value: u64) -> Self { - let shift = offset * 8; - Self((self.0 & !((u64::MAX as u128) << shift)) | ((value as u128) << shift)) - } -} +define_operand!(Operand64, 8, operands64); +define_operand!(Operand128, 16, operands128); From 29c3cbc2ad5f41371345b4d37c02a131323028b0 Mon Sep 17 00:00:00 2001 From: Henry Date: Mon, 17 Aug 2026 18:55:39 +0200 Subject: [PATCH 3/3] ci: fix build Signed-off-by: Henry --- examples/rust/src/print.twasm | Bin 0 -> 154 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 examples/rust/src/print.twasm diff --git a/examples/rust/src/print.twasm b/examples/rust/src/print.twasm new file mode 100644 index 0000000000000000000000000000000000000000..f0fa9a87c3b90554afabdd92675a76a7a92f0820 GIT binary patch literal 154 zcmYL@!3qK~3`A#IQBedxMHD~4Z&2_e#7GM6Ve7Ikg13IUX)lJ52@htH%k^~L9q}*5 z1;9IQjza6Y&>85nqCf_4Ht$pa8ovP>H51Jy!`S5sT#1-YDk)pUTI88@kcYlgbf4#x WQd(Y9V(SMOQ~oTQzdP*fH~ay3HX1|# literal 0 HcmV?d00001