diff --git a/crates/cranelift/src/func_environ/gc/enabled.rs b/crates/cranelift/src/func_environ/gc/enabled.rs index 0ed775f7f3c8..0a65d1c8a839 100644 --- a/crates/cranelift/src/func_environ/gc/enabled.rs +++ b/crates/cranelift/src/func_environ/gc/enabled.rs @@ -105,7 +105,7 @@ fn unbarriered_store_gc_ref( Ok(()) } -/// Emit CLIF to call the `gc_raw_alloc` libcall. +/// Emit CLIF to call the `gc_alloc_raw` libcall. #[cfg(any(feature = "gc-drc", feature = "gc-copying"))] fn emit_gc_raw_alloc( func_env: &mut FuncEnvironment<'_>, diff --git a/crates/cranelift/src/func_environ/gc/enabled/copying.rs b/crates/cranelift/src/func_environ/gc/enabled/copying.rs index 1c82ddea03bb..336f2df48741 100644 --- a/crates/cranelift/src/func_environ/gc/enabled/copying.rs +++ b/crates/cranelift/src/func_environ/gc/enabled/copying.rs @@ -1,7 +1,8 @@ //! Compiler for the copying (semi-space/Cheney) collector. //! -//! Allocation is performed via the `gc_alloc_raw` libcall (not inlined) for -//! now. Read and write barriers are unnecessary (e.g. no reference counting and +//! Allocation is performed inline with a bump pointer when possible, falling +//! back to the `gc_alloc_raw` libcall when the active semi-space is full. +//! Read and write barriers are unnecessary (e.g. no reference counting and //! no concurrent mutation during collection) but we do need stack maps so the //! collector can find and update roots when it relocates objects. @@ -11,11 +12,13 @@ use crate::func_environ::FuncEnvironment; use crate::translate::TargetEnvironment; use cranelift_codegen::ir::{self, InstBuilder}; use cranelift_frontend::FunctionBuilder; -use smallvec::SmallVec; -use wasmtime_environ::copying::{EXCEPTION_TAG_DEFINED_OFFSET, EXCEPTION_TAG_INSTANCE_OFFSET}; +use wasmtime_environ::copying::{ + ALIGN, EXCEPTION_TAG_DEFINED_OFFSET, EXCEPTION_TAG_INSTANCE_OFFSET, +}; use wasmtime_environ::{ - GcTypeLayouts, TypeIndex, VMGcKind, WasmHeapTopType, WasmHeapType, WasmRefType, WasmResult, - WasmStorageType, WasmValType, copying::CopyingTypeLayouts, + GcTypeLayouts, ModuleInternedTypeIndex, PtrSize, TypeIndex, VMGcKind, WasmHeapTopType, + WasmHeapType, WasmRefType, WasmResult, WasmStorageType, WasmValType, + copying::CopyingTypeLayouts, }; #[derive(Default)] @@ -24,6 +27,188 @@ pub struct CopyingCompiler { } impl CopyingCompiler { + /// Load the pointer to the `VMCopyingHeapData` from vmctx. + fn load_vmcopying_heap_data_ptr( + func_env: &mut FuncEnvironment<'_>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + let pointer_type = func_env.pointer_type(); + let vmctx = func_env.vmctx_val(&mut builder.cursor()); + builder.ins().load( + pointer_type, + ir::MemFlags::trusted().with_readonly().with_can_move(), + vmctx, + i32::from(func_env.offsets.ptr.vmctx_gc_heap_data()), + ) + } + + /// Load the current bump pointer and active-space end from a `*mut + /// VMCopyingHeapData`. + /// + /// Returns `(bump_ptr, active_space_end)` as `i32` values. + fn load_bump_state( + func_env: &mut FuncEnvironment<'_>, + builder: &mut FunctionBuilder, + ptr_to_heap_data: ir::Value, + ) -> (ir::Value, ir::Value) { + let bump_ptr = builder.ins().load( + ir::types::I32, + ir::MemFlags::trusted().with_can_move(), + ptr_to_heap_data, + i32::from(func_env.offsets.ptr.vmcopying_heap_data_bump_ptr()), + ); + let active_space_end = builder.ins().load( + ir::types::I32, + ir::MemFlags::trusted().with_readonly().with_can_move(), + ptr_to_heap_data, + i32::from(func_env.offsets.ptr.vmcopying_heap_data_active_space_end()), + ); + (bump_ptr, active_space_end) + } + + /// Round `size` (an `i32`) up to `ALIGN`, returning the result as an `i64`. + /// + /// Uses `i64` arithmetic so that overflow produces a value larger than any + /// valid heap index, which sends us to the slow allocation path instead of + /// wrapping around. + fn aligned_size(builder: &mut FunctionBuilder, size: ir::Value) -> ir::Value { + let size_64 = builder.ins().uextend(ir::types::I64, size); + let align_mask = builder.ins().iconst(ir::types::I64, i64::from(ALIGN - 1)); + let inv_align_mask = builder.ins().iconst(ir::types::I64, !i64::from(ALIGN - 1)); + let size_plus_mask = builder.ins().iadd(size_64, align_mask); + builder.ins().band(size_plus_mask, inv_align_mask) + } + + /// Emit inline bump allocation, falling back to `gc_alloc_raw` on failure. + /// + /// `size` must be an `i32` value >= `size_of(VMCopyingHeader)`. + /// + /// Returns `(gc_ref, raw_ptr_to_object)` where `gc_ref` is the `i32` GC + /// heap index and `raw_ptr_to_object` is a native pointer into the GC heap. + fn emit_inline_alloc( + &mut self, + func_env: &mut FuncEnvironment<'_>, + builder: &mut FunctionBuilder, + kind: VMGcKind, + ty: ModuleInternedTypeIndex, + size: ir::Value, + ) -> (ir::Value, ir::Value) { + debug_assert_ne!(kind, VMGcKind::ExternRef); + debug_assert!(!ty.is_reserved_value()); + assert_eq!(builder.func.dfg.value_type(size), ir::types::I32); + + let pointer_type = func_env.pointer_type(); + let current_block = builder.current_block().unwrap(); + let fast_block = builder.create_block(); + let slow_block = builder.create_block(); + let merge_block = builder.create_block(); + + builder.ensure_inserted_block(); + builder.insert_block_after(fast_block, current_block); + builder.insert_block_after(slow_block, fast_block); + builder.insert_block_after(merge_block, slow_block); + + let ptr_to_heap_data = Self::load_vmcopying_heap_data_ptr(func_env, builder); + let (bump_ptr, active_space_end) = + Self::load_bump_state(func_env, builder, ptr_to_heap_data); + let aligned_size_64 = Self::aligned_size(builder, size); + + // Compute `end_of_object = bump_ptr + aligned_size` (in i64) and check + // whether it fits within the active semi-space. + let bump_ptr_64 = builder.ins().uextend(ir::types::I64, bump_ptr); + let end_64 = builder.ins().iadd(bump_ptr_64, aligned_size_64); + let active_space_end_64 = builder.ins().uextend(ir::types::I64, active_space_end); + let fits = builder.ins().icmp( + ir::condcodes::IntCC::UnsignedLessThanOrEqual, + end_64, + active_space_end_64, + ); + builder.ins().brif(fits, fast_block, &[], slow_block, &[]); + + // Slow path: when there isn't enough room in the bump region, call the + // `gc_alloc_raw` libcall, which will collect or grow the GC heap as + // necessary. + { + builder.switch_to_block(slow_block); + builder.seal_block(slow_block); + builder.set_cold_block(slow_block); + let gc_ref = emit_gc_raw_alloc(func_env, builder, kind, ty, size, ALIGN); + let base = func_env.get_gc_heap_base(builder); + let heap_offset = uextend_i32_to_pointer_type(builder, pointer_type, gc_ref); + let obj_ptr = builder.ins().iadd(base, heap_offset); + builder + .ins() + .jump(merge_block, &[gc_ref.into(), obj_ptr.into()]); + } + + // Fast path: there is capacity for the requested object in the bump + // region, so finish the allocation inline, update our bump pointer, + // etc... + { + builder.switch_to_block(fast_block); + builder.seal_block(fast_block); + + // The old bump_ptr is the start of the new object. + let gc_ref = bump_ptr; + + // Update the bump pointer. + let end_of_object = builder.ins().ireduce(ir::types::I32, end_64); + builder.ins().store( + ir::MemFlags::trusted().with_alias_region(Some(ir::AliasRegion::Vmctx)), + end_of_object, + ptr_to_heap_data, + i32::from(func_env.offsets.ptr.vmcopying_heap_data_bump_ptr()), + ); + + // Compute the raw pointer to the new object. + let base = func_env.get_gc_heap_base(builder); + let heap_offset = uextend_i32_to_pointer_type(builder, pointer_type, gc_ref); + let obj_ptr = builder.ins().iadd(base, heap_offset); + + // Write `VMGcHeader::kind`. + let kind_val = builder + .ins() + .iconst(ir::types::I32, i64::from(kind.as_u32())); + builder.ins().store( + ir::MemFlags::trusted(), + kind_val, + obj_ptr, + i32::try_from(wasmtime_environ::VM_GC_HEADER_KIND_OFFSET).unwrap(), + ); + + // Write `VMGcHeader::type_index`. + let shared_ty = func_env.module_interned_to_shared_ty(&mut builder.cursor(), ty); + builder.ins().store( + ir::MemFlags::trusted(), + shared_ty, + obj_ptr, + i32::try_from(wasmtime_environ::VM_GC_HEADER_TYPE_INDEX_OFFSET).unwrap(), + ); + + // Write `VMCopyingHeader::object_size`. + builder.ins().istore32( + ir::MemFlags::trusted(), + aligned_size_64, + obj_ptr, + i32::try_from(wasmtime_environ::VM_GC_HEADER_SIZE).unwrap(), + ); + + builder + .ins() + .jump(merge_block, &[gc_ref.into(), obj_ptr.into()]); + } + + // Merge block: takes the GC ref and the raw pointer to the GC object as + // block parameters. + builder.switch_to_block(merge_block); + let gc_ref = builder.append_block_param(merge_block, ir::types::I32); + let ptr_to_object = builder.append_block_param(merge_block, pointer_type); + builder.seal_block(merge_block); + builder.declare_value_needs_stack_map(gc_ref); + + (gc_ref, ptr_to_object) + } + fn init_field( &mut self, func_env: &mut FuncEnvironment<'_>, @@ -85,28 +270,20 @@ impl GcCompiler for CopyingCompiler { let len_offset = gc_compiler(func_env)?.layouts().array_length_field_offset(); let array_layout = func_env.array_layout(interned_type_index).clone(); let base_size = array_layout.base_size; - let align = array_layout.align; let len_to_elems_delta = base_size.checked_sub(len_offset).unwrap(); // First, compute the array's total size. let len = init.len(&mut builder.cursor()); let size = emit_array_size(func_env, builder, &array_layout, len); - // Allocate via libcall. - let array_ref = emit_gc_raw_alloc( + // Allocate inline (with fallback to libcall). + let (array_ref, object_addr) = self.emit_inline_alloc( func_env, builder, VMGcKind::ArrayRef, interned_type_index, size, - align, ); - - // Write the array's length. - let base = func_env.get_gc_heap_base(builder); - let extended_array_ref = - uextend_i32_to_pointer_type(builder, func_env.pointer_type(), array_ref); - let object_addr = builder.ins().iadd(base, extended_array_ref); let len_addr = builder.ins().iadd_imm(object_addr, i64::from(len_offset)); let len = init.len(&mut builder.cursor()); builder @@ -142,26 +319,18 @@ impl GcCompiler for CopyingCompiler { let struct_layout = func_env.struct_or_exn_layout(interned_type_index); let struct_size = struct_layout.size; - let struct_align = struct_layout.align; - let field_offsets: SmallVec<[_; 8]> = struct_layout.fields.iter().copied().collect(); - assert_eq!(field_vals.len(), field_offsets.len()); let struct_size_val = builder.ins().iconst(ir::types::I32, i64::from(struct_size)); - let struct_ref = emit_gc_raw_alloc( + let (struct_ref, raw_ptr_to_struct) = self.emit_inline_alloc( func_env, builder, VMGcKind::StructRef, interned_type_index, struct_size_val, - struct_align, ); // Initialize fields. - let base = func_env.get_gc_heap_base(builder); - let extended_struct_ref = - uextend_i32_to_pointer_type(builder, func_env.pointer_type(), struct_ref); - let raw_ptr_to_struct = builder.ins().iadd(base, extended_struct_ref); initialize_struct_fields( func_env, builder, @@ -191,26 +360,18 @@ impl GcCompiler for CopyingCompiler { let exn_layout = func_env.struct_or_exn_layout(interned_type_index); let exn_size = exn_layout.size; - let exn_align = exn_layout.align; - let field_offsets: SmallVec<[_; 8]> = exn_layout.fields.iter().copied().collect(); - assert_eq!(field_vals.len(), field_offsets.len()); let exn_size_val = builder.ins().iconst(ir::types::I32, i64::from(exn_size)); - let exn_ref = emit_gc_raw_alloc( + let (exn_ref, raw_ptr_to_exn) = self.emit_inline_alloc( func_env, builder, VMGcKind::ExnRef, interned_type_index, exn_size_val, - exn_align, ); // Initialize fields. - let base = func_env.get_gc_heap_base(builder); - let extended_exn_ref = - uextend_i32_to_pointer_type(builder, func_env.pointer_type(), exn_ref); - let raw_ptr_to_exn = builder.ins().iadd(base, extended_exn_ref); initialize_struct_fields( func_env, builder, diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs index d73193ab56b1..5e5b9e7df417 100644 --- a/crates/environ/src/vmoffsets.rs +++ b/crates/environ/src/vmoffsets.rs @@ -480,6 +480,31 @@ pub trait PtrSize { self.vmctx_epoch_ptr() + self.size() } + /// Return the offset of the `bump_ptr` field within `VMCopyingHeapData`. + #[inline] + fn vmcopying_heap_data_bump_ptr(&self) -> u8 { + 0 + } + + /// Return the offset of the `active_space_end` field within + /// `VMCopyingHeapData`. + #[inline] + fn vmcopying_heap_data_active_space_end(&self) -> u8 { + 4 + } + + /// Return the size of `VMCopyingHeapData`. + #[inline] + fn size_of_vmcopying_heap_data(&self) -> u8 { + 8 + } + + /// Return the alignment of `VMCopyingHeapData`. + #[inline] + fn align_of_vmcopying_heap_data(&self) -> u8 { + 4 + } + /// The offset of the `type_ids` array pointer. #[inline] fn vmctx_type_ids_array(&self) -> u8 { diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs index a5d70100bc77..48f559160512 100644 --- a/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs +++ b/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs @@ -171,6 +171,18 @@ unsafe impl GcHeapObject for VMCopyingExternRef { } } +/// JIT-accessible bump-allocation state for the copying collector. +/// +/// NB: Layout is defined by constants in `wasmtime_environ::copying`. Keep in +/// sync! +#[repr(C)] +struct VMCopyingHeapData { + /// Current bump pointer (index into the GC heap). + bump_ptr: u32, + /// End of the active semi-space. + active_space_end: u32, +} + /// Get a typed reference to a copying-collector object from a raw `VMGcRef`. fn copying_ref(gc_ref: &VMGcRef) -> &TypedGcRef { debug_assert!(!gc_ref.is_i31()); @@ -207,17 +219,15 @@ struct CopyingHeap { /// memory is taken and updated when the memory is replaced. vmmemory: Option, - /// The bump "pointer" (really an index) for allocating new objects. + /// JIT-accessible allocation state: bump pointer and active-space end. /// - /// This is always within the active semi-space. - bump_ptr: u32, + /// NB: The bump pointer is written to by compiled Wasm code via the pointer + /// returned by `vmctx_gc_heap_data`. + vmctx_data: VMCopyingHeapData, /// The start of the active semi-space. active_space_start: u32, - /// The end of the active semi-space. - active_space_end: u32, - /// The start of the idle semi-space. idle_space_start: u32, @@ -254,9 +264,11 @@ impl CopyingHeap { no_gc_count: 0, memory: None, vmmemory: None, - bump_ptr: 0, + vmctx_data: VMCopyingHeapData { + bump_ptr: 0, + active_space_end: 0, + }, active_space_start: 0, - active_space_end: 0, idle_space_start: 0, idle_space_end: 0, worklist_ptr: 0, @@ -265,6 +277,22 @@ impl CopyingHeap { }) } + fn bump_ptr(&self) -> u32 { + self.vmctx_data.bump_ptr + } + + fn set_bump_ptr(&mut self, val: u32) { + self.vmctx_data.bump_ptr = val; + } + + fn active_space_end(&self) -> u32 { + self.vmctx_data.active_space_end + } + + fn set_active_space_end(&mut self, val: u32) { + self.vmctx_data.active_space_end = val; + } + fn capacity(&self) -> u32 { let len = self.vmmemory.as_ref().unwrap().current_length(); let len = u32::try_from(len).unwrap_or(u32::MAX); @@ -277,10 +305,10 @@ impl CopyingHeap { /// Initialize the semi-spaces for a heap of the given capacity. fn initialize_semi_spaces(&mut self) { debug_assert_eq!(self.active_space_start, 0); - debug_assert_eq!(self.active_space_end, 0); + debug_assert_eq!(self.active_space_end(), 0); debug_assert_eq!(self.idle_space_start, 0); debug_assert_eq!(self.idle_space_end, 0); - debug_assert_eq!(self.bump_ptr, 0); + debug_assert_eq!(self.bump_ptr(), 0); self.resize_semi_spaces(); self.reset_bump_ptr(); @@ -288,7 +316,7 @@ impl CopyingHeap { fn resize_semi_spaces(&mut self) { debug_assert_eq!( - self.active_space_end - self.active_space_start, + self.active_space_end() - self.active_space_start, self.idle_space_end - self.idle_space_start, "the active and idle spaces should be the same size" ); @@ -304,16 +332,16 @@ impl CopyingHeap { let capacity = self.capacity(); let halfway = capacity / 2; - debug_assert!(self.bump_ptr <= halfway); + debug_assert!(self.bump_ptr() <= halfway); debug_assert!(self.idle_space_start <= halfway); - debug_assert!(self.active_space_end <= halfway); + debug_assert!(self.active_space_end() <= halfway); - self.active_space_end = halfway; + self.set_active_space_end(halfway); self.idle_space_start = halfway; self.idle_space_end = capacity; debug_assert_eq!( - self.active_space_end - self.active_space_start, + self.active_space_end() - self.active_space_start, self.idle_space_end - self.idle_space_start, "the active and idle spaces should be the same size" ); @@ -329,11 +357,12 @@ impl CopyingHeap { // skip the first `ALIGN` bytes. This ensures that the active and idle // spaces are always equally sized, which is required to guarantee that // evacuating objects from one to the other will succeed. - self.bump_ptr = self.active_space_start; - if self.active_space_end - self.active_space_start >= ALIGN { - self.bump_ptr += ALIGN; + let mut bp = self.active_space_start; + if self.active_space_end() - self.active_space_start >= ALIGN { + bp += ALIGN; } - debug_assert!(self.bump_ptr.is_multiple_of(ALIGN)); + self.set_bump_ptr(bp); + debug_assert!(self.bump_ptr().is_multiple_of(ALIGN)); } /// Allocate `size` bytes from the active semi-space bump pointer. @@ -341,27 +370,27 @@ impl CopyingHeap { /// Returns `None` if there isn't enough room. fn allocate(&mut self, size: u32) -> Option { debug_assert!(size.is_multiple_of(ALIGN)); - debug_assert!(self.bump_ptr.is_multiple_of(ALIGN)); - debug_assert!(self.bump_ptr >= self.active_space_start); - debug_assert!(self.bump_ptr <= self.active_space_end); + debug_assert!(self.bump_ptr().is_multiple_of(ALIGN)); + debug_assert!(self.bump_ptr() >= self.active_space_start); + debug_assert!(self.bump_ptr() <= self.active_space_end()); - let result = self.bump_ptr; + let result = self.bump_ptr(); let new_bump_ptr = result.checked_add(size)?; - if new_bump_ptr > self.active_space_end { + if new_bump_ptr > self.active_space_end() { return None; } - self.bump_ptr = new_bump_ptr; - debug_assert!(self.bump_ptr.is_multiple_of(ALIGN)); - debug_assert!(self.bump_ptr >= self.active_space_start); - debug_assert!(self.bump_ptr <= self.active_space_end); + self.set_bump_ptr(new_bump_ptr); + debug_assert!(self.bump_ptr().is_multiple_of(ALIGN)); + debug_assert!(self.bump_ptr() >= self.active_space_start); + debug_assert!(self.bump_ptr() <= self.active_space_end()); Some(result) } /// Check whether an index is within the active semi-space. fn is_in_active_space(&self, index: u32) -> bool { - index >= self.active_space_start && index < self.active_space_end + index >= self.active_space_start && index < self.active_space_end() } /// Check whether an index is within the idle semi-space. @@ -372,13 +401,15 @@ impl CopyingHeap { /// Swap the active and idle semi-spaces. fn flip(&mut self) { debug_assert_eq!( - self.active_space_end - self.active_space_start, + self.active_space_end() - self.active_space_start, self.idle_space_end - self.idle_space_start, "the active and idle spaces should be the same size" ); mem::swap(&mut self.active_space_start, &mut self.idle_space_start); - mem::swap(&mut self.active_space_end, &mut self.idle_space_end); + let new_active_space_end = self.idle_space_end; + self.idle_space_end = self.active_space_end(); + self.set_active_space_end(new_active_space_end); self.reset_bump_ptr(); // The active idle list becomes the old active list; the active list is @@ -388,7 +419,7 @@ impl CopyingHeap { /// Initialize the worklist at the start of a collection. fn initialize_worklist(&mut self) { - self.worklist_ptr = self.bump_ptr; + self.worklist_ptr = self.bump_ptr(); } /// Pop the next item off the worklist, or return `None` if the worklist is @@ -396,14 +427,14 @@ impl CopyingHeap { fn worklist_pop(&mut self) -> Option { debug_assert!( self.is_in_active_space(self.worklist_ptr) - || self.worklist_ptr == self.active_space_end + || self.worklist_ptr == self.active_space_end() ); debug_assert!( - self.is_in_active_space(self.bump_ptr) || self.bump_ptr == self.active_space_end + self.is_in_active_space(self.bump_ptr()) || self.bump_ptr() == self.active_space_end() ); - debug_assert!(self.worklist_ptr <= self.bump_ptr); + debug_assert!(self.worklist_ptr <= self.bump_ptr()); - if self.worklist_ptr == self.bump_ptr { + if self.worklist_ptr == self.bump_ptr() { return None; } @@ -414,7 +445,7 @@ impl CopyingHeap { let obj_size = self.index(copying_ref(&result)).object_size(); self.worklist_ptr += obj_size; - debug_assert!(self.worklist_ptr <= self.bump_ptr); + debug_assert!(self.worklist_ptr <= self.bump_ptr()); Some(result) } @@ -432,7 +463,7 @@ impl CopyingHeap { let index = gc_ref.as_heap_index().unwrap().get(); debug_assert!(self.is_in_active_space(index)); let obj_size = self.index(copying_ref(gc_ref)).object_size(); - debug_assert_eq!(index + obj_size, self.bump_ptr); + debug_assert_eq!(index + obj_size, self.bump_ptr()); debug_assert!(self.worklist_ptr <= index); } @@ -580,13 +611,15 @@ unsafe impl GcHeap for CopyingHeap { fn detach(&mut self) -> crate::vm::Memory { assert!(self.is_attached()); + self.set_bump_ptr(0); + self.set_active_space_end(0); + let CopyingHeap { no_gc_count, memory, vmmemory, - bump_ptr, + vmctx_data: _, active_space_start, - active_space_end, idle_space_start, idle_space_end, worklist_ptr, @@ -600,9 +633,7 @@ unsafe impl GcHeap for CopyingHeap { *no_gc_count = 0; *vmmemory = None; - *bump_ptr = 0; *active_space_start = 0; - *active_space_end = 0; *idle_space_start = 0; *idle_space_end = 0; *worklist_ptr = 0; @@ -720,7 +751,11 @@ unsafe impl GcHeap for CopyingHeap { ); debug_assert!(layout.size() >= core::mem::size_of::()); - debug_assert_eq!(self.bump_ptr % ALIGN, 0, "bump_ptr is not aligned to ALIGN"); + debug_assert_eq!( + self.bump_ptr() % ALIGN, + 0, + "bump_ptr is not aligned to ALIGN" + ); debug_assert_eq!(header.reserved_u26(), 0); // We must have trace info for every GC type that we allocate in this @@ -830,7 +865,7 @@ unsafe impl GcHeap for CopyingHeap { } fn allocated_bytes(&self) -> usize { - usize::try_from(self.bump_ptr - self.active_space_start).unwrap() + usize::try_from(self.bump_ptr() - self.active_space_start).unwrap() } fn gc<'a>( @@ -848,9 +883,8 @@ unsafe impl GcHeap for CopyingHeap { } unsafe fn vmctx_gc_heap_data(&self) -> NonNull { - // The copying collector doesn't currently have vmctx GC heap - // data. Return a dangling pointer. - NonNull::dangling() + let ptr: *const VMCopyingHeapData = &self.vmctx_data; + NonNull::new(ptr as *mut u8).unwrap() } fn take_memory(&mut self) -> crate::vm::Memory { @@ -874,7 +908,7 @@ unsafe impl GcHeap for CopyingHeap { // If the heap was previously empty, reinitialize the semi-spaces from // scratch. - if self.active_space_end == 0 && self.idle_space_end == 0 { + if self.active_space_end() == 0 && self.idle_space_end == 0 { self.initialize_semi_spaces(); } else { // Otherwise the memory was grown: try to resize the semi-spaces @@ -982,11 +1016,11 @@ impl GarbageCollection<'_> for CopyingCollection<'_> { CopyingCollectionPhase::Collect => { log::trace!("Begin copying collection"); - assert!(self.heap.active_space_start <= self.heap.bump_ptr); - assert!(self.heap.bump_ptr <= self.heap.active_space_end); + assert!(self.heap.active_space_start <= self.heap.bump_ptr()); + assert!(self.heap.bump_ptr() <= self.heap.active_space_end()); assert!(self.heap.idle_space_start <= self.heap.idle_space_end); assert!( - self.heap.active_space_end <= self.heap.idle_space_start + self.heap.active_space_end() <= self.heap.idle_space_start || self.heap.idle_space_end <= self.heap.active_space_start ); @@ -997,11 +1031,11 @@ impl GarbageCollection<'_> for CopyingCollection<'_> { self.process_roots(); self.process_worklist(); - assert!(self.heap.active_space_start <= self.heap.bump_ptr); - assert!(self.heap.bump_ptr <= self.heap.active_space_end); + assert!(self.heap.active_space_start <= self.heap.bump_ptr()); + assert!(self.heap.bump_ptr() <= self.heap.active_space_end()); assert!(self.heap.idle_space_start <= self.heap.idle_space_end); assert!( - self.heap.active_space_end <= self.heap.idle_space_start + self.heap.active_space_end() <= self.heap.idle_space_start || self.heap.idle_space_end <= self.heap.active_space_start ); @@ -1009,7 +1043,7 @@ impl GarbageCollection<'_> for CopyingCollection<'_> { self.heap.resize_semi_spaces(); debug_assert_eq!( - self.heap.active_space_end - self.heap.active_space_start, + self.heap.active_space_end() - self.heap.active_space_start, self.heap.idle_space_end - self.heap.idle_space_start, "the active and idle spaces should be the same size" ); @@ -1033,6 +1067,7 @@ impl GarbageCollection<'_> for CopyingCollection<'_> { #[cfg(test)] mod tests { use super::*; + use wasmtime_environ::{HostPtr, PtrSize}; #[test] fn vm_copying_header_size_align() { @@ -1083,4 +1118,36 @@ mod tests { <= wasmtime_environ::copying::MIN_OBJECT_SIZE as usize, ); } + + #[test] + fn vm_copying_heap_data_bump_ptr_offset() { + assert_eq!( + HostPtr.vmcopying_heap_data_bump_ptr() as usize, + core::mem::offset_of!(VMCopyingHeapData, bump_ptr), + ); + } + + #[test] + fn vm_copying_heap_data_active_space_end_offset() { + assert_eq!( + HostPtr.vmcopying_heap_data_active_space_end() as usize, + core::mem::offset_of!(VMCopyingHeapData, active_space_end), + ); + } + + #[test] + fn vm_copying_heap_data_size() { + assert_eq!( + HostPtr.size_of_vmcopying_heap_data() as usize, + core::mem::size_of::(), + ); + } + + #[test] + fn vm_copying_heap_data_align() { + assert_eq!( + HostPtr.align_of_vmcopying_heap_data() as usize, + core::mem::align_of::(), + ); + } } diff --git a/tests/disas/gc/copying/array-new-fixed-of-gc-refs.wat b/tests/disas/gc/copying/array-new-fixed-of-gc-refs.wat index d73275c023b8..00693251b631 100644 --- a/tests/disas/gc/copying/array-new-fixed-of-gc-refs.wat +++ b/tests/disas/gc/copying/array-new-fixed-of-gc-refs.wat @@ -23,40 +23,70 @@ ;; stack_limit = gv2 ;; ;; block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32): -;; v44 = stack_addr.i64 ss2 -;; store notrap v2, v44 -;; v43 = stack_addr.i64 ss1 -;; store notrap v3, v43 -;; v42 = stack_addr.i64 ss0 -;; store notrap v4, v42 -;; @0025 v14 = iconst.i32 -1476395008 -;; @0025 v16 = load.i64 notrap aligned readonly can_move v0+40 -;; @0025 v17 = load.i32 notrap aligned readonly can_move v16 -;; v56 = iconst.i32 32 -;; @0025 v18 = iconst.i32 16 -;; @0025 v19 = call fn0(v0, v14, v17, v56, v18), stack_map=[i32 @ ss2+0, i32 @ ss1+0, i32 @ ss0+0] ; v14 = -1476395008, v56 = 32, v18 = 16 +;; v69 = stack_addr.i64 ss2 +;; store notrap v2, v69 +;; v68 = stack_addr.i64 ss1 +;; store notrap v3, v68 +;; v67 = stack_addr.i64 ss0 +;; store notrap v4, v67 +;; @0025 v14 = load.i64 notrap aligned readonly can_move v0+32 +;; @0025 v15 = load.i32 notrap aligned can_move v14 +;; @0025 v22 = uextend.i64 v15 +;; v65 = iconst.i64 32 +;; @0025 v23 = iadd v22, v65 ; v65 = 32 +;; @0025 v16 = load.i32 notrap aligned readonly can_move v14+4 +;; @0025 v24 = uextend.i64 v16 +;; @0025 v25 = icmp ule v23, v24 +;; @0025 brif v25, block2, block3 +;; +;; block2: +;; v189 = iconst.i32 32 +;; v95 = iadd.i32 v15, v189 ; v189 = 32 +;; @0025 store notrap aligned vmctx v95, v14 +;; v190 = iconst.i32 -1476395008 +;; v191 = load.i64 notrap aligned readonly can_move v0+8 +;; v192 = load.i64 notrap aligned readonly can_move v191+32 +;; @0025 v39 = iadd v192, v22 +;; @0025 store notrap aligned v190, v39 ; v190 = -1476395008 +;; v193 = load.i64 notrap aligned readonly can_move v0+40 +;; v194 = load.i32 notrap aligned readonly can_move v193 +;; @0025 store notrap aligned v194, v39+4 +;; v195 = iconst.i64 32 +;; @0025 istore32 notrap aligned v195, v39+8 ; v195 = 32 +;; @0025 jump block4(v15, v39) +;; +;; block3 cold: +;; @0025 v27 = iconst.i32 -1476395008 +;; @0025 v29 = load.i64 notrap aligned readonly can_move v0+40 +;; @0025 v30 = load.i32 notrap aligned readonly can_move v29 +;; v81 = iconst.i32 32 +;; @0025 v31 = iconst.i32 16 +;; @0025 v32 = call fn0(v0, v27, v30, v81, v31), stack_map=[i32 @ ss2+0, i32 @ ss1+0, i32 @ ss0+0] ; v27 = -1476395008, v81 = 32, v31 = 16 +;; @0025 v61 = load.i64 notrap aligned readonly can_move v0+8 +;; @0025 v33 = load.i64 notrap aligned readonly can_move v61+32 +;; @0025 v34 = uextend.i64 v32 +;; @0025 v35 = iadd v33, v34 +;; @0025 jump block4(v32, v35) +;; +;; block4(v44: i32, v45: i64): ;; @0025 v6 = iconst.i32 3 -;; @0025 v38 = load.i64 notrap aligned readonly can_move v0+8 -;; @0025 v20 = load.i64 notrap aligned readonly can_move v38+32 -;; @0025 v21 = uextend.i64 v19 -;; @0025 v22 = iadd v20, v21 -;; v37 = iconst.i64 16 -;; @0025 v23 = iadd v22, v37 ; v37 = 16 -;; @0025 store notrap aligned v6, v23 ; v6 = 3 -;; v33 = load.i32 notrap v44 -;; v58 = iconst.i64 20 -;; v63 = iadd v22, v58 ; v58 = 20 -;; @0025 store notrap aligned little v33, v63 -;; v32 = load.i32 notrap v43 -;; v66 = iconst.i64 24 -;; v71 = iadd v22, v66 ; v66 = 24 -;; @0025 store notrap aligned little v32, v71 -;; v31 = load.i32 notrap v42 -;; v87 = iconst.i64 28 -;; v92 = iadd v22, v87 ; v87 = 28 -;; @0025 store notrap aligned little v31, v92 -;; @0029 jump block1 +;; v60 = iconst.i64 16 +;; @0025 v46 = iadd v45, v60 ; v60 = 16 +;; @0025 store notrap aligned v6, v46 ; v6 = 3 +;; v56 = load.i32 notrap v69 +;; v98 = iconst.i64 20 +;; v103 = iadd v45, v98 ; v98 = 20 +;; @0025 store notrap aligned little v56, v103 +;; v55 = load.i32 notrap v68 +;; v106 = iconst.i64 24 +;; v111 = iadd v45, v106 ; v106 = 24 +;; @0025 store notrap aligned little v55, v111 +;; v54 = load.i32 notrap v67 +;; v127 = iconst.i64 28 +;; v132 = iadd v45, v127 ; v127 = 28 +;; @0025 store notrap aligned little v54, v132 +;; @0029 jump block1(v44) ;; -;; block1: -;; @0029 return v19 +;; block1(v5: i32): +;; @0029 return v5 ;; } diff --git a/tests/disas/gc/copying/array-new-fixed.wat b/tests/disas/gc/copying/array-new-fixed.wat index 726abb420411..3fbbe70777cd 100644 --- a/tests/disas/gc/copying/array-new-fixed.wat +++ b/tests/disas/gc/copying/array-new-fixed.wat @@ -20,31 +20,61 @@ ;; stack_limit = gv2 ;; ;; block0(v0: i64, v1: i64, v2: i64, v3: i64, v4: i64): -;; @0025 v14 = iconst.i32 -1476395008 -;; @0025 v16 = load.i64 notrap aligned readonly can_move v0+40 -;; @0025 v17 = load.i32 notrap aligned readonly can_move v16 -;; v45 = iconst.i32 48 -;; @0025 v18 = iconst.i32 16 -;; @0025 v19 = call fn0(v0, v14, v17, v45, v18) ; v14 = -1476395008, v45 = 48, v18 = 16 +;; @0025 v14 = load.i64 notrap aligned readonly can_move v0+32 +;; @0025 v15 = load.i32 notrap aligned can_move v14 +;; @0025 v22 = uextend.i64 v15 +;; v71 = iconst.i64 48 +;; @0025 v23 = iadd v22, v71 ; v71 = 48 +;; @0025 v16 = load.i32 notrap aligned readonly can_move v14+4 +;; @0025 v24 = uextend.i64 v16 +;; @0025 v25 = icmp ule v23, v24 +;; @0025 brif v25, block2, block3 +;; +;; block2: +;; v154 = iconst.i32 48 +;; v85 = iadd.i32 v15, v154 ; v154 = 48 +;; @0025 store notrap aligned vmctx v85, v14 +;; v155 = iconst.i32 -1476395008 +;; v156 = load.i64 notrap aligned readonly can_move v0+8 +;; v157 = load.i64 notrap aligned readonly can_move v156+32 +;; @0025 v39 = iadd v157, v22 +;; @0025 store notrap aligned v155, v39 ; v155 = -1476395008 +;; v158 = load.i64 notrap aligned readonly can_move v0+40 +;; v159 = load.i32 notrap aligned readonly can_move v158 +;; @0025 store notrap aligned v159, v39+4 +;; v160 = iconst.i64 48 +;; @0025 istore32 notrap aligned v160, v39+8 ; v160 = 48 +;; @0025 jump block4(v15, v39) +;; +;; block3 cold: +;; @0025 v27 = iconst.i32 -1476395008 +;; @0025 v29 = load.i64 notrap aligned readonly can_move v0+40 +;; @0025 v30 = load.i32 notrap aligned readonly can_move v29 +;; v70 = iconst.i32 48 +;; @0025 v31 = iconst.i32 16 +;; @0025 v32 = call fn0(v0, v27, v30, v70, v31) ; v27 = -1476395008, v70 = 48, v31 = 16 +;; @0025 v55 = load.i64 notrap aligned readonly can_move v0+8 +;; @0025 v33 = load.i64 notrap aligned readonly can_move v55+32 +;; @0025 v34 = uextend.i64 v32 +;; @0025 v35 = iadd v33, v34 +;; @0025 jump block4(v32, v35) +;; +;; block4(v44: i32, v45: i64): ;; @0025 v6 = iconst.i32 3 -;; @0025 v32 = load.i64 notrap aligned readonly can_move v0+8 -;; @0025 v20 = load.i64 notrap aligned readonly can_move v32+32 -;; @0025 v21 = uextend.i64 v19 -;; @0025 v22 = iadd v20, v21 -;; v31 = iconst.i64 16 -;; @0025 v23 = iadd v22, v31 ; v31 = 16 -;; @0025 store notrap aligned v6, v23 ; v6 = 3 -;; v37 = iconst.i64 24 -;; v51 = iadd v22, v37 ; v37 = 24 -;; @0025 store notrap aligned little v2, v51 -;; v34 = iconst.i64 32 -;; v58 = iadd v22, v34 ; v34 = 32 -;; @0025 store notrap aligned little v3, v58 -;; v73 = iconst.i64 40 -;; v78 = iadd v22, v73 ; v73 = 40 -;; @0025 store notrap aligned little v4, v78 -;; @0029 jump block1 +;; v54 = iconst.i64 16 +;; @0025 v46 = iadd v45, v54 ; v54 = 16 +;; @0025 store notrap aligned v6, v46 ; v6 = 3 +;; v62 = iconst.i64 24 +;; v92 = iadd v45, v62 ; v62 = 24 +;; @0025 store.i64 notrap aligned little v2, v92 +;; v59 = iconst.i64 32 +;; v99 = iadd v45, v59 ; v59 = 32 +;; @0025 store.i64 notrap aligned little v3, v99 +;; v114 = iconst.i64 40 +;; v119 = iadd v45, v114 ; v114 = 40 +;; @0025 store.i64 notrap aligned little v4, v119 +;; @0029 jump block1(v44) ;; -;; block1: -;; @0029 return v19 +;; block1(v5: i32): +;; @0029 return v5 ;; } diff --git a/tests/disas/gc/copying/array-new.wat b/tests/disas/gc/copying/array-new.wat index 8f5539acc1d7..82be522a689f 100644 --- a/tests/disas/gc/copying/array-new.wat +++ b/tests/disas/gc/copying/array-new.wat @@ -21,47 +21,83 @@ ;; ;; block0(v0: i64, v1: i64, v2: i64, v3: i32): ;; @0022 v6 = uextend.i64 v3 -;; v37 = iconst.i64 3 -;; v38 = ishl v6, v37 ; v37 = 3 -;; v35 = iconst.i64 32 -;; @0022 v8 = ushr v38, v35 ; v35 = 32 +;; v62 = iconst.i64 3 +;; v63 = ishl v6, v62 ; v62 = 3 +;; v60 = iconst.i64 32 +;; @0022 v8 = ushr v63, v60 ; v60 = 32 ;; @0022 trapnz v8, user17 ;; @0022 v5 = iconst.i32 24 -;; v44 = iconst.i32 3 -;; v45 = ishl v3, v44 ; v44 = 3 -;; @0022 v10 = uadd_overflow_trap v5, v45, user17 ; v5 = 24 -;; @0022 v12 = iconst.i32 -1476395008 -;; @0022 v14 = load.i64 notrap aligned readonly can_move v0+40 -;; @0022 v15 = load.i32 notrap aligned readonly can_move v14 -;; @0022 v16 = iconst.i32 16 -;; @0022 v17 = call fn0(v0, v12, v15, v10, v16) ; v12 = -1476395008, v16 = 16 -;; @0022 v33 = load.i64 notrap aligned readonly can_move v0+8 -;; @0022 v18 = load.i64 notrap aligned readonly can_move v33+32 -;; @0022 v19 = uextend.i64 v17 -;; @0022 v20 = iadd v18, v19 -;; v32 = iconst.i64 16 -;; @0022 v21 = iadd v20, v32 ; v32 = 16 -;; @0022 store notrap aligned v3, v21 -;; v49 = iconst.i64 24 -;; v54 = iadd v20, v49 ; v49 = 24 -;; @0022 v27 = uextend.i64 v10 -;; @0022 v28 = iadd v20, v27 -;; v36 = iconst.i64 8 -;; @0022 jump block2(v54) +;; v69 = iconst.i32 3 +;; v70 = ishl v3, v69 ; v69 = 3 +;; @0022 v10 = uadd_overflow_trap v5, v70, user17 ; v5 = 24 +;; @0022 v12 = load.i64 notrap aligned readonly can_move v0+32 +;; @0022 v13 = load.i32 notrap aligned can_move v12 +;; @0022 v20 = uextend.i64 v13 +;; @0022 v15 = uextend.i64 v10 +;; @0022 v16 = iconst.i64 15 +;; @0022 v18 = iadd v15, v16 ; v16 = 15 +;; @0022 v17 = iconst.i64 -16 +;; @0022 v19 = band v18, v17 ; v17 = -16 +;; @0022 v21 = iadd v20, v19 +;; @0022 v14 = load.i32 notrap aligned readonly can_move v12+4 +;; @0022 v22 = uextend.i64 v14 +;; @0022 v23 = icmp ule v21, v22 +;; @0022 brif v23, block2, block3 ;; -;; block2(v29: i64): -;; @0022 v30 = icmp eq v29, v28 -;; @0022 brif v30, block4, block3 +;; block2: +;; v78 = iconst.i32 15 +;; v79 = iadd.i32 v10, v78 ; v78 = 15 +;; v82 = iconst.i32 -16 +;; v83 = band v79, v82 ; v82 = -16 +;; v85 = iadd.i32 v13, v83 +;; @0022 store notrap aligned vmctx v85, v12 +;; v98 = iconst.i32 -1476395008 +;; v99 = load.i64 notrap aligned readonly can_move v0+8 +;; v100 = load.i64 notrap aligned readonly can_move v99+32 +;; @0022 v37 = iadd v100, v20 +;; @0022 store notrap aligned v98, v37 ; v98 = -1476395008 +;; v101 = load.i64 notrap aligned readonly can_move v0+40 +;; v102 = load.i32 notrap aligned readonly can_move v101 +;; @0022 store notrap aligned v102, v37+4 +;; v103 = band.i64 v18, v17 ; v17 = -16 +;; @0022 istore32 notrap aligned v103, v37+8 +;; @0022 jump block4(v13, v37) ;; -;; block3: -;; @0022 store.i64 notrap aligned little v2, v29 -;; v59 = iconst.i64 8 -;; v60 = iadd.i64 v29, v59 ; v59 = 8 -;; @0022 jump block2(v60) +;; block3 cold: +;; @0022 v25 = iconst.i32 -1476395008 +;; @0022 v27 = load.i64 notrap aligned readonly can_move v0+40 +;; @0022 v28 = load.i32 notrap aligned readonly can_move v27 +;; @0022 v29 = iconst.i32 16 +;; @0022 v30 = call fn0(v0, v25, v28, v10, v29) ; v25 = -1476395008, v29 = 16 +;; @0022 v56 = load.i64 notrap aligned readonly can_move v0+8 +;; @0022 v31 = load.i64 notrap aligned readonly can_move v56+32 +;; @0022 v32 = uextend.i64 v30 +;; @0022 v33 = iadd v31, v32 +;; @0022 jump block4(v30, v33) ;; -;; block4: -;; @0025 jump block1 +;; block4(v42: i32, v43: i64): +;; v55 = iconst.i64 16 +;; @0022 v44 = iadd v43, v55 ; v55 = 16 +;; @0022 store.i32 notrap aligned v3, v44 +;; v88 = iconst.i64 24 +;; v93 = iadd v43, v88 ; v88 = 24 +;; @0022 v51 = iadd v43, v15 +;; v61 = iconst.i64 8 +;; @0022 jump block5(v93) ;; -;; block1: -;; @0025 return v17 +;; block5(v52: i64): +;; @0022 v53 = icmp eq v52, v51 +;; @0022 brif v53, block7, block6 +;; +;; block6: +;; @0022 store.i64 notrap aligned little v2, v52 +;; v104 = iconst.i64 8 +;; v105 = iadd.i64 v52, v104 ; v104 = 8 +;; @0022 jump block5(v105) +;; +;; block7: +;; @0025 jump block1(v42) +;; +;; block1(v4: i32): +;; @0025 return v4 ;; } diff --git a/tests/disas/gc/copying/funcref-in-gc-heap-new.wat b/tests/disas/gc/copying/funcref-in-gc-heap-new.wat index 08b5e03b1953..c2fb4c5a7531 100644 --- a/tests/disas/gc/copying/funcref-in-gc-heap-new.wat +++ b/tests/disas/gc/copying/funcref-in-gc-heap-new.wat @@ -23,26 +23,56 @@ ;; stack_limit = gv2 ;; ;; block0(v0: i64, v1: i64, v2: i64): -;; @0020 v6 = iconst.i32 -1342177280 -;; @0020 v8 = load.i64 notrap aligned readonly can_move v0+40 -;; @0020 v9 = load.i32 notrap aligned readonly can_move v8 +;; @0020 v6 = load.i64 notrap aligned readonly can_move v0+32 +;; @0020 v7 = load.i32 notrap aligned can_move v6 +;; @0020 v14 = uextend.i64 v7 +;; v50 = iconst.i64 32 +;; @0020 v15 = iadd v14, v50 ; v50 = 32 +;; @0020 v8 = load.i32 notrap aligned readonly can_move v6+4 +;; @0020 v16 = uextend.i64 v8 +;; @0020 v17 = icmp ule v15, v16 +;; @0020 brif v17, block2, block3 +;; +;; block2: +;; v66 = iconst.i32 32 +;; v64 = iadd.i32 v7, v66 ; v66 = 32 +;; @0020 store notrap aligned vmctx v64, v6 +;; v67 = iconst.i32 -1342177280 +;; v68 = load.i64 notrap aligned readonly can_move v0+8 +;; v69 = load.i64 notrap aligned readonly can_move v68+32 +;; @0020 v31 = iadd v69, v14 +;; @0020 store notrap aligned v67, v31 ; v67 = -1342177280 +;; v70 = load.i64 notrap aligned readonly can_move v0+40 +;; v71 = load.i32 notrap aligned readonly can_move v70 +;; @0020 store notrap aligned v71, v31+4 +;; v72 = iconst.i64 32 +;; @0020 istore32 notrap aligned v72, v31+8 ; v72 = 32 +;; @0020 jump block4(v7, v31) +;; +;; block3 cold: +;; @0020 v19 = iconst.i32 -1342177280 +;; @0020 v21 = load.i64 notrap aligned readonly can_move v0+40 +;; @0020 v22 = load.i32 notrap aligned readonly can_move v21 ;; @0020 v4 = iconst.i32 32 -;; @0020 v10 = iconst.i32 16 -;; @0020 v11 = call fn0(v0, v6, v9, v4, v10) ; v6 = -1342177280, v4 = 32, v10 = 16 -;; v26 = stack_addr.i64 ss0 -;; store notrap v11, v26 -;; @0020 v17 = call fn1(v0, v2), stack_map=[i32 @ ss0+0] -;; @0020 v18 = ireduce.i32 v17 -;; @0020 v24 = load.i64 notrap aligned readonly can_move v0+8 -;; @0020 v12 = load.i64 notrap aligned readonly can_move v24+32 -;; @0020 v13 = uextend.i64 v11 -;; @0020 v14 = iadd v12, v13 -;; v22 = iconst.i64 16 -;; @0020 v15 = iadd v14, v22 ; v22 = 16 -;; @0020 store notrap aligned little v18, v15 -;; v19 = load.i32 notrap v26 +;; @0020 v23 = iconst.i32 16 +;; @0020 v24 = call fn0(v0, v19, v22, v4, v23) ; v19 = -1342177280, v4 = 32, v23 = 16 +;; @0020 v46 = load.i64 notrap aligned readonly can_move v0+8 +;; @0020 v25 = load.i64 notrap aligned readonly can_move v46+32 +;; @0020 v26 = uextend.i64 v24 +;; @0020 v27 = iadd v25, v26 +;; @0020 jump block4(v24, v27) +;; +;; block4(v36: i32, v37: i64): +;; v45 = stack_addr.i64 ss0 +;; store notrap v36, v45 +;; @0020 v40 = call fn1(v0, v2), stack_map=[i32 @ ss0+0] +;; @0020 v41 = ireduce.i32 v40 +;; v44 = iconst.i64 16 +;; @0020 v38 = iadd v37, v44 ; v44 = 16 +;; @0020 store notrap aligned little v41, v38 +;; v42 = load.i32 notrap v45 ;; @0023 jump block1 ;; ;; block1: -;; @0023 return v19 +;; @0023 return v42 ;; } diff --git a/tests/disas/gc/copying/struct-new-default.wat b/tests/disas/gc/copying/struct-new-default.wat index 5d0cf530968f..f2c6f96afe95 100644 --- a/tests/disas/gc/copying/struct-new-default.wat +++ b/tests/disas/gc/copying/struct-new-default.wat @@ -22,29 +22,59 @@ ;; stack_limit = gv2 ;; ;; block0(v0: i64, v1: i64): -;; @0021 v8 = iconst.i32 -1342177280 -;; @0021 v10 = load.i64 notrap aligned readonly can_move v0+40 -;; @0021 v11 = load.i32 notrap aligned readonly can_move v10 +;; @0021 v8 = load.i64 notrap aligned readonly can_move v0+32 +;; @0021 v9 = load.i32 notrap aligned can_move v8 +;; @0021 v16 = uextend.i64 v9 +;; v50 = iconst.i64 32 +;; @0021 v17 = iadd v16, v50 ; v50 = 32 +;; @0021 v10 = load.i32 notrap aligned readonly can_move v8+4 +;; @0021 v18 = uextend.i64 v10 +;; @0021 v19 = icmp ule v17, v18 +;; @0021 brif v19, block2, block3 +;; +;; block2: +;; v66 = iconst.i32 32 +;; v64 = iadd.i32 v9, v66 ; v66 = 32 +;; @0021 store notrap aligned vmctx v64, v8 +;; v67 = iconst.i32 -1342177280 +;; v68 = load.i64 notrap aligned readonly can_move v0+8 +;; v69 = load.i64 notrap aligned readonly can_move v68+32 +;; @0021 v33 = iadd v69, v16 +;; @0021 store notrap aligned v67, v33 ; v67 = -1342177280 +;; v70 = load.i64 notrap aligned readonly can_move v0+40 +;; v71 = load.i32 notrap aligned readonly can_move v70 +;; @0021 store notrap aligned v71, v33+4 +;; v72 = iconst.i64 32 +;; @0021 istore32 notrap aligned v72, v33+8 ; v72 = 32 +;; @0021 jump block4(v9, v33) +;; +;; block3 cold: +;; @0021 v21 = iconst.i32 -1342177280 +;; @0021 v23 = load.i64 notrap aligned readonly can_move v0+40 +;; @0021 v24 = load.i32 notrap aligned readonly can_move v23 ;; @0021 v6 = iconst.i32 32 -;; @0021 v12 = iconst.i32 16 -;; @0021 v13 = call fn0(v0, v8, v11, v6, v12) ; v8 = -1342177280, v6 = 32, v12 = 16 +;; @0021 v25 = iconst.i32 16 +;; @0021 v26 = call fn0(v0, v21, v24, v6, v25) ; v21 = -1342177280, v6 = 32, v25 = 16 +;; @0021 v46 = load.i64 notrap aligned readonly can_move v0+8 +;; @0021 v27 = load.i64 notrap aligned readonly can_move v46+32 +;; @0021 v28 = uextend.i64 v26 +;; @0021 v29 = iadd v27, v28 +;; @0021 jump block4(v26, v29) +;; +;; block4(v38: i32, v39: i64): ;; @0021 v3 = f32const 0.0 -;; @0021 v23 = load.i64 notrap aligned readonly can_move v0+8 -;; @0021 v14 = load.i64 notrap aligned readonly can_move v23+32 -;; @0021 v15 = uextend.i64 v13 -;; @0021 v16 = iadd v14, v15 -;; v22 = iconst.i64 16 -;; @0021 v17 = iadd v16, v22 ; v22 = 16 -;; @0021 store notrap aligned little v3, v17 ; v3 = 0.0 +;; v45 = iconst.i64 16 +;; @0021 v40 = iadd v39, v45 ; v45 = 16 +;; @0021 store notrap aligned little v3, v40 ; v3 = 0.0 ;; @0021 v4 = iconst.i32 0 -;; v21 = iconst.i64 20 -;; @0021 v18 = iadd v16, v21 ; v21 = 20 -;; @0021 istore8 notrap aligned little v4, v18 ; v4 = 0 -;; v20 = iconst.i64 24 -;; @0021 v19 = iadd v16, v20 ; v20 = 24 -;; @0021 store notrap aligned little v4, v19 ; v4 = 0 -;; @0024 jump block1 +;; v44 = iconst.i64 20 +;; @0021 v41 = iadd v39, v44 ; v44 = 20 +;; @0021 istore8 notrap aligned little v4, v41 ; v4 = 0 +;; v43 = iconst.i64 24 +;; @0021 v42 = iadd v39, v43 ; v43 = 24 +;; @0021 store notrap aligned little v4, v42 ; v4 = 0 +;; @0024 jump block1(v38) ;; -;; block1: -;; @0024 return v13 +;; block1(v2: i32): +;; @0024 return v2 ;; } diff --git a/tests/disas/gc/copying/struct-new.wat b/tests/disas/gc/copying/struct-new.wat index ee3b8c7a8d27..0eaaab491920 100644 --- a/tests/disas/gc/copying/struct-new.wat +++ b/tests/disas/gc/copying/struct-new.wat @@ -23,30 +23,60 @@ ;; stack_limit = gv2 ;; ;; block0(v0: i64, v1: i64, v2: f32, v3: i32, v4: i32): -;; v27 = stack_addr.i64 ss0 -;; store notrap v4, v27 -;; @002a v8 = iconst.i32 -1342177280 -;; @002a v10 = load.i64 notrap aligned readonly can_move v0+40 -;; @002a v11 = load.i32 notrap aligned readonly can_move v10 +;; v52 = stack_addr.i64 ss0 +;; store notrap v4, v52 +;; @002a v8 = load.i64 notrap aligned readonly can_move v0+32 +;; @002a v9 = load.i32 notrap aligned can_move v8 +;; @002a v16 = uextend.i64 v9 +;; v53 = iconst.i64 32 +;; @002a v17 = iadd v16, v53 ; v53 = 32 +;; @002a v10 = load.i32 notrap aligned readonly can_move v8+4 +;; @002a v18 = uextend.i64 v10 +;; @002a v19 = icmp ule v17, v18 +;; @002a brif v19, block2, block3 +;; +;; block2: +;; v69 = iconst.i32 32 +;; v67 = iadd.i32 v9, v69 ; v69 = 32 +;; @002a store notrap aligned vmctx v67, v8 +;; v70 = iconst.i32 -1342177280 +;; v71 = load.i64 notrap aligned readonly can_move v0+8 +;; v72 = load.i64 notrap aligned readonly can_move v71+32 +;; @002a v33 = iadd v72, v16 +;; @002a store notrap aligned v70, v33 ; v70 = -1342177280 +;; v73 = load.i64 notrap aligned readonly can_move v0+40 +;; v74 = load.i32 notrap aligned readonly can_move v73 +;; @002a store notrap aligned v74, v33+4 +;; v75 = iconst.i64 32 +;; @002a istore32 notrap aligned v75, v33+8 ; v75 = 32 +;; @002a jump block4(v9, v33) +;; +;; block3 cold: +;; @002a v21 = iconst.i32 -1342177280 +;; @002a v23 = load.i64 notrap aligned readonly can_move v0+40 +;; @002a v24 = load.i32 notrap aligned readonly can_move v23 ;; @002a v6 = iconst.i32 32 -;; @002a v12 = iconst.i32 16 -;; @002a v13 = call fn0(v0, v8, v11, v6, v12), stack_map=[i32 @ ss0+0] ; v8 = -1342177280, v6 = 32, v12 = 16 -;; @002a v25 = load.i64 notrap aligned readonly can_move v0+8 -;; @002a v14 = load.i64 notrap aligned readonly can_move v25+32 -;; @002a v15 = uextend.i64 v13 -;; @002a v16 = iadd v14, v15 -;; v24 = iconst.i64 16 -;; @002a v17 = iadd v16, v24 ; v24 = 16 -;; @002a store notrap aligned little v2, v17 -;; v23 = iconst.i64 20 -;; @002a v18 = iadd v16, v23 ; v23 = 20 -;; @002a istore8 notrap aligned little v3, v18 -;; v20 = load.i32 notrap v27 -;; v22 = iconst.i64 24 -;; @002a v19 = iadd v16, v22 ; v22 = 24 -;; @002a store notrap aligned little v20, v19 -;; @002d jump block1 +;; @002a v25 = iconst.i32 16 +;; @002a v26 = call fn0(v0, v21, v24, v6, v25), stack_map=[i32 @ ss0+0] ; v21 = -1342177280, v6 = 32, v25 = 16 +;; @002a v48 = load.i64 notrap aligned readonly can_move v0+8 +;; @002a v27 = load.i64 notrap aligned readonly can_move v48+32 +;; @002a v28 = uextend.i64 v26 +;; @002a v29 = iadd v27, v28 +;; @002a jump block4(v26, v29) +;; +;; block4(v38: i32, v39: i64): +;; v47 = iconst.i64 16 +;; @002a v40 = iadd v39, v47 ; v47 = 16 +;; @002a store.f32 notrap aligned little v2, v40 +;; v46 = iconst.i64 20 +;; @002a v41 = iadd v39, v46 ; v46 = 20 +;; @002a istore8.i32 notrap aligned little v3, v41 +;; v43 = load.i32 notrap v52 +;; v45 = iconst.i64 24 +;; @002a v42 = iadd v39, v45 ; v45 = 24 +;; @002a store notrap aligned little v43, v42 +;; @002d jump block1(v38) ;; -;; block1: -;; @002d return v13 +;; block1(v5: i32): +;; @002d return v5 ;; }