diff --git a/lib/compiler-singlepass/src/codegen_x64.rs b/lib/compiler-singlepass/src/codegen_x64.rs index 3c01738c13d..3b8a47d7cd6 100644 --- a/lib/compiler-singlepass/src/codegen_x64.rs +++ b/lib/compiler-singlepass/src/codegen_x64.rs @@ -7,9 +7,7 @@ use crate::{ use dynasmrt::{x64::X64Relocation, DynamicLabel, VecAssembler}; use smallvec::{smallvec, SmallVec}; use std::iter; -use wasmer_compiler::wasmparser::{ - MemoryImmediate, Operator, Type as WpType, TypeOrFuncType as WpTypeOrFuncType, -}; +use wasmer_compiler::wasmparser::{Operator, Type as WpType, TypeOrFuncType as WpTypeOrFuncType}; use wasmer_compiler::{ CallingConvention, CompiledFunction, CompiledFunctionFrameInfo, CustomSection, CustomSectionProtection, FunctionBody, FunctionBodyData, Relocation, RelocationTarget, @@ -530,50 +528,6 @@ impl<'a> FuncGen<'a> { Ok(()) } - /// Emits a memory operation. - fn memory_op Result<(), CodegenError>>( - &mut self, - addr: Location, - memarg: &MemoryImmediate, - check_alignment: bool, - value_size: usize, - cb: F, - ) -> Result<(), CodegenError> { - let need_check = match self.memory_styles[MemoryIndex::new(0)] { - MemoryStyle::Static { .. } => false, - MemoryStyle::Dynamic { .. } => true, - }; - - let offset = if self.module.num_imported_memories != 0 { - self.vmoffsets - .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) - } else { - self.vmoffsets - .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) - }; - let tmp_addr = self.machine.specific.pick_temp_gpr().unwrap(); - let begin = self.machine.specific.memory_op_begin( - addr, - memarg, - check_alignment, - value_size, - need_check, - self.module.num_imported_memories != 0, - offset as i32, - self.special_labels.heap_access_oob, - tmp_addr, - ); - cb(self, tmp_addr)?; - let end = self.machine.specific.memory_op_end(tmp_addr); - - self.machine.specific.mark_address_range_with_trap_code( - TrapCode::HeapAccessOutOfBounds, - begin, - end, - ); - Ok(()) - } - /// Emits a memory operation. fn op_memory(&mut self, cb: F) { let need_check = match self.memory_styles[MemoryIndex::new(0)] { @@ -2994,15 +2948,19 @@ impl<'a> FuncGen<'a> { false, )[0]; self.value_stack.push(ret); - - self.memory_op(target, memarg, false, 4, |this, addr| { - this.machine.specific.emit_relaxed_mov( - Size::S32, - Location::Memory(addr, 0), - ret, - ); - Ok(()) - })?; + self.op_memory( + |this, need_check, imported_memories, offset, heap_access_oob| { + this.machine.specific.i32_load( + target, + memarg, + ret, + need_check, + imported_memories, + offset, + heap_access_oob, + ); + }, + ); } Operator::F32Load { ref memarg } => { let target = self.pop_value_released(); @@ -3013,15 +2971,19 @@ impl<'a> FuncGen<'a> { self.value_stack.push(ret); self.fp_stack .push(FloatValue::new(self.value_stack.len() - 1)); - - self.memory_op(target, memarg, false, 4, |this, addr| { - this.machine.specific.emit_relaxed_mov( - Size::S32, - Location::Memory(addr, 0), - ret, - ); - Ok(()) - })?; + self.op_memory( + |this, need_check, imported_memories, offset, heap_access_oob| { + this.machine.specific.f32_load( + target, + memarg, + ret, + need_check, + imported_memories, + offset, + heap_access_oob, + ); + }, + ); } Operator::I32Load8U { ref memarg } => { let target = self.pop_value_released(); @@ -3030,16 +2992,19 @@ impl<'a> FuncGen<'a> { false, )[0]; self.value_stack.push(ret); - - self.memory_op(target, memarg, false, 1, |this, addr| { - this.machine.specific.emit_relaxed_zero_extension( - Size::S8, - Location::Memory(addr, 0), - Size::S32, - ret, - ); - Ok(()) - })?; + self.op_memory( + |this, need_check, imported_memories, offset, heap_access_oob| { + this.machine.specific.i32_load_8u( + target, + memarg, + ret, + need_check, + imported_memories, + offset, + heap_access_oob, + ); + }, + ); } Operator::I32Load8S { ref memarg } => { let target = self.pop_value_released(); @@ -3048,16 +3013,19 @@ impl<'a> FuncGen<'a> { false, )[0]; self.value_stack.push(ret); - - self.memory_op(target, memarg, false, 1, |this, addr| { - this.machine.specific.emit_relaxed_sign_extension( - Size::S8, - Location::Memory(addr, 0), - Size::S32, - ret, - ); - Ok(()) - })?; + self.op_memory( + |this, need_check, imported_memories, offset, heap_access_oob| { + this.machine.specific.i32_load_8s( + target, + memarg, + ret, + need_check, + imported_memories, + offset, + heap_access_oob, + ); + }, + ); } Operator::I32Load16U { ref memarg } => { let target = self.pop_value_released(); @@ -3066,16 +3034,19 @@ impl<'a> FuncGen<'a> { false, )[0]; self.value_stack.push(ret); - - self.memory_op(target, memarg, false, 2, |this, addr| { - this.machine.specific.emit_relaxed_zero_extension( - Size::S16, - Location::Memory(addr, 0), - Size::S32, - ret, - ); - Ok(()) - })?; + self.op_memory( + |this, need_check, imported_memories, offset, heap_access_oob| { + this.machine.specific.i32_load_16u( + target, + memarg, + ret, + need_check, + imported_memories, + offset, + heap_access_oob, + ); + }, + ); } Operator::I32Load16S { ref memarg } => { let target = self.pop_value_released(); @@ -3084,82 +3055,90 @@ impl<'a> FuncGen<'a> { false, )[0]; self.value_stack.push(ret); - - self.memory_op(target, memarg, false, 2, |this, addr| { - this.machine.specific.emit_relaxed_sign_extension( - Size::S16, - Location::Memory(addr, 0), - Size::S32, - ret, - ); - Ok(()) - })?; + self.op_memory( + |this, need_check, imported_memories, offset, heap_access_oob| { + this.machine.specific.i32_load_16s( + target, + memarg, + ret, + need_check, + imported_memories, + offset, + heap_access_oob, + ); + }, + ); } Operator::I32Store { ref memarg } => { let target_value = self.pop_value_released(); let target_addr = self.pop_value_released(); - - self.memory_op(target_addr, memarg, false, 4, |this, addr| { - this.machine.specific.emit_relaxed_mov( - Size::S32, - target_value, - Location::Memory(addr, 0), - ); - Ok(()) - })?; + self.op_memory( + |this, need_check, imported_memories, offset, heap_access_oob| { + this.machine.specific.i32_save( + target_value, + memarg, + target_addr, + need_check, + imported_memories, + offset, + heap_access_oob, + ); + }, + ); } Operator::F32Store { ref memarg } => { let target_value = self.pop_value_released(); let target_addr = self.pop_value_released(); let fp = self.fp_stack.pop1()?; let config_nan_canonicalization = self.config.enable_nan_canonicalization; - - self.memory_op(target_addr, memarg, false, 4, |this, addr| { - if !this.machine.arch_supports_canonicalize_nan() - || !config_nan_canonicalization - || fp.canonicalization.is_none() - { - this.machine.specific.emit_relaxed_mov( - Size::S32, - target_value, - Location::Memory(addr, 0), - ); - } else { - this.machine.canonicalize_nan( - Size::S32, + self.op_memory( + |this, need_check, imported_memories, offset, heap_access_oob| { + this.machine.specific.f32_save( target_value, - Location::Memory(addr, 0), + memarg, + target_addr, + config_nan_canonicalization && !fp.canonicalization.is_none(), + need_check, + imported_memories, + offset, + heap_access_oob, ); - } - - Ok(()) - })?; + }, + ); } Operator::I32Store8 { ref memarg } => { let target_value = self.pop_value_released(); let target_addr = self.pop_value_released(); - - self.memory_op(target_addr, memarg, false, 1, |this, addr| { - this.machine.specific.emit_relaxed_mov( - Size::S8, - target_value, - Location::Memory(addr, 0), - ); - Ok(()) - })?; + self.op_memory( + |this, need_check, imported_memories, offset, heap_access_oob| { + this.machine.specific.i32_save_8( + target_value, + memarg, + target_addr, + need_check, + imported_memories, + offset, + heap_access_oob, + ); + }, + ); } Operator::I32Store16 { ref memarg } => { let target_value = self.pop_value_released(); let target_addr = self.pop_value_released(); - - self.memory_op(target_addr, memarg, false, 2, |this, addr| { - this.machine.specific.emit_relaxed_mov( - Size::S16, - target_value, - Location::Memory(addr, 0), - ); - Ok(()) - })?; + self.op_memory( + |this, need_check, imported_memories, offset, heap_access_oob| { + this.machine.specific.i32_save_16( + target_value, + memarg, + target_addr, + need_check, + imported_memories, + offset, + heap_access_oob, + ); + }, + ); } Operator::I64Load { ref memarg } => { let target = self.pop_value_released(); @@ -3168,15 +3147,19 @@ impl<'a> FuncGen<'a> { false, )[0]; self.value_stack.push(ret); - - self.memory_op(target, memarg, false, 8, |this, addr| { - this.machine.specific.emit_relaxed_mov( - Size::S64, - Location::Memory(addr, 0), - ret, - ); - Ok(()) - })?; + self.op_memory( + |this, need_check, imported_memories, offset, heap_access_oob| { + this.machine.specific.i64_load( + target, + memarg, + ret, + need_check, + imported_memories, + offset, + heap_access_oob, + ); + }, + ); } Operator::F64Load { ref memarg } => { let target = self.pop_value_released(); @@ -3187,15 +3170,19 @@ impl<'a> FuncGen<'a> { self.value_stack.push(ret); self.fp_stack .push(FloatValue::new(self.value_stack.len() - 1)); - - self.memory_op(target, memarg, false, 8, |this, addr| { - this.machine.specific.emit_relaxed_mov( - Size::S64, - Location::Memory(addr, 0), - ret, - ); - Ok(()) - })?; + self.op_memory( + |this, need_check, imported_memories, offset, heap_access_oob| { + this.machine.specific.f64_load( + target, + memarg, + ret, + need_check, + imported_memories, + offset, + heap_access_oob, + ); + }, + ); } Operator::I64Load8U { ref memarg } => { let target = self.pop_value_released(); @@ -3204,16 +3191,19 @@ impl<'a> FuncGen<'a> { false, )[0]; self.value_stack.push(ret); - - self.memory_op(target, memarg, false, 1, |this, addr| { - this.machine.specific.emit_relaxed_zero_extension( - Size::S8, - Location::Memory(addr, 0), - Size::S64, - ret, - ); - Ok(()) - })?; + self.op_memory( + |this, need_check, imported_memories, offset, heap_access_oob| { + this.machine.specific.i64_load_8u( + target, + memarg, + ret, + need_check, + imported_memories, + offset, + heap_access_oob, + ); + }, + ); } Operator::I64Load8S { ref memarg } => { let target = self.pop_value_released(); @@ -3222,16 +3212,19 @@ impl<'a> FuncGen<'a> { false, )[0]; self.value_stack.push(ret); - - self.memory_op(target, memarg, false, 1, |this, addr| { - this.machine.specific.emit_relaxed_sign_extension( - Size::S8, - Location::Memory(addr, 0), - Size::S64, - ret, - ); - Ok(()) - })?; + self.op_memory( + |this, need_check, imported_memories, offset, heap_access_oob| { + this.machine.specific.i64_load_8s( + target, + memarg, + ret, + need_check, + imported_memories, + offset, + heap_access_oob, + ); + }, + ); } Operator::I64Load16U { ref memarg } => { let target = self.pop_value_released(); @@ -3240,16 +3233,19 @@ impl<'a> FuncGen<'a> { false, )[0]; self.value_stack.push(ret); - - self.memory_op(target, memarg, false, 2, |this, addr| { - this.machine.specific.emit_relaxed_zero_extension( - Size::S16, - Location::Memory(addr, 0), - Size::S64, - ret, - ); - Ok(()) - })?; + self.op_memory( + |this, need_check, imported_memories, offset, heap_access_oob| { + this.machine.specific.i64_load_16u( + target, + memarg, + ret, + need_check, + imported_memories, + offset, + heap_access_oob, + ); + }, + ); } Operator::I64Load16S { ref memarg } => { let target = self.pop_value_released(); @@ -3258,16 +3254,19 @@ impl<'a> FuncGen<'a> { false, )[0]; self.value_stack.push(ret); - - self.memory_op(target, memarg, false, 2, |this, addr| { - this.machine.specific.emit_relaxed_sign_extension( - Size::S16, - Location::Memory(addr, 0), - Size::S64, - ret, - ); - Ok(()) - })?; + self.op_memory( + |this, need_check, imported_memories, offset, heap_access_oob| { + this.machine.specific.i64_load_16s( + target, + memarg, + ret, + need_check, + imported_memories, + offset, + heap_access_oob, + ); + }, + ); } Operator::I64Load32U { ref memarg } => { let target = self.pop_value_released(); @@ -3276,30 +3275,19 @@ impl<'a> FuncGen<'a> { false, )[0]; self.value_stack.push(ret); - - self.memory_op(target, memarg, false, 4, |this, addr| { - match ret { - Location::GPR(_) => {} - Location::Memory(base, offset) => { - this.machine.specific.move_location( - Size::S32, - Location::Imm32(0), - Location::Memory(base, offset + 4), - ); // clear upper bits - } - _ => { - return Err(CodegenError { - message: "I64Load32U ret: unreachable code".to_string(), - }) - } - } - this.machine.specific.emit_relaxed_mov( - Size::S32, - Location::Memory(addr, 0), - ret, - ); - Ok(()) - })?; + self.op_memory( + |this, need_check, imported_memories, offset, heap_access_oob| { + this.machine.specific.i64_load_32u( + target, + memarg, + ret, + need_check, + imported_memories, + offset, + heap_access_oob, + ); + }, + ); } Operator::I64Load32S { ref memarg } => { let target = self.pop_value_released(); @@ -3308,94 +3296,108 @@ impl<'a> FuncGen<'a> { false, )[0]; self.value_stack.push(ret); - - self.memory_op(target, memarg, false, 4, |this, addr| { - this.machine.specific.emit_relaxed_sign_extension( - Size::S32, - Location::Memory(addr, 0), - Size::S64, - ret, - ); - Ok(()) - })?; + self.op_memory( + |this, need_check, imported_memories, offset, heap_access_oob| { + this.machine.specific.i64_load_32s( + target, + memarg, + ret, + need_check, + imported_memories, + offset, + heap_access_oob, + ); + }, + ); } Operator::I64Store { ref memarg } => { let target_value = self.pop_value_released(); let target_addr = self.pop_value_released(); - self.memory_op(target_addr, memarg, false, 8, |this, addr| { - this.machine.specific.emit_relaxed_mov( - Size::S64, - target_value, - Location::Memory(addr, 0), - ); - Ok(()) - })?; + self.op_memory( + |this, need_check, imported_memories, offset, heap_access_oob| { + this.machine.specific.i64_save( + target_value, + memarg, + target_addr, + need_check, + imported_memories, + offset, + heap_access_oob, + ); + }, + ); } Operator::F64Store { ref memarg } => { let target_value = self.pop_value_released(); let target_addr = self.pop_value_released(); let fp = self.fp_stack.pop1()?; let config_nan_canonicalization = self.config.enable_nan_canonicalization; - - self.memory_op(target_addr, memarg, false, 8, |this, addr| { - if !this.machine.arch_supports_canonicalize_nan() - || !config_nan_canonicalization - || fp.canonicalization.is_none() - { - this.machine.specific.emit_relaxed_mov( - Size::S64, - target_value, - Location::Memory(addr, 0), - ); - } else { - this.machine.canonicalize_nan( - Size::S64, + self.op_memory( + |this, need_check, imported_memories, offset, heap_access_oob| { + this.machine.specific.f64_save( target_value, - Location::Memory(addr, 0), + memarg, + target_addr, + config_nan_canonicalization && !fp.canonicalization.is_none(), + need_check, + imported_memories, + offset, + heap_access_oob, ); - } - Ok(()) - })?; + }, + ); } Operator::I64Store8 { ref memarg } => { let target_value = self.pop_value_released(); let target_addr = self.pop_value_released(); - - self.memory_op(target_addr, memarg, false, 1, |this, addr| { - this.machine.specific.emit_relaxed_mov( - Size::S8, - target_value, - Location::Memory(addr, 0), - ); - Ok(()) - })?; + self.op_memory( + |this, need_check, imported_memories, offset, heap_access_oob| { + this.machine.specific.i64_save_8( + target_value, + memarg, + target_addr, + need_check, + imported_memories, + offset, + heap_access_oob, + ); + }, + ); } Operator::I64Store16 { ref memarg } => { let target_value = self.pop_value_released(); let target_addr = self.pop_value_released(); - - self.memory_op(target_addr, memarg, false, 2, |this, addr| { - this.machine.specific.emit_relaxed_mov( - Size::S16, - target_value, - Location::Memory(addr, 0), - ); - Ok(()) - })?; + self.op_memory( + |this, need_check, imported_memories, offset, heap_access_oob| { + this.machine.specific.i64_save_16( + target_value, + memarg, + target_addr, + need_check, + imported_memories, + offset, + heap_access_oob, + ); + }, + ); } Operator::I64Store32 { ref memarg } => { let target_value = self.pop_value_released(); let target_addr = self.pop_value_released(); - - self.memory_op(target_addr, memarg, false, 4, |this, addr| { - this.machine.specific.emit_relaxed_mov( - Size::S32, - target_value, - Location::Memory(addr, 0), - ); - Ok(()) - })?; + self.op_memory( + |this, need_check, imported_memories, offset, heap_access_oob| { + this.machine.specific.i64_save_32( + target_value, + memarg, + target_addr, + need_check, + imported_memories, + offset, + heap_access_oob, + ); + }, + ); } Operator::Unreachable => { self.mark_trappable(); diff --git a/lib/compiler-singlepass/src/machine.rs b/lib/compiler-singlepass/src/machine.rs index a86ae402b74..3f59a069aac 100644 --- a/lib/compiler-singlepass/src/machine.rs +++ b/lib/compiler-singlepass/src/machine.rs @@ -184,21 +184,6 @@ pub trait MachineSpecific { fn arch_supports_canonicalize_nan(&self) -> bool; /// Cannonicalize a NaN (or panic if not supported) fn canonicalize_nan(&mut self, sz: Size, input: Location, output: Location); - /// prepare to do a memory opcode - fn memory_op_begin( - &mut self, - addr: Location, - memarg: &MemoryImmediate, - check_alignment: bool, - value_size: usize, - need_check: bool, - imported_memories: bool, - offset: i32, - heap_access_oob: Label, - tmp_addr: R, - ) -> usize; - /// finished do a memory opcode - fn memory_op_end(&mut self, tmp_addr: R) -> usize; /// emit an Illegal Opcode fn emit_illegal_op(&mut self); @@ -321,8 +306,6 @@ pub trait MachineSpecific { fn emit_relaxed_mov(&mut self, sz: Size, src: Location, dst: Location); /// relaxed cmp: compare from anywhere and anywhere fn emit_relaxed_cmp(&mut self, sz: Size, src: Location, dst: Location); - /// relaxed atomic xchg: atomic exchange of anywhere and anywhere - fn emit_relaxed_atomic_xchg(&mut self, sz: Size, src: Location, dst: Location); /// Emit a memory fence. Can be nothing for x86_64 or a DMB on ARM64 for example fn emit_memory_fence(&mut self); /// relaxed move with zero extension @@ -453,6 +436,61 @@ pub trait MachineSpecific { fn i32_rol(&mut self, loc_a: Location, loc_b: Location, ret: Location); /// i32 Roll Right fn i32_ror(&mut self, loc_a: Location, loc_b: Location, ret: Location); + /// i32 load + fn i32_load( + &mut self, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i32 load of an unsigned 8bits + fn i32_load_8u( + &mut self, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i32 load of an signed 8bits + fn i32_load_8s( + &mut self, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i32 load of an unsigned 16bits + fn i32_load_16u( + &mut self, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i32 load of an signed 16bits + fn i32_load_16s( + &mut self, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); /// i32 atomic load fn i32_atomic_load( &mut self, @@ -486,6 +524,39 @@ pub trait MachineSpecific { offset: i32, heap_access_oob: Label, ); + /// i32 save + fn i32_save( + &mut self, + value: Location, + memarg: &MemoryImmediate, + addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i32 save of the lower 8bits + fn i32_save_8( + &mut self, + value: Location, + memarg: &MemoryImmediate, + addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i32 save of the lower 16bits + fn i32_save_16( + &mut self, + value: Location, + memarg: &MemoryImmediate, + addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); /// i32 atomic save fn i32_atomic_save( &mut self, @@ -891,6 +962,83 @@ pub trait MachineSpecific { fn i64_rol(&mut self, loc_a: Location, loc_b: Location, ret: Location); /// i64 Roll Right fn i64_ror(&mut self, loc_a: Location, loc_b: Location, ret: Location); + /// i64 load + fn i64_load( + &mut self, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 load of an unsigned 8bits + fn i64_load_8u( + &mut self, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 load of an signed 8bits + fn i64_load_8s( + &mut self, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 load of an unsigned 32bits + fn i64_load_32u( + &mut self, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 load of an signed 32bits + fn i64_load_32s( + &mut self, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 load of an signed 16bits + fn i64_load_16u( + &mut self, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 load of an signed 16bits + fn i64_load_16s( + &mut self, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); /// i64 atomic load fn i64_atomic_load( &mut self, @@ -935,6 +1083,50 @@ pub trait MachineSpecific { offset: i32, heap_access_oob: Label, ); + /// i64 save + fn i64_save( + &mut self, + value: Location, + memarg: &MemoryImmediate, + addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 save of the lower 8bits + fn i64_save_8( + &mut self, + value: Location, + memarg: &MemoryImmediate, + addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 save of the lower 16bits + fn i64_save_16( + &mut self, + value: Location, + memarg: &MemoryImmediate, + addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 save of the lower 32bits + fn i64_save_32( + &mut self, + value: Location, + memarg: &MemoryImmediate, + addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); /// i64 atomic save fn i64_atomic_save( &mut self, @@ -1320,6 +1512,52 @@ pub trait MachineSpecific { heap_access_oob: Label, ); + /// load an F32 + fn f32_load( + &mut self, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// f32 save + fn f32_save( + &mut self, + value: Location, + memarg: &MemoryImmediate, + addr: Location, + canonicalize: bool, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// load an F64 + fn f64_load( + &mut self, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// f64 save + fn f64_save( + &mut self, + value: Location, + memarg: &MemoryImmediate, + addr: Location, + canonicalize: bool, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); /// Convert a F64 from I64, signed or unsigned fn convert_f64_i64(&mut self, loc: Location, signed: bool, ret: Location); /// Convert a F64 from I32, signed or unsigned diff --git a/lib/compiler-singlepass/src/machine_x64.rs b/lib/compiler-singlepass/src/machine_x64.rs index 7fc89f155c7..73ef570efc7 100644 --- a/lib/compiler-singlepass/src/machine_x64.rs +++ b/lib/compiler-singlepass/src/machine_x64.rs @@ -1535,6 +1535,10 @@ impl MachineX86_64 { self.release_gpr(tmp_out); } } + + fn emit_relaxed_atomic_xchg(&mut self, sz: Size, src: Location, dst: Location) { + self.emit_relaxed_binop(Assembler::emit_xchg, sz, src, dst); + } } impl MachineSpecific for MachineX86_64 { @@ -2216,120 +2220,6 @@ impl MachineSpecific for MachineX86_64 { self.assembler.emit_pop(size, loc); } - fn memory_op_begin( - &mut self, - addr: Location, - memarg: &MemoryImmediate, - check_alignment: bool, - value_size: usize, - need_check: bool, - imported_memories: bool, - offset: i32, - heap_access_oob: Label, - tmp_addr: GPR, - ) -> usize { - self.reserve_gpr(tmp_addr); - - // Reusing `tmp_addr` for temporary indirection here, since it's not used before the last reference to `{base,bound}_loc`. - let (base_loc, bound_loc) = if imported_memories { - // Imported memories require one level of indirection. - self.move_location( - Size::S64, - Location::Memory(Machine::get_vmctx_reg(), offset), - Location::GPR(tmp_addr), - ); - (Location::Memory(tmp_addr, 0), Location::Memory(tmp_addr, 8)) - } else { - ( - Location::Memory(Machine::get_vmctx_reg(), offset), - Location::Memory(Machine::get_vmctx_reg(), offset + 8), - ) - }; - - let tmp_base = self.pick_temp_gpr().unwrap(); - self.reserve_gpr(tmp_base); - let tmp_bound = self.pick_temp_gpr().unwrap(); - self.reserve_gpr(tmp_bound); - - // Load base into temporary register. - self.move_location(Size::S64, base_loc, Location::GPR(tmp_base)); - - // Load bound into temporary register, if needed. - if need_check { - self.move_location(Size::S64, bound_loc, Location::GPR(tmp_bound)); - - // Wasm -> Effective. - // Assuming we never underflow - should always be true on Linux/macOS and Windows >=8, - // since the first page from 0x0 to 0x1000 is not accepted by mmap. - - // This `lea` calculates the upper bound allowed for the beginning of the word. - // Since the upper bound of the memory is (exclusively) `tmp_bound + tmp_base`, - // the maximum allowed beginning of word is (inclusively) - // `tmp_bound + tmp_base - value_size`. - self.location_address( - Size::S64, - Location::Memory2(tmp_bound, tmp_base, Multiplier::One, -(value_size as i32)), - Location::GPR(tmp_bound), - ); - } - - // Load effective address. - // `base_loc` and `bound_loc` becomes INVALID after this line, because `tmp_addr` - // might be reused. - self.move_location(Size::S32, addr, Location::GPR(tmp_addr)); - - // Add offset to memory address. - if memarg.offset != 0 { - self.location_add( - Size::S32, - Location::Imm32(memarg.offset), - Location::GPR(tmp_addr), - true, - ); - - // Trap if offset calculation overflowed. - self.jmp_on_overflow(heap_access_oob); - } - - // Wasm linear memory -> real memory - self.location_add( - Size::S64, - Location::GPR(tmp_base), - Location::GPR(tmp_addr), - false, - ); - - if need_check { - // Trap if the end address of the requested area is above that of the linear memory. - self.location_cmp(Size::S64, Location::GPR(tmp_bound), Location::GPR(tmp_addr)); - - // `tmp_bound` is inclusive. So trap only if `tmp_addr > tmp_bound`. - self.jmp_on_above(heap_access_oob); - } - - self.release_gpr(tmp_bound); - self.release_gpr(tmp_base); - - let align = memarg.align; - if check_alignment && align != 1 { - self.location_test( - Size::S32, - Location::Imm32((align - 1).into()), - Location::GPR(tmp_addr), - ); - self.jmp_on_different(heap_access_oob); - } - - self.get_offset().0 - } - - fn memory_op_end(&mut self, tmp_addr: GPR) -> usize { - let end = self.get_offset().0; - self.release_gpr(tmp_addr); - - end - } - fn emit_memory_fence(&mut self) { // nothing on x86_64 } @@ -2364,9 +2254,6 @@ impl MachineSpecific for MachineX86_64 { fn emit_relaxed_cmp(&mut self, sz: Size, src: Location, dst: Location) { self.emit_relaxed_binop(Assembler::emit_cmp, sz, src, dst); } - fn emit_relaxed_atomic_xchg(&mut self, sz: Size, src: Location, dst: Location) { - self.emit_relaxed_binop(Assembler::emit_xchg, sz, src, dst); - } fn emit_relaxed_zero_extension( &mut self, sz_src: Size, @@ -2685,7 +2572,7 @@ impl MachineSpecific for MachineX86_64 { fn i32_ror(&mut self, loc_a: Location, loc_b: Location, ret: Location) { self.emit_shift_i32(Assembler::emit_ror, loc_a, loc_b, ret); } - fn i32_atomic_load( + fn i32_load( &mut self, addr: Location, memarg: &MemoryImmediate, @@ -2698,18 +2585,23 @@ impl MachineSpecific for MachineX86_64 { self.memory_op( addr, memarg, - true, + false, 4, need_check, imported_memories, offset, heap_access_oob, |this, addr| { - this.emit_relaxed_mov(Size::S32, Location::Memory(addr, 0), ret); + this.emit_relaxed_binop( + Assembler::emit_mov, + Size::S32, + Location::Memory(addr, 0), + ret, + ); }, ); } - fn i32_atomic_load_8u( + fn i32_load_8u( &mut self, addr: Location, memarg: &MemoryImmediate, @@ -2722,14 +2614,15 @@ impl MachineSpecific for MachineX86_64 { self.memory_op( addr, memarg, - true, + false, 1, need_check, imported_memories, offset, heap_access_oob, |this, addr| { - this.emit_relaxed_zero_extension( + this.emit_relaxed_zx_sx( + Assembler::emit_movzx, Size::S8, Location::Memory(addr, 0), Size::S32, @@ -2738,7 +2631,7 @@ impl MachineSpecific for MachineX86_64 { }, ); } - fn i32_atomic_load_16u( + fn i32_load_8s( &mut self, addr: Location, memarg: &MemoryImmediate, @@ -2751,15 +2644,16 @@ impl MachineSpecific for MachineX86_64 { self.memory_op( addr, memarg, - true, - 2, + false, + 1, need_check, imported_memories, offset, heap_access_oob, |this, addr| { - this.emit_relaxed_zero_extension( - Size::S16, + this.emit_relaxed_zx_sx( + Assembler::emit_movsx, + Size::S8, Location::Memory(addr, 0), Size::S32, ret, @@ -2767,83 +2661,93 @@ impl MachineSpecific for MachineX86_64 { }, ); } - fn i32_atomic_save( + fn i32_load_16u( &mut self, - value: Location, + addr: Location, memarg: &MemoryImmediate, - target_addr: Location, + ret: Location, need_check: bool, imported_memories: bool, offset: i32, heap_access_oob: Label, ) { self.memory_op( - target_addr, + addr, memarg, - true, - 4, + false, + 2, need_check, imported_memories, offset, heap_access_oob, |this, addr| { - this.emit_relaxed_atomic_xchg(Size::S32, value, Location::Memory(addr, 0)); + this.emit_relaxed_zx_sx( + Assembler::emit_movzx, + Size::S16, + Location::Memory(addr, 0), + Size::S32, + ret, + ); }, ); } - fn i32_atomic_save_8( + fn i32_load_16s( &mut self, - value: Location, + addr: Location, memarg: &MemoryImmediate, - target_addr: Location, + ret: Location, need_check: bool, imported_memories: bool, offset: i32, heap_access_oob: Label, ) { self.memory_op( - target_addr, + addr, memarg, - true, - 1, + false, + 2, need_check, imported_memories, offset, heap_access_oob, |this, addr| { - this.emit_relaxed_atomic_xchg(Size::S8, value, Location::Memory(addr, 0)); + this.emit_relaxed_zx_sx( + Assembler::emit_movsx, + Size::S16, + Location::Memory(addr, 0), + Size::S32, + ret, + ); }, ); } - fn i32_atomic_save_16( + fn i32_atomic_load( &mut self, - value: Location, + addr: Location, memarg: &MemoryImmediate, - target_addr: Location, + ret: Location, need_check: bool, imported_memories: bool, offset: i32, heap_access_oob: Label, ) { self.memory_op( - target_addr, + addr, memarg, true, - 2, + 4, need_check, imported_memories, offset, heap_access_oob, |this, addr| { - this.emit_relaxed_atomic_xchg(Size::S16, value, Location::Memory(addr, 0)); + this.emit_relaxed_mov(Size::S32, Location::Memory(addr, 0), ret); }, ); } - // i32 atomic Add with i32 - fn i32_atomic_add( + fn i32_atomic_load_8u( &mut self, - loc: Location, - target: Location, + addr: Location, memarg: &MemoryImmediate, ret: Location, need_check: bool, @@ -2851,33 +2755,28 @@ impl MachineSpecific for MachineX86_64 { offset: i32, heap_access_oob: Label, ) { - let value = self.acquire_temp_gpr().unwrap(); - self.move_location(Size::S32, loc, Location::GPR(value)); self.memory_op( - target, + addr, memarg, true, - 4, + 1, need_check, imported_memories, offset, heap_access_oob, |this, addr| { - this.assembler.emit_lock_xadd( - Size::S32, - Location::GPR(value), + this.emit_relaxed_zero_extension( + Size::S8, Location::Memory(addr, 0), + Size::S32, + ret, ); }, ); - self.move_location(Size::S32, Location::GPR(value), ret); - self.release_gpr(value); } - // i32 atomic Add with u8 - fn i32_atomic_add_8u( + fn i32_atomic_load_16u( &mut self, - loc: Location, - target: Location, + addr: Location, memarg: &MemoryImmediate, ret: Location, need_check: bool, @@ -2885,82 +2784,306 @@ impl MachineSpecific for MachineX86_64 { offset: i32, heap_access_oob: Label, ) { - let value = self.acquire_temp_gpr().unwrap(); - self.move_location_extend(Size::S8, false, loc, Size::S32, Location::GPR(value)); self.memory_op( - target, + addr, memarg, true, - 1, + 2, need_check, imported_memories, offset, heap_access_oob, |this, addr| { - this.assembler.emit_lock_xadd( - Size::S8, - Location::GPR(value), + this.emit_relaxed_zero_extension( + Size::S16, Location::Memory(addr, 0), + Size::S32, + ret, ); }, ); - self.move_location(Size::S32, Location::GPR(value), ret); - self.release_gpr(value); } - // i32 atomic Add with u16 - fn i32_atomic_add_16u( + fn i32_save( &mut self, - loc: Location, - target: Location, + target_value: Location, memarg: &MemoryImmediate, - ret: Location, + target_addr: Location, need_check: bool, imported_memories: bool, offset: i32, heap_access_oob: Label, ) { - let value = self.acquire_temp_gpr().unwrap(); - self.move_location_extend(Size::S16, false, loc, Size::S32, Location::GPR(value)); self.memory_op( - target, + target_addr, memarg, - true, - 2, + false, + 4, need_check, imported_memories, offset, heap_access_oob, |this, addr| { - this.assembler.emit_lock_xadd( - Size::S16, - Location::GPR(value), + this.emit_relaxed_binop( + Assembler::emit_mov, + Size::S32, + target_value, Location::Memory(addr, 0), ); }, ); - self.move_location(Size::S32, Location::GPR(value), ret); - self.release_gpr(value); } - // i32 atomic Sub with i32 - fn i32_atomic_sub( + fn i32_save_8( &mut self, - loc: Location, - target: Location, + target_value: Location, memarg: &MemoryImmediate, - ret: Location, + target_addr: Location, need_check: bool, imported_memories: bool, offset: i32, heap_access_oob: Label, ) { - let value = self.acquire_temp_gpr().unwrap(); - self.location_neg(Size::S32, false, loc, Size::S32, Location::GPR(value)); self.memory_op( - target, + target_addr, memarg, - true, - 4, - need_check, + false, + 1, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_relaxed_binop( + Assembler::emit_mov, + Size::S8, + target_value, + Location::Memory(addr, 0), + ); + }, + ); + } + fn i32_save_16( + &mut self, + target_value: Location, + memarg: &MemoryImmediate, + target_addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.memory_op( + target_addr, + memarg, + false, + 2, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_relaxed_binop( + Assembler::emit_mov, + Size::S16, + target_value, + Location::Memory(addr, 0), + ); + }, + ); + } + fn i32_atomic_save( + &mut self, + value: Location, + memarg: &MemoryImmediate, + target_addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.memory_op( + target_addr, + memarg, + true, + 4, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_relaxed_atomic_xchg(Size::S32, value, Location::Memory(addr, 0)); + }, + ); + } + fn i32_atomic_save_8( + &mut self, + value: Location, + memarg: &MemoryImmediate, + target_addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.memory_op( + target_addr, + memarg, + true, + 1, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_relaxed_atomic_xchg(Size::S8, value, Location::Memory(addr, 0)); + }, + ); + } + fn i32_atomic_save_16( + &mut self, + value: Location, + memarg: &MemoryImmediate, + target_addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.memory_op( + target_addr, + memarg, + true, + 2, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_relaxed_atomic_xchg(Size::S16, value, Location::Memory(addr, 0)); + }, + ); + } + // i32 atomic Add with i32 + fn i32_atomic_add( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + let value = self.acquire_temp_gpr().unwrap(); + self.move_location(Size::S32, loc, Location::GPR(value)); + self.memory_op( + target, + memarg, + true, + 4, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.assembler.emit_lock_xadd( + Size::S32, + Location::GPR(value), + Location::Memory(addr, 0), + ); + }, + ); + self.move_location(Size::S32, Location::GPR(value), ret); + self.release_gpr(value); + } + // i32 atomic Add with u8 + fn i32_atomic_add_8u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + let value = self.acquire_temp_gpr().unwrap(); + self.move_location_extend(Size::S8, false, loc, Size::S32, Location::GPR(value)); + self.memory_op( + target, + memarg, + true, + 1, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.assembler.emit_lock_xadd( + Size::S8, + Location::GPR(value), + Location::Memory(addr, 0), + ); + }, + ); + self.move_location(Size::S32, Location::GPR(value), ret); + self.release_gpr(value); + } + // i32 atomic Add with u16 + fn i32_atomic_add_16u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + let value = self.acquire_temp_gpr().unwrap(); + self.move_location_extend(Size::S16, false, loc, Size::S32, Location::GPR(value)); + self.memory_op( + target, + memarg, + true, + 2, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.assembler.emit_lock_xadd( + Size::S16, + Location::GPR(value), + Location::Memory(addr, 0), + ); + }, + ); + self.move_location(Size::S32, Location::GPR(value), ret); + self.release_gpr(value); + } + // i32 atomic Sub with i32 + fn i32_atomic_sub( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + let value = self.acquire_temp_gpr().unwrap(); + self.location_neg(Size::S32, false, loc, Size::S32, Location::GPR(value)); + self.memory_op( + target, + memarg, + true, + 4, + need_check, imported_memories, offset, heap_access_oob, @@ -3865,26 +3988,247 @@ impl MachineSpecific for MachineX86_64 { } else { self.assembler.emit_popcnt(Size::S64, loc, ret); } - } - _ => { - unreachable!(); - } - } - } - fn i64_shl(&mut self, loc_a: Location, loc_b: Location, ret: Location) { - self.emit_shift_i64(Assembler::emit_shl, loc_a, loc_b, ret); - } - fn i64_shr(&mut self, loc_a: Location, loc_b: Location, ret: Location) { - self.emit_shift_i64(Assembler::emit_shr, loc_a, loc_b, ret); - } - fn i64_sar(&mut self, loc_a: Location, loc_b: Location, ret: Location) { - self.emit_shift_i64(Assembler::emit_sar, loc_a, loc_b, ret); - } - fn i64_rol(&mut self, loc_a: Location, loc_b: Location, ret: Location) { - self.emit_shift_i64(Assembler::emit_rol, loc_a, loc_b, ret); + } + _ => { + unreachable!(); + } + } + } + fn i64_shl(&mut self, loc_a: Location, loc_b: Location, ret: Location) { + self.emit_shift_i64(Assembler::emit_shl, loc_a, loc_b, ret); + } + fn i64_shr(&mut self, loc_a: Location, loc_b: Location, ret: Location) { + self.emit_shift_i64(Assembler::emit_shr, loc_a, loc_b, ret); + } + fn i64_sar(&mut self, loc_a: Location, loc_b: Location, ret: Location) { + self.emit_shift_i64(Assembler::emit_sar, loc_a, loc_b, ret); + } + fn i64_rol(&mut self, loc_a: Location, loc_b: Location, ret: Location) { + self.emit_shift_i64(Assembler::emit_rol, loc_a, loc_b, ret); + } + fn i64_ror(&mut self, loc_a: Location, loc_b: Location, ret: Location) { + self.emit_shift_i64(Assembler::emit_ror, loc_a, loc_b, ret); + } + fn i64_load( + &mut self, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.memory_op( + addr, + memarg, + false, + 8, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_relaxed_binop( + Assembler::emit_mov, + Size::S64, + Location::Memory(addr, 0), + ret, + ); + }, + ); + } + fn i64_load_8u( + &mut self, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.memory_op( + addr, + memarg, + false, + 1, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_relaxed_zx_sx( + Assembler::emit_movzx, + Size::S8, + Location::Memory(addr, 0), + Size::S64, + ret, + ); + }, + ); + } + fn i64_load_8s( + &mut self, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.memory_op( + addr, + memarg, + false, + 1, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_relaxed_zx_sx( + Assembler::emit_movsx, + Size::S8, + Location::Memory(addr, 0), + Size::S64, + ret, + ); + }, + ); + } + fn i64_load_16u( + &mut self, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.memory_op( + addr, + memarg, + false, + 2, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_relaxed_zx_sx( + Assembler::emit_movzx, + Size::S16, + Location::Memory(addr, 0), + Size::S64, + ret, + ); + }, + ); + } + fn i64_load_16s( + &mut self, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.memory_op( + addr, + memarg, + false, + 2, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_relaxed_zx_sx( + Assembler::emit_movsx, + Size::S16, + Location::Memory(addr, 0), + Size::S64, + ret, + ); + }, + ); + } + fn i64_load_32u( + &mut self, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.memory_op( + addr, + memarg, + false, + 4, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + match ret { + Location::GPR(_) => {} + Location::Memory(base, offset) => { + this.assembler.emit_mov( + Size::S32, + Location::Imm32(0), + Location::Memory(base, offset + 4), + ); // clear upper bits + } + _ => { + unreachable!(); + } + } + this.emit_relaxed_binop( + Assembler::emit_mov, + Size::S32, + Location::Memory(addr, 0), + ret, + ); + }, + ); } - fn i64_ror(&mut self, loc_a: Location, loc_b: Location, ret: Location) { - self.emit_shift_i64(Assembler::emit_ror, loc_a, loc_b, ret); + fn i64_load_32s( + &mut self, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.memory_op( + addr, + memarg, + false, + 4, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_relaxed_zx_sx( + Assembler::emit_movsx, + Size::S32, + Location::Memory(addr, 0), + Size::S64, + ret, + ); + }, + ); } fn i64_atomic_load( &mut self, @@ -4010,6 +4354,122 @@ impl MachineSpecific for MachineX86_64 { }, ); } + fn i64_save( + &mut self, + target_value: Location, + memarg: &MemoryImmediate, + target_addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.memory_op( + target_addr, + memarg, + false, + 8, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_relaxed_binop( + Assembler::emit_mov, + Size::S64, + target_value, + Location::Memory(addr, 0), + ); + }, + ); + } + fn i64_save_8( + &mut self, + target_value: Location, + memarg: &MemoryImmediate, + target_addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.memory_op( + target_addr, + memarg, + false, + 1, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_relaxed_binop( + Assembler::emit_mov, + Size::S8, + target_value, + Location::Memory(addr, 0), + ); + }, + ); + } + fn i64_save_16( + &mut self, + target_value: Location, + memarg: &MemoryImmediate, + target_addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.memory_op( + target_addr, + memarg, + false, + 2, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_relaxed_binop( + Assembler::emit_mov, + Size::S16, + target_value, + Location::Memory(addr, 0), + ); + }, + ); + } + fn i64_save_32( + &mut self, + target_value: Location, + memarg: &MemoryImmediate, + target_addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.memory_op( + target_addr, + memarg, + false, + 4, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_relaxed_binop( + Assembler::emit_mov, + Size::S32, + target_value, + Location::Memory(addr, 0), + ); + }, + ); + } fn i64_atomic_save( &mut self, value: Location, @@ -5073,6 +5533,135 @@ impl MachineSpecific for MachineX86_64 { self.release_gpr(compare); } + fn f32_load( + &mut self, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.memory_op( + addr, + memarg, + false, + 4, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_relaxed_binop( + Assembler::emit_mov, + Size::S32, + Location::Memory(addr, 0), + ret, + ); + }, + ); + } + fn f32_save( + &mut self, + target_value: Location, + memarg: &MemoryImmediate, + target_addr: Location, + canonicalize: bool, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + let canonicalize = canonicalize && self.arch_supports_canonicalize_nan(); + self.memory_op( + target_addr, + memarg, + false, + 4, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + if !canonicalize { + this.emit_relaxed_binop( + Assembler::emit_mov, + Size::S32, + target_value, + Location::Memory(addr, 0), + ); + } else { + this.canonicalize_nan(Size::S32, target_value, Location::Memory(addr, 0)); + } + }, + ); + } + fn f64_load( + &mut self, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.memory_op( + addr, + memarg, + false, + 8, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_relaxed_binop( + Assembler::emit_mov, + Size::S64, + Location::Memory(addr, 0), + ret, + ); + }, + ); + } + fn f64_save( + &mut self, + target_value: Location, + memarg: &MemoryImmediate, + target_addr: Location, + canonicalize: bool, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + let canonicalize = canonicalize && self.arch_supports_canonicalize_nan(); + self.memory_op( + target_addr, + memarg, + false, + 8, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + if !canonicalize { + this.emit_relaxed_binop( + Assembler::emit_mov, + Size::S64, + target_value, + Location::Memory(addr, 0), + ); + } else { + this.canonicalize_nan(Size::S64, target_value, Location::Memory(addr, 0)); + } + }, + ); + } + fn convert_f64_i64(&mut self, loc: Location, signed: bool, ret: Location) { if self.assembler.arch_has_fconverti() { let tmp_out = self.acquire_temp_simd().unwrap();