From cbd169add25fbe226b6d61e4f59919f996a0e3a3 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Fri, 19 Nov 2021 10:32:34 +0000 Subject: [PATCH 01/12] Upgrade rkyv to 0.7 --- Cargo.lock | 42 +++++++++++++++++++++++---- lib/compiler/Cargo.toml | 4 +-- lib/compiler/src/section.rs | 2 +- lib/engine-dylib/Cargo.toml | 2 +- lib/engine-dylib/src/serialize.rs | 16 +++++----- lib/engine-universal/Cargo.toml | 2 +- lib/engine-universal/src/serialize.rs | 16 +++++----- lib/types/Cargo.toml | 2 +- lib/types/src/indexes.rs | 24 +++++++-------- lib/types/src/module.rs | 15 +++++----- lib/vm/Cargo.toml | 2 +- 11 files changed, 78 insertions(+), 49 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 815f04419e0..23a613570a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -210,6 +210,27 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631" +[[package]] +name = "bytecheck" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb738a1e65989ecdcd5bba16079641bd7209688fa546e1064832fd6e012fd32a" +dependencies = [ + "bytecheck_derive", + "ptr_meta", +] + +[[package]] +name = "bytecheck_derive" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b4dff26fdc9f847dab475c9fec16f2cba82d5aa1f09981b87c44520721e10a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "byteorder" version = "1.4.3" @@ -1772,23 +1793,34 @@ dependencies = [ "winapi", ] +[[package]] +name = "rend" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0351a2e529ee30d571ef31faa5a4e0b9addaad087697b77efb20d2809e41c7" +dependencies = [ + "bytecheck", +] + [[package]] name = "rkyv" -version = "0.6.7" +version = "0.7.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb135b3e5e3311f0a254bfb00333f4bac9ef1d89888b84242a89eb8722b09a07" +checksum = "e419b2e30d088b21c4bf3072561535305df8066e89937ad05fc205b99874c23c" dependencies = [ - "memoffset", + "bytecheck", + "hashbrown 0.11.2", "ptr_meta", + "rend", "rkyv_derive", "seahash", ] [[package]] name = "rkyv_derive" -version = "0.6.7" +version = "0.7.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba8f489f6b6d8551bb15904293c1ad58a6abafa7d8390d15f7ed05a2afcd87d5" +checksum = "2ae58c4ba80f15f2f0842f4c61729e92c4e33a09bd78196c2b1ab9b0771a3ddf" dependencies = [ "proc-macro2", "quote", diff --git a/lib/compiler/Cargo.toml b/lib/compiler/Cargo.toml index 215c49bdc7b..ce07cc76460 100644 --- a/lib/compiler/Cargo.toml +++ b/lib/compiler/Cargo.toml @@ -20,8 +20,8 @@ hashbrown = { version = "0.11", optional = true } serde = { version = "1.0", features = ["derive"], optional = true } thiserror = "1.0" serde_bytes = { version = "0.11", optional = true } -smallvec = "1.6" -rkyv = { version = "0.6.1", optional = true } +smallvec = "1.6" +rkyv = { version = "0.7.20", optional = true } loupe = "0.1" [features] diff --git a/lib/compiler/src/section.rs b/lib/compiler/src/section.rs index 98063705d99..1ac45eeda4f 100644 --- a/lib/compiler/src/section.rs +++ b/lib/compiler/src/section.rs @@ -22,7 +22,7 @@ use wasmer_types::entity::entity_impl; )] #[cfg_attr( feature = "enable-rkyv", - archive(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) + archive_attr(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) )] #[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug, MemoryUsage)] pub struct SectionIndex(u32); diff --git a/lib/engine-dylib/Cargo.toml b/lib/engine-dylib/Cargo.toml index 1b356e07020..aa9e1a356b5 100644 --- a/lib/engine-dylib/Cargo.toml +++ b/lib/engine-dylib/Cargo.toml @@ -23,7 +23,7 @@ leb128 = "0.2" libloading = "0.7" tempfile = "3.1" which = "4.0" -rkyv = "0.6.1" +rkyv = "0.7.20" loupe = "0.1" [features] diff --git a/lib/engine-dylib/src/serialize.rs b/lib/engine-dylib/src/serialize.rs index 7021834a8a2..b087318575d 100644 --- a/lib/engine-dylib/src/serialize.rs +++ b/lib/engine-dylib/src/serialize.rs @@ -1,10 +1,8 @@ use loupe::MemoryUsage; use rkyv::{ - archived_value, - de::{adapters::SharedDeserializerAdapter, deserializers::AllocDeserializer}, - ser::adapters::SharedSerializerAdapter, - ser::{serializers::WriteSerializer, Serializer as RkyvSerializer}, - Archive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize, + archived_value, de::deserializers::SharedDeserializeMap, ser::serializers::AllocSerializer, + ser::Serializer as RkyvSerializer, Archive, Deserialize as RkyvDeserialize, + Serialize as RkyvSerialize, }; use serde::{Deserialize, Serialize}; use std::error::Error; @@ -59,11 +57,11 @@ impl ModuleMetadata { } pub fn serialize(&mut self) -> Result, CompileError> { - let mut serializer = SharedSerializerAdapter::new(WriteSerializer::new(vec![])); + let mut serializer = AllocSerializer::<4096>::default(); let pos = serializer.serialize_value(self).map_err(to_compile_error)? as u64; - let mut serialized_data = serializer.into_inner().into_inner(); + let mut serialized_data = serializer.into_serializer().into_inner(); serialized_data.extend_from_slice(&pos.to_le_bytes()); - Ok(serialized_data) + Ok(serialized_data.to_vec()) } pub unsafe fn deserialize(metadata_slice: &[u8]) -> Result { @@ -86,7 +84,7 @@ impl ModuleMetadata { pub fn deserialize_from_archive( archived: &ArchivedModuleMetadata, ) -> Result { - let mut deserializer = SharedDeserializerAdapter::new(AllocDeserializer); + let mut deserializer = SharedDeserializeMap::new(); RkyvDeserialize::deserialize(archived, &mut deserializer) .map_err(|e| DeserializeError::CorruptedBinary(format!("{:?}", e))) } diff --git a/lib/engine-universal/Cargo.toml b/lib/engine-universal/Cargo.toml index 683acc8fb23..9bee0203f9f 100644 --- a/lib/engine-universal/Cargo.toml +++ b/lib/engine-universal/Cargo.toml @@ -19,7 +19,7 @@ wasmer-engine = { path = "../engine", version = "2.0.0" } region = "3.0" cfg-if = "1.0" leb128 = "0.2" -rkyv = "0.6.1" +rkyv = "0.7.20" loupe = "0.1" [target.'cfg(target_os = "windows")'.dependencies] diff --git a/lib/engine-universal/src/serialize.rs b/lib/engine-universal/src/serialize.rs index 6e3c768d452..4a8d62c1336 100644 --- a/lib/engine-universal/src/serialize.rs +++ b/lib/engine-universal/src/serialize.rs @@ -1,10 +1,8 @@ use loupe::MemoryUsage; use rkyv::{ - archived_value, - de::{adapters::SharedDeserializerAdapter, deserializers::AllocDeserializer}, - ser::adapters::SharedSerializerAdapter, - ser::{serializers::WriteSerializer, Serializer as RkyvSerializer}, - Archive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize, + archived_value, de::deserializers::SharedDeserializeMap, ser::serializers::AllocSerializer, + ser::Serializer as RkyvSerializer, Archive, Deserialize as RkyvDeserialize, + Serialize as RkyvSerialize, }; use wasmer_compiler::{ CompileModuleInfo, CompiledFunctionFrameInfo, CustomSection, Dwarf, FunctionBody, @@ -49,13 +47,13 @@ impl SerializableModule { /// The bytes will have the following format: /// RKYV serialization (any length) + POS (8 bytes) pub fn serialize(&self) -> Result, SerializeError> { - let mut serializer = SharedSerializerAdapter::new(WriteSerializer::new(vec![])); + let mut serializer = AllocSerializer::<4096>::default(); let pos = serializer .serialize_value(self) .map_err(to_serialize_error)? as u64; - let mut serialized_data = serializer.into_inner().into_inner(); + let mut serialized_data = serializer.into_serializer().into_inner(); serialized_data.extend_from_slice(&pos.to_le_bytes()); - Ok(serialized_data) + Ok(serialized_data.to_vec()) } /// Deserialize a Module from a slice. @@ -98,7 +96,7 @@ impl SerializableModule { pub fn deserialize_from_archive( archived: &ArchivedSerializableModule, ) -> Result { - let mut deserializer = SharedDeserializerAdapter::new(AllocDeserializer); + let mut deserializer = SharedDeserializeMap::new(); RkyvDeserialize::deserialize(archived, &mut deserializer) .map_err(|e| DeserializeError::CorruptedBinary(format!("{:?}", e))) } diff --git a/lib/types/Cargo.toml b/lib/types/Cargo.toml index 11c5d3b5411..3c3ea3b9bb0 100644 --- a/lib/types/Cargo.toml +++ b/lib/types/Cargo.toml @@ -14,7 +14,7 @@ edition = "2018" serde = { version = "1.0", features = ["derive", "rc"], optional = true, default-features = false } thiserror = "1.0" indexmap = { version = "1.6", features = ["serde-1"] } -rkyv = { version = "0.6.1", optional = true } +rkyv = { version = "0.7.20", optional = true } loupe = { version = "0.1", features = ["enable-indexmap"] } [features] diff --git a/lib/types/src/indexes.rs b/lib/types/src/indexes.rs index af1975ecc76..b7fd2008ba7 100644 --- a/lib/types/src/indexes.rs +++ b/lib/types/src/indexes.rs @@ -16,7 +16,7 @@ use serde::{Deserialize, Serialize}; )] #[cfg_attr( feature = "enable-rkyv", - archive(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) + archive_attr(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) )] pub struct LocalFunctionIndex(u32); entity_impl!(LocalFunctionIndex); @@ -44,7 +44,7 @@ entity_impl!(LocalMemoryIndex); )] #[cfg_attr( feature = "enable-rkyv", - archive(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) + archive_attr(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) )] pub struct LocalGlobalIndex(u32); entity_impl!(LocalGlobalIndex); @@ -60,7 +60,7 @@ entity_impl!(ArchivedLocalGlobalIndex); )] #[cfg_attr( feature = "enable-rkyv", - archive(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) + archive_attr(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) )] pub struct FunctionIndex(u32); entity_impl!(FunctionIndex); @@ -76,7 +76,7 @@ entity_impl!(ArchivedFunctionIndex); )] #[cfg_attr( feature = "enable-rkyv", - archive(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) + archive_attr(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) )] pub struct TableIndex(u32); entity_impl!(TableIndex); @@ -92,7 +92,7 @@ entity_impl!(ArchivedTableIndex); )] #[cfg_attr( feature = "enable-rkyv", - archive(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) + archive_attr(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) )] pub struct GlobalIndex(u32); entity_impl!(GlobalIndex); @@ -108,7 +108,7 @@ entity_impl!(ArchivedGlobalIndex); )] #[cfg_attr( feature = "enable-rkyv", - archive(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) + archive_attr(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) )] pub struct MemoryIndex(u32); entity_impl!(MemoryIndex); @@ -124,7 +124,7 @@ entity_impl!(ArchivedMemoryIndex); )] #[cfg_attr( feature = "enable-rkyv", - archive(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) + archive_attr(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) )] pub struct SignatureIndex(u32); entity_impl!(SignatureIndex); @@ -140,7 +140,7 @@ entity_impl!(ArchivedSignatureIndex); )] #[cfg_attr( feature = "enable-rkyv", - archive(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) + archive_attr(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) )] pub struct DataIndex(u32); entity_impl!(DataIndex); @@ -156,7 +156,7 @@ entity_impl!(ArchivedDataIndex); )] #[cfg_attr( feature = "enable-rkyv", - archive(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) + archive_attr(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) )] pub struct ElemIndex(u32); entity_impl!(ElemIndex); @@ -172,7 +172,7 @@ entity_impl!(ArchivedElemIndex); )] #[cfg_attr( feature = "enable-rkyv", - archive(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) + archive_attr(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) )] pub struct CustomSectionIndex(u32); entity_impl!(CustomSectionIndex); @@ -188,7 +188,7 @@ entity_impl!(ArchivedCustomSectionIndex); )] #[cfg_attr( feature = "enable-rkyv", - archive(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) + archive_attr(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) )] pub enum ExportIndex { /// Function export. @@ -210,7 +210,7 @@ pub enum ExportIndex { )] #[cfg_attr( feature = "enable-rkyv", - archive(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) + archive_attr(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) )] pub enum ImportIndex { /// Function import. diff --git a/lib/types/src/module.rs b/lib/types/src/module.rs index a4a02ed7d3c..0c1d0c75803 100644 --- a/lib/types/src/module.rs +++ b/lib/types/src/module.rs @@ -15,18 +15,17 @@ use crate::{ }; use indexmap::IndexMap; use loupe::MemoryUsage; +use rkyv::ser::ScratchSpace; #[cfg(feature = "enable-rkyv")] use rkyv::{ - de::SharedDeserializer, ser::Serializer, ser::SharedSerializer, Archive, Archived, - Deserialize as RkyvDeserialize, Fallible, Serialize as RkyvSerialize, + de::SharedDeserializeRegistry, ser::Serializer, ser::SharedSerializeRegistry, Archive, + Archived, Deserialize as RkyvDeserialize, Fallible, Serialize as RkyvSerialize, }; #[cfg(feature = "enable-serde")] use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::fmt; use std::iter::ExactSizeIterator; -#[cfg(feature = "enable-rkyv")] -use std::mem::MaybeUninit; use std::sync::atomic::{AtomicUsize, Ordering::SeqCst}; use std::sync::Arc; @@ -228,20 +227,22 @@ impl Archive for ModuleInfo { type Archived = ::Archived; type Resolver = ::Resolver; - fn resolve(&self, pos: usize, resolver: Self::Resolver, out: &mut MaybeUninit) { + unsafe fn resolve(&self, pos: usize, resolver: Self::Resolver, out: *mut Self::Archived) { ArchivableModuleInfo::from(self).resolve(pos, resolver, out) } } #[cfg(feature = "enable-rkyv")] -impl RkyvSerialize for ModuleInfo { +impl RkyvSerialize + for ModuleInfo +{ fn serialize(&self, serializer: &mut S) -> Result { ArchivableModuleInfo::from(self).serialize(serializer) } } #[cfg(feature = "enable-rkyv")] -impl RkyvDeserialize +impl RkyvDeserialize for Archived { fn deserialize(&self, deserializer: &mut D) -> Result { diff --git a/lib/vm/Cargo.toml b/lib/vm/Cargo.toml index b817d364201..4f1219ef580 100644 --- a/lib/vm/Cargo.toml +++ b/lib/vm/Cargo.toml @@ -21,7 +21,7 @@ more-asserts = "0.2" cfg-if = "1.0" backtrace = "0.3" serde = { version = "1.0", features = ["derive", "rc"] } -rkyv = { version = "0.6.1", optional = true} +rkyv = { version = "0.7.20", optional = true } loupe = { version = "0.1", features = ["enable-indexmap"] } [target.'cfg(target_os = "windows")'.dependencies] From 9e0fe7d6950439739232c0feefbf3a64d24dda86 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Fri, 19 Nov 2021 14:32:52 +0000 Subject: [PATCH 02/12] Use BTreeMap in ModuleInfo to avoid non-deterministic serialization --- lib/types/src/module.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/types/src/module.rs b/lib/types/src/module.rs index 0c1d0c75803..135e17ce1e6 100644 --- a/lib/types/src/module.rs +++ b/lib/types/src/module.rs @@ -23,7 +23,7 @@ use rkyv::{ }; #[cfg(feature = "enable-serde")] use serde::{Deserialize, Serialize}; -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::fmt; use std::iter::ExactSizeIterator; use std::sync::atomic::{AtomicUsize, Ordering::SeqCst}; @@ -141,10 +141,10 @@ pub struct ArchivableModuleInfo { exports: ArchivableIndexMap, start_function: Option, table_initializers: Vec, - passive_elements: HashMap>, - passive_data: HashMap>, + passive_elements: BTreeMap>, + passive_data: BTreeMap>, global_initializers: PrimaryMap, - function_names: HashMap, + function_names: BTreeMap, signatures: PrimaryMap, functions: PrimaryMap, tables: PrimaryMap, @@ -167,10 +167,10 @@ impl From for ArchivableModuleInfo { exports: ArchivableIndexMap::from(it.exports), start_function: it.start_function, table_initializers: it.table_initializers, - passive_elements: it.passive_elements, - passive_data: it.passive_data, + passive_elements: it.passive_elements.into_iter().collect(), + passive_data: it.passive_data.into_iter().collect(), global_initializers: it.global_initializers, - function_names: it.function_names, + function_names: it.function_names.into_iter().collect(), signatures: it.signatures, functions: it.functions, tables: it.tables, @@ -196,10 +196,10 @@ impl From for ModuleInfo { exports: it.exports.into(), start_function: it.start_function, table_initializers: it.table_initializers, - passive_elements: it.passive_elements, - passive_data: it.passive_data, + passive_elements: it.passive_elements.into_iter().collect(), + passive_data: it.passive_data.into_iter().collect(), global_initializers: it.global_initializers, - function_names: it.function_names, + function_names: it.function_names.into_iter().collect(), signatures: it.signatures, functions: it.functions, tables: it.tables, From 57f876b92295fca5152cfbe3c3ab05cfc1ccb68a Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Mon, 22 Nov 2021 15:30:15 +0000 Subject: [PATCH 03/12] Make DWARF frametable generation deterministic in Cranelift --- lib/compiler-cranelift/src/compiler.rs | 94 ++++++++++++-------------- 1 file changed, 44 insertions(+), 50 deletions(-) diff --git a/lib/compiler-cranelift/src/compiler.rs b/lib/compiler-cranelift/src/compiler.rs index c836b9ce106..6a887752f16 100644 --- a/lib/compiler-cranelift/src/compiler.rs +++ b/lib/compiler-cranelift/src/compiler.rs @@ -92,14 +92,13 @@ impl Compiler for CraneliftCompiler { // FDEs will cause some issues in Linux. None } else { - use std::sync::Mutex; match target.triple().default_calling_convention() { Ok(CallingConvention::SystemV) => { match isa.create_systemv_cie() { Some(cie) => { let mut dwarf_frametable = FrameTable::default(); let cie_id = dwarf_frametable.add_cie(cie); - Some((Arc::new(Mutex::new(dwarf_frametable)), cie_id)) + Some((dwarf_frametable, cie_id)) } // Even though we are in a SystemV system, Cranelift doesn't support it None => None, @@ -134,7 +133,7 @@ impl Compiler for CraneliftCompiler { #[cfg(all(target_arch = "x86_64", target_os = "linux"))] let probestack_trampoline_relocation_target = SectionIndex::new(custom_sections.len() - 1); - let functions = function_body_inputs + let (functions, fdes): (Vec, Vec<_>) = function_body_inputs .iter() .collect::)>>() .par_iter() @@ -190,31 +189,25 @@ impl Compiler for CraneliftCompiler { CompileError::Codegen(pretty_error(&context.func, Some(&*isa), error)) })?; - let unwind_info = match compiled_function_unwind_info(&*isa, &context)? { + let (unwind_info, fde) = match compiled_function_unwind_info(&*isa, &context)? { #[cfg(feature = "unwind")] CraneliftUnwindInfo::FDE(fde) => { - if let Some((dwarf_frametable, cie_id)) = &dwarf_frametable { - dwarf_frametable - .lock() - .expect("Can't write into DWARF frametable") - .add_fde( - *cie_id, - fde.to_fde(Address::Symbol { - // The symbol is the kind of relocation. - // "0" is used for functions - symbol: WriterRelocate::FUNCTION_SYMBOL, - // We use the addend as a way to specify the - // function index - addend: i.index() as _, - }), - ); + if dwarf_frametable.is_some() { + let fde = fde.to_fde(Address::Symbol { + // The symbol is the kind of relocation. + // "0" is used for functions + symbol: WriterRelocate::FUNCTION_SYMBOL, + // We use the addend as a way to specify the + // function index + addend: i.index() as _, + }); // The unwind information is inserted into the dwarf section - Some(CompiledFunctionUnwindInfo::Dwarf) + (Some(CompiledFunctionUnwindInfo::Dwarf), Some(fde)) } else { - None + (None, None) } } - other => other.maybe_into_to_windows_unwind(), + other => (other.maybe_into_to_windows_unwind(), None), }; let range = reader.range(); @@ -223,40 +216,41 @@ impl Compiler for CraneliftCompiler { // We transform the Cranelift JumpTable's into compiler JumpTables let func_jt_offsets = transform_jump_table(context.func.jt_offsets); - Ok(CompiledFunction { - body: FunctionBody { - body: code_buf, - unwind_info, + Ok(( + CompiledFunction { + body: FunctionBody { + body: code_buf, + unwind_info, + }, + jt_offsets: func_jt_offsets, + relocations: reloc_sink.func_relocs, + frame_info: CompiledFunctionFrameInfo { + address_map, + traps: trap_sink.traps, + }, }, - jt_offsets: func_jt_offsets, - relocations: reloc_sink.func_relocs, - frame_info: CompiledFunctionFrameInfo { - address_map, - traps: trap_sink.traps, - }, - }) + fde, + )) }) .collect::, CompileError>>()? .into_iter() - .collect::>(); + .unzip(); #[cfg(feature = "unwind")] - let dwarf = { - let dwarf = if let Some((dwarf_frametable, _cie_id)) = dwarf_frametable { - let mut eh_frame = EhFrame(WriterRelocate::new(target.triple().endianness().ok())); - dwarf_frametable - .lock() - .unwrap() - .write_eh_frame(&mut eh_frame) - .unwrap(); + let dwarf = if let Some((mut dwarf_frametable, cie_id)) = dwarf_frametable { + for fde in fdes { + if let Some(fde) = fde { + dwarf_frametable.add_fde(cie_id, fde); + } + } + let mut eh_frame = EhFrame(WriterRelocate::new(target.triple().endianness().ok())); + dwarf_frametable.write_eh_frame(&mut eh_frame).unwrap(); - let eh_frame_section = eh_frame.0.into_section(); - custom_sections.push(eh_frame_section); - Some(Dwarf::new(SectionIndex::new(custom_sections.len() - 1))) - } else { - None - }; - dwarf + let eh_frame_section = eh_frame.0.into_section(); + custom_sections.push(eh_frame_section); + Some(Dwarf::new(SectionIndex::new(custom_sections.len() - 1))) + } else { + None }; #[cfg(not(feature = "unwind"))] let dwarf = None; @@ -289,7 +283,7 @@ impl Compiler for CraneliftCompiler { .collect::>(); Ok(Compilation::new( - functions, + functions.into_iter().collect(), custom_sections, function_call_trampolines, dynamic_function_trampolines, From a61893bbb320454087c287fb84acf436fa1db3f6 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Mon, 22 Nov 2021 15:30:38 +0000 Subject: [PATCH 04/12] Use BTreeMap when serializing an IndexMap --- lib/types/src/archives.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/types/src/archives.rs b/lib/types/src/archives.rs index 0a7d84a58ee..48e30272e37 100644 --- a/lib/types/src/archives.rs +++ b/lib/types/src/archives.rs @@ -3,19 +3,21 @@ use core::hash::Hash; use indexmap::IndexMap; use rkyv::{Archive, Deserialize, Serialize}; #[cfg(feature = "std")] -use std::{collections::HashMap, hash::Hash}; +use std::{collections::BTreeMap, hash::Hash}; #[derive(Serialize, Deserialize, Archive)] /// Rkyv Archivable IndexMap -pub struct ArchivableIndexMap { - indices: HashMap, +pub struct ArchivableIndexMap { + indices: BTreeMap, entries: Vec<(K, V)>, } -impl From> for ArchivableIndexMap { +impl From> + for ArchivableIndexMap +{ fn from(it: IndexMap) -> ArchivableIndexMap { let mut r = ArchivableIndexMap { - indices: HashMap::new(), + indices: BTreeMap::new(), entries: Vec::new(), }; let mut i: u64 = 0; @@ -28,7 +30,9 @@ impl From> for Archiv } } -impl Into> for ArchivableIndexMap { +impl Into> + for ArchivableIndexMap +{ fn into(self) -> IndexMap { let mut r = IndexMap::new(); for (k, v) in self.entries.into_iter() { From b32e8d92c1242aa5b35d954928ff08a2294fe1dd Mon Sep 17 00:00:00 2001 From: Anbang Wen Date: Fri, 12 Mar 2021 11:40:10 -0800 Subject: [PATCH 05/12] add a compiler test to check for deterministic This is to test #2173, the empty test always pass while the table one fails sometimes. --- tests/compilers/deterministic.rs | 44 ++++++++++++++++++++++++++++++++ tests/compilers/main.rs | 1 + 2 files changed, 45 insertions(+) create mode 100644 tests/compilers/deterministic.rs diff --git a/tests/compilers/deterministic.rs b/tests/compilers/deterministic.rs new file mode 100644 index 00000000000..9c9070ce5c0 --- /dev/null +++ b/tests/compilers/deterministic.rs @@ -0,0 +1,44 @@ +use anyhow::Result; +use wasmer::{wat2wasm, Module}; + +fn compile_and_compare(wasm: &[u8]) -> Result<()> { + let store = Default::default(); + + // compile for first time + let module = Module::new(&store, wasm)?; + let first = module.serialize()?; + + // compile for second time + let module = Module::new(&store, wasm)?; + let second = module.serialize()?; + + assert!(first == second); + + Ok(()) +} + +#[test] +fn deterministic_empty() -> Result<()> { + let wasm_bytes = wat2wasm( + br#" + (module) + "#, + )?; + + compile_and_compare(&wasm_bytes) +} + +#[test] +fn deterministic_table() -> Result<()> { + let wasm_bytes = wat2wasm( + br#" +(module + (table 2 funcref) + (func $f1) + (func $f2) + (elem (i32.const 0) $f1 $f2)) +"#, + )?; + + compile_and_compare(&wasm_bytes) +} diff --git a/tests/compilers/main.rs b/tests/compilers/main.rs index 290d31c8fd0..9ba00794f68 100644 --- a/tests/compilers/main.rs +++ b/tests/compilers/main.rs @@ -6,6 +6,7 @@ extern crate compiler_test_derive; mod config; +mod deterministic; mod imports; mod issues; mod metering; From 4e46c80c48991aa5d0b4da9863efcb0469750c62 Mon Sep 17 00:00:00 2001 From: Anbang Wen Date: Wed, 17 Mar 2021 18:00:31 -0700 Subject: [PATCH 06/12] fuzz: add a test for for deterministic compilation --- fuzz/Cargo.toml | 5 ++ fuzz/fuzz_targets/deterministic.rs | 77 ++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 fuzz/fuzz_targets/deterministic.rs diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 891de355229..4b52f18a3ba 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -57,3 +57,8 @@ required-features = ["universal", "cranelift"] name = "dylib_cranelift" path = "fuzz_targets/dylib_cranelift.rs" required-features = ["dylib", "cranelift"] + +[[bin]] +name = "deterministic" +path = "fuzz_targets/deterministic.rs" +required-features = ["dylib", "cranelift"] \ No newline at end of file diff --git a/fuzz/fuzz_targets/deterministic.rs b/fuzz/fuzz_targets/deterministic.rs new file mode 100644 index 00000000000..b06615ef53f --- /dev/null +++ b/fuzz/fuzz_targets/deterministic.rs @@ -0,0 +1,77 @@ +#![no_main] + +use libfuzzer_sys::{arbitrary, arbitrary::Arbitrary, fuzz_target}; +use wasm_smith::{Config, ConfiguredModule}; +use wasmer::{CompilerConfig, Engine, Module, Store}; +use wasmer_compiler_cranelift::Cranelift; +use wasmer_compiler_llvm::LLVM; +use wasmer_compiler_singlepass::Singlepass; +use wasmer_engine_jit::JIT; +use wasmer_engine_native::Native; + +#[derive(Arbitrary, Debug, Default, Copy, Clone)] +struct NoImportsConfig; +impl Config for NoImportsConfig { + fn max_imports(&self) -> usize { + 0 + } + fn max_memory_pages(&self) -> u32 { + // https://github.com/wasmerio/wasmer/issues/2187 + 65535 + } + fn allow_start_export(&self) -> bool { + false + } +} + +fn compile_and_compare(name: &str, engine: impl Engine, wasm: &[u8]) { + let store = Store::new(&engine); + + // compile for first time + let module = Module::new(&store, wasm).unwrap(); + let first = module.serialize().unwrap(); + + // compile for second time + let module = Module::new(&store, wasm).unwrap(); + let second = module.serialize().unwrap(); + + if first != second { + panic!("non-deterministic compilation from {}", name); + } +} + +fuzz_target!(|module: ConfiguredModule| { + let wasm_bytes = module.to_bytes(); + + let mut compiler = Cranelift::default(); + compiler.canonicalize_nans(true); + compiler.enable_verifier(); + compile_and_compare( + "jit-cranelift", + JIT::new(compiler.clone()).engine(), + &wasm_bytes, + ); + compile_and_compare( + "native-cranelift", + Native::new(compiler).engine(), + &wasm_bytes, + ); + + // let mut compiler = LLVM::default(); + // compiler.canonicalize_nans(true); + // compiler.enable_verifier(); + // compile_and_compare("jit-llvm", JIT::new(compiler.clone()).engine(), &wasm_bytes); + // compile_and_compare("native-llvm", Native::new(compiler).engine(), &wasm_bytes); + + // let compiler = Singlepass::default(); + // compile_and_compare( + // "jit-singlepass", + // JIT::new(compiler.clone()).engine(), + // &wasm_bytes, + // ); + // compile_and_compare( + // "native-singlepass", + // Native::new(compiler).engine(), + // &wasm_bytes, + // ); +}); From dd8fd6db9fb67df732a5256eb4a3edd1c7e33fc6 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Mon, 22 Nov 2021 17:45:50 +0000 Subject: [PATCH 07/12] Fix fuzz target --- fuzz/Cargo.toml | 2 +- fuzz/fuzz_targets/deterministic.rs | 50 ++++++++++++++++-------------- 2 files changed, 28 insertions(+), 24 deletions(-) diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 4b52f18a3ba..e28680e2a09 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -61,4 +61,4 @@ required-features = ["dylib", "cranelift"] [[bin]] name = "deterministic" path = "fuzz_targets/deterministic.rs" -required-features = ["dylib", "cranelift"] \ No newline at end of file +required-features = ["universal", "dylib", "cranelift", "llvm", "singlepass"] \ No newline at end of file diff --git a/fuzz/fuzz_targets/deterministic.rs b/fuzz/fuzz_targets/deterministic.rs index b06615ef53f..b613d5449d5 100644 --- a/fuzz/fuzz_targets/deterministic.rs +++ b/fuzz/fuzz_targets/deterministic.rs @@ -6,8 +6,8 @@ use wasmer::{CompilerConfig, Engine, Module, Store}; use wasmer_compiler_cranelift::Cranelift; use wasmer_compiler_llvm::LLVM; use wasmer_compiler_singlepass::Singlepass; -use wasmer_engine_jit::JIT; -use wasmer_engine_native::Native; +use wasmer_engine_dylib::Dylib; +use wasmer_engine_universal::Universal; #[derive(Arbitrary, Debug, Default, Copy, Clone)] struct NoImportsConfig; @@ -47,31 +47,35 @@ fuzz_target!(|module: ConfiguredModule| { compiler.canonicalize_nans(true); compiler.enable_verifier(); compile_and_compare( - "jit-cranelift", - JIT::new(compiler.clone()).engine(), + "universal-cranelift", + Universal::new(compiler.clone()).engine(), &wasm_bytes, ); + //compile_and_compare( + // "dylib-cranelift", + // Dylib::new(compiler).engine(), + // &wasm_bytes, + //); + + let mut compiler = LLVM::default(); + compiler.canonicalize_nans(true); + compiler.enable_verifier(); compile_and_compare( - "native-cranelift", - Native::new(compiler).engine(), + "universal-llvm", + Universal::new(compiler.clone()).engine(), &wasm_bytes, ); + //compile_and_compare("dylib-llvm", Dylib::new(compiler).engine(), &wasm_bytes); - // let mut compiler = LLVM::default(); - // compiler.canonicalize_nans(true); - // compiler.enable_verifier(); - // compile_and_compare("jit-llvm", JIT::new(compiler.clone()).engine(), &wasm_bytes); - // compile_and_compare("native-llvm", Native::new(compiler).engine(), &wasm_bytes); - - // let compiler = Singlepass::default(); - // compile_and_compare( - // "jit-singlepass", - // JIT::new(compiler.clone()).engine(), - // &wasm_bytes, - // ); - // compile_and_compare( - // "native-singlepass", - // Native::new(compiler).engine(), - // &wasm_bytes, - // ); + let compiler = Singlepass::default(); + compile_and_compare( + "universal-singlepass", + Universal::new(compiler.clone()).engine(), + &wasm_bytes, + ); + //compile_and_compare( + // "dylib-singlepass", + // Dylib::new(compiler).engine(), + // &wasm_bytes, + //); }); From c7e04f142066634ab75879fbb213fd44fb15ef6a Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Mon, 22 Nov 2021 17:50:24 +0000 Subject: [PATCH 08/12] Fix build with rkyv disabled --- lib/types/src/module.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/types/src/module.rs b/lib/types/src/module.rs index 135e17ce1e6..b9fb247c8bd 100644 --- a/lib/types/src/module.rs +++ b/lib/types/src/module.rs @@ -15,15 +15,17 @@ use crate::{ }; use indexmap::IndexMap; use loupe::MemoryUsage; -use rkyv::ser::ScratchSpace; #[cfg(feature = "enable-rkyv")] use rkyv::{ - de::SharedDeserializeRegistry, ser::Serializer, ser::SharedSerializeRegistry, Archive, - Archived, Deserialize as RkyvDeserialize, Fallible, Serialize as RkyvSerialize, + de::SharedDeserializeRegistry, ser::ScratchSpace, ser::Serializer, + ser::SharedSerializeRegistry, Archive, Archived, Deserialize as RkyvDeserialize, Fallible, + Serialize as RkyvSerialize, }; #[cfg(feature = "enable-serde")] use serde::{Deserialize, Serialize}; -use std::collections::{BTreeMap, HashMap}; +#[cfg(feature = "enable-rkyv")] +use std::collections::BTreeMap; +use std::collections::HashMap; use std::fmt; use std::iter::ExactSizeIterator; use std::sync::atomic::{AtomicUsize, Ordering::SeqCst}; From 60db3b8931e5adaac215837762936776f0766828 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Mon, 22 Nov 2021 17:52:25 +0000 Subject: [PATCH 09/12] Remove unneeded BTreeMap from ArchivableIndexMap --- lib/types/src/archives.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/types/src/archives.rs b/lib/types/src/archives.rs index 48e30272e37..8ff259716b6 100644 --- a/lib/types/src/archives.rs +++ b/lib/types/src/archives.rs @@ -3,12 +3,11 @@ use core::hash::Hash; use indexmap::IndexMap; use rkyv::{Archive, Deserialize, Serialize}; #[cfg(feature = "std")] -use std::{collections::BTreeMap, hash::Hash}; +use std::hash::Hash; #[derive(Serialize, Deserialize, Archive)] /// Rkyv Archivable IndexMap pub struct ArchivableIndexMap { - indices: BTreeMap, entries: Vec<(K, V)>, } @@ -17,12 +16,10 @@ impl From> { fn from(it: IndexMap) -> ArchivableIndexMap { let mut r = ArchivableIndexMap { - indices: BTreeMap::new(), entries: Vec::new(), }; let mut i: u64 = 0; for (k, v) in it.into_iter() { - r.indices.insert(k.clone(), i); r.entries.push((k, v)); i += 1; } From 54b4495b3f9a9e3dd60cc1bf00d20d07eb777bab Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Mon, 22 Nov 2021 19:33:02 +0000 Subject: [PATCH 10/12] Fix build --- lib/api/src/js/externals/function.rs | 2 +- lib/api/src/sys/externals/function.rs | 2 +- lib/types/src/archives.rs | 2 -- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/api/src/js/externals/function.rs b/lib/api/src/js/externals/function.rs index d9ccea88be4..c9bf5853575 100644 --- a/lib/api/src/js/externals/function.rs +++ b/lib/api/src/js/externals/function.rs @@ -1288,7 +1288,7 @@ mod inner { #[test] fn test_into_array() { - assert_eq!(().into_array(), []); + assert_eq!(().into_array(), [0i128; 0]); assert_eq!((1).into_array(), [1]); assert_eq!((1i32, 2i64).into_array(), [1, 2]); assert_eq!( diff --git a/lib/api/src/sys/externals/function.rs b/lib/api/src/sys/externals/function.rs index 773faacd4ef..3146c823854 100644 --- a/lib/api/src/sys/externals/function.rs +++ b/lib/api/src/sys/externals/function.rs @@ -1472,7 +1472,7 @@ mod inner { #[test] fn test_into_array() { - assert_eq!(().into_array(), []); + assert_eq!(().into_array(), [0i128; 0]); assert_eq!((1).into_array(), [1]); assert_eq!((1i32, 2i64).into_array(), [1, 2]); assert_eq!( diff --git a/lib/types/src/archives.rs b/lib/types/src/archives.rs index 8ff259716b6..f620520b5ea 100644 --- a/lib/types/src/archives.rs +++ b/lib/types/src/archives.rs @@ -18,10 +18,8 @@ impl From> let mut r = ArchivableIndexMap { entries: Vec::new(), }; - let mut i: u64 = 0; for (k, v) in it.into_iter() { r.entries.push((k, v)); - i += 1; } r } From 4944ddf75464a534080b1c2f6a7e5bce018def14 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Mon, 22 Nov 2021 21:46:39 +0000 Subject: [PATCH 11/12] Fix cranelift build without unwind --- lib/compiler-cranelift/src/compiler.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/compiler-cranelift/src/compiler.rs b/lib/compiler-cranelift/src/compiler.rs index 6a887752f16..3fbc998095d 100644 --- a/lib/compiler-cranelift/src/compiler.rs +++ b/lib/compiler-cranelift/src/compiler.rs @@ -207,7 +207,13 @@ impl Compiler for CraneliftCompiler { (None, None) } } + #[cfg(feature = "unwind")] other => (other.maybe_into_to_windows_unwind(), None), + + // This is a bit hacky, but necessary since gimli is not + // available when the "unwind" feature is disabled. + #[cfg(not(feature = "unwind"))] + other => (other.maybe_into_to_windows_unwind(), None::<()>), }; let range = reader.range(); From 4732b171126ec1b0e397208145cd0c888a2166bf Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Mon, 22 Nov 2021 23:12:27 +0000 Subject: [PATCH 12/12] Fix ignore for windows test --- tests/ignores.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ignores.txt b/tests/ignores.txt index 4661fe0493b..fd9bebcb7f1 100644 --- a/tests/ignores.txt +++ b/tests/ignores.txt @@ -70,7 +70,7 @@ cranelift spec::simd::simd_i8x16_arith2 cranelift spec::simd::simd_int_to_int_extend # Windows doesn't overcommit and fails to allocate 4GB of memory -windows wast::wasmer::max_size_of_memory +windows wasmer::max_size_of_memory # Frontends