From a02ff59a0f32074f92bd25e2b7775994c6e771bb Mon Sep 17 00:00:00 2001 From: Manos Pitsidianakis Date: Fri, 8 Jul 2022 14:46:54 +0300 Subject: [PATCH 1/3] cli: re-introduce create-exe functionality --- Cargo.lock | 9 +- lib/api/src/js/module_info_polyfill.rs | 10 +- lib/cli/Cargo.toml | 1 + lib/cli/src/cli.rs | 9 + lib/cli/src/commands.rs | 4 + lib/cli/src/commands/create_exe.rs | 293 ++++++++++++++++++ lib/cli/src/commands/wasmer_create_exe_main.c | 176 +++++++++++ lib/compiler/src/engine/resolver.rs | 10 +- lib/compiler/src/translator/environ.rs | 3 +- lib/object/src/error.rs | 3 + lib/object/src/lib.rs | 3 +- lib/object/src/module.rs | 68 ++++ lib/types/src/lib.rs | 2 +- lib/types/src/module.rs | 76 +++-- lib/types/src/serialize.rs | 15 + 15 files changed, 642 insertions(+), 40 deletions(-) create mode 100644 lib/cli/src/commands/create_exe.rs create mode 100644 lib/cli/src/commands/wasmer_create_exe_main.c diff --git a/Cargo.lock b/Cargo.lock index eb9f28714e9..15c08559d80 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1911,9 +1911,9 @@ dependencies = [ [[package]] name = "rkyv" -version = "0.7.38" +version = "0.7.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "517a3034eb2b1499714e9d1e49b2367ad567e07639b69776d35e259d9c27cca6" +checksum = "cec2b3485b07d96ddfd3134767b8a447b45ea4eb91448d0a35180ec0ffd5ed15" dependencies = [ "bytecheck", "hashbrown 0.12.1", @@ -1926,9 +1926,9 @@ dependencies = [ [[package]] name = "rkyv_derive" -version = "0.7.38" +version = "0.7.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "505c209ee04111a006431abf39696e640838364d67a107c559ababaf6fd8c9dd" +checksum = "6eaedadc88b53e36dd32d940ed21ae4d850d5916f2581526921f553a72ac34c4" dependencies = [ "proc-macro2", "quote", @@ -2927,6 +2927,7 @@ dependencies = [ "wasmer-compiler-llvm", "wasmer-compiler-singlepass", "wasmer-emscripten", + "wasmer-object", "wasmer-types", "wasmer-vfs", "wasmer-vm", diff --git a/lib/api/src/js/module_info_polyfill.rs b/lib/api/src/js/module_info_polyfill.rs index ce8cdab2732..80f7cccfc21 100644 --- a/lib/api/src/js/module_info_polyfill.rs +++ b/lib/api/src/js/module_info_polyfill.rs @@ -40,11 +40,11 @@ impl ModuleInfoPolyfill { field: &str, ) -> WasmResult<()> { self.info.imports.insert( - ( - String::from(module), - String::from(field), - self.info.imports.len() as u32, - ), + wasmer_types::ImportKey { + module: String::from(module), + field: String::from(field), + import_idx: self.info.imports.len() as u32, + }, import, ); Ok(()) diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index e32976a2460..d43d15c523d 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -38,6 +38,7 @@ wasmer-wast = { version = "=2.3.0", path = "../../tests/lib/wast", optional = tr wasmer-cache = { version = "=2.3.0", path = "../cache", optional = true } wasmer-types = { version = "=2.3.0", path = "../types" } wasmer-vfs = { version = "=2.3.0", path = "../vfs", default-features = false, features = ["host-fs"] } +wasmer-object = { version = "=2.3.0", path = "../object" } atty = "0.2" colored = "2.0" anyhow = "1.0" diff --git a/lib/cli/src/cli.rs b/lib/cli/src/cli.rs index 89b8ee971bc..ad938a9ae8d 100644 --- a/lib/cli/src/cli.rs +++ b/lib/cli/src/cli.rs @@ -4,6 +4,8 @@ use crate::commands::Binfmt; #[cfg(feature = "compiler")] use crate::commands::Compile; +#[cfg(feature = "compiler")] +use crate::commands::CreateExe; #[cfg(feature = "wast")] use crate::commands::Wast; use crate::commands::{Cache, Config, Inspect, Run, SelfUpdate, Validate}; @@ -44,6 +46,11 @@ enum WasmerCLIOptions { #[structopt(name = "compile")] Compile(Compile), + /// Compile a WebAssembly binary into a native executable + #[cfg(feature = "compiler")] + #[structopt(name = "create-exe")] + CreateExe(CreateExe), + /// Get various configuration information needed /// to compile programs which use Wasmer #[structopt(name = "config")] @@ -77,6 +84,8 @@ impl WasmerCLIOptions { Self::Validate(validate) => validate.execute(), #[cfg(feature = "compiler")] Self::Compile(compile) => compile.execute(), + #[cfg(feature = "compiler")] + Self::CreateExe(create_exe) => create_exe.execute(), Self::Config(config) => config.execute(), Self::Inspect(inspect) => inspect.execute(), #[cfg(feature = "wast")] diff --git a/lib/cli/src/commands.rs b/lib/cli/src/commands.rs index 2757b3e02d3..3313370f2b2 100644 --- a/lib/cli/src/commands.rs +++ b/lib/cli/src/commands.rs @@ -5,6 +5,8 @@ mod cache; #[cfg(feature = "compiler")] mod compile; mod config; +#[cfg(feature = "compiler")] +mod create_exe; mod inspect; mod run; mod self_update; @@ -16,6 +18,8 @@ mod wast; pub use binfmt::*; #[cfg(feature = "compiler")] pub use compile::*; +#[cfg(feature = "compiler")] +pub use create_exe::*; #[cfg(feature = "wast")] pub use wast::*; pub use {cache::*, config::*, inspect::*, run::*, self_update::*, validate::*}; diff --git a/lib/cli/src/commands/create_exe.rs b/lib/cli/src/commands/create_exe.rs new file mode 100644 index 00000000000..d240fa09a25 --- /dev/null +++ b/lib/cli/src/commands/create_exe.rs @@ -0,0 +1,293 @@ +//! Create a standalone native executable for a given Wasm file. + +use crate::store::CompilerOptions; +use anyhow::{Context, Result}; +use std::env; +use std::fs; +use std::fs::File; +use std::io::prelude::*; +use std::io::BufWriter; +use std::path::{Path, PathBuf}; +use std::process::Command; +use structopt::StructOpt; +use wasmer::*; +use wasmer_object::{emit_serialized, get_object_for_target}; + +const WASMER_MAIN_C_SOURCE: &[u8] = include_bytes!("wasmer_create_exe_main.c"); + +#[derive(Debug, StructOpt)] +/// The options for the `wasmer create-exe` subcommand +pub struct CreateExe { + /// Input file + #[structopt(name = "FILE", parse(from_os_str))] + path: PathBuf, + + /// Output file + #[structopt(name = "OUTPUT PATH", short = "o", parse(from_os_str))] + output: PathBuf, + + /// Compilation Target triple + #[structopt(long = "target")] + target_triple: Option, + + #[structopt(flatten)] + compiler: CompilerOptions, + + #[structopt(short = "m", multiple = true, number_of_values = 1)] + cpu_features: Vec, + + /// Additional libraries to link against. + /// This is useful for fixing linker errors that may occur on some systems. + #[structopt(short = "l", multiple = true, number_of_values = 1)] + libraries: Vec, +} + +impl CreateExe { + /// Runs logic for the `compile` subcommand + pub fn execute(&self) -> Result<()> { + let target = self + .target_triple + .as_ref() + .map(|target_triple| { + let mut features = self + .cpu_features + .clone() + .into_iter() + .fold(CpuFeature::set(), |a, b| a | b); + // Cranelift requires SSE2, so we have this "hack" for now to facilitate + // usage + features |= CpuFeature::SSE2; + Target::new(target_triple.clone(), features) + }) + .unwrap_or_default(); + let (store, compiler_type) = self.compiler.get_store_for_target(target.clone())?; + + println!("Compiler: {}", compiler_type.to_string()); + println!("Target: {}", target.triple()); + + let working_dir = tempfile::tempdir()?; + let starting_cd = env::current_dir()?; + let output_path = starting_cd.join(&self.output); + env::set_current_dir(&working_dir)?; + + #[cfg(not(windows))] + let wasm_object_path = PathBuf::from("wasm.o"); + #[cfg(windows)] + let wasm_object_path = PathBuf::from("wasm.obj"); + + let wasm_module_path = starting_cd.join(&self.path); + + let module = + Module::from_file(&store, &wasm_module_path).context("failed to compile Wasm")?; + let bytes = module.serialize()?; + let mut obj = get_object_for_target(target.triple())?; + emit_serialized(&mut obj, &bytes, target.triple())?; + let mut writer = BufWriter::new(File::create(&wasm_object_path)?); + obj.write_stream(&mut writer) + .map_err(|err| anyhow::anyhow!(err.to_string()))?; + writer.flush()?; + drop(writer); + + self.compile_c(wasm_object_path, output_path)?; + + eprintln!( + "✔ Native executable compiled successfully to `{}`.", + self.output.display(), + ); + + Ok(()) + } + + fn compile_c(&self, wasm_object_path: PathBuf, output_path: PathBuf) -> anyhow::Result<()> { + // write C src to disk + let c_src_path = Path::new("wasmer_main.c"); + #[cfg(not(windows))] + let c_src_obj = PathBuf::from("wasmer_main.o"); + #[cfg(windows)] + let c_src_obj = PathBuf::from("wasmer_main.obj"); + + { + let mut c_src_file = fs::OpenOptions::new() + .create_new(true) + .write(true) + .open(&c_src_path) + .context("Failed to open C source code file")?; + c_src_file.write_all(WASMER_MAIN_C_SOURCE)?; + } + run_c_compile(c_src_path, &c_src_obj, self.target_triple.clone()) + .context("Failed to compile C source code")?; + LinkCode { + object_paths: vec![c_src_obj, wasm_object_path], + output_path, + additional_libraries: self.libraries.clone(), + target: self.target_triple.clone(), + ..Default::default() + } + .run() + .context("Failed to link objects together")?; + + Ok(()) + } +} + +fn get_wasmer_dir() -> anyhow::Result { + Ok(PathBuf::from( + env::var("WASMER_DIR") + .or_else(|e| { + option_env!("WASMER_INSTALL_PREFIX") + .map(str::to_string) + .ok_or(e) + }) + .context("Trying to read env var `WASMER_DIR`")?, + )) +} + +fn get_wasmer_include_directory() -> anyhow::Result { + let mut path = get_wasmer_dir()?; + path.push("include"); + Ok(path) +} + +/// path to the static libwasmer +fn get_libwasmer_path() -> anyhow::Result { + let mut path = get_wasmer_dir()?; + path.push("lib"); + + // TODO: prefer headless Wasmer if/when it's a separate library. + #[cfg(not(windows))] + path.push("libwasmer.a"); + #[cfg(windows)] + path.push("wasmer.lib"); + + Ok(path) +} + +/// Compile the C code. +fn run_c_compile( + path_to_c_src: &Path, + output_name: &Path, + target: Option, +) -> anyhow::Result<()> { + #[cfg(not(windows))] + let c_compiler = "cc"; + // We must use a C++ compiler on Windows because wasm.h uses `static_assert` + // which isn't available in `clang` on Windows. + #[cfg(windows)] + let c_compiler = "clang++"; + + let mut command = Command::new(c_compiler); + let command = command + .arg("-O2") + .arg("-c") + .arg(path_to_c_src) + .arg("-I") + .arg(get_wasmer_include_directory()?); + + let command = if let Some(target) = target { + command.arg("-target").arg(format!("{}", target)) + } else { + command + }; + + let output = command.arg("-o").arg(output_name).output()?; + + if !output.status.success() { + bail!( + "C code compile failed with: stdout: {}\n\nstderr: {}", + std::str::from_utf8(&output.stdout) + .expect("stdout is not utf8! need to handle arbitrary bytes"), + std::str::from_utf8(&output.stderr) + .expect("stderr is not utf8! need to handle arbitrary bytes") + ); + } + Ok(()) +} + +/// Data used to run a linking command for generated artifacts. +#[derive(Debug)] +struct LinkCode { + /// Path to the linker used to run the linking command. + linker_path: PathBuf, + /// String used as an optimization flag. + optimization_flag: String, + /// Paths of objects to link. + object_paths: Vec, + /// Additional libraries to link against. + additional_libraries: Vec, + /// Path to the output target. + output_path: PathBuf, + /// Path to the dir containing the static libwasmer library. + libwasmer_path: PathBuf, + /// The target to link the executable for. + target: Option, +} + +impl Default for LinkCode { + fn default() -> Self { + #[cfg(not(windows))] + let linker = "cc"; + #[cfg(windows)] + let linker = "clang"; + Self { + linker_path: PathBuf::from(linker), + optimization_flag: String::from("-O2"), + object_paths: vec![], + additional_libraries: vec![], + output_path: PathBuf::from("a.out"), + libwasmer_path: get_libwasmer_path().unwrap(), + target: None, + } + } +} + +impl LinkCode { + fn run(&self) -> anyhow::Result<()> { + let mut command = Command::new(&self.linker_path); + let command = command + .arg(&self.optimization_flag) + .args( + self.object_paths + .iter() + .map(|path| path.canonicalize().unwrap()), + ) + .arg( + &self + .libwasmer_path + .canonicalize() + .context("Failed to find libwasmer")?, + ); + let command = if let Some(target) = &self.target { + command.arg("-target").arg(format!("{}", target)) + } else { + command + }; + // Add libraries required per platform. + // We need userenv, sockets (Ws2_32), advapi32 for some system calls and bcrypt for random numbers. + #[cfg(windows)] + let command = command + .arg("-luserenv") + .arg("-lWs2_32") + .arg("-ladvapi32") + .arg("-lbcrypt"); + // On unix we need dlopen-related symbols, libmath for a few things, and pthreads. + #[cfg(not(windows))] + let command = command.arg("-ldl").arg("-lm").arg("-pthread"); + let link_aganist_extra_libs = self + .additional_libraries + .iter() + .map(|lib| format!("-l{}", lib)); + let command = command.args(link_aganist_extra_libs); + let output = command.arg("-o").arg(&self.output_path).output()?; + + if !output.status.success() { + bail!( + "linking failed with: stdout: {}\n\nstderr: {}", + std::str::from_utf8(&output.stdout) + .expect("stdout is not utf8! need to handle arbitrary bytes"), + std::str::from_utf8(&output.stderr) + .expect("stderr is not utf8! need to handle arbitrary bytes") + ); + } + Ok(()) + } +} diff --git a/lib/cli/src/commands/wasmer_create_exe_main.c b/lib/cli/src/commands/wasmer_create_exe_main.c new file mode 100644 index 00000000000..1fa6a834fdb --- /dev/null +++ b/lib/cli/src/commands/wasmer_create_exe_main.c @@ -0,0 +1,176 @@ + +#include "wasmer.h" +//#include "my_wasm.h" + +#include +#include +#include + +#define own + +// TODO: make this define templated so that the Rust code can toggle it on/off +#define WASI + +extern size_t WASMER_MODULE_LENGTH asm("WASMER_MODULE_LENGTH"); +extern char WASMER_MODULE_DATA asm("WASMER_MODULE_DATA"); + +static void print_wasmer_error() { + int error_len = wasmer_last_error_length(); + printf("Error len: `%d`\n", error_len); + char *error_str = (char *)malloc(error_len); + wasmer_last_error_message(error_str, error_len); + printf("%s\n", error_str); + free(error_str); +} + +#ifdef WASI +static void pass_mapdir_arg(wasi_config_t *wasi_config, char *mapdir) { + int colon_location = strchr(mapdir, ':') - mapdir; + if (colon_location == 0) { + // error malformed argument + fprintf(stderr, "Expected mapdir argument of the form alias:directory\n"); + exit(-1); + } + + char *alias = (char *)malloc(colon_location + 1); + memcpy(alias, mapdir, colon_location); + alias[colon_location] = '\0'; + + int dir_len = strlen(mapdir) - colon_location; + char *dir = (char *)malloc(dir_len + 1); + memcpy(dir, &mapdir[colon_location + 1], dir_len); + dir[dir_len] = '\0'; + + wasi_config_mapdir(wasi_config, alias, dir); + free(alias); + free(dir); +} + +// We try to parse out `--dir` and `--mapdir` ahead of time and process those +// specially. All other arguments are passed to the guest program. +static void handle_arguments(wasi_config_t *wasi_config, int argc, + char *argv[]) { + for (int i = 1; i < argc; ++i) { + // We probably want special args like `--dir` and `--mapdir` to not be + // passed directly + if (strcmp(argv[i], "--dir") == 0) { + // next arg is a preopen directory + if ((i + 1) < argc) { + i++; + wasi_config_preopen_dir(wasi_config, argv[i]); + } else { + fprintf(stderr, "--dir expects a following argument specifying which " + "directory to preopen\n"); + exit(-1); + } + } else if (strcmp(argv[i], "--mapdir") == 0) { + // next arg is a mapdir + if ((i + 1) < argc) { + i++; + pass_mapdir_arg(wasi_config, argv[i]); + } else { + fprintf(stderr, + "--mapdir expects a following argument specifying which " + "directory to preopen in the form alias:directory\n"); + exit(-1); + } + } else if (strncmp(argv[i], "--dir=", strlen("--dir=")) == 0) { + // this arg is a preopen dir + char *dir = argv[i] + strlen("--dir="); + wasi_config_preopen_dir(wasi_config, dir); + } else if (strncmp(argv[i], "--mapdir=", strlen("--mapdir=")) == 0) { + // this arg is a mapdir + char *mapdir = argv[i] + strlen("--mapdir="); + pass_mapdir_arg(wasi_config, mapdir); + } else { + // guest argument + wasi_config_arg(wasi_config, argv[i]); + } + } +} +#endif + +int main(int argc, char *argv[]) { + wasm_config_t *config = wasm_config_new(); + wasm_engine_t *engine = wasm_engine_new_with_config(config); + wasm_store_t *store = wasm_store_new(engine); + + wasm_byte_vec_t module_byte_vec = { + .size = WASMER_MODULE_LENGTH, + .data = (const char*)&WASMER_MODULE_DATA, + }; + wasm_module_t *module = wasm_module_deserialize(store, &module_byte_vec); + + if (!module) { + fprintf(stderr, "Failed to create module\n"); + print_wasmer_error(); + return -1; + } + + // We have now finished the memory buffer book keeping and we have a valid + // Module. + +#ifdef WASI + wasi_config_t *wasi_config = wasi_config_new(argv[0]); + handle_arguments(wasi_config, argc, argv); + + wasi_env_t *wasi_env = wasi_env_new(store, wasi_config); + if (!wasi_env) { + fprintf(stderr, "Error building WASI env!\n"); + print_wasmer_error(); + return 1; + } +#endif + + wasm_importtype_vec_t import_types; + wasm_module_imports(module, &import_types); + + wasm_extern_vec_t imports; + wasm_extern_vec_new_uninitialized(&imports, import_types.size); + wasm_importtype_vec_delete(&import_types); + +#ifdef WASI + bool get_imports_result = wasi_get_imports(store, wasi_env, module, &imports); + + if (!get_imports_result) { + fprintf(stderr, "Error getting WASI imports!\n"); + print_wasmer_error(); + + return 1; + } +#endif + + wasm_instance_t *instance = wasm_instance_new(store, module, &imports, NULL); + + if (!instance) { + fprintf(stderr, "Failed to create instance\n"); + print_wasmer_error(); + return -1; + } + +#ifdef WASI + own wasm_func_t *start_function = wasi_get_start_function(instance); + if (!start_function) { + fprintf(stderr, "`_start` function not found\n"); + print_wasmer_error(); + return -1; + } + + wasm_val_vec_t args = WASM_EMPTY_VEC; + wasm_val_vec_t results = WASM_EMPTY_VEC; + own wasm_trap_t *trap = wasm_func_call(start_function, &args, &results); + if (trap) { + fprintf(stderr, "Trap is not NULL: TODO:\n"); + return -1; + } + wasi_env_delete(wasi_env); +#endif + + // TODO: handle non-WASI start (maybe with invoke?) + + wasm_instance_delete(instance); + wasm_module_delete(module); + wasm_store_delete(store); + wasm_engine_delete(engine); + return 0; +} diff --git a/lib/compiler/src/engine/resolver.rs b/lib/compiler/src/engine/resolver.rs index a97044adb8b..207380bb819 100644 --- a/lib/compiler/src/engine/resolver.rs +++ b/lib/compiler/src/engine/resolver.rs @@ -64,7 +64,15 @@ pub fn resolve_imports( let mut memory_imports = PrimaryMap::with_capacity(module.num_imported_memories); let mut global_imports = PrimaryMap::with_capacity(module.num_imported_globals); - for ((module_name, field, import_idx), import_index) in module.imports.iter() { + for ( + wasmer_types::ImportKey { + module: module_name, + field, + import_idx, + }, + import_index, + ) in module.imports.iter() + { let import_extern = get_extern_from_import(module, import_index); let resolved = if let Some(r) = imports.get(*import_idx as usize) { r diff --git a/lib/compiler/src/translator/environ.rs b/lib/compiler/src/translator/environ.rs index 64cfdf02a74..e172d92b063 100644 --- a/lib/compiler/src/translator/environ.rs +++ b/lib/compiler/src/translator/environ.rs @@ -108,7 +108,8 @@ impl<'data> ModuleEnvironment<'data> { String::from(module), String::from(field), self.module.imports.len().try_into().unwrap(), - ), + ) + .into(), import, ); Ok(()) diff --git a/lib/object/src/error.rs b/lib/object/src/error.rs index 78fa3c6654d..910afbe4d9b 100644 --- a/lib/object/src/error.rs +++ b/lib/object/src/error.rs @@ -17,4 +17,7 @@ pub enum ObjectError { /// The object was provided a not-supported architecture #[error("Error when writing the object: {0}")] Write(#[from] ObjectWriteError), + /// The module provided could not be serialized into bytes + #[error("Error when serializing the given module: {0}")] + Serialize(#[from] wasmer_types::SerializeError), } diff --git a/lib/object/src/lib.rs b/lib/object/src/lib.rs index 97771837bf8..db674a5e91b 100644 --- a/lib/object/src/lib.rs +++ b/lib/object/src/lib.rs @@ -23,4 +23,5 @@ mod error; mod module; pub use crate::error::ObjectError; -pub use crate::module::{emit_compilation, emit_data, get_object_for_target}; +pub use crate::module::{emit_compilation, emit_data, emit_serialized, get_object_for_target}; +pub use object::{self, Object}; diff --git a/lib/object/src/module.rs b/lib/object/src/module.rs index d9b4d94c135..20e65b168bd 100644 --- a/lib/object/src/module.rs +++ b/lib/object/src/module.rs @@ -384,3 +384,71 @@ pub fn emit_compilation( Ok(()) } + +/// Emit the compilation result into an existing object. +/// +/// # Usage +/// +/// ```rust +/// # use wasmer_compiler::SymbolRegistry; +/// # use wasmer_types::{Compilation, Triple}; +/// # use wasmer_object::ObjectError; +/// use wasmer_object::{get_object_for_target, emit_compilation}; +/// +/// # fn emit_compilation_into_object( +/// # triple: &Triple, +/// # compilation: Compilation, +/// # symbol_registry: impl SymbolRegistry, +/// # ) -> Result<(), ObjectError> { +/// let mut object = get_object_for_target(&triple)?; +/// emit_compilation(&mut object, compilation, &symbol_registry, &triple)?; +/// # Ok(()) +/// # } +/// ``` +pub fn emit_serialized( + obj: &mut Object, + sercomp: &[u8], + triple: &Triple, +) -> Result<(), ObjectError> { + obj.set_mangling(object::write::Mangling::None); + //let module_name = module.compile_info.module.name.clone(); + let len_name = "WASMER_MODULE_LENGTH"; + let data_name = "WASMER_MODULE_DATA"; + //let metadata_name = "WASMER_MODULE_METADATA"; + + let align = match triple.architecture { + Architecture::X86_64 => 1, + // In Arm64 is recommended a 4-byte alignment + Architecture::Aarch64(_) => 4, + _ => 1, + }; + + let len = sercomp.len(); + let section_id = obj.section_id(StandardSection::Data); + let symbol_id = obj.add_symbol(ObjSymbol { + name: len_name.as_bytes().to_vec(), + value: 0, + size: len.to_le_bytes().len() as _, + kind: SymbolKind::Data, + scope: SymbolScope::Dynamic, + weak: false, + section: SymbolSection::Section(section_id), + flags: SymbolFlags::None, + }); + obj.add_symbol_data(symbol_id, section_id, &len.to_le_bytes(), align); + + let section_id = obj.section_id(StandardSection::Data); + let symbol_id = obj.add_symbol(ObjSymbol { + name: data_name.as_bytes().to_vec(), + value: 0, + size: sercomp.len() as _, + kind: SymbolKind::Data, + scope: SymbolScope::Dynamic, + weak: false, + section: SymbolSection::Section(section_id), + flags: SymbolFlags::None, + }); + obj.add_symbol_data(symbol_id, section_id, sercomp, align); + + Ok(()) +} diff --git a/lib/types/src/lib.rs b/lib/types/src/lib.rs index 707c5d05027..74b12e3e26d 100644 --- a/lib/types/src/lib.rs +++ b/lib/types/src/lib.rs @@ -92,7 +92,7 @@ pub use crate::initializers::{ DataInitializer, DataInitializerLocation, OwnedDataInitializer, TableInitializer, }; pub use crate::memory::{Memory32, Memory64, MemorySize}; -pub use crate::module::{ExportsIterator, ImportsIterator, ModuleInfo}; +pub use crate::module::{ExportsIterator, ImportKey, ImportsIterator, ModuleInfo}; pub use crate::units::{ Bytes, PageCountOutOfRange, Pages, WASM_MAX_PAGES, WASM_MIN_PAGES, WASM_PAGE_SIZE, }; diff --git a/lib/types/src/module.rs b/lib/types/src/module.rs index afdbc9ec6d8..e109eb1a194 100644 --- a/lib/types/src/module.rs +++ b/lib/types/src/module.rs @@ -45,6 +45,28 @@ impl Default for ModuleId { } } +/// Hash key of an import +#[derive(Debug, Hash, Eq, PartialEq, Clone, Default, RkyvSerialize, RkyvDeserialize, Archive)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub struct ImportKey { + /// Module name + pub module: String, + /// Field name + pub field: String, + /// Import index + pub import_idx: u32, +} + +impl From<(String, String, u32)> for ImportKey { + fn from((module, field, import_idx): (String, String, u32)) -> Self { + Self { + module, + field, + import_idx, + } + } +} + /// A translated WebAssembly module, excluding the function bodies and /// memory initializers. #[derive(Debug, Clone, Default)] @@ -67,7 +89,7 @@ pub struct ModuleInfo { /// Keeping the `index_of_the_import` is important, as there can be /// two same references to the same import, and we don't want to confuse /// them. - pub imports: IndexMap<(String, String, u32), ImportIndex>, + pub imports: IndexMap, /// Exported entities. pub exports: IndexMap, @@ -128,7 +150,7 @@ pub struct ModuleInfo { #[derive(RkyvSerialize, RkyvDeserialize, Archive)] pub struct ArchivableModuleInfo { name: Option, - imports: IndexMap<(String, String, u32), ImportIndex>, + imports: IndexMap, exports: IndexMap, start_function: Option, table_initializers: Vec, @@ -320,31 +342,31 @@ impl ModuleInfo { /// Get the import types of the module pub fn imports(&'_ self) -> ImportsIterator + '_> { - let iter = self - .imports - .iter() - .map(move |((module, field, _), import_index)| { - let extern_type = match import_index { - ImportIndex::Function(i) => { - let signature = self.functions.get(*i).unwrap(); - let func_type = self.signatures.get(*signature).unwrap(); - ExternType::Function(func_type.clone()) - } - ImportIndex::Table(i) => { - let table_type = self.tables.get(*i).unwrap(); - ExternType::Table(*table_type) - } - ImportIndex::Memory(i) => { - let memory_type = self.memories.get(*i).unwrap(); - ExternType::Memory(*memory_type) - } - ImportIndex::Global(i) => { - let global_type = self.globals.get(*i).unwrap(); - ExternType::Global(*global_type) - } - }; - ImportType::new(module, field, extern_type) - }); + let iter = + self.imports + .iter() + .map(move |(ImportKey { module, field, .. }, import_index)| { + let extern_type = match import_index { + ImportIndex::Function(i) => { + let signature = self.functions.get(*i).unwrap(); + let func_type = self.signatures.get(*signature).unwrap(); + ExternType::Function(func_type.clone()) + } + ImportIndex::Table(i) => { + let table_type = self.tables.get(*i).unwrap(); + ExternType::Table(*table_type) + } + ImportIndex::Memory(i) => { + let memory_type = self.memories.get(*i).unwrap(); + ExternType::Memory(*memory_type) + } + ImportIndex::Global(i) => { + let global_type = self.globals.get(*i).unwrap(); + ExternType::Global(*global_type) + } + }; + ImportType::new(module, field, extern_type) + }); ImportsIterator::new(iter, self.imports.len()) } diff --git a/lib/types/src/serialize.rs b/lib/types/src/serialize.rs index e32cd0927de..da7e5b9e4aa 100644 --- a/lib/types/src/serialize.rs +++ b/lib/types/src/serialize.rs @@ -34,6 +34,21 @@ pub struct SerializableCompilation { pub libcall_trampoline_len: u32, } +impl SerializableCompilation { + /// Serialize a Compilation into bytes + /// The bytes will have the following format: + /// RKYV serialization (any length) + POS (8 bytes) + pub fn serialize(&self) -> Result, SerializeError> { + let mut serializer = AllocSerializer::<4096>::default(); + let pos = serializer + .serialize_value(self) + .map_err(to_serialize_error)? as u64; + let mut serialized_data = serializer.into_serializer().into_inner(); + serialized_data.extend_from_slice(&pos.to_le_bytes()); + Ok(serialized_data.to_vec()) + } +} + /// Serializable struct that is able to serialize from and to a `ArtifactInfo`. #[derive(Archive, RkyvDeserialize, RkyvSerialize)] #[allow(missing_docs)] From 8215b65494b50f5db657e874c94719c39635c77f Mon Sep 17 00:00:00 2001 From: Manos Pitsidianakis Date: Tue, 12 Jul 2022 11:58:57 +0300 Subject: [PATCH 2/3] cli: initialize wasienv memory in create-exe --- lib/c-api/src/wasm_c_api/wasi/mod.rs | 10 +++++++- lib/cli/src/commands/create_exe.rs | 19 ++++++++------- lib/cli/src/commands/wasmer_create_exe_main.c | 23 ++++++++++++++++++- 3 files changed, 42 insertions(+), 10 deletions(-) diff --git a/lib/c-api/src/wasm_c_api/wasi/mod.rs b/lib/c-api/src/wasm_c_api/wasi/mod.rs index 54ed50ed46b..4501b83f44f 100644 --- a/lib/c-api/src/wasm_c_api/wasi/mod.rs +++ b/lib/c-api/src/wasm_c_api/wasi/mod.rs @@ -4,7 +4,7 @@ pub use super::unstable::wasi::wasi_get_unordered_imports; use super::{ - externals::{wasm_extern_t, wasm_extern_vec_t, wasm_func_t}, + externals::{wasm_extern_t, wasm_extern_vec_t, wasm_func_t, wasm_memory_t}, instance::wasm_instance_t, module::wasm_module_t, store::{wasm_store_t, StoreRef}, @@ -197,6 +197,14 @@ pub unsafe extern "C" fn wasi_env_new( #[no_mangle] pub extern "C" fn wasi_env_delete(_state: Option>) {} +/// Set the memory on a [`wasi_env_t`]. +#[no_mangle] +pub unsafe extern "C" fn wasi_env_set_memory(env: &mut wasi_env_t, memory: &wasm_memory_t) { + let mut store_mut = env.store.store_mut(); + let wasi_env = env.inner.data_mut(&mut store_mut); + wasi_env.set_memory(memory.extern_.memory()); +} + #[no_mangle] pub unsafe extern "C" fn wasi_env_read_stdout( env: &mut wasi_env_t, diff --git a/lib/cli/src/commands/create_exe.rs b/lib/cli/src/commands/create_exe.rs index d240fa09a25..1d8f468fe63 100644 --- a/lib/cli/src/commands/create_exe.rs +++ b/lib/cli/src/commands/create_exe.rs @@ -242,6 +242,14 @@ impl Default for LinkCode { impl LinkCode { fn run(&self) -> anyhow::Result<()> { + let libwasmer_path = self + .libwasmer_path + .canonicalize() + .context("Failed to find libwasmer")?; + println!( + "Using path `{}` as libwasmer path.", + libwasmer_path.display() + ); let mut command = Command::new(&self.linker_path); let command = command .arg(&self.optimization_flag) @@ -250,12 +258,7 @@ impl LinkCode { .iter() .map(|path| path.canonicalize().unwrap()), ) - .arg( - &self - .libwasmer_path - .canonicalize() - .context("Failed to find libwasmer")?, - ); + .arg(&libwasmer_path); let command = if let Some(target) = &self.target { command.arg("-target").arg(format!("{}", target)) } else { @@ -272,11 +275,11 @@ impl LinkCode { // On unix we need dlopen-related symbols, libmath for a few things, and pthreads. #[cfg(not(windows))] let command = command.arg("-ldl").arg("-lm").arg("-pthread"); - let link_aganist_extra_libs = self + let link_against_extra_libs = self .additional_libraries .iter() .map(|lib| format!("-l{}", lib)); - let command = command.args(link_aganist_extra_libs); + let command = command.args(link_against_extra_libs); let output = command.arg("-o").arg(&self.output_path).output()?; if !output.status.success() { diff --git a/lib/cli/src/commands/wasmer_create_exe_main.c b/lib/cli/src/commands/wasmer_create_exe_main.c index 1fa6a834fdb..a58fb5c73c2 100644 --- a/lib/cli/src/commands/wasmer_create_exe_main.c +++ b/lib/cli/src/commands/wasmer_create_exe_main.c @@ -149,6 +149,24 @@ int main(int argc, char *argv[]) { } #ifdef WASI + // Read the exports. + wasm_extern_vec_t exports; + wasm_instance_exports(instance, &exports); + wasm_memory_t* mem = NULL; + for (size_t i = 0; i < exports.size; i++) { + mem = wasm_extern_as_memory(exports.data[i]); + if (mem) { + break; + } + } + + if (!mem) { + fprintf(stderr, "Failed to create instance: Could not find memory in exports\n"); + print_wasmer_error(); + return -1; + } + wasi_env_set_memory(wasi_env, mem); + own wasm_func_t *start_function = wasi_get_start_function(instance); if (!start_function) { fprintf(stderr, "`_start` function not found\n"); @@ -163,11 +181,14 @@ int main(int argc, char *argv[]) { fprintf(stderr, "Trap is not NULL: TODO:\n"); return -1; } - wasi_env_delete(wasi_env); #endif // TODO: handle non-WASI start (maybe with invoke?) +#ifdef WASI + wasi_env_delete(wasi_env); + wasm_extern_vec_delete(&exports); +#endif wasm_instance_delete(instance); wasm_module_delete(module); wasm_store_delete(store); From 11d8ea255af25bd3d7c688fb3fa4596539588517 Mon Sep 17 00:00:00 2001 From: Manos Pitsidianakis Date: Wed, 27 Jul 2022 20:19:25 +0300 Subject: [PATCH 3/3] compiler: move MetadataHeader to wasmer-types --- .../src/artifact_builders/artifact_builder.rs | 3 +- lib/compiler/src/engine/artifact.rs | 3 +- lib/compiler/src/traits.rs | 73 +------------------ 3 files changed, 5 insertions(+), 74 deletions(-) diff --git a/lib/compiler/src/artifact_builders/artifact_builder.rs b/lib/compiler/src/artifact_builders/artifact_builder.rs index 7a3c9132a17..321434b031a 100644 --- a/lib/compiler/src/artifact_builders/artifact_builder.rs +++ b/lib/compiler/src/artifact_builders/artifact_builder.rs @@ -7,14 +7,13 @@ use crate::ArtifactCreate; #[cfg(feature = "compiler")] use crate::EngineInner; use crate::Features; -use crate::MetadataHeader; -#[cfg(feature = "compiler")] use crate::{ModuleEnvironment, ModuleMiddlewareChain}; use enumset::EnumSet; use std::mem; use wasmer_types::entity::PrimaryMap; #[cfg(feature = "compiler")] use wasmer_types::CompileModuleInfo; +use wasmer_types::MetadataHeader; use wasmer_types::SerializeError; use wasmer_types::{ CompileError, CpuFeature, CustomSection, Dwarf, FunctionIndex, LocalFunctionIndex, MemoryIndex, diff --git a/lib/compiler/src/engine/artifact.rs b/lib/compiler/src/engine/artifact.rs index 778c6224a78..876b8957792 100644 --- a/lib/compiler/src/engine/artifact.rs +++ b/lib/compiler/src/engine/artifact.rs @@ -9,13 +9,14 @@ use crate::Features; use crate::ModuleEnvironment; use crate::{ register_frame_info, resolve_imports, FunctionExtent, GlobalFrameInfoRegistration, - InstantiationError, MetadataHeader, RuntimeError, Tunables, + InstantiationError, RuntimeError, Tunables, }; use crate::{Engine, EngineInner}; use enumset::EnumSet; use std::sync::Arc; use std::sync::Mutex; use wasmer_types::entity::{BoxedSlice, PrimaryMap}; +use wasmer_types::MetadataHeader; use wasmer_types::{ CompileError, CpuFeature, DataInitializer, DeserializeError, FunctionIndex, LocalFunctionIndex, MemoryIndex, ModuleInfo, OwnedDataInitializer, SerializableModule, SerializeError, diff --git a/lib/compiler/src/traits.rs b/lib/compiler/src/traits.rs index e8da762451e..8b0cb748794 100644 --- a/lib/compiler/src/traits.rs +++ b/lib/compiler/src/traits.rs @@ -3,14 +3,13 @@ use crate::Features; use enumset::EnumSet; use std::any::Any; -use std::convert::TryInto; +use std::fs; use std::path::Path; -use std::{fs, mem}; use wasmer_types::entity::PrimaryMap; +use wasmer_types::SerializeError; use wasmer_types::{ CpuFeature, MemoryIndex, MemoryStyle, ModuleInfo, OwnedDataInitializer, TableIndex, TableStyle, }; -use wasmer_types::{DeserializeError, SerializeError}; /// An `Artifact` is the product that the `Engine` /// implementation produce and use. @@ -87,71 +86,3 @@ impl dyn ArtifactCreate + 'static { self.upcast_any_mut().downcast_mut::() } } - -/// Metadata header which holds an ABI version and the length of the remaining -/// metadata. -#[repr(C)] -#[derive(Clone, Copy)] -pub struct MetadataHeader { - magic: [u8; 8], - version: u32, - len: u32, -} - -impl MetadataHeader { - /// Current ABI version. Increment this any time breaking changes are made - /// to the format of the serialized data. - const CURRENT_VERSION: u32 = 1; - - /// Magic number to identify wasmer metadata. - const MAGIC: [u8; 8] = *b"WASMER\0\0"; - - /// Length of the metadata header. - pub const LEN: usize = 16; - - /// Alignment of the metadata. - pub const ALIGN: usize = 16; - - /// Creates a new header for metadata of the given length. - pub fn new(len: usize) -> Self { - Self { - magic: Self::MAGIC, - version: Self::CURRENT_VERSION, - len: len.try_into().expect("metadata exceeds maximum length"), - } - } - - /// Convert the header into its bytes representation. - pub fn into_bytes(self) -> [u8; 16] { - unsafe { mem::transmute(self) } - } - - /// Parses the header and returns the length of the metadata following it. - pub fn parse(bytes: &[u8]) -> Result { - if bytes.as_ptr() as usize % 16 != 0 { - return Err(DeserializeError::CorruptedBinary( - "misaligned metadata".to_string(), - )); - } - let bytes: [u8; 16] = bytes - .get(..16) - .ok_or_else(|| { - DeserializeError::CorruptedBinary("invalid metadata header".to_string()) - })? - .try_into() - .unwrap(); - let header: Self = unsafe { mem::transmute(bytes) }; - if header.magic != Self::MAGIC { - return Err(DeserializeError::Incompatible( - "The provided bytes were not serialized by Wasmer".to_string(), - )); - } - if header.version != Self::CURRENT_VERSION { - return Err(DeserializeError::Incompatible( - "The provided bytes were serialized by an incompatible version of Wasmer" - .to_string(), - )); - } - Ok(header.len as usize) - } -}