Merge #1719

1719: Turn on testing native engine as part of make test. r=nlewycky a=nlewycky

Add support for "on native" and "on jit" to the ignores.txt.

Extend test-no-traps feature to cover tests/compiler/traps.rs and enable it when testing with the native engine.

Turn on testing native engine as part of make test.


Co-authored-by: Nick Lewycky <nick@wasmer.io>
Co-authored-by: nlewycky <nick@wasmer.io>

Cargo.toml

@@ -131,9 +131,11 @@ test-llvm = [
 
 test-native = [
     "native",
+    "test-generator/test-native",
 ]
 test-jit = [
     "jit",
+    "test-generator/test-jit",
 ]
 
 # Specifies that we're running in coverage testing mode. This disables tests

Makefile

@@ -8,11 +8,13 @@ else
 	UNAME_S := 
 endif
 
-compilers :=
+# Which compilers we build. These have dependencies that may not on the system.
+compilers := cranelift
+
+# Which engines we test. We always build all engines.
+engines := jit
 
 ifeq ($(ARCH), x86_64)
-	# In X64, Cranelift is enabled
-	compilers += cranelift
 	# LLVM could be enabled if not in Windows
 	ifneq ($(OS), Windows_NT)
 		# Singlepass doesn't work yet on Windows
@@ -29,10 +31,14 @@ ifeq ($(ARCH), x86_64)
 				compilers += llvm
 			endif
 		endif
+
+		# Native engine doesn't work yet on Windows
+		engines += native
 	endif
 endif
 
 compilers := $(filter-out ,$(compilers))
+engines := $(filter-out ,$(engines))
 
 ifneq ($(OS), Windows_NT)
 	bold := $(shell tput bold)
@@ -94,7 +100,7 @@ build-capi-llvm:
 # Testing #
 ###########
 
-test: $(foreach compiler,$(compilers),test-$(compiler)-jit) test-packages test-examples test-deprecated
+test: $(foreach engine,$(engines),$(foreach compiler,$(compilers),test-$(compiler)-$(engine))) test-packages test-examples test-deprecated
 
 # Singlepass and native engine don't work together, this rule does nothing.
 test-singlepass-native:

examples/engine_headless.rs

@@ -147,7 +147,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 }
 
 #[test]
-#[cfg(not(windows))]
+#[cfg(not(any(windows, target_arch = "aarch64")))]
 fn test_engine_headless() -> Result<(), Box<dyn std::error::Error>> {
     main()
 }

examples/engine_native.rs

@@ -87,6 +87,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 }
 
 #[test]
+#[cfg(not(target_arch = "aarch64"))]
 fn test_engine_native() -> Result<(), Box<dyn std::error::Error>> {
     main()
 }

lib/compiler-cranelift/Cargo.toml

@@ -15,7 +15,7 @@ edition = "2018"
 wasmer-compiler = { path = "../compiler", version = "1.0.0-alpha4", features = ["translator"], default-features = false }
 wasmer-vm = { path = "../vm", version = "1.0.0-alpha4" }
 wasmer-types = { path = "../wasmer-types", version = "1.0.0-alpha4", default-features = false, features = ["std"] }
-cranelift-codegen = { version = "0.65", default-features = false }
+cranelift-codegen = { version = "0.65", default-features = false, features = ["x86", "arm64"] }
 cranelift-frontend = { version = "0.65", default-features = false }
 tracing = "0.1"
 hashbrown = { version = "0.8", optional = true }

tests/compilers/traps.rs

@@ -38,7 +38,14 @@ fn test_trap_return() -> Result<()> {
 }
 
 #[test]
-#[cfg_attr(feature = "test-singlepass", ignore)]
+#[cfg_attr(
+    any(
+        feature = "test-singlepass",
+        feature = "test-native",
+        target_arch = "aarch64",
+    ),
+    ignore
+)]
 fn test_trap_trace() -> Result<()> {
     let store = get_store();
     let wat = r#"
@@ -118,7 +125,14 @@ fn test_trap_trace_cb() -> Result<()> {
 }
 
 #[test]
-#[cfg_attr(feature = "test-singlepass", ignore)]
+#[cfg_attr(
+    any(
+        feature = "test-singlepass",
+        feature = "test-native",
+        target_arch = "aarch64",
+    ),
+    ignore
+)]
 fn test_trap_stack_overflow() -> Result<()> {
     let store = get_store();
     let wat = r#"
@@ -149,7 +163,15 @@ fn test_trap_stack_overflow() -> Result<()> {
 }
 
 #[test]
-#[cfg_attr(any(feature = "test-singlepass", feature = "test-llvm"), ignore)]
+#[cfg_attr(
+    any(
+        feature = "test-singlepass",
+        feature = "test-llvm",
+        feature = "test-native",
+        target_arch = "aarch64",
+    ),
+    ignore
+)]
 fn trap_display_pretty() -> Result<()> {
     let store = get_store();
     let wat = r#"
@@ -182,7 +204,15 @@ RuntimeError: unreachable
 }
 
 #[test]
-#[cfg_attr(any(feature = "test-singlepass", feature = "test-llvm"), ignore)]
+#[cfg_attr(
+    any(
+        feature = "test-singlepass",
+        feature = "test-llvm",
+        feature = "test-native",
+        target_arch = "aarch64",
+    ),
+    ignore
+)]
 fn trap_display_multi_module() -> Result<()> {
     let store = get_store();
     let wat = r#"
@@ -387,7 +417,14 @@ fn mismatched_arguments() -> Result<()> {
 }
 
 #[test]
-#[cfg_attr(any(feature = "test-singlepass", feature = "test-llvm"), ignore)]
+#[cfg_attr(
+    any(
+        feature = "test-singlepass",
+        feature = "test-llvm",
+        feature = "test-native"
+    ),
+    ignore
+)]
 fn call_signature_mismatch() -> Result<()> {
     let store = get_store();
     let binary = r#"
@@ -418,7 +455,15 @@ RuntimeError: indirect call type mismatch
 }
 
 #[test]
-#[cfg_attr(any(feature = "test-singlepass", feature = "test-llvm"), ignore)]
+#[cfg_attr(
+    any(
+        feature = "test-singlepass",
+        feature = "test-llvm",
+        feature = "test-native",
+        target_arch = "aarch64",
+    ),
+    ignore
+)]
 fn start_trap_pretty() -> Result<()> {
     let store = get_store();
     let wat = r#"
@@ -450,6 +495,7 @@ RuntimeError: unreachable
 }
 
 #[test]
+#[cfg_attr(feature = "test-native", ignore)]
 fn present_after_module_drop() -> Result<()> {
     let store = get_store();
     let module = Module::new(&store, r#"(func (export "foo") unreachable)"#)?;

tests/compilers/wast.rs

@@ -64,6 +64,20 @@ pub fn run_wast(wast_path: &str, compiler: &str) -> anyhow::Result<()> {
     }
     let store = get_store(features, try_nan_canonicalization);
     let mut wast = Wast::new_with_spectest(store);
+    // `bulk-memory-operations/bulk.wast` checks for a message that
+    // specifies which element is uninitialized, but our traps don't
+    // shepherd that information out.
+    wast.allow_trap_message("uninitialized element 2", "uninitialized element");
+    if compiler == "cranelift" && cfg!(feature = "test-native") {
+        wast.allow_trap_message("call stack exhausted", "out of bounds memory access");
+        wast.allow_trap_message("indirect call type mismatch", "call stack exhausted");
+        wast.allow_trap_message("integer divide by zero", "call stack exhausted");
+        wast.allow_trap_message("integer overflow", "call stack exhausted");
+        wast.allow_trap_message("invalid conversion to integer", "call stack exhausted");
+        wast.allow_trap_message("undefined element", "call stack exhausted");
+        wast.allow_trap_message("uninitialized element", "call stack exhausted");
+        wast.allow_trap_message("unreachable", "call stack exhausted");
+    }
     if cfg!(feature = "coverage") {
         wast.disable_assert_and_exhaustion();
     }

tests/ignores.txt

@@ -22,6 +22,29 @@ singlepass on windows # Singlepass is not yet supported on Windows
 cranelift::spec::skip_stack_guard_page on darwin
 llvm::spec::skip_stack_guard_page on darwin
 
+# TODO(https://github.com/wasmerio/wasmer/issues/1727): Traps in native engine
+cranelift::spec::linking on native
+# TODO(https://github.com/wasmerio/wasmer/pull/1710): Library is dropped too soon.
+llvm::spec::linking on native
+
+# https://github.com/wasmerio/wasmer/issues/1722
+llvm::spec::skip_stack_guard_page on native
+
+# Cranelift on arm failures
+cranelift::spec::align on aarch64
+cranelift::spec::memory_trap on aarch64
+cranelift::spec::simd::simd_bit_shift on aarch64
+cranelift::spec::simd::simd_boolean on aarch64
+cranelift::spec::simd::simd_const on aarch64
+cranelift::spec::simd::simd_i16x8_arith on aarch64
+cranelift::spec::simd::simd_i16x8_sat_arith on aarch64
+cranelift::spec::simd::simd_i32x4_arith on aarch64
+cranelift::spec::simd::simd_i64x2_arith on aarch64
+cranelift::spec::simd::simd_i8x16_arith on aarch64
+cranelift::spec::simd::simd_i8x16_sat_arith on aarch64
+cranelift::spec::simd::simd_lane on aarch64
+cranelift::spec::skip_stack_guard_page on aarch64
+
 # Frontends
 
 ## WASI

tests/lib/test-generator/Cargo.toml

@@ -7,3 +7,7 @@ publish = false
 [dependencies]
 anyhow = "1.0"
 target-lexicon = "0.10"
+
+[features]
+test-native = []
+test-jit = []

tests/lib/test-generator/src/lib.rs

@@ -47,6 +47,13 @@ pub fn build_ignores_from_textfile(path: PathBuf) -> anyhow::Result<Ignores> {
     let file = File::open(path)?;
     let reader = BufReader::new(file);
     let host = Triple::host().to_string();
+    let engine = if cfg!(feature = "test-native") {
+        Some("native")
+    } else if cfg!(feature = "test-jit") {
+        Some("jit")
+    } else {
+        None
+    };
     for line in reader.lines() {
         let line = line.unwrap();
         // If the line has a `#` we discard all the content that comes after
@@ -75,8 +82,10 @@ pub fn build_ignores_from_textfile(path: PathBuf) -> anyhow::Result<Ignores> {
         }
 
         // We skip the ignore if doesn't apply to the current
-        // host target
-        if target.map(|t| !host.contains(&t)).unwrap_or(false) {
+        // host target or engine
+        if target.clone().map(|t| !host.contains(&t)).unwrap_or(false)
+            && target.clone() != engine.map(str::to_string)
+        {
             continue;
         }
 

tests/lib/wast/src/wast.rs

@@ -18,6 +18,9 @@ pub struct Wast {
     instances: HashMap<String, Instance>,
     /// Allowed failures (ideally this should be empty)
     allowed_instantiation_failures: HashSet<String>,
+    /// If the (expected from .wast, actual) message pair is in this list,
+    /// treat the strings as matching.
+    match_trap_messages: HashMap<String, String>,
     /// If the current module was an allowed failure, we allow test to fail
     current_is_allowed_failure: bool,
     /// The wasm Store
@@ -37,6 +40,7 @@ impl Wast {
             store,
             import_object,
             allowed_instantiation_failures: HashSet::new(),
+            match_trap_messages: HashMap::new(),
             current_is_allowed_failure: false,
             instances: HashMap::new(),
             fail_fast: true,
@@ -52,6 +56,12 @@ impl Wast {
         }
     }
 
+    /// A list of alternative messages to permit for a trap failure.
+    pub fn allow_trap_message(&mut self, expected: &str, allowed: &str) {
+        self.match_trap_messages
+            .insert(expected.into(), allowed.into());
+    }
+
     /// Do not run any code in assert_trap or assert_exhaustion.
     pub fn disable_assert_and_exhaustion(&mut self) {
         self.disable_assert_trap_exhaustion = true;
@@ -132,7 +142,7 @@ impl Wast {
             Ok(values) => bail!("expected trap, got {:?}", values),
             Err(t) => format!("{}", t),
         };
-        if Self::matches_message_assert_trap(expected, &actual) {
+        if self.matches_message_assert_trap(expected, &actual) {
             return Ok(());
         }
         bail!("expected '{}', got '{}'", expected, actual)
@@ -414,12 +424,13 @@ impl Wast {
     }
 
     // Checks if the `assert_trap` message matches the expected one
-    fn matches_message_assert_trap(expected: &str, actual: &str) -> bool {
+    fn matches_message_assert_trap(&self, expected: &str, actual: &str) -> bool {
         actual.contains(expected)
-            // `bulk-memory-operations/bulk.wast` checks for a message that
-            // specifies which element is uninitialized, but our traps don't
-            // shepherd that information out.
-            || (expected.contains("uninitialized element 2") && actual.contains("uninitialized element"))
+            || self
+                .match_trap_messages
+                .get(expected)
+                .map(|alternative| actual.contains(alternative))
+                .unwrap_or(false)
     }
 }