Update and revamp wasm32 SIMD intrinsics

.github/workflows/main.yml

@@ -77,7 +77,7 @@ jobs:
         - mips64-unknown-linux-gnuabi64
         - mips64el-unknown-linux-gnuabi64
         - s390x-unknown-linux-gnu
-        - wasm32-unknown-unknown
+        - wasm32-wasi
         - i586-unknown-linux-gnu
         - x86_64-linux-android
         - arm-linux-androideabi
@@ -129,7 +129,7 @@ jobs:
           disable_assert_instr: true
         - target: s390x-unknown-linux-gnu
           os: ubuntu-latest
-        - target: wasm32-unknown-unknown
+        - target: wasm32-wasi
           os: ubuntu-latest
         - target: aarch64-unknown-linux-gnu
           os: ubuntu-latest

ci/docker/wasm32-wasi/Dockerfile

@@ -0,0 +1,16 @@
+FROM ubuntu:20.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update -y && apt-get install -y --no-install-recommends \
+  ca-certificates \
+  curl \
+  xz-utils \
+  clang
+
+RUN curl -L https://github.com/bytecodealliance/wasmtime/releases/download/dev/wasmtime-dev-x86_64-linux.tar.xz | tar xJf -
+ENV PATH=$PATH:/wasmtime-dev-x86_64-linux
+
+ENV CARGO_TARGET_WASM32_WASI_RUNNER="wasmtime \
+  --enable-simd \
+  --mapdir .::/checkout/target/wasm32-wasi/release/deps \
+  --"

ci/run.sh

@@ -44,6 +44,16 @@ cargo_test() {
     fi
     cmd="$cmd ${subcmd} --target=$TARGET $1"
     cmd="$cmd -- $2"
+
+    # wasm targets can't catch panics so if a test failures make sure the test
+    # harness isn't trying to capture output, otherwise we won't get any useful
+    # output.
+    case ${TARGET} in
+        wasm32*)
+            cmd="$cmd --nocapture"
+            ;;
+    esac
+
     $cmd
 }
 
@@ -72,20 +82,11 @@ case ${TARGET} in
         export RUSTFLAGS="${RUSTFLAGS} -C target-feature=+avx"
         cargo_test "--release"
         ;;
-    wasm32-unknown-unknown*)
-        # Attempt to actually run some SIMD tests in node.js. Unfortunately
-        # though node.js (transitively through v8) doesn't have support for the
-        # full SIMD spec yet, only some functions. As a result only pass in
-        # some target features and a special `--cfg`
-        # FIXME: broken
-        #export RUSTFLAGS="${RUSTFLAGS} -C target-feature=+simd128 --cfg only_node_compatible_functions"
-        #cargo_test "--release"
-
-        # After that passes make sure that all intrinsics compile, passing in
-        # the extra feature to compile in non-node-compatible SIMD.
-        # FIXME: broken
-        #export RUSTFLAGS="${RUSTFLAGS} -C target-feature=+simd128,+unimplemented-simd128"
-        #cargo_test "--release --no-run"
+    wasm32*)
+        prev="$RUSTFLAGS"
+        export RUSTFLAGS="${RUSTFLAGS} -C target-feature=+simd128,+unimplemented-simd128"
+        cargo_test "--release"
+        export RUSTFLAGS="$prev"
         ;;
     # FIXME: don't build anymore
     #mips-*gnu* | mipsel-*gnu*)
@@ -111,7 +112,7 @@ case ${TARGET} in
 
 esac
 
-if [ "$NORUN" != "1" ] && [ "$NOSTD" != 1 ] && [ "$TARGET" != "wasm32-unknown-unknown" ]; then
+if [ "$NORUN" != "1" ] && [ "$NOSTD" != 1 ]; then
     # Test examples
     (
         cd examples

crates/assert-instr-macro/src/lib.rs

@@ -122,6 +122,13 @@ pub fn assert_instr(
             // generate some code that's hopefully very tight in terms of
             // codegen but is otherwise unique to prevent code from being
             // folded.
+            //
+            // This is avoided on Wasm32 right now since these functions aren't
+            // inlined which breaks our tests since each intrinsic looks like it
+            // calls functions. Turns out functions aren't similar enough to get
+            // merged on wasm32 anyway. This bug is tracked at
+            // rust-lang/rust#74320.
+            #[cfg(not(target_arch = "wasm32"))]
             ::stdarch_test::_DONT_DEDUP.store(
                 std::mem::transmute(#shim_name_str.as_bytes().as_ptr()),
                 std::sync::atomic::Ordering::Relaxed,
@@ -131,8 +138,7 @@ pub fn assert_instr(
     };
 
     let tokens: TokenStream = quote! {
-        #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
-        #[cfg_attr(not(target_arch = "wasm32"), test)]
+        #[test]
         #[allow(non_snake_case)]
         fn #assert_name() {
             #to_test
@@ -146,11 +152,6 @@ pub fn assert_instr(
                                    #instr);
         }
     };
-    // why? necessary now to get tests to work?
-    let tokens: TokenStream = tokens
-        .to_string()
-        .parse()
-        .expect("cannot parse tokenstream");
 
     let tokens: TokenStream = quote! {
         #item

crates/core_arch/Cargo.toml

@@ -26,8 +26,5 @@ maintenance = { status = "experimental" }
 stdarch-test = { version = "0.*", path = "../stdarch-test" }
 std_detect = { version = "0.*", path = "../std_detect" }
 
-[target.wasm32-unknown-unknown.dev-dependencies]
-wasm-bindgen-test = "0.2.47"
-
 [package.metadata.docs.rs]
 rustdoc-args = [ "--cfg", "dox" ]

crates/core_arch/build.rs

@@ -1,3 +1,17 @@
+use std::env;
+
 fn main() {
     println!("cargo:rustc-cfg=core_arch_docs");
+
+    // Used to tell our `#[assert_instr]` annotations that all simd intrinsics
+    // are available to test their codegen, since some are gated behind an extra
+    // `-Ctarget-feature=+unimplemented-simd128` that doesn't have any
+    // equivalent in `#[target_feature]` right now.
+    println!("cargo:rerun-if-env-changed=RUSTFLAGS");
+    if env::var("RUSTFLAGS")
+        .unwrap_or_default()
+        .contains("unimplemented-simd128")
+    {
+        println!("cargo:rustc-cfg=all_simd");
+    }
 }

crates/core_arch/src/lib.rs

@@ -1,9 +1,12 @@
 #![doc(include = "core_arch_docs.md")]
+#![allow(improper_ctypes_definitions)]
 #![allow(dead_code)]
 #![allow(unused_features)]
+#![allow(incomplete_features)]
 #![feature(
     const_fn,
     const_fn_union,
+    const_generics,
     custom_inner_attributes,
     link_llvm_intrinsics,
     platform_intrinsics,
@@ -32,9 +35,12 @@
     adx_target_feature,
     rtm_target_feature,
     f16c_target_feature,
-    external_doc
+    external_doc,
+    allow_internal_unstable,
+    decl_macro
 )]
 #![cfg_attr(test, feature(test, abi_vectorcall, untagged_unions))]
+#![cfg_attr(all(test, target_arch = "wasm32"), feature(wasm_simd))]
 #![deny(clippy::missing_inline_in_public_items)]
 #![allow(
     clippy::inline_always,
@@ -66,13 +72,10 @@ extern crate std_detect;
 #[cfg(test)]
 extern crate stdarch_test;
 
-#[cfg(all(test, target_arch = "wasm32"))]
-extern crate wasm_bindgen_test;
-
 #[path = "mod.rs"]
 mod core_arch;
 
-pub use self::core_arch::arch::*;
+pub use self::core_arch::arch;
 
 #[allow(unused_imports)]
 use core::{ffi, hint, intrinsics, marker, mem, ops, ptr, sync};

crates/core_arch/src/mod.rs

@@ -57,14 +57,110 @@ pub mod arch {
 
     /// Platform-specific intrinsics for the `wasm32` platform.
     ///
-
-    /// # Availability
+    /// This module provides intrinsics specific to the WebAssembly
+    /// architecture. Here you'll find intrinsics necessary for leveraging
+    /// WebAssembly proposals such as [atomics] and [simd]. These proposals are
+    /// evolving over time and as such the support here is unstable and requires
+    /// the nightly channel. As WebAssembly proposals stabilize these functions
+    /// will also become stable.
     ///
-    /// Note that intrinsics gated by `target_feature = "atomics"` or `target_feature = "simd128"`
-    /// are only available **when the standard library itself is compiled with the the respective
-    /// target feature**. This version of the standard library is not obtainable via `rustup`,
-    /// but rather will require the standard library to be compiled from source.
-    /// See the [module documentation](../index.html) for more details.
+    /// [atomics]: https://github.com/webassembly/threads
+    /// [simd]: https://github.com/webassembly/simd
+    ///
+    /// See the [module documentation](../index.html) for general information
+    /// about the `arch` module and platform intrinsics.
+    ///
+    /// ## Atomics
+    ///
+    /// The [threads proposal][atomics] for WebAssembly adds a number of
+    /// instructions for dealing with multithreaded programs. Atomic
+    /// instructions can all be generated through `std::sync::atomic` types, but
+    /// some instructions have no equivalent in Rust such as
+    /// `memory.atomic.notify` so this module will provide these intrinsics.
+    ///
+    /// At this time, however, these intrinsics are only available **when the
+    /// standard library itself is compiled with atomics**. Compiling with
+    /// atomics is not enabled by default and requires passing
+    /// `-Ctarget-feature=+atomics` to rustc. The standard library shipped via
+    /// `rustup` is not compiled with atomics. To get access to these intrinsics
+    /// you'll need to compile the standard library from source with the
+    /// requisite compiler flags.
+    ///
+    /// ## SIMD
+    ///
+    /// The [simd proposal][simd] for WebAssembly adds a new `v128` type for a
+    /// 128-bit SIMD register. It also adds a large array of instructions to
+    /// operate on the `v128` type to perform data processing. The SIMD proposal
+    /// has been in progress for quite some time and many instructions have come
+    /// and gone. This module attempts to keep up with the proposal, but if you
+    /// notice anything awry please feel free to [open an
+    /// issue](https://github.com/rust-lang/stdarch/issues/new).
+    ///
+    /// It's important to be aware that the current state of development of SIMD
+    /// in WebAssembly is still somewhat early days. There's lots of pieces to
+    /// demo and prototype with, but discussions and support are still in
+    /// progress. There's a number of pitfalls and gotchas in various places,
+    /// which will attempt to be documented here, but there may be others
+    /// lurking!
+    ///
+    /// Using SIMD is intended to be similar to as you would on `x86_64`, for
+    /// example. You'd write a function such as:
+    ///
+    /// ```rust,ignore
+    /// #[cfg(target_arch = "wasm32")]
+    /// #[target_feature(enable = "simd128")]
+    /// unsafe fn uses_simd() {
+    ///     use std::arch::wasm32::*;
+    ///     // ...
+    /// }
+    /// ```
+    ///
+    /// Unlike `x86_64`, however, WebAssembly does not currently have dynamic
+    /// detection at runtime as to whether SIMD is supported (this is one of the
+    /// motivators for the [conditional sections proposal][condsections], but
+    /// that is still pretty early days). This means that your binary will
+    /// either have SIMD and can only run on engines which support SIMD, or it
+    /// will not have SIMD at all. For compatibility the standard library itself
+    /// does not use any SIMD internally. Determining how best to ship your
+    /// WebAssembly binary with SIMD is largely left up to you as it can can be
+    /// pretty nuanced depending on your situation.
+    ///
+    /// [condsections]: https://github.com/webassembly/conditional-sections
+    ///
+    /// To enable SIMD support at compile time you need to do one of two things:
+    ///
+    /// * First you can annotate functions with `#[target_feature(enable =
+    ///   "simd128")]`. This causes just that one function to have SIMD support
+    ///   available to it, and intrinsics will get inlined as usual in this
+    ///   situation.
+    ///
+    /// * Second you can compile your program with `-Ctarget-feature=+simd128`.
+    ///   This compilation flag blanket enables SIMD support for your entire
+    ///   compilation. Note that this does not include the standard library
+    ///   unless you recompile the standard library.
+    ///
+    /// If you enable SIMD via either of these routes then you'll have a
+    /// WebAssembly binary that uses SIMD instructions, and you'll need to ship
+    /// that accordingly. Also note that if you call SIMD intrinsics but don't
+    /// enable SIMD via either of these mechanisms, you'll still have SIMD
+    /// generated in your program. This means to generate a binary without SIMD
+    /// you'll need to avoid both options above plus calling into any intrinsics
+    /// in this module.
+    ///
+    /// > **Note**: Due to
+    /// > [rust-lang/rust#74320](https://github.com/rust-lang/rust/issues/74320)
+    /// > it's recommended to compile your entire program with SIMD support
+    /// > (using `RUSTFLAGS`) or otherwise functions may not be inlined
+    /// > correctly.
+    ///
+    /// > **Note**: LLVM's SIMD support is actually split into two features:
+    /// > `simd128` and `unimplemented-simd128`. Rust code can enable `simd128`
+    /// > with `#[target_feature]` (and test for it with `#[cfg(target_feature =
+    /// > "simd128")]`, but it cannot enable `unimplemented-simd128`. The only
+    /// > way to enable this feature is to compile with
+    /// > `-Ctarget-feature=+simd128,+unimplemented-simd128`. This second
+    /// > feature enables more recent instructions implemented in LLVM which
+    /// > haven't always had enough time to make their way to runtimes.
     #[cfg(any(target_arch = "wasm32", dox))]
     #[doc(cfg(target_arch = "wasm32"))]
     #[stable(feature = "simd_wasm32", since = "1.33.0")]