[wip] Build musl for tests

.github/workflows/main.yml

@@ -4,44 +4,103 @@ on: [push, pull_request]
 env:
   RUSTDOCFLAGS: -Dwarnings
   RUSTFLAGS: -Dwarnings
+  RUST_BACKTRACE: full
 
 jobs:
   docker:
     name: Docker
-    runs-on: ubuntu-latest
+    timeout-minutes: 10
     strategy:
+      fail-fast: false
       matrix:
-        target:
-        - aarch64-unknown-linux-gnu
-        - arm-unknown-linux-gnueabi
-        - arm-unknown-linux-gnueabihf
-        - armv7-unknown-linux-gnueabihf
-        # - i686-unknown-linux-gnu
-        # MIPS targets disabled since they are dropped to tier 3.
-        # See https://github.com/rust-lang/compiler-team/issues/648
-        #- mips-unknown-linux-gnu
-        #- mips64-unknown-linux-gnuabi64
-        #- mips64el-unknown-linux-gnuabi64
-        - powerpc-unknown-linux-gnu
-        - powerpc64-unknown-linux-gnu
-        - powerpc64le-unknown-linux-gnu
-        - x86_64-unknown-linux-gnu
+        include:
+        - target: aarch64-apple-darwin
+          os: macos-latest
+        - target: aarch64-unknown-linux-gnu
+          os: ubuntu-latest
+        - target: aarch64-pc-windows-msvc
+          os: windows-latest
+        - target: arm-unknown-linux-gnueabi
+          os: ubuntu-latest
+        - target: arm-unknown-linux-gnueabihf
+          os: ubuntu-latest
+        - target: armv7-unknown-linux-gnueabihf
+          os: ubuntu-latest
+        - target: i586-unknown-linux-gnu
+          os: ubuntu-latest
+        - target: i686-unknown-linux-gnu
+          os: ubuntu-latest
+        - target: powerpc-unknown-linux-gnu
+          os: ubuntu-latest
+        - target: powerpc64-unknown-linux-gnu
+          os: ubuntu-latest
+        - target: powerpc64le-unknown-linux-gnu
+          os: ubuntu-latest
+        - target: riscv64gc-unknown-linux-gnu
+          os: ubuntu-latest
+        - target: thumbv6m-none-eabi
+          os: ubuntu-latest
+        - target: thumbv7em-none-eabi
+          os: ubuntu-latest
+        - target: thumbv7em-none-eabihf
+          os: ubuntu-latest
+        - target: thumbv7m-none-eabi
+          os: ubuntu-latest
+        - target: x86_64-unknown-linux-gnu
+          os: ubuntu-latest
+        - target: x86_64-apple-darwin
+          os: macos-13
+        - target: i686-pc-windows-msvc
+          os: windows-latest
+        - target: x86_64-pc-windows-msvc
+          os: windows-latest
+        - target: i686-pc-windows-gnu
+          os: windows-latest
+        - target: x86_64-pc-windows-gnu
+          os: windows-latest
+    runs-on: ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@master
-    - name: Install Rust
-      run: rustup update nightly && rustup default nightly
-    - run: rustup target add ${{ matrix.target }}
-    - run: rustup target add x86_64-unknown-linux-musl
-    - run: cargo generate-lockfile
-    - run: ./ci/run-docker.sh ${{ matrix.target }}
+    - name: Print runner information
+      run: uname -a
+    - uses: actions/checkout@v4
+    - name: Install Rust (rustup)
+      shell: bash
+      run: |
+        rustup update nightly --no-self-update
+        rustup default nightly
+        rustup target add ${{ matrix.target }}
+        rustup component add llvm-tools-preview
+    - uses: Swatinem/rust-cache@v2
+      with:
+        key: ${{ matrix.target }}
+
+    - name: Download musl source
+      run: ./ci/download-musl.sh
+      shell: bash
+
+    # Non-linux tests just use our raw script
+    - name: Run tests locally
+      run: ./ci/run.sh ${{ matrix.target }}
+      if: matrix.os != 'ubuntu-latest'
+      shell: bash
+
+    # Otherwise we use our docker containers to run builds
+    - name: Run in Docker
+      if: matrix.os == 'ubuntu-latest'
+      run: |
+        rustup target add x86_64-unknown-linux-musl
+        cargo generate-lockfile && ./ci/run-docker.sh ${{ matrix.target }}
 
   rustfmt:
     name: Rustfmt
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@master
     - name: Install Rust
-      run: rustup update stable && rustup default stable && rustup component add rustfmt
+      run: |
+        rustup update stable --no-self-update
+        rustup default stable
+        rustup component add rustfmt
     - run: cargo fmt -- --check
 
   wasm:
@@ -50,8 +109,10 @@ jobs:
     steps:
     - uses: actions/checkout@master
     - name: Install Rust
-      run: rustup update nightly && rustup default nightly
+      run: rustup update nightly --no-self-update && rustup default nightly
     - run: rustup target add wasm32-unknown-unknown
+    - name: Download MUSL source
+      run: ./ci/download-musl.sh
     - run: cargo build --target wasm32-unknown-unknown
 
   cb:
@@ -60,7 +121,7 @@ jobs:
     steps:
     - uses: actions/checkout@master
     - name: Install Rust
-      run: rustup update nightly && rustup default nightly
+      run: rustup update nightly --no-self-update && rustup default nightly
     - run: cargo build -p cb
 
   benchmarks:
@@ -69,7 +130,9 @@ jobs:
     steps:
     - uses: actions/checkout@master
     - name: Install Rust
-      run: rustup update nightly && rustup default nightly
+      run: rustup update nightly --no-self-update && rustup default nightly
+    - name: Download MUSL source
+      run: ./ci/download-musl.sh
     - run: cargo bench --all
 
   success:

.gitignore

@@ -1,8 +1,9 @@
-**/*.rs.bk
+**.bk
 .#*
 /bin
 /math/src
 /math/target
 /target
-/tests
 Cargo.lock
+musl/
+**.tar.gz

Cargo.toml

@@ -27,6 +27,7 @@ members = [
   "crates/compiler-builtins-smoke-test",
   "crates/libm-bench",
   "crates/libm-test",
+  "crates/musl-math-sys",
 ]
 default-members = [
   ".",

ci/docker/i586-unknown-linux-gnu/Dockerfile

@@ -0,0 +1,5 @@
+FROM ubuntu:24.04
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    gcc-multilib libc6-dev ca-certificates

ci/docker/riscv64gc-unknown-linux-gnu/Dockerfile

@@ -0,0 +1,15 @@
+FROM ubuntu:24.04
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    gcc libc6-dev qemu-user-static ca-certificates \
+    gcc-riscv64-linux-gnu libc6-dev-riscv64-cross \
+    qemu-system-riscv64
+
+ENV TOOLCHAIN_PREFIX=riscv64-linux-gnu-
+ENV CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_LINKER="$TOOLCHAIN_PREFIX"gcc \
+    CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_RUNNER=qemu-riscv64-static \
+    AR_riscv64gc_unknown_linux_gnu="$TOOLCHAIN_PREFIX"ar \
+    CC_riscv64gc_unknown_linux_gnu="$TOOLCHAIN_PREFIX"gcc \
+    QEMU_LD_PREFIX=/usr/riscv64-linux-gnu \
+    RUST_TEST_THREADS=1

ci/download-musl.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+set -eux
+
+fname=musl-1.2.5.tar.gz
+sha=a9a118bbe84d8764da0ea0d28b3ab3fae8477fc7e4085d90102b8596fc7c75e4
+
+mkdir musl
+curl "https://musl.libc.org/releases/$fname" -O
+
+echo "$(uname -s)"
+
+case "$(uname -s)" in
+  MINGW*)
+    # Need to extract the second line because certutil does human output
+    fsha=$(certutil -hashfile "$fname" SHA256 | sed -n '2p')
+    [ "$sha" = "$fsha" ] || exit 1
+  ;;
+  *)
+    echo "$sha  $fname" | shasum -a 256 --check || exit 1
+  ;;
+esac
+
+tar -xzf "$fname" -C musl --strip-components 1
+rm "$fname"

ci/run.sh

@@ -17,6 +17,8 @@ $cmd --release
 $cmd --features 'unstable'
 $cmd --release --features 'unstable'
 
-# also run the reference tests
-$cmd --features 'unstable libm-test/musl-bitwise-tests'
-$cmd --release --features 'unstable libm-test/musl-bitwise-tests'
+if [ "$(uname -a)" = "Linux" ]; then
+  # also run the reference tests
+  $cmd --features 'unstable libm-test/musl-bitwise-tests'
+  $cmd --release --features 'unstable libm-test/musl-bitwise-tests'
+fi

crates/libm-test/Cargo.toml

@@ -14,5 +14,13 @@ musl-bitwise-tests = ["rand"]
 [dependencies]
 libm = { path = "../.." }
 
+# We can't build musl on MSVC or wasm
+[target.'cfg(not(any(target_env = "msvc", target_family = "wasm")))'.dependencies]
+musl-math-sys = { path = "../musl-math-sys" }
+
+[dev-dependencies]
+rand = "0.8.5"
+rand_chacha = "0.3.1"
+
 [build-dependencies]
 rand = { version = "0.8.5", optional = true }

crates/libm-test/build.rs

@@ -1,8 +1,96 @@
+use std::fmt::Write;
+use std::path::PathBuf;
+use std::{env, fs};
+
 fn main() {
+    let cfg = Config::from_env();
+
+    emit_optimization_cfg(&cfg);
+    emit_cfg_shorthands(&cfg);
+    list_all_tests(&cfg);
+
     #[cfg(feature = "musl-bitwise-tests")]
     musl_reference_tests::generate();
 }
 
+#[allow(dead_code)]
+struct Config {
+    manifest_dir: PathBuf,
+    out_dir: PathBuf,
+    opt_level: u8,
+    target_arch: String,
+    target_env: String,
+    target_family: String,
+    target_os: String,
+    target_string: String,
+    target_vendor: String,
+    target_features: Vec<String>,
+}
+
+impl Config {
+    fn from_env() -> Self {
+        let target_features = env::var("CARGO_CFG_TARGET_FEATURE")
+            .map(|feats| feats.split(',').map(ToOwned::to_owned).collect())
+            .unwrap_or_default();
+
+        Self {
+            manifest_dir: PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()),
+            out_dir: PathBuf::from(env::var("OUT_DIR").unwrap()),
+            opt_level: env::var("OPT_LEVEL").unwrap().parse().unwrap(),
+            target_arch: env::var("CARGO_CFG_TARGET_ARCH").unwrap(),
+            target_env: env::var("CARGO_CFG_TARGET_ENV").unwrap(),
+            target_family: env::var("CARGO_CFG_TARGET_FAMILY").unwrap(),
+            target_os: env::var("CARGO_CFG_TARGET_OS").unwrap(),
+            target_string: env::var("TARGET").unwrap(),
+            target_vendor: env::var("CARGO_CFG_TARGET_VENDOR").unwrap(),
+            target_features,
+        }
+    }
+}
+
+/// Some tests are extremely slow. Emit a config option to
+fn emit_optimization_cfg(cfg: &Config) {
+    println!("cargo::rustc-check-cfg=cfg(optimizations_enabled)");
+
+    if cfg.opt_level >= 2 {
+        println!("cargo::rustc-cfg=optimizations_enabled");
+    }
+}
+
+/// Turn lengthy configuration into
+fn emit_cfg_shorthands(cfg: &Config) {
+    println!("cargo::rustc-check-cfg=cfg(x86_no_sse)");
+    if cfg.target_arch == "x86" && !cfg.target_features.iter().any(|f| f == "sse") {
+        // Shorthand to detect i586 targets
+        println!("cargo::rustc-cfg=x86_no_sse");
+    }
+}
+
+fn list_all_tests(cfg: &Config) {
+    let math_src = cfg.manifest_dir.join("../../src/math");
+
+    let mut files = fs::read_dir(math_src)
+        .unwrap()
+        .map(|f| f.unwrap().path())
+        .filter(|entry| entry.is_file())
+        .map(|f| f.file_stem().unwrap().to_str().unwrap().to_owned())
+        .collect::<Vec<_>>();
+    files.sort();
+
+    let mut s = "pub const MATH_FILES: &[&str] = &[".to_owned();
+    for f in files {
+        if f == "mod" {
+            // skip mod.rs
+            continue;
+        }
+        write!(s, "\"{f}\",").unwrap();
+    }
+    write!(s, "];").unwrap();
+
+    let outfile = cfg.out_dir.join("all_files.rs");
+    fs::write(outfile, s).unwrap();
+}
+
 #[cfg(feature = "musl-bitwise-tests")]
 mod musl_reference_tests {
     use rand::seq::SliceRandom;