diff --git a/default.nix b/default.nix index edaf9775271..e6a595beea6 100644 --- a/default.nix +++ b/default.nix @@ -32,9 +32,11 @@ let llvmPackages_12.bintools rustc-nightly cargo-nightly - xargo wasmtime rust-bindgen + python3 + ] ++ pkgs.lib.optional pkgs.stdenv.isDarwin [ + libiconv ]; llvmEnv = '' @@ -152,46 +154,97 @@ in rec { rts = let - # We run this on motoko-rts-tests, to get the union of all - # dependencies - rustDeps = nixpkgs.rustPlatform-nightly.fetchCargoTarball { + # Build Rust package cargo-vendor-tools + cargoVendorTools = nixpkgs.rustPlatform.buildRustPackage rec { + name = "cargo-vendor-tools"; + src = ./rts/cargo-vendor-tools; + cargoSha256 = "0zi3fiq9sy6c9dv7fd2xc9lan85d16gfax47n6g6f5q5c1zb5r47"; + }; + + # Path to vendor-rust-std-deps, provided by cargo-vendor-tools + vendorRustStdDeps = "${cargoVendorTools}/bin/vendor-rust-std-deps"; + + # SHA256 of Rust std deps + rustStdDepsHash = "0wxx8prh66i19vd5078iky6x5bzs6ppz7c1vbcyx9h4fg0f7pfj6"; + + # Vendor directory for Rust std deps + rustStdDeps = nixpkgs.stdenvNoCC.mkDerivation { + name = "rustc-std-deps"; + + nativeBuildInputs = with nixpkgs; [ + curl + ]; + + buildCommand = '' + mkdir $out + cd $out + ${vendorRustStdDeps} ${nixpkgs.rustc-nightly} . + ''; + + outputHash = rustStdDepsHash; + outputHashAlgo = "sha256"; + outputHashMode = "recursive"; + }; + + # Vendor tarball of the RTS + rtsDeps = nixpkgs.rustPlatform.fetchCargoTarball { name = "motoko-rts-deps"; src = subpath ./rts; sourceRoot = "rts/motoko-rts-tests"; - sha256 = "0jyp3j8n5bj5cy1fd26d7h55zmc4v14qc2w8adxqwmsv5riqz41g"; + sha256 = "129gfmn96vm7di903pxirg7zybl83q6nkwiqr3rsy7l1q8667kxx"; copyLockfile = true; }; + + # Unpacked RTS deps + rtsDepsUnpacked = nixpkgs.stdenvNoCC.mkDerivation { + name = rtsDeps.name + "-unpacked"; + buildCommand = '' + tar xf ${rtsDeps} + mv *.tar.gz $out + ''; + }; + + # All dependencies needed to build the RTS, including Rust std deps, to + # allow `cargo -Zbuild-std`. (rust-lang/wg-cargo-std-aware#23) + allDeps = nixpkgs.stdenvNoCC.mkDerivation { + name = "merged-rust-deps"; + + buildCommand = '' + mkdir -p $out + cp -r ${rtsDepsUnpacked}/* $out/ + cp -r ${rustStdDeps}/* $out/ + ''; + }; in stdenv.mkDerivation { name = "moc-rts"; src = subpath ./rts; - nativeBuildInputs = [ nixpkgs.makeWrapper nixpkgs.removeReferencesTo ]; + + nativeBuildInputs = [ nixpkgs.makeWrapper nixpkgs.removeReferencesTo nixpkgs.cacert ]; buildInputs = rtsBuildInputs; preBuild = '' - export XARGO_HOME=$PWD/xargo-home export CARGO_HOME=$PWD/cargo-home - # this replicates logic from nixpkgs’ pkgs/build-support/rust/default.nix + # This replicates logic from nixpkgs’ pkgs/build-support/rust/default.nix mkdir -p $CARGO_HOME - echo "Using vendored sources from ${rustDeps}" - unpackFile ${rustDeps} + echo "Using vendored sources from ${rtsDeps}" + unpackFile ${allDeps} cat > $CARGO_HOME/config <<__END__ [source."crates-io"] "replace-with" = "vendored-sources" [source."vendored-sources"] - "directory" = "$(stripHash ${rustDeps})" + "directory" = "$(stripHash ${allDeps})" __END__ ${llvmEnv} export TOMMATHSRC=${nixpkgs.sources.libtommath} export MUSLSRC=${nixpkgs.sources.musl-wasi}/libc-top-half/musl export MUSL_WASI_SYSROOT=${musl-wasi-sysroot} - ''; doCheck = true; @@ -206,11 +259,16 @@ rec { cp mo-rts-debug.wasm $out/rts ''; - # This needs to be self-contained. Remove mention of - # nix path in debug message. + # This needs to be self-contained. Remove mention of nix path in debug + # message. preFixup = '' - remove-references-to -t ${nixpkgs.rustc-nightly} -t ${rustDeps} $out/rts/mo-rts.wasm $out/rts/mo-rts-debug.wasm + remove-references-to \ + -t ${nixpkgs.rustc-nightly} \ + -t ${rtsDeps} \ + -t ${rustStdDeps} \ + $out/rts/mo-rts.wasm $out/rts/mo-rts-debug.wasm ''; + allowedRequisites = []; }; @@ -440,7 +498,6 @@ rec { filecheck = nixpkgs.linkFarm "FileCheck" [ { name = "bin/FileCheck"; path = "${nixpkgs.llvm}/bin/FileCheck";} ]; - inherit (nixpkgs) xargo; # gitMinimal is used by nix/gitSource.nix; building it here warms the nix cache inherit (nixpkgs) gitMinimal; diff --git a/nix/default.nix b/nix/default.nix index 7e68f71465e..3a64fed5524 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -46,7 +46,7 @@ let # Rust nightly (self: super: let moz_overlay = import self.sources.nixpkgs-mozilla self super; - rust-channel = moz_overlay.rustChannelOf { date = "2020-07-22"; channel = "nightly"; }; + rust-channel = moz_overlay.rustChannelOf { date = "2021-10-25"; channel = "nightly"; }; in rec { rustc-nightly = rust-channel.rust.override { targets = [ @@ -62,7 +62,6 @@ let rustc = rustc-nightly; cargo = cargo-nightly; }; - xargo = self.callPackage ./xargo.nix {}; }) # wasm-profiler diff --git a/nix/xargo.nix b/nix/xargo.nix deleted file mode 100644 index c84995221b6..00000000000 --- a/nix/xargo.nix +++ /dev/null @@ -1,31 +0,0 @@ -# xargo is used to build motoko-rts for wasm32. We need to make a shared Wasm -# library for the RTS (that's what moc-ld supports) but Rust ships wasm32 -# libraries (core and std) without PIC relocation model, so we use xargo to make -# PIC versions of core and std. - -{ rustPlatform-nightly, fetchFromGitHub, lib }: - -rustPlatform-nightly.buildRustPackage rec { - name = "xargo"; - - src = fetchFromGitHub { - owner = "japaric"; - repo = "${name}"; - rev = "16035a7c401262824edcb87e1401fe4b05a5ccc0"; - sha256 = "0m1dg7vwmmlpqp20p219gsm7zbnnii6lik6hc2vvfsdmnygf271l"; - fetchSubmodules = true; - }; - - cargoSha256 = "171a7xm47qdcdd6n67plyvnsxp00hn6skx5vzxd8a2kmblfqn5gy"; - - doCheck = false; - USER = "nobody"; # for xargo tests (if we would run them) - - meta = with lib; { - description = "The sysroot manager that lets you build and customize std"; - homepage = "https://github.com/japaric/xargo"; - license = licenses.mit; - maintainers = []; - platforms = platforms.unix; - }; -} diff --git a/rts/Makefile b/rts/Makefile index 415881dc19a..eb5fd2efac1 100644 --- a/rts/Makefile +++ b/rts/Makefile @@ -223,11 +223,11 @@ $(TOMMATH_BINDINGS_RS): | _build $(RTS_RUST_WASM_A): $(TOMMATH_BINDINGS_RS) $(RTS_RUST_FILES) $(RTS_CARGO_FILES) | _build/wasm - cd motoko-rts && xargo build --release --target=wasm32-unknown-emscripten + cd motoko-rts && cargo build --release --target=wasm32-unknown-emscripten -Zbuild-std=core,alloc cp motoko-rts/target/wasm32-unknown-emscripten/release/libmotoko_rts.a $@ $(RTS_RUST_DEBUG_WASM_A): $(TOMMATH_BINDINGS_RS) $(RTS_RUST_FILES) $(RTS_CARGO_FILES) | _build/wasm - cd motoko-rts && xargo build --target=wasm32-unknown-emscripten + cd motoko-rts && cargo build --target=wasm32-unknown-emscripten -Zbuild-std=core,alloc cp motoko-rts/target/wasm32-unknown-emscripten/debug/libmotoko_rts.a $@ # @@ -286,5 +286,4 @@ clean: mo-rts-debug.wasm \ motoko-rts/target \ motoko-rts-tests/target \ - motoko-rts/xargo-home \ motoko-rts/cargo-home diff --git a/rts/README.md b/rts/README.md index 3e464e8b6fd..d1ba8ffc1df 100644 --- a/rts/README.md +++ b/rts/README.md @@ -58,31 +58,36 @@ See `motoko-rts/src/bigint.rs` for the technical details. Rust build ---------- -The Rust parts are built from `motoko-rts`, using `xargo` and `cargo`. +To build Motoko RTS in nix we need pre-fetch Rust dependencies. This works in +`nix-build` by: -To build this in nix, we need pre-fetch some dependencies (currently -`compiler_builtins` and `libc`). This works in `nix-build` by: - - * Adding`compiler_builtins` as a dependency in `Cargo.toml` (even though not - needed), so that it shows up with a hash in `Cargo.lock` - - * Building a directory with vendored sources in `default.nix` (see `rustDeps`) + * Building a directory with vendored sources in `default.nix` * Configuring `cargo` to use that vendored directory (see `preBuild`) If you change dependencies (e.g. bump versions, add more crates), - 1. Add them to `Cargo.toml` - 2. Make sure that `Cargo.lock` is up to date - 3. In `default.nix`, invalidate the `sha256` of `rustDeps` (e.g. change one + 1. Make sure that `motoko-rts-tests/Cargo.lock` is up to date. This can be + done by running `cargo build --target=wasm32-wasi` in `motoko-rts-tests/` + directory. + 2. In `default.nix`, invalidate the `sha256` of `rtsDeps` (e.g. change one character) - 4. Run `nix-build -A rts`. You should get an error message about the actual + 3. Run `nix-build -A rts`. You should get an error message about the actual checksum. - 5. Set that as `sha256` of `rustDeps` in `default.nix` + 4. Set that as `sha256` of `rtsDeps` in `default.nix` Warning: nix will happily use a stale version of the dependencies if you do not do step 3. +**Updating rustc**: + +1. Update Rust version in `nix/default.nix`, in the line with + `moz_overlay.rustChannelOf { ... }`. +2. Invalidate `rustStdDepsHash` in `default.nix`. +3. Run `nix-build -A rts`. You should get an error message about the expected + value of `rustStdDepsHash`. +4. Update `rustStdDepsHash` with the expected value in the error message. + Running RTS tests ----------------- diff --git a/rts/cargo-vendor-tools/Cargo.lock b/rts/cargo-vendor-tools/Cargo.lock new file mode 100644 index 00000000000..e372c4c585a --- /dev/null +++ b/rts/cargo-vendor-tools/Cargo.lock @@ -0,0 +1,230 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "ansi_term" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +dependencies = [ + "winapi", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array", +] + +[[package]] +name = "cargo-vendor-tools" +version = "0.1.0" +dependencies = [ + "clap", + "serde_json", + "sha2", + "toml", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "clap" +version = "2.33.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" +dependencies = [ + "ansi_term", + "atty", + "bitflags", + "strsim", + "textwrap", + "unicode-width", + "vec_map", +] + +[[package]] +name = "cpufeatures" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469" +dependencies = [ + "libc", +] + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + +[[package]] +name = "generic-array" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "itoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" + +[[package]] +name = "libc" +version = "0.2.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "869d572136620d55835903746bcb5cdc54cb2851fd0aeec53220b4bb65ef3013" + +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "serde" +version = "1.0.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f12d06de37cf59146fbdecab66aa99f9fe4f78722e3607577a5375d66bd0c913" + +[[package]] +name = "serde_json" +version = "1.0.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f690853975602e1bfe1ccbf50504d67174e3bcf340f23b5ea9992e0587a52d8" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b69f9a4c9740d74c5baa3fd2e547f9525fa8088a8a958e0ca2409a514e33f5fa" +dependencies = [ + "block-buffer", + "cfg-if", + "cpufeatures", + "digest", + "opaque-debug", +] + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "toml" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +dependencies = [ + "serde", +] + +[[package]] +name = "typenum" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec" + +[[package]] +name = "unicode-width" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "version_check" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/rts/cargo-vendor-tools/Cargo.toml b/rts/cargo-vendor-tools/Cargo.toml new file mode 100644 index 00000000000..7b63e2002cc --- /dev/null +++ b/rts/cargo-vendor-tools/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "cargo-vendor-tools" +version = "0.1.0" +authors = ["dfinity "] +edition = "2018" + +[[bin]] +name = "vendor-rust-std-deps" +path = "src/vendor_rust_std_deps.rs" + +[dependencies] +clap = "2.33.3" +serde_json = "1.0.64" +sha2 = "0.9.5" +toml = "0.5.8" diff --git a/rts/cargo-vendor-tools/rustfmt.toml b/rts/cargo-vendor-tools/rustfmt.toml new file mode 100644 index 00000000000..66aa257537b --- /dev/null +++ b/rts/cargo-vendor-tools/rustfmt.toml @@ -0,0 +1 @@ +struct_lit_width = 40 diff --git a/rts/cargo-vendor-tools/src/vendor_rust_std_deps.rs b/rts/cargo-vendor-tools/src/vendor_rust_std_deps.rs new file mode 100644 index 00000000000..3c3bf7e7848 --- /dev/null +++ b/rts/cargo-vendor-tools/src/vendor_rust_std_deps.rs @@ -0,0 +1,235 @@ +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output, Stdio}; + +use clap::{App, Arg}; +use serde_json as json; +use sha2::{Digest, Sha256}; + +static DESCR: &str = " +Given a Rust toolchain path and a vendor directory, this program vendors Rust std dependencies in +the given vendor directory. This directory can then be used to build packages with `-Zbuild-std` +parameters in nix. +"; + +fn main() { + let args = App::new("vendor-rust-std-deps") + .long_about(DESCR) + .arg( + Arg::with_name("rust_install_path") + .value_name("RUST_INSTALL_PATH") + .takes_value(true) + .required(true), + ) + .arg( + Arg::with_name("vendor_dir") + .value_name("VENDOR_DIR") + .takes_value(true) + .required(true), + ) + .get_matches(); + + let rust_install_path = args.value_of("rust_install_path").unwrap(); + let vendor_dir = args.value_of("vendor_dir").unwrap(); + + let compiler_cargo_lock_path = format!( + "{}/{}", + rust_install_path, "lib/rustlib/src/rust/Cargo.lock" + ); + + println!("Reading compiler Cargo.lock ..."); + let compiler_deps = cargo_lock_deps(&compiler_cargo_lock_path); + + println!( + "Fetching and unpacking compiler dependencies in {} ...", + vendor_dir + ); + + let total_deps = compiler_deps.len(); + for (dep_idx, Dep { url, name, tarball_checksum }) in compiler_deps.into_iter().enumerate() { + println!(" {} ({}/{})", name, dep_idx + 1, total_deps); + + // + // Fetch the package tarball + // + + // curl --location + let Output { status, stdout: curl_stdout, stderr } = Command::new("curl") + .args(&["--location", &url, "--insecure"]) + .output() + .unwrap(); + + if !status.success() { + panic!( + "`curl --location {}` returned {:?}:\n{}", + url, + status.code(), + String::from_utf8_lossy(&stderr) + ); + } + + let package_vendor_dir = format!("{}/{}", vendor_dir, name); + + // + // Create package vendor dir + // + + std::fs::create_dir_all(&package_vendor_dir).unwrap(); + + // + // Unpack package + // + + let mut tar_cmd = Command::new("tar") + .args(&["xz"]) + .current_dir(&vendor_dir) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + + let mut tar_stdin = tar_cmd.stdin.take().unwrap(); + tar_stdin.write_all(&curl_stdout).unwrap(); + drop(tar_stdin); + + let Output { status, stdout: _, stderr } = tar_cmd.wait_with_output().unwrap(); + + if !status.success() { + panic!( + "`tar xz` returned {:?}:\n{}", + status.code(), + String::from_utf8_lossy(&stderr) + ); + } + + // + // Generate file checksums + // + + let mut file_checksums: json::map::Map = Default::default(); + generate_file_checksums( + &package_vendor_dir, + PathBuf::from(&package_vendor_dir).as_ref(), + &mut file_checksums, + ); + + let mut json_map = json::map::Map::new(); + json_map.insert("files".to_owned(), json::Value::Object(file_checksums)); + json_map.insert( + "package".to_owned(), + json::Value::String(tarball_checksum.to_owned()), + ); + + let mut checksum_file = + std::fs::File::create(format!("{}/.cargo-checksum.json", package_vendor_dir)).unwrap(); + json::ser::to_writer_pretty(&mut checksum_file, &json_map).unwrap(); + } +} + +#[derive(Debug)] +struct Dep { + /// URL to fetch the package tarball + url: String, + + /// - + name: String, + + /// Expected checksum (SHA256) of the tarball + tarball_checksum: String, +} + +fn cargo_lock_deps(lock_file_path: &str) -> Vec { + let mut deps = vec![]; + + let lock_file = std::fs::read_to_string(lock_file_path).unwrap(); + + let toml_value: toml::value::Value = toml::de::from_str(&lock_file).unwrap(); + + let packages = toml_value.as_table().unwrap().get("package").unwrap(); + + for package in packages.as_array().unwrap() { + let package_tbl = package.as_table().unwrap(); + if let Some(source) = package_tbl.get("source") { + if source.as_str().unwrap() == "registry+https://github.com/rust-lang/crates.io-index" { + let name = package_tbl.get("name").unwrap().as_str().unwrap(); + let version = package_tbl.get("version").unwrap().as_str().unwrap(); + let checksum = package_tbl.get("checksum").unwrap().as_str().unwrap(); + deps.push(Dep { + url: format!( + "https://crates.io/api/v1/crates/{}/{}/download", + name, version + ), + name: format!("{}-{}", name, version), + tarball_checksum: checksum.to_owned(), + }); + } + } + } + + deps +} + +static IGNORED_FILES: [&str; 7] = [ + ".", + "..", + ".gitattributes", + ".gitignore", + ".cargo-ok", + ".cargo-checksum.json", + ".cargo_vcs_info.json", +]; + +fn generate_file_checksums( + root: &str, + dir: &Path, + file_checksums: &mut json::map::Map, +) { + for entry in std::fs::read_dir(dir).unwrap() { + let entry = entry.unwrap(); + let file_name = entry.file_name(); + + if IGNORED_FILES + .iter() + .any(|ignored_file| *ignored_file == file_name) + { + continue; + } + + match entry.file_type() { + Err(err) => { + eprintln!( + "Error while getting type of file {:?}: {:?}", + file_name, err + ); + } + Ok(file_type) => { + if file_type.is_dir() { + let mut dir_path = PathBuf::new(); + dir_path.push(dir); + dir_path.push(entry.file_name()); + generate_file_checksums(root, &dir_path, file_checksums); + } else { + let mut file_path = PathBuf::new(); + file_path.push(dir); + file_path.push(entry.file_name()); + + let rel_file_path = file_path.strip_prefix(root).unwrap(); + + let file_contents = std::fs::read(&file_path).unwrap(); + + let mut hasher = Sha256::new(); + hasher.update(&file_contents); + let hash = hasher.finalize(); + let hash_str = format!("{:x}", hash); + + let old = file_checksums.insert( + rel_file_path.to_string_lossy().to_string(), + json::Value::String(hash_str), + ); + assert!(old.is_none()); + } + } + } + } +} diff --git a/rts/motoko-rts-tests/Cargo.lock b/rts/motoko-rts-tests/Cargo.lock index 6b60a63f4e4..834253a1868 100644 --- a/rts/motoko-rts-tests/Cargo.lock +++ b/rts/motoko-rts-tests/Cargo.lock @@ -1,5 +1,7 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + [[package]] name = "autocfg" version = "1.0.1" @@ -24,12 +26,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" -[[package]] -name = "compiler_builtins" -version = "0.1.39" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3748f82c7d366a0b4950257d19db685d4958d2fa27c6d164a3f069fec42b748b" - [[package]] name = "fxhash" version = "0.2.1" @@ -66,7 +62,6 @@ checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb" name = "motoko-rts" version = "0.1.0" dependencies = [ - "compiler_builtins", "libc", "motoko-rts-macros", "static_assertions", diff --git a/rts/motoko-rts-tests/src/main.rs b/rts/motoko-rts-tests/src/main.rs index f18df6ac569..9f77525e034 100644 --- a/rts/motoko-rts-tests/src/main.rs +++ b/rts/motoko-rts-tests/src/main.rs @@ -1,4 +1,4 @@ -#![feature(ptr_offset_from, map_first_last, clamp)] +#![feature(map_first_last)] mod bigint; mod bitmap; diff --git a/rts/motoko-rts/Cargo.lock b/rts/motoko-rts/Cargo.lock index 1e4ffb001c7..dc5205045cf 100644 --- a/rts/motoko-rts/Cargo.lock +++ b/rts/motoko-rts/Cargo.lock @@ -1,10 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -[[package]] -name = "compiler_builtins" -version = "0.1.39" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3748f82c7d366a0b4950257d19db685d4958d2fa27c6d164a3f069fec42b748b" +version = 3 [[package]] name = "libc" @@ -16,7 +12,6 @@ checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb" name = "motoko-rts" version = "0.1.0" dependencies = [ - "compiler_builtins", "libc", "motoko-rts-macros", "static_assertions", diff --git a/rts/motoko-rts/Cargo.toml b/rts/motoko-rts/Cargo.toml index 575b327b100..b7eaa74f564 100644 --- a/rts/motoko-rts/Cargo.toml +++ b/rts/motoko-rts/Cargo.toml @@ -23,20 +23,6 @@ libc = { version = "0.2.81", default_features = false } motoko-rts-macros = { path = "../motoko-rts-macros" } static_assertions = "1.1.0" -# Added here so that it ends up in Cargo.lock, so that nix will pre-fetch it -[dependencies.compiler_builtins] -version = "0.1.39" -# Without this feature we get dozens of duplicate symbol errors when generating -# the final shared .wasm file: -# -# wasm-ld: error: duplicate symbol: __multi3 -# >>> defined in _build/wasm/libmotoko_rts.a(compiler_builtins-d709bd899857aa61.compiler_builtins.3abndchk-cgu.0.rcgu.o) -# >>> defined in _build/wasm/libmotoko_rts.a(compiler_builtins-06d1ead628e1f468.compiler_builtins.6moz1ltd-cgu.0.rcgu.o) -# -# It seems like we're linking multiple versions of compiler_builtins in the same -# shared library, which we should fix at some point. TODO -features = ["mangled-names"] - [profile.dev] panic = "abort" diff --git a/rts/motoko-rts/Xargo.toml b/rts/motoko-rts/Xargo.toml deleted file mode 100644 index d3910abede1..00000000000 --- a/rts/motoko-rts/Xargo.toml +++ /dev/null @@ -1,6 +0,0 @@ -[dependencies.core] -stage = 0 - -[dependencies.compiler_builtins] -stage = 1 -version = "0.1.39" diff --git a/rts/motoko-rts/native/Cargo.toml b/rts/motoko-rts/native/Cargo.toml index 9c8913e32ff..5e22f655e31 100644 --- a/rts/motoko-rts/native/Cargo.toml +++ b/rts/motoko-rts/native/Cargo.toml @@ -13,10 +13,6 @@ libc = { version = "0.2.73", default_features = false } motoko-rts-macros = { path = "../../motoko-rts-macros" } static_assertions = "1.1.0" -[dependencies.compiler_builtins] -version = "0.1.39" -features = ["mangled-names"] - [profile.dev] panic = "abort" diff --git a/rts/motoko-rts/src/lib.rs b/rts/motoko-rts/src/lib.rs index df68ab06b74..e036b0d3096 100644 --- a/rts/motoko-rts/src/lib.rs +++ b/rts/motoko-rts/src/lib.rs @@ -2,13 +2,7 @@ #![no_std] // TODO (osa): Some of these are stabilized, we need to update rustc -#![feature( - arbitrary_self_types, - assoc_char_funcs, - core_intrinsics, - panic_info_message, - ptr_offset_from -)] +#![feature(arbitrary_self_types, core_intrinsics, panic_info_message)] #[macro_use] mod print;