diff --git a/Cargo.lock b/Cargo.lock index c919f7beaa07a8..6a76c30c4a8d02 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -369,6 +369,9 @@ name = "cc" version = "1.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22a9137b95ea06864e018375b72adfb7db6e6f68cfc8df5a04d00288050485ee" +dependencies = [ + "jobserver", +] [[package]] name = "cfg-if" @@ -744,6 +747,7 @@ dependencies = [ "walkdir", "winapi 0.3.9", "winres", + "zstd", ] [[package]] @@ -979,6 +983,7 @@ dependencies = [ "hyper", "libc", "log", + "lzzzz", "netif", "nix", "notify", @@ -1985,6 +1990,15 @@ version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" +[[package]] +name = "jobserver" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.49" @@ -2212,6 +2226,15 @@ dependencies = [ "syn 1.0.65", ] +[[package]] +name = "lzzzz" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6d891cedd3b1659c206a60ff8afd15bccd7c2754b157f8a164861989e042b88" +dependencies = [ + "cc", +] + [[package]] name = "malloc_buf" version = "0.0.6" @@ -5079,3 +5102,32 @@ dependencies = [ "syn 1.0.65", "synstructure", ] + +[[package]] +name = "zstd" +version = "0.9.2+zstd.1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2390ea1bf6c038c39674f22d95f0564725fc06034a47129179810b2fc58caa54" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "4.1.3+zstd.1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e99d81b99fb3c2c2c794e3fe56c305c63d5173a16a46b5850b07c935ffc7db79" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "1.6.2+zstd.1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2daf2f248d9ea44454bfcb2516534e8b8ad2fc91bf818a1885495fc42bc8ac9f" +dependencies = [ + "cc", + "libc", +] diff --git a/Cargo.toml b/Cargo.toml index e8ee6b8ed50f6b..3f206b2232e3fe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -88,6 +88,12 @@ opt-level = 3 opt-level = 3 [profile.bench.package.tokio] opt-level = 3 +[profile.bench.package.zstd] +opt-level = 3 +[profile.bench.package.lzzzz] +opt-level = 3 +[profile.bench.package.zstd-sys] +opt-level = 3 # NB: the `bench` and `release` profiles must remain EXACTLY the same. [profile.release.package.rand] @@ -130,3 +136,9 @@ opt-level = 3 opt-level = 3 [profile.release.package.tokio] opt-level = 3 +[profile.release.package.zstd] +opt-level = 3 +[profile.release.package.lzzzz] +opt-level = 3 +[profile.release.package.zstd-sys] +opt-level = 3 diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 15a82b5eb3e595..7fd34e2f177e83 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -33,6 +33,7 @@ deno_websocket = { version = "0.36.0", path = "../ext/websocket" } deno_webstorage = { version = "0.26.0", path = "../ext/webstorage" } regex = "=1.5.4" serde = { version = "=1.0.133", features = ["derive"] } +zstd = '=0.9.2' [target.'cfg(windows)'.build-dependencies] winapi = "=0.3.9" @@ -85,6 +86,7 @@ text_lines = "=0.4.1" tokio = { version = "=1.14", features = ["full"] } uuid = { version = "=0.8.2", features = ["v4", "serde"] } walkdir = "=2.3.2" +zstd = '=0.9.2' [target.'cfg(windows)'.dependencies] fwdansi = "=1.1.0" diff --git a/cli/build.rs b/cli/build.rs index 48c55e628d6c31..87c68848044150 100644 --- a/cli/build.rs +++ b/cli/build.rs @@ -38,7 +38,30 @@ fn create_snapshot( let snapshot = js_runtime.snapshot(); let snapshot_slice: &[u8] = &*snapshot; println!("Snapshot size: {}", snapshot_slice.len()); - std::fs::write(&snapshot_path, snapshot_slice).unwrap(); + + let compressed_snapshot_with_size = { + let mut vec = vec![]; + + vec.extend_from_slice( + &u32::try_from(snapshot.len()) + .expect("snapshot larger than 4gb") + .to_le_bytes(), + ); + + vec.extend_from_slice( + &zstd::block::compress(snapshot_slice, 22) + .expect("snapshot compression failed"), + ); + + vec + }; + + println!( + "Snapshot compressed size: {}", + compressed_snapshot_with_size.len() + ); + + std::fs::write(&snapshot_path, compressed_snapshot_with_size).unwrap(); println!("Snapshot written to: {} ", snapshot_path.display()); } diff --git a/cli/tsc.rs b/cli/tsc.rs index 3dfb975880c480..a2265e36f1459d 100644 --- a/cli/tsc.rs +++ b/cli/tsc.rs @@ -51,11 +51,25 @@ pub static SHARED_GLOBALS_LIB: &str = pub static WINDOW_LIB: &str = include_str!("dts/lib.deno.window.d.ts"); pub static UNSTABLE_NS_LIB: &str = include_str!("dts/lib.deno.unstable.d.ts"); -pub static COMPILER_SNAPSHOT: &[u8] = - include_bytes!(concat!(env!("OUT_DIR"), "/COMPILER_SNAPSHOT.bin")); +pub static COMPILER_SNAPSHOT: Lazy> = Lazy::new( + #[cold] + #[inline(never)] + || { + static COMPRESSED_COMPILER_SNAPSHOT: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/COMPILER_SNAPSHOT.bin")); + + zstd::block::decompress( + &COMPRESSED_COMPILER_SNAPSHOT[4..], + u32::from_le_bytes(COMPRESSED_COMPILER_SNAPSHOT[0..4].try_into().unwrap()) + as usize, + ) + .unwrap() + .into_boxed_slice() + }, +); pub fn compiler_snapshot() -> Snapshot { - Snapshot::Static(COMPILER_SNAPSHOT) + Snapshot::Static(&*COMPILER_SNAPSHOT) } macro_rules! inc { diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 67fc6a82d70610..8c88ce2e31da49 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -39,6 +39,8 @@ deno_webidl = { version = "0.31.0", path = "../ext/webidl" } deno_websocket = { version = "0.36.0", path = "../ext/websocket" } deno_webstorage = { version = "0.26.0", path = "../ext/webstorage" } +lzzzz = '=0.8.0' + [target.'cfg(windows)'.build-dependencies] winres = "0.1.11" winapi = "0.3.9" @@ -70,6 +72,7 @@ http = "0.2.4" hyper = { version = "0.14.12", features = ["server", "stream", "http1", "http2", "runtime"] } libc = "0.2.106" log = "0.4.14" +lzzzz = '=0.8.0' netif = "0.1.0" notify = "=5.0.0-pre.12" once_cell = "=1.9.0" diff --git a/runtime/build.rs b/runtime/build.rs index d1e8517b8c1b0e..e2fe21b9e78892 100644 --- a/runtime/build.rs +++ b/runtime/build.rs @@ -37,7 +37,32 @@ mod not_docs { let snapshot = js_runtime.snapshot(); let snapshot_slice: &[u8] = &*snapshot; println!("Snapshot size: {}", snapshot_slice.len()); - std::fs::write(&snapshot_path, snapshot_slice).unwrap(); + + let compressed_snapshot_with_size = { + let mut vec = vec![]; + + vec.extend_from_slice( + &u32::try_from(snapshot.len()) + .expect("snapshot larger than 4gb") + .to_le_bytes(), + ); + + lzzzz::lz4_hc::compress_to_vec( + snapshot_slice, + &mut vec, + lzzzz::lz4_hc::CLEVEL_MAX, + ) + .expect("snapshot compression failed"); + + vec + }; + + println!( + "Snapshot compressed size: {}", + compressed_snapshot_with_size.len() + ); + + std::fs::write(&snapshot_path, compressed_snapshot_with_size).unwrap(); println!("Snapshot written to: {} ", snapshot_path.display()); } diff --git a/runtime/js.rs b/runtime/js.rs index 58053ac02fca6d..c259ff5eec9b8e 100644 --- a/runtime/js.rs +++ b/runtime/js.rs @@ -1,14 +1,35 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. use deno_core::Snapshot; use log::debug; +use once_cell::sync::Lazy; -pub static CLI_SNAPSHOT: &[u8] = - include_bytes!(concat!(env!("OUT_DIR"), "/CLI_SNAPSHOT.bin")); +pub static CLI_SNAPSHOT: Lazy> = Lazy::new( + #[cold] + #[inline(never)] + || { + static COMPRESSED_CLI_SNAPSHOT: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/CLI_SNAPSHOT.bin")); + + let size = + u32::from_le_bytes(COMPRESSED_CLI_SNAPSHOT[0..4].try_into().unwrap()) + as usize; + let mut vec = Vec::with_capacity(size); + + // SAFETY: vec is allocated with exact snapshot size (+ alignment) + // SAFETY: non zeroed bytes are overwritten with decompressed snapshot + unsafe { + vec.set_len(size); + } + + lzzzz::lz4::decompress(&COMPRESSED_CLI_SNAPSHOT[4..], &mut vec).unwrap(); + + vec.into_boxed_slice() + }, +); pub fn deno_isolate_init() -> Snapshot { debug!("Deno isolate init with snapshots."); - let data = CLI_SNAPSHOT; - Snapshot::Static(data) + Snapshot::Static(&*CLI_SNAPSHOT) } #[cfg(test)]