diff --git a/.gitignore b/.gitignore index 32e703fb..b6c2687f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ .direnv /target -/vorpal-build -/vorpal-cli +/vorpal diff --git a/Cargo.lock b/Cargo.lock index 3c5597e5..e6cd74f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -38,6 +38,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "anstream" version = "0.6.14" @@ -93,6 +108,26 @@ version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +[[package]] +name = "async-compression" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd066d0b4ef8ecb03a55319dc13aa6910616d0f44008a045bb1835af830abff5" +dependencies = [ + "brotli", + "bzip2", + "deflate64", + "flate2", + "futures-core", + "futures-io", + "memchr", + "pin-project-lite", + "tokio", + "xz2", + "zstd", + "zstd-safe", +] + [[package]] name = "async-stream" version = "0.3.5" @@ -243,6 +278,27 @@ dependencies = [ "generic-array", ] +[[package]] +name = "brotli" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bstr" version = "1.9.1" @@ -271,6 +327,27 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "cc" version = "1.0.99" @@ -409,6 +486,12 @@ dependencies = [ "typenum", ] +[[package]] +name = "deflate64" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83ace6c86376be0b6cdcf3fb41882e81d94b31587573d1cfa9d01cd06bba210d" + [[package]] name = "der" version = "0.7.9" @@ -559,6 +642,21 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.30" @@ -566,6 +664,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -574,6 +673,23 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +[[package]] +name = "futures-executor" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" + [[package]] name = "futures-macro" version = "0.3.30" @@ -603,9 +719,13 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ + "futures-channel", "futures-core", + "futures-io", "futures-macro", + "futures-sink", "futures-task", + "memchr", "pin-project-lite", "pin-utils", "slab", @@ -1118,6 +1238,17 @@ version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +[[package]] +name = "lzma-sys" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "matchit" version = "0.7.3" @@ -1494,6 +1625,20 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "process-stream" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c061a7af7d7370dfe98c231848492686b574a65d245a2e4239d49dcee878ad7" +dependencies = [ + "async-stream", + "async-trait", + "futures", + "tap", + "tokio", + "tokio-stream", +] + [[package]] name = "prost" version = "0.12.6" @@ -1586,6 +1731,15 @@ dependencies = [ "getrandom", ] +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -2002,15 +2156,10 @@ dependencies = [ ] [[package]] -name = "tar" -version = "0.4.41" +name = "tap" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb797dad5fb5b76fcf519e702f4a589483b5ef06567f160c392832c1f5e44909" -dependencies = [ - "filetime", - "libc", - "xattr", -] +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" @@ -2146,6 +2295,21 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-tar" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5714c010ca3e5c27114c1cdeb9d14641ace49874aa5626d7149e47aedace75" +dependencies = [ + "filetime", + "futures-core", + "libc", + "redox_syscall 0.3.5", + "tokio", + "tokio-stream", + "xattr", +] + [[package]] name = "tokio-util" version = "0.7.11" @@ -2427,6 +2591,7 @@ name = "vorpal" version = "0.1.0" dependencies = [ "anyhow", + "async-compression", "clap", "dirs", "flate2", @@ -2434,17 +2599,17 @@ dependencies = [ "git2", "hex", "infer", + "process-stream", "prost", "rand", "reqwest", "rsa", "rusqlite", "sha256", - "tar", - "tempfile", "tera", "tokio", "tokio-stream", + "tokio-tar", "tonic", "tonic-build", "tracing", @@ -2746,6 +2911,15 @@ dependencies = [ "rustix", ] +[[package]] +name = "xz2" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +dependencies = [ + "lzma-sys", +] + [[package]] name = "zerocopy" version = "0.7.34" @@ -2771,3 +2945,31 @@ name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + +[[package]] +name = "zstd" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.10+zstd.1.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index d33ff4fc..607366b7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ path = "src/bin/main.rs" [dependencies] anyhow = "1.0" +async-compression = { version = "0.4", features = ["all"] } clap = { version = "4.5", features = ["derive"] } dirs = "5.0" flate2 = "1.0" @@ -16,17 +17,17 @@ futures-util = "0.3" git2 = "0.18" hex = "0.4" infer = "0.3" +process-stream = "0.3" prost = "0.12" rand = "0.8" rsa = { version = "0.9", features = ["sha2"] } rusqlite = { version = "0.31", features = ["bundled"] } sha256 = "1.5" reqwest = { version = "0.12", features = ["json"] } -tar = "0.4" -tempfile = "3.10" tera = { version = "1", default-features = false } tokio = { version = "1.37", features = ["full"] } tokio-stream = "0.1" +tokio-tar = "0.3" tonic = "0.11" url = "2.5" uuid = { version = "1.8", features = ["v7"] } diff --git a/example/rust/Cargo.lock b/example/rust/Cargo.lock index f3d23b59..6e3652a7 100644 --- a/example/rust/Cargo.lock +++ b/example/rust/Cargo.lock @@ -38,6 +38,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "anstream" version = "0.6.14" @@ -70,9 +85,9 @@ dependencies = [ [[package]] name = "anstyle-query" -version = "1.0.3" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a64c907d4e79225ac72e2a354c9ce84d50ebb4586dee56c82b3ee73004f537f5" +checksum = "ad186efb764318d35165f1758e7dcef3b10628e26d41a44bc5550652e6804391" dependencies = [ "windows-sys 0.52.0", ] @@ -93,6 +108,26 @@ version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +[[package]] +name = "async-compression" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd066d0b4ef8ecb03a55319dc13aa6910616d0f44008a045bb1835af830abff5" +dependencies = [ + "brotli", + "bzip2", + "deflate64", + "flate2", + "futures-core", + "futures-io", + "memchr", + "pin-project-lite", + "tokio", + "xz2", + "zstd", + "zstd-safe", +] + [[package]] name = "async-stream" version = "0.3.5" @@ -157,7 +192,7 @@ dependencies = [ "futures-util", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.28", + "hyper 0.14.29", "itoa", "matchit", "memchr", @@ -243,6 +278,27 @@ dependencies = [ "generic-array", ] +[[package]] +name = "brotli" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bstr" version = "1.9.1" @@ -271,11 +327,32 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "cc" -version = "1.0.98" +version = "1.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f" +checksum = "96c51067fd44124faa7f870b4b1c969379ad32b2ba805aa959430ceaa384f695" dependencies = [ "jobserver", "libc", @@ -290,9 +367,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "4.5.4" +version = "4.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" +checksum = "a9689a29b593160de5bc4aacab7b5d54fb52231de70122626c178e6a368994c7" dependencies = [ "clap_builder", "clap_derive", @@ -300,9 +377,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.2" +version = "4.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" +checksum = "2e5387378c84f6faa26890ebf9f0a92989f8873d4d380467bcd0d8d8620424df" dependencies = [ "anstream", "anstyle", @@ -312,9 +389,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.4" +version = "4.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" +checksum = "c780290ccf4fb26629baa7a1081e68ced113f1d3ec302fa5948f1c381ebf06c6" dependencies = [ "heck", "proc-macro2", @@ -324,9 +401,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" +checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" [[package]] name = "colorchoice" @@ -409,6 +486,12 @@ dependencies = [ "typenum", ] +[[package]] +name = "deflate64" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83ace6c86376be0b6cdcf3fb41882e81d94b31587573d1cfa9d01cd06bba210d" + [[package]] name = "der" version = "0.7.9" @@ -559,6 +642,21 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.30" @@ -566,6 +664,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -574,6 +673,23 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +[[package]] +name = "futures-executor" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" + [[package]] name = "futures-macro" version = "0.3.30" @@ -603,9 +719,13 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ + "futures-channel", "futures-core", + "futures-io", "futures-macro", + "futures-sink", "futures-task", + "memchr", "pin-project-lite", "pin-utils", "slab", @@ -827,9 +947,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "0.14.28" +version = "0.14.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +checksum = "f361cde2f109281a220d4307746cdfd5ee3f410da58a70377762396775634b33" dependencies = [ "bytes", "futures-channel", @@ -875,7 +995,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper 0.14.28", + "hyper 0.14.29", "pin-project-lite", "tokio", "tokio-io-timeout", @@ -1118,6 +1238,17 @@ version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +[[package]] +name = "lzma-sys" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "matchit" version = "0.7.3" @@ -1179,6 +1310,16 @@ dependencies = [ "tempfile", ] +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num-bigint-dig" version = "0.8.4" @@ -1301,6 +1442,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + [[package]] name = "parking_lot" version = "0.12.3" @@ -1478,6 +1625,20 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "process-stream" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c061a7af7d7370dfe98c231848492686b574a65d245a2e4239d49dcee878ad7" +dependencies = [ + "async-stream", + "async-trait", + "futures", + "tap", + "tokio", + "tokio-stream", +] + [[package]] name = "prost" version = "0.12.6" @@ -1570,6 +1731,15 @@ dependencies = [ "getrandom", ] +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -1866,6 +2036,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "signal-hook-registry" version = "1.4.2" @@ -1977,15 +2156,10 @@ dependencies = [ ] [[package]] -name = "tar" -version = "0.4.40" +name = "tap" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b16afcea1f22891c49a00c751c7b63b2233284064f11a200fc624137c51e2ddb" -dependencies = [ - "filetime", - "libc", - "xattr", -] +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" @@ -2035,6 +2209,16 @@ dependencies = [ "syn", ] +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if", + "once_cell", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -2111,6 +2295,21 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-tar" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5714c010ca3e5c27114c1cdeb9d14641ace49874aa5626d7149e47aedace75" +dependencies = [ + "filetime", + "futures-core", + "libc", + "redox_syscall 0.3.5", + "tokio", + "tokio-stream", + "xattr", +] + [[package]] name = "tokio-util" version = "0.7.11" @@ -2138,7 +2337,7 @@ dependencies = [ "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.28", + "hyper 0.14.29", "hyper-timeout", "percent-encoding", "pin-project", @@ -2225,6 +2424,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +dependencies = [ + "nu-ansi-term", + "sharded-slab", + "smallvec", + "thread_local", + "tracing-core", + "tracing-log", ] [[package]] @@ -2329,9 +2554,9 @@ dependencies = [ [[package]] name = "utf8parse" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" @@ -2343,6 +2568,12 @@ dependencies = [ "getrandom", ] +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + [[package]] name = "vcpkg" version = "0.2.15" @@ -2360,6 +2591,7 @@ name = "vorpal" version = "0.1.0" dependencies = [ "anyhow", + "async-compression", "clap", "dirs", "flate2", @@ -2367,19 +2599,21 @@ dependencies = [ "git2", "hex", "infer", + "process-stream", "prost", "rand", "reqwest", "rsa", "rusqlite", "sha256", - "tar", - "tempfile", "tera", "tokio", "tokio-stream", + "tokio-tar", "tonic", "tonic-build", + "tracing", + "tracing-subscriber", "url", "uuid", "walkdir", @@ -2495,6 +2729,22 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.8" @@ -2504,6 +2754,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-sys" version = "0.48.0" @@ -2664,6 +2920,15 @@ dependencies = [ "rustix", ] +[[package]] +name = "xz2" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +dependencies = [ + "lzma-sys", +] + [[package]] name = "zerocopy" version = "0.7.34" @@ -2689,3 +2954,31 @@ name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + +[[package]] +name = "zstd" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.10+zstd.1.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/example/rust/vorpal.rs b/example/rust/vorpal.rs index afc281e7..ff02a7d8 100644 --- a/example/rust/vorpal.rs +++ b/example/rust/vorpal.rs @@ -1,5 +1,4 @@ use anyhow::Result; -use std::env; use vorpal::api::build_service_client::BuildServiceClient; use vorpal::api::{PackageRequest, PackageSource, PackageSourceKind}; @@ -7,31 +6,29 @@ use vorpal::api::{PackageRequest, PackageSource, PackageSourceKind}; pub async fn main() -> Result<(), anyhow::Error> { let mut client = BuildServiceClient::connect("http://[::1]:23151").await?; - let example_rust = client + client .package(PackageRequest { build_deps: Vec::new(), build_phase: r#" - cat vorpal.rs - "# - .to_string(), - install_phase: r#" - mkdir -p $OUTPUT - cp vorpal.rs $OUTPUT/build.rs + cd coreutils-9.5 + test -f configure || ./bootstrap + ./configure --prefix=$OUTPUT + make "# .to_string(), + install_phase: "make install".to_string(), install_deps: Vec::new(), - name: "example-rust".to_string(), + name: "coreutils".to_string(), source: Some(PackageSource { - hash: None, - ignore_paths: vec![".git".to_string(), "target".to_string()], - kind: PackageSourceKind::Local.into(), - uri: env::current_dir()?.to_string_lossy().to_string(), + hash: Some( + "af6d643afd6241ec35c7781b7f999b97a66c84bea4710ad2bb15e75a5caf11b4".to_string(), + ), + ignore_paths: vec![], + kind: PackageSourceKind::Http.into(), + uri: "https://ftp.gnu.org/gnu/coreutils/coreutils-9.5.tar.gz".to_string(), }), }) - .await? - .into_inner(); - - println!("{:?}", example_rust); + .await?; Ok(()) } diff --git a/flake.nix b/flake.nix index 34672474..75147030 100644 --- a/flake.nix +++ b/flake.nix @@ -34,7 +34,7 @@ packages = { default = buildRustPackage { buildInputs = [openssl] ++ lib.optionals pkgs.stdenv.isDarwin [CoreServices SystemConfiguration Security]; - cargoSha256 = "sha256-FliwadZfdF0+O3BRTbaR1nks8SmIDIi6I2dV9iyTZFw="; + cargoSha256 = "sha256-VrAN1joJAs7puM76oyHIByNxovhDrp6t2Xw2FiG5tT4="; checkPhase = '' ${pkgs.cargo}/bin/cargo clippy -- -D warnings ${pkgs.rust-bin.nightly.latest.default}/bin/cargo fmt --check --verbose diff --git a/justfile b/justfile index c1a594ad..61667253 100644 --- a/justfile +++ b/justfile @@ -1,32 +1,24 @@ _default: just --list -_start profile: - nix run '.#{{ profile }}' - -build profile="default": - nix build \ - --json \ - --no-link \ - --print-build-logs \ - '.#{{ profile }}' +build: + nix build --json --no-link --print-build-logs . check: nix flake check -start: - just _start "vorpal-build" +clean: + rm -rf ~/.vorpal + mkdir -p ~/.vorpal -start-build: - just _start "vorpal-build" +start: clean + nix run ".#start-dev" -package profile="default": +package: #!/usr/bin/env bash set -euxo pipefail - DERIVATION=$(just _build "{{ profile }}") - OUTPUT=$(echo $DERIVATION | jq -r .[0].outputs.out) - install --mode 755 $OUTPUT/bin/vorpal-build . - install --mode 755 $OUTPUT/bin/vorpal-cli . + OUTPUT=$(just build | jq -r .[0].outputs.out) + install --mode 755 $OUTPUT/bin/vorpal . update: nix flake update diff --git a/src/service/build/mod.rs b/src/service/build/mod.rs index 0529a0b1..ef58a93a 100644 --- a/src/service/build/mod.rs +++ b/src/service/build/mod.rs @@ -30,7 +30,7 @@ pub async fn start(port: u16) -> Result<(), anyhow::Error> { let private_key_path = store::get_private_key_path(); let public_key_path = store::get_public_key_path(); if !private_key_path.exists() && !public_key_path.exists() { - let key_dir = store::get_private_key_path(); + let key_dir = store::get_key_dir_path(); fs::create_dir_all(&key_dir).await?; info!("Key directory: {:?}", key_dir); notary::generate_keys()?; @@ -54,7 +54,7 @@ pub async fn start(port: u16) -> Result<(), anyhow::Error> { info!("Database path: {:?}", db_path.display()); if let Err(e) = db.close() { - eprintln!("Failed to close database: {:?}", e) + return Err(e.1.into()); } let addr = format!("[::1]:{}", port).parse()?; diff --git a/src/service/build/run_build.rs b/src/service/build/run_build.rs index ffe891dd..ce18a2ab 100644 --- a/src/service/build/run_build.rs +++ b/src/service/build/run_build.rs @@ -2,13 +2,12 @@ use crate::api::{BuildRequest, BuildResponse}; use crate::database; use crate::service::build::sandbox_default; use crate::store; +use process_stream::{Process, ProcessExt, StreamExt}; use std::os::unix::fs::PermissionsExt; -use tempfile::TempDir; use tera::Tera; use tokio::fs; -use tokio::process::Command; use tonic::{Request, Response, Status}; -use tracing::{error, info, span, Level}; +use tracing::{info, span, Level}; use walkdir::WalkDir; pub async fn run(request: Request) -> Result, Status> { @@ -22,15 +21,11 @@ pub async fn run(request: Request) -> Result) -> Result) -> Result) -> Result 0 { - error!("sandbox stderr: {}", sandbox_stderr); - return Err(Status::internal("Build failed")); + while let Some(output) = stream.next().await { + info!("{output}") } if sandbox_output_path.is_dir() { for entry in WalkDir::new(&sandbox_output_path) { - let entry = entry.map_err(|e| { - error!("Failed to walk sandbox output: {:?}", e); - Status::internal("Failed to walk sandbox output") - })?; + let entry = entry + .map_err(|e| Status::internal(format!("Failed to walk sandbox output: {:?}", e)))?; let output_path = entry.path().strip_prefix(&sandbox_output_path).unwrap(); let output_store_path = store_output_path.join(output_path); if entry.path().is_dir() { @@ -183,8 +173,14 @@ pub async fn run(request: Request) -> Result::new()) .map_err(|_| Status::internal("Failed to get sandbox output files"))?; - store::compress_files(&store_output_path, &store_output_tar, &store_output_files) - .map_err(|_| Status::internal("Failed to compress sandbox output"))?; + if let Err(err) = + store::compress_tar_gz(&store_output_path, &store_output_tar, &store_output_files).await + { + return Err(Status::internal(format!( + "Failed to compress sandbox output: {:?}", + err + ))); + } } else { fs::copy(&sandbox_output_path, &store_output_path).await?; } diff --git a/src/service/build/run_prepare.rs b/src/service/build/run_prepare.rs index 87473979..3c452bcd 100644 --- a/src/service/build/run_prepare.rs +++ b/src/service/build/run_prepare.rs @@ -6,12 +6,12 @@ use futures_util::StreamExt; use rsa::pss::{Signature, VerifyingKey}; use rsa::sha2::Sha256; use rsa::signature::Verifier; -use std::fs::File; -use std::io::Write; use std::os::unix::fs::PermissionsExt; use tokio::fs; +use tokio::fs::File; +use tokio::io::AsyncWriteExt; use tonic::{Response, Status, Streaming}; -use tracing::{error, info}; +use tracing::info; pub async fn run( mut stream: Streaming, @@ -75,68 +75,63 @@ pub async fn run( let source_tar_path = store::get_source_tar_path(&source_name, &source_hash); if !source_tar_path.exists() { - let mut source_tar = File::create(&source_tar_path)?; - if let Err(e) = source_tar.write_all(&source_data) { - error!("Failed source file: {}", e) + let mut source_tar = File::create(&source_tar_path).await?; + if let Err(e) = source_tar.write_all(&source_data).await { + return Err(Status::internal(format!( + "Failed to write source tar: {}", + e + ))); } else { let metadata = fs::metadata(&source_tar_path).await?; let mut permissions = metadata.permissions(); permissions.set_mode(0o444); - fs::set_permissions(&source_tar_path, permissions).await?; + fs::set_permissions(source_tar_path.clone(), permissions).await?; let file_name = source_tar_path.file_name().unwrap(); - info!("Source tar: {}", file_name.to_string_lossy()); + info!("source tar created: {}", file_name.to_string_lossy()); } fs::create_dir_all(&source_dir_path).await?; - store::unpack_source(&source_dir_path, &source_tar_path).map_err(|e| { - error!("Failed to unpack source: {:?}", e); - Status::internal("Failed to unpack source") - })?; + if let Err(err) = store::unpack_tar_gz(&source_dir_path, &source_tar_path).await { + return Err(Status::internal(format!( + "Failed to unpack source tar: {}", + err + ))); + } - let source_files = - store::get_file_paths(&source_dir_path, &Vec::<&str>::new()).map_err(|e| { - error!("Failed to get source files: {}", e); - Status::internal("Failed to get source files") - })?; + let source_files = store::get_file_paths(&source_dir_path, &Vec::<&str>::new()) + .map_err(|e| Status::internal(format!("Failed to get source files: {:?}", e)))?; info!("source files: {:?}", source_files); - let source_files_hashes = store::get_file_hashes(&source_files).map_err(|e| { - error!("Failed to get source files hashes: {}", e); - Status::internal("Failed to get source files hashes") - })?; + let source_files_hashes = store::get_file_hashes(&source_files) + .map_err(|e| Status::internal(format!("Failed to get source file hashes: {:?}", e)))?; - let source_hash_computed = store::get_source_hash(&source_files_hashes).map_err(|e| { - error!("Failed to get source hash: {}", e); - Status::internal("Failed to get source hash") - })?; + let source_hash_computed = store::get_source_hash(&source_files_hashes) + .map_err(|e| Status::internal(format!("Failed to get source hash: {:?}", e)))?; info!("message source hash: {}", source_hash); info!("computed source hash: {}", source_hash_computed); if source_hash != source_hash_computed { - error!("source hash mismatch"); - return Err(Status::internal("Source hash mismatch")); + return Err(Status::internal("source hash mismatch")); } - database::insert_source(&db, &source_hash, &source_name).map_err(|e| { - error!("Failed to insert source: {:?}", e); - Status::internal("Failed to insert source") - })?; + database::insert_source(&db, &source_hash, &source_name) + .map_err(|e| Status::internal(format!("Failed to insert source: {:?}", e)))?; fs::remove_dir_all(source_dir_path).await?; } let source_id = database::find_source(&db, &source_hash, &source_name) .map(|source| source.id) - .map_err(|e| { - error!("Failed to find source: {:?}", e); - Status::internal("Failed to find source") - })?; + .map_err(|e| Status::internal(format!("Failed to find source: {:?}", e.to_string())))?; if let Err(e) = db.close() { - error!("Failed to close database: {:?}", e) + return Err(Status::internal(format!( + "Failed to close database: {:?}", + e.1 + ))); } let response = PrepareResponse { source_id }; diff --git a/src/service/proxy/mod.rs b/src/service/proxy/mod.rs index ef39cb06..0c6e5554 100644 --- a/src/service/proxy/mod.rs +++ b/src/service/proxy/mod.rs @@ -38,7 +38,7 @@ pub async fn start(port: u16) -> Result<(), anyhow::Error> { let addr = format!("[::1]:{}", port).parse()?; let proxy = service::Proxy::default(); - println!("Proxy listening on: {}", addr); + info!("Proxy listening on: {}", addr); Server::builder() .add_service(BuildServiceServer::new(proxy)) diff --git a/src/service/proxy/package/mod.rs b/src/service/proxy/package/mod.rs index 56f4cbbd..de51a4dd 100644 --- a/src/service/proxy/package/mod.rs +++ b/src/service/proxy/package/mod.rs @@ -4,42 +4,43 @@ use crate::api::{ }; use crate::notary; use crate::store; +use async_compression::tokio::bufread::BzDecoder; use git2::build::RepoBuilder; use git2::{Cred, RemoteCallbacks}; use reqwest; use std::fs::Permissions; use std::os::unix::fs::PermissionsExt; use std::path::Path; -use tempfile::{tempdir, NamedTempFile}; use tokio::fs; -use tokio::fs::{copy, create_dir_all, read, set_permissions, write, File}; +use tokio::fs::{ + copy, create_dir_all, read, remove_dir_all, remove_file, set_permissions, write, File, +}; use tokio::io::AsyncWriteExt; use tokio_stream; +use tokio_tar::Archive; use tonic::{Request, Response, Status}; -use tracing::{error, info}; +use tracing::info; use url::Url; pub async fn run(request: Request) -> Result, Status> { let req = request.into_inner(); - let req_source = req.source.as_ref().ok_or_else(|| { - error!("Source is required"); - Status::invalid_argument("Source is required") - })?; + let req_source = req + .source + .as_ref() + .ok_or_else(|| Status::invalid_argument("source is required"))?; info!("Preparing: {}", req.name); - let (source_id, source_hash) = prepare(&req.name, req_source).await.map_err(|e| { - error!("Failed to prepare: {:?}", e); - Status::internal(e.to_string()) - })?; + let (source_id, source_hash) = prepare(&req.name, req_source) + .await + .map_err(|e| Status::internal(e.to_string()))?; info!("Building: {}-{}", req.name, source_hash); - build(source_id, &source_hash, &req).await.map_err(|e| { - error!("Failed to build: {:?}", e); - Status::internal(e.to_string()) - })?; + build(source_id, &source_hash, &req) + .await + .map_err(|e| Status::internal(format!("Failed to build package: {}", e)))?; let response = PackageResponse { source_id: source_id.to_string(), @@ -52,7 +53,7 @@ pub async fn run(request: Request) -> Result>( source_name: &str, source_hash: &str, - source_tar: P, + source_tar: &P, ) -> Result<(i32, String), anyhow::Error> { let data = read(source_tar).await?; @@ -61,7 +62,7 @@ async fn prepare_source>( info!("Source tar signature: {}", signature); let mut request_chunks = vec![]; - let request_chunks_size = 2 * 1024 * 1024; // default grpc limit + let request_chunks_size = 8192; // default grpc limit for chunk in data.chunks(request_chunks_size) { request_chunks.push(PrepareRequest { @@ -80,18 +81,19 @@ async fn prepare_source>( info!("Source ID: {}", res.source_id); + remove_file(&source_tar).await?; + Ok((res.source_id, source_hash.to_string())) } async fn prepare(name: &str, source: &PackageSource) -> Result<(i32, String), anyhow::Error> { - let work_dir = tempdir()?; - let workdir_path = work_dir.path().canonicalize()?; + let workdir = store::create_temp_dir().await?; + let workdir_path = workdir.canonicalize()?; info!("Preparing working dir: {:?}", workdir_path); if source.kind == PackageSourceKind::Unknown as i32 { - error!("Unknown source kind"); - return Err(anyhow::anyhow!("Unknown source kind")); + return Err(anyhow::anyhow!("unknown source kind")); } if source.kind == PackageSourceKind::Git as i32 { @@ -128,8 +130,7 @@ async fn prepare(name: &str, source: &PackageSource) -> Result<(i32, String), an let url = Url::parse(&source.uri)?; if url.scheme() != "http" && url.scheme() != "https" { - error!("Invalid HTTP source URL"); - return Err(anyhow::anyhow!("Invalid HTTP source URL")); + return Err(anyhow::anyhow!("invalid HTTP source URL")); } let response = reqwest::get(url.as_str()).await?.bytes().await?; @@ -139,10 +140,16 @@ async fn prepare(name: &str, source: &PackageSource) -> Result<(i32, String), an info!("Preparing source kind: {:?}", source_kind); if let "application/gzip" = source_kind.mime_type() { - let temp_file = NamedTempFile::new()?; + let temp_file = store::create_temp_file("tar.gz").await?; write(&temp_file, response_bytes).await?; - store::unpack_source(&workdir_path, temp_file.path())?; - info!("Prepared packed source: {:?}", workdir_path); + store::unpack_tar_gz(&workdir_path, &temp_file).await?; + remove_file(&temp_file).await?; + info!("Prepared gzip source: {:?}", workdir_path); + } else if let "application/x-bzip2" = source_kind.mime_type() { + let bz_decoder = BzDecoder::new(response_bytes); + let mut archive = Archive::new(bz_decoder); + archive.unpack(&workdir_path).await?; + info!("Prepared bzip2 source: {:?}", workdir_path); } else { let source_file_name = url.path_segments().unwrap().last(); let source_file = source_file_name.unwrap(); @@ -161,8 +168,8 @@ async fn prepare(name: &str, source: &PackageSource) -> Result<(i32, String), an info!("Preparing source kind: {:?}", source_kind); if source_kind.mime_type() == "application/gzip" { - info!("Preparing packed source: {:?}", work_dir); - store::unpack_source(&workdir_path, &source_path)?; + info!("Preparing packed source: {:?}", workdir); + store::unpack_tar_gz(&workdir_path, &source_path).await?; } } @@ -227,17 +234,19 @@ async fn prepare(name: &str, source: &PackageSource) -> Result<(i32, String), an } } - let source_tar = NamedTempFile::new()?; - let source_tar_path = source_tar.path().canonicalize()?; + let source_tar = store::create_temp_file("tar.gz").await?; + let source_tar_path = source_tar.canonicalize()?; info!("Creating source tar: {:?}", source_tar); - store::compress_files(&workdir_path, &source_tar_path, &workdir_files)?; + store::compress_tar_gz(&workdir_path, &source_tar_path, &workdir_files).await?; set_permissions(&source_tar, Permissions::from_mode(0o444)).await?; info!("Source tar: {}", source_tar_path.display()); + remove_dir_all(&workdir_path).await?; + prepare_source(name, &workdir_hash, &source_tar_path).await } @@ -273,7 +282,7 @@ async fn build( fs::create_dir_all(&store_path_dir).await?; - store::unpack_source(&store_path_dir, &store_path_tar)?; + store::unpack_tar_gz(&store_path_dir, &store_path_tar).await?; info!("Unpacked source: {}", store_path_dir.display()); @@ -282,7 +291,7 @@ async fn build( let mut store_tar = File::create(&store_path_tar).await?; if let Err(e) = store_tar.write(&response_data.package_data).await { - eprintln!("Failed source file: {}", e) + return Err(anyhow::anyhow!("Failed to write tar: {:?}", e)); } else { let metadata = fs::metadata(&store_path_tar).await?; let mut permissions = metadata.permissions(); @@ -298,7 +307,7 @@ async fn build( fs::create_dir_all(&store_path_dir).await?; - store::unpack_source(&store_path_dir, &store_path_tar)?; + store::unpack_tar_gz(&store_path_dir, &store_path_tar).await?; info!("Unpacked source: {}", store_path_dir.display()); } diff --git a/src/store/mod.rs b/src/store/mod.rs index 7ff264b3..de675981 100644 --- a/src/store/mod.rs +++ b/src/store/mod.rs @@ -1,17 +1,17 @@ use anyhow::Result; -use flate2::read::GzDecoder; -use flate2::write::GzEncoder; -use flate2::Compression; +use async_compression::tokio::{bufread::GzipDecoder, write::GzipEncoder}; use sha256::{digest, try_digest}; use std::env; use std::ffi::OsStr; -use std::fs; -use std::fs::File; -use std::io::BufReader; +use std::os::unix::fs::MetadataExt; use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; -use tar::Archive; -use tar::Builder; +use tokio::fs; +use tokio::fs::File; +use tokio::io::AsyncWriteExt; +use tokio::io::BufReader; +use tokio_tar::Archive; +use tokio_tar::Builder; use tracing::info; use uuid::Uuid; use walkdir::WalkDir; @@ -42,7 +42,7 @@ pub fn get_public_key_path() -> PathBuf { get_key_dir_path().join("public").with_extension("pem") } -pub fn get_temp_dir_path() -> PathBuf { +pub fn get_temp_path() -> PathBuf { env::temp_dir().join(Uuid::now_v7().to_string()) } @@ -50,6 +50,18 @@ pub fn get_store_dir_name(name: &str, hash: &str) -> String { format!("{}-{}", name, hash) } +pub async fn create_temp_dir() -> Result { + let temp_dir = get_temp_path(); + fs::create_dir(&temp_dir).await?; + Ok(temp_dir) +} + +pub async fn create_temp_file(extension: &str) -> Result { + let temp_file = get_temp_path().with_extension(extension); + File::create(&temp_file).await?; + Ok(temp_file) +} + pub fn get_source_tar_path(source_name: &str, source_hash: &str) -> PathBuf { let store_dir = get_store_dir_path(); let store_dir_name = get_store_dir_name(source_name, source_hash); @@ -136,7 +148,7 @@ where Ok(digest(combined)) } -pub fn compress_files<'a, P1, P2, P3, I>( +pub async fn compress_tar_gz<'a, P1, P2, P3, I>( source: P1, source_output: P2, source_files: I, @@ -147,8 +159,8 @@ where P3: AsRef + 'a, I: IntoIterator, { - let tar = File::create(source_output)?; - let tar_encoder = GzEncoder::new(tar.try_clone()?, Compression::default()); + let tar = File::create(source_output).await?; + let tar_encoder = GzipEncoder::new(tar); let mut tar_builder = Builder::new(tar_encoder); let source = source.as_ref(); @@ -166,47 +178,42 @@ where info!("Adding: {}", relative_path.display()); if path.is_file() { - tar_builder.append_path_with_name(path, relative_path)?; + tar_builder + .append_path_with_name(path, relative_path) + .await?; } else if path.is_dir() { - tar_builder.append_dir(relative_path, path)?; + tar_builder.append_dir_all(relative_path, path).await?; } } - tar_builder.finish()?; + let mut output = tar_builder.into_inner().await?; + + output.shutdown().await?; - Ok(tar) + Ok(output.into_inner()) } -pub fn set_files_permissions<'a, P, I>(files: I) -> Result<(), anyhow::Error> -where - P: AsRef + 'a, - I: IntoIterator, -{ +pub async fn unpack_tar_gz(target_dir: &PathBuf, source_tar: &Path) -> Result<(), anyhow::Error> { + let tar_gz = File::open(source_tar).await?; + let buf_reader = BufReader::new(tar_gz); + let gz_decoder = GzipDecoder::new(buf_reader); + let mut archive = Archive::new(gz_decoder); + Ok(archive.unpack(target_dir).await?) +} + +pub async fn set_files_permissions(files: &[PathBuf]) -> Result<(), anyhow::Error> { for file in files { - let permissions = fs::metadata(file)?.permissions(); + let permissions = fs::metadata(file).await?; if permissions.mode() & 0o111 != 0 { - fs::set_permissions(file, fs::Permissions::from_mode(0o555))?; + fs::set_permissions(file, std::fs::Permissions::from_mode(0o555)).await?; } else { - fs::set_permissions(file, fs::Permissions::from_mode(0o444))?; + fs::set_permissions(file, std::fs::Permissions::from_mode(0o444)).await?; } } Ok(()) } -pub fn unpack_source(target_dir: P1, source_tar: P2) -> Result<(), anyhow::Error> -where - P1: AsRef, - P2: AsRef, -{ - let tar_gz = File::open(source_tar)?; - let buf_reader = BufReader::new(tar_gz); - let gz_decoder = GzDecoder::new(buf_reader); - let mut archive = Archive::new(gz_decoder); - archive.unpack(target_dir)?; - Ok(()) -} - #[cfg(test)] mod tests { use super::*;