From 9632a3e1d4324cc3d35be6f306258f5038457a1b Mon Sep 17 00:00:00 2001 From: Cole Mickens Date: Wed, 27 Mar 2024 08:26:12 -0700 Subject: [PATCH 01/27] scripts: init add build/serve scripts for testing flakehub-push changes --- scripts/build-musl.sh | 11 +++++++++++ scripts/run-miniserve.sh | 8 ++++++++ 2 files changed, 19 insertions(+) create mode 100755 scripts/build-musl.sh create mode 100755 scripts/run-miniserve.sh diff --git a/scripts/build-musl.sh b/scripts/build-musl.sh new file mode 100755 index 00000000..32173c12 --- /dev/null +++ b/scripts/build-musl.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -x +set -euo pipefail + +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +cd "${DIR}/.." + +export CARGO_BUILD_TARGET="x86_64-unknown-linux-musl"; +export CARGO_BUILD_RUSTFLAGS="-C target-feature=+crt-static"; + +cargo watch -x build diff --git a/scripts/run-miniserve.sh b/scripts/run-miniserve.sh new file mode 100755 index 00000000..da1bfb89 --- /dev/null +++ b/scripts/run-miniserve.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -x +set -euo pipefail + +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +cd "${DIR}/.." + +miniserve -i :: --port 9898 ./target/x86_64-unknown-linux-musl/debug From ecbf8fb61c9010a143e013785eadcaec6732a8c1 Mon Sep 17 00:00:00 2001 From: Cole Mickens Date: Wed, 27 Mar 2024 12:39:24 -0700 Subject: [PATCH 02/27] refactor, wip, hopefully self-comments are mostly removed --- Cargo.lock | 888 ++++++++++++++++++++------------------ Cargo.toml | 3 +- flake.lock | 50 +-- scripts/build-musl.sh | 1 + src/cli/mod.rs | 329 +------------- src/error.rs | 4 +- src/flake_info.rs | 450 ++++++++++++------- src/flakehub_auth_fake.rs | 36 ++ src/flakehub_client.rs | 113 +++++ src/github/graphql/mod.rs | 2 + src/gitlab/mod.rs | 14 + src/main.rs | 110 ++++- src/push.rs | 388 ----------------- src/push_context.rs | 416 ++++++++++++++++++ src/release_metadata.rs | 1 + src/s3.rs | 44 ++ 16 files changed, 1517 insertions(+), 1332 deletions(-) create mode 100644 src/flakehub_auth_fake.rs create mode 100644 src/flakehub_client.rs create mode 100644 src/gitlab/mod.rs delete mode 100644 src/push.rs create mode 100644 src/push_context.rs create mode 100644 src/s3.rs diff --git a/Cargo.lock b/Cargo.lock index 262da99e..c0d9ad4b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,9 +19,9 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "ahash" -version = "0.8.6" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", "getrandom", @@ -32,9 +32,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] @@ -47,47 +47,48 @@ checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" [[package]] name = "anstream" -version = "0.6.5" +version = "0.6.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d664a92ecae85fd0a7392615844904654d1d5f5514837f471ddef4a057aba1b6" +checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b" dependencies = [ "anstyle", "anstyle-parse", "anstyle-query", "anstyle-wincon", "colorchoice", + "is_terminal_polyfill", "utf8parse", ] [[package]] name = "anstyle" -version = "1.0.4" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" +checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" [[package]] name = "anstyle-parse" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" +checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" +checksum = "a64c907d4e79225ac72e2a354c9ce84d50ebb4586dee56c82b3ee73004f537f5" dependencies = [ "windows-sys 0.52.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.2" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" +checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19" dependencies = [ "anstyle", "windows-sys 0.52.0", @@ -95,9 +96,9 @@ dependencies = [ [[package]] name = "arc-swap" -version = "1.6.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" [[package]] name = "arrayvec" @@ -113,13 +114,13 @@ checksum = "eab1c04a571841102f5345a8fc0f6bb3d31c315dec879b5c6e42e40ce7ffa34e" [[package]] name = "async-trait" -version = "0.1.74" +version = "0.1.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" +checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.61", ] [[package]] @@ -130,15 +131,15 @@ checksum = "c59bdb34bc650a32731b31bd8f0829cc15d24a708ee31559e0bb34f2bc320cba" [[package]] name = "autocfg" -version = "1.1.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "backtrace" -version = "0.3.69" +version = "0.3.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" dependencies = [ "addr2line", "cc", @@ -151,9 +152,15 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.5" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "base64" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bitflags" @@ -163,29 +170,29 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" dependencies = [ "serde", ] [[package]] name = "bstr" -version = "1.8.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "542f33a8835a0884b006a0c3df3dadd99c0c3f296ed26c2fdc8028e01ad6230c" +checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706" dependencies = [ "memchr", - "regex-automata 0.4.3", + "regex-automata 0.4.6", "serde", ] [[package]] name = "bumpalo" -version = "3.14.0" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "byteorder" @@ -195,9 +202,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" [[package]] name = "bytesize" @@ -207,12 +214,9 @@ checksum = "a3e368af43e418a04d52505cf3dbc23dda4e3407ae2fa99fd0e4f308ce546acc" [[package]] name = "cc" -version = "1.0.83" +version = "1.0.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" -dependencies = [ - "libc", -] +checksum = "099a5357d84c4c61eb35fc8eafa9a79a902c2f76911e5747ced4e032edd8d9b4" [[package]] name = "cfg-if" @@ -222,9 +226,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "4.4.11" +version = "4.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfaff671f6b22ca62406885ece523383b9b64022e341e53e009a62ebc47a45f2" +checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" dependencies = [ "clap_builder", "clap_derive", @@ -232,9 +236,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.11" +version = "4.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a216b506622bb1d316cd51328dce24e07bdff4a6128a47c7e7fad11878d5adbb" +checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" dependencies = [ "anstream", "anstyle", @@ -244,33 +248,33 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.4.7" +version = "4.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" +checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.61", ] [[package]] name = "clap_lex" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" +checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" [[package]] name = "clru" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8191fa7302e03607ff0e237d4246cc043ff5b3cb9409d995172ba3bea16b807" +checksum = "cbd0f76e066e64fdc5631e3bb46381254deab9ef1158292f27c8c57e3bf3fe59" [[package]] name = "color-eyre" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a667583cca8c4f8436db8de46ea8233c42a7d9ae424a82d338f2e4675229204" +checksum = "55146f5e46f237f7423d74111267d4597b59b0dad0ffaf7303bce9945d843ad5" dependencies = [ "backtrace", "color-spantrace", @@ -296,9 +300,9 @@ dependencies = [ [[package]] name = "colorchoice" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" [[package]] name = "combine" @@ -331,20 +335,19 @@ checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "crc32fast" -version = "1.3.2" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" dependencies = [ "cfg-if", ] [[package]] name = "crossbeam" -version = "0.8.2" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2801af0d36612ae591caa9568261fddce32ce6e08a7275ea334a06a4ad021a2c" +checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" dependencies = [ - "cfg-if", "crossbeam-channel", "crossbeam-deque", "crossbeam-epoch", @@ -354,55 +357,46 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.9" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c3242926edf34aec4ac3a77108ad4854bffaa2e4ddc1824124ce59231302d5" +checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95" dependencies = [ - "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fca89a0e215bab21874660c67903c5f143333cab1da83d041c7ded6053774751" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" dependencies = [ - "cfg-if", "crossbeam-epoch", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.16" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d2fe95351b870527a5d09bf563ed3c97c0cffb87cf1c78a591bf48bb218d9aa" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "autocfg", - "cfg-if", "crossbeam-utils", - "memoffset", ] [[package]] name = "crossbeam-queue" -version = "0.3.9" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9bcf5bdbfdd6030fb4a1c497b5d5fc5921aa2f60d359a17e249c0e6df3de153" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" dependencies = [ - "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.17" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d96137f14f244c37f989d9fff8f95e6c18b918e71f36638f8c49112e4c78f" -dependencies = [ - "cfg-if", -] +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" [[package]] name = "dashmap" @@ -411,7 +405,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "lock_api", "once_cell", "parking_lot_core", @@ -419,9 +413,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eb30d70a07a3b04884d2677f06bec33509dc67ca60d92949e5535352d3191dc" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" dependencies = [ "powerfmt", ] @@ -434,30 +428,24 @@ checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b" [[package]] name = "either" -version = "1.9.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +checksum = "a47c1c47d2f5964e29c61246e81db715514cd532db6b5116a25ea3c03d6780a2" [[package]] name = "encoding_rs" -version = "0.8.33" +version = "0.8.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" dependencies = [ "cfg-if", ] -[[package]] -name = "equivalent" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" - [[package]] name = "errno" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" dependencies = [ "libc", "windows-sys 0.52.0", @@ -481,9 +469,9 @@ checksum = "a2a2b11eda1d40935b26cf18f6833c526845ae8c41e58d09af6adeb6f0269183" [[package]] name = "fastrand" -version = "2.0.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" [[package]] name = "filetime" @@ -493,7 +481,7 @@ checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.4.1", "windows-sys 0.52.0", ] @@ -501,7 +489,7 @@ dependencies = [ name = "flakehub-push" version = "0.1.0" dependencies = [ - "base64", + "base64 0.21.7", "clap", "color-eyre", "flate2", @@ -509,6 +497,7 @@ dependencies = [ "gix", "gix-ref", "graphql_client", + "http", "reqwest", "ring 0.16.20", "semver", @@ -528,9 +517,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.28" +version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" +checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" dependencies = [ "crc32fast", "miniz_oxide", @@ -553,24 +542,24 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", ] [[package]] name = "futures-core" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-io" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-lite" @@ -587,32 +576,32 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.61", ] [[package]] name = "futures-sink" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-util" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-core", "futures-io", @@ -627,9 +616,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.11" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "libc", @@ -831,7 +820,7 @@ version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbd06203b1a9b33a78c88252a625031b094d9e1b647260070c25b09910c0a804" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.5.0", "bstr", "gix-path", "libc", @@ -891,9 +880,9 @@ dependencies = [ [[package]] name = "gix-dir" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6943a1f213ad7a060a0548ece229be53f3c2151534b126446ce3533eaf5f14c" +checksum = "3d6fcd56ffa1133f35525af890226ad0d3b2e607b4490360c94b1869e278eba3" dependencies = [ "bstr", "gix-discover", @@ -986,7 +975,7 @@ version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "682bdc43cb3c00dbedfcc366de2a849b582efd8d886215dbad2ea662ec156bb5" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.5.0", "bstr", "gix-features", "gix-path", @@ -1011,7 +1000,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ddf80e16f3c19ac06ce415a38b8591993d3f73aede049cb561becb5b3a8e242" dependencies = [ "gix-hash", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "parking_lot", ] @@ -1031,11 +1020,11 @@ dependencies = [ [[package]] name = "gix-index" -version = "0.32.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3383122cf18655ef4c097c0b935bba5eb56983947959aaf3b0ceb1949d4dd371" +checksum = "881ab3b1fa57f497601a5add8289e72a7ae09471fc0b9bbe483b628ae8e418a1" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.5.0", "bstr", "filetime", "fnv", @@ -1047,7 +1036,7 @@ dependencies = [ "gix-object", "gix-traverse", "gix-utils", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "itoa", "libc", "memmap2", @@ -1076,7 +1065,7 @@ checksum = "1dff438f14e67e7713ab9332f5fd18c8f20eb7eb249494f6c2bf170522224032" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.61", ] [[package]] @@ -1098,7 +1087,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54ba98f8c8c06870dfc167d192ca38a38261867b836cb89ac80bc9176dba975e" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.5.0", "gix-commitgraph", "gix-date", "gix-hash", @@ -1213,11 +1202,11 @@ dependencies = [ [[package]] name = "gix-pathspec" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d479789f3abd10f68a709454ce04cd68b54092ee882c8622ae3aa1bb9bf8496c" +checksum = "ea9f934a111e0efdf93ae06e3648427e60e783099fbebd6a53a7a2ffb10a1e65" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.5.0", "bstr", "gix-attributes", "gix-config-value", @@ -1347,7 +1336,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fddc27984a643b20dd03e97790555804f98cf07404e0e552c0ad8133266a79a1" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.5.0", "gix-path", "libc", "serde", @@ -1440,7 +1429,7 @@ version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4029ec209b0cc480d209da3837a42c63801dd8548f09c1f4502c60accb62aeb" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.5.0", "gix-commitgraph", "gix-date", "gix-hash", @@ -1489,9 +1478,9 @@ dependencies = [ [[package]] name = "gix-worktree" -version = "0.33.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "359a87dfef695b5f91abb9a424c947edca82768f34acfc269659f66174a510b4" +checksum = "f06ca5dd164678914fc9280ba9d1ffeb66499ccc16ab1278c513828beee88401" dependencies = [ "bstr", "gix-attributes", @@ -1582,7 +1571,7 @@ checksum = "a40f793251171991c4eb75bd84bc640afa8b68ff6907bc89d3b712a22f700506" dependencies = [ "graphql-introspection-query", "graphql-parser", - "heck", + "heck 0.4.1", "lazy_static", "proc-macro2", "quote", @@ -1602,25 +1591,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "h2" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", -] - [[package]] name = "hashbrown" version = "0.12.3" @@ -1629,9 +1599,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.14.3" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ "ahash", "allocator-api2", @@ -1643,11 +1613,17 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + [[package]] name = "hermit-abi" -version = "0.3.3" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" [[package]] name = "home" @@ -1660,9 +1636,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.11" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" dependencies = [ "bytes", "fnv", @@ -1671,12 +1647,24 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.6" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" dependencies = [ "bytes", + "futures-core", "http", + "http-body", "pin-project-lite", ] @@ -1686,54 +1674,66 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - [[package]] name = "human_format" -version = "1.0.3" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86cce260d758a9aa3d7c4b99d55c815a540f8a37514ba6046ab6be402a157cb0" +checksum = "5c3b1f728c459d27b12448862017b96ad4767b1ec2ec5e6434e99f1577f085b8" [[package]] name = "hyper" -version = "0.14.28" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" dependencies = [ "bytes", "futures-channel", - "futures-core", "futures-util", - "h2", "http", "http-body", "httparse", - "httpdate", "itoa", "pin-project-lite", - "socket2", + "smallvec", "tokio", - "tower-service", - "tracing", "want", ] [[package]] name = "hyper-rustls" -version = "0.24.2" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" dependencies = [ "futures-util", "http", "hyper", + "hyper-util", "rustls", + "rustls-pki-types", "tokio", "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "pin-project-lite", + "socket2", + "tokio", + "tower", + "tower-service", + "tracing", ] [[package]] @@ -1762,16 +1762,6 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" -[[package]] -name = "indexmap" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" -dependencies = [ - "equivalent", - "hashbrown 0.14.3", -] - [[package]] name = "io-close" version = "0.3.7" @@ -1788,17 +1778,23 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +[[package]] +name = "is_terminal_polyfill" +version = "1.70.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" + [[package]] name = "itoa" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "js-sys" -version = "0.3.66" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" dependencies = [ "wasm-bindgen", ] @@ -1831,21 +1827,21 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.153" +version = "0.2.154" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346" [[package]] name = "linux-raw-sys" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "lock_api" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" dependencies = [ "autocfg", "scopeguard", @@ -1853,9 +1849,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.20" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" [[package]] name = "matchers" @@ -1868,20 +1864,20 @@ dependencies = [ [[package]] name = "maybe-async" -version = "0.2.7" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f1b8c13cb1f814b634a96b2c725449fe7ed464a7b8781de8688be5ffbd3f305" +checksum = "5cf92c10c7e361d6b99666ec1c6f9805b0bea2c3bd8c78dc6fe98ac5bd78db11" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.61", ] [[package]] name = "memchr" -version = "2.6.4" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" [[package]] name = "memmap2" @@ -1892,15 +1888,6 @@ dependencies = [ "libc", ] -[[package]] -name = "memoffset" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" -dependencies = [ - "autocfg", -] - [[package]] name = "mime" version = "0.3.17" @@ -1909,9 +1896,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "miniz_oxide" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" dependencies = [ "adler", ] @@ -1937,6 +1924,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num_cpus" version = "1.16.0" @@ -1949,18 +1942,18 @@ dependencies = [ [[package]] name = "num_threads" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" dependencies = [ "libc", ] [[package]] name = "object" -version = "0.32.1" +version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" dependencies = [ "memchr", ] @@ -1997,9 +1990,9 @@ checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" [[package]] name = "parking_lot" -version = "0.12.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +checksum = "7e4af0ca4f6caed20e900d564c242b8e5d4903fdacf31d3daf527b66fe6f42fb" dependencies = [ "lock_api", "parking_lot_core", @@ -2007,15 +2000,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.9" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.1", "smallvec", - "windows-targets 0.48.5", + "windows-targets 0.52.5", ] [[package]] @@ -2024,11 +2017,31 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "pin-project" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.61", +] + [[package]] name = "pin-project-lite" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" [[package]] name = "pin-utils" @@ -2050,9 +2063,9 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro2" -version = "1.0.78" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" +checksum = "8ad3d49ab951a01fbaafe34f2ec74122942fe18a3f9814c3268f1bb72042131b" dependencies = [ "unicode-ident", ] @@ -2069,9 +2082,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.35" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] @@ -2108,9 +2121,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.8.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" dependencies = [ "either", "rayon-core", @@ -2118,9 +2131,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" dependencies = [ "crossbeam-deque", "crossbeam-utils", @@ -2135,16 +2148,25 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" +dependencies = [ + "bitflags 2.5.0", +] + [[package]] name = "regex" -version = "1.10.2" +version = "1.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.3", - "regex-syntax 0.8.2", + "regex-automata 0.4.6", + "regex-syntax 0.8.3", ] [[package]] @@ -2158,13 +2180,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.3" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.2", + "regex-syntax 0.8.3", ] [[package]] @@ -2175,26 +2197,26 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" +checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" [[package]] name = "reqwest" -version = "0.11.23" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" +checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10" dependencies = [ - "base64", + "base64 0.22.1", "bytes", - "encoding_rs", "futures-core", "futures-util", - "h2", "http", "http-body", + "http-body-util", "hyper", "hyper-rustls", + "hyper-util", "ipnet", "js-sys", "log", @@ -2205,10 +2227,11 @@ dependencies = [ "rustls", "rustls-native-certs", "rustls-pemfile", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", - "system-configuration", + "sync_wrapper", "tokio", "tokio-rustls", "tokio-socks", @@ -2239,31 +2262,32 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.7" +version = "0.17.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", + "cfg-if", "getrandom", "libc", "spin 0.9.8", "untrusted 0.9.0", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "rustc-demangle" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustix" -version = "0.38.32" +version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.5.0", "errno", "libc", "linux-raw-sys", @@ -2272,52 +2296,63 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.11" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fecbfb7b1444f477b345853b1fce097a2c6fb637b2bfb87e6bc5db0f043fae4" +checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" dependencies = [ "log", - "ring 0.17.7", + "ring 0.17.8", + "rustls-pki-types", "rustls-webpki", - "sct", + "subtle", + "zeroize", ] [[package]] name = "rustls-native-certs" -version = "0.6.3" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +checksum = "8f1fb85efa936c42c6d5fc28d2629bb51e4b2f4b8a5211e297d599cc5a093792" dependencies = [ "openssl-probe", "rustls-pemfile", + "rustls-pki-types", "schannel", "security-framework", ] [[package]] name = "rustls-pemfile" -version = "1.0.4" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" dependencies = [ - "base64", + "base64 0.22.1", + "rustls-pki-types", ] +[[package]] +name = "rustls-pki-types" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" + [[package]] name = "rustls-webpki" -version = "0.101.7" +version = "0.102.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +checksum = "f3bce581c0dd41bce533ce695a1437fa16a7ab5ac3ccfa99fe1a620a7885eabf" dependencies = [ - "ring 0.17.7", + "ring 0.17.8", + "rustls-pki-types", "untrusted 0.9.0", ] [[package]] name = "ryu" -version = "1.0.16" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "same-file" @@ -2330,11 +2365,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -2343,23 +2378,13 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "sct" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" -dependencies = [ - "ring 0.17.7", - "untrusted 0.9.0", -] - [[package]] name = "security-framework" -version = "2.9.2" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.5.0", "core-foundation", "core-foundation-sys", "libc", @@ -2368,9 +2393,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.9.1" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7" dependencies = [ "core-foundation-sys", "libc", @@ -2378,38 +2403,38 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.20" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" dependencies = [ "serde", ] [[package]] name = "serde" -version = "1.0.193" +version = "1.0.201" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +checksum = "780f1cebed1629e4753a1a38a3c72d30b97ec044f0aef68cb26650a3c5cf363c" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.193" +version = "1.0.201" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +checksum = "c5e405930b9796f1c00bee880d03fc7e0bb4b9a11afc776885ffe84320da2865" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.61", ] [[package]] name = "serde_json" -version = "1.0.108" +version = "1.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" dependencies = [ "itoa", "ryu", @@ -2461,9 +2486,9 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" dependencies = [ "libc", ] @@ -2479,28 +2504,28 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.2" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" dependencies = [ "serde", ] [[package]] name = "socket2" -version = "0.5.5" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "spdx" -version = "0.10.2" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b19b32ed6d899ab23174302ff105c1577e45a06b08d4fe0a9dd13ce804bbbf71" +checksum = "29ef1a0fa1e39ac22972c8db23ff89aea700ab96aa87114e1fb55937a631a0c9" dependencies = [ "smallvec", ] @@ -2525,9 +2550,15 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "strsim" -version = "0.10.0" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "syn" @@ -2542,9 +2573,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.48" +version = "2.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" +checksum = "c993ed8ccba56ae856363b1845da7266a7cb78e1d146c8a32d54b45a8b831fc9" dependencies = [ "proc-macro2", "quote", @@ -2552,25 +2583,10 @@ dependencies = [ ] [[package]] -name = "system-configuration" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" -dependencies = [ - "bitflags 1.3.2", - "core-foundation", - "system-configuration-sys", -] - -[[package]] -name = "system-configuration-sys" -version = "0.5.0" +name = "sync_wrapper" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" -dependencies = [ - "core-foundation-sys", - "libc", -] +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" [[package]] name = "tar" @@ -2596,29 +2612,29 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.56" +version = "1.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" +checksum = "579e9083ca58dd9dcf91a9923bb9054071b9ebbd800b342194c9feb0ee89fc18" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.56" +version = "1.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" +checksum = "e2470041c06ec3ac1ab38d0356a6119054dedaea53e12fbefc0de730a1c08524" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.61", ] [[package]] name = "thread_local" -version = "1.1.7" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" dependencies = [ "cfg-if", "once_cell", @@ -2626,13 +2642,14 @@ dependencies = [ [[package]] name = "time" -version = "0.3.31" +version = "0.3.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" dependencies = [ "deranged", "itoa", "libc", + "num-conv", "num_threads", "powerfmt", "serde", @@ -2648,10 +2665,11 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.16" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26197e33420244aeb70c3e8c78376ca46571bc4e701e4791c2cd9f57dcb3a43f" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" dependencies = [ + "num-conv", "time-core", ] @@ -2672,9 +2690,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.35.0" +version = "1.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d45b238a16291a4e1584e61820b8ae57d696cc5015c459c229ccc6990cc1c" +checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" dependencies = [ "backtrace", "bytes", @@ -2698,16 +2716,17 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.61", ] [[package]] name = "tokio-rustls" -version = "0.24.1" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" dependencies = [ "rustls", + "rustls-pki-types", "tokio", ] @@ -2725,18 +2744,39 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.10" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" dependencies = [ "bytes", "futures-core", "futures-sink", "pin-project-lite", "tokio", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tower-layer", + "tower-service", "tracing", ] +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + [[package]] name = "tower-service" version = "0.3.2" @@ -2749,6 +2789,7 @@ version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -2762,7 +2803,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.61", ] [[package]] @@ -2822,18 +2863,18 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "uluru" -version = "3.0.0" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "794a32261a1f5eb6a4462c81b59cec87b5c27d5deea7dd1ac8fc781c41d226db" +checksum = "7c8a2469e56e6e5095c82ccd3afb98dad95f7af7929aab6d8ba8d6e0f73657da" dependencies = [ "arrayvec", ] [[package]] name = "unicode-bidi" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-bom" @@ -2849,9 +2890,9 @@ checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" dependencies = [ "tinyvec", ] @@ -2927,9 +2968,9 @@ checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" [[package]] name = "walkdir" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" dependencies = [ "same-file", "winapi-util", @@ -2952,9 +2993,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.89" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -2962,24 +3003,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.89" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.61", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.39" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac36a15a220124ac510204aec1c3e5db8a22ab06fd6706d881dc6149f8ed9a12" +checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" dependencies = [ "cfg-if", "js-sys", @@ -2989,9 +3030,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.89" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2999,28 +3040,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.89" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.61", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.89" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" [[package]] name = "wasm-streams" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" +checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129" dependencies = [ "futures-util", "js-sys", @@ -3031,9 +3072,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.66" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" dependencies = [ "js-sys", "wasm-bindgen", @@ -3057,11 +3098,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" dependencies = [ - "winapi", + "windows-sys 0.52.0", ] [[package]] @@ -3085,7 +3126,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.0", + "windows-targets 0.52.5", ] [[package]] @@ -3105,17 +3146,18 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" dependencies = [ - "windows_aarch64_gnullvm 0.52.0", - "windows_aarch64_msvc 0.52.0", - "windows_i686_gnu 0.52.0", - "windows_i686_msvc 0.52.0", - "windows_x86_64_gnu 0.52.0", - "windows_x86_64_gnullvm 0.52.0", - "windows_x86_64_msvc 0.52.0", + "windows_aarch64_gnullvm 0.52.5", + "windows_aarch64_msvc 0.52.5", + "windows_i686_gnu 0.52.5", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.5", + "windows_x86_64_gnu 0.52.5", + "windows_x86_64_gnullvm 0.52.5", + "windows_x86_64_msvc 0.52.5", ] [[package]] @@ -3126,9 +3168,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" [[package]] name = "windows_aarch64_msvc" @@ -3138,9 +3180,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" [[package]] name = "windows_i686_gnu" @@ -3150,9 +3192,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.0" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" [[package]] name = "windows_i686_msvc" @@ -3162,9 +3210,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" [[package]] name = "windows_x86_64_gnu" @@ -3174,9 +3222,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" [[package]] name = "windows_x86_64_gnullvm" @@ -3186,9 +3234,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" [[package]] name = "windows_x86_64_msvc" @@ -3198,24 +3246,24 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" [[package]] name = "winnow" -version = "0.6.6" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0c976aaaa0e1f90dbb21e9587cdaf1d9679a1cde8875c0d6bd83ab96a208352" +checksum = "c3c52e9c97a68071b23e836c9380edae937f17b9c4667bd021973efc689f618d" dependencies = [ "memchr", ] [[package]] name = "winreg" -version = "0.50.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" dependencies = [ "cfg-if", "windows-sys 0.48.0", @@ -3223,9 +3271,9 @@ dependencies = [ [[package]] name = "xattr" -version = "1.1.3" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7dae5072fe1f8db8f8d29059189ac175196e410e40ba42d5d4684ae2f750995" +checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" dependencies = [ "libc", "linux-raw-sys", @@ -3234,20 +3282,26 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.31" +version = "0.7.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c4061bedbb353041c12f413700357bec76df2c7e2ca8e4df8bac24c6bf68e3d" +checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.31" +version = "0.7.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3c129550b3e6de3fd0ba67ba5c81818f9805e58b8d7fee80a3a59d2c9fc601a" +checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.61", ] + +[[package]] +name = "zeroize" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" diff --git a/Cargo.toml b/Cargo.toml index 63facfb8..84415b88 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ clap = { version = "4.3.4", features = ["derive", "env"] } color-eyre = { version = "0.6.2", default-features = false, features = [ "track-caller", "issue-url", "tracing-error", "capture-spantrace", "color-spantrace" ] } graphql_client = { version = "0.13.0" } tokio = { version = "1.21.0", default-features = false, features = ["time", "io-std", "process", "fs", "signal", "tracing", "rt-multi-thread", "macros", "io-util", "parking_lot" ] } -reqwest = { version = "0.11.11", default-features = false, features = ["rustls-tls-native-roots", "stream", "socks", "json"] } +reqwest = { version = "0.12.3", default-features = false, features = ["rustls-tls-native-roots", "stream", "socks", "json"] } serde = { version = "1.0.164", features = ["derive"] } serde_json = "1.0.97" gix = { version = "0.62.0", features = ["async-network-client", "serde"] } @@ -32,6 +32,7 @@ uuid = { version = "1.4.0", features = ["serde", "v7", "rand", "std"] } semver = { version = "1.0.18", features = ["serde"] } thiserror = "1.0.56" url = { version = "2.5.0", features = ["serde"] } +http = "1.1.0" [profile.release] strip = true # Automatically strip symbols from the binary. diff --git a/flake.lock b/flake.lock index e2ddc1a5..1c0b995c 100644 --- a/flake.lock +++ b/flake.lock @@ -8,12 +8,12 @@ "rust-overlay": "rust-overlay" }, "locked": { - "lastModified": 1695511445, - "narHash": "sha256-mnE14re43v3/Jc50Jv0BKPMtEk7FEtDSligP6B5HwlI=", - "rev": "3de322e06fc88ada5e3589dc8a375b73e749f512", - "revCount": 411, + "lastModified": 1697588719, + "narHash": "sha256-n9ALgm3S+ygpzjesBkB9qutEtM4dtIkhn8WnstCPOew=", + "rev": "da6b58e270d339a78a6e95728012ec2eea879612", + "revCount": 440, "type": "tarball", - "url": "https://api.flakehub.com/f/pinned/ipetkov/crane/0.14.1/018ac45c-ff5e-7076-b956-d478a0336516/source.tar.gz" + "url": "https://api.flakehub.com/f/pinned/ipetkov/crane/0.14.3/018b402e-8337-76a6-9764-1748a79a54fd/source.tar.gz" }, "original": { "type": "tarball", @@ -23,11 +23,11 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1673956053, - "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "lastModified": 1696267196, + "narHash": "sha256-AAQ/2sD+0D18bb8hKuEEVpHUYD1GmO2Uh/taFamn6XQ=", "owner": "edolstra", "repo": "flake-compat", - "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "rev": "4f910c9827911b1ec2bf26b5a062cd09f8d89f85", "type": "github" }, "original": { @@ -41,11 +41,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1685518550, - "narHash": "sha256-o2d0KcvaXzTrPRIo0kOLV0/QXHhDQ5DTi+OxcjO8xqY=", + "lastModified": 1694529238, + "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=", "owner": "numtide", "repo": "flake-utils", - "rev": "a1720a10a6cfe8234c0e93907ffe81be440f4cef", + "rev": "ff7b65b44d01cf9ba6a71320833626af21126384", "type": "github" }, "original": { @@ -59,11 +59,11 @@ "systems": "systems_2" }, "locked": { - "lastModified": 1681202837, - "narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=", + "lastModified": 1705309234, + "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=", "owner": "numtide", "repo": "flake-utils", - "rev": "cfacdce06f30d2b68473a46042957675eebb3401", + "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26", "type": "github" }, "original": { @@ -74,12 +74,12 @@ }, "nixpkgs": { "locked": { - "lastModified": 1696604326, - "narHash": "sha256-YXUNI0kLEcI5g8lqGMb0nh67fY9f2YoJsILafh6zlMo=", - "rev": "87828a0e03d1418e848d3dd3f3014a632e4a4f64", - "revCount": 533189, + "lastModified": 1711703276, + "narHash": "sha256-iMUFArF0WCatKK6RzfUJknjem0H9m4KgorO/p3Dopkk=", + "rev": "d8fe5e6c92d0d190646fb9f1056741a229980089", + "revCount": 604424, "type": "tarball", - "url": "https://api.flakehub.com/f/pinned/NixOS/nixpkgs/0.1.533189%2Brev-87828a0e03d1418e848d3dd3f3014a632e4a4f64/018b0dc8-e84f-7c59-b5d6-16849c3b2074/source.tar.gz" + "url": "https://api.flakehub.com/f/pinned/NixOS/nixpkgs/0.1.604424%2Brev-d8fe5e6c92d0d190646fb9f1056741a229980089/018e8c76-3928-7110-9c82-93066ec52d25/source.tar.gz" }, "original": { "type": "tarball", @@ -99,11 +99,11 @@ "nixpkgs": ["crane", "nixpkgs"] }, "locked": { - "lastModified": 1685759304, - "narHash": "sha256-I3YBH6MS3G5kGzNuc1G0f9uYfTcNY9NYoRc3QsykLk4=", + "lastModified": 1696299134, + "narHash": "sha256-RS77cAa0N+Sfj5EmKbm5IdncNXaBCE1BSSQvUE8exvo=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "c535b4f3327910c96dcf21851bbdd074d0760290", + "rev": "611ccdceed92b4d94ae75328148d84ee4a5b462d", "type": "github" }, "original": { @@ -118,11 +118,11 @@ "nixpkgs": ["nixpkgs"] }, "locked": { - "lastModified": 1687141659, - "narHash": "sha256-ckvEuxejYmFTyFF0u9CWV8h5u+ubuxA7vYrOw/GXRXg=", + "lastModified": 1711937855, + "narHash": "sha256-jlfDBRtsLoqRNFxtQtG47wsrwVsQSV4AqoMgWG6Bvng=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "86302751ef371597d48951983e1a2f04fe78d4ff", + "rev": "3f46a51b47f56c24b4d7e8db8fb8e73118923f96", "type": "github" }, "original": { diff --git a/scripts/build-musl.sh b/scripts/build-musl.sh index 32173c12..3935222c 100755 --- a/scripts/build-musl.sh +++ b/scripts/build-musl.sh @@ -7,5 +7,6 @@ cd "${DIR}/.." export CARGO_BUILD_TARGET="x86_64-unknown-linux-musl"; export CARGO_BUILD_RUSTFLAGS="-C target-feature=+crt-static"; +# export CARGO_PROFILE_DEV_STRIP="debuginfo" cargo watch -x build diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 31fb429e..77384a75 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -8,13 +8,7 @@ use std::{ }; use crate::{ - build_http_client, - github::{ - get_actions_id_bearer_token, - graphql::{GithubGraphqlDataQuery, MAX_LABEL_LENGTH, MAX_NUM_TOTAL_LABELS}, - }, - push::push_new_release, - release_metadata::RevisionInfo, + build_http_client, flakehub_client::Tarball, github::graphql::{GithubGraphqlDataQuery, MAX_LABEL_LENGTH, MAX_NUM_TOTAL_LABELS}, release_metadata::{ReleaseMetadata, RevisionInfo} }; #[derive(Debug, clap::Parser)] @@ -254,8 +248,7 @@ impl clap::builder::TypedValueParser for U64ToNoneParser { } impl FlakeHubPushCli { - #[tracing::instrument(skip_all)] - pub(crate) fn populate_from_github_actions_environment(&mut self) { + pub(crate) fn backfill_from_github_env(&mut self) { if self.git_root.0.is_none() { let env_key = "GITHUB_WORKSPACE"; if let Ok(env_val) = std::env::var(env_key) { @@ -281,319 +274,7 @@ impl FlakeHubPushCli { } } - #[tracing::instrument( - name = "flakehub_push" - skip_all, - fields( - host = %self.host, - visibility = ?self.visibility, - visibility_alt = ?self.visibility_alt, - name = self.name.0, - tag = tracing::field::Empty, - rolling_minor = tracing::field::Empty, - rolling = self.rolling, - directory = tracing::field::Empty, - repository = tracing::field::Empty, - git_root = tracing::field::Empty, - commit_count = tracing::field::Empty, - mirror = self.mirror, - jwt_issuer_uri = tracing::field::Empty, - extra_labels = self.extra_labels.join(","), - spdx_expression = tracing::field::Empty, - error_on_conflict = self.error_on_conflict, - include_output_paths = self.include_output_paths, - ) - )] - pub(crate) async fn execute(mut self) -> color_eyre::Result { - let span = tracing::Span::current(); - tracing::trace!("Executing"); - - let is_github_actions = std::env::var("GITHUB_ACTION").ok().is_some(); - if is_github_actions { - tracing::debug!("Running inside Github Actions, enriching arguments with GitHub Actions environment data"); - self.populate_from_github_actions_environment() - } - - let Self { - host, - visibility, - visibility_alt, - name, - tag, - rolling, - rolling_minor, - github_token, - directory, - repository, - git_root, - mirror, - jwt_issuer_uri, - instrumentation: _, - extra_labels, - spdx_expression, - extra_tags, - error_on_conflict, - include_output_paths, - .. - } = self; - - // Check for the misspelled env var value first (FLAKEHUB_PUSH_VISIBLITY) and use the properly - // spelled env var value (FLAKEHUB_PUSH_VISIBILITY) if not. - let visibility = - match (visibility_alt, visibility) { - (Some(v), _) => v, - (None, Some(v)) => v, - (None, None) => return Err(eyre!( - "Could not determine the flake's desired visibility. Use `--visibility` to set this to one of the following: public, unlisted, private.", - )), - }; - - let mut labels: HashSet<_> = extra_labels.into_iter().filter(|v| !v.is_empty()).collect(); - let extra_tags: HashSet<_> = extra_tags.into_iter().filter(|v| !v.is_empty()).collect(); - - if !extra_tags.is_empty() { - let message = "`extra-tags` is deprecated and will be removed in the future. Please use `extra-labels` instead."; - tracing::warn!("{message}"); - - if is_github_actions { - println!("::warning::{message}"); - } - - if labels.is_empty() { - labels = extra_tags; - } else { - let message = - "Both `extra-tags` and `extra-labels` were set; `extra-tags` will be ignored."; - tracing::warn!("{message}"); - - if is_github_actions { - println!("::warning::{message}"); - } - } - } - - let git_root = if let Some(git_root) = git_root.0 { - git_root.clone() - } else { - std::env::current_dir().map(PathBuf::from).wrap_err("Could not determine current `git_root`. Pass `--git-root` or set `FLAKEHUB_PUSH_GIT_ROOT`, or run `flakehub-push` with the git root as the current working directory")? - }; - - let git_root = git_root - .canonicalize() - .wrap_err("Failed to canonicalize `--git-root` argument")?; - - let subdir = if let Some(directory) = &directory.0 { - let absolute_directory = if directory.is_absolute() { - directory.clone() - } else { - git_root.join(directory) - }; - let canonical_directory = absolute_directory - .canonicalize() - .wrap_err("Failed to canonicalize `--directory` argument")?; - - Path::new( - canonical_directory - .strip_prefix(git_root.clone()) - .wrap_err( - "Specified `--directory` was not a directory inside the `--git-root`", - )?, - ) - .into() - } else { - PathBuf::new() - }; - - let Some(repository) = repository.0 else { - return Err(eyre!("Could not determine repository name, pass `--repository` formatted like `determinatesystems/flakehub-push`")); - }; - - // If the upload name is supplied by the user, ensure that it contains exactly - // one slash and no whitespace. Default to the repository name. - let upload_name = if let Some(name) = name.0 { - let num_slashes = name.matches('/').count(); - - if num_slashes == 0 - || num_slashes > 1 - || !name.is_ascii() - || name.contains(char::is_whitespace) - { - return Err(eyre!("The argument `--name` must be in the format of `owner-name/repo-name` and cannot contain whitespace or other special characters")); - } else { - name - } - } else { - repository.clone() - }; - - let mut repository_split = repository.split('/'); - let project_owner = repository_split - .next() - .ok_or_else(|| eyre!("Could not determine owner, pass `--repository` formatted like `determinatesystems/flakehub-push`"))? - .to_string(); - let project_name = repository_split.next() - .ok_or_else(|| eyre!("Could not determine project, pass `--repository` formatted like `determinatesystems/flakehub-push`"))? - .to_string(); - if repository_split.next().is_some() { - Err(eyre!("Could not determine the owner/project, pass `--repository` formatted like `determinatesystems/flakehub-push`. The passed value has too many slashes (/) to be a valid repository"))?; - } - - let mut spdx_expression = spdx_expression.0; - - #[allow(unused_assignments)] - // Since we return an error outside github actions right now, `commit_count` throws an unused warning since we don't actually use it. - let RevisionInfo { - mut commit_count, - revision, - } = RevisionInfo::from_git_root(&git_root)?; - - let github_graphql_data_result = if let Some(github_token) = &github_token.0 { - tracing::debug!("Got Github token, enriching with GitHub API data"); - - let github_api_client = build_http_client().build()?; - - // Take the opportunity to be able to populate/encrich data from the GitHub API since we need it for project/owner_id anywys - let github_graphql_data_result = GithubGraphqlDataQuery::get( - &github_api_client, - github_token, - &project_owner, - &project_name, - &revision, - ) - .await?; - - // On GitHub Actions, typically shallow clones are used which would report 1 for the commit count. Override it with the result from the API. - // Since users can't set this as a command line flag, that's fine. - tracing::trace!( - "Updating `commit_count` from {} to {} via GitHub API", - commit_count - .map(|v| v.to_string()) - .unwrap_or_else(|| "".into()), - github_graphql_data_result.rev_count as usize - ); - commit_count = Some(github_graphql_data_result.rev_count as usize); - - // If the user didn't pass `--spdx-expression` from command line, enrich it with Github's data. - spdx_expression = if spdx_expression.is_none() { - if let Some(spdx_string) = &github_graphql_data_result.spdx_identifier { - tracing::debug!("Recieved SPDX identifier `{}` from GitHub API", spdx_string); - let parsed = spdx::Expression::parse(spdx_string) - .wrap_err("Invalid SPDX license identifier reported from the GitHub API, either you are using a non-standard license or GitHub has returned a value that cannot be validated")?; - span.record("spdx_expression", tracing::field::display(&parsed)); - Some(parsed) - } else { - None - } - } else { - // Provide the user notice if the SPDX expression passed differs from the one detected on GitHub -- It's probably something they care about. - if github_graphql_data_result.spdx_identifier - != spdx_expression.as_ref().map(|v| v.to_string()) - { - tracing::warn!( - "SPDX identifier `{}` was passed via argument, but GitHub's API suggests it may be `{}`", - spdx_expression.as_ref().map(|v| v.to_string()).unwrap_or_else(|| "None".to_string()), - github_graphql_data_result.spdx_identifier.clone().unwrap_or_else(|| "None".to_string()), - ) - } - spdx_expression - }; - - // Extend the labels provided by the user with those from GitHub. - labels = labels - .into_iter() - .chain(github_graphql_data_result.topics.iter().cloned()) - .collect::>(); - - Some(github_graphql_data_result) - } else { - None - }; - - let upload_bearer_token = match jwt_issuer_uri.0 { - None if is_github_actions => get_actions_id_bearer_token(&host) - .await - .wrap_err("Getting upload bearer token from GitHub")?, - None => { - // TODO: Accept a `--flakehub-token` arg to use a bearer token once FlakeHub supports it. - return Err(eyre!( - "`flakehub-push` currently only runs inside Github Actions" - )); - } - Some(jwt_issuer_uri) => { - tracing::warn!("Running in a development-only context, pushing to https://api.flakehub.com will not work!"); - let client = build_http_client().build()?; - - let mut claims = github_actions_oidc_claims::Claims::make_dummy(); - // FIXME: we should probably fill in more of these claims. - claims.aud = "flakehub-localhost".to_string(); - claims.iss = "flakehub-push-dev".to_string(); - claims.repository = repository.clone(); - claims.repository_owner = project_owner.to_string(); - - if let Some(github_graphql_data_result) = github_graphql_data_result { - claims.repository_id = github_graphql_data_result.project_id.to_string(); - claims.repository_owner_id = github_graphql_data_result.owner_id.to_string(); - } else { - return Err(eyre!("No populated GitHub API data (`--github-token` was likely not passed), cannot create the required JWT")); - } - - let response = client - .post(jwt_issuer_uri) - .header("Content-Type", "application/json") - .json(&claims) - .send() - .await - .wrap_err("Sending request to JWT issuer")?; - #[derive(serde::Deserialize)] - struct Response { - token: String, - } - let response_deserialized: Response = response - .json() - .await - .wrap_err("Getting token from JWT issuer's response")?; - response_deserialized.token - } - }; - - // Here we merge explicitly user-supplied labels and the labels ("topics") - // associated with the repo. Duplicates are excluded and all - // are converted to lower case. - let labels: Vec = labels - .into_iter() - .take(MAX_NUM_TOTAL_LABELS) - .map(|s| s.trim().to_lowercase()) - .filter(|t: &String| { - !t.is_empty() - && t.len() <= MAX_LABEL_LENGTH - && t.chars().all(|c| c.is_alphanumeric() || c == '-') - }) - .collect(); - - let Some(commit_count) = commit_count else { - return Err(eyre!("Could not determine commit count, this is normally determined via the `--git-root` argument or via the GitHub API")); - }; - - push_new_release( - &host, - &upload_bearer_token, - &git_root, - &subdir, - revision, - commit_count, - upload_name, - mirror, - visibility, - tag.0, - rolling, - rolling_minor.0, - labels, - spdx_expression, - error_on_conflict, - include_output_paths, - ) - .await?; - - Ok(ExitCode::SUCCESS) + pub(crate) fn backfill_from_gitlab_env(&mut self) { + todo!(); } -} +} \ No newline at end of file diff --git a/src/error.rs b/src/error.rs index d094fc95..3eccbfb7 100644 --- a/src/error.rs +++ b/src/error.rs @@ -3,10 +3,10 @@ pub(crate) enum Error { /// Unauthorized, with a single line message detailing the nature of the problem. #[error("Unauthorized: {0}")] Unauthorized(String), - #[error("{upload_name}/{rolling_prefix_or_tag} already exists")] + #[error("{upload_name}/{release_version} already exists")] Conflict { upload_name: String, - rolling_prefix_or_tag: String, + release_version: String, }, } diff --git a/src/flake_info.rs b/src/flake_info.rs index bcaf3beb..b741fedb 100644 --- a/src/flake_info.rs +++ b/src/flake_info.rs @@ -1,188 +1,314 @@ -use std::{io::Write, path::Path}; +use std::{io::Write, path::{Path, PathBuf}}; -use color_eyre::eyre::{eyre, WrapErr}; +use color_eyre::eyre::{eyre, Result, WrapErr}; +use serde::Deserialize; use tokio::io::AsyncWriteExt; +use crate::flakehub_client::Tarball; + // The UUID embedded in our flake that we'll replace with the flake URL of the flake we're trying to // get outputs from. const FLAKE_URL_PLACEHOLDER_UUID: &str = "c9026fc0-ced9-48e0-aa3c-fc86c4c86df1"; +const README_FILENAME_LOWERCASE: &str = "readme.md"; + +// TODO: can't we just do this sanity checking in from_dir? +// // TODO(colemickens): can we move this to a method on FlakeMetadata? +// // maybe FlakeMetadata should know its dir? probably inside the json already +// #[tracing::instrument( +// skip_all, +// fields( +// directory = %directory.display(), +// ) +// )] +// pub(crate) async fn check_flake_evaluates(directory: &Path) -> color_eyre::Result<()> { +// let output = tokio::process::Command::new("nix") +// .arg("flake") +// .arg("show") +// .arg("--all-systems") +// .arg("--json") +// .arg("--no-write-lock-file") +// .arg(directory) +// .output() +// .await +// .wrap_err_with(|| { +// eyre!( +// "Failed to execute `nix flake show --all-systems --json --no-write-lock-file {}`", +// directory.display() +// ) +// })?; + +// if !output.status.success() { +// let command = format!( +// "nix flake show --all-systems --json --no-write-lock-file {}", +// directory.display(), +// ); +// let msg = format!( +// "\ +// Failed to execute command `{command}`{maybe_status} \n\ +// stdout: {stdout}\n\ +// stderr: {stderr}\n\ +// ", +// stdout = String::from_utf8_lossy(&output.stdout), +// stderr = String::from_utf8_lossy(&output.stderr), +// maybe_status = if let Some(status) = output.status.code() { +// format!(" with status {status}") +// } else { +// String::new() +// } +// ); +// return Err(eyre!(msg))?; +// } + +// Ok(()) +// } -#[tracing::instrument( - skip_all, - fields( - directory = %directory.display(), - ) -)] -pub(crate) async fn get_flake_tarball( - directory: &Path, - last_modified: u64, -) -> color_eyre::Result> { - let mut tarball_builder = tar::Builder::new(vec![]); - tarball_builder.follow_symlinks(false); - tarball_builder.force_mtime(last_modified); - - tracing::trace!("Creating tarball"); - // `tar` works according to the current directory (yay) - // So we change dir and restory it after - // TODO: Fix this - let current_dir = std::env::current_dir().wrap_err("Could not get current directory")?; - std::env::set_current_dir( - directory - .parent() - .ok_or_else(|| eyre!("Getting parent directory"))?, - )?; - let dirname = directory - .file_name() - .ok_or_else(|| eyre!("No file name of directory"))?; - tarball_builder - .append_dir_all(dirname, dirname) - .wrap_err_with(|| eyre!("Adding `{}` to tarball", directory.display()))?; - std::env::set_current_dir(current_dir).wrap_err("Could not set current directory")?; - - let tarball = tarball_builder.into_inner().wrap_err("Creating tarball")?; - tracing::trace!("Created tarball, compressing..."); - let mut gzip_encoder = flate2::write::GzEncoder::new(vec![], flate2::Compression::default()); - gzip_encoder - .write_all(&tarball[..]) - .wrap_err("Adding tarball to gzip")?; - let compressed_tarball = gzip_encoder.finish().wrap_err("Creating gzip")?; - tracing::trace!("Compressed tarball"); - - Ok(compressed_tarball) + +#[derive(Debug)] +pub struct FlakeMetadata { + pub(crate) source_dir: std::path::PathBuf, + pub(crate) flake_locked_url: String, + pub(crate) metadata_json: serde_json::Value, } -#[tracing::instrument( - skip_all, - fields( - directory = %directory.display(), - ) -)] -pub(crate) async fn check_flake_evaluates(directory: &Path) -> color_eyre::Result<()> { - let output = tokio::process::Command::new("nix") - .arg("flake") - .arg("show") - .arg("--all-systems") - .arg("--json") - .arg("--no-write-lock-file") - .arg(directory) - .output() - .await - .wrap_err_with(|| { +#[derive(Debug, Deserialize)] +pub struct FlakeOutputs(pub serde_json::Value); + +impl FlakeMetadata { + pub async fn from_dir(directory: &Path) -> Result { + // TODO(colemickens): the de-duped block below runs `--no-update-lock-file`, this one is `--no-write-lock-file` + let output = tokio::process::Command::new("nix") + .arg("flake") + .arg("metadata") + .arg("--json") + .arg("--no-write-lock-file") + .arg(directory) + .output() + .await + .wrap_err_with(|| { + eyre!( + "Failed to execute `nix flake metadata --json {}`", + directory.display() + ) + })?; + + let metadata_json: serde_json::Value = serde_json::from_slice(&output.stdout).wrap_err_with(|| { eyre!( - "Failed to execute `nix flake show --all-systems --json --no-write-lock-file {}`", + "Parsing `nix flake metadata --json {}` as JSON", directory.display() ) })?; - if !output.status.success() { - let command = format!( - "nix flake show --all-systems --json --no-write-lock-file {}", - directory.display(), - ); - let msg = format!( - "\ - Failed to execute command `{command}`{maybe_status} \n\ - stdout: {stdout}\n\ - stderr: {stderr}\n\ - ", - stdout = String::from_utf8_lossy(&output.stdout), - stderr = String::from_utf8_lossy(&output.stderr), - maybe_status = if let Some(status) = output.status.code() { - format!(" with status {status}") - } else { - String::new() + /* + if flake_dir.join("flake.lock").exists() { + let output = tokio::process::Command::new("nix") + .arg("flake") + .arg("metadata") + .arg("--json") + .arg("--no-update-lock-file") + .arg(&flake_dir) + .output() + .await + .wrap_err_with(|| { + eyre!( + "Failed to execute `nix flake metadata --json --no-update-lock-file {}`", + flake_dir.display() + ) + })?; + + if !output.status.success() { + let command = format!( + "nix flake metadata --json --no-update-lock-file {}", + flake_dir.display(), + ); + let msg = format!( + "\ + Failed to execute command `{command}`{maybe_status} \n\ + stdout: {stdout}\n\ + stderr: {stderr}\n\ + ", + stdout = String::from_utf8_lossy(&output.stdout), + stderr = String::from_utf8_lossy(&output.stderr), + maybe_status = if let Some(status) = output.status.code() { + format!(" with status {status}") + } else { + String::new() + } + ); + return Err(eyre!(msg))?; + } + } + */ + + // determine flake's store (sub)dir: + + let flake_locked_url = metadata_json + .get("url") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| { + eyre!("Could not get `url` attribute from `nix flake metadata --json` output") + })?; + tracing::debug!("Locked URL = {}", flake_locked_url); + let flake_metadata_value_path = metadata_json + .get("path") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| { + eyre!("Could not get `path` attribute from `nix flake metadata --json` output") + })?; + let flake_metadata_value_resolved_dir = metadata_json + .pointer("/resolved/dir") + .and_then(serde_json::Value::as_str); + + let source = match flake_metadata_value_resolved_dir { + Some(flake_metadata_value_resolved_dir) => { + Path::new(flake_metadata_value_path).join(flake_metadata_value_resolved_dir) } - ); - return Err(eyre!(msg))?; + None => PathBuf::from(flake_metadata_value_path), + }; + + Ok(FlakeMetadata { + source_dir: source, + flake_locked_url: flake_locked_url.to_string(), + metadata_json: metadata_json, + // TODO(Colemickens): remove this, we want to use get_source() + //dir: directory, + // move the source determination to from_dir so we just know it as a property + // rename 'dir' to 'source' + }) } - Ok(()) -} + pub fn flake_tarball(&self) -> Result { + let last_modified = if let Some(last_modified) = self.metadata_json.get("lastModified") { + last_modified.as_u64().ok_or_else(|| { + eyre!("`nix flake metadata --json` does not have a integer `lastModified` field") + })? + } else { + return Err(eyre!( + "`nix flake metadata` did not return a `lastModified` attribute" + )); + }; + tracing::debug!("lastModified = {}", last_modified); + + let mut tarball_builder = tar::Builder::new(vec![]); + tarball_builder.follow_symlinks(false); + tarball_builder.force_mtime(last_modified); + + tracing::trace!("Creating tarball"); + // `tar` works according to the current directory (yay) + // So we change dir and restory it after + // TODO: Fix this + let source = self.source_dir; // refactor to be known when we create struct with from_dir + let current_dir = std::env::current_dir().wrap_err("Could not get current directory")?; + std::env::set_current_dir( + source + .parent() + .ok_or_else(|| eyre!("Getting parent directory"))?, + )?; + let dirname = self.source_dir + .file_name() + .ok_or_else(|| eyre!("No file name of directory"))?; + tarball_builder + .append_dir_all(dirname, dirname) + .wrap_err_with(|| eyre!("Adding `{}` to tarball", self.source_dir.display()))?; + std::env::set_current_dir(current_dir).wrap_err("Could not set current directory")?; + + let tarball = tarball_builder.into_inner().wrap_err("Creating tarball")?; + tracing::trace!("Created tarball, compressing..."); + let mut gzip_encoder = flate2::write::GzEncoder::new(vec![], flate2::Compression::default()); + gzip_encoder + .write_all(&tarball[..]) + .wrap_err("Adding tarball to gzip")?; + let compressed_tarball = gzip_encoder.finish().wrap_err("Creating gzip")?; + tracing::trace!("Compressed tarball"); + + let flake_tarball_hash = { + let mut context = ring::digest::Context::new(&ring::digest::SHA256); + context.update(&compressed_tarball); + context.finish() + }; + let flake_tarball_hash_base64 = { + // TODO: Use URL_SAFE_NO_PAD + use base64::{engine::general_purpose::STANDARD, Engine as _}; + STANDARD.encode(flake_tarball_hash) + }; + + let tarball = Tarball { + bytes: compressed_tarball, + hash_base64: flake_tarball_hash_base64 + }; + + Ok(tarball) + } + + pub async fn outputs(&self, include_output_paths: bool) -> Result { + let tempdir = tempfile::Builder::new() + .prefix("flakehub_push_outputs") + .tempdir() + .wrap_err("Creating tempdir")?; + + let flake_contents = include_str!("mixed-flake.nix") + .replace( + FLAKE_URL_PLACEHOLDER_UUID, + &self.flake_locked_url.escape_default().to_string(), + ) + .replace( + "INCLUDE_OUTPUT_PATHS", + if include_output_paths { + "true" + } else { + "false" + }, + ); + + let mut flake = tokio::fs::File::create(tempdir.path().join("flake.nix")).await?; + flake.write_all(flake_contents.as_bytes()).await?; + + let mut cmd = tokio::process::Command::new("nix"); + cmd.arg("eval"); + cmd.arg("--json"); + cmd.arg("--no-write-lock-file"); + cmd.arg(format!("{}#contents", tempdir.path().display())); + let output = cmd + .output() + .await + .wrap_err_with(|| eyre!("Failed to get flake outputs from tarball {}", &self.flake_locked_url))?; -#[tracing::instrument( - skip_all, - fields( - directory = %directory.display(), - ) -)] -pub(crate) async fn get_flake_metadata(directory: &Path) -> color_eyre::Result { - let output = tokio::process::Command::new("nix") - .arg("flake") - .arg("metadata") - .arg("--json") - .arg("--no-write-lock-file") - .arg(directory) - .output() - .await - .wrap_err_with(|| { + if !output.status.success() { + return Err(eyre!( + "Failed to get flake outputs from tarball {}: {}", + &self.flake_locked_url, + String::from_utf8(output.stderr).unwrap() + )); + } + + let output_json = serde_json::from_slice(&output.stdout).wrap_err_with(|| { eyre!( - "Failed to execute `nix flake metadata --json {}`", - directory.display() + "Parsing flake outputs from {} as JSON: {}", + &self.flake_locked_url, + String::from_utf8(output.stdout).unwrap(), ) })?; - let output_json = serde_json::from_slice(&output.stdout).wrap_err_with(|| { - eyre!( - "Parsing `nix flake metadata --json {}` as JSON", - directory.display() - ) - })?; + Ok(output_json) + } - Ok(output_json) -} + + #[tracing::instrument(skip_all, fields(readme_dir))] + pub(crate) async fn get_readme_contents(&self) -> Result> { + let mut read_dir = tokio::fs::read_dir(&self.source_dir).await?; -#[tracing::instrument(skip_all, fields(flake_url,))] -pub(crate) async fn get_flake_outputs( - flake_url: &str, - include_output_paths: bool, -) -> color_eyre::Result { - let tempdir = tempfile::Builder::new() - .prefix("flakehub_push_outputs") - .tempdir() - .wrap_err("Creating tempdir")?; - - let flake_contents = include_str!("mixed-flake.nix") - .replace( - FLAKE_URL_PLACEHOLDER_UUID, - &flake_url.escape_default().to_string(), - ) - .replace( - "INCLUDE_OUTPUT_PATHS", - if include_output_paths { - "true" - } else { - "false" - }, - ); - - let mut flake = tokio::fs::File::create(tempdir.path().join("flake.nix")).await?; - flake.write_all(flake_contents.as_bytes()).await?; - - let mut cmd = tokio::process::Command::new("nix"); - cmd.arg("eval"); - cmd.arg("--json"); - cmd.arg("--no-write-lock-file"); - cmd.arg(format!("{}#contents", tempdir.path().display())); - let output = cmd - .output() - .await - .wrap_err_with(|| eyre!("Failed to get flake outputs from tarball {}", flake_url))?; - - if !output.status.success() { - return Err(eyre!( - "Failed to get flake outputs from tarball {}: {}", - flake_url, - String::from_utf8(output.stderr).unwrap() - )); + let readme_path: Option = { + while let Some(entry) = read_dir.next_entry().await? { + if entry.file_name().to_ascii_lowercase() == README_FILENAME_LOWERCASE { + Some(Some(entry.path())); + } + } + None + }; + let readme = if let Some(readme_path) = readme_path { + Some(tokio::fs::read_to_string(&readme_path).await?) + } else { + None + }; + Ok(readme) } - - let output_json = serde_json::from_slice(&output.stdout).wrap_err_with(|| { - eyre!( - "Parsing flake outputs from {} as JSON: {}", - flake_url, - String::from_utf8(output.stdout).unwrap(), - ) - })?; - - Ok(output_json) } diff --git a/src/flakehub_auth_fake.rs b/src/flakehub_auth_fake.rs new file mode 100644 index 00000000..f13fcb76 --- /dev/null +++ b/src/flakehub_auth_fake.rs @@ -0,0 +1,36 @@ +use color_eyre::eyre::{eyre, Context, Result}; + +use crate::github::graphql::GithubGraphqlDataResult; + +pub async fn get_fake_bearer_token(jwt_issuer_uri: &str, repository: &str, project_owner: &str, github_graphql_data_result: GithubGraphqlDataResult) -> Result { + tracing::warn!("running outside github/gitlab - minting a dev-signed JWT"); + + let client = reqwest::Client::new(); + + let mut claims = github_actions_oidc_claims::Claims::make_dummy(); + // FIXME: we should probably fill in more of these claims. + claims.aud = "flakehub-localhost".to_string(); + claims.iss = "flakehub-push-dev".to_string(); + claims.repository = repository.to_string(); + claims.repository_owner = project_owner.to_string(); + + claims.repository_id = github_graphql_data_result.project_id.to_string(); + claims.repository_owner_id = github_graphql_data_result.owner_id.to_string(); + + let response = client + .post(jwt_issuer_uri) + .header("Content-Type", "application/json") + .json(&claims) + .send() + .await + .wrap_err("Sending request to JWT issuer")?; + #[derive(serde::Deserialize)] + struct Response { + token: String, + } + let response_deserialized: Response = response + .json() + .await + .wrap_err("Getting token from JWT issuer's response")?; + Ok(response_deserialized.token) +} diff --git a/src/flakehub_client.rs b/src/flakehub_client.rs new file mode 100644 index 00000000..d3bde310 --- /dev/null +++ b/src/flakehub_client.rs @@ -0,0 +1,113 @@ +use std::str::FromStr; + +use color_eyre::eyre::{eyre, Context, Result}; +use http::{StatusCode}; +use reqwest::{header::HeaderMap, Response}; +use uuid::Uuid; + +use crate::release_metadata::ReleaseMetadata; + + +pub struct FlakeHubClient { + host: url::Url, + bearer_token: String, + client: reqwest::Client, +} + +pub struct Tarball { + pub hash_base64: String, + pub bytes: Vec, +} + +// TODO(colemickens): static init +pub fn flakehub_headers() -> HeaderMap { + let mut header_map = HeaderMap::new(); + + header_map.insert( + reqwest::header::CONTENT_TYPE, + reqwest::header::HeaderValue::from_str("application/json").unwrap(), + ); + // TODO(colemickens): tube > ngrok, remove + header_map.insert( + reqwest::header::HeaderName::from_static("ngrok-skip-browser-warning"), + reqwest::header::HeaderValue::from_str("please").unwrap(), + ); + header_map +} + +impl FlakeHubClient { + pub fn new(host: url::Url, token: String) -> Result { + let builder = reqwest::ClientBuilder::new(); + builder.user_agent("flakehub-push"); + + let client = builder.build()?; + + let client = Self { + client: client, + bearer_token: token, + host: host, + }; + + Ok(client) + } + pub async fn release_stage(&self, upload_name: String, release_version: String, release_metadata: ReleaseMetadata, tarball: Tarball) -> Result { + let flake_tarball_len = tarball.bytes.len(); + let flake_tarball_hash_base64 = tarball.hash_base64; + let relative_url = &format!("upload/{upload_name}/{release_version}/{flake_tarball_len}/{flake_tarball_hash_base64}"); + + + let release_metadata_post_url = format!("{}/{}", self.host, relative_url); + // TODO(colemickens): better join + + tracing::debug!( + url = %release_metadata_post_url, + "Computed release metadata POST URL" + ); + + let response = self.client + .post(release_metadata_post_url) + .bearer_auth(&self.bearer_token) + .headers(flakehub_headers()) + .json(&release_metadata) + .send() + .await + .unwrap(); + + Ok(response) + } + + pub async fn release_publish(&self, release_uuidv7: Uuid) -> Result<()> { + let publish_post_url = format!("{}/publish/{}", self.host, release_uuidv7); + // TODO(colemickens): fix url joining + + + tracing::debug!(url = %publish_post_url, "Computed publish POST URL"); + + let publish_response = self.client + .post(publish_post_url) + .bearer_auth(&self.bearer_token) + .headers(flakehub_headers()) + .send() + .await + .wrap_err("Publishing release")?; + + let publish_response_status = publish_response.status(); + tracing::trace!( + status = tracing::field::display(publish_response_status), + "Got publish POST response" + ); + + if publish_response_status != 200 { + return Err(eyre!( + "\ + Status {publish_response_status} from publish POST\n\ + {}\ + ", + String::from_utf8_lossy(&publish_response.bytes().await.unwrap()) + )); + } + + // TODO: return the actual response object? + Ok(()) + } +} \ No newline at end of file diff --git a/src/github/graphql/mod.rs b/src/github/graphql/mod.rs index 596726a3..2f5a2247 100644 --- a/src/github/graphql/mod.rs +++ b/src/github/graphql/mod.rs @@ -130,6 +130,7 @@ impl GithubGraphqlDataQuery { .collect(); Ok(GithubGraphqlDataResult { + revision: revision.to_string(), rev_count, spdx_identifier, project_id, @@ -141,6 +142,7 @@ impl GithubGraphqlDataQuery { #[derive(Debug)] pub(crate) struct GithubGraphqlDataResult { + pub(crate) revision: String, pub(crate) rev_count: i64, pub(crate) spdx_identifier: Option, pub(crate) project_id: i64, diff --git a/src/gitlab/mod.rs b/src/gitlab/mod.rs new file mode 100644 index 00000000..95b70b3b --- /dev/null +++ b/src/gitlab/mod.rs @@ -0,0 +1,14 @@ +use color_eyre::eyre::{eyre, WrapErr}; + +#[tracing::instrument(skip_all, fields(audience = tracing::field::Empty))] +pub(crate) async fn get_runner_bearer_token(host: &url::Url) -> color_eyre::Result { + // github allows you to at-runtime change the audience of the token + // gitlab requires job-level audience/token config, and makes it available via envvar + + let maybe_token = std::env::var("GITLAB_JWT_ID_TOKEN"); + let token = maybe_token.wrap_err("Failed to get a JWT from GitLab. You must configure id_token in the jobs.")?; + + // TODO(colemickens): valdiate the audience of the gitlab token matches `host` + + Ok(token) +} diff --git a/src/main.rs b/src/main.rs index 39c6830e..4eb11ff2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,16 +1,31 @@ -use std::{fmt::Display, io::IsTerminal}; +use std::{fmt::Display, io::IsTerminal, path::{Path, PathBuf}, process::ExitCode}; use clap::Parser; +use color_eyre::eyre::{eyre, Result, WrapErr}; use error::Error; +use http::StatusCode; +use uuid::Uuid; + +use crate::{flakehub_client::FlakeHubClient, github::graphql::GithubGraphqlDataQuery, push_context::PushContext, release_metadata::ReleaseMetadata}; mod cli; mod error; mod flake_info; +mod flakehub_auth_fake; +mod flakehub_client; mod github; -mod push; +mod gitlab; +mod push_context; mod release_metadata; +mod s3; + +const DEFAULT_ROLLING_PREFIX: &str = "0.1"; + +pub(crate) fn build_http_client() -> reqwest::ClientBuilder { + reqwest::Client::builder().user_agent("flakehub-push") +} #[tokio::main] -async fn main() -> color_eyre::Result { +async fn main() -> Result { color_eyre::config::HookBuilder::default() .issue_url(concat!(env!("CARGO_PKG_REPOSITORY"), "/issues/new")) .add_issue_metadata("version", env!("CARGO_PKG_VERSION")) @@ -36,15 +51,88 @@ async fn main() -> color_eyre::Result { let cli = cli::FlakeHubPushCli::parse(); cli.instrumentation.setup()?; - match cli.execute().await { - Ok(exit) => Ok(exit), - Err(error) => { - if let Some(known_error) = error.downcast_ref::() { - known_error.maybe_github_actions_annotation() + let ctx: PushContext = PushContext::from_cli_and_env(&mut cli).await?; + drop(cli); // drop cli so we force ourselves to use ctx + + let fhclient = FlakeHubClient::new(ctx.flakehub_host, ctx.auth_token)?; + + // "upload.rs" - stage the release + let stage_response = fhclient.release_stage( + ctx.upload_name, + ctx.release_version, + ctx.metadata, + ctx.tarball, + ).await?; + + // handle the response here, rather than in client, so we can do special behavior + // TODO(colemickens/review): move this intoo release_create with another flag? + // or "ReleaseOptions->error_on_conflict" and have a way to override options for a release + // creation?? + let release_metadata_post_response_status = stage_response.status(); + tracing::trace!( + status = tracing::field::display(release_metadata_post_response_status), + "Got release metadata POST response" + ); + + match release_metadata_post_response_status { + StatusCode::OK => (), + StatusCode::CONFLICT => { + tracing::info!( + "Release for revision `{revision}` of {upload_name}/{release_version} already exists; flakehub-push will not upload it again", + revision = ctx.metadata.revision, + upload_name = ctx.upload_name, + release_version = ctx.release_version, + ); + if ctx.error_if_release_conflicts { + return Err(Error::Conflict { + upload_name: ctx.upload_name, + release_version: ctx.release_version, + })?; + } else { + return Ok(()); } - Err(error) } + StatusCode::UNAUTHORIZED => { + let body = stage_response.bytes().await?; + let message = serde_json::from_slice::(&body)?; + + return Err(Error::Unauthorized(message))?; + } + _ => { + let body = stage_response.bytes().await?; + let message = serde_json::from_slice::(&body)?; + return Err(eyre!( + "\ + Status {release_metadata_post_response_status} from metadata POST\n\ + {}\ + ", + message + )); + } + } + + #[derive(serde::Deserialize)] + struct StageResult { + s3_upload_url: String, + uuid: Uuid, } + let stage_result: StageResult = stage_response + .json() + .await + .wrap_err("Decoding release metadata POST response")?; + + // upload tarball to s3 + s3::upload_release_to_s3(stage_result.s3_upload_url, ctx.tarball).await?; + + // "publish.rs" - publish the release after upload + let publish_result = fhclient.release_publish(stage_result.uuid).await?; + + tracing::info!( + "Successfully released new version of {}/{}", ctx.upload_name, ctx.release_version + ); + + let exitcode = ExitCode::SUCCESS; + return Ok(exitcode); } #[derive(Debug, Clone, Copy, clap::ValueEnum, serde::Serialize, serde::Deserialize)] @@ -67,7 +155,3 @@ impl Display for Visibility { } } } - -pub(crate) fn build_http_client() -> reqwest::ClientBuilder { - reqwest::Client::builder().user_agent("flakehub-push") -} diff --git a/src/push.rs b/src/push.rs deleted file mode 100644 index 3732a0c3..00000000 --- a/src/push.rs +++ /dev/null @@ -1,388 +0,0 @@ -use color_eyre::eyre::{eyre, WrapErr}; -use reqwest::{header::HeaderMap, StatusCode}; -use std::{ - path::{Path, PathBuf}, - str::FromStr, -}; -use tokio::io::AsyncWriteExt; -use uuid::Uuid; - -use crate::{ - build_http_client, - error::Error, - flake_info::{check_flake_evaluates, get_flake_metadata, get_flake_outputs, get_flake_tarball}, - release_metadata::ReleaseMetadata, - Visibility, -}; - -const DEFAULT_ROLLING_PREFIX: &str = "0.1"; - -#[tracing::instrument( - skip_all, - fields( - %host, - flake_root, - subdir, - revision, - revision_count, - repository, - upload_name, - mirror, - %visibility, - tag, - rolling, - rolling_minor, - labels = labels.join(","), - mirror, - spdx_expression, - error_if_release_conflicts, - include_output_paths, - project_id, - owner_id, - ) -)] -#[allow(clippy::too_many_arguments)] -pub(crate) async fn push_new_release( - host: &url::Url, - upload_bearer_token: &str, - flake_root: &Path, - subdir: &Path, - revision: String, - revision_count: usize, - upload_name: String, - mirror: bool, - visibility: Visibility, - tag: Option, - rolling: bool, - rolling_minor: Option, - labels: Vec, - spdx_expression: Option, - error_if_release_conflicts: bool, - include_output_paths: bool, -) -> color_eyre::Result<()> { - let span = tracing::Span::current(); - span.record("upload_name", tracing::field::display(upload_name.clone())); - - let rolling_prefix_or_tag = match (rolling_minor.as_ref(), tag) { - (Some(_), _) if !rolling => { - return Err(eyre!( - "You must enable `rolling` to upload a release with a specific `rolling-minor`." - )); - } - (Some(minor), _) => format!("0.{minor}"), - (None, _) if rolling => DEFAULT_ROLLING_PREFIX.to_string(), - (None, Some(tag)) => { - let version_only = tag.strip_prefix('v').unwrap_or(&tag); - // Ensure the version respects semver - semver::Version::from_str(version_only).wrap_err_with(|| eyre!("Failed to parse version `{tag}` as semver, see https://semver.org/ for specifications"))?; - tag - } - (None, None) => { - return Err(eyre!("Could not determine tag or rolling minor version, `--tag`, `GITHUB_REF_NAME`, or `--rolling-minor` must be set")); - } - }; - - tracing::info!("Preparing release of {upload_name}/{rolling_prefix_or_tag}"); - - let tempdir = tempfile::Builder::new() - .prefix("flakehub_push") - .tempdir() - .wrap_err("Creating tempdir")?; - - let flake_dir = flake_root.join(subdir); - - check_flake_evaluates(&flake_dir) - .await - .wrap_err("Checking flake evaluates")?; - let flake_metadata = get_flake_metadata(&flake_dir) - .await - .wrap_err("Getting flake metadata")?; - tracing::debug!("Got flake metadata: {:?}", flake_metadata); - - // FIXME: bail out if flake_metadata denotes a dirty tree. - - let flake_locked_url = flake_metadata - .get("url") - .and_then(serde_json::Value::as_str) - .ok_or_else(|| { - eyre!("Could not get `url` attribute from `nix flake metadata --json` output") - })?; - tracing::debug!("Locked URL = {}", flake_locked_url); - let flake_metadata_value_path = flake_metadata - .get("path") - .and_then(serde_json::Value::as_str) - .ok_or_else(|| { - eyre!("Could not get `path` attribute from `nix flake metadata --json` output") - })?; - let flake_metadata_value_resolved_dir = flake_metadata - .pointer("/resolved/dir") - .and_then(serde_json::Value::as_str); - - let flake_outputs = get_flake_outputs(flake_locked_url, include_output_paths).await?; - tracing::debug!("Got flake outputs: {:?}", flake_outputs); - - let source = match flake_metadata_value_resolved_dir { - Some(flake_metadata_value_resolved_dir) => { - Path::new(flake_metadata_value_path).join(flake_metadata_value_resolved_dir) - } - None => PathBuf::from(flake_metadata_value_path), - }; - span.record("source", tracing::field::display(source.clone().display())); - tracing::debug!("Found source"); - - if flake_dir.join("flake.lock").exists() { - let output = tokio::process::Command::new("nix") - .arg("flake") - .arg("metadata") - .arg("--json") - .arg("--no-update-lock-file") - .arg(&flake_dir) - .output() - .await - .wrap_err_with(|| { - eyre!( - "Failed to execute `nix flake metadata --json --no-update-lock-file {}`", - flake_dir.display() - ) - })?; - - if !output.status.success() { - let command = format!( - "nix flake metadata --json --no-update-lock-file {}", - flake_dir.display(), - ); - let msg = format!( - "\ - Failed to execute command `{command}`{maybe_status} \n\ - stdout: {stdout}\n\ - stderr: {stderr}\n\ - ", - stdout = String::from_utf8_lossy(&output.stdout), - stderr = String::from_utf8_lossy(&output.stderr), - maybe_status = if let Some(status) = output.status.code() { - format!(" with status {status}") - } else { - String::new() - } - ); - return Err(eyre!(msg))?; - } - } - - let last_modified = if let Some(last_modified) = flake_metadata.get("lastModified") { - last_modified.as_u64().ok_or_else(|| { - eyre!("`nix flake metadata --json` does not have a integer `lastModified` field") - })? - } else { - return Err(eyre!( - "`nix flake metadata` did not return a `lastModified` attribute" - )); - }; - tracing::debug!("lastModified = {}", last_modified); - - let flake_tarball = get_flake_tarball(&source, last_modified) - .await - .wrap_err("Making release tarball")?; - - let flake_tarball_len: usize = flake_tarball.len(); - let flake_tarball_hash = { - let mut context = ring::digest::Context::new(&ring::digest::SHA256); - context.update(&flake_tarball); - context.finish() - }; - let flake_tarball_hash_base64 = { - // TODO: Use URL_SAFE_NO_PAD - use base64::{engine::general_purpose::STANDARD, Engine as _}; - STANDARD.encode(flake_tarball_hash) - }; - tracing::debug!( - flake_tarball_len, - flake_tarball_hash_base64, - "Got tarball metadata" - ); - - let flake_tarball_path = tempdir.path().join("release.tar.gz"); - let mut tempfile = tokio::fs::File::create(&flake_tarball_path) - .await - .wrap_err("Creating release.tar.gz")?; - tempfile - .write_all(&flake_tarball) - .await - .wrap_err("Writing compressed tarball to tempfile")?; - - let release_metadata = ReleaseMetadata::build( - &source, - subdir, - revision, - revision_count, - flake_metadata, - flake_outputs, - upload_name.clone(), - mirror, - visibility, - labels, - spdx_expression, - ) - .await - .wrap_err("Building release metadata")?; - - let flakehub_client = build_http_client().build()?; - - let rolling_minor_with_postfix_or_tag = if rolling_minor.is_some() || rolling { - format!( - "{rolling_prefix_or_tag}.{}+rev-{}", - release_metadata.commit_count, release_metadata.revision - ) - } else { - rolling_prefix_or_tag.to_string() // This will always be the tag since `self.rolling_prefix` was empty. - }; - - let release_metadata_post_url = host.join(&format!("upload/{upload_name}/{rolling_minor_with_postfix_or_tag}/{flake_tarball_len}/{flake_tarball_hash_base64}"))?; - tracing::debug!( - url = %release_metadata_post_url, - "Computed release metadata POST URL" - ); - - let flakehub_headers = { - let mut header_map = HeaderMap::new(); - - header_map.insert( - reqwest::header::CONTENT_TYPE, - reqwest::header::HeaderValue::from_str("application/json").unwrap(), - ); - header_map.insert( - reqwest::header::HeaderName::from_static("ngrok-skip-browser-warning"), - reqwest::header::HeaderValue::from_str("please").unwrap(), - ); - header_map - }; - - let release_metadata_post_response = flakehub_client - .post(release_metadata_post_url) - .bearer_auth(upload_bearer_token) - .headers(flakehub_headers.clone()) - .json(&release_metadata) - .send() - .await - .wrap_err("Sending release metadata")?; - - let release_metadata_post_response_status = release_metadata_post_response.status(); - tracing::trace!( - status = tracing::field::display(release_metadata_post_response_status), - "Got release metadata POST response" - ); - - match release_metadata_post_response_status { - StatusCode::OK => (), - StatusCode::CONFLICT => { - tracing::info!( - "Release for revision `{revision}` of {upload_name}/{rolling_prefix_or_tag} already exists; flakehub-push will not upload it again", - revision = release_metadata.revision - ); - if error_if_release_conflicts { - return Err(Error::Conflict { - upload_name, - rolling_prefix_or_tag, - })?; - } else { - return Ok(()); - } - } - StatusCode::UNAUTHORIZED => { - let body = &release_metadata_post_response.bytes().await?; - let message = serde_json::from_slice::(body)?; - - return Err(Error::Unauthorized(message))?; - } - _ => { - let body = &release_metadata_post_response.bytes().await?; - let message = serde_json::from_slice::(body)?; - return Err(eyre!( - "\ - Status {release_metadata_post_response_status} from metadata POST\n\ - {}\ - ", - message - )); - } - } - - #[derive(serde::Deserialize)] - struct Result { - s3_upload_url: String, - uuid: Uuid, - } - - let release_metadata_post_result: Result = release_metadata_post_response - .json() - .await - .wrap_err("Decoding release metadata POST response")?; - - let tarball_put_response = flakehub_client - .put(release_metadata_post_result.s3_upload_url) - .headers({ - let mut header_map = HeaderMap::new(); - header_map.insert( - reqwest::header::CONTENT_LENGTH, - reqwest::header::HeaderValue::from_str(&format!("{}", flake_tarball_len)).unwrap(), - ); - header_map.insert( - reqwest::header::HeaderName::from_static("x-amz-checksum-sha256"), - reqwest::header::HeaderValue::from_str(&flake_tarball_hash_base64).unwrap(), - ); - header_map.insert( - reqwest::header::CONTENT_TYPE, - reqwest::header::HeaderValue::from_str("application/gzip").unwrap(), - ); - header_map - }) - .body(flake_tarball) - .send() - .await - .wrap_err("Sending tarball PUT")?; - - let tarball_put_response_status = tarball_put_response.status(); - tracing::trace!( - status = tracing::field::display(tarball_put_response_status), - "Got tarball PUT response" - ); - if !tarball_put_response_status.is_success() { - return Err(eyre!( - "Got {tarball_put_response_status} status from PUT request" - )); - } - - // Make the release we just uploaded visible. - let publish_post_url = host.join(&format!("publish/{}", release_metadata_post_result.uuid))?; - tracing::debug!(url = %publish_post_url, "Computed publish POST URL"); - - let publish_response = flakehub_client - .post(publish_post_url) - .bearer_auth(upload_bearer_token) - .headers(flakehub_headers) - .send() - .await - .wrap_err("Publishing release")?; - - let publish_response_status = publish_response.status(); - tracing::trace!( - status = tracing::field::display(publish_response_status), - "Got publish POST response" - ); - - if publish_response_status != 200 { - return Err(eyre!( - "\ - Status {publish_response_status} from publish POST\n\ - {}\ - ", - String::from_utf8_lossy(&publish_response.bytes().await.unwrap()) - )); - } - - tracing::info!( - "Successfully released new version of {upload_name}/{rolling_minor_with_postfix_or_tag}" - ); - - Ok(()) -} diff --git a/src/push_context.rs b/src/push_context.rs new file mode 100644 index 00000000..665a34d9 --- /dev/null +++ b/src/push_context.rs @@ -0,0 +1,416 @@ +use std::{ + collections::HashSet, + path::{Path, PathBuf}, + str::FromStr, +}; + +use color_eyre::eyre::{eyre, Context, Result}; +use spdx::Expression; + +use crate::{ + cli::FlakeHubPushCli, + flake_info, flakehub_auth_fake, + flakehub_client::Tarball, + github::graphql::{ + GithubGraphqlDataQuery, GithubGraphqlDataResult, MAX_LABEL_LENGTH, MAX_NUM_TOTAL_LABELS, + }, + release_metadata::{ReleaseMetadata, RevisionInfo}, + DEFAULT_ROLLING_PREFIX, +}; + +pub struct GitContext { + pub spdx_expression: Option, + pub repo_topics: Vec, + pub revision_info: RevisionInfo, +} + +impl GitContext { + pub fn from_cli_and_github( + cli: &FlakeHubPushCli, + github_graphql_data_result: &GithubGraphqlDataResult, + ) -> Result { + // do everything we need to get data from github here + + // step: validate spdx, backfill from GitHub API + let spdx_expression = if cli.spdx_expression.0.is_none() { + if let Some(spdx_string) = &github_graphql_data_result.spdx_identifier { + tracing::debug!("Recieved SPDX identifier `{}` from GitHub API", spdx_string); + let parsed = spdx::Expression::parse(spdx_string) + .wrap_err("Invalid SPDX license identifier reported from the GitHub API, either you are using a non-standard license or GitHub has returned a value that cannot be validated")?; + //span.record("spdx_expression", tracing::field::display(&parsed)); + Some(parsed) + } else { + None + } + } else { + // Provide the user notice if the SPDX expression passed differs from the one detected on GitHub -- It's probably something they care about. + if github_graphql_data_result.spdx_identifier + != cli.spdx_expression.0.as_ref().map(|v| v.to_string()) + { + tracing::warn!( + "SPDX identifier `{}` was passed via argument, but GitHub's API suggests it may be `{}`", + cli.spdx_expression.0.as_ref().map(|v| v.to_string()).unwrap_or_else(|| "None".to_string()), + github_graphql_data_result.spdx_identifier.clone().unwrap_or_else(|| "None".to_string()), + ) + } + cli.spdx_expression.0 + }; + + let ctx = GitContext { + spdx_expression: spdx_expression, + repo_topics: github_graphql_data_result.topics, + revision_info: RevisionInfo { + // TODO(colemickens): type coherency here... :/ (as is bad) + commit_count: Some(github_graphql_data_result.rev_count as usize), + revision: github_graphql_data_result.revision, + }, + }; + Ok(ctx) + } +} + +pub(crate) struct PushContext { + pub(crate) flakehub_host: url::Url, + pub(crate) auth_token: String, + + // url components + pub(crate) upload_name: String, // {org}/{project} + pub(crate) release_version: String, + + // internal behavior changes + pub(crate) error_if_release_conflicts: bool, + + // the goods + pub(crate) metadata: ReleaseMetadata, + pub(crate) tarball: Tarball, +} + +impl PushContext { + pub async fn from_cli_and_env(cli: &mut FlakeHubPushCli) -> Result { + // Take the opportunity to be able to populate/encrich data from the GitHub API + // this is used to augment user/discovered data, and is used for the faked JWT for local flakehub-push testing + + let client = reqwest::Client::new(); + + let is_github = std::env::var("GITHUB_ACTION").ok().is_some(); + let is_gitlab = std::env::var("GITLAB_CI").ok().is_some(); + + // "backfill" env vars from the environment, for the first time, anyway... + if is_github { + cli.backfill_from_github_env(); + } + if is_gitlab { + cli.backfill_from_gitlab_env(); + } + + let visibility = match (cli.visibility_alt, cli.visibility) { + (Some(v), _) => v, + (None, Some(v)) => v, + (None, None) => return Err(eyre!( + "Could not determine the flake's desired visibility. Use `--visibility` to set this to one of the following: public, unlisted, private.", + )), + }; + + // STEP: determine and check 'repository' and 'upload_name' + // If the upload name is supplied by the user, ensure that it contains exactly + // one slash and no whitespace. Default to the repository name. + let Some(ref repository) = cli.repository.0 else { + return Err(eyre!("Could not determine repository name, pass `--repository` formatted like `determinatesystems/flakehub-push`")); + }; + let upload_name = if let Some(ref name) = cli.name.0 { + let num_slashes = name.matches('/').count(); + + if num_slashes == 0 + || num_slashes > 1 + || !name.is_ascii() + || name.contains(char::is_whitespace) + { + return Err(eyre!("The argument `--name` must be in the format of `owner-name/repo-name` and cannot contain whitespace or other special characters")); + } else { + name.to_string() + } + } else { + repository.clone() + }; + let mut repository_split = repository.split('/'); + let project_owner = repository_split + .next() + .ok_or_else(|| eyre!("Could not determine owner, pass `--repository` formatted like `determinatesystems/flakehub-push`"))? + .to_string(); + let project_name = repository_split.next() + .ok_or_else(|| eyre!("Could not determine project, pass `--repository` formatted like `determinatesystems/flakehub-push`"))? + .to_string(); + if repository_split.next().is_some() { + Err(eyre!("Could not determine the owner/project, pass `--repository` formatted like `determinatesystems/flakehub-push`. The passed value has too many slashes (/) to be a valid repository"))?; + } + + // TODO(colemickens): pretty sure there's a better way to write this: + let local_git_root = (match &cli.git_root.0 { + Some(gr) => Ok(gr.to_owned()), + None => std::env::current_dir().map(PathBuf::from) + }).wrap_err("Could not determine current `git_root`. Pass `--git-root` or set `FLAKEHUB_PUSH_GIT_ROOT`, or run `flakehub-push` with the git root as the current working directory")?; + + let local_git_root = local_git_root + .canonicalize() + .wrap_err("Failed to canonicalize `--git-root` argument")?; + let local_rev_info = RevisionInfo::from_git_root(&local_git_root)?; + + let (token, git_ctx) = match (is_github, is_gitlab, &cli.jwt_issuer_uri.0) { + (true, false, None) => { + let github_token = cli + .github_token + .0 + .clone() + .expect("failed to get github token when running in GitHub Actions"); + + let github_graphql_data_result = GithubGraphqlDataQuery::get( + &client, + &github_token, + &project_owner, + &project_name, + &local_rev_info.revision, + ) + .await?; + + let git_ctx = GitContext::from_cli_and_github(&cli, &github_graphql_data_result)?; + + let token = crate::github::get_actions_id_bearer_token(&cli.host) + .await + .wrap_err("Getting upload bearer token from GitHub")?; + + (token, git_ctx) + } + (false, true, None) => { + let token = crate::gitlab::get_runner_bearer_token(&cli.host) + .await + .wrap_err("Getting upload bearer token from GitLab")?; + // let git_ctx = GitContext::from_cli_and_gitlab(cli); // TODO: !!!!! + let git_ctx = todo!(); + (token, git_ctx) + } + (false, false, Some(u)) => { + // local, fake github, we need the ... github ids to get a faked token + let github_token = cli + .github_token + .0 + .clone() + .expect("failed to get github token when running locally"); + + let github_graphql_data_result = GithubGraphqlDataQuery::get( + &client, + &github_token, + &project_owner, + &project_name, + &local_rev_info.revision, + ) + .await?; + + let git_ctx: GitContext = + GitContext::from_cli_and_github(&cli, &github_graphql_data_result)?; + + let token = flakehub_auth_fake::get_fake_bearer_token( + u, + &project_owner, + &project_name, + github_graphql_data_result, + ) + .await?; + (token, git_ctx) + } + (_, _, Some(_)) => { + // we're in GitHub or GitLab and jwt_issuer_uri was specified, invalid + return Err(eyre!( + "specifying the jwt_issuer_uri when running in GitHub or GitLab is invalid" + )); + } + _ => { + // who knows what's going on, invalid + return Err(eyre!("can't determine execution environment")); + } + }; + + // STEP: resolve "subdir" (use --directory flag from cli) + // TODO(colemickens): do we really need both in our source, "subdir" is part of release_metadata tho + // NOTE(colemickens,self): flake_dir can probably be evne more "intenral" only used for tarring/flake_info + let subdir = if let Some(directory) = &cli.directory.0 { + let absolute_directory = if directory.is_absolute() { + directory.clone() + } else { + local_git_root.join(directory) + }; + let canonical_directory = absolute_directory + .canonicalize() + .wrap_err("Failed to canonicalize `--directory` argument")?; + + Path::new( + canonical_directory + .strip_prefix(local_git_root.clone()) + .wrap_err( + "Specified `--directory` was not a directory inside the `--git-root`", + )?, + ) + .into() + } else { + PathBuf::new() + }; + + let rolling_prefix_or_tag = match (cli.rolling_minor.0.as_ref(), &cli.tag.0) { + (Some(_), _) if !cli.rolling => { + return Err(eyre!( + "You must enable `rolling` to upload a release with a specific `rolling-minor`." + )); + } + (Some(minor), _) => format!("0.{minor}"), + (None, _) if cli.rolling => DEFAULT_ROLLING_PREFIX.to_string(), + (None, Some(tag)) => { + let version_only = tag.strip_prefix('v').unwrap_or(&tag); + // Ensure the version respects semver + semver::Version::from_str(version_only).wrap_err_with(|| eyre!("Failed to parse version `{tag}` as semver, see https://semver.org/ for specifications"))?; + tag.to_string() + } + (None, None) => { + return Err(eyre!("Could not determine tag or rolling minor version, `--tag`, `GITHUB_REF_NAME`, or `--rolling-minor` must be set")); + } + }; + + // TODO(don't use revision_info, maybe, so we don't have this extra shadowy local var) + let Some(commit_count) = git_ctx.revision_info.commit_count else { + return Err(eyre!("Could not determine commit count, this is normally determined via the `--git-root` argument or via the GitHub API")); + }; + + let rolling_minor_with_postfix_or_tag = if cli.rolling_minor.0.is_some() || cli.rolling { + format!( + "{rolling_prefix_or_tag}.{}+rev-{}", + commit_count, git_ctx.revision_info.revision + ) + } else { + rolling_prefix_or_tag.to_string() // This will always be the tag since `self.rolling_prefix` was empty. + }; + + // STEP: calculate labels + // - they can specify: extra_labels on cli + // TODO: merge execution_environment's labels + cli extra_labels + let merged_labels = { + let mut labels: HashSet<_> = cli + .extra_labels + .clone() + .into_iter() + .filter(|v| !v.is_empty()) + .collect(); + let extra_tags: HashSet<_> = cli + .extra_tags + .clone() + .into_iter() + .filter(|v| !v.is_empty()) + .collect(); + + if !extra_tags.is_empty() { + let message = "`extra-tags` is deprecated and will be removed in the future. Please use `extra-labels` instead."; + tracing::warn!("{message}"); + + // TODO(colemickens): restore this cleanly, have something on git_ctx that logs? rename git_ctx? + // if is_github_actions { + // println!("::warning::{message}"); + // } + + if labels.is_empty() { + labels = extra_tags; + } else { + let message = + "Both `extra-tags` and `extra-labels` were set; `extra-tags` will be ignored."; + tracing::warn!("{message}"); + + // TODO(colemickens): restore this cleanly, have something on git_ctx that logs? rename git_ctx? + // if is_github_actions { + // println!("::warning::{message}"); + // } + } + } + + // Get the "topic" labels from git_ctx, extend local mut labels + let topics = git_ctx.repo_topics; + labels = labels + .into_iter() + .chain(topics.iter().cloned()) + .collect::>(); + + // Here we merge explicitly user-supplied labels and the labels ("topics") + // associated with the repo. Duplicates are excluded and all + // are converted to lower case. + let merged_labels: Vec = labels + .into_iter() + .take(MAX_NUM_TOTAL_LABELS) + .map(|s| s.trim().to_lowercase()) + .filter(|t: &String| { + !t.is_empty() + && t.len() <= MAX_LABEL_LENGTH + && t.chars().all(|c| c.is_alphanumeric() || c == '-') + }) + .collect(); + + merged_labels + }; + + // flake_dir is an absolute path of flake_root(aka git_root)/subdir + let flake_dir = local_git_root.join(&subdir); + // TODO: depending on what the user called us with, this flake_dir isn't even necessarily canonicalized, is this a sec/traversal issue? + + // FIXME: bail out if flake_metadata denotes a dirty tree. + // (Todo make sure invariant checks are right, de-duped properly) + let flake_metadata = flake_info::FlakeMetadata::from_dir(&flake_dir) + .await + .wrap_err("Getting flake metadata")?; + tracing::debug!("Got flake metadata: {:?}", flake_metadata); + + let flake_outputs = flake_metadata.outputs(cli.include_output_paths).await?; + tracing::debug!("Got flake outputs: {:?}", flake_outputs); + + let description = flake_metadata + .metadata_json + .get("description") + .and_then(serde_json::Value::as_str) + .map(|s| s.to_string()); + + let readme = flake_metadata.get_readme_contents().await?; + + let release_metadata = ReleaseMetadata { + commit_count: commit_count, + description: description, + outputs: flake_outputs.0, + raw_flake_metadata: flake_metadata.metadata_json.clone(), + readme: readme, + repo: repository.to_string(), + revision: git_ctx.revision_info.revision, + visibility: visibility, + mirrored: cli.mirror, + source_subdirectory: Some( + subdir + .to_str() + .map(|d| d.to_string()) + .ok_or(eyre!("Directory {:?} is not a valid UTF-8 string", subdir))?, + ), + spdx_identifier: git_ctx.spdx_expression, + labels: merged_labels, + }; + + let flake_tarball = flake_metadata + .flake_tarball() + //.await //weird, tar is not async? + .wrap_err("Making release tarball")?; + + let ctx = Self { + flakehub_host: cli.host.clone(), + auth_token: token, + + upload_name: upload_name, + release_version: rolling_minor_with_postfix_or_tag, + + error_if_release_conflicts: cli.error_on_conflict, + + // this is the payload we send + metadata: release_metadata, + tarball: flake_tarball, + }; + Ok(ctx) + } +} diff --git a/src/release_metadata.rs b/src/release_metadata.rs index ce905594..927122f7 100644 --- a/src/release_metadata.rs +++ b/src/release_metadata.rs @@ -5,6 +5,7 @@ use crate::Visibility; const README_FILENAME_LOWERCASE: &str = "readme.md"; +// TODO(colemickens): fhlib? #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub(crate) struct ReleaseMetadata { pub(crate) commit_count: usize, diff --git a/src/s3.rs b/src/s3.rs new file mode 100644 index 00000000..d8ed63d6 --- /dev/null +++ b/src/s3.rs @@ -0,0 +1,44 @@ +use color_eyre::eyre::{eyre, Result, WrapErr}; +use reqwest::header::HeaderMap; + +use crate::flakehub_client::Tarball; + +pub async fn upload_release_to_s3(presigned_s3_url: String, tarball: Tarball) -> Result<()> { + let client = reqwest::Client::new(); + let tarball_put_response = client + .put(presigned_s3_url) + .headers({ + let mut header_map = HeaderMap::new(); + header_map.insert( + reqwest::header::CONTENT_LENGTH, + reqwest::header::HeaderValue::from_str(&format!("{}", tarball.bytes.len())).unwrap(), + ); + header_map.insert( + reqwest::header::HeaderName::from_static("x-amz-checksum-sha256"), + reqwest::header::HeaderValue::from_str(&tarball.hash_base64).unwrap(), + ); + header_map.insert( + reqwest::header::CONTENT_TYPE, + reqwest::header::HeaderValue::from_str("application/gzip").unwrap(), + ); + header_map + }) + .body(tarball.bytes) + .send() + .await + .wrap_err("Sending tarball PUT")?; + + let tarball_put_response_status = tarball_put_response.status(); + tracing::trace!( + status = tracing::field::display(tarball_put_response_status), + "Got tarball PUT response" + ); + if !tarball_put_response_status.is_success() { + return Err(eyre!( + "Got {tarball_put_response_status} status from PUT request" + )); + } + + Ok(()) +} + From b3e67ba2860c016f155a3f147bfe2ad5d3afaf10 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Tue, 16 Apr 2024 07:28:58 -0700 Subject: [PATCH 03/27] WIP: fixup compile + auto cargo fmt sorry --- src/flake_info.rs | 56 +++++++++++++++++++++++------------------- src/flakehub_client.rs | 42 +++++++++++++++++-------------- src/main.rs | 40 +++++++++++++++++++----------- src/push_context.rs | 6 ++--- 4 files changed, 83 insertions(+), 61 deletions(-) diff --git a/src/flake_info.rs b/src/flake_info.rs index b741fedb..f7f94f22 100644 --- a/src/flake_info.rs +++ b/src/flake_info.rs @@ -1,4 +1,7 @@ -use std::{io::Write, path::{Path, PathBuf}}; +use std::{ + io::Write, + path::{Path, PathBuf}, +}; use color_eyre::eyre::{eyre, Result, WrapErr}; use serde::Deserialize; @@ -62,7 +65,6 @@ const README_FILENAME_LOWERCASE: &str = "readme.md"; // Ok(()) // } - #[derive(Debug)] pub struct FlakeMetadata { pub(crate) source_dir: std::path::PathBuf, @@ -91,12 +93,13 @@ impl FlakeMetadata { ) })?; - let metadata_json: serde_json::Value = serde_json::from_slice(&output.stdout).wrap_err_with(|| { - eyre!( - "Parsing `nix flake metadata --json {}` as JSON", - directory.display() - ) - })?; + let metadata_json: serde_json::Value = serde_json::from_slice(&output.stdout) + .wrap_err_with(|| { + eyre!( + "Parsing `nix flake metadata --json {}` as JSON", + directory.display() + ) + })?; /* if flake_dir.join("flake.lock").exists() { @@ -140,7 +143,7 @@ impl FlakeMetadata { */ // determine flake's store (sub)dir: - + let flake_locked_url = metadata_json .get("url") .and_then(serde_json::Value::as_str) @@ -187,39 +190,41 @@ impl FlakeMetadata { )); }; tracing::debug!("lastModified = {}", last_modified); - + let mut tarball_builder = tar::Builder::new(vec![]); tarball_builder.follow_symlinks(false); tarball_builder.force_mtime(last_modified); - + tracing::trace!("Creating tarball"); // `tar` works according to the current directory (yay) // So we change dir and restory it after // TODO: Fix this - let source = self.source_dir; // refactor to be known when we create struct with from_dir + let source = &self.source_dir; // refactor to be known when we create struct with from_dir let current_dir = std::env::current_dir().wrap_err("Could not get current directory")?; std::env::set_current_dir( source .parent() .ok_or_else(|| eyre!("Getting parent directory"))?, )?; - let dirname = self.source_dir + let dirname = self + .source_dir .file_name() .ok_or_else(|| eyre!("No file name of directory"))?; tarball_builder .append_dir_all(dirname, dirname) .wrap_err_with(|| eyre!("Adding `{}` to tarball", self.source_dir.display()))?; std::env::set_current_dir(current_dir).wrap_err("Could not set current directory")?; - + let tarball = tarball_builder.into_inner().wrap_err("Creating tarball")?; tracing::trace!("Created tarball, compressing..."); - let mut gzip_encoder = flate2::write::GzEncoder::new(vec![], flate2::Compression::default()); + let mut gzip_encoder = + flate2::write::GzEncoder::new(vec![], flate2::Compression::default()); gzip_encoder .write_all(&tarball[..]) .wrap_err("Adding tarball to gzip")?; let compressed_tarball = gzip_encoder.finish().wrap_err("Creating gzip")?; tracing::trace!("Compressed tarball"); - + let flake_tarball_hash = { let mut context = ring::digest::Context::new(&ring::digest::SHA256); context.update(&compressed_tarball); @@ -230,12 +235,12 @@ impl FlakeMetadata { use base64::{engine::general_purpose::STANDARD, Engine as _}; STANDARD.encode(flake_tarball_hash) }; - + let tarball = Tarball { bytes: compressed_tarball, - hash_base64: flake_tarball_hash_base64 + hash_base64: flake_tarball_hash_base64, }; - + Ok(tarball) } @@ -267,10 +272,12 @@ impl FlakeMetadata { cmd.arg("--json"); cmd.arg("--no-write-lock-file"); cmd.arg(format!("{}#contents", tempdir.path().display())); - let output = cmd - .output() - .await - .wrap_err_with(|| eyre!("Failed to get flake outputs from tarball {}", &self.flake_locked_url))?; + let output = cmd.output().await.wrap_err_with(|| { + eyre!( + "Failed to get flake outputs from tarball {}", + &self.flake_locked_url + ) + })?; if !output.status.success() { return Err(eyre!( @@ -289,9 +296,8 @@ impl FlakeMetadata { })?; Ok(output_json) - } + } - #[tracing::instrument(skip_all, fields(readme_dir))] pub(crate) async fn get_readme_contents(&self) -> Result> { let mut read_dir = tokio::fs::read_dir(&self.source_dir).await?; diff --git a/src/flakehub_client.rs b/src/flakehub_client.rs index d3bde310..b2f12b0d 100644 --- a/src/flakehub_client.rs +++ b/src/flakehub_client.rs @@ -1,13 +1,12 @@ use std::str::FromStr; use color_eyre::eyre::{eyre, Context, Result}; -use http::{StatusCode}; +use http::StatusCode; use reqwest::{header::HeaderMap, Response}; use uuid::Uuid; use crate::release_metadata::ReleaseMetadata; - pub struct FlakeHubClient { host: url::Url, bearer_token: String, @@ -37,8 +36,7 @@ pub fn flakehub_headers() -> HeaderMap { impl FlakeHubClient { pub fn new(host: url::Url, token: String) -> Result { - let builder = reqwest::ClientBuilder::new(); - builder.user_agent("flakehub-push"); + let builder = reqwest::ClientBuilder::new().user_agent("flakehub-push"); let client = builder.build()?; @@ -50,21 +48,27 @@ impl FlakeHubClient { Ok(client) } - pub async fn release_stage(&self, upload_name: String, release_version: String, release_metadata: ReleaseMetadata, tarball: Tarball) -> Result { + pub async fn release_stage( + &self, + upload_name: &str, + release_version: &str, + release_metadata: &ReleaseMetadata, + tarball: &Tarball, + ) -> Result { let flake_tarball_len = tarball.bytes.len(); - let flake_tarball_hash_base64 = tarball.hash_base64; + let flake_tarball_hash_base64 = &tarball.hash_base64; let relative_url = &format!("upload/{upload_name}/{release_version}/{flake_tarball_len}/{flake_tarball_hash_base64}"); - - + let release_metadata_post_url = format!("{}/{}", self.host, relative_url); // TODO(colemickens): better join - + tracing::debug!( url = %release_metadata_post_url, "Computed release metadata POST URL" ); - - let response = self.client + + let response = self + .client .post(release_metadata_post_url) .bearer_auth(&self.bearer_token) .headers(flakehub_headers()) @@ -75,28 +79,28 @@ impl FlakeHubClient { Ok(response) } - + pub async fn release_publish(&self, release_uuidv7: Uuid) -> Result<()> { let publish_post_url = format!("{}/publish/{}", self.host, release_uuidv7); // TODO(colemickens): fix url joining - tracing::debug!(url = %publish_post_url, "Computed publish POST URL"); - - let publish_response = self.client + + let publish_response = self + .client .post(publish_post_url) .bearer_auth(&self.bearer_token) .headers(flakehub_headers()) .send() .await .wrap_err("Publishing release")?; - + let publish_response_status = publish_response.status(); tracing::trace!( status = tracing::field::display(publish_response_status), "Got publish POST response" ); - + if publish_response_status != 200 { return Err(eyre!( "\ @@ -107,7 +111,7 @@ impl FlakeHubClient { )); } - // TODO: return the actual response object? + // TODO: return the actual response object? Ok(()) } -} \ No newline at end of file +} diff --git a/src/main.rs b/src/main.rs index 4eb11ff2..fa02a7da 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,9 @@ -use std::{fmt::Display, io::IsTerminal, path::{Path, PathBuf}, process::ExitCode}; +use std::{ + fmt::Display, + io::IsTerminal, + path::{Path, PathBuf}, + process::ExitCode, +}; use clap::Parser; use color_eyre::eyre::{eyre, Result, WrapErr}; @@ -6,7 +11,10 @@ use error::Error; use http::StatusCode; use uuid::Uuid; -use crate::{flakehub_client::FlakeHubClient, github::graphql::GithubGraphqlDataQuery, push_context::PushContext, release_metadata::ReleaseMetadata}; +use crate::{ + flakehub_client::FlakeHubClient, github::graphql::GithubGraphqlDataQuery, + push_context::PushContext, release_metadata::ReleaseMetadata, +}; mod cli; mod error; mod flake_info; @@ -48,7 +56,7 @@ async fn main() -> Result { }) .install()?; - let cli = cli::FlakeHubPushCli::parse(); + let mut cli = cli::FlakeHubPushCli::parse(); cli.instrumentation.setup()?; let ctx: PushContext = PushContext::from_cli_and_env(&mut cli).await?; @@ -57,12 +65,14 @@ async fn main() -> Result { let fhclient = FlakeHubClient::new(ctx.flakehub_host, ctx.auth_token)?; // "upload.rs" - stage the release - let stage_response = fhclient.release_stage( - ctx.upload_name, - ctx.release_version, - ctx.metadata, - ctx.tarball, - ).await?; + let stage_response = fhclient + .release_stage( + &ctx.upload_name, + &ctx.release_version, + &ctx.metadata, + &ctx.tarball, + ) + .await?; // handle the response here, rather than in client, so we can do special behavior // TODO(colemickens/review): move this intoo release_create with another flag? @@ -79,7 +89,7 @@ async fn main() -> Result { StatusCode::CONFLICT => { tracing::info!( "Release for revision `{revision}` of {upload_name}/{release_version} already exists; flakehub-push will not upload it again", - revision = ctx.metadata.revision, + revision = &ctx.metadata.revision, upload_name = ctx.upload_name, release_version = ctx.release_version, ); @@ -89,7 +99,7 @@ async fn main() -> Result { release_version: ctx.release_version, })?; } else { - return Ok(()); + todo!(); } } StatusCode::UNAUTHORIZED => { @@ -126,13 +136,15 @@ async fn main() -> Result { // "publish.rs" - publish the release after upload let publish_result = fhclient.release_publish(stage_result.uuid).await?; - + tracing::info!( - "Successfully released new version of {}/{}", ctx.upload_name, ctx.release_version + "Successfully released new version of {}/{}", + ctx.upload_name, + ctx.release_version ); let exitcode = ExitCode::SUCCESS; - return Ok(exitcode); + Ok(exitcode) } #[derive(Debug, Clone, Copy, clap::ValueEnum, serde::Serialize, serde::Deserialize)] diff --git a/src/push_context.rs b/src/push_context.rs index 665a34d9..e5ca3997 100644 --- a/src/push_context.rs +++ b/src/push_context.rs @@ -53,16 +53,16 @@ impl GitContext { github_graphql_data_result.spdx_identifier.clone().unwrap_or_else(|| "None".to_string()), ) } - cli.spdx_expression.0 + cli.spdx_expression.0.clone() }; let ctx = GitContext { spdx_expression: spdx_expression, - repo_topics: github_graphql_data_result.topics, + repo_topics: github_graphql_data_result.topics.clone(), revision_info: RevisionInfo { // TODO(colemickens): type coherency here... :/ (as is bad) commit_count: Some(github_graphql_data_result.rev_count as usize), - revision: github_graphql_data_result.revision, + revision: github_graphql_data_result.revision.clone(), }, }; Ok(ctx) From 292de1db0bbf8b980fca01d7892a731800950dc3 Mon Sep 17 00:00:00 2001 From: Cole Mickens Date: Tue, 16 Apr 2024 08:10:10 -0700 Subject: [PATCH 04/27] refactor: flake_info: restore sanity checks --- src/flake_info.rs | 184 ++++++++++++++++++++++---------------------- src/push_context.rs | 8 ++ 2 files changed, 98 insertions(+), 94 deletions(-) diff --git a/src/flake_info.rs b/src/flake_info.rs index f7f94f22..870e1565 100644 --- a/src/flake_info.rs +++ b/src/flake_info.rs @@ -14,57 +14,6 @@ use crate::flakehub_client::Tarball; const FLAKE_URL_PLACEHOLDER_UUID: &str = "c9026fc0-ced9-48e0-aa3c-fc86c4c86df1"; const README_FILENAME_LOWERCASE: &str = "readme.md"; -// TODO: can't we just do this sanity checking in from_dir? -// // TODO(colemickens): can we move this to a method on FlakeMetadata? -// // maybe FlakeMetadata should know its dir? probably inside the json already -// #[tracing::instrument( -// skip_all, -// fields( -// directory = %directory.display(), -// ) -// )] -// pub(crate) async fn check_flake_evaluates(directory: &Path) -> color_eyre::Result<()> { -// let output = tokio::process::Command::new("nix") -// .arg("flake") -// .arg("show") -// .arg("--all-systems") -// .arg("--json") -// .arg("--no-write-lock-file") -// .arg(directory) -// .output() -// .await -// .wrap_err_with(|| { -// eyre!( -// "Failed to execute `nix flake show --all-systems --json --no-write-lock-file {}`", -// directory.display() -// ) -// })?; - -// if !output.status.success() { -// let command = format!( -// "nix flake show --all-systems --json --no-write-lock-file {}", -// directory.display(), -// ); -// let msg = format!( -// "\ -// Failed to execute command `{command}`{maybe_status} \n\ -// stdout: {stdout}\n\ -// stderr: {stderr}\n\ -// ", -// stdout = String::from_utf8_lossy(&output.stdout), -// stderr = String::from_utf8_lossy(&output.stderr), -// maybe_status = if let Some(status) = output.status.code() { -// format!(" with status {status}") -// } else { -// String::new() -// } -// ); -// return Err(eyre!(msg))?; -// } - -// Ok(()) -// } - #[derive(Debug)] pub struct FlakeMetadata { pub(crate) source_dir: std::path::PathBuf, @@ -75,9 +24,13 @@ pub struct FlakeMetadata { #[derive(Debug, Deserialize)] pub struct FlakeOutputs(pub serde_json::Value); +// notes: there are three distinct `nix` invocations throughout FlakeMetadata: +// 1. to get the basic flake metadata (used for last_modified, initial sanity check, store output path(s)) +// 2. check_evaluates make sure all the outputs evaluate (TODO(review): seems a tad aggressive almost?) +// 3. check_lock_if_exists makes sure the user has accidentally pushed a flake with incoherent `flake.{nix,lock}` + impl FlakeMetadata { pub async fn from_dir(directory: &Path) -> Result { - // TODO(colemickens): the de-duped block below runs `--no-update-lock-file`, this one is `--no-write-lock-file` let output = tokio::process::Command::new("nix") .arg("flake") .arg("metadata") @@ -101,27 +54,106 @@ impl FlakeMetadata { ) })?; - /* - if flake_dir.join("flake.lock").exists() { + let flake_locked_url = metadata_json + .get("url") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| { + eyre!("Could not get `url` attribute from `nix flake metadata --json` output") + })?; + tracing::debug!("Locked URL = {}", flake_locked_url); + let flake_metadata_value_path = metadata_json + .get("path") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| { + eyre!("Could not get `path` attribute from `nix flake metadata --json` output") + })?; + let flake_metadata_value_resolved_dir = metadata_json + .pointer("/resolved/dir") + .and_then(serde_json::Value::as_str); + + let source = match flake_metadata_value_resolved_dir { + Some(flake_metadata_value_resolved_dir) => { + Path::new(flake_metadata_value_path).join(flake_metadata_value_resolved_dir) + } + None => PathBuf::from(flake_metadata_value_path), + }; + + Ok(FlakeMetadata { + source_dir: source, + flake_locked_url: flake_locked_url.to_string(), + metadata_json: metadata_json, + }) + } + + /// check_evalutes checks that the flake evaluates + /// (note it is not necessary for the target to have a flake.lock) + pub async fn check_evaluates(&self) -> Result<()> { + let output = tokio::process::Command::new("nix") + .arg("flake") + .arg("show") + .arg("--all-systems") + .arg("--json") + .arg("--no-write-lock-file") + .arg(&self.source_dir) + .output() + .await + .wrap_err_with(|| { + eyre!( + "Failed to execute `nix flake show --all-systems --json --no-write-lock-file {}`", + self.source_dir.display() + ) + })?; + + if !output.status.success() { + let command = format!( + "nix flake show --all-systems --json --no-write-lock-file {}", + self.source_dir.display(), + ); + let msg = format!( + "\ + Failed to execute command `{command}`{maybe_status} \n\ + stdout: {stdout}\n\ + stderr: {stderr}\n\ + ", + stdout = String::from_utf8_lossy(&output.stdout), + stderr = String::from_utf8_lossy(&output.stderr), + maybe_status = if let Some(status) = output.status.code() { + format!(" with status {status}") + } else { + String::new() + } + ); + return Err(eyre!(msg))?; + } + + Ok(()) + } + + /// check_lock_if_exists is specifically to check locked flakes to make sure the flake.lock + /// has not "drifted" from flake.nix. This would happen if the user added a new flake.nix input, + /// and committed/pushed that without the corresponding update to the flake.lock. Importantly, + /// this does not ensure anything about the recentness of the locked revs. + pub async fn check_lock_if_exists(&self) -> Result<()> { + if self.source_dir.join("flake.lock").exists() { let output = tokio::process::Command::new("nix") .arg("flake") .arg("metadata") .arg("--json") .arg("--no-update-lock-file") - .arg(&flake_dir) + .arg(&self.source_dir) .output() .await .wrap_err_with(|| { eyre!( "Failed to execute `nix flake metadata --json --no-update-lock-file {}`", - flake_dir.display() + self.source_dir.display() ) })?; if !output.status.success() { let command = format!( "nix flake metadata --json --no-update-lock-file {}", - flake_dir.display(), + self.source_dir.display(), ); let msg = format!( "\ @@ -140,43 +172,7 @@ impl FlakeMetadata { return Err(eyre!(msg))?; } } - */ - - // determine flake's store (sub)dir: - - let flake_locked_url = metadata_json - .get("url") - .and_then(serde_json::Value::as_str) - .ok_or_else(|| { - eyre!("Could not get `url` attribute from `nix flake metadata --json` output") - })?; - tracing::debug!("Locked URL = {}", flake_locked_url); - let flake_metadata_value_path = metadata_json - .get("path") - .and_then(serde_json::Value::as_str) - .ok_or_else(|| { - eyre!("Could not get `path` attribute from `nix flake metadata --json` output") - })?; - let flake_metadata_value_resolved_dir = metadata_json - .pointer("/resolved/dir") - .and_then(serde_json::Value::as_str); - - let source = match flake_metadata_value_resolved_dir { - Some(flake_metadata_value_resolved_dir) => { - Path::new(flake_metadata_value_path).join(flake_metadata_value_resolved_dir) - } - None => PathBuf::from(flake_metadata_value_path), - }; - - Ok(FlakeMetadata { - source_dir: source, - flake_locked_url: flake_locked_url.to_string(), - metadata_json: metadata_json, - // TODO(Colemickens): remove this, we want to use get_source() - //dir: directory, - // move the source determination to from_dir so we just know it as a property - // rename 'dir' to 'source' - }) + Ok(()) } pub fn flake_tarball(&self) -> Result { diff --git a/src/push_context.rs b/src/push_context.rs index e5ca3997..40677ccc 100644 --- a/src/push_context.rs +++ b/src/push_context.rs @@ -362,6 +362,14 @@ impl PushContext { .wrap_err("Getting flake metadata")?; tracing::debug!("Got flake metadata: {:?}", flake_metadata); + // sanity checks + flake_metadata.check_evaluates() + .await + .wrap_err("failed to evaluate all system attrs of the flake")?; + flake_metadata.check_lock_if_exists() + .await + .wrap_err("failed to evaluate all system attrs of the flake")?; + let flake_outputs = flake_metadata.outputs(cli.include_output_paths).await?; tracing::debug!("Got flake outputs: {:?}", flake_outputs); From 417639b4c4d09f7f7980e5372b2f47d7c23effe1 Mon Sep 17 00:00:00 2001 From: Cole Mickens Date: Tue, 16 Apr 2024 09:59:26 -0700 Subject: [PATCH 05/27] refactor: more cleanup, fixes, stash before some gitlab stuff --- src/flake_info.rs | 2 + src/flakehub_auth_fake.rs | 8 ++- src/flakehub_client.rs | 14 ++-- src/github/graphql/mod.rs | 4 ++ src/main.rs | 11 ++- src/push_context.rs | 64 ++++++++--------- src/release_metadata.rs | 143 +------------------------------------- src/revision_info.rs | 85 ++++++++++++++++++++++ 8 files changed, 143 insertions(+), 188 deletions(-) create mode 100644 src/revision_info.rs diff --git a/src/flake_info.rs b/src/flake_info.rs index 870e1565..ae8cceb7 100644 --- a/src/flake_info.rs +++ b/src/flake_info.rs @@ -85,6 +85,8 @@ impl FlakeMetadata { }) } + // TODO(colemickens): consider a nix_cmd() that wraps this stuff up and just takes the string args[] and handles errors etc + /// check_evalutes checks that the flake evaluates /// (note it is not necessary for the target to have a flake.lock) pub async fn check_evaluates(&self) -> Result<()> { diff --git a/src/flakehub_auth_fake.rs b/src/flakehub_auth_fake.rs index f13fcb76..946e6197 100644 --- a/src/flakehub_auth_fake.rs +++ b/src/flakehub_auth_fake.rs @@ -2,13 +2,16 @@ use color_eyre::eyre::{eyre, Context, Result}; use crate::github::graphql::GithubGraphqlDataResult; -pub async fn get_fake_bearer_token(jwt_issuer_uri: &str, repository: &str, project_owner: &str, github_graphql_data_result: GithubGraphqlDataResult) -> Result { +pub async fn get_fake_bearer_token(jwt_issuer_uri: &str, project_owner: &str, repository: &str, github_graphql_data_result: GithubGraphqlDataResult) -> Result { tracing::warn!("running outside github/gitlab - minting a dev-signed JWT"); let client = reqwest::Client::new(); let mut claims = github_actions_oidc_claims::Claims::make_dummy(); // FIXME: we should probably fill in more of these claims. + + // TODO(review): on the contrary, I think we should ditch this, and we should basically use forge_login-esque functionality for this going forward + // this would remove the entire need for the fake JWT server, since we are ourselves a JWT issuer claims.aud = "flakehub-localhost".to_string(); claims.iss = "flakehub-push-dev".to_string(); claims.repository = repository.to_string(); @@ -17,6 +20,8 @@ pub async fn get_fake_bearer_token(jwt_issuer_uri: &str, repository: &str, proje claims.repository_id = github_graphql_data_result.project_id.to_string(); claims.repository_owner_id = github_graphql_data_result.owner_id.to_string(); + tracing::debug!(?claims); + let response = client .post(jwt_issuer_uri) .header("Content-Type", "application/json") @@ -24,6 +29,7 @@ pub async fn get_fake_bearer_token(jwt_issuer_uri: &str, repository: &str, proje .send() .await .wrap_err("Sending request to JWT issuer")?; + #[derive(serde::Deserialize)] struct Response { token: String, diff --git a/src/flakehub_client.rs b/src/flakehub_client.rs index b2f12b0d..af4d819d 100644 --- a/src/flakehub_client.rs +++ b/src/flakehub_client.rs @@ -57,15 +57,18 @@ impl FlakeHubClient { ) -> Result { let flake_tarball_len = tarball.bytes.len(); let flake_tarball_hash_base64 = &tarball.hash_base64; - let relative_url = &format!("upload/{upload_name}/{release_version}/{flake_tarball_len}/{flake_tarball_hash_base64}"); + let relative_url: &String = &format!("upload/{upload_name}/{release_version}/{flake_tarball_len}/{flake_tarball_hash_base64}"); - let release_metadata_post_url = format!("{}/{}", self.host, relative_url); - // TODO(colemickens): better join + let release_metadata_post_url = self.host.join(relative_url)?; tracing::debug!( url = %release_metadata_post_url, "Computed release metadata POST URL" ); + + tracing::debug!(?release_metadata); //TODO colemickens: sanity check this against main fhp + tracing::debug!("repo={}", release_metadata.repo); + tracing::debug!("upload_name={}", upload_name); let response = self .client @@ -81,8 +84,8 @@ impl FlakeHubClient { } pub async fn release_publish(&self, release_uuidv7: Uuid) -> Result<()> { - let publish_post_url = format!("{}/publish/{}", self.host, release_uuidv7); - // TODO(colemickens): fix url joining + let relative_url = format!("publish/{}", release_uuidv7); + let publish_post_url = self.host.join(&relative_url)?; tracing::debug!(url = %publish_post_url, "Computed publish POST URL"); @@ -111,7 +114,6 @@ impl FlakeHubClient { )); } - // TODO: return the actual response object? Ok(()) } } diff --git a/src/github/graphql/mod.rs b/src/github/graphql/mod.rs index 2f5a2247..6e3c7259 100644 --- a/src/github/graphql/mod.rs +++ b/src/github/graphql/mod.rs @@ -38,6 +38,9 @@ impl GithubGraphqlDataQuery { revision: revision.to_string(), max_num_topics: MAX_NUM_EXTRA_TOPICS, }; + + tracing::debug!(?variables); // TODO remove + let query = GithubGraphqlDataQuery::build_query(variables); let reqwest_response = reqwest_client .post(GITHUB_ENDPOINT) @@ -47,6 +50,7 @@ impl GithubGraphqlDataQuery { .await .wrap_err("Failed to issue RevCountQuery request to Github's GraphQL API")?; + let response_status = reqwest_response.status(); let response: graphql_client::Response< ::ResponseData, diff --git a/src/main.rs b/src/main.rs index fa02a7da..d79dbaa8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -24,6 +24,7 @@ mod github; mod gitlab; mod push_context; mod release_metadata; +mod revision_info; mod s3; const DEFAULT_ROLLING_PREFIX: &str = "0.1"; @@ -75,9 +76,7 @@ async fn main() -> Result { .await?; // handle the response here, rather than in client, so we can do special behavior - // TODO(colemickens/review): move this intoo release_create with another flag? - // or "ReleaseOptions->error_on_conflict" and have a way to override options for a release - // creation?? + // TODO(review): what do we things, some sort of ReleaseFlags and indeed handle this in the client? let release_metadata_post_response_status = stage_response.status(); tracing::trace!( status = tracing::field::display(release_metadata_post_response_status), @@ -99,7 +98,8 @@ async fn main() -> Result { release_version: ctx.release_version, })?; } else { - todo!(); + // we're just done, and happy about it: + return Ok(ExitCode::SUCCESS); } } StatusCode::UNAUTHORIZED => { @@ -143,8 +143,7 @@ async fn main() -> Result { ctx.release_version ); - let exitcode = ExitCode::SUCCESS; - Ok(exitcode) + Ok(ExitCode::SUCCESS) } #[derive(Debug, Clone, Copy, clap::ValueEnum, serde::Serialize, serde::Deserialize)] diff --git a/src/push_context.rs b/src/push_context.rs index 40677ccc..0ea49410 100644 --- a/src/push_context.rs +++ b/src/push_context.rs @@ -8,14 +8,9 @@ use color_eyre::eyre::{eyre, Context, Result}; use spdx::Expression; use crate::{ - cli::FlakeHubPushCli, - flake_info, flakehub_auth_fake, - flakehub_client::Tarball, - github::graphql::{ + build_http_client, cli::FlakeHubPushCli, flake_info, flakehub_auth_fake, flakehub_client::Tarball, github::graphql::{ GithubGraphqlDataQuery, GithubGraphqlDataResult, MAX_LABEL_LENGTH, MAX_NUM_TOTAL_LABELS, - }, - release_metadata::{ReleaseMetadata, RevisionInfo}, - DEFAULT_ROLLING_PREFIX, + }, release_metadata::ReleaseMetadata, revision_info::RevisionInfo, DEFAULT_ROLLING_PREFIX }; pub struct GitContext { @@ -29,8 +24,6 @@ impl GitContext { cli: &FlakeHubPushCli, github_graphql_data_result: &GithubGraphqlDataResult, ) -> Result { - // do everything we need to get data from github here - // step: validate spdx, backfill from GitHub API let spdx_expression = if cli.spdx_expression.0.is_none() { if let Some(spdx_string) = &github_graphql_data_result.spdx_identifier { @@ -67,6 +60,12 @@ impl GitContext { }; Ok(ctx) } + + pub fn from_cli_and_gitlab( + cli: &FlakeHubPushCli, + ) -> Result { + todo!(); + } } pub(crate) struct PushContext { @@ -90,7 +89,7 @@ impl PushContext { // Take the opportunity to be able to populate/encrich data from the GitHub API // this is used to augment user/discovered data, and is used for the faked JWT for local flakehub-push testing - let client = reqwest::Client::new(); + let client = build_http_client().build()?; let is_github = std::env::var("GITHUB_ACTION").ok().is_some(); let is_gitlab = std::env::var("GITLAB_CI").ok().is_some(); @@ -114,6 +113,11 @@ impl PushContext { // STEP: determine and check 'repository' and 'upload_name' // If the upload name is supplied by the user, ensure that it contains exactly // one slash and no whitespace. Default to the repository name. + + // Oh yeesh: + // upload_name is set from --repository if --name is not passed + // but --repository is used for the gh_owner/gh_repo, .... ?? only for github? + // I _HATE_ that we accept user input for this stuff when we don't even validate it, we just blindly re-display it let Some(ref repository) = cli.repository.0 else { return Err(eyre!("Could not determine repository name, pass `--repository` formatted like `determinatesystems/flakehub-push`")); }; @@ -144,7 +148,6 @@ impl PushContext { Err(eyre!("Could not determine the owner/project, pass `--repository` formatted like `determinatesystems/flakehub-push`. The passed value has too many slashes (/) to be a valid repository"))?; } - // TODO(colemickens): pretty sure there's a better way to write this: let local_git_root = (match &cli.git_root.0 { Some(gr) => Ok(gr.to_owned()), None => std::env::current_dir().map(PathBuf::from) @@ -157,6 +160,7 @@ impl PushContext { let (token, git_ctx) = match (is_github, is_gitlab, &cli.jwt_issuer_uri.0) { (true, false, None) => { + // GITHUB CI let github_token = cli .github_token .0 @@ -181,15 +185,15 @@ impl PushContext { (token, git_ctx) } (false, true, None) => { + // GITLAB CI let token = crate::gitlab::get_runner_bearer_token(&cli.host) .await .wrap_err("Getting upload bearer token from GitLab")?; - // let git_ctx = GitContext::from_cli_and_gitlab(cli); // TODO: !!!!! - let git_ctx = todo!(); + let git_ctx = GitContext::from_cli_and_gitlab(cli)?; (token, git_ctx) } (false, false, Some(u)) => { - // local, fake github, we need the ... github ids to get a faked token + // LOCAL, DEV (aka emulating GITHUB) let github_token = cli .github_token .0 @@ -218,7 +222,7 @@ impl PushContext { (token, git_ctx) } (_, _, Some(_)) => { - // we're in GitHub or GitLab and jwt_issuer_uri was specified, invalid + // we're in (GitHub|GitLab) and jwt_issuer_uri was specified, invalid return Err(eyre!( "specifying the jwt_issuer_uri when running in GitHub or GitLab is invalid" )); @@ -229,9 +233,7 @@ impl PushContext { } }; - // STEP: resolve "subdir" (use --directory flag from cli) - // TODO(colemickens): do we really need both in our source, "subdir" is part of release_metadata tho - // NOTE(colemickens,self): flake_dir can probably be evne more "intenral" only used for tarring/flake_info + // STEP: resolve/canonicalize "subdir" let subdir = if let Some(directory) = &cli.directory.0 { let absolute_directory = if directory.is_absolute() { directory.clone() @@ -273,8 +275,8 @@ impl PushContext { } }; - // TODO(don't use revision_info, maybe, so we don't have this extra shadowy local var) let Some(commit_count) = git_ctx.revision_info.commit_count else { + // TODO(review): shouldn't this maybe only be fatal whenever we are rolling_minor||rolling? return Err(eyre!("Could not determine commit count, this is normally determined via the `--git-root` argument or via the GitHub API")); }; @@ -288,8 +290,6 @@ impl PushContext { }; // STEP: calculate labels - // - they can specify: extra_labels on cli - // TODO: merge execution_environment's labels + cli extra_labels let merged_labels = { let mut labels: HashSet<_> = cli .extra_labels @@ -308,10 +308,9 @@ impl PushContext { let message = "`extra-tags` is deprecated and will be removed in the future. Please use `extra-labels` instead."; tracing::warn!("{message}"); - // TODO(colemickens): restore this cleanly, have something on git_ctx that logs? rename git_ctx? - // if is_github_actions { - // println!("::warning::{message}"); - // } + if is_github { + println!("::warning::{message}"); + } if labels.is_empty() { labels = extra_tags; @@ -320,10 +319,9 @@ impl PushContext { "Both `extra-tags` and `extra-labels` were set; `extra-tags` will be ignored."; tracing::warn!("{message}"); - // TODO(colemickens): restore this cleanly, have something on git_ctx that logs? rename git_ctx? - // if is_github_actions { - // println!("::warning::{message}"); - // } + if is_github { + println!("::warning::{message}"); + } } } @@ -353,10 +351,9 @@ impl PushContext { // flake_dir is an absolute path of flake_root(aka git_root)/subdir let flake_dir = local_git_root.join(&subdir); - // TODO: depending on what the user called us with, this flake_dir isn't even necessarily canonicalized, is this a sec/traversal issue? + // TODO(review): depending on what the user called us with, this flake_dir isn't even necessarily canonicalized, is this a sec/traversal issue? // FIXME: bail out if flake_metadata denotes a dirty tree. - // (Todo make sure invariant checks are right, de-duped properly) let flake_metadata = flake_info::FlakeMetadata::from_dir(&flake_dir) .await .wrap_err("Getting flake metadata")?; @@ -387,7 +384,8 @@ impl PushContext { outputs: flake_outputs.0, raw_flake_metadata: flake_metadata.metadata_json.clone(), readme: readme, - repo: repository.to_string(), + // TODO(colemickens): remove this confusing, redundant field (FH-267) + repo: upload_name.to_string(), revision: git_ctx.revision_info.revision, visibility: visibility, mirrored: cli.mirror, @@ -403,7 +401,6 @@ impl PushContext { let flake_tarball = flake_metadata .flake_tarball() - //.await //weird, tar is not async? .wrap_err("Making release tarball")?; let ctx = Self { @@ -415,7 +412,6 @@ impl PushContext { error_if_release_conflicts: cli.error_on_conflict, - // this is the payload we send metadata: release_metadata, tarball: flake_tarball, }; diff --git a/src/release_metadata.rs b/src/release_metadata.rs index 927122f7..9a13118b 100644 --- a/src/release_metadata.rs +++ b/src/release_metadata.rs @@ -5,7 +5,6 @@ use crate::Visibility; const README_FILENAME_LOWERCASE: &str = "readme.md"; -// TODO(colemickens): fhlib? #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub(crate) struct ReleaseMetadata { pub(crate) commit_count: usize, @@ -30,132 +29,7 @@ pub(crate) struct ReleaseMetadata { pub(crate) labels: Vec, } -#[derive(Clone)] -pub(crate) struct RevisionInfo { - pub(crate) commit_count: Option, - pub(crate) revision: String, -} - -impl RevisionInfo { - pub(crate) fn from_git_root(git_root: &Path) -> color_eyre::Result { - let gix_repository = gix::open(git_root).wrap_err("Opening the Git repository")?; - let gix_repository_head = gix_repository - .head() - .wrap_err("Getting the HEAD revision of the repository")?; - - let revision = match gix_repository_head.kind { - gix::head::Kind::Symbolic(gix_ref::Reference { - name: _, target, .. - }) => match target { - gix_ref::Target::Peeled(object_id) => object_id, - gix_ref::Target::Symbolic(_) => { - return Err(eyre!( - "Symbolic revision pointing to a symbolic revision is not supported at this time" - )) - } - }, - gix::head::Kind::Detached { - target: object_id, .. - } => object_id, - gix::head::Kind::Unborn(_) => { - return Err(eyre!( - "Newly initialized repository detected, at least one commit is necessary" - )) - } - }; - - let commit_count = gix_repository - .rev_walk([revision]) - .all() - .map(|rev_iter| rev_iter.count()) - .ok(); - let revision = revision.to_hex().to_string(); - - Ok(Self { - commit_count, - revision, - }) - } -} - -impl ReleaseMetadata { - // FIXME - #[allow(clippy::too_many_arguments)] - #[tracing::instrument(skip_all, fields( - flake_store_path = %flake_store_path.display(), - subdir = %subdir.display(), - description = tracing::field::Empty, - readme = tracing::field::Empty, - %revision, - %commit_count, - spdx_identifier = tracing::field::Empty, - visibility = ?visibility, - ))] - pub(crate) async fn build( - flake_store_path: &Path, - subdir: &Path, - revision: String, - commit_count: usize, - flake_metadata: serde_json::Value, - flake_outputs: serde_json::Value, - upload_name: String, - mirror: bool, - visibility: Visibility, - labels: Vec, - spdx_identifier: Option, - ) -> color_eyre::Result { - let span = tracing::Span::current(); - - if let Some(spdx_identifier) = &spdx_identifier { - span.record("spdx_identifier", tracing::field::display(spdx_identifier)); - } - - assert!(subdir.is_relative()); - - let description = if let Some(description) = flake_metadata.get("description") { - let description_value = description - .as_str() - .ok_or_else(|| { - eyre!("`nix flake metadata --json` does not have a string `description` field") - })? - .to_string(); - span.record("description", tracing::field::display(&description_value)); - Some(description_value) - } else { - None - }; - - let readme_path = get_readme(flake_store_path).await?; - let readme = if let Some(readme_path) = readme_path { - span.record("readme", tracing::field::display(readme_path.display())); - Some(tokio::fs::read_to_string(&readme_path).await?) - } else { - None - }; - - tracing::trace!("Collected ReleaseMetadata information"); - - Ok(ReleaseMetadata { - description, - repo: upload_name.to_string(), - raw_flake_metadata: flake_metadata.clone(), - readme, - revision, - commit_count, - visibility, - outputs: flake_outputs, - source_subdirectory: Some( - subdir - .to_str() - .map(|d| d.to_string()) - .ok_or(eyre!("Directory {:?} is not a valid UTF-8 string", subdir))?, - ), - mirrored: mirror, - spdx_identifier, - labels, - }) - } -} +// TODO(review,colemickens): I don't really undersatnd why these are nededed??? we don't need the OptionString-y stuff since this isn't GHA adjacent? fn option_string_to_spdx<'de, D>(deserializer: D) -> Result, D::Error> where @@ -185,17 +59,4 @@ where } else { serializer.serialize_none() } -} - -#[tracing::instrument(skip_all, fields(readme_dir))] -async fn get_readme(readme_dir: &Path) -> color_eyre::Result> { - let mut read_dir = tokio::fs::read_dir(readme_dir).await?; - - while let Some(entry) = read_dir.next_entry().await? { - if entry.file_name().to_ascii_lowercase() == README_FILENAME_LOWERCASE { - return Ok(Some(entry.path())); - } - } - - Ok(None) -} +} \ No newline at end of file diff --git a/src/revision_info.rs b/src/revision_info.rs new file mode 100644 index 00000000..f56cff8f --- /dev/null +++ b/src/revision_info.rs @@ -0,0 +1,85 @@ +use color_eyre::eyre::{eyre, WrapErr}; +use std::path::{Path, PathBuf}; + +use crate::Visibility; + +const README_FILENAME_LOWERCASE: &str = "readme.md"; + + +#[derive(Clone)] +pub(crate) struct RevisionInfo { + pub(crate) commit_count: Option, + pub(crate) revision: String, +} + +impl RevisionInfo { + pub(crate) fn from_git_root(git_root: &Path) -> color_eyre::Result { + let gix_repository = gix::open(git_root).wrap_err("Opening the Git repository")?; + let gix_repository_head = gix_repository + .head() + .wrap_err("Getting the HEAD revision of the repository")?; + + let revision = match gix_repository_head.kind { + gix::head::Kind::Symbolic(gix_ref::Reference { + name: _, target, .. + }) => match target { + gix_ref::Target::Peeled(object_id) => object_id, + gix_ref::Target::Symbolic(_) => { + return Err(eyre!( + "Symbolic revision pointing to a symbolic revision is not supported at this time" + )) + } + }, + gix::head::Kind::Detached { + target: object_id, .. + } => object_id, + gix::head::Kind::Unborn(_) => { + return Err(eyre!( + "Newly initialized repository detected, at least one commit is necessary" + )) + } + }; + + let commit_count = gix_repository + .rev_walk([revision]) + .all() + .map(|rev_iter| rev_iter.count()) + .ok(); + let revision = revision.to_hex().to_string(); + + Ok(Self { + commit_count, + revision, + }) + } +} + +fn option_string_to_spdx<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::de::Deserializer<'de>, +{ + let spdx_identifier: Option<&str> = serde::Deserialize::deserialize(deserializer)?; + + if let Some(spdx_identifier) = spdx_identifier { + spdx::Expression::parse(spdx_identifier) + .map_err(serde::de::Error::custom) + .map(Option::Some) + } else { + Ok(None) + } +} + +fn option_spdx_serialize( + spdx_identifier: &Option, + serializer: S, +) -> Result +where + S: serde::ser::Serializer, +{ + if let Some(spdx_identifier) = spdx_identifier { + let spdx_string = spdx_identifier.to_string(); + serializer.serialize_str(&spdx_string) + } else { + serializer.serialize_none() + } +} From 1e72c991b5e45291772ec12297d663a25be34caf Mon Sep 17 00:00:00 2001 From: Cole Mickens Date: Tue, 16 Apr 2024 13:53:56 -0700 Subject: [PATCH 06/27] refactor/gitlab: stash some more notes for (p)review --- src/cli/mod.rs | 31 +++++++++++++++++++-- src/flake_info.rs | 7 ----- src/flakehub_client.rs | 2 +- src/gitlab/mod.rs | 6 ++-- src/main.rs | 2 +- src/push_context.rs | 63 ++++++++++++++++++++++++++++++++++-------- 6 files changed, 86 insertions(+), 25 deletions(-) diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 77384a75..819fea99 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -8,7 +8,7 @@ use std::{ }; use crate::{ - build_http_client, flakehub_client::Tarball, github::graphql::{GithubGraphqlDataQuery, MAX_LABEL_LENGTH, MAX_NUM_TOTAL_LABELS}, release_metadata::{ReleaseMetadata, RevisionInfo} + build_http_client, flakehub_client::Tarball, github::graphql::{GithubGraphqlDataQuery, MAX_LABEL_LENGTH, MAX_NUM_TOTAL_LABELS}, release_metadata::{ReleaseMetadata} }; #[derive(Debug, clap::Parser)] @@ -249,6 +249,8 @@ impl clap::builder::TypedValueParser for U64ToNoneParser { impl FlakeHubPushCli { pub(crate) fn backfill_from_github_env(&mut self) { + // https://docs.github.com/en/actions/learn-github-actions/variables + if self.git_root.0.is_none() { let env_key = "GITHUB_WORKSPACE"; if let Ok(env_val) = std::env::var(env_key) { @@ -275,6 +277,31 @@ impl FlakeHubPushCli { } pub(crate) fn backfill_from_gitlab_env(&mut self) { - todo!(); + // https://docs.gitlab.com/ee/ci/variables/predefined_variables.html + + if self.git_root.0.is_none() { + let env_key: &str = "CI_PROJECT_DIR"; + if let Ok(env_val) = std::env::var(env_key) { + tracing::debug!(git_root = %env_val, "Set via `${env_key}`"); + self.git_root.0 = Some(PathBuf::from(env_val)); + } + } + + if self.repository.0.is_none() { + let env_key = "CI_PROJECT_ID"; + if let Ok(env_val) = std::env::var(env_key) { + tracing::debug!(repository = %env_val, "Set via `${env_key}`"); + self.repository.0 = Some(env_val); + } + } + + // TODO(review): this... isn't really a "tag" for github either, but I think maybe that's intentional? + if self.tag.0.is_none() { + let env_key = "CI_COMMIT_REF_NAME"; + if let Ok(env_val) = std::env::var(env_key) { + tracing::debug!(repository = %env_val, "Set via `${env_key}`"); + self.tag.0 = Some(env_val); + } + } } } \ No newline at end of file diff --git a/src/flake_info.rs b/src/flake_info.rs index ae8cceb7..0f79c5b6 100644 --- a/src/flake_info.rs +++ b/src/flake_info.rs @@ -24,11 +24,6 @@ pub struct FlakeMetadata { #[derive(Debug, Deserialize)] pub struct FlakeOutputs(pub serde_json::Value); -// notes: there are three distinct `nix` invocations throughout FlakeMetadata: -// 1. to get the basic flake metadata (used for last_modified, initial sanity check, store output path(s)) -// 2. check_evaluates make sure all the outputs evaluate (TODO(review): seems a tad aggressive almost?) -// 3. check_lock_if_exists makes sure the user has accidentally pushed a flake with incoherent `flake.{nix,lock}` - impl FlakeMetadata { pub async fn from_dir(directory: &Path) -> Result { let output = tokio::process::Command::new("nix") @@ -85,8 +80,6 @@ impl FlakeMetadata { }) } - // TODO(colemickens): consider a nix_cmd() that wraps this stuff up and just takes the string args[] and handles errors etc - /// check_evalutes checks that the flake evaluates /// (note it is not necessary for the target to have a flake.lock) pub async fn check_evaluates(&self) -> Result<()> { diff --git a/src/flakehub_client.rs b/src/flakehub_client.rs index af4d819d..4f432f63 100644 --- a/src/flakehub_client.rs +++ b/src/flakehub_client.rs @@ -18,7 +18,7 @@ pub struct Tarball { pub bytes: Vec, } -// TODO(colemickens): static init +// TODO(future): static init pub fn flakehub_headers() -> HeaderMap { let mut header_map = HeaderMap::new(); diff --git a/src/gitlab/mod.rs b/src/gitlab/mod.rs index 95b70b3b..9ea0bb17 100644 --- a/src/gitlab/mod.rs +++ b/src/gitlab/mod.rs @@ -1,14 +1,14 @@ use color_eyre::eyre::{eyre, WrapErr}; #[tracing::instrument(skip_all, fields(audience = tracing::field::Empty))] -pub(crate) async fn get_runner_bearer_token(host: &url::Url) -> color_eyre::Result { +pub(crate) async fn get_runner_bearer_token() -> color_eyre::Result { // github allows you to at-runtime change the audience of the token // gitlab requires job-level audience/token config, and makes it available via envvar let maybe_token = std::env::var("GITLAB_JWT_ID_TOKEN"); let token = maybe_token.wrap_err("Failed to get a JWT from GitLab. You must configure id_token in the jobs.")?; - - // TODO(colemickens): valdiate the audience of the gitlab token matches `host` + + // for now, don't bother validating audience, since flakehub will validate the audience as part of authn anyway Ok(token) } diff --git a/src/main.rs b/src/main.rs index d79dbaa8..66d6c68a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -135,7 +135,7 @@ async fn main() -> Result { s3::upload_release_to_s3(stage_result.s3_upload_url, ctx.tarball).await?; // "publish.rs" - publish the release after upload - let publish_result = fhclient.release_publish(stage_result.uuid).await?; + let _publish_result = fhclient.release_publish(stage_result.uuid).await?; tracing::info!( "Successfully released new version of {}/{}", diff --git a/src/push_context.rs b/src/push_context.rs index 0ea49410..8cf88d46 100644 --- a/src/push_context.rs +++ b/src/push_context.rs @@ -8,9 +8,16 @@ use color_eyre::eyre::{eyre, Context, Result}; use spdx::Expression; use crate::{ - build_http_client, cli::FlakeHubPushCli, flake_info, flakehub_auth_fake, flakehub_client::Tarball, github::graphql::{ + build_http_client, + cli::FlakeHubPushCli, + flake_info, flakehub_auth_fake, + flakehub_client::Tarball, + github::graphql::{ GithubGraphqlDataQuery, GithubGraphqlDataResult, MAX_LABEL_LENGTH, MAX_NUM_TOTAL_LABELS, - }, release_metadata::ReleaseMetadata, revision_info::RevisionInfo, DEFAULT_ROLLING_PREFIX + }, + release_metadata::ReleaseMetadata, + revision_info::RevisionInfo, + DEFAULT_ROLLING_PREFIX, }; pub struct GitContext { @@ -63,8 +70,35 @@ impl GitContext { pub fn from_cli_and_gitlab( cli: &FlakeHubPushCli, + // local_revision_info likely ) -> Result { - todo!(); + // Gitlab token during jobs for calling api: `CI_JOB_TOKEN` + // TODO(colemickens): graphql is a bit of a distraction right now, come back, just use REST for now + + // repo_topics: GET /projects/:id/labels + // https://docs.gitlab.com/ee/api/labels.html + // TODO(colemickens): implement + + // revision_info: GET /projects/:id/repository/commits + // https://docs.gitlab.com/ee/api/commits.html + // not sure this has what we need: + // - hitting projects endpoint with ?statistics=true gives a commit_count but it's unclear what that is + // -- https://gitlab.com/api/v4/projects/54746217?statistics=true + // -- https://gitlab.com/api/v4/projects/54746217/repository/commits/171d5b1d13b326e6870df1c38575c39c58acfcd4 (?stats does nothing) + // ---- unclear: https://forum.gitlab.com/t/gitlab-api-projects-statistics-true-commit-count/10435/3 + // ---- another user wondering: https://forum.gitlab.com/t/api-to-get-total-commits-count/93118 + + // spdx_expression: can't find any evidence GitLab tries to surface this info + + let ctx = GitContext { + spdx_expression: None, + repo_topics: vec![], + revision_info: RevisionInfo { + commit_count: None, + revision: "foo".to_string(), + }, + }; + Ok(ctx) } } @@ -94,7 +128,7 @@ impl PushContext { let is_github = std::env::var("GITHUB_ACTION").ok().is_some(); let is_gitlab = std::env::var("GITLAB_CI").ok().is_some(); - // "backfill" env vars from the environment, for the first time, anyway... + // "backfill" env vars from the environment if is_github { cli.backfill_from_github_env(); } @@ -113,14 +147,17 @@ impl PushContext { // STEP: determine and check 'repository' and 'upload_name' // If the upload name is supplied by the user, ensure that it contains exactly // one slash and no whitespace. Default to the repository name. + // notes for future readers: + // upload_name is derived from repository, unless set + // upload_name is then used for upload_name (and repository) there-after + // *except* in GitHub paths, where it's used to query the authoritative git_ctx and locally to fill the fake jwt - // Oh yeesh: - // upload_name is set from --repository if --name is not passed - // but --repository is used for the gh_owner/gh_repo, .... ?? only for github? - // I _HATE_ that we accept user input for this stuff when we don't even validate it, we just blindly re-display it let Some(ref repository) = cli.repository.0 else { return Err(eyre!("Could not determine repository name, pass `--repository` formatted like `determinatesystems/flakehub-push`")); }; + + // If the upload name is supplied by the user, ensure that it contains exactly + // one slash and no whitespace. Default to the repository name. let upload_name = if let Some(ref name) = cli.name.0 { let num_slashes = name.matches('/').count(); @@ -158,6 +195,8 @@ impl PushContext { .wrap_err("Failed to canonicalize `--git-root` argument")?; let local_rev_info = RevisionInfo::from_git_root(&local_git_root)?; + // "cli" and "git_ctx" are the user/env supplied info, augmented with data we might have fetched from github/gitlab apis + let (token, git_ctx) = match (is_github, is_gitlab, &cli.jwt_issuer_uri.0) { (true, false, None) => { // GITHUB CI @@ -186,7 +225,7 @@ impl PushContext { } (false, true, None) => { // GITLAB CI - let token = crate::gitlab::get_runner_bearer_token(&cli.host) + let token = crate::gitlab::get_runner_bearer_token() .await .wrap_err("Getting upload bearer token from GitLab")?; let git_ctx = GitContext::from_cli_and_gitlab(cli)?; @@ -360,10 +399,12 @@ impl PushContext { tracing::debug!("Got flake metadata: {:?}", flake_metadata); // sanity checks - flake_metadata.check_evaluates() + flake_metadata + .check_evaluates() .await .wrap_err("failed to evaluate all system attrs of the flake")?; - flake_metadata.check_lock_if_exists() + flake_metadata + .check_lock_if_exists() .await .wrap_err("failed to evaluate all system attrs of the flake")?; From 320731d61dc7255d09fcc95f843dabef9b6e2b6c Mon Sep 17 00:00:00 2001 From: Cole Mickens Date: Wed, 17 Apr 2024 07:30:52 -0700 Subject: [PATCH 07/27] gitlab: as much as we can seemingly get from the api --- Cargo.lock | 543 +++++++++++++++++++++++++++++++++++++++++--- Cargo.toml | 1 + src/push_context.rs | 35 +-- 3 files changed, 537 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c0d9ad4b..dc9141dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -45,6 +45,21 @@ version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "0.6.14" @@ -150,6 +165,12 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.21.7" @@ -224,6 +245,19 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "num-traits", + "serde", + "windows-targets 0.52.5", +] + [[package]] name = "clap" version = "4.5.4" @@ -243,7 +277,7 @@ dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim", + "strsim 0.11.1", ] [[package]] @@ -342,6 +376,17 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "cron" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8c3e73077b4b4a6ab1ea5047c37c57aee77657bc8ecd6f29b0af082d0b0c07" +dependencies = [ + "chrono", + "nom", + "once_cell", +] + [[package]] name = "crossbeam" version = "0.8.4" @@ -398,6 +443,41 @@ version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.10.0", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core", + "quote", + "syn 1.0.109", +] + [[package]] name = "dashmap" version = "5.5.3" @@ -420,6 +500,37 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "derive_builder" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d07adf7be193b71cc36b193d0f5fe60b918a3a9db4dad0449f57bcfd519704a3" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f91d4cfa921f1c05904dc3c57b4a32c38aed3340cce209f3a6fd1478babafc4" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "derive_builder_macro" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f0314b72bed045f3a68671b3c86328386762c93f82d98c65c3cb5e5f573dd68" +dependencies = [ + "derive_builder_core", + "syn 1.0.109", +] + [[package]] name = "dunce" version = "1.0.4" @@ -441,6 +552,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" version = "0.3.9" @@ -494,11 +611,12 @@ dependencies = [ "color-eyre", "flate2", "github-actions-oidc-claims", + "gitlab", "gix", "gix-ref", - "graphql_client", - "http", - "reqwest", + "graphql_client 0.13.0", + "http 1.1.0", + "reqwest 0.12.4", "ring 0.16.20", "semver", "serde", @@ -640,6 +758,32 @@ dependencies = [ "serde", ] +[[package]] +name = "gitlab" +version = "0.1610.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6c802fc7eb82ff5ba2e4447c5acd0f18ec1b7bb95dbe95b6d77639e25be7cbe" +dependencies = [ + "async-trait", + "base64 0.13.1", + "bytes", + "chrono", + "cron", + "derive_builder", + "futures-util", + "graphql_client 0.11.0", + "http 0.2.12", + "itertools", + "log", + "percent-encoding", + "reqwest 0.11.27", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror", + "url", +] + [[package]] name = "gix" version = "0.62.0" @@ -1552,15 +1696,43 @@ dependencies = [ "thiserror", ] +[[package]] +name = "graphql_client" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fc16d75d169fddb720d8f1c7aed6413e329e1584079b9734ff07266a193f5bc" +dependencies = [ + "graphql_query_derive 0.11.0", + "serde", + "serde_json", +] + [[package]] name = "graphql_client" version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cdf7b487d864c2939b23902291a5041bc4a84418268f25fda1c8d4e15ad8fa" dependencies = [ - "graphql_query_derive", + "graphql_query_derive 0.13.0", + "serde", + "serde_json", +] + +[[package]] +name = "graphql_client_codegen" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f290ecfa3bea3e8a157899dc8a1d96ee7dd6405c18c8ddd213fc58939d18a0e9" +dependencies = [ + "graphql-introspection-query", + "graphql-parser", + "heck 0.4.1", + "lazy_static", + "proc-macro2", + "quote", "serde", "serde_json", + "syn 1.0.109", ] [[package]] @@ -1580,17 +1752,47 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "graphql_query_derive" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a755cc59cda2641ea3037b4f9f7ef40471c329f55c1fa2db6fa0bb7ae6c1f7ce" +dependencies = [ + "graphql_client_codegen 0.11.0", + "proc-macro2", + "syn 1.0.109", +] + [[package]] name = "graphql_query_derive" version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00bda454f3d313f909298f626115092d348bc231025699f557b27e248475f48c" dependencies = [ - "graphql_client_codegen", + "graphql_client_codegen 0.13.0", "proc-macro2", "syn 1.0.109", ] +[[package]] +name = "h2" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -1634,6 +1836,17 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http" version = "1.1.0" @@ -1645,6 +1858,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + [[package]] name = "http-body" version = "1.0.0" @@ -1652,7 +1876,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" dependencies = [ "bytes", - "http", + "http 1.1.0", ] [[package]] @@ -1663,8 +1887,8 @@ checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" dependencies = [ "bytes", "futures-core", - "http", - "http-body", + "http 1.1.0", + "http-body 1.0.0", "pin-project-lite", ] @@ -1674,12 +1898,42 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + [[package]] name = "human_format" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c3b1f728c459d27b12448862017b96ad4767b1ec2ec5e6434e99f1577f085b8" +[[package]] +name = "hyper" +version = "0.14.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + [[package]] name = "hyper" version = "1.3.1" @@ -1689,8 +1943,8 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http", - "http-body", + "http 1.1.0", + "http-body 1.0.0", "httparse", "itoa", "pin-project-lite", @@ -1699,6 +1953,20 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http 0.2.12", + "hyper 0.14.28", + "rustls 0.21.12", + "tokio", + "tokio-rustls 0.24.1", +] + [[package]] name = "hyper-rustls" version = "0.26.0" @@ -1706,13 +1974,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" dependencies = [ "futures-util", - "http", - "hyper", + "http 1.1.0", + "hyper 1.3.1", "hyper-util", - "rustls", + "rustls 0.22.4", "rustls-pki-types", "tokio", - "tokio-rustls", + "tokio-rustls 0.25.0", "tower-service", ] @@ -1725,9 +1993,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http", - "http-body", - "hyper", + "http 1.1.0", + "http-body 1.0.0", + "hyper 1.3.1", "pin-project-lite", "socket2", "tokio", @@ -1736,6 +2004,35 @@ dependencies = [ "tracing", ] +[[package]] +name = "iana-time-zone" +version = "0.1.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "0.5.0" @@ -1762,6 +2059,16 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" +[[package]] +name = "indexmap" +version = "2.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +dependencies = [ + "equivalent", + "hashbrown 0.14.5", +] + [[package]] name = "io-close" version = "0.3.7" @@ -1784,6 +2091,15 @@ version = "1.70.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.11" @@ -1894,6 +2210,12 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.7.2" @@ -1914,6 +2236,16 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -1930,6 +2262,15 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "num_cpus" version = "1.16.0" @@ -2201,6 +2542,47 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" +[[package]] +name = "reqwest" +version = "0.11.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" +dependencies = [ + "base64 0.21.7", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.28", + "hyper-rustls 0.24.2", + "ipnet", + "js-sys", + "log", + "mime", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls 0.21.12", + "rustls-pemfile 1.0.4", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-rustls 0.24.1", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", + "winreg 0.50.0", +] + [[package]] name = "reqwest" version = "0.12.4" @@ -2211,11 +2593,11 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http", - "http-body", + "http 1.1.0", + "http-body 1.0.0", "http-body-util", - "hyper", - "hyper-rustls", + "hyper 1.3.1", + "hyper-rustls 0.26.0", "hyper-util", "ipnet", "js-sys", @@ -2224,16 +2606,16 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls", + "rustls 0.22.4", "rustls-native-certs", - "rustls-pemfile", + "rustls-pemfile 2.1.2", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-rustls", + "tokio-rustls 0.25.0", "tokio-socks", "tokio-util", "tower-service", @@ -2242,7 +2624,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "winreg", + "winreg 0.52.0", ] [[package]] @@ -2294,6 +2676,18 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring 0.17.8", + "rustls-webpki 0.101.7", + "sct", +] + [[package]] name = "rustls" version = "0.22.4" @@ -2303,7 +2697,7 @@ dependencies = [ "log", "ring 0.17.8", "rustls-pki-types", - "rustls-webpki", + "rustls-webpki 0.102.3", "subtle", "zeroize", ] @@ -2315,12 +2709,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f1fb85efa936c42c6d5fc28d2629bb51e4b2f4b8a5211e297d599cc5a093792" dependencies = [ "openssl-probe", - "rustls-pemfile", + "rustls-pemfile 2.1.2", "rustls-pki-types", "schannel", "security-framework", ] +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", +] + [[package]] name = "rustls-pemfile" version = "2.1.2" @@ -2337,6 +2740,16 @@ version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring 0.17.8", + "untrusted 0.9.0", +] + [[package]] name = "rustls-webpki" version = "0.102.3" @@ -2378,6 +2791,16 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring 0.17.8", + "untrusted 0.9.0", +] + [[package]] name = "security-framework" version = "2.11.0" @@ -2548,6 +2971,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "strsim" version = "0.11.1" @@ -2588,6 +3017,27 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tar" version = "0.4.40" @@ -2719,13 +3169,23 @@ dependencies = [ "syn 2.0.61", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.12", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" dependencies = [ - "rustls", + "rustls 0.22.4", "rustls-pki-types", "tokio", ] @@ -3080,6 +3540,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "0.25.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" + [[package]] name = "winapi" version = "0.3.9" @@ -3111,6 +3577,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets 0.52.5", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -3259,6 +3734,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "winreg" version = "0.52.0" diff --git a/Cargo.toml b/Cargo.toml index 84415b88..ce24999a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,7 @@ semver = { version = "1.0.18", features = ["serde"] } thiserror = "1.0.56" url = { version = "2.5.0", features = ["serde"] } http = "1.1.0" +gitlab = "0.1610.0" [profile.release] strip = true # Automatically strip symbols from the binary. diff --git a/src/push_context.rs b/src/push_context.rs index 8cf88d46..4f9af31a 100644 --- a/src/push_context.rs +++ b/src/push_context.rs @@ -5,6 +5,7 @@ use std::{ }; use color_eyre::eyre::{eyre, Context, Result}; +use gitlab::api::Query; use spdx::Expression; use crate::{ @@ -70,14 +71,21 @@ impl GitContext { pub fn from_cli_and_gitlab( cli: &FlakeHubPushCli, - // local_revision_info likely + local_revision_info: RevisionInfo, ) -> Result { // Gitlab token during jobs for calling api: `CI_JOB_TOKEN` - // TODO(colemickens): graphql is a bit of a distraction right now, come back, just use REST for now + let gitlab_host = std::env::var("CI_SERVER_FQDN") + .wrap_err("couldn't determinte CI_SERVER_FQDN")?; + let gitlab_token = std::env::var("CI_JOB_TOKEN") + .wrap_err("couldn't determinte CI_JOB_TOKEN")?; - // repo_topics: GET /projects/:id/labels - // https://docs.gitlab.com/ee/api/labels.html - // TODO(colemickens): implement + let gitlab_client = gitlab::Gitlab::new(gitlab_host, gitlab_token)?; + + // gitlab -> repo labels + let labels_endpoint = gitlab::api::projects::labels::Labels::builder().build().unwrap(); + let repo_labels = gitlab::api::paged( + labels_endpoint, + gitlab::api::Pagination::Limit(MAX_LABEL_LENGTH)).query(&gitlab_client)?; // revision_info: GET /projects/:id/repository/commits // https://docs.gitlab.com/ee/api/commits.html @@ -87,16 +95,16 @@ impl GitContext { // -- https://gitlab.com/api/v4/projects/54746217/repository/commits/171d5b1d13b326e6870df1c38575c39c58acfcd4 (?stats does nothing) // ---- unclear: https://forum.gitlab.com/t/gitlab-api-projects-statistics-true-commit-count/10435/3 // ---- another user wondering: https://forum.gitlab.com/t/api-to-get-total-commits-count/93118 + // - also checked the graphql api pretty extensively + // -- not finding anything obvious, not seeing the equivalent of the related github graphql query // spdx_expression: can't find any evidence GitLab tries to surface this info + let spdx_expression = &cli.spdx_expression.0; let ctx = GitContext { - spdx_expression: None, - repo_topics: vec![], - revision_info: RevisionInfo { - commit_count: None, - revision: "foo".to_string(), - }, + spdx_expression: spdx_expression.clone(), + repo_topics: repo_labels, + revision_info: local_revision_info, }; Ok(ctx) } @@ -228,7 +236,7 @@ impl PushContext { let token = crate::gitlab::get_runner_bearer_token() .await .wrap_err("Getting upload bearer token from GitLab")?; - let git_ctx = GitContext::from_cli_and_gitlab(cli)?; + let git_ctx = GitContext::from_cli_and_gitlab(cli, local_rev_info)?; (token, git_ctx) } (false, false, Some(u)) => { @@ -314,8 +322,9 @@ impl PushContext { } }; + // TODO(review): shouldn't this maybe only be fatal whenever we are rolling_minor||rolling? + // TODO!!! doubly so since we're not going to be able to get commit_count from the gitlab api? let Some(commit_count) = git_ctx.revision_info.commit_count else { - // TODO(review): shouldn't this maybe only be fatal whenever we are rolling_minor||rolling? return Err(eyre!("Could not determine commit count, this is normally determined via the `--git-root` argument or via the GitHub API")); }; From da0c7c3d6118cc1ee727f699216c181c9ad55f44 Mon Sep 17 00:00:00 2001 From: Cole Mickens Date: Wed, 17 Apr 2024 09:02:44 -0700 Subject: [PATCH 08/27] cargo fmt --- src/cli/mod.rs | 7 +++++-- src/flake_info.rs | 40 +++++++++++++++++++-------------------- src/flakehub_auth_fake.rs | 9 +++++++-- src/flakehub_client.rs | 2 +- src/github/graphql/mod.rs | 1 - src/gitlab/mod.rs | 5 +++-- src/push_context.rs | 18 +++++++++++------- src/release_metadata.rs | 2 +- src/revision_info.rs | 1 - src/s3.rs | 4 ++-- 10 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 819fea99..d50ec096 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -8,7 +8,10 @@ use std::{ }; use crate::{ - build_http_client, flakehub_client::Tarball, github::graphql::{GithubGraphqlDataQuery, MAX_LABEL_LENGTH, MAX_NUM_TOTAL_LABELS}, release_metadata::{ReleaseMetadata} + build_http_client, + flakehub_client::Tarball, + github::graphql::{GithubGraphqlDataQuery, MAX_LABEL_LENGTH, MAX_NUM_TOTAL_LABELS}, + release_metadata::ReleaseMetadata, }; #[derive(Debug, clap::Parser)] @@ -304,4 +307,4 @@ impl FlakeHubPushCli { } } } -} \ No newline at end of file +} diff --git a/src/flake_info.rs b/src/flake_info.rs index 0f79c5b6..e0ffc102 100644 --- a/src/flake_info.rs +++ b/src/flake_info.rs @@ -98,30 +98,30 @@ impl FlakeMetadata { self.source_dir.display() ) })?; - - if !output.status.success() { - let command = format!( - "nix flake show --all-systems --json --no-write-lock-file {}", - self.source_dir.display(), - ); - let msg = format!( - "\ + + if !output.status.success() { + let command = format!( + "nix flake show --all-systems --json --no-write-lock-file {}", + self.source_dir.display(), + ); + let msg = format!( + "\ Failed to execute command `{command}`{maybe_status} \n\ stdout: {stdout}\n\ stderr: {stderr}\n\ ", - stdout = String::from_utf8_lossy(&output.stdout), - stderr = String::from_utf8_lossy(&output.stderr), - maybe_status = if let Some(status) = output.status.code() { - format!(" with status {status}") - } else { - String::new() - } - ); - return Err(eyre!(msg))?; - } - - Ok(()) + stdout = String::from_utf8_lossy(&output.stdout), + stderr = String::from_utf8_lossy(&output.stderr), + maybe_status = if let Some(status) = output.status.code() { + format!(" with status {status}") + } else { + String::new() + } + ); + return Err(eyre!(msg))?; + } + + Ok(()) } /// check_lock_if_exists is specifically to check locked flakes to make sure the flake.lock diff --git a/src/flakehub_auth_fake.rs b/src/flakehub_auth_fake.rs index 946e6197..751d3677 100644 --- a/src/flakehub_auth_fake.rs +++ b/src/flakehub_auth_fake.rs @@ -2,7 +2,12 @@ use color_eyre::eyre::{eyre, Context, Result}; use crate::github::graphql::GithubGraphqlDataResult; -pub async fn get_fake_bearer_token(jwt_issuer_uri: &str, project_owner: &str, repository: &str, github_graphql_data_result: GithubGraphqlDataResult) -> Result { +pub async fn get_fake_bearer_token( + jwt_issuer_uri: &str, + project_owner: &str, + repository: &str, + github_graphql_data_result: GithubGraphqlDataResult, +) -> Result { tracing::warn!("running outside github/gitlab - minting a dev-signed JWT"); let client = reqwest::Client::new(); @@ -29,7 +34,7 @@ pub async fn get_fake_bearer_token(jwt_issuer_uri: &str, project_owner: &str, re .send() .await .wrap_err("Sending request to JWT issuer")?; - + #[derive(serde::Deserialize)] struct Response { token: String, diff --git a/src/flakehub_client.rs b/src/flakehub_client.rs index 4f432f63..99f36407 100644 --- a/src/flakehub_client.rs +++ b/src/flakehub_client.rs @@ -65,7 +65,7 @@ impl FlakeHubClient { url = %release_metadata_post_url, "Computed release metadata POST URL" ); - + tracing::debug!(?release_metadata); //TODO colemickens: sanity check this against main fhp tracing::debug!("repo={}", release_metadata.repo); tracing::debug!("upload_name={}", upload_name); diff --git a/src/github/graphql/mod.rs b/src/github/graphql/mod.rs index 6e3c7259..84f14ec7 100644 --- a/src/github/graphql/mod.rs +++ b/src/github/graphql/mod.rs @@ -50,7 +50,6 @@ impl GithubGraphqlDataQuery { .await .wrap_err("Failed to issue RevCountQuery request to Github's GraphQL API")?; - let response_status = reqwest_response.status(); let response: graphql_client::Response< ::ResponseData, diff --git a/src/gitlab/mod.rs b/src/gitlab/mod.rs index 9ea0bb17..dd26a81c 100644 --- a/src/gitlab/mod.rs +++ b/src/gitlab/mod.rs @@ -4,9 +4,10 @@ use color_eyre::eyre::{eyre, WrapErr}; pub(crate) async fn get_runner_bearer_token() -> color_eyre::Result { // github allows you to at-runtime change the audience of the token // gitlab requires job-level audience/token config, and makes it available via envvar - + let maybe_token = std::env::var("GITLAB_JWT_ID_TOKEN"); - let token = maybe_token.wrap_err("Failed to get a JWT from GitLab. You must configure id_token in the jobs.")?; + let token = maybe_token + .wrap_err("Failed to get a JWT from GitLab. You must configure id_token in the jobs.")?; // for now, don't bother validating audience, since flakehub will validate the audience as part of authn anyway diff --git a/src/push_context.rs b/src/push_context.rs index 4f9af31a..e8c3c6a9 100644 --- a/src/push_context.rs +++ b/src/push_context.rs @@ -74,18 +74,22 @@ impl GitContext { local_revision_info: RevisionInfo, ) -> Result { // Gitlab token during jobs for calling api: `CI_JOB_TOKEN` - let gitlab_host = std::env::var("CI_SERVER_FQDN") - .wrap_err("couldn't determinte CI_SERVER_FQDN")?; - let gitlab_token = std::env::var("CI_JOB_TOKEN") - .wrap_err("couldn't determinte CI_JOB_TOKEN")?; + let gitlab_host = + std::env::var("CI_SERVER_FQDN").wrap_err("couldn't determinte CI_SERVER_FQDN")?; + let gitlab_token = + std::env::var("CI_JOB_TOKEN").wrap_err("couldn't determinte CI_JOB_TOKEN")?; let gitlab_client = gitlab::Gitlab::new(gitlab_host, gitlab_token)?; // gitlab -> repo labels - let labels_endpoint = gitlab::api::projects::labels::Labels::builder().build().unwrap(); + let labels_endpoint = gitlab::api::projects::labels::Labels::builder() + .build() + .unwrap(); let repo_labels = gitlab::api::paged( labels_endpoint, - gitlab::api::Pagination::Limit(MAX_LABEL_LENGTH)).query(&gitlab_client)?; + gitlab::api::Pagination::Limit(MAX_LABEL_LENGTH), + ) + .query(&gitlab_client)?; // revision_info: GET /projects/:id/repository/commits // https://docs.gitlab.com/ee/api/commits.html @@ -99,7 +103,7 @@ impl GitContext { // -- not finding anything obvious, not seeing the equivalent of the related github graphql query // spdx_expression: can't find any evidence GitLab tries to surface this info - let spdx_expression = &cli.spdx_expression.0; + let spdx_expression = &cli.spdx_expression.0; let ctx = GitContext { spdx_expression: spdx_expression.clone(), diff --git a/src/release_metadata.rs b/src/release_metadata.rs index 9a13118b..677458b6 100644 --- a/src/release_metadata.rs +++ b/src/release_metadata.rs @@ -59,4 +59,4 @@ where } else { serializer.serialize_none() } -} \ No newline at end of file +} diff --git a/src/revision_info.rs b/src/revision_info.rs index f56cff8f..dc51f6e3 100644 --- a/src/revision_info.rs +++ b/src/revision_info.rs @@ -5,7 +5,6 @@ use crate::Visibility; const README_FILENAME_LOWERCASE: &str = "readme.md"; - #[derive(Clone)] pub(crate) struct RevisionInfo { pub(crate) commit_count: Option, diff --git a/src/s3.rs b/src/s3.rs index d8ed63d6..013d2e74 100644 --- a/src/s3.rs +++ b/src/s3.rs @@ -11,7 +11,8 @@ pub async fn upload_release_to_s3(presigned_s3_url: String, tarball: Tarball) -> let mut header_map = HeaderMap::new(); header_map.insert( reqwest::header::CONTENT_LENGTH, - reqwest::header::HeaderValue::from_str(&format!("{}", tarball.bytes.len())).unwrap(), + reqwest::header::HeaderValue::from_str(&format!("{}", tarball.bytes.len())) + .unwrap(), ); header_map.insert( reqwest::header::HeaderName::from_static("x-amz-checksum-sha256"), @@ -41,4 +42,3 @@ pub async fn upload_release_to_s3(presigned_s3_url: String, tarball: Tarball) -> Ok(()) } - From b1b39870ef269f03e3e90ad00e34fb9c6b1629c4 Mon Sep 17 00:00:00 2001 From: Cole Mickens Date: Wed, 17 Apr 2024 09:02:59 -0700 Subject: [PATCH 09/27] cargo fix --- src/cli/mod.rs | 13 +++---------- src/flakehub_auth_fake.rs | 2 +- src/flakehub_client.rs | 2 +- src/gitlab/mod.rs | 2 +- src/main.rs | 5 ++--- src/release_metadata.rs | 4 ++-- src/revision_info.rs | 4 ++-- 7 files changed, 12 insertions(+), 20 deletions(-) diff --git a/src/cli/mod.rs b/src/cli/mod.rs index d50ec096..0485ca35 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -1,18 +1,11 @@ mod instrumentation; -use color_eyre::eyre::{eyre, WrapErr}; + use std::{ - collections::HashSet, - path::{Path, PathBuf}, - process::ExitCode, + path::{PathBuf}, }; -use crate::{ - build_http_client, - flakehub_client::Tarball, - github::graphql::{GithubGraphqlDataQuery, MAX_LABEL_LENGTH, MAX_NUM_TOTAL_LABELS}, - release_metadata::ReleaseMetadata, -}; + #[derive(Debug, clap::Parser)] #[clap(version)] diff --git a/src/flakehub_auth_fake.rs b/src/flakehub_auth_fake.rs index 751d3677..a1fae662 100644 --- a/src/flakehub_auth_fake.rs +++ b/src/flakehub_auth_fake.rs @@ -1,4 +1,4 @@ -use color_eyre::eyre::{eyre, Context, Result}; +use color_eyre::eyre::{Context, Result}; use crate::github::graphql::GithubGraphqlDataResult; diff --git a/src/flakehub_client.rs b/src/flakehub_client.rs index 99f36407..b934f4c9 100644 --- a/src/flakehub_client.rs +++ b/src/flakehub_client.rs @@ -1,7 +1,7 @@ use std::str::FromStr; use color_eyre::eyre::{eyre, Context, Result}; -use http::StatusCode; + use reqwest::{header::HeaderMap, Response}; use uuid::Uuid; diff --git a/src/gitlab/mod.rs b/src/gitlab/mod.rs index dd26a81c..78a08b10 100644 --- a/src/gitlab/mod.rs +++ b/src/gitlab/mod.rs @@ -1,4 +1,4 @@ -use color_eyre::eyre::{eyre, WrapErr}; +use color_eyre::eyre::{WrapErr}; #[tracing::instrument(skip_all, fields(audience = tracing::field::Empty))] pub(crate) async fn get_runner_bearer_token() -> color_eyre::Result { diff --git a/src/main.rs b/src/main.rs index 66d6c68a..6a509197 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,6 @@ use std::{ fmt::Display, io::IsTerminal, - path::{Path, PathBuf}, process::ExitCode, }; @@ -12,8 +11,8 @@ use http::StatusCode; use uuid::Uuid; use crate::{ - flakehub_client::FlakeHubClient, github::graphql::GithubGraphqlDataQuery, - push_context::PushContext, release_metadata::ReleaseMetadata, + flakehub_client::FlakeHubClient, + push_context::PushContext, }; mod cli; mod error; diff --git a/src/release_metadata.rs b/src/release_metadata.rs index 677458b6..73ce2cfc 100644 --- a/src/release_metadata.rs +++ b/src/release_metadata.rs @@ -1,5 +1,5 @@ -use color_eyre::eyre::{eyre, WrapErr}; -use std::path::{Path, PathBuf}; + + use crate::Visibility; diff --git a/src/revision_info.rs b/src/revision_info.rs index dc51f6e3..02539971 100644 --- a/src/revision_info.rs +++ b/src/revision_info.rs @@ -1,7 +1,7 @@ use color_eyre::eyre::{eyre, WrapErr}; -use std::path::{Path, PathBuf}; +use std::path::{Path}; + -use crate::Visibility; const README_FILENAME_LOWERCASE: &str = "readme.md"; From 482e85526ba041b04a3e8b74d2108c9a7811ae43 Mon Sep 17 00:00:00 2001 From: Cole Mickens Date: Wed, 17 Apr 2024 09:09:52 -0700 Subject: [PATCH 10/27] STASH AGAIN: this time cargo clippy --fix is making my code not compile From 4b6634d07b084a8902c9519e6fd083448e33436c Mon Sep 17 00:00:00 2001 From: Cole Mickens Date: Wed, 17 Apr 2024 09:14:08 -0700 Subject: [PATCH 11/27] more clippy and fmt --- flake.nix | 1 + src/cli/mod.rs | 7 +------ src/error.rs | 25 +++++++++++++------------ src/flake_info.rs | 7 ++++--- src/flakehub_client.rs | 11 ++++------- src/gitlab/mod.rs | 2 +- src/main.rs | 13 +++---------- src/push_context.rs | 16 ++++++++-------- src/release_metadata.rs | 5 ----- src/revision_info.rs | 36 +----------------------------------- 10 files changed, 36 insertions(+), 87 deletions(-) diff --git a/flake.nix b/flake.nix index 8e021206..ac108d39 100644 --- a/flake.nix +++ b/flake.nix @@ -75,6 +75,7 @@ nodejs_latest nodePackages_latest.pnpm + bacon ] ++ inputs.self.packages.${system}.flakehub-push.buildInputs ++ pkgs.lib.optionals pkgs.stdenv.isDarwin (with pkgs.darwin.apple_sdk.frameworks; [ Security ]); diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 0485ca35..3d66e571 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -1,11 +1,6 @@ mod instrumentation; - -use std::{ - path::{PathBuf}, -}; - - +use std::path::PathBuf; #[derive(Debug, clap::Parser)] #[clap(version)] diff --git a/src/error.rs b/src/error.rs index 3eccbfb7..d1eabe44 100644 --- a/src/error.rs +++ b/src/error.rs @@ -17,16 +17,17 @@ impl Error { } } - /// Output a Github Actions annotation command if desired. - // Note: These may only be one line! Any further lines will not be printed! - pub(crate) fn maybe_github_actions_annotation(&self) { - if std::env::var("GITHUB_ACTIONS").is_ok() { - match self { - Error::Unauthorized(message) => { - println!("::error title=Unauthorized::{message}") - } - Error::Conflict { .. } => println!("::error title=Conflict::{self}"), - } - } - } + // TODO(colemickens/review): was this used? where? + // /// Output a Github Actions annotation command if desired. + // // Note: These may only be one line! Any further lines will not be printed! + // pub(crate) fn maybe_github_actions_annotation(&self) { + // if std::env::var("GITHUB_ACTIONS").is_ok() { + // match self { + // Error::Unauthorized(message) => { + // println!("::error title=Unauthorized::{message}") + // } + // Error::Conflict { .. } => println!("::error title=Conflict::{self}"), + // } + // } + // } } diff --git a/src/flake_info.rs b/src/flake_info.rs index e0ffc102..9f670f74 100644 --- a/src/flake_info.rs +++ b/src/flake_info.rs @@ -76,7 +76,7 @@ impl FlakeMetadata { Ok(FlakeMetadata { source_dir: source, flake_locked_url: flake_locked_url.to_string(), - metadata_json: metadata_json, + metadata_json, }) } @@ -294,12 +294,13 @@ impl FlakeMetadata { let mut read_dir = tokio::fs::read_dir(&self.source_dir).await?; let readme_path: Option = { + let mut readme_path = None; while let Some(entry) = read_dir.next_entry().await? { if entry.file_name().to_ascii_lowercase() == README_FILENAME_LOWERCASE { - Some(Some(entry.path())); + readme_path = Some(entry.path()); } } - None + readme_path }; let readme = if let Some(readme_path) = readme_path { Some(tokio::fs::read_to_string(&readme_path).await?) diff --git a/src/flakehub_client.rs b/src/flakehub_client.rs index b934f4c9..2723be93 100644 --- a/src/flakehub_client.rs +++ b/src/flakehub_client.rs @@ -1,7 +1,4 @@ -use std::str::FromStr; - use color_eyre::eyre::{eyre, Context, Result}; - use reqwest::{header::HeaderMap, Response}; use uuid::Uuid; @@ -35,15 +32,15 @@ pub fn flakehub_headers() -> HeaderMap { } impl FlakeHubClient { - pub fn new(host: url::Url, token: String) -> Result { + pub fn new(host: url::Url, bearer_token: String) -> Result { let builder = reqwest::ClientBuilder::new().user_agent("flakehub-push"); let client = builder.build()?; let client = Self { - client: client, - bearer_token: token, - host: host, + client, + bearer_token, + host, }; Ok(client) diff --git a/src/gitlab/mod.rs b/src/gitlab/mod.rs index 78a08b10..6875fe22 100644 --- a/src/gitlab/mod.rs +++ b/src/gitlab/mod.rs @@ -1,4 +1,4 @@ -use color_eyre::eyre::{WrapErr}; +use color_eyre::eyre::WrapErr; #[tracing::instrument(skip_all, fields(audience = tracing::field::Empty))] pub(crate) async fn get_runner_bearer_token() -> color_eyre::Result { diff --git a/src/main.rs b/src/main.rs index 6a509197..51bd84bd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,8 +1,4 @@ -use std::{ - fmt::Display, - io::IsTerminal, - process::ExitCode, -}; +use std::{fmt::Display, io::IsTerminal, process::ExitCode}; use clap::Parser; use color_eyre::eyre::{eyre, Result, WrapErr}; @@ -10,10 +6,7 @@ use error::Error; use http::StatusCode; use uuid::Uuid; -use crate::{ - flakehub_client::FlakeHubClient, - push_context::PushContext, -}; +use crate::{flakehub_client::FlakeHubClient, push_context::PushContext}; mod cli; mod error; mod flake_info; @@ -134,7 +127,7 @@ async fn main() -> Result { s3::upload_release_to_s3(stage_result.s3_upload_url, ctx.tarball).await?; // "publish.rs" - publish the release after upload - let _publish_result = fhclient.release_publish(stage_result.uuid).await?; + fhclient.release_publish(stage_result.uuid).await?; tracing::info!( "Successfully released new version of {}/{}", diff --git a/src/push_context.rs b/src/push_context.rs index e8c3c6a9..1e581d15 100644 --- a/src/push_context.rs +++ b/src/push_context.rs @@ -58,7 +58,7 @@ impl GitContext { }; let ctx = GitContext { - spdx_expression: spdx_expression, + spdx_expression, repo_topics: github_graphql_data_result.topics.clone(), revision_info: RevisionInfo { // TODO(colemickens): type coherency here... :/ (as is bad) @@ -227,7 +227,7 @@ impl PushContext { ) .await?; - let git_ctx = GitContext::from_cli_and_github(&cli, &github_graphql_data_result)?; + let git_ctx = GitContext::from_cli_and_github(cli, &github_graphql_data_result)?; let token = crate::github::get_actions_id_bearer_token(&cli.host) .await @@ -261,7 +261,7 @@ impl PushContext { .await?; let git_ctx: GitContext = - GitContext::from_cli_and_github(&cli, &github_graphql_data_result)?; + GitContext::from_cli_and_github(cli, &github_graphql_data_result)?; let token = flakehub_auth_fake::get_fake_bearer_token( u, @@ -316,7 +316,7 @@ impl PushContext { (Some(minor), _) => format!("0.{minor}"), (None, _) if cli.rolling => DEFAULT_ROLLING_PREFIX.to_string(), (None, Some(tag)) => { - let version_only = tag.strip_prefix('v').unwrap_or(&tag); + let version_only = tag.strip_prefix('v').unwrap_or(tag); // Ensure the version respects semver semver::Version::from_str(version_only).wrap_err_with(|| eyre!("Failed to parse version `{tag}` as semver, see https://semver.org/ for specifications"))?; tag.to_string() @@ -433,11 +433,11 @@ impl PushContext { let readme = flake_metadata.get_readme_contents().await?; let release_metadata = ReleaseMetadata { - commit_count: commit_count, - description: description, + commit_count, + description, outputs: flake_outputs.0, raw_flake_metadata: flake_metadata.metadata_json.clone(), - readme: readme, + readme, // TODO(colemickens): remove this confusing, redundant field (FH-267) repo: upload_name.to_string(), revision: git_ctx.revision_info.revision, @@ -461,7 +461,7 @@ impl PushContext { flakehub_host: cli.host.clone(), auth_token: token, - upload_name: upload_name, + upload_name, release_version: rolling_minor_with_postfix_or_tag, error_if_release_conflicts: cli.error_on_conflict, diff --git a/src/release_metadata.rs b/src/release_metadata.rs index 73ce2cfc..340b103c 100644 --- a/src/release_metadata.rs +++ b/src/release_metadata.rs @@ -1,10 +1,5 @@ - - - use crate::Visibility; -const README_FILENAME_LOWERCASE: &str = "readme.md"; - #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub(crate) struct ReleaseMetadata { pub(crate) commit_count: usize, diff --git a/src/revision_info.rs b/src/revision_info.rs index 02539971..cc892c8d 100644 --- a/src/revision_info.rs +++ b/src/revision_info.rs @@ -1,9 +1,5 @@ use color_eyre::eyre::{eyre, WrapErr}; -use std::path::{Path}; - - - -const README_FILENAME_LOWERCASE: &str = "readme.md"; +use std::path::Path; #[derive(Clone)] pub(crate) struct RevisionInfo { @@ -52,33 +48,3 @@ impl RevisionInfo { }) } } - -fn option_string_to_spdx<'de, D>(deserializer: D) -> Result, D::Error> -where - D: serde::de::Deserializer<'de>, -{ - let spdx_identifier: Option<&str> = serde::Deserialize::deserialize(deserializer)?; - - if let Some(spdx_identifier) = spdx_identifier { - spdx::Expression::parse(spdx_identifier) - .map_err(serde::de::Error::custom) - .map(Option::Some) - } else { - Ok(None) - } -} - -fn option_spdx_serialize( - spdx_identifier: &Option, - serializer: S, -) -> Result -where - S: serde::ser::Serializer, -{ - if let Some(spdx_identifier) = spdx_identifier { - let spdx_string = spdx_identifier.to_string(); - serializer.serialize_str(&spdx_string) - } else { - serializer.serialize_none() - } -} From dd01368e027e86f5696f55721d353f0307efa438 Mon Sep 17 00:00:00 2001 From: Cole Mickens Date: Wed, 17 Apr 2024 09:28:38 -0700 Subject: [PATCH 12/27] move stage conflict handling to flakehub_client --- src/flakehub_client.rs | 64 ++++++++++++++++++++++++++++++++++-- src/main.rs | 73 +++++++++--------------------------------- 2 files changed, 76 insertions(+), 61 deletions(-) diff --git a/src/flakehub_client.rs b/src/flakehub_client.rs index 2723be93..4941ece9 100644 --- a/src/flakehub_client.rs +++ b/src/flakehub_client.rs @@ -1,8 +1,10 @@ use color_eyre::eyre::{eyre, Context, Result}; -use reqwest::{header::HeaderMap, Response}; +use http::StatusCode; +use reqwest::header::HeaderMap; use uuid::Uuid; use crate::release_metadata::ReleaseMetadata; +use crate::Error; pub struct FlakeHubClient { host: url::Url, @@ -15,6 +17,12 @@ pub struct Tarball { pub bytes: Vec, } +#[derive(serde::Deserialize)] +pub(crate) struct StageResult { + pub(crate) s3_upload_url: String, + pub(crate) uuid: Uuid, +} + // TODO(future): static init pub fn flakehub_headers() -> HeaderMap { let mut header_map = HeaderMap::new(); @@ -51,7 +59,8 @@ impl FlakeHubClient { release_version: &str, release_metadata: &ReleaseMetadata, tarball: &Tarball, - ) -> Result { + error_if_release_conflicts: bool, + ) -> Result> { let flake_tarball_len = tarball.bytes.len(); let flake_tarball_hash_base64 = &tarball.hash_base64; let relative_url: &String = &format!("upload/{upload_name}/{release_version}/{flake_tarball_len}/{flake_tarball_hash_base64}"); @@ -77,7 +86,56 @@ impl FlakeHubClient { .await .unwrap(); - Ok(response) + let release_metadata_post_response_status = response.status(); + tracing::trace!( + status = tracing::field::display(release_metadata_post_response_status), + "Got release metadata POST response" + ); + + match release_metadata_post_response_status { + StatusCode::OK => (), + StatusCode::CONFLICT => { + tracing::info!( + "Release for revision `{revision}` of {upload_name}/{release_version} already exists; flakehub-push will not upload it again", + revision = &release_metadata.revision, + upload_name = upload_name, + release_version = &release_version, + ); + if error_if_release_conflicts { + return Err(Error::Conflict { + upload_name: upload_name.to_string(), + release_version: release_version.to_string(), + })?; + } else { + // we're just done, and happy about it: + return Ok(None); + } + } + StatusCode::UNAUTHORIZED => { + let body = response.bytes().await?; + let message = serde_json::from_slice::(&body)?; + + return Err(Error::Unauthorized(message))?; + } + _ => { + let body = response.bytes().await?; + let message = serde_json::from_slice::(&body)?; + return Err(eyre!( + "\ + Status {release_metadata_post_response_status} from metadata POST\n\ + {}\ + ", + message + )); + } + } + + let stage_result: StageResult = response + .json() + .await + .wrap_err("Decoding release metadata POST response")?; + + Ok(Some(stage_result)) } pub async fn release_publish(&self, release_uuidv7: Uuid) -> Result<()> { diff --git a/src/main.rs b/src/main.rs index 51bd84bd..d2d07b1c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,8 @@ -use std::{fmt::Display, io::IsTerminal, process::ExitCode}; +use std::{ + fmt::Display, + io::IsTerminal, + process::ExitCode, +}; use clap::Parser; use color_eyre::eyre::{eyre, Result, WrapErr}; @@ -6,7 +10,10 @@ use error::Error; use http::StatusCode; use uuid::Uuid; -use crate::{flakehub_client::FlakeHubClient, push_context::PushContext}; +use crate::{ + flakehub_client::{FlakeHubClient, StageResult}, + push_context::PushContext, +}; mod cli; mod error; mod flake_info; @@ -58,70 +65,20 @@ async fn main() -> Result { let fhclient = FlakeHubClient::new(ctx.flakehub_host, ctx.auth_token)?; // "upload.rs" - stage the release - let stage_response = fhclient + let stage_result: Option = fhclient .release_stage( &ctx.upload_name, &ctx.release_version, &ctx.metadata, &ctx.tarball, + ctx.error_if_release_conflicts, ) .await?; - // handle the response here, rather than in client, so we can do special behavior - // TODO(review): what do we things, some sort of ReleaseFlags and indeed handle this in the client? - let release_metadata_post_response_status = stage_response.status(); - tracing::trace!( - status = tracing::field::display(release_metadata_post_response_status), - "Got release metadata POST response" - ); - - match release_metadata_post_response_status { - StatusCode::OK => (), - StatusCode::CONFLICT => { - tracing::info!( - "Release for revision `{revision}` of {upload_name}/{release_version} already exists; flakehub-push will not upload it again", - revision = &ctx.metadata.revision, - upload_name = ctx.upload_name, - release_version = ctx.release_version, - ); - if ctx.error_if_release_conflicts { - return Err(Error::Conflict { - upload_name: ctx.upload_name, - release_version: ctx.release_version, - })?; - } else { - // we're just done, and happy about it: - return Ok(ExitCode::SUCCESS); - } - } - StatusCode::UNAUTHORIZED => { - let body = stage_response.bytes().await?; - let message = serde_json::from_slice::(&body)?; - - return Err(Error::Unauthorized(message))?; - } - _ => { - let body = stage_response.bytes().await?; - let message = serde_json::from_slice::(&body)?; - return Err(eyre!( - "\ - Status {release_metadata_post_response_status} from metadata POST\n\ - {}\ - ", - message - )); - } - } - - #[derive(serde::Deserialize)] - struct StageResult { - s3_upload_url: String, - uuid: Uuid, - } - let stage_result: StageResult = stage_response - .json() - .await - .wrap_err("Decoding release metadata POST response")?; + let stage_result = match stage_result { + Some(stage_result) => stage_result, + None => return Ok(ExitCode::SUCCESS), + }; // upload tarball to s3 s3::upload_release_to_s3(stage_result.s3_upload_url, ctx.tarball).await?; From bb111f63d1fe2b9f43ab4d6a13a4f53090d85487 Mon Sep 17 00:00:00 2001 From: Cole Mickens Date: Wed, 17 Apr 2024 10:02:37 -0700 Subject: [PATCH 13/27] cargo clippy fix and fmt --- src/main.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/main.rs b/src/main.rs index d2d07b1c..2b7e1f69 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,14 +1,8 @@ -use std::{ - fmt::Display, - io::IsTerminal, - process::ExitCode, -}; +use std::{fmt::Display, io::IsTerminal, process::ExitCode}; use clap::Parser; -use color_eyre::eyre::{eyre, Result, WrapErr}; +use color_eyre::eyre::Result; use error::Error; -use http::StatusCode; -use uuid::Uuid; use crate::{ flakehub_client::{FlakeHubClient, StageResult}, From 13a45a2df8e19f890be93d825f7e5330ef80143c Mon Sep 17 00:00:00 2001 From: Cole Mickens Date: Wed, 17 Apr 2024 10:02:57 -0700 Subject: [PATCH 14/27] refactor: restore github error surfacing --- src/error.rs | 24 +++++++++++------------- src/main.rs | 12 ++++++++++++ 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/src/error.rs b/src/error.rs index d1eabe44..cafb6ee7 100644 --- a/src/error.rs +++ b/src/error.rs @@ -17,17 +17,15 @@ impl Error { } } - // TODO(colemickens/review): was this used? where? - // /// Output a Github Actions annotation command if desired. - // // Note: These may only be one line! Any further lines will not be printed! - // pub(crate) fn maybe_github_actions_annotation(&self) { - // if std::env::var("GITHUB_ACTIONS").is_ok() { - // match self { - // Error::Unauthorized(message) => { - // println!("::error title=Unauthorized::{message}") - // } - // Error::Conflict { .. } => println!("::error title=Conflict::{self}"), - // } - // } - // } + // Note: These may only be one line! Any further lines will not be printed! + pub(crate) fn maybe_github_actions_annotation(&self) { + if std::env::var("GITHUB_ACTIONS").is_ok() { + match self { + Error::Unauthorized(message) => { + println!("::error title=Unauthorized::{message}") + } + Error::Conflict { .. } => println!("::error title=Conflict::{self}"), + } + } + } } diff --git a/src/main.rs b/src/main.rs index 2b7e1f69..ba48eab8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -50,6 +50,18 @@ async fn main() -> Result { }) .install()?; + match execute().await { + Ok(exit) => Ok(exit), + Err(error) => { + if let Some(known_error) = error.downcast_ref::() { + known_error.maybe_github_actions_annotation() + } + Err(error) + } + } +} + +async fn execute() -> Result { let mut cli = cli::FlakeHubPushCli::parse(); cli.instrumentation.setup()?; From da72bdf737f219e2dfb9f928e2f2d30b9de5bab7 Mon Sep 17 00:00:00 2001 From: Cole Mickens Date: Wed, 17 Apr 2024 10:26:29 -0700 Subject: [PATCH 15/27] refactor: remove testing scripts --- scripts/build-musl.sh | 12 ------------ scripts/run-miniserve.sh | 8 -------- 2 files changed, 20 deletions(-) delete mode 100755 scripts/build-musl.sh delete mode 100755 scripts/run-miniserve.sh diff --git a/scripts/build-musl.sh b/scripts/build-musl.sh deleted file mode 100755 index 3935222c..00000000 --- a/scripts/build-musl.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -set -x -set -euo pipefail - -DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -cd "${DIR}/.." - -export CARGO_BUILD_TARGET="x86_64-unknown-linux-musl"; -export CARGO_BUILD_RUSTFLAGS="-C target-feature=+crt-static"; -# export CARGO_PROFILE_DEV_STRIP="debuginfo" - -cargo watch -x build diff --git a/scripts/run-miniserve.sh b/scripts/run-miniserve.sh deleted file mode 100755 index da1bfb89..00000000 --- a/scripts/run-miniserve.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -set -x -set -euo pipefail - -DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -cd "${DIR}/.." - -miniserve -i :: --port 9898 ./target/x86_64-unknown-linux-musl/debug From 1f3e424a74d65a83f8d2710146b626ba8db2de0a Mon Sep 17 00:00:00 2001 From: Cole Mickens Date: Wed, 17 Apr 2024 14:07:04 -0700 Subject: [PATCH 16/27] fake_auth: fix fakeidp jwt usage, take Url for issuer --- src/cli/mod.rs | 8 +++----- src/flakehub_auth_fake.rs | 16 +++++++--------- src/push_context.rs | 2 +- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 3d66e571..ec221b9f 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -45,11 +45,9 @@ pub(crate) struct FlakeHubPushCli { // This should only be used by DeterminateSystems #[clap(long, env = "FLAKEHUB_PUSH_MIRROR", default_value_t = false)] pub(crate) mirror: bool, - /// URL of a JWT mock server (like https://github.com/ruiyang/jwt-mock-server) which can issue tokens. - /// - /// Used instead of ACTIONS_ID_TOKEN_REQUEST_URL/ACTIONS_ID_TOKEN_REQUEST_TOKEN when developing locally. - #[clap(long, env = "FLAKEHUB_PUSH_JWT_ISSUER_URI", value_parser = StringToNoneParser, default_value = "")] - pub(crate) jwt_issuer_uri: OptionString, + + /// URL of a JWT mock server (like https://github.com/spectare/fakeidp) which can issue tokens. + pub(crate) jwt_issuer_uri: Option, /// User-supplied labels, merged with any associated with GitHub repository (if possible) #[clap( diff --git a/src/flakehub_auth_fake.rs b/src/flakehub_auth_fake.rs index a1fae662..61063e2f 100644 --- a/src/flakehub_auth_fake.rs +++ b/src/flakehub_auth_fake.rs @@ -3,7 +3,7 @@ use color_eyre::eyre::{Context, Result}; use crate::github::graphql::GithubGraphqlDataResult; pub async fn get_fake_bearer_token( - jwt_issuer_uri: &str, + jwt_issuer_uri: &url::Url, project_owner: &str, repository: &str, github_graphql_data_result: GithubGraphqlDataResult, @@ -27,21 +27,19 @@ pub async fn get_fake_bearer_token( tracing::debug!(?claims); + let token_gen_endpoint = jwt_issuer_uri.join("/token")?; + let response = client - .post(jwt_issuer_uri) + .post(token_gen_endpoint) .header("Content-Type", "application/json") .json(&claims) .send() .await .wrap_err("Sending request to JWT issuer")?; - #[derive(serde::Deserialize)] - struct Response { - token: String, - } - let response_deserialized: Response = response - .json() + let token = response + .text() .await .wrap_err("Getting token from JWT issuer's response")?; - Ok(response_deserialized.token) + Ok(token) } diff --git a/src/push_context.rs b/src/push_context.rs index 1e581d15..58b56577 100644 --- a/src/push_context.rs +++ b/src/push_context.rs @@ -209,7 +209,7 @@ impl PushContext { // "cli" and "git_ctx" are the user/env supplied info, augmented with data we might have fetched from github/gitlab apis - let (token, git_ctx) = match (is_github, is_gitlab, &cli.jwt_issuer_uri.0) { + let (token, git_ctx) = match (is_github, is_gitlab, &cli.jwt_issuer_uri) { (true, false, None) => { // GITHUB CI let github_token = cli From 1d7250a13b611a14446da1fecaa638da6d7fdc0c Mon Sep 17 00:00:00 2001 From: Cole Mickens Date: Wed, 8 May 2024 07:54:32 -0700 Subject: [PATCH 17/27] push_context: drop unused gitlab bits --- src/push_context.rs | 39 ++++++--------------------------------- 1 file changed, 6 insertions(+), 33 deletions(-) diff --git a/src/push_context.rs b/src/push_context.rs index 58b56577..98a589de 100644 --- a/src/push_context.rs +++ b/src/push_context.rs @@ -5,7 +5,6 @@ use std::{ }; use color_eyre::eyre::{eyre, Context, Result}; -use gitlab::api::Query; use spdx::Expression; use crate::{ @@ -69,45 +68,16 @@ impl GitContext { Ok(ctx) } - pub fn from_cli_and_gitlab( + pub async fn from_cli_and_gitlab( cli: &FlakeHubPushCli, local_revision_info: RevisionInfo, ) -> Result { - // Gitlab token during jobs for calling api: `CI_JOB_TOKEN` - let gitlab_host = - std::env::var("CI_SERVER_FQDN").wrap_err("couldn't determinte CI_SERVER_FQDN")?; - let gitlab_token = - std::env::var("CI_JOB_TOKEN").wrap_err("couldn't determinte CI_JOB_TOKEN")?; - - let gitlab_client = gitlab::Gitlab::new(gitlab_host, gitlab_token)?; - - // gitlab -> repo labels - let labels_endpoint = gitlab::api::projects::labels::Labels::builder() - .build() - .unwrap(); - let repo_labels = gitlab::api::paged( - labels_endpoint, - gitlab::api::Pagination::Limit(MAX_LABEL_LENGTH), - ) - .query(&gitlab_client)?; - - // revision_info: GET /projects/:id/repository/commits - // https://docs.gitlab.com/ee/api/commits.html - // not sure this has what we need: - // - hitting projects endpoint with ?statistics=true gives a commit_count but it's unclear what that is - // -- https://gitlab.com/api/v4/projects/54746217?statistics=true - // -- https://gitlab.com/api/v4/projects/54746217/repository/commits/171d5b1d13b326e6870df1c38575c39c58acfcd4 (?stats does nothing) - // ---- unclear: https://forum.gitlab.com/t/gitlab-api-projects-statistics-true-commit-count/10435/3 - // ---- another user wondering: https://forum.gitlab.com/t/api-to-get-total-commits-count/93118 - // - also checked the graphql api pretty extensively - // -- not finding anything obvious, not seeing the equivalent of the related github graphql query - // spdx_expression: can't find any evidence GitLab tries to surface this info let spdx_expression = &cli.spdx_expression.0; let ctx = GitContext { spdx_expression: spdx_expression.clone(), - repo_topics: repo_labels, + repo_topics: vec![], revision_info: local_revision_info, }; Ok(ctx) @@ -240,7 +210,9 @@ impl PushContext { let token = crate::gitlab::get_runner_bearer_token() .await .wrap_err("Getting upload bearer token from GitLab")?; - let git_ctx = GitContext::from_cli_and_gitlab(cli, local_rev_info)?; + + let git_ctx = GitContext::from_cli_and_gitlab(cli, local_rev_info).await?; + (token, git_ctx) } (false, false, Some(u)) => { @@ -469,6 +441,7 @@ impl PushContext { metadata: release_metadata, tarball: flake_tarball, }; + Ok(ctx) } } From 84f81af0901a44d5925b96cec463f373de4519c4 Mon Sep 17 00:00:00 2001 From: Cole Mickens Date: Wed, 8 May 2024 13:41:06 -0700 Subject: [PATCH 18/27] initial feedback --- src/cli/mod.rs | 3 ++- src/error.rs | 1 + src/flakehub_auth_fake.rs | 9 +++++---- src/flakehub_client.rs | 2 +- src/main.rs | 10 +++++----- src/push_context.rs | 1 + 6 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/cli/mod.rs b/src/cli/mod.rs index ec221b9f..138a2ec9 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -47,7 +47,8 @@ pub(crate) struct FlakeHubPushCli { pub(crate) mirror: bool, /// URL of a JWT mock server (like https://github.com/spectare/fakeidp) which can issue tokens. - pub(crate) jwt_issuer_uri: Option, + #[clap(long)] + pub(crate) jwt_issuer_uri: Option, /// User-supplied labels, merged with any associated with GitHub repository (if possible) #[clap( diff --git a/src/error.rs b/src/error.rs index cafb6ee7..3eccbfb7 100644 --- a/src/error.rs +++ b/src/error.rs @@ -17,6 +17,7 @@ impl Error { } } + /// Output a Github Actions annotation command if desired. // Note: These may only be one line! Any further lines will not be printed! pub(crate) fn maybe_github_actions_annotation(&self) { if std::env::var("GITHUB_ACTIONS").is_ok() { diff --git a/src/flakehub_auth_fake.rs b/src/flakehub_auth_fake.rs index 61063e2f..818eed3f 100644 --- a/src/flakehub_auth_fake.rs +++ b/src/flakehub_auth_fake.rs @@ -3,7 +3,7 @@ use color_eyre::eyre::{Context, Result}; use crate::github::graphql::GithubGraphqlDataResult; pub async fn get_fake_bearer_token( - jwt_issuer_uri: &url::Url, + jwt_issuer_uri: &str, project_owner: &str, repository: &str, github_graphql_data_result: GithubGraphqlDataResult, @@ -18,8 +18,8 @@ pub async fn get_fake_bearer_token( // TODO(review): on the contrary, I think we should ditch this, and we should basically use forge_login-esque functionality for this going forward // this would remove the entire need for the fake JWT server, since we are ourselves a JWT issuer claims.aud = "flakehub-localhost".to_string(); - claims.iss = "flakehub-push-dev".to_string(); - claims.repository = repository.to_string(); + claims.iss = jwt_issuer_uri.to_string(); + claims.repository = format!("{project_owner}/{repository}"); claims.repository_owner = project_owner.to_string(); claims.repository_id = github_graphql_data_result.project_id.to_string(); @@ -27,7 +27,8 @@ pub async fn get_fake_bearer_token( tracing::debug!(?claims); - let token_gen_endpoint = jwt_issuer_uri.join("/token")?; + let issuer_url = url::Url::parse(&jwt_issuer_uri)?; + let token_gen_endpoint = issuer_url.join("/token")?; let response = client .post(token_gen_endpoint) diff --git a/src/flakehub_client.rs b/src/flakehub_client.rs index 4941ece9..4bf90bec 100644 --- a/src/flakehub_client.rs +++ b/src/flakehub_client.rs @@ -159,7 +159,7 @@ impl FlakeHubClient { "Got publish POST response" ); - if publish_response_status != 200 { + if publish_response_status != StatusCode::OK { return Err(eyre!( "\ Status {publish_response_status} from publish POST\n\ diff --git a/src/main.rs b/src/main.rs index ba48eab8..7311287a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -62,11 +62,11 @@ async fn main() -> Result { } async fn execute() -> Result { - let mut cli = cli::FlakeHubPushCli::parse(); - cli.instrumentation.setup()?; - - let ctx: PushContext = PushContext::from_cli_and_env(&mut cli).await?; - drop(cli); // drop cli so we force ourselves to use ctx + let ctx = { + let mut cli = cli::FlakeHubPushCli::parse(); + cli.instrumentation.setup()?; + PushContext::from_cli_and_env(&mut cli).await? + }; let fhclient = FlakeHubClient::new(ctx.flakehub_host, ctx.auth_token)?; diff --git a/src/push_context.rs b/src/push_context.rs index 98a589de..cda4ae2f 100644 --- a/src/push_context.rs +++ b/src/push_context.rs @@ -72,6 +72,7 @@ impl GitContext { cli: &FlakeHubPushCli, local_revision_info: RevisionInfo, ) -> Result { + // TODO(future): investigate library to sniff out SPDX expression based on repo contents // spdx_expression: can't find any evidence GitLab tries to surface this info let spdx_expression = &cli.spdx_expression.0; From 6015b2a4984cc0139eeaaf6a932434834b9e70f7 Mon Sep 17 00:00:00 2001 From: Cole Mickens Date: Wed, 8 May 2024 13:41:25 -0700 Subject: [PATCH 19/27] move git_context out --- src/flakehub_auth_fake.rs | 2 +- src/git_context.rs | 77 +++++++++++++++++++++++++++++++++++++ src/main.rs | 1 + src/push_context.rs | 81 ++------------------------------------- 4 files changed, 83 insertions(+), 78 deletions(-) create mode 100644 src/git_context.rs diff --git a/src/flakehub_auth_fake.rs b/src/flakehub_auth_fake.rs index 818eed3f..da9d9972 100644 --- a/src/flakehub_auth_fake.rs +++ b/src/flakehub_auth_fake.rs @@ -27,7 +27,7 @@ pub async fn get_fake_bearer_token( tracing::debug!(?claims); - let issuer_url = url::Url::parse(&jwt_issuer_uri)?; + let issuer_url = url::Url::parse(jwt_issuer_uri)?; let token_gen_endpoint = issuer_url.join("/token")?; let response = client diff --git a/src/git_context.rs b/src/git_context.rs new file mode 100644 index 00000000..767f0d16 --- /dev/null +++ b/src/git_context.rs @@ -0,0 +1,77 @@ + + +use color_eyre::eyre::{Context, Result}; +use spdx::Expression; + +use crate::{ + cli::FlakeHubPushCli, + github::graphql::{ + GithubGraphqlDataResult, + }, + revision_info::RevisionInfo, +}; + +pub struct GitContext { + pub spdx_expression: Option, + pub repo_topics: Vec, + pub revision_info: RevisionInfo, +} + +impl GitContext { + pub fn from_cli_and_github( + cli: &FlakeHubPushCli, + github_graphql_data_result: &GithubGraphqlDataResult, + ) -> Result { + // step: validate spdx, backfill from GitHub API + let spdx_expression = if cli.spdx_expression.0.is_none() { + if let Some(spdx_string) = &github_graphql_data_result.spdx_identifier { + tracing::debug!("Recieved SPDX identifier `{}` from GitHub API", spdx_string); + let parsed = spdx::Expression::parse(spdx_string) + .wrap_err("Invalid SPDX license identifier reported from the GitHub API, either you are using a non-standard license or GitHub has returned a value that cannot be validated")?; + //span.record("spdx_expression", tracing::field::display(&parsed)); + Some(parsed) + } else { + None + } + } else { + // Provide the user notice if the SPDX expression passed differs from the one detected on GitHub -- It's probably something they care about. + if github_graphql_data_result.spdx_identifier + != cli.spdx_expression.0.as_ref().map(|v| v.to_string()) + { + tracing::warn!( + "SPDX identifier `{}` was passed via argument, but GitHub's API suggests it may be `{}`", + cli.spdx_expression.0.as_ref().map(|v| v.to_string()).unwrap_or_else(|| "None".to_string()), + github_graphql_data_result.spdx_identifier.clone().unwrap_or_else(|| "None".to_string()), + ) + } + cli.spdx_expression.0.clone() + }; + + let ctx = GitContext { + spdx_expression, + repo_topics: github_graphql_data_result.topics.clone(), + revision_info: RevisionInfo { + // TODO(colemickens): type coherency here... :/ (as is bad) + commit_count: Some(github_graphql_data_result.rev_count as usize), + revision: github_graphql_data_result.revision.clone(), + }, + }; + Ok(ctx) + } + + pub async fn from_cli_and_gitlab( + cli: &FlakeHubPushCli, + local_revision_info: RevisionInfo, + ) -> Result { + // TODO(future): investigate library to sniff out SPDX expression based on repo contents + // spdx_expression: can't find any evidence GitLab tries to surface this info + let spdx_expression = &cli.spdx_expression.0; + + let ctx = GitContext { + spdx_expression: spdx_expression.clone(), + repo_topics: vec![], + revision_info: local_revision_info, + }; + Ok(ctx) + } +} diff --git a/src/main.rs b/src/main.rs index 7311287a..29893e43 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,6 +15,7 @@ mod flakehub_auth_fake; mod flakehub_client; mod github; mod gitlab; +mod git_context; mod push_context; mod release_metadata; mod revision_info; diff --git a/src/push_context.rs b/src/push_context.rs index cda4ae2f..dd429a6f 100644 --- a/src/push_context.rs +++ b/src/push_context.rs @@ -5,86 +5,13 @@ use std::{ }; use color_eyre::eyre::{eyre, Context, Result}; -use spdx::Expression; use crate::{ - build_http_client, - cli::FlakeHubPushCli, - flake_info, flakehub_auth_fake, - flakehub_client::Tarball, - github::graphql::{ - GithubGraphqlDataQuery, GithubGraphqlDataResult, MAX_LABEL_LENGTH, MAX_NUM_TOTAL_LABELS, - }, - release_metadata::ReleaseMetadata, - revision_info::RevisionInfo, - DEFAULT_ROLLING_PREFIX, + build_http_client, cli::FlakeHubPushCli, flake_info, flakehub_auth_fake, flakehub_client::Tarball, git_context::GitContext, github::graphql::{ + GithubGraphqlDataQuery, MAX_LABEL_LENGTH, MAX_NUM_TOTAL_LABELS, + }, release_metadata::ReleaseMetadata, revision_info::RevisionInfo, DEFAULT_ROLLING_PREFIX }; -pub struct GitContext { - pub spdx_expression: Option, - pub repo_topics: Vec, - pub revision_info: RevisionInfo, -} - -impl GitContext { - pub fn from_cli_and_github( - cli: &FlakeHubPushCli, - github_graphql_data_result: &GithubGraphqlDataResult, - ) -> Result { - // step: validate spdx, backfill from GitHub API - let spdx_expression = if cli.spdx_expression.0.is_none() { - if let Some(spdx_string) = &github_graphql_data_result.spdx_identifier { - tracing::debug!("Recieved SPDX identifier `{}` from GitHub API", spdx_string); - let parsed = spdx::Expression::parse(spdx_string) - .wrap_err("Invalid SPDX license identifier reported from the GitHub API, either you are using a non-standard license or GitHub has returned a value that cannot be validated")?; - //span.record("spdx_expression", tracing::field::display(&parsed)); - Some(parsed) - } else { - None - } - } else { - // Provide the user notice if the SPDX expression passed differs from the one detected on GitHub -- It's probably something they care about. - if github_graphql_data_result.spdx_identifier - != cli.spdx_expression.0.as_ref().map(|v| v.to_string()) - { - tracing::warn!( - "SPDX identifier `{}` was passed via argument, but GitHub's API suggests it may be `{}`", - cli.spdx_expression.0.as_ref().map(|v| v.to_string()).unwrap_or_else(|| "None".to_string()), - github_graphql_data_result.spdx_identifier.clone().unwrap_or_else(|| "None".to_string()), - ) - } - cli.spdx_expression.0.clone() - }; - - let ctx = GitContext { - spdx_expression, - repo_topics: github_graphql_data_result.topics.clone(), - revision_info: RevisionInfo { - // TODO(colemickens): type coherency here... :/ (as is bad) - commit_count: Some(github_graphql_data_result.rev_count as usize), - revision: github_graphql_data_result.revision.clone(), - }, - }; - Ok(ctx) - } - - pub async fn from_cli_and_gitlab( - cli: &FlakeHubPushCli, - local_revision_info: RevisionInfo, - ) -> Result { - // TODO(future): investigate library to sniff out SPDX expression based on repo contents - // spdx_expression: can't find any evidence GitLab tries to surface this info - let spdx_expression = &cli.spdx_expression.0; - - let ctx = GitContext { - spdx_expression: spdx_expression.clone(), - repo_topics: vec![], - revision_info: local_revision_info, - }; - Ok(ctx) - } -} - pub(crate) struct PushContext { pub(crate) flakehub_host: url::Url, pub(crate) auth_token: String, @@ -414,7 +341,7 @@ impl PushContext { // TODO(colemickens): remove this confusing, redundant field (FH-267) repo: upload_name.to_string(), revision: git_ctx.revision_info.revision, - visibility: visibility, + visibility, mirrored: cli.mirror, source_subdirectory: Some( subdir From 2b0e573b26b05711e11b3cd770e141da72de39c2 Mon Sep 17 00:00:00 2001 From: Cole Mickens Date: Wed, 8 May 2024 13:52:04 -0700 Subject: [PATCH 20/27] more local token claim fixups --- src/flakehub_auth_fake.rs | 2 +- src/push_context.rs | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/flakehub_auth_fake.rs b/src/flakehub_auth_fake.rs index da9d9972..a0158bc1 100644 --- a/src/flakehub_auth_fake.rs +++ b/src/flakehub_auth_fake.rs @@ -19,7 +19,7 @@ pub async fn get_fake_bearer_token( // this would remove the entire need for the fake JWT server, since we are ourselves a JWT issuer claims.aud = "flakehub-localhost".to_string(); claims.iss = jwt_issuer_uri.to_string(); - claims.repository = format!("{project_owner}/{repository}"); + claims.repository = repository.to_string(); claims.repository_owner = project_owner.to_string(); claims.repository_id = github_graphql_data_result.project_id.to_string(); diff --git a/src/push_context.rs b/src/push_context.rs index dd429a6f..c1c77109 100644 --- a/src/push_context.rs +++ b/src/push_context.rs @@ -95,10 +95,11 @@ impl PushContext { Err(eyre!("Could not determine the owner/project, pass `--repository` formatted like `determinatesystems/flakehub-push`. The passed value has too many slashes (/) to be a valid repository"))?; } - let local_git_root = (match &cli.git_root.0 { + let maybe_git_root = match &cli.git_root.0 { Some(gr) => Ok(gr.to_owned()), None => std::env::current_dir().map(PathBuf::from) - }).wrap_err("Could not determine current `git_root`. Pass `--git-root` or set `FLAKEHUB_PUSH_GIT_ROOT`, or run `flakehub-push` with the git root as the current working directory")?; + }; + let local_git_root = maybe_git_root.wrap_err("Could not determine current `git_root`. Pass `--git-root` or set `FLAKEHUB_PUSH_GIT_ROOT`, or run `flakehub-push` with the git root as the current working directory")?; let local_git_root = local_git_root .canonicalize() @@ -166,7 +167,7 @@ impl PushContext { let token = flakehub_auth_fake::get_fake_bearer_token( u, &project_owner, - &project_name, + &format!("{}/{}", project_owner, project_name), github_graphql_data_result, ) .await?; From 2c53facd489849a32dd865863a8df1367a50d8ea Mon Sep 17 00:00:00 2001 From: Cole Mickens Date: Wed, 8 May 2024 13:52:04 -0700 Subject: [PATCH 21/27] move conflict handling out of client into main --- src/flakehub_client.rs | 60 +++------------------------------------- src/main.rs | 62 +++++++++++++++++++++++++++++++++++++----- 2 files changed, 59 insertions(+), 63 deletions(-) diff --git a/src/flakehub_client.rs b/src/flakehub_client.rs index 4bf90bec..6c2aad26 100644 --- a/src/flakehub_client.rs +++ b/src/flakehub_client.rs @@ -1,10 +1,10 @@ use color_eyre::eyre::{eyre, Context, Result}; use http::StatusCode; use reqwest::header::HeaderMap; +use reqwest::Response; use uuid::Uuid; use crate::release_metadata::ReleaseMetadata; -use crate::Error; pub struct FlakeHubClient { host: url::Url, @@ -59,8 +59,7 @@ impl FlakeHubClient { release_version: &str, release_metadata: &ReleaseMetadata, tarball: &Tarball, - error_if_release_conflicts: bool, - ) -> Result> { + ) -> Result { let flake_tarball_len = tarball.bytes.len(); let flake_tarball_hash_base64 = &tarball.hash_base64; let relative_url: &String = &format!("upload/{upload_name}/{release_version}/{flake_tarball_len}/{flake_tarball_hash_base64}"); @@ -76,7 +75,7 @@ impl FlakeHubClient { tracing::debug!("repo={}", release_metadata.repo); tracing::debug!("upload_name={}", upload_name); - let response = self + self .client .post(release_metadata_post_url) .bearer_auth(&self.bearer_token) @@ -84,58 +83,7 @@ impl FlakeHubClient { .json(&release_metadata) .send() .await - .unwrap(); - - let release_metadata_post_response_status = response.status(); - tracing::trace!( - status = tracing::field::display(release_metadata_post_response_status), - "Got release metadata POST response" - ); - - match release_metadata_post_response_status { - StatusCode::OK => (), - StatusCode::CONFLICT => { - tracing::info!( - "Release for revision `{revision}` of {upload_name}/{release_version} already exists; flakehub-push will not upload it again", - revision = &release_metadata.revision, - upload_name = upload_name, - release_version = &release_version, - ); - if error_if_release_conflicts { - return Err(Error::Conflict { - upload_name: upload_name.to_string(), - release_version: release_version.to_string(), - })?; - } else { - // we're just done, and happy about it: - return Ok(None); - } - } - StatusCode::UNAUTHORIZED => { - let body = response.bytes().await?; - let message = serde_json::from_slice::(&body)?; - - return Err(Error::Unauthorized(message))?; - } - _ => { - let body = response.bytes().await?; - let message = serde_json::from_slice::(&body)?; - return Err(eyre!( - "\ - Status {release_metadata_post_response_status} from metadata POST\n\ - {}\ - ", - message - )); - } - } - - let stage_result: StageResult = response - .json() - .await - .wrap_err("Decoding release metadata POST response")?; - - Ok(Some(stage_result)) + .wrap_err("Publishing release") } pub async fn release_publish(&self, release_uuidv7: Uuid) -> Result<()> { diff --git a/src/main.rs b/src/main.rs index 29893e43..82ffc517 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,8 +1,9 @@ use std::{fmt::Display, io::IsTerminal, process::ExitCode}; use clap::Parser; -use color_eyre::eyre::Result; +use color_eyre::eyre::{eyre, Result}; use error::Error; +use http::StatusCode; use crate::{ flakehub_client::{FlakeHubClient, StageResult}, @@ -72,19 +73,66 @@ async fn execute() -> Result { let fhclient = FlakeHubClient::new(ctx.flakehub_host, ctx.auth_token)?; // "upload.rs" - stage the release - let stage_result: Option = fhclient + let stage_result = fhclient .release_stage( &ctx.upload_name, &ctx.release_version, &ctx.metadata, &ctx.tarball, - ctx.error_if_release_conflicts, ) - .await?; + .await; + + let stage_result: StageResult = match stage_result { + Err(e) => { return Err(e)?; }, + Ok(response) => { + let response_status = response.status(); + let stage_result = match response_status { + StatusCode::OK => { + let stage_result: StageResult = response + .json() + .await + .map_err(|_| eyre!("Decoding release metadata POST response"))?; + + stage_result + }, + StatusCode::CONFLICT => { + tracing::info!( + "Release for revision `{revision}` of {upload_name}/{release_version} already exists; flakehub-push will not upload it again", + revision = &ctx.metadata.revision, + upload_name = ctx.upload_name, + release_version = &ctx.release_version, + ); + if ctx.error_if_release_conflicts { + return Err(Error::Conflict { + upload_name: ctx.upload_name.to_string(), + release_version: ctx.release_version.to_string(), + })?; + } else { + // we're just done, and happy about it: + return Ok(ExitCode::SUCCESS); + } + } + StatusCode::UNAUTHORIZED => { + let body = response.bytes().await?; + let message = serde_json::from_slice::(&body)?; - let stage_result = match stage_result { - Some(stage_result) => stage_result, - None => return Ok(ExitCode::SUCCESS), + return Err(Error::Unauthorized(message))?; + } + _ => { + let body = response.bytes().await?; + let message = serde_json::from_slice::(&body)?; + return Err(eyre!( + "\ + Status {} from metadata POST\n\ + {}\ + ", + response_status, + message + )); + } + }; + stage_result + }, }; // upload tarball to s3 From 3883a6c6e43a3ef4c6126f1f7621166d258ae4b4 Mon Sep 17 00:00:00 2001 From: Cole Mickens Date: Wed, 8 May 2024 14:34:17 -0700 Subject: [PATCH 22/27] add ExecutionEnvironment enum --- src/push_context.rs | 47 ++++++++++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/src/push_context.rs b/src/push_context.rs index c1c77109..edae0ad5 100644 --- a/src/push_context.rs +++ b/src/push_context.rs @@ -12,6 +12,13 @@ use crate::{ }, release_metadata::ReleaseMetadata, revision_info::RevisionInfo, DEFAULT_ROLLING_PREFIX }; +#[derive(Clone)] +pub enum ExecutionEnvironment { + GitHub, + GitLab, + Local, +} + pub(crate) struct PushContext { pub(crate) flakehub_host: url::Url, pub(crate) auth_token: String, @@ -35,16 +42,24 @@ impl PushContext { let client = build_http_client().build()?; - let is_github = std::env::var("GITHUB_ACTION").ok().is_some(); - let is_gitlab = std::env::var("GITLAB_CI").ok().is_some(); + let exec_env = if std::env::var("GITHUB_ACTION").ok().is_some() { + ExecutionEnvironment::GitHub + } else if std::env::var("GITLAB_CI").ok().is_some() { + ExecutionEnvironment::GitLab + } else { + ExecutionEnvironment::Local + }; - // "backfill" env vars from the environment - if is_github { - cli.backfill_from_github_env(); - } - if is_gitlab { - cli.backfill_from_gitlab_env(); - } + match exec_env.clone() { + ExecutionEnvironment::GitHub => { + // TODO(colemickens): we were back-filling from github, in local paths before, check + cli.backfill_from_github_env(); + }, + ExecutionEnvironment::GitLab => { + cli.backfill_from_gitlab_env(); + }, + _ => {}, + }; let visibility = match (cli.visibility_alt, cli.visibility) { (Some(v), _) => v, @@ -108,8 +123,8 @@ impl PushContext { // "cli" and "git_ctx" are the user/env supplied info, augmented with data we might have fetched from github/gitlab apis - let (token, git_ctx) = match (is_github, is_gitlab, &cli.jwt_issuer_uri) { - (true, false, None) => { + let (token, git_ctx) = match (exec_env.clone(), &cli.jwt_issuer_uri) { + (ExecutionEnvironment::GitHub, None) => { // GITHUB CI let github_token = cli .github_token @@ -134,7 +149,7 @@ impl PushContext { (token, git_ctx) } - (false, true, None) => { + (ExecutionEnvironment::GitLab, None) => { // GITLAB CI let token = crate::gitlab::get_runner_bearer_token() .await @@ -144,7 +159,7 @@ impl PushContext { (token, git_ctx) } - (false, false, Some(u)) => { + (ExecutionEnvironment::Local, Some(u)) => { // LOCAL, DEV (aka emulating GITHUB) let github_token = cli .github_token @@ -173,7 +188,7 @@ impl PushContext { .await?; (token, git_ctx) } - (_, _, Some(_)) => { + (_, Some(_)) => { // we're in (GitHub|GitLab) and jwt_issuer_uri was specified, invalid return Err(eyre!( "specifying the jwt_issuer_uri when running in GitHub or GitLab is invalid" @@ -261,7 +276,7 @@ impl PushContext { let message = "`extra-tags` is deprecated and will be removed in the future. Please use `extra-labels` instead."; tracing::warn!("{message}"); - if is_github { + if matches!(&exec_env, ExecutionEnvironment::GitHub) { println!("::warning::{message}"); } @@ -272,7 +287,7 @@ impl PushContext { "Both `extra-tags` and `extra-labels` were set; `extra-tags` will be ignored."; tracing::warn!("{message}"); - if is_github { + if matches!(exec_env, ExecutionEnvironment::GitHub) { println!("::warning::{message}"); } } From 14bfd30c453fbebdbc52f24a8f0c78e986d466b6 Mon Sep 17 00:00:00 2001 From: Cole Mickens Date: Wed, 8 May 2024 14:34:17 -0700 Subject: [PATCH 23/27] misc other tracing cleanups --- src/flakehub_auth_fake.rs | 2 -- src/flakehub_client.rs | 7 +------ src/git_context.rs | 8 +------- src/main.rs | 12 +++++++----- src/push_context.rs | 20 +++++++++++++------- 5 files changed, 22 insertions(+), 27 deletions(-) diff --git a/src/flakehub_auth_fake.rs b/src/flakehub_auth_fake.rs index a0158bc1..173f027c 100644 --- a/src/flakehub_auth_fake.rs +++ b/src/flakehub_auth_fake.rs @@ -25,8 +25,6 @@ pub async fn get_fake_bearer_token( claims.repository_id = github_graphql_data_result.project_id.to_string(); claims.repository_owner_id = github_graphql_data_result.owner_id.to_string(); - tracing::debug!(?claims); - let issuer_url = url::Url::parse(jwt_issuer_uri)?; let token_gen_endpoint = issuer_url.join("/token")?; diff --git a/src/flakehub_client.rs b/src/flakehub_client.rs index 6c2aad26..5011555b 100644 --- a/src/flakehub_client.rs +++ b/src/flakehub_client.rs @@ -71,12 +71,7 @@ impl FlakeHubClient { "Computed release metadata POST URL" ); - tracing::debug!(?release_metadata); //TODO colemickens: sanity check this against main fhp - tracing::debug!("repo={}", release_metadata.repo); - tracing::debug!("upload_name={}", upload_name); - - self - .client + self.client .post(release_metadata_post_url) .bearer_auth(&self.bearer_token) .headers(flakehub_headers()) diff --git a/src/git_context.rs b/src/git_context.rs index 767f0d16..15e1f8e0 100644 --- a/src/git_context.rs +++ b/src/git_context.rs @@ -1,14 +1,8 @@ - - use color_eyre::eyre::{Context, Result}; use spdx::Expression; use crate::{ - cli::FlakeHubPushCli, - github::graphql::{ - GithubGraphqlDataResult, - }, - revision_info::RevisionInfo, + cli::FlakeHubPushCli, github::graphql::GithubGraphqlDataResult, revision_info::RevisionInfo, }; pub struct GitContext { diff --git a/src/main.rs b/src/main.rs index 82ffc517..3f3ce4e7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -14,9 +14,9 @@ mod error; mod flake_info; mod flakehub_auth_fake; mod flakehub_client; +mod git_context; mod github; mod gitlab; -mod git_context; mod push_context; mod release_metadata; mod revision_info; @@ -83,7 +83,9 @@ async fn execute() -> Result { .await; let stage_result: StageResult = match stage_result { - Err(e) => { return Err(e)?; }, + Err(e) => { + return Err(e)?; + } Ok(response) => { let response_status = response.status(); let stage_result = match response_status { @@ -92,9 +94,9 @@ async fn execute() -> Result { .json() .await .map_err(|_| eyre!("Decoding release metadata POST response"))?; - + stage_result - }, + } StatusCode::CONFLICT => { tracing::info!( "Release for revision `{revision}` of {upload_name}/{release_version} already exists; flakehub-push will not upload it again", @@ -132,7 +134,7 @@ async fn execute() -> Result { } }; stage_result - }, + } }; // upload tarball to s3 diff --git a/src/push_context.rs b/src/push_context.rs index edae0ad5..a590bf8e 100644 --- a/src/push_context.rs +++ b/src/push_context.rs @@ -7,9 +7,15 @@ use std::{ use color_eyre::eyre::{eyre, Context, Result}; use crate::{ - build_http_client, cli::FlakeHubPushCli, flake_info, flakehub_auth_fake, flakehub_client::Tarball, git_context::GitContext, github::graphql::{ - GithubGraphqlDataQuery, MAX_LABEL_LENGTH, MAX_NUM_TOTAL_LABELS, - }, release_metadata::ReleaseMetadata, revision_info::RevisionInfo, DEFAULT_ROLLING_PREFIX + build_http_client, + cli::FlakeHubPushCli, + flake_info, flakehub_auth_fake, + flakehub_client::Tarball, + git_context::GitContext, + github::graphql::{GithubGraphqlDataQuery, MAX_LABEL_LENGTH, MAX_NUM_TOTAL_LABELS}, + release_metadata::ReleaseMetadata, + revision_info::RevisionInfo, + DEFAULT_ROLLING_PREFIX, }; #[derive(Clone)] @@ -54,11 +60,11 @@ impl PushContext { ExecutionEnvironment::GitHub => { // TODO(colemickens): we were back-filling from github, in local paths before, check cli.backfill_from_github_env(); - }, + } ExecutionEnvironment::GitLab => { cli.backfill_from_gitlab_env(); - }, - _ => {}, + } + _ => {} }; let visibility = match (cli.visibility_alt, cli.visibility) { @@ -112,7 +118,7 @@ impl PushContext { let maybe_git_root = match &cli.git_root.0 { Some(gr) => Ok(gr.to_owned()), - None => std::env::current_dir().map(PathBuf::from) + None => std::env::current_dir().map(PathBuf::from), }; let local_git_root = maybe_git_root.wrap_err("Could not determine current `git_root`. Pass `--git-root` or set `FLAKEHUB_PUSH_GIT_ROOT`, or run `flakehub-push` with the git root as the current working directory")?; From cd781d1a2013138a585edbea3890843cc27da02d Mon Sep 17 00:00:00 2001 From: Cole Mickens Date: Wed, 8 May 2024 14:58:27 -0700 Subject: [PATCH 24/27] drop nrgok header stuff --- src/flakehub_client.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/flakehub_client.rs b/src/flakehub_client.rs index 5011555b..706b4c24 100644 --- a/src/flakehub_client.rs +++ b/src/flakehub_client.rs @@ -31,11 +31,6 @@ pub fn flakehub_headers() -> HeaderMap { reqwest::header::CONTENT_TYPE, reqwest::header::HeaderValue::from_str("application/json").unwrap(), ); - // TODO(colemickens): tube > ngrok, remove - header_map.insert( - reqwest::header::HeaderName::from_static("ngrok-skip-browser-warning"), - reqwest::header::HeaderValue::from_str("please").unwrap(), - ); header_map } From 5fe737e715755b0dce7c9b9cc7142d5d5ffd71bd Mon Sep 17 00:00:00 2001 From: Cole Mickens Date: Wed, 8 May 2024 15:02:16 -0700 Subject: [PATCH 25/27] clippy compliance --- src/main.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main.rs b/src/main.rs index 3f3ce4e7..f43dd629 100644 --- a/src/main.rs +++ b/src/main.rs @@ -88,7 +88,7 @@ async fn execute() -> Result { } Ok(response) => { let response_status = response.status(); - let stage_result = match response_status { + match response_status { StatusCode::OK => { let stage_result: StageResult = response .json() @@ -132,8 +132,7 @@ async fn execute() -> Result { message )); } - }; - stage_result + } } }; From a4e58e3c32c4e55ddc6bf8ef07cc8e85fbe625d0 Mon Sep 17 00:00:00 2001 From: Cole Mickens Date: Wed, 8 May 2024 15:04:45 -0700 Subject: [PATCH 26/27] review comments --- src/git_context.rs | 1 - src/push_context.rs | 7 ++----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/git_context.rs b/src/git_context.rs index 15e1f8e0..13a53cd3 100644 --- a/src/git_context.rs +++ b/src/git_context.rs @@ -22,7 +22,6 @@ impl GitContext { tracing::debug!("Recieved SPDX identifier `{}` from GitHub API", spdx_string); let parsed = spdx::Expression::parse(spdx_string) .wrap_err("Invalid SPDX license identifier reported from the GitHub API, either you are using a non-standard license or GitHub has returned a value that cannot be validated")?; - //span.record("spdx_expression", tracing::field::display(&parsed)); Some(parsed) } else { None diff --git a/src/push_context.rs b/src/push_context.rs index a590bf8e..2b718875 100644 --- a/src/push_context.rs +++ b/src/push_context.rs @@ -58,7 +58,6 @@ impl PushContext { match exec_env.clone() { ExecutionEnvironment::GitHub => { - // TODO(colemickens): we were back-filling from github, in local paths before, check cli.backfill_from_github_env(); } ExecutionEnvironment::GitLab => { @@ -188,7 +187,7 @@ impl PushContext { let token = flakehub_auth_fake::get_fake_bearer_token( u, &project_owner, - &format!("{}/{}", project_owner, project_name), + repository, github_graphql_data_result, ) .await?; @@ -248,8 +247,7 @@ impl PushContext { } }; - // TODO(review): shouldn't this maybe only be fatal whenever we are rolling_minor||rolling? - // TODO!!! doubly so since we're not going to be able to get commit_count from the gitlab api? + // TODO(future): (FH-282): change this so commit_count is only set authoritatively, is an explicit error if not set, when rolling, for gitlab let Some(commit_count) = git_ctx.revision_info.commit_count else { return Err(eyre!("Could not determine commit count, this is normally determined via the `--git-root` argument or via the GitHub API")); }; @@ -325,7 +323,6 @@ impl PushContext { // flake_dir is an absolute path of flake_root(aka git_root)/subdir let flake_dir = local_git_root.join(&subdir); - // TODO(review): depending on what the user called us with, this flake_dir isn't even necessarily canonicalized, is this a sec/traversal issue? // FIXME: bail out if flake_metadata denotes a dirty tree. let flake_metadata = flake_info::FlakeMetadata::from_dir(&flake_dir) From 9b4b3f8238116776289b33e65a5b0ef5aa9cddaf Mon Sep 17 00:00:00 2001 From: Cole Mickens Date: Wed, 8 May 2024 15:18:30 -0700 Subject: [PATCH 27/27] comment cleanup --- src/flakehub_auth_fake.rs | 4 ---- src/git_context.rs | 1 - 2 files changed, 5 deletions(-) diff --git a/src/flakehub_auth_fake.rs b/src/flakehub_auth_fake.rs index 173f027c..d6a14554 100644 --- a/src/flakehub_auth_fake.rs +++ b/src/flakehub_auth_fake.rs @@ -13,10 +13,6 @@ pub async fn get_fake_bearer_token( let client = reqwest::Client::new(); let mut claims = github_actions_oidc_claims::Claims::make_dummy(); - // FIXME: we should probably fill in more of these claims. - - // TODO(review): on the contrary, I think we should ditch this, and we should basically use forge_login-esque functionality for this going forward - // this would remove the entire need for the fake JWT server, since we are ourselves a JWT issuer claims.aud = "flakehub-localhost".to_string(); claims.iss = jwt_issuer_uri.to_string(); claims.repository = repository.to_string(); diff --git a/src/git_context.rs b/src/git_context.rs index 13a53cd3..db64e8e1 100644 --- a/src/git_context.rs +++ b/src/git_context.rs @@ -44,7 +44,6 @@ impl GitContext { spdx_expression, repo_topics: github_graphql_data_result.topics.clone(), revision_info: RevisionInfo { - // TODO(colemickens): type coherency here... :/ (as is bad) commit_count: Some(github_graphql_data_result.rev_count as usize), revision: github_graphql_data_result.revision.clone(), },