From 4cdbbaf4e19e344a2003ecd97226511db98456d1 Mon Sep 17 00:00:00 2001 From: Bradford Larsen Date: Tue, 31 Jan 2023 17:10:23 -0500 Subject: [PATCH] Checkpoint work on GitHub enumeration --- Cargo.lock | 565 +++++++++++++++++- Cargo.toml | 6 +- src/bin/noseyparker/args.rs | 56 +- src/bin/noseyparker/cmd_github.rs | 109 ++++ src/bin/noseyparker/main.rs | 3 +- src/github/mod.rs | 208 +++++++ src/github/models/mod.rs | 287 +++++++++ src/lib.rs | 1 + tests/snapshots/cli__noseyparker_help-2.snap | 2 + .../snapshots/cli__noseyparker_no_args-3.snap | 1 + 10 files changed, 1233 insertions(+), 5 deletions(-) create mode 100644 src/bin/noseyparker/cmd_github.rs create mode 100644 src/github/mod.rs create mode 100644 src/github/models/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 74caa2913..59584e411 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -40,6 +40,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anes" version = "0.1.6" @@ -118,6 +127,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "base64" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" + [[package]] name = "bit-set" version = "0.5.3" @@ -193,6 +208,12 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +[[package]] +name = "bytes" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" + [[package]] name = "bytesize" version = "1.1.0" @@ -235,6 +256,21 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-integer", + "num-traits", + "time 0.1.45", + "wasm-bindgen", + "winapi", +] + [[package]] name = "ciborium" version = "0.2.0" @@ -338,6 +374,16 @@ dependencies = [ "cc", ] +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + [[package]] name = "compact_str" version = "0.6.1" @@ -368,6 +414,22 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + [[package]] name = "cpufeatures" version = "0.2.5" @@ -531,6 +593,50 @@ dependencies = [ "syn", ] +[[package]] +name = "cxx" +version = "1.0.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "322296e2f2e5af4270b54df9e85a02ff037e271af20ba3e7fe1575515dc840b8" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "017a1385b05d631e7875b1f151c9f012d37b53491e2a87f65bff5c262b2111d8" +dependencies = [ + "cc", + "codespan-reporting", + "once_cell", + "proc-macro2", + "quote", + "scratch", + "syn", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c26bbb078acf09bc1ecda02d4223f03bdd28bd4874edcb0379138efc499ce971" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357f40d1f06a24b60ae1fe122542c1fb05d28d32acb2aed064e84bc2ad1e252e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "dashmap" version = "5.4.0" @@ -644,6 +750,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" +[[package]] +name = "encoding_rs" +version = "0.8.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" +dependencies = [ + "cfg-if", +] + [[package]] name = "errno" version = "0.2.8" @@ -787,6 +902,45 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures-channel" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e5317663a9089767a1ec00a487df42e0ca174b61b4483213ac24448e4664df5" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608" + +[[package]] +name = "futures-sink" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364" + +[[package]] +name = "futures-task" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf79a1bf610b10f42aea489289c5a2c478a786509693b80cd39c44ccd936366" + +[[package]] +name = "futures-util" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c1d6de3acfef38d2be4b1f543f553131788603495be83da675e180c8d6b7bd1" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "pin-utils", +] + [[package]] name = "generic-array" version = "0.14.6" @@ -805,7 +959,7 @@ checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", ] [[package]] @@ -924,7 +1078,7 @@ dependencies = [ "bstr 1.1.0", "itoa 1.0.5", "thiserror", - "time", + "time 0.3.17", ] [[package]] @@ -1362,6 +1516,25 @@ dependencies = [ "walkdir", ] +[[package]] +name = "h2" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "half" version = "1.8.2" @@ -1431,12 +1604,83 @@ dependencies = [ "winapi", ] +[[package]] +name = "http" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +dependencies = [ + "bytes", + "fnv", + "itoa 1.0.5", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + [[package]] name = "human_format" version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86cce260d758a9aa3d7c4b99d55c815a540f8a37514ba6046ab6be402a157cb0" +[[package]] +name = "hyper" +version = "0.14.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "034711faac9d2166cb1baf1a2fb0b60b1f277f8492fd72176c17f3515e1abd3c" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa 1.0.5", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "hyperscan" version = "0.3.2" @@ -1468,6 +1712,30 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "iana-time-zone" +version = "0.1.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "winapi", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +dependencies = [ + "cxx", + "cxx-build", +] + [[package]] name = "idna" version = "0.3.0" @@ -1607,6 +1875,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "ipnet" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" + [[package]] name = "is-terminal" version = "0.4.2" @@ -1732,6 +2006,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "link-cplusplus" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" +dependencies = [ + "cc", +] + [[package]] name = "linked-hash-map" version = "0.5.6" @@ -1796,6 +2079,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1811,6 +2100,36 @@ dependencies = [ "adler", ] +[[package]] +name = "mio" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" +dependencies = [ + "libc", + "log", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys", +] + +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nix" version = "0.26.2" @@ -1848,6 +2167,7 @@ dependencies = [ "assert_fs", "atty", "bstr 1.1.0", + "chrono", "clap 4.1.4", "console", "criterion", @@ -1875,15 +2195,19 @@ dependencies = [ "proptest", "rayon", "regex", + "reqwest", "rlimit", "rusqlite", + "secrecy", "serde", "serde_json", "serde_yaml", "sha1", + "tokio", "tracing", "tracing-log", "tracing-subscriber", + "url", "walkdir", ] @@ -1897,6 +2221,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.15" @@ -2123,6 +2457,12 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "pkg-config" version = "0.3.26" @@ -2424,6 +2764,43 @@ dependencies = [ "winapi", ] +[[package]] +name = "reqwest" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21eed90ec8570952d53b772ecf8f206aa1ec9a3d76b2521c56c42973f2d91ee9" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "rlimit" version = "0.9.1" @@ -2503,12 +2880,59 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" +dependencies = [ + "windows-sys", +] + [[package]] name = "scopeguard" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "scratch" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" + +[[package]] +name = "secrecy" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bd1c54ea06cfd2f6b63219704de0b9b4f72dcc2b8fdef820be6cd799780e91e" +dependencies = [ + "zeroize", +] + +[[package]] +name = "security-framework" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.16" @@ -2546,6 +2970,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa 1.0.5", + "ryu", + "serde", +] + [[package]] name = "serde_yaml" version = "0.9.17" @@ -2631,12 +3067,31 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "420acb44afdae038210c99e69aae24109f32f15500aa708e81d46c9f29d55fcf" +[[package]] +name = "slab" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +dependencies = [ + "autocfg", +] + [[package]] name = "smallvec" version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +[[package]] +name = "socket2" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "static_assertions" version = "1.1.0" @@ -2745,6 +3200,17 @@ dependencies = [ "once_cell", ] +[[package]] +name = "time" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" +dependencies = [ + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", + "winapi", +] + [[package]] name = "time" version = "0.3.17" @@ -2799,6 +3265,52 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" +[[package]] +name = "tokio" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af" +dependencies = [ + "autocfg", + "bytes", + "libc", + "memchr", + "mio", + "pin-project-lite", + "socket2", + "windows-sys", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + [[package]] name = "tracing" version = "0.1.37" @@ -2857,6 +3369,12 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "try-lock" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" + [[package]] name = "typenum" version = "1.16.0" @@ -2981,6 +3499,22 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -3012,6 +3546,18 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.83" @@ -3154,6 +3700,15 @@ version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi", +] + [[package]] name = "yaml-rust" version = "0.4.5" @@ -3168,3 +3723,9 @@ name = "yansi" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" + +[[package]] +name = "zeroize" +version = "1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c394b5bd0c6f669e7275d9c20aa90ae064cb22e75a1cad54e1b34088034b149f" diff --git a/Cargo.toml b/Cargo.toml index fb1b64e67..ab2cd1e76 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,7 @@ rule_profiling = [] anyhow = { version = "1.0" } atty = "0.2" bstr = { version = "1.0.1", features = ["serde"] } +chrono = "0.4.23" clap = { version = "4.0", features = ["cargo", "derive", "env", "unicode", "wrap_help"] } console = "0.15.2" git-discover = "0.12.1" @@ -54,15 +55,19 @@ pretty_assertions = "1.3" prettytable-rs = "0.10.0" rayon = "1.5" regex = "1.7" +reqwest = { version = "0.11.13", features = ["json"] } rlimit = "0.9.0" rusqlite = { version = "0.28", features = ["bundled", "backup"] } +secrecy = "0.8.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_yaml = "0.9" sha1 = "0.10" +tokio = "1.23.0" tracing = "0.1.37" tracing-log = "0.1.3" tracing-subscriber = { version = "0.3.16", features = ["tracing-log", "ansi"] } +url = "2.3.1" walkdir = "2.3" [dev-dependencies] @@ -131,7 +136,6 @@ opt-level = 3 # tempfile # filesystem # termcolor # reporting # tinytemplate # templating -# tokio # toml # data format, configuration # tree_magic # content type guesser # unicode-normalization diff --git a/src/bin/noseyparker/args.rs b/src/bin/noseyparker/args.rs index 46a9ce12b..a2c5e384d 100644 --- a/src/bin/noseyparker/args.rs +++ b/src/bin/noseyparker/args.rs @@ -66,13 +66,18 @@ pub enum Command { Scan(ScanArgs), /// Summarize scan findings - #[command(display_order = 2, alias="summarise")] + #[command(display_order = 2, alias = "summarise")] Summarize(SummarizeArgs), /// Report detailed scan findings #[command(display_order = 3)] Report(ReportArgs), + /// Query GitHub + #[command(display_order = 4, name = "github")] + GitHub(GitHubArgs), + + #[command(display_order = 30)] /// Manage datastores Datastore(DatastoreArgs), @@ -146,6 +151,55 @@ impl std::fmt::Display for Mode { } } +// ----------------------------------------------------------------------------- +// `github` command +// ----------------------------------------------------------------------------- +#[derive(Args, Debug)] +pub struct GitHubArgs { + #[command(subcommand)] + pub command: GitHubCommand, +} + +#[derive(Subcommand, Debug)] +pub enum GitHubCommand { + /// Interact with GitHub repositories + #[command(subcommand)] + Repos(GitHubReposCommand), +} + +#[derive(Subcommand, Debug)] +pub enum GitHubReposCommand { + /// List repositories belonging to a specific user or organization + List(GitHubReposListArgs), +} + +#[derive(Args, Debug)] +pub struct GitHubReposListArgs { + #[command(flatten)] + pub repo_specifiers: GitHubRepoSpecifiers, + + #[command(flatten)] + pub output_args: OutputArgs, +} + +#[derive(Args, Debug)] +pub struct GitHubRepoSpecifiers { + /// Select repositories belonging to the specified user + #[arg(long)] + pub user: Vec, + + /// Select repositories belonging to the specified organization + #[arg(long, visible_alias = "org")] + pub organization: Vec, +} + +impl GitHubRepoSpecifiers { + pub fn is_empty(&self) -> bool { + self.user.is_empty() && self.organization.is_empty() + } +} + + // ----------------------------------------------------------------------------- // `rules` command // ----------------------------------------------------------------------------- diff --git a/src/bin/noseyparker/cmd_github.rs b/src/bin/noseyparker/cmd_github.rs new file mode 100644 index 000000000..944212495 --- /dev/null +++ b/src/bin/noseyparker/cmd_github.rs @@ -0,0 +1,109 @@ +use anyhow::{Context, Result, bail}; +// use chrono::{DateTime, Utc}; +// use reqwest::Url; +// use serde::Deserialize; +// use std::collections::BTreeMap; + +use crate::args; +use noseyparker::github; + +pub fn run(global_args: &args::GlobalArgs, args: &args::GitHubArgs) -> Result<()> { + use args::GitHubCommand::*; + use args::GitHubReposCommand::*; + match &args.command { + Repos(List(args)) => list_repos(global_args, args) + } +} + +fn list_repos(_global_args: &args::GlobalArgs, args: &args::GitHubReposListArgs) -> Result<()> { + if args.repo_specifiers.is_empty() { + bail!("No repositories specified"); + } + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .context("Failed to initialize async runtime")?; + + let client = github::Client::new() + .context("Failed to initialize GitHub client")?; + + let rate_limit = runtime.block_on(client.rate_limit())?; + println!("{:#?}", rate_limit); + + for username in &args.repo_specifiers.user { + let user = runtime.block_on(client.user(username))?; + println!("{:#?}", user); + let repos = runtime.block_on(client.user_repos(username))?; + for repo in repos.iter() { + println!("{:#?}", repo); + } + } + + let rate_limit = runtime.block_on(client.rate_limit())?; + println!("{:#?}", rate_limit); + + return Ok(()); + + /* + let mut writer = args + .output_args + .get_writer() + .context("Failed to open output destination for writing")?; + + let run_inner = move || -> std::io::Result<()> { + match &args.output_args.format { + args::OutputFormat::Human => { + // writeln!(writer)?; + // let table = summary_table(summary); + // // FIXME: this doesn't preserve ANSI styling on the table + // table.print(&mut writer)?; + } + args::OutputFormat::Json => { + // serde_json::to_writer_pretty(&mut writer, &summary)?; + } + args::OutputFormat::Jsonl => { + // for entry in summary.0.iter() { + // serde_json::to_writer(&mut writer, entry)?; + // writeln!(&mut writer)?; + // } + } + } + Ok(()) + }; + match run_inner() { + // Ignore SIGPIPE errors, like those that can come from piping to `head` + Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => { Ok(()) } + Err(e) => Err(e)?, + Ok(()) => Ok(()), + } + */ +} + +/* +pub fn summary_table(summary: MatchSummary) -> prettytable::Table { + use prettytable::format::{FormatBuilder, LinePosition, LineSeparator}; + use prettytable::row; + + let f = FormatBuilder::new() + // .column_separator('│') + // .separators(&[LinePosition::Title], LineSeparator::new('─', '┼', '├', '┤')) + .column_separator(' ') + .separators(&[LinePosition::Title], LineSeparator::new('─', '─', '─', '─')) + .padding(1, 1) + .build(); + + let mut table: prettytable::Table = summary + .0 + .into_iter() + .map(|e| row![ + l -> &e.rule_name, + r -> HumanCount(e.distinct_count.try_into().unwrap()), + r -> HumanCount(e.total_count.try_into().unwrap()) + ]) + .collect(); + table.set_format(f); + table.set_titles(row![lb -> "Rule", cb -> "Distinct Matches", cb -> "Total Matches"]); + table +} +*/ diff --git a/src/bin/noseyparker/main.rs b/src/bin/noseyparker/main.rs index 87eee6ad2..2db329c73 100644 --- a/src/bin/noseyparker/main.rs +++ b/src/bin/noseyparker/main.rs @@ -3,6 +3,7 @@ use tracing::debug; mod args; mod cmd_datastore; +mod cmd_github; mod cmd_report; mod cmd_rules; mod cmd_scan; @@ -23,7 +24,6 @@ fn configure_tracing(global_args: &args::GlobalArgs) -> Result<()> { .with_max_level(filter.as_log()) .init()?; - // a builder for `FmtSubscriber`. let subscriber = tracing_subscriber::FmtSubscriber::builder() .with_max_level(filter) .with_ansi(global_args.use_color()) @@ -63,6 +63,7 @@ fn try_main() -> Result<()> { match &args.command { args::Command::Datastore(args) => cmd_datastore::run(global_args, args), + args::Command::GitHub(args) => cmd_github::run(global_args, args), args::Command::Rules(args) => cmd_rules::run(global_args, args), args::Command::Scan(args) => cmd_scan::run(global_args, args), args::Command::Summarize(args) => cmd_summarize::run(global_args, args), diff --git a/src/github/mod.rs b/src/github/mod.rs new file mode 100644 index 000000000..2fec3c10d --- /dev/null +++ b/src/github/mod.rs @@ -0,0 +1,208 @@ +use chrono::{DateTime, Utc, TimeZone, Duration}; +use reqwest; +use reqwest::{header, header::HeaderValue, StatusCode, IntoUrl, Url}; +use secrecy::{ExposeSecret, SecretString}; + +pub mod models; + +// ------------------------------------------------------------------------------------------------- +// Result +// ------------------------------------------------------------------------------------------------- +pub type Result = std::result::Result; + +// ------------------------------------------------------------------------------------------------- +// Error +// ------------------------------------------------------------------------------------------------- +#[derive(Debug)] +pub enum Error { + RateLimited { + /// The client error returned by GitHub + client_error: models::ClientError, + + /// The duration to wait until trying again + wait: Option, + }, + UrlParseError(url::ParseError), + UrlSlashError(String), + ReqwestError(reqwest::Error), +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Error::RateLimited{..} => write!(f, "request was rate-limited"), + Error::UrlParseError(e) => write!(f, "error parsing URL: {}", e), + Error::UrlSlashError(p) => write!(f, "error building URL: component {:?} contains a slash", p), + Error::ReqwestError(e) => write!(f, "error making request: {}", e), + } + } +} + +impl std::error::Error for Error { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Error::RateLimited{..} => None, + Error::UrlParseError(e) => Some(e), + Error::UrlSlashError(_) => None, + Error::ReqwestError(e) => Some(e), + } + } +} + +// ------------------------------------------------------------------------------------------------- +// Auth +// ------------------------------------------------------------------------------------------------- +/// Supported forms of authentication +pub enum Auth { + /// No authentication + Unauthenticated, + + /// Authenticate with a GitHub Personal Access Token + PersonalToken(SecretString), +} + +// ------------------------------------------------------------------------------------------------- +// ClientBuilder +// ------------------------------------------------------------------------------------------------- +pub struct ClientBuilder { + base_url: Option, + auth: Option, +} + +impl ClientBuilder { + pub fn new() -> Self { + ClientBuilder { + base_url: None, + auth: None, + } + } + + pub fn base_url(mut self, url: T) -> Result { + self.base_url = Some(url.into_url().map_err(Error::ReqwestError)?); + Ok(self) + } + + pub fn auth(mut self, auth: Auth) -> Self { + self.auth = Some(auth); + self + } + + pub fn build(self) -> Result { + let base_url = self.base_url.unwrap_or_else(|| { + Url::parse("https://api.github.com").expect("default base URL should parse") + }); + let auth = self.auth.unwrap_or(Auth::Unauthenticated); + let inner = reqwest::ClientBuilder::new() + .user_agent("noseyparker") + .build() + .map_err(Error::ReqwestError)?; + Ok(Client { + base_url, + auth, + inner, + }) + } +} + +// ------------------------------------------------------------------------------------------------- +// Client +// ------------------------------------------------------------------------------------------------- +pub struct Client { + base_url: reqwest::Url, + inner: reqwest::Client, + auth: Auth, +} + +// TODO: deserialization of results +// TODO: debug logging +// TODO: rate limiting support via headers +// TODO: pagination support; per_page query parameter +// TODO: retry combinators? +// TODO: graceful error handling / HTTP response code handling +impl Client { + pub fn new() -> Result { + ClientBuilder::new().build() + } + + pub async fn rate_limit(&self) -> Result { + let response = self.get(&["rate_limit"]).await?; + let body = response.json().await.map_err(Error::ReqwestError)?; + Ok(body) + } + + pub async fn user(&self, username: &str) -> Result { + let response = self.get(&["users", username]).await?; + let body = response.json().await.map_err(Error::ReqwestError)?; + Ok(body) + } + +// pub async fn user_repos(&self, user: &models::User) -> Result> { + pub async fn user_repos(&self, username: &str) -> Result> { + let response = self.get(&["users", username, "repos"]).await?; + let body = response.json().await.map_err(Error::ReqwestError)?; + Ok(body) + } + + async fn get(&self, path_parts: &[&str]) -> Result { + let url = { + let mut buf = String::new(); + for p in path_parts { + buf.push_str("/"); + if p.contains('/') { + return Err(Error::UrlSlashError(p.to_string())); + } + buf.push_str(p); + } + self.base_url.clone().join(&buf).map_err(Error::UrlParseError)? + }; + println!("GET {}", url); + let request_builder = self.inner.get(url) + .header(header::ACCEPT, "application/vnd.github+json") + .header("X-GitHub-Api-Version", "2022-11-28"); + + let request_builder = match &self.auth { + Auth::PersonalToken(token) => { + request_builder.bearer_auth(token.expose_secret()) + } + Auth::Unauthenticated => request_builder + }; + + let response = request_builder.send().await.map_err(Error::ReqwestError)?; + + // Check for rate limiting. + // Instead of using an HTTP 429 response code, GitHub uses 403 and sets the + // `x-ratelimit-remaining` header to 0. + if response.status() == StatusCode::FORBIDDEN { + if let Some(b"0") = response.headers().get("x-ratelimit-remaining").map(HeaderValue::as_bytes) { + println!("{:#?}", response.headers()); + let date: Option> = match response.headers().get("date") { + Some(v) => match v.to_str() { + Ok(v) => DateTime::parse_from_rfc2822(v).ok().map(|v| v.with_timezone(&Utc)), + Err(_) => None, + } + None => None, + }; + let reset_time: Option> = match response.headers().get("x-ratelimit-reset") { + Some(v) => match v.to_str() { + Ok(v) => v.parse::().ok().and_then(|v| Utc.timestamp_opt(v, 0).single()), + Err(_) => None, + } + None => None, + }; + // N.B. can convert `wait` to `std::time::Duration` with the `.to_std()` method + let wait: Option = match (date, reset_time) { + (Some(t1), Some(t2)) => Some(t2 - t1), + _ => None, + }; + + let client_error = response.json().await.map_err(Error::ReqwestError)?; + return Err(Error::RateLimited { + client_error, + wait, + }); + } + } + + Ok(response.error_for_status().map_err(Error::ReqwestError)?) + } +} diff --git a/src/github/models/mod.rs b/src/github/models/mod.rs new file mode 100644 index 000000000..cb7f1a6b9 --- /dev/null +++ b/src/github/models/mod.rs @@ -0,0 +1,287 @@ +use serde::Deserialize; +use url::Url; + +// ------------------------------------------------------------------------------------------------- +// ClientError +// ------------------------------------------------------------------------------------------------- +#[derive(Debug, Deserialize)] +pub struct ClientError { + pub message: String, + pub documentation_url: Option, + pub errors: Option>, +} + +// ------------------------------------------------------------------------------------------------- +// Error +// ------------------------------------------------------------------------------------------------- +#[derive(Debug, Deserialize)] +pub struct Error { + pub resource: String, + pub field: String, + pub code: ErrorCode, +} + +// ------------------------------------------------------------------------------------------------- +// ErrorCode +// ------------------------------------------------------------------------------------------------- +#[derive(Debug, Deserialize)] +pub enum ErrorCode { + Missing, + MissingField, + Invalid, + AlreadyExists, + Unprocessable, +} + +// ------------------------------------------------------------------------------------------------- +// RateLimit +// ------------------------------------------------------------------------------------------------- +#[derive(Debug, Deserialize)] +pub struct RateLimitOverview { + pub resources: Resources, + pub rate: Rate, +} + +// ------------------------------------------------------------------------------------------------- +// Resource +// ------------------------------------------------------------------------------------------------- +#[derive(Debug, Deserialize)] +pub struct Resources { + pub core: Rate, + pub search: Rate, + pub graphql: Option, + pub source_import: Option, + pub integration_manifest: Option, + pub code_scanning_upload: Option, + pub actions_runner_registration: Option, + pub scim: Option, + pub dependency_snapshots: Option, +} + +// ------------------------------------------------------------------------------------------------- +// Rate +// ------------------------------------------------------------------------------------------------- +#[derive(Debug, Deserialize)] +pub struct Rate { + pub limit: i64, + pub remaining: i64, + pub reset: i64, + pub used: i64, +} + +// ------------------------------------------------------------------------------------------------- +// User +// ------------------------------------------------------------------------------------------------- +#[derive(Debug, Deserialize)] +pub struct User { + pub login: String, + pub id: i64, + pub node_id: String, + pub avatar_url: String, + pub gravatar_id: Option, + pub url: String, + pub html_url: String, + pub followers_url: String, + pub following_url: String, + pub gists_url: String, + pub starred_url: String, + pub subscriptions_url: String, + pub organizations_url: String, + pub repos_url: String, + pub events_url: String, + pub received_events_url: String, + #[serde(rename = "type")] + pub user_type: String, + pub site_admin: bool, + pub name: Option, + pub company: Option, + pub blog: Option, + pub location: Option, + pub email: Option, + pub hireable: Option, + pub bio: Option, + pub twitter_username: Option, + pub public_repos: i64, + pub public_gists: i64, + pub followers: i64, + pub following: i64, + pub created_at: String, + pub updated_at: String, + // pub plan: Option>, + pub suspended_at: Option, + pub private_gists: Option, + pub total_private_repos: Option, + pub owned_private_repos: Option, + pub disk_usage: Option, + pub collaborators: Option, + + pub business_plus: Option, + pub ldap_dn: Option, + pub two_factor_authentication: Option, +} + +// ------------------------------------------------------------------------------------------------- +// Gist +// ------------------------------------------------------------------------------------------------- +/* +#[derive(Debug, Deserialize)] +pub struct Gist { + pub comments: u64, + pub comments_url: Url, + pub commits_url: Url, + pub created_at: DateTime, + pub description: Option, + pub files: BTreeMap, + pub forks_url: Url, + pub git_pull_url: Url, + pub git_push_url: Url, + pub html_url: Url, + pub id: String, + pub node_id: String, + pub updated_at: DateTime, + pub url: Url, +} +*/ + +// ------------------------------------------------------------------------------------------------- +// GistFile +// ------------------------------------------------------------------------------------------------- +// This is the same as octocrab::models::gists::Gist, except it doesn't have `content` or `truncated` +/* +#[derive(Debug, Deserialize)] +pub struct GistFile { + pub filename: String, + pub language: Option, + pub r#type: String, + pub raw_url: Url, + pub size: u64, +} +*/ + +// ------------------------------------------------------------------------------------------------- +// Page +// ------------------------------------------------------------------------------------------------- +/* +pub struct Page { + items: Vec, + next: Option, + prev: Option, + last: Option, + first: Option, +} + +// See . +use anyhow::Result; +impl Page { + pub fn from_response(response: &reqwest::Response) -> Result { + let link = response.headers().get(reqwest::header::LINK); + let items = + let next = None; + let prev = None; + let last = None; + let first = None; + Ok(Page { + items, + next, + prev, + last, + first, + }) + } +} +*/ + +// ------------------------------------------------------------------------------------------------- +// Repository +// ------------------------------------------------------------------------------------------------- +#[derive(Debug, Deserialize)] +pub struct Repository { + pub id: i32, + pub node_id: String, + pub name: String, + pub full_name: String, + // pub owner: Box, + pub private: bool, + pub html_url: String, + pub description: Option, + pub fork: bool, + pub url: String, + pub archive_url: String, + pub assignees_url: String, + pub blobs_url: String, + pub branches_url: String, + pub collaborators_url: String, + pub comments_url: String, + pub commits_url: String, + pub compare_url: String, + pub contents_url: String, + pub contributors_url: String, + pub deployments_url: String, + pub downloads_url: String, + pub events_url: String, + pub forks_url: String, + pub git_commits_url: String, + pub git_refs_url: String, + pub git_tags_url: String, + pub git_url: Option, + pub issue_comment_url: String, + pub issue_events_url: String, + pub issues_url: String, + pub keys_url: String, + pub labels_url: String, + pub languages_url: String, + pub merges_url: String, + pub milestones_url: String, + pub notifications_url: String, + pub pulls_url: String, + pub releases_url: String, + pub ssh_url: Option, + pub stargazers_url: String, + pub statuses_url: String, + pub subscribers_url: String, + pub subscription_url: String, + pub tags_url: String, + pub teams_url: String, + pub trees_url: String, + pub clone_url: Option, + pub mirror_url: Option>, + pub hooks_url: String, + pub svn_url: Option, + pub homepage: Option>, + pub language: Option>, + pub forks_count: Option, + pub stargazers_count: Option, + pub watchers_count: Option, + /// The size of the repository. Size is calculated hourly. When a repository is initially created, the size is 0. + pub size: Option, + pub default_branch: Option, + pub open_issues_count: Option, + pub is_template: Option, + pub topics: Option>, + pub has_issues: Option, + pub has_projects: Option, + pub has_wiki: Option, + pub has_pages: Option, + pub has_downloads: Option, + pub has_discussions: Option, + pub archived: Option, + pub disabled: Option, + pub visibility: Option, + pub pushed_at: Option>, + pub created_at: Option>, + pub updated_at: Option>, + // pub permissions: Option>, + pub role_name: Option, + pub temp_clone_token: Option, + pub delete_branch_on_merge: Option, + pub subscribers_count: Option, + pub network_count: Option, + // pub code_of_conduct: Option>, + // pub license: Option>>, + pub forks: Option, + pub open_issues: Option, + pub watchers: Option, + pub allow_forking: Option, + pub web_commit_signoff_required: Option, + // pub security_and_analysis: Option>>, +} diff --git a/src/lib.rs b/src/lib.rs index b34d5f739..2167fcb70 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,6 +4,7 @@ pub mod blob_id_set; pub mod bstring_escape; pub mod datastore; pub mod defaults; +pub mod github; pub mod input_enumerator; pub mod location; pub mod match_type; diff --git a/tests/snapshots/cli__noseyparker_help-2.snap b/tests/snapshots/cli__noseyparker_help-2.snap index 257cbd5f5..0f9805b3f 100644 --- a/tests/snapshots/cli__noseyparker_help-2.snap +++ b/tests/snapshots/cli__noseyparker_help-2.snap @@ -13,6 +13,8 @@ Commands: Summarize scan findings report Report detailed scan findings + github + Query GitHub datastore Manage datastores rules diff --git a/tests/snapshots/cli__noseyparker_no_args-3.snap b/tests/snapshots/cli__noseyparker_no_args-3.snap index 94dfc971c..fb0548e5e 100644 --- a/tests/snapshots/cli__noseyparker_no_args-3.snap +++ b/tests/snapshots/cli__noseyparker_no_args-3.snap @@ -10,6 +10,7 @@ Commands: scan Scan content for secrets summarize Summarize scan findings report Report detailed scan findings + github Query GitHub datastore Manage datastores rules Manage rules help Print this message or the help of the given subcommand(s)