Checkpoint work on GitHub enumeration

Cargo.toml

@@ -33,6 +33,7 @@ rule_profiling = []
 anyhow = { version = "1.0" }
 atty = "0.2"
 bstr = { version = "1.0.1", features = ["serde"] }
+chrono = "0.4.23"
 clap = { version = "4.0", features = ["cargo", "derive", "env", "unicode", "wrap_help"] }
 console = "0.15.2"
 git-discover = "0.12.1"
@@ -54,15 +55,19 @@ pretty_assertions = "1.3"
 prettytable-rs = "0.10.0"
 rayon = "1.5"
 regex = "1.7"
+reqwest = { version = "0.11.13", features = ["json"] }
 rlimit = "0.9.0"
 rusqlite = { version = "0.28", features = ["bundled", "backup"] }
+secrecy = "0.8.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 serde_yaml = "0.9"
 sha1 = "0.10"
+tokio = "1.23.0"
 tracing = "0.1.37"
 tracing-log = "0.1.3"
 tracing-subscriber = { version = "0.3.16", features = ["tracing-log", "ansi"] }
+url = "2.3.1"
 walkdir = "2.3"
 
 [dev-dependencies]
@@ -131,7 +136,6 @@ opt-level = 3
 # tempfile                                      # filesystem
 # termcolor                                     # reporting
 # tinytemplate                                  # templating
-# tokio
 # toml                                          # data format, configuration
 # tree_magic                                    # content type guesser
 # unicode-normalization

src/bin/noseyparker/args.rs

@@ -66,13 +66,18 @@ pub enum Command {
     Scan(ScanArgs),
 
     /// Summarize scan findings
-    #[command(display_order = 2, alias="summarise")]
+    #[command(display_order = 2, alias = "summarise")]
     Summarize(SummarizeArgs),
 
     /// Report detailed scan findings
     #[command(display_order = 3)]
     Report(ReportArgs),
 
+    /// Query GitHub
+    #[command(display_order = 4, name = "github")]
+    GitHub(GitHubArgs),
+
+
     #[command(display_order = 30)]
     /// Manage datastores
     Datastore(DatastoreArgs),
@@ -146,6 +151,55 @@ impl std::fmt::Display for Mode {
     }
 }
 
+// -----------------------------------------------------------------------------
+// `github` command
+// -----------------------------------------------------------------------------
+#[derive(Args, Debug)]
+pub struct GitHubArgs {
+    #[command(subcommand)]
+    pub command: GitHubCommand,
+}
+
+#[derive(Subcommand, Debug)]
+pub enum GitHubCommand {
+    /// Interact with GitHub repositories
+    #[command(subcommand)]
+    Repos(GitHubReposCommand),
+}
+
+#[derive(Subcommand, Debug)]
+pub enum GitHubReposCommand {
+    /// List repositories belonging to a specific user or organization
+    List(GitHubReposListArgs),
+}
+
+#[derive(Args, Debug)]
+pub struct GitHubReposListArgs {
+    #[command(flatten)]
+    pub repo_specifiers: GitHubRepoSpecifiers,
+
+    #[command(flatten)]
+    pub output_args: OutputArgs,
+}
+
+#[derive(Args, Debug)]
+pub struct GitHubRepoSpecifiers {
+    /// Select repositories belonging to the specified user
+    #[arg(long)]
+    pub user: Vec<String>,
+
+    /// Select repositories belonging to the specified organization
+    #[arg(long, visible_alias = "org")]
+    pub organization: Vec<String>,
+}
+
+impl GitHubRepoSpecifiers {
+    pub fn is_empty(&self) -> bool {
+        self.user.is_empty() && self.organization.is_empty()
+    }
+}
+
+
 // -----------------------------------------------------------------------------
 // `rules` command
 // -----------------------------------------------------------------------------

src/bin/noseyparker/cmd_github.rs

@@ -0,0 +1,109 @@
+use anyhow::{Context, Result, bail};
+// use chrono::{DateTime, Utc};
+// use reqwest::Url;
+// use serde::Deserialize;
+// use std::collections::BTreeMap;
+
+use crate::args;
+use noseyparker::github;
+
+pub fn run(global_args: &args::GlobalArgs, args: &args::GitHubArgs) -> Result<()> {
+    use args::GitHubCommand::*;
+    use args::GitHubReposCommand::*;
+    match &args.command {
+        Repos(List(args)) => list_repos(global_args, args)
+    }
+}
+
+fn list_repos(_global_args: &args::GlobalArgs, args: &args::GitHubReposListArgs) -> Result<()> {
+    if args.repo_specifiers.is_empty() {
+        bail!("No repositories specified");
+    }
+
+    let runtime = tokio::runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()
+        .context("Failed to initialize async runtime")?;
+
+    let client = github::Client::new()
+        .context("Failed to initialize GitHub client")?;
+
+    let rate_limit = runtime.block_on(client.rate_limit())?;
+    println!("{:#?}", rate_limit);
+
+    for username in &args.repo_specifiers.user {
+        let user = runtime.block_on(client.user(username))?;
+        println!("{:#?}", user);
+        let repos = runtime.block_on(client.user_repos(username))?;
+        for repo in repos.iter() {
+            println!("{:#?}", repo);
+        }
+    }
+
+    let rate_limit = runtime.block_on(client.rate_limit())?;
+    println!("{:#?}", rate_limit);
+
+    return Ok(());
+
+    /*
+    let mut writer = args
+        .output_args
+        .get_writer()
+        .context("Failed to open output destination for writing")?;
+
+    let run_inner = move || -> std::io::Result<()> {
+        match &args.output_args.format {
+            args::OutputFormat::Human => {
+                // writeln!(writer)?;
+                // let table = summary_table(summary);
+                // // FIXME: this doesn't preserve ANSI styling on the table
+                // table.print(&mut writer)?;
+            }
+            args::OutputFormat::Json => {
+                // serde_json::to_writer_pretty(&mut writer, &summary)?;
+            }
+            args::OutputFormat::Jsonl => {
+                // for entry in summary.0.iter() {
+                //     serde_json::to_writer(&mut writer, entry)?;
+                //     writeln!(&mut writer)?;
+                // }
+            }
+        }
+        Ok(())
+    };
+    match run_inner() {
+        // Ignore SIGPIPE errors, like those that can come from piping to `head`
+        Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => { Ok(()) }
+        Err(e) => Err(e)?,
+        Ok(()) => Ok(()),
+    }
+    */
+}
+
+/*
+pub fn summary_table(summary: MatchSummary) -> prettytable::Table {
+    use prettytable::format::{FormatBuilder, LinePosition, LineSeparator};
+    use prettytable::row;
+
+    let f = FormatBuilder::new()
+        // .column_separator('│')
+        // .separators(&[LinePosition::Title], LineSeparator::new('─', '┼', '├', '┤'))
+        .column_separator(' ')
+        .separators(&[LinePosition::Title], LineSeparator::new('─', '─', '─', '─'))
+        .padding(1, 1)
+        .build();
+
+    let mut table: prettytable::Table = summary
+        .0
+        .into_iter()
+        .map(|e| row![
+             l -> &e.rule_name,
+             r -> HumanCount(e.distinct_count.try_into().unwrap()),
+             r -> HumanCount(e.total_count.try_into().unwrap())
+        ])
+        .collect();
+    table.set_format(f);
+    table.set_titles(row![lb -> "Rule", cb -> "Distinct Matches", cb -> "Total Matches"]);
+    table
+}
+*/

src/bin/noseyparker/main.rs

@@ -3,6 +3,7 @@ use tracing::debug;
 
 mod args;
 mod cmd_datastore;
+mod cmd_github;
 mod cmd_report;
 mod cmd_rules;
 mod cmd_scan;
@@ -23,7 +24,6 @@ fn configure_tracing(global_args: &args::GlobalArgs) -> Result<()> {
         .with_max_level(filter.as_log())
         .init()?;
 
-    // a builder for `FmtSubscriber`.
     let subscriber = tracing_subscriber::FmtSubscriber::builder()
       .with_max_level(filter)
       .with_ansi(global_args.use_color())
@@ -63,6 +63,7 @@ fn try_main() -> Result<()> {
 
     match &args.command {
         args::Command::Datastore(args) => cmd_datastore::run(global_args, args),
+        args::Command::GitHub(args) => cmd_github::run(global_args, args),
         args::Command::Rules(args) => cmd_rules::run(global_args, args),
         args::Command::Scan(args) => cmd_scan::run(global_args, args),
         args::Command::Summarize(args) => cmd_summarize::run(global_args, args),