git-scrape

#!perl
# mengwong 20200619

# USAGE: git-scrape username    2>&1 | tee username.org

# download all of username's repositories' clone_urls
# keep asking for more pages until we get 0 back
# then clone all the actual repositories
# the output logging format is org-friendly: open it in emacs for better hierarchy
# see also https://github.com/mengwong/git-scrape
# 
# requires npm i -g json

use Cwd;
use File::Basename;

my ($cwdbase) = getcwd() =~ /([^\/]+)$/;
my $username = shift;
if (defined $username or not length $username) {
  $username = $cwdbase;
}

print "* git-scrape $username\n";

my @jsons;
my $n = 1;
my $nobreak = 0;

# maybe we are already in the relevant directory
unless ($cwdbase eq $username) {
  print "mkdir $username; cd $username;\n";
  mkdir $username;
  chdir $username;
} else {
  print "already in $username/\n";
}

print "* what repositories does $username have?\n";

do {
  my $torun = "curl -s 'https://api.github.com/users/$username/repos?page=$n' | json -a clone_url";
  my @urls = qx($torun);
  print "page $n returned " . @urls . " urls\n";
  push (@jsons, @urls);
  $nobreak = @urls;
  $n++;
}
while ($nobreak);

print "** $username has " . @jsons . " repositories\n";
print "- $_" for @jsons;

print "* cloning " . @jsons . " repositories\n";

for my $repo (@jsons) {
  chomp $repo;
  print "** $repo\n";
  # clone or pull?
  my ($basename) = $repo =~ /([^\/]+)\.git$/;
  if (-d $basename) {
    print "git pull\n";
    system("cd $basename; git pull");
  } else {
    system("git clone $repo");
  }
}

print "\n* done!\nin future, repeat the command to pull\n";