Dynamically pull repo information for HPC lessons

.github/workflows/build-and-deploy.yml

@@ -3,6 +3,9 @@ name: check, build, deploy hpc-carpentry.org
 on:
   push:
     branches: main
+  pull_request:
+    branches:
+      - main
 
 jobs:
   build-website:
@@ -40,6 +43,14 @@ jobs:
           curl --remote-name-all https://feeds.carpentries.org/all_instructors_by_airport.geojson &&\
           cp all_instructors_by_airport.geojson files/.
 
+      - name: Create repository data files
+        uses: eessi/github-action-eessi@v1
+        shell: bash
+        env:
+          GITHUB_PAT: ${{ secrets.JEKYLL_PAT }}
+        run: |
+          module load R
+          make data
 
 #      - name: Create PDF file of some pages
 #        uses: docker://pandoc/latex:latest

Makefile

@@ -20,6 +20,13 @@ site :
 install :
 	bundle install
 
+## lesson data  : pull lesson data from GitHub
+# (requires GitHub PAT provided via GITHUB_PAT env var)
+data:
+	R -q -e "source('R/hpc-carpentry_lessons.R')"
+## help-wanted: list of issues that have the label "help wanted"
+	R -q -e "source('R/help_wanted_issues.R')"
+
 #-------------------------------------------------------------------------------
 
 ## clean      : clean up junk files.

R/README.md

@@ -0,0 +1 @@
+The files in this directory are based off of https://github.com/carpentries/feeds.carpentries.org (also MIT licence). If they require an update you should check there first.

R/help_wanted_issues.R

@@ -0,0 +1,182 @@
+source("R/utils.R")
+
+new_tbl_github_issues <- function(url = character(0),
+                                  title = character(0),
+                                  type = character(0),
+                                  labels = character(0),
+                                  label_colors = character(0),
+                                  font_colors = character(0),
+                                  created_at = character(0),
+                                  updated_at = character(0),
+                                  empty = FALSE
+                                  ) {
+
+  stopifnot(rlang::is_scalar_logical(empty))
+
+  if (empty) {
+    stopifnot(rlang::is_character(url, n = 0L))
+    stopifnot(rlang::is_character(title, n = 0L))
+    stopifnot(rlang::is_character(type, n = 0L))
+    stopifnot(rlang::is_character(labels, n = 0L))
+    stopifnot(rlang::is_character(label_colors, n = 0L))
+    stopifnot(rlang::is_character(font_colors, n = 0L))
+    stopifnot(rlang::is_character(created_at, n = 0L))
+    stopifnot(rlang::is_character(updated_at, n = 0L))
+  } else {
+    stopifnot(rlang::is_scalar_character(url))
+    stopifnot(rlang::is_scalar_character(title))
+    stopifnot(rlang::is_scalar_character(type))
+    stopifnot(rlang::is_character(labels))
+    stopifnot(rlang::is_character(label_colors))
+    stopifnot(rlang::is_character(font_colors))
+    stopifnot(rlang::is_scalar_character(created_at))
+    stopifnot(rlang::is_scalar_character(updated_at))
+  }
+
+  tibble::tibble(
+    url,
+    title,
+    type,
+    labels,
+    label_colors,
+    font_colors,
+    created_at,
+    updated_at
+  )
+
+}
+
+get_gh_issues_raw <- function(owner, repo, labels) {
+  if (!is.null(labels)) {
+    stopifnot(identical(length(labels), 1L))
+  }
+  gh::gh(
+    "GET /repos/:owner/:repo/issues",
+    owner = owner,
+    repo = repo,
+    labels = labels
+  )
+}
+
+extract_issue_info <- function(issues) {
+
+  if (identical(length(issues), 0L)) {
+    return(new_tbl_github_issues(empty = TRUE))
+  }
+
+  issues %>%
+    purrr::map_df(function(.x) {
+      new_tbl_github_issues(
+        url = .x$html_url,
+        title = .x$title,
+        type = dplyr::case_when(
+          grepl("/pull/[0-9]+$", .x$html_url) ~ "PR",
+          TRUE ~ "issue"
+        ),
+        labels = purrr::map_chr(.x$labels, "name") %>%
+          paste(., collapse = ","),
+        label_colors = purrr::map_chr(.x$labels, "color") %>%
+          paste0("#", ., collapse = ","),
+        font_colors = purrr::map_chr(.x$labels, "color") %>%
+          paste0("#", .) %>%
+          font_color(.) %>%
+          paste(., collapse = ","),
+        created_at = .x$created_at,
+        updated_at = .x$updated_at
+      )
+    })
+}
+
+get_gh_issues <- function(owner, repo, labels) {
+  get_gh_issues_raw(owner, repo, labels) %>%
+    extract_issue_info() %>%
+    dplyr::mutate(
+      org = owner,
+      repo = repo,
+      full_repo = paste0(owner, "/", repo)
+    )
+}
+
+keep_opted_in <- function(orgs) {
+
+  # at_opted_in <- airtabler::airtable(
+  #   base = "appeZJGnGremE1MYm",
+  #   tables = "Repositories"
+  # )
+
+  # opted_in <- at_opted_in$Repositories$select_all() %>%
+  #   dplyr::mutate(lesson_program = tolower(lesson_program))
+  #
+  # dplyr::left_join(
+    # orgs, opted_in,
+    # by = c(
+      # "carpentries_org" = "lesson_program",
+      # "repo" = "repository"
+    # )
+  # ) %>%
+  # dplyr::filter(display_help_wanted)
+  # message(sQuote(orgs))
+  orgs
+}
+
+keep_other_repos <- function(orgs) {
+  other_repos <- tibble::tribble(
+    ~carpentries_org, ~repo,
+    "carpentries-incubator", "hpc-intro",
+    "hpc-carpentry", "hpc-carpentry.github.io",
+  )
+
+  dplyr::inner_join(
+    orgs, other_repos,
+    by = c("carpentries_org", "repo")
+  )
+}
+
+
+list_organizations <- c(
+  "HPC Carpentry" = "hpc-carpentry"
+  # would like to include the incubator but need to filter the repo according to topic
+  # "The Carpentries Incubator" = "carpentries-incubator"
+)
+
+list_help_wanted <- purrr::imap_dfr(
+  list_organizations,
+  function(.x, .y) {
+    orgs <- get_list_repos(
+      .x, ignore_archived = TRUE,
+      ignore_pattern = "^\\d{4}-\\d{2}-\\d{2}"
+    )
+
+    lessons <- orgs %>%
+      keep_opted_in()
+
+    other_repos <- orgs %>%
+      keep_other_repos()
+
+    dplyr::bind_rows(
+      lessons,
+      other_repos
+    )  %>%
+      dplyr::distinct(carpentries_org, repo, .keep_all = TRUE) %>%
+      purrr::pmap_df(function(carpentries_org, repo, description, ...) {
+        message("  repo: ", repo, appendLF = FALSE)
+        res <- purrr::map_dfr(
+          c("help wanted",  "good first issue"),
+          ~ get_gh_issues(
+            owner = carpentries_org, repo = repo, labels = .x
+          )
+        ) %>%
+          dplyr::distinct(url, .keep_all = TRUE)
+        message(" -- n issues: ", nrow(res))
+        res %>%
+          dplyr::mutate(
+            description = description,
+            ## remove GitHub emoji from repo description
+            clean_description = gsub(":([a-z0-9_]+):", "", description),
+            org_name = .y)
+      })
+  }
+)
+
+
+jsonlite::write_json(list_help_wanted, "_data/help_wanted_issues.json")

R/hpc-carpentry_lessons.R

@@ -0,0 +1,76 @@
+source("R/utils.R")
+
+LIFE_CYCLE_TAGS <- c("pre-alpha", "alpha", "beta", "stable")
+# The tags below will be filtered out in the json
+COMMON_TAGS <- c(
+  "carpentries",
+  "carpentries-incubator",
+  "carpentries-lesson",
+  "carpentryconnect",
+  "data-carpentry",
+  "datacarpentry",
+  "education",
+  "lesson"
+)
+
+check_missing_repo_info <- function(.d, field) {
+  if (any(!nzchar(.d[[field]]))) {
+    paste0(
+      "Missing repo ", sQuote(field), " for: \n",
+      paste0("  - ", .d$repo_url[!nzchar(.d[[field]])], collapse = "\n"),
+      "\n"
+    )
+  }
+}
+
+check_repo_info <- function(.d, fields) {
+  tryCatch({
+    out <- purrr::map(
+      fields, ~ check_missing_repo_info(.d, .)
+    )
+    msgs <- purrr::keep(out, ~ !is.null(.))
+
+    if (length(msgs)) {
+      stop(msgs, call. = FALSE)
+    }
+
+    cli::cli_alert_success("No issues detected!")
+  },
+  error = function(err) {
+    stop(err$message, call. = FALSE)
+  })
+}
+
+make_community_lessons_feed <- function(path, ...) {
+
+  carp_inc <- get_org_topics("carpentries-incubator")
+  hpc_carp <- get_org_topics("hpc-carpentry")
+
+  res <- dplyr::bind_rows(carp_inc, hpc_carp) %>%
+    dplyr::select(-private) %>%
+    dplyr::filter(grepl("hpc-carpentry", github_topics)) %>%
+    dplyr::filter(grepl("lesson", github_topics)) %>%
+    extract_tag(
+      life_cycle_tag,
+      LIFE_CYCLE_TAGS,
+      approach = "include",
+      allow_multiple = FALSE,
+      allow_empty = FALSE
+    ) %>%
+    extract_tag(
+      lesson_tags,
+      COMMON_TAGS,
+      approach = "exclude",
+      allow_multiple = TRUE,
+      allow_empty = TRUE
+    )
+
+  ## checks
+  check_repo_info(res, c("description", "rendered_site"))
+
+  res %>%
+    jsonlite::write_json(path = path)
+
+}
+
+make_community_lessons_feed("_data/hpc_lessons.json")