diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml
index ea839cb47..d46f4da2e 100644
--- a/.github/workflows/build-and-deploy.yml
+++ b/.github/workflows/build-and-deploy.yml
@@ -35,11 +35,16 @@ jobs:
yaml-lint -q -n $(find _posts -regex ".*.md\|.*html") &&\
yaml-lint -q -n $(find pages -regex ".*.md\|.*html")
- - name: Get GeoJSON for instructor map
- run: |
- curl --remote-name-all https://feeds.carpentries.org/all_instructors_by_airport.geojson &&\
- cp all_instructors_by_airport.geojson files/.
+ - name: Setup EESSI to give us an R module
+ uses: eessi/github-action-eessi@v1
+ - name: Create repository data files
+ shell: bash
+ env:
+ GITHUB_PAT: ${{ secrets.JEKYLL_PAT }}
+ run: |
+ module load R
+ make data
# - name: Create PDF file of some pages
# uses: docker://pandoc/latex:latest
diff --git a/Gemfile b/Gemfile
index 4b5ecef9f..83d687db0 100644
--- a/Gemfile
+++ b/Gemfile
@@ -4,5 +4,5 @@ gem 'faraday', '0.17.3'
group :jekyll_plugins do
gem 'github-pages'
- gem 'jekyll-get-json', "~> 0.0.2"
+# gem 'jekyll-get-json', "~> 0.0.2"
end
diff --git a/Makefile b/Makefile
index a5baa8c97..c7594341a 100644
--- a/Makefile
+++ b/Makefile
@@ -20,6 +20,13 @@ site :
install :
bundle install
+## lesson data : pull lesson data from GitHub
+# (requires GitHub PAT provided via GITHUB_PAT env var)
+data:
+ R -q -e "source('feeds/hpc-carpentry_lessons.R')"
+## help-wanted: list of issues that have the label "help wanted"
+ R -q -e "source('feeds/help_wanted_issues.R')"
+
#-------------------------------------------------------------------------------
## clean : clean up junk files.
diff --git a/README.md b/README.md
index 4f0b34e77..07c7187f1 100644
--- a/README.md
+++ b/README.md
@@ -1,55 +1,67 @@
# The HPC Carpentry Website
-## NOTE: THIS IS AN ALPHA CREATION, A CLONE OF THE CARPENTRIES WEBSITE AND MAY NOT BE COMPLETELY CLEANED OF THE CARPENTRIES CONTENT
-
-This is the repository for the [HPC Carpentry website](http://www.hpc-carpentry.org) (and directly based on
-the [Carpentries website](https://carpentries.org)).
-Please submit additions and fixes as pull requests to [our GitHub repository](https://github.com/hpc-carpentry/hpc-carpentry.github.io).
-
-* [Setup](#setup)
-* [Previewing](#previewing)
-* [Development](#development)
- * [Write a Blog Post](#blog)
- * [Create a New Page](#page)
- * [Add a Workshop](#workshop)
-* [The Details](#details)
-
-Lessons are not stored in this repository:
-please see the [HPC Carpentry lessons page](https://hpc-carpentry.org/lessons/), for links to the many individual lesson repositories.
-You can find the official Carpentries lessons at [Software Carpentry lessons page](https://software-carpentry.org/lessons/), the [Data Carpentry lessons page](https://datacarpentry.org/lessons/), or the [Library Carpentry lessons page](https://librarycarpentry.org/lessons/).
-
-> HPC Carpentry, like The Carpentries (Software, Data, and Library Carpentry), is an open project,
-> and we welcome contributions of all kinds.
-> By contributing,
-> you are agreeing that The Carpentries may redistribute your work
-> under [these licenses](http://software-carpentry.org/license/),
-> and to abide by [our code of conduct](http://docs.carpentries.org/topic_folders/policies/code-of-conduct.html).
+
+This is the repository for the [HPC Carpentry website](
+http://www.hpc-carpentry.org) (and directly based on the [Carpentries website](
+https://carpentries.org)). Please submit additions and fixes as pull requests
+to [our GitHub repository](
+https://github.com/hpc-carpentry/hpc-carpentry.github.io).
+
+* [Setup](#setup)
+* [Previewing](#previewing)
+* [Development](#development)
+ * [Write a Blog Post](#blog)
+ * [Create a New Page](#page)
+ * [Add a Workshop](#workshop)
+* [The Details](#details)
+
+*Lessons are not stored in this repository.*
+Please see the [HPC Carpentry lessons page](
+https://hpc-carpentry.org/lessons/), for links to the many individual lesson
+repositories. You can find the official Carpentries lessons at the
+[Software Carpentry lessons page](https://software-carpentry.org/lessons/), the
+[Data Carpentry lessons page](https://datacarpentry.org/lessons/), or the
+[Library Carpentry lessons page](https://librarycarpentry.org/lessons/).
+
+> HPC Carpentry, like The Carpentries (Software, Data, and Library Carpentry),
+> is an open project, and we welcome contributions of all kinds.
+> By contributing, you are agreeing that The Carpentries may redistribute your
+> work under [these licenses](http://software-carpentry.org/license/),
+> and to abide by [our code of conduct](
+> http://docs.carpentries.org/topic_folders/policies/code-of-conduct.html).
## Setup
-The website uses [Jekyll](http://jekyllrb.com/), a static website generator written in Ruby.
-You need to have Version 2.7.1 or higher of Ruby and the package manager Bundler.
-(The package manager is used to make sure you use exactly the same versions of the Ruby Gems as we do.)
-After checking out the repository, please run:
+The website uses [Jekyll](http://jekyllrb.com/), a static website generator
+written in Ruby. You need to have Version 2.7.1 or higher of Ruby and the
+package manager Bundler. (The package manager is used to make sure you use
+exactly the same versions of the Ruby Gems as we do.)
+After checking out the repository, please install Jekyll and dependencies
+by running
```
$ bundle install
```
-to install Jekyll and the software it depends on.
-You may consult [Using Jekyll with Pages](https://help.github.com/articles/using-jekyll-with-pages/) for further instructions.
+You may consult [Using Jekyll with Pages](
+https://help.github.com/articles/using-jekyll-with-pages/) for further
+instructions.
## Previewing
-Please do **not** use `jekyll build` or `jekyll serve` directly to build or view the website.
-Instead, you should use the following commands:
+Please do **not** use `jekyll build` or `jekyll serve` directly to build or
+view the website. Instead, you should use the following commands:
-* `make` or `make commands`: list available commands.
-* `make serve`: build files locally and run a server at [http://0.0.0.0:4000/](http://0.0.0.0:4000/) for viewing.
- This is the best way to preview the site.
-* `make site`: build files locally, but do not serve them dynamically.
-* `make clean` removes the `_site` directory and any Emacs editor backup files littering the source directories.
+* `make` or `make commands`: list available commands.
+* `make data`: (optionally) generate the dynamic data (see [Data Files](#data)
+ to see the requirements for this step)
+* `make serve`: build files locally and run a server at
+ [http://0.0.0.0:4000/](http://0.0.0.0:4000/) for viewing. This is the best
+ way to preview the site.
+* `make site`: build files locally, but do not serve them dynamically.
+* `make clean` removes the `_site` directory and any Emacs editor backup files
+ littering the source directories.
The [details](#details) describes a few more advanced commands as well.
@@ -58,13 +70,14 @@ The [details](#details) describes a few more advanced commands as well.
To **write a blog post**,
-create a file called `_posts/YYYY/MM/YYYY-MM-DD-some-title.html` or `_posts/YYYY/MM/YYYY-MM-DD-some-title.md`
-(for HTML and Markdown respectively).
-YYYY is the 4-digit year of the post, MM the 2-digit month, and DD the 2-digit day;
-`some-title` can be any hyphenated string of words that do not include special characters such as quotes.
+create a file called `_posts/YYYY/MM/YYYY-MM-DD-some-title.html` or
+`_posts/YYYY/MM/YYYY-MM-DD-some-title.md` (for HTML and Markdown respectively).
+YYYY is the 4-digit year of the post, MM the 2-digit month, and DD the 2-digit
+day; `some-title` can be any hyphenated string of words that do not include
+special characters such as quotes.
Please do *not* use underscores or periods in the names.
-When published,
-your blog post will appear as `https://hpc-carpentry.org/blog/YYYY/MM/some-title.html`.
+When published, your blog post will appear as
+`https://hpc-carpentry.org/blog/YYYY/MM/some-title.html`.
The YAML header of a blog post must look like this:
@@ -79,10 +92,10 @@ category: ["Some Category", "Some Other Category"]
---
~~~
-where `YYYY-MM-DD` is replaced by the post's date and `hh:mm:ss` by the post's time.
-Note that the time *must* be quoted so that the colons it contains do not confuse Jekyll's YAML parser.
-Note also that `authors` is a list---if the post has more than one author,
-please format the list like this:
+where `YYYY-MM-DD` is replaced by the post's date and `hh:mm:ss` by the post's
+time. Note that the time *must* be quoted so that the colons it contains do not
+confuse Jekyll's YAML parser. Note also that `authors` is a *list*. If the post
+has more than one author, please format the list like this:
~~~
...
@@ -108,34 +121,51 @@ You must then also add the page to `_data/navigation.yml`,
which is used to generate the site's pull-down navigation menu.
-To **add a workshop**,
-fill in the [workshop request form](https://amy.carpentries.org/forms/workshop/) online.
-You should fill in this form even for self-organized workshops in order to get your workshop into the Carpentries database.
+To **add a workshop**, fill in the [workshop request form](
+https://amy.carpentries.org/forms/workshop/) online. You should fill in this
+form even for self-organized workshops in order to get your workshop into the
+Carpentries database.
## The Details
### How is the site built and rendered?
-The website is build with a GitHub Actions (see [this file](https://github.com/hpc-carpentry/hpc-carpentry.github.io/blob/main/.github/workflows/build-and-deploy.yml)).
-
+The website is built with a GitHub Action, configured in
+[`.github/workflows/build-and-deploy.yml`](
+https://github.com/hpc-carpentry/hpc-carpentry.github.io/blob/main/.github/workflows/build-and-deploy.yml).
Each time a commit is pushed to the default branch of the repository (`main`)
-and every 6 hours, the GitHub Action does the following:
+and every 24 hours, the GitHub Action does the following:
+
+1. Validate the YAML headers of all the pages and blog posts.
+1. Update the time-dependent content (list of lessons, list of "Help Wanted"
+ issues, etc.). For details on how these data files can be generated see the
+ next section.
+1. Build and deploy the website to .
+
+### Data Files
-1. it validates the YAML headers of all the pages and blog posts
-1. it builds the website 1 using the latest versions of the Carpentries [data
- feeds](https://feeds.carpentries.org) to generate the dynamic content on the
- site (list of community members, list of workshops, etc.). For this, we use
- the [Jekyll Get JSON](https://github.com/brockfanning/jekyll-get-json)
- plugin.
+The data files for the workshops and the instructors are generated by our
+GitHub Action using the scripts found in the `feeds` folder.
-### Data Files
+To generate these feeds locally,
+
+```
+make data
+```
-The data files for the workshops and the instructors are generated every 6 hours
-from AMY (via the Carpentries redash server) by the script in the
-[feeds.carpentries.org repository](https://github.com/carpentries/feeds.carpentries.org).
-These files are available at .
+To successfully execute this step you need an appropriate R environment (with
+packages `colorspace`, `deplyr`, `gh`, `jsonlite`, `purrr`, and `tibble` — see
+[feeds/README.md](
+https://github.com/hpc-carpentry/hpc-carpentry.github.io/blob/main/)) and the
+environment variable `GITHUB_PAT` configured with your
+[GitHub Personal Authentication Token](
+https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token)
+so that GitHub does not limit your API usage. When generating this PAT, you
+should *not* need to set any scopes (not even "repo"): this process is
+read-only, and our work is public.
### Styles
-The files in the `_sass` and `assets` directories control the appearance of this site.
+The files in the `_sass` and `assets` directories control the appearance of
+this site.
diff --git a/_config.yml b/_config.yml
index 9f7ec871b..4936bc351 100644
--- a/_config.yml
+++ b/_config.yml
@@ -111,6 +111,7 @@ exclude:
- README.md
- INSTALL.md
- Makefile
+ - feeds/
- vendor/
# Build settings
@@ -119,14 +120,12 @@ exclude:
plugins:
- jekyll-paginate
- jekyll-redirect-from
- - jekyll-get-json
+# For HPC Carpentry, we generate the json files as part of a GitHub Action
+# so we don't (currently) need this dependency
+# - jekyll-get-json
## external data sources
-jekyll_get_json:
-# - data: community_lessons
-# json: 'https://feeds.carpentries.org/community_lessons.json'
- - data: help_wanted_issues
- json: 'https://feeds.carpentries.org/help_wanted_issues.json'
+# jekyll_get_json:
# - data: all_badged_people
# json: 'https://feeds.carpentries.org/all_badged_people.json'
# - data: badges_stats
diff --git a/_data/community_lessons.json b/_data/community_lessons.json
deleted file mode 100644
index 19032c393..000000000
--- a/_data/community_lessons.json
+++ /dev/null
@@ -1,101 +0,0 @@
-[
- {
- "hpc_carpentry_org": "carpentries-incubator",
- "repo": "hpc-intro",
- "repo_url": "https://github.com/hpc-carpentry/hpc-intro",
- "full_name": "hpc-carpentry/hpc-intro",
- "description": "Intro to HPC lesson materials",
- "rendered_site": "http://hpc-carpentry.org/hpc-intro/",
- "github_topics": [
- "carpentries-incubator",
- "lesson"
- ],
- "life_cycle_tag": [
- "alpha"
- ],
- "lesson_tags": [
- "HPC",
- "english"
- ]
- },
- {
- "hpc_carpentry_org": "carpentries-incubator",
- "repo": "hpc-shell",
- "repo_url": "https://github.com/hpc-carpentry/hpc-shell",
- "full_name": "hpc-carpentry/hpc-shell",
- "description": "Materials to teach terminal fundamentals for HPC users",
- "rendered_site": "http://hpc-carpentry.org/hpc-shell/",
- "github_topics": [
- "hpc-carpentry-lab",
- "lesson"
- ],
- "life_cycle_tag": [
- "alpha"
- ],
- "lesson_tags": [
- "HPC",
- "english",
- "shell"
- ]
- },
- {
- "hpc_carpentry_org": "carpentries-incubator",
- "repo": "hpc-parallel-novice",
- "repo_url": "https://github.com/hpc-carpentry/hpc-parallel-novice",
- "full_name": "hpc-carpentry/hpc-parallel-novice",
- "description": "Materials to teach terminal fundamentals for HPC users",
- "rendered_site": "http://hpc-carpentry.org/hpc-parallel-novice/",
- "github_topics": [
- "hpc-carpentry-lab",
- "lesson"
- ],
- "life_cycle_tag": [
- "alpha"
- ],
- "lesson_tags": [
- "HPC",
- "english",
- "shell"
- ]
- },
- {
- "hpc_carpentry_org": "carpentries-incubator",
- "repo": "hpc-python",
- "repo_url": "https://github.com/hpc-carpentry/hpc-python",
- "full_name": "hpc-carpentry/hpc-python",
- "description": "HPC Python lesson materials",
- "rendered_site": "http://hpc-carpentry.org/hpc-python/",
- "github_topics": [
- "hpc-carpentry-lab",
- "lesson"
- ],
- "life_cycle_tag": [
- "alpha"
- ],
- "lesson_tags": [
- "HPC",
- "english",
- "python"
- ]
- },
- {
- "hpc_carpentry_org": "carpentries-incubator",
- "repo": "hpc-chapel",
- "repo_url": "https://github.com/hpc-carpentry/hpc-chapel",
- "full_name": "hpc-carpentry/hpc-chapel",
- "description": "HPC Chapel lesson materials",
- "rendered_site": "http://hpc-carpentry.org/hpc-chapel/",
- "github_topics": [
- "hpc-carpentry-lab",
- "lesson"
- ],
- "life_cycle_tag": [
- "pre-alpha"
- ],
- "lesson_tags": [
- "HPC",
- "english",
- "Chapel"
- ]
- }
-]
diff --git a/feeds/README.md b/feeds/README.md
new file mode 100644
index 000000000..493cf3cdf
--- /dev/null
+++ b/feeds/README.md
@@ -0,0 +1,52 @@
+# HPC Carpentry Feeds
+
+The files in this directory are based off of [carpentries/feeds.carpentries.org](
+https://github.com/carpentries/feeds.carpentries.org) (MIT licenced).
+If they require an update you should check that repository first.
+
+## Dependencies
+
+### R
+
+These [R](https://www.r-project.org) scripts depend on the following libraries:
+* [colorspace](https://cran.r-project.org/web/packages/colorspace/)
+* [dplyr](https://cran.r-project.org/web/packages/dplyr/)
+* [gh](https://cran.r-project.org/web/packages/gh/)
+* [jsonlite](https://cran.r-project.org/web/packages/jsonlite/)
+* [purrr](https://cran.r-project.org/web/packages/purrr/)
+* [tibble](https://cran.r-project.org/web/packages/tibble/)
+
+If you have a Conda environment for local development and testing, install the
+packages:
+
+```bash
+conda install -c conda-forge r-colorspace r-dplyr r-gh r-jsonlite r-purrr r-tibble
+```
+
+### GitHub
+
+GitHub may limit your API access if you are not authenticated. To avoid this,
+generate a basic [Personal Access Token](
+https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token)
+with *no roles* associated (not even "repo"). Copy the token, and export it
+under the new environment variable `GITHUB_PAT`.
+
+### EESSI Project
+
+For CI platforms, installing a full R environment is time-consuming. A faster
+alternative is to use the [EESSI](https://eessi.github.io/docs/) filesystem
+overlay.
+
+There is a GitHub Action for this: `eessi/github-action-eessi`. This is used in
+our `/.github/workflows/build-and-deploy.yml`.
+
+## Build
+
+To build the data feeds locally, from the top-level directory of this
+repository, run
+
+```bash
+make data
+```
+
+This will create (or update) JSON files under `_data` with the latest content.
diff --git a/feeds/help_wanted_issues.R b/feeds/help_wanted_issues.R
new file mode 100644
index 000000000..cb154f6e8
--- /dev/null
+++ b/feeds/help_wanted_issues.R
@@ -0,0 +1,165 @@
+source("feeds/utils.R")
+
+new_tbl_github_issues <- function(url = character(0),
+ title = character(0),
+ type = character(0),
+ labels = character(0),
+ label_colors = character(0),
+ font_colors = character(0),
+ created_at = character(0),
+ updated_at = character(0),
+ empty = FALSE
+ ) {
+
+ stopifnot(rlang::is_scalar_logical(empty))
+
+ if (empty) {
+ stopifnot(rlang::is_character(url, n = 0L))
+ stopifnot(rlang::is_character(title, n = 0L))
+ stopifnot(rlang::is_character(type, n = 0L))
+ stopifnot(rlang::is_character(labels, n = 0L))
+ stopifnot(rlang::is_character(label_colors, n = 0L))
+ stopifnot(rlang::is_character(font_colors, n = 0L))
+ stopifnot(rlang::is_character(created_at, n = 0L))
+ stopifnot(rlang::is_character(updated_at, n = 0L))
+ } else {
+ stopifnot(rlang::is_scalar_character(url))
+ stopifnot(rlang::is_scalar_character(title))
+ stopifnot(rlang::is_scalar_character(type))
+ stopifnot(rlang::is_character(labels))
+ stopifnot(rlang::is_character(label_colors))
+ stopifnot(rlang::is_character(font_colors))
+ stopifnot(rlang::is_scalar_character(created_at))
+ stopifnot(rlang::is_scalar_character(updated_at))
+ }
+
+ tibble::tibble(
+ url,
+ title,
+ type,
+ labels,
+ label_colors,
+ font_colors,
+ created_at,
+ updated_at
+ )
+
+}
+
+get_gh_issues_raw <- function(owner, repo, labels) {
+ if (!is.null(labels)) {
+ stopifnot(identical(length(labels), 1L))
+ }
+ gh::gh(
+ "GET /repos/:owner/:repo/issues",
+ owner = owner,
+ repo = repo,
+ labels = labels
+ )
+}
+
+extract_issue_info <- function(issues) {
+
+ if (identical(length(issues), 0L)) {
+ return(new_tbl_github_issues(empty = TRUE))
+ }
+
+ issues %>%
+ purrr::map_df(function(.x) {
+ new_tbl_github_issues(
+ url = .x$html_url,
+ title = .x$title,
+ type = dplyr::case_when(
+ grepl("/pull/[0-9]+$", .x$html_url) ~ "PR",
+ TRUE ~ "issue"
+ ),
+ labels = purrr::map_chr(.x$labels, "name") %>%
+ paste(., collapse = ","),
+ label_colors = purrr::map_chr(.x$labels, "color") %>%
+ paste0("#", ., collapse = ","),
+ font_colors = purrr::map_chr(.x$labels, "color") %>%
+ paste0("#", .) %>%
+ font_color(.) %>%
+ paste(., collapse = ","),
+ created_at = .x$created_at,
+ updated_at = .x$updated_at
+ )
+ })
+}
+
+get_gh_issues <- function(owner, repo, labels) {
+ get_gh_issues_raw(owner, repo, labels) %>%
+ extract_issue_info() %>%
+ dplyr::mutate(
+ org = owner,
+ repo = repo,
+ full_repo = paste0(owner, "/", repo)
+ )
+}
+
+keep_hpc_carpentry_repos <- function(orgs) {
+ dplyr::filter(
+ orgs, carpentries_org == "hpc-carpentry"
+ )
+}
+
+keep_other_repos <- function(orgs) {
+ # Repos that wish to opt in should be manually added here
+ other_repos <- tibble::tribble(
+ ~carpentries_org, ~repo,
+ "carpentries-incubator", "hpc-intro",
+ )
+
+ dplyr::inner_join(
+ orgs, other_repos,
+ by = c("carpentries_org", "repo")
+ )
+}
+
+
+list_organizations <- c(
+ "HPC Carpentry" = "hpc-carpentry",
+ "The Carpentries Incubator" = "carpentries-incubator"
+)
+
+list_help_wanted <- purrr::imap_dfr(
+ list_organizations,
+ function(.x, .y) {
+ orgs <- get_list_repos(
+ .x, ignore_archived = TRUE,
+ ignore_pattern = "^\\d{4}-\\d{2}-\\d{2}"
+ )
+
+ hpc_carpentry_repos <- orgs %>%
+ keep_hpc_carpentry_repos()
+
+ other_repos <- orgs %>%
+ keep_other_repos()
+
+ dplyr::bind_rows(
+ hpc_carpentry_repos,
+ other_repos
+ ) %>%
+ dplyr::distinct(carpentries_org, repo, .keep_all = TRUE) %>%
+ purrr::pmap_df(function(carpentries_org, repo, description, ...) {
+ message(" repo: ", repo, appendLF = FALSE)
+ res <- purrr::map_dfr(
+ c("help wanted", "good first issue"),
+ ~ get_gh_issues(
+ owner = carpentries_org, repo = repo, labels = .x
+ )
+ ) %>%
+ dplyr::distinct(url, .keep_all = TRUE)
+ message(" -- n issues: ", nrow(res))
+ res %>%
+ dplyr::mutate(
+ description = description,
+ ## remove GitHub emoji from repo description
+ clean_description = gsub(":([a-z0-9_]+):", "", description),
+ org_name = .y)
+ })
+ }
+)
+
+
+jsonlite::write_json(list_help_wanted, "_data/help_wanted_issues.json")
diff --git a/feeds/hpc-carpentry_lessons.R b/feeds/hpc-carpentry_lessons.R
new file mode 100644
index 000000000..85fc061fd
--- /dev/null
+++ b/feeds/hpc-carpentry_lessons.R
@@ -0,0 +1,77 @@
+source("feeds/utils.R")
+
+LIFE_CYCLE_TAGS <- c("pre-alpha", "alpha", "beta", "stable")
+# The tags below will be filtered out in the json
+COMMON_TAGS <- c(
+ "carpentries",
+ "carpentries-incubator",
+ "carpentries-lesson",
+ "carpentryconnect",
+ "data-carpentry",
+ "hpc-carpentry",
+ "datacarpentry",
+ "education",
+ "lesson"
+)
+
+check_missing_repo_info <- function(.d, field) {
+ if (any(!nzchar(.d[[field]]))) {
+ paste0(
+ "Missing repo ", sQuote(field), " for: \n",
+ paste0(" - ", .d$repo_url[!nzchar(.d[[field]])], collapse = "\n"),
+ "\n"
+ )
+ }
+}
+
+check_repo_info <- function(.d, fields) {
+ tryCatch({
+ out <- purrr::map(
+ fields, ~ check_missing_repo_info(.d, .)
+ )
+ msgs <- purrr::keep(out, ~ !is.null(.))
+
+ if (length(msgs)) {
+ stop(msgs, call. = FALSE)
+ }
+
+ cli::cli_alert_success("No issues detected!")
+ },
+ error = function(err) {
+ stop(err$message, call. = FALSE)
+ })
+}
+
+make_community_lessons_feed <- function(path, ...) {
+
+ carp_inc <- get_org_topics("carpentries-incubator")
+ hpc_carp <- get_org_topics("hpc-carpentry")
+
+ res <- dplyr::bind_rows(carp_inc, hpc_carp) %>%
+ dplyr::select(-private) %>%
+ dplyr::filter(grepl("hpc-carpentry", github_topics)) %>%
+ dplyr::filter(grepl("lesson", github_topics)) %>%
+ extract_tag(
+ life_cycle_tag,
+ LIFE_CYCLE_TAGS,
+ approach = "include",
+ allow_multiple = FALSE,
+ allow_empty = FALSE
+ ) %>%
+ extract_tag(
+ lesson_tags,
+ COMMON_TAGS,
+ approach = "exclude",
+ allow_multiple = TRUE,
+ allow_empty = TRUE
+ )
+
+ ## checks
+ check_repo_info(res, c("description", "rendered_site"))
+
+ res %>%
+ jsonlite::write_json(path = path)
+
+}
+
+make_community_lessons_feed("_data/hpc_lessons.json")
diff --git a/feeds/utils.R b/feeds/utils.R
new file mode 100644
index 000000000..47004cff5
--- /dev/null
+++ b/feeds/utils.R
@@ -0,0 +1,170 @@
+library(gh)
+library(jsonlite)
+library(purrr)
+
+`%<<%` <- function(x, y) {
+ if (identical(length(x), 0L)) return(y)
+ if (is.null(x) || identical(x, "") ||
+ is.na(x)) return(y)
+ x
+}
+
+use_pat <- function() {
+ if (Sys.getenv("GITHUB_PAT") != "" || Sys.getenv("GITHUB_TOKEN") != "") {
+ cli::cli_alert_success("Using GitHub token!")
+ } else {
+ cli::cli_alert_danger("No GitHub token detected.")
+ }
+}
+use_pat()
+
+
+get_list_repos <- function(org, ignore_archived = FALSE,
+ ignore_pattern = NULL, ...) {
+
+ init_res <- gh::gh("GET /orgs/:org/repos", org = org)
+ res <- list()
+ test <- TRUE
+ i <- 1
+
+ while (test) {
+ message("Getting page: ", i, " for ", sQuote(org))
+ res <- append(res, init_res)
+
+ init_res <- tryCatch({
+ gh::gh_next(init_res)
+ },
+ error = function(e) {
+ test <<- FALSE
+ NULL
+ })
+ i <- i+1
+ }
+ # message(sQuote(res))
+ res <- purrr::map_df(res, function(.x) {
+ list(
+ carpentries_org = tolower(.x$owner$login) %<<% "",
+ repo = .x$name,
+ repo_url = .x$html_url,
+ full_name = .x$full_name,
+ description = .x$description %<<% "",
+ rendered_site = .x$homepage %<<% "",
+ private = .x$private,
+ archived = .x$archived
+ )
+ }) %>%
+ dplyr::filter(
+ !private,
+ tolower(carpentries_org) == tolower(org)
+ )
+
+ if (ignore_archived) {
+ res <- res %>%
+ dplyr::filter(!archived)
+ }
+
+ if (!is.null(ignore_pattern)) {
+ res <- res %>%
+ dplyr::filter(!grepl(ignore_pattern, repo, ...))
+ }
+
+ res %>%
+ dplyr::select(-archived)
+}
+
+
+font_color <- function(hexcode) {
+ rgb <- colorspace::hex2RGB(hexcode)
+ rgbR <- rgb@coords[, "R"]
+ rgbG <- rgb@coords[, "G"]
+ rgbB <- rgb@coords[, "B"]
+ luma <- ((0.299 * rgbR) + (0.587 * rgbG) + (0.114 * rgbB))
+ res <- rep("#ffffff", length(hexcode))
+ res[luma > .5] <- "#222222"
+ res
+}
+
+
+get_github_topics <- function(owner, repo) {
+ res <- gh::gh(
+ "GET /repos/:owner/:repo/topics",
+ owner = owner, repo = repo,
+ .send_headers = c("Accept" = "application/vnd.github.mercy-preview+json")
+ )
+
+ purrr::map_chr(res[["names"]], ~ .)
+}
+
+get_org_topics <- function(org) {
+
+ get_list_repos(org) %>%
+ dplyr::filter(
+ !private,
+ carpentries_org == org
+ ) %>%
+ dplyr::mutate(
+ github_topics = purrr::pmap(., function(carpentries_org, repo, ...) {
+ get_github_topics(carpentries_org, repo) %<<% ""
+ })
+ )
+}
+
+
+##' @param data the data frame that contains the column `github_topics` from
+##' which the tags should be extracted
+##' @param new_col_name name of the new column that will contain the extracted
+##' tags
+##' @param dict the dictionary of tags (as a character vector) against which the
+##' content of the `github_topics` column will be compared
+##' @param approach when set to `include` the tag(s) that match(es) the content
+##' of the vector specified in `dict` will be extracted to create the content
+##' of the new column; when set to `exclude` the tag(s) that match(es) the
+##' content of the vector specified in `dict` will be excluded to create the
+##' content of the new column. For instance if the column `github_topics` has
+##' the values `c("tag1", "tag2")` and `dict` is `"tag1"`, the new column will
+##' have the value `tag1` when using `include` and `tag2` when using
+##' `exclude`.
+##' @param allow_multiple can the resulting new column contain more than one
+##' tag?
+##' @param allow_empty can the resulting new column contain an empty value?
+extract_tag <- function(data,
+ new_col_name,
+ dict,
+ approach = c("include", "exclude"),
+ allow_multiple,
+ allow_empty
+ ) {
+
+ approach <- match.arg(approach)
+
+ if (identical(approach, "include")) {
+ f1 <- intersect
+ f2 <- setdiff
+ } else if (identical(approach, "exclude")) {
+ f1 <- setdiff
+ f2 <- intersect
+ } else {
+ stop("invalid value for approach")
+ }
+
+ data %>%
+ dplyr::mutate(!!rlang::quo_name(rlang::enquo(new_col_name)) := purrr::pmap(.,
+ function(github_topics, full_name, ...) {
+ extracted_tag <- f1(github_topics, dict)
+ if ((!allow_multiple) && length(extracted_tag) > 1) {
+ stop("More than one tag detected for: ", full_name)
+ }
+ if (length(extracted_tag) == 0) {
+ if (! allow_empty) {
+ stop("No tag found among (", paste(dict, collapse = ", "), ") ",
+ "for repo: ", full_name)
+ }
+ return("")
+ }
+ extracted_tag
+ })) %>%
+ dplyr::mutate(github_topics = purrr::pmap(., function(github_topics, ...) {
+ f2(github_topics, dict)
+ }))
+
+}
diff --git a/pages/community-lessons.md b/pages/community-lessons.md
index 328bf7dc8..4abf0ea17 100644
--- a/pages/community-lessons.md
+++ b/pages/community-lessons.md
@@ -22,21 +22,27 @@ permalink: "/community-lessons/"
The HPC Carpentry community is committed to a collaborative and open process for
lesson development and to sharing teaching materials. We
-provide two avenues for community members to share lesson
-materials - **The Carpentries Incubator** and **HPC Carpentry Lab**.
+provide three avenues for community members to share lesson
+materials - **The Carpentries Incubator**, **HPC Carpentry** and our
+**HPC Listings**.
[The Carpentries Incubator](#the-carpentries-incubator) is for:
* Collaborative lesson development (from conceptual to stable lessons).
-* Providing visibility for lessons that are being worked on.
+* Providing wider visibility for lessons that are being worked on.
-[HPC Carpentry Lab](#hpc-carpentry-lab) is for:
+[HPC Carpentry](#hpc-carpentry) is for:
* Repository of peer-reviewed, short-format, lessons that use the teaching
approach and lesson design from The Carpentries.
* Getting peer-review on the content of the lesson in the way traditional
journal peer-review wouldn’t be able to provide.
+[HPC Listings](#hpc-listings) is for:
+* Sharing of HPC-relevant, short-format, lessons that use the teaching
+ approach and lesson design from The Carpentries.
+* Collaborative lesson development and peer review.
+
People already familiar with The Carpentries teaching practices can teach
-The Carpentries Incubator or HPC Carpentry Lab lessons in meetups, in classes,
+The Carpentries Incubator or HPC Carpentry lessons in meetups, in classes,
or as complements to a "standard" 2-day Carpentries workshop.
These lessons can also be used by independent learners, outside of workshops.
@@ -44,47 +50,66 @@ These lessons can also be used by independent learners, outside of workshops.
The Carpentries Incubator is a place to share Carpentries-style teaching
materials at all stages of development, to collaborate on lesson development,
-and receive feedback from other community members.
+and receive feedback from other Carpentries community members.
Lessons in The Carpentries Incubator are developed and supported by community
-members and are not officially endorsed by The Carpentries or HPC Carpentry. We
+members and are not officially endorsed by The Carpentries (or HPC Carpentry
+unless they are part of our curriculum). We
encourage you to browse the Incubator lessons for materials that meet your
needs and to use these materials freely (all lessons should be licensed
[CC-BY 4.0](https://creativecommons.org/licenses/by/4.0/)).
-If you are interested in developing or submitting a lesson to the HPC Carpentry
+If you are interested in developing or submitting a lesson to the Carpentries
Incubator, [contact us]({{ site.contact }}).
Please read the information in The Carpentries' [Development of Lessons page](
https://carpentries.org/involved-lessons/) if you would like to contribute to
the development of a lesson already present in The Carpentries Incubator.
-## Lessons in The Carpentries Incubator
+### _Lessons in The Carpentries Incubator_:
+
+
-{% assign lesson_list = site.data.community_lessons | where: "hpc_carpentry_org","the-carpentries-incubator" %}
+{% assign lesson_list = site.data.hpc_lessons | where: "carpentries_org","carpentries-incubator" %}
{% include lesson_table %}
-## HPC Carpentry Lab
+## HPC Carpentry
-The HPC Carpentry Lab is a place for sharing high-quality, peer-reviewed lessons
+The [HPC Carpentry GitHub organisation](https://github.com/hpc-carpentry) is a
+place for sharing HPC-oriented, high-quality, peer-reviewed lessons
that follow best practices in pedagogy and the general teaching practices used in
Carpentries workshops.
-Lessons in HPC Carpentry Lab have been peer-reviewed.
-We encourage you to browse the Lab lessons for materials that meet your needs and
+Lessons in HPC Carpentry have been peer-reviewed and each lesson includes
+an indication of the level of maturity of the content.
+We encourage you to browse the lessons for materials that meet your needs and
to use these materials freely (all lessons are
licensed [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/)).
-
@@ -99,13 +124,13 @@ submit it first to the Incubator.
## List of Community Developed Lessons by Topic
-{% assign lesson_topics = site.data.community_lessons | map: "lesson_tags" | uniq | sort%}
+{% assign lesson_topics = site.data.hpc_lessons | map: "lesson_tags" | uniq | sort%}
{% for t in lesson_topics %}
### {{ t | capitalize | replace: "-", " "}}
-{% assign lesson_with_tag = site.data.community_lessons | where_exp: "item", "item.lesson_tags contains t" %}
+{% assign lesson_with_tag = site.data.hpc_lessons | where_exp: "item", "item.lesson_tags contains t" %}
{% for l in lesson_with_tag %}
- [{{l.description}}](#{{ l.description | slugify }}) {{l.carpentries_org}}
diff --git a/pages/help-wanted-issues.md b/pages/help-wanted-issues.md
index 76313ad82..c0f4c71b1 100644
--- a/pages/help-wanted-issues.md
+++ b/pages/help-wanted-issues.md
@@ -20,18 +20,27 @@ permalink: "/help-wanted-issues/"
Information for Lesson Maintainers
-{% assign help_wanted = site.data.help_wanted_issues | where: "org_name", "The Carpentries Incubator" %}
+{% assign help_wanted = site.data.help_wanted_issues %}
-{% assign hpc_repos = site.data.lessons %}
+{% comment %}
+This was a way to get all the organizations automatically, but probably better to curate ordering
{% assign orgs = help_wanted | map: "org_name" | uniq %}
+{% endcomment %}
-{% for each_repo in hpc_repos %}
+{% assign orgs = "The Carpentries Incubator, HPC Carpentry" | split: ", " %}
-{% assign org_repos = help_wanted | where: "repo", each_repo.repo %}
+
+{% for each_org in orgs %}
+
+{% assign org_repos = help_wanted | where: "org_name", each_org %}
{% assign grouped_org_repos = org_repos | group_by: "full_repo" %}
+{% if org_repos.size > 0 %}
+
+
{{ each_org }}
+
{% for r in grouped_org_repos %}
{% assign repo_desc = r.items[0].clean_description %}
@@ -54,7 +63,7 @@ permalink: "/help-wanted-issues/"
{{ i.title}}
-{% for l in labels %}{{ l }}{% endfor %}
+{% for l in labels %}{{ l }} {% endfor %}