Merge pull request #1132 from parente/github-actions

Switch to GitHub Actions for CI/CD

.github/workflows/docker.yml

@@ -0,0 +1,62 @@
+name: Build Docker Images
+
+on:
+  push:
+    paths-ignore:
+      - "*.md"
+      - "binder/**"
+      - "docs/**"
+      - "examples/**"
+
+jobs:
+  build:
+    name: Build Docker Images
+    runs-on: ubuntu-latest
+    if: "!contains(github.event.head_commit.message , 'ci skip')"
+    steps:
+      - name: Clone Main Repo
+        uses: actions/checkout@v2
+        with:
+          path: main
+      - name: Clone Wiki
+        uses: actions/checkout@v2
+        with:
+          repository: ${{github.repository}}.wiki
+          path: wiki
+      - name: Set Up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Install Dev Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          make -C main dev-env lint-install
+      - name: Lint Dockerfiles
+        run: make -C main lint-all
+      - name: Build Docker Images
+        run: make -C main build-test-all
+      - name: Run Post-Build Hooks
+        run: make -C main hook-all
+        env:
+          COMMIT_MSG: "${{github.event.head_commit.message}}"
+          WIKI_PATH: ../wiki
+
+  push:
+    name: Push Docker Images
+    runs-on: ubuntu-latest
+    needs:
+      - build
+    if: github.ref == 'refs/heads/master'
+    steps:
+      - name: Login to Docker Hub
+        run: >
+          echo '${{secrets.DOCKERHUB_PASSWORD}}' | docker login --username
+          '${{secrets.DOCKERHUB_USERNAME}}'' --password-stdin
+      - name: Push Images to DockerHub
+        run: make -C main push-all
+      - name: Push Wiki to GitHub
+        run: make -C main git-commit
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_REPOSITORY: ${{github.repository}}.wiki
+          LOCAL_PATH: ../wiki

.github/workflows/sphinx.yml

@@ -0,0 +1,45 @@
+name: Build Sphinx Documentation
+
+on:
+  push:
+    paths:
+      - "docs/**"
+      - ".github/workflows/sphinx.yml"
+
+jobs:
+  build:
+    name: Build Sphinx Documentation
+    runs-on: ubuntu-latest
+    if: "!contains(github.event.head_commit.message , 'ci skip')"
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v2
+      - name: Set Up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.x
+      - name: Install Dev Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          make dev-env
+      - name: Build Documentation
+        run: make docs
+
+  gettext:
+    name: Update Translation Source Strings
+    runs-on: ubuntu-latest
+    needs:
+      - build
+    if: github.ref == 'refs/heads/master'
+    steps:
+      - name: Extract Source Strings
+        working-directory: docs
+        run: |
+          make gettext
+          sphinx-intl update -p _build/gettext -l en
+      - name: Push Strings to Master
+        run: make git-commit
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_REPOSITORY: ${{ github.repository }}
+          LOCAL_PATH: ./docs/locale/en

Makefile

@@ -4,9 +4,8 @@
 
 # Use bash for inline if-statements in arch_patch target
 SHELL:=bash
-OWNER:=jupyter
 ARCH:=$(shell uname -m)
-DIFF_RANGE?=master...HEAD
+OWNER?=jupyter
 
 # Need to list the images in build dependency order
 ifeq ($(ARCH),ppc64le)
@@ -76,22 +75,34 @@ dev/%: ## run a foreground container for a stack
 dev-env: ## install libraries required to build docs and run tests
 	pip install -r requirements-dev.txt
 
-lint/%: ARGS?=
-lint/%: ## lint the dockerfile(s) for a stack
-	@echo "Linting Dockerfiles in $(notdir $@)..."
-	@git ls-files --exclude='Dockerfile*' --ignored $(notdir $@) | grep -v ppc64 | xargs -L 1 $(HADOLINT) $(ARGS)
-	@echo "Linting done!"
-
-lint-all: $(foreach I,$(ALL_IMAGES),lint/$(I) ) ## lint all stacks
+docs: ## build HTML documentation
+	make -C docs html
 
-lint-build-test-all: $(foreach I,$(ALL_IMAGES),lint/$(I) arch_patch/$(I) build/$(I) test/$(I) ) ## lint, build and test all stacks
+git-commit: LOCAL_PATH?=.
+git-commit: export GITHUB_SHA?=$(shell git rev-parse HEAD)
+git-commit: GITHUB_REPOSITORY?=jupyter/docker-stacks
+git-commit: ## commit outstading git changes and push to remote
+	@git config --global user.name "GitHub Actions"
+	@git config --global user.email "actions@users.noreply.github.com"
+	@git remote add publisher https://$${GITHUB_TOKEN}@github.com/$${GITHUB_REPOSITORY}.git
 
-lint-install: ## install hadolint
-	@echo "Installing hadolint at $(HADOLINT) ..."
-	@curl -sL -o $(HADOLINT) "https://github.com/hadolint/hadolint/releases/download/v1.18.0/hadolint-$(shell uname -s)-$(shell uname -m)"
-	@chmod 700 $(HADOLINT)
-	@echo "Installation done!"
-	@$(HADOLINT) --version	
+	@cd $(LOCAL_PATH)
+	@git checkout master
+	@git add -A -- .
+	@git commit -m "[ci skip] Automated publish for $${GITHUB_SHA}" || exit 0
+	@git push -u publisher master
+
+hook/%: export COMMIT_MSG?=$(shell git log -1 --pretty=%B)
+hook/%: export GITHUB_SHA?=$(shell git rev-parse HEAD)
+hook/%: export WIKI_PATH?=../wiki
+hook/%: ## run post-build hooks for an image
+	BUILD_TIMESTAMP="$$(date -u +%FT%TZ)" \
+	DOCKER_REPO="$(OWNER)/$(notdir $@)" \
+	IMAGE_NAME="$(OWNER)/$(notdir $@):latest" \
+	IMAGE_SHORT_NAME="$(notdir $@)" \
+	$(SHELL) $(notdir $@)/hooks/run_hook
+
+hook-all: $(foreach I,$(ALL_IMAGES),hook/$(I) ) ## run post-build hooks for all images
 
 img-clean: img-rm-dang img-rm ## clean dangling and jupyter images
 
@@ -107,20 +118,33 @@ img-rm-dang: ## remove dangling images (tagged None)
 	@echo "Removing dangling images ..."
 	-docker rmi --force $(shell docker images -f "dangling=true" -q) 2> /dev/null
 
-docs: ## build HTML documentation
-	make -C docs html
+lint/%: ARGS?=
+lint/%: ## lint the dockerfile(s) for a stack
+	@echo "Linting Dockerfiles in $(notdir $@)..."
+	@git ls-files --exclude='Dockerfile*' --ignored $(notdir $@) | grep -v ppc64 | xargs -L 1 $(HADOLINT) $(ARGS)
+	@echo "Linting done!"
 
-n-docs-diff: ## number of docs/ files changed since branch from master
-	@git diff --name-only $(DIFF_RANGE) -- docs/ ':!docs/locale' | wc -l | awk '{print $$1}'
+lint-all: $(foreach I,$(ALL_IMAGES),lint/$(I) ) ## lint all stacks
 
+lint-build-test-all: $(foreach I,$(ALL_IMAGES),lint/$(I) arch_patch/$(I) build/$(I) test/$(I) ) ## lint, build and test all stacks
 
-n-other-diff: ## number of files outside docs/ changed since branch from master
-	@git diff --name-only $(DIFF_RANGE) -- ':!docs/' | wc -l | awk '{print $$1}'
+lint-install: ## install hadolint
+	@echo "Installing hadolint at $(HADOLINT) ..."
+	@curl -sL -o $(HADOLINT) "https://github.com/hadolint/hadolint/releases/download/v1.18.0/hadolint-$(shell uname -s)-$(shell uname -m)"
+	@chmod 700 $(HADOLINT)
+	@echo "Installation done!"
+	@$(HADOLINT) --version	
 
 pull/%: DARGS?=
 pull/%: ## pull a jupyter image
 	docker pull $(DARGS) $(OWNER)/$(notdir $@)
 
+push/%: DARGS?=
+push/%: ## push all tags for a jupyter image
+	docker push $(DARGS) $(OWNER)/$(notdir $@)
+
+push-all: $(foreach I,$(ALL_IMAGES),push/$(I) ) ## push all tagged images
+
 run/%: DARGS?=
 run/%: ## run a bash in interactive mode in a stack
 	docker run -it --rm $(DARGS) $(OWNER)/$(notdir $@) $(SHELL)
@@ -129,20 +153,6 @@ run-sudo/%: DARGS?=
 run-sudo/%: ## run a bash in interactive mode as root in a stack
 	docker run -it --rm -u root $(DARGS) $(OWNER)/$(notdir $@) $(SHELL)
 
-tx-en: ## rebuild en locale strings and push to master (req: GH_TOKEN)
-	@git config --global user.email "travis@travis-ci.org"
-	@git config --global user.name "Travis CI"
-	@git checkout master
-
-	@make -C docs clean gettext
-	@cd docs && sphinx-intl update -p _build/gettext -l en
-
-	@git add docs/locale/en
-	@git commit -m "[ci skip] Update en source strings (build: $$TRAVIS_JOB_NUMBER)"
-
-	@git remote add origin-tx https://$${GH_TOKEN}@github.com/jupyter/docker-stacks.git
-	@git push -u origin-tx master
-
 test/%: ## run tests against a stack (only common tests or common tests + specific tests)
 	@if [ ! -d "$(notdir $@)/test" ]; then TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest -m "not info" test; \
 	else TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest -m "not info" test $(notdir $@)/test; fi

all-spark-notebook/hooks/manifest.tmpl → all-spark-notebook/hooks/run_hook

@@ -1,9 +1,23 @@
+#!/bin/bash
+set -e
+
+# Apply tags
+GIT_SHA_TAG=${GITHUB_SHA:0:12}
+docker tag $IMAGE_NAME "$DOCKER_REPO:$GIT_SHA_TAG"
+
+# Update index
+INDEX_ROW="|\`${BUILD_TIMESTAMP}\`|\`jupyter/${IMAGE_SHORT_NAME}:${GIT_SHA_TAG}\`|[Git diff](https://github.com/jupyter/docker-stacks/commit/${GITHUB_SHA})<br />[Dockerfile](https://github.com/jupyter/docker-stacks/blob/${GITHUB_SHA}/${IMAGE_SHORT_NAME}/Dockerfile)<br />[Build manifest](./${IMAGE_SHORT_NAME}-${GIT_SHA_TAG})|"
+sed "/|-|/a ${INDEX_ROW}" -i "${WIKI_PATH}/Home.md"
+
+# Build manifest
+MANIFEST_FILE="${WIKI_PATH}/manifests/${IMAGE_SHORT_NAME}-${GIT_SHA_TAG}.md"
+mkdir -p $(dirname "$MANIFEST_FILE")
+
 cat << EOF > "$MANIFEST_FILE"
 * Build datetime: ${BUILD_TIMESTAMP}
-* DockerHub build code: ${BUILD_CODE}
 * Docker image: ${DOCKER_REPO}:${GIT_SHA_TAG}
 * Docker image size: $(docker images ${IMAGE_NAME} --format "{{.Size}}")
-* Git commit SHA: [${SOURCE_COMMIT}](https://github.com/jupyter/docker-stacks/commit/${SOURCE_COMMIT})
+* Git commit SHA: [${GITHUB_SHA}](https://github.com/jupyter/docker-stacks/commit/${GITHUB_SHA})
 * Git commit message:
 \`\`\`
 ${COMMIT_MSG}