From 4ee0f745c45e26a82354b5ee9952d8f873451b1e Mon Sep 17 00:00:00 2001 From: Mick McGrath Date: Wed, 6 Apr 2022 10:02:31 -0400 Subject: [PATCH 1/9] add logic to handle aws token for sso accounts --- .circleci/config.yml | 3 ++- scripts/aws/setup.sh | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5a6dad4d..dacd12c1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,7 @@ workflows: filters: branches: only: - - master + - handle-aws-sso tags: only: - "/.*/" @@ -45,6 +45,7 @@ jobs: export AWS_SECRET_ACCESS_KEY="${BITOPS_AWS_SECRET_ACCESS_KEY}" # publish setup + export BITOPS_DOCKER_IMAGE_PUBLISH_TAG="handle-aws-sso" export BITOPS_DOCKER_IMAGE_PUBLISH_SKIP_SHA="1" export BITOPS_DOCKER_IMAGE_NAME="${CIRCLE_PROJECT_REPONAME}" # bitops export BITOPS_GIT_TAG="${CIRCLE_TAG}" diff --git a/scripts/aws/setup.sh b/scripts/aws/setup.sh index 689b18a1..d409db12 100755 --- a/scripts/aws/setup.sh +++ b/scripts/aws/setup.sh @@ -12,6 +12,10 @@ aws_access_key_id = "${AWS_ACCESS_KEY_ID}" aws_secret_access_key = "${AWS_SECRET_ACCESS_KEY}" EOF +if [ -n "$AWS_SESSION_TOKEN" ]; then + echo "aws_session_token = $AWS_SESSION_TOKEN" >> /root/.aws/credentials +fi + cat < /root/.aws/config [default] region = "$AWS_DEFAULT_REGION" From fd3ff5b0b7b8092a9e2e7f74ada3b180f060f671 Mon Sep 17 00:00:00 2001 From: Mick McGrath Date: Wed, 6 Apr 2022 10:14:17 -0400 Subject: [PATCH 2/9] add docs for session token --- docs/cloud-configuration/configuration-aws.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/cloud-configuration/configuration-aws.md b/docs/cloud-configuration/configuration-aws.md index 692faf23..f3bc1e0d 100644 --- a/docs/cloud-configuration/configuration-aws.md +++ b/docs/cloud-configuration/configuration-aws.md @@ -22,6 +22,15 @@ AWS Access. See [AWS official documentation](https://docs.aws.amazon.com/general AWS Access. See [AWS official documentation](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) +------------------- +### aws_session_token +* **BitOps Property:** [TODO](https://github.com/bitovi/bitops/issues/15) +* **Environment Variable:** `AWS_SESSION_TOKEN` +* **default:** `""` +* **required:** no + +AWS Access. See [AWS official documentation](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) + ------------------- ### aws_default_region * **BitOps Property:** [TODO](https://github.com/bitovi/bitops/issues/15) From 6a9f82210e7ee2e668aead0ee1fe5e5f5305776f Mon Sep 17 00:00:00 2001 From: Mick McGrath Date: Wed, 6 Apr 2022 10:18:16 -0400 Subject: [PATCH 3/9] test pipeline runner config --- .circleci/config.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dacd12c1..2171786b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,9 +8,6 @@ workflows: branches: only: - handle-aws-sso - tags: - only: - - "/.*/" docs: jobs: - deploy-docs: From 091e19839d42b91777d9a18fc5d4908672c695f4 Mon Sep 17 00:00:00 2001 From: Mick McGrath Date: Wed, 6 Apr 2022 10:48:25 -0400 Subject: [PATCH 4/9] add github actions for build and publish --- .circleci/config.yml | 47 ----------- .github/workflows/build-and-publish.yml | 29 +++++++ scripts/ci/publish-circleci.sh | 67 ++++++++++++++++ scripts/ci/publish.sh | 102 ++++++++++++++---------- 4 files changed, 155 insertions(+), 90 deletions(-) create mode 100644 .github/workflows/build-and-publish.yml create mode 100755 scripts/ci/publish-circleci.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 2171786b..e39b8b62 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,13 +1,5 @@ version: 2.1 workflows: - bitops: - jobs: - - build: - context: bitops - filters: - branches: - only: - - handle-aws-sso docs: jobs: - deploy-docs: @@ -15,45 +7,6 @@ workflows: branches: only: - master -jobs: - build: - machine: - image: ubuntu-1604:201903-01 - steps: - - checkout - - run: - name: Install dependencies - command: | - ./scripts/ci/install.sh - - run: - name: Run tests - command: | - ./scripts/ci/test.sh - - deploy: - name: Publish docker image - command: | - ### - ### SETUP ENV VARS - ### - - # registry setup - export REGISTRY_URL="bitovi/bitops" - export AWS_ACCESS_KEY_ID="${BITOPS_AWS_ACCESS_KEY_ID}" - export AWS_SECRET_ACCESS_KEY="${BITOPS_AWS_SECRET_ACCESS_KEY}" - - # publish setup - export BITOPS_DOCKER_IMAGE_PUBLISH_TAG="handle-aws-sso" - export BITOPS_DOCKER_IMAGE_PUBLISH_SKIP_SHA="1" - export BITOPS_DOCKER_IMAGE_NAME="${CIRCLE_PROJECT_REPONAME}" # bitops - export BITOPS_GIT_TAG="${CIRCLE_TAG}" - export BITOPS_GIT_SHA="${CIRCLE_SHA1}" - export BITOPS_GIT_BRANCH="${CIRCLE_BRANCH}" - export BITOPS_GIT_BASE_BRANCH="master" - - ### - ### PUBLISH - ### - ./scripts/ci/publish.sh deploy-docs: docker: diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml new file mode 100644 index 00000000..b2a5d126 --- /dev/null +++ b/.github/workflows/build-and-publish.yml @@ -0,0 +1,29 @@ +name: Build and Publish + +on: + push: + branches: [ handle-aws-sso ] + # branches: [ master ] + tags: + - "*" + paths-ignore: + - "docs/**" + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + # - run: | + # ./scripts/ci/install.sh + # - run: | + # ./scripts/ci/test.sh + - name: Publish Docker + env: + REGISTRY_URL: "bitovi/bitops" + DEFAULT_BRANCH: "master" + DOCKER_USER: ${{ secrets.DOCKER_USER}} + DOCKER_PASS: ${{ secrets.DOCKER_PASS}} + run: | + echo "running scripts/ci/publish.sh" + ./scripts/ci/publish.sh diff --git a/scripts/ci/publish-circleci.sh b/scripts/ci/publish-circleci.sh new file mode 100755 index 00000000..29524b6b --- /dev/null +++ b/scripts/ci/publish-circleci.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +set -xe + + +#### +#### validation +#### +if [ -z "$REGISTRY_URL" ]; then + >&2 echo "{\"script\":\"scripts/ci/publish.sh\", \"error\":\"REGISTRY_URL required\"}" + exit 1 +fi + + + + +#### +#### docker login +#### +if [ -n "$BITOPS_PUBLISH_ECR" ]; then + ./scripts/ci/docker-login-ecr.sh + +else + echo "$DOCKER_PASS" | docker login --username="$DOCKER_USER" --password-stdin + echo "logged into dockerhub registry" +fi + +#### +#### docker build +#### +./scripts/ci/docker-build.sh + + +#### +#### set up tagging +#### + + +# allow custom branching +if [ -n "$BITOPS_DOCKER_IMAGE_PUBLISH_TAG" ]; then + echo "{\"script\":\"scripts/ci/publish.sh\", \"tag\": \"${BITOPS_DOCKER_IMAGE_PUBLISH_TAG}\"}" + docker tag ${BITOPS_DOCKER_IMAGE_NAME}:latest ${REGISTRY_URL}:${BITOPS_DOCKER_IMAGE_PUBLISH_TAG} +else + # handle git tag + if [ -n "$BITOPS_GIT_TAG" ]; then + echo "{\"script\":\"scripts/ci/publish.sh\", \"tag\": \"${BITOPS_GIT_TAG}\"}" + docker tag ${BITOPS_DOCKER_IMAGE_NAME}:latest ${REGISTRY_URL}:${BITOPS_GIT_TAG} + + # if master, tag latest + elif [ "$BITOPS_GIT_BRANCH" == "$BITOPS_GIT_BASE_BRANCH" ]; then + echo "{\"script\":\"scripts/ci/publish.sh\", \"tag\": \"${latest}\"}" + docker tag ${BITOPS_DOCKER_IMAGE_NAME}:latest ${REGISTRY_URL}:latest + + + # fall back to the sha + elif [ -z "$BITOPS_DOCKER_IMAGE_PUBLISH_SKIP_SHA" ]; then + echo "{\"script\":\"scripts/ci/publish.sh\", \"tag\": \"${BITOPS_GIT_SHA}\"}" + docker tag ${BITOPS_DOCKER_IMAGE_NAME}:latest ${REGISTRY_URL}:${BITOPS_GIT_SHA} + + # don't tag anything + else + echo "{\"script\":\"scripts/ci/publish.sh\", \"tag\": \"\"}" + fi +fi + + +# push everything +docker push ${REGISTRY_URL} diff --git a/scripts/ci/publish.sh b/scripts/ci/publish.sh index 29524b6b..71de637c 100755 --- a/scripts/ci/publish.sh +++ b/scripts/ci/publish.sh @@ -1,5 +1,6 @@ -#!/usr/bin/env bash -set -xe +#!/bin/bash + +set -e #### @@ -10,58 +11,73 @@ if [ -z "$REGISTRY_URL" ]; then exit 1 fi - - - #### #### docker login #### -if [ -n "$BITOPS_PUBLISH_ECR" ]; then - ./scripts/ci/docker-login-ecr.sh +echo "$DOCKER_PASS" | docker login --username="$DOCKER_USER" --password-stdin +echo "logged into dockerhub registry" -else - echo "$DOCKER_PASS" | docker login --username="$DOCKER_USER" --password-stdin - echo "logged into dockerhub registry" -fi -#### -#### docker build -#### -./scripts/ci/docker-build.sh +### +### PUBLISH - environment setup +### +#Defining the Default branch variable +if [ -z "$DEFAULT_BRANCH" ]; then + DEFAULT_BRANCH="main" +fi -#### -#### set up tagging -#### + +REPO_NAME=$(echo $GITHUB_REPOSITORY | sed 's/^.*\///') +ORG_NAME=$(echo $GITHUB_REPOSITORY | sed 's/\/.*//') +TAG_OR_HEAD="$(echo $GITHUB_REF | cut -d / -f2)" +BRANCH_OR_TAG_NAME=$(echo $GITHUB_REF | cut -d / -f3) +echo "REPO_NAME: $REPO_NAME" +echo "ORG_NAME: $ORG_NAME" +echo "TAG_OR_HEAD: $TAG_OR_HEAD" +echo "BRANCH_OR_TAG_NAME: $BRANCH_OR_TAG_NAME" -# allow custom branching -if [ -n "$BITOPS_DOCKER_IMAGE_PUBLISH_TAG" ]; then - echo "{\"script\":\"scripts/ci/publish.sh\", \"tag\": \"${BITOPS_DOCKER_IMAGE_PUBLISH_TAG}\"}" - docker tag ${BITOPS_DOCKER_IMAGE_NAME}:latest ${REGISTRY_URL}:${BITOPS_DOCKER_IMAGE_PUBLISH_TAG} -else - # handle git tag - if [ -n "$BITOPS_GIT_TAG" ]; then - echo "{\"script\":\"scripts/ci/publish.sh\", \"tag\": \"${BITOPS_GIT_TAG}\"}" - docker tag ${BITOPS_DOCKER_IMAGE_NAME}:latest ${REGISTRY_URL}:${BITOPS_GIT_TAG} - - # if master, tag latest - elif [ "$BITOPS_GIT_BRANCH" == "$BITOPS_GIT_BASE_BRANCH" ]; then - echo "{\"script\":\"scripts/ci/publish.sh\", \"tag\": \"${latest}\"}" - docker tag ${BITOPS_DOCKER_IMAGE_NAME}:latest ${REGISTRY_URL}:latest - - - # fall back to the sha - elif [ -z "$BITOPS_DOCKER_IMAGE_PUBLISH_SKIP_SHA" ]; then - echo "{\"script\":\"scripts/ci/publish.sh\", \"tag\": \"${BITOPS_GIT_SHA}\"}" - docker tag ${BITOPS_DOCKER_IMAGE_NAME}:latest ${REGISTRY_URL}:${BITOPS_GIT_SHA} - - # don't tag anything +# if tag, use tag +# if default branch, use `latest` +# if otherwise, use branch name +if [ -z "$IMAGE_TAG" ]; then + if [ -n "$USE_COMMIT_HASH_FOR_ARTIFACTS" ]; then + IMAGE_TAG="$GITHUB_SHA" else - echo "{\"script\":\"scripts/ci/publish.sh\", \"tag\": \"\"}" + if [ "$TAG_OR_HEAD" == "tags" ]; then + IMAGE_TAG="$BRANCH_OR_TAG_NAME" + elif [ "$TAG_OR_HEAD" == "heads" ] && [ "$BRANCH_OR_TAG_NAME" == "$DEFAULT_BRANCH" ]; then + IMAGE_TAG="latest" + elif [ "$TAG_OR_HEAD" == "pull" ]; then + IMAGE_TAG="pr-${BRANCH_OR_TAG_NAME}" + else + IMAGE_TAG="$BRANCH_OR_TAG_NAME" + fi fi fi -# push everything -docker push ${REGISTRY_URL} +### +### PUBLISH DOCKER +### +echo "###" +echo "### PUBLISH DOCKER" +echo "###" + +#Defining the Image name variable +IMAGE_NAME="$REPO_NAME" + + + +#Building the docker image... +echo "Building the docker image" +docker build -t ${IMAGE_NAME} . + +#docker image deploy function +aws ecr get-login-password --region ${AWS_DEFAULT_REGION} | docker login --username AWS --password-stdin ${REGISTRY_URL} +echo "docker tag ${IMAGE_NAME} ${REGISTRY_URL}:${IMAGE_TAG}" +docker tag ${IMAGE_NAME} ${REGISTRY_URL}:${IMAGE_TAG} + +echo "Pushing the docker image to the ecr repository..." +docker push ${REGISTRY_URL}:${IMAGE_TAG} \ No newline at end of file From 7719b8fe4c15b04e3faf7b92c779bfcf1b92679e Mon Sep 17 00:00:00 2001 From: Mick McGrath Date: Wed, 6 Apr 2022 10:51:30 -0400 Subject: [PATCH 5/9] remove aws cli from docker login --- scripts/ci/publish.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/ci/publish.sh b/scripts/ci/publish.sh index 71de637c..ab337341 100755 --- a/scripts/ci/publish.sh +++ b/scripts/ci/publish.sh @@ -75,7 +75,6 @@ echo "Building the docker image" docker build -t ${IMAGE_NAME} . #docker image deploy function -aws ecr get-login-password --region ${AWS_DEFAULT_REGION} | docker login --username AWS --password-stdin ${REGISTRY_URL} echo "docker tag ${IMAGE_NAME} ${REGISTRY_URL}:${IMAGE_TAG}" docker tag ${IMAGE_NAME} ${REGISTRY_URL}:${IMAGE_TAG} From 3a42462f1211d62c4ed2f2982caf956f857e1d18 Mon Sep 17 00:00:00 2001 From: Mick McGrath Date: Wed, 6 Apr 2022 11:01:39 -0400 Subject: [PATCH 6/9] add docs pipeline for GHA and prep for master merge --- .circleci/config.yml | 22 ---------------------- .github/workflows/build-and-publish.yml | 3 +-- .github/workflows/publish-docs.yml | 24 ++++++++++++++++++++++++ 3 files changed, 25 insertions(+), 24 deletions(-) delete mode 100644 .circleci/config.yml create mode 100644 .github/workflows/publish-docs.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index e39b8b62..00000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,22 +0,0 @@ -version: 2.1 -workflows: - docs: - jobs: - - deploy-docs: - filters: - branches: - only: - - master - - deploy-docs: - docker: - - image: cimg/python:3.7 - steps: - - add_ssh_keys: - fingerprints: - - "0a:83:f3:84:67:25:97:33:af:42:38:77:dc:29:60:8f" - - checkout - - run: pip install mkdocs mkdocs-material - - deploy: - name: Deploy docs - command: mkdocs gh-deploy -c -m "[ci skip]" diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index b2a5d126..74e9febd 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -2,8 +2,7 @@ name: Build and Publish on: push: - branches: [ handle-aws-sso ] - # branches: [ master ] + branches: [ master ] tags: - "*" paths-ignore: diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml new file mode 100644 index 00000000..ce4b77dc --- /dev/null +++ b/.github/workflows/publish-docs.yml @@ -0,0 +1,24 @@ +name: Publish Docs + +on: + push: + branches: [ master ] + tags: + - "*" + paths: + - "docs/**" + +jobs: + publish: + runs-on: cimg/python:3.7 + steps: + - uses: actions/checkout@v2 + + # TODO: this was from circleci - do we need it for GHA? + # - add_ssh_keys: + # fingerprints: + # - "0a:83:f3:84:67:25:97:33:af:42:38:77:dc:29:60:8f" + - name: Publish Docs + run: | + pip install mkdocs mkdocs-material + mkdocs gh-deploy -c -m "[ci skip]" \ No newline at end of file From 1bd3293f4a8e7be52edd9575899469d01693ee37 Mon Sep 17 00:00:00 2001 From: Mick McGrath Date: Wed, 6 Apr 2022 11:09:49 -0400 Subject: [PATCH 7/9] remove extra circleci script --- .github/workflows/build-and-publish.yml | 2 +- scripts/ci/publish-circleci.sh | 67 ------------------------- 2 files changed, 1 insertion(+), 68 deletions(-) delete mode 100755 scripts/ci/publish-circleci.sh diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 74e9febd..37414c73 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -13,9 +13,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + # TODO: when we have tests # - run: | # ./scripts/ci/install.sh - # - run: | # ./scripts/ci/test.sh - name: Publish Docker env: diff --git a/scripts/ci/publish-circleci.sh b/scripts/ci/publish-circleci.sh deleted file mode 100755 index 29524b6b..00000000 --- a/scripts/ci/publish-circleci.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env bash -set -xe - - -#### -#### validation -#### -if [ -z "$REGISTRY_URL" ]; then - >&2 echo "{\"script\":\"scripts/ci/publish.sh\", \"error\":\"REGISTRY_URL required\"}" - exit 1 -fi - - - - -#### -#### docker login -#### -if [ -n "$BITOPS_PUBLISH_ECR" ]; then - ./scripts/ci/docker-login-ecr.sh - -else - echo "$DOCKER_PASS" | docker login --username="$DOCKER_USER" --password-stdin - echo "logged into dockerhub registry" -fi - -#### -#### docker build -#### -./scripts/ci/docker-build.sh - - -#### -#### set up tagging -#### - - -# allow custom branching -if [ -n "$BITOPS_DOCKER_IMAGE_PUBLISH_TAG" ]; then - echo "{\"script\":\"scripts/ci/publish.sh\", \"tag\": \"${BITOPS_DOCKER_IMAGE_PUBLISH_TAG}\"}" - docker tag ${BITOPS_DOCKER_IMAGE_NAME}:latest ${REGISTRY_URL}:${BITOPS_DOCKER_IMAGE_PUBLISH_TAG} -else - # handle git tag - if [ -n "$BITOPS_GIT_TAG" ]; then - echo "{\"script\":\"scripts/ci/publish.sh\", \"tag\": \"${BITOPS_GIT_TAG}\"}" - docker tag ${BITOPS_DOCKER_IMAGE_NAME}:latest ${REGISTRY_URL}:${BITOPS_GIT_TAG} - - # if master, tag latest - elif [ "$BITOPS_GIT_BRANCH" == "$BITOPS_GIT_BASE_BRANCH" ]; then - echo "{\"script\":\"scripts/ci/publish.sh\", \"tag\": \"${latest}\"}" - docker tag ${BITOPS_DOCKER_IMAGE_NAME}:latest ${REGISTRY_URL}:latest - - - # fall back to the sha - elif [ -z "$BITOPS_DOCKER_IMAGE_PUBLISH_SKIP_SHA" ]; then - echo "{\"script\":\"scripts/ci/publish.sh\", \"tag\": \"${BITOPS_GIT_SHA}\"}" - docker tag ${BITOPS_DOCKER_IMAGE_NAME}:latest ${REGISTRY_URL}:${BITOPS_GIT_SHA} - - # don't tag anything - else - echo "{\"script\":\"scripts/ci/publish.sh\", \"tag\": \"\"}" - fi -fi - - -# push everything -docker push ${REGISTRY_URL} From b89b7d1508a9598d5ba77263257e96003fe2f605 Mon Sep 17 00:00:00 2001 From: Mick McGrath Date: Wed, 6 Apr 2022 11:38:10 -0400 Subject: [PATCH 8/9] more meaningful docker publish step name --- .github/workflows/build-and-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 37414c73..b89fb0ac 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -17,7 +17,7 @@ jobs: # - run: | # ./scripts/ci/install.sh # ./scripts/ci/test.sh - - name: Publish Docker + - name: Publish Docker Image env: REGISTRY_URL: "bitovi/bitops" DEFAULT_BRANCH: "master" From 6e710684e0b1a219bcb0fcf314564c05ca45a9ab Mon Sep 17 00:00:00 2001 From: Mick McGrath Date: Wed, 6 Apr 2022 13:11:16 -0400 Subject: [PATCH 9/9] watch mkdocs.yml for docs pipeline --- .github/workflows/publish-docs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml index ce4b77dc..8a6065e1 100644 --- a/.github/workflows/publish-docs.yml +++ b/.github/workflows/publish-docs.yml @@ -7,6 +7,7 @@ on: - "*" paths: - "docs/**" + - "mkdocs.yml" jobs: publish: