diff --git a/.circleci/_set_up_deploy_envs.sh b/.circleci/_set_up_deploy_envs.sh deleted file mode 100755 index 639e46a3..00000000 --- a/.circleci/_set_up_deploy_envs.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash -set -o nounset - -if [[ ${CIRCLE_TAG} =~ v[0-9]+(\.[0-9]+)*(\-snapshot) ]]; then - echo "Setting snap shot env vars for ${CIRCLE_TAG}" - - echo 'export RELEASE_TAG=$RELEASE_TAG_SNAPSHOT' >> $BASH_ENV - echo 'export HELM_VALUE_FILENAME=$K8_HELM_VALUE_FILENAME_SNAPSHOT' >> $BASH_ENV - echo 'export K8_CLUSTER_SERVER=$K8_CLUSTER_SERVER_SNAPSHOT' >> $BASH_ENV - echo 'export K8_RELEASE_NAME=$K8_RELEASE_NAME_SNAPSHOT' >> $BASH_ENV - echo 'export K8_NAMESPACE=$K8_NAMESPACE_SNAPSHOT' >> $BASH_ENV - echo 'export K8_USER_NAME=$K8_USER_NAME_SNAPSHOT' >> $BASH_ENV - echo 'export K8_USER_TOKEN=$K8_USER_TOKEN_SNAPSHOT' >> $BASH_ENV - echo 'export K8_HELM_CHART_NAME=$K8_HELM_CHART_NAME_SNAPSHOT' >> $BASH_ENV - echo 'export HELM_VALUE_SET_VALUES="--set central.centralhub.centralledger.containers.api.image.repository=$DOCKER_ORG/$CIRCLE_PROJECT_REPONAME --set central.centralhub.centralledger.containers.api.image.tag=$CIRCLE_TAG --set central.centralhub.centralledger.containers.admin.image.repository=$DOCKER_ORG/$CIRCLE_PROJECT_REPONAME --set central.centralhub.centralledger.containers.admin.image.tag=$CIRCLE_TAG"' >> $BASH_ENV - - exit 0 -fi - -if [[ ${CIRCLE_TAG} =~ v[0-9]+(\.[0-9]+)* ]]; then - echo "Setting prod env vars for ${CIRCLE_TAG}" - - echo 'export RELEASE_TAG=$RELEASE_TAG_PROD' >> $BASH_ENV - echo 'export HELM_VALUE_FILENAME=$K8_HELM_VALUE_FILENAME_PROD' >> $BASH_ENV - echo 'export K8_CLUSTER_SERVER=$K8_CLUSTER_SERVER_PROD' >> $BASH_ENV - echo 'export K8_RELEASE_NAME=$K8_RELEASE_NAME_PROD' >> $BASH_ENV - echo 'export K8_NAMESPACE=$K8_NAMESPACE_PROD' >> $BASH_ENV - echo 'export K8_USER_NAME=$K8_USER_NAME_PROD' >> $BASH_ENV - echo 'export K8_USER_TOKEN=$K8_USER_TOKEN_PROD' >> $BASH_ENV - echo 'export K8_HELM_CHART_NAME=$K8_HELM_CHART_NAME_PROD' >> $BASH_ENV - echo 'export HELM_VALUE_SET_VALUES="--set central.centralhub.centralledger.containers.api.image.repository=$DOCKER_ORG/$CIRCLE_PROJECT_REPONAME --set central.centralhub.centralledger.containers.api.image.tag=$CIRCLE_TAG --set central.centralhub.centralledger.containers.admin.image.repository=$DOCKER_ORG/$CIRCLE_PROJECT_REPONAME --set central.centralhub.centralledger.containers.admin.image.tag=$CIRCLE_TAG"' >> $BASH_ENV - - exit 0 -fi - -echo "No valid match found for CIRCLE_TAG: ${CIRCLE_TAG}" -exit 1 \ No newline at end of file diff --git a/.circleci/config.yml b/.circleci/config.yml index b7641e0a..aa10077f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,8 +1,14 @@ # CircleCI v2.1 Config version: 2.1 + +## +# orbs +# +# Orbs used in this pipeline +### orbs: anchore: anchore/anchore-engine@1.6.0 - + deploy-kube: mojaloop/deployment@0.1.6 ## # defaults @@ -29,116 +35,12 @@ defaults_awsCliDependencies: &defaults_awsCliDependencies | pip install --upgrade awscli==1.14.5 s3cmd==2.0.1 python-magic apk -v --purge del py-pip -defaults_build_docker_build: &defaults_build_docker_build - name: Build Docker $CIRCLE_TAG image - command: | - echo "Building Docker image: $CIRCLE_TAG" - docker build -t $DOCKER_ORG/$CIRCLE_PROJECT_REPONAME:$CIRCLE_TAG . - -defaults_build_docker_login: &defaults_build_docker_login - name: Login to Docker Hub - command: | - docker login -u $DOCKER_USER -p $DOCKER_PASS - -defaults_build_docker_publish: &defaults_build_docker_publish - name: Publish Docker image $CIRCLE_TAG & Latest tag to Docker Hub - command: | - echo "Publishing $DOCKER_ORG/$CIRCLE_PROJECT_REPONAME:$CIRCLE_TAG" - docker push $DOCKER_ORG/$CIRCLE_PROJECT_REPONAME:$CIRCLE_TAG - echo "Publishing $DOCKER_ORG/$CIRCLE_PROJECT_REPONAME:$RELEASE_TAG" - docker push $DOCKER_ORG/$CIRCLE_PROJECT_REPONAME:$RELEASE_TAG - -defaults_deploy_config_kubernetes_cluster: &defaults_deploy_config_kubernetes_cluster - name: Configure Kubernetes cluster - command: | - echo "Configure Kubernetes cluster ${K8_CLUSTER_NAME}" - kubectl config set-cluster $K8_CLUSTER_NAME --server=$K8_CLUSTER_SERVER --insecure-skip-tls-verify=true - -defaults_deploy_config_kubernetes_context: &defaults_deploy_config_kubernetes_context - name: Confi gure Kubernetes context - command: | - echo "Configure Kubernetes context ${K8_CLUSTER_NAME}" - kubectl config set-context $K8_CLUSTER_NAME --cluster=$K8_CLUSTER_NAME --user=$K8_USER_NAME --namespace=$K8_NAMESPACE - -defaults_deploy_config_kubernetes_credentials: &defaults_deploy_config_kubernetes_credentials - name: Configure Kubernetes credentails - command: | - echo "Configure Kubernetes credentials ${K8_USER_NAME}" - if [ ! -z "$K8_USER_TOKEN" ]; - then - echo "Configure Kubernetes credentials ${K8_USER_NAME} using Token" - kubectl config set-credentials $K8_USER_NAME --token=$K8_USER_TOKEN - else - echo "Configure Kubernetes credentials ${K8_USER_NAME} using Certs" - kubectl config set-credentials $K8_USER_NAME --client-certificate=$CIRCLE_WORKING_DIRECTORY/$AWS_S3_DIR_DEVOPS_DEPLOYMENT_CONFIG_KEYS/$K8_USER_PEM_CERT_FILENAME --client-key=$CIRCLE_WORKING_DIRECTORY/$AWS_S3_DIR_DEVOPS_DEPLOYMENT_CONFIG_KEYS/$K8_USER_PEM_KEY_FILENAME - fi - -defaults_deploy_configure_helm: &defaults_deploy_configure_helm - name: Configure Helm - command: | - helm init --client-only - -defaults_deploy_install_or_upgrade_helm_chart: &defaults_deploy_install_or_upgrade_helm_chart - name: Install or Upgrade Helm Chart - command: | - echo "Install or Upgrade Chart ${K8_RELEASE_NAME} for Docker Image ${DOCKER_ORG}/${CIRCLE_PROJECT_REPONAME}:${CIRCLE_TAG}" - if [ -z "$(helm list -q | grep -E "^${K8_RELEASE_NAME}$")" ] && [ "$(helm list -q | grep -E "^${K8_RELEASE_NAME}$")" != "Error: Unauthorized" ]; - then - echo "Installing ${K8_RELEASE_NAME} new release" - helm install --namespace=$K8_NAMESPACE --name=$K8_RELEASE_NAME --repo=$K8_HELM_REPO --version $K8_HELM_CHART_VERSION $HELM_VALUE_SET_VALUES -f $CIRCLE_WORKING_DIRECTORY/$AWS_S3_DIR_DEVOPS_DEPLOYMENT_CONFIG_HELM/$HELM_VALUE_FILENAME $K8_HELM_CHART_NAME - else - echo "Upgrading ${K8_RELEASE_NAME} release" - helm upgrade $K8_RELEASE_NAME --repo=$K8_HELM_REPO --version $K8_HELM_CHART_VERSION --reuse-values $HELM_VALUE_SET_VALUES -f $CIRCLE_WORKING_DIRECTORY/$AWS_S3_DIR_DEVOPS_DEPLOYMENT_CONFIG_HELM/$HELM_VALUE_FILENAME $K8_HELM_CHART_NAME - fi - -defaults_deploy_prequisites: &defaults_deploy_prequisites - name: Copy deployment pre-requisites from S3 bucket - command: | - if [ -z "$K8_USER_TOKEN" ]; - then - echo "Copying K8 keys into $AWS_S3_DIR_DEVOPS_DEPLOYMENT_CONFIG_KEYS folder" - mkdir $CIRCLE_WORKING_DIRECTORY/$AWS_S3_DIR_DEVOPS_DEPLOYMENT_CONFIG_KEYS - aws s3 cp $AWS_S3_URI_DEVOPS_DEPLOYMENT_CONFIG/$AWS_S3_DIR_DEVOPS_DEPLOYMENT_CONFIG_KEYS/$K8_USER_PEM_KEY_FILENAME $CIRCLE_WORKING_DIRECTORY/$AWS_S3_DIR_DEVOPS_DEPLOYMENT_CONFIG_KEYS/ - aws s3 cp $AWS_S3_URI_DEVOPS_DEPLOYMENT_CONFIG/$AWS_S3_DIR_DEVOPS_DEPLOYMENT_CONFIG_KEYS/$K8_USER_PEM_CERT_FILENAME $CIRCLE_WORKING_DIRECTORY/$AWS_S3_DIR_DEVOPS_DEPLOYMENT_CONFIG_KEYS/ - else - echo "Skipping K8 keys into $AWS_S3_DIR_DEVOPS_DEPLOYMENT_CONFIG_KEYS folder" - fi - - echo "Copying Helm value file into $AWS_S3_DIR_DEVOPS_DEPLOYMENT_CONFIG_HELM folder for $K8_RELEASE_NAME release" - mkdir $CIRCLE_WORKING_DIRECTORY/$AWS_S3_DIR_DEVOPS_DEPLOYMENT_CONFIG_HELM - aws s3 cp $AWS_S3_URI_DEVOPS_DEPLOYMENT_CONFIG/$AWS_S3_DIR_DEVOPS_DEPLOYMENT_CONFIG_HELM/$HELM_VALUE_FILENAME $CIRCLE_WORKING_DIRECTORY/$AWS_S3_DIR_DEVOPS_DEPLOYMENT_CONFIG_HELM/ - -defaults_deploy_set_kubernetes_context: &defaults_deploy_set_kubernetes_context - name: Set Kubernetes context - command: | - echo "Configure Kubernetes context ${K8_CLUSTER_NAME}" - kubectl config use-context $K8_CLUSTER_NAME - - defaults_license_scanner: &defaults_license_scanner name: Install and set up license-scanner command: | git clone https://github.com/mojaloop/license-scanner /tmp/license-scanner cd /tmp/license-scanner && make build default-files set-up - -defaults_build_docker_build_release: &defaults_build_docker_build_release - name: Build Docker $RELEASE_TAG image - command: | - echo "Building Docker image: $RELEASE_TAG" - docker build -t $DOCKER_ORG/$CIRCLE_PROJECT_REPONAME:$RELEASE_TAG . - - - -defaults_slack_announcement: &defaults_slack_announcement - name: Slack announcement for tag releases - command: | - curl -X POST \ - $SLACK_WEBHOOK_ANNOUNCEMENT \ - -H 'Content-type: application/json' \ - -H 'cache-control: no-cache' \ - -d "{ \"text\": \"*${CIRCLE_PROJECT_REPONAME}* - Release \`${CIRCLE_TAG}\`: https://github.com/mojaloop/${CIRCLE_PROJECT_REPONAME}/releases/tag/${CIRCLE_TAG}\"}" - ## # Executors # @@ -154,11 +56,6 @@ executors: machine: image: ubuntu-1604:201903-01 - helm-kube: - working_directory: /home/circleci/project - docker: - - image: hypnoglow/kubernetes-helm - ## # Jobs # @@ -305,9 +202,12 @@ jobs: steps: - checkout - run: - <<: *defaults_build_docker_build + name: Build Docker $CIRCLE_TAG image + command: | + echo "Building Docker image: $CIRCLE_TAG" + docker build -t $DOCKER_ORG/$CIRCLE_PROJECT_REPONAME:$CIRCLE_TAG . - run: - name: save docker image to workspace + name: Save docker image to workspace command: docker save -o /tmp/docker-image.tar $DOCKER_ORG/$CIRCLE_PROJECT_REPONAME:$CIRCLE_TAG - persist_to_workspace: root: /tmp @@ -326,7 +226,7 @@ jobs: <<: *defaults_license_scanner - run: name: Run the license-scanner - command: cd /tmp/license-scanner && mode=docker dockerImage=$DOCKER_ORG/$CIRCLE_PROJECT_REPONAME:$CIRCLE_TAG make run + command: cd /tmp/license-scanner && mode=docker dockerImages=$DOCKER_ORG/$CIRCLE_PROJECT_REPONAME:$CIRCLE_TAG make run - store_artifacts: path: /tmp/license-scanner/results prefix: licenses @@ -336,6 +236,9 @@ jobs: steps: - setup_remote_docker - checkout + - run: + name: Install AWS CLI dependencies + command: *defaults_awsCliDependencies - attach_workspace: at: /tmp - run: @@ -344,8 +247,7 @@ jobs: - anchore/analyze_local_image: dockerfile_path: ./Dockerfile image_name: ${DOCKER_ORG}/${CIRCLE_PROJECT_REPONAME}:${CIRCLE_TAG} - policy_bundle_file_path: .circleci/.anchore/policy_bundle.json - # Anchore bug: if policy_failure is `true`, reports don't get written - we manually check for + # Anchore bug: if policy_failure is `true`, reports don't get written - we manually check for failures below policy_failure: false timeout: '500' - run: @@ -362,15 +264,20 @@ jobs: printf "\n%s\n" "The following vulnerabilities were found:" jq '[.vulnerabilities | group_by(.package) | .[] | {package: .[0].package, vuln: [.[].vuln]}]' anchore-reports/*vuln*.json fi - - # Check the status of the policy.json for whether or not our tests failed - failCount=$(cat anchore-reports/*policy*.json | grep '\"status\": \"fail\"' | wc -l) - if [ $failCount -gt 0 ]; then - echo "Failed with a policy failure count of: ${failCount}" - # TODO: Enable this when we want to increase the strictness of our security policies - # exit 1 - fi - + - run: + name: Upload Anchore reports to s3 + command: | + aws s3 cp anchore-reports ${AWS_S3_DIR_ANCHORE_REPORTS}/${CIRCLE_PROJECT_REPONAME}/ --recursive + aws s3 rm ${AWS_S3_DIR_ANCHORE_REPORTS}/latest/ --recursive --exclude "*" --include "${CIRCLE_PROJECT_REPONAME}*" + aws s3 cp anchore-reports ${AWS_S3_DIR_ANCHORE_REPORTS}/latest/ --recursive + + # TODO: Enable this when we want to increase the strictness of our security policies + # failCount=$(cat anchore-reports/*policy*.json | grep 'fail' | wc -l) + # echo "FailCount is: ${failCount}" + # if [ $failCount -gt 0 ]; then + # printf "Failed with a policy failure count of: ${failCount}" + # exit 1 + # fi - store_artifacts: path: anchore-reports @@ -384,43 +291,38 @@ jobs: name: Load the pre-built docker image from workspace command: docker load -i /tmp/docker-image.tar - run: - <<: *defaults_build_docker_login - - run: - name: setup environment vars for publish/deploy - command: ./.circleci/_set_up_deploy_envs.sh + name: Login to Docker Hub + command: docker login -u $DOCKER_USER -p $DOCKER_PASS - run: name: Re-tag pre built image command: | docker tag $DOCKER_ORG/$CIRCLE_PROJECT_REPONAME:$CIRCLE_TAG $DOCKER_ORG/$CIRCLE_PROJECT_REPONAME:$RELEASE_TAG - run: - <<: *defaults_build_docker_publish + name: Publish Docker image $CIRCLE_TAG & Latest tag to Docker Hub + command: | + echo "Publishing $DOCKER_ORG/$CIRCLE_PROJECT_REPONAME:$CIRCLE_TAG" + docker push $DOCKER_ORG/$CIRCLE_PROJECT_REPONAME:$CIRCLE_TAG + echo "Publishing $DOCKER_ORG/$CIRCLE_PROJECT_REPONAME:$RELEASE_TAG" + docker push $DOCKER_ORG/$CIRCLE_PROJECT_REPONAME:$RELEASE_TAG - run: - <<: *defaults_slack_announcement + name: Slack announcement for tag releases + command: | + curl -X POST \ + $SLACK_WEBHOOK_ANNOUNCEMENT \ + -H 'Content-type: application/json' \ + -H 'cache-control: no-cache' \ + -d "{\"text\": \"*${CIRCLE_PROJECT_REPONAME}* - Release \`${CIRCLE_TAG}\`: https://github.com/mojaloop/${CIRCLE_PROJECT_REPONAME}/releases/tag/${CIRCLE_TAG}\"}" deploy: - executor: helm-kube + executor: deploy-kube/helm-kube steps: - checkout - - run: - name: Install AWS CLI dependencies - command: *defaults_awsCliDependencies - - run: - name: setup environment vars for release - command: ./.circleci/_set_up_deploy_envs.sh - - run: - <<: *defaults_deploy_prequisites - - run: - <<: *defaults_deploy_config_kubernetes_cluster - - run: - <<: *defaults_deploy_config_kubernetes_credentials - - run: - <<: *defaults_deploy_config_kubernetes_context - - run: - <<: *defaults_deploy_set_kubernetes_context - - run: - <<: *defaults_deploy_configure_helm - - run: - <<: *defaults_deploy_install_or_upgrade_helm_chart + - deploy-kube/setup_and_run: + helm_set_values: | + --set .account-lookup-service.account-lookup-service.containers.api.image.repository=$DOCKER_ORG/$CIRCLE_PROJECT_REPONAME \ + --set .account-lookup-service.account-lookup-service.containers.api.image.tag=$CIRCLE_TAG \ + --set .account-lookup-service.account-lookup-service.containers.admin.image.repository=$DOCKER_ORG/$CIRCLE_PROJECT_REPONAME \ + --set .account-lookup-service.account-lookup-service.containers.admin.image.tag=$CIRCLE_TAG ## # Workflows