Merge pull request #1 from Sage/initial_code

Buildkite plugin for shared SBC code

docker-compose.yml

@@ -0,0 +1,11 @@
+version: '3.4'
+services:
+  tests:
+    image: buildkite/plugin-tester
+    volumes:
+      - ".:/plugin"
+  lint:
+    image: buildkite/plugin-linter
+    command: ['--id', 'Sage/sbc-shared']
+    volumes:
+      - ".:/plugin:ro"

hooks/command

@@ -0,0 +1,6 @@
+set -euo pipefail
+
+if [[ -n $BUILDKITE_COMMAND ]]; then
+  echo "Step command detected for $BUILDKITE_LABEL .  Executing command"
+  eval $BUILDKITE_COMMAND
+fi

hooks/post-command

@@ -0,0 +1,51 @@
+set -euo pipefail
+
+. "$(dirname $BASH_SOURCE)/../lib/functions.sh"
+
+ACTION="$BUILDKITE_PLUGIN_SBC_SHARED_ACTION"
+
+if [[ $ACTION == "push_image" ]];then
+  echo "--- :floppy_disk: Push $ENVIRONMENT image for $APP"
+
+  # Push the docker image
+  push_image --account_id $ACCOUNT_ID --app $APP --tag $DOCKER_TAG
+
+  # For dev, other images like database and test and master should be pushed.
+  if [[ "$ENVIRONMENT" == "qa" ]]; then
+    INITIAL_BK_BRANCH="$BK_BRANCH"
+
+    # Push a special image based on a merge to the default branch (e.g. master or main)
+    if [[ "$BK_BRANCH" == "$BUILDKITE_PIPELINE_DEFAULT_BRANCH" ]]; then
+      echo "$BUILDKITE_PIPELINE_DEFAULT_BRANCH test image "
+      BK_BRANCH="test-$BK_BRANCH"
+      push_image --account_id $AWS_ACCOUNT_ID --app $APP --tag test
+    fi
+
+    if [[ "$HAS_DB_IMAGE" == "true" ]]; then
+      echo "DB image"
+      BK_BRANCH="database-$INITIAL_BK_BRANCH"
+      push_image --account_id $AWS_ACCOUNT_ID --app $APP --tag database
+    fi
+  fi
+elif [[ $ACTION == "push_param" ]];then
+ config_path=$ENVIRONMENT/$LANDSCAPE
+
+  # change directory from which all param logic expects to be run from.
+  cd configuration
+ ./push.sh $config_path
+elif [[ $ACTION == "publish_gem" ]];then
+  echo ""
+elif [[ $ACTION == "build" ]];then
+  # Load a predetermined custom functions file that a repository may need to use to extend the build process
+  CUSTOM_FILE=".buildkite/custom_functions.sh"
+  if [ -f "$CUSTOM_FILE" ]; then
+    echo 'Loading custom file'
+    . $CUSTOM_FILE
+  fi
+
+  # execute the predetermined build script that a repository must have.
+  . ".buildkite/build.sh"
+else
+  echo "Unsupport action name of $ACTION"
+  exit 1
+fi

hooks/pre-command

@@ -0,0 +1,46 @@
+set -euo pipefail
+
+. "$(dirname $BASH_SOURCE)/../lib/functions.sh"
+
+prefix="${BUILDKITE_PLUGIN_SBC_SHARED_REPO_PREFIX:-sageone}"
+setup $prefix
+
+set +u
+# Validation of config will be done by each relevant action
+export ENVIRONMENT="${BUILDKITE_PLUGIN_SBC_SHARED_ENVIRONMENT}"
+if [[ -z $ENVIRONMENT ]]; then
+  unset ENVIRONMENT
+fi
+
+export LANDSCAPE="${BUILDKITE_PLUGIN_SBC_SHARED_LANDSCAPE}"
+if [[ -z $LANDSCAPE ]]; then
+  unset LANDSCAPE
+fi
+
+export ACCOUNT_ID="${BUILDKITE_PLUGIN_SBC_SHARED_ACCOUNT_ID}"
+if [[ -z $ACCOUNT_ID ]]; then
+  unset ACCOUNT_ID
+fi
+
+export GEM_HOST="${BUILDKITE_PLUGIN_SBC_SHARED_GEM_HOST}"
+if [[ -z $GEM_HOST ]]; then
+  unset GEM_HOST
+fi
+
+# Only export the AWS_REGION if it was specified to be overriden.
+if [[ -n $BUILDKITE_PLUGIN_SBC_SHARED_REGION ]]; then
+  export AWS_REGION="${BUILDKITE_PLUGIN_SBC_SHARED_REGION}"
+fi
+
+# If the target image needs to have a custom tag other than the GH tag/branch
+export TARGET_TAG="${BUILDKITE_PLUGIN_SBC_SHARED_TARGET_TAG:-}"
+
+set -u
+export DOCKER_TAG="${BUILDKITE_PLUGIN_SBC_SHARED_TAG:-application}"
+export HAS_DB_IMAGE="${BUILDKITE_PLUGIN_SBC_SHARED_HAS_DB_IMAGE:-false}"
+
+# copy scripts to be used by pipeline in order to be invoked within a docker container.
+ACTION="$BUILDKITE_PLUGIN_SBC_SHARED_ACTION"
+if [[ $ACTION == "publish_gem" ]];then
+  cp "$(dirname $BASH_SOURCE)/../lib/release_jfrog.sh" .buildkite/release.sh
+fi

lib/functions.sh

@@ -0,0 +1,163 @@
+varx () {
+  if [[ -z "${!1}" ]]; then
+    echo "$1 is not set."
+    exit 1
+  fi
+}
+
+setup() {
+  if [ -z $1 ]; then
+    echo "Please define a repo prefix name (e.g. setup sageone)"
+    exit 1
+  fi
+
+  echo "Setting up variables for $1"
+
+  # Setup the env that contains the application name and repo name
+  export APP=$(cat .buildkite/.application)
+  export REPO=$1/$APP
+
+  # Setup the proper docker tag to be used depending on GH tag and/or branch
+  export BK_BRANCH="${BUILDKITE_BRANCH:-$BUILDKITE_TAG}"
+
+  # Setup CI branch and time used by various other tools. E.g. ssm pusher
+  export CI_STRING_TIME=$(date '+%Y-%m-%d %H:%M:%S')
+  export CI_BRANCH=$BK_BRANCH
+
+  # Change the output of the docker build process to not be truncated in BK
+  if [[ "$BUILDKITE" == "true" ]]; then
+    export BUILDKIT_PROGRESS=plain
+  fi
+
+  export BK_ECR=268539851198.dkr.ecr.eu-west-1.amazonaws.com/sageone/buildkite
+}
+
+# convert --<switch> to a variable
+switches() {
+  echo "Processing switches"
+  while [ $# -gt 0 ]; do
+   if [[ $1 == *"--"* ]]; then
+        v="${1/--/}"
+        export $v="$2"
+   fi
+
+  shift
+  done
+}
+
+# validate list of switch names exist as a set variable
+validate_switches() {
+  echo "Validating switches"
+
+  arr=("$@")
+
+  set +u
+  for item in "${arr[@]}"
+  do
+    if [ -z "${!item}" ]; then
+      echo "--$item is not set"
+      echo $@
+      set -u
+      exit 1
+    fi
+  done
+  set -u
+}
+
+# target => (Optional) set the target build stage to build
+# tag => tag for the docker image
+# file => source docker file to build from
+# cache_id => cache identifier from where it was built from.  Typically GH branch name
+buildx() {
+  target=
+  switches "$@"
+  validate_switches tag file cache_id
+  varx REPO
+
+  echo "--- :building_construction: Build $tag"
+
+  local OPTIONAL_TARGET=
+  if [[ -n $target ]]; then
+    OPTIONAL_TARGET="--target $target"
+  fi
+
+  docker buildx build \
+    -f $file \
+    --build-arg BUILDKIT_INLINE_CACHE=1 \
+    --build-arg CI_BRANCH \
+    --build-arg CI_STRING_TIME \
+    --cache-from $BK_ECR:$APP-$tag-cache-$cache_id \
+    --cache-from $BK_ECR:$APP-$tag-cache-master \
+    --secret id=railslts,env=BUNDLE_GEMS__RAILSLTS__COM \
+    --secret id=jfrog,env=BUNDLE_SAGEONEGEMS__JFROG__IO \
+    --ssh default $OPTIONAL_TARGET \
+    -t $REPO:$tag \
+    .
+}
+
+# app => name of the application
+# target => (Optional) set the target build stage to build
+# tag => tag for the docker image
+# file => source docker file to build from
+# cache_id => cache identifier from where it was built from.  Typically GH branch name
+buildx_and_cachex () {
+  target=
+  switches "$@"
+  validate_switches app tag cache_id file
+  varx REPO
+
+  local OPTIONAL_TARGET=
+  if [[ -n $target ]]; then
+    OPTIONAL_TARGET="--target $target"
+  fi
+
+  buildx --app $app $OPTIONAL_TARGET --tag $tag --file $file --cache_id $cache_id
+
+  cachex --app $app --tag $tag --cache_id $cache_id
+}
+
+# Push an image into the BK ECR
+pushx () {
+  switches "$@"
+  validate_switches app tag
+  varx REPO
+  varx BUILDKITE_BUILD_NUMBER
+
+  echo "--- :floppy_disk: Push $tag"
+  local BUILD_IMAGE_NAME=$BK_ECR:$app-$tag-build-$BUILDKITE_BUILD_NUMBER
+  docker tag  $REPO:$tag $BUILD_IMAGE_NAME
+  docker push $BUILD_IMAGE_NAME
+}
+
+# Push an image into the BK ECR for caching builds
+cachex () {
+  switches "$@"
+  validate_switches app tag cache_id
+  varx REPO
+
+  echo "--- :s3: Cache $tag"
+  local BUILD_IMAGE_NAME=$BK_ECR:$app-$tag-cache-$cache_id
+  docker tag $REPO:$tag $BUILD_IMAGE_NAME
+  docker push $BUILD_IMAGE_NAME
+}
+
+# Push an image into a target ECR for deployments
+push_image () {
+  switches "$@"
+  validate_switches account_id app tag
+  varx BUILDKITE_BUILD_NUMBER
+  varx AWS_REGION
+  varx BK_BRANCH
+
+  # If the override ENV option was specified in the pipeline, use that tag value.
+  # This supports custom tags like `last-successful-build` that don't match the GH tag/branch that triggered the commit
+  local target_tag=${TARGET_TAG:-$BK_BRANCH}
+
+  echo "Pushing image for $app using tag: $target_tag"
+
+  SOURCE_IMAGE=$BK_ECR:$app-$tag-build-$BUILDKITE_BUILD_NUMBER
+  TARGET_IMAGE=$account_id.dkr.ecr.$AWS_REGION.amazonaws.com/$REPO:$target_tag
+  docker pull $SOURCE_IMAGE
+  docker tag $SOURCE_IMAGE $TARGET_IMAGE
+  docker push $TARGET_IMAGE
+}

lib/release_jfrog.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+set -e
+
+bundle exec rake build $APP.gemspec
+
+export RUBYGEMS_HOST=${GEM_HOST:-"https://sageonegems.jfrog.io/sageonegems/api/gems/gems-local"}
+
+echo "Gems Host: $RUBYGEMS_HOST"
+
+mkdir -p ~/.gem
+curl -u $ART_USER:$ART_PASS $RUBYGEMS_HOST/api/v1/api_key.yaml > ~/.gem/credentials
+chmod 600 ~/.gem/credentials
+
+GEMS_PATH=${GEM_PATH:-"pkg/*.gem"}
+
+echo "Gem Path: $GEMS_PATH"
+
+gem push /usr/src/app/$GEMS_PATH
+
+echo "Push Complete"

plugin.yml

@@ -0,0 +1,26 @@
+name: SBC Shared Build Tools
+description: Common functions to used to build and deliver SBC docker images
+author: https://github.com/Sage
+requirements:
+  - aws
+configuration:
+  properties:
+    action:
+      type: string
+    environment:
+      type: string
+    landscape:
+      type: string
+    tag:
+      type: string
+    target_tag:
+      type: string
+    repo_prefix:
+      type: string
+    region:
+      type: string
+    gem_host:
+      type: string
+  required:
+    - action
+  additionalProperties: false