Updating buildpack bump pipeline

kubecf-build-pipelines/buildpacks/config.yml

@@ -4,7 +4,7 @@ build-image-resource-repo: https://github.com/concourse/docker-image-resource
 build-image-resource-branch: master
 kubecf-repo: git@github.com:SUSE/kubecf.git
 kubecf-branch: master
-kubecf-ops-set-suse-buildpacks: deploy/helm/kubecf/assets/operations/buildpacks/set_suse_buildpacks.yaml
+kubecf-values: deploy/helm/kubecf/values.yaml
 fissile-linux-s3-bucket: cf-opensusefs2
 stemcell-repository: splatform/fissile-stemcell-sle
 stemcell-versions-s3-bucket: cf-operators

kubecf-build-pipelines/buildpacks/pipeline.yml.erb

@@ -12,61 +12,61 @@ releases = [
     name: "go-buildpack",
     uri: "git@github.com:SUSE/cf-go-buildpack-release.git",
     trigger_path: "config/blobs.yml",
-    name_in_role_manifest: "suse-go-buildpack"
+    name_in_values_yaml: "suse-go-buildpack"
   },
   {
     name: "nodejs-buildpack",
     uri: "git@github.com:SUSE/cf-nodejs-buildpack-release.git",
     trigger_path: "config/blobs.yml",
-    name_in_role_manifest: "suse-nodejs-buildpack"
+    name_in_values_yaml: "suse-nodejs-buildpack"
   },
   {
     name: "python-buildpack",
     uri: "git@github.com:SUSE/cf-python-buildpack-release.git",
     trigger_path: "config/blobs.yml",
-    name_in_role_manifest: "suse-python-buildpack"
+    name_in_values_yaml: "suse-python-buildpack"
   },
   {
     name: "staticfile-buildpack",
     uri: "git@github.com:SUSE/cf-staticfile-buildpack-release.git",
     trigger_path: "config/blobs.yml",
-    name_in_role_manifest: "suse-staticfile-buildpack"
+    name_in_values_yaml: "suse-staticfile-buildpack"
   },
   {
     name: "nginx-buildpack",
     uri: "git@github.com:SUSE/cf-nginx-buildpack-release.git",
     trigger_path: "config/blobs.yml",
-    name_in_role_manifest: "suse-nginx-buildpack"
+    name_in_values_yaml: "suse-nginx-buildpack"
   },
   {
     name: "java-buildpack",
     uri: "git@github.com:SUSE/cf-java-buildpack-release.git",
     trigger_path: "config/blobs.yml",
-    name_in_role_manifest: "suse-java-buildpack"
+    name_in_values_yaml: "suse-java-buildpack"
   },
   {
     name: "dotnet-core-buildpack",
     uri:  "git@github.com:SUSE/cf-dotnet-core-buildpack-release.git",
     trigger_path: "config/blobs.yml",
-    name_in_role_manifest: "suse-dotnet-core-buildpack"
+    name_in_values_yaml: "suse-dotnet-core-buildpack"
   },
   {
     name: "php-buildpack",
     uri: "git@github.com:SUSE/cf-php-buildpack-release.git",
     trigger_path: "config/blobs.yml",
-    name_in_role_manifest: "suse-php-buildpack"
+    name_in_values_yaml: "suse-php-buildpack"
   },
   {
     name: "ruby-buildpack",
     uri: "git@github.com:SUSE/cf-ruby-buildpack-release.git",
     trigger_path: "config/blobs.yml",
-    name_in_role_manifest: "suse-ruby-buildpack"
+    name_in_values_yaml: "suse-ruby-buildpack"
   },
   {
     name: "binary-buildpack",
     uri: "git@github.com:SUSE/cf-binary-buildpack-release.git",
     trigger_path: "config/blobs.yml",
-    name_in_role_manifest: "suse-binary-buildpack"
+    name_in_values_yaml: "suse-binary-buildpack"
   }
 ]
 %>
@@ -142,8 +142,8 @@ jobs:
     - task: build
       privileged: true
       input_mapping:
-        s3.stemcell-version: s3.fissile-stemcell-version 
-        gh_release: <%= release[:name] %>-gh-release
+        s3.stemcell-version: s3.fissile-stemcell-version
+        suse_final_release: suse-final-release-<%= release[:name] %>
       params:
         STEMCELL_REPOSITORY: <%= stemcell_repository %>
         STEMCELL_VERSIONED_FILE: <%= stemcell_version_file %>
@@ -158,16 +158,16 @@ jobs:
       privileged: true
       input_mapping:
         suse_final_release: suse-final-release-<%= release[:name] %>
-        gh_release: <%= release[:name] %>-gh-release
+        built_image: built_image
       params:
         GITHUB_TOKEN: <%= github_access_token %>
         GITHUB_PRIVATE_KEY: |
          <%= github_private_key.gsub("\n", "\n         ") %>
         GIT_MAIL: <%= github_username %>
         GIT_USER: suse-cf-ci-bot
         RELEASE_NAME: <%= release[:name] %>
-        NAME_IN_ROLE_MANIFEST: <%= release[:name_in_role_manifest] %>
+        BUILDPACK_NAME: <%= release[:name_in_values_yaml] %>
         KUBECF_BRANCH: <%= kubecf_branch %>
-        KUBECF_OPS_SET_SUSE_BUILDPACKS: <%= kubecf_ops_set_suse_buildpacks %>
+        KUBECF_VALUES: <%= kubecf_values %>
       file: ci/kubecf-build-pipelines/buildpacks/tasks/create_pr.yml
 <% end %>

kubecf-build-pipelines/buildpacks/tasks/build.sh

@@ -29,10 +29,10 @@ docker pull "${stemcell_image}"
 # Build the releases.
 base_dir=$(pwd)
 # Get version from the GitHub release that triggered this task
-pushd gh_release
+pushd suse_final_release
 RELEASE_VERSION=$(cat version)
-RELEASE_URL=$(cat body | grep -o "Release Tarball: .*" | sed 's/Release Tarball: //')
-RELEASE_SHA=$(sha1sum ${base_dir}/s3.*/*.tgz | cut -d' ' -f1)
+RELEASE_URL=$(cat url)
+RELEASE_SHA=$(sha1sum *.tgz | cut -d' ' -f1)
 popd
 
 tasks_dir="$(dirname $0)"

kubecf-build-pipelines/buildpacks/tasks/build.yml

@@ -10,7 +10,9 @@ inputs:
 - name: build-image-resource
 - name: s3.stemcell-version
 - name: s3.fissile-linux
-- name: gh_release
+- name: suse_final_release
+outputs:
+- name: built_image
 params:
   STEMCELL_REPOSITORY: ~
   STEMCELL_VERSIONED_FILE: ~

kubecf-build-pipelines/buildpacks/tasks/build_release.sh

@@ -31,7 +31,7 @@ function build_release() {
   )
 
   built_image=$(fissile build release-images --dry-run "${build_args[@]}" | cut -d' ' -f3)
-
+  echo $built_image > built_image/image
   export DOCKER_CLI_EXPERIMENTAL=enabled
   # Only build and push the container image if doesn't exits already.
   if docker manifest inspect "${built_image}" 2>&1 | grep --quiet "no such manifest"; then

kubecf-build-pipelines/buildpacks/tasks/create_pr.sh

@@ -2,45 +2,59 @@
 
 set -o errexit -o nounset
 
-# Updates release information in role-manifest
+# Updates release information in values.yaml
 # Looks for a release like:
 #
-# - type: replace
-#   path: /releases/name=go-buildpack
-#   value:
-#     name: suse-go-buildpack
-#     url: "https://s3.amazonaws.com/suse-final-releases/go-buildpack-release-1.8.42.1.tgz"
-#     version: "1.8.42.1"
-#     sha1: "f811bef86bfba4532d6a7f9653444c7901c59989"
+# suse-go-buildpack:
+#   url: registry.suse.com/cap-staging
+#   version: "1.9.4.1"
+#   stemcell:
+#     os: SLE_15_SP1
+#     version: 23.1-7.0.0_374.gb8e8e6af
+#   file: suse-go-buildpack/packages/go-buildpack-sle15/go-buildpack-sle15-v1.9.4.1-1.1-436eaf5d.zip
 
 function update_buildpack_info() {
 
-KUBECF_OPS_SET_SUSE_BUILDPACKS=$1
-RELEASE=$2
-NEW_URL=$3
-NEW_VERSION=$4
-NEW_SHA=$5
+BUILDPACK_NAME=$1
+KUBECF_VALUES=$2
+BUILT_IMAGE=$3
+NEW_FILE_NAME=$4
 
 PYTHON_CODE=$(cat <<EOF 
 #!/usr/bin/python3
 
 import ruamel.yaml
 
+def represent_none(self, data):
+    return self.represent_scalar(u'tag:yaml.org,2002:null', u'~')
+
+# Replaces the filename at the end of the original 'file'.
+def get_new_filename():
+    new_file = values['releases']["${BUILDPACK_NAME}"]['file'].split("/")[:3]
+    new_file.append("${NEW_FILE_NAME}")
+    return "/".join(new_file)
+
 yaml = ruamel.yaml.YAML()
 yaml.preserve_quotes = True
+yaml.representer.add_representer(type(None), represent_none)
+
+# Breaking down the built_url to retrieve individual values.
+NEW_URL = "/".join("${BUILT_IMAGE}".split("/", 2)[:2])
+BUILT_IMAGE = "${BUILT_IMAGE}".split("-")
+NEW_VERSION = BUILT_IMAGE[-1]
+NEW_STEMCELL_OS = BUILT_IMAGE[3].split(":")[1]
+NEW_STEMCELL_VERSION = "-".join(BUILT_IMAGE[4:6])
 
-with open("${KUBECF_OPS_SET_SUSE_BUILDPACKS}") as fp:
-    buildpacks = yaml.load(fp)
+with open("${KUBECF_VALUES}") as fp:
+    values = yaml.load(fp)
 
-for buildpack in buildpacks:
-    if buildpack['value']['name'] == "${RELEASE}":
-        buildpack['value']['url'] = "${NEW_URL}"
-        buildpack['value']['version'] = "${NEW_VERSION}"
-        buildpack['value']['sha1'] = "${NEW_SHA}"
-        break
+values['releases']["${BUILDPACK_NAME}"]['url'] = NEW_URL
+values['releases']["${BUILDPACK_NAME}"]['stemcell']['os'] = NEW_STEMCELL_OS
+values['releases']["${BUILDPACK_NAME}"]['stemcell']['version'] = NEW_STEMCELL_VERSION
+values['releases']["${BUILDPACK_NAME}"]['file'] = get_new_filename()
 
-with open("${KUBECF_OPS_SET_SUSE_BUILDPACKS}", 'w') as f:
-    yaml.dump(buildpacks, f)
+with open("${KUBECF_VALUES}", 'w') as f:
+    yaml.dump(values, f)
 
 EOF
 )
@@ -62,27 +76,23 @@ chmod 0600 ~/.ssh/id_ecdsa
 git config --global user.email "$GIT_MAIL"
 git config --global user.name "$GIT_USER"
 
-base_dir=$(pwd)
-# Get version from the GitHub release that triggered this task
-pushd gh_release
-RELEASE_VERSION=$(cat version)
-RELEASE_URL=$(cat body | grep -o "Release Tarball: .*" | sed 's/Release Tarball: //')
-RELEASE_SHA=$(sha1sum ${base_dir}/suse_final_release/*.tgz | cut -d' ' -f1)
-popd
+RELEASE_VERSION=$(cat suse_final_release/version)
+BUILT_IMAGE=$(cat built_image/image)
+NEW_FILE=$(tar -zxOf suse_final_release/*.tgz packages | tar -ztf - | grep zip | cut -d'/' -f3)
 
-COMMIT_TITLE="Bump ${NAME_IN_ROLE_MANIFEST} release to ${RELEASE_VERSION}"
+COMMIT_TITLE="Bump ${BUILDPACK_NAME} release to ${RELEASE_VERSION}"
 
 # Update release in kubecf repo
 cp -r kubecf/. updated-kubecf/
 cd updated-kubecf
 
 git pull
-export GIT_BRANCH_NAME="bump_${NAME_IN_ROLE_MANIFEST}-`date +%Y%m%d%H%M%S`"
+export GIT_BRANCH_NAME="bump_${BUILDPACK_NAME}-`date +%Y%m%d%H%M%S`"
 git checkout -b "${GIT_BRANCH_NAME}"
 
-update_buildpack_info "${KUBECF_OPS_SET_SUSE_BUILDPACKS}" "${NAME_IN_ROLE_MANIFEST}" "${RELEASE_URL}" "${RELEASE_VERSION}" "${RELEASE_SHA}"
+update_buildpack_info "${BUILDPACK_NAME}" "${KUBECF_VALUES}" "${BUILT_IMAGE}" "${NEW_FILE}"
 
-git commit "${KUBECF_OPS_SET_SUSE_BUILDPACKS}" -m "${COMMIT_TITLE}"
+git commit "${KUBECF_VALUES}" -m "${COMMIT_TITLE}"
 
 # Open a Pull Request
 PR_MESSAGE=`echo -e "${COMMIT_TITLE}"`

kubecf-build-pipelines/buildpacks/tasks/create_pr.yml

@@ -7,16 +7,16 @@ image_resource:
     tag: kubecf
 inputs:
 - name: ci
-- name: gh_release
 - name: suse_final_release
 - name: kubecf
+- name: built_image
 params:
   GITHUB_TOKEN: ~
   GIT_MAIL: ~
   GIT_USER: ~
   RELEASE_NAME: ~
-  NAME_IN_ROLE_MANIFEST: ~
+  BUILDPACK_NAME: ~
   KUBECF_BRANCH: ~
-  KUBECF_OPS_SET_SUSE_BUILDPACKS: ~
+  KUBECF_VALUES: ~
 run:
   path: ci/kubecf-build-pipelines/buildpacks/tasks/create_pr.sh