Skip to content

Commit

Permalink
builder-base: add update-shasums script to make updating easier (aws#275
Browse files Browse the repository at this point in the history
)
  • Loading branch information
jaxesn authored Dec 16, 2021
1 parent 331c65c commit 3b4ac4d
Show file tree
Hide file tree
Showing 8 changed files with 177 additions and 38 deletions.
2 changes: 1 addition & 1 deletion builder-base/goss-amd64-checksum
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1994069f78d44a76d463b76fef42c7e4ab412df14703a51643642ecdefa2c029 packer-provisioner-goss-v3.0.3-linux-amd64.tar.gz
1994069f78d44a76d463b76fef42c7e4ab412df14703a51643642ecdefa2c029 packer-provisioner-goss-v3.0.3-linux-amd64.tar.gz
2 changes: 1 addition & 1 deletion builder-base/govc-amd64-checksum
Original file line number Diff line number Diff line change
@@ -1 +1 @@
faf409c1ecbb23fb4a4a6d2df9092e047b94ed14d8a55768e23957388c6a2117 govc_linux_amd64.gz
faf409c1ecbb23fb4a4a6d2df9092e047b94ed14d8a55768e23957388c6a2117 govc_linux_amd64.gz
50 changes: 17 additions & 33 deletions builder-base/install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ if [[ "$CI" == "true" ]]; then
BASE_DIR=$(pwd)/builder-base
fi

source $BASE_DIR/versions.sh

# Only add dependencies needed to build the builder base in this first part
yum upgrade -y
Expand All @@ -50,7 +51,6 @@ yum install -y \
unzip \
wget

GOLANG_VERSION="${GOLANG_VERSION:-1.16.12}"
GOLANG_MAJOR_VERSION=${GOLANG_VERSION%.*}
GOLANG_SDK_ROOT=/root/sdk/go${GOLANG_VERSION}
GOLANG_MAJOR_VERSION_BIN=${GOPATH}/go${GOLANG_MAJOR_VERSION}/bin
Expand All @@ -60,7 +60,7 @@ wget \
--progress dot:giga \
--max-redirect=1 \
--domains go.dev \
https://go.dev/dl/go${GOLANG_VERSION}.linux-$TARGETARCH.tar.gz -O go${GOLANG_VERSION}.linux-$TARGETARCH.tar.gz
$GOLANG_DOWNLOAD_URL -O go${GOLANG_VERSION}.linux-$TARGETARCH.tar.gz
sha256sum -c $BASE_DIR/golang-$TARGETARCH-checksum
tar -C ${GOLANG_SDK_ROOT} -xzf go${GOLANG_VERSION}.linux-$TARGETARCH.tar.gz --strip-components=1
for binary in go gofmt; do
Expand All @@ -86,16 +86,14 @@ rm awscli-exe-linux-$ARCH.zip
rm -rf /aws

if [ $TARGETARCH == 'amd64' ]; then
BUILDKIT_VERSION="${BUILDKIT_VERSION:-v0.9.0}"
wget \
--progress dot:giga \
https://github.com/moby/buildkit/releases/download/$BUILDKIT_VERSION/buildkit-$BUILDKIT_VERSION.linux-$TARGETARCH.tar.gz
$BUILDKIT_DOWNLOAD_URL
sha256sum -c $BASE_DIR/buildkit-$TARGETARCH-checksum
tar -C /usr -xzf buildkit-$BUILDKIT_VERSION.linux-$TARGETARCH.tar.gz
rm -rf buildkit-$BUILDKIT_VERSION.linux-$TARGETARCH.tar.gz

GITHUB_CLI_VERSION="${GITHUB_CLI_VERSION:-1.8.0}"
wget --progress dot:giga https://github.com/cli/cli/releases/download/v${GITHUB_CLI_VERSION}/gh_${GITHUB_CLI_VERSION}_linux_$TARGETARCH.tar.gz
wget --progress dot:giga $GITHUB_CLI_DOWNLOAD_URL
sha256sum -c $BASE_DIR/github-cli-$TARGETARCH-checksum
tar -xzf gh_${GITHUB_CLI_VERSION}_linux_$TARGETARCH.tar.gz
mv gh_${GITHUB_CLI_VERSION}_linux_$TARGETARCH/bin/gh $USR_BIN
Expand Down Expand Up @@ -133,18 +131,14 @@ yum install -y \
which

# needed to parse eks-d release yaml to get latest artifacts
YQ_VERSION="${YQ_VERSION:-v4.7.1}"
wget \
--progress dot:giga \
https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_$TARGETARCH.tar.gz
$YQ_DOWNLOAD_URL
sha256sum -c $BASE_DIR/yq-$TARGETARCH-checksum
tar -xzf yq_linux_$TARGETARCH.tar.gz
mv yq_linux_$TARGETARCH $USR_BIN/yq
rm yq_linux_$TARGETARCH.tar.gz

# Bash 4.3 is required to run kubernetes make test
OVERRIDE_BASH_VERSION="${OVERRIDE_BASH_VERSION:-4.3}"
wget http://ftp.gnu.org/gnu/bash/bash-$OVERRIDE_BASH_VERSION.tar.gz
wget $BASH_DOWNLOAD_URL
tar -xf bash-$OVERRIDE_BASH_VERSION.tar.gz
sha256sum -c $BASE_DIR/bash-checksum
cd bash-$OVERRIDE_BASH_VERSION
Expand Down Expand Up @@ -178,11 +172,10 @@ pip3 install "ansible==$ANSIBLE_VERSION"
PYWINRM_VERSION="${PYWINRM_VERSION:-0.4.1}"
pip3 install "pywinrm==$PYWINRM_VERSION"

PACKER_VERSION="${PACKER_VERSION:-1.7.2}"
rm -rf /usr/sbin/packer
wget \
--progress dot:giga \
https://releases.hashicorp.com/packer/$PACKER_VERSION/packer_${PACKER_VERSION}_linux_$TARGETARCH.zip
$PACKER_DOWNLOAD_URL
sha256sum -c $BASE_DIR/packer-$TARGETARCH-checksum
unzip -o packer_${PACKER_VERSION}_linux_$TARGETARCH.zip -d $USR_BIN
rm -rf packer_${PACKER_VERSION}_linux_$TARGETARCH.zip
Expand All @@ -194,19 +187,12 @@ rm -rf packer_${PACKER_VERSION}_linux_$TARGETARCH.zip
# to properly find core go packages
GO111MODULE=on go get github.com/google/go-licenses@v0.0.0-20210816172045-3099c18c36e1

if [ $TARGETARCH == 'amd64' ]; then
ARCH='x64'
fi

NODEJS_VERSION="${NODEJS_VERSION:-v15.11.0}"
wget --progress dot:giga \
https://nodejs.org/dist/$NODEJS_VERSION/node-$NODEJS_VERSION-linux-$ARCH.tar.gz
wget --progress dot:giga $NODEJS_DOWNLOAD_URL
sha256sum -c ${BASE_DIR}/nodejs-$TARGETARCH-checksum
tar -C /usr --strip-components=1 -xzf node-$NODEJS_VERSION-linux-$ARCH.tar.gz node-$NODEJS_VERSION-linux-$ARCH
rm -rf node-$NODEJS_VERSION-linux-$ARCH.tar.gz
tar -C /usr --strip-components=1 -xzf $NODEJS_FILENAME $NDOEJS_FOLDER
rm -rf $NODEJS_FILENAME

HELM_VERSION="${HELM_VERSION:-3.7.1}"
curl -O https://get.helm.sh/helm-v${HELM_VERSION}-linux-$TARGETARCH.tar.gz
curl -O $HELM_DOWNLOAD_URL
sha256sum -c $BASE_DIR/helm-$TARGETARCH-checksum
tar -xzvf helm-v${HELM_VERSION}-linux-$TARGETARCH.tar.gz linux-$TARGETARCH/helm
rm -f helm-v${HELM_VERSION}-linux-$TARGETARCH.tar.gz
Expand Down Expand Up @@ -237,30 +223,28 @@ setupgo "${GOLANG117_VERSION:-1.17.5}"

useradd -ms /bin/bash -u 1100 imagebuilder
mkdir -p /home/imagebuilder/.packer.d/plugins
GOSS_VERSION="${GOSS_VERSION:-3.0.3}"
wget \
--progress dot:giga \
https://github.com/YaleUniversity/packer-provisioner-goss/releases/download/v${GOSS_VERSION}/packer-provisioner-goss-v${GOSS_VERSION}-linux-$TARGETARCH.tar.gz
$GOSS_DOWNLOAD_URL
sha256sum -c $BASE_DIR/goss-$TARGETARCH-checksum
tar -C /home/imagebuilder/.packer.d/plugins -xzf packer-provisioner-goss-v${GOSS_VERSION}-linux-$TARGETARCH.tar.gz
rm -rf packer-provisioner-goss-v${GOSS_VERSION}-linux-$TARGETARCH.tar.gz

GOVC_VERSION="${GOVC_VERSION:-0.24.0}"
wget \
--progress dot:giga \
https://github.com/vmware/govmomi/releases/download/v${GOVC_VERSION}/govc_linux_$TARGETARCH.gz
$GOVC_DOWNLOAD_URL
sha256sum -c $BASE_DIR/govc-$TARGETARCH-checksum
gzip -d govc_linux_$TARGETARCH.gz
mv govc_linux_$TARGETARCH $USR_BIN/govc
chmod +x $USR_BIN/govc

# Install hugo for docs
HUGOVERSION=0.85.0
wget https://github.com/gohugoio/hugo/releases/download/v${HUGOVERSION}/hugo_extended_${HUGOVERSION}_Linux-64bit.tar.gz

wget $HUGO_DOWNLOAD_URL
sha256sum -c ${BASE_DIR}/hugo-$TARGETARCH-checksum
tar -xf hugo_extended_${HUGOVERSION}_Linux-64bit.tar.gz
tar -xf hugo_extended_${HUGO_VERSION}_Linux-64bit.tar.gz
mv hugo $USR_BIN/hugo
rm -rf hugo_extended_${HUGOVERSION}_Linux-64bit.tar.gz LICENSE README.md
rm -rf hugo_extended_${HUGO_VERSION}_Linux-64bit.tar.gz LICENSE README.md

SKOPEO_VERSION="${SKOPEO_VERSION:-v1.5.0}"
git clone https://github.com/containers/skopeo
Expand Down
2 changes: 1 addition & 1 deletion builder-base/packer-amd64-checksum
Original file line number Diff line number Diff line change
@@ -1 +1 @@
9429c3a6f80b406dbddb9b30a4e468aeac59ab6ae4d09618c8d70c4f4188442e packer_1.7.2_linux_amd64.zip
9429c3a6f80b406dbddb9b30a4e468aeac59ab6ae4d09618c8d70c4f4188442e packer_1.7.2_linux_amd64.zip
77 changes: 77 additions & 0 deletions builder-base/update_shasums.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
#!/usr/bin/env bash
# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -e
set -o pipefail
set -x

for TARGETARCH in arm64 amd64; do
source ./versions.sh

# GOLANG
go_file_name="go$GOLANG_VERSION.linux-$TARGETARCH.tar.gz"
sha256=$(curl -sSL --retry 5 https://go.dev/dl/?mode=json\&include=all | jq -r ".[] | select(.version==\"go$GOLANG_VERSION\") | .files[] | select(.filename==\"$go_file_name\").sha256")
echo "$sha256 $go_file_name" > golang-$TARGETARCH-checksum

# YQ
readarray -t yq_checksum_order <<< $(curl -sSL --retry 5 $YQ_CHECKSUM_ORDER_URL)
yq_checksums=$(curl -sSL --retry 5 $YQ_CHECKSUM_URL | grep -r yq_linux_$TARGETARCH | cut -d ":" -f 2)
IFS=' ' read -r -a yq_checksums_ar <<< "$yq_checksums"

yq_checksum_index=-1
for i in "${!yq_checksum_order[@]}"; do
if [[ "${yq_checksum_order[$i]}" = "SHA-256" ]]; then
yq_checksum_index=$(($i+1))
fi
done
echo $yq_checksum_index
echo "${yq_checksums_ar[*]}"
sha256="${yq_checksums_ar[$yq_checksum_index]}"
echo "$sha256 yq_linux_$TARGETARCH" > yq-$TARGETARCH-checksum
done

# BUILDKIT
sha256=$(curl -sSL --retry 5 $BUILDKIT_DOWNLOAD_URL | sha256sum | awk '{print $1}')
echo "$sha256 buildkit-$BUILDKIT_VERSION.linux-$TARGETARCH.tar.gz" > buildkit-$TARGETARCH-checksum

# GITHUB CLI
echo "$(curl -sSL --retry 5 $GITHUB_CLI_CHECKSUM_URL | grep -r gh_${GITHUB_CLI_VERSION}_linux_$TARGETARCH.tar.gz | cut -d ":" -f 2)" > github-cli-$TARGETARCH-checksum

# PACKER
echo "$(curl -sSL --retry 5 $PACKER_CHECKSUM_URL | grep -r packer_${PACKER_VERSION}_linux_$TARGETARCH.zip | cut -d ":" -f 2)" > packer-$TARGETARCH-checksum

# NODEJS
echo "$(curl -sSL --retry 5 $NODEJS_CHECKSUM_URL | grep -r $NODEJS_FILENAME | cut -d ":" -f 2)" > nodejs-$TARGETARCH-checksum

# HELM
sha256=$(curl -sSL --retry 5 $HELM_CHECKSUM_URL)
echo "$sha256 helm-v${HELM_VERSION}-linux-$TARGETARCH.tar.gz" > helm-$TARGETARCH-checksum

# GOSS
# TODO: Later versions push a sha256sum file to github so when we upgrade we can start using it instead
sha256=$(curl -sSL --retry 5 $GOSS_DOWNLOAD_URL | sha256sum | awk '{print $1}')
echo "$sha256 packer-provisioner-goss-v${GOSS_VERSION}-linux-$TARGETARCH.tar.gz" > goss-$TARGETARCH-checksum

# GOVC
# TODO: Later versions push a sha256sum file to github so when we upgrade we can start using it instead
sha256=$(curl -sSL --retry 5 $GOVC_DOWNLOAD_URL | sha256sum | awk '{print $1}')
echo "$sha256 govc_linux_$TARGETARCH.gz" > govc-$TARGETARCH-checksum

# HUGO
echo "$(curl -sSL --retry 5 $HUGO_CHECKSUM_URL | grep -r $HUGO_FILENAME | cut -d ":" -f 2)" > hugo-$TARGETARCH-checksum

# BASH
sha256=$(curl -sSL --retry 5 $BASH_DOWNLOAD_URL | sha256sum | awk '{print $1}')
echo "$sha256 bash-$OVERRIDE_BASH_VERSION.tar.gz" > bash-checksum
78 changes: 78 additions & 0 deletions builder-base/versions.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
#!/usr/bin/env bash
# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This script is used to install the necessary dependencies on the pod
# building the builder-base as well as into the builder-base itself
# Note: since we run the build in fargate we do not have access to an overlayfs
# so we use a single script from the dockerfile instead of layers to avoid
# layer duplicate and running out of disk space
# This does make local builds painful. Its recommended to add new additions
# in their own script/layer while testing and then when you are done add
# to here

GOLANG_VERSION="${GOLANG_VERSION:-1.16.12}"
GOLANG_DOWNLOAD_URL="https://go.dev/dl/go$GOLANG_VERSION.linux-$TARGETARCH.tar.gz"

BUILDKIT_VERSION="${BUILDKIT_VERSION:-v0.9.0}"
BUILDKIT_DOWNLOAD_URL="https://github.com/moby/buildkit/releases/download/$BUILDKIT_VERSION/buildkit-$BUILDKIT_VERSION.linux-$TARGETARCH.tar.gz"

GITHUB_CLI_VERSION="${GITHUB_CLI_VERSION:-1.8.0}"
GITHUB_CLI_DOWNLOAD_URL="https://github.com/cli/cli/releases/download/v${GITHUB_CLI_VERSION}/gh_${GITHUB_CLI_VERSION}_linux_$TARGETARCH.tar.gz"
GITHUB_CLI_CHECKSUM_URL="https://github.com/cli/cli/releases/download/v${GITHUB_CLI_VERSION}/gh_${GITHUB_CLI_VERSION}_checksums.txt"

OVERRIDE_BASH_VERSION="${OVERRIDE_BASH_VERSION:-4.3}"
BASH_DOWNLOAD_URL="http://ftp.gnu.org/gnu/bash/bash-$OVERRIDE_BASH_VERSION.tar.gz"

PACKER_VERSION="${PACKER_VERSION:-1.7.2}"
PACKER_DOWNLOAD_URL="https://releases.hashicorp.com/packer/$PACKER_VERSION/packer_${PACKER_VERSION}_linux_$TARGETARCH.zip"
PACKER_CHECKSUM_URL="https://releases.hashicorp.com/packer/$PACKER_VERSION/packer_${PACKER_VERSION}_SHA256SUMS"

NODEJS_VERSION="${NODEJS_VERSION:-v15.11.0}"
if [ $TARGETARCH == 'amd64' ]; then
NODEJS_FILENAME="node-$NODEJS_VERSION-linux-x64.tar.gz"
NDOEJS_FOLDER="node-$NODEJS_VERSION-linux-x64"
else
NODEJS_FILENAME="node-$NODEJS_VERSION-linux-arm64.tar.gz"
NDOEJS_FOLDER="node-$NODEJS_VERSION-linux-arm64"
fi
NODEJS_DOWNLOAD_URL="https://nodejs.org/dist/$NODEJS_VERSION/$NODEJS_FILENAME"
NODEJS_CHECKSUM_URL="https://nodejs.org/dist/$NODEJS_VERSION/SHASUMS256.txt"

HELM_VERSION="${HELM_VERSION:-3.7.1}"
HELM_DOWNLOAD_URL="https://get.helm.sh/helm-v${HELM_VERSION}-linux-$TARGETARCH.tar.gz"
HELM_CHECKSUM_URL="$HELM_DOWNLOAD_URL.sha256"

GOSS_VERSION="${GOSS_VERSION:-3.0.3}"
GOSS_DOWNLOAD_URL="https://github.com/YaleUniversity/packer-provisioner-goss/releases/download/v${GOSS_VERSION}/packer-provisioner-goss-v${GOSS_VERSION}-linux-$TARGETARCH.tar.gz"
GOSS_CHECKSUM_URL="https://github.com/YaleUniversity/packer-provisioner-goss/releases/download/v${GOSS_VERSION}/packer-provisioner-goss-v${GOSS_VERSION}_SHA256SUMS"

GOVC_VERSION="${GOVC_VERSION:-0.24.0}"
GOVC_DOWNLOAD_URL="https://github.com/vmware/govmomi/releases/download/v${GOVC_VERSION}/govc_linux_$TARGETARCH.gz"
GOVC_CHECKSUM_URL="https://github.com/vmware/govmomi/releases/download/v${GOVC_VERSION}/checksums.txt"

HUGO_VERSION=0.85.0
if [ $TARGETARCH == 'amd64' ]; then
HUGO_FILENAME="hugo_extended_${HUGO_VERSION}_Linux-64bit.tar.gz"
else
HUGO_FILENAME="nhugo_extended_${HUGO_VERSION}_Linux-<nonexistent>.tar.gz"
fi
HUGO_DOWNLOAD_URL="https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/$HUGO_FILENAME"
HUGO_CHECKSUM_URL="https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_${HUGO_VERSION}_checksums.txt"


YQ_VERSION="${YQ_VERSION:-v4.7.1}"
YQ_DOWNLOAD_URL="https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_$TARGETARCH"
YQ_CHECKSUM_URL="https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/checksums"
YQ_CHECKSUM_ORDER_URL="https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/checksums_hashes_order"
2 changes: 1 addition & 1 deletion builder-base/yq-amd64-checksum
Original file line number Diff line number Diff line change
@@ -1 +1 @@
4eee4d884da169b2bfdab545d2b149b4d6a0050e86e3906088f8fae6ee509394 yq_linux_amd64.tar.gz
16a443be2913c206b95e7bf53c086ba213a77955a31f2f134f41a529c5c62aa4 yq_linux_amd64
2 changes: 1 addition & 1 deletion builder-base/yq-arm64-checksum
Original file line number Diff line number Diff line change
@@ -1 +1 @@
92fa8586a821a44a450e7c864dc1961afdbc4196f2132e76abb9f34e09a30802 yq_linux_arm64.tar.gz
291b4b481012b6066ebb9f6672957b91a3fff45a667b0bef7a61d8e07a70b855 yq_linux_arm64

0 comments on commit 3b4ac4d

Please sign in to comment.