From ae02c5f9d5ff3adc4e47ba991fa1ea3c41318ee5 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 15 Apr 2021 16:40:59 +0100 Subject: [PATCH] packer cache support for the 7.x and 7.latestMinor branches (#25091) --- .ci/packer_cache.sh | 129 ++++++++++++++++++++++---------------------- 1 file changed, 63 insertions(+), 66 deletions(-) mode change 100644 => 100755 .ci/packer_cache.sh diff --git a/.ci/packer_cache.sh b/.ci/packer_cache.sh old mode 100644 new mode 100755 index 86f3a51c3273..e4248118fae4 --- a/.ci/packer_cache.sh +++ b/.ci/packer_cache.sh @@ -9,75 +9,23 @@ source /usr/local/bin/bash_standard_lib.sh # shellcheck disable=SC1091 source ./dev-tools/common.bash -# Docker images used on Dockerfiles 2019-07-12 -# aerospike:3.9.0 -# alpine:edge -# apache/couchdb:1.7 -# busybox:latest -# ceph/daemon:master-6373c6a-jewel-centos-7-x86_64 -# cockroachdb/cockroach:v19.1.1 -# consul:1.4.2 -# coredns/coredns:1.5.0 -# couchbase:4.5.1 -# debian:latest -# debian:stretch -# docker.elastic.co/beats/metricbeat:6.5.4 -# docker.elastic.co/beats/metricbeat:7.2.0 -# envoyproxy/envoy:v1.7.0 -# exekias/localkube-image -# haproxy:1.8 -# httpd:2.4.20 -# java:8-jdk-alpine -# jplock/zookeeper:3.4.8 -# maven:3.3-jdk-8 -# memcached:1.4.35-alpine -# microsoft/mssql-server-linux:2017-GA -# mongo:3.4 -# mysql:5.7.12 -# nats:1.3.0 -# nginx:1.9 -# oraclelinux:7 -# postgres:9.5.3 -# prom/prometheus:v2.6.0 -# python:3.6-alpine -# quay.io/coreos/etcd:v3.3.10 -# rabbitmq:3.7.4-management -# redis:3.2.12-alpine -# redis:3.2.4-alpine -# store/oracle/database-enterprise:12.2.0.1 -# traefik:1.6-alpine -# tsouza/nginx-php-fpm:php-7.1 -# ubuntu:16.04 -# ubuntu:trusty +###################### +############ FUNCTIONS +###################### +function getBeatsVersion() { + grep 'defaultBeatVersion' libbeat/version/version.go | cut -d= -f2 | sed 's#"##g' | tr -d " " +} -get_go_version - -DOCKER_IMAGES="docker.elastic.co/observability-ci/database-instantclient:12.2.0.1 -docker.elastic.co/observability-ci/database-enterprise:12.2.0.1 -docker.elastic.co/beats-dev/fpm:1.11.0 -docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-arm -docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-base-arm-debian9 -docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-darwin -docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main -docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main-debian7 -docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main-debian8 -docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main-debian9 -docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-mips -docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-ppc -docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-s390x -docker.elastic.co/elasticsearch/elasticsearch:8.0.0-SNAPSHOT -docker.elastic.co/kibana/kibana:8.0.0-SNAPSHOT -docker.elastic.co/logstash/logstash:8.0.0-SNAPSHOT -golang:${GO_VERSION} -golang:1.14.12-stretch -centos:7 -" -if [ -x "$(command -v docker)" ]; then - for image in ${DOCKER_IMAGES} - do +function dockerPullCommonImages() { + DOCKER_IMAGES="docker.elastic.co/observability-ci/database-instantclient:12.2.0.1 + docker.elastic.co/observability-ci/database-enterprise:12.2.0.1 + docker.elastic.co/beats-dev/fpm:1.11.0 + golang:1.14.12-stretch + centos:7 + " + for image in ${DOCKER_IMAGES} ; do (retry 2 docker pull ${image}) || echo "Error pulling ${image} Docker image. Continuing." done - docker tag \ docker.elastic.co/observability-ci/database-instantclient:12.2.0.1 \ store/oracle/database-instantclient:12.2.0.1 \ @@ -86,4 +34,53 @@ if [ -x "$(command -v docker)" ]; then docker.elastic.co/observability-ci/database-enterprise:12.2.0.1 \ store/oracle/database-enterprise:12.2.0.1 \ || echo "Error setting the Oracle Database tag" +} + +function dockerPullImages() { + SNAPSHOT=$1 + get_go_version + + DOCKER_IMAGES=" + docker.elastic.co/elasticsearch/elasticsearch:${SNAPSHOT} + docker.elastic.co/kibana/kibana:${SNAPSHOT} + docker.elastic.co/logstash/logstash:${SNAPSHOT} + docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-arm + docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-base-arm-debian9 + docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-darwin + docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main + docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main-debian7 + docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main-debian8 + docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main-debian9 + docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-mips + docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-ppc + docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-s390x + golang:${GO_VERSION} + " + for image in ${DOCKER_IMAGES} + do + (retry 2 docker pull ${image}) || echo "Error pulling ${image} Docker image. Continuing." + done +} + +################# +############ MAIN +################# +if [ -x "$(command -v docker)" ]; then + set -x + echo "Docker pull common docker images" + dockerPullCommonImages + + ## GitHub api returns up to 100 entries. + ## Probably we need a different approach to search the latest minor. + latestMinor=$(curl -s https://api.github.com/repos/elastic/beats/branches\?per_page=100 | jq -r '.[].name' | grep "^7." | tail -1) + + for branch in master 7.x $latestMinor ; do + if [ "$branch" != "master" ] ; then + echo "Checkout the branch $branch" + git checkout "$branch" + fi + + VERSION=$(getBeatsVersion) + dockerPullImages "${VERSION}-SNAPSHOT" + done fi