From 1c07b69c327bf63eb9f425f47328fb970eb549a8 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Tue, 11 Jan 2022 15:33:46 -0500 Subject: [PATCH] Change docker image from CentOS 7 to Ubuntu 20.04 (#29681) * Switch to Ubuntu 20.04. * Fix Dockerfile. * Change to amd64 in Dockerfile.tmpl. * Add missing curl and ca-certificate deps. Change back to x86_64. * Fix issues with setcap and xz extraction. * Add changelog, fix remaining issues. * Fix synthetics deps * Fix apt-get. Remove todo from packages.yml. Co-authored-by: Andrew Cholakian (cherry picked from commit febc7ddbfa4b8c7f39fb8a05b8d90864f0199e93) --- .ci/packer_cache.sh | 2 +- CHANGELOG.next.asciidoc | 4 ++ auditbeat/Dockerfile | 2 +- dev-tools/packaging/packages.yml | 18 ++--- .../docker/Dockerfile.elastic-agent.tmpl | 68 ++++++++++++------- .../templates/docker/Dockerfile.tmpl | 67 ++++++++++++------ filebeat/Dockerfile | 2 +- heartbeat/Dockerfile | 2 +- libbeat/Dockerfile | 2 +- metricbeat/Dockerfile | 2 +- packetbeat/Dockerfile | 2 +- x-pack/functionbeat/Dockerfile | 2 +- x-pack/libbeat/Dockerfile | 2 +- 13 files changed, 111 insertions(+), 64 deletions(-) diff --git a/.ci/packer_cache.sh b/.ci/packer_cache.sh index 1627343c612..9b444376924 100755 --- a/.ci/packer_cache.sh +++ b/.ci/packer_cache.sh @@ -21,7 +21,7 @@ function dockerPullCommonImages() { docker.elastic.co/observability-ci/database-enterprise:12.2.0.1 docker.elastic.co/beats-dev/fpm:1.11.0 golang:1.14.12-stretch - centos:7 + ubuntu:20.04 " for image in ${DOCKER_IMAGES} ; do (retry 2 docker pull ${image}) || echo "Error pulling ${image} Docker image. Continuing." diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 79392444ac4..451d7573c79 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -27,6 +27,10 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Remove `auto` from the available options of `setup.ilm.enabled` and set the default value to `true`. {pull}28671[28671] - Remove deprecated `--template` and `--ilm-policy` flags. Use `--index-management` instead. {pull}28870[28870] - Remove options `logging.files.suffix` and default to datetime endings. {pull}28927[28927] +- Remove Journalbeat. Use `journald` input of Filebeat instead. {pull}29131[29131] +- `include_matches` option of `journald` input no longer accepts a list of string. {pull}29294[29294] +- Add job.name in pods controlled by Jobs {pull}28954[28954] +- Change Docker base image from CentOS 7 to Ubuntu 20.04 {pull}29681[29681] *Auditbeat* diff --git a/auditbeat/Dockerfile b/auditbeat/Dockerfile index 061539b6dcf..08cff3f06a7 100644 --- a/auditbeat/Dockerfile +++ b/auditbeat/Dockerfile @@ -2,7 +2,7 @@ FROM golang:1.17.5 RUN \ apt-get update \ - && apt-get install -y --no-install-recommends \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ python3 \ python3-pip \ python3-venv \ diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index 09efabf2826..9221f1ac0c0 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -475,8 +475,8 @@ shared: - &agent_docker_spec <<: *agent_binary_spec extra_vars: - from: 'centos:7' - buildFrom: 'centos:7' + from: 'ubuntu:20.04' + buildFrom: 'ubuntu:20.04' dockerfile: 'Dockerfile.elastic-agent.tmpl' docker_entrypoint: 'docker-entrypoint.elastic-agent.tmpl' user: '{{ .BeatName }}' @@ -495,8 +495,8 @@ shared: - &agent_docker_arm_spec <<: *agent_docker_spec extra_vars: - from: 'arm64v8/centos:7' - buildFrom: 'arm64v8/centos:7' + from: 'arm64v8/ubuntu:20.04' + buildFrom: 'arm64v8/ubuntu:20.04' - &agent_docker_cloud_spec <<: *agent_docker_spec @@ -653,8 +653,8 @@ shared: - &docker_spec <<: *binary_spec extra_vars: - from: 'centos:7' - buildFrom: 'centos:7' + from: 'ubuntu:20.04' + buildFrom: 'ubuntu:20.04' user: '{{ .BeatName }}' linux_capabilities: '' files: @@ -666,8 +666,8 @@ shared: - &docker_arm_spec <<: *docker_spec extra_vars: - from: 'arm64v8/centos:7' - buildFrom: 'arm64v8/centos:7' + from: 'arm64v8/ubuntu:20.04' + buildFrom: 'arm64v8/ubuntu:20.04' - &docker_ubi_spec extra_vars: @@ -1230,4 +1230,4 @@ specs: <<: *elastic_license_for_binaries files: '{{.BeatName}}{{.BinaryExt}}': - source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} \ No newline at end of file + source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} diff --git a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl index 38f7934a9d7..644df5bd73a 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl @@ -27,42 +27,51 @@ RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/data/elastic-agent-{{ commit_s {{- end }} true -{{- if .linux_capabilities }} -# Since the beat is stored at the other end of a symlink we must follow the symlink first -# For security reasons setcap does not support symlinks. This is smart in the general case -# but in our specific case since we're building a trusted image from trusted binaries this is -# fine. Thus, we use readlink to follow the link and setcap on the actual binary -RUN readlink -f {{ $beatBinary }} | xargs setcap {{ .linux_capabilities }} -{{- end }} - FROM {{ .from }} ENV BEAT_SETUID_AS={{ .user }} {{- if contains .from "ubi-minimal" }} -RUN for iter in {1..10}; do microdnf update -y && microdnf install -y shadow-utils jq && microdnf clean all && exit_code=0 && break || exit_code=$? && echo "microdnf error: retry $iter in 10s" && sleep 10; done; (exit $exit_code) +RUN for iter in {1..10}; do microdnf update -y && microdnf install -y findutils shadow-utils && microdnf clean all && exit_code=0 && break || exit_code=$? && echo "microdnf error: retry $iter in 10s" && sleep 10; done; (exit $exit_code) {{- else }} -# Installing jq needs to be installed after epel-release and cannot be in the same yum install command. -RUN case $(arch) in aarch64) YUM_FLAGS="-x bind-license";; esac; \ - for iter in {1..10}; do \ - yum update -y $YUM_FLAGS && \ - yum install -y epel-release && \ - yum update -y $YUM_FLAGS && \ - yum install -y jq && \ - yum clean all && \ - exit_code=0 && break || exit_code=$? && echo "yum error: retry $iter in 10s" && sleep 10; \ + +RUN for iter in {1..10}; do \ + apt-get update -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends --yes ca-certificates curl libcap2-bin xz-utils && \ + apt-get clean all && \ + exit_code=0 && break || exit_code=$? && echo "apt-get error: retry $iter in 10s" && sleep 10; \ done; \ (exit $exit_code) {{- end }} {{- if (and (contains .image_name "-complete") (not (contains .from "ubi-minimal"))) }} -RUN for iter in {1..10}; do \ - yum -y install atk gtk gdk xrandr pango libXcomposite libXcursor libXdamage \ - libXext libXi libXtst libXScrnSaver libXrandr GConf2 \ - alsa-lib atk gtk3 ipa-gothic-fonts xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils \ - xorg-x11-fonts-cyrillic xorg-x11-fonts-Type1 xorg-x11-fonts-misc \ - yum clean all && \ - exit_code=0 && break || exit_code=$? && echo "yum error: retry $iter in 10s" && sleep 10; \ +RUN apt-get update -y && \ + for iter in {1..10}; do \ + DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends --yes \ + libglib2.0-0\ + libnss3\ + libnspr4\ + libatk1.0-0\ + libatk-bridge2.0-0\ + libcups2\ + libdrm2\ + libdbus-1-3\ + libxcb1\ + libxkbcommon0\ + libx11-6\ + libxcomposite1\ + libxdamage1\ + libxext6\ + libxfixes3\ + libxrandr2\ + libgbm1\ + libpango-1.0-0\ + libcairo2\ + libasound2\ + libatspi2.0-0\ + libxshmfence1 && \ + apt-get clean all && \ + exit_code=0 && break || exit_code=$? && echo "apt-get error: retry $iter in 10s" && sleep 10; \ done; \ (exit $exit_code) ENV NODE_PATH={{ $beatHome }}/.node @@ -145,6 +154,14 @@ COPY --from=home {{ $beatHome }}/NOTICE.txt /licenses COPY --from=home /opt /opt {{- end }} +{{- if .linux_capabilities }} +# Since the beat is stored at the other end of a symlink we must follow the symlink first +# For security reasons setcap does not support symlinks. This is smart in the general case +# but in our specific case since we're building a trusted image from trusted binaries this is +# fine. Thus, we use readlink to follow the link and setcap on the actual binary +RUN readlink -f {{ $beatBinary }} | xargs setcap {{ .linux_capabilities }} +{{- end }} + {{- if eq .user "root" }} {{- if contains .image_name "-cloud" }} # Generate folder for a stub command that will be overwritten at runtime @@ -213,3 +230,4 @@ RUN echo -e '#!/bin/sh\nexec /usr/local/bin/docker-entrypoint' > /app/apm.sh && {{- else }} ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/docker-entrypoint"] {{- end }} + diff --git a/dev-tools/packaging/templates/docker/Dockerfile.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.tmpl index 91a636f50b0..9309516bd80 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.tmpl @@ -18,36 +18,52 @@ RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/logs && \ {{- end }} chmod 0775 {{ $beatHome }}/data {{ $beatHome }}/logs -{{- if .linux_capabilities }} -# Since the beat is stored at the other end of a symlink we must follow the symlink first -# For security reasons setcap does not support symlinks. This is smart in the general case -# but in our specific case since we're building a trusted image from trusted binaries this is -# fine. Thus, we use readlink to follow the link and setcap on the actual binary -RUN readlink -f {{ $beatBinary }} | xargs setcap {{ .linux_capabilities }} -{{- end }} - FROM {{ .from }} {{- if contains .from "ubi-minimal" }} RUN microdnf -y update && \ - microdnf install shadow-utils && \ + microdnf install findutils shadow-utils && \ microdnf clean all {{- else }} -# FIXME: Package bind-license failed to update in arm -RUN case $(arch) in aarch64) YUM_FLAGS="-x bind-license";; esac; \ - yum -y update $YUM_FLAGS \ - {{- if (eq .BeatName "heartbeat") }} - && yum -y install epel-release \ - && yum -y install atk gtk gdk xrandr pango libXcomposite libXcursor libXdamage \ - libXext libXi libXtst libXScrnSaver libXrandr GConf2 \ - alsa-lib atk gtk3 ipa-gothic-fonts xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils \ - xorg-x11-fonts-cyrillic xorg-x11-fonts-Type1 xorg-x11-fonts-misc \ - {{- end }} - && yum clean all && rm -rf /var/cache/yum - # See https://access.redhat.com/discussions/3195102 for why rm is needed +RUN for iter in {1..10}; do \ + apt-get update -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends --yes ca-certificates curl libcap2-bin xz-utils && \ + apt-get clean all && \ + exit_code=0 && break || exit_code=$? && echo "apt-get error: retry $iter in 10s" && sleep 10; \ + done; \ + (exit $exit_code) {{- end }} {{- if (and (eq .BeatName "heartbeat") (not (contains .from "ubi-minimal"))) }} +RUN apt-get update -y && \ + for iter in {1..10}; do \ + DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends --yes \ + libglib2.0-0\ + libnss3\ + libnspr4\ + libatk1.0-0\ + libatk-bridge2.0-0\ + libcups2\ + libdrm2\ + libdbus-1-3\ + libxcb1\ + libxkbcommon0\ + libx11-6\ + libxcomposite1\ + libxdamage1\ + libxext6\ + libxfixes3\ + libxrandr2\ + libgbm1\ + libpango-1.0-0\ + libcairo2\ + libasound2\ + libatspi2.0-0\ + libxshmfence1 && \ + apt-get clean all && \ + exit_code=0 && break || exit_code=$? && echo "apt-get error: retry $iter in 10s" && sleep 10; \ + done; \ + (exit $exit_code) ENV NODE_PATH={{ $beatHome }}/.node RUN echo \ $NODE_PATH \ @@ -93,6 +109,7 @@ RUN set -e ; \ TINI_BIN=""; \ TINI_SHA256=""; \ TINI_VERSION="v0.19.0"; \ + echo "The arch value is $(arch)"; \ case "$(arch)" in \ x86_64) \ TINI_BIN="tini-amd64"; \ @@ -120,6 +137,14 @@ RUN mkdir /licenses COPY --from=home {{ $beatHome }}/LICENSE.txt /licenses COPY --from=home {{ $beatHome }}/NOTICE.txt /licenses +{{- if .linux_capabilities }} +# Since the beat is stored at the other end of a symlink we must follow the symlink first +# For security reasons setcap does not support symlinks. This is smart in the general case +# but in our specific case since we're building a trusted image from trusted binaries this is +# fine. Thus, we use readlink to follow the link and setcap on the actual binary +RUN readlink -f {{ $beatBinary }} | xargs setcap {{ .linux_capabilities }} +{{- end }} + {{- if ne .user "root" }} RUN groupadd --gid 1000 {{ .BeatName }} RUN useradd -M --uid 1000 --gid 1000 --groups 0 --home {{ $beatHome }} {{ .user }} diff --git a/filebeat/Dockerfile b/filebeat/Dockerfile index 763572e5a99..3b17f95e998 100644 --- a/filebeat/Dockerfile +++ b/filebeat/Dockerfile @@ -2,7 +2,7 @@ FROM golang:1.17.5 RUN \ apt-get update \ - && apt-get install -y --no-install-recommends \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ libsystemd-dev \ netcat \ rsync \ diff --git a/heartbeat/Dockerfile b/heartbeat/Dockerfile index 355510ff31f..f4a1faae369 100644 --- a/heartbeat/Dockerfile +++ b/heartbeat/Dockerfile @@ -2,7 +2,7 @@ FROM golang:1.17.5 RUN \ apt-get update \ - && apt-get install -y --no-install-recommends \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ netcat \ python3 \ python3-pip \ diff --git a/libbeat/Dockerfile b/libbeat/Dockerfile index 3ff2e7a8ce4..97a1b634384 100644 --- a/libbeat/Dockerfile +++ b/libbeat/Dockerfile @@ -2,7 +2,7 @@ FROM golang:1.17.5 RUN \ apt-get update \ - && apt-get install -y --no-install-recommends \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ netcat \ libpcap-dev \ python3 \ diff --git a/metricbeat/Dockerfile b/metricbeat/Dockerfile index d9166f054f1..e1c97b72d36 100644 --- a/metricbeat/Dockerfile +++ b/metricbeat/Dockerfile @@ -2,7 +2,7 @@ FROM golang:1.17.5 RUN \ apt update \ - && apt install -qq -y --no-install-recommends \ + && DEBIAN_FRONTEND=noninteractive apt-get install -qq -y --no-install-recommends \ netcat \ python3 \ python3-dev \ diff --git a/packetbeat/Dockerfile b/packetbeat/Dockerfile index 6223e3d70f0..0a5a4a84128 100644 --- a/packetbeat/Dockerfile +++ b/packetbeat/Dockerfile @@ -2,7 +2,7 @@ FROM golang:1.17.5 RUN \ apt-get update \ - && apt-get install -y --no-install-recommends \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ python3 \ python3-pip \ python3-venv \ diff --git a/x-pack/functionbeat/Dockerfile b/x-pack/functionbeat/Dockerfile index f07760a8487..5a75f945955 100644 --- a/x-pack/functionbeat/Dockerfile +++ b/x-pack/functionbeat/Dockerfile @@ -2,7 +2,7 @@ FROM golang:1.17.5 RUN \ apt-get update \ - && apt-get install -y --no-install-recommends \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ netcat \ rsync \ python3 \ diff --git a/x-pack/libbeat/Dockerfile b/x-pack/libbeat/Dockerfile index 12ce0e09203..78c387b04d9 100644 --- a/x-pack/libbeat/Dockerfile +++ b/x-pack/libbeat/Dockerfile @@ -2,7 +2,7 @@ FROM golang:1.17.5 RUN \ apt-get update \ - && apt-get install -y --no-install-recommends \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ netcat \ rsync \ python3 \