Change docker image from CentOS 7 to Ubuntu 20.04 (#29681)

* Switch to Ubuntu 20.04.

* Fix Dockerfile.

* Change to amd64 in Dockerfile.tmpl.

* Add missing curl and ca-certificate deps. Change back to x86_64.

* Fix issues with setcap and xz extraction.

* Add changelog, fix remaining issues.

* Fix synthetics deps

* Fix apt-get. Remove todo from packages.yml.

Co-authored-by: Andrew Cholakian <andrewvc@elastic.co>
(cherry picked from commit febc7dd)

.ci/packer_cache.sh

@@ -21,7 +21,7 @@ function dockerPullCommonImages() {
   docker.elastic.co/observability-ci/database-enterprise:12.2.0.1
   docker.elastic.co/beats-dev/fpm:1.11.0
   golang:1.14.12-stretch
-  centos:7
+  ubuntu:20.04
   "
   for image in ${DOCKER_IMAGES} ; do
     (retry 2 docker pull ${image}) || echo "Error pulling ${image} Docker image. Continuing."

CHANGELOG.next.asciidoc

@@ -27,6 +27,10 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Remove `auto` from the available options of `setup.ilm.enabled` and set the default value to `true`. {pull}28671[28671]
 - Remove deprecated `--template` and `--ilm-policy` flags. Use `--index-management` instead. {pull}28870[28870]
 - Remove options `logging.files.suffix` and default to datetime endings. {pull}28927[28927]
+- Remove Journalbeat. Use `journald` input of Filebeat instead. {pull}29131[29131]
+- `include_matches` option of `journald` input no longer accepts a list of string. {pull}29294[29294]
+- Add job.name in pods controlled by Jobs {pull}28954[28954]
+- Change Docker base image from CentOS 7 to Ubuntu 20.04 {pull}29681[29681]
 
 *Auditbeat*
 

auditbeat/Dockerfile

@@ -2,7 +2,7 @@ FROM golang:1.17.5
 
 RUN \
     apt-get update \
-      && apt-get install -y --no-install-recommends \
+      && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
          python3 \
          python3-pip \
          python3-venv \

dev-tools/packaging/packages.yml

@@ -475,8 +475,8 @@ shared:
   - &agent_docker_spec
     <<: *agent_binary_spec
     extra_vars:
-      from: 'centos:7'
-      buildFrom: 'centos:7'
+      from: 'ubuntu:20.04'
+      buildFrom: 'ubuntu:20.04'
       dockerfile: 'Dockerfile.elastic-agent.tmpl'
       docker_entrypoint: 'docker-entrypoint.elastic-agent.tmpl'
       user: '{{ .BeatName }}'
@@ -495,8 +495,8 @@ shared:
   - &agent_docker_arm_spec
     <<: *agent_docker_spec
     extra_vars:
-      from: 'arm64v8/centos:7'
-      buildFrom: 'arm64v8/centos:7'
+      from: 'arm64v8/ubuntu:20.04'
+      buildFrom: 'arm64v8/ubuntu:20.04'
 
   - &agent_docker_cloud_spec
     <<: *agent_docker_spec
@@ -653,8 +653,8 @@ shared:
   - &docker_spec
     <<: *binary_spec
     extra_vars:
-      from: 'centos:7'
-      buildFrom: 'centos:7'
+      from: 'ubuntu:20.04'
+      buildFrom: 'ubuntu:20.04'
       user: '{{ .BeatName }}'
       linux_capabilities: ''
     files:
@@ -666,8 +666,8 @@ shared:
   - &docker_arm_spec
     <<: *docker_spec
     extra_vars:
-      from: 'arm64v8/centos:7'
-      buildFrom: 'arm64v8/centos:7'
+      from: 'arm64v8/ubuntu:20.04'
+      buildFrom: 'arm64v8/ubuntu:20.04'
 
   - &docker_ubi_spec
     extra_vars:
@@ -1230,4 +1230,4 @@ specs:
         <<: *elastic_license_for_binaries
         files:
           '{{.BeatName}}{{.BinaryExt}}':
-            source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
+            source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}

dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl

@@ -27,42 +27,51 @@ RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/data/elastic-agent-{{ commit_s
 {{- end }}
     true
 
-{{- if .linux_capabilities }}
-# Since the beat is stored at the other end of a symlink we must follow the symlink first
-# For security reasons setcap does not support symlinks. This is smart in the general case
-# but in our specific case since we're building a trusted image from trusted binaries this is
-# fine. Thus, we use readlink to follow the link and setcap on the actual binary
-RUN readlink -f {{ $beatBinary }} | xargs setcap {{ .linux_capabilities }}
-{{- end }}
-
 FROM {{ .from }}
 
 ENV BEAT_SETUID_AS={{ .user }}
 
 {{- if contains .from "ubi-minimal" }}
-RUN for iter in {1..10}; do microdnf update -y && microdnf install -y shadow-utils jq &&  microdnf clean all && exit_code=0 && break || exit_code=$? && echo "microdnf error: retry $iter in 10s" && sleep 10; done; (exit $exit_code)
+RUN for iter in {1..10}; do microdnf update -y && microdnf install -y findutils shadow-utils && microdnf clean all && exit_code=0 && break || exit_code=$? && echo "microdnf error: retry $iter in 10s" && sleep 10; done; (exit $exit_code)
 {{- else }}
-# Installing jq needs to be installed after epel-release and cannot be in the same yum install command.
-RUN case $(arch) in aarch64) YUM_FLAGS="-x bind-license";; esac; \
-    for iter in {1..10}; do \
-        yum update -y $YUM_FLAGS && \
-        yum install -y epel-release &&  \
-        yum update -y $YUM_FLAGS && \
-        yum install -y jq && \
-        yum clean all && \
-        exit_code=0 && break || exit_code=$? && echo "yum error: retry $iter in 10s" && sleep 10; \
+
+RUN for iter in {1..10}; do \
+        apt-get update -y && \
+        DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends --yes ca-certificates curl libcap2-bin xz-utils && \
+        apt-get clean all && \
+        exit_code=0 && break || exit_code=$? && echo "apt-get error: retry $iter in 10s" && sleep 10; \
     done; \
     (exit $exit_code)
 {{- end }}
 
 {{- if (and (contains .image_name "-complete") (not (contains .from "ubi-minimal"))) }}
-RUN for iter in {1..10}; do \
-        yum -y install atk gtk gdk xrandr pango libXcomposite libXcursor libXdamage \
-        libXext libXi libXtst libXScrnSaver libXrandr GConf2 \
-        alsa-lib atk gtk3 ipa-gothic-fonts xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils \
-        xorg-x11-fonts-cyrillic xorg-x11-fonts-Type1 xorg-x11-fonts-misc \
-        yum clean all && \
-        exit_code=0 && break || exit_code=$? && echo "yum error: retry $iter in 10s" && sleep 10; \
+RUN apt-get update -y && \
+    for iter in {1..10}; do \
+        DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends --yes \
+        libglib2.0-0\
+        libnss3\
+        libnspr4\
+        libatk1.0-0\
+        libatk-bridge2.0-0\
+        libcups2\
+        libdrm2\
+        libdbus-1-3\
+        libxcb1\
+        libxkbcommon0\
+        libx11-6\
+        libxcomposite1\
+        libxdamage1\
+        libxext6\
+        libxfixes3\
+        libxrandr2\
+        libgbm1\
+        libpango-1.0-0\
+        libcairo2\
+        libasound2\
+        libatspi2.0-0\
+        libxshmfence1 && \
+        apt-get clean all && \
+        exit_code=0 && break || exit_code=$? && echo "apt-get error: retry $iter in 10s" && sleep 10; \
     done; \
     (exit $exit_code)
 ENV NODE_PATH={{ $beatHome }}/.node
@@ -145,6 +154,14 @@ COPY --from=home {{ $beatHome }}/NOTICE.txt /licenses
 COPY --from=home /opt /opt
 {{- end }}
 
+{{- if .linux_capabilities }}
+# Since the beat is stored at the other end of a symlink we must follow the symlink first
+# For security reasons setcap does not support symlinks. This is smart in the general case
+# but in our specific case since we're building a trusted image from trusted binaries this is
+# fine. Thus, we use readlink to follow the link and setcap on the actual binary
+RUN readlink -f {{ $beatBinary }} | xargs setcap {{ .linux_capabilities }}
+{{- end }}
+
 {{- if eq .user "root" }}
 {{- if contains .image_name "-cloud" }}
 # Generate folder for a stub command that will be overwritten at runtime
@@ -213,3 +230,4 @@ RUN echo -e '#!/bin/sh\nexec /usr/local/bin/docker-entrypoint' > /app/apm.sh &&
 {{- else }}
 ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/docker-entrypoint"]
 {{- end }}
+

dev-tools/packaging/templates/docker/Dockerfile.tmpl

@@ -18,36 +18,52 @@ RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/logs && \
 {{- end }}
     chmod 0775 {{ $beatHome }}/data {{ $beatHome }}/logs
 
-{{- if .linux_capabilities }}
-# Since the beat is stored at the other end of a symlink we must follow the symlink first
-# For security reasons setcap does not support symlinks. This is smart in the general case
-# but in our specific case since we're building a trusted image from trusted binaries this is
-# fine. Thus, we use readlink to follow the link and setcap on the actual binary
-RUN readlink -f {{ $beatBinary }} | xargs setcap {{ .linux_capabilities }}
-{{- end }}
-
 FROM {{ .from }}
 
 {{- if contains .from "ubi-minimal" }}
 RUN microdnf -y update && \
-    microdnf install shadow-utils && \
+    microdnf install findutils shadow-utils && \
     microdnf clean all
 {{- else }}
-# FIXME: Package bind-license failed to update in arm
-RUN case $(arch) in aarch64) YUM_FLAGS="-x bind-license";; esac; \
-    yum -y update $YUM_FLAGS \
-  {{- if (eq .BeatName "heartbeat") }}
-    && yum -y install epel-release \
-    && yum -y install atk gtk gdk xrandr pango libXcomposite libXcursor libXdamage \
-        libXext libXi libXtst libXScrnSaver libXrandr GConf2 \
-        alsa-lib atk gtk3 ipa-gothic-fonts xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils \
-        xorg-x11-fonts-cyrillic xorg-x11-fonts-Type1 xorg-x11-fonts-misc \
-  {{- end }}
-  && yum clean all && rm -rf /var/cache/yum
-  # See https://access.redhat.com/discussions/3195102 for why rm is needed
+RUN for iter in {1..10}; do \
+        apt-get update -y && \
+        DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends --yes ca-certificates curl libcap2-bin xz-utils && \
+        apt-get clean all && \
+        exit_code=0 && break || exit_code=$? && echo "apt-get error: retry $iter in 10s" && sleep 10; \
+    done; \
+    (exit $exit_code)
 {{- end }}
 
 {{- if (and (eq .BeatName "heartbeat") (not (contains .from "ubi-minimal"))) }}
+RUN apt-get update -y && \
+    for iter in {1..10}; do \
+        DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends --yes \
+        libglib2.0-0\
+        libnss3\
+        libnspr4\
+        libatk1.0-0\
+        libatk-bridge2.0-0\
+        libcups2\
+        libdrm2\
+        libdbus-1-3\
+        libxcb1\
+        libxkbcommon0\
+        libx11-6\
+        libxcomposite1\
+        libxdamage1\
+        libxext6\
+        libxfixes3\
+        libxrandr2\
+        libgbm1\
+        libpango-1.0-0\
+        libcairo2\
+        libasound2\
+        libatspi2.0-0\
+        libxshmfence1 && \
+        apt-get clean all && \
+        exit_code=0 && break || exit_code=$? && echo "apt-get error: retry $iter in 10s" && sleep 10; \
+    done; \
+    (exit $exit_code)
 ENV NODE_PATH={{ $beatHome }}/.node
 RUN echo \
     $NODE_PATH \
@@ -93,6 +109,7 @@ RUN set -e ; \
   TINI_BIN=""; \
   TINI_SHA256=""; \
   TINI_VERSION="v0.19.0"; \
+  echo "The arch value is $(arch)"; \
   case "$(arch)" in \
     x86_64) \
         TINI_BIN="tini-amd64"; \
@@ -120,6 +137,14 @@ RUN mkdir /licenses
 COPY --from=home {{ $beatHome }}/LICENSE.txt /licenses
 COPY --from=home {{ $beatHome }}/NOTICE.txt /licenses
 
+{{- if .linux_capabilities }}
+# Since the beat is stored at the other end of a symlink we must follow the symlink first
+# For security reasons setcap does not support symlinks. This is smart in the general case
+# but in our specific case since we're building a trusted image from trusted binaries this is
+# fine. Thus, we use readlink to follow the link and setcap on the actual binary
+RUN readlink -f {{ $beatBinary }} | xargs setcap {{ .linux_capabilities }}
+{{- end }}
+
 {{- if ne .user "root" }}
 RUN groupadd --gid 1000 {{ .BeatName }}
 RUN useradd -M --uid 1000 --gid 1000 --groups 0 --home {{ $beatHome }} {{ .user }}

filebeat/Dockerfile

@@ -2,7 +2,7 @@ FROM golang:1.17.5
 
 RUN \
     apt-get update \
-      && apt-get install -y --no-install-recommends \
+      && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
          libsystemd-dev \
          netcat \
          rsync \

heartbeat/Dockerfile

@@ -2,7 +2,7 @@ FROM golang:1.17.5
 
 RUN \
     apt-get update \
-      && apt-get install -y --no-install-recommends \
+      && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
          netcat \
          python3 \
          python3-pip \

libbeat/Dockerfile

@@ -2,7 +2,7 @@ FROM golang:1.17.5
 
 RUN \
     apt-get update \
-      && apt-get install -y --no-install-recommends \
+      && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
          netcat \
          libpcap-dev \
          python3 \

metricbeat/Dockerfile

@@ -2,7 +2,7 @@ FROM golang:1.17.5
 
 RUN \
     apt update \
-      && apt install -qq -y --no-install-recommends \
+      && DEBIAN_FRONTEND=noninteractive apt-get install -qq -y --no-install-recommends \
          netcat \
          python3 \
          python3-dev \

packetbeat/Dockerfile

@@ -2,7 +2,7 @@ FROM golang:1.17.5
 
 RUN \
     apt-get update \
-      && apt-get install -y --no-install-recommends \
+      && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
          python3 \
          python3-pip \
          python3-venv \

x-pack/functionbeat/Dockerfile

@@ -2,7 +2,7 @@ FROM golang:1.17.5
 
 RUN \
     apt-get update \
-      && apt-get install -y --no-install-recommends \
+      && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
          netcat \
          rsync \
          python3 \

x-pack/libbeat/Dockerfile

@@ -2,7 +2,7 @@ FROM golang:1.17.5
 
 RUN \
     apt-get update \
-      && apt-get install -y --no-install-recommends \
+      && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
          netcat \
          rsync \
          python3 \