From cf4c99bf7d38edd205c1784e61383b8e273be599 Mon Sep 17 00:00:00 2001 From: Jeff Cheng <83052155+jeffreyc-splunk@users.noreply.github.com> Date: Mon, 27 Mar 2023 15:33:18 -0400 Subject: [PATCH] Update installer script for amazonlinux 2023 and rhel 9 (#2865) * Update installer script for amazonlinux 2023 and rhel 9 * Update linux-installer.md * Update test images * Create test matrix for distros --- .github/workflows/installer-script-test.yml | 41 ++++++++++++++----- docs/getting-started/linux-installer.md | 8 ++-- .../packaging/installer/install.sh | 34 +++++++++++---- .../tests/images/rpm/Dockerfile.amazonlinux-2 | 2 +- .../images/rpm/Dockerfile.amazonlinux-2023 | 20 +++++++++ .../tests/images/rpm/Dockerfile.centos-8 | 6 +-- .../tests/images/rpm/Dockerfile.centos-9 | 20 +++++++++ .../tests/images/rpm/Dockerfile.oraclelinux-9 | 20 +++++++++ .../packaging/tests/installer_test.py | 19 ++++++--- 9 files changed, 137 insertions(+), 33 deletions(-) create mode 100644 internal/buildscripts/packaging/tests/images/rpm/Dockerfile.amazonlinux-2023 create mode 100644 internal/buildscripts/packaging/tests/images/rpm/Dockerfile.centos-9 create mode 100644 internal/buildscripts/packaging/tests/images/rpm/Dockerfile.oraclelinux-9 diff --git a/.github/workflows/installer-script-test.yml b/.github/workflows/installer-script-test.yml index 3690dad0ed4..de28c742dfc 100644 --- a/.github/workflows/installer-script-test.yml +++ b/.github/workflows/installer-script-test.yml @@ -19,14 +19,35 @@ concurrency: cancel-in-progress: true jobs: + installer-test-matrix: + runs-on: ubuntu-20.04 + steps: + - name: Check out code + uses: actions/checkout@v3 + + - name: Get distros + id: get-matrix + run: | + # create test matrix for distro images + dockerfiles=$(ls internal/buildscripts/packaging/tests/images/deb/Dockerfile.* internal/buildscripts/packaging/tests/images/rpm/Dockerfile.* | cut -d '.' -f2- | sort -u) + if [ -z "$dockerfiles" ]; then + echo "Failed to get dockerfiles from internal/buildscripts/packaging/tests/images!" >&2 + exit 1 + fi + distro=$(for d in $dockerfiles; do echo -n "\"$d\","; done) + matrix="{\"DISTRO\": [${distro%,}]}" + echo "$matrix" | jq + echo "matrix=${matrix}" >> $GITHUB_OUTPUT + outputs: + matrix: ${{ steps.get-matrix.outputs.matrix }} + linux-installer-script-test: - name: linux-installer-script-test # Use 20.04.5 until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16450 is resolved runs-on: ubuntu-20.04 + needs: installer-test-matrix strategy: - matrix: - PACKAGE_TYPE: [ "deb", "rpm" ] - INSTRUMENTATION: [ "true", "false" ] + matrix: ${{ fromJSON(needs.installer-test-matrix.outputs.matrix) }} + fail-fast: false env: PYTHON_VERSION: '3.10' PIP_VERSION: '22.0.4' @@ -57,12 +78,12 @@ jobs: timeout-minutes: 45 run: | mkdir -p ${{ env.RESULT_PATH }} - if [ "${{ matrix.INSTRUMENTATION }}" = "true" ]; then - markers="${{ matrix.PACKAGE_TYPE }} and instrumentation" - else - markers="${{ matrix.PACKAGE_TYPE }} and not instrumentation" + distro="${{ matrix.DISTRO }}" + if [[ "$distro" = "amazonlinux-2" ]]; then + # workaround for pytest substring matching + distro="amazonlinux-2 and not amazonlinux-2023" fi - pytest -n2 --verbose -m "$markers" \ + pytest -n auto --verbose -k "$distro" \ --junitxml=${{ env.RESULT_PATH }}/results.xml \ --html=${{ env.RESULT_PATH }}/results.html \ --self-contained-html \ @@ -71,5 +92,5 @@ jobs: - name: Uploading artifacts uses: actions/upload-artifact@v3 with: - name: linux-installer-script-test-${{ matrix.PACKAGE_TYPE }}-${{ matrix.INSTRUMENTATION }}-result + name: linux-installer-script-test-${{ matrix.DISTRO }}-result path: ${{ env.RESULT_PATH }} diff --git a/docs/getting-started/linux-installer.md b/docs/getting-started/linux-installer.md index 2911c80cd16..906876bea94 100644 --- a/docs/getting-started/linux-installer.md +++ b/docs/getting-started/linux-installer.md @@ -5,10 +5,10 @@ For non-containerized Linux environments, an installer script is available. The script deploys and configures: -- Splunk OpenTelemetry Collector for Linux +- Splunk OpenTelemetry Collector for Linux (**x86_64/amd64 and aarch64/arm64 platforms only**) - [SignalFx Smart Agent and collectd bundle](https://github.com/signalfx/signalfx-agent/releases) (**x86_64/amd64 platforms only**) - [Fluentd (via the TD Agent)](https://www.fluentd.org/) - - Optional, **enabled** by default + - Optional, **enabled** by default for supported Linux distributions - See the [Fluentd Configuration](#fluentd-configuration) section for additional information, including how to skip installation. - [Splunk OpenTelemetry Auto Instrumentation for Java](https://github.com/signalfx/splunk-otel-collector/tree/main/instrumentation#linux-java-auto-instrumentation) - Optional, **disabled** by default @@ -18,8 +18,8 @@ script deploys and configures: Currently, the following Linux distributions and versions are supported: -- Amazon Linux: 2 -- CentOS / Red Hat / Oracle: 7, 8 +- Amazon Linux: 2, 2023 (**Note:** Log collection with Fluentd not currently supported for Amazon Linux 2023.) +- CentOS / Red Hat / Oracle: 7, 8, 9 - Debian: 9, 10, 11 - SUSE: 12, 15 (**Note:** Only for Collector versions v0.34.0 or higher. Log collection with Fluentd not currently supported.) - Ubuntu: 16.04, 18.04, 20.04, 22.04 diff --git a/internal/buildscripts/packaging/installer/install.sh b/internal/buildscripts/packaging/installer/install.sh index 3f058f5e161..4fe2e29a3c0 100755 --- a/internal/buildscripts/packaging/installer/install.sh +++ b/internal/buildscripts/packaging/installer/install.sh @@ -729,7 +729,7 @@ distro_is_supported() { ;; amzn) case "$distro_version" in - 2) + 2|2023) return 0 ;; esac @@ -743,7 +743,7 @@ distro_is_supported() { ;; centos|ol|rhel) case "$distro_version" in - 7*|8*) + 7*|8*|9*) return 0 ;; esac @@ -752,6 +752,21 @@ distro_is_supported() { return 1 } +fluentd_supported() { + case "$distro" in + amzn) + if [ "$distro_version" != "2" ]; then + return 1 + fi + ;; + sles|opensuse*) + return 1 + ;; + esac + + return 0 +} + parse_args_and_install() { local access_token= local api_url= @@ -863,7 +878,12 @@ parse_args_and_install() { uninstall="true" ;; --with-fluentd) - with_fluentd="true" + if ! fluentd_supported; then + echo "WARNING: Ignoring the --with-fluentd option since fluentd is currently not supported for ${distro}:${distro_version}." >&2 + with_fluentd="false" + else + with_fluentd="true" + fi ;; --without-fluentd) with_fluentd="false" @@ -972,11 +992,9 @@ parse_args_and_install() { exit 1 fi - case "$distro" in - sles|opensuse*) - with_fluentd="false" - ;; - esac + if ! fluentd_supported; then + with_fluentd="false" + fi ensure_not_installed "$with_fluentd" "$with_instrumentation" diff --git a/internal/buildscripts/packaging/tests/images/rpm/Dockerfile.amazonlinux-2 b/internal/buildscripts/packaging/tests/images/rpm/Dockerfile.amazonlinux-2 index fce892dae7f..9894257c23d 100644 --- a/internal/buildscripts/packaging/tests/images/rpm/Dockerfile.amazonlinux-2 +++ b/internal/buildscripts/packaging/tests/images/rpm/Dockerfile.amazonlinux-2 @@ -4,7 +4,7 @@ FROM amazonlinux:2 ENV container docker -RUN yum install -y curl procps initscripts systemd wget +RUN yum install -y curl procps initscripts systemd wget tar RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i = \ "systemd-tmpfiles-setup.service" ] || rm -f $i; done); \ diff --git a/internal/buildscripts/packaging/tests/images/rpm/Dockerfile.amazonlinux-2023 b/internal/buildscripts/packaging/tests/images/rpm/Dockerfile.amazonlinux-2023 new file mode 100644 index 00000000000..19a5a64382b --- /dev/null +++ b/internal/buildscripts/packaging/tests/images/rpm/Dockerfile.amazonlinux-2023 @@ -0,0 +1,20 @@ +# A amazonlinux2023 image with systemd enabled. Must be run with: +# `-d --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:ro` flags +FROM amazonlinux:2023 + +ENV container docker + +RUN yum install -y procps initscripts systemd wget tar + +RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i = \ + "systemd-tmpfiles-setup.service" ] || rm -f $i; done); \ + rm -f /lib/systemd/system/multi-user.target.wants/*;\ + rm -f /lib/systemd/system/local-fs.target.wants/*; \ + rm -f /lib/systemd/system/sockets.target.wants/*udev*; \ + rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \ + rm -f /lib/systemd/system/basic.target.wants/*;\ + rm -f /lib/systemd/system/anaconda.target.wants/*; + +VOLUME [ "/sys/fs/cgroup" ] + +CMD ["/usr/sbin/init"] diff --git a/internal/buildscripts/packaging/tests/images/rpm/Dockerfile.centos-8 b/internal/buildscripts/packaging/tests/images/rpm/Dockerfile.centos-8 index b9afa24c66f..92df82b774f 100644 --- a/internal/buildscripts/packaging/tests/images/rpm/Dockerfile.centos-8 +++ b/internal/buildscripts/packaging/tests/images/rpm/Dockerfile.centos-8 @@ -1,12 +1,10 @@ # A centos8 image with systemd enabled. Must be run with: # `-d --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:ro` flags -FROM centos:8 +FROM quay.io/centos/centos:stream8 ENV container docker -RUN sed -i 's|\$releasever|8-stream|g' /etc/yum.repos.d/CentOS*.repo -RUN dnf install -y --allowerasing centos-stream-release -RUN dnf install -y curl procps initscripts systemd wget +RUN dnf install -y procps initscripts systemd wget RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i = \ "systemd-tmpfiles-setup.service" ] || rm -f $i; done); \ diff --git a/internal/buildscripts/packaging/tests/images/rpm/Dockerfile.centos-9 b/internal/buildscripts/packaging/tests/images/rpm/Dockerfile.centos-9 new file mode 100644 index 00000000000..d667743b442 --- /dev/null +++ b/internal/buildscripts/packaging/tests/images/rpm/Dockerfile.centos-9 @@ -0,0 +1,20 @@ +# A centos9 image with systemd enabled. Must be run with: +# `-d --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:ro` flags +FROM quay.io/centos/centos:stream9 + +ENV container docker + +RUN dnf install -y procps initscripts systemd wget + +RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i = \ + "systemd-tmpfiles-setup.service" ] || rm -f $i; done); \ + rm -f /lib/systemd/system/multi-user.target.wants/*;\ + rm -f /lib/systemd/system/local-fs.target.wants/*; \ + rm -f /lib/systemd/system/sockets.target.wants/*udev*; \ + rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \ + rm -f /lib/systemd/system/basic.target.wants/*;\ + rm -f /lib/systemd/system/anaconda.target.wants/*; + +VOLUME [ "/sys/fs/cgroup" ] + +CMD ["/usr/sbin/init"] diff --git a/internal/buildscripts/packaging/tests/images/rpm/Dockerfile.oraclelinux-9 b/internal/buildscripts/packaging/tests/images/rpm/Dockerfile.oraclelinux-9 new file mode 100644 index 00000000000..31e333c5bd9 --- /dev/null +++ b/internal/buildscripts/packaging/tests/images/rpm/Dockerfile.oraclelinux-9 @@ -0,0 +1,20 @@ +# A oraclelinux9 image with systemd enabled. Must be run with: +# `-d --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:ro` flags +FROM oraclelinux:9 + +ENV container docker + +RUN dnf install -y curl procps initscripts systemd wget + +RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i = \ + "systemd-tmpfiles-setup.service" ] || rm -f $i; done); \ + rm -f /lib/systemd/system/multi-user.target.wants/*;\ + rm -f /lib/systemd/system/local-fs.target.wants/*; \ + rm -f /lib/systemd/system/sockets.target.wants/*udev*; \ + rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \ + rm -f /lib/systemd/system/basic.target.wants/*;\ + rm -f /lib/systemd/system/anaconda.target.wants/*; + +VOLUME [ "/sys/fs/cgroup" ] + +CMD ["/usr/sbin/init"] diff --git a/internal/buildscripts/packaging/tests/installer_test.py b/internal/buildscripts/packaging/tests/installer_test.py index bcd455fd8b7..fd090859a46 100644 --- a/internal/buildscripts/packaging/tests/installer_test.py +++ b/internal/buildscripts/packaging/tests/installer_test.py @@ -100,6 +100,13 @@ def verify_uninstall(container, distro): assert container.exec_run(f"rpm -q {pkg}").exit_code != 0 +def fluentd_supported(distro): + if "opensuse" in distro or distro == "amazonlinux-2023": + return False + + return True + + @pytest.mark.installer @pytest.mark.parametrize( "distro", @@ -139,7 +146,7 @@ def test_installer_mode(distro, version, mode): # verify collector service status assert wait_for(lambda: service_is_running(container, service_owner=SERVICE_OWNER)) - if "opensuse" not in distro: + if fluentd_supported(distro): assert container.exec_run("systemctl status td-agent").exit_code == 0 # test support bundle script @@ -148,7 +155,7 @@ def test_installer_mode(distro, version, mode): verify_uninstall(container, distro) finally: - if "opensuse" not in distro: + if fluentd_supported(distro): run_container_cmd(container, "journalctl -u td-agent --no-pager") if container.exec_run("test -f /var/log/td-agent/td-agent.log").exit_code == 0: run_container_cmd(container, "cat /var/log/td-agent/td-agent.log") @@ -187,13 +194,13 @@ def test_installer_ballast(distro, version): # verify collector service status assert wait_for(lambda: service_is_running(container, service_owner=SERVICE_OWNER)) - if "opensuse" not in distro: + if fluentd_supported(distro): assert container.exec_run("systemctl status td-agent").exit_code == 0 verify_uninstall(container, distro) finally: - if "opensuse" not in distro: + if fluentd_supported(distro): run_container_cmd(container, "journalctl -u td-agent --no-pager") if container.exec_run("test -f /var/log/td-agent/td-agent.log").exit_code == 0: run_container_cmd(container, "cat /var/log/td-agent/td-agent.log") @@ -244,13 +251,13 @@ def test_installer_service_owner(distro, version): config_owner = container.exec_run("stat -c '%U:%G' /etc/otel").output.decode("utf-8") assert config_owner.strip() == f"{service_owner}:{service_owner}" - if "opensuse" not in distro: + if fluentd_supported(distro): assert container.exec_run("systemctl status td-agent").exit_code == 0 verify_uninstall(container, distro) finally: - if "opensuse" not in distro: + if fluentd_supported(distro): run_container_cmd(container, "journalctl -u td-agent --no-pager") run_container_cmd(container, f"journalctl -u {SERVICE_NAME} --no-pager")