From 965b0874416fe9e8379cc5b81b52119e4b23773b Mon Sep 17 00:00:00 2001 From: jeffreyc-splunk Date: Fri, 28 Jul 2023 09:31:05 -0400 Subject: [PATCH] Revert "Revert "OTL-2195: Disable fluentd by default in installer scripts (#3369)" (#3420)" This reverts commit f6dc932544b394f09635685afb57d66ec0069d1a. --- CHANGELOG.md | 5 +++ docs/getting-started/linux-installer.md | 33 ++++++++++++------- docs/getting-started/windows-installer.md | 20 +++++++---- .../packaging/installer/install.ps1 | 6 ++-- .../packaging/installer/install.sh | 4 +-- .../packaging/tests/installer_test.py | 33 +++++++++++-------- 6 files changed, 65 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b09c5b5f2..8b2760c1d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## Unreleased +### 🛑 Breaking changes 🛑 + +- (Splunk) Fluentd installation ***disabled*** by default for the Linux and Windows installer scripts ([#3369](https://github.com/signalfx/splunk-otel-collector/pull/3369)) + - Specify the `--with-fluentd` (Linux) or `with_fluentd = 1` (Windows) option to enable installation + ## v0.81.1 ### 🧰 Bug fixes 🧰 diff --git a/docs/getting-started/linux-installer.md b/docs/getting-started/linux-installer.md index 85af7417c5..b2d4bc742f 100644 --- a/docs/getting-started/linux-installer.md +++ b/docs/getting-started/linux-installer.md @@ -7,15 +7,17 @@ script deploys and configures: - Splunk OpenTelemetry Collector for Linux (**x86_64/amd64 and aarch64/arm64 platforms only**) - [SignalFx Smart Agent and collectd bundle](https://github.com/signalfx/signalfx-agent/releases) (**x86_64/amd64 platforms only**) -- [Fluentd (via the TD Agent)](https://www.fluentd.org/) - - Optional, **enabled** by default for supported Linux distributions - - See the [Fluentd Configuration](#fluentd-configuration) section for additional information, including how to skip installation. +- Log Collection with [Fluentd (via the TD Agent)](https://www.fluentd.org/) + - Optional, **disabled** by default + - See the [Fluentd Configuration](#fluentd-configuration) section for additional information, including how to enable installation for [supported platforms](#supported-platforms). - [Splunk OpenTelemetry Auto Instrumentation for Java](https://github.com/signalfx/splunk-otel-collector/tree/main/instrumentation#linux-java-auto-instrumentation) - Optional, **disabled** by default - See the [Auto Instrumentation](#auto-instrumentation) section for additional information, including how to enable installation. > IMPORTANT: systemd is required to use this script. +### Supported Platforms + Currently, the following Linux distributions and versions are supported: - Amazon Linux: 2, 2023 (**Note:** Log collection with Fluentd not currently supported for Amazon Linux 2023.) @@ -85,7 +87,7 @@ sudo sh /tmp/splunk-otel-collector.sh --realm SPLUNK_REALM --memory SPLUNK_MEMOR ``` By default, apt/yum/zypper repo definition files will be created to download -the Collector and Fluentd deb/rpm packages from +the Collector and Fluentd (if enabled) deb/rpm packages from [https://splunk.jfrog.io/splunk](https://splunk.jfrog.io/splunk) and [https://packages.treasuredata.com](https://packages.treasuredata.com), respectively. To skip these steps and use pre-configured repos on the target @@ -95,7 +97,7 @@ packages, specify the `--skip-collector-repo` and/or ```sh curl -sSL https://dl.signalfx.com/splunk-otel-collector.sh > /tmp/splunk-otel-collector.sh && \ -sudo sh /tmp/splunk-otel-collector.sh --realm SPLUNK_REALM --skip-collector-repo --skip-fluentd-repo \ +sudo sh /tmp/splunk-otel-collector.sh --realm SPLUNK_REALM --with-fluentd --skip-collector-repo --skip-fluentd-repo \ -- SPLUNK_ACCESS_TOKEN ``` @@ -183,14 +185,21 @@ To upgrade the Collector, run the following commands on your system (requires ### Fluentd Configuration -> If log collection is not required, run the installer script with the -> `--without-fluentd` option to skip installation of Fluentd and the -> plugins/dependencies listed below. +If log collection with Fluentd is required and your platform is [supported]( +#supported-platforms), run the installer script with the `--with-fluentd` +option to install/configure Fluentd and the plugins/dependencies listed below. +For example: + +```sh +curl -sSL https://dl.signalfx.com/splunk-otel-collector.sh > /tmp/splunk-otel-collector.sh && \ +sudo sh /tmp/splunk-otel-collector.sh --realm SPLUNK_REALM --with-fluentd -- SPLUNK_ACCESS_TOKEN +``` -By default, the Fluentd service will be installed and configured to forward log -events with the `@SPLUNK` label to the Collector (see below for how to add -custom Fluentd log sources), and the Collector will send these events to the -HEC ingest endpoint determined by the `--realm SPLUNK_REALM` option, e.g. +The Fluentd deb/rpm package (`td-agent`) will be installed, and the `td-agent` +service will be configured to forward log events with the `@SPLUNK` label to +the Collector (see below for how to add custom Fluentd log sources). The +Collector will then send these events to the HEC ingest endpoint determined by +the `--realm SPLUNK_REALM` option, i.e. `https://ingest.SPLUNK_REALM.signalfx.com/v1/log`. The following Fluentd plugins will also be installed: diff --git a/docs/getting-started/windows-installer.md b/docs/getting-started/windows-installer.md index a2ffdd7cbd..6547f56716 100644 --- a/docs/getting-started/windows-installer.md +++ b/docs/getting-started/windows-installer.md @@ -6,7 +6,7 @@ For Windows 64-bit environments, an installer script is available. The script deploys and configures: - Splunk OpenTelemetry Collector for Windows -- [Fluentd (via the TD Agent)](https://www.fluentd.org/) +- **Optional**: Log Collection with [Fluentd (via the TD Agent)](#fluentd-configuration) (disabled, by default) Currently, the following Windows versions are supported and requires PowerShell 3.0 or newer: @@ -176,10 +176,18 @@ as `Administrator`: ### Fluentd Configuration -By default, the Fluentd service will be installed and configured to forward log -events with the `@SPLUNK` label to the Collector (see below for how to add -custom Fluentd log sources), and the Collector will send these events to the -HEC ingest endpoint determined by the `realm = ` option, e.g. +If log collection with [Fluentd](https://www.fluentd.org/) is required, run the +installer script with the `with_fluentd = 1` option. For example (replace the +`` values for your configuration): + +```powershell +& {Set-ExecutionPolicy Bypass -Scope Process -Force; $script = ((New-Object System.Net.WebClient).DownloadString('https://dl.signalfx.com/splunk-otel-collector.ps1')); $params = @{access_token = ""; realm = ""; with_fluentd = 1}; Invoke-Command -ScriptBlock ([scriptblock]::Create(". {$script} $(&{$args} @params)"))} +``` + +Fluentd will be downloaded, installed, and configured to forward log events +with the `@SPLUNK` label to the Collector (see below for how to add custom +Fluentd log sources), and the Collector will send these events to the HEC +ingest endpoint determined by the `realm = ` option, e.g. `https://ingest.SPLUNK_REALM.signalfx.com/v1/log`. To configure the Collector to send log events to a custom HEC endpoint URL, you @@ -190,7 +198,7 @@ can specify the following parameters for the installer script: For example (replace the `` values for your configuration): ```powershell -& {Set-ExecutionPolicy Bypass -Scope Process -Force; $script = ((New-Object System.Net.WebClient).DownloadString('https://dl.signalfx.com/splunk-otel-collector.ps1')); $params = @{access_token = ""; realm = ""; hec_url = ""; hec_token = ""}; Invoke-Command -ScriptBlock ([scriptblock]::Create(". {$script} $(&{$args} @params)"))} +& {Set-ExecutionPolicy Bypass -Scope Process -Force; $script = ((New-Object System.Net.WebClient).DownloadString('https://dl.signalfx.com/splunk-otel-collector.ps1')); $params = @{access_token = ""; realm = ""; with_fluentd = 1; hec_url = ""; hec_token = ""}; Invoke-Command -ScriptBlock ([scriptblock]::Create(". {$script} $(&{$args} @params)"))} ``` The main Fluentd configuration file will be installed to diff --git a/internal/buildscripts/packaging/installer/install.ps1 b/internal/buildscripts/packaging/installer/install.ps1 index fb2e17e95b..f7d02a9df0 100644 --- a/internal/buildscripts/packaging/installer/install.ps1 +++ b/internal/buildscripts/packaging/installer/install.ps1 @@ -61,9 +61,9 @@ .EXAMPLE .\install.ps1 -access_token "ACCESSTOKEN" -hec_token "HECTOKEN" .PARAMETER with_fluentd - (OPTIONAL) Whether to install and configure fluentd to forward log events to the collector (default: $true) + (OPTIONAL) Whether to install and configure fluentd to forward log events to the collector (default: $false) .EXAMPLE - .\install.ps1 -access_token "ACCESSTOKEN" -with_fluentd $false + .\install.ps1 -access_token "ACCESSTOKEN" -with_fluentd $true .PARAMETER with_dotnet_instrumentation (OPTIONAL) Whether to install and configure .NET tracing to forward .NET application traces to the local collector (default: $false) .EXAMPLE @@ -116,7 +116,7 @@ param ( [string]$hec_token = "", [bool]$insecure = $false, [string]$collector_version = "", - [bool]$with_fluentd = $true, + [bool]$with_fluentd = $false, [bool]$with_dotnet_instrumentation = $false, [string]$bundle_dir = "", [ValidateSet('test','beta','release')][string]$stage = "release", diff --git a/internal/buildscripts/packaging/installer/install.sh b/internal/buildscripts/packaging/installer/install.sh index 0fc6bd89ae..b94bea2b4b 100755 --- a/internal/buildscripts/packaging/installer/install.sh +++ b/internal/buildscripts/packaging/installer/install.sh @@ -677,7 +677,7 @@ Options: (default: https://ingest.REALM.signalfx.com/v2/trace) --uninstall Removes the Splunk OpenTelemetry Collector for Linux. --with[out]-fluentd Whether to install and configure fluentd to forward log events to the collector. - (default: --with-fluentd) + (default: --without-fluentd) --with[out]-instrumentation Whether to install and configure the splunk-otel-auto-instrumentation package. (default: --without-instrumentation) --deployment-environment Set the 'deployment.environment' resource attribute to the specified value. @@ -848,7 +848,7 @@ parse_args_and_install() { local trace_url= local uninstall="false" local mode="agent" - local with_fluentd="true" + local with_fluentd="false" local collector_config_path= local skip_collector_repo="false" local skip_fluentd_repo="false" diff --git a/internal/buildscripts/packaging/tests/installer_test.py b/internal/buildscripts/packaging/tests/installer_test.py index e91cb8f41f..a1266a5a49 100644 --- a/internal/buildscripts/packaging/tests/installer_test.py +++ b/internal/buildscripts/packaging/tests/installer_test.py @@ -155,6 +155,14 @@ def test_installer_default(distro, arch, mode): run_container_cmd(container, install_cmd, env={"VERIFY_ACCESS_TOKEN": "false"}, timeout=INSTALLER_TIMEOUT) time.sleep(5) + # verify td-agent is not installed + if distro in DEB_DISTROS: + assert container.exec_run("dpkg -s td-agent").exit_code != 0 + else: + assert container.exec_run("rpm -q td-agent").exit_code != 0 + + assert container.exec_run("systemctl status td-agent").exit_code != 0 + # verify splunk-otel-auto-instrumentation is not installed if distro in DEB_DISTROS: assert container.exec_run("dpkg -s splunk-otel-auto-instrumentation").exit_code != 0 @@ -167,19 +175,12 @@ def test_installer_default(distro, arch, mode): # verify collector service status assert wait_for(lambda: service_is_running(container, service_owner=SERVICE_OWNER)) - if fluentd_supported(distro, arch): - assert container.exec_run("systemctl status td-agent").exit_code == 0 - # test support bundle script verify_support_bundle(container) verify_uninstall(container, distro) finally: - if fluentd_supported(distro, arch): - run_container_cmd(container, "journalctl -u td-agent --no-pager") - if container.exec_run("test -f /var/log/td-agent/td-agent.log").exit_code == 0: - run_container_cmd(container, "cat /var/log/td-agent/td-agent.log") run_container_cmd(container, f"journalctl -u {SERVICE_NAME} --no-pager") @@ -201,7 +202,7 @@ def test_installer_custom(distro, arch): install_cmd = " ".join(( get_installer_cmd(), - "--without-fluentd", + "--with-fluentd", "--memory 256 --ballast 64", f"--service-user {service_owner} --service-group {service_owner}", f"--collector-config {custom_config}", @@ -238,16 +239,22 @@ def test_installer_custom(distro, arch): config_owner = container.exec_run("stat -c '%U:%G' /etc/otel").output.decode("utf-8") assert config_owner.strip() == f"{service_owner}:{service_owner}" - # verify td-agent was not installed - if distro in DEB_DISTROS: - assert container.exec_run("dpkg -s td-agent").exit_code != 0 - else: - assert container.exec_run("rpm -q td-agent").exit_code != 0 + if fluentd_supported(distro, arch): + # verify td-agent was installed + if distro in DEB_DISTROS: + assert container.exec_run("dpkg -s td-agent").exit_code == 0 + else: + assert container.exec_run("rpm -q td-agent").exit_code == 0 + assert container.exec_run("systemctl status td-agent").exit_code == 0 verify_uninstall(container, distro) finally: run_container_cmd(container, f"journalctl -u {SERVICE_NAME} --no-pager") + if fluentd_supported(distro, arch): + run_container_cmd(container, "journalctl -u td-agent --no-pager") + if container.exec_run("test -f /var/log/td-agent/td-agent.log").exit_code == 0: + run_container_cmd(container, "cat /var/log/td-agent/td-agent.log") @pytest.mark.installer