AMLII-933 - Linux log file tailing e2e test

.gitlab/e2e.yml

@@ -238,6 +238,12 @@ new-e2e-npm-main:
   # Temporary, until we manage to stabilize those tests.
   allow_failure: true
 
+new-e2e-log-agent-main:
+  extends: .new_e2e_template
+  rules: !reference [.on_main]
+  variables:
+    TARGETS: ./tests/agent-metric-logs
+
 #   ^    If you create a new job here that extends `.new_e2e_template`,
 #  /!\   do not forget to add it in the `dependencies` statement of the
 # /___\  `e2e_test_junit_upload` job in the `.gitlab/e2e_test_junit_upload.yml` file

.gitlab/e2e_test_junit_upload.yml

@@ -15,6 +15,7 @@ e2e_test_junit_upload:
     - new-e2e-language-detection-main
     - new-e2e-agent-platform-install-script-debian-a7-x64
     - new-e2e-npm-main
+    - new-e2e-agent-metric-logs-main
   script:
     - set +x
     - export DATADOG_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.datadog_api_key_org2 --with-decryption --query "Parameter.Value" --out text)

test/new-e2e/tests/agent-metric-logs/linux-log-agent/01_file_tailing_test.go

@@ -0,0 +1,161 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2016-present Datadog, Inc.
+
+package logagent
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e"
+	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/params"
+	"github.com/DataDog/test-infra-definitions/components/datadog/agentparams"
+)
+
+// vmFakeintakeSuite defines a test suite for the log agent interacting with a virtual machine and fake intake.
+type vmFakeintakeSuite struct {
+	e2e.Suite[e2e.FakeIntakeEnv]
+}
+
+// logsExampleStackDef returns the stack definition required for the log agent test suite.
+func logsExampleStackDef() *e2e.StackDefinition[e2e.FakeIntakeEnv] {
+	config :=
+		`logs:
+  - type: file
+    path: '/var/log/hello-world.log'
+    service: hello
+    source: custom_log
+`
+	return e2e.FakeIntakeStackDef(
+		e2e.WithAgentParams(
+			agentparams.WithLogs(),
+			agentparams.WithIntegration("custom_logs.d", config)))
+
+}
+
+// TestE2EVMFakeintakeSuite runs the E2E test suite for the log agent with a VM and fake intake.
+func TestE2EVMFakeintakeSuite(t *testing.T) {
+	e2e.Run(t, &vmFakeintakeSuite{}, logsExampleStackDef(), params.WithDevMode())
+}
+
+func (s *vmFakeintakeSuite) TestLinuxLogTailing() {
+	// Clean up once test is finished running
+	s.cleanUp()
+	defer s.cleanUp()
+
+	// Flush server and reset aggregators
+	s.Env().Fakeintake.FlushServerAndResetAggregators()
+	defer s.Env().Fakeintake.FlushServerAndResetAggregators()
+
+	// Run test cases
+	s.T().Run("LogCollection", func(t *testing.T) {
+		s.LogCollection()
+	})
+
+	s.T().Run("LogPermission", func(t *testing.T) {
+		s.LogPermission()
+	})
+
+	s.T().Run("LogRotation", func(t *testing.T) {
+		s.LogRotation()
+	})
+}
+
+func (s *vmFakeintakeSuite) LogCollection() {
+	t := s.T()
+	fakeintake := s.Env().Fakeintake
+
+	// Create a new log file
+	s.Env().VM.Execute("sudo touch /var/log/hello-world.log")
+
+	// Part 1: Ensure no logs are present in fakeintake
+	s.EventuallyWithT(func(c *assert.CollectT) {
+		logs, err := fakeintake.FilterLogs("hello")
+		require.NoError(t, err, "Unable to filter logs by the service 'hello'.")
+		require.Empty(t, logs, "Logs were found when none were expected.")
+
+		// If logs are found, print their content for debugging
+		if len(logs) != 0 {
+			cat, _ := s.Env().VM.ExecuteWithError("cat /var/log/hello-world.log")
+			t.Logf("Logs detected when none were expected: %v", cat)
+			require.Empty(t, logs, "Logs were found when none were expected.")
+		}
+	}, 5*time.Minute, 10*time.Second)
+
+	// Part 2: Adjust permissions of new log file
+	_, err := s.Env().VM.ExecuteWithError("sudo chmod 777 /var/log/hello-world.log")
+	require.NoError(t, err, "Unable to adjust permissions for the log file '/var/log/hello-world.log'.")
+
+	// Generate log
+	generateLog(s, t, "hello-world")
+
+	// Part 3: Assert that logs are found in intake after generation
+	checkLogs(s, "hello", "hello-world")
+}
+
+func (s *vmFakeintakeSuite) LogPermission() {
+	t := s.T()
+
+	// Part 4: Block permission and check the Agent status
+	s.Env().VM.Execute("sudo chmod 000 /var/log/hello-world.log")
+	s.EventuallyWithT(func(c *assert.CollectT) {
+		// Check the Agent status
+		statusOutput, err := s.Env().VM.ExecuteWithError("sudo datadog-agent status | grep -A 10 'custom_logs'")
+		require.NoError(t, err, "Issue running agent status: %s", err)
+
+		if strings.Contains(statusOutput, "Status: OK") {
+			require.Fail(t, "log file is unexpectedly accessible")
+		}
+
+		require.Contains(t, statusOutput, "denied", "Log file is correctly inaccessible")
+	}, 3*time.Minute, 10*time.Second)
+
+	// Part 5: Restore permissions
+	s.Env().VM.Execute("sudo chmod 777 /var/log/hello-world.log")
+
+	// Part 6: Restart the agent, generate new logs
+	s.Env().VM.Execute("sudo service datadog-agent restart")
+
+	generateLog(s, s.T(), "hello-world")
+
+	// Check the Agent status
+	s.EventuallyWithT(func(c *assert.CollectT) {
+		statusOutput, err := s.Env().VM.ExecuteWithError("sudo datadog-agent status | grep -A 10 'custom_logs'")
+		require.NoError(t, err, "Issue running agent status: %s", err)
+		require.Contains(t, statusOutput, "Status: OK", "Expecting log file to be accessible but it is inaccessible instead")
+	}, 5*time.Minute, 2*time.Second)
+}
+
+func (s *vmFakeintakeSuite) LogRotation() {
+	t := s.T()
+
+	// Part 7: Rotate the log file and check if the agent is tailing the new log file.
+	// Rotate the log file
+	s.Env().VM.Execute("sudo mv /var/log/hello-world.log /var/log/hello-world.log.old && sudo touch /var/log/hello-world.log")
+
+	// Verify the old log file's existence after rotation
+	_, err := s.Env().VM.ExecuteWithError("ls /var/log/hello-world.log.old")
+	require.NoError(t, err, "Failed to find the old log file after rotation")
+
+	// Grant new log file permission
+	s.Env().VM.Execute("sudo chmod 777 /var/log/hello-world.log")
+
+	// Check if agent is tailing new log file via agent status
+	s.EventuallyWithT(func(c *assert.CollectT) {
+		newStatusOutput, err := s.Env().VM.ExecuteWithError("sudo datadog-agent status | grep -A 10 'custom_logs'")
+		require.NoErrorf(t, err, "Issue running agent status. Is the agent running?\n %s", newStatusOutput)
+		assert.Containsf(t, newStatusOutput, "Path: /var/log/hello-world.log", "The agent is not tailing the expected log file,instead: \n %s", newStatusOutput)
+	}, 5*time.Minute, 10*time.Second)
+
+	// Generate new log
+	generateLog(s, t, "hello-world-new-content")
+
+	// Verify Log's content is generated and submitted
+	checkLogs(s, "hello", "hello-world-new-content")
+}

test/new-e2e/tests/agent-metric-logs/linux-log-agent/file_tailing_utils_test.go

@@ -0,0 +1,125 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2016-present Datadog, Inc.
+
+package logagent
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	fi "github.com/DataDog/datadog-agent/test/fakeintake/client"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// generateLog generates and verifies log contents.
+func generateLog(s *vmFakeintakeSuite, t *testing.T, content string) {
+	// Determine the OS and set the appropriate log path and command.
+	var logPath, cmd, checkCmd string
+
+	osType, err := s.getOSType()
+	require.NoErrorf(t, err, "Failed to get OS type: %s", err)
+
+	switch osType {
+	case "windows":
+		t.Log("Generating Windows log.")
+		logPath = "C:\\logs\\hello-world.log"
+		cmd = fmt.Sprintf("echo %s > %s", strings.Repeat(content, 10), logPath)
+		checkCmd = fmt.Sprintf("Get-Content %s", logPath)
+	default: // Assuming Linux if not Windows.
+		t.Log("Generating Linux log.")
+		logPath = "/var/log/hello-world.log"
+		cmd = fmt.Sprintf("echo %s > %s", strings.Repeat(content, 10), logPath)
+		checkCmd = fmt.Sprintf("cat %s", logPath)
+	}
+
+	s.Env().VM.Execute(cmd)
+
+	// Check if the log has been generated.
+	s.EventuallyWithT(func(c *assert.CollectT) {
+		output := s.Env().VM.Execute(checkCmd)
+		if strings.Contains(output, content) {
+			t.Logf("Finished generating %s log.", osType)
+		} else {
+			require.Fail(t, "Log not yet generated.")
+		}
+	}, 5*time.Minute, 2*time.Second)
+}
+
+// checkLogs checks and verifies logs inside the intake.
+func checkLogs(fakeintake *vmFakeintakeSuite, service, content string) {
+	client := fakeintake.Env().Fakeintake
+	t := fakeintake.T()
+
+	fakeintake.EventuallyWithT(func(c *assert.CollectT) {
+		names, err := client.GetLogServiceNames()
+		assert.NoErrorf(t, err, "Error found: %s", err)
+
+		if len(names) > 0 {
+			logs, err := client.FilterLogs(service)
+			assert.NoErrorf(t, err, "Error found: %s", err)
+			assert.NotEmpty(t, logs, "No logs with service matching '%s' found, instead got '%s'", service, names)
+
+			logs, err = client.FilterLogs(service, fi.WithMessageContaining(content))
+			assert.NoErrorf(t, err, "Error found: %s", err)
+			assert.True(t, len(logs) > 0, "Expected at least 1 log with content: '%s', but received %v logs.", content, len(logs))
+		}
+	}, 10*time.Minute, 10*time.Second)
+
+}
+
+func (s *vmFakeintakeSuite) getOSType() (string, error) {
+	// Get Linux OS.
+	output, err := s.Env().VM.ExecuteWithError("cat /etc/os-release")
+	if err == nil && strings.Contains(output, "ID=ubuntu") {
+		return "linux", nil
+	}
+
+	// Get Windows OS.
+	output, err = s.Env().VM.ExecuteWithError("wmic os get Caption")
+	if err == nil && strings.Contains(output, "Windows") {
+		return "windows", nil
+	}
+
+	return "", errors.New("unable to determine OS type.")
+}
+
+// cleanUp cleans up any existing log files.
+func (s *vmFakeintakeSuite) cleanUp() {
+	t := s.T()
+	osType, err := s.getOSType()
+	if err != nil {
+		t.Logf("Failed to determine OS type: %v", err)
+		return
+	}
+
+	var checkCmd string
+
+	switch osType {
+	case "linux":
+		s.Env().VM.Execute("sudo rm -f /var/log/hello-world.log")
+		s.Env().VM.Execute("sudo rm -f /var/log/hello-world.log.old")
+		checkCmd = "ls /var/log/hello-world.log /var/log/hello-world.log.old 2>/dev/null || echo 'Files do not exist'"
+	case "windows":
+		s.Env().VM.Execute("if (Test-Path C:\\logs\\hello-world.log) { Remove-Item -Path C:\\logs\\hello-world.log -Force }")
+		s.Env().VM.Execute("if (Test-Path C:\\logs\\hello-world.log.old) { Remove-Item -Path C:\\logs\\hello-world.log.old -Force }")
+		checkCmd = "if (Test-Path C:\\logs\\hello-world.log) { Get-ChildItem -Path C:\\logs\\hello-world.log } elseif (Test-Path C:\\logs\\hello-world.log.old) { Get-ChildItem -Path C:\\logs\\hello-world.log.old } else { Write-Output 'Files do not exist' }"
+	default:
+		t.Logf("Unsupported OS type: %s", osType)
+		return
+	}
+
+	s.EventuallyWithT(func(c *assert.CollectT) {
+		output, err := s.Env().VM.ExecuteWithError(checkCmd)
+		if err != nil {
+			require.NoErrorf(t, err, "Having issue cleaning log files, retrying... %s", output)
+		} else {
+			t.Log("Successfully cleaned up.")
+		}
+	}, 5*time.Minute, 2*time.Second)
+}