From 59b38cd0029471bcc67f798633286239251f561f Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 23 Nov 2021 16:13:42 +0100
Subject: [PATCH] Get output files when they have a suffix

---
 .../autodiscover_test.go                      | 32 +++++++++++++++++--
 .../testdata/templates/filebeat.yml.tmpl      |  2 ++
 .../testdata/templates/heartbeat.yml.tmpl     |  2 ++
 .../testdata/templates/metricbeat.yml.tmpl    |  2 ++
 4 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/e2e/_suites/kubernetes-autodiscover/autodiscover_test.go b/e2e/_suites/kubernetes-autodiscover/autodiscover_test.go
index c4ad9868b8..097e85a3d7 100644
--- a/e2e/_suites/kubernetes-autodiscover/autodiscover_test.go
+++ b/e2e/_suites/kubernetes-autodiscover/autodiscover_test.go
@@ -366,12 +366,14 @@ func (m *podsManager) waitForEventsCondition(podName string, conditionFn func(ct
 	localPath := filepath.Join(tmpDir, "events")
 	exp := backoff.WithContext(backoff.NewConstantBackOff(1*time.Second), ctx)
 	return backoff.Retry(func() error {
-		_, err := m.kubectl.Run(ctx, "cp", "--no-preserve", containerPath, localPath)
+		err := m.copyEvents(ctx, containerPath, localPath)
 		if err != nil {
-			log.Debugf("Failed to copy events from %s to %s: %s", containerPath, localPath, err)
-			return err
+			return fmt.Errorf("failed to copy events from %s: %w", containerPath, err)
 		}
 		ok, err := conditionFn(ctx, localPath)
+		if err != nil {
+			return fmt.Errorf("events condition failed: %w", err)
+		}
 		if !ok {
 			return fmt.Errorf("events do not satisfy condition")
 		}
@@ -379,6 +381,30 @@ func (m *podsManager) waitForEventsCondition(podName string, conditionFn func(ct
 	}, exp)
 }
 
+func (m *podsManager) copyEvents(ctx context.Context, containerPath string, localPath string) error {
+	today := time.Now().Format("20060102")
+	paths := []string{
+		containerPath,
+
+		// Format used since 8.0.
+		containerPath + "-" + today + ".ndjson",
+	}
+
+	var err error
+	var output string
+	for _, containerPath := range paths {
+		// This command always succeeds, so check if the local path has been created.
+		os.Remove(localPath)
+		output, _ = m.kubectl.Run(ctx, "cp", "--no-preserve", containerPath, localPath)
+		if _, err = os.Stat(localPath); os.IsNotExist(err) {
+			continue
+		}
+		return nil
+	}
+	log.Debugf("Failed to copy events from %s to %s: %s", containerPath, localPath, output)
+	return err
+}
+
 func (m *podsManager) getPodInstances(ctx context.Context, podName string) (instances []string, err error) {
 	span, _ := apm.StartSpanOptions(m.ctx, "Getting pod instances", "pod.instances.get", apm.SpanOptions{
 		Parent: apm.SpanFromContext(m.ctx).TraceContext(),
diff --git a/e2e/_suites/kubernetes-autodiscover/testdata/templates/filebeat.yml.tmpl b/e2e/_suites/kubernetes-autodiscover/testdata/templates/filebeat.yml.tmpl
index 05ae5d2b6f..eeecccc756 100644
--- a/e2e/_suites/kubernetes-autodiscover/testdata/templates/filebeat.yml.tmpl
+++ b/e2e/_suites/kubernetes-autodiscover/testdata/templates/filebeat.yml.tmpl
@@ -44,6 +44,8 @@ data:
     output.file:
       path: /tmp
       filename: beats-events
+      rotate_every_kb: 100000
+      rotate_on_startup: false
 ---
 apiVersion: apps/v1
 kind: DaemonSet
diff --git a/e2e/_suites/kubernetes-autodiscover/testdata/templates/heartbeat.yml.tmpl b/e2e/_suites/kubernetes-autodiscover/testdata/templates/heartbeat.yml.tmpl
index 31f9e625ba..8d3d963d9c 100644
--- a/e2e/_suites/kubernetes-autodiscover/testdata/templates/heartbeat.yml.tmpl
+++ b/e2e/_suites/kubernetes-autodiscover/testdata/templates/heartbeat.yml.tmpl
@@ -41,6 +41,8 @@ data:
     output.file:
       path: /tmp
       filename: beats-events
+      rotate_every_kb: 100000
+      rotate_on_startup: false
 ---
 # Deploy singleton instance in the whole cluster for some unique data sources, like kube-state-metrics
 apiVersion: apps/v1
diff --git a/e2e/_suites/kubernetes-autodiscover/testdata/templates/metricbeat.yml.tmpl b/e2e/_suites/kubernetes-autodiscover/testdata/templates/metricbeat.yml.tmpl
index a21a9ea43e..d39ece59e3 100644
--- a/e2e/_suites/kubernetes-autodiscover/testdata/templates/metricbeat.yml.tmpl
+++ b/e2e/_suites/kubernetes-autodiscover/testdata/templates/metricbeat.yml.tmpl
@@ -18,6 +18,8 @@ data:
     output.file:
       path: /tmp
       filename: beats-events
+      rotate_every_kb: 100000
+      rotate_on_startup: false
 ---
 apiVersion: apps/v1
 kind: DaemonSet