Enable x-pack security for testing against snapshots (#28131)

auditbeat/docker-compose.yml

@@ -8,7 +8,7 @@ services:
     environment:
       - ES_HOST=elasticsearch
       - ES_PORT=9200
-      - ES_USER=beats
+      - ES_USER=auditbeat_user
       - ES_PASS=testing
       - KIBANA_HOST=kibana
       - KIBANA_PORT=5601
@@ -32,8 +32,15 @@ services:
     extends:
       file: ../testing/environments/${TESTING_ENVIRONMENT}.yml
       service: elasticsearch
+    healthcheck:
+      test: ["CMD-SHELL", "curl -u admin:testing -s http://localhost:9200/_cat/health?h=status | grep -q green"]
+      retries: 300
+      interval: 1s
 
   kibana:
     extends:
       file: ../testing/environments/${TESTING_ENVIRONMENT}.yml
       service: kibana
+    healthcheck:
+      test: ["CMD-SHELL", "curl -u beats:testing -s http://localhost:5601/api/status?v8format=true | grep -q '\"overall\":{\"level\":\"available\"'"]
+      retries: 600

auditbeat/tests/system/test_base.py

@@ -41,7 +41,7 @@ def test_index_management(self):
         """
         dirs = [self.temp_dir("auditbeat_test")]
         with PathCleanup(dirs):
-            es = Elasticsearch([self.get_elasticsearch_url()])
+            es = self.get_elasticsearch_instance()
 
             self.render_config_template(
                 modules=[{
@@ -50,7 +50,8 @@ def test_index_management(self):
                         "paths": dirs,
                     }
                 }],
-                elasticsearch={"host": self.get_elasticsearch_url()})
+                elasticsearch=self.get_elasticsearch_template_config()
+            )
             self.run_beat(extra_args=["setup", "--index-management"], exit_code=0)
 
             assert self.log_contains('Loaded index template')
@@ -67,16 +68,15 @@ def test_dashboards(self):
             kibana_dir = os.path.join(self.beat_path, "build", "kibana")
             shutil.copytree(kibana_dir, os.path.join(self.working_dir, "kibana"))
 
-            es = Elasticsearch([self.get_elasticsearch_url()])
             self.render_config_template(
                 modules=[{
                     "name": "file_integrity",
                     "extras": {
                         "paths": dirs,
                     }
                 }],
-                elasticsearch={"host": self.get_elasticsearch_url()},
-                kibana={"host": self.get_kibana_url()},
+                elasticsearch=self.get_elasticsearch_template_config(),
+                kibana=self.get_kibana_template_config(),
             )
             self.run_beat(extra_args=["setup", "--dashboards"], exit_code=0)
 

dev-tools/cmd/dashboards/export_dashboards.go

@@ -66,7 +66,8 @@ func main() {
 		user = u.User.Username()
 		pass, _ = u.User.Password()
 	}
-
+	user = "beats"
+	pass = "testing"
 	transport := httpcommon.DefaultHTTPTransportSettings()
 	transport.Timeout = kibanaTimeout
 

dev-tools/mage/integtest_docker.go

@@ -112,6 +112,10 @@ func (d *DockerIntegrationTester) Test(dir string, mageTarget string, env map[st
 		// Use the host machine's pkg cache to minimize external downloads.
 		"-v", goPkgCache + ":" + dockerGoPkgCache + ":ro",
 		"-e", "GOPROXY=file://" + dockerGoPkgCache + ",direct",
+		// Do not set ES_USER or ES_PATH in this file unless you intend to override
+		// values set in all individual docker-compose files
+		//		"-e", "ES_USER=admin",
+		//		"-e", "ES_PASS=testing",
 	}
 	args, err = addUidGidEnvArgs(args)
 	if err != nil {

dev-tools/mage/pytest.go

@@ -133,6 +133,14 @@ func PythonTest(params PythonTestArgs) error {
 	pytestOptions := []string{
 		"--timeout=90",
 		"--durations=20",
+		// Enable -x to stop at the first failing test
+		// "-x",
+		// Enable --tb=long to produce long tracebacks
+		//"--tb=long",
+		// Enable -v to produce verbose output
+		//"-v",
+		// Don't capture test output
+		//"-s",
 	}
 	if mg.Verbose() {
 		pytestOptions = append(pytestOptions, "-v")

filebeat/docker-compose.yml

@@ -40,6 +40,10 @@ services:
     extends:
       file: ${ES_BEATS}/testing/environments/${TESTING_ENVIRONMENT}.yml
       service: elasticsearch
+    healthcheck:
+      test: ["CMD-SHELL", "curl -u admin:testing -s http://localhost:9200/_cat/health?h=status | grep -q green"]
+      retries: 300
+      interval: 1s
 
   kafka:
     build: ${ES_BEATS}/testing/environments/docker/kafka
@@ -53,6 +57,9 @@ services:
     extends:
       file: ${ES_BEATS}/testing/environments/${TESTING_ENVIRONMENT}.yml
       service: kibana
+    healthcheck:
+      test: ["CMD-SHELL", "curl -u beats:testing -s http://localhost:5601/api/status?v8format=true | grep -q '\"overall\":{\"level\":\"available\"'"]
+      retries: 600
 
   mosquitto:
     build: ${ES_BEATS}/testing/environments/docker/mosquitto

filebeat/fileset/modules_integration_test.go

@@ -258,7 +258,9 @@ func TestLoadMultiplePipelinesWithRollback(t *testing.T) {
 
 func getTestingElasticsearch(t eslegtest.TestLogger) *eslegclient.Connection {
 	conn, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{
-		URL: eslegtest.GetURL(),
+		URL:      eslegtest.GetURL(),
+		Username: eslegtest.GetUser(),
+		Password: eslegtest.GetPass(),
 	})
 	if err != nil {
 		t.Fatal(err)

filebeat/tests/system/config/filebeat_modules.yml.j2

@@ -12,8 +12,10 @@ filebeat.overwrite_pipelines: true
 filebeat.config.modules:
   path: {{ beat.working_dir + '/modules.d/*.yml' }}
 
-output.elasticsearch.hosts: ["{{ elasticsearch_url }}"]
+output.elasticsearch.hosts: ["{{ elasticsearch.host }}"]
 output.elasticsearch.index: {{ index_name }}
+output.elasticsearch.username: {{ elasticsearch.user }}
+output.elasticsearch.password: {{ elasticsearch.pass }}
 
 setup.template.name: {{ index_name }}
 setup.template.pattern: {{ index_name }}*

filebeat/tests/system/test_base.py

@@ -1,13 +1,19 @@
 import os
 import unittest
 from filebeat import BaseTest
-from elasticsearch import Elasticsearch
 from beat.beat import INTEGRATION_TESTS
 from beat import common_tests
 
 
 class Test(BaseTest, common_tests.TestExportsMixin, common_tests.TestDashboardMixin):
 
+    def setUp(self):
+        super(Test, self).setUp()
+        self.render_config_template(
+            elasticsearch=self.get_elasticsearch_template_config(),
+        )
+        self.es = self.get_elasticsearch_instance()
+
     def test_base(self):
         """
         Test if the basic fields exist.
@@ -32,12 +38,11 @@ def test_index_management(self):
         """
         Test that the template can be loaded with `setup --index-management`
         """
-        es = Elasticsearch([self.get_elasticsearch_url()])
         self.render_config_template(
-            elasticsearch={"host": self.get_elasticsearch_url()},
+            elasticsearch=self.get_elasticsearch_template_config(),
         )
         exit_code = self.run_beat(extra_args=["setup", "--index-management"])
 
         assert exit_code == 0
         assert self.log_contains('Loaded index template')
-        assert len(es.cat.templates(name='filebeat-*', h='name')) > 0
+        assert len(self.es.cat.templates(name='filebeat-*', h='name')) > 0

filebeat/tests/system/test_crawler.py

@@ -19,7 +19,7 @@ def test_fetched_lines(self):
         """
 
         self.render_config_template(
-            path=os.path.abspath(self.working_dir) + "/log/*"
+            path=os.path.abspath(self.working_dir) + "/log/*",
         )
         os.mkdir(self.working_dir + "/log/")
 
@@ -55,7 +55,7 @@ def test_unfinished_line_and_continue(self):
         """
 
         self.render_config_template(
-            path=os.path.abspath(self.working_dir) + "/log/*"
+            path=os.path.abspath(self.working_dir) + "/log/*",
         )
         os.mkdir(self.working_dir + "/log/")
 
@@ -162,7 +162,7 @@ def test_file_renaming(self):
         """
 
         self.render_config_template(
-            path=os.path.abspath(self.working_dir) + "/log/*"
+            path=os.path.abspath(self.working_dir) + "/log/*",
         )
         os.mkdir(self.working_dir + "/log/")
 
@@ -339,7 +339,7 @@ def test_new_line_on_existing_file(self):
         """
 
         self.render_config_template(
-            path=os.path.abspath(self.working_dir) + "/log/*"
+            path=os.path.abspath(self.working_dir) + "/log/*",
         )
         os.mkdir(self.working_dir + "/log/")
 
@@ -373,7 +373,7 @@ def test_multiple_appends(self):
         """
 
         self.render_config_template(
-            path=os.path.abspath(self.working_dir) + "/log/*"
+            path=os.path.abspath(self.working_dir) + "/log/*",
         )
         os.mkdir(self.working_dir + "/log/")
 
@@ -419,7 +419,7 @@ def test_new_line_on_open_file(self):
         """
 
         self.render_config_template(
-            path=os.path.abspath(self.working_dir) + "/log/*"
+            path=os.path.abspath(self.working_dir) + "/log/*",
         )
         os.mkdir(self.working_dir + "/log/")
 
@@ -457,7 +457,7 @@ def test_tail_files(self):
 
         self.render_config_template(
             path=os.path.abspath(self.working_dir) + "/log/*",
-            tail_files="true"
+            tail_files="true",
         )
         os.mkdir(self.working_dir + "/log/")
 
@@ -501,7 +501,7 @@ def test_utf8(self):
 
         self.render_config_template(
             path=os.path.abspath(self.working_dir) + "/log/*",
-            encoding="utf-8"
+            encoding="utf-8",
         )
         os.mkdir(self.working_dir + "/log/")
 
@@ -613,7 +613,7 @@ def test_include_lines(self):
 
         self.render_config_template(
             path=os.path.abspath(self.working_dir) + "/log/*",
-            include_lines=["^ERR", "^WARN"]
+            include_lines=["^ERR", "^WARN"],
         )
         os.mkdir(self.working_dir + "/log/")
 
@@ -648,9 +648,8 @@ def test_default_include_exclude_lines(self):
         """
         Checks if all the log lines are exported by default
         """
-
         self.render_config_template(
-            path=os.path.abspath(self.working_dir) + "/log/*"
+            path=os.path.abspath(self.working_dir) + "/log/*",
         )
         os.mkdir(self.working_dir + "/log/")
 
@@ -688,7 +687,7 @@ def test_exclude_lines(self):
 
         self.render_config_template(
             path=os.path.abspath(self.working_dir) + "/log/*",
-            exclude_lines=["^DBG"]
+            exclude_lines=["^DBG"],
         )
         os.mkdir(self.working_dir + "/log/")
 
@@ -727,7 +726,7 @@ def test_include_exclude_lines(self):
         self.render_config_template(
             path=os.path.abspath(self.working_dir) + "/log/*",
             exclude_lines=["^DBG"],
-            include_lines=["apache"]
+            include_lines=["apache"],
         )
         os.mkdir(self.working_dir + "/log/")
 

filebeat/tests/system/test_modules.py

@@ -5,7 +5,6 @@
 import glob
 import subprocess
 
-from elasticsearch import Elasticsearch
 import json
 import logging
 from parameterized import parameterized
@@ -118,9 +117,7 @@ def load_fileset_test_cases():
 class Test(BaseTest):
 
     def init(self):
-        self.elasticsearch_url = self.get_elasticsearch_url()
-        print("Using elasticsearch: {}".format(self.elasticsearch_url))
-        self.es = Elasticsearch([self.elasticsearch_url])
+        self.es = self.get_elasticsearch_instance(user='admin')
         logging.getLogger("urllib3").setLevel(logging.WARNING)
         logging.getLogger("elasticsearch").setLevel(logging.ERROR)
 
@@ -146,7 +143,7 @@ def test_fileset_file(self, module, fileset, test_file):
             template_name="filebeat_modules",
             output=cfgfile,
             index_name=self.index_name,
-            elasticsearch_url=self.elasticsearch_url,
+            elasticsearch=self.get_elasticsearch_template_config(user='admin')
         )
 
         self.run_on_file(

filebeat/tests/system/test_pipeline.py

@@ -2,7 +2,6 @@
 from beat.beat import INTEGRATION_TESTS
 import os
 import unittest
-from elasticsearch import Elasticsearch
 import json
 import logging
 
@@ -12,8 +11,7 @@ class Test(BaseTest):
     def init(self):
         self.elasticsearch_url = self.get_elasticsearch_url()
         self.kibana_url = self.get_kibana_url()
-        print("Using elasticsearch: {}".format(self.elasticsearch_url))
-        self.es = Elasticsearch([self.elasticsearch_url])
+        self.es = self.get_elasticsearch_instance()
         logging.getLogger("urllib3").setLevel(logging.WARNING)
         logging.getLogger("elasticsearch").setLevel(logging.ERROR)
 
@@ -47,10 +45,13 @@ def test_input_pipeline_config(self):
 
         self.render_config_template(
             path=os.path.abspath(self.working_dir) + "/log/*",
-            elasticsearch=dict(
-                host=self.elasticsearch_url,
-                pipeline="estest",
-                index=index_name),
+            elasticsearch={
+                'host': self.elasticsearch_url,
+                'pipeline': "estest",
+                'index': index_name,
+                'user': os.getenv("ES_USER"),
+                'pass': os.getenv("ES_PASS")
+            },
             pipeline="test",
             setup_template_name=index_name,
             setup_template_pattern=index_name + "*",

filebeat/tests/system/test_reload_modules.py

@@ -6,7 +6,6 @@
 
 from filebeat import BaseTest
 from beat.beat import INTEGRATION_TESTS
-from elasticsearch import Elasticsearch
 
 
 moduleConfigTemplate = """
@@ -27,7 +26,7 @@ class Test(BaseTest):
     def setUp(self):
         super(BaseTest, self).setUp()
         if INTEGRATION_TESTS:
-            self.es = Elasticsearch([self.get_elasticsearch_url()])
+            self.es = self.get_elasticsearch_instance()
 
         # Copy system module
         shutil.copytree(os.path.join(self.beat_path, "tests", "system", "module", "test"),
@@ -72,7 +71,7 @@ def test_reload_writes_pipeline(self):
             reload_path=self.working_dir + "/configs/*.yml",
             reload_type="modules",
             inputs=False,
-            elasticsearch={"host": self.get_elasticsearch_url()}
+            elasticsearch=self.get_elasticsearch_template_config(),
         )
 
         proc = self.start_beat()