Enable x-pack security for testing against snapshots

auditbeat/docker-compose.yml

@@ -8,7 +8,7 @@ services:
     environment:
       - ES_HOST=elasticsearch
       - ES_PORT=9200
-      - ES_USER=beats
+      - ES_USER=auditbeat_user
       - ES_PASS=testing
       - KIBANA_HOST=kibana
       - KIBANA_PORT=5601

auditbeat/tests/system/test_base.py

@@ -41,7 +41,7 @@ def test_index_management(self):
         """
         dirs = [self.temp_dir("auditbeat_test")]
         with PathCleanup(dirs):
-            es = Elasticsearch([self.get_elasticsearch_url()])
+            es = self.get_elasticsearch_instance()
 
             self.render_config_template(
                 modules=[{
@@ -50,7 +50,8 @@ def test_index_management(self):
                         "paths": dirs,
                     }
                 }],
-                elasticsearch={"host": self.get_elasticsearch_url()})
+                elasticsearch=self.get_elasticsearch_template_config()
+            )
             self.run_beat(extra_args=["setup", "--index-management"], exit_code=0)
 
             assert self.log_contains('Loaded index template')
@@ -67,16 +68,15 @@ def test_dashboards(self):
             kibana_dir = os.path.join(self.beat_path, "build", "kibana")
             shutil.copytree(kibana_dir, os.path.join(self.working_dir, "kibana"))
 
-            es = Elasticsearch([self.get_elasticsearch_url()])
             self.render_config_template(
                 modules=[{
                     "name": "file_integrity",
                     "extras": {
                         "paths": dirs,
                     }
                 }],
-                elasticsearch={"host": self.get_elasticsearch_url()},
-                kibana={"host": self.get_kibana_url()},
+                elasticsearch=self.get_elasticsearch_template_config(),
+                kibana=self.get_kibana_template_config(),
             )
             self.run_beat(extra_args=["setup", "--dashboards"], exit_code=0)
 

dev-tools/cmd/dashboards/export_dashboards.go

@@ -66,7 +66,8 @@ func main() {
 		user = u.User.Username()
 		pass, _ = u.User.Password()
 	}
-
+	user = "beats"
+	pass = "testing"
 	transport := httpcommon.DefaultHTTPTransportSettings()
 	transport.Timeout = kibanaTimeout
 

dev-tools/mage/integtest_docker.go

@@ -112,6 +112,10 @@ func (d *DockerIntegrationTester) Test(_ string, mageTarget string, env map[stri
 		// Use the host machine's pkg cache to minimize external downloads.
 		"-v", goPkgCache + ":" + dockerGoPkgCache + ":ro",
 		"-e", "GOPROXY=file://" + dockerGoPkgCache + ",direct",
+		// Do not set ES_USER or ES_PATH in this file unless you intend to override
+		// values set in all individual docker-compose files
+		//		"-e", "ES_USER=admin",
+		//		"-e", "ES_PASS=testing",
 	}
 	args, err = addUidGidEnvArgs(args)
 	if err != nil {

dev-tools/mage/pytest.go

@@ -133,6 +133,14 @@ func PythonTest(params PythonTestArgs) error {
 	pytestOptions := []string{
 		"--timeout=90",
 		"--durations=20",
+		// Enable -x to stop at the first failing test
+		// "-x",
+		// Enable --tb=long to produce long tracebacks
+		//"--tb=long",
+		// Enable -v to produce verbose output
+		//"-v",
+		// Don't capture test output
+		//"-s",
 	}
 	if mg.Verbose() {
 		pytestOptions = append(pytestOptions, "-v")

filebeat/fileset/modules_integration_test.go

@@ -258,7 +258,9 @@ func TestLoadMultiplePipelinesWithRollback(t *testing.T) {
 
 func getTestingElasticsearch(t eslegtest.TestLogger) *eslegclient.Connection {
 	conn, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{
-		URL: eslegtest.GetURL(),
+		URL:      eslegtest.GetURL(),
+		Username: eslegtest.GetUser(),
+		Password: eslegtest.GetPass(),
 	})
 	if err != nil {
 		t.Fatal(err)

filebeat/tests/system/config/filebeat_modules.yml.j2

@@ -12,8 +12,10 @@ filebeat.overwrite_pipelines: true
 filebeat.config.modules:
   path: {{ beat.working_dir + '/modules.d/*.yml' }}
 
-output.elasticsearch.hosts: ["{{ elasticsearch_url }}"]
+output.elasticsearch.hosts: ["{{ elasticsearch.host }}"]
 output.elasticsearch.index: {{ index_name }}
+output.elasticsearch.username: {{ elasticsearch.user }}
+output.elasticsearch.password: {{ elasticsearch.pass }}
 
 setup.template.name: {{ index_name }}
 setup.template.pattern: {{ index_name }}*

filebeat/tests/system/test_base.py

@@ -1,13 +1,19 @@
 import os
 import unittest
 from filebeat import BaseTest
-from elasticsearch import Elasticsearch
 from beat.beat import INTEGRATION_TESTS
 from beat import common_tests
 
 
 class Test(BaseTest, common_tests.TestExportsMixin, common_tests.TestDashboardMixin):
 
+    def setUp(self):
+        super(Test, self).setUp()
+        self.render_config_template(
+            elasticsearch=self.get_elasticsearch_template_config(),
+        )
+        self.es = self.get_elasticsearch_instance()
+
     def test_base(self):
         """
         Test if the basic fields exist.
@@ -32,12 +38,11 @@ def test_index_management(self):
         """
         Test that the template can be loaded with `setup --index-management`
         """
-        es = Elasticsearch([self.get_elasticsearch_url()])
         self.render_config_template(
-            elasticsearch={"host": self.get_elasticsearch_url()},
+            elasticsearch=self.get_elasticsearch_template_config(),
         )
         exit_code = self.run_beat(extra_args=["setup", "--index-management"])
 
         assert exit_code == 0
         assert self.log_contains('Loaded index template')
-        assert len(es.cat.templates(name='filebeat-*', h='name')) > 0
+        assert len(self.es.cat.templates(name='filebeat-*', h='name')) > 0

filebeat/tests/system/test_crawler.py

@@ -19,7 +19,7 @@ def test_fetched_lines(self):
         """
 
         self.render_config_template(
-            path=os.path.abspath(self.working_dir) + "/log/*"
+            path=os.path.abspath(self.working_dir) + "/log/*",
         )
         os.mkdir(self.working_dir + "/log/")
 
@@ -55,7 +55,7 @@ def test_unfinished_line_and_continue(self):
         """
 
         self.render_config_template(
-            path=os.path.abspath(self.working_dir) + "/log/*"
+            path=os.path.abspath(self.working_dir) + "/log/*",
         )
         os.mkdir(self.working_dir + "/log/")
 
@@ -162,7 +162,7 @@ def test_file_renaming(self):
         """
 
         self.render_config_template(
-            path=os.path.abspath(self.working_dir) + "/log/*"
+            path=os.path.abspath(self.working_dir) + "/log/*",
         )
         os.mkdir(self.working_dir + "/log/")
 
@@ -339,7 +339,7 @@ def test_new_line_on_existing_file(self):
         """
 
         self.render_config_template(
-            path=os.path.abspath(self.working_dir) + "/log/*"
+            path=os.path.abspath(self.working_dir) + "/log/*",
         )
         os.mkdir(self.working_dir + "/log/")
 
@@ -373,7 +373,7 @@ def test_multiple_appends(self):
         """
 
         self.render_config_template(
-            path=os.path.abspath(self.working_dir) + "/log/*"
+            path=os.path.abspath(self.working_dir) + "/log/*",
         )
         os.mkdir(self.working_dir + "/log/")
 
@@ -419,7 +419,7 @@ def test_new_line_on_open_file(self):
         """
 
         self.render_config_template(
-            path=os.path.abspath(self.working_dir) + "/log/*"
+            path=os.path.abspath(self.working_dir) + "/log/*",
         )
         os.mkdir(self.working_dir + "/log/")
 
@@ -457,7 +457,7 @@ def test_tail_files(self):
 
         self.render_config_template(
             path=os.path.abspath(self.working_dir) + "/log/*",
-            tail_files="true"
+            tail_files="true",
         )
         os.mkdir(self.working_dir + "/log/")
 
@@ -501,7 +501,7 @@ def test_utf8(self):
 
         self.render_config_template(
             path=os.path.abspath(self.working_dir) + "/log/*",
-            encoding="utf-8"
+            encoding="utf-8",
         )
         os.mkdir(self.working_dir + "/log/")
 
@@ -613,7 +613,7 @@ def test_include_lines(self):
 
         self.render_config_template(
             path=os.path.abspath(self.working_dir) + "/log/*",
-            include_lines=["^ERR", "^WARN"]
+            include_lines=["^ERR", "^WARN"],
         )
         os.mkdir(self.working_dir + "/log/")
 
@@ -648,9 +648,8 @@ def test_default_include_exclude_lines(self):
         """
         Checks if all the log lines are exported by default
         """
-
         self.render_config_template(
-            path=os.path.abspath(self.working_dir) + "/log/*"
+            path=os.path.abspath(self.working_dir) + "/log/*",
         )
         os.mkdir(self.working_dir + "/log/")
 
@@ -688,7 +687,7 @@ def test_exclude_lines(self):
 
         self.render_config_template(
             path=os.path.abspath(self.working_dir) + "/log/*",
-            exclude_lines=["^DBG"]
+            exclude_lines=["^DBG"],
         )
         os.mkdir(self.working_dir + "/log/")
 
@@ -727,7 +726,7 @@ def test_include_exclude_lines(self):
         self.render_config_template(
             path=os.path.abspath(self.working_dir) + "/log/*",
             exclude_lines=["^DBG"],
-            include_lines=["apache"]
+            include_lines=["apache"],
         )
         os.mkdir(self.working_dir + "/log/")
 

filebeat/tests/system/test_modules.py

@@ -5,7 +5,6 @@
 import glob
 import subprocess
 
-from elasticsearch import Elasticsearch
 import json
 import logging
 from parameterized import parameterized
@@ -61,9 +60,7 @@ def load_fileset_test_cases():
 class Test(BaseTest):
 
     def init(self):
-        self.elasticsearch_url = self.get_elasticsearch_url()
-        print("Using elasticsearch: {}".format(self.elasticsearch_url))
-        self.es = Elasticsearch([self.elasticsearch_url])
+        self.es = self.get_elasticsearch_instance(user='admin')
         logging.getLogger("urllib3").setLevel(logging.WARNING)
         logging.getLogger("elasticsearch").setLevel(logging.ERROR)
 
@@ -89,7 +86,7 @@ def test_fileset_file(self, module, fileset, test_file):
             template_name="filebeat_modules",
             output=cfgfile,
             index_name=self.index_name,
-            elasticsearch_url=self.elasticsearch_url,
+            elasticsearch=self.get_elasticsearch_template_config(user='admin')
         )
 
         self.run_on_file(

filebeat/tests/system/test_pipeline.py

@@ -2,7 +2,6 @@
 from beat.beat import INTEGRATION_TESTS
 import os
 import unittest
-from elasticsearch import Elasticsearch
 import json
 import logging
 
@@ -12,8 +11,7 @@ class Test(BaseTest):
     def init(self):
         self.elasticsearch_url = self.get_elasticsearch_url()
         self.kibana_url = self.get_kibana_url()
-        print("Using elasticsearch: {}".format(self.elasticsearch_url))
-        self.es = Elasticsearch([self.elasticsearch_url])
+        self.es = self.get_elasticsearch_instance()
         logging.getLogger("urllib3").setLevel(logging.WARNING)
         logging.getLogger("elasticsearch").setLevel(logging.ERROR)
 
@@ -47,10 +45,13 @@ def test_input_pipeline_config(self):
 
         self.render_config_template(
             path=os.path.abspath(self.working_dir) + "/log/*",
-            elasticsearch=dict(
-                host=self.elasticsearch_url,
-                pipeline="estest",
-                index=index_name),
+            elasticsearch={
+                'host': self.elasticsearch_url,
+                'pipeline': "estest",
+                'index': index_name,
+                'user': os.getenv("ES_USER"),
+                'pass': os.getenv("ES_PASS")
+            },
             pipeline="test",
             setup_template_name=index_name,
             setup_template_pattern=index_name + "*",

filebeat/tests/system/test_reload_modules.py

@@ -6,7 +6,6 @@
 
 from filebeat import BaseTest
 from beat.beat import INTEGRATION_TESTS
-from elasticsearch import Elasticsearch
 
 
 moduleConfigTemplate = """
@@ -27,7 +26,7 @@ class Test(BaseTest):
     def setUp(self):
         super(BaseTest, self).setUp()
         if INTEGRATION_TESTS:
-            self.es = Elasticsearch([self.get_elasticsearch_url()])
+            self.es = self.get_elasticsearch_instance()
 
         # Copy system module
         shutil.copytree(os.path.join(self.beat_path, "tests", "system", "module", "test"),
@@ -72,7 +71,7 @@ def test_reload_writes_pipeline(self):
             reload_path=self.working_dir + "/configs/*.yml",
             reload_type="modules",
             inputs=False,
-            elasticsearch={"host": self.get_elasticsearch_url()}
+            elasticsearch=self.get_elasticsearch_template_config(),
         )
 
         proc = self.start_beat()

filebeat/tests/system/test_setup.py

@@ -3,8 +3,6 @@
 import yaml
 from shutil import copytree, copyfile
 
-from elasticsearch import Elasticsearch
-
 from filebeat import BaseTest
 
 INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False)
@@ -15,7 +13,7 @@ class Test(BaseTest):
     def init(self):
         self.elasticsearch_url = self.get_elasticsearch_url()
         print("Using elasticsearch: {}".format(self.elasticsearch_url))
-        self.es = Elasticsearch([self.elasticsearch_url])
+        self.es = self.get_elasticsearch_instance()
 
     @unittest.skipIf(not INTEGRATION_TESTS,
                      "integration tests are disabled, run with INTEGRATION_TESTS=1 to enable them.")
@@ -28,9 +26,7 @@ def test_setup_modules_d_config(self):
         self.init()
         self.render_config_template(
             modules=True,
-            elasticsearch={
-                "host": self.get_elasticsearch_url(),
-            },
+            elasticsearch=self.get_elasticsearch_template_config(),
         )
 
         self._setup_dummy_module()

heartbeat/docker-compose.yml

@@ -8,6 +8,8 @@ services:
     - REDIS_HOST=redis
     - REDIS_PORT=6379
     - ES_HOST=elasticsearch
+    - ES_USER=heartbeat_user
+    - ES_PASS=testing
     - ES_PORT=9200
     working_dir: /go/src/github.com/elastic/beats/heartbeat
     volumes:

heartbeat/tests/system/config/heartbeat.yml.j2

@@ -105,6 +105,8 @@ queue.mem:
 {%- if elasticsearch %}
 output.elasticsearch:
   hosts: ["{{ elasticsearch.host }}"]
+  username: {{ elasticsearch.user }}
+  password: {{ elasticsearch.pass }}
 {%- else %}
 output.file:
   path: '{{ output_file_path|default(beat.working_dir + "/output") }}'