From 0490c5d453001c0e931b968791ed57041799d38d Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 27 Sep 2021 15:37:44 +0200 Subject: [PATCH 01/52] Enable xpack security for snapshot testing --- testing/environments/Makefile | 2 ++ testing/environments/snapshot.yml | 18 ++++++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/testing/environments/Makefile b/testing/environments/Makefile index bef47686095..6387289ac01 100644 --- a/testing/environments/Makefile +++ b/testing/environments/Makefile @@ -9,6 +9,8 @@ start: stop: ${BASE_COMMAND} down -v +status: + ${BASE_COMMAND} ps up: ${BASE_COMMAND} build diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index c9215fe874b..d7cfcf28d0c 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -5,21 +5,21 @@ services: elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:8.0.0-2f347a19-SNAPSHOT healthcheck: - test: ["CMD-SHELL", "curl -s http://localhost:9200/_cat/health?h=status | grep -q green"] + test: ["CMD-SHELL", "curl -u elastic:changeme -s http://localhost:9200/_cat/health?h=status | grep -q green"] retries: 300 interval: 1s environment: - "ES_JAVA_OPTS=-Xms1g -Xmx1g" + - "ELASTIC_PASSWORD=changeme" - "network.host=" - "transport.host=127.0.0.1" - "http.host=0.0.0.0" - - "xpack.security.enabled=false" + - "xpack.security.enabled=true" - "script.context.template.max_compilations_rate=unlimited" - "script.context.ingest.cache_max_size=2000" - "script.context.processor_conditional.cache_max_size=2000" - "script.context.template.cache_max_size=2000" - "action.destructive_requires_name=false" - # Disable geoip updates to prevent golden file test failures when the database # changes and prevent race conditions between tests and database updates. - "ingest.geoip.downloader.enabled=false" volumes: @@ -35,7 +35,7 @@ services: retries: 600 interval: 1s volumes: - - ./docker/logstash/pipeline:/usr/share/logstash/pipeline:ro + - ./docker/logstash/pipeline-xpack:/usr/share/logstash/pipeline:ro - ./docker/logstash/pki:/etc/pki:ro kibana: @@ -44,3 +44,13 @@ services: test: ["CMD-SHELL", "curl -s http://localhost:5601/api/status?v8format=true | grep -q '\"overall\":{\"level\":\"available\"'"] retries: 600 interval: 1s + environment: + - "ELASTICSEARCH_USERNAME=elastic" + - "ELASTICSEARCH_PASSWORD=changeme" + - "XPACK_SECURITY_ENCRYPTIONKEY=wZSVeczkXAmebqNgfcKEzNMmQCBZKkSH" + - "XPACK_XPACK_MAIN_TELEMETRY_ENABLED=false" + - "XPACK_REPORTING_ENCRYPTIONKEY=xCyqJUFqrUJJKxjZVGfnhrRkyqqaKeAG" + + + + From 2b0fefe2505610910ca17eef0fa50538fd1ffa2b Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 27 Sep 2021 15:38:50 +0200 Subject: [PATCH 02/52] Restore accidentially deleted comment --- testing/environments/snapshot.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index d7cfcf28d0c..3c6129dea42 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -20,6 +20,7 @@ services: - "script.context.processor_conditional.cache_max_size=2000" - "script.context.template.cache_max_size=2000" - "action.destructive_requires_name=false" + # Disable geoip updates to prevent golden file test failures when the database # changes and prevent race conditions between tests and database updates. - "ingest.geoip.downloader.enabled=false" volumes: From cf15cb8c349eb52ff0c2d0b9d11b8692986b38f8 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 27 Sep 2021 15:39:34 +0200 Subject: [PATCH 03/52] Include logstash config for xpack security --- .../logstash/pipeline-xpack/default.conf | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 testing/environments/docker/logstash/pipeline-xpack/default.conf diff --git a/testing/environments/docker/logstash/pipeline-xpack/default.conf b/testing/environments/docker/logstash/pipeline-xpack/default.conf new file mode 100644 index 00000000000..3619905a615 --- /dev/null +++ b/testing/environments/docker/logstash/pipeline-xpack/default.conf @@ -0,0 +1,26 @@ +input { + beats { + port => 5044 + ssl => false + } + + beats { + port => 5055 + ssl => true + ssl_certificate => "/etc/pki/tls/certs/logstash.crt" + ssl_key => "/etc/pki/tls/private/logstash.key" + } +} + + +output { + elasticsearch { + hosts => ["${ES_HOST:elasticsearch}:${ES_PORT:9200}"] + index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}" + user => elastic + password => changeme + } + + # Used for easier debugging + #stdout { codec => rubydebug { metadata => true } } +} From 01d1d102d811778f56daa93f19035f3bac05e517 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 27 Sep 2021 15:49:05 +0200 Subject: [PATCH 04/52] Clean up spacing --- testing/environments/snapshot.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 3c6129dea42..98f981fef30 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -52,6 +52,3 @@ services: - "XPACK_XPACK_MAIN_TELEMETRY_ENABLED=false" - "XPACK_REPORTING_ENCRYPTIONKEY=xCyqJUFqrUJJKxjZVGfnhrRkyqqaKeAG" - - - From 16dbf065e7ba1edc3269d5261c36e53e2e249f4e Mon Sep 17 00:00:00 2001 From: Mike Place Date: Tue, 28 Sep 2021 11:20:21 +0200 Subject: [PATCH 05/52] Fix up Filebeat tests to work with xpack security enabled --- filebeat/tests/system/test_base.py | 18 +++++++++++++++++- filebeat/tests/system/test_modules.py | 2 +- filebeat/tests/system/test_pipeline.py | 2 +- filebeat/tests/system/test_reload_modules.py | 2 +- filebeat/tests/system/test_setup.py | 2 +- heartbeat/tests/system/test_base.py | 2 +- libbeat/tests/system/base.py | 2 +- libbeat/tests/system/test_cmd_version.py | 2 +- libbeat/tests/system/test_monitoring.py | 2 +- .../system/config/filebeat_modules.yml.j2 | 2 ++ 10 files changed, 27 insertions(+), 9 deletions(-) diff --git a/filebeat/tests/system/test_base.py b/filebeat/tests/system/test_base.py index 6082c07f609..15717a4cd47 100644 --- a/filebeat/tests/system/test_base.py +++ b/filebeat/tests/system/test_base.py @@ -32,7 +32,7 @@ def test_index_management(self): """ Test that the template can be loaded with `setup --index-management` """ - es = Elasticsearch([self.get_elasticsearch_url()]) + es = Elasticsearch([self.get_elasticsearch_url()], http_auth=('elastic', 'changeme')) self.render_config_template( elasticsearch={"host": self.get_elasticsearch_url()}, ) @@ -41,3 +41,19 @@ def test_index_management(self): assert exit_code == 0 assert self.log_contains('Loaded index template') assert len(es.cat.templates(name='filebeat-*', h='name')) > 0 + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + def test_template_migration(self): + """ + Test that the template can be loaded with `setup --template` + """ + es = Elasticsearch([self.get_elasticsearch_url()], http_auth=('elastic', 'changeme')) + self.render_config_template( + elasticsearch={"host": self.get_elasticsearch_url()}, + ) + exit_code = self.run_beat(extra_args=["setup", "--template", + "-E", "setup.template.overwrite=true", "-E", "migration.6_to_7.enabled=true"]) + + assert exit_code == 0 + assert self.log_contains('Loaded index template') + assert len(es.cat.templates(name='filebeat-*', h='name')) > 0 diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py index b552560f83c..d7e1d81db51 100644 --- a/filebeat/tests/system/test_modules.py +++ b/filebeat/tests/system/test_modules.py @@ -63,7 +63,7 @@ class Test(BaseTest): def init(self): self.elasticsearch_url = self.get_elasticsearch_url() print("Using elasticsearch: {}".format(self.elasticsearch_url)) - self.es = Elasticsearch([self.elasticsearch_url]) + self.es = Elasticsearch([self.elasticsearch_url], http_auth=('elastic', 'changeme')) logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("elasticsearch").setLevel(logging.ERROR) diff --git a/filebeat/tests/system/test_pipeline.py b/filebeat/tests/system/test_pipeline.py index afb3219e62d..29adca082f0 100644 --- a/filebeat/tests/system/test_pipeline.py +++ b/filebeat/tests/system/test_pipeline.py @@ -13,7 +13,7 @@ def init(self): self.elasticsearch_url = self.get_elasticsearch_url() self.kibana_url = self.get_kibana_url() print("Using elasticsearch: {}".format(self.elasticsearch_url)) - self.es = Elasticsearch([self.elasticsearch_url]) + self.es = Elasticsearch([self.elasticsearch_url], http_auth=('elastic', 'changeme')) logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("elasticsearch").setLevel(logging.ERROR) diff --git a/filebeat/tests/system/test_reload_modules.py b/filebeat/tests/system/test_reload_modules.py index b22294e7d9a..9ef8988a9cf 100644 --- a/filebeat/tests/system/test_reload_modules.py +++ b/filebeat/tests/system/test_reload_modules.py @@ -27,7 +27,7 @@ class Test(BaseTest): def setUp(self): super(BaseTest, self).setUp() if INTEGRATION_TESTS: - self.es = Elasticsearch([self.get_elasticsearch_url()]) + self.es = Elasticsearch([self.get_elasticsearch_url()], http_auth=('elastic', 'changeme')) # Copy system module shutil.copytree(os.path.join(self.beat_path, "tests", "system", "module", "test"), diff --git a/filebeat/tests/system/test_setup.py b/filebeat/tests/system/test_setup.py index bd1a96a9194..4dc8d92772f 100644 --- a/filebeat/tests/system/test_setup.py +++ b/filebeat/tests/system/test_setup.py @@ -15,7 +15,7 @@ class Test(BaseTest): def init(self): self.elasticsearch_url = self.get_elasticsearch_url() print("Using elasticsearch: {}".format(self.elasticsearch_url)) - self.es = Elasticsearch([self.elasticsearch_url]) + self.es = Elasticsearch([self.elasticsearch_url], http_auth=('elastic', 'changeme')) @unittest.skipIf(not INTEGRATION_TESTS, "integration tests are disabled, run with INTEGRATION_TESTS=1 to enable them.") diff --git a/heartbeat/tests/system/test_base.py b/heartbeat/tests/system/test_base.py index 172960209d3..4889b9f16e0 100644 --- a/heartbeat/tests/system/test_base.py +++ b/heartbeat/tests/system/test_base.py @@ -198,7 +198,7 @@ def test_index_management(self): """ Test that the template can be loaded with `setup --index-management` """ - es = Elasticsearch([self.get_elasticsearch_url()]) + es = Elasticsearch([self.get_elasticsearch_url()], http_auth=('elastic', 'changeme')) self.render_config_template( monitors=[{ "type": "http", diff --git a/libbeat/tests/system/base.py b/libbeat/tests/system/base.py index a768d80e31e..5a222337a55 100644 --- a/libbeat/tests/system/base.py +++ b/libbeat/tests/system/base.py @@ -26,5 +26,5 @@ def es_client(self): if self._es: return self._es - self._es = Elasticsearch([self.get_elasticsearch_url()]) + self._es = Elasticsearch([self.get_elasticsearch_url()], http_auth=('elastic', 'changeme')) return self._es diff --git a/libbeat/tests/system/test_cmd_version.py b/libbeat/tests/system/test_cmd_version.py index 240b8759668..b841df17507 100644 --- a/libbeat/tests/system/test_cmd_version.py +++ b/libbeat/tests/system/test_cmd_version.py @@ -18,7 +18,7 @@ def setUp(self): self.elasticsearch_url = self.get_elasticsearch_url() print("Using elasticsearch: {}".format(self.elasticsearch_url)) - self.es = Elasticsearch([self.elasticsearch_url]) + self.es = Elasticsearch([self.elasticsearch_url], http_auth=('elastic', 'changeme')) logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("elasticsearch").setLevel(logging.ERROR) diff --git a/libbeat/tests/system/test_monitoring.py b/libbeat/tests/system/test_monitoring.py index 2232b19712f..7c79f486971 100644 --- a/libbeat/tests/system/test_monitoring.py +++ b/libbeat/tests/system/test_monitoring.py @@ -19,7 +19,7 @@ def setUp(self): super(BaseTest, self).setUp() self.es = Elasticsearch([self.get_elasticsearch_url()]) - self.es_monitoring = Elasticsearch([self.get_elasticsearch_monitoring_url()]) + self.es_monitoring = Elasticsearch([self.get_elasticsearch_monitoring_url()], http_auth=('elastic', 'changeme')) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") @pytest.mark.tag('integration') diff --git a/x-pack/filebeat/tests/system/config/filebeat_modules.yml.j2 b/x-pack/filebeat/tests/system/config/filebeat_modules.yml.j2 index 6df0f3ba0d9..def6543dc25 100644 --- a/x-pack/filebeat/tests/system/config/filebeat_modules.yml.j2 +++ b/x-pack/filebeat/tests/system/config/filebeat_modules.yml.j2 @@ -14,6 +14,8 @@ filebeat.registry: output.elasticsearch.hosts: ["{{ elasticsearch_url }}"] output.elasticsearch.index: {{ index_name }} +output.elasticsearch.username: elastic +output.elasticsearch.password: changeme setup.template.name: {{ index_name }} setup.template.pattern: {{ index_name }}* From 2093f15c56a1f7da430f67fe75b30f0e77ce78bf Mon Sep 17 00:00:00 2001 From: Mike Place Date: Wed, 29 Sep 2021 14:22:31 +0200 Subject: [PATCH 06/52] More test fixing for enabling xpack security --- filebeat/fileset/modules_integration_test.go | 4 +++- heartbeat/tests/system/config/heartbeat.yml.j2 | 2 ++ libbeat/docker-compose.yml | 2 +- libbeat/esleg/eslegtest/util.go | 4 ++-- libbeat/template/load_integration_test.go | 2 ++ libbeat/tests/system/config/libbeat.yml.j2 | 2 ++ 6 files changed, 12 insertions(+), 4 deletions(-) diff --git a/filebeat/fileset/modules_integration_test.go b/filebeat/fileset/modules_integration_test.go index 6b9e86cac84..80cbb19c621 100644 --- a/filebeat/fileset/modules_integration_test.go +++ b/filebeat/fileset/modules_integration_test.go @@ -258,7 +258,9 @@ func TestLoadMultiplePipelinesWithRollback(t *testing.T) { func getTestingElasticsearch(t eslegtest.TestLogger) *eslegclient.Connection { conn, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ - URL: eslegtest.GetURL(), + URL: eslegtest.GetURL(), + Username: "elastic", + Password: "changeme", }) if err != nil { t.Fatal(err) diff --git a/heartbeat/tests/system/config/heartbeat.yml.j2 b/heartbeat/tests/system/config/heartbeat.yml.j2 index 44dfe3a836d..e282dd5dae0 100644 --- a/heartbeat/tests/system/config/heartbeat.yml.j2 +++ b/heartbeat/tests/system/config/heartbeat.yml.j2 @@ -105,6 +105,8 @@ queue.mem: {%- if elasticsearch %} output.elasticsearch: hosts: ["{{ elasticsearch.host }}"] + username: elastic + password: changeme {%- else %} output.file: path: '{{ output_file_path|default(beat.working_dir + "/output") }}' diff --git a/libbeat/docker-compose.yml b/libbeat/docker-compose.yml index c96b40e3ea8..999e1b637a6 100644 --- a/libbeat/docker-compose.yml +++ b/libbeat/docker-compose.yml @@ -67,7 +67,7 @@ services: file: ${ES_BEATS}/testing/environments/${TESTING_ENVIRONMENT}.yml service: elasticsearch healthcheck: - test: ["CMD-SHELL", "curl -s http://localhost:9200/_cat/health?h=status | grep -q green"] + test: ["CMD-SHELL", "curl -u elastic:changeme -s http://localhost:9200/_cat/health?h=status | grep -q green"] elasticsearchssl: extends: file: ${ES_BEATS}/testing/environments/${TESTING_ENVIRONMENT}.yml diff --git a/libbeat/esleg/eslegtest/util.go b/libbeat/esleg/eslegtest/util.go index 28f33fde2dc..8c1519c8762 100644 --- a/libbeat/esleg/eslegtest/util.go +++ b/libbeat/esleg/eslegtest/util.go @@ -76,12 +76,12 @@ func getEsPort() string { // GetUser returns the Elasticsearch testing user. func GetUser() string { - return getEnv("ES_USER", "") + return getEnv("ES_USER", "elastic") } // GetPass returns the Elasticsearch testing user's password. func GetPass() string { - return getEnv("ES_PASS", "") + return getEnv("ES_PASS", "changeme") } func getEnv(name, def string) string { diff --git a/libbeat/template/load_integration_test.go b/libbeat/template/load_integration_test.go index 371991810e4..c0567b19e55 100644 --- a/libbeat/template/load_integration_test.go +++ b/libbeat/template/load_integration_test.go @@ -408,6 +408,8 @@ func getTestingElasticsearch(t eslegtest.TestLogger) *eslegclient.Connection { conn, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ URL: eslegtest.GetURL(), Transport: httpcommon.DefaultHTTPTransportSettings(), + Username: eslegtest.GetUser(), + Password: eslegtest.GetPass() }) if err != nil { t.Fatal(err) diff --git a/libbeat/tests/system/config/libbeat.yml.j2 b/libbeat/tests/system/config/libbeat.yml.j2 index edfc178eea8..62f92f49e74 100644 --- a/libbeat/tests/system/config/libbeat.yml.j2 +++ b/libbeat/tests/system/config/libbeat.yml.j2 @@ -74,6 +74,8 @@ setup.kibana.host: "{{ kibana.host }}" output: elasticsearch: hosts: ["{{ elasticsearch.host }}"] + username: "elastic" + password: "changeme" {% if elasticsearch.pipeline %} pipeline: {{elasticsearch.pipeline}} {% endif %} From 06a216e4bb8fed358e5c7604c54a007096724ae4 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Wed, 29 Sep 2021 14:50:22 +0200 Subject: [PATCH 07/52] Fix lint error --- libbeat/template/load_integration_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libbeat/template/load_integration_test.go b/libbeat/template/load_integration_test.go index c0567b19e55..aeb11fc4ba0 100644 --- a/libbeat/template/load_integration_test.go +++ b/libbeat/template/load_integration_test.go @@ -408,8 +408,8 @@ func getTestingElasticsearch(t eslegtest.TestLogger) *eslegclient.Connection { conn, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ URL: eslegtest.GetURL(), Transport: httpcommon.DefaultHTTPTransportSettings(), - Username: eslegtest.GetUser(), - Password: eslegtest.GetPass() + Username: eslegtest.GetUser(), + Password: eslegtest.GetPass(), }) if err != nil { t.Fatal(err) From 99c7303a372f76998fae094b0abb13b8fcc275f7 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Thu, 30 Sep 2021 10:46:21 +0200 Subject: [PATCH 08/52] Setting xpack username and pass --- filebeat/fileset/modules_integration_test.go | 4 ++-- libbeat/esleg/eslegtest/util.go | 14 ++++++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/filebeat/fileset/modules_integration_test.go b/filebeat/fileset/modules_integration_test.go index 80cbb19c621..aec3c740593 100644 --- a/filebeat/fileset/modules_integration_test.go +++ b/filebeat/fileset/modules_integration_test.go @@ -259,8 +259,8 @@ func TestLoadMultiplePipelinesWithRollback(t *testing.T) { func getTestingElasticsearch(t eslegtest.TestLogger) *eslegclient.Connection { conn, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ URL: eslegtest.GetURL(), - Username: "elastic", - Password: "changeme", + Username: eslegtest.GetXPackUser(), + Password: eslegtest.GetXPackPass(), }) if err != nil { t.Fatal(err) diff --git a/libbeat/esleg/eslegtest/util.go b/libbeat/esleg/eslegtest/util.go index 8c1519c8762..bac04827204 100644 --- a/libbeat/esleg/eslegtest/util.go +++ b/libbeat/esleg/eslegtest/util.go @@ -76,12 +76,22 @@ func getEsPort() string { // GetUser returns the Elasticsearch testing user. func GetUser() string { - return getEnv("ES_USER", "elastic") + return getEnv("ES_USER", "") } // GetPass returns the Elasticsearch testing user's password. func GetPass() string { - return getEnv("ES_PASS", "changeme") + return getEnv("ES_PASS", "") +} + +// GetXPackUser returns the admin user for Elasticsearch +func GetXPackUser() string { + return getEnv("ES_XPACK_USER", "elastic") +} + +// GetXpackPass returns the admin pass for Elasticsearch +func GetXPackPass() string { + return getEnv("ES_XPACK_PASS", "changeme") } func getEnv(name, def string) string { From 7b4468c49a5736228df3056753864629ca4699f5 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Thu, 30 Sep 2021 17:24:58 +0200 Subject: [PATCH 09/52] Moving xpack auth into env --- dev-tools/mage/pytest.go | 1 + filebeat/tests/system/test_modules.py | 3 +-- filebeat/tests/system/test_pipeline.py | 3 +-- filebeat/tests/system/test_reload_modules.py | 3 +-- filebeat/tests/system/test_setup.py | 4 +-- heartbeat/tests/system/test_base.py | 6 ++--- libbeat/tests/system/beat/beat.py | 27 +++++++++++++++++--- libbeat/tests/system/test_cmd_version.py | 3 +-- libbeat/tests/system/test_monitoring.py | 5 ++-- 9 files changed, 34 insertions(+), 21 deletions(-) diff --git a/dev-tools/mage/pytest.go b/dev-tools/mage/pytest.go index d4b8dd3fcee..f97dede19f8 100644 --- a/dev-tools/mage/pytest.go +++ b/dev-tools/mage/pytest.go @@ -133,6 +133,7 @@ func PythonTest(params PythonTestArgs) error { pytestOptions := []string{ "--timeout=90", "--durations=20", + "-x", } if mg.Verbose() { pytestOptions = append(pytestOptions, "-v") diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py index d7e1d81db51..d456c2ce1f8 100644 --- a/filebeat/tests/system/test_modules.py +++ b/filebeat/tests/system/test_modules.py @@ -5,7 +5,6 @@ import glob import subprocess -from elasticsearch import Elasticsearch import json import logging from parameterized import parameterized @@ -63,7 +62,7 @@ class Test(BaseTest): def init(self): self.elasticsearch_url = self.get_elasticsearch_url() print("Using elasticsearch: {}".format(self.elasticsearch_url)) - self.es = Elasticsearch([self.elasticsearch_url], http_auth=('elastic', 'changeme')) + self.es = self.get_elasticsearch_instance(url=self.elasticsearch_url) logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("elasticsearch").setLevel(logging.ERROR) diff --git a/filebeat/tests/system/test_pipeline.py b/filebeat/tests/system/test_pipeline.py index 29adca082f0..4527a88e752 100644 --- a/filebeat/tests/system/test_pipeline.py +++ b/filebeat/tests/system/test_pipeline.py @@ -2,7 +2,6 @@ from beat.beat import INTEGRATION_TESTS import os import unittest -from elasticsearch import Elasticsearch import json import logging @@ -13,7 +12,7 @@ def init(self): self.elasticsearch_url = self.get_elasticsearch_url() self.kibana_url = self.get_kibana_url() print("Using elasticsearch: {}".format(self.elasticsearch_url)) - self.es = Elasticsearch([self.elasticsearch_url], http_auth=('elastic', 'changeme')) + self.es = self.get_elasticsearch_instance(url=self.elasticsearch_url) logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("elasticsearch").setLevel(logging.ERROR) diff --git a/filebeat/tests/system/test_reload_modules.py b/filebeat/tests/system/test_reload_modules.py index 9ef8988a9cf..22886c98052 100644 --- a/filebeat/tests/system/test_reload_modules.py +++ b/filebeat/tests/system/test_reload_modules.py @@ -6,7 +6,6 @@ from filebeat import BaseTest from beat.beat import INTEGRATION_TESTS -from elasticsearch import Elasticsearch moduleConfigTemplate = """ @@ -27,7 +26,7 @@ class Test(BaseTest): def setUp(self): super(BaseTest, self).setUp() if INTEGRATION_TESTS: - self.es = Elasticsearch([self.get_elasticsearch_url()], http_auth=('elastic', 'changeme')) + self.es = self.get_elasticsearch_instance() # Copy system module shutil.copytree(os.path.join(self.beat_path, "tests", "system", "module", "test"), diff --git a/filebeat/tests/system/test_setup.py b/filebeat/tests/system/test_setup.py index 4dc8d92772f..3ea31152242 100644 --- a/filebeat/tests/system/test_setup.py +++ b/filebeat/tests/system/test_setup.py @@ -3,8 +3,6 @@ import yaml from shutil import copytree, copyfile -from elasticsearch import Elasticsearch - from filebeat import BaseTest INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) @@ -15,7 +13,7 @@ class Test(BaseTest): def init(self): self.elasticsearch_url = self.get_elasticsearch_url() print("Using elasticsearch: {}".format(self.elasticsearch_url)) - self.es = Elasticsearch([self.elasticsearch_url], http_auth=('elastic', 'changeme')) + self.es = self.get_elasticsearch_instance(url=self.elasticsearch_url) @unittest.skipIf(not INTEGRATION_TESTS, "integration tests are disabled, run with INTEGRATION_TESTS=1 to enable them.") diff --git a/heartbeat/tests/system/test_base.py b/heartbeat/tests/system/test_base.py index 4889b9f16e0..dfd47aba9d7 100644 --- a/heartbeat/tests/system/test_base.py +++ b/heartbeat/tests/system/test_base.py @@ -2,7 +2,6 @@ import unittest from heartbeat import BaseTest -from elasticsearch import Elasticsearch from beat.beat import INTEGRATION_TESTS from beat import common_tests from time import sleep @@ -198,13 +197,14 @@ def test_index_management(self): """ Test that the template can be loaded with `setup --index-management` """ - es = Elasticsearch([self.get_elasticsearch_url()], http_auth=('elastic', 'changeme')) + es_url = self.get_elasticsearch_url() + es = self.get_elasticsearch_instance(url=es_url) self.render_config_template( monitors=[{ "type": "http", "urls": ["http://localhost:9200"], }], - elasticsearch={"host": self.get_elasticsearch_url()}, + elasticsearch={"host": es_url}, ) exit_code = self.run_beat(extra_args=["setup", "--index-management"]) diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index e7a2c60640a..ea93119f2a5 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -16,6 +16,8 @@ from .compose import ComposeMixin +from elasticsearch import Elasticsearch + BEAT_REQUIRED_FIELDS = ["@timestamp", "agent.type", "agent.name", "agent.version"] @@ -669,8 +671,7 @@ def output_count(self, pred, output_file=None): def get_elasticsearch_url(self): """ - Returns an elasticsearch.Elasticsearch instance built from the - env variables like the integration tests. + Returns a string with the Elasticsearch URL """ return "http://{host}:{port}".format( host=os.getenv("ES_HOST", "localhost"), @@ -679,14 +680,32 @@ def get_elasticsearch_url(self): def get_elasticsearch_url_ssl(self): """ - Returns an elasticsearch.Elasticsearch instance built from the - env variables like the integration tests. + Returns a string with the Elasticsearch URL """ return "https://{host}:{port}".format( host=os.getenv("ES_HOST_SSL", "localhost"), port=os.getenv("ES_PORT_SSL", "9205"), ) + def get_elasticsearch_instance(self, xpack_security=True, ssl=False, url=None): + """ + Returns an elasticsearch.Elasticsearch instance built from the + env variables like the integration tests. + """ + if url is None: + if ssl: + url = self.get_elasticsearch_url_ssl() + else: + url = self.get_elasticsearch_url() + + if xpack_security: + username = os.getenv("ES_XPACK_USER", "elastic") + password = os.getenv("ES_XPACK_PASS", "changeme") + es_instance = Elasticsearch([url], http_auth=(username, password)) + else: + es_instance = Elasticsearch([url]) + return es_instance + def get_kibana_url(self): """ Returns kibana host URL diff --git a/libbeat/tests/system/test_cmd_version.py b/libbeat/tests/system/test_cmd_version.py index b841df17507..fb98f53c577 100644 --- a/libbeat/tests/system/test_cmd_version.py +++ b/libbeat/tests/system/test_cmd_version.py @@ -1,5 +1,4 @@ from base import BaseTest -from elasticsearch import Elasticsearch, TransportError import logging import os @@ -18,7 +17,7 @@ def setUp(self): self.elasticsearch_url = self.get_elasticsearch_url() print("Using elasticsearch: {}".format(self.elasticsearch_url)) - self.es = Elasticsearch([self.elasticsearch_url], http_auth=('elastic', 'changeme')) + self.es = self.get_elasticsearch_instance(url=self.elasticsearch_url) logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("elasticsearch").setLevel(logging.ERROR) diff --git a/libbeat/tests/system/test_monitoring.py b/libbeat/tests/system/test_monitoring.py index 7c79f486971..1fd2bc415d1 100644 --- a/libbeat/tests/system/test_monitoring.py +++ b/libbeat/tests/system/test_monitoring.py @@ -7,7 +7,6 @@ import unittest from base import BaseTest -from elasticsearch import Elasticsearch INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) @@ -18,8 +17,8 @@ class Test(BaseTest): def setUp(self): super(BaseTest, self).setUp() - self.es = Elasticsearch([self.get_elasticsearch_url()]) - self.es_monitoring = Elasticsearch([self.get_elasticsearch_monitoring_url()], http_auth=('elastic', 'changeme')) + self.es = self.get_elasticsearch_instance() + self.es_monitoring = self.get_elasticsearch_instance(url=self.get_elasticsearch_monitoring_url()) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") @pytest.mark.tag('integration') From 24214b2bd5563cc720473381691033fa5efb4c6b Mon Sep 17 00:00:00 2001 From: Mike Place Date: Thu, 30 Sep 2021 17:28:36 +0200 Subject: [PATCH 10/52] Centralize ES config in test_base --- filebeat/tests/system/test_base.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/filebeat/tests/system/test_base.py b/filebeat/tests/system/test_base.py index 15717a4cd47..5f5d8769bae 100644 --- a/filebeat/tests/system/test_base.py +++ b/filebeat/tests/system/test_base.py @@ -1,7 +1,6 @@ import os import unittest from filebeat import BaseTest -from elasticsearch import Elasticsearch from beat.beat import INTEGRATION_TESTS from beat import common_tests @@ -32,9 +31,10 @@ def test_index_management(self): """ Test that the template can be loaded with `setup --index-management` """ - es = Elasticsearch([self.get_elasticsearch_url()], http_auth=('elastic', 'changeme')) + es_url = self.get_elasticsearch_url() + es = self.get_elasticsearch_instance(url=es_url) self.render_config_template( - elasticsearch={"host": self.get_elasticsearch_url()}, + elasticsearch={"host": es_url}, ) exit_code = self.run_beat(extra_args=["setup", "--index-management"]) @@ -47,9 +47,10 @@ def test_template_migration(self): """ Test that the template can be loaded with `setup --template` """ - es = Elasticsearch([self.get_elasticsearch_url()], http_auth=('elastic', 'changeme')) + es_url = self.get_elasticsearch_url() + es = self.get_elasticsearch_instance(url=es_url) self.render_config_template( - elasticsearch={"host": self.get_elasticsearch_url()}, + elasticsearch={"host": es_url}, ) exit_code = self.run_beat(extra_args=["setup", "--template", "-E", "setup.template.overwrite=true", "-E", "migration.6_to_7.enabled=true"]) From 9b460bef0f2c54cc83781c9841939bb914054c44 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Thu, 30 Sep 2021 17:31:41 +0200 Subject: [PATCH 11/52] Use get_elasticsearch_instance in base.py --- libbeat/tests/system/base.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libbeat/tests/system/base.py b/libbeat/tests/system/base.py index 5a222337a55..c5f72d02008 100644 --- a/libbeat/tests/system/base.py +++ b/libbeat/tests/system/base.py @@ -1,7 +1,6 @@ import os from datetime import datetime from beat.beat import TestCase -from elasticsearch import Elasticsearch, NotFoundError class BaseTest(TestCase): @@ -26,5 +25,5 @@ def es_client(self): if self._es: return self._es - self._es = Elasticsearch([self.get_elasticsearch_url()], http_auth=('elastic', 'changeme')) + self._es = self.get_elasticsearch_instance() return self._es From cc0949d112c19ee123fb6a6e30f6db2e115a0de3 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Fri, 1 Oct 2021 19:23:33 +0200 Subject: [PATCH 12/52] Splitting out auth by module --- auditbeat/docker-compose.yml | 2 +- auditbeat/tests/system/test_base.py | 12 +- filebeat/tests/system/test_base.py | 20 ++-- filebeat/tests/system/test_crawler.py | 104 ++++++++++++++++-- heartbeat/docker-compose.yml | 2 + .../tests/system/config/heartbeat.yml.j2 | 4 +- heartbeat/tests/system/test_base.py | 10 +- libbeat/esleg/eslegtest/util.go | 10 -- .../logstash/logstash_integration_test.go | 4 +- libbeat/template/load_integration_test.go | 2 +- libbeat/tests/system/beat/beat.py | 8 +- libbeat/tests/system/config/libbeat.yml.j2 | 4 +- metricbeat/docker-compose.yml | 2 + testing/environments/docker/README.md | 6 + .../docker/elasticsearch/roles.yml | 31 ++++++ .../environments/docker/elasticsearch/users | 8 ++ .../docker/elasticsearch/users_roles | 11 ++ testing/environments/snapshot.yml | 12 +- 18 files changed, 200 insertions(+), 52 deletions(-) create mode 100644 testing/environments/docker/README.md create mode 100644 testing/environments/docker/elasticsearch/roles.yml create mode 100644 testing/environments/docker/elasticsearch/users create mode 100644 testing/environments/docker/elasticsearch/users_roles diff --git a/auditbeat/docker-compose.yml b/auditbeat/docker-compose.yml index 6a0e252106b..fc3e701f85d 100644 --- a/auditbeat/docker-compose.yml +++ b/auditbeat/docker-compose.yml @@ -8,7 +8,7 @@ services: environment: - ES_HOST=elasticsearch - ES_PORT=9200 - - ES_USER=beats + - ES_USER=auditbeat_user - ES_PASS=testing - KIBANA_HOST=kibana - KIBANA_PORT=5601 diff --git a/auditbeat/tests/system/test_base.py b/auditbeat/tests/system/test_base.py index 2f7f645c750..cd76b3daee8 100644 --- a/auditbeat/tests/system/test_base.py +++ b/auditbeat/tests/system/test_base.py @@ -40,8 +40,10 @@ def test_index_management(self): Test that the template can be loaded with `setup --index-management` """ dirs = [self.temp_dir("auditbeat_test")] + es_user = os.getenv('ES_USER') + es_pass = os.getenv('ES_PASS') with PathCleanup(dirs): - es = Elasticsearch([self.get_elasticsearch_url()]) + es = self.get_elasticsearch_instance(url=self.get_elasticsearch_url(), user=es_user) self.render_config_template( modules=[{ @@ -50,7 +52,7 @@ def test_index_management(self): "paths": dirs, } }], - elasticsearch={"host": self.get_elasticsearch_url()}) + elasticsearch={"host": self.get_elasticsearch_url(), "user": es_user, "pass": es_pass}) self.run_beat(extra_args=["setup", "--index-management"], exit_code=0) assert self.log_contains('Loaded index template') @@ -67,7 +69,9 @@ def test_dashboards(self): kibana_dir = os.path.join(self.beat_path, "build", "kibana") shutil.copytree(kibana_dir, os.path.join(self.working_dir, "kibana")) - es = Elasticsearch([self.get_elasticsearch_url()]) + es_user = os.getenv('ES_USER') + es_pass = os.getenv('ES_PASS') + es = self.get_elasticsearch_instance(url=self.get_elasticsearch_url(), user=es_user) self.render_config_template( modules=[{ "name": "file_integrity", @@ -75,7 +79,7 @@ def test_dashboards(self): "paths": dirs, } }], - elasticsearch={"host": self.get_elasticsearch_url()}, + elasticsearch={"host": self.get_elasticsearch_url(), "user": es_user, "pass": es_pass}, kibana={"host": self.get_kibana_url()}, ) self.run_beat(extra_args=["setup", "--dashboards"], exit_code=0) diff --git a/filebeat/tests/system/test_base.py b/filebeat/tests/system/test_base.py index 5f5d8769bae..22ebfb1da6b 100644 --- a/filebeat/tests/system/test_base.py +++ b/filebeat/tests/system/test_base.py @@ -7,6 +7,17 @@ class Test(BaseTest, common_tests.TestExportsMixin, common_tests.TestDashboardMixin): + def setUp(self): + super(Test, self).setUp() + self.render_config_template( + elasticsearch={ + "host": self.get_elasticsearch_url(), + "user": "filebeat_user", + "pass": os.getenv('ES_PASS') + }, + ) + self.es = self.get_elasticsearch_instance(url=self.get_elasticsearch_url(), user='filebeat_user') + def test_base(self): """ Test if the basic fields exist. @@ -40,21 +51,16 @@ def test_index_management(self): assert exit_code == 0 assert self.log_contains('Loaded index template') - assert len(es.cat.templates(name='filebeat-*', h='name')) > 0 + assert len(self.es.cat.templates(name='filebeat-*', h='name')) > 0 @unittest.skipUnless(INTEGRATION_TESTS, "integration test") def test_template_migration(self): """ Test that the template can be loaded with `setup --template` """ - es_url = self.get_elasticsearch_url() - es = self.get_elasticsearch_instance(url=es_url) - self.render_config_template( - elasticsearch={"host": es_url}, - ) exit_code = self.run_beat(extra_args=["setup", "--template", "-E", "setup.template.overwrite=true", "-E", "migration.6_to_7.enabled=true"]) assert exit_code == 0 assert self.log_contains('Loaded index template') - assert len(es.cat.templates(name='filebeat-*', h='name')) > 0 + assert len(self.es.cat.templates(name='filebeat-*', h='name')) > 0 diff --git a/filebeat/tests/system/test_crawler.py b/filebeat/tests/system/test_crawler.py index f3b5d0877a6..694b057927a 100644 --- a/filebeat/tests/system/test_crawler.py +++ b/filebeat/tests/system/test_crawler.py @@ -19,7 +19,12 @@ def test_fetched_lines(self): """ self.render_config_template( - path=os.path.abspath(self.working_dir) + "/log/*" + path=os.path.abspath(self.working_dir) + "/log/*", + elasticsearch={ + "host": self.get_elasticsearch_url(), + "user": "filebeat_user", + "pass": os.getenv('ES_PASS') + }, ) os.mkdir(self.working_dir + "/log/") @@ -55,7 +60,12 @@ def test_unfinished_line_and_continue(self): """ self.render_config_template( - path=os.path.abspath(self.working_dir) + "/log/*" + path=os.path.abspath(self.working_dir) + "/log/*", + elasticsearch={ + "host": self.get_elasticsearch_url(), + "user": "filebeat_user", + "pass": os.getenv('ES_PASS') + }, ) os.mkdir(self.working_dir + "/log/") @@ -114,6 +124,11 @@ def test_partial_line(self): self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/*", + elasticsearch={ + "host": self.get_elasticsearch_url(), + "user": "filebeat_user", + "pass": os.getenv('ES_PASS') + }, ) os.mkdir(self.working_dir + "/log/") @@ -162,7 +177,12 @@ def test_file_renaming(self): """ self.render_config_template( - path=os.path.abspath(self.working_dir) + "/log/*" + path=os.path.abspath(self.working_dir) + "/log/*", + elasticsearch={ + "host": self.get_elasticsearch_url(), + "user": "filebeat_user", + "pass": os.getenv('ES_PASS') + }, ) os.mkdir(self.working_dir + "/log/") @@ -214,6 +234,11 @@ def test_file_disappear(self): self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/*", clean_removed="false", + elasticsearch={ + "host": self.get_elasticsearch_url(), + "user": "filebeat_user", + "pass": os.getenv('ES_PASS') + }, ) os.mkdir(self.working_dir + "/log/") @@ -274,6 +299,11 @@ def test_file_disappear_appear(self): close_removed="true", scan_frequency="0.1s", clean_removed="false", + elasticsearch={ + "host": self.get_elasticsearch_url(), + "user": "filebeat_user", + "pass": os.getenv('ES_PASS') + }, ) os.mkdir(self.working_dir + "/log/") @@ -339,7 +369,12 @@ def test_new_line_on_existing_file(self): """ self.render_config_template( - path=os.path.abspath(self.working_dir) + "/log/*" + path=os.path.abspath(self.working_dir) + "/log/*", + elasticsearch={ + "host": self.get_elasticsearch_url(), + "user": "filebeat_user", + "pass": os.getenv('ES_PASS') + }, ) os.mkdir(self.working_dir + "/log/") @@ -373,7 +408,12 @@ def test_multiple_appends(self): """ self.render_config_template( - path=os.path.abspath(self.working_dir) + "/log/*" + path=os.path.abspath(self.working_dir) + "/log/*", + elasticsearch={ + "host": self.get_elasticsearch_url(), + "user": "filebeat_user", + "pass": os.getenv('ES_PASS') + }, ) os.mkdir(self.working_dir + "/log/") @@ -419,7 +459,12 @@ def test_new_line_on_open_file(self): """ self.render_config_template( - path=os.path.abspath(self.working_dir) + "/log/*" + path=os.path.abspath(self.working_dir) + "/log/*", + elasticsearch={ + "host": self.get_elasticsearch_url(), + "user": "filebeat_user", + "pass": os.getenv('ES_PASS') + }, ) os.mkdir(self.working_dir + "/log/") @@ -457,7 +502,12 @@ def test_tail_files(self): self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/*", - tail_files="true" + tail_files="true", + elasticsearch={ + "host": self.get_elasticsearch_url(), + "user": "filebeat_user", + "pass": os.getenv('ES_PASS') + }, ) os.mkdir(self.working_dir + "/log/") @@ -501,7 +551,12 @@ def test_utf8(self): self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/*", - encoding="utf-8" + encoding="utf-8", + elasticsearch={ + "host": self.get_elasticsearch_url(), + "user": "filebeat_user", + "pass": os.getenv('ES_PASS') + }, ) os.mkdir(self.working_dir + "/log/") @@ -613,7 +668,12 @@ def test_include_lines(self): self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/*", - include_lines=["^ERR", "^WARN"] + include_lines=["^ERR", "^WARN"], + elasticsearch={ + "host": self.get_elasticsearch_url(), + "user": "filebeat_user", + "pass": os.getenv('ES_PASS') + }, ) os.mkdir(self.working_dir + "/log/") @@ -650,7 +710,12 @@ def test_default_include_exclude_lines(self): """ self.render_config_template( - path=os.path.abspath(self.working_dir) + "/log/*" + path=os.path.abspath(self.working_dir) + "/log/*", + elasticsearch={ + "host": self.get_elasticsearch_url(), + "user": "filebeat_user", + "pass": os.getenv('ES_PASS') + }, ) os.mkdir(self.working_dir + "/log/") @@ -688,7 +753,12 @@ def test_exclude_lines(self): self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/*", - exclude_lines=["^DBG"] + exclude_lines=["^DBG"], + elasticsearch={ + "host": self.get_elasticsearch_url(), + "user": "filebeat_user", + "pass": os.getenv('ES_PASS') + }, ) os.mkdir(self.working_dir + "/log/") @@ -727,7 +797,12 @@ def test_include_exclude_lines(self): self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/*", exclude_lines=["^DBG"], - include_lines=["apache"] + include_lines=["apache"], + elasticsearch={ + "host": self.get_elasticsearch_url(), + "user": "filebeat_user", + "pass": os.getenv('ES_PASS') + }, ) os.mkdir(self.working_dir + "/log/") @@ -768,6 +843,11 @@ def test_file_no_permission(self): self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/*", + elasticsearch={ + "host": self.get_elasticsearch_url(), + "user": "filebeat_user", + "pass": os.getenv('ES_PASS') + }, ) os.mkdir(self.working_dir + "/log/") diff --git a/heartbeat/docker-compose.yml b/heartbeat/docker-compose.yml index c7da39a8798..e3374592b4e 100644 --- a/heartbeat/docker-compose.yml +++ b/heartbeat/docker-compose.yml @@ -8,6 +8,8 @@ services: - REDIS_HOST=redis - REDIS_PORT=6379 - ES_HOST=elasticsearch + - ES_USER=heartbeat_user + - ES_PASS=testing - ES_PORT=9200 working_dir: /go/src/github.com/elastic/beats/heartbeat volumes: diff --git a/heartbeat/tests/system/config/heartbeat.yml.j2 b/heartbeat/tests/system/config/heartbeat.yml.j2 index e282dd5dae0..6736ab8452e 100644 --- a/heartbeat/tests/system/config/heartbeat.yml.j2 +++ b/heartbeat/tests/system/config/heartbeat.yml.j2 @@ -105,8 +105,8 @@ queue.mem: {%- if elasticsearch %} output.elasticsearch: hosts: ["{{ elasticsearch.host }}"] - username: elastic - password: changeme + username: {{ elasticsearch.user }} + password: {{ elasticsearch.pass }} {%- else %} output.file: path: '{{ output_file_path|default(beat.working_dir + "/output") }}' diff --git a/heartbeat/tests/system/test_base.py b/heartbeat/tests/system/test_base.py index dfd47aba9d7..ecfd0b2107f 100644 --- a/heartbeat/tests/system/test_base.py +++ b/heartbeat/tests/system/test_base.py @@ -198,13 +198,19 @@ def test_index_management(self): Test that the template can be loaded with `setup --index-management` """ es_url = self.get_elasticsearch_url() - es = self.get_elasticsearch_instance(url=es_url) + es_user = os.getenv('ES_USER') + es_pass = os.getenv('ES_PASS') + es = self.get_elasticsearch_instance(url=es_url, user=es_user) self.render_config_template( monitors=[{ "type": "http", "urls": ["http://localhost:9200"], }], - elasticsearch={"host": es_url}, + elasticsearch={ + "host": es_url, + "user": es_user, + "pass": es_pass, + } ) exit_code = self.run_beat(extra_args=["setup", "--index-management"]) diff --git a/libbeat/esleg/eslegtest/util.go b/libbeat/esleg/eslegtest/util.go index bac04827204..28f33fde2dc 100644 --- a/libbeat/esleg/eslegtest/util.go +++ b/libbeat/esleg/eslegtest/util.go @@ -84,16 +84,6 @@ func GetPass() string { return getEnv("ES_PASS", "") } -// GetXPackUser returns the admin user for Elasticsearch -func GetXPackUser() string { - return getEnv("ES_XPACK_USER", "elastic") -} - -// GetXpackPass returns the admin pass for Elasticsearch -func GetXPackPass() string { - return getEnv("ES_XPACK_PASS", "changeme") -} - func getEnv(name, def string) string { if v := os.Getenv(name); len(v) > 0 { return v diff --git a/libbeat/outputs/logstash/logstash_integration_test.go b/libbeat/outputs/logstash/logstash_integration_test.go index e76f4703e67..d5123a91872 100644 --- a/libbeat/outputs/logstash/logstash_integration_test.go +++ b/libbeat/outputs/logstash/logstash_integration_test.go @@ -102,8 +102,8 @@ func esConnect(t *testing.T, index string) *esConnection { Timestamp: ts, }) - username := os.Getenv("ES_USER") - password := os.Getenv("ES_PASS") + username := os.Getenv("ES_XPACK_USER", "elastic") + password := os.Getenv("ES_XPACK_PASS", "changeme") transport := httpcommon.DefaultHTTPTransportSettings() transport.Timeout = 60 * time.Second client, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ diff --git a/libbeat/template/load_integration_test.go b/libbeat/template/load_integration_test.go index aeb11fc4ba0..bfc0c18b37a 100644 --- a/libbeat/template/load_integration_test.go +++ b/libbeat/template/load_integration_test.go @@ -408,7 +408,7 @@ func getTestingElasticsearch(t eslegtest.TestLogger) *eslegclient.Connection { conn, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ URL: eslegtest.GetURL(), Transport: httpcommon.DefaultHTTPTransportSettings(), - Username: eslegtest.GetUser(), + Username: eslegtest.GetXPackUser(), Password: eslegtest.GetPass(), }) if err != nil { diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index ea93119f2a5..345a33e9f72 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -687,7 +687,7 @@ def get_elasticsearch_url_ssl(self): port=os.getenv("ES_PORT_SSL", "9205"), ) - def get_elasticsearch_instance(self, xpack_security=True, ssl=False, url=None): + def get_elasticsearch_instance(self, security=True, ssl=False, url=None, user=None): """ Returns an elasticsearch.Elasticsearch instance built from the env variables like the integration tests. @@ -698,9 +698,9 @@ def get_elasticsearch_instance(self, xpack_security=True, ssl=False, url=None): else: url = self.get_elasticsearch_url() - if xpack_security: - username = os.getenv("ES_XPACK_USER", "elastic") - password = os.getenv("ES_XPACK_PASS", "changeme") + if security: + username = os.getenv("ES_USER", user) + password = os.getenv("ES_PASS") es_instance = Elasticsearch([url], http_auth=(username, password)) else: es_instance = Elasticsearch([url]) diff --git a/libbeat/tests/system/config/libbeat.yml.j2 b/libbeat/tests/system/config/libbeat.yml.j2 index 62f92f49e74..9b0baff0475 100644 --- a/libbeat/tests/system/config/libbeat.yml.j2 +++ b/libbeat/tests/system/config/libbeat.yml.j2 @@ -74,8 +74,8 @@ setup.kibana.host: "{{ kibana.host }}" output: elasticsearch: hosts: ["{{ elasticsearch.host }}"] - username: "elastic" - password: "changeme" + username: {{ elasticsearch.user }} + password: {{ elasticsearch.pass }} {% if elasticsearch.pipeline %} pipeline: {{elasticsearch.pipeline}} {% endif %} diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml index 299bbb4f4b1..96fbc149cfa 100644 --- a/metricbeat/docker-compose.yml +++ b/metricbeat/docker-compose.yml @@ -4,6 +4,8 @@ services: beat: build: ${PWD}/. environment: + - ES_USER=metricbeat_user + - ES_PASS=testing - BEAT_STRICT_PERMS=false - TEST_ENVIRONMENT=false working_dir: /go/src/github.com/elastic/beats/metricbeat diff --git a/testing/environments/docker/README.md b/testing/environments/docker/README.md new file mode 100644 index 00000000000..2ecb6c1688a --- /dev/null +++ b/testing/environments/docker/README.md @@ -0,0 +1,6 @@ +# XPack security + +This directory contains default usernames and passwords with roles configured +according to the Beats documentation. + +The default password for all accounts is `changeme`. \ No newline at end of file diff --git a/testing/environments/docker/elasticsearch/roles.yml b/testing/environments/docker/elasticsearch/roles.yml new file mode 100644 index 00000000000..ea8e5ffea7e --- /dev/null +++ b/testing/environments/docker/elasticsearch/roles.yml @@ -0,0 +1,31 @@ +--- +beats: + cluster: ['manage_index_templates', 'monitor', 'manage_ingest_pipelines', 'manage_ilm', 'manage_security', 'manage_api_key'] + indices: + - names: ['filebeat-*', 'shrink-filebeat-*'] + privileges: ['all'] +filebeat: + cluster: ['manage_index_templates', 'monitor', 'manage_ingest_pipelines', 'manage_ilm'] + indices: + - names: ['filebeat-*', 'shrink-filebeat-*'] + privileges: ['all'] +heartbeat: + cluster: ['manage_index_templates', 'monitor', 'manage_ingest_pipelines', 'manage_ilm'] + indices: + - names: ['heartbeat-*', 'shrink-heartbeat-*'] + privileges: ['all'] +auditbeat: + cluster: ['manage_index_templates', 'monitor', 'manage_ingest_pipelines', 'manage_ilm'] + indices: + - names: ['auditbeat-*', 'shrink-auditbeat-*'] + privileges: ['all'] +journalbeat: + cluster: ['manage_index_templates', 'monitor', 'manage_ingest_pipelines', 'manage_ilm'] + indices: + - names: ['journalbeat-*', 'shrink-journalbeat-*'] + privileges: ['all'] +metricbeat: + cluster: ['manage_index_templates', 'monitor', 'manage_ingest_pipelines', 'manage_ilm'] + indices: + - names: ['metricbeat-*', 'shrink-metricbeat-*'] + privileges: ['all'] \ No newline at end of file diff --git a/testing/environments/docker/elasticsearch/users b/testing/environments/docker/elasticsearch/users new file mode 100644 index 00000000000..b912ebffd77 --- /dev/null +++ b/testing/environments/docker/elasticsearch/users @@ -0,0 +1,8 @@ +admin:$2a$10$3y5UdMFkcUWtBfDhAJtYieGwZobnb6GNxCBlTt4ymMkEgImZk.vl2 +beats:$2a$10$3y5UdMFkcUWtBfDhAJtYieGwZobnb6GNxCBlTt4ymMkEgImZk.vl2 +filebeat_user:$2a$10$3y5UdMFkcUWtBfDhAJtYieGwZobnb6GNxCBlTt4ymMkEgImZk.vl2 +heartbeat_user:$2a$10$3y5UdMFkcUWtBfDhAJtYieGwZobnb6GNxCBlTt4ymMkEgImZk.vl2 +kibana_system_user:$2a$10$3y5UdMFkcUWtBfDhAJtYieGwZobnb6GNxCBlTt4ymMkEgImZk.vl2 +metricbeat_user:$2a$10$3y5UdMFkcUWtBfDhAJtYieGwZobnb6GNxCBlTt4ymMkEgImZk.vl2 +auditbeat_user:$2a$10$3y5UdMFkcUWtBfDhAJtYieGwZobnb6GNxCBlTt4ymMkEgImZk.vl2 +journalbeat_user:$2a$10$3y5UdMFkcUWtBfDhAJtYieGwZobnb6GNxCBlTt4ymMkEgImZk.vl2 diff --git a/testing/environments/docker/elasticsearch/users_roles b/testing/environments/docker/elasticsearch/users_roles new file mode 100644 index 00000000000..f5e7c2d41c7 --- /dev/null +++ b/testing/environments/docker/elasticsearch/users_roles @@ -0,0 +1,11 @@ +beats:beats +beats_system:beats,filebeat_user,heartbeat_user,metricbeat_user,auditbeat_user,journalbeat_user +filebeat:filebeat_user +heartbeat:heartbeat_user +ingest_admin:apm_server_user +kibana_system:kibana_system_user +kibana_user:apm_server_user,apm_user_ro,beats,filebeat_user,heartbeat_user,metricbeat_user,auditbeat_user,journalbeat_user +metricbeat:metricbeat_user +auditbeat:auditbeat_user +journalbeat:journalbeat_user +superuser:admin diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 98f981fef30..2b465874912 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -5,12 +5,11 @@ services: elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:8.0.0-2f347a19-SNAPSHOT healthcheck: - test: ["CMD-SHELL", "curl -u elastic:changeme -s http://localhost:9200/_cat/health?h=status | grep -q green"] + test: ["CMD-SHELL", "curl -u admin:testing -s http://localhost:9200/_cat/health?h=status | grep -q green"] retries: 300 interval: 1s environment: - "ES_JAVA_OPTS=-Xms1g -Xmx1g" - - "ELASTIC_PASSWORD=changeme" - "network.host=" - "transport.host=127.0.0.1" - "http.host=0.0.0.0" @@ -28,6 +27,9 @@ services: - "./GeoLite2-ASN.mmdb:/usr/share/elasticsearch/config/ingest-geoip/GeoLite2-ASN.mmdb:ro" - "./GeoLite2-City.mmdb:/usr/share/elasticsearch/config/ingest-geoip/GeoLite2-City.mmdb:ro" - "./GeoLite2-Country.mmdb:/usr/share/elasticsearch/config/ingest-geoip/GeoLite2-Country.mmdb:ro" + - "./docker/elasticsearch/roles.yml:/usr/share/elasticsearch/config/roles.yml" + - "./docker/elasticsearch/users:/usr/share/elasticsearch/config/users" + - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles" logstash: image: docker.elastic.co/logstash/logstash@sha256:e01cf165142edf8d67485115b938c94deeda66153e9516aa2ce69ee417c5fc33 @@ -44,10 +46,10 @@ services: healthcheck: test: ["CMD-SHELL", "curl -s http://localhost:5601/api/status?v8format=true | grep -q '\"overall\":{\"level\":\"available\"'"] retries: 600 - interval: 1s + interval: 10s environment: - - "ELASTICSEARCH_USERNAME=elastic" - - "ELASTICSEARCH_PASSWORD=changeme" + - "ELASTICSEARCH_USERNAME=kibana_system_user" + - "ELASTICSEARCH_PASSWORD=testing" - "XPACK_SECURITY_ENCRYPTIONKEY=wZSVeczkXAmebqNgfcKEzNMmQCBZKkSH" - "XPACK_XPACK_MAIN_TELEMETRY_ENABLED=false" - "XPACK_REPORTING_ENCRYPTIONKEY=xCyqJUFqrUJJKxjZVGfnhrRkyqqaKeAG" From aed46fa626bf4f913643ab31b8fe4b876567d8dd Mon Sep 17 00:00:00 2001 From: Mike Place Date: Thu, 21 Oct 2021 14:16:38 +0200 Subject: [PATCH 13/52] Fixups in filebeat testing --- dev-tools/mage/pytest.go | 3 + filebeat/tests/system/test_crawler.py | 81 --------------------------- libbeat/tests/system/beat/beat.py | 2 +- 3 files changed, 4 insertions(+), 82 deletions(-) diff --git a/dev-tools/mage/pytest.go b/dev-tools/mage/pytest.go index f97dede19f8..09e77b9a01c 100644 --- a/dev-tools/mage/pytest.go +++ b/dev-tools/mage/pytest.go @@ -122,6 +122,7 @@ func PythonTest(params PythonTestArgs) error { pytestEnv := map[string]string{ // activate sets this. Not sure if it's ever needed. "VIRTUAL_ENV": ve, + "ES_PASS": "changeme", } if IsInIntegTestEnv() { pytestEnv["INTEGRATION_TESTS"] = "1" @@ -134,6 +135,8 @@ func PythonTest(params PythonTestArgs) error { "--timeout=90", "--durations=20", "-x", + "-v", + "-s", } if mg.Verbose() { pytestOptions = append(pytestOptions, "-v") diff --git a/filebeat/tests/system/test_crawler.py b/filebeat/tests/system/test_crawler.py index 694b057927a..51f7a979590 100644 --- a/filebeat/tests/system/test_crawler.py +++ b/filebeat/tests/system/test_crawler.py @@ -20,11 +20,6 @@ def test_fetched_lines(self): self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/*", - elasticsearch={ - "host": self.get_elasticsearch_url(), - "user": "filebeat_user", - "pass": os.getenv('ES_PASS') - }, ) os.mkdir(self.working_dir + "/log/") @@ -61,11 +56,6 @@ def test_unfinished_line_and_continue(self): self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/*", - elasticsearch={ - "host": self.get_elasticsearch_url(), - "user": "filebeat_user", - "pass": os.getenv('ES_PASS') - }, ) os.mkdir(self.working_dir + "/log/") @@ -124,11 +114,6 @@ def test_partial_line(self): self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/*", - elasticsearch={ - "host": self.get_elasticsearch_url(), - "user": "filebeat_user", - "pass": os.getenv('ES_PASS') - }, ) os.mkdir(self.working_dir + "/log/") @@ -178,11 +163,6 @@ def test_file_renaming(self): self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/*", - elasticsearch={ - "host": self.get_elasticsearch_url(), - "user": "filebeat_user", - "pass": os.getenv('ES_PASS') - }, ) os.mkdir(self.working_dir + "/log/") @@ -234,11 +214,6 @@ def test_file_disappear(self): self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/*", clean_removed="false", - elasticsearch={ - "host": self.get_elasticsearch_url(), - "user": "filebeat_user", - "pass": os.getenv('ES_PASS') - }, ) os.mkdir(self.working_dir + "/log/") @@ -299,11 +274,6 @@ def test_file_disappear_appear(self): close_removed="true", scan_frequency="0.1s", clean_removed="false", - elasticsearch={ - "host": self.get_elasticsearch_url(), - "user": "filebeat_user", - "pass": os.getenv('ES_PASS') - }, ) os.mkdir(self.working_dir + "/log/") @@ -370,11 +340,6 @@ def test_new_line_on_existing_file(self): self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/*", - elasticsearch={ - "host": self.get_elasticsearch_url(), - "user": "filebeat_user", - "pass": os.getenv('ES_PASS') - }, ) os.mkdir(self.working_dir + "/log/") @@ -409,11 +374,6 @@ def test_multiple_appends(self): self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/*", - elasticsearch={ - "host": self.get_elasticsearch_url(), - "user": "filebeat_user", - "pass": os.getenv('ES_PASS') - }, ) os.mkdir(self.working_dir + "/log/") @@ -460,11 +420,6 @@ def test_new_line_on_open_file(self): self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/*", - elasticsearch={ - "host": self.get_elasticsearch_url(), - "user": "filebeat_user", - "pass": os.getenv('ES_PASS') - }, ) os.mkdir(self.working_dir + "/log/") @@ -503,11 +458,6 @@ def test_tail_files(self): self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/*", tail_files="true", - elasticsearch={ - "host": self.get_elasticsearch_url(), - "user": "filebeat_user", - "pass": os.getenv('ES_PASS') - }, ) os.mkdir(self.working_dir + "/log/") @@ -552,11 +502,6 @@ def test_utf8(self): self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/*", encoding="utf-8", - elasticsearch={ - "host": self.get_elasticsearch_url(), - "user": "filebeat_user", - "pass": os.getenv('ES_PASS') - }, ) os.mkdir(self.working_dir + "/log/") @@ -669,11 +614,6 @@ def test_include_lines(self): self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/*", include_lines=["^ERR", "^WARN"], - elasticsearch={ - "host": self.get_elasticsearch_url(), - "user": "filebeat_user", - "pass": os.getenv('ES_PASS') - }, ) os.mkdir(self.working_dir + "/log/") @@ -708,14 +648,8 @@ def test_default_include_exclude_lines(self): """ Checks if all the log lines are exported by default """ - self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/*", - elasticsearch={ - "host": self.get_elasticsearch_url(), - "user": "filebeat_user", - "pass": os.getenv('ES_PASS') - }, ) os.mkdir(self.working_dir + "/log/") @@ -754,11 +688,6 @@ def test_exclude_lines(self): self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/*", exclude_lines=["^DBG"], - elasticsearch={ - "host": self.get_elasticsearch_url(), - "user": "filebeat_user", - "pass": os.getenv('ES_PASS') - }, ) os.mkdir(self.working_dir + "/log/") @@ -798,11 +727,6 @@ def test_include_exclude_lines(self): path=os.path.abspath(self.working_dir) + "/log/*", exclude_lines=["^DBG"], include_lines=["apache"], - elasticsearch={ - "host": self.get_elasticsearch_url(), - "user": "filebeat_user", - "pass": os.getenv('ES_PASS') - }, ) os.mkdir(self.working_dir + "/log/") @@ -843,11 +767,6 @@ def test_file_no_permission(self): self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/*", - elasticsearch={ - "host": self.get_elasticsearch_url(), - "user": "filebeat_user", - "pass": os.getenv('ES_PASS') - }, ) os.mkdir(self.working_dir + "/log/") diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index 345a33e9f72..f9ed3fa9d74 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -700,7 +700,7 @@ def get_elasticsearch_instance(self, security=True, ssl=False, url=None, user=No if security: username = os.getenv("ES_USER", user) - password = os.getenv("ES_PASS") + password = os.getenv("ES_PASS", "") es_instance = Elasticsearch([url], http_auth=(username, password)) else: es_instance = Elasticsearch([url]) From 396c8645eb61c3d21d1c146f56fa5e4c2d00106b Mon Sep 17 00:00:00 2001 From: Mike Place Date: Thu, 21 Oct 2021 17:28:51 +0200 Subject: [PATCH 14/52] Fix go integration tests for filebeat --- dev-tools/mage/integtest_docker.go | 2 ++ filebeat/fileset/modules_integration_test.go | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dev-tools/mage/integtest_docker.go b/dev-tools/mage/integtest_docker.go index e9767f68436..4bdcb5949bf 100644 --- a/dev-tools/mage/integtest_docker.go +++ b/dev-tools/mage/integtest_docker.go @@ -112,6 +112,8 @@ func (d *DockerIntegrationTester) Test(_ string, mageTarget string, env map[stri // Use the host machine's pkg cache to minimize external downloads. "-v", goPkgCache + ":" + dockerGoPkgCache + ":ro", "-e", "GOPROXY=file://" + dockerGoPkgCache + ",direct", + "-e", "ES_USER=admin", + "-e", "ES_PASS=testing", } args, err = addUidGidEnvArgs(args) if err != nil { diff --git a/filebeat/fileset/modules_integration_test.go b/filebeat/fileset/modules_integration_test.go index aec3c740593..005385de5bf 100644 --- a/filebeat/fileset/modules_integration_test.go +++ b/filebeat/fileset/modules_integration_test.go @@ -259,8 +259,8 @@ func TestLoadMultiplePipelinesWithRollback(t *testing.T) { func getTestingElasticsearch(t eslegtest.TestLogger) *eslegclient.Connection { conn, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ URL: eslegtest.GetURL(), - Username: eslegtest.GetXPackUser(), - Password: eslegtest.GetXPackPass(), + Username: eslegtest.GetUser(), + Password: eslegtest.GetPass(), }) if err != nil { t.Fatal(err) From e36195e62f7c32e49cf89383175492c8d04ec3e6 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 25 Oct 2021 10:01:32 +0200 Subject: [PATCH 15/52] Additional libbeat test fixes --- libbeat/docker-compose.yml | 7 +++---- libbeat/licenser/elastic_fetcher_integration_test.go | 2 +- libbeat/outputs/logstash/logstash_integration_test.go | 6 ++---- libbeat/template/load_integration_test.go | 2 +- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/libbeat/docker-compose.yml b/libbeat/docker-compose.yml index 999e1b637a6..2d7007f5e1e 100644 --- a/libbeat/docker-compose.yml +++ b/libbeat/docker-compose.yml @@ -30,7 +30,7 @@ services: # - ES_KERBEROS_HOST=elasticsearch_kerberos.elastic - ES_PORT_SSL=9200 - ES_SUPERUSER_USER=admin - - ES_SUPERUSER_PASS=changeme + - ES_SUPERUSER_PASS=testing volumes: - ${PWD}/..:/go/src/github.com/elastic/beats/ # Used for docker integration tests: @@ -67,13 +67,13 @@ services: file: ${ES_BEATS}/testing/environments/${TESTING_ENVIRONMENT}.yml service: elasticsearch healthcheck: - test: ["CMD-SHELL", "curl -u elastic:changeme -s http://localhost:9200/_cat/health?h=status | grep -q green"] + test: ["CMD-SHELL", "curl -u admin:testing -s http://localhost:9200/_cat/health?h=status | grep -q green"] elasticsearchssl: extends: file: ${ES_BEATS}/testing/environments/${TESTING_ENVIRONMENT}.yml service: elasticsearch healthcheck: - test: ["CMD", "curl", "-u", "admin:changeme", "-f", "https://localhost:9200", "--insecure"] + test: ["CMD", "curl", "-u", "admin:testing", "-f", "https://localhost:9200", "--insecure"] retries: 1200 interval: 5s start_period: 60s @@ -96,7 +96,6 @@ services: - ${ES_BEATS}/testing/environments/docker/elasticsearch/pki:/usr/share/elasticsearch/config/pki:ro expose: - 9200 - command: bash -c "bin/elasticsearch-users useradd admin -r superuser -p changeme | /usr/local/bin/docker-entrypoint.sh eswrapper" # This host name is static because of the certificate. logstash: diff --git a/libbeat/licenser/elastic_fetcher_integration_test.go b/libbeat/licenser/elastic_fetcher_integration_test.go index c12f6651bfe..0040c0d49f1 100644 --- a/libbeat/licenser/elastic_fetcher_integration_test.go +++ b/libbeat/licenser/elastic_fetcher_integration_test.go @@ -43,7 +43,7 @@ func getTestClient() *eslegclient.Connection { host := "http://" + cli.GetEnvOr("ES_HOST", elasticsearchHost) + ":" + cli.GetEnvOr("ES_POST", elasticsearchPort) client, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ URL: host, - Username: "myelastic", // NOTE: I will refactor this in a followup PR + Username: "elastic", Password: "changeme", CompressionLevel: 3, Transport: transport, diff --git a/libbeat/outputs/logstash/logstash_integration_test.go b/libbeat/outputs/logstash/logstash_integration_test.go index d5123a91872..28c2f935958 100644 --- a/libbeat/outputs/logstash/logstash_integration_test.go +++ b/libbeat/outputs/logstash/logstash_integration_test.go @@ -102,14 +102,12 @@ func esConnect(t *testing.T, index string) *esConnection { Timestamp: ts, }) - username := os.Getenv("ES_XPACK_USER", "elastic") - password := os.Getenv("ES_XPACK_PASS", "changeme") transport := httpcommon.DefaultHTTPTransportSettings() transport.Timeout = 60 * time.Second client, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ URL: host, - Username: username, - Password: password, + Username: "elastic", + Password: "changeme", Transport: transport, }) if err != nil { diff --git a/libbeat/template/load_integration_test.go b/libbeat/template/load_integration_test.go index bfc0c18b37a..aeb11fc4ba0 100644 --- a/libbeat/template/load_integration_test.go +++ b/libbeat/template/load_integration_test.go @@ -408,7 +408,7 @@ func getTestingElasticsearch(t eslegtest.TestLogger) *eslegclient.Connection { conn, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ URL: eslegtest.GetURL(), Transport: httpcommon.DefaultHTTPTransportSettings(), - Username: eslegtest.GetXPackUser(), + Username: eslegtest.GetUser(), Password: eslegtest.GetPass(), }) if err != nil { From fe1f840d08562e1bb3b7210b159dc4eb5670a554 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 25 Oct 2021 14:29:39 +0200 Subject: [PATCH 16/52] Fixing heartbeat tests --- dev-tools/mage/integtest_docker.go | 6 ++++-- dev-tools/mage/pytest.go | 2 +- heartbeat/tests/system/test_base.py | 8 +++++++- x-pack/filebeat/docker-compose.yml | 2 ++ 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/dev-tools/mage/integtest_docker.go b/dev-tools/mage/integtest_docker.go index 4bdcb5949bf..182796fe446 100644 --- a/dev-tools/mage/integtest_docker.go +++ b/dev-tools/mage/integtest_docker.go @@ -112,8 +112,10 @@ func (d *DockerIntegrationTester) Test(_ string, mageTarget string, env map[stri // Use the host machine's pkg cache to minimize external downloads. "-v", goPkgCache + ":" + dockerGoPkgCache + ":ro", "-e", "GOPROXY=file://" + dockerGoPkgCache + ",direct", - "-e", "ES_USER=admin", - "-e", "ES_PASS=testing", + // Do not set ES_USER or ES_PATH in this file unless you intend to override + // values set in all individual docker-compose files + // "-e", "ES_USER=admin", + // "-e", "ES_PASS=testing", } args, err = addUidGidEnvArgs(args) if err != nil { diff --git a/dev-tools/mage/pytest.go b/dev-tools/mage/pytest.go index 09e77b9a01c..2cf5e0539b7 100644 --- a/dev-tools/mage/pytest.go +++ b/dev-tools/mage/pytest.go @@ -122,7 +122,7 @@ func PythonTest(params PythonTestArgs) error { pytestEnv := map[string]string{ // activate sets this. Not sure if it's ever needed. "VIRTUAL_ENV": ve, - "ES_PASS": "changeme", + // "ES_PASS": "changeme", } if IsInIntegTestEnv() { pytestEnv["INTEGRATION_TESTS"] = "1" diff --git a/heartbeat/tests/system/test_base.py b/heartbeat/tests/system/test_base.py index ecfd0b2107f..8017d1db927 100644 --- a/heartbeat/tests/system/test_base.py +++ b/heartbeat/tests/system/test_base.py @@ -200,7 +200,13 @@ def test_index_management(self): es_url = self.get_elasticsearch_url() es_user = os.getenv('ES_USER') es_pass = os.getenv('ES_PASS') - es = self.get_elasticsearch_instance(url=es_url, user=es_user) + #es_user = 'heartbeat_user' + #es_pass = 'testing' + print('ES USER: {}'.format(es_user)) + print('ES PASS: {}'.format(es_pass)) + print('ES OS USER: {}'.format(os.getenv('ES_USER'))) + print('ES OS PASS: {}'.format(os.getenv('ES_PASS'))) + es = self.get_elasticsearch_instance(url=es_url, user='beats') self.render_config_template( monitors=[{ "type": "http", diff --git a/x-pack/filebeat/docker-compose.yml b/x-pack/filebeat/docker-compose.yml index 0c0b477a611..3224c88bc18 100644 --- a/x-pack/filebeat/docker-compose.yml +++ b/x-pack/filebeat/docker-compose.yml @@ -9,6 +9,8 @@ services: - BEAT_STRICT_PERMS=false - ES_HOST=elasticsearch - ES_PORT=9200 + - ES_USER=beats + - ES_PASS=testing working_dir: /go/src/github.com/elastic/beats/x-pack/filebeat volumes: - ${PWD}/../..:/go/src/github.com/elastic/beats/ From 300bd48d2251dbdc3df06182a18992d1bb427045 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 25 Oct 2021 15:31:30 +0200 Subject: [PATCH 17/52] Fix libbeat tests for xpack auth --- libbeat/tests/system/test_cmd_test.py | 1 + libbeat/tests/system/test_cmd_version.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/libbeat/tests/system/test_cmd_test.py b/libbeat/tests/system/test_cmd_test.py index 38f15ef095f..d3d75f07be6 100644 --- a/libbeat/tests/system/test_cmd_test.py +++ b/libbeat/tests/system/test_cmd_test.py @@ -62,6 +62,7 @@ def test_output(self): assert self.log_contains('TLS... WARN secure connection disabled') assert self.log_contains('talk to server... OK') + @unittest.skipIf(not INTEGRATION_TESTS, "integration test") def test_wrong_output(self): """ Test test wrong output works diff --git a/libbeat/tests/system/test_cmd_version.py b/libbeat/tests/system/test_cmd_version.py index fb98f53c577..ace84b99062 100644 --- a/libbeat/tests/system/test_cmd_version.py +++ b/libbeat/tests/system/test_cmd_version.py @@ -17,7 +17,7 @@ def setUp(self): self.elasticsearch_url = self.get_elasticsearch_url() print("Using elasticsearch: {}".format(self.elasticsearch_url)) - self.es = self.get_elasticsearch_instance(url=self.elasticsearch_url) + self.es = self.get_elasticsearch_instance(url=self.elasticsearch_url, user='beats') logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("elasticsearch").setLevel(logging.ERROR) From 06403f6894b0a9eb111d95426b78eed7bcc13515 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Tue, 26 Oct 2021 12:30:42 +0200 Subject: [PATCH 18/52] Additional test fixups --- libbeat/docker-compose.yml | 2 +- .../licenser/elastic_fetcher_integration_test.go | 4 ++-- .../logstash/logstash_integration_test.go | 16 +++++++++------- libbeat/tests/system/test_ca_pinning.py | 4 ++-- .../system/test_cmd_setup_index_management.py | 6 +++++- .../docker/logstash/pipeline-xpack/default.conf | 4 ++-- 6 files changed, 21 insertions(+), 15 deletions(-) diff --git a/libbeat/docker-compose.yml b/libbeat/docker-compose.yml index 2d7007f5e1e..ffd7649bb2e 100644 --- a/libbeat/docker-compose.yml +++ b/libbeat/docker-compose.yml @@ -21,7 +21,7 @@ services: - KIBANA_PASS=testing - ES_HOST=elasticsearch - ES_PORT=9200 - - ES_USER=beats + - ES_USER=admin - ES_PASS=testing - ES_MONITORING_HOST=elasticsearch_monitoring - ES_MONITORING_PORT=9200 diff --git a/libbeat/licenser/elastic_fetcher_integration_test.go b/libbeat/licenser/elastic_fetcher_integration_test.go index 0040c0d49f1..b1a21d937cd 100644 --- a/libbeat/licenser/elastic_fetcher_integration_test.go +++ b/libbeat/licenser/elastic_fetcher_integration_test.go @@ -43,8 +43,8 @@ func getTestClient() *eslegclient.Connection { host := "http://" + cli.GetEnvOr("ES_HOST", elasticsearchHost) + ":" + cli.GetEnvOr("ES_POST", elasticsearchPort) client, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ URL: host, - Username: "elastic", - Password: "changeme", + Username: "admin", + Password: "testing", CompressionLevel: 3, Transport: transport, }) diff --git a/libbeat/outputs/logstash/logstash_integration_test.go b/libbeat/outputs/logstash/logstash_integration_test.go index 28c2f935958..ef5078eecc5 100644 --- a/libbeat/outputs/logstash/logstash_integration_test.go +++ b/libbeat/outputs/logstash/logstash_integration_test.go @@ -106,8 +106,8 @@ func esConnect(t *testing.T, index string) *esConnection { transport.Timeout = 60 * time.Second client, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ URL: host, - Username: "elastic", - Password: "changeme", + Username: "admin", + Password: "testing", Transport: transport, }) if err != nil { @@ -172,11 +172,13 @@ func newTestElasticsearchOutput(t *testing.T, test string) *testOutputer { bulkSize := 0 config, _ := common.NewConfigFrom(map[string]interface{}{ - "hosts": []string{getElasticsearchHost()}, - "index": connection.index, - "bulk_max_size": &bulkSize, - "username": os.Getenv("ES_USER"), - "password": os.Getenv("ES_PASS"), + "hosts": []string{getElasticsearchHost()}, + "index": connection.index, + "bulk_max_size": &bulkSize, + //"username": os.Getenv("ES_USER"), + //"password": os.Getenv("ES_PASS"), + "username": "admin", + "password": "testing", "template.enabled": false, }) diff --git a/libbeat/tests/system/test_ca_pinning.py b/libbeat/tests/system/test_ca_pinning.py index 4c1480b82a0..55a9e9ea13c 100644 --- a/libbeat/tests/system/test_ca_pinning.py +++ b/libbeat/tests/system/test_ca_pinning.py @@ -35,7 +35,7 @@ def test_sending_events_with_a_good_sha256(self): elasticsearch={ "hosts": self.get_elasticsearch_url_ssl(), "username": "admin", - "password": "changeme", + "password": "testing", "ssl.certificate_authorities": [ca], "ssl.ca_sha256": "8hZS8gpciuzlu+7Xi0sdv8T7RKRRxG1TWKumUQsDam0=", }, @@ -67,7 +67,7 @@ def test_sending_events_with_a_bad_sha256(self): elasticsearch={ "hosts": self.get_elasticsearch_url_ssl(), "username": "admin", - "password": "changeme", + "password": "testing", "ssl.certificate_authorities": [ca], "ssl.ca_sha256": "not-good-sha", }, diff --git a/libbeat/tests/system/test_cmd_setup_index_management.py b/libbeat/tests/system/test_cmd_setup_index_management.py index cd6d19eefbe..47028962b2c 100644 --- a/libbeat/tests/system/test_cmd_setup_index_management.py +++ b/libbeat/tests/system/test_cmd_setup_index_management.py @@ -40,8 +40,12 @@ def tearDown(self): policies=[self.policy_name, self.custom_policy]) def render_config(self, **kwargs): + print('OS ES USER: {}'.format(os.getenv('ES_USER'))) + print('OS ES pass: {}'.format(os.getenv('ES_PASS'))) self.render_config_template( - elasticsearch={"hosts": self.get_elasticsearch_url()}, + # Note that the template is such that we need to pass in 'username' as opposed to 'user' and + # 'password' instead of 'pass'. + elasticsearch={"hosts": self.get_elasticsearch_url(), "username": "admin", "password": "testing"}, es_template_name=self.index_name, **kwargs ) diff --git a/testing/environments/docker/logstash/pipeline-xpack/default.conf b/testing/environments/docker/logstash/pipeline-xpack/default.conf index 3619905a615..01d46fc4c4b 100644 --- a/testing/environments/docker/logstash/pipeline-xpack/default.conf +++ b/testing/environments/docker/logstash/pipeline-xpack/default.conf @@ -17,8 +17,8 @@ output { elasticsearch { hosts => ["${ES_HOST:elasticsearch}:${ES_PORT:9200}"] index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}" - user => elastic - password => changeme + user => admin + password => testing } # Used for easier debugging From 82e6cb2f30e26e3a5dd9983d74b2db473befd3cc Mon Sep 17 00:00:00 2001 From: Mike Place Date: Wed, 27 Oct 2021 11:46:50 +0200 Subject: [PATCH 19/52] Add auth to cmd test --- libbeat/tests/system/test_cmd_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/tests/system/test_cmd_test.py b/libbeat/tests/system/test_cmd_test.py index d3d75f07be6..e4d916841cb 100644 --- a/libbeat/tests/system/test_cmd_test.py +++ b/libbeat/tests/system/test_cmd_test.py @@ -52,7 +52,7 @@ def test_output(self): self.render_config_template("mockbeat", os.path.join(self.working_dir, "mockbeat.yml"), - elasticsearch={"hosts": self.get_elasticsearch_url()}) + elasticsearch={"hosts": self.get_elasticsearch_url(), "username": "admin", "password": "testing"}) exit_code = self.run_beat( extra_args=["test", "output"], config="mockbeat.yml") From 12740153bce09a2529f48d38470915f339c1b962 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Wed, 27 Oct 2021 11:47:37 +0200 Subject: [PATCH 20/52] Add auth to dashboard tests --- libbeat/tests/system/test_dashboard.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libbeat/tests/system/test_dashboard.py b/libbeat/tests/system/test_dashboard.py index 338e832b045..cdff5fe656e 100644 --- a/libbeat/tests/system/test_dashboard.py +++ b/libbeat/tests/system/test_dashboard.py @@ -58,10 +58,13 @@ def test_load_dashboard(self): "-E", "setup.kibana.protocol=http", "-E", "setup.kibana.host=" + self.get_kibana_host(), "-E", "setup.kibana.port=" + self.get_kibana_port(), + "-E", "setup.kibana.username=beats", + "-E", "setup.kibana.password=testing", "-E", "output.elasticsearch.hosts=['" + self.get_host() + "']", + "-E", "output.elasticsearch.username=admin", + "-E", "output.elasticsearch.password=testing", "-E", "output.file.enabled=false"] ) - beat.check_wait(exit_code=0) assert self.log_contains("Kibana dashboards successfully loaded") is True From bbe9c811b2e3e7fc3db11571daba7a962997af32 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Wed, 27 Oct 2021 11:47:51 +0200 Subject: [PATCH 21/52] Add auth to ILM tests --- libbeat/tests/system/test_ilm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libbeat/tests/system/test_ilm.py b/libbeat/tests/system/test_ilm.py index 672f51327f6..a9009a840c0 100644 --- a/libbeat/tests/system/test_ilm.py +++ b/libbeat/tests/system/test_ilm.py @@ -36,7 +36,7 @@ def tearDown(self): def render_config(self, **kwargs): self.render_config_template( - elasticsearch={"hosts": self.get_elasticsearch_url()}, + elasticsearch={"hosts": self.get_elasticsearch_url(), "username": "admin", "password": "testing"}, es_template_name=self.index_name, **kwargs ) @@ -186,7 +186,7 @@ def tearDown(self): def render_config(self, **kwargs): self.render_config_template( - elasticsearch={"hosts": self.get_elasticsearch_url()}, + elasticsearch={"hosts": self.get_elasticsearch_url(), "username": "admin", "password": "testing"}, es_template_name=self.index_name, **kwargs ) From f66e5b9daf593d3ff970aa1684c60eb7c395535d Mon Sep 17 00:00:00 2001 From: Mike Place Date: Wed, 27 Oct 2021 11:48:22 +0200 Subject: [PATCH 22/52] Add auth to template tests --- libbeat/tests/system/test_template.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libbeat/tests/system/test_template.py b/libbeat/tests/system/test_template.py index 6a81a6bd036..a78681766b3 100644 --- a/libbeat/tests/system/test_template.py +++ b/libbeat/tests/system/test_template.py @@ -97,7 +97,7 @@ def test_json_template(self): print(path) self.render_config_template( - elasticsearch={"hosts": self.get_host()}, + elasticsearch={"hosts": self.get_host(), "username": "admin", "password": "testing"}, template_overwrite="true", template_json_enabled="true", template_json_path=path, @@ -136,7 +136,7 @@ def tearDown(self): def render_config(self, **kwargs): self.render_config_template( - elasticsearch={"hosts": self.get_elasticsearch_url()}, + elasticsearch={"hosts": self.get_elasticsearch_url(), "username": "admin", "password": "testing"}, **kwargs ) @@ -197,7 +197,7 @@ def tearDown(self): def render_config(self, **kwargs): self.render_config_template( - elasticsearch={"hosts": self.get_elasticsearch_url()}, + elasticsearch={"hosts": self.get_elasticsearch_url(), "username": "admin", "password": "testing"}, **kwargs ) From 729e7183d2ebd4872248da0cf207ed0ddca12149 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Wed, 10 Nov 2021 10:40:00 +0100 Subject: [PATCH 23/52] Additional test fixes --- dev-tools/cmd/dashboards/export_dashboards.go | 3 ++- libbeat/dashboards/importer.go | 2 ++ libbeat/dashboards/kibana_loader.go | 6 ++++++ libbeat/docker-compose.yml | 16 +++++++------- libbeat/kibana/client.go | 8 ++++++- libbeat/tests/system/test_dashboard.py | 21 ++++++++++++++++--- testing/environments/snapshot.yml | 6 ++++-- 7 files changed, 47 insertions(+), 15 deletions(-) diff --git a/dev-tools/cmd/dashboards/export_dashboards.go b/dev-tools/cmd/dashboards/export_dashboards.go index 364fae9e0f5..d1ab9b084e4 100644 --- a/dev-tools/cmd/dashboards/export_dashboards.go +++ b/dev-tools/cmd/dashboards/export_dashboards.go @@ -66,7 +66,8 @@ func main() { user = u.User.Username() pass, _ = u.User.Password() } - + user = "beats" + pass = "testing" transport := httpcommon.DefaultHTTPTransportSettings() transport.Timeout = kibanaTimeout diff --git a/libbeat/dashboards/importer.go b/libbeat/dashboards/importer.go index b27ec695cad..a6502c566fd 100644 --- a/libbeat/dashboards/importer.go +++ b/libbeat/dashboards/importer.go @@ -125,7 +125,9 @@ func (imp Importer) ImportDir(dirType string, dir string) error { return fmt.Errorf("The directory %s is empty, nothing to import", dir) } for _, file := range files { + imp.loader.statusMsg("start file %s", file) err = imp.ImportFile(dirType, file) + imp.loader.statusMsg("finish file %s", file) if err != nil { errors = append(errors, fmt.Sprintf(" error loading %s: %s", file, err)) } diff --git a/libbeat/dashboards/kibana_loader.go b/libbeat/dashboards/kibana_loader.go index 7d0a2ef737d..ab9c90e2e32 100644 --- a/libbeat/dashboards/kibana_loader.go +++ b/libbeat/dashboards/kibana_loader.go @@ -102,16 +102,20 @@ func (loader KibanaLoader) ImportIndexFile(file string) error { // read json file reader, err := ioutil.ReadFile(file) + loader.statusMsg("Finished file read for %s", file) if err != nil { return fmt.Errorf("fail to read index-pattern from file %s: %v", file, err) } var indexContent common.MapStr err = json.Unmarshal(reader, &indexContent) + + loader.statusMsg("Finished json unmarshal for %s", file) if err != nil { return fmt.Errorf("fail to unmarshal the index content from file %s: %v", file, err) } + loader.statusMsg("Finished tasks for %s", file) return loader.ImportIndex(indexContent) } @@ -138,7 +142,9 @@ func (loader KibanaLoader) ImportIndex(pattern common.MapStr) error { // ImportDashboard imports the dashboard file func (loader KibanaLoader) ImportDashboard(file string) error { + loader.statusMsg("dashboard attempt starting") if loader.version.LessThan(kibana.MinimumRequiredVersionSavedObjects) { + loader.statusMsg("loader version mismatch fail") return fmt.Errorf("Kibana version must be at least " + kibana.MinimumRequiredVersionSavedObjects.String()) } diff --git a/libbeat/docker-compose.yml b/libbeat/docker-compose.yml index ffd7649bb2e..8073cab58ff 100644 --- a/libbeat/docker-compose.yml +++ b/libbeat/docker-compose.yml @@ -49,7 +49,7 @@ services: elasticsearch_monitoring: { condition: service_healthy } elasticsearchssl: { condition: service_healthy } logstash: { condition: service_healthy } - kafka: { condition: service_healthy } +# kafka: { condition: service_healthy } redis: { condition: service_healthy } sredis: { condition: service_healthy } kibana: { condition: service_healthy } @@ -121,13 +121,13 @@ services: - REDIS_HOST=redis - REDIS_PORT=6379 - kafka: - build: ${ES_BEATS}/testing/environments/docker/kafka - expose: - - 9092 - - 2181 - environment: - - ADVERTISED_HOST=kafka +# kafka: +# build: ${ES_BEATS}/testing/environments/docker/kafka +# expose: +# - 9092 +# - 2181 +# environment: +# - ADVERTISED_HOST=kafka # elasticsearch_kerberos.elastic: # build: ${ES_BEATS}/testing/environments/docker/elasticsearch_kerberos diff --git a/libbeat/kibana/client.go b/libbeat/kibana/client.go index 111db2e3816..6ceb3d19886 100644 --- a/libbeat/kibana/client.go +++ b/libbeat/kibana/client.go @@ -285,6 +285,9 @@ func (conn *Connection) SendWithContext(ctx context.Context, method, extraPath s req.Header.Set("Accept", "application/json") req.Header.Set("kbn-xsrf", "1") + log := logp.NewLogger("kibana_conn") + log.Infof("kibana headers: %s", req) + return conn.RoundTrip(req) } @@ -351,6 +354,8 @@ func (client *Client) readVersion() error { func (client *Client) GetVersion() common.Version { return client.Version } func (client *Client) ImportMultiPartFormFile(url string, params url.Values, filename string, contents string) error { + log := logp.NewLogger("kibana_foo") + log.Infof("starting import multiport form") buf := &bytes.Buffer{} w := multipart.NewWriter(buf) @@ -370,8 +375,9 @@ func (client *Client) ImportMultiPartFormFile(url string, params url.Values, fil headers := http.Header{} headers.Add("Content-Type", w.FormDataContentType()) - + log.Infof("starting conn") statusCode, response, err := client.Connection.Request("POST", url, params, headers, buf) + log.Infof("finished conn") if err != nil || statusCode >= 300 { return fmt.Errorf("returned %d to import file: %v. Response: %s", statusCode, err, response) } diff --git a/libbeat/tests/system/test_dashboard.py b/libbeat/tests/system/test_dashboard.py index cdff5fe656e..ebe12c4aa11 100644 --- a/libbeat/tests/system/test_dashboard.py +++ b/libbeat/tests/system/test_dashboard.py @@ -33,7 +33,11 @@ def test_load_without_dashboard(self): "-E", "setup.kibana.protocol=http", "-E", "setup.kibana.host=" + self.get_kibana_host(), "-E", "setup.kibana.port=" + self.get_kibana_port(), + "-E", "setup.kibana.username=beats", + "-E", "setup.kibana.password=testing", "-E", "output.elasticsearch.hosts=['" + self.get_host() + "']", + "-E", "output.elasticsearch.username=admin", + "-E", "output.elasticsearch.password=testing", "-E", "output.file.enabled=false"] ) @@ -94,8 +98,12 @@ def test_load_dashboard_into_space(self, create_space=True): "-E", "setup.kibana.protocol=http", "-E", "setup.kibana.host=" + self.get_kibana_host(), "-E", "setup.kibana.port=" + self.get_kibana_port(), + "-E", "setup.kibana.username=beats", + "-E", "setup.kibana.password=testing", "-E", "setup.kibana.space.id=foo-bar", "-E", "output.elasticsearch.hosts=['" + self.get_host() + "']", + "-E", "output.elasticsearch.username=admin", + "-E", "output.elasticsearch.password=testing", "-E", "output.file.enabled=false"] ) @@ -121,7 +129,11 @@ def test_load_only_index_patterns(self): "-E", "setup.kibana.protocol=http", "-E", "setup.kibana.host=" + self.get_kibana_host(), "-E", "setup.kibana.port=" + self.get_kibana_port(), + "-E", "setup.kibana.username=beats", + "-E", "setup.kibana.password=testing", "-E", "output.elasticsearch.hosts=['" + self.get_host() + "']", + "-E", "output.elasticsearch.username=admin", + "-E", "output.elasticsearch.password=testing", "-E", "output.file.enabled=false"] ) @@ -144,6 +156,8 @@ def test_export_dashboard_cmd_export_dashboard_by_id(self): "-E", "setup.kibana.protocol=http", "-E", "setup.kibana.host=" + self.get_kibana_host(), "-E", "setup.kibana.port=" + self.get_kibana_port(), + "-E", "setup.kibana.username=beats", + "-E", "setup.kibana.password=testing", "-id", "Metricbeat-system-overview", "-folder", "system-overview"] ) @@ -165,6 +179,8 @@ def test_export_dashboard_cmd_export_dashboard_by_id_unknown_id(self): "-E", "setup.kibana.protocol=http", "-E", "setup.kibana.host=" + self.get_kibana_host(), "-E", "setup.kibana.port=" + self.get_kibana_port(), + "-E", "setup.kibana.username=beats", + "-E", "setup.kibana.password=testing", "-id", "No-such-dashboard", "-folder", "system-overview"] ) @@ -190,7 +206,6 @@ def test_dev_tool_export_dashboard_by_id(self): p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) content, err = p.communicate() - assert p.returncode == 0 self._check_if_dashboard_exported(folder_name) @@ -269,7 +284,7 @@ def create_kibana_space(self): "kbn-xsrf": "1" } - r = requests.post(url, json=data, headers=headers) + r = requests.post(url, json=data, headers=headers, auth=("beats", "testing")) if r.status_code != 200 and r.status_code != 409: self.fail('Bad Kibana status code when creating space: {}'.format(r.status_code)) @@ -277,7 +292,7 @@ def get_version(self): url = "http://" + self.get_kibana_host() + ":" + self.get_kibana_port() + \ "/api/status" - r = requests.get(url) + r = requests.get(url, auth=("beats", "testing")) body = r.json() version = body["version"]["number"] diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 2b465874912..6759d99da34 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -44,13 +44,15 @@ services: kibana: image: docker.elastic.co/kibana/kibana:8.0.0-2f347a19-SNAPSHOT healthcheck: - test: ["CMD-SHELL", "curl -s http://localhost:5601/api/status?v8format=true | grep -q '\"overall\":{\"level\":\"available\"'"] + test: ["CMD-SHELL", "curl -u beats:testing -s http://localhost:5601/api/status?v8format=true | grep -q '\"overall\":{\"level\":\"available\"'"] retries: 600 interval: 10s environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" + - "XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=gwaXhuYzE6l3r1wh5ZdSkJvtK6uSw11d" - "XPACK_SECURITY_ENCRYPTIONKEY=wZSVeczkXAmebqNgfcKEzNMmQCBZKkSH" - - "XPACK_XPACK_MAIN_TELEMETRY_ENABLED=false" +# - "XPACK_XPACK_MAIN_TELEMETRY_ENABLED=false" - "XPACK_REPORTING_ENCRYPTIONKEY=xCyqJUFqrUJJKxjZVGfnhrRkyqqaKeAG" + - "LOGGING_ROOT_LEVEL=all" From 34d21e964bdfa1f46b23458452ea3af2cd18bf7a Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 29 Nov 2021 12:31:33 +0100 Subject: [PATCH 24/52] Series of fixes for filebeat --- dev-tools/mage/pytest.go | 3 ++- .../system/config/filebeat_modules.yml.j2 | 4 +++- filebeat/tests/system/test_base.py | 18 +++++------------- filebeat/tests/system/test_modules.py | 9 ++++++--- filebeat/tests/system/test_pipeline.py | 11 +++++++---- filebeat/tests/system/test_reload_modules.py | 6 +++++- libbeat/tests/system/beat/beat.py | 2 +- libbeat/tests/system/beat/common_tests.py | 14 +++++++++++--- libbeat/tests/system/config/libbeat.yml.j2 | 2 ++ 9 files changed, 42 insertions(+), 27 deletions(-) diff --git a/dev-tools/mage/pytest.go b/dev-tools/mage/pytest.go index 2cf5e0539b7..340cb0f10fb 100644 --- a/dev-tools/mage/pytest.go +++ b/dev-tools/mage/pytest.go @@ -134,7 +134,8 @@ func PythonTest(params PythonTestArgs) error { pytestOptions := []string{ "--timeout=90", "--durations=20", - "-x", + // "-x", + "--tb=long", "-v", "-s", } diff --git a/filebeat/tests/system/config/filebeat_modules.yml.j2 b/filebeat/tests/system/config/filebeat_modules.yml.j2 index 710a3609ea4..d4a41e3d554 100644 --- a/filebeat/tests/system/config/filebeat_modules.yml.j2 +++ b/filebeat/tests/system/config/filebeat_modules.yml.j2 @@ -12,8 +12,10 @@ filebeat.overwrite_pipelines: true filebeat.config.modules: path: {{ beat.working_dir + '/modules.d/*.yml' }} -output.elasticsearch.hosts: ["{{ elasticsearch_url }}"] +output.elasticsearch.hosts: ["{{ elasticsearch.hosts }}"] output.elasticsearch.index: {{ index_name }} +output.elasticsearch.username: {{ elasticsearch.user }} +output.elasticsearch.password: {{ elasticsearch.pass }} setup.template.name: {{ index_name }} setup.template.pattern: {{ index_name }}* diff --git a/filebeat/tests/system/test_base.py b/filebeat/tests/system/test_base.py index 22ebfb1da6b..40f857989a7 100644 --- a/filebeat/tests/system/test_base.py +++ b/filebeat/tests/system/test_base.py @@ -45,22 +45,14 @@ def test_index_management(self): es_url = self.get_elasticsearch_url() es = self.get_elasticsearch_instance(url=es_url) self.render_config_template( - elasticsearch={"host": es_url}, + elasticsearch={ + "host": es_url, + "user": "filebeat_user", + "pass": os.getenv('ES_PASS'), + } ) exit_code = self.run_beat(extra_args=["setup", "--index-management"]) assert exit_code == 0 assert self.log_contains('Loaded index template') assert len(self.es.cat.templates(name='filebeat-*', h='name')) > 0 - - @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - def test_template_migration(self): - """ - Test that the template can be loaded with `setup --template` - """ - exit_code = self.run_beat(extra_args=["setup", "--template", - "-E", "setup.template.overwrite=true", "-E", "migration.6_to_7.enabled=true"]) - - assert exit_code == 0 - assert self.log_contains('Loaded index template') - assert len(self.es.cat.templates(name='filebeat-*', h='name')) > 0 diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py index d456c2ce1f8..ec04c3708a4 100644 --- a/filebeat/tests/system/test_modules.py +++ b/filebeat/tests/system/test_modules.py @@ -61,8 +61,7 @@ class Test(BaseTest): def init(self): self.elasticsearch_url = self.get_elasticsearch_url() - print("Using elasticsearch: {}".format(self.elasticsearch_url)) - self.es = self.get_elasticsearch_instance(url=self.elasticsearch_url) + self.es = self.get_elasticsearch_instance(url=self.elasticsearch_url, user='admin') logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("elasticsearch").setLevel(logging.ERROR) @@ -88,7 +87,11 @@ def test_fileset_file(self, module, fileset, test_file): template_name="filebeat_modules", output=cfgfile, index_name=self.index_name, - elasticsearch_url=self.elasticsearch_url, + elasticsearch={ + "hosts": self.elasticsearch_url, + "user": "admin", + "pass": os.getenv('ES_PASS'), + } ) self.run_on_file( diff --git a/filebeat/tests/system/test_pipeline.py b/filebeat/tests/system/test_pipeline.py index 4527a88e752..0295af15321 100644 --- a/filebeat/tests/system/test_pipeline.py +++ b/filebeat/tests/system/test_pipeline.py @@ -46,10 +46,13 @@ def test_input_pipeline_config(self): self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/*", - elasticsearch=dict( - host=self.elasticsearch_url, - pipeline="estest", - index=index_name), + elasticsearch={ + 'host': self.elasticsearch_url, + 'pipeline': "estest", + 'index': index_name, + 'user': "filebeat_user", + 'pass': os.getenv("ES_PASS") + }, pipeline="test", setup_template_name=index_name, setup_template_pattern=index_name + "*", diff --git a/filebeat/tests/system/test_reload_modules.py b/filebeat/tests/system/test_reload_modules.py index 22886c98052..50116c53021 100644 --- a/filebeat/tests/system/test_reload_modules.py +++ b/filebeat/tests/system/test_reload_modules.py @@ -71,7 +71,11 @@ def test_reload_writes_pipeline(self): reload_path=self.working_dir + "/configs/*.yml", reload_type="modules", inputs=False, - elasticsearch={"host": self.get_elasticsearch_url()} + elasticsearch={ + "host": self.get_elasticsearch_url(), + "user": "filebeat_user", + "pass": os.getenv('ES_PASS') + } ) proc = self.start_beat() diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index f9ed3fa9d74..2e37a0c489a 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -699,7 +699,7 @@ def get_elasticsearch_instance(self, security=True, ssl=False, url=None, user=No url = self.get_elasticsearch_url() if security: - username = os.getenv("ES_USER", user) + username = user or os.getenv("ES_USER", user) password = os.getenv("ES_PASS", "") es_instance = Elasticsearch([url], http_auth=(username, password)) else: diff --git a/libbeat/tests/system/beat/common_tests.py b/libbeat/tests/system/beat/common_tests.py index f5bccf2968d..2dd318dcfd4 100644 --- a/libbeat/tests/system/beat/common_tests.py +++ b/libbeat/tests/system/beat/common_tests.py @@ -111,8 +111,16 @@ def test_dashboards(self): es = Elasticsearch([self.get_elasticsearch_url()]) self.render_config_template( - elasticsearch={"host": self.get_elasticsearch_url()}, - kibana={"host": self.get_kibana_url()}, + elasticsearch={ + "host": self.get_elasticsearch_url(), + "user": "filebeat_user", + "pass": os.getenv('ES_PASS') + }, + kibana={ + "host": self.get_kibana_url(), + "user": "beats", + "pass": "testing", + }, ) exit_code = self.run_beat(extra_args=["setup", "--dashboards"]) @@ -126,7 +134,7 @@ def is_saved_object_api_available(self): def get_version(self): url = self.get_kibana_url() + "/api/status" - r = requests.get(url) + r = requests.get(url, auth=(os.getenv('ES_USER'), os.getenv('ES_PASS'))) body = r.json() version = body["version"]["number"] diff --git a/libbeat/tests/system/config/libbeat.yml.j2 b/libbeat/tests/system/config/libbeat.yml.j2 index 9b0baff0475..95af9436069 100644 --- a/libbeat/tests/system/config/libbeat.yml.j2 +++ b/libbeat/tests/system/config/libbeat.yml.j2 @@ -63,6 +63,8 @@ queue.mem: {% if kibana -%} setup.kibana.host: "{{ kibana.host }}" +setup.kibana.username: "{{ kibana.user }}" +setup.kibana.password: "{{ kibana.pass }}" {%- endif %} #================================ Outputs ===================================== From f4d5441d3603e83acc5ec9f127f6f5924339d73b Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 29 Nov 2021 12:42:36 +0100 Subject: [PATCH 25/52] Fixup of auditbeat integration tests --- auditbeat/tests/system/test_base.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/auditbeat/tests/system/test_base.py b/auditbeat/tests/system/test_base.py index cd76b3daee8..b36bf552990 100644 --- a/auditbeat/tests/system/test_base.py +++ b/auditbeat/tests/system/test_base.py @@ -79,8 +79,15 @@ def test_dashboards(self): "paths": dirs, } }], - elasticsearch={"host": self.get_elasticsearch_url(), "user": es_user, "pass": es_pass}, - kibana={"host": self.get_kibana_url()}, + elasticsearch={ + "host": self.get_elasticsearch_url(), + "user": es_user, + "pass": es_pass}, + kibana={ + "host": self.get_kibana_url(), + "user": es_user, + "pass": es_pass, + }, ) self.run_beat(extra_args=["setup", "--dashboards"], exit_code=0) From 91349cf8055c023d512449c847ab32907658c2ea Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 29 Nov 2021 13:04:24 +0100 Subject: [PATCH 26/52] Additional fixes --- filebeat/tests/system/test_modules.py | 2 +- filebeat/tests/system/test_reload_modules.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py index ec04c3708a4..4b33523388b 100644 --- a/filebeat/tests/system/test_modules.py +++ b/filebeat/tests/system/test_modules.py @@ -91,7 +91,7 @@ def test_fileset_file(self, module, fileset, test_file): "hosts": self.elasticsearch_url, "user": "admin", "pass": os.getenv('ES_PASS'), - } + } ) self.run_on_file( diff --git a/filebeat/tests/system/test_reload_modules.py b/filebeat/tests/system/test_reload_modules.py index 50116c53021..3ac300f440e 100644 --- a/filebeat/tests/system/test_reload_modules.py +++ b/filebeat/tests/system/test_reload_modules.py @@ -75,7 +75,7 @@ def test_reload_writes_pipeline(self): "host": self.get_elasticsearch_url(), "user": "filebeat_user", "pass": os.getenv('ES_PASS') - } + } ) proc = self.start_beat() From cf771efc6b140a8fd98e74bb472fdc47d6ad25e8 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 29 Nov 2021 13:10:19 +0100 Subject: [PATCH 27/52] Linting --- libbeat/tests/system/beat/common_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/tests/system/beat/common_tests.py b/libbeat/tests/system/beat/common_tests.py index 2dd318dcfd4..c7cf6d32aad 100644 --- a/libbeat/tests/system/beat/common_tests.py +++ b/libbeat/tests/system/beat/common_tests.py @@ -120,7 +120,7 @@ def test_dashboards(self): "host": self.get_kibana_url(), "user": "beats", "pass": "testing", - }, + }, ) exit_code = self.run_beat(extra_args=["setup", "--dashboards"]) From 4652aea15b6ac3b4e7c4f00828df27eb9a1b625a Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 29 Nov 2021 13:33:10 +0100 Subject: [PATCH 28/52] Linting --- auditbeat/tests/system/test_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auditbeat/tests/system/test_base.py b/auditbeat/tests/system/test_base.py index b36bf552990..998d9141784 100644 --- a/auditbeat/tests/system/test_base.py +++ b/auditbeat/tests/system/test_base.py @@ -52,7 +52,7 @@ def test_index_management(self): "paths": dirs, } }], - elasticsearch={"host": self.get_elasticsearch_url(), "user": es_user, "pass": es_pass}) + elasticsearch={"host": self.get_elasticsearch_url(), "user": es_user, "pass": es_pass}) self.run_beat(extra_args=["setup", "--index-management"], exit_code=0) assert self.log_contains('Loaded index template') From a346733c98519e07e0022ecefb39770d7a3d083d Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 29 Nov 2021 14:36:44 +0100 Subject: [PATCH 29/52] Lint and test fix --- filebeat/tests/system/test_setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/filebeat/tests/system/test_setup.py b/filebeat/tests/system/test_setup.py index 3ea31152242..b4193daf75a 100644 --- a/filebeat/tests/system/test_setup.py +++ b/filebeat/tests/system/test_setup.py @@ -28,6 +28,8 @@ def test_setup_modules_d_config(self): modules=True, elasticsearch={ "host": self.get_elasticsearch_url(), + "user": "filebeat_user", + "pass": os.getenv('ES_PASS'), }, ) From 43a7b69a1c8346a68a6469b4c769c2904e95a020 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 29 Nov 2021 16:18:16 +0100 Subject: [PATCH 30/52] Remove debugging --- heartbeat/tests/system/test_base.py | 6 ------ libbeat/kibana/client.go | 7 ------- testing/environments/docker/README.md | 2 +- testing/environments/docker/elasticsearch/roles.yml | 2 +- 4 files changed, 2 insertions(+), 15 deletions(-) diff --git a/heartbeat/tests/system/test_base.py b/heartbeat/tests/system/test_base.py index 8017d1db927..ccd62f36af6 100644 --- a/heartbeat/tests/system/test_base.py +++ b/heartbeat/tests/system/test_base.py @@ -200,12 +200,6 @@ def test_index_management(self): es_url = self.get_elasticsearch_url() es_user = os.getenv('ES_USER') es_pass = os.getenv('ES_PASS') - #es_user = 'heartbeat_user' - #es_pass = 'testing' - print('ES USER: {}'.format(es_user)) - print('ES PASS: {}'.format(es_pass)) - print('ES OS USER: {}'.format(os.getenv('ES_USER'))) - print('ES OS PASS: {}'.format(os.getenv('ES_PASS'))) es = self.get_elasticsearch_instance(url=es_url, user='beats') self.render_config_template( monitors=[{ diff --git a/libbeat/kibana/client.go b/libbeat/kibana/client.go index 6ceb3d19886..d7da444fa8b 100644 --- a/libbeat/kibana/client.go +++ b/libbeat/kibana/client.go @@ -285,9 +285,6 @@ func (conn *Connection) SendWithContext(ctx context.Context, method, extraPath s req.Header.Set("Accept", "application/json") req.Header.Set("kbn-xsrf", "1") - log := logp.NewLogger("kibana_conn") - log.Infof("kibana headers: %s", req) - return conn.RoundTrip(req) } @@ -354,8 +351,6 @@ func (client *Client) readVersion() error { func (client *Client) GetVersion() common.Version { return client.Version } func (client *Client) ImportMultiPartFormFile(url string, params url.Values, filename string, contents string) error { - log := logp.NewLogger("kibana_foo") - log.Infof("starting import multiport form") buf := &bytes.Buffer{} w := multipart.NewWriter(buf) @@ -375,9 +370,7 @@ func (client *Client) ImportMultiPartFormFile(url string, params url.Values, fil headers := http.Header{} headers.Add("Content-Type", w.FormDataContentType()) - log.Infof("starting conn") statusCode, response, err := client.Connection.Request("POST", url, params, headers, buf) - log.Infof("finished conn") if err != nil || statusCode >= 300 { return fmt.Errorf("returned %d to import file: %v. Response: %s", statusCode, err, response) } diff --git a/testing/environments/docker/README.md b/testing/environments/docker/README.md index 2ecb6c1688a..8ecb7bb5241 100644 --- a/testing/environments/docker/README.md +++ b/testing/environments/docker/README.md @@ -3,4 +3,4 @@ This directory contains default usernames and passwords with roles configured according to the Beats documentation. -The default password for all accounts is `changeme`. \ No newline at end of file +The default password for all accounts is `testing`. \ No newline at end of file diff --git a/testing/environments/docker/elasticsearch/roles.yml b/testing/environments/docker/elasticsearch/roles.yml index ea8e5ffea7e..2f324761053 100644 --- a/testing/environments/docker/elasticsearch/roles.yml +++ b/testing/environments/docker/elasticsearch/roles.yml @@ -28,4 +28,4 @@ metricbeat: cluster: ['manage_index_templates', 'monitor', 'manage_ingest_pipelines', 'manage_ilm'] indices: - names: ['metricbeat-*', 'shrink-metricbeat-*'] - privileges: ['all'] \ No newline at end of file + privileges: ['all'] From ff7f8bdc25e7e039827e0212c0e38d60c9e96120 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 29 Nov 2021 16:21:57 +0100 Subject: [PATCH 31/52] Re-enabled kafka container --- libbeat/docker-compose.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/libbeat/docker-compose.yml b/libbeat/docker-compose.yml index 8073cab58ff..ffd7649bb2e 100644 --- a/libbeat/docker-compose.yml +++ b/libbeat/docker-compose.yml @@ -49,7 +49,7 @@ services: elasticsearch_monitoring: { condition: service_healthy } elasticsearchssl: { condition: service_healthy } logstash: { condition: service_healthy } -# kafka: { condition: service_healthy } + kafka: { condition: service_healthy } redis: { condition: service_healthy } sredis: { condition: service_healthy } kibana: { condition: service_healthy } @@ -121,13 +121,13 @@ services: - REDIS_HOST=redis - REDIS_PORT=6379 -# kafka: -# build: ${ES_BEATS}/testing/environments/docker/kafka -# expose: -# - 9092 -# - 2181 -# environment: -# - ADVERTISED_HOST=kafka + kafka: + build: ${ES_BEATS}/testing/environments/docker/kafka + expose: + - 9092 + - 2181 + environment: + - ADVERTISED_HOST=kafka # elasticsearch_kerberos.elastic: # build: ${ES_BEATS}/testing/environments/docker/elasticsearch_kerberos From 15492ebba8f6152585cce65970cdfee40bba99dd Mon Sep 17 00:00:00 2001 From: Mike Place Date: Tue, 30 Nov 2021 12:03:00 +0100 Subject: [PATCH 32/52] Move auditbeat to template generator --- auditbeat/tests/system/test_base.py | 7 +++---- libbeat/tests/system/beat/beat.py | 9 ++++++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/auditbeat/tests/system/test_base.py b/auditbeat/tests/system/test_base.py index 998d9141784..51779238bd9 100644 --- a/auditbeat/tests/system/test_base.py +++ b/auditbeat/tests/system/test_base.py @@ -40,10 +40,8 @@ def test_index_management(self): Test that the template can be loaded with `setup --index-management` """ dirs = [self.temp_dir("auditbeat_test")] - es_user = os.getenv('ES_USER') - es_pass = os.getenv('ES_PASS') with PathCleanup(dirs): - es = self.get_elasticsearch_instance(url=self.get_elasticsearch_url(), user=es_user) + es = self.get_elasticsearch_instance(url=self.get_elasticsearch_url()) self.render_config_template( modules=[{ @@ -52,7 +50,8 @@ def test_index_management(self): "paths": dirs, } }], - elasticsearch={"host": self.get_elasticsearch_url(), "user": es_user, "pass": es_pass}) + elasticsearch=self.get_elasticsearch_template_config() + ) self.run_beat(extra_args=["setup", "--index-management"], exit_code=0) assert self.log_contains('Loaded index template') diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index 2e37a0c489a..43de50a24c7 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -687,6 +687,13 @@ def get_elasticsearch_url_ssl(self): port=os.getenv("ES_PORT_SSL", "9205"), ) + def get_elasticsearch_template_config(self): + return { + "host": self.get_elasticsearch_url(), + "user": os.getenv("ES_USER", ""), + "pass": os.getenv("ES_PASS", "") + } + def get_elasticsearch_instance(self, security=True, ssl=False, url=None, user=None): """ Returns an elasticsearch.Elasticsearch instance built from the @@ -699,7 +706,7 @@ def get_elasticsearch_instance(self, security=True, ssl=False, url=None, user=No url = self.get_elasticsearch_url() if security: - username = user or os.getenv("ES_USER", user) + username = user or os.getenv("ES_USER", "") password = os.getenv("ES_PASS", "") es_instance = Elasticsearch([url], http_auth=(username, password)) else: From 9307cad2ae15c068d1f93036dd90a85ba5c959c6 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Tue, 30 Nov 2021 12:13:57 +0100 Subject: [PATCH 33/52] Add kibana template config test helper --- auditbeat/tests/system/test_base.py | 16 +++------------ libbeat/tests/system/beat/beat.py | 30 +++++++++++++++++++++++++---- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/auditbeat/tests/system/test_base.py b/auditbeat/tests/system/test_base.py index 51779238bd9..4bb744fe80b 100644 --- a/auditbeat/tests/system/test_base.py +++ b/auditbeat/tests/system/test_base.py @@ -50,7 +50,7 @@ def test_index_management(self): "paths": dirs, } }], - elasticsearch=self.get_elasticsearch_template_config() + elasticsearch = self.get_elasticsearch_template_config() ) self.run_beat(extra_args=["setup", "--index-management"], exit_code=0) @@ -68,9 +68,6 @@ def test_dashboards(self): kibana_dir = os.path.join(self.beat_path, "build", "kibana") shutil.copytree(kibana_dir, os.path.join(self.working_dir, "kibana")) - es_user = os.getenv('ES_USER') - es_pass = os.getenv('ES_PASS') - es = self.get_elasticsearch_instance(url=self.get_elasticsearch_url(), user=es_user) self.render_config_template( modules=[{ "name": "file_integrity", @@ -78,15 +75,8 @@ def test_dashboards(self): "paths": dirs, } }], - elasticsearch={ - "host": self.get_elasticsearch_url(), - "user": es_user, - "pass": es_pass}, - kibana={ - "host": self.get_kibana_url(), - "user": es_user, - "pass": es_pass, - }, + elasticsearch = self.get_elasticsearch_template_config(), + kibana = self.get_kibana_template_config(), ) self.run_beat(extra_args=["setup", "--dashboards"], exit_code=0) diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index 43de50a24c7..933c9a30040 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -687,13 +687,21 @@ def get_elasticsearch_url_ssl(self): port=os.getenv("ES_PORT_SSL", "9205"), ) - def get_elasticsearch_template_config(self): - return { + + def get_elasticsearch_template_config(self, security=True): + """ + Returns a template suitable for a Beats config + """ + template = { "host": self.get_elasticsearch_url(), - "user": os.getenv("ES_USER", ""), - "pass": os.getenv("ES_PASS", "") } + if security: + template["user"] = os.getenv("ES_USER", "") + template["pass"] = os.getenv("ES_PASS", "") + + return template + def get_elasticsearch_instance(self, security=True, ssl=False, url=None, user=None): """ Returns an elasticsearch.Elasticsearch instance built from the @@ -722,6 +730,20 @@ def get_kibana_url(self): port=os.getenv("KIBANA_PORT", "5601"), ) + def get_kibana_template_config(self, security=True): + """ + Returns a Kibana template suitable for a Beat + """ + template = { + "host": self.get_kibana_url() + } + + if security: + template["user"] = os.getenv("ES_USER", "") + template["pass"] = os.getenv("ES_PASS", "") + + return template + def assert_fields_are_documented(self, evt): """ Assert that all keys present in evt are documented in fields.yml. From a929fb22aabe4074a9c9b65ddb39efffcbe6e705 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Tue, 30 Nov 2021 12:14:58 +0100 Subject: [PATCH 34/52] Lint auditbeat --- auditbeat/tests/system/test_base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/auditbeat/tests/system/test_base.py b/auditbeat/tests/system/test_base.py index 4bb744fe80b..4d55a3b5239 100644 --- a/auditbeat/tests/system/test_base.py +++ b/auditbeat/tests/system/test_base.py @@ -50,7 +50,7 @@ def test_index_management(self): "paths": dirs, } }], - elasticsearch = self.get_elasticsearch_template_config() + elasticsearch=self.get_elasticsearch_template_config() ) self.run_beat(extra_args=["setup", "--index-management"], exit_code=0) @@ -75,8 +75,8 @@ def test_dashboards(self): "paths": dirs, } }], - elasticsearch = self.get_elasticsearch_template_config(), - kibana = self.get_kibana_template_config(), + elasticsearch=self.get_elasticsearch_template_config(), + kibana=self.get_kibana_template_config(), ) self.run_beat(extra_args=["setup", "--dashboards"], exit_code=0) From ff8a05d6e71429bb1aee554bb9961d9d5bca59ba Mon Sep 17 00:00:00 2001 From: Mike Place Date: Tue, 30 Nov 2021 13:45:40 +0100 Subject: [PATCH 35/52] Cleaning up filebeat tests --- auditbeat/tests/system/test_base.py | 2 +- .../tests/system/config/filebeat_modules.yml.j2 | 2 +- filebeat/tests/system/test_base.py | 16 +++------------- filebeat/tests/system/test_modules.py | 9 ++------- filebeat/tests/system/test_pipeline.py | 5 ++--- filebeat/tests/system/test_reload_modules.py | 6 +----- filebeat/tests/system/test_setup.py | 8 ++------ libbeat/tests/system/beat/beat.py | 8 ++++---- 8 files changed, 16 insertions(+), 40 deletions(-) diff --git a/auditbeat/tests/system/test_base.py b/auditbeat/tests/system/test_base.py index 4d55a3b5239..73a7d8ae9f0 100644 --- a/auditbeat/tests/system/test_base.py +++ b/auditbeat/tests/system/test_base.py @@ -41,7 +41,7 @@ def test_index_management(self): """ dirs = [self.temp_dir("auditbeat_test")] with PathCleanup(dirs): - es = self.get_elasticsearch_instance(url=self.get_elasticsearch_url()) + es = self.get_elasticsearch_instance() self.render_config_template( modules=[{ diff --git a/filebeat/tests/system/config/filebeat_modules.yml.j2 b/filebeat/tests/system/config/filebeat_modules.yml.j2 index d4a41e3d554..93ded5a1379 100644 --- a/filebeat/tests/system/config/filebeat_modules.yml.j2 +++ b/filebeat/tests/system/config/filebeat_modules.yml.j2 @@ -12,7 +12,7 @@ filebeat.overwrite_pipelines: true filebeat.config.modules: path: {{ beat.working_dir + '/modules.d/*.yml' }} -output.elasticsearch.hosts: ["{{ elasticsearch.hosts }}"] +output.elasticsearch.hosts: ["{{ elasticsearch.host }}"] output.elasticsearch.index: {{ index_name }} output.elasticsearch.username: {{ elasticsearch.user }} output.elasticsearch.password: {{ elasticsearch.pass }} diff --git a/filebeat/tests/system/test_base.py b/filebeat/tests/system/test_base.py index 40f857989a7..61a38c6c895 100644 --- a/filebeat/tests/system/test_base.py +++ b/filebeat/tests/system/test_base.py @@ -10,13 +10,9 @@ class Test(BaseTest, common_tests.TestExportsMixin, common_tests.TestDashboardMi def setUp(self): super(Test, self).setUp() self.render_config_template( - elasticsearch={ - "host": self.get_elasticsearch_url(), - "user": "filebeat_user", - "pass": os.getenv('ES_PASS') - }, + elasticsearch=self.get_elasticsearch_template_config(), ) - self.es = self.get_elasticsearch_instance(url=self.get_elasticsearch_url(), user='filebeat_user') + self.es = self.get_elasticsearch_instance() def test_base(self): """ @@ -42,14 +38,8 @@ def test_index_management(self): """ Test that the template can be loaded with `setup --index-management` """ - es_url = self.get_elasticsearch_url() - es = self.get_elasticsearch_instance(url=es_url) self.render_config_template( - elasticsearch={ - "host": es_url, - "user": "filebeat_user", - "pass": os.getenv('ES_PASS'), - } + elasticsearch=self.get_elasticsearch_template_config(), ) exit_code = self.run_beat(extra_args=["setup", "--index-management"]) diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py index 4b33523388b..6dd64b1f349 100644 --- a/filebeat/tests/system/test_modules.py +++ b/filebeat/tests/system/test_modules.py @@ -60,8 +60,7 @@ def load_fileset_test_cases(): class Test(BaseTest): def init(self): - self.elasticsearch_url = self.get_elasticsearch_url() - self.es = self.get_elasticsearch_instance(url=self.elasticsearch_url, user='admin') + self.es = self.get_elasticsearch_instance(user='admin') logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("elasticsearch").setLevel(logging.ERROR) @@ -87,11 +86,7 @@ def test_fileset_file(self, module, fileset, test_file): template_name="filebeat_modules", output=cfgfile, index_name=self.index_name, - elasticsearch={ - "hosts": self.elasticsearch_url, - "user": "admin", - "pass": os.getenv('ES_PASS'), - } + elasticsearch=self.get_elasticsearch_template_config(user='admin') ) self.run_on_file( diff --git a/filebeat/tests/system/test_pipeline.py b/filebeat/tests/system/test_pipeline.py index 0295af15321..83cc25ff7d4 100644 --- a/filebeat/tests/system/test_pipeline.py +++ b/filebeat/tests/system/test_pipeline.py @@ -11,8 +11,7 @@ class Test(BaseTest): def init(self): self.elasticsearch_url = self.get_elasticsearch_url() self.kibana_url = self.get_kibana_url() - print("Using elasticsearch: {}".format(self.elasticsearch_url)) - self.es = self.get_elasticsearch_instance(url=self.elasticsearch_url) + self.es = self.get_elasticsearch_instance() logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("elasticsearch").setLevel(logging.ERROR) @@ -50,7 +49,7 @@ def test_input_pipeline_config(self): 'host': self.elasticsearch_url, 'pipeline': "estest", 'index': index_name, - 'user': "filebeat_user", + 'user': os.getenv("ES_USER"), 'pass': os.getenv("ES_PASS") }, pipeline="test", diff --git a/filebeat/tests/system/test_reload_modules.py b/filebeat/tests/system/test_reload_modules.py index 3ac300f440e..5b8e08f49f4 100644 --- a/filebeat/tests/system/test_reload_modules.py +++ b/filebeat/tests/system/test_reload_modules.py @@ -71,11 +71,7 @@ def test_reload_writes_pipeline(self): reload_path=self.working_dir + "/configs/*.yml", reload_type="modules", inputs=False, - elasticsearch={ - "host": self.get_elasticsearch_url(), - "user": "filebeat_user", - "pass": os.getenv('ES_PASS') - } + elasticsearch=self.get_elasticsearch_template_config(), ) proc = self.start_beat() diff --git a/filebeat/tests/system/test_setup.py b/filebeat/tests/system/test_setup.py index b4193daf75a..7422c8d3329 100644 --- a/filebeat/tests/system/test_setup.py +++ b/filebeat/tests/system/test_setup.py @@ -13,7 +13,7 @@ class Test(BaseTest): def init(self): self.elasticsearch_url = self.get_elasticsearch_url() print("Using elasticsearch: {}".format(self.elasticsearch_url)) - self.es = self.get_elasticsearch_instance(url=self.elasticsearch_url) + self.es = self.get_elasticsearch_instance() @unittest.skipIf(not INTEGRATION_TESTS, "integration tests are disabled, run with INTEGRATION_TESTS=1 to enable them.") @@ -26,11 +26,7 @@ def test_setup_modules_d_config(self): self.init() self.render_config_template( modules=True, - elasticsearch={ - "host": self.get_elasticsearch_url(), - "user": "filebeat_user", - "pass": os.getenv('ES_PASS'), - }, + elasticsearch=self.get_elasticsearch_template_config(), ) self._setup_dummy_module() diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index 933c9a30040..58622653828 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -688,7 +688,7 @@ def get_elasticsearch_url_ssl(self): ) - def get_elasticsearch_template_config(self, security=True): + def get_elasticsearch_template_config(self, security=True, user=None): """ Returns a template suitable for a Beats config """ @@ -697,7 +697,7 @@ def get_elasticsearch_template_config(self, security=True): } if security: - template["user"] = os.getenv("ES_USER", "") + template["user"] = user or os.getenv("ES_USER", "") template["pass"] = os.getenv("ES_PASS", "") return template @@ -730,7 +730,7 @@ def get_kibana_url(self): port=os.getenv("KIBANA_PORT", "5601"), ) - def get_kibana_template_config(self, security=True): + def get_kibana_template_config(self, security=True, user=None): """ Returns a Kibana template suitable for a Beat """ @@ -739,7 +739,7 @@ def get_kibana_template_config(self, security=True): } if security: - template["user"] = os.getenv("ES_USER", "") + template["user"] = user or os.getenv("ES_USER", "") template["pass"] = os.getenv("ES_PASS", "") return template From 0dbd3a8a188a60d1055d8e01d3c98029e4f9eac7 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Tue, 30 Nov 2021 13:54:45 +0100 Subject: [PATCH 36/52] Clean up heartbeat tests --- heartbeat/tests/system/test_base.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/heartbeat/tests/system/test_base.py b/heartbeat/tests/system/test_base.py index ccd62f36af6..7819ccb291e 100644 --- a/heartbeat/tests/system/test_base.py +++ b/heartbeat/tests/system/test_base.py @@ -197,20 +197,13 @@ def test_index_management(self): """ Test that the template can be loaded with `setup --index-management` """ - es_url = self.get_elasticsearch_url() - es_user = os.getenv('ES_USER') - es_pass = os.getenv('ES_PASS') - es = self.get_elasticsearch_instance(url=es_url, user='beats') + es = self.get_elasticsearch_instance() self.render_config_template( monitors=[{ "type": "http", "urls": ["http://localhost:9200"], }], - elasticsearch={ - "host": es_url, - "user": es_user, - "pass": es_pass, - } + elasticsearch=self.get_elasticsearch_template_config() ) exit_code = self.run_beat(extra_args=["setup", "--index-management"]) From f73950ad99a309c71df18a46d0495ca20e88d8c5 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Tue, 30 Nov 2021 16:03:13 +0100 Subject: [PATCH 37/52] Refining tests for libbeat --- dev-tools/mage/pytest.go | 2 +- libbeat/dashboards/kibana_loader.go | 5 ---- libbeat/docker-compose.yml | 2 +- libbeat/tests/system/beat/common_tests.py | 12 ++------ libbeat/tests/system/config/mockbeat.yml.j2 | 30 +++++++++++++++---- libbeat/tests/system/test_ca_pinning.py | 20 ++++++------- .../system/test_cmd_setup_index_management.py | 6 ++-- 7 files changed, 41 insertions(+), 36 deletions(-) diff --git a/dev-tools/mage/pytest.go b/dev-tools/mage/pytest.go index 340cb0f10fb..790317cda77 100644 --- a/dev-tools/mage/pytest.go +++ b/dev-tools/mage/pytest.go @@ -134,7 +134,7 @@ func PythonTest(params PythonTestArgs) error { pytestOptions := []string{ "--timeout=90", "--durations=20", - // "-x", + "-x", "--tb=long", "-v", "-s", diff --git a/libbeat/dashboards/kibana_loader.go b/libbeat/dashboards/kibana_loader.go index ab9c90e2e32..fe0aad61f03 100644 --- a/libbeat/dashboards/kibana_loader.go +++ b/libbeat/dashboards/kibana_loader.go @@ -102,15 +102,12 @@ func (loader KibanaLoader) ImportIndexFile(file string) error { // read json file reader, err := ioutil.ReadFile(file) - loader.statusMsg("Finished file read for %s", file) if err != nil { return fmt.Errorf("fail to read index-pattern from file %s: %v", file, err) } var indexContent common.MapStr err = json.Unmarshal(reader, &indexContent) - - loader.statusMsg("Finished json unmarshal for %s", file) if err != nil { return fmt.Errorf("fail to unmarshal the index content from file %s: %v", file, err) } @@ -142,9 +139,7 @@ func (loader KibanaLoader) ImportIndex(pattern common.MapStr) error { // ImportDashboard imports the dashboard file func (loader KibanaLoader) ImportDashboard(file string) error { - loader.statusMsg("dashboard attempt starting") if loader.version.LessThan(kibana.MinimumRequiredVersionSavedObjects) { - loader.statusMsg("loader version mismatch fail") return fmt.Errorf("Kibana version must be at least " + kibana.MinimumRequiredVersionSavedObjects.String()) } diff --git a/libbeat/docker-compose.yml b/libbeat/docker-compose.yml index ffd7649bb2e..2d7007f5e1e 100644 --- a/libbeat/docker-compose.yml +++ b/libbeat/docker-compose.yml @@ -21,7 +21,7 @@ services: - KIBANA_PASS=testing - ES_HOST=elasticsearch - ES_PORT=9200 - - ES_USER=admin + - ES_USER=beats - ES_PASS=testing - ES_MONITORING_HOST=elasticsearch_monitoring - ES_MONITORING_PORT=9200 diff --git a/libbeat/tests/system/beat/common_tests.py b/libbeat/tests/system/beat/common_tests.py index c7cf6d32aad..0d76e155bc2 100644 --- a/libbeat/tests/system/beat/common_tests.py +++ b/libbeat/tests/system/beat/common_tests.py @@ -111,16 +111,8 @@ def test_dashboards(self): es = Elasticsearch([self.get_elasticsearch_url()]) self.render_config_template( - elasticsearch={ - "host": self.get_elasticsearch_url(), - "user": "filebeat_user", - "pass": os.getenv('ES_PASS') - }, - kibana={ - "host": self.get_kibana_url(), - "user": "beats", - "pass": "testing", - }, + elasticsearch=self.get_elasticsearch_template_config(), + kibana=self.get_kibana_template_config(), ) exit_code = self.run_beat(extra_args=["setup", "--dashboards"]) diff --git a/libbeat/tests/system/config/mockbeat.yml.j2 b/libbeat/tests/system/config/mockbeat.yml.j2 index 8aee28a276f..046f1654eb7 100644 --- a/libbeat/tests/system/config/mockbeat.yml.j2 +++ b/libbeat/tests/system/config/mockbeat.yml.j2 @@ -37,12 +37,32 @@ output: {% endfor -%} {%- endif %} - {% if elasticsearch -%} +{% if elasticsearch -%} +output: elasticsearch: - {% for k, v in elasticsearch.items() -%} - {{ k }}: {{ v }} - {% endfor -%} - {%- endif %} + hosts: ["{{ elasticsearch.host }}"] + username: {{ elasticsearch.user }} + password: {{ elasticsearch.pass }} + {% if elasticsearch.pipeline %} + pipeline: {{elasticsearch.pipeline}} + {% endif %} + {% if elasticsearch.index %} + index: {{elasticsearch.index}} + {% endif %} + {% if elasticsearch.ilm %} + ilm.enabled: {{elasticsearch.ilm}} + {% endif %} + {% if elasticsearch.timeout %} + timeout: {{elasticsearch.timeout}} + {% endif %} + {% if elasticsearch.ssl_certificate_authorities %} + ssl.certificate_authorities: {{elasticsearch.ssl_certificate_authorities}} + {% endif %} + {% if elasticsearch.ssl_ca_sha256 %} + ssl.ca_sha256: {{ elasticsearch.ssl_ca_sha256 }} + {% endif %} +{%- endif %} + # Redis as output # Options: diff --git a/libbeat/tests/system/test_ca_pinning.py b/libbeat/tests/system/test_ca_pinning.py index 55a9e9ea13c..e7969f70b2c 100644 --- a/libbeat/tests/system/test_ca_pinning.py +++ b/libbeat/tests/system/test_ca_pinning.py @@ -33,11 +33,11 @@ def test_sending_events_with_a_good_sha256(self): self.render_config_template( elasticsearch={ - "hosts": self.get_elasticsearch_url_ssl(), - "username": "admin", - "password": "testing", - "ssl.certificate_authorities": [ca], - "ssl.ca_sha256": "8hZS8gpciuzlu+7Xi0sdv8T7RKRRxG1TWKumUQsDam0=", + "host": self.get_elasticsearch_url_ssl(), + "user": "beats", + "pass": "testing", + "ssl_certificate_authorities": [ca], + "ssl_ca_sha256": "8hZS8gpciuzlu+7Xi0sdv8T7RKRRxG1TWKumUQsDam0=", }, ) @@ -65,11 +65,11 @@ def test_sending_events_with_a_bad_sha256(self): self.render_config_template( elasticsearch={ - "hosts": self.get_elasticsearch_url_ssl(), - "username": "admin", - "password": "testing", - "ssl.certificate_authorities": [ca], - "ssl.ca_sha256": "not-good-sha", + "host": self.get_elasticsearch_url_ssl(), + "user": "beats", + "pass": "testing", + "ssl_certificate_authorities": [ca], + "ssl_ca_sha256": "not-good-sha", }, ) diff --git a/libbeat/tests/system/test_cmd_setup_index_management.py b/libbeat/tests/system/test_cmd_setup_index_management.py index 47028962b2c..8dffde675b6 100644 --- a/libbeat/tests/system/test_cmd_setup_index_management.py +++ b/libbeat/tests/system/test_cmd_setup_index_management.py @@ -27,7 +27,7 @@ def setUp(self): self.custom_policy = self.beat_name + "_bar" self.custom_template = self.beat_name + "_foobar" - self.es = self.es_client() + self.es = self.get_elasticsearch_instance() self.idxmgmt = IdxMgmt(self.es, self.index_name) self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy], policies=[self.policy_name, self.custom_policy]) @@ -40,12 +40,10 @@ def tearDown(self): policies=[self.policy_name, self.custom_policy]) def render_config(self, **kwargs): - print('OS ES USER: {}'.format(os.getenv('ES_USER'))) - print('OS ES pass: {}'.format(os.getenv('ES_PASS'))) self.render_config_template( # Note that the template is such that we need to pass in 'username' as opposed to 'user' and # 'password' instead of 'pass'. - elasticsearch={"hosts": self.get_elasticsearch_url(), "username": "admin", "password": "testing"}, + elasticsearch=self.get_elasticsearch_template_config(), es_template_name=self.index_name, **kwargs ) From 71130eeaf88114c0b59a0a4a39a2f4a295821412 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Tue, 30 Nov 2021 17:03:09 +0100 Subject: [PATCH 38/52] Lint --- libbeat/tests/system/beat/beat.py | 1 - 1 file changed, 1 deletion(-) diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index 58622653828..4f151bdd936 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -687,7 +687,6 @@ def get_elasticsearch_url_ssl(self): port=os.getenv("ES_PORT_SSL", "9205"), ) - def get_elasticsearch_template_config(self, security=True, user=None): """ Returns a template suitable for a Beats config From 1cf01cab3d9087915b6a59c9805b2d9bcb843447 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Wed, 1 Dec 2021 15:46:01 +0100 Subject: [PATCH 39/52] Fixups for libbeat tests --- dev-tools/mage/pytest.go | 2 +- libbeat/docker-compose.yml | 7 ++++++- libbeat/tests/system/test_ca_pinning.py | 2 +- libbeat/tests/system/test_cmd_test.py | 8 ++++++-- libbeat/tests/system/test_ilm.py | 4 ++-- libbeat/tests/system/test_keystore.py | 13 +++++++------ libbeat/tests/system/test_template.py | 10 +++++----- .../tests/system/config/filebeat_modules.yml.j2 | 6 +++--- 8 files changed, 31 insertions(+), 21 deletions(-) diff --git a/dev-tools/mage/pytest.go b/dev-tools/mage/pytest.go index 790317cda77..340cb0f10fb 100644 --- a/dev-tools/mage/pytest.go +++ b/dev-tools/mage/pytest.go @@ -134,7 +134,7 @@ func PythonTest(params PythonTestArgs) error { pytestOptions := []string{ "--timeout=90", "--durations=20", - "-x", + // "-x", "--tb=long", "-v", "-s", diff --git a/libbeat/docker-compose.yml b/libbeat/docker-compose.yml index 2d7007f5e1e..0cb37e69ae1 100644 --- a/libbeat/docker-compose.yml +++ b/libbeat/docker-compose.yml @@ -21,7 +21,9 @@ services: - KIBANA_PASS=testing - ES_HOST=elasticsearch - ES_PORT=9200 - - ES_USER=beats + # ES_USER must be admin in order for the Go Integration tests to + # function because they require indices:data/read/search + - ES_USER=admin - ES_PASS=testing - ES_MONITORING_HOST=elasticsearch_monitoring - ES_MONITORING_PORT=9200 @@ -94,6 +96,9 @@ services: - "xpack.security.authc.realms.file.file1.order=0" volumes: - ${ES_BEATS}/testing/environments/docker/elasticsearch/pki:/usr/share/elasticsearch/config/pki:ro + - ${ES_BEATS}/testing/environments/docker/elasticsearch/roles.yml:/usr/share/elasticsearch/config/roles.yml + - ${ES_BEATS}/testing/environments/docker/elasticsearch/users:/usr/share/elasticsearch/config/users + - ${ES_BEATS}/testing/environments/docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles expose: - 9200 diff --git a/libbeat/tests/system/test_ca_pinning.py b/libbeat/tests/system/test_ca_pinning.py index e7969f70b2c..484e90b0337 100644 --- a/libbeat/tests/system/test_ca_pinning.py +++ b/libbeat/tests/system/test_ca_pinning.py @@ -34,7 +34,7 @@ def test_sending_events_with_a_good_sha256(self): self.render_config_template( elasticsearch={ "host": self.get_elasticsearch_url_ssl(), - "user": "beats", + "user": "admin", "pass": "testing", "ssl_certificate_authorities": [ca], "ssl_ca_sha256": "8hZS8gpciuzlu+7Xi0sdv8T7RKRRxG1TWKumUQsDam0=", diff --git a/libbeat/tests/system/test_cmd_test.py b/libbeat/tests/system/test_cmd_test.py index e4d916841cb..944d7791fb6 100644 --- a/libbeat/tests/system/test_cmd_test.py +++ b/libbeat/tests/system/test_cmd_test.py @@ -52,7 +52,7 @@ def test_output(self): self.render_config_template("mockbeat", os.path.join(self.working_dir, "mockbeat.yml"), - elasticsearch={"hosts": self.get_elasticsearch_url(), "username": "admin", "password": "testing"}) + elasticsearch=self.get_elasticsearch_template_config()) exit_code = self.run_beat( extra_args=["test", "output"], config="mockbeat.yml") @@ -70,7 +70,11 @@ def test_wrong_output(self): self.render_config_template("mockbeat", os.path.join(self.working_dir, "mockbeat.yml"), - elasticsearch={"hosts": '["badhost:9200"]'}) + elasticsearch={ + "host": 'badhost:9200', + "user": 'admin', + "pass": 'testing' + }) exit_code = self.run_beat( extra_args=["test", "output"], config="mockbeat.yml") diff --git a/libbeat/tests/system/test_ilm.py b/libbeat/tests/system/test_ilm.py index a9009a840c0..dd9f6290508 100644 --- a/libbeat/tests/system/test_ilm.py +++ b/libbeat/tests/system/test_ilm.py @@ -36,7 +36,7 @@ def tearDown(self): def render_config(self, **kwargs): self.render_config_template( - elasticsearch={"hosts": self.get_elasticsearch_url(), "username": "admin", "password": "testing"}, + elasticsearch=self.get_elasticsearch_template_config(), es_template_name=self.index_name, **kwargs ) @@ -186,7 +186,7 @@ def tearDown(self): def render_config(self, **kwargs): self.render_config_template( - elasticsearch={"hosts": self.get_elasticsearch_url(), "username": "admin", "password": "testing"}, + elasticsearch=self.get_elasticsearch_template_config(), es_template_name=self.index_name, **kwargs ) diff --git a/libbeat/tests/system/test_keystore.py b/libbeat/tests/system/test_keystore.py index b0589123c83..44fe8d1a051 100644 --- a/libbeat/tests/system/test_keystore.py +++ b/libbeat/tests/system/test_keystore.py @@ -43,12 +43,12 @@ def test_keystore_with_key_not_present(self): key = "elasticsearch_host" self.render_config_template(keystore_path=self.keystore_path, elasticsearch={ - 'hosts': "${%s}:9200" % key + 'host': "${%s}:9200" % key }) exit_code = self.run_beat() assert self.log_contains( - "missing field accessing 'output.elasticsearch.hosts'") + "missing field accessing 'output.elasticsearch.hosts") assert exit_code == 1 def test_keystore_with_nested_key(self): @@ -80,9 +80,10 @@ def test_export_config_with_keystore(self): key = "asecret" secret = "asecretvalue" - self.render_config_template(keystore_path=self.keystore_path, elasticsearch={ - 'hosts': "${%s}" % key - }) + self.render_config_template( + keystore_path=self.keystore_path, + elasticsearch=self.get_elasticsearch_template_config() + ) exit_code = self.run_beat(extra_args=["keystore", "create"]) assert exit_code == 0 @@ -92,4 +93,4 @@ def test_export_config_with_keystore(self): assert exit_code == 0 assert self.log_contains(secret) == False - assert self.log_contains("${%s}" % key) +# assert self.log_contains("${%s}" % key) diff --git a/libbeat/tests/system/test_template.py b/libbeat/tests/system/test_template.py index a78681766b3..2780738344c 100644 --- a/libbeat/tests/system/test_template.py +++ b/libbeat/tests/system/test_template.py @@ -32,7 +32,7 @@ def test_index_not_modified(self): Test that beat starts running if elasticsearch output is set """ self.render_config_template( - elasticsearch={"hosts": "localhost:9200"}, + elasticsearch=self.get_elasticsearch_template_config(), ) proc = self.start_beat() @@ -74,7 +74,7 @@ def test_index_with_pattern_name(self): Test that beat starts running if elasticsearch output with modified index and pattern and name are set """ self.render_config_template( - elasticsearch={"hosts": "localhost:9200"}, + elasticsearch=self.get_elasticsearch_template_config(), es_template_name="test", es_template_pattern="test-*", ) @@ -97,7 +97,7 @@ def test_json_template(self): print(path) self.render_config_template( - elasticsearch={"hosts": self.get_host(), "username": "admin", "password": "testing"}, + elasticsearch=self.get_elasticsearch_template_config(), template_overwrite="true", template_json_enabled="true", template_json_path=path, @@ -136,7 +136,7 @@ def tearDown(self): def render_config(self, **kwargs): self.render_config_template( - elasticsearch={"hosts": self.get_elasticsearch_url(), "username": "admin", "password": "testing"}, + elasticsearch=self.get_elasticsearch_template_config(), **kwargs ) @@ -197,7 +197,7 @@ def tearDown(self): def render_config(self, **kwargs): self.render_config_template( - elasticsearch={"hosts": self.get_elasticsearch_url(), "username": "admin", "password": "testing"}, + elasticsearch=self.get_elasticsearch_template_config(), **kwargs ) diff --git a/x-pack/filebeat/tests/system/config/filebeat_modules.yml.j2 b/x-pack/filebeat/tests/system/config/filebeat_modules.yml.j2 index def6543dc25..800dbb0d46d 100644 --- a/x-pack/filebeat/tests/system/config/filebeat_modules.yml.j2 +++ b/x-pack/filebeat/tests/system/config/filebeat_modules.yml.j2 @@ -12,10 +12,10 @@ filebeat.registry: {% endif %} {%endif%} -output.elasticsearch.hosts: ["{{ elasticsearch_url }}"] +output.elasticsearch.hosts: ["{{ elasticsearch.host }}"] output.elasticsearch.index: {{ index_name }} -output.elasticsearch.username: elastic -output.elasticsearch.password: changeme +output.elasticsearch.username: {{ elasticsearch.user }} +output.elasticsearch.password: {{ elasticsearch.pass }} setup.template.name: {{ index_name }} setup.template.pattern: {{ index_name }}* From 62f7ede963a31b176bbd34a4f4d1d18a70d2c446 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Thu, 2 Dec 2021 09:58:21 +0100 Subject: [PATCH 40/52] Minor cleanup --- dev-tools/mage/pytest.go | 11 +++++++---- libbeat/dashboards/importer.go | 2 -- libbeat/dashboards/kibana_loader.go | 1 - .../logstash/logstash_integration_test.go | 18 +++++++++--------- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/dev-tools/mage/pytest.go b/dev-tools/mage/pytest.go index 340cb0f10fb..eb0d605c80c 100644 --- a/dev-tools/mage/pytest.go +++ b/dev-tools/mage/pytest.go @@ -122,7 +122,6 @@ func PythonTest(params PythonTestArgs) error { pytestEnv := map[string]string{ // activate sets this. Not sure if it's ever needed. "VIRTUAL_ENV": ve, - // "ES_PASS": "changeme", } if IsInIntegTestEnv() { pytestEnv["INTEGRATION_TESTS"] = "1" @@ -134,10 +133,14 @@ func PythonTest(params PythonTestArgs) error { pytestOptions := []string{ "--timeout=90", "--durations=20", + // Enable -x to stop at the first failing test // "-x", - "--tb=long", - "-v", - "-s", + // Enable --tb=long to produce long tracebacks + //"--tb=long", + // Enable -v to produce verbose output + //"-v", + // Don't capture test output + //"-s", } if mg.Verbose() { pytestOptions = append(pytestOptions, "-v") diff --git a/libbeat/dashboards/importer.go b/libbeat/dashboards/importer.go index a6502c566fd..b27ec695cad 100644 --- a/libbeat/dashboards/importer.go +++ b/libbeat/dashboards/importer.go @@ -125,9 +125,7 @@ func (imp Importer) ImportDir(dirType string, dir string) error { return fmt.Errorf("The directory %s is empty, nothing to import", dir) } for _, file := range files { - imp.loader.statusMsg("start file %s", file) err = imp.ImportFile(dirType, file) - imp.loader.statusMsg("finish file %s", file) if err != nil { errors = append(errors, fmt.Sprintf(" error loading %s: %s", file, err)) } diff --git a/libbeat/dashboards/kibana_loader.go b/libbeat/dashboards/kibana_loader.go index fe0aad61f03..7d0a2ef737d 100644 --- a/libbeat/dashboards/kibana_loader.go +++ b/libbeat/dashboards/kibana_loader.go @@ -112,7 +112,6 @@ func (loader KibanaLoader) ImportIndexFile(file string) error { return fmt.Errorf("fail to unmarshal the index content from file %s: %v", file, err) } - loader.statusMsg("Finished tasks for %s", file) return loader.ImportIndex(indexContent) } diff --git a/libbeat/outputs/logstash/logstash_integration_test.go b/libbeat/outputs/logstash/logstash_integration_test.go index ef5078eecc5..e76f4703e67 100644 --- a/libbeat/outputs/logstash/logstash_integration_test.go +++ b/libbeat/outputs/logstash/logstash_integration_test.go @@ -102,12 +102,14 @@ func esConnect(t *testing.T, index string) *esConnection { Timestamp: ts, }) + username := os.Getenv("ES_USER") + password := os.Getenv("ES_PASS") transport := httpcommon.DefaultHTTPTransportSettings() transport.Timeout = 60 * time.Second client, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ URL: host, - Username: "admin", - Password: "testing", + Username: username, + Password: password, Transport: transport, }) if err != nil { @@ -172,13 +174,11 @@ func newTestElasticsearchOutput(t *testing.T, test string) *testOutputer { bulkSize := 0 config, _ := common.NewConfigFrom(map[string]interface{}{ - "hosts": []string{getElasticsearchHost()}, - "index": connection.index, - "bulk_max_size": &bulkSize, - //"username": os.Getenv("ES_USER"), - //"password": os.Getenv("ES_PASS"), - "username": "admin", - "password": "testing", + "hosts": []string{getElasticsearchHost()}, + "index": connection.index, + "bulk_max_size": &bulkSize, + "username": os.Getenv("ES_USER"), + "password": os.Getenv("ES_PASS"), "template.enabled": false, }) From 81b0331e6d3d53b51cbe71a4c31044efb9cfe657 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Tue, 7 Dec 2021 13:42:10 +0100 Subject: [PATCH 41/52] Lint --- libbeat/tests/system/test_cmd_setup_index_management.py | 1 - 1 file changed, 1 deletion(-) diff --git a/libbeat/tests/system/test_cmd_setup_index_management.py b/libbeat/tests/system/test_cmd_setup_index_management.py index a79ae71f154..6c2512b2d74 100644 --- a/libbeat/tests/system/test_cmd_setup_index_management.py +++ b/libbeat/tests/system/test_cmd_setup_index_management.py @@ -33,7 +33,6 @@ def setUp(self): policies=[self.policy_name, self.custom_policy], data_streams=[self.data_stream]) - logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("elasticsearch").setLevel(logging.ERROR) From 0150043411263c22ac46d9fd281bbfc5f5b44ecc Mon Sep 17 00:00:00 2001 From: Mike Place Date: Thu, 9 Dec 2021 13:06:39 +0100 Subject: [PATCH 42/52] Update libbeat/tests/system/test_keystore.py Co-authored-by: Tiago Queiroz --- libbeat/tests/system/test_keystore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/tests/system/test_keystore.py b/libbeat/tests/system/test_keystore.py index 44fe8d1a051..f1dbd837b7c 100644 --- a/libbeat/tests/system/test_keystore.py +++ b/libbeat/tests/system/test_keystore.py @@ -48,7 +48,7 @@ def test_keystore_with_key_not_present(self): exit_code = self.run_beat() assert self.log_contains( - "missing field accessing 'output.elasticsearch.hosts") + "missing field accessing 'output.elasticsearch.hosts'") assert exit_code == 1 def test_keystore_with_nested_key(self): From 9681b4bdbba994f77586fe4f3e59c8c6f368bf30 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Thu, 9 Dec 2021 13:06:53 +0100 Subject: [PATCH 43/52] Update libbeat/tests/system/test_keystore.py Co-authored-by: Tiago Queiroz --- libbeat/tests/system/test_keystore.py | 1 - 1 file changed, 1 deletion(-) diff --git a/libbeat/tests/system/test_keystore.py b/libbeat/tests/system/test_keystore.py index f1dbd837b7c..3cfa322bfd3 100644 --- a/libbeat/tests/system/test_keystore.py +++ b/libbeat/tests/system/test_keystore.py @@ -93,4 +93,3 @@ def test_export_config_with_keystore(self): assert exit_code == 0 assert self.log_contains(secret) == False -# assert self.log_contains("${%s}" % key) From ca21c3eee483e829222a83617b010b0e9ca8e6d7 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 13 Dec 2021 11:53:53 +0100 Subject: [PATCH 44/52] Resolve merge conflict --- testing/environments/snapshot.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 6759d99da34..5e3dca1fbca 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -14,7 +14,8 @@ services: - "transport.host=127.0.0.1" - "http.host=0.0.0.0" - "xpack.security.enabled=true" - - "script.context.template.max_compilations_rate=unlimited" + # We want something as unlimited compilation rate, but 'unlimited' is not valid. + - "script.max_compilations_rate=100000/1m" - "script.context.ingest.cache_max_size=2000" - "script.context.processor_conditional.cache_max_size=2000" - "script.context.template.cache_max_size=2000" From 1c7f804f86262fa74f7fd5fdcde870b987413fa2 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 13 Dec 2021 13:50:56 +0100 Subject: [PATCH 45/52] Sync snapshot with master branch --- testing/environments/snapshot.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index f49ea21c130..33ec4cd1286 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -16,9 +16,6 @@ services: - "xpack.security.enabled=true" # We want something as unlimited compilation rate, but 'unlimited' is not valid. - "script.max_compilations_rate=100000/1m" - - "script.context.ingest.cache_max_size=2000" - - "script.context.processor_conditional.cache_max_size=2000" - - "script.context.template.cache_max_size=2000" - "action.destructive_requires_name=false" # Disable geoip updates to prevent golden file test failures when the database # changes and prevent race conditions between tests and database updates. From 6153e65f5c2545a2510b69fa812fa3adca5cafb5 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Tue, 14 Dec 2021 11:15:34 +0100 Subject: [PATCH 46/52] Fix assert for keystore test --- libbeat/tests/system/test_keystore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/tests/system/test_keystore.py b/libbeat/tests/system/test_keystore.py index 3cfa322bfd3..66295b5d34e 100644 --- a/libbeat/tests/system/test_keystore.py +++ b/libbeat/tests/system/test_keystore.py @@ -48,7 +48,7 @@ def test_keystore_with_key_not_present(self): exit_code = self.run_beat() assert self.log_contains( - "missing field accessing 'output.elasticsearch.hosts'") + "missing field accessing 'output.elasticsearch.hosts.0'") assert exit_code == 1 def test_keystore_with_nested_key(self): From 692a9f9c091a0bbc387283053f6317abf84fcd6b Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Thu, 20 Jan 2022 13:31:11 +0100 Subject: [PATCH 47/52] fix kibana config and users roles --- testing/environments/docker/elasticsearch/users_roles | 2 +- x-pack/libbeat/docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/environments/docker/elasticsearch/users_roles b/testing/environments/docker/elasticsearch/users_roles index f5e7c2d41c7..36dd721ecb5 100644 --- a/testing/environments/docker/elasticsearch/users_roles +++ b/testing/environments/docker/elasticsearch/users_roles @@ -4,7 +4,7 @@ filebeat:filebeat_user heartbeat:heartbeat_user ingest_admin:apm_server_user kibana_system:kibana_system_user -kibana_user:apm_server_user,apm_user_ro,beats,filebeat_user,heartbeat_user,metricbeat_user,auditbeat_user,journalbeat_user +kibana_admin:apm_server_user,apm_user_ro,beats,filebeat_user,heartbeat_user,metricbeat_user,auditbeat_user,journalbeat_user metricbeat:metricbeat_user auditbeat:auditbeat_user journalbeat:journalbeat_user diff --git a/x-pack/libbeat/docker-compose.yml b/x-pack/libbeat/docker-compose.yml index d89e6a30746..42fa3efa953 100644 --- a/x-pack/libbeat/docker-compose.yml +++ b/x-pack/libbeat/docker-compose.yml @@ -54,4 +54,4 @@ services: retries: 1200 interval: 5s start_period: 60s - command: /usr/local/bin/kibana-docker --xpack.security.enabled=true --elasticsearch.username=myelastic --elasticsearch.password=changeme + command: /usr/local/bin/kibana-docker From 1c759e6081418579af56e3227d2cd3f9eb9994c2 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Thu, 20 Jan 2022 17:23:15 +0100 Subject: [PATCH 48/52] fix docker-compose healthcheck --- auditbeat/docker-compose.yml | 4 ++++ filebeat/docker-compose.yml | 4 ++++ heartbeat/docker-compose.yml | 4 ++++ libbeat/docker-compose.yml | 5 +++++ packetbeat/docker-compose.yml | 4 ++++ testing/environments/snapshot.yml | 10 ++++++---- x-pack/filebeat/docker-compose.yml | 4 ++++ x-pack/functionbeat/docker-compose.yml | 4 ++++ 8 files changed, 35 insertions(+), 4 deletions(-) diff --git a/auditbeat/docker-compose.yml b/auditbeat/docker-compose.yml index fc3e701f85d..70b5fe82de9 100644 --- a/auditbeat/docker-compose.yml +++ b/auditbeat/docker-compose.yml @@ -32,6 +32,10 @@ services: extends: file: ../testing/environments/${TESTING_ENVIRONMENT}.yml service: elasticsearch + healthcheck: + test: ["CMD-SHELL", "curl -u admin:testing -s http://localhost:9200/_cat/health?h=status | grep -q green"] + retries: 300 + interval: 1s kibana: extends: diff --git a/filebeat/docker-compose.yml b/filebeat/docker-compose.yml index 19302ae1e6f..99e4d7e1318 100644 --- a/filebeat/docker-compose.yml +++ b/filebeat/docker-compose.yml @@ -40,6 +40,10 @@ services: extends: file: ${ES_BEATS}/testing/environments/${TESTING_ENVIRONMENT}.yml service: elasticsearch + healthcheck: + test: ["CMD-SHELL", "curl -u admin:testing -s http://localhost:9200/_cat/health?h=status | grep -q green"] + retries: 300 + interval: 1s kafka: build: ${ES_BEATS}/testing/environments/docker/kafka diff --git a/heartbeat/docker-compose.yml b/heartbeat/docker-compose.yml index e3374592b4e..ace731f7bbb 100644 --- a/heartbeat/docker-compose.yml +++ b/heartbeat/docker-compose.yml @@ -30,6 +30,10 @@ services: extends: file: ${ES_BEATS}/testing/environments/${TESTING_ENVIRONMENT}.yml service: elasticsearch + healthcheck: + test: ["CMD-SHELL", "curl -u admin:testing -s http://localhost:9200/_cat/health?h=status | grep -q green"] + retries: 300 + interval: 1s redis: build: ${PWD}/tests/docker_support/redis diff --git a/libbeat/docker-compose.yml b/libbeat/docker-compose.yml index 0cb37e69ae1..685b093e353 100644 --- a/libbeat/docker-compose.yml +++ b/libbeat/docker-compose.yml @@ -63,6 +63,10 @@ services: extends: file: ${ES_BEATS}/testing/environments/${TESTING_ENVIRONMENT}.yml service: elasticsearch + healthcheck: + test: ["CMD-SHELL", "curl -u admin:testing -s http://localhost:9200/_cat/health?h=status | grep -q green"] + retries: 300 + interval: 1s elasticsearch_monitoring: extends: @@ -70,6 +74,7 @@ services: service: elasticsearch healthcheck: test: ["CMD-SHELL", "curl -u admin:testing -s http://localhost:9200/_cat/health?h=status | grep -q green"] + elasticsearchssl: extends: file: ${ES_BEATS}/testing/environments/${TESTING_ENVIRONMENT}.yml diff --git a/packetbeat/docker-compose.yml b/packetbeat/docker-compose.yml index 8abfad19410..3c5d839aef2 100644 --- a/packetbeat/docker-compose.yml +++ b/packetbeat/docker-compose.yml @@ -30,6 +30,10 @@ services: extends: file: ../testing/environments/${TESTING_ENVIRONMENT}.yml service: elasticsearch + healthcheck: + test: ["CMD-SHELL", "curl -u admin:testing -s http://localhost:9200/_cat/health?h=status | grep -q green"] + retries: 300 + interval: 1s kibana: extends: diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 4c9b7ab4027..cc17be448a0 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -4,10 +4,12 @@ version: '2.3' services: elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:8.1.0-f5a18001-SNAPSHOT - healthcheck: - test: ["CMD-SHELL", "curl -u admin:testing -s http://localhost:9200/_cat/health?h=status | grep -q green"] - retries: 300 - interval: 1s + # When extend is used it merges healthcheck.tests, see: + # https://github.com/docker/compose/issues/8962 + # healthcheck: + # test: ["CMD-SHELL", "curl -u admin:testing -s http://localhost:9200/_cat/health?h=status | grep -q green"] + # retries: 300 + # interval: 1s environment: - "ES_JAVA_OPTS=-Xms1g -Xmx1g" - "network.host=" diff --git a/x-pack/filebeat/docker-compose.yml b/x-pack/filebeat/docker-compose.yml index 3224c88bc18..c53bb1ca983 100644 --- a/x-pack/filebeat/docker-compose.yml +++ b/x-pack/filebeat/docker-compose.yml @@ -28,4 +28,8 @@ services: extends: file: ${ES_BEATS}/testing/environments/${STACK_ENVIRONMENT}.yml service: elasticsearch + healthcheck: + test: ["CMD-SHELL", "curl -u admin:testing -s http://localhost:9200/_cat/health?h=status | grep -q green"] + retries: 300 + interval: 1s diff --git a/x-pack/functionbeat/docker-compose.yml b/x-pack/functionbeat/docker-compose.yml index e49a7cdac29..aa6cc364a7e 100644 --- a/x-pack/functionbeat/docker-compose.yml +++ b/x-pack/functionbeat/docker-compose.yml @@ -22,3 +22,7 @@ services: extends: file: ${ES_BEATS}/testing/environments/${TESTING_ENVIRONMENT}.yml service: elasticsearch + healthcheck: + test: ["CMD-SHELL", "curl -u admin:testing -s http://localhost:9200/_cat/health?h=status | grep -q green"] + retries: 300 + interval: 1s From 361908e80762bf49393a4c0b6b6ed7f722c5dfe5 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Thu, 20 Jan 2022 18:14:54 +0100 Subject: [PATCH 49/52] fix merge issues --- libbeat/tests/system/config/mockbeat.yml.j2 | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/libbeat/tests/system/config/mockbeat.yml.j2 b/libbeat/tests/system/config/mockbeat.yml.j2 index 63319991de8..f56162b3acb 100644 --- a/libbeat/tests/system/config/mockbeat.yml.j2 +++ b/libbeat/tests/system/config/mockbeat.yml.j2 @@ -61,13 +61,9 @@ output: {% if elasticsearch.ssl_ca_sha256 %} ssl.ca_sha256: {{ elasticsearch.ssl_ca_sha256 }} {% endif %} - # merge conflict - {% for k, v in elasticsearch.items() -%} - {{ k }}: {{ v }} - {% endfor -%} - # older versions have to be allowed because mockbeat is on v9.9.9 - allow_older_versions: true - # merge conflict + + # older versions have to be allowed because mockbeat is on v9.9.9 + allow_older_versions: true {%- endif %} From af6442fef4e84b97c351147972a3866d8d7898ae Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Mon, 24 Jan 2022 20:19:51 +0100 Subject: [PATCH 50/52] fix yml identation --- libbeat/tests/system/config/mockbeat.yml.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libbeat/tests/system/config/mockbeat.yml.j2 b/libbeat/tests/system/config/mockbeat.yml.j2 index f56162b3acb..9e1c21f488a 100644 --- a/libbeat/tests/system/config/mockbeat.yml.j2 +++ b/libbeat/tests/system/config/mockbeat.yml.j2 @@ -62,8 +62,8 @@ output: ssl.ca_sha256: {{ elasticsearch.ssl_ca_sha256 }} {% endif %} - # older versions have to be allowed because mockbeat is on v9.9.9 - allow_older_versions: true + # older versions have to be allowed because mockbeat is on v9.9.9 + allow_older_versions: true {%- endif %} From 862abd2e35a966e5f11c37b2222f5a172b2d3b2c Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Tue, 25 Jan 2022 13:19:41 +0100 Subject: [PATCH 51/52] fix: libbeat and x-pack/libbeat integTests --- libbeat/docker-compose.yml | 3 +++ testing/environments/snapshot.yml | 4 ---- x-pack/libbeat/docker-compose.yml | 4 ++-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/libbeat/docker-compose.yml b/libbeat/docker-compose.yml index 685b093e353..73c8f1e2dfe 100644 --- a/libbeat/docker-compose.yml +++ b/libbeat/docker-compose.yml @@ -172,3 +172,6 @@ services: extends: file: ${ES_BEATS}/testing/environments/${TESTING_ENVIRONMENT}.yml service: kibana + healthcheck: + test: ["CMD-SHELL", "curl -u beats:testing -s http://localhost:5601/api/status?v8format=true | grep -q '\"overall\":{\"level\":\"available\"'"] + retries: 600 diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index cc17be448a0..4bbc0c5a16f 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -43,10 +43,6 @@ services: kibana: image: docker.elastic.co/kibana/kibana:8.1.0-f5a18001-SNAPSHOT - healthcheck: - test: ["CMD-SHELL", "curl -u beats:testing -s http://localhost:5601/api/status?v8format=true | grep -q '\"overall\":{\"level\":\"available\"'"] - retries: 600 - interval: 10s environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" diff --git a/x-pack/libbeat/docker-compose.yml b/x-pack/libbeat/docker-compose.yml index 42fa3efa953..b7c84484d58 100644 --- a/x-pack/libbeat/docker-compose.yml +++ b/x-pack/libbeat/docker-compose.yml @@ -29,7 +29,7 @@ services: file: ${ES_BEATS}/testing/environments/${TESTING_ENVIRONMENT}.yml service: elasticsearch healthcheck: - test: ["CMD-SHELL", "curl -u myelastic:changeme -f http://localhost:9200/_cat/health?h=status | grep -q green"] + test: ["CMD-SHELL", "curl -u kibana_system_user:testing -f http://localhost:9200/_cat/health?h=status | grep -q green"] retries: 1200 interval: 5s start_period: 60s @@ -50,7 +50,7 @@ services: file: ${ES_BEATS}/testing/environments/${TESTING_ENVIRONMENT}.yml service: kibana healthcheck: - test: ["CMD-SHELL", "curl -s -u myelastic:changeme -f http://localhost:5601/api/status?v8format=true | grep -q '\"overall\":{\"level\":\"available\"'"] + test: ["CMD-SHELL", "curl -s -u kibana_system_user:testing -f http://localhost:5601/api/status?v8format=true | grep -q '\"overall\":{\"level\":\"available\"'"] retries: 1200 interval: 5s start_period: 60s From 66c384a73e66fb453426a279f619579e90cac55c Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Tue, 25 Jan 2022 16:21:06 +0100 Subject: [PATCH 52/52] fix kibana healthcheck for all beats --- auditbeat/docker-compose.yml | 3 +++ filebeat/docker-compose.yml | 3 +++ packetbeat/docker-compose.yml | 3 +++ testing/environments/snapshot.yml | 5 +++++ 4 files changed, 14 insertions(+) diff --git a/auditbeat/docker-compose.yml b/auditbeat/docker-compose.yml index 70b5fe82de9..adf33888988 100644 --- a/auditbeat/docker-compose.yml +++ b/auditbeat/docker-compose.yml @@ -41,3 +41,6 @@ services: extends: file: ../testing/environments/${TESTING_ENVIRONMENT}.yml service: kibana + healthcheck: + test: ["CMD-SHELL", "curl -u beats:testing -s http://localhost:5601/api/status?v8format=true | grep -q '\"overall\":{\"level\":\"available\"'"] + retries: 600 diff --git a/filebeat/docker-compose.yml b/filebeat/docker-compose.yml index 99e4d7e1318..a73f0bc39d6 100644 --- a/filebeat/docker-compose.yml +++ b/filebeat/docker-compose.yml @@ -57,6 +57,9 @@ services: extends: file: ${ES_BEATS}/testing/environments/${TESTING_ENVIRONMENT}.yml service: kibana + healthcheck: + test: ["CMD-SHELL", "curl -u beats:testing -s http://localhost:5601/api/status?v8format=true | grep -q '\"overall\":{\"level\":\"available\"'"] + retries: 600 mosquitto: build: ${ES_BEATS}/testing/environments/docker/mosquitto diff --git a/packetbeat/docker-compose.yml b/packetbeat/docker-compose.yml index 3c5d839aef2..038d3e37450 100644 --- a/packetbeat/docker-compose.yml +++ b/packetbeat/docker-compose.yml @@ -39,3 +39,6 @@ services: extends: file: ../testing/environments/${TESTING_ENVIRONMENT}.yml service: kibana + healthcheck: + test: ["CMD-SHELL", "curl -u beats:testing -s http://localhost:5601/api/status?v8format=true | grep -q '\"overall\":{\"level\":\"available\"'"] + retries: 600 diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 4bbc0c5a16f..d335efc04fa 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -51,4 +51,9 @@ services: # - "XPACK_XPACK_MAIN_TELEMETRY_ENABLED=false" - "XPACK_REPORTING_ENCRYPTIONKEY=xCyqJUFqrUJJKxjZVGfnhrRkyqqaKeAG" - "LOGGING_ROOT_LEVEL=all" + # When extend is used it merges healthcheck.tests, see: + # https://github.com/docker/compose/issues/8962 + # healthcheck: + # test: ["CMD-SHELL", "curl -u beats:testing -s http://localhost:5601/api/status?v8format=true | grep -q '\"overall\":{\"level\":\"available\"'"] + # retries: 600