diff --git a/README.md b/README.md index 4665f5ab..b401cb14 100644 --- a/README.md +++ b/README.md @@ -35,21 +35,27 @@ collections: You will need the following Ansible collections installed -* community.general (probably already present) +* `community.general` (probably already present) + +You will need these packages / libraries installed. Some very basic packages like `openssl` get handled by the collection if needed. The following list contains packages and libraries which only apply to special cases or need for you to decide on the installation method. + +* `passlib` Python library if you do not disable password hashing for logstash user and you want to use logstash role from this collection. It should be installed with pip on the Ansible controller. You may want the following Ansible roles installed. There other ways to achieve what they are doing but using them is easy and convenient. -* geerlingguy.redis -* openssl if you want to use Elastic Security +* `geerlingguy.redis` if you want to use logstash role ### Supported systems We test the collection on the following Linux distributions. Each one with Elastic Stack 7 and 8. +* Rocky Linux 9 * Rocky Linux 8 -* Ubuntu 20.04 LTS * Ubuntu 22.04 LTS +* Ubuntu 20.04 LTS * Debian 11 +* Debian 10 +* CentOS 8 We know from personal experience, that the collections work in following combinations. Missing tests mostly come from incompatibilties between the distribution and our testing environment, not from problems with the collection itself. @@ -57,9 +63,6 @@ We know from personal experience, that the collections work in following combina ### Known Issues -There are known issues with the following Linux distributions. - -* Rocky Linux 9: The GnuPG key used by Elastic seems to be incompatible with this version of Rocky. ## Usage diff --git a/docs/role-beats.md b/docs/role-beats.md index 2f45e14f..73722a6c 100644 --- a/docs/role-beats.md +++ b/docs/role-beats.md @@ -10,7 +10,6 @@ Requirements You need to have the beats you want to install available in your software repositories. We provide a [role](./role-repos.md) for just that but if you have other ways of managing software, just make sure it's available. Alternatively you can install the Beats yourself. -* `cryptography` >= 2.5 * `community.crypto` collection: ansible-galaxy collection install community.crypto Role Variables @@ -87,7 +86,6 @@ beats_filebeat_journald_inputs: * *beats_loglevel*: Level of logging (for all beats) (Default: `info`) * *beats_logpath*: If logging to file, where to put logfiles (Default: `/var/log/beats`) * *beats_fields*: Fields that are added to every input in the configuration -* *beats_manage_unzip*: Install `unzip` via package manager (Default: `true`) The following variables only apply if you use this role together with our other Elastic Stack roles. diff --git a/docs/role-elasticsearch.md b/docs/role-elasticsearch.md index 2db6c39b..bfcdf3d1 100644 --- a/docs/role-elasticsearch.md +++ b/docs/role-elasticsearch.md @@ -9,11 +9,6 @@ If you use the role to set up security you, can use its CA to create certificate Please note that setting `elasticsearch_bootstrap_pw` as variable will only take effect when initialising Elasticsearch. Changes after starting elasticsearch for the first time will not change the bootstrap password for the instance and will lead to breaking tests. -Requirements ------------- - -* `cryptography` >= 2.5 - Role Variables -------------- diff --git a/docs/role-kibana.md b/docs/role-kibana.md index 28abf012..07d17653 100644 --- a/docs/role-kibana.md +++ b/docs/role-kibana.md @@ -5,11 +5,6 @@ Ansible Role: Kibana This roles installs and configures Kibana. -Requirements ------------- - -* `cryptography` >= 2.5 - Role Variables -------------- diff --git a/docs/role-logstash.md b/docs/role-logstash.md index 010dff1a..f489a25d 100644 --- a/docs/role-logstash.md +++ b/docs/role-logstash.md @@ -19,7 +19,10 @@ Requirements ------------ * `community.general` collection -* `cryptography` >= 2.5 + +You will need these packages / libraries installed. Some very basic packages like `openssl` get handled by the collection if needed. The following list contains packages and libraries which only apply to special cases or need for you to decide on the installation method. + +* `passlib` Python library if you do not disable password hashing for logstash user. It should be installed with pip on the Ansible controller. You need to have the Elastic Repos configured on your system. You can use our [role](./role-repos.md) @@ -69,7 +72,7 @@ Aside from `logstash.yml` we can manage Logstashs pipelines. * *logstash_password_hash*: Generate and use a hash from your `logstash_password` (default: `true`) * *logstash_password_hash_algorithm*: Password hashing algorithms. Value must be same as `xpack.security.authc.password_hashing.algorithm` (default: `bcrypt`) * *logstash_password_salt_length*: base64 encoded Salt character lenght. This value must be integer and must be compatible to the selected password hashing algorithms (default: `22`) -**logstash_password_hash_salt_seed*: A seed to generate random but idempotent salt on the elasticstack ca host. The salt will be used to create idempotent logstash hashed user password (default: `SeedChangeMe`) +* *logstash_password_hash_salt_seed*: A seed to generate random but idempotent salt on the elasticstack ca host. The salt will be used to create idempotent logstash hashed user password (default: `SeedChangeMe`) * *logstash_password*: Password of Elasticsearch user. It must be at least 6 characters long (default: `password`) * *logstash_user_indices*: Indices the user has access to (default: `'"ecs-logstash*", "logstash*", "logs*"'`) * *logstash_reset_writer_role*: Reset user and role with every run: (default: `true`) diff --git a/molecule/elasticsearch_cluster-oss/molecule.yml b/molecule/elasticsearch_cluster-oss/molecule.yml index 56dd07f2..50f31e05 100644 --- a/molecule/elasticsearch_cluster-oss/molecule.yml +++ b/molecule/elasticsearch_cluster-oss/molecule.yml @@ -1,6 +1,8 @@ --- dependency: name: galaxy + options: + requirements-file: requirements.yml driver: name: docker platforms: diff --git a/molecule/elasticsearch_default/molecule.yml b/molecule/elasticsearch_default/molecule.yml index 0b2fca9d..25c5022b 100644 --- a/molecule/elasticsearch_default/molecule.yml +++ b/molecule/elasticsearch_default/molecule.yml @@ -1,6 +1,8 @@ --- dependency: name: galaxy + options: + requirements-file: requirements.yml driver: name: docker platforms: diff --git a/molecule/elasticsearch_no-security/molecule.yml b/molecule/elasticsearch_no-security/molecule.yml index 83ba94da..9855d8e2 100644 --- a/molecule/elasticsearch_no-security/molecule.yml +++ b/molecule/elasticsearch_no-security/molecule.yml @@ -1,6 +1,8 @@ --- dependency: name: galaxy + options: + requirements-file: requirements.yml driver: name: docker platforms: diff --git a/molecule/elasticsearch_roles_calculation/molecule.yml b/molecule/elasticsearch_roles_calculation/molecule.yml index 7f5a962d..6e7d35a3 100644 --- a/molecule/elasticsearch_roles_calculation/molecule.yml +++ b/molecule/elasticsearch_roles_calculation/molecule.yml @@ -1,6 +1,8 @@ --- dependency: name: galaxy + options: + requirements-file: requirements.yml driver: name: docker platforms: diff --git a/molecule/elasticstack_default/converge.yml b/molecule/elasticstack_default/converge.yml index 8cde64e3..718dc256 100644 --- a/molecule/elasticstack_default/converge.yml +++ b/molecule/elasticstack_default/converge.yml @@ -40,6 +40,9 @@ - name: Include logstash ansible.builtin.include_role: name: logstash + - name: Include kibana + ansible.builtin.include_role: + name: kibana - name: Include Beats ansible.builtin.include_role: name: beats @@ -54,9 +57,3 @@ ansible.builtin.service: name: rsyslog state: started - - name: Include kibana - ansible.builtin.include_role: - name: kibana - - name: Include Beats - ansible.builtin.include_role: - name: beats diff --git a/molecule/elasticstack_default/molecule.yml b/molecule/elasticstack_default/molecule.yml index f92ae58b..d658c84d 100644 --- a/molecule/elasticstack_default/molecule.yml +++ b/molecule/elasticstack_default/molecule.yml @@ -1,6 +1,8 @@ --- dependency: name: galaxy + options: + requirements-file: requirements.yml driver: name: docker platforms: diff --git a/molecule/logstash_full_stack-oss/molecule.yml b/molecule/logstash_full_stack-oss/molecule.yml index e053a2f7..ebd40da8 100644 --- a/molecule/logstash_full_stack-oss/molecule.yml +++ b/molecule/logstash_full_stack-oss/molecule.yml @@ -1,6 +1,8 @@ --- dependency: name: galaxy + options: + requirements-file: requirements.yml driver: name: docker platforms: diff --git a/molecule/logstash_pipelines/molecule.yml b/molecule/logstash_pipelines/molecule.yml index 204e1b92..e2627f02 100644 --- a/molecule/logstash_pipelines/molecule.yml +++ b/molecule/logstash_pipelines/molecule.yml @@ -1,6 +1,8 @@ --- dependency: name: galaxy + options: + requirements-file: requirements.yml driver: name: docker platforms: diff --git a/molecule/logstash_specific_version/molecule.yml b/molecule/logstash_specific_version/molecule.yml index a4024619..5b4cf3c0 100644 --- a/molecule/logstash_specific_version/molecule.yml +++ b/molecule/logstash_specific_version/molecule.yml @@ -1,6 +1,8 @@ --- dependency: name: galaxy + options: + requirements-file: requirements.yml driver: name: docker platforms: diff --git a/molecule/repos_default/molecule.yml b/molecule/repos_default/molecule.yml index dbbe9ea0..3c857a61 100644 --- a/molecule/repos_default/molecule.yml +++ b/molecule/repos_default/molecule.yml @@ -1,6 +1,8 @@ --- dependency: name: galaxy + options: + requirements-file: requirements.yml driver: name: docker platforms: diff --git a/molecule/repos_oss/molecule.yml b/molecule/repos_oss/molecule.yml index a083fcd5..9527212b 100644 --- a/molecule/repos_oss/molecule.yml +++ b/molecule/repos_oss/molecule.yml @@ -1,6 +1,8 @@ --- dependency: name: galaxy + options: + requirements-file: requirements.yml driver: name: docker platforms: diff --git a/requirements-test.txt b/requirements-test.txt index c67fdccb..ac24d2cf 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -3,4 +3,4 @@ ansible-lint molecule molecule-plugins[docker] pytest -passlib \ No newline at end of file +passlib diff --git a/roles/beats/defaults/main.yml b/roles/beats/defaults/main.yml index 7f9550fc..5708617b 100644 --- a/roles/beats/defaults/main.yml +++ b/roles/beats/defaults/main.yml @@ -10,7 +10,6 @@ elasticstack_beats_port: 5044 beats_logging: file beats_logpath: /var/log/beats beats_loglevel: info -beats_manage_unzip: true # Use TLS without Elastic X-Pack # diff --git a/roles/beats/tasks/beats-security.yml b/roles/beats/tasks/beats-security.yml index 0a19489c..ef034ee3 100644 --- a/roles/beats/tasks/beats-security.yml +++ b/roles/beats/tasks/beats-security.yml @@ -1,11 +1,15 @@ --- -- name: Install unzip for certificate handling +- name: Install packages for security tasks ansible.builtin.package: - name: unzip - when: beats_manage_unzip | bool + name: + - unzip + - python3-cryptography + - openssl tags: + - certificates - renew_ca + - renew_kibana_cert - renew_beats_cert - name: Ensure beats certificate exists diff --git a/roles/elasticsearch/tasks/elasticsearch-keystore.yml b/roles/elasticsearch/tasks/elasticsearch-keystore.yml new file mode 100644 index 00000000..31481fc8 --- /dev/null +++ b/roles/elasticsearch/tasks/elasticsearch-keystore.yml @@ -0,0 +1,184 @@ +--- + +- name: Create keystore + ansible.builtin.command: /usr/share/elasticsearch/bin/elasticsearch-keystore create + args: + creates: /etc/elasticsearch/elasticsearch.keystore + +- name: Check for bootstrap password + ansible.builtin.command: /usr/share/elasticsearch/bin/elasticsearch-keystore list + changed_when: false + register: elasticsearch_keystore + +- name: Set bootstrap password # noqa: risky-shell-pipe + ansible.builtin.shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + echo "{{ elasticsearch_bootstrap_pw }}" | + /usr/share/elasticsearch/bin/elasticsearch-keystore + add -x 'bootstrap.password' + when: "'bootstrap.password' not in elasticsearch_keystore.stdout_lines" + changed_when: false + no_log: true + notify: + - Restart Elasticsearch + ignore_errors: "{{ ansible_check_mode }}" + +- name: Get xpack.security.http.ssl.keystore.secure_password # noqa: risky-shell-pipe + ansible.builtin.shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + /usr/share/elasticsearch/bin/elasticsearch-keystore + show 'xpack.security.http.ssl.keystore.secure_password' + when: + - "'xpack.security.http.ssl.keystore.secure_password' in elasticsearch_keystore.stdout_lines" + - elasticsearch_http_security + register: elasticsearch_http_ssl_keystore_secure_password + ignore_errors: "{{ ansible_check_mode }}" + no_log: true + changed_when: false + +- name: Set xpack.security.http.ssl.keystore.secure_password # noqa: risky-shell-pipe + ansible.builtin.shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + echo "{{ elasticsearch_tls_key_passphrase }}" | + /usr/share/elasticsearch/bin/elasticsearch-keystore + add -f -x 'xpack.security.http.ssl.keystore.secure_password' + changed_when: false + no_log: true + when: + - elasticsearch_http_ssl_keystore_secure_password.stdout is undefined or elasticsearch_tls_key_passphrase != elasticsearch_http_ssl_keystore_secure_password.stdout + - elasticsearch_http_security + notify: + - Restart Elasticsearch + +- name: Remove xpack.security.http.ssl.keystore.secure_password # noqa: risky-shell-pipe + ansible.builtin.shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + /usr/share/elasticsearch/bin/elasticsearch-keystore + remove 'xpack.security.http.ssl.keystore.secure_password' + changed_when: false + no_log: true + when: + - "'xpack.security.http.ssl.keystore.secure_password' in elasticsearch_keystore.stdout_lines" + - not elasticsearch_http_security + notify: + - Restart Elasticsearch + +- name: Get xpack.security.http.ssl.truststore.secure_password # noqa: risky-shell-pipe + ansible.builtin.shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + /usr/share/elasticsearch/bin/elasticsearch-keystore + show 'xpack.security.http.ssl.truststore.secure_password' + when: + - "'xpack.security.http.ssl.truststore.secure_password' in elasticsearch_keystore.stdout_lines" + - elasticsearch_http_security + register: elasticsearch_http_ssl_truststore_secure_password + ignore_errors: "{{ ansible_check_mode }}" + no_log: true + changed_when: false + +- name: Set xpack.security.http.ssl.truststore.secure_password # noqa: risky-shell-pipe + ansible.builtin.shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + echo "{{ elasticsearch_tls_key_passphrase }}" | + /usr/share/elasticsearch/bin/elasticsearch-keystore + add -f -x 'xpack.security.http.ssl.truststore.secure_password' + changed_when: false + no_log: true + when: + - elasticsearch_http_ssl_truststore_secure_password.stdout is undefined or elasticsearch_tls_key_passphrase != elasticsearch_http_ssl_truststore_secure_password.stdout + - elasticsearch_http_security + notify: + - Restart Elasticsearch + +- name: Remove xpack.security.http.ssl.truststore.secure_password # noqa: risky-shell-pipe + ansible.builtin.shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + /usr/share/elasticsearch/bin/elasticsearch-keystore + remove 'xpack.security.http.ssl.truststore.secure_password' + changed_when: false + no_log: true + when: + - "'xpack.security.http.ssl.truststore.secure_password' in elasticsearch_keystore.stdout_lines" + - not elasticsearch_http_security + notify: + - Restart Elasticsearch + +- name: Get xpack.security.transport.ssl.keystore.secure_password # noqa: risky-shell-pipe + ansible.builtin.shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + /usr/share/elasticsearch/bin/elasticsearch-keystore + show 'xpack.security.transport.ssl.keystore.secure_password' + when: + - "'xpack.security.transport.ssl.keystore.secure_password' in elasticsearch_keystore.stdout_lines" + - elasticsearch_security + register: elasticsearch_transport_ssl_keystore_secure_password + ignore_errors: "{{ ansible_check_mode }}" + no_log: true + changed_when: false + +- name: Set xpack.security.transport.ssl.keystore.secure_password # noqa: risky-shell-pipe + ansible.builtin.shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + echo "{{ elasticsearch_tls_key_passphrase }}" | + /usr/share/elasticsearch/bin/elasticsearch-keystore + add -f -x 'xpack.security.transport.ssl.keystore.secure_password' + changed_when: false + no_log: true + when: + - elasticsearch_transport_ssl_keystore_secure_password.stdout is undefined or elasticsearch_tls_key_passphrase != elasticsearch_transport_ssl_keystore_secure_password.stdout + - elasticsearch_security + notify: + - Restart Elasticsearch + +- name: Remove xpack.security.transport.ssl.keystore.secure_password # noqa: risky-shell-pipe + ansible.builtin.shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + /usr/share/elasticsearch/bin/elasticsearch-keystore + remove 'xpack.security.transport.ssl.keystore.secure_password' + changed_when: false + no_log: true + when: + - "'xpack.security.transport.ssl.keystore.secure_password' in elasticsearch_keystore.stdout_lines" + - not elasticsearch_security + notify: + - Restart Elasticsearch + +- name: Get xpack.security.transport.ssl.truststore.secure_password # noqa: risky-shell-pipe + ansible.builtin.shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + /usr/share/elasticsearch/bin/elasticsearch-keystore + show 'xpack.security.transport.ssl.truststore.secure_password' + when: + - "'xpack.security.transport.ssl.truststore.secure_password' in elasticsearch_keystore.stdout_lines" + - elasticsearch_security + register: elasticsearch_transport_ssl_truststore_secure_password + ignore_errors: "{{ ansible_check_mode }}" + no_log: true + changed_when: false + +- name: Set xpack.security.transport.ssl.truststore.secure_password # noqa: risky-shell-pipe + ansible.builtin.shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + echo "{{ elasticsearch_tls_key_passphrase }}" | + /usr/share/elasticsearch/bin/elasticsearch-keystore + add -f -x 'xpack.security.transport.ssl.truststore.secure_password' + changed_when: false + no_log: true + when: + - elasticsearch_transport_ssl_truststore_secure_password.stdout is undefined or elasticsearch_tls_key_passphrase != elasticsearch_transport_ssl_truststore_secure_password.stdout + - elasticsearch_security + notify: + - Restart Elasticsearch + +- name: Remove xpack.security.transport.ssl.truststore.secure_password # noqa: risky-shell-pipe + ansible.builtin.shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + /usr/share/elasticsearch/bin/elasticsearch-keystore + remove 'xpack.security.transport.ssl.truststore.secure_password' + changed_when: false + no_log: true + when: + - "'xpack.security.transport.ssl.truststore.secure_password' in elasticsearch_keystore.stdout_lines" + - not elasticsearch_security + notify: + - Restart Elasticsearch diff --git a/roles/elasticsearch/tasks/elasticsearch-security.yml b/roles/elasticsearch/tasks/elasticsearch-security.yml index 65483fc3..0b1a95ac 100644 --- a/roles/elasticsearch/tasks/elasticsearch-security.yml +++ b/roles/elasticsearch/tasks/elasticsearch-security.yml @@ -1,5 +1,17 @@ --- +- name: Install packages for security tasks + ansible.builtin.package: + name: + - unzip + - python3-cryptography + - openssl + tags: + - certificates + - renew_ca + - renew_kibana_cert + - renew_es_cert + - name: Set elasticstack_ca variable if not already done by user ansible.builtin.set_fact: elasticstack_ca: "{{ groups['elasticsearch'][0] }}" @@ -222,188 +234,8 @@ - renew_ca - renew_es_cert -- name: Create keystore - ansible.builtin.command: /usr/share/elasticsearch/bin/elasticsearch-keystore create - args: - creates: /etc/elasticsearch/elasticsearch.keystore - -- name: Check for bootstrap password - ansible.builtin.command: /usr/share/elasticsearch/bin/elasticsearch-keystore list - changed_when: false - register: elasticsearch_keystore - -- name: Set bootstrap password # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - echo "{{ elasticsearch_bootstrap_pw }}" | - /usr/share/elasticsearch/bin/elasticsearch-keystore - add -x 'bootstrap.password' - when: "'bootstrap.password' not in elasticsearch_keystore.stdout_lines" - changed_when: true - no_log: "{{ elasticstack_no_log }}" - notify: - - Restart Elasticsearch - ignore_errors: "{{ ansible_check_mode }}" - -- name: Get xpack.security.http.ssl.keystore.secure_password # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - /usr/share/elasticsearch/bin/elasticsearch-keystore - show 'xpack.security.http.ssl.keystore.secure_password' - when: - - "'xpack.security.http.ssl.keystore.secure_password' in elasticsearch_keystore.stdout_lines" - - elasticsearch_http_security - register: elasticsearch_http_ssl_keystore_secure_password - ignore_errors: "{{ ansible_check_mode }}" - no_log: "{{ elasticstack_no_log }}" - changed_when: false - -- name: Set xpack.security.http.ssl.keystore.secure_password # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - echo "{{ elasticsearch_tls_key_passphrase }}" | - /usr/share/elasticsearch/bin/elasticsearch-keystore - add -f -x 'xpack.security.http.ssl.keystore.secure_password' - changed_when: true - no_log: "{{ elasticstack_no_log }}" - when: - - elasticsearch_http_ssl_keystore_secure_password.stdout is undefined or elasticsearch_tls_key_passphrase != elasticsearch_http_ssl_keystore_secure_password.stdout - - elasticsearch_http_security - notify: - - Restart Elasticsearch - -- name: Remove xpack.security.http.ssl.keystore.secure_password # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - /usr/share/elasticsearch/bin/elasticsearch-keystore - remove 'xpack.security.http.ssl.keystore.secure_password' - changed_when: true - no_log: "{{ elasticstack_no_log }}" - when: - - "'xpack.security.http.ssl.keystore.secure_password' in elasticsearch_keystore.stdout_lines" - - not elasticsearch_http_security - notify: - - Restart Elasticsearch - -- name: Get xpack.security.http.ssl.truststore.secure_password # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - /usr/share/elasticsearch/bin/elasticsearch-keystore - show 'xpack.security.http.ssl.truststore.secure_password' - when: - - "'xpack.security.http.ssl.truststore.secure_password' in elasticsearch_keystore.stdout_lines" - - elasticsearch_http_security - register: elasticsearch_http_ssl_truststore_secure_password - ignore_errors: "{{ ansible_check_mode }}" - no_log: "{{ elasticstack_no_log }}" - changed_when: false - -- name: Set xpack.security.http.ssl.truststore.secure_password # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - echo "{{ elasticsearch_tls_key_passphrase }}" | - /usr/share/elasticsearch/bin/elasticsearch-keystore - add -f -x 'xpack.security.http.ssl.truststore.secure_password' - changed_when: true - no_log: "{{ elasticstack_no_log }}" - when: - - elasticsearch_http_ssl_truststore_secure_password.stdout is undefined or elasticsearch_tls_key_passphrase != elasticsearch_http_ssl_truststore_secure_password.stdout - - elasticsearch_http_security - notify: - - Restart Elasticsearch - -- name: Remove xpack.security.http.ssl.truststore.secure_password # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - /usr/share/elasticsearch/bin/elasticsearch-keystore - remove 'xpack.security.http.ssl.truststore.secure_password' - changed_when: true - no_log: "{{ elasticstack_no_log }}" - when: - - "'xpack.security.http.ssl.truststore.secure_password' in elasticsearch_keystore.stdout_lines" - - not elasticsearch_http_security - notify: - - Restart Elasticsearch - -- name: Get xpack.security.transport.ssl.keystore.secure_password # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - /usr/share/elasticsearch/bin/elasticsearch-keystore - show 'xpack.security.transport.ssl.keystore.secure_password' - when: - - "'xpack.security.transport.ssl.keystore.secure_password' in elasticsearch_keystore.stdout_lines" - - elasticsearch_security - register: elasticsearch_transport_ssl_keystore_secure_password - ignore_errors: "{{ ansible_check_mode }}" - no_log: "{{ elasticstack_no_log }}" - changed_when: false - -- name: Set xpack.security.transport.ssl.keystore.secure_password # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - echo "{{ elasticsearch_tls_key_passphrase }}" | - /usr/share/elasticsearch/bin/elasticsearch-keystore - add -f -x 'xpack.security.transport.ssl.keystore.secure_password' - changed_when: true - no_log: "{{ elasticstack_no_log }}" - when: - - elasticsearch_transport_ssl_keystore_secure_password.stdout is undefined or elasticsearch_tls_key_passphrase != elasticsearch_transport_ssl_keystore_secure_password.stdout - - elasticsearch_security - notify: - - Restart Elasticsearch - -- name: Remove xpack.security.transport.ssl.keystore.secure_password # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - /usr/share/elasticsearch/bin/elasticsearch-keystore - remove 'xpack.security.transport.ssl.keystore.secure_password' - changed_when: true - no_log: "{{ elasticstack_no_log }}" - when: - - "'xpack.security.transport.ssl.keystore.secure_password' in elasticsearch_keystore.stdout_lines" - - not elasticsearch_security - notify: - - Restart Elasticsearch - -- name: Get xpack.security.transport.ssl.truststore.secure_password # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - /usr/share/elasticsearch/bin/elasticsearch-keystore - show 'xpack.security.transport.ssl.truststore.secure_password' - when: - - "'xpack.security.transport.ssl.truststore.secure_password' in elasticsearch_keystore.stdout_lines" - - elasticsearch_security - register: elasticsearch_transport_ssl_truststore_secure_password - ignore_errors: "{{ ansible_check_mode }}" - no_log: "{{ elasticstack_no_log }}" - changed_when: false - -- name: Set xpack.security.transport.ssl.truststore.secure_password # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - echo "{{ elasticsearch_tls_key_passphrase }}" | - /usr/share/elasticsearch/bin/elasticsearch-keystore - add -f -x 'xpack.security.transport.ssl.truststore.secure_password' - changed_when: true - no_log: "{{ elasticstack_no_log }}" - when: - - elasticsearch_transport_ssl_truststore_secure_password.stdout is undefined or elasticsearch_tls_key_passphrase != elasticsearch_transport_ssl_truststore_secure_password.stdout - - elasticsearch_security - notify: - - Restart Elasticsearch - -- name: Remove xpack.security.transport.ssl.truststore.secure_password # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - /usr/share/elasticsearch/bin/elasticsearch-keystore - remove 'xpack.security.transport.ssl.truststore.secure_password' - changed_when: true - no_log: "{{ elasticstack_no_log }}" - when: - - "'xpack.security.transport.ssl.truststore.secure_password' in elasticsearch_keystore.stdout_lines" - - not elasticsearch_security - notify: - - Restart Elasticsearch +- name: Import Tasks elasticsearch-keystore.yml + ansible.builtin.import_tasks: elasticsearch-keystore.yml - name: Create ca and certificates on elasticstack_ca host when: inventory_hostname == elasticstack_ca @@ -551,20 +383,19 @@ elasticsearch_http_protocol: "https" when: elasticsearch_http_security -- name: Check for cluster status with bootstrap password # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - curl -ks - {{ elasticsearch_http_protocol }}://elastic:{{ elasticsearch_bootstrap_pw }}@localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health?pretty | - grep status | - cut -d\" -f4 +- name: Check for cluster status with bootstrap password + ansible.builtin.uri: + url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health?pretty" + user: elastic + password: "{{ elasticsearch_bootstrap_pw }}" + validate_certs: false register: elasticsearch_cluster_status_bootstrap changed_when: false no_log: "{{ elasticstack_no_log }}" when: - not elasticsearch_passwords_file.stat.exists | bool - groups['elasticsearch'] | length > 1 - until: elasticsearch_cluster_status_bootstrap.stdout == "green" + until: elasticsearch_cluster_status_bootstrap.json.status == "green" retries: 5 delay: 10 @@ -579,20 +410,19 @@ delegate_to: "{{ elasticstack_ca }}" when: elasticsearch_passwords_file.stat.exists | bool -- name: Check for cluster status with elastic password # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - curl -ks - {{ elasticsearch_http_protocol }}://elastic:{{ elasticstack_password.stdout }}@localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health?pretty | - grep status | - cut -d\" -f4 +- name: Check for cluster status with elastic password + ansible.builtin.uri: + url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health?pretty" + user: elastic + password: "{{ elasticstack_password.stdout }}" + validate_certs: false register: elasticsearch_cluster_status changed_when: false no_log: "{{ elasticstack_no_log }}" when: - elasticsearch_passwords_file.stat.exists | bool - groups['elasticsearch'] | length > 1 - until: elasticsearch_cluster_status.stdout == "green" + until: elasticsearch_cluster_status.json.status == "green" retries: 20 delay: 10 diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml index b2517ab3..e783665d 100644 --- a/roles/elasticsearch/tasks/main.yml +++ b/roles/elasticsearch/tasks/main.yml @@ -183,7 +183,7 @@ - name: Force systemd to reread configs on container ansible.builtin.systemd: daemon_reload: true - when: ansible_virtualization_type == "container" + when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker" # Free up some space to let elsticsearch allocate replica in GitHub Action - name: Remove cache # noqa: risky-shell-pipe @@ -192,7 +192,7 @@ rm -rf /var/cache/* failed_when: false changed_when: false - when: ansible_virtualization_type == "container" + when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker" - name: Import Tasks elasticsearch-security.yml ansible.builtin.import_tasks: elasticsearch-security.yml @@ -213,20 +213,15 @@ - name: Handle cluster setup without security when: not elasticsearch_security | bool block: - - name: Check for cluster status without security # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - curl -s - http://localhost:{{ elastic_elasticsearch_http_port }}/_cluster/health?pretty | - grep status | - cut -d\" -f4 - register: es_cluster_status - changed_when: false - no_log: "{{ elasticstack_no_log }}" + - name: Check for cluster status without security + ansible.builtin.uri: + url: "http://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health?pretty" + register: elasticsearch_cluster_status ignore_errors: "{{ ansible_check_mode }}" - until: es_cluster_status.stdout == "green" + until: elasticsearch_cluster_status.json.status == "green" retries: 5 delay: 10 + no_log: "{{ elasticstack_no_log }}" - name: Leave a file showing that the cluster is set up ansible.builtin.template: diff --git a/roles/elasticsearch/templates/elasticsearch.yml.j2 b/roles/elasticsearch/templates/elasticsearch.yml.j2 index b71c3ee8..c809f309 100644 --- a/roles/elasticsearch/templates/elasticsearch.yml.j2 +++ b/roles/elasticsearch/templates/elasticsearch.yml.j2 @@ -62,6 +62,9 @@ xpack.security.http.ssl.keystore.path: certs/{{ ansible_hostname }}.p12 xpack.security.http.ssl.truststore.path: certs/{{ ansible_hostname }}.p12 {% endif %} {% endif %} +{% if not elasticsearch_security | bool %} +xpack.security.enabled: false +{% endif %} {% endif %} {% if elasticsearch_fs_repo is defined %} diff --git a/roles/kibana/tasks/kibana-security.yml b/roles/kibana/tasks/kibana-security.yml index d7058afb..4bb14fbd 100644 --- a/roles/kibana/tasks/kibana-security.yml +++ b/roles/kibana/tasks/kibana-security.yml @@ -1,8 +1,11 @@ --- -- name: Make sure openssl is installed +- name: Install packages for security tasks ansible.builtin.package: - name: openssl + name: + - unzip + - python3-cryptography + - openssl tags: - certificates - renew_ca diff --git a/roles/logstash/defaults/main.yml b/roles/logstash/defaults/main.yml index 27f8a16d..8ff97804 100644 --- a/roles/logstash/defaults/main.yml +++ b/roles/logstash/defaults/main.yml @@ -47,6 +47,7 @@ logstash_sniffing: false # logstash security logstash_password_hash: true logstash_password_hash_algorithm: bcrypt +logstash_password_salt_length: 22 logstash_user: logstash_writer logstash_password: password logstash_password_hash_salt_length: 22 diff --git a/roles/logstash/tasks/logstash-security.yml b/roles/logstash/tasks/logstash-security.yml index f0cae832..5a93e9d9 100644 --- a/roles/logstash/tasks/logstash-security.yml +++ b/roles/logstash/tasks/logstash-security.yml @@ -1,8 +1,11 @@ --- -- name: Install unzip for certificate handling +- name: Install packages for security tasks ansible.builtin.package: - name: unzip + name: + - unzip + - python3-cryptography + - openssl tags: - certificates - renew_ca @@ -366,54 +369,16 @@ - renew_ca - renew_logstash_cert -- name: Place logstash_writer role configuration on ca node - ansible.builtin.template: - dest: /root/logstash_writer_role - src: logstash_writer_role.j2 - owner: root - group: root - mode: 0600 - delegate_to: "{{ elasticstack_ca }}" - run_once: true - - name: Check the length of logstash user password ansible.builtin.fail: msg: logstash user password must be at least 6 characters long. when: logstash_password | length < 6 -- name: Create logstash password hash salt - ansible.builtin.copy: - content: "{{ lookup('password', '/dev/null', chars=['ascii_lowercase', 'digits'], length=logstash_password_hash_salt_length, seed=logstash_password_hash_salt_seed)}}" - dest: /root/logstash_password_hash_salt - owner: root - group: root - mode: 0600 - delegate_to: "{{ elasticstack_ca }}" - when: logstash_password_hash | bool and inventory_hostname == elasticstack_ca - -- name: Read password hash salt - ansible.builtin.slurp: - src: /root/logstash_password_hash_salt - register: logstash_password_hash_salt_file - delegate_to: "{{ elasticstack_ca }}" - when: logstash_password_hash | bool and inventory_hostname == elasticstack_ca - - name: Set password hash salt as a fact ansible.builtin.set_fact: - logstash_password_hash_salt: "{{ logstash_password_hash_salt_file['content'] | b64decode }}" - delegate_to: "{{ elasticstack_ca }}" + logstash_password_hash_salt: "{{ lookup('password', '/dev/null', chars=['ascii_lowercase', 'digits'], length=logstash_password_hash_salt_length, seed=logstash_password_hash_salt_seed) }}" when: logstash_password_hash | bool and inventory_hostname == elasticstack_ca -- name: Place logstash_writer user configuration on ca node - ansible.builtin.template: - dest: /root/logstash_writer_user - src: logstash_writer_user.j2 - owner: root - group: root - mode: 0600 - delegate_to: "{{ elasticstack_ca }}" - run_once: true - - name: Fetch Elastic password # noqa: risky-shell-pipe ansible.builtin.shell: > if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; @@ -427,94 +392,81 @@ - configuration - logstash_configuration -- name: Check for logstash_writer role # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - curl --cacert {{ elasticstack_ca_dir }}/ca.crt - -u elastic:{{ logstash_elasticstack_password.stdout }} - https://{{ hostvars[groups['elasticsearch'][0]].ansible_default_ipv4.address }}:{{ elasticstack_elasticsearch_http_port }}/_security/role/logstash_writer - | grep "enabled" - delegate_to: "{{ elasticstack_ca }}" - failed_when: false - changed_when: false - register: logstash_writer_role_present - no_log: "{{ elasticstack_no_log }}" - run_once: true - -# we doubled the task and didn't use a more sophisticated way to just change -# the URI because we expect this task to be removed when ES 7 is out of -# support +- name: Set elasticsearch security-api base url for elasticsearch > 7 + ansible.builtin.set_fact: + security_api_base_url: "https://{{ hostvars[elasticstack_ca].ansible_default_ipv4.address }}:{{ elasticstack_elasticsearch_http_port }}/_security/" + when: elasticstack_release | int > 7 -- name: Put logstash_writer role into Elasticsearch < 8 - ansible.builtin.command: > - curl -T /root/logstash_writer_role --header 'Content-Type: application/json' - --cacert {{ elasticstack_ca_dir }}/ca.crt - -u elastic:{{ logstash_elasticstack_password.stdout }} - https://{{ hostvars[groups['elasticsearch'][0]].ansible_default_ipv4.address }}:{{ elasticstack_elasticsearch_http_port }}/_xpack/security/role/logstash_writer +- name: Set elasticsearch security-api base url for elasticsearch < 8 + ansible.builtin.set_fact: + security_api_base_url: "https://{{ hostvars[elasticstack_ca].ansible_default_ipv4.address }}:{{ elasticstack_elasticsearch_http_port }}/_xpack/security/" + when: elasticstack_release | int < 8 + +- name: Check for logstash_writer role + ansible.builtin.uri: + url: "{{ security_api_base_url }}role/logstash_writer" + ca_path: "{{ elasticstack_ca_dir }}/ca.crt" + user: elastic + password: "{{ logstash_elasticstack_password.stdout }}" + register: check_logstash_writer_role_response delegate_to: "{{ elasticstack_ca }}" + failed_when: false changed_when: false no_log: "{{ elasticstack_no_log }}" run_once: true - when: - - logstash_writer_role_present.rc > 0 or logstash_reset_writer_role | bool - - elasticstack_release | int < 8 -- name: Put logstash_writer role into Elasticsearch > 7 - ansible.builtin.command: > - curl -T /root/logstash_writer_role --header 'Content-Type: application/json' - --cacert {{ elasticstack_ca_dir }}/ca.crt - -u elastic:{{ logstash_elasticstack_password.stdout }} - https://{{ hostvars[groups['elasticsearch'][0]].ansible_default_ipv4.address }}:{{ elasticstack_elasticsearch_http_port }}/_security/role/logstash_writer +- name: Set logstash_writer_role_present + ansible.builtin.set_fact: + logstash_writer_role_present: true + when: check_logstash_writer_role_response.json.logstash_writer is defined + +- name: Put logstash_writer role into Elasticsearch if not present + ansible.builtin.uri: + url: "{{ security_api_base_url }}role/logstash_writer" + ca_path: "{{ elasticstack_ca_dir }}/ca.crt" + user: elastic + password: "{{ logstash_elasticstack_password.stdout }}" + method: PUT + headers: + Content-Type: application/json + body: "{{ lookup('template', 'logstash_writer_role.j2') }}" + body_format: json + register: put_logstash_writer_role_response + when: logstash_writer_role_present is not defined delegate_to: "{{ elasticstack_ca }}" - changed_when: false + failed_when: not put_logstash_writer_role_response.json.role.created | bool run_once: true - no_log: "{{ elasticstack_no_log }}" - when: - - logstash_writer_role_present.rc > 0 or logstash_reset_writer_role | bool - - elasticstack_release | int > 7 -- name: Check for logstash_writer user # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - curl --cacert {{ elasticstack_ca_dir }}/ca.crt - -u elastic:{{ logstash_elasticstack_password.stdout }} - https://{{ hostvars[groups['elasticsearch'][0]].ansible_default_ipv4.address }}:{{ elasticstack_elasticsearch_http_port }}/_security/user/{{ logstash_user }} - | grep "enabled" +- name: Check for logstash_writer user + ansible.builtin.uri: + url: "{{ security_api_base_url }}user/{{ logstash_user }}" + ca_path: "{{ elasticstack_ca_dir }}/ca.crt" + user: elastic + password: "{{ logstash_elasticstack_password.stdout }}" + register: check_logstash_writer_user_response delegate_to: "{{ elasticstack_ca }}" failed_when: false changed_when: false - no_log: "{{ elasticstack_no_log }}" - register: logstash_writer_user_present run_once: true -# we doubled the task and didn't use a more sophisticated way to just change -# the URI because we expect this task to be removed when ES 7 is out of -# support - -- name: Put logstash_writer user into Elasticsearch < 8 - ansible.builtin.command: > - curl -T /root/logstash_writer_user --header 'Content-Type: application/json' - --cacert {{ elasticstack_ca_dir }}/ca.crt - -u elastic:{{ logstash_elasticstack_password.stdout }} - https://{{ hostvars[groups['elasticsearch'][0]].ansible_default_ipv4.address }}:{{ elasticstack_elasticsearch_http_port }}/_xpack/security/user/{{ logstash_user }} - delegate_to: "{{ elasticstack_ca }}" - changed_when: false - no_log: "{{ elasticstack_no_log }}" - run_once: true - when: - - logstash_writer_user_present.rc > 0 - - elasticstack_release | int < 8 - -- name: Put logstash_writer user into Elasticsearch > 7 - ansible.builtin.command: > - curl -T /root/logstash_writer_user --header 'Content-Type: application/json' - --cacert {{ elasticstack_ca_dir }}/ca.crt - -u elastic:{{ logstash_elasticstack_password.stdout }} - https://{{ hostvars[groups['elasticsearch'][0]].ansible_default_ipv4.address }}:{{ elasticstack_elasticsearch_http_port }}/_security/user/{{ logstash_user }} +- name: Set logstash_writer_user_present + ansible.builtin.set_fact: + logstash_writer_user_present: true + when: check_logstash_writer_user_response.json.logstash_writer.username is defined and check_logstash_writer_user_response.json.logstash_writer.username == "logstash_writer" + +- name: Put logstash_writer user into Elasticsearch if not present + ansible.builtin.uri: + url: "{{ security_api_base_url }}user/{{ logstash_user }}" + ca_path: "{{ elasticstack_ca_dir }}/ca.crt" + user: elastic + password: "{{ logstash_elasticstack_password.stdout }}" + method: PUT + headers: + Content-Type: application/json + body: "{{ lookup('template', 'logstash_writer_user.j2') }}" + body_format: json + register: put_logstash_writer_user_response + when: logstash_writer_user_present is not defined delegate_to: "{{ elasticstack_ca }}" run_once: true - no_log: "{{ elasticstack_no_log }}" - changed_when: false - when: - - logstash_writer_user_present.rc > 0 - - elasticstack_release | int > 7 + failed_when: not put_logstash_writer_user_response.json.created diff --git a/roles/logstash/templates/logstash_writer_user.j2 b/roles/logstash/templates/logstash_writer_user.j2 index 7335bf68..5c21a745 100644 --- a/roles/logstash/templates/logstash_writer_user.j2 +++ b/roles/logstash/templates/logstash_writer_user.j2 @@ -1,7 +1,9 @@ { {% if logstash_password_hash | bool %} -{# using a fixed salt is neccessary for idempotency, will be created on elasticstack CA host #} - "password_hash" : "{{ logstash_password | password_hash( hashtype=logstash_password_hash_algorithm, salt=logstash_password_hash_salt, ident='2a' ) }}", +{# using a fixed salt is neccessary for idempotency, will be generated as a set fact. +rounds specifies the bcrypt version. The default version in Ansible module is 12. The acceptable one is 10 on elasticsearch 7. +On elasticsearch 8, the 12 and 10 versions will work, so we should use 10 until the support of 7 stops #} + "password_hash" : "{{ logstash_password | password_hash( hashtype=logstash_password_hash_algorithm, salt=logstash_password_hash_salt, ident='2a', rounds=10 ) }}", {% else %} "password" : "{{ logstash_password }}", {% endif %}