Kibana config always points its elasticsearch.hosts to a "logging" VM (#1347)

CHANGELOG-0.7.md

@@ -18,6 +18,7 @@
 - [#1462](https://github.com/epiphany-platform/epiphany/issues/1461) - [BUG] Error upgrading AWS/RHEL/calico cluster from 0.6 to 0.7
 - [#802](https://github.com/epiphany-platform/epiphany/issues/802) - Docker version is hard-coded in installation tasks
 - [#1495](https://github.com/epiphany-platform/epiphany/issues/1495) - Offline installation is broken for CentOS 7.8 environments
+- [#1347](https://github.com/epiphany-platform/epiphany/issues/1347) - [BUG] Kibana config always points its elasticsearch.hosts to a "logging" VM
 
 ### Added
 

core/src/epicli/data/common/ansible/playbooks/roles/filebeat/tasks/configure-filebeat.yml

@@ -7,12 +7,6 @@
     src: filebeat.yml.j2
   register: modify_filebeat_yml
 
-- name: Enable auditd module
-  shell: >-
-    filebeat modules enable auditd 
-  register: enable_module
-  changed_when: "'Enabled auditd' in enable_module.stdout"
-
 - name: Set Filebeat to be started after Docker
   block:
     - name: Create directory (filebeat.service.d)
@@ -37,6 +31,11 @@
 
 - name: Start/restart and enable filebeat service
   block:
+    - name: Enable auditd module
+      command: filebeat modules enable auditd
+      register: enable_module
+      changed_when: "'Enabled auditd' in enable_module.stdout"
+
     - name: Restart filebeat service
       systemd:
         name: filebeat
@@ -51,16 +50,18 @@
         state: started
         enabled: true
 
-    - name: Verify Filebeat is running
+    - name: Verify filebeat is running
       command: systemctl is-active filebeat
       changed_when: false
-  when:
-    - groups['logging'][0] is defined
+      retries: 10
+      delay: 1
+      register: result
+      until: result is succeeded
+  when: groups['logging'][0] is defined
 
 - name: Stop and disable filebeat service
   systemd:
     name: filebeat
     state: stopped
     enabled: false
-  when:
-    - groups['logging'][0] is undefined
+  when: groups['logging'][0] is undefined

core/src/epicli/data/common/ansible/playbooks/roles/filebeat/tasks/main.yml

@@ -2,29 +2,15 @@
 - name: Include installation tasks
   include_tasks: install-filebeat.yml
 
-- name: Set facts for filebeat.yml template
-  block: 
-
-    - name: Set flag if logging output exists
-      set_fact:
-        logging_output_exists: False 
-
-    - name: Set flag if logging output exists
-      set_fact:
-        logging_output_exists: True
-      when:
-        - groups["logging"] is defined
-        - groups["logging"] | length > 0
-
-    - name: Set value for setup.kibana.host
-      set_fact:
-        setup_kibana_host: >-
-          {{ hostvars[groups['kibana'][0]].ansible_default_ipv4.address + ':5601' }}
-      when:
-        - groups['kibana'][0] is defined
+- name: Set value for setup.kibana.host
+  set_fact:
+    setup_kibana_host: >-
+      {{ hostvars[groups['kibana'][0]].ansible_default_ipv4.address + ':5601' }}
+  when:
+    - groups['kibana'][0] is defined
 
 - name: Include auditd configuration tasks
   include_tasks: configure-auditd.yml
 
 - name: Include filebeat configuration tasks
-  include_tasks: configure-filebeat.yml
+  include_tasks: configure-filebeat.yml

core/src/epicli/data/common/ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2

@@ -198,27 +198,29 @@ setup.template.settings:
 #================================ Outputs =====================================
 
 # Configure what output to use when sending the data collected by the beat.
+# Default Elasticsearch output is array of 'logging' hosts. If not exist, 'hosts' remains empty to manual configuration.
 
 #-------------------------- Elasticsearch output ------------------------------
 output.elasticsearch:
   # Array of hosts to connect to.
-{% if logging_output_exists %}
-  hosts: 
-{% for host in groups['logging'] %}
-  - "https://{{hostvars[host]['ansible_hostname']}}:9200"
-{% endfor %}
+{% if groups['logging'] is defined %}
+  hosts:
+    # Logging hosts:
+    {% for host in groups['logging'] -%}
+    - "https://{{hostvars[host]['ansible_hostname']}}:9200"
+    {% endfor %}
+
   protocol: "https"
   ssl.verification_mode: none
   username: logstash
   password: logstash
 {% else %}
   hosts: []
-{% endif %}
-
-  # Optional protocol and basic auth credentials.
   #protocol: "https"
+  #ssl.verification_mode: none
   #username: "elastic"
   #password: "changeme"
+{% endif %}
 
 #----------------------------- Logstash output --------------------------------
 #output.logstash:

core/src/epicli/data/common/ansible/playbooks/roles/kibana/tasks/main.yml

@@ -1,5 +1,6 @@
 ---
-- include_tasks: "{{ ansible_os_family }}.yml"
+- name: Include Kibana package installation task
+  include_tasks: "{{ ansible_os_family }}.yml"
 
 - name: Update Kibana configuration file
   template:
@@ -10,7 +11,8 @@
     mode: 0644
   register: change_config
 
-- include_tasks: setup-logging.yml
+- name: Include logging configuration tasks
+  include_tasks: setup-logging.yml
 
 - name: Restart Kibana service
   systemd:
@@ -29,7 +31,7 @@
     host: "{{ ansible_default_ipv4.address | default(ansible_all_ipv4_addresses[0]) }}"
     port: 5601
     delay: 5
-  
+
 - name: Wait for kibana to be ready
   uri:
     url: http://{{ ansible_default_ipv4.address | default(ansible_all_ipv4_addresses[0]) }}:5601/api/kibana/settings

core/src/epicli/data/common/ansible/playbooks/roles/kibana/tasks/setup-logging.yml

@@ -28,4 +28,4 @@
     owner: root
     group: root
     mode: 0644
-    src: logrotate.conf.j2
+    src: logrotate.conf.j2

core/src/epicli/data/common/ansible/playbooks/roles/kibana/templates/kibana.yml.j2

@@ -15,11 +15,21 @@
 
 # Description:
 # Default Kibana configuration for Open Distro.
+
 server.host: "{{ ansible_default_ipv4.address | default(ansible_all_ipv4_addresses[0]) }}"
-elasticsearch.hosts: 
-{% for host in groups['logging'] %}
+elasticsearch.hosts:
+{% if 'logging' in group_names %}
+  # Logging hosts:
+  {% for host in groups['logging'] %}
+  - "https://{{hostvars[host]['ansible_hostname']}}:9200"
+  {% endfor %}
+{% elif 'opendistro_for_elasticsearch' in group_names %}
+  # Opendistro (data) hosts:
+  {% for host in groups['opendistro_for_elasticsearch'] %}
   - "https://{{hostvars[host]['ansible_hostname']}}:9200"
-{% endfor %}
+  {% endfor %}
+{% endif %}
+
 elasticsearch.ssl.verificationMode: none
 elasticsearch.username: kibanaserver
 elasticsearch.password: kibanaserver

docs/home/howto/DATABASES.md

@@ -543,7 +543,8 @@ DROP EXTENSION IF EXISTS pgaudit;
 
 ## How to start working with OpenDistro for Elasticsearch
 
-OpenDistro for Elasticsearch is [an Apache 2.0-licensed distribution of Elasticsearch enhanced with enterprise security, alerting, SQL](https://opendistro.github.io/for-elasticsearch/). In order to start working with OpenDistro change machines count to value greater than 0 in your cluster configuration:
+OpenDistro for Elasticsearch is [an Apache 2.0-licensed distribution of Elasticsearch enhanced with enterprise security, alerting, SQL](https://opendistro.github.io/for-elasticsearch/).
+In order to start working with OpenDistro change machines count to value greater than 0 in your cluster configuration:
 
 ```yaml
 kind: epiphany-cluster
@@ -576,6 +577,23 @@ specification:
 
 Result of this configuration will be one or more independent nodes of OpenDistro.
 
+By default Kibana is deployed only for `logging` component. If you want to deploy Kibana for `opendistro_for_elasticsearch` you have to modify feature mapping. Use below configuration in your manifest.
+```yaml
+kind: configuration/feature-mapping
+title: "Feature mapping to roles"
+name: default
+specification:
+  roles_mapping:
+    opendistro_for_elasticsearch:
+      - opendistro-for-elasticsearch
+      - node-exporter
+      - filebeat
+      - firewall
+      - kibana
+```
+
+Filebeat running on `opendistro_for_elasticsearch` hosts will always point to centralized logging hosts (./LOGGING.md).
+
 ## How to start working with Apache Ignite Stateful setup
 
 Apache Ignite can be installed in Epiphany if `count` property for `ignite` feature is greater than 0.

docs/home/howto/LOGGING.md

@@ -1,32 +1,48 @@
 ## Centralized logging setup
 
-For centralized logging Epiphany uses [OpenDistro for Elasticsearch](https://opendistro.github.io/for-elasticsearch/). In order to enable centralized logging, there is required to use `logging` role on feature mapping level - as of 0.5.0 this is a default configuration.
+For centralized logging Epiphany uses [OpenDistro for Elasticsearch](https://opendistro.github.io/for-elasticsearch/).
+In order to enable centralized logging, be sure that `count` property for `logging` feature is greater than 0 in your configuration manifest.
 
+```yaml
+kind: epiphany-cluster
+...
+specification:
+  ...
+  components:
+    kubernetes_master:
+      count: 1
+    kubernetes_node:
+      count: 0
+    ...
+    logging:
+      count: 1
+    ...
+```
+
+### Default feature mapping for logging:
 ```yaml
     ...
     logging:
       - logging
       - kibana
+      - node-exporter
       - filebeat
       - firewall
     ...
 ```
+The `logging` role replaced `elasticsearch` role. This change was done to enable Elasticsearch usage also for data storage - not only for logs as it was till 0.5.0.
 
-The `logging` role replaced `elasticsearch` role in logging feature. This change was done to enable Elasticsearch usage for data storage - not only for logs as it was till 0.5.0.
-
-Default configuration of `logging` role is the same as [opendistro_for_elasticsearch](./DATABASES.md#how-to-start-working-with-opendistro-for-elasticsearch) which is used for logs storage. In order to modify centralized logging configuration adjust and use following defaults:
+Default configuration of `logging` and `opendistro_for_elasticsearch` roles is identical (./DATABASES.md#how-to-start-working-with-opendistro-for-elasticsearch). To modify configuration of centralized logging adjust and use the following defaults in your manifest:
 
 ```yaml
 kind: configuration/logging
 title: Logging Config
 name: default
 specification:
-  opendistro_version_redhat: "1.3.0"
-  elasticsearch_oss_version_debian: "7.3.2"
-  opendistro_version_debian: "1.3.0*"
   cluster_name: EpiphanyElastic
   clustered: True
   paths:
     data: /var/lib/elasticsearch
+    repo: /var/lib/elasticsearch-snapshots
     logs: /var/log/elasticsearch
-```
+```