ODFE: Switch to Garbage-First Garbage Collector (G1GC) (#1718)

* Switch to Garbage-First Garbage Collector (G1GC)

* Fixes after testing

* Fix 'versions' variables

core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/defaults/main.yml

@@ -1,14 +1,22 @@
 ---
 host_ip: "{{ ansible_default_ipv4.address | default(ansible_all_ipv4_addresses[0]) }}"
-current_group_name: "opendistro_for_elasticsearch"
 # Required and used for upgrade Open Distro for Elasticsearch cluster:
+versions:
+  RedHat:
+    elasticsearch_oss: "7.8.0"
+    opendistro: "1.9.0*"
+  Debian:
+    elasticsearch_oss: "7.8.0"
+    opendistro: "1.9.0*"
 specification:
-  versions:
-    RedHat:
-      elasticsearch_oss: "7.8.0"
-      opendistro: "1.9.0*"
-    Debian:
-      elasticsearch_oss: "7.8.0"
-      opendistro: "1.9.0*"
   es_user: admin
   es_password: admin
+  cluster_name: EpiphanyElastic
+  clustered: True
+  paths:
+    data: /var/lib/elasticsearch
+    repo: /var/lib/elasticsearch-snapshots
+    logs: /var/log/elasticsearch
+# JVM settings
+  init_heap_space: -Xms1g
+  max_heap_space: -Xmx1g

core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/configure-es.yml

@@ -7,20 +7,30 @@
     group: elasticsearch
     mode: u=rwx,go=
 
-- name: Create Elasticsearch configuration file
+- name: Provide jvm configuration file
+  template:
+    src: jvm.options.j2
+    dest: /etc/elasticsearch/jvm.options
+    owner: elasticsearch
+    group: root
+    mode: ug=rw,o=
+  register: change_jvm_config
+
+- name: Provide Elasticsearch configuration file
   template:
     src: elasticsearch.yml.j2
     dest: /etc/elasticsearch/elasticsearch.yml
     owner: elasticsearch
     group: root
-    mode: 0644
+    mode: ug=rw,o=
   register: change_config
 
 - name: Restart elasticsearch service
   systemd:
     name: elasticsearch
     state: restarted
-  when: change_config.changed
+  when:
+    - change_config.changed or change_jvm_config.changed
 
 - name: Enable and start elasticsearch service
   systemd:

core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/install-es-Debian.yml

@@ -2,7 +2,6 @@
 - name: Install Open Distro for Elasticsearch packages
   apt:
     name:
-      - openjdk-8-jre-headless
-      - elasticsearch-oss={{ specification.versions[ansible_os_family].elasticsearch_oss }}
+      - elasticsearch-oss={{ versions[ansible_os_family].elasticsearch_oss }}
     state: present
   register: install_elasticsearch_package

core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/install-es-RedHat.yml

@@ -2,8 +2,7 @@
 - name: Install Open Distro for Elasticsearch packages
   yum:
     name:
-      - java-1.8.0-openjdk-headless
-      - elasticsearch-oss-{{ specification.versions[ansible_os_family].elasticsearch_oss }}
+      - elasticsearch-oss-{{ versions[ansible_os_family].elasticsearch_oss }}
     update_cache: yes
     state: present
   register: install_elasticsearch_package

core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/install-opendistro-Debian.yml

@@ -3,11 +3,11 @@
 - name: Install Open Distro for Elasticsearch packages
   apt:
     name:
-      - opendistro-alerting={{ specification.versions[ansible_os_family].opendistro }}
-      - opendistro-index-management={{ specification.versions[ansible_os_family].opendistro }}
-      - opendistro-job-scheduler={{ specification.versions[ansible_os_family].opendistro }}
-      - opendistro-performance-analyzer={{ specification.versions[ansible_os_family].opendistro }}
-      - opendistro-security={{ specification.versions[ansible_os_family].opendistro }}
-      - opendistro-sql={{ specification.versions[ansible_os_family].opendistro }}
+      - opendistro-alerting={{ versions[ansible_os_family].opendistro }}
+      - opendistro-index-management={{ versions[ansible_os_family].opendistro }}
+      - opendistro-job-scheduler={{ versions[ansible_os_family].opendistro }}
+      - opendistro-performance-analyzer={{ versions[ansible_os_family].opendistro }}
+      - opendistro-security={{ versions[ansible_os_family].opendistro }}
+      - opendistro-sql={{ versions[ansible_os_family].opendistro }}
     state: present
   register: install_elasticsearch_package

core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/install-opendistro-RedHat.yml

@@ -3,12 +3,12 @@
 - name: Install Open Distro for Elasticsearch packages
   yum:
     name:
-      - opendistro-alerting-{{ specification.versions[ansible_os_family].opendistro }}
-      - opendistro-index-management-{{ specification.versions[ansible_os_family].opendistro }}
-      - opendistro-job-scheduler-{{ specification.versions[ansible_os_family].opendistro }}
-      - opendistro-performance-analyzer-{{ specification.versions[ansible_os_family].opendistro }}
-      - opendistro-security-{{ specification.versions[ansible_os_family].opendistro }}
-      - opendistro-sql-{{ specification.versions[ansible_os_family].opendistro }}
+      - opendistro-alerting-{{ versions[ansible_os_family].opendistro }}
+      - opendistro-index-management-{{ versions[ansible_os_family].opendistro }}
+      - opendistro-job-scheduler-{{ versions[ansible_os_family].opendistro }}
+      - opendistro-performance-analyzer-{{ versions[ansible_os_family].opendistro }}
+      - opendistro-security-{{ versions[ansible_os_family].opendistro }}
+      - opendistro-sql-{{ versions[ansible_os_family].opendistro }}
     update_cache: yes
     state: present
   register: install_elasticsearch_package

core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/main.yml

@@ -19,5 +19,7 @@
 - name: Include opendistro-* plugins installation task
   include_tasks: "install-opendistro-{{ ansible_os_family }}.yml"
 
-- name: Include configuration tasks in a context of {{ current_group_name }}
+- name: Include configuration tasks
   include_tasks: configure-es.yml
+  vars:
+    current_group_name: "opendistro_for_elasticsearch"

core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/templates/jvm.options.j2

@@ -0,0 +1,77 @@
+## JVM configuration
+
+################################################################
+## IMPORTANT: JVM heap size
+################################################################
+##
+## You should always set the min and max JVM heap
+## size to the same value. For example, to set
+## the heap to 4 GB, set:
+##
+## -Xms4g
+## -Xmx4g
+##
+## See https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html
+## for more information
+##
+################################################################
+
+# Xms represents the initial size of total heap space
+# Xmx represents the maximum size of total heap space
+
+{{ specification.init_heap_space }}
+{{ specification.max_heap_space }}
+
+################################################################
+## Expert settings
+################################################################
+##
+## All settings below this section are considered
+## expert settings. Don't tamper with them unless
+## you understand what you are doing
+##
+################################################################
+
+## GC configuration
+8-13:-XX:+UseConcMarkSweepGC
+8-13:-XX:CMSInitiatingOccupancyFraction=75
+8-13:-XX:+UseCMSInitiatingOccupancyOnly
+
+## G1GC Configuration
+# NOTE: G1 GC is only supported on JDK version 10 or later
+# to use G1GC, uncomment the next two lines and update the version on the
+# following three lines to your version of the JDK
+# 10-13:-XX:-UseConcMarkSweepGC
+# 10-13:-XX:-UseCMSInitiatingOccupancyOnly
+14-:-XX:+UseG1GC
+14-:-XX:G1ReservePercent=25
+14-:-XX:InitiatingHeapOccupancyPercent=30
+
+## JVM temporary directory
+-Djava.io.tmpdir=${ES_TMPDIR}
+
+## heap dumps
+
+# generate a heap dump when an allocation from the Java heap fails
+# heap dumps are created in the working directory of the JVM
+-XX:+HeapDumpOnOutOfMemoryError
+
+# specify an alternative path for heap dumps; ensure the directory exists and
+# has sufficient space
+-XX:HeapDumpPath=/var/lib/elasticsearch
+
+# specify an alternative path for JVM fatal error logs
+-XX:ErrorFile=/var/log/elasticsearch/hs_err_pid%p.log
+
+## JDK 8 GC logging
+8:-XX:+PrintGCDetails
+8:-XX:+PrintGCDateStamps
+8:-XX:+PrintTenuringDistribution
+8:-XX:+PrintGCApplicationStoppedTime
+8:-Xloggc:/var/log/elasticsearch/gc.log
+8:-XX:+UseGCLogFileRotation
+8:-XX:NumberOfGCLogFiles=32
+8:-XX:GCLogFileSize=64m
+
+# JDK 9+ GC logging
+9-:-Xlog:gc*,gc+age=trace,safepoint:file=/var/log/elasticsearch/gc.log:utctime,pid,tags:filecount=32,filesize=64m

core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch.yml

@@ -18,13 +18,13 @@
   debug:
     msg:
       - "Installed version: {{ ansible_facts.packages['elasticsearch-oss'][0].version }}"
-      - "Target version: {{ specification.versions[ansible_os_family].elasticsearch_oss }}"
+      - "Target version: {{ versions[ansible_os_family].elasticsearch_oss }}"
 
 - name: Open Distro for Elasticsearch | Upgrade elasticsearch-oss
   include_tasks: opendistro_for_elasticsearch/upgrade_elasticsearch.yml
   vars:
     es_http_port: 9200
     es_transport_port: 9300
   when:
-    - "specification.versions[ansible_os_family].elasticsearch_oss
+    - "versions[ansible_os_family].elasticsearch_oss
       is version(ansible_facts.packages['elasticsearch-oss'][0].version, '>')"

core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/upgrade_elasticsearch.yml

@@ -46,6 +46,11 @@
     name: opendistro_for_elasticsearch
     tasks_from: install-es-{{ ansible_os_family }}.yml
 
+- name: Include configuration tasks
+  include_role:
+    name: opendistro_for_elasticsearch
+    tasks_from: configure-es.yml
+
 - name: Open Distro for Elasticsearch - plugins | Upgrade
   include_tasks: opendistro_for_elasticsearch/upgrade_plugins.yml
 

core/src/epicli/data/common/ansible/playbooks/upgrade.yml

@@ -117,6 +117,8 @@
     - import_role:
         name: upgrade
         tasks_from: opendistro_for_elasticsearch
+      vars:
+        current_group_name: logging
 
 - hosts: opendistro_for_elasticsearch
   become: true

core/src/epicli/data/common/defaults/configuration/opendistro-for-elasticsearch.yml

@@ -2,13 +2,6 @@ kind: configuration/opendistro-for-elasticsearch
 title: OpenDistro for Elasticsearch Config
 name: default
 specification:
-  versions:
-    RedHat:
-      elasticsearch_oss: "7.8.0"
-      opendistro: "1.9.0*"
-    Debian:
-      elasticsearch_oss: "7.8.0"
-      opendistro: "1.9.0*"
   cluster_name: EpiphanyElastic
   clustered: True
   paths: