From 87da2cb6e6c8f18b254e6b2f99590c0d76995a91 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Wed, 27 Apr 2022 11:22:44 -0600 Subject: [PATCH] Resolve performance degredation when we went to OpenSearch 1.3 by using the G1GC garbage collector. see: * https://discuss.opendistrocommunity.dev/t/slow-indexing-performance/9366/4 * https://github.com/opensearch-project/OpenSearch/issues/2916 * https://marknienaber.medium.com/jvm-tuning-with-g1-gc-76f27535f054 * https://github.com/opensearch-project/OpenSearch/issues/2985 --- Dockerfiles/opensearch.Dockerfile | 2 ++ docker-compose-standalone.yml | 4 ++-- docker-compose.yml | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Dockerfiles/opensearch.Dockerfile b/Dockerfiles/opensearch.Dockerfile index 3c6ee4ef6..e48d279e9 100644 --- a/Dockerfiles/opensearch.Dockerfile +++ b/Dockerfiles/opensearch.Dockerfile @@ -38,6 +38,8 @@ RUN yum install -y openssl util-linux procps && \ /usr/share/opensearch/bin/opensearch-plugin remove opensearch-security --purge && \ /usr/share/opensearch/bin/opensearch-plugin remove opensearch-performance-analyzer --purge && \ echo -e 'cluster.name: "docker-cluster"\nnetwork.host: 0.0.0.0\ncompatibility.override_main_response_version: true\nbootstrap.memory_lock: true' > /usr/share/opensearch/config/opensearch.yml && \ + sed -i "s/#[[:space:]]*\([0-9]*-[0-9]*:-XX:-\(UseConcMarkSweepGC\|UseCMSInitiatingOccupancyOnly\)\)/\1/" /usr/share/opensearch/config/jvm.options && \ + sed -i "s/^[0-9][0-9]*\(-:-XX:\(+UseG1GC\|G1ReservePercent\|InitiatingHeapOccupancyPercent\)\)/$($JAVA_HOME/bin/java -version 2>&1 | grep version | awk '{print $3}' | tr -d '\"' | cut -d. -f1)\1/" /usr/share/opensearch/config/jvm.options && \ chown -R $PUSER:$PGROUP /usr/share/opensearch/config/opensearch.yml && \ sed -i "s/^\([[:space:]]*\)\(performance-analyzer-agent-cli\)/\1# \2/" /usr/share/opensearch/opensearch-docker-entrypoint.sh && \ sed -i '/[^#].*OPENSEARCH_HOME\/bin\/opensearch.*/i /usr/local/bin/jdk-cacerts-auto-import.sh || true' /usr/share/opensearch/opensearch-docker-entrypoint.sh diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml index 86bff71ba..da3af9d9c 100644 --- a/docker-compose-standalone.yml +++ b/docker-compose-standalone.yml @@ -152,7 +152,7 @@ services: logger.level : 'WARN' bootstrap.memory_lock : 'true' MAX_LOCKED_MEMORY : 'unlimited' - OPENSEARCH_JAVA_OPTS : '-Xms4g -Xmx4g -Xss256k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true' + OPENSEARCH_JAVA_OPTS : '-server -Xms4g -Xmx4g -Xss256k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true' VIRTUAL_HOST : 'os.malcolm.local' discovery.type : 'single-node' discovery.zen.minimum_master_nodes : 1 @@ -252,7 +252,7 @@ services: << : *common-beats-variables << : *common-lookup-variables OS_HOSTS : 'opensearch:9200' - LS_JAVA_OPTS : '-Xms2g -Xmx2g -Xss1536k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true' + LS_JAVA_OPTS : '-server -Xms2g -Xmx2g -Xss1536k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true' depends_on: - opensearch expose: diff --git a/docker-compose.yml b/docker-compose.yml index 6e5870a4c..cec61de5c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -155,7 +155,7 @@ services: logger.level : 'INFO' bootstrap.memory_lock : 'true' MAX_LOCKED_MEMORY : 'unlimited' - OPENSEARCH_JAVA_OPTS : '-Xms4g -Xmx4g -Xss256k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true' + OPENSEARCH_JAVA_OPTS : '-server -Xms4g -Xmx4g -Xss256k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true' VIRTUAL_HOST : 'os.malcolm.local' discovery.type : 'single-node' discovery.zen.minimum_master_nodes : 1 @@ -264,7 +264,7 @@ services: << : *common-beats-variables << : *common-lookup-variables OS_HOSTS : 'opensearch:9200' - LS_JAVA_OPTS : '-Xms2g -Xmx2g -Xss1536k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true' + LS_JAVA_OPTS : '-server -Xms2g -Xmx2g -Xss1536k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true' depends_on: - opensearch expose: