From cfd8e29980dfa50cb62930d67e6427403f9bcf0f Mon Sep 17 00:00:00 2001 From: Julien Mailleret <8582351+jmlrt@users.noreply.github.com> Date: Fri, 20 Nov 2020 12:30:21 +0100 Subject: [PATCH] [meta] upgrade test (#907) This PR is adding a new upgrade test for every chart. This test is deploying the oldest **release** version of charts which can be upgraded to latest version, then upgrading using unreleased charts in the repo. The goal is to detect when some PR is introducing a breaking change. - Note 1: An upgrade test is already existing for Elasticsearch but had been removed from master branch because upgrade from 7.x to Elasticsearch 8.0.0-SNAPSHOT used by master branch isn't working. The new test allows overriding the Docker image version used during upgrade so we can still test upgrade on master but use the latest 7.x Docker image instead of 8.0.0-SNAPSHOT. * Note 2: Metricbeat chart introduced a breaking change in #516 and can be upgraded from any previous version currently, so Metricbeat test is commented and should be uncommented after 7.10.0 release so we can check if new PR are introducing new breaking changes. * Note 3: Currently charts upgrade are only tested with default values and so cover a small part of charts codebase, we should update it to test optional resources (ie: enable service or ingress, ...). Fix #396 --- apm-server/examples/upgrade/Makefile | 15 ++++ apm-server/examples/upgrade/README.md | 21 ++++++ apm-server/examples/upgrade/test/goss.yaml | 6 ++ apm-server/examples/upgrade/values.yaml | 12 +++ elasticsearch/examples/upgrade/Makefile | 16 ++++ elasticsearch/examples/upgrade/README.md | 17 +++++ elasticsearch/examples/upgrade/test/goss.yaml | 17 +++++ elasticsearch/examples/upgrade/values.yaml | 2 + filebeat/examples/upgrade/Makefile | 15 ++++ filebeat/examples/upgrade/README.md | 21 ++++++ filebeat/examples/upgrade/test/goss.yaml | 45 ++++++++++++ filebeat/examples/upgrade/values.yaml | 4 + helpers/matrix.yml | 12 ++- helpers/upgrade.sh | 73 +++++++++++++++++++ kibana/examples/upgrade/Makefile | 16 ++++ kibana/examples/upgrade/README.md | 21 ++++++ kibana/examples/upgrade/test/goss.yaml | 14 ++++ kibana/examples/upgrade/values.yaml | 2 + logstash/examples/upgrade/Makefile | 15 ++++ logstash/examples/upgrade/README.md | 19 +++++ logstash/examples/upgrade/test/goss.yaml | 43 +++++++++++ logstash/examples/upgrade/values.yaml | 1 + metricbeat/examples/upgrade/Makefile | 17 +++++ metricbeat/examples/upgrade/README.md | 21 ++++++ .../examples/upgrade/test/goss-metrics.yaml | 42 +++++++++++ metricbeat/examples/upgrade/test/goss.yaml | 46 ++++++++++++ metricbeat/examples/upgrade/values.yaml | 4 + 27 files changed, 534 insertions(+), 3 deletions(-) create mode 100644 apm-server/examples/upgrade/Makefile create mode 100644 apm-server/examples/upgrade/README.md create mode 100644 apm-server/examples/upgrade/test/goss.yaml create mode 100644 apm-server/examples/upgrade/values.yaml create mode 100644 elasticsearch/examples/upgrade/Makefile create mode 100644 elasticsearch/examples/upgrade/README.md create mode 100644 elasticsearch/examples/upgrade/test/goss.yaml create mode 100644 elasticsearch/examples/upgrade/values.yaml create mode 100644 filebeat/examples/upgrade/Makefile create mode 100644 filebeat/examples/upgrade/README.md create mode 100644 filebeat/examples/upgrade/test/goss.yaml create mode 100644 filebeat/examples/upgrade/values.yaml create mode 100755 helpers/upgrade.sh create mode 100644 kibana/examples/upgrade/Makefile create mode 100644 kibana/examples/upgrade/README.md create mode 100644 kibana/examples/upgrade/test/goss.yaml create mode 100644 kibana/examples/upgrade/values.yaml create mode 100644 logstash/examples/upgrade/Makefile create mode 100644 logstash/examples/upgrade/README.md create mode 100644 logstash/examples/upgrade/test/goss.yaml create mode 100644 logstash/examples/upgrade/values.yaml create mode 100644 metricbeat/examples/upgrade/Makefile create mode 100644 metricbeat/examples/upgrade/README.md create mode 100644 metricbeat/examples/upgrade/test/goss-metrics.yaml create mode 100644 metricbeat/examples/upgrade/test/goss.yaml create mode 100644 metricbeat/examples/upgrade/values.yaml diff --git a/apm-server/examples/upgrade/Makefile b/apm-server/examples/upgrade/Makefile new file mode 100644 index 000000000..02ea4dbcb --- /dev/null +++ b/apm-server/examples/upgrade/Makefile @@ -0,0 +1,15 @@ +default: test + +include ../../../helpers/examples.mk + +CHART := apm-server +RELEASE := helm-apm-server-upgrade +FROM := 7.6.0 # 7.6.0 is the first release for apm-server + +install: + ../../../helpers/upgrade.sh --chart $(CHART) --release $(RELEASE) --from $(FROM) + +test: install goss + +purge: + helm del $(RELEASE) diff --git a/apm-server/examples/upgrade/README.md b/apm-server/examples/upgrade/README.md new file mode 100644 index 000000000..835b3c7ed --- /dev/null +++ b/apm-server/examples/upgrade/README.md @@ -0,0 +1,21 @@ +# Upgrade + +This example will deploy APM Server chart using an old chart version, +then upgrade it. + + +## Usage + +* Add the Elastic Helm charts repo: `helm repo add elastic https://helm.elastic.co` + +* Deploy [Elasticsearch Helm chart][]: `helm install elasticsearch elastic/elasticsearch` + +* Deploy and upgrade APM Server chart with the default values: `make install` + + +## Testing + +You can also run [goss integration tests][] using `make test`. + + +[goss integration tests]: https://github.com/elastic/helm-charts/tree/master/apm-server/examples/upgrade/test/goss.yaml diff --git a/apm-server/examples/upgrade/test/goss.yaml b/apm-server/examples/upgrade/test/goss.yaml new file mode 100644 index 000000000..5dbc51660 --- /dev/null +++ b/apm-server/examples/upgrade/test/goss.yaml @@ -0,0 +1,6 @@ +http: + http://localhost:8200?pretty: + status: 200 + timeout: 2000 + body: + - "8.0.0" diff --git a/apm-server/examples/upgrade/values.yaml b/apm-server/examples/upgrade/values.yaml new file mode 100644 index 000000000..4b66615c5 --- /dev/null +++ b/apm-server/examples/upgrade/values.yaml @@ -0,0 +1,12 @@ +--- +apmConfig: + apm-server.yml: | + apm-server: + host: "0.0.0.0:8200" + + queue: {} + output.file: + enabled: false + + output.elasticsearch: + hosts: ["http://upgrade-master:9200"] diff --git a/elasticsearch/examples/upgrade/Makefile b/elasticsearch/examples/upgrade/Makefile new file mode 100644 index 000000000..58dd7d4a8 --- /dev/null +++ b/elasticsearch/examples/upgrade/Makefile @@ -0,0 +1,16 @@ +default: test + +include ../../../helpers/examples.mk + +CHART := elasticsearch +RELEASE := helm-es-upgrade +FROM := 7.4.0 # versions before 7.4.O aren't compatible with Kubernetes >= 1.16.0 +TO := 7.10.0 # upgrade from 7.x to 8.0.0-SNAPSHOT currently doesn't work + +install: + ../../../helpers/upgrade.sh --chart $(CHART) --release $(RELEASE) --from $(FROM) --to $(TO) + +test: install goss + +purge: + helm del $(RELEASE) diff --git a/elasticsearch/examples/upgrade/README.md b/elasticsearch/examples/upgrade/README.md new file mode 100644 index 000000000..85977f52e --- /dev/null +++ b/elasticsearch/examples/upgrade/README.md @@ -0,0 +1,17 @@ +# Upgrade + +This example will deploy a 3 node Elasticsearch cluster chart using an old chart +version, then upgrade it. + + +## Usage + +* Deploy and upgrade Elasticsearch chart with the default values: `make install` + + +## Testing + +You can also run [goss integration tests][] using `make test`. + + +[goss integration tests]: https://github.com/elastic/helm-charts/tree/master/elasticsearch/examples/upgrade/test/goss.yaml diff --git a/elasticsearch/examples/upgrade/test/goss.yaml b/elasticsearch/examples/upgrade/test/goss.yaml new file mode 100644 index 000000000..9f8a0b0f7 --- /dev/null +++ b/elasticsearch/examples/upgrade/test/goss.yaml @@ -0,0 +1,17 @@ +http: + http://localhost:9200/_cluster/health: + status: 200 + timeout: 2000 + body: + - 'green' + - '"number_of_nodes":3' + - '"number_of_data_nodes":3' + + http://localhost:9200: + status: 200 + timeout: 2000 + body: + - '"number" : "7.10.0"' + - '"cluster_name" : "upgrade"' + - '"name" : "upgrade-master-0"' + - 'You Know, for Search' diff --git a/elasticsearch/examples/upgrade/values.yaml b/elasticsearch/examples/upgrade/values.yaml new file mode 100644 index 000000000..de0283af4 --- /dev/null +++ b/elasticsearch/examples/upgrade/values.yaml @@ -0,0 +1,2 @@ +--- +clusterName: upgrade diff --git a/filebeat/examples/upgrade/Makefile b/filebeat/examples/upgrade/Makefile new file mode 100644 index 000000000..252b39250 --- /dev/null +++ b/filebeat/examples/upgrade/Makefile @@ -0,0 +1,15 @@ +default: test + +include ../../../helpers/examples.mk + +CHART := filebeat +RELEASE := helm-filebeat-upgrade +FROM := 7.9.0 # registry file version 1 not supported error with previous version + +install: + ../../../helpers/upgrade.sh --chart $(CHART) --release $(RELEASE) --from $(FROM) + +test: install goss + +purge: + helm del $(RELEASE) diff --git a/filebeat/examples/upgrade/README.md b/filebeat/examples/upgrade/README.md new file mode 100644 index 000000000..fa3ee3b85 --- /dev/null +++ b/filebeat/examples/upgrade/README.md @@ -0,0 +1,21 @@ +# Upgrade + +This example will deploy Filebeat chart using an old chart version, +then upgrade it. + + +## Usage + +* Add the Elastic Helm charts repo: `helm repo add elastic https://helm.elastic.co` + +* Deploy [Elasticsearch Helm chart][]: `helm install elasticsearch elastic/elasticsearch` + +* Deploy and upgrade Filebeat chart with the default values: `make install` + + +## Testing + +You can also run [goss integration tests][] using `make test`. + + +[goss integration tests]: https://github.com/elastic/helm-charts/tree/master/filebeat/examples/upgrade/test/goss.yaml diff --git a/filebeat/examples/upgrade/test/goss.yaml b/filebeat/examples/upgrade/test/goss.yaml new file mode 100644 index 000000000..b71a24325 --- /dev/null +++ b/filebeat/examples/upgrade/test/goss.yaml @@ -0,0 +1,45 @@ +port: + tcp:5066: + listening: true + ip: + - "127.0.0.1" + +mount: + /usr/share/filebeat/data: + exists: true + /run/docker.sock: + exists: true + /var/lib/docker/containers: + exists: true + opts: + - ro + /usr/share/filebeat/filebeat.yml: + exists: true + opts: + - ro + +user: + filebeat: + exists: true + uid: 1000 + gid: 1000 + +http: + http://upgrade-master:9200/_cat/indices: + status: 200 + timeout: 2000 + body: + - "filebeat-8.0.0" + +file: + /usr/share/filebeat/filebeat.yml: + exists: true + contains: + - "add_kubernetes_metadata" + - "output.elasticsearch" + +command: + cd /usr/share/filebeat && filebeat test output: + exit-status: 0 + stdout: + - "elasticsearch: http://upgrade-master:9200" diff --git a/filebeat/examples/upgrade/values.yaml b/filebeat/examples/upgrade/values.yaml new file mode 100644 index 000000000..8b230601e --- /dev/null +++ b/filebeat/examples/upgrade/values.yaml @@ -0,0 +1,4 @@ +--- +extraEnvs: + - name: ELASTICSEARCH_HOSTS + value: upgrade-master:9200 diff --git a/helpers/matrix.yml b/helpers/matrix.yml index 7dc71f9d3..a8fd93a32 100644 --- a/helpers/matrix.yml +++ b/helpers/matrix.yml @@ -11,28 +11,34 @@ ES_SUITE: - multi - oss - security + - upgrade KIBANA_SUITE: - default - oss - security + - upgrade FILEBEAT_SUITE: - default - oss - security + - upgrade METRICBEAT_SUITE: - default - oss - security + #- upgrade TODO: uncomment after 7.10.0 release LOGSTASH_SUITE: - default - oss - elasticsearch - security + - upgrade APM_SERVER_SUITE: - default - oss - security + - upgrade KUBERNETES_VERSION: - - '1.15' - - '1.16' - - '1.17' + - "1.15" + - "1.16" + - "1.17" diff --git a/helpers/upgrade.sh b/helpers/upgrade.sh new file mode 100755 index 000000000..269b64d8a --- /dev/null +++ b/helpers/upgrade.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# +# upgrade.sh deploy some Helm chart to a specific released version, +# then upgrade it. +# +# An optional version can be specified for Docker image tag to use for upgrade. +# This is required for master branch because upgrade from Elasticsearch 7.X +# to 8.0.0-SNAPSHOT doesn't work. +# +set -euo pipefail + +TO="" + +usage() { + cat <<-EOF + USAGE: + $0 --chart --release --from [--to ] + $0 --help + + OPTIONS: + --chart + Name of the Elastic Helm chart to install (ie: elasticsearch) + --release + Name of the Helm release to install (ie: helm-upgrade-elasticsearch) + --from + Version to use for first install (ie: 7.7.0) + --to + Version of the Docker images to use for upgrade (ie: 7.10.0) + EOF + exit 1 +} + +while [[ $# -gt 0 ]] +do + key="$1" + + case $key in + --help) + usage + ;; + --chart) + CHART="$2" + shift 2 + ;; + --release) + RELEASE="$2" + shift 2 + ;; + --from) + FROM="$2" + shift 2 + ;; + --to) + TO="--set imageTag=$2" + shift 2 + ;; + *) + log "Unrecognized argument: '$key'" + usage + ;; + esac +done + +helm repo add elastic https://helm.elastic.co + +# Initial install +printf "Installing %s %s\n" "$RELEASE" "$FROM" +helm upgrade --wait --timeout=1200s --install --version "$FROM" --values values.yaml "$RELEASE" elastic/"$CHART" + +# Upgrade +printf "Upgrading %s\n" "$RELEASE" +# shellcheck disable=SC2086 +helm upgrade --wait --timeout=1200s --install --set terminationGracePeriod=121 $TO --values values.yaml "$RELEASE" ../../ diff --git a/kibana/examples/upgrade/Makefile b/kibana/examples/upgrade/Makefile new file mode 100644 index 000000000..6fcb2103e --- /dev/null +++ b/kibana/examples/upgrade/Makefile @@ -0,0 +1,16 @@ +default: test + +include ../../../helpers/examples.mk + +CHART := kibana +RELEASE := helm-kibana-upgrade +FROM := 7.4.0 # versions before 7.4.O aren't compatible with Kubernetes >= 1.16.0 +TO := 7.10.0 # required to use with Elasticsearch 7.10.0 + +install: + ../../../helpers/upgrade.sh --chart $(CHART) --release $(RELEASE) --from $(FROM) --to $(TO) + +test: install goss + +purge: + helm del $(RELEASE) diff --git a/kibana/examples/upgrade/README.md b/kibana/examples/upgrade/README.md new file mode 100644 index 000000000..ad28f65c6 --- /dev/null +++ b/kibana/examples/upgrade/README.md @@ -0,0 +1,21 @@ +# Upgrade + +This example will deploy Kibana chart using an old chart version, +then upgrade it. + + +## Usage + +* Add the Elastic Helm charts repo: `helm repo add elastic https://helm.elastic.co` + +* Deploy [Elasticsearch Helm chart][]: `helm install elasticsearch elastic/elasticsearch` + +* Deploy and upgrade Kibana chart with the default values: `make install` + + +## Testing + +You can also run [goss integration tests][] using `make test`. + + +[goss integration tests]: https://github.com/elastic/helm-charts/tree/master/kibana/examples/upgrade/test/goss.yaml diff --git a/kibana/examples/upgrade/test/goss.yaml b/kibana/examples/upgrade/test/goss.yaml new file mode 100644 index 000000000..5b3a1b28a --- /dev/null +++ b/kibana/examples/upgrade/test/goss.yaml @@ -0,0 +1,14 @@ +http: + http://localhost:5601/api/status: + status: 200 + timeout: 2000 + body: + - '"number":"7.10.0"' + + http://localhost:5601/app/kibana: + status: 200 + timeout: 2000 + + http://helm-kibana-upgrade-kibana:5601/app/kibana: + status: 200 + timeout: 2000 diff --git a/kibana/examples/upgrade/values.yaml b/kibana/examples/upgrade/values.yaml new file mode 100644 index 000000000..01d99c838 --- /dev/null +++ b/kibana/examples/upgrade/values.yaml @@ -0,0 +1,2 @@ +--- +elasticsearchHosts: "http://upgrade-master:9200" diff --git a/logstash/examples/upgrade/Makefile b/logstash/examples/upgrade/Makefile new file mode 100644 index 000000000..db739a141 --- /dev/null +++ b/logstash/examples/upgrade/Makefile @@ -0,0 +1,15 @@ +default: test + +include ../../../helpers/examples.mk + +CHART := logstash +RELEASE := helm-logstash-upgrade +FROM := 7.9.0 # upgrade from version < 7.9.0 is failing due to headless service breaking change + +install: + ../../../helpers/upgrade.sh --chart $(CHART) --release $(RELEASE) --from $(FROM) + +test: install goss + +purge: + helm del $(RELEASE) diff --git a/logstash/examples/upgrade/README.md b/logstash/examples/upgrade/README.md new file mode 100644 index 000000000..c8986a070 --- /dev/null +++ b/logstash/examples/upgrade/README.md @@ -0,0 +1,19 @@ +# Upgrade + +This example will deploy Logstash chart using an old chart version, +then upgrade it. + + +## Usage + +* Add the Elastic Helm charts repo: `helm repo add elastic https://helm.elastic.co` + +* Deploy and upgrade Logstash chart with the default values: `make install` + + +## Testing + +You can also run [goss integration tests][] using `make test`. + + +[goss integration tests]: https://github.com/elastic/helm-charts/tree/master/logstash/examples/upgrade/test/goss.yaml diff --git a/logstash/examples/upgrade/test/goss.yaml b/logstash/examples/upgrade/test/goss.yaml new file mode 100644 index 000000000..f58d38d05 --- /dev/null +++ b/logstash/examples/upgrade/test/goss.yaml @@ -0,0 +1,43 @@ +user: + logstash: + exists: true + uid: 1000 + gid: 1000 + +http: + http://localhost:9600?pretty: + status: 200 + timeout: 2000 + body: + - '"host" : "helm-logstash-upgrade-logstash-0"' + - '"version" : "8.0.0"' + - '"http_address" : "0.0.0.0:9600"' + - '"name" : "helm-logstash-upgrade-logstash-0"' + - '"status" : "green"' + - '"workers" : 1' + - '"batch_size" : 125' + - '"batch_delay" : 50' + +file: + /usr/share/logstash/config/logstash.yml: + exists: true + mode: "0644" + owner: logstash + group: root + filetype: file + contains: + - 'http.host: "0.0.0.0"' + - 'xpack.monitoring.elasticsearch.hosts: [ "http://elasticsearch:9200" ]' + /usr/share/logstash/pipeline/logstash.conf: + exists: true + mode: "0644" + owner: logstash + group: root + filetype: file + contains: + - "input {" + - "beats {" + - "port => 5044" + - "output {" + - "stdout {" + - "codec => rubydebug" diff --git a/logstash/examples/upgrade/values.yaml b/logstash/examples/upgrade/values.yaml new file mode 100644 index 000000000..ed97d539c --- /dev/null +++ b/logstash/examples/upgrade/values.yaml @@ -0,0 +1 @@ +--- diff --git a/metricbeat/examples/upgrade/Makefile b/metricbeat/examples/upgrade/Makefile new file mode 100644 index 000000000..d2d8d8509 --- /dev/null +++ b/metricbeat/examples/upgrade/Makefile @@ -0,0 +1,17 @@ +default: test + +include ../../../helpers/examples.mk + +CHART := metricbeat +RELEASE := helm-metricbeat-upgrade +#FROM := 7.10.0 # upgrade from version < 7.10.0 is failing due to selector + # breaking change in https://github.com/elastic/helm-charts/pull/516 + +install: + ../../../helpers/upgrade.sh --chart $(CHART) --release $(RELEASE) --from $(FROM) + +#TODO: uncomment after 7.10.0 release +test: #install goss + +purge: + helm del $(RELEASE) diff --git a/metricbeat/examples/upgrade/README.md b/metricbeat/examples/upgrade/README.md new file mode 100644 index 000000000..4c1821b1f --- /dev/null +++ b/metricbeat/examples/upgrade/README.md @@ -0,0 +1,21 @@ +# Upgrade + +This example will deploy Metricbeat chart using an old chart version, +then upgrade it. + + +## Usage + +* Add the Elastic Helm charts repo: `helm repo add elastic https://helm.elastic.co` + +* Deploy [Elasticsearch Helm chart][]: `helm install elasticsearch elastic/elasticsearch` + +* Deploy and upgrade Metricbeat chart with the default values: `make install` + + +## Testing + +You can also run [goss integration tests][] using `make test`. + + +[goss integration tests]: https://github.com/elastic/helm-charts/tree/master/metricbeat/examples/upgrade/test/goss.yaml diff --git a/metricbeat/examples/upgrade/test/goss-metrics.yaml b/metricbeat/examples/upgrade/test/goss-metrics.yaml new file mode 100644 index 000000000..ed1d30db3 --- /dev/null +++ b/metricbeat/examples/upgrade/test/goss-metrics.yaml @@ -0,0 +1,42 @@ +port: + tcp:5066: + listening: true + ip: + - "127.0.0.1" + +mount: + /usr/share/metricbeat/metricbeat.yml: + exists: true + opts: + - ro + +user: + metricbeat: + exists: true + uid: 1000 + gid: 1000 + +http: + http://upgrade-master:9200/_cat/indices: + status: 200 + timeout: 2000 + body: + - "metricbeat-7.10.0" + + ? "http://upgrade-master:9200/_search?q=metricset.name:state_container%20AND%20kubernetes.container.name:metricbeat" + : status: 200 + timeout: 2000 + body: + - "metricbeat-7.10.0" + +file: + /usr/share/metricbeat/metricbeat.yml: + exists: true + contains: + - "output.elasticsearch" + +command: + cd /usr/share/metricbeat && metricbeat test output: + exit-status: 0 + stdout: + - "elasticsearch: http://upgrade-master:9200" diff --git a/metricbeat/examples/upgrade/test/goss.yaml b/metricbeat/examples/upgrade/test/goss.yaml new file mode 100644 index 000000000..a0480583c --- /dev/null +++ b/metricbeat/examples/upgrade/test/goss.yaml @@ -0,0 +1,46 @@ +port: + tcp:5066: + listening: true + ip: + - "127.0.0.1" + +mount: + /usr/share/metricbeat/data: + exists: true + /run/docker.sock: + exists: true + /usr/share/metricbeat/metricbeat.yml: + exists: true + opts: + - ro + +user: + metricbeat: + exists: true + uid: 1000 + gid: 1000 + +http: + http://upgrade-master:9200/_cat/indices: + status: 200 + timeout: 2000 + body: + - "metricbeat-8.0.0" + ? "http://upgrade-master:9200/_search?q=metricset.name:container%20AND%20kubernetes.container.name:metricbeat" + : status: 200 + timeout: 2000 + body: + - "metricbeat-8.0.0" + +file: + /usr/share/metricbeat/metricbeat.yml: + exists: true + contains: + - "add_kubernetes_metadata" + - "output.elasticsearch" + +command: + cd /usr/share/metricbeat && metricbeat test output: + exit-status: 0 + stdout: + - "elasticsearch: http://upgrade-master:9200" diff --git a/metricbeat/examples/upgrade/values.yaml b/metricbeat/examples/upgrade/values.yaml new file mode 100644 index 000000000..8b230601e --- /dev/null +++ b/metricbeat/examples/upgrade/values.yaml @@ -0,0 +1,4 @@ +--- +extraEnvs: + - name: ELASTICSEARCH_HOSTS + value: upgrade-master:9200