diff --git a/.ci/jobs/elastic+helm-charts+pull-request+cluster-cleanup.yml b/.ci/jobs/elastic+helm-charts+pull-request+cluster-cleanup.yml index cbd0d55eb..e9dd14363 100644 --- a/.ci/jobs/elastic+helm-charts+pull-request+cluster-cleanup.yml +++ b/.ci/jobs/elastic+helm-charts+pull-request+cluster-cleanup.yml @@ -3,6 +3,7 @@ name: elastic+helm-charts+pull-request+cluster-cleanup display-name: elastic / helm-charts - pull-request - cluster cleanup description: Pull request - cluster cleanup + concurrent: true scm: - git: refspec: +refs/pull/*:refs/remotes/origin/pr/* diff --git a/.ci/jobs/elastic+helm-charts+pull-request+cluster-creation.yml b/.ci/jobs/elastic+helm-charts+pull-request+cluster-creation.yml index d1b491080..be63bfbcd 100644 --- a/.ci/jobs/elastic+helm-charts+pull-request+cluster-creation.yml +++ b/.ci/jobs/elastic+helm-charts+pull-request+cluster-creation.yml @@ -3,6 +3,7 @@ name: elastic+helm-charts+pull-request+cluster-creation display-name: elastic / helm-charts - pull-request - cluster creation description: Pull request - cluster creation + concurrent: true scm: - git: refspec: +refs/pull/*:refs/remotes/origin/pr/* diff --git a/.ci/jobs/elastic+helm-charts+pull-request+integration-apm-server.yml b/.ci/jobs/elastic+helm-charts+pull-request+integration-apm-server.yml index 55b331e9d..3a8caa100 100644 --- a/.ci/jobs/elastic+helm-charts+pull-request+integration-apm-server.yml +++ b/.ci/jobs/elastic+helm-charts+pull-request+integration-apm-server.yml @@ -3,6 +3,7 @@ name: elastic+helm-charts+pull-request+integration-apm-server display-name: elastic / helm-charts - pull-request - integration apm-server description: Pull request - integration apm-server + concurrent: true scm: - git: refspec: +refs/pull/*:refs/remotes/origin/pr/* diff --git a/.ci/jobs/elastic+helm-charts+pull-request+integration-elasticsearch.yml b/.ci/jobs/elastic+helm-charts+pull-request+integration-elasticsearch.yml index a476a1d85..459c4636a 100644 --- a/.ci/jobs/elastic+helm-charts+pull-request+integration-elasticsearch.yml +++ b/.ci/jobs/elastic+helm-charts+pull-request+integration-elasticsearch.yml @@ -3,6 +3,7 @@ name: elastic+helm-charts+pull-request+integration-elasticsearch display-name: elastic / helm-charts - pull-request - integration elasticsearch description: Pull request - integration elasticsearch + concurrent: true scm: - git: refspec: +refs/pull/*:refs/remotes/origin/pr/* diff --git a/.ci/jobs/elastic+helm-charts+pull-request+integration-filebeat.yml b/.ci/jobs/elastic+helm-charts+pull-request+integration-filebeat.yml index 3a1621fbb..d49a5560a 100644 --- a/.ci/jobs/elastic+helm-charts+pull-request+integration-filebeat.yml +++ b/.ci/jobs/elastic+helm-charts+pull-request+integration-filebeat.yml @@ -3,6 +3,7 @@ name: elastic+helm-charts+pull-request+integration-filebeat display-name: elastic / helm-charts - pull-request - integration filebeat description: Pull request - integration filebeat + concurrent: true scm: - git: refspec: +refs/pull/*:refs/remotes/origin/pr/* diff --git a/.ci/jobs/elastic+helm-charts+pull-request+integration-kibana.yml b/.ci/jobs/elastic+helm-charts+pull-request+integration-kibana.yml index ed2e6ec4a..0127166ff 100644 --- a/.ci/jobs/elastic+helm-charts+pull-request+integration-kibana.yml +++ b/.ci/jobs/elastic+helm-charts+pull-request+integration-kibana.yml @@ -3,6 +3,7 @@ name: elastic+helm-charts+pull-request+integration-kibana display-name: elastic / helm-charts - pull-request - integration kibana description: Pull request - integration kibana + concurrent: true scm: - git: refspec: +refs/pull/*:refs/remotes/origin/pr/* diff --git a/.ci/jobs/elastic+helm-charts+pull-request+integration-logstash.yml b/.ci/jobs/elastic+helm-charts+pull-request+integration-logstash.yml index bf7066426..5631b5ae6 100644 --- a/.ci/jobs/elastic+helm-charts+pull-request+integration-logstash.yml +++ b/.ci/jobs/elastic+helm-charts+pull-request+integration-logstash.yml @@ -3,6 +3,7 @@ name: elastic+helm-charts+pull-request+integration-logstash display-name: elastic / helm-charts - pull-request - integration logstash description: Pull request - integration logstash + concurrent: true scm: - git: refspec: +refs/pull/*:refs/remotes/origin/pr/* diff --git a/.ci/jobs/elastic+helm-charts+pull-request+integration-metricbeat.yml b/.ci/jobs/elastic+helm-charts+pull-request+integration-metricbeat.yml index 34c6ac2fb..f37ec5ca0 100644 --- a/.ci/jobs/elastic+helm-charts+pull-request+integration-metricbeat.yml +++ b/.ci/jobs/elastic+helm-charts+pull-request+integration-metricbeat.yml @@ -3,6 +3,7 @@ name: elastic+helm-charts+pull-request+integration-metricbeat display-name: elastic / helm-charts - pull-request - integration metricbeat description: Pull request - integration metricbeat + concurrent: true scm: - git: refspec: +refs/pull/*:refs/remotes/origin/pr/* diff --git a/.ci/jobs/elastic+helm-charts+pull-request+lint-python.yml b/.ci/jobs/elastic+helm-charts+pull-request+lint-python.yml index 5ec00f723..a956fd5c5 100644 --- a/.ci/jobs/elastic+helm-charts+pull-request+lint-python.yml +++ b/.ci/jobs/elastic+helm-charts+pull-request+lint-python.yml @@ -3,6 +3,7 @@ name: elastic+helm-charts+pull-request+lint-python display-name: elastic / helm-charts - pull-request - lint python description: Pull request - lint python + concurrent: true scm: - git: refspec: +refs/pull/*:refs/remotes/origin/pr/* diff --git a/.ci/jobs/elastic+helm-charts+pull-request+template-testing.yml b/.ci/jobs/elastic+helm-charts+pull-request+template-testing.yml index 4f0a3ae93..29a1ed7e2 100644 --- a/.ci/jobs/elastic+helm-charts+pull-request+template-testing.yml +++ b/.ci/jobs/elastic+helm-charts+pull-request+template-testing.yml @@ -3,6 +3,7 @@ name: elastic+helm-charts+pull-request+template-testing display-name: elastic / helm-charts - pull-request - template testing description: Pull request - template testing + concurrent: true scm: - git: refspec: +refs/pull/*:refs/remotes/origin/pr/* diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 61de9e089..46fd7c261 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -18,7 +18,7 @@ e.g. `helm get elasticsearch` (replace `elasticsearch` with the name of your hel *Be careful to obfuscate every secrets (credentials, token, public IP, ...) that could be visible in the output before copy-pasting.* -*If you find some secrets in plain text in `helm get release` output you should use [Kubernetes Secrets](https://kubernetes.io/docs/concepts/configuration/secret/) to managed them is a secure way (see [Security Example](https://github.com/elastic/helm-charts/blob/master/elasticsearch/examples/security/security.yml#L23-L38)).* +*If you find some secrets in plain text in `helm get release` output you should use [Kubernetes Secrets](https://kubernetes.io/docs/concepts/configuration/secret/) to managed them is a secure way (see [Security Example](https://github.com/elastic/helm-charts/blob/master/elasticsearch/examples/security/values.yaml#L23-L38)).*
Output of helm get release diff --git a/BREAKING_CHANGES.md b/BREAKING_CHANGES.md index e4cddb1d2..90c04c386 100644 --- a/BREAKING_CHANGES.md +++ b/BREAKING_CHANGES.md @@ -3,6 +3,11 @@ + - [7.10.0](#7100) + - [Migration to Helm 3](#migration-to-helm-3) + - [Metricbeat upgrade](#metricbeat-upgrade) + - [Elasticsearch upgrade with persistence.labels.enabled](#elasticsearch-upgrade-with-persistencelabelsenabled) + - [Rendered manifests contain a resource that already exists error](#rendered-manifests-contain-a-resource-that-already-exists-error) - [7.9.3 - 2020/10/22](#793---20201022) - [Fix Logstash headless Service (end)](#fix-logstash-headless-service-end) - [6.8.13 - 2020/10/22](#6813---20201022) @@ -39,6 +44,87 @@ +## 7.10.0 + +### Migration to Helm 3 + +Starting from the 7.10.0 release, Helm 3 is fully supported in Elastic Helm charts +and Helm 2 is deprecated. + +In most cases, [Helm 2to3][] can be used to migrate from previous charts +releases deployed with Helm 2: + +```shell +# Install Helm 3 + +# Install 2to3 plugin +helm plugin install https://github.com/helm/helm-2to3.git + +# Migrate Helm 2 local config +helm3 2to3 move config + +# Migrate Helm 2 releases +helm3 2to3 convert + +# Upgrade to 7.10.0 +helm upgrade elastic/ --version 7.10.0 + +# Cleanup Helm 2 datas +helm3 2to3 cleanup +``` + +Migration to Helm 3 with 7.10.0 charts release should work smoothly for the +following charts. + +- apm-server >= 7.6.0 +- elasticsearch >= 7.4.0 (except when `persistence.labels.enabled` is true) +- filebeat >= 7.9.0 +- kibana >= 7.4.0 +- logstash >= 7.9.0 + +#### Metricbeat upgrade + +Metricbeat 7.10.0 introduce a breaking change in [#516][] to make it compatible +with Helm 3. + +The removing of some `heritage` labels in Metricbeat deployment make upgrade +fail with the following error: + +``` +UPGRADE FAILED +Error: Deployment.apps "mb-metricbeat-metrics" is invalid: spec.selector: Invalid value: v1.LabelSelector{MatchLabels:map[string]string{"app":"mb-metricbeat-metrics", "release":"mb"}, MatchExpressions:[]v1.LabelSelectorRequirement(nil)}: field is immutable +Error: UPGRADE FAILED: Deployment.apps "mb-metricbeat-metrics" is invalid: spec.selector: Invalid value: v1.LabelSelector{MatchLabels:map[string]string{"app":"mb-metricbeat-metrics", "release":"mb"}, MatchExpressions:[]v1.LabelSelectorRequirement(nil)}: field is immutable +``` + +Unfortunately using `helm upgrade --force` with Helm 3 is not enough. This chart +will need to be uninstalled and re-installed. + +#### Elasticsearch upgrade with persistence.labels.enabled + +If you are using `persistence.labels.enabled=true` with Elasticsearch, upgrade +will fail even with `--force`. + +You'll need to deploy a new release with the same `clusterName` using Helm 3, +migrate your data, then remove the old release. + +#### Rendered manifests contain a resource that already exists error + +We experimented some `rendered manifests contain a resource that already exists` +errors with some charts upgrade, mostly for charts deploying `ClusterRole` and +`ClusterRoleBinding`resources. + +Helm 3 is automatically adding some new annotations (`meta.helm.sh/release-name` +and `meta.helm.sh/release-namespace`) and labels (`app.kubernetes.io/managed-by`) +to the charts resources during upgrade. However it sometimes fail to add the annotations with the below error: + +``` +Error: UPGRADE FAILED: rendered manifests contain a resource that already exists. Unable to continue with update: ClusterRole "apm-apm-server-cluster-role" in namespace "" exists and cannot be imported into the current release: invalid ownership metadata; label validation error: missing key "app.kubernetes.io/managed-by": must be set to "Helm"; annotation validation error: missing key "meta.helm.sh/release-name": must be set to "apm"; annotation validation error: missing key "meta.helm.sh/release-namespace": must be set to "default" +``` + +The workaround is to manually add these annotations and labels to the existing failing resources +using `kubectl edit` for example, then relaunch the upgrade command. + + ## 7.9.3 - 2020/10/22 ### Fix Logstash headless Service (end) @@ -57,7 +143,7 @@ See [7.9.3 - 2020/10/22](#793---20201022) and [7.9.1 - 2020/09/03](#791---202009 ### Fix Logstash headless Service [#776][] fixed an issue with headless `Service` when using `extraPorts` value -(see [Add headless Service for StatefulSet](#add-headless-service-for-statefulset) +(see [Add headless Service for StatefulSet](#add-headless-service-for-statefulset) for more details). Unfortunately, it introduced a new bug when using a `NodePort` `Service` ([#807][]). This is fixed by [#839][] in 7.9.3 (and 6.8.13). @@ -290,6 +376,7 @@ volumeClaimTemplate: [#352]: https://github.com/elastic/helm-charts/pull/352 [#437]: https://github.com/elastic/helm-charts/pull/437 [#458]: https://github.com/elastic/helm-charts/pull/458 +[#516]: https://github.com/elastic/helm/pull/516 [#540]: https://github.com/elastic/helm-charts/pull/540 [#568]: https://github.com/elastic/helm-charts/pull/568 [#572]: https://github.com/elastic/helm-charts/pull/572 @@ -307,6 +394,7 @@ volumeClaimTemplate: [elastic elasticsearch chart]: https://github.com/elastic/helm-charts/tree/master/elasticsearch [elastic helm repo]: https://helm.elastic.co [github releases]: https://github.com/elastic/helm-charts/releases +[helm 2to3]: https://helm.sh/blog/migrate-from-helm-v2-to-helm-v3/ [migration guide]: https://github.com/elastic/helm-charts/blob/master/elasticsearch/examples/migration/README.md [new branching model]: https://github.com/elastic/helm-charts/blob/master/CONTRIBUTING.md#branching [kube-state-metrics]: https://github.com/helm/charts/tree/master/stable/kube-state-metrics diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b8879472..a9310835f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,29 @@ # Changelog +## 7.10.0 + +* 7.10.0 as default version. + + +| PR | Author | Title | +| --- | --- | --- | +| [#927](https://github.com/elastic/helm-charts/pull/927) | [@jmlrt](https://github.com/jmlrt) | [meta] Remove version from dev install section title | +| [#922](https://github.com/elastic/helm-charts/pull/922) | [@jmlrt](https://github.com/jmlrt) | [meta] Remove support for k8s <1.14 & helm <2.17.0 (#916) [7.10] | +| [#919](https://github.com/elastic/helm-charts/pull/919) | [@jmlrt](https://github.com/jmlrt) | [meta] Upgrade test (#907) [7.10] | +| [#914](https://github.com/elastic/helm-charts/pull/914) | [@jmlrt](https://github.com/jmlrt) | [meta] Initiate 7.10 branch | +| [#897](https://github.com/elastic/helm-charts/pull/897) | [@cospeedster](https://github.com/cospeedster) | [elasticsearch] Fix spelling | +| [#911](https://github.com/elastic/helm-charts/pull/911) | [@jmlrt](https://github.com/jmlrt) | [elasticsearch] Update test hook annotations | +| [#910](https://github.com/elastic/helm-charts/pull/910) | [@jmlrt](https://github.com/jmlrt) | [meta] Add link to eck chart doc | +| [#902](https://github.com/elastic/helm-charts/pull/902) | [@jmlrt](https://github.com/jmlrt) | [meta] Helm 3 (#516) | +| [#891](https://github.com/elastic/helm-charts/pull/891) | [@jmlrt](https://github.com/jmlrt) | [meta] Increase helm timeout | +| [#890](https://github.com/elastic/helm-charts/pull/890) | [@jmlrt](https://github.com/jmlrt) | [meta] Update rbac.authorization.k8s.io api | +| [#887](https://github.com/elastic/helm-charts/pull/887) | [@nkammah](https://github.com/nkammah) | [meta] Add warning comment placeholder (7.x branch) | +| [#882](https://github.com/elastic/helm-charts/pull/882) | [@jmlrt](https://github.com/jmlrt) | [metricbeat] Use relocated stable repo for kube-state-metrics | +| [#880](https://github.com/elastic/helm-charts/pull/880) | [@jmlrt](https://github.com/jmlrt) | [meta] Add support for helm 2.17.0 and k8s 1.17 | +| [#854](https://github.com/elastic/helm-charts/pull/854) | [@jmlrt](https://github.com/jmlrt) | [elasticsearch] Add coordinator node to multi test | +| [#860](https://github.com/elastic/helm-charts/pull/860) | [@nkammah](https://github.com/nkammah) | [meta] Simplify doc in 7.x branch | + + ## 7.9.3 * 7.9.3 as default version. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bb964b421..eb82e009e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -195,8 +195,6 @@ Charts are released from version branchs (example `7.7` branch). [Elastic Helm repository][] is updated only during releases. -The current release process is documented in [release.md][]. - ## Testing @@ -267,6 +265,5 @@ make goss [serverspec]: https://serverspec.org [templating test example]: https://github.com/elastic/helm-charts/blob/master/elasticsearch/tests/elasticsearch_test.py [templating tests section]: #templating-tests -[release.md]: https://github.com/elastic/helm-charts/blob/master/helpers/release.md [releases section]: #releases [requirements.txt]: https://github.com/elastic/helm-charts/blob/master/requirements.txt diff --git a/README.md b/README.md index 660a6fbe9..9dfa5edee 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,6 @@ - [Support Matrix](#support-matrix) - [Kubernetes Versions](#kubernetes-versions) - [Helm versions](#helm-versions) - - [Helm 3 beta](#helm-3-beta) - [ECK](#eck) @@ -36,14 +35,14 @@ The `master` version of these charts are intended to support the latest pre-rele versions of our products, and therefore may or may not work with current released versions. -| Chart | Docker documentation |Latest 7 Version|Latest 6 Version| -|--------------------------------------------|---------------------------------------------------------------------------------|-----------|-----------| -| [APM-Server](./apm-server/README.md) | https://www.elastic.co/guide/en/apm/server/current/running-on-docker.html |[`7.9.3`][apm-7] |[`6.8.13`][apm-6] | -| [Elasticsearch](./elasticsearch/README.md) | https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html |[`7.9.3`][elasticsearch-7] |[`6.8.13`][elasticsearch-6] | -| [Filebeat](./filebeat/README.md) | https://www.elastic.co/guide/en/beats/filebeat/current/running-on-docker.html |[`7.9.3`][filebeat-7] |[`6.8.13`][filebeat-6] | -| [Kibana](./kibana/README.md) | https://www.elastic.co/guide/en/kibana/current/docker.html |[`7.9.3`][kibana-7] |[`6.8.13`][kibana-6] | -| [Logstash](./logstash/README.md) | https://www.elastic.co/guide/en/logstash/current/docker.html |[`7.9.3`][logstash-7] |[`6.8.13`][logstash-6] | -| [Metricbeat](./metricbeat/README.md) | https://www.elastic.co/guide/en/beats/metricbeat/current/running-on-docker.html |[`7.9.3`][metricbeat-7] |[`6.8.13`][metricbeat-6] | +| Chart | Docker documentation | Latest 7 Version | Latest 6 Version | +|--------------------------------------------|---------------------------------------------------------------------------------|----------------------------|-----------------------------| +| [APM-Server](./apm-server/README.md) | https://www.elastic.co/guide/en/apm/server/current/running-on-docker.html | [`7.10.0`][apm-7] | [`6.8.13`][apm-6] | +| [Elasticsearch](./elasticsearch/README.md) | https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html | [`7.10.0`][elasticsearch-7] | [`6.8.13`][elasticsearch-6] | +| [Filebeat](./filebeat/README.md) | https://www.elastic.co/guide/en/beats/filebeat/current/running-on-docker.html | [`7.10.0`][filebeat-7] | [`6.8.13`][filebeat-6] | +| [Kibana](./kibana/README.md) | https://www.elastic.co/guide/en/kibana/current/docker.html | [`7.10.0`][kibana-7] | [`6.8.13`][kibana-6] | +| [Logstash](./logstash/README.md) | https://www.elastic.co/guide/en/logstash/current/docker.html | [`7.10.0`][logstash-7] | [`6.8.13`][logstash-6] | +| [Metricbeat](./metricbeat/README.md) | https://www.elastic.co/guide/en/beats/metricbeat/current/running-on-docker.html | [`7.10.0`][metricbeat-7] | [`6.8.13`][metricbeat-6] | ## Supported Configurations @@ -77,40 +76,37 @@ exact versions are defined under `KUBERNETES_VERSIONS` in ### Helm versions While we are checking backward compatibility, the charts are only tested with -Helm version mentioned in [helm-tester Dockerfile][] (currently 2.16.12). +Helm version mentioned in [helm-tester Dockerfile][] (currently 3.4.1). -#### Helm 3 beta - -While we don't have automated tests for [Helm 3][] yet, we fixed the main -blockers to use it. We now have enough feedbacks from internal and external -users to add support in beta. ## ECK In addition of these Helm charts, Elastic also provides [Elastic Cloud on Kubernetes][] which is based on [Operator pattern][] and is Elastic recommended way to deploy Elasticsearch, Kibana and APM Server on -Kubernetes. +Kubernetes. There is a dedicated Helm chart for ECK which can be found +[in ECK repo][eck-chart] ([documentation][eck-chart-doc]). [currently tested]: https://devops-ci.elastic.co/job/elastic+helm-charts+master/ +[eck-chart]: https://github.com/elastic/cloud-on-k8s/tree/master/deploy +[eck-chart-doc]: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-install-helm.html [elastic cloud on kubernetes]: https://github.com/elastic/cloud-on-k8s [elastic helm repo]: https://helm.elastic.co [github releases]: https://github.com/elastic/helm-charts/releases -[helm 3]: https://v3.helm.sh [helm-tester Dockerfile]: https://github.com/elastic/helm-charts/blob/master/helpers/helm-tester/Dockerfile [helpers/matrix.yml]: https://github.com/elastic/helm-charts/blob/master/helpers/matrix.yml [operator pattern]: https://kubernetes.io/docs/concepts/extend-kubernetes/operator/ [elasticsearch-771]: https://github.com/elastic/helm-charts/tree/7.7.1/elasticsearch/ -[apm-7]: https://github.com/elastic/helm-charts/tree/7.9.3/apm-server/README.md +[apm-7]: https://github.com/elastic/helm-charts/tree/7.10.0/apm-server/README.md [apm-6]: https://github.com/elastic/helm-charts/tree/6.8.13/apm-server/README.md -[elasticsearch-7]: https://github.com/elastic/helm-charts/tree/7.9.3/elasticsearch/README.md +[elasticsearch-7]: https://github.com/elastic/helm-charts/tree/7.10.0/elasticsearch/README.md [elasticsearch-6]: https://github.com/elastic/helm-charts/tree/6.8.13/elasticsearch/README.md -[filebeat-7]: https://github.com/elastic/helm-charts/tree/7.9.3/filebeat/README.md +[filebeat-7]: https://github.com/elastic/helm-charts/tree/7.10.0/filebeat/README.md [filebeat-6]: https://github.com/elastic/helm-charts/tree/6.8.13/filebeat/README.md -[kibana-7]: https://github.com/elastic/helm-charts/tree/7.9.3/kibana/README.md +[kibana-7]: https://github.com/elastic/helm-charts/tree/7.10.0/kibana/README.md [kibana-6]: https://github.com/elastic/helm-charts/tree/6.8.13/kibana/README.md -[logstash-7]: https://github.com/elastic/helm-charts/tree/7.9.3/logstash/README.md +[logstash-7]: https://github.com/elastic/helm-charts/tree/7.10.0/logstash/README.md [logstash-6]: https://github.com/elastic/helm-charts/tree/6.8.13/logstash/README.md -[metricbeat-7]: https://github.com/elastic/helm-charts/tree/7.9.3/metricbeat/README.md +[metricbeat-7]: https://github.com/elastic/helm-charts/tree/7.10.0/metricbeat/README.md [metricbeat-6]: https://github.com/elastic/helm-charts/tree/6.8.13/metricbeat/README.md diff --git a/apm-server/README.md b/apm-server/README.md index 22eef6497..c1b3cd576 100644 --- a/apm-server/README.md +++ b/apm-server/README.md @@ -8,7 +8,9 @@ The design and code is less mature than official GA features and is being provided as-is with no warranties. Alpha features are not subject to the support SLA of official GA features (see [supported configurations][] for more details). + **Warning**: This branch is used for development, please use the latest [7.x][] release for released version. + @@ -32,8 +34,8 @@ SLA of official GA features (see [supported configurations][] for more details). ## Requirements -* Kubernetes >= 1.9 -* [Helm][] >= 2.8.0 +* Kubernetes >= 1.14 +* [Helm][] >= 2.17.0 See [supported configurations][] for more details. @@ -46,8 +48,8 @@ See [supported configurations][] for more details. `helm repo add elastic https://helm.elastic.co` * Install it: - - with Helm 2: `helm install --name apm-server elastic/apm-server` - - with [Helm 3 (beta)][]: `helm install apm-server elastic/apm-server` + - with Helm 3: `helm install apm-server elastic/apm-server` + - with Helm 2 (deprecated): `helm install --name apm-server elastic/apm-server` ### Install development version using master branch @@ -55,8 +57,8 @@ See [supported configurations][] for more details. * Clone the git repo: `git clone git@github.com:elastic/helm-charts.git` * Install it: - - with Helm 2: `helm install --name apm-server ./helm-charts/apm-server --set imageTag=8.0.0-SNAPSHOT` - - with [Helm 3 (beta)][]: `helm install apm-server ./helm-charts/apm-server --set imageTag=8.0.0-SNAPSHOT` + - with Helm 3: `helm install apm-server ./helm-charts/apm-server --set imageTag=8.0.0-SNAPSHOT` + - with Helm 2 (deprecated): `helm install --name apm-server ./helm-charts/apm-server --set imageTag=8.0.0-SNAPSHOT` ## Upgrading @@ -80,42 +82,42 @@ as a reference. They are also used in the automated testing of this chart. ## Configuration -| Parameter | Description | Default | -|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------| -| `affinity` | Configurable [affinity][] | `{}` | -| `apmConfig` | Allows you to add any config files in `/usr/share/apm-server/config` such as `apm-server.yml` | see [values.yaml][] | -| `autoscaling` | Enable the [horizontal pod autoscaler][] | `enabled: false` | -| `envFrom` | Templatable string to be passed to the [environment from variables][] which will be appended to the `envFrom:` definition for the container | `[]` | -| `extraContainers` | Templatable string of additional containers to be passed to the `tpl` function | `""` | -| `extraEnvs` | Extra [environment variables][] which will be appended to the `env:` definition for the container | `[]` | -| `extraInitContainers` | Templatable string of additional containers to be passed to the `tpl` function | `""` | -| `extraVolumeMounts` | List of additional `volumeMounts` | `[]` | -| `extraVolumes` | List of additional `volumes` | `[]` | -| `fullnameOverride` | Overrides the full name of the resources. If not set the name will default to `.Release.Name` - `.Values.nameOverride` or `.Chart.Name` | `""` | -| `imagePullPolicy` | The Kubernetes [imagePullPolicy][] value | `IfNotPresent` | -| `imagePullSecrets` | Configuration for [imagePullSecrets][] so that you can use a private registry for your image | `[]` | -| `imageTag` | The APM Server Docker image tag | `8.0.0-SNAPSHOT` | -| `image` | The APM Server Docker image | `docker.elastic.co/apm/apm-server` | -| `ingress` | Configurable [ingress][] to expose the APM Server service | see [values.yaml][] | -| `labels` | Configurable [labels][] applied to all APM server pods | `{}` | -| `lifecycle` | Configurable [lifecycle hooks][] | `false` | -| `livenessProbe` | Parameters to pass to liveness [probe][] checks for values such as timeouts and thresholds | see [values.yaml][] | -| `managedServiceAccount` | Whether the `serviceAccount` should be managed by this Helm chart. Set this to `false` in order to manage your own service account and related roles | `true` | -| `nameOverride` | Overrides the chart name for resources. If not set the name will default to `.Chart.Name` | `""` | -| `nodeSelector` | Configurable [nodeSelector][] | `{}` | -| `podAnnotations` | Configurable [annotations][] applied to all APM Server pods | `{}` | -| `podSecurityContext` | Configurable [podSecurityContext][] for APM Server pod execution environment | see [values.yaml][] | -| `priorityClassName` | The name of the [PriorityClass][]. No default is supplied as the `PriorityClass` must be created first | `""` | -| `readinessProbe` | Parameters to pass to readiness [probe][] checks for values such as timeouts and thresholds | see [values.yaml][] | -| `replicas` | Number of APM servers to run | `1` | -| `resources` | Allows you to set the [resources][] for the `Deployment` | see [values.yaml][] | -| `secretMounts` | Allows you easily mount a secret as a file inside the `Deployment`. Useful for mounting certificates and other secrets. See [values.yaml][] for an example | `[]` | -| `serviceAccount` | Custom [serviceAccount][] that APM Server will use during execution. By default will use the `serviceAccount` created by this chart | `""` | -| `serviceAccountAnnotations` | Annotations to be added to the ServiceAccount that is created by this chart. | `{}` -| `service` | Configurable [service][] to expose the APM Server service. See [values.yaml][] for an example | see [values.yaml][] | -| `terminationGracePeriod` | Termination period (in seconds) to wait before killing APM Server pod process on pod shutdown | `30` | -| `tolerations` | Configurable [tolerations][] | `[]` | -| `updateStrategy` | Allows you to change the default [updateStrategy][] for the deployment | see [values.yaml][] | +| Parameter | Description | Default | +|-----------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------| +| `affinity` | Configurable [affinity][] | `{}` | +| `apmConfig` | Allows you to add any config files in `/usr/share/apm-server/config` such as `apm-server.yml` | see [values.yaml][] | +| `autoscaling` | Enable the [horizontal pod autoscaler][] | see [values.yaml][] | +| `envFrom` | Templatable string to be passed to the [environment from variables][] which will be appended to the `envFrom:` definition for the container | `[]` | +| `extraContainers` | Templatable string of additional containers to be passed to the `tpl` function | `""` | +| `extraEnvs` | Extra [environment variables][] which will be appended to the `env:` definition for the container | `[]` | +| `extraInitContainers` | Templatable string of additional containers to be passed to the `tpl` function | `""` | +| `extraVolumeMounts` | List of additional `volumeMounts` | `[]` | +| `extraVolumes` | List of additional `volumes` | `[]` | +| `fullnameOverride` | Overrides the full name of the resources. If not set the name will default to `.Release.Name` - `.Values.nameOverride` or `.Chart.Name` | `""` | +| `imagePullPolicy` | The Kubernetes [imagePullPolicy][] value | `IfNotPresent` | +| `imagePullSecrets` | Configuration for [imagePullSecrets][] so that you can use a private registry for your image | `[]` | +| `imageTag` | The APM Server Docker image tag | `8.0.0-SNAPSHOT` | +| `image` | The APM Server Docker image | `docker.elastic.co/apm/apm-server` | +| `ingress` | Configurable [ingress][] to expose the APM Server service | see [values.yaml][] | +| `labels` | Configurable [labels][] applied to all APM server pods | `{}` | +| `lifecycle` | Configurable [lifecycle hooks][] | `false` | +| `livenessProbe` | Parameters to pass to liveness [probe][] checks for values such as timeouts and thresholds | see [values.yaml][] | +| `managedServiceAccount` | Whether the `serviceAccount` should be managed by this Helm chart. Set this to `false` in order to manage your own service account and related roles | `true` | +| `nameOverride` | Overrides the chart name for resources. If not set the name will default to `.Chart.Name` | `""` | +| `nodeSelector` | Configurable [nodeSelector][] | `{}` | +| `podAnnotations` | Configurable [annotations][] applied to all APM Server pods | `{}` | +| `podSecurityContext` | Configurable [podSecurityContext][] for APM Server pod execution environment | see [values.yaml][] | +| `priorityClassName` | The name of the [PriorityClass][]. No default is supplied as the `PriorityClass` must be created first | `""` | +| `readinessProbe` | Parameters to pass to readiness [probe][] checks for values such as timeouts and thresholds | see [values.yaml][] | +| `replicas` | Number of APM servers to run | `1` | +| `resources` | Allows you to set the [resources][] for the `Deployment` | see [values.yaml][] | +| `secretMounts` | Allows you easily mount a secret as a file inside the `Deployment`. Useful for mounting certificates and other secrets. See [values.yaml][] for an example | `[]` | +| `serviceAccount` | Custom [serviceAccount][] that APM Server will use during execution. By default will use the `serviceAccount` created by this chart | `""` | +| `serviceAccountAnnotations` | Annotations to be added to the ServiceAccount that is created by this chart. | `{}` | +| `service` | Configurable [service][] to expose the APM Server service. See [values.yaml][] for an example | see [values.yaml][] | +| `terminationGracePeriod` | Termination period (in seconds) to wait before killing APM Server pod process on pod shutdown | `30` | +| `tolerations` | Configurable [tolerations][] | `[]` | +| `updateStrategy` | Allows you to change the default [updateStrategy][] for the deployment | see [values.yaml][] | ## FAQ @@ -158,7 +160,6 @@ about our development and testing process. [examples/oss]: https://github.com/elastic/helm-charts/tree/master/apm-server/examples/oss [examples/security]: https://github.com/elastic/helm-charts/tree/master/apm-server/examples/security [helm]: https://helm.sh -[helm 3 (beta)]: https://github.com/elastic/helm-charts/tree/master/README.md#helm-3-beta [horizontal pod autoscaler]: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/ [imagePullPolicy]: https://kubernetes.io/docs/concepts/containers/images/#updating-images [imagePullSecrets]: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-pod-that-uses-your-secret diff --git a/apm-server/examples/default/Makefile b/apm-server/examples/default/Makefile index 23a7eedc6..5658ff024 100644 --- a/apm-server/examples/default/Makefile +++ b/apm-server/examples/default/Makefile @@ -5,9 +5,9 @@ include ../../../helpers/examples.mk RELEASE := helm-apm-server-default install: - helm upgrade --wait --timeout=600 --install $(RELEASE) ../../ + helm upgrade --wait --timeout=$(TIMEOUT) --install $(RELEASE) ../../ test: install goss purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/apm-server/examples/oss/Makefile b/apm-server/examples/oss/Makefile index 175317b98..686864878 100644 --- a/apm-server/examples/oss/Makefile +++ b/apm-server/examples/oss/Makefile @@ -5,9 +5,9 @@ include ../../../helpers/examples.mk RELEASE := helm-apm-server-oss install: - helm upgrade --wait --timeout=600 --install $(RELEASE) --values values.yaml ../../ + helm upgrade --wait --timeout=$(TIMEOUT) --install --values values.yaml $(RELEASE) ../../ test: install goss purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/apm-server/examples/security/Makefile b/apm-server/examples/security/Makefile index 2e9be1f86..47cd9dd3b 100644 --- a/apm-server/examples/security/Makefile +++ b/apm-server/examples/security/Makefile @@ -5,9 +5,9 @@ include ../../../helpers/examples.mk RELEASE := helm-apm-server-security install: - helm upgrade --wait --timeout=600 --install $(RELEASE) --values values.yaml ../../ + helm upgrade --wait --timeout=$(TIMEOUT) --install --values values.yaml $(RELEASE) ../../ test: install goss purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/apm-server/examples/upgrade/Makefile b/apm-server/examples/upgrade/Makefile new file mode 100644 index 000000000..fa16a4d48 --- /dev/null +++ b/apm-server/examples/upgrade/Makefile @@ -0,0 +1,16 @@ +default: test + +include ../../../helpers/examples.mk + +CHART := apm-server +RELEASE := helm-apm-server-upgrade +FROM := 7.6.0 # 7.6.0 is the first release for apm-server + +install: + ../../../helpers/upgrade.sh --chart $(CHART) --release $(RELEASE) --from $(FROM) + kubectl rollout status deployment $(RELEASE)-apm-server + +test: install goss + +purge: + helm del $(RELEASE) diff --git a/apm-server/examples/upgrade/README.md b/apm-server/examples/upgrade/README.md new file mode 100644 index 000000000..835b3c7ed --- /dev/null +++ b/apm-server/examples/upgrade/README.md @@ -0,0 +1,21 @@ +# Upgrade + +This example will deploy APM Server chart using an old chart version, +then upgrade it. + + +## Usage + +* Add the Elastic Helm charts repo: `helm repo add elastic https://helm.elastic.co` + +* Deploy [Elasticsearch Helm chart][]: `helm install elasticsearch elastic/elasticsearch` + +* Deploy and upgrade APM Server chart with the default values: `make install` + + +## Testing + +You can also run [goss integration tests][] using `make test`. + + +[goss integration tests]: https://github.com/elastic/helm-charts/tree/master/apm-server/examples/upgrade/test/goss.yaml diff --git a/apm-server/examples/upgrade/test/goss.yaml b/apm-server/examples/upgrade/test/goss.yaml new file mode 100644 index 000000000..5dbc51660 --- /dev/null +++ b/apm-server/examples/upgrade/test/goss.yaml @@ -0,0 +1,6 @@ +http: + http://localhost:8200?pretty: + status: 200 + timeout: 2000 + body: + - "8.0.0" diff --git a/apm-server/examples/upgrade/values.yaml b/apm-server/examples/upgrade/values.yaml new file mode 100644 index 000000000..4b66615c5 --- /dev/null +++ b/apm-server/examples/upgrade/values.yaml @@ -0,0 +1,12 @@ +--- +apmConfig: + apm-server.yml: | + apm-server: + host: "0.0.0.0:8200" + + queue: {} + output.file: + enabled: false + + output.elasticsearch: + hosts: ["http://upgrade-master:9200"] diff --git a/apm-server/templates/_helpers.tpl b/apm-server/templates/_helpers.tpl index abc1361ea..d36af2029 100755 --- a/apm-server/templates/_helpers.tpl +++ b/apm-server/templates/_helpers.tpl @@ -19,23 +19,6 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this {{- end -}} {{- end -}} -{{/* -Return the appropriate apiVersion for ingress. -*/}} -{{- define "apm.ingress.apiVersion" -}} -{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} -{{- print "extensions/v1beta1" -}} -{{- else -}} -{{- print "networking.k8s.io/v1beta1" -}} -{{- end -}} -{{- end -}} -{{- define "apm.autoscaling.apiVersion" -}} -{{- if semverCompare "<1.12-0" .Capabilities.KubeVersion.GitVersion -}} -{{- print "autoscaling/v2beta1" -}} -{{- else -}} -{{- print "autoscaling/v2beta2" -}} -{{- end -}} -{{- end -}} {{/* Use the fullname if the serviceAccount value is not set */}} diff --git a/apm-server/templates/clusterrole.yaml b/apm-server/templates/clusterrole.yaml index 6e0d2bc98..2d8e1ebf8 100644 --- a/apm-server/templates/clusterrole.yaml +++ b/apm-server/templates/clusterrole.yaml @@ -1,5 +1,5 @@ {{- if .Values.managedServiceAccount }} -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: {{ template "apm.serviceAccount" . }}-cluster-role diff --git a/apm-server/templates/clusterrolebinding.yaml b/apm-server/templates/clusterrolebinding.yaml index 3060496bf..c65b2be8d 100644 --- a/apm-server/templates/clusterrolebinding.yaml +++ b/apm-server/templates/clusterrolebinding.yaml @@ -1,5 +1,5 @@ {{- if .Values.managedServiceAccount }} -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: {{ template "apm.serviceAccount" . }}-cluster-role-binding diff --git a/apm-server/templates/hpa.yaml b/apm-server/templates/hpa.yaml index 928f25789..b78724124 100644 --- a/apm-server/templates/hpa.yaml +++ b/apm-server/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: {{ template "apm.autoscaling.apiVersion" . }} +apiVersion: autoscaling/v2beta2 kind: HorizontalPodAutoscaler metadata: name: {{ template "apm.fullname" . }} @@ -14,4 +14,13 @@ spec: apiVersion: apps/v1 kind: Deployment name: {{ template "apm.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.averageCpuUtilization }} {{- end }} diff --git a/apm-server/templates/ingress.yaml b/apm-server/templates/ingress.yaml index bfaa0f0e5..64ebcced9 100644 --- a/apm-server/templates/ingress.yaml +++ b/apm-server/templates/ingress.yaml @@ -2,7 +2,7 @@ {{- $fullName := include "apm.fullname" . -}} {{- $servicePort := .Values.service.port -}} {{- $ingressPath := .Values.ingress.path -}} -apiVersion: {{ template "apm.ingress.apiVersion" . }} +apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: name: {{ template "apm.fullname" . }} diff --git a/apm-server/tests/apmserver_test.py b/apm-server/tests/apmserver_test.py index 2ce3b70b1..a3b1419df 100644 --- a/apm-server/tests/apmserver_test.py +++ b/apm-server/tests/apmserver_test.py @@ -341,3 +341,13 @@ def test_setting_fullnameOverride(): ] == project ) + + +def test_enabling_horizontal_pod_autoscaler(): + config = """ +autoscaling: + enabled: true +""" + r = helm_template(config) + + assert "horizontalpodautoscaler" in r diff --git a/apm-server/values.yaml b/apm-server/values.yaml index 6d5038007..3402fce6d 100755 --- a/apm-server/values.yaml +++ b/apm-server/values.yaml @@ -140,6 +140,9 @@ fullnameOverride: "" autoscaling: enabled: false + minReplicas: 1 + maxReplicas: 3 + averageCpuUtilization: 50 ingress: enabled: false diff --git a/elasticsearch/README.md b/elasticsearch/README.md index 06255be13..f0062d542 100644 --- a/elasticsearch/README.md +++ b/elasticsearch/README.md @@ -3,6 +3,7 @@ This Helm chart is a lightweight way to configure and run our official [Elasticsearch Docker image][]. + **Warning**: This branch is used for development, please use the latest [7.x][] release for released version. @@ -40,8 +41,8 @@ This Helm chart is a lightweight way to configure and run our official ## Requirements -* [Helm][] >=2.8.0 and <3.0.0 -* Kubernetes >=1.8 +* Kubernetes >= 1.14 +* [Helm][] >= 2.17.0 * Minimum cluster requirements include the following to run this chart with default settings. All of these settings are configurable. * Three Kubernetes nodes to respect the default "hard" affinity settings @@ -58,8 +59,8 @@ See [supported configurations][] for more details. `helm repo add elastic https://helm.elastic.co` * Install it: - - with Helm 2: `helm install --name elasticsearch elastic/elasticsearch` - - with [Helm 3 (beta)][]: `helm install elasticsearch elastic/elasticsearch` + - with Helm 3: `helm install elasticsearch elastic/elasticsearch` + - with Helm 2 (deprecated): `helm install --name elasticsearch elastic/elasticsearch` ### Install development version using master branch @@ -67,8 +68,8 @@ See [supported configurations][] for more details. * Clone the git repo: `git clone git@github.com:elastic/helm-charts.git` * Install it: - - with Helm 2: `helm install --name elasticsearch ./helm-charts/elasticsearch --set imageTag=8.0.0-SNAPSHOT` - - with [Helm 3 (beta)][]: `helm install elasticsearch ./helm-charts/elasticsearch --set imageTag=8.0.0-SNAPSHOT` + - with Helm 3: `helm install elasticsearch ./helm-charts/elasticsearch --set imageTag=8.0.0-SNAPSHOT` + - with Helm 2 (deprecated): `helm install --name elasticsearch ./helm-charts/elasticsearch --set imageTag=8.0.0-SNAPSHOT` ## Upgrading @@ -124,7 +125,7 @@ support multiple versions with minimal changes. | `httpPort` | The http port that Kubernetes will use for the healthchecks and the service. If you change this you will also need to set [http.port][] in `extraEnvs` | `9200` | | `imagePullPolicy` | The Kubernetes [imagePullPolicy][] value | `IfNotPresent` | | `imagePullSecrets` | Configuration for [imagePullSecrets][] so that you can use a private registry for your image | `[]` | -| `imageTag` | The Elasticsearch Docker image tag | `8.0.0-SNAPSHOT` | +| `imageTag` | The Elasticsearch Docker image tag | `8.0.0-SNAPSHOT` | | `image` | The Elasticsearch Docker image | `docker.elastic.co/elasticsearch/elasticsearch` | | `ingress` | Configurable [ingress][] to expose the Elasticsearch service. See [values.yaml][] for an example | see [values.yaml][] | | `initResources` | Allows you to set the [resources][] for the `initContainer` in the StatefulSet | `{}` | @@ -156,7 +157,7 @@ support multiple versions with minimal changes. | `secretMounts` | Allows you easily mount a secret as a file inside the StatefulSet. Useful for mounting certificates and other secrets. See [values.yaml][] for an example | `[]` | | `securityContext` | Allows you to set the [securityContext][] for the container | see [values.yaml][] | | `service.annotations` | [LoadBalancer annotations][] that Kubernetes will use for the service. This will configure load balancer if `service.type` is `LoadBalancer` | `{}` | -| `service.externalTrafficPolicy` | Some cloud providers allow you to specify the [LoadBalancer externalTrafficPolicy][]. Kubernetes will use this to preserve the client source IP. This will configure load balancer if `service.type` is `LoadBalancer` | `""` | +| `service.externalTrafficPolicy` | Some cloud providers allow you to specify the [LoadBalancer externalTrafficPolicy][]. Kubernetes will use this to preserve the client source IP. This will configure load balancer if `service.type` is `LoadBalancer` | `""` | | `service.httpPortName` | The name of the http port within the service | `http` | | `service.labelsHeadless` | Labels to be added to headless service | `{}` | | `service.labels` | Labels to be added to non-headless service | `{}` | @@ -412,7 +413,6 @@ about our development and testing process. [examples/security]: https://github.com/elastic/helm-charts/tree/master/elasticsearch/examples/security [gke]: https://cloud.google.com/kubernetes-engine [helm]: https://helm.sh -[helm 3 (beta)]: https://github.com/elastic/helm-charts/tree/master/README.md#helm-3-beta [helm/charts stable]: https://github.com/helm/charts/tree/master/stable/elasticsearch/ [how to install plugins guide]: https://github.com/elastic/helm-charts/tree/master/elasticsearch/README.md#how-to-install-plugins [how to use the keystore]: https://github.com/elastic/helm-charts/tree/master/elasticsearch/README.md#how-to-use-the-keystore diff --git a/elasticsearch/examples/config/Makefile b/elasticsearch/examples/config/Makefile index a3f96174b..9ae9c3788 100644 --- a/elasticsearch/examples/config/Makefile +++ b/elasticsearch/examples/config/Makefile @@ -1,10 +1,12 @@ default: test + include ../../../helpers/examples.mk RELEASE := helm-es-config +TIMEOUT := 1200s install: - helm upgrade --wait --timeout=600 --install $(RELEASE) --values ./values.yaml ../../ + helm upgrade --wait --timeout=$(TIMEOUT) --install --values values.yaml $(RELEASE) ../../ secrets: kubectl delete secret elastic-config-credentials elastic-config-secret elastic-config-slack elastic-config-custom-path || true @@ -16,4 +18,4 @@ secrets: test: secrets install goss purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/elasticsearch/examples/config/test/goss.yaml b/elasticsearch/examples/config/test/goss.yaml index 848701370..51c4907f5 100644 --- a/elasticsearch/examples/config/test/goss.yaml +++ b/elasticsearch/examples/config/test/goss.yaml @@ -3,7 +3,7 @@ http: status: 200 timeout: 2000 body: - - 'green' + - "green" - '"number_of_nodes":1' - '"number_of_data_nodes":1' @@ -12,8 +12,7 @@ http: timeout: 2000 body: - '"cluster_name" : "config"' - - '"name" : "config-master-0"' - - 'You Know, for Search' + - "You Know, for Search" command: "elasticsearch-keystore list": diff --git a/elasticsearch/examples/default/Makefile b/elasticsearch/examples/default/Makefile index 5f5215c6b..389bf99e3 100644 --- a/elasticsearch/examples/default/Makefile +++ b/elasticsearch/examples/default/Makefile @@ -3,14 +3,12 @@ default: test include ../../../helpers/examples.mk RELEASE := helm-es-default +TIMEOUT := 1200s install: - helm upgrade --wait --timeout=600 --install $(RELEASE) ../../ - -restart: - helm upgrade --set terminationGracePeriod=121 --wait --timeout=600 --install $(RELEASE) ../../ + helm upgrade --wait --timeout=$(TIMEOUT) --install $(RELEASE) ../../ test: install goss purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/elasticsearch/examples/default/test/goss.yaml b/elasticsearch/examples/default/test/goss.yaml index eacfc0d5e..70c27ef87 100644 --- a/elasticsearch/examples/default/test/goss.yaml +++ b/elasticsearch/examples/default/test/goss.yaml @@ -1,13 +1,13 @@ kernel-param: vm.max_map_count: - value: '262144' + value: "262144" http: http://elasticsearch-master:9200/_cluster/health: status: 200 timeout: 2000 body: - - 'green' + - "green" - '"number_of_nodes":3' - '"number_of_data_nodes":3' @@ -17,8 +17,7 @@ http: body: - '"number" : "8.0.0-SNAPSHOT"' - '"cluster_name" : "elasticsearch"' - - '"name" : "elasticsearch-master-0"' - - 'You Know, for Search' + - "You Know, for Search" file: /usr/share/elasticsearch/data: diff --git a/elasticsearch/examples/docker-for-mac/Makefile b/elasticsearch/examples/docker-for-mac/Makefile index 398545e64..18fd053d8 100644 --- a/elasticsearch/examples/docker-for-mac/Makefile +++ b/elasticsearch/examples/docker-for-mac/Makefile @@ -1,12 +1,13 @@ default: test RELEASE := helm-es-docker-for-mac +TIMEOUT := 1200s install: - helm upgrade --wait --timeout=900 --install --values values.yaml $(RELEASE) ../../ + helm upgrade --wait --timeout=$(TIMEOUT) --install --values values.yaml $(RELEASE) ../../ test: install helm test $(RELEASE) purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/elasticsearch/examples/kubernetes-kind/Makefile b/elasticsearch/examples/kubernetes-kind/Makefile index af816a9d8..9e5602d49 100644 --- a/elasticsearch/examples/kubernetes-kind/Makefile +++ b/elasticsearch/examples/kubernetes-kind/Makefile @@ -1,16 +1,17 @@ default: test RELEASE := helm-es-kind +TIMEOUT := 1200s install: - helm upgrade --wait --timeout=900 --install --values values.yaml $(RELEASE) ../../ + helm upgrade --wait --timeout=$(TIMEOUT) --install --values values.yaml $(RELEASE) ../../ install-local-path: kubectl apply -f https://raw.githubusercontent.com/rancher/local-path-provisioner/master/deploy/local-path-storage.yaml - helm upgrade --wait --timeout=900 --install --values values-local-path.yaml $(RELEASE) ../../ + helm upgrade --wait --timeout=$(TIMEOUT) --install --values values-local-path.yaml $(RELEASE) ../../ test: install helm test $(RELEASE) purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/elasticsearch/examples/microk8s/Makefile b/elasticsearch/examples/microk8s/Makefile index 2c7d3d394..2d0012d8a 100644 --- a/elasticsearch/examples/microk8s/Makefile +++ b/elasticsearch/examples/microk8s/Makefile @@ -1,12 +1,13 @@ default: test RELEASE := helm-es-microk8s +TIMEOUT := 1200s install: - helm upgrade --wait --timeout=900 --install --values values.yaml $(RELEASE) ../../ + helm upgrade --wait --timeout=$(TIMEOUT) --install --values values.yaml $(RELEASE) ../../ test: install helm test $(RELEASE) purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/elasticsearch/examples/migration/Makefile b/elasticsearch/examples/migration/Makefile index 3b1dac105..020906f92 100644 --- a/elasticsearch/examples/migration/Makefile +++ b/elasticsearch/examples/migration/Makefile @@ -1,10 +1,10 @@ PREFIX := helm-es-migration data: - helm upgrade --wait --timeout=600 --install --values ./data.yml $(PREFIX)-data ../../ + helm upgrade --wait --timeout=$(TIMEOUT) --install --values data.yaml $(PREFIX)-data ../../ master: - helm upgrade --wait --timeout=600 --install --values ./master.yml $(PREFIX)-master ../../ + helm upgrade --wait --timeout=$(TIMEOUT) --install --values master.yaml $(PREFIX)-master ../../ client: - helm upgrade --wait --timeout=600 --install --values ./client.yml $(PREFIX)-client ../../ + helm upgrade --wait --timeout=$(TIMEOUT) --install --values client.yaml $(PREFIX)-client ../../ diff --git a/elasticsearch/examples/migration/README.md b/elasticsearch/examples/migration/README.md index 843345f3e..74adbe202 100644 --- a/elasticsearch/examples/migration/README.md +++ b/elasticsearch/examples/migration/README.md @@ -56,7 +56,7 @@ cluster is healthy before continuing: ``` * Deploy new data nodes which will join the existing cluster. Take a look at the -configuration in [data.yml][]: +configuration in [data.yaml][]: ``` make data @@ -102,7 +102,7 @@ nodes as to not lose quorum: ``` * Now deploy a single new master so that we have 3 masters again. See -[master.yml][] for the configuration: +[master.yaml][] for the configuration: ``` make master @@ -114,7 +114,7 @@ nodes as to not lose quorum: kubectl scale statefulsets my-release-elasticsearch-master --replicas=1 ``` -* Edit the masters in [masters.yml][] to 2 and redeploy: +* Edit the masters in [masters.yaml][] to 2 and redeploy: ``` make master @@ -126,7 +126,7 @@ nodes as to not lose quorum: kubectl scale statefulsets my-release-elasticsearch-master --replicas=0 ``` -* Edit the [masters.yml][] to have 3 replicas and remove the +* Edit the [masters.yaml][] to have 3 replicas and remove the `discovery.zen.ping.unicast.hosts` entry from `extraEnvs` then redeploy the masters. This will make sure all 3 masters are running in the new cluster and are pointing at each other for discovery: @@ -160,8 +160,8 @@ client nodes: working correctly you can cleanup leftover resources from your old cluster. [basic license]: https://www.elastic.co/subscriptions -[data.yml]: https://github.com/elastic/helm-charts/blob/master/elasticsearch/examples/migration/data.yml +[data.yaml]: https://github.com/elastic/helm-charts/blob/master/elasticsearch/examples/migration/data.yaml [helm/charts]: https://github.com/helm/charts/tree/master/stable/elasticsearch -[master.yml]: https://github.com/elastic/helm-charts/blob/master/elasticsearch/examples/migration/master.yml +[master.yaml]: https://github.com/elastic/helm-charts/blob/master/elasticsearch/examples/migration/master.yaml [restoring to a different cluster guide]: https://www.elastic.co/guide/en/elasticsearch/reference/6.8/modules-snapshots.html#_restoring_to_a_different_cluster [rolling upgrades guide]: https://www.elastic.co/guide/en/elasticsearch/reference/6.8/rolling-upgrades.html diff --git a/elasticsearch/examples/migration/client.yml b/elasticsearch/examples/migration/client.yaml similarity index 100% rename from elasticsearch/examples/migration/client.yml rename to elasticsearch/examples/migration/client.yaml diff --git a/elasticsearch/examples/migration/data.yml b/elasticsearch/examples/migration/data.yaml similarity index 100% rename from elasticsearch/examples/migration/data.yml rename to elasticsearch/examples/migration/data.yaml diff --git a/elasticsearch/examples/migration/master.yml b/elasticsearch/examples/migration/master.yaml similarity index 100% rename from elasticsearch/examples/migration/master.yml rename to elasticsearch/examples/migration/master.yaml diff --git a/elasticsearch/examples/minikube/Makefile b/elasticsearch/examples/minikube/Makefile index 97109ce8c..1021d9867 100644 --- a/elasticsearch/examples/minikube/Makefile +++ b/elasticsearch/examples/minikube/Makefile @@ -1,12 +1,13 @@ default: test RELEASE := helm-es-minikube +TIMEOUT := 1200s install: - helm upgrade --wait --timeout=900 --install --values values.yaml $(RELEASE) ../../ + helm upgrade --wait --timeout=$(TIMEOUT) --install --values values.yaml $(RELEASE) ../../ test: install helm test $(RELEASE) purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/elasticsearch/examples/multi/Makefile b/elasticsearch/examples/multi/Makefile index 9682966cd..243e50435 100644 --- a/elasticsearch/examples/multi/Makefile +++ b/elasticsearch/examples/multi/Makefile @@ -4,15 +4,16 @@ include ../../../helpers/examples.mk PREFIX := helm-es-multi RELEASE := helm-es-multi-master +TIMEOUT := 1200s install: - helm upgrade --wait --timeout=600 --install --values ./master.yml $(PREFIX)-master ../../ - helm upgrade --wait --timeout=600 --install --values ./data.yml $(PREFIX)-data ../../ - helm upgrade --wait --timeout=600 --install --values ./client.yml $(PREFIX)-client ../../ + helm upgrade --wait --timeout=$(TIMEOUT) --install --values master.yaml $(PREFIX)-master ../../ + helm upgrade --wait --timeout=$(TIMEOUT) --install --values data.yaml $(PREFIX)-data ../../ + helm upgrade --wait --timeout=$(TIMEOUT) --install --values client.yaml $(PREFIX)-client ../../ test: install goss purge: - helm del --purge $(PREFIX)-master - helm del --purge $(PREFIX)-data - helm del --purge $(PREFIX)-client + helm del $(PREFIX)-master + helm del $(PREFIX)-data + helm del $(PREFIX)-client diff --git a/elasticsearch/examples/multi/README.md b/elasticsearch/examples/multi/README.md index 38ae1ccd5..2b9875cef 100644 --- a/elasticsearch/examples/multi/README.md +++ b/elasticsearch/examples/multi/README.md @@ -23,8 +23,7 @@ releases: You can also run [goss integration tests][] using `make test` -[client values]: https://github.com/elastic/helm-charts/tree/master/elasticsearch/examples/multi/client.yml -[data values]: https://github.com/elastic/helm-charts/tree/master/elasticsearch/examples/multi/data.yml +[client values]: https://github.com/elastic/helm-charts/tree/master/elasticsearch/examples/multi/client.yaml +[data values]: https://github.com/elastic/helm-charts/tree/master/elasticsearch/examples/multi/data.yaml [goss integration tests]: https://github.com/elastic/helm-charts/tree/master/elasticsearch/examples/multi/test/goss.yaml -[master values]: https://github.com/elastic/helm-charts/tree/master/elasticsearch/examples/multi/master.yml - +[master values]: https://github.com/elastic/helm-charts/tree/master/elasticsearch/examples/multi/master.yaml diff --git a/elasticsearch/examples/multi/client.yml b/elasticsearch/examples/multi/client.yaml similarity index 100% rename from elasticsearch/examples/multi/client.yml rename to elasticsearch/examples/multi/client.yaml diff --git a/elasticsearch/examples/multi/data.yml b/elasticsearch/examples/multi/data.yaml similarity index 100% rename from elasticsearch/examples/multi/data.yml rename to elasticsearch/examples/multi/data.yaml diff --git a/elasticsearch/examples/multi/master.yml b/elasticsearch/examples/multi/master.yaml similarity index 100% rename from elasticsearch/examples/multi/master.yml rename to elasticsearch/examples/multi/master.yaml diff --git a/elasticsearch/examples/openshift/Makefile b/elasticsearch/examples/openshift/Makefile index 6e495916d..078c33c9d 100644 --- a/elasticsearch/examples/openshift/Makefile +++ b/elasticsearch/examples/openshift/Makefile @@ -1,15 +1,13 @@ default: test + include ../../../helpers/examples.mk RELEASE := elasticsearch -template: - helm template --values ./values.yaml ../../ - install: - helm upgrade --wait --timeout=600 --install $(RELEASE) --values ./values.yaml ../../ + helm upgrade --wait --timeout=$(TIMEOUT) --install --values values.yaml $(RELEASE) ../../ test: install goss purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/elasticsearch/examples/openshift/test/goss.yaml b/elasticsearch/examples/openshift/test/goss.yaml index 88b4bd83a..97daa9bb0 100644 --- a/elasticsearch/examples/openshift/test/goss.yaml +++ b/elasticsearch/examples/openshift/test/goss.yaml @@ -3,7 +3,7 @@ http: status: 200 timeout: 2000 body: - - 'green' + - "green" - '"number_of_nodes":3' - '"number_of_data_nodes":3' @@ -13,5 +13,4 @@ http: body: - '"number" : "8.0.0-SNAPSHOT"' - '"cluster_name" : "elasticsearch"' - - '"name" : "elasticsearch-master-0"' - - 'You Know, for Search' + - "You Know, for Search" diff --git a/elasticsearch/examples/oss/Makefile b/elasticsearch/examples/oss/Makefile index e274659c6..e7425e74a 100644 --- a/elasticsearch/examples/oss/Makefile +++ b/elasticsearch/examples/oss/Makefile @@ -1,12 +1,14 @@ default: test + include ../../../helpers/examples.mk RELEASE := helm-es-oss +TIMEOUT := 1200s install: - helm upgrade --wait --timeout=600 --install $(RELEASE) --values ./values.yaml ../../ + helm upgrade --wait --timeout=$(TIMEOUT) --install --values values.yaml $(RELEASE) ../../ test: install goss purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/elasticsearch/examples/oss/test/goss.yaml b/elasticsearch/examples/oss/test/goss.yaml index b50702357..97ea1a81b 100644 --- a/elasticsearch/examples/oss/test/goss.yaml +++ b/elasticsearch/examples/oss/test/goss.yaml @@ -13,5 +13,4 @@ http: body: - '"number" : "8.0.0-SNAPSHOT"' - '"cluster_name" : "oss"' - - '"name" : "oss-master-0"' - 'You Know, for Search' diff --git a/elasticsearch/examples/security/Makefile b/elasticsearch/examples/security/Makefile index 71937071d..522b720ca 100644 --- a/elasticsearch/examples/security/Makefile +++ b/elasticsearch/examples/security/Makefile @@ -4,15 +4,16 @@ include ../../../helpers/examples.mk RELEASE := helm-es-security ELASTICSEARCH_IMAGE := docker.elastic.co/elasticsearch/elasticsearch:$(STACK_VERSION) +TIMEOUT := 1200s install: - helm upgrade --wait --timeout=600 --install --values ./security.yml $(RELEASE) ../../ + helm upgrade --wait --timeout=$(TIMEOUT) --install --values values.yaml $(RELEASE) ../../ + +test: secrets install goss purge: kubectl delete secrets elastic-credentials elastic-certificates elastic-certificate-pem elastic-certificate-crt|| true - helm del --purge $(RELEASE) - -test: secrets install goss + helm del $(RELEASE) pull-elasticsearch-image: docker pull $(ELASTICSEARCH_IMAGE) @@ -33,5 +34,5 @@ secrets: kubectl create secret generic elastic-certificates --from-file=elastic-certificates.p12 && \ kubectl create secret generic elastic-certificate-pem --from-file=elastic-certificate.pem && \ kubectl create secret generic elastic-certificate-crt --from-file=elastic-certificate.crt && \ - kubectl create secret generic elastic-credentials --from-literal=password=$$password --from-literal=username=elastic && \ + kubectl create secret generic elastic-credentials --from-literal=password=$$password --from-literal=username=elastic && \ rm -f elastic-certificates.p12 elastic-certificate.pem elastic-certificate.crt elastic-stack-ca.p12 diff --git a/elasticsearch/examples/security/test/goss.yaml b/elasticsearch/examples/security/test/goss.yaml index 00be5fbb4..731d9115f 100644 --- a/elasticsearch/examples/security/test/goss.yaml +++ b/elasticsearch/examples/security/test/goss.yaml @@ -3,10 +3,10 @@ http: status: 200 timeout: 2000 allow-insecure: true - username: '{{ .Env.ELASTIC_USERNAME }}' - password: '{{ .Env.ELASTIC_PASSWORD }}' + username: "{{ .Env.ELASTIC_USERNAME }}" + password: "{{ .Env.ELASTIC_PASSWORD }}" body: - - 'green' + - "green" - '"number_of_nodes":3' - '"number_of_data_nodes":3' @@ -14,32 +14,31 @@ http: status: 200 timeout: 2000 allow-insecure: true - username: '{{ .Env.ELASTIC_USERNAME }}' - password: '{{ .Env.ELASTIC_PASSWORD }}' + username: "{{ .Env.ELASTIC_USERNAME }}" + password: "{{ .Env.ELASTIC_PASSWORD }}" body: - '"cluster_name" : "security"' - - '"name" : "security-master-0"' - - 'You Know, for Search' + - "You Know, for Search" https://localhost:9200/_license: status: 200 timeout: 2000 allow-insecure: true - username: '{{ .Env.ELASTIC_USERNAME }}' - password: '{{ .Env.ELASTIC_PASSWORD }}' + username: "{{ .Env.ELASTIC_USERNAME }}" + password: "{{ .Env.ELASTIC_PASSWORD }}" body: - - 'active' - - 'basic' + - "active" + - "basic" file: /usr/share/elasticsearch/config/elasticsearch.yml: exists: true contains: - - 'xpack.security.enabled: true' - - 'xpack.security.transport.ssl.enabled: true' - - 'xpack.security.transport.ssl.verification_mode: certificate' - - 'xpack.security.transport.ssl.keystore.path: /usr/share/elasticsearch/config/certs/elastic-certificates.p12' - - 'xpack.security.transport.ssl.truststore.path: /usr/share/elasticsearch/config/certs/elastic-certificates.p12' - - 'xpack.security.http.ssl.enabled: true' - - 'xpack.security.http.ssl.truststore.path: /usr/share/elasticsearch/config/certs/elastic-certificates.p12' - - 'xpack.security.http.ssl.keystore.path: /usr/share/elasticsearch/config/certs/elastic-certificates.p12' + - "xpack.security.enabled: true" + - "xpack.security.transport.ssl.enabled: true" + - "xpack.security.transport.ssl.verification_mode: certificate" + - "xpack.security.transport.ssl.keystore.path: /usr/share/elasticsearch/config/certs/elastic-certificates.p12" + - "xpack.security.transport.ssl.truststore.path: /usr/share/elasticsearch/config/certs/elastic-certificates.p12" + - "xpack.security.http.ssl.enabled: true" + - "xpack.security.http.ssl.truststore.path: /usr/share/elasticsearch/config/certs/elastic-certificates.p12" + - "xpack.security.http.ssl.keystore.path: /usr/share/elasticsearch/config/certs/elastic-certificates.p12" diff --git a/elasticsearch/examples/security/security.yml b/elasticsearch/examples/security/values.yaml similarity index 100% rename from elasticsearch/examples/security/security.yml rename to elasticsearch/examples/security/values.yaml diff --git a/elasticsearch/examples/upgrade/Makefile b/elasticsearch/examples/upgrade/Makefile new file mode 100644 index 000000000..01e56e4e6 --- /dev/null +++ b/elasticsearch/examples/upgrade/Makefile @@ -0,0 +1,17 @@ +default: test + +include ../../../helpers/examples.mk + +CHART := elasticsearch +RELEASE := helm-es-upgrade +FROM := 7.4.0 # versions before 7.4.O aren't compatible with Kubernetes >= 1.16.0 +TO := 7.10.0 # upgrade from 7.x to 8.0.0-SNAPSHOT currently doesn't work + +install: + ../../../helpers/upgrade.sh --chart $(CHART) --release $(RELEASE) --from $(FROM) --to $(TO) + kubectl rollout status statefulset elasticsearch-master + +test: install goss + +purge: + helm del $(RELEASE) diff --git a/elasticsearch/examples/upgrade/README.md b/elasticsearch/examples/upgrade/README.md new file mode 100644 index 000000000..85977f52e --- /dev/null +++ b/elasticsearch/examples/upgrade/README.md @@ -0,0 +1,17 @@ +# Upgrade + +This example will deploy a 3 node Elasticsearch cluster chart using an old chart +version, then upgrade it. + + +## Usage + +* Deploy and upgrade Elasticsearch chart with the default values: `make install` + + +## Testing + +You can also run [goss integration tests][] using `make test`. + + +[goss integration tests]: https://github.com/elastic/helm-charts/tree/master/elasticsearch/examples/upgrade/test/goss.yaml diff --git a/elasticsearch/examples/upgrade/test/goss.yaml b/elasticsearch/examples/upgrade/test/goss.yaml new file mode 100644 index 000000000..5887b5f43 --- /dev/null +++ b/elasticsearch/examples/upgrade/test/goss.yaml @@ -0,0 +1,16 @@ +http: + http://localhost:9200/_cluster/health: + status: 200 + timeout: 2000 + body: + - "green" + - '"number_of_nodes":3' + - '"number_of_data_nodes":3' + + http://localhost:9200: + status: 200 + timeout: 2000 + body: + - '"number" : "7.10.0"' + - '"cluster_name" : "upgrade"' + - "You Know, for Search" diff --git a/elasticsearch/examples/upgrade/values.yaml b/elasticsearch/examples/upgrade/values.yaml new file mode 100644 index 000000000..de0283af4 --- /dev/null +++ b/elasticsearch/examples/upgrade/values.yaml @@ -0,0 +1,2 @@ +--- +clusterName: upgrade diff --git a/elasticsearch/templates/NOTES.txt b/elasticsearch/templates/NOTES.txt index 3841adafc..73edf425a 100755 --- a/elasticsearch/templates/NOTES.txt +++ b/elasticsearch/templates/NOTES.txt @@ -1,4 +1,4 @@ 1. Watch all cluster members come up. $ kubectl get pods --namespace={{ .Release.Namespace }} -l app={{ template "elasticsearch.uname" . }} -w 2. Test cluster health using Helm test. - $ helm test {{ .Release.Name }} --cleanup + $ helm test {{ .Release.Name }} diff --git a/elasticsearch/templates/_helpers.tpl b/elasticsearch/templates/_helpers.tpl index 0b0fe9ae1..91a15535a 100755 --- a/elasticsearch/templates/_helpers.tpl +++ b/elasticsearch/templates/_helpers.tpl @@ -63,25 +63,3 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this {{- end -}} {{- end -}} {{- end -}} - -{{/* -Return the appropriate apiVersion for statefulset. -*/}} -{{- define "elasticsearch.statefulset.apiVersion" -}} -{{- if semverCompare "<1.9-0" .Capabilities.KubeVersion.GitVersion -}} -{{- print "apps/v1beta2" -}} -{{- else -}} -{{- print "apps/v1" -}} -{{- end -}} -{{- end -}} - -{{/* -Return the appropriate apiVersion for ingress. -*/}} -{{- define "elasticsearch.ingress.apiVersion" -}} -{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} -{{- print "extensions/v1beta1" -}} -{{- else -}} -{{- print "networking.k8s.io/v1beta1" -}} -{{- end -}} -{{- end -}} diff --git a/elasticsearch/templates/ingress.yaml b/elasticsearch/templates/ingress.yaml index b863ff400..e34b475e1 100644 --- a/elasticsearch/templates/ingress.yaml +++ b/elasticsearch/templates/ingress.yaml @@ -2,7 +2,7 @@ {{- $fullName := include "elasticsearch.uname" . -}} {{- $servicePort := .Values.httpPort -}} {{- $ingressPath := .Values.ingress.path -}} -apiVersion: {{ template "elasticsearch.ingress.apiVersion" . }} +apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: name: {{ $fullName }} diff --git a/elasticsearch/templates/statefulset.yaml b/elasticsearch/templates/statefulset.yaml index 0450a0cbc..70dd1a3cc 100644 --- a/elasticsearch/templates/statefulset.yaml +++ b/elasticsearch/templates/statefulset.yaml @@ -1,5 +1,5 @@ --- -apiVersion: {{ template "elasticsearch.statefulset.apiVersion" . }} +apiVersion: apps/v1 kind: StatefulSet metadata: name: {{ template "elasticsearch.uname" . }} @@ -28,7 +28,6 @@ spec: name: {{ template "elasticsearch.uname" . }} {{- if .Values.persistence.labels.enabled }} labels: - heritage: {{ .Release.Service | quote }} release: {{ .Release.Name | quote }} chart: "{{ .Chart.Name }}" app: "{{ template "elasticsearch.uname" . }}" @@ -47,7 +46,6 @@ spec: metadata: name: "{{ template "elasticsearch.uname" . }}" labels: - heritage: {{ .Release.Service | quote }} release: {{ .Release.Name | quote }} chart: "{{ .Chart.Name }}" app: "{{ template "elasticsearch.uname" . }}" @@ -154,9 +152,8 @@ spec: imagePullSecrets: {{ toYaml .Values.imagePullSecrets | indent 8 }} {{- end }} - {{- if semverCompare ">1.13-0" .Capabilities.KubeVersion.GitVersion }} enableServiceLinks: {{ .Values.enableServiceLinks }} - {{- end }} + {{- if or (.Values.extraInitContainers) (.Values.sysctlInitContainer.enabled) (.Values.keystore) }} initContainers: {{- if .Values.sysctlInitContainer.enabled }} - name: configure-sysctl @@ -214,6 +211,7 @@ spec: {{ tpl .Values.extraInitContainers . | indent 6 }} {{- else }} {{ toYaml .Values.extraInitContainers | indent 6 }} + {{- end }} {{- end }} {{- end }} containers: diff --git a/elasticsearch/templates/test/test-elasticsearch-health.yaml b/elasticsearch/templates/test/test-elasticsearch-health.yaml index a278b140a..dd5eaa043 100644 --- a/elasticsearch/templates/test/test-elasticsearch-health.yaml +++ b/elasticsearch/templates/test/test-elasticsearch-health.yaml @@ -4,7 +4,8 @@ kind: Pod metadata: name: "{{ .Release.Name }}-{{ randAlpha 5 | lower }}-test" annotations: - "helm.sh/hook": test-success + "helm.sh/hook": test + "helm.sh/hook-delete-policy": hook-succeeded spec: securityContext: {{ toYaml .Values.podSecurityContext | indent 4 }} diff --git a/elasticsearch/tests/elasticsearch_test.py b/elasticsearch/tests/elasticsearch_test.py index 293453dd7..ac2a58c1e 100755 --- a/elasticsearch/tests/elasticsearch_test.py +++ b/elasticsearch/tests/elasticsearch_test.py @@ -371,10 +371,7 @@ def test_sysctl_init_container_disabled(): enabled: false """ r = helm_template(config) - initContainers = r["statefulset"][uname]["spec"]["template"]["spec"][ - "initContainers" - ] - assert initContainers is None + assert "initContainers" not in r["statefulset"][uname]["spec"]["template"]["spec"] def test_sysctl_init_container_enabled(): @@ -445,7 +442,10 @@ def test_enabling_persistence_label_in_volumeclaimtemplate(): "volumeClaimTemplates" ][0]["metadata"]["labels"] statefulset_labels = r["statefulset"][uname]["metadata"]["labels"] - assert volume_claim_template_labels == statefulset_labels + expected_labels = statefulset_labels + # heritage label shouldn't be present in volumeClaimTemplates labels + expected_labels.pop("heritage") + assert volume_claim_template_labels == expected_labels def test_adding_a_secret_mount(): diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index 0cd018022..1402af0e2 100755 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -122,7 +122,7 @@ podSecurityPolicy: persistence: enabled: true labels: - # Add default labels for the volumeClaimTemplate fo the StatefulSet + # Add default labels for the volumeClaimTemplate of the StatefulSet enabled: false annotations: {} diff --git a/filebeat/README.md b/filebeat/README.md index e65594d14..90246116b 100644 --- a/filebeat/README.md +++ b/filebeat/README.md @@ -3,7 +3,9 @@ This Helm chart is a lightweight way to configure and run our official [Filebeat Docker image][]. + **Warning**: This branch is used for development, please use the latest [7.x][] release for released version. + @@ -29,8 +31,8 @@ This Helm chart is a lightweight way to configure and run our official ## Requirements -* [Helm][] >=2.8.0 and <3.0.0 -* Kubernetes >=1.9 +* Kubernetes >= 1.14 +* [Helm][] >= 2.17.0 See [supported configurations][] for more details. @@ -43,8 +45,8 @@ See [supported configurations][] for more details. `helm repo add elastic https://helm.elastic.co` * Install it: - - with Helm 2: `helm install --name filebeat elastic/filebeat` - - with [Helm 3 (beta)][]: `helm install filebeat elastic/filebeat` + - with Helm 3: `helm install filebeat elastic/filebeat` + - with Helm 2 (deprecated): `helm install --name filebeat elastic/filebeat` ### Install development version using master branch @@ -52,8 +54,8 @@ See [supported configurations][] for more details. * Clone the git repo: `git clone git@github.com:elastic/helm-charts.git` * Install it: - - with Helm 2: `helm install --name filebeat ./helm-charts/filebeat --set imageTag=8.0.0-SNAPSHOT` - - with [Helm 3 (beta)][]: `helm install filebeat ./helm-charts/filebeat --set imageTag=8.0.0-SNAPSHOT` + - with Helm 3: `helm install filebeat ./helm-charts/filebeat --set imageTag=8.0.0-SNAPSHOT` + - with Helm 2 (deprecated): `helm install --name filebeat ./helm-charts/filebeat --set imageTag=8.0.0-SNAPSHOT` ## Upgrading @@ -228,7 +230,6 @@ about our development and testing process. [filebeat oss docker image]: https://www.docker.elastic.co/r/beats/filebeat-oss [filebeat outputs]: https://www.elastic.co/guide/en/beats/filebeat/current/configuring-output.html [helm]: https://helm.sh -[helm 3 (beta)]: https://github.com/elastic/helm-charts/tree/master/README.md#helm-3-beta [hostNetwork]: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#host-namespaces [dnsConfig]: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/ [hostPath]: https://kubernetes.io/docs/concepts/storage/volumes/#hostpath diff --git a/filebeat/examples/default/Makefile b/filebeat/examples/default/Makefile index 6cfa6a648..b39ece967 100644 --- a/filebeat/examples/default/Makefile +++ b/filebeat/examples/default/Makefile @@ -5,9 +5,9 @@ include ../../../helpers/examples.mk RELEASE := helm-filebeat-default install: - helm upgrade --wait --timeout=600 --install $(RELEASE) ../../ + helm upgrade --wait --timeout=$(TIMEOUT) --install $(RELEASE) ../../ test: install goss purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/filebeat/examples/oss/Makefile b/filebeat/examples/oss/Makefile index e6b665412..3caa17af1 100644 --- a/filebeat/examples/oss/Makefile +++ b/filebeat/examples/oss/Makefile @@ -5,9 +5,9 @@ include ../../../helpers/examples.mk RELEASE := helm-filebeat-oss install: - helm upgrade --wait --timeout=600 --install $(RELEASE) --values values.yaml ../../ + helm upgrade --wait --timeout=$(TIMEOUT) --install --values values.yaml $(RELEASE) ../../ test: install goss purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/filebeat/examples/security/Makefile b/filebeat/examples/security/Makefile index 6d79d4231..7bec9abec 100644 --- a/filebeat/examples/security/Makefile +++ b/filebeat/examples/security/Makefile @@ -5,9 +5,9 @@ include ../../../helpers/examples.mk RELEASE := helm-filebeat-security install: - helm upgrade --wait --timeout=600 --install $(RELEASE) --values values.yaml ../../ + helm upgrade --wait --timeout=$(TIMEOUT) --install --values values.yaml $(RELEASE) ../../ test: install goss purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/filebeat/examples/upgrade/Makefile b/filebeat/examples/upgrade/Makefile new file mode 100644 index 000000000..054b53c23 --- /dev/null +++ b/filebeat/examples/upgrade/Makefile @@ -0,0 +1,16 @@ +default: test + +include ../../../helpers/examples.mk + +CHART := filebeat +RELEASE := helm-filebeat-upgrade +FROM := 7.9.0 # registry file version 1 not supported error with previous version + +install: + ../../../helpers/upgrade.sh --chart $(CHART) --release $(RELEASE) --from $(FROM) + kubectl rollout status daemonset $(RELEASE)-filebeat + +test: install goss + +purge: + helm del $(RELEASE) diff --git a/filebeat/examples/upgrade/README.md b/filebeat/examples/upgrade/README.md new file mode 100644 index 000000000..fa3ee3b85 --- /dev/null +++ b/filebeat/examples/upgrade/README.md @@ -0,0 +1,21 @@ +# Upgrade + +This example will deploy Filebeat chart using an old chart version, +then upgrade it. + + +## Usage + +* Add the Elastic Helm charts repo: `helm repo add elastic https://helm.elastic.co` + +* Deploy [Elasticsearch Helm chart][]: `helm install elasticsearch elastic/elasticsearch` + +* Deploy and upgrade Filebeat chart with the default values: `make install` + + +## Testing + +You can also run [goss integration tests][] using `make test`. + + +[goss integration tests]: https://github.com/elastic/helm-charts/tree/master/filebeat/examples/upgrade/test/goss.yaml diff --git a/filebeat/examples/upgrade/test/goss.yaml b/filebeat/examples/upgrade/test/goss.yaml new file mode 100644 index 000000000..b71a24325 --- /dev/null +++ b/filebeat/examples/upgrade/test/goss.yaml @@ -0,0 +1,45 @@ +port: + tcp:5066: + listening: true + ip: + - "127.0.0.1" + +mount: + /usr/share/filebeat/data: + exists: true + /run/docker.sock: + exists: true + /var/lib/docker/containers: + exists: true + opts: + - ro + /usr/share/filebeat/filebeat.yml: + exists: true + opts: + - ro + +user: + filebeat: + exists: true + uid: 1000 + gid: 1000 + +http: + http://upgrade-master:9200/_cat/indices: + status: 200 + timeout: 2000 + body: + - "filebeat-8.0.0" + +file: + /usr/share/filebeat/filebeat.yml: + exists: true + contains: + - "add_kubernetes_metadata" + - "output.elasticsearch" + +command: + cd /usr/share/filebeat && filebeat test output: + exit-status: 0 + stdout: + - "elasticsearch: http://upgrade-master:9200" diff --git a/filebeat/examples/upgrade/values.yaml b/filebeat/examples/upgrade/values.yaml new file mode 100644 index 000000000..8b230601e --- /dev/null +++ b/filebeat/examples/upgrade/values.yaml @@ -0,0 +1,4 @@ +--- +extraEnvs: + - name: ELASTICSEARCH_HOSTS + value: upgrade-master:9200 diff --git a/filebeat/templates/clusterrole.yaml b/filebeat/templates/clusterrole.yaml index 1742199ca..36dc5d310 100644 --- a/filebeat/templates/clusterrole.yaml +++ b/filebeat/templates/clusterrole.yaml @@ -1,5 +1,5 @@ {{- if .Values.managedServiceAccount }} -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: {{ template "filebeat.serviceAccount" . }}-cluster-role diff --git a/filebeat/templates/clusterrolebinding.yaml b/filebeat/templates/clusterrolebinding.yaml index 45436b76b..887775c2c 100644 --- a/filebeat/templates/clusterrolebinding.yaml +++ b/filebeat/templates/clusterrolebinding.yaml @@ -1,5 +1,5 @@ {{- if .Values.managedServiceAccount }} -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: {{ template "filebeat.serviceAccount" . }}-cluster-role-binding diff --git a/filebeat/tests/filebeat_test.py b/filebeat/tests/filebeat_test.py index db247ebbd..9dbb19e43 100644 --- a/filebeat/tests/filebeat_test.py +++ b/filebeat/tests/filebeat_test.py @@ -6,7 +6,6 @@ project = "filebeat" name = "release-name-" + project -name = "RELEASE-NAME-" + project def test_defaults(): diff --git a/helpers/common.mk b/helpers/common.mk index d98b0c61f..f7debead3 100644 --- a/helpers/common.mk +++ b/helpers/common.mk @@ -13,18 +13,10 @@ build: ## Build helm-tester docker image .PHONY: deps deps: ## Update helm charts dependencies - sed --in-place '/charts\//d' ./.helmignore helm dependency update -.PHONY: helm -helm: ## Deploy helm on k8s cluster - kubectl get cs - kubectl create clusterrolebinding add-on-cluster-admin --clusterrole=cluster-admin --serviceaccount=kube-system:default || true - helm init --wait --upgrade - .PHONY: lint lint: ## Lint helm templates - grep 'charts/' ./.helmignore || echo 'charts/' >> ./.helmignore helm lint --strict ./ .PHONY: lint-python @@ -44,4 +36,4 @@ test: build ## Run all tests in a docker container docker run --rm -i --user "$$(id -u):$$(id -g)" -v $$(pwd)/../:/app -w /app/$$(basename $$(pwd)) helm-tester make test-all .PHONY: test-all ## Run all tests -test-all: lint deps template pytest +test-all: deps lint template pytest diff --git a/helpers/examples.mk b/helpers/examples.mk index 66c9f2ae1..130756f4e 100644 --- a/helpers/examples.mk +++ b/helpers/examples.mk @@ -2,6 +2,7 @@ GOSS_VERSION := v0.3.6 GOSS_FILE ?= goss.yaml GOSS_SELECTOR ?= release=$(RELEASE) STACK_VERSION := 8.0.0-SNAPSHOT +TIMEOUT := 900s .PHONY: help help: ## Display this help @@ -9,7 +10,7 @@ help: ## Display this help .PHONY: goss goss: ## Run goss tests - GOSS_CONTAINER=$$(kubectl get --no-headers=true pods -l $(GOSS_SELECTOR) -o custom-columns=:metadata.name | sed -n 1p ) && \ + GOSS_CONTAINER=$$(kubectl get --no-headers=true pods -l $(GOSS_SELECTOR) -o custom-columns=:metadata.name --field-selector=status.phase=Running --sort-by=.metadata.creationTimestamp | tail -1 ) && \ echo Testing with pod: $$GOSS_CONTAINER && \ kubectl cp test/$(GOSS_FILE) $$GOSS_CONTAINER:/tmp/$(GOSS_FILE) && \ kubectl exec $$GOSS_CONTAINER -- sh -c "cd /tmp/ && curl -s -L https://github.com/aelsabbahy/goss/releases/download/$(GOSS_VERSION)/goss-linux-amd64 -o goss && chmod +rx ./goss && ./goss --gossfile $(GOSS_FILE) validate --retry-timeout 300s --sleep 5s --color --format documentation" diff --git a/helpers/helm-tester/Dockerfile b/helpers/helm-tester/Dockerfile index 08706c4eb..96ad4c483 100644 --- a/helpers/helm-tester/Dockerfile +++ b/helpers/helm-tester/Dockerfile @@ -1,13 +1,13 @@ FROM python:3.7 -ENV HELM_VERSION=2.16.12 +ENV HELM_VERSION=3.4.1 -RUN wget --no-verbose https://storage.googleapis.com/kubernetes-helm/helm-v${HELM_VERSION}-linux-amd64.tar.gz && \ +RUN wget --no-verbose https://get.helm.sh/helm-v${HELM_VERSION}-linux-amd64.tar.gz && \ tar xfv helm-v${HELM_VERSION}-linux-amd64.tar.gz && \ mv linux-amd64/helm /usr/local/bin/ && \ - rm -rf linux-amd64 && \ - HOME=/ helm init --client-only && \ - chmod 777 -R /.helm + mkdir --parents --mode=777 /.config/helm && \ + HOME=/ helm repo add stable https://charts.helm.sh/stable && \ + rm -rf helm-v${HELM_VERSION}-linux-amd64.tar.gz linux-amd64 COPY requirements.txt /usr/src/app/ RUN pip install --no-cache-dir -r /usr/src/app/requirements.txt diff --git a/helpers/helpers.py b/helpers/helpers.py index 9af447a94..4cc96756e 100644 --- a/helpers/helpers.py +++ b/helpers/helpers.py @@ -9,7 +9,7 @@ def helm_template(config): with tempfile.NamedTemporaryFile() as temp: with open(temp.name, "w") as values: values.write(config) - helm_cmd = "helm template -f {0} --namespace default ./".format(temp.name) + helm_cmd = "helm template release-name -f {0} ./".format(temp.name) result = yaml.load_all(check_output(helm_cmd.split())) results = {} diff --git a/helpers/matrix.yml b/helpers/matrix.yml index 45a53c353..aa2f4b3c4 100644 --- a/helpers/matrix.yml +++ b/helpers/matrix.yml @@ -11,27 +11,34 @@ ES_SUITE: - multi - oss - security + - upgrade KIBANA_SUITE: - default - oss - security + - upgrade FILEBEAT_SUITE: - default - oss - security + - upgrade METRICBEAT_SUITE: - default - oss - security + - upgrade LOGSTASH_SUITE: - default - oss - elasticsearch - security + - upgrade APM_SERVER_SUITE: - default - oss - security + - upgrade KUBERNETES_VERSION: - - '1.15' - - '1.16' + - "1.15" + - "1.16" + - "1.17" diff --git a/helpers/release.md b/helpers/release.md deleted file mode 100644 index 590323a82..000000000 --- a/helpers/release.md +++ /dev/null @@ -1,10 +0,0 @@ -# Release process - -* Update the [changelog](/CHANGELOG.md) -* Update the stack and chart versions in [bumper.py](/helpers/bumper.py) and run the script. This will update the versions in all the right places -* Open a pull request and wait for a green build before merging -* Create a [new release](https://github.com/elastic/helm-charts/releases/new) and include the latest changelog entry -* Run the [release script](/helpers/release.py) to build and upload the artifact - ``` - GCS_BUCKET=elastic-helm-charts python release.py - ``` diff --git a/helpers/release.py b/helpers/release.py deleted file mode 100644 index 49e140143..000000000 --- a/helpers/release.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python2 - -import glob -import os -import subprocess -import yaml - -try: - raw_input -except NameError: # Python 3 - raw_input = input - -os.chdir(os.path.join(os.path.dirname(__file__), "..")) - -bucket = "gs://" + os.environ["GCS_BUCKET"] - - -def run(cmd): - if "DEBUG" in os.environ: - print(" ".join(cmd)) - else: - subprocess.check_call(cmd) - - -# Cleanup existing releases -for release in glob.glob("*/*.tgz"): - print("Removing: " + release) - os.remove(release) - -for filepath in glob.iglob("*/Chart.yaml"): - chart = os.path.split(os.path.dirname(filepath))[-1] - - # Download dependencies - run(["helm", "dependency", "update", chart]) - - # Package up the chart - run(["helm", "package", chart, "--destination", chart]) - - # Upload it to the GCS bucket if it doesn't exist - source = "{chart}/{chart}-*.tgz".format(**locals()) - destination = "{bucket}/helm/{chart}/".format(**locals()) - run(["gsutil", "cp", "-n", source, destination]) - -# Grab the current index so we can merge it with the latest releases -run(["gsutil", "cp", bucket + "/index.yaml", "index.yaml.old"]) - -# Merge it with the old index to include the older releases -run( - [ - "helm", - "repo", - "index", - "--merge", - "index.yaml.old", - "--url", - "https://helm.elastic.co/helm/", - ".", - ] -) - -with open("index.yaml", "r") as index: - print("=" * 80) - print(index.read()) - print("=" * 80) - -answer = raw_input('Upload new index.yaml? ("y" or "yes")\n') -if answer in ["y", "yes"]: - run(["gsutil", "cp", "index.yaml", bucket + "/index.yaml"]) diff --git a/helpers/terraform/Dockerfile b/helpers/terraform/Dockerfile index c8e07a69e..3540dfb60 100644 --- a/helpers/terraform/Dockerfile +++ b/helpers/terraform/Dockerfile @@ -3,7 +3,7 @@ FROM centos:7 ENV VAULT_VERSION 0.9.3 ENV TERRAFORM_VERSION=0.11.7 ENV KUBECTL_VERSION=1.16.10 -ENV HELM_VERSION=2.16.12 +ENV HELM_VERSION=3.4.1 ENV DOCKER_VERSION=18.09.7 ENV JQ_VERSION=1.6 @@ -22,6 +22,7 @@ RUN yum -y install \ RUN curl -O https://releases.hashicorp.com/vault/${VAULT_VERSION}/vault_${VAULT_VERSION}_linux_amd64.zip && \ unzip vault_${VAULT_VERSION}_linux_amd64.zip -d /usr/local/bin/ && \ chmod +x /usr/local/bin/vault && \ + rm -f vault_${VAULT_VERSION}_linux_amd64.zip && \ vault version RUN curl -O https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \ @@ -34,17 +35,17 @@ RUN curl -O https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL chmod a+x /usr/local/bin/kubectl && \ kubectl version --client -RUN curl -O https://storage.googleapis.com/kubernetes-helm/helm-v${HELM_VERSION}-linux-amd64.tar.gz && \ +RUN curl -O https://get.helm.sh/helm-v${HELM_VERSION}-linux-amd64.tar.gz && \ tar xfv helm-v${HELM_VERSION}-linux-amd64.tar.gz && \ mv linux-amd64/helm /usr/local/bin/ && \ - rm -rf linux-amd64 && \ + rm -rf helm-v${HELM_VERSION}-linux-amd64.tar.gz linux-amd64 && \ helm version --client RUN curl -O https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz && \ tar xfv docker* && \ mv docker/docker /usr/local/bin && \ - rm -rf docker/ && \ - docker + rm -rf docker-${DOCKER_VERSION}.tgz docker/ && \ + docker -v RUN curl -O -L https://github.com/stedolan/jq/releases/download/jq-${JQ_VERSION}/jq-linux64 && \ mv jq-linux64 /usr/local/bin/jq && \ diff --git a/helpers/terraform/Makefile b/helpers/terraform/Makefile index a6dcff24b..8d448c3e3 100644 --- a/helpers/terraform/Makefile +++ b/helpers/terraform/Makefile @@ -65,8 +65,8 @@ creds: credentials.json ## Get gke credentials kubectl create namespace $(NAMESPACE) || true kubectl config set-context $$(kubectl config current-context) --namespace=$(NAMESPACE) -.PHONY: k8s -k8s: apply creds ## Configure gke cluster +.PHONY: up +up: apply creds ## Configure gke cluster kubectl get cs .PHONY: k8s-staging-registry @@ -77,15 +77,10 @@ k8s-staging-registry: creds ## Create the staging registry auth secret in k8s --docker-username="devops-ci" \ --docker-password="$$DOCKER_PASSWORD" -.PHONY: up -up: k8s ## Install helm on gke cluster - kubectl create clusterrolebinding add-on-cluster-admin --clusterrole=cluster-admin --serviceaccount=kube-system:default || true - for i in 1 2 3 4 5; do helm init --wait --upgrade && break || sleep 5; done - .PHONY: integration integration: creds ## Deploy helm chart and run integration tests cd ../../$(CHART)/ && \ - helm init --client-only && \ + helm repo add stable https://charts.helm.sh/stable && \ helm dependency update && \ cd ./examples/$(SUITE) && \ make @@ -98,4 +93,3 @@ build: ## Build helm-charts docker image pull-private-images: ## Pull private images used in testing cd ../../elasticsearch/examples/security/ && \ make pull-elasticsearch-image - diff --git a/helpers/upgrade.sh b/helpers/upgrade.sh new file mode 100755 index 000000000..269b64d8a --- /dev/null +++ b/helpers/upgrade.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# +# upgrade.sh deploy some Helm chart to a specific released version, +# then upgrade it. +# +# An optional version can be specified for Docker image tag to use for upgrade. +# This is required for master branch because upgrade from Elasticsearch 7.X +# to 8.0.0-SNAPSHOT doesn't work. +# +set -euo pipefail + +TO="" + +usage() { + cat <<-EOF + USAGE: + $0 --chart --release --from [--to ] + $0 --help + + OPTIONS: + --chart + Name of the Elastic Helm chart to install (ie: elasticsearch) + --release + Name of the Helm release to install (ie: helm-upgrade-elasticsearch) + --from + Version to use for first install (ie: 7.7.0) + --to + Version of the Docker images to use for upgrade (ie: 7.10.0) + EOF + exit 1 +} + +while [[ $# -gt 0 ]] +do + key="$1" + + case $key in + --help) + usage + ;; + --chart) + CHART="$2" + shift 2 + ;; + --release) + RELEASE="$2" + shift 2 + ;; + --from) + FROM="$2" + shift 2 + ;; + --to) + TO="--set imageTag=$2" + shift 2 + ;; + *) + log "Unrecognized argument: '$key'" + usage + ;; + esac +done + +helm repo add elastic https://helm.elastic.co + +# Initial install +printf "Installing %s %s\n" "$RELEASE" "$FROM" +helm upgrade --wait --timeout=1200s --install --version "$FROM" --values values.yaml "$RELEASE" elastic/"$CHART" + +# Upgrade +printf "Upgrading %s\n" "$RELEASE" +# shellcheck disable=SC2086 +helm upgrade --wait --timeout=1200s --install --set terminationGracePeriod=121 $TO --values values.yaml "$RELEASE" ../../ diff --git a/kibana/README.md b/kibana/README.md index fb156ab53..72e08cc3d 100644 --- a/kibana/README.md +++ b/kibana/README.md @@ -3,6 +3,7 @@ This Helm chart is a lightweight way to configure and run our official [Kibana Docker image][]. + **Warning**: This branch is used for development, please use the latest [7.x][] release for released version. @@ -32,8 +33,8 @@ This Helm chart is a lightweight way to configure and run our official ## Requirements -* [Helm][] >=2.8.0 and <3.0.0 -* Kubernetes >=1.9 +* Kubernetes >= 1.14 +* [Helm][] >= 2.17.0 See [supported configurations][] for more details. @@ -46,8 +47,8 @@ See [supported configurations][] for more details. `helm repo add elastic https://helm.elastic.co` * Install it: - - with Helm 2: `helm install --name kibana elastic/kibana` - - with [Helm 3 (beta)][]: `helm install kibana elastic/kibana` + - with Helm 3: `helm install kibana elastic/kibana` + - with Helm 2 (deprecated): `helm install --name kibana elastic/kibana` ### Install development version using master branch @@ -55,8 +56,8 @@ See [supported configurations][] for more details. * Clone the git repo: `git clone git@github.com:elastic/helm-charts.git` * Install it: - - with Helm 2: `helm install --name kibana ./helm-charts/kibana --set imageTag=8.0.0-SNAPSHOT` - - with [Helm 3 (beta)][]: `helm install kibana ./helm-charts/kibana --set imageTag=8.0.0-SNAPSHOT` + - with Helm 3: `helm install kibana ./helm-charts/kibana --set imageTag=8.0.0-SNAPSHOT` + - with Helm 2 (deprecated): `helm install --name kibana ./helm-charts/kibana --set imageTag=8.0.0-SNAPSHOT` ## Upgrading @@ -89,7 +90,7 @@ as a reference. They are also used in the automated testing of this chart. | `httpPort` | The http port that Kubernetes will use for the healthchecks and the service | `5601` | | `imagePullPolicy` | The Kubernetes [imagePullPolicy][]value | `IfNotPresent` | | `imagePullSecrets` | Configuration for [imagePullSecrets][] so that you can use a private registry for your image | `[]` | -| `imageTag` | The Kibana Docker image tag | `8.0.0-SNAPSHOT` | +| `imageTag` | The Kibana Docker image tag | `8.0.0-SNAPSHOT` | | `image` | The Kibana Docker image | `docker.elastic.co/kibana/kibana` | | `ingress` | Configurable [ingress][] to expose the Kibana service. | see [values.yaml][] | | `kibanaConfig` | Allows you to add any config files in `/usr/share/kibana/config/` such as `kibana.yml` See [values.yaml][] for an example of the formatting | `{}` | @@ -215,7 +216,6 @@ about our development and testing process. [examples/security]: https://github.com/elastic/helm-charts/tree/master/kibana/examples/security [gke]: https://cloud.google.com/kubernetes-engine [helm]: https://helm.sh -[helm 3 (beta)]: https://github.com/elastic/helm-charts/tree/master/README.md#helm-3-beta [imagePullPolicy]: https://kubernetes.io/docs/concepts/containers/images/#updating-images [imagePullSecrets]: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-pod-that-uses-your-secret [ingress]: https://kubernetes.io/docs/concepts/services-networking/ingress/ diff --git a/kibana/examples/default/Makefile b/kibana/examples/default/Makefile index 39caa3072..d181571fa 100644 --- a/kibana/examples/default/Makefile +++ b/kibana/examples/default/Makefile @@ -1,13 +1,13 @@ default: test + include ../../../helpers/examples.mk RELEASE := helm-kibana-default install: - echo "Goss container: $(GOSS_CONTAINER)" - helm upgrade --wait --timeout=600 --install $(RELEASE) ../../ + helm upgrade --wait --timeout=$(TIMEOUT) --install $(RELEASE) ../../ test: install goss purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/kibana/examples/openshift/Makefile b/kibana/examples/openshift/Makefile index 9dccc65ed..35b66d02b 100644 --- a/kibana/examples/openshift/Makefile +++ b/kibana/examples/openshift/Makefile @@ -4,12 +4,12 @@ include ../../../helpers/examples.mk RELEASE := kibana template: - helm template --values ./values.yml ../../ + helm template --values values.yaml ../../ -install: - helm upgrade --wait --timeout=600 --install --values ./values.yml $(RELEASE) ../../ +install: + helm upgrade --wait --timeout=$(TIMEOUT) --install --values values.yaml $(RELEASE) ../../ test: install goss - + purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/kibana/examples/oss/Makefile b/kibana/examples/oss/Makefile index 77c6412db..c4804946a 100644 --- a/kibana/examples/oss/Makefile +++ b/kibana/examples/oss/Makefile @@ -1,12 +1,13 @@ default: test + include ../../../helpers/examples.mk RELEASE := helm-kibana-oss install: - helm upgrade --wait --timeout=600 --install --values ./values.yml $(RELEASE) ../../ + helm upgrade --wait --timeout=$(TIMEOUT) --install --values values.yaml $(RELEASE) ../../ test: install goss purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/kibana/examples/oss/values.yml b/kibana/examples/oss/values.yaml similarity index 100% rename from kibana/examples/oss/values.yml rename to kibana/examples/oss/values.yaml diff --git a/kibana/examples/security/Makefile b/kibana/examples/security/Makefile index d3a365fd7..034c8c5cc 100644 --- a/kibana/examples/security/Makefile +++ b/kibana/examples/security/Makefile @@ -1,16 +1,17 @@ default: test + include ../../../helpers/examples.mk RELEASE := helm-kibana-security install: - helm upgrade --wait --timeout=600 --install --values ./security.yml $(RELEASE) ../../ + helm upgrade --wait --timeout=$(TIMEOUT) --install --values values.yaml $(RELEASE) ../../ test: secrets install goss purge: kubectl delete secret kibana || true - helm del --purge $(RELEASE) + helm del $(RELEASE) secrets: encryptionkey=$$(docker run --rm busybox:1.31.1 /bin/sh -c "< /dev/urandom tr -dc _A-Za-z0-9 | head -c50") && \ diff --git a/kibana/examples/security/README.md b/kibana/examples/security/README.md index bceca9cd0..8bc5e0199 100644 --- a/kibana/examples/security/README.md +++ b/kibana/examples/security/README.md @@ -25,4 +25,4 @@ You can also run [goss integration tests][] using `make test` [elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/master/elasticsearch/examples/security/ [goss integration tests]: https://github.com/elastic/helm-charts/tree/master/kibana/examples/security/test/goss.yaml -[values]: https://github.com/elastic/helm-charts/tree/master/kibana/examples/security/security.yml +[values]: https://github.com/elastic/helm-charts/tree/master/kibana/examples/security/values.yaml diff --git a/kibana/examples/security/security.yml b/kibana/examples/security/values.yaml similarity index 100% rename from kibana/examples/security/security.yml rename to kibana/examples/security/values.yaml diff --git a/kibana/examples/upgrade/Makefile b/kibana/examples/upgrade/Makefile new file mode 100644 index 000000000..988c56c00 --- /dev/null +++ b/kibana/examples/upgrade/Makefile @@ -0,0 +1,17 @@ +default: test + +include ../../../helpers/examples.mk + +CHART := kibana +RELEASE := helm-kibana-upgrade +FROM := 7.4.0 # versions before 7.4.O aren't compatible with Kubernetes >= 1.16.0 +TO := 7.10.0 # required to use with Elasticsearch 7.10.0 + +install: + ../../../helpers/upgrade.sh --chart $(CHART) --release $(RELEASE) --from $(FROM) --to $(TO) + kubectl rollout status deployment $(RELEASE)-kibana + +test: install goss + +purge: + helm del $(RELEASE) diff --git a/kibana/examples/upgrade/README.md b/kibana/examples/upgrade/README.md new file mode 100644 index 000000000..ad28f65c6 --- /dev/null +++ b/kibana/examples/upgrade/README.md @@ -0,0 +1,21 @@ +# Upgrade + +This example will deploy Kibana chart using an old chart version, +then upgrade it. + + +## Usage + +* Add the Elastic Helm charts repo: `helm repo add elastic https://helm.elastic.co` + +* Deploy [Elasticsearch Helm chart][]: `helm install elasticsearch elastic/elasticsearch` + +* Deploy and upgrade Kibana chart with the default values: `make install` + + +## Testing + +You can also run [goss integration tests][] using `make test`. + + +[goss integration tests]: https://github.com/elastic/helm-charts/tree/master/kibana/examples/upgrade/test/goss.yaml diff --git a/kibana/examples/upgrade/test/goss.yaml b/kibana/examples/upgrade/test/goss.yaml new file mode 100644 index 000000000..5b3a1b28a --- /dev/null +++ b/kibana/examples/upgrade/test/goss.yaml @@ -0,0 +1,14 @@ +http: + http://localhost:5601/api/status: + status: 200 + timeout: 2000 + body: + - '"number":"7.10.0"' + + http://localhost:5601/app/kibana: + status: 200 + timeout: 2000 + + http://helm-kibana-upgrade-kibana:5601/app/kibana: + status: 200 + timeout: 2000 diff --git a/kibana/examples/upgrade/values.yaml b/kibana/examples/upgrade/values.yaml new file mode 100644 index 000000000..01d99c838 --- /dev/null +++ b/kibana/examples/upgrade/values.yaml @@ -0,0 +1,2 @@ +--- +elasticsearchHosts: "http://upgrade-master:9200" diff --git a/kibana/templates/_helpers.tpl b/kibana/templates/_helpers.tpl index d2ab927b6..407e29680 100755 --- a/kibana/templates/_helpers.tpl +++ b/kibana/templates/_helpers.tpl @@ -19,17 +19,6 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this {{- end -}} {{- end -}} -{{/* -Return the appropriate apiVersion for ingress. -*/}} -{{- define "kibana.ingress.apiVersion" -}} -{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} -{{- print "extensions/v1beta1" -}} -{{- else -}} -{{- print "networking.k8s.io/v1beta1" -}} -{{- end -}} -{{- end -}} - {{/* Common labels */}} diff --git a/kibana/templates/ingress.yaml b/kibana/templates/ingress.yaml index e62977495..fd4eae24a 100644 --- a/kibana/templates/ingress.yaml +++ b/kibana/templates/ingress.yaml @@ -2,7 +2,7 @@ {{- $fullName := include "kibana.fullname" . -}} {{- $servicePort := .Values.service.port -}} {{- $ingressPath := .Values.ingress.path -}} -apiVersion: {{ template "kibana.ingress.apiVersion" . }} +apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: name: {{ $fullName }} diff --git a/kibana/templates/service.yaml b/kibana/templates/service.yaml index dfb254c0c..fbced7344 100644 --- a/kibana/templates/service.yaml +++ b/kibana/templates/service.yaml @@ -26,7 +26,7 @@ spec: nodePort: {{ .Values.service.nodePort }} {{- end }} protocol: TCP - name: http + name: {{ .Values.service.httpPortName | default "http" }} targetPort: {{ .Values.httpPort }} selector: app: {{ .Chart.Name }} diff --git a/kibana/tests/kibana_test.py b/kibana/tests/kibana_test.py index 38a9fb154..d58da0301 100644 --- a/kibana/tests/kibana_test.py +++ b/kibana/tests/kibana_test.py @@ -629,3 +629,16 @@ def test_adding_loadBalancerIP(): r = helm_template(config) assert r["service"][name]["spec"]["loadBalancerIP"] == "12.5.11.79" + + +def test_service_port_name(): + r = helm_template("") + + config = """ + service: + httpPortName: istio + """ + + r = helm_template(config) + + assert r["service"][name]["spec"]["ports"][0]["name"] == "istio" diff --git a/kibana/values.yaml b/kibana/values.yaml index 2b8e4fcf5..cf6980fed 100755 --- a/kibana/values.yaml +++ b/kibana/values.yaml @@ -107,6 +107,7 @@ service: # service.beta.kubernetes.io/cce-load-balancer-internal-vpc: "true" loadBalancerSourceRanges: [] # 0.0.0.0/0 + httpPortName: http ingress: enabled: false diff --git a/logstash/README.md b/logstash/README.md index 66b16a215..5a3306253 100644 --- a/logstash/README.md +++ b/logstash/README.md @@ -8,6 +8,7 @@ The design and code is less mature than official GA features and is being provided as-is with no warranties. Alpha features are not subject to the support SLA of official GA features (see [supported configurations][] for more details). + **Warning**: This branch is used for development, please use the latest [7.x][] release for released version. @@ -33,8 +34,8 @@ SLA of official GA features (see [supported configurations][] for more details). ## Requirements -* [Helm][] >=2.8.0 and <3.0.0 -* Kubernetes >=1.8 +* Kubernetes >= 1.14 +* [Helm][] >= 2.17.0 See [supported configurations][] for more details. @@ -46,8 +47,8 @@ See [supported configurations][] for more details. `helm repo add elastic https://helm.elastic.co` * Install it: - - with Helm 2: `helm install --name logstash elastic/logstash` - - with [Helm 3 (beta)][]: `helm install logstash elastic/logstash` + - with Helm 3: `helm install logstash elastic/logstash` + - with Helm 2 (deprecated): `helm install --name logstash elastic/logstash` ### Install development version using master branch @@ -55,8 +56,8 @@ See [supported configurations][] for more details. * Clone the git repo: `git clone git@github.com:elastic/helm-charts.git` * Install it: - - with Helm 2: `helm install --name logstash ./helm-charts/logstash --set imageTag=8.0.0-SNAPSHOT` - - with [Helm 3 (beta)][]: `helm install logstash ./helm-charts/logstash --set imageTag=8.0.0-SNAPSHOT` + - with Helm 3: `helm install logstash ./helm-charts/logstash --set imageTag=8.0.0-SNAPSHOT` + - with Helm 2 (deprecated): `helm install --name logstash ./helm-charts/logstash --set imageTag=8.0.0-SNAPSHOT` ## Upgrading @@ -201,7 +202,6 @@ about our development and testing process. [examples]: https://github.com/elastic/helm-charts/tree/master/logstash/examples [examples/oss]: https://github.com/elastic/helm-charts/tree/master/logstash/examples/oss [helm]: https://helm.sh -[helm 3 (beta)]: https://github.com/elastic/helm-charts/tree/master/README.md#helm-3-beta [http input plugin]: https://www.elastic.co/guide/en/logstash/current/plugins-inputs-http.html [imagePullPolicy]: https://kubernetes.io/docs/concepts/containers/images/#updating-images [imagePullSecrets]: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-pod-that-uses-your-secret diff --git a/logstash/examples/default/Makefile b/logstash/examples/default/Makefile index 03d9592f1..03d77f816 100644 --- a/logstash/examples/default/Makefile +++ b/logstash/examples/default/Makefile @@ -3,14 +3,12 @@ default: test include ../../../helpers/examples.mk RELEASE := helm-logstash-default +TIMEOUT := 1200s install: - helm upgrade --wait --timeout=900 --install $(RELEASE) ../../ - -restart: - helm upgrade --set terminationGracePeriod=121 --wait --timeout=900 --install $(RELEASE) ../../ + helm upgrade --wait --timeout=$(TIMEOUT) --install $(RELEASE) ../../ test: install goss purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/logstash/examples/default/test/goss.yaml b/logstash/examples/default/test/goss.yaml index 13b15955d..5fc0d1b80 100644 --- a/logstash/examples/default/test/goss.yaml +++ b/logstash/examples/default/test/goss.yaml @@ -9,10 +9,8 @@ http: status: 200 timeout: 2000 body: - - '"host" : "helm-logstash-default-logstash-0"' - '"version" : "8.0.0"' - '"http_address" : "0.0.0.0:9600"' - - '"name" : "helm-logstash-default-logstash-0"' - '"status" : "green"' - '"workers" : 1' - '"batch_size" : 125' @@ -35,9 +33,9 @@ file: group: root filetype: file contains: - - 'input {' - - 'beats {' - - 'port => 5044' - - 'output {' - - 'stdout {' - - 'codec => rubydebug' + - "input {" + - "beats {" + - "port => 5044" + - "output {" + - "stdout {" + - "codec => rubydebug" diff --git a/logstash/examples/elasticsearch/Makefile b/logstash/examples/elasticsearch/Makefile index f78c42f4e..6b914df90 100644 --- a/logstash/examples/elasticsearch/Makefile +++ b/logstash/examples/elasticsearch/Makefile @@ -3,15 +3,13 @@ default: test include ../../../helpers/examples.mk RELEASE := helm-logstash-elasticsearch +TIMEOUT := 1200s install: - helm upgrade --wait --timeout=900 --install $(RELEASE) --values ./values.yaml ../../ - -restart: - helm upgrade --set terminationGracePeriod=121 --wait --timeout=900 --install $(RELEASE) ../../ + helm upgrade --wait --timeout=$(TIMEOUT) --install --values values.yaml $(RELEASE) ../../ test: install goss purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) kubectl delete $$(kubectl get pvc -l release=$(RELEASE) -o name) diff --git a/logstash/examples/elasticsearch/test/goss.yaml b/logstash/examples/elasticsearch/test/goss.yaml index e24fdadd0..8dd73f80d 100644 --- a/logstash/examples/elasticsearch/test/goss.yaml +++ b/logstash/examples/elasticsearch/test/goss.yaml @@ -21,10 +21,8 @@ http: status: 200 timeout: 2000 body: - - '"host" : "helm-logstash-elasticsearch-logstash-0"' - '"version" : "8.0.0"' - '"http_address" : "0.0.0.0:9600"' - - '"name" : "helm-logstash-elasticsearch-logstash-0"' - '"status" : "green"' - '"workers" : 1' - '"batch_size" : 125' @@ -33,7 +31,7 @@ http: status: 200 timeout: 2000 body: - - 'logstash' + - "logstash" file: /usr/share/logstash/config/logstash.yml: @@ -43,8 +41,8 @@ file: group: logstash filetype: file contains: - - 'http.host: 0.0.0.0' - - 'xpack.monitoring.enabled: false' + - "http.host: 0.0.0.0" + - "xpack.monitoring.enabled: false" /usr/share/logstash/pipeline/uptime.conf: exists: true mode: "0644" diff --git a/logstash/examples/oss/Makefile b/logstash/examples/oss/Makefile index 856234431..8f96d8286 100644 --- a/logstash/examples/oss/Makefile +++ b/logstash/examples/oss/Makefile @@ -3,14 +3,12 @@ default: test include ../../../helpers/examples.mk RELEASE := helm-logstash-oss +TIMEOUT := 1200s install: - helm upgrade --wait --timeout=900 --install $(RELEASE) --values ./values.yaml ../../ - -restart: - helm upgrade --set terminationGracePeriod=121 --wait --timeout=900 --install $(RELEASE) ../../ + helm upgrade --wait --timeout=$(TIMEOUT) --install --values values.yaml $(RELEASE) ../../ test: install goss purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/logstash/examples/oss/test/goss.yaml b/logstash/examples/oss/test/goss.yaml index 29cf3b528..86e49b021 100644 --- a/logstash/examples/oss/test/goss.yaml +++ b/logstash/examples/oss/test/goss.yaml @@ -9,10 +9,8 @@ http: status: 200 timeout: 2000 body: - - '"host" : "helm-logstash-oss-logstash-0"' - '"version" : "8.0.0"' - '"http_address" : "0.0.0.0:9600"' - - '"name" : "helm-logstash-oss-logstash-0"' - '"status" : "green"' - '"workers" : 1' - '"batch_size" : 125' diff --git a/logstash/examples/security/Makefile b/logstash/examples/security/Makefile index 8bfee0711..d5bfcb2f8 100644 --- a/logstash/examples/security/Makefile +++ b/logstash/examples/security/Makefile @@ -3,12 +3,13 @@ default: test include ../../../helpers/examples.mk RELEASE := helm-logstash-security +TIMEOUT := 1200s install: - helm upgrade --wait --timeout=900 --install $(RELEASE) --values values.yaml ../../ + helm upgrade --wait --timeout=$(TIMEOUT) --install --values values.yaml $(RELEASE) ../../ test: install goss purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) kubectl delete $$(kubectl get pvc -l release=$(RELEASE) -o name) diff --git a/logstash/examples/security/test/goss.yaml b/logstash/examples/security/test/goss.yaml index f6a2fbe0a..3730a2a6c 100644 --- a/logstash/examples/security/test/goss.yaml +++ b/logstash/examples/security/test/goss.yaml @@ -21,10 +21,8 @@ http: status: 200 timeout: 2000 body: - - '"host" : "helm-logstash-security-logstash-0"' - '"version" : "8.0.0"' - '"http_address" : "0.0.0.0:9600"' - - '"name" : "helm-logstash-security-logstash-0"' - '"status" : "green"' - '"workers" : 1' - '"batch_size" : 125' @@ -33,10 +31,10 @@ http: status: 200 timeout: 2000 body: - - 'logstash' + - "logstash" allow-insecure: true - username: '{{ .Env.ELASTICSEARCH_USERNAME }}' - password: '{{ .Env.ELASTICSEARCH_PASSWORD }}' + username: "{{ .Env.ELASTICSEARCH_USERNAME }}" + password: "{{ .Env.ELASTICSEARCH_PASSWORD }}" file: /usr/share/logstash/config/logstash.yml: @@ -46,10 +44,10 @@ file: group: logstash filetype: file contains: - - 'http.host: 0.0.0.0' - - 'xpack.monitoring.enabled: true' + - "http.host: 0.0.0.0" + - "xpack.monitoring.enabled: true" - 'xpack.monitoring.elasticsearch.hosts: ["https://security-master:9200"]' - - 'xpack.monitoring.elasticsearch.ssl.certificate_authority: /usr/share/logstash/config/certs/elastic-certificate.crt' + - "xpack.monitoring.elasticsearch.ssl.certificate_authority: /usr/share/logstash/config/certs/elastic-certificate.crt" /usr/share/logstash/pipeline/uptime.conf: exists: true mode: "0644" @@ -58,7 +56,7 @@ file: filetype: file contains: - 'input { exec { command => "uptime" interval => 30 } }' - - 'output { elasticsearch {' + - "output { elasticsearch {" - 'hosts => ["https://security-master:9200"]' - 'cacert => "/usr/share/logstash/config/certs/elastic-certificate.crt"' - 'index => "logstash"' diff --git a/logstash/examples/upgrade/Makefile b/logstash/examples/upgrade/Makefile new file mode 100644 index 000000000..e5ee63671 --- /dev/null +++ b/logstash/examples/upgrade/Makefile @@ -0,0 +1,16 @@ +default: test + +include ../../../helpers/examples.mk + +CHART := logstash +RELEASE := helm-logstash-upgrade +FROM := 7.9.0 # upgrade from version < 7.9.0 is failing due to headless service breaking change + +install: + ../../../helpers/upgrade.sh --chart $(CHART) --release $(RELEASE) --from $(FROM) + kubectl rollout status statefulset $(RELEASE)-logstash + +test: install goss + +purge: + helm del $(RELEASE) diff --git a/logstash/examples/upgrade/README.md b/logstash/examples/upgrade/README.md new file mode 100644 index 000000000..c8986a070 --- /dev/null +++ b/logstash/examples/upgrade/README.md @@ -0,0 +1,19 @@ +# Upgrade + +This example will deploy Logstash chart using an old chart version, +then upgrade it. + + +## Usage + +* Add the Elastic Helm charts repo: `helm repo add elastic https://helm.elastic.co` + +* Deploy and upgrade Logstash chart with the default values: `make install` + + +## Testing + +You can also run [goss integration tests][] using `make test`. + + +[goss integration tests]: https://github.com/elastic/helm-charts/tree/master/logstash/examples/upgrade/test/goss.yaml diff --git a/logstash/examples/upgrade/test/goss.yaml b/logstash/examples/upgrade/test/goss.yaml new file mode 100644 index 000000000..5fc0d1b80 --- /dev/null +++ b/logstash/examples/upgrade/test/goss.yaml @@ -0,0 +1,41 @@ +user: + logstash: + exists: true + uid: 1000 + gid: 1000 + +http: + http://localhost:9600?pretty: + status: 200 + timeout: 2000 + body: + - '"version" : "8.0.0"' + - '"http_address" : "0.0.0.0:9600"' + - '"status" : "green"' + - '"workers" : 1' + - '"batch_size" : 125' + - '"batch_delay" : 50' + +file: + /usr/share/logstash/config/logstash.yml: + exists: true + mode: "0644" + owner: logstash + group: root + filetype: file + contains: + - 'http.host: "0.0.0.0"' + - 'xpack.monitoring.elasticsearch.hosts: [ "http://elasticsearch:9200" ]' + /usr/share/logstash/pipeline/logstash.conf: + exists: true + mode: "0644" + owner: logstash + group: root + filetype: file + contains: + - "input {" + - "beats {" + - "port => 5044" + - "output {" + - "stdout {" + - "codec => rubydebug" diff --git a/logstash/examples/upgrade/values.yaml b/logstash/examples/upgrade/values.yaml new file mode 100644 index 000000000..ed97d539c --- /dev/null +++ b/logstash/examples/upgrade/values.yaml @@ -0,0 +1 @@ +--- diff --git a/logstash/templates/_helpers.tpl b/logstash/templates/_helpers.tpl index d240a026e..82aad6075 100755 --- a/logstash/templates/_helpers.tpl +++ b/logstash/templates/_helpers.tpl @@ -18,25 +18,3 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} {{- end -}} {{- end -}} - -{{/* -Return the appropriate apiVersion for statefulset. -*/}} -{{- define "logstash.statefulset.apiVersion" -}} -{{- if semverCompare "<1.9-0" .Capabilities.KubeVersion.GitVersion -}} -{{- print "apps/v1beta2" -}} -{{- else -}} -{{- print "apps/v1" -}} -{{- end -}} -{{- end -}} - -{{/* -Return the appropriate apiVersion for ingress. -*/}} -{{- define "logstash.ingress.apiVersion" -}} -{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} -{{- print "extensions/v1beta1" -}} -{{- else -}} -{{- print "networking.k8s.io/v1beta1" -}} -{{- end -}} -{{- end -}} diff --git a/logstash/templates/ingress.yaml b/logstash/templates/ingress.yaml index cef5666b9..a4aeb145e 100644 --- a/logstash/templates/ingress.yaml +++ b/logstash/templates/ingress.yaml @@ -1,6 +1,6 @@ {{- if .Values.ingress.enabled -}} {{- $fullName := include "logstash.fullname" . -}} -apiVersion: {{ template "logstash.ingress.apiVersion" . }} +apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: name: {{ $fullName }} @@ -28,6 +28,6 @@ spec: backend: serviceName: {{ $fullName }} servicePort: {{ .servicePort }} - {{- end }} + {{- end }} {{- end }} {{- end }} diff --git a/logstash/templates/service.yaml b/logstash/templates/service.yaml index 6cbca1ee4..6540c8c61 100644 --- a/logstash/templates/service.yaml +++ b/logstash/templates/service.yaml @@ -16,7 +16,6 @@ spec: selector: app: "{{ template "logstash.fullname" . }}" chart: "{{ .Chart.Name }}" - heritage: {{ .Release.Service | quote }} release: {{ .Release.Name | quote }} ports: {{ toYaml .Values.service.ports | indent 4 }} diff --git a/logstash/templates/serviceaccount.yaml b/logstash/templates/serviceaccount.yaml index 4508878b8..bb72735b6 100644 --- a/logstash/templates/serviceaccount.yaml +++ b/logstash/templates/serviceaccount.yaml @@ -17,4 +17,10 @@ metadata: chart: "{{ .Chart.Name }}" heritage: {{ .Release.Service | quote }} release: {{ .Release.Name | quote }} + {{- if .Values.rbac.annotations }} + annotations: + {{- range $key, $value := .Values.rbac.annotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} {{- end -}} diff --git a/logstash/templates/statefulset.yaml b/logstash/templates/statefulset.yaml index 9200cfc0c..55548eced 100644 --- a/logstash/templates/statefulset.yaml +++ b/logstash/templates/statefulset.yaml @@ -1,5 +1,5 @@ --- -apiVersion: {{ template "logstash.statefulset.apiVersion" . }} +apiVersion: apps/v1 kind: StatefulSet metadata: name: {{ template "logstash.fullname" . }} diff --git a/logstash/tests/logstash_test.py b/logstash/tests/logstash_test.py index caee32c29..1555b960c 100755 --- a/logstash/tests/logstash_test.py +++ b/logstash/tests/logstash_test.py @@ -760,6 +760,8 @@ def test_pod_security_policy(): rbac: create: true serviceAccountName: "" + annotations: + "eks.amazonaws.com/role-arn": "test-rbac-annotations" podSecurityPolicy: create: true diff --git a/logstash/values.yaml b/logstash/values.yaml index 3ed7fa239..ff242820c 100755 --- a/logstash/values.yaml +++ b/logstash/values.yaml @@ -89,6 +89,10 @@ rbac: create: false serviceAccountAnnotations: {} serviceAccountName: "" + annotations: {} + #annotation1: "value1" + #annotation2: "value2" + #annotation3: "value3" podSecurityPolicy: create: false diff --git a/metricbeat/README.md b/metricbeat/README.md index 3b8c0004f..9d1ee2ee0 100644 --- a/metricbeat/README.md +++ b/metricbeat/README.md @@ -3,6 +3,7 @@ This Helm chart is a lightweight way to configure and run our official [Metricbeat Docker image][]. + **Warning**: This branch is used for development, please use the latest [7.x][] release for released version. @@ -31,8 +32,8 @@ This Helm chart is a lightweight way to configure and run our official ## Requirements -* [Helm][] >=2.8.0 and <3.0.0 -* Kubernetes >=1.9 +* Kubernetes >= 1.14 +* [Helm][] >= 2.17.0 See [supported configurations][] for more details. @@ -45,8 +46,8 @@ See [supported configurations][] for more details. `helm repo add elastic https://helm.elastic.co` * Install it: - - with Helm 2: `helm install --name metricbeat elastic/metricbeat` - - with [Helm 3 (beta)][]: `helm install metricbeat elastic/metricbeat` + - with Helm 3: `helm install metricbeat elastic/metricbeat` + - with Helm 2 (deprecated): `helm install --name metricbeat elastic/metricbeat` ### Install development version using master branch @@ -54,8 +55,8 @@ See [supported configurations][] for more details. * Clone the git repo: `git clone git@github.com:elastic/helm-charts.git` * Install it: - - with Helm 2: `helm install --name metricbeat ./helm-charts/metricbeat --set imageTag=8.0.0-SNAPSHOT` - - with [Helm 3 (beta)][]: `helm install metricbeat ./helm-charts/metricbeat --set imageTag=8.0.0-SNAPSHOT` + - with Helm 3: `helm install metricbeat ./helm-charts/metricbeat --set imageTag=8.0.0-SNAPSHOT` + - with Helm 2 (deprecated): `helm install --name metricbeat ./helm-charts/metricbeat --set imageTag=8.0.0-SNAPSHOT` ## Upgrading @@ -223,7 +224,6 @@ about our development and testing process. [examples/oss]: https://github.com/elastic/helm-charts/tree/master/metricbeat/examples/oss [examples/security]: https://github.com/elastic/helm-charts/tree/master/metricbeat/examples/security [helm]: https://helm.sh -[helm 3 (beta)]: https://github.com/elastic/helm-charts/tree/master/README.md#helm-3-beta [hostPath]: https://kubernetes.io/docs/concepts/storage/volumes/#hostpath [hostNetwork]: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#host-namespaces [imagePullPolicy]: https://kubernetes.io/docs/concepts/containers/images/#updating-images diff --git a/metricbeat/examples/default/Makefile b/metricbeat/examples/default/Makefile index 833f62973..bcd4fb77b 100644 --- a/metricbeat/examples/default/Makefile +++ b/metricbeat/examples/default/Makefile @@ -6,7 +6,7 @@ RELEASE = helm-metricbeat-default GOSS_SELECTOR = release=$(RELEASE),app=helm-metricbeat-default-metricbeat install: - helm upgrade --wait --timeout=600 --install $(RELEASE) ../../ + helm upgrade --wait --timeout=$(TIMEOUT) --install $(RELEASE) ../../ test-metrics: GOSS_FILE=goss-metrics.yaml make goss GOSS_SELECTOR=release=$(RELEASE),app=helm-metricbeat-default-metricbeat-metrics @@ -14,4 +14,4 @@ test-metrics: test: install goss test-metrics purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/metricbeat/examples/oss/Makefile b/metricbeat/examples/oss/Makefile index 0e4828ed0..534201044 100644 --- a/metricbeat/examples/oss/Makefile +++ b/metricbeat/examples/oss/Makefile @@ -6,12 +6,12 @@ RELEASE := helm-metricbeat-oss GOSS_SELECTOR = release=$(RELEASE),app=helm-metricbeat-oss-metricbeat install: - helm upgrade --wait --timeout=600 --install $(RELEASE) --values values.yaml ../../ - -purge: - helm del --purge $(RELEASE) + helm upgrade --wait --timeout=$(TIMEOUT) --install --values values.yaml $(RELEASE) ../../ test-metrics: GOSS_FILE=goss-metrics.yaml make goss GOSS_SELECTOR=release=$(RELEASE),app=helm-metricbeat-oss-metricbeat-metrics test: install goss test-metrics + +purge: + helm del $(RELEASE) diff --git a/metricbeat/examples/security/Makefile b/metricbeat/examples/security/Makefile index 3f92e7fe2..2433d8e95 100644 --- a/metricbeat/examples/security/Makefile +++ b/metricbeat/examples/security/Makefile @@ -6,12 +6,12 @@ RELEASE := helm-metricbeat-security GOSS_SELECTOR = release=$(RELEASE),app=helm-metricbeat-security-metricbeat install: - helm upgrade --wait --timeout=600 --install $(RELEASE) --values values.yaml ../../ - -purge: - helm del --purge $(RELEASE) + helm upgrade --wait --timeout=$(TIMEOUT) --install --values values.yaml $(RELEASE) ../../ test-metrics: GOSS_FILE=goss-metrics.yaml make goss GOSS_SELECTOR=release=$(RELEASE),app=helm-metricbeat-security-metricbeat-metrics test: install goss test-metrics + +purge: + helm del $(RELEASE) diff --git a/metricbeat/examples/upgrade/Makefile b/metricbeat/examples/upgrade/Makefile new file mode 100644 index 000000000..5cd564fcc --- /dev/null +++ b/metricbeat/examples/upgrade/Makefile @@ -0,0 +1,19 @@ +default: test + +include ../../../helpers/examples.mk + +CHART := metricbeat +RELEASE := helm-metricbeat-upgrade +FROM := 7.10.0 # upgrade from version < 7.10.0 is failing due to selector + # breaking change in https://github.com/elastic/helm-charts/pull/516 + +install: + ../../../helpers/upgrade.sh --chart $(CHART) --release $(RELEASE) --from $(FROM) + kubectl rollout status daemonset $(RELEASE)-metricbeat + kubectl rollout status deployment $(RELEASE)-metricbeat-metrics + kubectl rollout status deployment $(RELEASE)-kube-state-metrics + +test: install goss + +purge: + helm del $(RELEASE) diff --git a/metricbeat/examples/upgrade/README.md b/metricbeat/examples/upgrade/README.md new file mode 100644 index 000000000..4c1821b1f --- /dev/null +++ b/metricbeat/examples/upgrade/README.md @@ -0,0 +1,21 @@ +# Upgrade + +This example will deploy Metricbeat chart using an old chart version, +then upgrade it. + + +## Usage + +* Add the Elastic Helm charts repo: `helm repo add elastic https://helm.elastic.co` + +* Deploy [Elasticsearch Helm chart][]: `helm install elasticsearch elastic/elasticsearch` + +* Deploy and upgrade Metricbeat chart with the default values: `make install` + + +## Testing + +You can also run [goss integration tests][] using `make test`. + + +[goss integration tests]: https://github.com/elastic/helm-charts/tree/master/metricbeat/examples/upgrade/test/goss.yaml diff --git a/metricbeat/examples/upgrade/test/goss-metrics.yaml b/metricbeat/examples/upgrade/test/goss-metrics.yaml new file mode 100644 index 000000000..ed1d30db3 --- /dev/null +++ b/metricbeat/examples/upgrade/test/goss-metrics.yaml @@ -0,0 +1,42 @@ +port: + tcp:5066: + listening: true + ip: + - "127.0.0.1" + +mount: + /usr/share/metricbeat/metricbeat.yml: + exists: true + opts: + - ro + +user: + metricbeat: + exists: true + uid: 1000 + gid: 1000 + +http: + http://upgrade-master:9200/_cat/indices: + status: 200 + timeout: 2000 + body: + - "metricbeat-7.10.0" + + ? "http://upgrade-master:9200/_search?q=metricset.name:state_container%20AND%20kubernetes.container.name:metricbeat" + : status: 200 + timeout: 2000 + body: + - "metricbeat-7.10.0" + +file: + /usr/share/metricbeat/metricbeat.yml: + exists: true + contains: + - "output.elasticsearch" + +command: + cd /usr/share/metricbeat && metricbeat test output: + exit-status: 0 + stdout: + - "elasticsearch: http://upgrade-master:9200" diff --git a/metricbeat/examples/upgrade/test/goss.yaml b/metricbeat/examples/upgrade/test/goss.yaml new file mode 100644 index 000000000..a0480583c --- /dev/null +++ b/metricbeat/examples/upgrade/test/goss.yaml @@ -0,0 +1,46 @@ +port: + tcp:5066: + listening: true + ip: + - "127.0.0.1" + +mount: + /usr/share/metricbeat/data: + exists: true + /run/docker.sock: + exists: true + /usr/share/metricbeat/metricbeat.yml: + exists: true + opts: + - ro + +user: + metricbeat: + exists: true + uid: 1000 + gid: 1000 + +http: + http://upgrade-master:9200/_cat/indices: + status: 200 + timeout: 2000 + body: + - "metricbeat-8.0.0" + ? "http://upgrade-master:9200/_search?q=metricset.name:container%20AND%20kubernetes.container.name:metricbeat" + : status: 200 + timeout: 2000 + body: + - "metricbeat-8.0.0" + +file: + /usr/share/metricbeat/metricbeat.yml: + exists: true + contains: + - "add_kubernetes_metadata" + - "output.elasticsearch" + +command: + cd /usr/share/metricbeat && metricbeat test output: + exit-status: 0 + stdout: + - "elasticsearch: http://upgrade-master:9200" diff --git a/metricbeat/examples/upgrade/values.yaml b/metricbeat/examples/upgrade/values.yaml new file mode 100644 index 000000000..8b230601e --- /dev/null +++ b/metricbeat/examples/upgrade/values.yaml @@ -0,0 +1,4 @@ +--- +extraEnvs: + - name: ELASTICSEARCH_HOSTS + value: upgrade-master:9200 diff --git a/metricbeat/requirements.lock b/metricbeat/requirements.lock index 63f207bdf..3cbe95eaf 100644 --- a/metricbeat/requirements.lock +++ b/metricbeat/requirements.lock @@ -1,6 +1,6 @@ dependencies: - name: kube-state-metrics - repository: https://kubernetes-charts.storage.googleapis.com + repository: https://charts.helm.sh/stable version: 2.4.1 digest: sha256:948dca129bc7c16b138ed8bcbdf666c324d812e43af59d475b8bb74a53e99778 -generated: "2020-04-16T16:14:03.9537312+03:00" +generated: "2020-10-30T18:58:57.381827+01:00" diff --git a/metricbeat/templates/clusterrole.yaml b/metricbeat/templates/clusterrole.yaml index bbc209db7..851153f8e 100644 --- a/metricbeat/templates/clusterrole.yaml +++ b/metricbeat/templates/clusterrole.yaml @@ -1,5 +1,5 @@ {{- if .Values.managedServiceAccount }} -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: {{ template "metricbeat.serviceAccount" . }}-cluster-role diff --git a/metricbeat/templates/clusterrolebinding.yaml b/metricbeat/templates/clusterrolebinding.yaml index dc785b2e0..e95a98db8 100644 --- a/metricbeat/templates/clusterrolebinding.yaml +++ b/metricbeat/templates/clusterrolebinding.yaml @@ -1,5 +1,5 @@ {{- if .Values.managedServiceAccount }} -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: {{ template "metricbeat.serviceAccount" . }}-cluster-role-binding diff --git a/metricbeat/templates/deployment.yaml b/metricbeat/templates/deployment.yaml index e9d682b2e..cc97f20e5 100644 --- a/metricbeat/templates/deployment.yaml +++ b/metricbeat/templates/deployment.yaml @@ -30,7 +30,6 @@ spec: selector: matchLabels: app: '{{ template "metricbeat.fullname" . }}-metrics' - heritage: '{{ .Release.Service }}' release: '{{ .Release.Name }}' template: metadata: @@ -45,7 +44,6 @@ spec: labels: app: '{{ template "metricbeat.fullname" . }}-metrics' chart: '{{ .Chart.Name }}-{{ .Chart.Version }}' - heritage: '{{ .Release.Service }}' release: '{{ .Release.Name }}' {{- if .Values.deployment.labels }} {{- range $key, $value := .Values.deployment.labels }}