From e5deb84b2cb9ca669732ca7ebb456e82e9841cd2 Mon Sep 17 00:00:00 2001 From: Julien Mailleret Date: Tue, 28 Apr 2020 19:26:45 +0200 Subject: [PATCH 1/2] FAQ and examples improvements (#598) - Add missing section in contributing guide - Add note about ECK - Add more FAQ topics - Document all examples - NIT: fixes typos... --- CONTRIBUTING.md | 16 ++ README.md | 22 +- apm-server/README.md | 42 +-- apm-server/examples/default/README.md | 27 ++ apm-server/examples/oss/README.md | 27 ++ apm-server/examples/security/README.md | 28 ++ elasticsearch/README.md | 271 ++++++++---------- elasticsearch/examples/config/README.md | 26 +- elasticsearch/examples/default/README.md | 25 ++ .../examples/docker-for-mac/README.md | 23 ++ .../examples/kubernetes-kind/README.md | 30 +- elasticsearch/examples/microk8s/README.md | 20 +- elasticsearch/examples/minikube/README.md | 38 +++ elasticsearch/examples/multi/README.md | 27 ++ elasticsearch/examples/openshift/README.md | 24 ++ elasticsearch/examples/oss/README.md | 23 ++ elasticsearch/examples/security/README.md | 29 ++ elasticsearch/examples/upgrade/README.md | 19 ++ elasticsearch/values.yaml | 2 +- filebeat/README.md | 55 ++-- filebeat/examples/default/README.md | 27 ++ filebeat/examples/oss/README.md | 27 ++ filebeat/examples/security/README.md | 28 ++ kibana/README.md | 111 ++++--- kibana/examples/default/README.md | 27 ++ kibana/examples/openshift/README.md | 26 ++ kibana/examples/oss/README.md | 27 ++ kibana/examples/security/README.md | 28 ++ logstash/README.md | 34 +-- logstash/examples/default/README.md | 17 ++ logstash/examples/elasticsearch/README.md | 28 ++ logstash/examples/oss/README.md | 17 ++ metricbeat/README.md | 85 ++++-- metricbeat/examples/oss/README.md | 27 ++ metricbeat/examples/security/README.md | 28 ++ 35 files changed, 1022 insertions(+), 289 deletions(-) create mode 100644 apm-server/examples/default/README.md create mode 100644 apm-server/examples/oss/README.md create mode 100644 apm-server/examples/security/README.md create mode 100644 elasticsearch/examples/default/README.md create mode 100644 elasticsearch/examples/docker-for-mac/README.md create mode 100644 elasticsearch/examples/minikube/README.md create mode 100644 elasticsearch/examples/multi/README.md create mode 100644 elasticsearch/examples/openshift/README.md create mode 100644 elasticsearch/examples/oss/README.md create mode 100644 elasticsearch/examples/security/README.md create mode 100644 elasticsearch/examples/upgrade/README.md create mode 100644 filebeat/examples/default/README.md create mode 100644 filebeat/examples/oss/README.md create mode 100644 filebeat/examples/security/README.md create mode 100644 kibana/examples/default/README.md create mode 100644 kibana/examples/openshift/README.md create mode 100644 kibana/examples/oss/README.md create mode 100644 kibana/examples/security/README.md create mode 100644 logstash/examples/default/README.md create mode 100644 logstash/examples/elasticsearch/README.md create mode 100644 logstash/examples/oss/README.md create mode 100644 metricbeat/examples/oss/README.md create mode 100644 metricbeat/examples/security/README.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 477c275db..89f4722a8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,6 +12,7 @@ - [Commits and Merging](#commits-and-merging) - [Rebasing and fixing merge conflicts](#rebasing-and-fixing-merge-conflicts) - [What Goes Into a Pull Request](#what-goes-into-a-pull-request) +- [Submitting a Pull Request](#submitting-a-pull-request) - [Releases](#releases) - [Testing](#testing) - [Templating tests](#templating-tests) @@ -169,6 +170,21 @@ appropriate. * See [Submitting a Pull Request](#submitting-a-pull-request) for more info. +## Submitting a Pull Request + +Push your local changes to your forked copy of the repository and submit a Pull +Request. In the Pull Request, describe what your changes do and mention the +number of the issue where discussion has taken place, e.g., “Closes #123″. + +Always submit your pull against `master` unless the bug is only present in an +older version. If the bug affects both master and another branch say so in your +pull. + +Then sit back and wait. There will probably be discussion about the Pull Request +and, if any changes are needed, we'll work with you to get your Pull Request +merged into Kibana. + + ## Releases Just like with the rest of the stack, all versions in this helm chart repo are diff --git a/README.md b/README.md index 8bab50ce9..ada0377f1 100644 --- a/README.md +++ b/README.md @@ -16,12 +16,12 @@ added below. | Chart | Docker documentation | |--------------------------------------------|---------------------------------------------------------------------------------| -| [APM-Server](./apm-server/README.md) | https://www.elastic.co/guide/en/apm/server/current/running-on-docker.html | -| [Elasticsearch](./elasticsearch/README.md) | https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html | -| [Filebeat](./filebeat/README.md) | https://www.elastic.co/guide/en/beats/filebeat/current/running-on-docker.html | -| [Kibana](./kibana/README.md) | https://www.elastic.co/guide/en/kibana/current/docker.html | -| [Logstash](./logstash/README.md) | https://www.elastic.co/guide/en/logstash/current/docker.html | -| [Metricbeat](./metricbeat/README.md) | https://www.elastic.co/guide/en/beats/metricbeat/current/running-on-docker.html | +| [APM-Server](./apm-server/README.md) | https://www.elastic.co/guide/en/apm/server/7.7/running-on-docker.html | +| [Elasticsearch](./elasticsearch/README.md) | https://www.elastic.co/guide/en/elasticsearch/reference/7.7/docker.html | +| [Filebeat](./filebeat/README.md) | https://www.elastic.co/guide/en/beats/filebeat/7.7/running-on-docker.html | +| [Kibana](./kibana/README.md) | https://www.elastic.co/guide/en/kibana/7.7/docker.html | +| [Logstash](./logstash/README.md) | https://www.elastic.co/guide/en/logstash/7.7/docker.html | +| [Metricbeat](./metricbeat/README.md) | https://www.elastic.co/guide/en/beats/metricbeat/7.7/running-on-docker.html | ## Kubernetes Versions @@ -35,7 +35,17 @@ While we are checking backward compatibility, the charts are only tested with Helm version mentioned in [helm-tester Dockerfile][] (currently 2.16.6). Note that we don't support [Helm 3][] version. +## ECK + +In addition of these Helm charts, Elastic also provides +[Elastic Cloud on Kubernetes][] which is based on [Operator pattern][] and is +Elastic recommended way to deploy Elasticsearch, Kibana and APM Server on +Kubernetes. + + [currently tested]: https://devops-ci.elastic.co/job/elastic+helm-charts+7.7/ +[elastic cloud on kubernetes]: https://github.com/elastic/cloud-on-k8s [helm 3]: https://v3.helm.sh [helm-tester Dockerfile]: https://github.com/elastic/helm-charts/blob/7.7/helpers/helm-tester/Dockerfile [helpers/matrix.yml]: https://github.com/elastic/helm-charts/blob/7.7/helpers/matrix.yml +[operator pattern]: https://kubernetes.io/docs/concepts/extend-kubernetes/operator/ diff --git a/apm-server/README.md b/apm-server/README.md index af9e7616a..359f96d45 100644 --- a/apm-server/README.md +++ b/apm-server/README.md @@ -10,8 +10,9 @@ - [Upgrading](#upgrading) - [Usage notes](#usage-notes) - [Configuration](#configuration) -- [Examples](#examples) - - [Default](#default) +- [FAQ](#faq) + - [How to use APM Server with Elasticsearch with security (authentication and TLS) enabled?](#how-to-use-apm-server-with-elasticsearch-with-security-authentication-and-tls-enabled) + - [How to install OSS version of APM Server?](#how-to-install-oss-version-of-apm-server) - [Contributing](#contributing) @@ -75,6 +76,9 @@ can easily be overridden in the config value `apmConfig.apm-server.yml`. * Automated testing of this chart is currently only run against GKE (Google Kubernetes Engine). +* This repo includes a number of [examples][] configurations which can be used +as a reference. They are also used in the automated testing of this chart. + ## Configuration @@ -115,28 +119,23 @@ Kubernetes Engine). | `updateStrategy` | Allows you to change the default [updateStrategy][] for the deployment | see [values.yaml][] | -## Examples - -In [examples][] you will find some example configurations. These examples are -used for the automated testing of this Helm chart. +## FAQ -### Default +### How to use APM Server with Elasticsearch with security (authentication and TLS) enabled? -* Deploy the [default Elasticsearch Helm chart][]. +This Helm chart can use existing [Kubernetes secrets][] to setup +credentials or certificates for examples. These secrets should be created +outside of this chart and accessed using [environment variables][] and volumes. -* Deploy APM Server with the default values: +An example can be found in [examples/security][]. - ``` - cd examples/default - make - ``` +### How to install OSS version of APM Server? -* You can now setup a port forward for Elasticsearch to observe APM indices: +Deploying OSS version of Elasticsearch can be done by setting `image` value to +[APM Server OSS Docker image][] - ``` - kubectl port-forward svc/elasticsearch-master 9200 - curl localhost:9200/_cat/indices - ``` +An example of APM Server deployment using OSS version can be found in +[examples/oss][]. ## Contributing @@ -150,15 +149,20 @@ about our development and testing process. [CONTRIBUTING.md]: https://github.com/elastic/helm-charts/blob/master/CONTRIBUTING.md [affinity]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity [annotations]: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ -[apm server docker image]: https://www.elastic.co/guide/en/apm/server/current/running-on-docker.html +[apm server docker image]: https://www.elastic.co/guide/en/apm/server/7.7/running-on-docker.html +[apm server oss docker image]: https://www.docker.elastic.co/#apm-server-7-6-2-oss [default elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/README.md#default [environment variables]: https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#using-environment-variables-inside-of-your-config +[environment from variables]: https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#configure-all-key-value-pairs-in-a-configmap-as-container-environment-variables [examples]: https://github.com/elastic/helm-charts/tree/7.7/apm-server/examples +[examples/oss]: https://github.com/elastic/helm-charts/tree/7.7/apm-server/examples/oss +[examples/security]: https://github.com/elastic/helm-charts/tree/7.7/apm-server/examples/security [helm]: https://helm.sh [horizontal pod autoscaler]: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/ [imagePullPolicy]: https://kubernetes.io/docs/concepts/containers/images/#updating-images [imagePullSecrets]: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-pod-that-uses-your-secret [ingress]: https://kubernetes.io/docs/concepts/services-networking/ingress/ +[kubernetes secrets]: https://kubernetes.io/docs/concepts/configuration/secret/ [labels]: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ [lifecycle hooks]: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/ [nodeSelector]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector diff --git a/apm-server/examples/default/README.md b/apm-server/examples/default/README.md new file mode 100644 index 000000000..8af8a5038 --- /dev/null +++ b/apm-server/examples/default/README.md @@ -0,0 +1,27 @@ +# Default + +This example deploy APM Server 7.7.0-SNAPSHOT using [default values][]. + + +## Usage + +* Deploy [Elasticsearch Helm chart][]. + +* Deploy APM Server chart with the default values: `make install` + +* You can now setup a port forward to query APM indices: + + ``` + kubectl port-forward svc/elasticsearch-master 9200 + curl localhost:9200/_cat/indices + ``` + + +## Testing + +You can also run [goss integration tests][] using `make test` + + +[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/default/ +[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/apm-server/examples/default/test/goss.yaml +[default values]: https://github.com/elastic/helm-charts/tree/7.7/apm-server/values.yaml diff --git a/apm-server/examples/oss/README.md b/apm-server/examples/oss/README.md new file mode 100644 index 000000000..e32234f8a --- /dev/null +++ b/apm-server/examples/oss/README.md @@ -0,0 +1,27 @@ +# OSS + +This example deploy APM Server 7.7.0-SNAPSHOT using [APM Server OSS][] version. + + +## Usage + +* Deploy [Elasticsearch Helm chart][]. + +* Deploy APM Server chart with the default values: `make install` + +* You can now setup a port forward to query APM indices: + + ``` + kubectl port-forward svc/oss-master 9200 + curl localhost:9200/_cat/indices + ``` + + +## Testing + +You can also run [goss integration tests][] using `make test` + + +[apm server oss]: https://www.elastic.co/downloads/apm-oss +[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/oss/ +[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/apm-server/examples/oss/test/goss.yaml diff --git a/apm-server/examples/security/README.md b/apm-server/examples/security/README.md new file mode 100644 index 000000000..517e53f86 --- /dev/null +++ b/apm-server/examples/security/README.md @@ -0,0 +1,28 @@ +# Security + +This example deploy APM Server 7.7.0-SNAPSHOT using authentication and TLS to connect to +Elasticsearch (see [values][]). + + +## Usage + +* Deploy [Elasticsearch Helm chart][]. + +* Deploy APM Server chart with security: `make install` + +* You can now setup a port forward to query APM indices: + + ``` + kubectl port-forward svc/security-master 9200 + curl -u elastic:changeme https://localhost:9200/_cat/indices + ``` + + +## Testing + +You can also run [goss integration tests][] using `make test` + + +[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/security/ +[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/apm-server/examples/security/test/goss.yaml +[values]: https://github.com/elastic/helm-charts/tree/7.7/apm-server/examples/security/values.yaml diff --git a/elasticsearch/README.md b/elasticsearch/README.md index aaecfdb79..2cd44eb36 100644 --- a/elasticsearch/README.md +++ b/elasticsearch/README.md @@ -9,26 +9,22 @@ - [Using the 7.7 branch](#using-the-77-branch) - [Upgrading](#upgrading) - [Usage notes](#usage-notes) -- [Migration from helm/charts stable](#migration-from-helmcharts-stable) - [Configuration](#configuration) - [Deprecated](#deprecated) -- [Try it out](#try-it-out) - - [Default](#default) - - [Multi](#multi) - - [Security](#security) - [FAQ](#faq) + - [How to deploy this chart on a specific K8S distribution?](#how-to-deploy-this-chart-on-a-specific-k8s-distribution) + - [How to deploy dedicated nodes types?](#how-to-deploy-dedicated-nodes-types) + - [Clustering and Node Discovery](#clustering-and-node-discovery) + - [How to deploy clusters with security (authentication and TLS) enabled?](#how-to-deploy-clusters-with-security-authentication-and-tls-enabled) + - [How to migrate from helm/charts stable chart?](#how-to-migrate-from-helmcharts-stable-chart) + - [How to install OSS version of Elasticsearch?](#how-to-install-oss-version-of-elasticsearch) - [How to install plugins?](#how-to-install-plugins) - [How to use the keystore?](#how-to-use-the-keystore) - [Basic example](#basic-example) - [Multiple keys](#multiple-keys) - [Custom paths and keys](#custom-paths-and-keys) - [How to enable snapshotting?](#how-to-enable-snapshotting) -- [Local development environments](#local-development-environments) - - [Minikube](#minikube) - - [Docker for Mac - Kubernetes](#docker-for-mac---kubernetes) - - [KIND - Kubernetes](#kind---kubernetes) - - [MicroK8S](#microk8s) -- [Clustering and Node Discovery](#clustering-and-node-discovery) + - [How to configure templates post-deployment?](#how-to-configure-templates-post-deployment) - [Contributing](#contributing) @@ -112,12 +108,6 @@ inside of the container. Doing this makes it much easier for this chart to support multiple versions with minimal changes. -## Migration from helm/charts stable - -If you currently have a cluster deployed with the [helm/charts stable][] chart -you can follow the [migration guide][]. - - ## Configuration | Parameter | Description | Default | @@ -145,7 +135,7 @@ you can follow the [migration guide][]. | `initResources` | Allows you to set the [resources][] for the `initContainer` in the StatefulSet | `{}` | | `keystore` | Allows you map Kubernetes secrets into the keystore. See the [config example][] and [how to use the keystore][] | `[]` | | `labels` | Configurable [labels][] applied to all Elasticsearch pods | `{}` | -| `lifecycle` | Allows you to add lifecycle configuration. See [values.yaml][] for an example of the formatting | `{}` | +| `lifecycle` | Allows you to add [lifecycle hooks][]. See [values.yaml][] for an example of the formatting | `{}` | | `masterService` | The service name used to connect to the masters. You only need to set this if your master `nodeGroup` is set to something other than `master`. See [Clustering and Node Discovery][] for more information | `""` | | `masterTerminationFix` | A workaround needed for Elasticsearch < 7.2 to prevent master status being lost during restarts [#63][] | `false` | | `maxUnavailable` | The [maxUnavailable][] value for the pod disruption budget. By default this will prevent Kubernetes from having more than 1 unhealthy pod in the node group | `1` | @@ -195,47 +185,80 @@ you can follow the [migration guide][]. | `fsGroup` | The Group ID (GID) for [securityContext][] so that the Elasticsearch user can read from the persistent volume | `""` | -## Try it out +## FAQ -In [examples][] you will find some example configurations. These examples are -used for the automated testing of this Helm chart. +### How to deploy this chart on a specific K8S distribution? -### Default +This chart is designed to run on production scale Kubernetes clusters with +multiple nodes, lots of memory and persistent storage. For that reason it can be +a bit tricky to run them against local Kubernetes environments such as +[Minikube][]. -To deploy a cluster with all default values and run the integration tests: +This chart is highly tested with [GKE][], but some K8S distribution also +requires specific configurations. -``` -cd examples/default -make -``` +We provide examples of configuration for the following K8S providers: -### Multi +- [Docker for Mac][] +- [KIND][] +- [Minikube][] +- [MicroK8S][] +- [OpenShift][] -A cluster with dedicated node types: +### How to deploy dedicated nodes types? -``` -cd examples/multi -make -``` +All the Elasticsearch pods deployed share the same configuration. If you need to +deploy dedicated [nodes types][] (for example dedicated master and data nodes), +you can deploy multiple releases of this chart with different configurations +while they share the same `clusterName` value. -### Security +For each Helm release, the nodes types can then be defined using `roles` value. -A cluster with node to node security and https enabled. This example uses -autogenerated certificates and password, for a production deployment you want to -generate SSL certificates following the [official docs][node-certificates]. +An example of Elasticsearch cluster using 2 different Helm releases for master +and data nodes can be found in [examples/multi][]. -Generate the certificates and install Elasticsearch: +#### Clustering and Node Discovery -``` -cd examples/security -make +This chart facilitates Elasticsearch node discovery and services by creating two +`Service` definitions in Kubernetes, one with the name `$clusterName-$nodeGroup` +and another named `$clusterName-$nodeGroup-headless`. +Only `Ready` pods are a part of the `$clusterName-$nodeGroup` service, while all +pods ( `Ready` or not) are a part of `$clusterName-$nodeGroup-headless`. -# Run a curl command to interact with the cluster -kubectl exec -ti security-master-0 -- sh -c 'curl -u $ELASTIC_USERNAME:$ELASTIC_PASSWORD -k https://localhost:9200/_cluster/health?pretty' -``` +If your group of master nodes has the default `nodeGroup: master` then you can +just add new groups of nodes with a different `nodeGroup` and they will +automatically discover the correct master. If your master nodes have a different +`nodeGroup` name then you will need to set `masterService` to +`$clusterName-$masterNodeGroup`. +The chart value for `masterService` is used to populate +`discovery.zen.ping.unicast.hosts` , which Elasticsearch nodes will use to +contact master nodes and form a cluster. +Therefore, to add a group of nodes to an existing cluster, setting +`masterService` to the desired `Service` name of the related cluster is +sufficient. -## FAQ +### How to deploy clusters with security (authentication and TLS) enabled? + +This Helm chart can use existing [Kubernetes secrets][] to setup +credentials or certificates for examples. These secrets should be created +outside of this chart and accessed using [environment variables][] and volumes. + +An example of Elasticsearch cluster using security can be found in +[examples/security][]. + +### How to migrate from helm/charts stable chart? + +If you currently have a cluster deployed with the [helm/charts stable][] chart +you can follow the [migration guide][]. + +### How to install OSS version of Elasticsearch? + +Deploying OSS version of Elasticsearch can be done by setting `image` value to +[Elasticsearch OSS Docker image][] + +An example of Elasticsearch cluster using OSS version can be found in +[examples/oss][]. ### How to install plugins? @@ -334,92 +357,32 @@ following the [how to use the keystore][] guide. there are plans to have Elasticsearch manage automated snapshots with [Snapshot Lifecycle Management][]. - -## Local development environments - -This chart is designed to run on production scale Kubernetes clusters with -multiple nodes, lots of memory and persistent storage. For that reason it can be -a bit tricky to run them against local Kubernetes environments such as minikube. -Below are some examples of how to get this working locally. - -### Minikube - -This chart also works successfully on [minikube][] in addition to typical hosted -Kubernetes environments. An example `values.yaml` file for minikube is provided -under `examples/`. - -In order to properly support the required persistent volume claims for the -Elasticsearch `StatefulSet` , the `default-storageclass` and -`storage-provisioner` minikube addons must be enabled: - -``` -minikube addons enable default-storageclass -minikube addons enable storage-provisioner -cd examples/minikube -make -``` - -Note that if `helm` or `kubectl` timeouts occur, you may consider creating a -minikube VM with more CPU cores or memory allocated. - -### Docker for Mac - Kubernetes - -It is also possible to run this chart with the built in Kubernetes cluster that -comes with [docker-for-mac][]: - -``` -cd examples/docker-for-mac -make +### How to configure templates post-deployment? + +You can use `postStart` [lifecycle hooks][] to run code triggered after a +container is created. + +Here is an example of `postStart` hook to configure templates: + +```yaml +lifecycle: + postStart: + exec: + command: + - bash + - -c + - | + #!/bin/bash + # Add a template to adjust number of shards/replicas + TEMPLATE_NAME=my_template + INDEX_PATTERN="logstash-*" + SHARD_COUNT=8 + REPLICA_COUNT=1 + ES_URL=http://localhost:9200 + while [[ "$(curl -s -o /dev/null -w '%{http_code}\n' $ES_URL)" != "200" ]]; do sleep 1; done + curl -XPUT "$ES_URL/_template/$TEMPLATE_NAME" -H 'Content-Type: application/json' -d'{"index_patterns":['\""$INDEX_PATTERN"\"'],"settings":{"number_of_shards":'$SHARD_COUNT',"number_of_replicas":'$REPLICA_COUNT'}}' ``` -### KIND - Kubernetes - -It is also possible to run this chart using a Kubernetes [KIND][] (Kubernetes in -Docker) cluster: - -``` -cd examples/kubernetes-kind -make -``` - -### MicroK8S - -It is also possible to run this chart using [MicroK8S][]: - -``` -microk8s.enable dns -microk8s.enable helm -microk8s.enable storage -cd examples/microk8s -make -``` - - -## Clustering and Node Discovery - -This chart facilitates Elasticsearch node discovery and services by creating two -`Service` definitions in Kubernetes, one with the name `$clusterName-$nodeGroup` -and another named `$clusterName-$nodeGroup-headless`. -Only `Ready` pods are a part of the `$clusterName-$nodeGroup` service, while all -pods ( `Ready` or not) are a part of `$clusterName-$nodeGroup-headless`. - -If your group of master nodes has the default `nodeGroup: master` then you can -just add new groups of nodes with a different `nodeGroup` and they will -automatically discover the correct master. If your master nodes have a different -`nodeGroup` name then you will need to set `masterService` to -`$clusterName-$masterNodeGroup`. - -The chart value for `masterService` is used to populate -`discovery.zen.ping.unicast.hosts` , which Elasticsearch nodes will use to -contact master nodes and form a cluster. -Therefore, to add a group of nodes to an existing cluster, setting -`masterService` to the desired `Service` name of the related cluster is -sufficient. - -For an example of deploying both a group master nodes and data nodes using -multiple releases of this chart, see the accompanying values files in -`examples/multi`. - ## Contributing @@ -434,57 +397,67 @@ about our development and testing process. [alternate scheduler]: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/#specify-schedulers-for-pods [annotations]: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ [anti-affinity]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity -[cluster.name]: https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster.name.html +[cluster.name]: https://www.elastic.co/guide/en/elasticsearch/reference/7.7/cluster.name.html [clustering and node discovery]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/README.md#clustering-and-node-discovery [config example]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/config/values.yaml -[curator]: https://www.elastic.co/guide/en/elasticsearch/client/curator/current/snapshot.html -[custom docker image]: https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html#_c_customized_image +[curator]: https://www.elastic.co/guide/en/elasticsearch/client/curator/7.7/snapshot.html +[custom docker image]: https://www.elastic.co/guide/en/elasticsearch/reference/7.7/docker.html#_c_customized_image [deploys statefulsets serially]: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#pod-management-policies -[discovery.zen.minimum_master_nodes]: https://www.elastic.co/guide/en/elasticsearch/reference/current/discovery-settings.html#minimum_master_nodes -[docker-for-mac]: https://docs.docker.com/docker-for-mac/kubernetes/ -[elasticsearch cluster health status params]: https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html#request-params -[elasticsearch docker image]: https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html +[discovery.zen.minimum_master_nodes]: https://www.elastic.co/guide/en/elasticsearch/reference/7.7/discovery-settings.html#minimum_master_nodes +[docker for mac]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/docker-for-mac +[elasticsearch cluster health status params]: https://www.elastic.co/guide/en/elasticsearch/reference/7.7/cluster-health.html#request-params +[elasticsearch docker image]: https://www.elastic.co/guide/en/elasticsearch/reference/7.7/docker.html +[elasticsearch oss docker image]: https://www.docker.elastic.co/#elasticsearch-7-6-2-oss [environment variables]: https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#using-environment-variables-inside-of-your-config +[environment from variables]: https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#configure-all-key-value-pairs-in-a-configmap-as-container-environment-variables [examples]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/ +[examples/multi]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/multi +[examples/oss]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/oss +[examples/security]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/security +[gke]: https://cloud.google.com/kubernetes-engine [helm]: https://helm.sh [helm/charts stable]: https://github.com/helm/charts/tree/master/stable/elasticsearch/ [how to install plugins guide]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/README.md#how-to-install-plugins [how to use the keystore]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/README.md#how-to-use-the-keystore -[http.port]: https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#_settings +[http.port]: https://www.elastic.co/guide/en/elasticsearch/reference/7.7/modules-http.html#_settings [imagePullPolicy]: https://kubernetes.io/docs/concepts/containers/images/#updating-images [imagePullSecrets]: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-pod-that-uses-your-secret [ingress]: https://kubernetes.io/docs/concepts/services-networking/ingress/ -[java options]: https://www.elastic.co/guide/en/elasticsearch/reference/current/jvm-options.html -[jvm heap size]: https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html -[kind]: https://github.com/kubernetes-sigs/kind +[java options]: https://www.elastic.co/guide/en/elasticsearch/reference/7.7/jvm-options.html +[jvm heap size]: https://www.elastic.co/guide/en/elasticsearch/reference/7.7/heap-size.html +[kind]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/kubernetes-kind +[kubernetes secrets]: https://kubernetes.io/docs/concepts/configuration/secret/ [labels]: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ +[lifecycle hooks]: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/ [loadBalancer annotations]: https://kubernetes.io/docs/concepts/services-networking/service/#ssl-support-on-aws [loadBalancer]: https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer [maxUnavailable]: https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget [migration guide]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/migration/README.md -[minikube]: https://kubernetes.io/docs/setup/minikube/ -[microk8s]: https://microk8s.io +[minikube]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/minikube +[microk8s]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/microk8s [multi]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/multi/ -[network.host elasticsearch setting]: https://www.elastic.co/guide/en/elasticsearch/reference/current/network.host.html +[network.host elasticsearch setting]: https://www.elastic.co/guide/en/elasticsearch/reference/7.7/network.host.html [node affinity settings]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#node-affinity-beta-feature -[node-certificates]: https://www.elastic.co/guide/en/elasticsearch/reference/current/configuring-tls.html#node-certificates +[node-certificates]: https://www.elastic.co/guide/en/elasticsearch/reference/7.7/configuring-tls.html#node-certificates [nodePort]: https://kubernetes.io/docs/concepts/services-networking/service/#nodeport +[nodes types]: https://www.elastic.co/guide/en/elasticsearch/reference/7.7/modules-node.html [nodeSelector]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector +[openshift]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/openshift [parent readme]: https://github.com/elastic/helm-charts/tree/7.7/README.md [priorityClass]: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass [probe]: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/ [resources]: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ -[roles]: https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html +[roles]: https://www.elastic.co/guide/en/elasticsearch/reference/7.7/modules-node.html [secret]: https://kubernetes.io/docs/concepts/configuration/secret/#using-secrets [securityContext]: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ [service types]: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types [snapshot lifecycle management]: https://github.com/elastic/elasticsearch/issues/38461 -[snapshot plugin]: https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository.html -[snapshot repository]: https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html -[sysctl vm.max_map_count]: https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html#vm-max-map-count +[snapshot plugin]: https://www.elastic.co/guide/en/elasticsearch/plugins/7.7/repository.html +[snapshot repository]: https://www.elastic.co/guide/en/elasticsearch/reference/7.7/modules-snapshots.html +[sysctl vm.max_map_count]: https://www.elastic.co/guide/en/elasticsearch/reference/7.7/vm-max-map-count.html#vm-max-map-count [terminationGracePeriod]: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods [tolerations]: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ -[transport port configuration]: https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-transport.html#_transport_settings +[transport port configuration]: https://www.elastic.co/guide/en/elasticsearch/reference/7.7/modules-transport.html#_transport_settings [updateStrategy]: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/ [values.yaml]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/values.yaml [volumeClaimTemplate for statefulsets]: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#stable-storage diff --git a/elasticsearch/examples/config/README.md b/elasticsearch/examples/config/README.md index d98d836bf..98d1360f5 100644 --- a/elasticsearch/examples/config/README.md +++ b/elasticsearch/examples/config/README.md @@ -1,3 +1,27 @@ # Config -An example testing suite for testing some of the optional features of this chart. +This example deploy a single node Elasticsearch 7.7.0-SNAPSHOT with authentication and +custom [values][]. + + +## Usage + +* Create the required secrets: `make secrets` + +* Deploy Elasticsearch chart with the default values: `make install` + +* You can now setup a port forward to query Elasticsearch API: + + ``` + kubectl port-forward svc/config-master 9200 + curl -u elastic:changeme http://localhost:9200/_cat/indices + ``` + + +## Testing + +You can also run [goss integration tests][] using `make test` + + +[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/config/test/goss.yaml +[values]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/config/values.yaml diff --git a/elasticsearch/examples/default/README.md b/elasticsearch/examples/default/README.md new file mode 100644 index 000000000..495e2a1ac --- /dev/null +++ b/elasticsearch/examples/default/README.md @@ -0,0 +1,25 @@ +# Default + +This example deploy a 3 nodes Elasticsearch 7.7.0-SNAPSHOT cluster using +[default values][]. + + +## Usage + +* Deploy Elasticsearch chart with the default values: `make install` + +* You can now setup a port forward to query Elasticsearch API: + + ``` + kubectl port-forward svc/elasticsearch-master 9200 + curl localhost:9200/_cat/indices + ``` + + +## Testing + +You can also run [goss integration tests][] using `make test` + + +[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/default/test/goss.yaml +[default values]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/values.yaml diff --git a/elasticsearch/examples/docker-for-mac/README.md b/elasticsearch/examples/docker-for-mac/README.md new file mode 100644 index 000000000..569bd416d --- /dev/null +++ b/elasticsearch/examples/docker-for-mac/README.md @@ -0,0 +1,23 @@ +# Docker for Mac + +This example deploy a 3 nodes Elasticsearch 7.7.0-SNAPSHOT cluster on [Docker for Mac][] +using [custom values][]. + +Note that this configuration should be used for test only and isn't recommended +for production. + + +## Usage + +* Deploy Elasticsearch chart with the default values: `make install` + +* You can now setup a port forward to query Elasticsearch API: + + ``` + kubectl port-forward svc/elasticsearch-master 9200 + curl localhost:9200/_cat/indices + ``` + + +[custom values]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/docker-for-mac/values.yaml +[docker for mac]: https://docs.docker.com/docker-for-mac/kubernetes/ diff --git a/elasticsearch/examples/kubernetes-kind/README.md b/elasticsearch/examples/kubernetes-kind/README.md index 3dfbe2f75..76ce26646 100644 --- a/elasticsearch/examples/kubernetes-kind/README.md +++ b/elasticsearch/examples/kubernetes-kind/README.md @@ -1,13 +1,13 @@ # KIND -An example of configuration for deploying Elasticsearch chart on [Kind][]. - -You can use `make install` to deploy it. +This example deploy a 3 nodes Elasticsearch 7.7.0-SNAPSHOT cluster on [Kind][] +using [custom values][]. Note that this configuration should be used for test only and isn't recommended for production. -## Current issue + +## Requirements There is currently an [kind issue][] with mount points created from PVCs not writable by non-root users. [kubernetes-sigs/kind#1157][] should fix it in a @@ -17,8 +17,22 @@ Meanwhile, the workaround is to install manually [Rancher Local Path Provisioner][] and use `local-path` storage class for Elasticsearch volumes (see [Makefile][] instructions). -[Kind]: https://kind.sigs.k8s.io/ -[Kind issue]: https://github.com/kubernetes-sigs/kind/issues/830 -[Kubernetes-sigs/kind#1157]: https://github.com/kubernetes-sigs/kind/pull/1157 -[Rancher Local Path Provisioner]: https://github.com/rancher/local-path-provisioner + +## Usage + +* Deploy Elasticsearch chart with the default values: `make install` + +* You can now setup a port forward to query Elasticsearch API: + + ``` + kubectl port-forward svc/elasticsearch-master 9200 + curl localhost:9200/_cat/indices + ``` + + +[custom values]: https://github.com/elastic/helm-charts/blob/7.7/elasticsearch/examples/kubernetes-kind/values.yaml +[kind]: https://kind.sigs.k8s.io/ +[kind issue]: https://github.com/kubernetes-sigs/kind/issues/830 +[kubernetes-sigs/kind#1157]: https://github.com/kubernetes-sigs/kind/pull/1157 +[rancher local path provisioner]: https://github.com/rancher/local-path-provisioner [Makefile]: https://github.com/elastic/helm-charts/blob/7.7/elasticsearch/examples/kubernetes-kind/Makefile#L5 diff --git a/elasticsearch/examples/microk8s/README.md b/elasticsearch/examples/microk8s/README.md index 161279cb3..487899d85 100644 --- a/elasticsearch/examples/microk8s/README.md +++ b/elasticsearch/examples/microk8s/README.md @@ -1,10 +1,12 @@ # MicroK8S -An example of configuration for deploying Elasticsearch chart on [MicroK8S][]. +This example deploy a 3 nodes Elasticsearch 7.7.0-SNAPSHOT cluster on [MicroK8S][] +using [custom values][]. Note that this configuration should be used for test only and isn't recommended for production. + ## Requirements The following MicroK8S [addons][] need to be enabled: @@ -12,5 +14,19 @@ The following MicroK8S [addons][] need to be enabled: - `helm` - `storage` + +## Usage + +* Deploy Elasticsearch chart with the default values: `make install` + +* You can now setup a port forward to query Elasticsearch API: + + ``` + kubectl port-forward svc/elasticsearch-master 9200 + curl localhost:9200/_cat/indices + ``` + + +[addons]: https://microk8s.io/docs/addons +[custom values]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/microk8s/values.yaml [MicroK8S]: https://microk8s.io -[Addons]: https://microk8s.io/docs/addons \ No newline at end of file diff --git a/elasticsearch/examples/minikube/README.md b/elasticsearch/examples/minikube/README.md new file mode 100644 index 000000000..08f4f4fc4 --- /dev/null +++ b/elasticsearch/examples/minikube/README.md @@ -0,0 +1,38 @@ +# Minikube + +This example deploy a 3 nodes Elasticsearch 7.7.0-SNAPSHOT cluster on [Minikube][] +using [custom values][]. + +If helm or kubectl timeouts occur, you may consider creating a minikube VM with +more CPU cores or memory allocated. + +Note that this configuration should be used for test only and isn't recommended +for production. + + +## Requirements + +In order to properly support the required persistent volume claims for the +Elasticsearch StatefulSet, the `default-storageclass` and `storage-provisioner` +minikube addons must be enabled. + +``` +minikube addons enable default-storageclass +minikube addons enable storage-provisioner +``` + + +## Usage + +* Deploy Elasticsearch chart with the default values: `make install` + +* You can now setup a port forward to query Elasticsearch API: + + ``` + kubectl port-forward svc/elasticsearch-master 9200 + curl localhost:9200/_cat/indices + ``` + + +[custom values]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/minikube/values.yaml +[minikube]: https://minikube.sigs.k8s.io/docs/ diff --git a/elasticsearch/examples/multi/README.md b/elasticsearch/examples/multi/README.md new file mode 100644 index 000000000..410391dbd --- /dev/null +++ b/elasticsearch/examples/multi/README.md @@ -0,0 +1,27 @@ +# Multi + +This example deploy an Elasticsearch 7.7.0-SNAPSHOT cluster composed of 2 different Helm +releases: + +- `helm-es-multi-master` for the 3 master nodes using [master values][] +- `helm-es-multi-data` for the 3 data nodes using [data values][] + +## Usage + +* Deploy the 2 Elasticsearch releases: `make install` + +* You can now setup a port forward to query Elasticsearch API: + + ``` + kubectl port-forward svc/multi-master 9200 + curl -u elastic:changeme http://localhost:9200/_cat/indices + ``` + +## Testing + +You can also run [goss integration tests][] using `make test` + + +[data values]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/multi/data.yaml +[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/multi/test/goss.yaml +[master values]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/multi/master.yaml diff --git a/elasticsearch/examples/openshift/README.md b/elasticsearch/examples/openshift/README.md new file mode 100644 index 000000000..53a924a25 --- /dev/null +++ b/elasticsearch/examples/openshift/README.md @@ -0,0 +1,24 @@ +# OpenShift + +This example deploy a 3 nodes Elasticsearch 7.7.0-SNAPSHOT cluster on [OpenShift][] +using [custom values][]. + +## Usage + +* Deploy Elasticsearch chart with the default values: `make install` + +* You can now setup a port forward to query Elasticsearch API: + + ``` + kubectl port-forward svc/elasticsearch-master 9200 + curl localhost:9200/_cat/indices + ``` + +## Testing + +You can also run [goss integration tests][] using `make test` + + +[custom values]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/openshift/values.yaml +[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/openshift/test/goss.yaml +[openshift]: https://www.openshift.com/ diff --git a/elasticsearch/examples/oss/README.md b/elasticsearch/examples/oss/README.md new file mode 100644 index 000000000..67a36b114 --- /dev/null +++ b/elasticsearch/examples/oss/README.md @@ -0,0 +1,23 @@ +# OSS + +This example deploy a 3 nodes Elasticsearch 7.7.0-SNAPSHOT cluster using +[Elasticsearch OSS][] version. + +## Usage + +* Deploy Elasticsearch chart with the default values: `make install` + +* You can now setup a port forward to query Elasticsearch API: + + ``` + kubectl port-forward svc/oss-master 9200 + curl localhost:9200/_cat/indices + ``` + +## Testing + +You can also run [goss integration tests][] using `make test` + + +[elasticsearch oss]: https://www.elastic.co/downloads/elasticsearch-oss +[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/oss/test/goss.yaml diff --git a/elasticsearch/examples/security/README.md b/elasticsearch/examples/security/README.md new file mode 100644 index 000000000..1fdcf0d94 --- /dev/null +++ b/elasticsearch/examples/security/README.md @@ -0,0 +1,29 @@ +# Security + +This example deploy a 3 nodes Elasticsearch 7.7.0-SNAPSHOT with authentication and +autogenerated certificates for TLS (see [values][]). + +Note that this configuration should be used for test only. For a production +deployment you should generate SSL certificates following the [official docs][]. + +## Usage + +* Create the required secrets: `make secrets` + +* Deploy Elasticsearch chart with the default values: `make install` + +* You can now setup a port forward to query Elasticsearch API: + + ``` + kubectl port-forward svc/security-master 9200 + curl -u elastic:changeme https://localhost:9200/_cat/indices + ``` + +## Testing + +You can also run [goss integration tests][] using `make test` + + +[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/security/test/goss.yaml +[official docs]: https://www.elastic.co/guide/en/elasticsearch/reference/7.7/configuring-tls.html#node-certificates +[values]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/security/security.yaml diff --git a/elasticsearch/examples/upgrade/README.md b/elasticsearch/examples/upgrade/README.md new file mode 100644 index 000000000..28cdb46e9 --- /dev/null +++ b/elasticsearch/examples/upgrade/README.md @@ -0,0 +1,19 @@ +# Upgrade + +This example deploy a 3 nodes Elasticsearch cluster using [7.0.0-alpha1][] chart +version, then upgrade it to 7.7.0-SNAPSHOT version. + + +## Usage + +Running `make install` command will do both 7.0.0-alpha1 install and 7.7.0-SNAPSHOT +upgrade + + +## Testing + +You can also run [goss integration tests][] using `make test` + + +[7.0.0-alpha1]: https://github.com/elastic/helm-charts/releases/tag/7.0.0-alpha1 +[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/upgrade/test/goss.yaml diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index a4de388a8..1f7b0bab3 100755 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -204,7 +204,7 @@ readinessProbe: successThreshold: 3 timeoutSeconds: 5 -# https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html#request-params wait_for_status +# https://www.elastic.co/guide/en/elasticsearch/reference/7.7/cluster-health.html#request-params wait_for_status clusterHealthCheckParams: "wait_for_status=green&timeout=1s" ## Use an alternate scheduler. diff --git a/filebeat/README.md b/filebeat/README.md index 98e2047eb..ff34d96f4 100644 --- a/filebeat/README.md +++ b/filebeat/README.md @@ -10,8 +10,10 @@ - [Upgrading](#upgrading) - [Usage notes](#usage-notes) - [Configuration](#configuration) -- [Examples](#examples) - - [Default](#default) +- [FAQ](#faq) + - [How to use Filebeat with Elasticsearch with security (authentication and TLS) enabled?](#how-to-use-filebeat-with-elasticsearch-with-security-authentication-and-tls-enabled) + - [How to install OSS version of Filebeat?](#how-to-install-oss-version-of-filebeat) + - [Why is Filebeat host.name field set to Kubernetes pod name?](#why-is-filebeat-hostname-field-set-to-kubernetes-pod-name) - [Contributing](#contributing) @@ -84,6 +86,8 @@ pods on the same node isn't possible with `hostNetwork` However Filebeat does recommend activating it. If your kubernetes provider is compatible with `hostNetwork` and you don't need to run multiple Filebeat DaemonSets, you can activate it by setting `hostNetworking: true` in [values.yaml][]. +* This repo includes a number of [examples][] configurations which can be used +as a reference. They are also used in the automated testing of this chart. ## Configuration @@ -122,28 +126,36 @@ activate it by setting `hostNetworking: true` in [values.yaml][]. | `updateStrategy` | The [updateStrategy][] for the `DaemonSet`. By default Kubernetes will kill and recreate pods on updates. Setting this to `OnDelete` will require that pods be deleted manually | `RollingUpdate` | -## Examples +## FAQ -In [examples][] you will find some example configurations. These examples are -used for the automated testing of this Helm chart. +### How to use Filebeat with Elasticsearch with security (authentication and TLS) enabled? -### Default +This Helm chart can use existing [Kubernetes secrets][] to setup +credentials or certificates for examples. These secrets should be created +outside of this chart and accessed using [environment variables][] and volumes. -* Deploy the [default Filebeat Helm chart][]. -* Deploy Filebeat with the default values: +An example can be found in [examples/security][]. - ``` - cd examples/default - make - ``` +### How to install OSS version of Filebeat? -* You can now setup a port forward for Filebeat to observe Filebeat -indices: +Deploying OSS version of Elasticsearch can be done by setting `image` value to +[Filebeat OSS Docker image][] - ``` - kubectl port-forward svc/filebeat-master 9200 - curl localhost:9200/_cat/indices - ``` +An example of Filebeat deployment using OSS version can be found in +[examples/oss][]. + +### Why is Filebeat host.name field set to Kubernetes pod name? + +The default Filebeat configuration is using Filebeat pod name for +`agent.hostname` and `host.name` fields. The `hostname` of the Kubernetes nodes +can be find in `kubernetes.node.name` field. If you would like to have +`agent.hostname` and `host.name` fields set to the hostname of the nodes, you'll +need to set `daemonset.hostNetworking` value to true. + +Note that enabling [hostNetwork][] make Filebeat pod use the host network +namespace which gives it access to the host loopback device, services listening +on localhost, could be used to snoop on network activity of other pods on the +same node. ## Contributing @@ -159,13 +171,18 @@ about our development and testing process. [annotations]: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ [default Filebeat Helm chart]: https://github.com/elastic/helm-charts/tree/7.7/filebeat/README.md#default [environment variables]: https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#using-environment-variables-inside-of-your-config +[environment from variables]: https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#configure-all-key-value-pairs-in-a-configmap-as-container-environment-variables [examples]: https://github.com/elastic/helm-charts/tree/7.7/filebeat/examples -[filebeat docker image]: https://www.elastic.co/guide/en/beats/filebeat/current/running-on-docker.html +[examples/oss]: https://github.com/elastic/helm-charts/tree/7.7/filebeat/examples/oss +[examples/security]: https://github.com/elastic/helm-charts/tree/7.7/filebeat/examples/security +[filebeat docker image]: https://www.elastic.co/guide/en/beats/filebeat/7.7/running-on-docker.html +[filebeat oss docker image]: https://www.docker.elastic.co/#filebeat-7-6-2-oss [helm]: https://helm.sh [hostNetwork]: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#host-namespaces [hostPath]: https://kubernetes.io/docs/concepts/storage/volumes/#hostpath [imagePullPolicy]: https://kubernetes.io/docs/concepts/containers/images/#updating-images [imagePullSecrets]: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-pod-that-uses-your-secret +[kubernetes secrets]: https://kubernetes.io/docs/concepts/configuration/secret/ [labels]: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ [parent readme]: https://github.com/elastic/helm-charts/tree/7.7/README.md [nodeSelector]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector diff --git a/filebeat/examples/default/README.md b/filebeat/examples/default/README.md new file mode 100644 index 000000000..aee7abde5 --- /dev/null +++ b/filebeat/examples/default/README.md @@ -0,0 +1,27 @@ +# Default + +This example deploy Filebeat 7.7.0-SNAPSHOT using [default values][]. + + +## Usage + +* Deploy [Elasticsearch Helm chart][]. + +* Deploy Filebeat chart with the default values: `make install` + +* You can now setup a port forward to query Filebeat indices: + + ``` + kubectl port-forward svc/elasticsearch-master 9200 + curl localhost:9200/_cat/indices + ``` + + +## Testing + +You can also run [goss integration tests][] using `make test` + + +[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/default/ +[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/filebeat/examples/default/test/goss.yaml +[default values]: https://github.com/elastic/helm-charts/tree/7.7/filebeat/values.yaml diff --git a/filebeat/examples/oss/README.md b/filebeat/examples/oss/README.md new file mode 100644 index 000000000..b964ab23a --- /dev/null +++ b/filebeat/examples/oss/README.md @@ -0,0 +1,27 @@ +# OSS + +This example deploy Filebeat 7.7.0-SNAPSHOT using [Filebeat OSS][] version. + + +## Usage + +* Deploy [Elasticsearch Helm chart][]. + +* Deploy Filebeat chart with the default values: `make install` + +* You can now setup a port forward to query Filebeat indices: + + ``` + kubectl port-forward svc/oss-master 9200 + curl localhost:9200/_cat/indices + ``` + + +## Testing + +You can also run [goss integration tests][] using `make test` + + +[filebeat oss]: https://www.elastic.co/downloads/beats/filebeat-oss +[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/oss/ +[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/filebeat/examples/oss/test/goss.yaml diff --git a/filebeat/examples/security/README.md b/filebeat/examples/security/README.md new file mode 100644 index 000000000..c88e277ca --- /dev/null +++ b/filebeat/examples/security/README.md @@ -0,0 +1,28 @@ +# Security + +This example deploy Filebeat 7.7.0-SNAPSHOT using authentication and TLS to connect to +Elasticsearch (see [values][]). + + +## Usage + +* Deploy [Elasticsearch Helm chart][]. + +* Deploy Filebeat chart with security: `make install` + +* You can now setup a port forward to query Filebeat indices: + + ``` + kubectl port-forward svc/security-master 9200 + curl -u elastic:changeme https://localhost:9200/_cat/indices + ``` + + +## Testing + +You can also run [goss integration tests][] using `make test` + + +[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/security/ +[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/filebeat/examples/security/test/goss.yaml +[values]: https://github.com/elastic/helm-charts/tree/7.7/filebeat/examples/security/values.yaml diff --git a/kibana/README.md b/kibana/README.md index ebb79db5d..2ad7efd63 100644 --- a/kibana/README.md +++ b/kibana/README.md @@ -8,13 +8,15 @@ - [Using Helm repository](#using-helm-repository) - [Using the 7.7 branch](#using-the-77-branch) - [Upgrading](#upgrading) +- [Usage notes](#usage-notes) - [Configuration](#configuration) - [Deprecated](#deprecated) -- [Examples](#examples) - - [Default](#default) - - [Security](#security) - [FAQ](#faq) + - [How to deploy this chart on a specific K8S distribution?](#how-to-deploy-this-chart-on-a-specific-k8s-distribution) + - [How to use Kibana with security (authentication and TLS) enabled?](#how-to-use-kibana-with-security-authentication-and-tls-enabled) + - [How to install OSS version of Kibana?](#how-to-install-oss-version-of-kibana) - [How to install plugins?](#how-to-install-plugins) + - [How to import objects post-deployment?](#how-to-import-objects-post-deployment) - [Contributing](#contributing) @@ -69,6 +71,15 @@ Please always check [CHANGELOG.md][] and [BREAKING_CHANGES.md][] before upgrading to a new chart version. +## Usage notes + +* Automated testing of this chart is currently only run against GKE (Google +Kubernetes Engine). + +* This repo includes a number of [examples][] configurations which can be used +as a reference. They are also used in the automated testing of this chart. + + ## Configuration | Parameter | Description | Default | @@ -89,7 +100,7 @@ upgrading to a new chart version. | `ingress` | Configurable [ingress][] to expose the Kibana service. | see [values.yaml][] | | `kibanaConfig` | Allows you to add any config files in `/usr/share/kibana/config/` such as `kibana.yml` See [values.yaml][] for an example of the formatting | `{}` | | `labels` | Configurable [labels][] applied to all Kibana pods | `{}` | -| `lifecycle` | Allows you to add lifecycle configuration. See [values.yaml][] for an example of the formatting | `{}` | +| `lifecycle` | Allows you to add [lifecycle hooks][]. See [values.yaml][] for an example of the formatting | `{}` | | `nameOverride` | Overrides the chart name for resources. If not set the name will default to `.Chart.Name` | `""` | | `nodeSelector` | Configurable [nodeSelector][] so that you can target specific nodes for your Kibana instances | `{}` | | `podAnnotations` | Configurable [annotations][] applied to all Kibana pods | `{}` | @@ -114,55 +125,32 @@ upgrading to a new chart version. | `elasticsearchURL` | The URL used to connect to Elasticsearch. needs to be used for Kibana versions < 6.6 | `""` | -## Examples - -In [examples][] you will find some example configurations. These examples are -used for the automated testing of this Helm chart. - -### Default - -* Deploy the [default Elasticsearch Helm chart][]. -* Deploy Kibana with the default values: - - ``` - cd examples/default - make - ``` - -* You can now setup a port forward and access Kibana at http://localhost:5601: +## FAQ - ``` - kubectl port-forward deployment/helm-kibana-default-kibana 5601 - ``` +### How to deploy this chart on a specific K8S distribution? -### Security +This chart is highly tested with [GKE][], but some K8S distribution also +requires specific configurations. -* Deploy a [security enabled Elasticsearch cluster][]. -* Deploy Kibana with the security example: +We provide examples of configuration for the following K8S providers: - ``` - cd examples/security - make - ``` +- [OpenShift][] -* Setup a port forward and access Kibana at https://localhost:5601: +### How to use Kibana with security (authentication and TLS) enabled? - ``` - # Setup the port forward - kubectl port-forward deployment/helm-kibana-security-kibana 5601 +This Helm chart can use existing [Kubernetes secrets][] to setup +credentials or certificates for examples. These secrets should be created +outside of this chart and accessed using [environment variables][] and volumes. - # Run this in a seperate terminal - # Get the auto generated password - password=$(kubectl get secret elastic-credentials -o jsonpath='{.data.password}' | base64 --decode) - echo password +An example can be found in [examples/security][]. - # Test Kibana is working with curl or access it with your browser at https://localhost:5601 - # The example certificate is self signed so you may see a warning about the certificate - curl -I -k -u elastic:$password https://localhost:5601/app/kibana - ``` +### How to install OSS version of Kibana? +Deploying OSS version of Elasticsearch can be done by setting `image` value to +[kibana OSS Docker image][] -## FAQ +An example of APM Server deployment using OSS version can be found in +[examples/oss][]. ### How to install plugins? @@ -189,6 +177,29 @@ random times. 2. Mutating the state of a running Docker image (by installing plugins) goes against best practices of containers and immutable infrastructure. +### How to import objects post-deployment? + +You can use `postStart` [lifecycle hooks][] to run code triggered after a +container is created. + +Here is an example of `postStart` hook to import an index-pattern and a +dashboard: + +```yaml +lifecycle: + postStart: + exec: + command: + - bash + - -c + - | + #!/bin/bash + # Import a dashboard + KB_URL=http://localhost:5601 + while [[ "$(curl -s -o /dev/null -w '%{http_code}\n' -L $KB_URL)" != "200" ]]; do sleep 1; done + curl -XPOST "$KB_URL/api/kibana/dashboards/import" -H "Content-Type: application/json" -H 'kbn-xsrf: true' -d'"objects":[{"type":"index-pattern","id":"my-pattern","attributes":{"title":"my-pattern-*"}},{"type":"dashboard","id":"my-dashboard","attributes":{"title":"Look at my dashboard"}}]}' +``` + ## Contributing @@ -199,27 +210,35 @@ about our development and testing process. [BREAKING_CHANGES.md]: https://github.com/elastic/helm-charts/blob/master/BREAKING_CHANGES.md [CHANGELOG.md]: https://github.com/elastic/helm-charts/blob/master/CHANGELOG.md [CONTRIBUTING.md]: https://github.com/elastic/helm-charts/blob/master/CONTRIBUTING.md +[affinity]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity [annotations]: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ [default elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/README.md#default [environment variables]: https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#using-environment-variables-inside-of-your-config -[kibana docker image]: https://www.elastic.co/guide/en/kibana/current/docker.html +[environment from variables]: https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#configure-all-key-value-pairs-in-a-configmap-as-container-environment-variables [examples]: https://github.com/elastic/helm-charts/tree/7.7/kibana/examples +[examples/oss]: https://github.com/elastic/helm-charts/tree/7.7/kibana/examples/oss +[examples/security]: https://github.com/elastic/helm-charts/tree/7.7/kibana/examples/security [helm]: https://helm.sh [imagePullPolicy]: https://kubernetes.io/docs/concepts/containers/images/#updating-images [imagePullSecrets]: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-pod-that-uses-your-secret [ingress]: https://kubernetes.io/docs/concepts/services-networking/ingress/ +[kibana docker image]: https://www.elastic.co/guide/en/kibana/7.7/docker.html +[kibana oss docker image]: https://www.docker.elastic.co/#kibana-7-6-2-oss +[kubernetes secrets]: https://kubernetes.io/docs/concepts/configuration/secret/ [labels]: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ +[lifecycle hooks]: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/ [nodeSelector]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector +[openshift]: https://github.com/elastic/helm-charts/tree/7.7/kibana/examples/openshift [parent readme]: https://github.com/elastic/helm-charts/tree/7.7/README.md [priorityClass]: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass [probe]: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/ [resources]: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ [security enabled elasticsearch cluster]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/README.md#security [securityContext]: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod -[server.host]: https://www.elastic.co/guide/en/kibana/current/settings.html +[server.host]: https://www.elastic.co/guide/en/kibana/7.7/settings.html [service]: https://kubernetes.io/docs/concepts/services-networking/service/ [serviceAccount]: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ -[standard upgrade]: https://www.elastic.co/guide/en/kibana/current/upgrade-standard.html +[standard upgrade]: https://www.elastic.co/guide/en/kibana/7.7/upgrade-standard.html [tolerations]: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ [updateStrategy]: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#updating-a-deployment [values.yaml]: https://github.com/elastic/helm-charts/tree/7.7/kibana/values.yaml diff --git a/kibana/examples/default/README.md b/kibana/examples/default/README.md new file mode 100644 index 000000000..b06e0188e --- /dev/null +++ b/kibana/examples/default/README.md @@ -0,0 +1,27 @@ +# Default + +This example deploy Kibana 7.7.0-SNAPSHOT using [default values][]. + + +## Usage + +* Deploy [Elasticsearch Helm chart][]. + +* Deploy Kibana chart with the default values: `make install` + +* You can now setup a port forward to query Kibana indices: + + ``` + kubectl port-forward svc/elasticsearch-master 9200 + curl localhost:9200/_cat/indices + ``` + + +## Testing + +You can also run [goss integration tests][] using `make test` + + +[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/default/ +[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/kibana/examples/default/test/goss.yaml +[default values]: https://github.com/elastic/helm-charts/tree/7.7/kibana/values.yaml diff --git a/kibana/examples/openshift/README.md b/kibana/examples/openshift/README.md new file mode 100644 index 000000000..d8eefbb54 --- /dev/null +++ b/kibana/examples/openshift/README.md @@ -0,0 +1,26 @@ +# OpenShift + +This example deploy Kibana 7.7.0-SNAPSHOT on [OpenShift][] using [custom values][]. + +## Usage + +* Deploy [Elasticsearch Helm chart][]. + +* Deploy Kibana chart with the default values: `make install` + +* You can now setup a port forward to query Elasticsearch API: + + ``` + kubectl port-forward svc/elasticsearch-master 9200 + curl localhost:9200/_cat/indices + ``` + +## Testing + +You can also run [goss integration tests][] using `make test` + + +[custom values]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/openshift/values.yaml +[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/openshift/ +[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/openshift/test/goss.yaml +[openshift]: https://www.openshift.com/ diff --git a/kibana/examples/oss/README.md b/kibana/examples/oss/README.md new file mode 100644 index 000000000..a4d5896ff --- /dev/null +++ b/kibana/examples/oss/README.md @@ -0,0 +1,27 @@ +# OSS + +This example deploy Kibana 7.7.0-SNAPSHOT using [Kibana OSS][] version. + + +## Usage + +* Deploy [Elasticsearch Helm chart][]. + +* Deploy Kibana chart with the default values: `make install` + +* You can now setup a port forward to query Kibana indices: + + ``` + kubectl port-forward svc/oss-master 9200 + curl localhost:9200/_cat/indices + ``` + + +## Testing + +You can also run [goss integration tests][] using `make test` + + +[kibana oss]: https://www.elastic.co/downloads/kibana-oss +[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/oss/ +[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/kibana/examples/oss/test/goss.yaml diff --git a/kibana/examples/security/README.md b/kibana/examples/security/README.md new file mode 100644 index 000000000..e54c091e6 --- /dev/null +++ b/kibana/examples/security/README.md @@ -0,0 +1,28 @@ +# Security + +This example deploy Kibana 7.7.0-SNAPSHOT using authentication and TLS to connect to +Elasticsearch (see [values][]). + + +## Usage + +* Deploy [Elasticsearch Helm chart][]. + +* Deploy Kibana chart with security: `make install` + +* You can now setup a port forward to query Kibana indices: + + ``` + kubectl port-forward svc/security-master 9200 + curl -u elastic:changeme https://localhost:9200/_cat/indices + ``` + + +## Testing + +You can also run [goss integration tests][] using `make test` + + +[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/security/ +[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/kibana/examples/security/test/goss.yaml +[values]: https://github.com/elastic/helm-charts/tree/7.7/kibana/examples/security/security.yaml diff --git a/logstash/README.md b/logstash/README.md index 4000b5668..a9369f659 100644 --- a/logstash/README.md +++ b/logstash/README.md @@ -10,9 +10,8 @@ - [Upgrading](#upgrading) - [Usage notes](#usage-notes) - [Configuration](#configuration) -- [Try it out](#try-it-out) - - [Default](#default) - [FAQ](#faq) + - [How to install OSS version of Logstash?](#how-to-install-oss-version-of-logstash) - [How to install plugins?](#how-to-install-plugins) - [Contributing](#contributing) @@ -145,22 +144,15 @@ using `http.host: 127.0.0.1`, default probes should be disabled or overrided | `volumeClaimTemplate` | Configuration for the [volumeClaimTemplate for StatefulSets][]. You will want to adjust the storage (default `30Gi` ) and the `storageClassName` if you are using a different storage class | see [values.yaml][] | -## Try it out - -In [examples][] you will find some example configurations. These examples are -used for the automated testing of this Helm chart. +## FAQ -### Default +### How to install OSS version of Logstash? -To deploy a cluster with all default values and run the integration tests: +Deploying OSS version of Elasticsearch can be done by setting `image` value to +[Logstash OSS Docker image][] -``` -cd examples/default -make -``` - - -## FAQ +An example of APM Server deployment using OSS version can be found in +[examples/oss][]. ### How to install plugins? @@ -200,26 +192,30 @@ about our development and testing process. [annotations]: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ [anti-affinity]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity [deploys statefulsets serially]: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#pod-management-policies -[custom docker image]: https://www.elastic.co/guide/en/logstash/current/docker-config.html#_custom_images +[custom docker image]: https://www.elastic.co/guide/en/logstash/7.7/docker-config.html#_custom_images [environment variables]: https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#using-environment-variables-inside-of-your-config +[environment from variables]: https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#configure-all-key-value-pairs-in-a-configmap-as-container-environment-variables [examples]: https://github.com/elastic/helm-charts/tree/7.7/logstash/examples +[examples/oss]: https://github.com/elastic/helm-charts/tree/7.7/logstash/examples/oss [helm]: https://helm.sh [imagePullPolicy]: https://kubernetes.io/docs/concepts/containers/images/#updating-images [imagePullSecrets]: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-pod-that-uses-your-secret +[kubernetes secrets]: https://kubernetes.io/docs/concepts/configuration/secret/ [labels]: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ -[logstash docker image]: https://www.elastic.co/guide/en/logstash/current/docker.html +[logstash docker image]: https://www.elastic.co/guide/en/logstash/7.7/docker.html +[logstash oss docker image]: https://www.docker.elastic.co/#logstash-7-6-2-oss [maxUnavailable]: https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget [node affinity settings]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#node-affinity-beta-feature [nodeSelector]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector -[note]: https://www.elastic.co/guide/en/logstash/current/docker-config.html#docker-env-config +[note]: https://www.elastic.co/guide/en/logstash/7.7/docker-config.html#docker-env-config [parent readme]: https://github.com/elastic/helm-charts/tree/7.7/README.md [priorityClass]: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass [probe]: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/ [resources]: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ [updateStrategy]: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/ [securityContext]: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod +[service]: https://kubernetes.io/docs/concepts/services-networking/service/ [terminationGracePeriod]: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods [tolerations]: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ [values.yaml]: https://github.com/elastic/helm-charts/tree/7.7/logstash/values.yaml -[updateStrategy]: https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets [volumeClaimTemplate for statefulsets]: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#stable-storage diff --git a/logstash/examples/default/README.md b/logstash/examples/default/README.md new file mode 100644 index 000000000..ba2fa62cf --- /dev/null +++ b/logstash/examples/default/README.md @@ -0,0 +1,17 @@ +# Default + +This example deploy Logstash 7.7.0-SNAPSHOT using [default values][]. + + +## Usage + +* Deploy Logstash chart with the default values: `make install` + + +## Testing + +You can also run [goss integration tests][] using `make test` + + +[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/logstash/examples/default/test/goss.yaml +[default values]: https://github.com/elastic/helm-charts/tree/7.7/logstash/values.yaml diff --git a/logstash/examples/elasticsearch/README.md b/logstash/examples/elasticsearch/README.md new file mode 100644 index 000000000..58d9850e9 --- /dev/null +++ b/logstash/examples/elasticsearch/README.md @@ -0,0 +1,28 @@ +# Elasticsearch + +This example deploy Logstash 7.7.0-SNAPSHOT which connects to Elasticsearch (see +[values][]). + + +## Usage + +* Deploy [Elasticsearch Helm chart][]. + +* Deploy Logstash chart: `make install` + +* You can now setup a port forward to query Logstash indices: + + ``` + kubectl port-forward svc/elasticsearch-master 9200 + curl localhost:9200/_cat/indices + ``` + + +## Testing + +You can also run [goss integration tests][] using `make test` + + +[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/default/ +[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/logstash/examples/elasticsearch/test/goss.yaml +[values]: https://github.com/elastic/helm-charts/tree/7.7/logstash/examples/elasticsearch/values.yaml diff --git a/logstash/examples/oss/README.md b/logstash/examples/oss/README.md new file mode 100644 index 000000000..8350847e6 --- /dev/null +++ b/logstash/examples/oss/README.md @@ -0,0 +1,17 @@ +# OSS + +This example deploy Logstash 7.7.0-SNAPSHOT using [Logstash OSS][] version. + + +## Usage + +* Deploy Logstash chart with the default values: `make install` + + +## Testing + +You can also run [goss integration tests][] using `make test` + + +[logstash oss]: https://www.elastic.co/downloads/logstash-oss +[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/logstash/examples/oss/test/goss.yaml diff --git a/metricbeat/README.md b/metricbeat/README.md index 1febca509..bc305a4e1 100644 --- a/metricbeat/README.md +++ b/metricbeat/README.md @@ -8,10 +8,14 @@ - [Using Helm repository](#using-helm-repository) - [Using the 7.7 branch](#using-the-77-branch) - [Upgrading](#upgrading) +- [Usage notes](#usage-notes) - [Configuration](#configuration) - [Deprecated](#deprecated) -- [Examples](#examples) - - [Default](#default) +- [FAQ](#faq) + - [How to use Metricbeat with Elasticsearch with security (authentication and TLS) enabled?](#how-to-use-metricbeat-with-elasticsearch-with-security-authentication-and-tls-enabled) + - [How to install OSS version of Metricbeat?](#how-to-install-oss-version-of-metricbeat) + - [How to use Kubelet read-only port instead of secure port?](#how-to-use-kubelet-read-only-port-instead-of-secure-port) + - [Why is Metricbeat host.name field set to Kubernetes pod name?](#why-is-metricbeat-hostname-field-set-to-kubernetes-pod-name) - [Contributing](#contributing) @@ -66,6 +70,25 @@ Please always check [CHANGELOG.md][] and [BREAKING_CHANGES.md][] before upgrading to a new chart version. +## Usage notes + +* The default Metricbeat configuration file for this chart is configured to use +an Elasticsearch endpoint. Without any additional changes, Metricbeat will send +documents to the service URL that the Elasticsearch Helm chart sets up by +default. You may either set the `ELASTICSEARCH_HOSTS` environment variable in +`extraEnvs` to override this endpoint or modify the default `metricbeatConfig` +to change this behavior. +* This chart disables the [HostNetwork][] setting by default for compatibility +reasons with the majority of kubernetes providers and scenarios. Some kubernetes +providers may not allow enabling `hostNetwork` and deploying multiple Metricbeat +pods on the same node isn't possible with `hostNetwork` However Metricbeat does +recommend activating it. If your kubernetes provider is compatible with +`hostNetwork` and you don't need to run multiple Metricbeat DaemonSets, you can +activate it by setting `hostNetworking: true` in [values.yaml][]. +* This repo includes a number of [examples][] configurations which can be used +as a reference. They are also used in the automated testing of this chart. + + ## Configuration | Parameter | Description | Default | @@ -131,28 +154,48 @@ upgrading to a new chart version. | `tolerations` | Configurable [tolerations][] for both Metricbeat DaemonSet and Deployment | `[]` | -## Examples +## FAQ + +### How to use Metricbeat with Elasticsearch with security (authentication and TLS) enabled? + +This Helm chart can use existing [Kubernetes secrets][] to setup +credentials or certificates for examples. These secrets should be created +outside of this chart and accessed using [environment variables][] and volumes. + +An example can be found in [examples/security][]. + +### How to install OSS version of Metricbeat? + +Deploying OSS version of Elasticsearch can be done by setting `image` value to +[Metricbeat OSS Docker image][] + +An example of Metricbeat deployment using OSS version can be found in +[examples/oss][]. -In [examples][] you will find some example configurations. These examples are -used for the automated testing of this Helm chart. +### How to use Kubelet read-only port instead of secure port? -### Default +Default Metricbeat configuration has been switched to Kubelet secure port +(10250/TCP) instead of read-only port (10255/TCP) in [#471][] because read-only +port usage is now discouraged and not enabled by default in most Kubernetes +configurations. -* Deploy the [default Elasticsearch Helm chart][]. -* Deploy Metricbeat with the default values: +However, if you need to use read-only port, you can replace +`hosts: ["https://${NODE_NAME}:10250"]` by `hosts: ["${NODE_NAME}:10255"]` and +comment `bearer_token_file` and `ssl.verification_mode` in +`daemonset.metricbeatConfig` in [values.yaml][]. - ``` - cd examples/default - make - ``` +### Why is Metricbeat host.name field set to Kubernetes pod name? -* You can now setup a port forward for Elasticsearch to observe Metricbeat -indices: +The default Metricbeat configuration is using Metricbeat pod name for +`agent.hostname` and `host.name` fields. The `hostname` of the Kubernetes nodes +can be find in `kubernetes.node.name` field. If you would like to have +`agent.hostname` and `host.name` fields set to the hostname of the nodes, you'll +need to set `daemonset.hostNetworking` value to true. - ``` - kubectl port-forward svc/elasticsearch-master 9200 - curl localhost:9200/_cat/indices - ``` +Note that enabling [hostNetwork][] make Metricbeat pod use the host network +namespace which gives it access to the host loopback device, services listening +on localhost, could be used to snoop on network activity of other pods on the +same node. ## Contributing @@ -171,13 +214,17 @@ about our development and testing process. [environment variables]: https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#using-environment-variables-inside-of-your-config [environment from variables]: https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#configure-all-key-value-pairs-in-a-configmap-as-container-environment-variables [examples]: https://github.com/elastic/helm-charts/tree/7.7/metricbeat/examples +[examples/oss]: https://github.com/elastic/helm-charts/tree/7.7/metricbeat/examples/oss +[examples/security]: https://github.com/elastic/helm-charts/tree/7.7/metricbeat/examples/security [helm]: https://helm.sh [hostPath]: https://kubernetes.io/docs/concepts/storage/volumes/#hostpath +[hostNetwork]: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#host-namespaces [imagePullPolicy]: https://kubernetes.io/docs/concepts/containers/images/#updating-images [imagePullSecrets]: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-pod-that-uses-your-secret [kube-state-metrics]: https://github.com/helm/charts/tree/7.7/stable/kube-state-metrics +[kubernetes secrets]: https://kubernetes.io/docs/concepts/configuration/secret/ [labels]: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ -[metricbeat docker image]: https://www.elastic.co/guide/en/beats/metricbeat/current/running-on-docker.html +[metricbeat docker image]: https://www.elastic.co/guide/en/beats/metricbeat/7.7/running-on-docker.html [priorityClass]: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass [nodeSelector]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector [parent readme]: https://github.com/elastic/helm-charts/tree/7.7/README.md diff --git a/metricbeat/examples/oss/README.md b/metricbeat/examples/oss/README.md new file mode 100644 index 000000000..c8b8ee916 --- /dev/null +++ b/metricbeat/examples/oss/README.md @@ -0,0 +1,27 @@ +# OSS + +This example deploy Metricbeat 7.7.0-SNAPSHOT using [Metricbeat OSS][] version. + + +## Usage + +* Deploy [Elasticsearch Helm chart][]. + +* Deploy Metricbeat chart with the default values: `make install` + +* You can now setup a port forward to query Metricbeat indices: + + ``` + kubectl port-forward svc/oss-master 9200 + curl localhost:9200/_cat/indices + ``` + + +## Testing + +You can also run [goss integration tests][] using `make test` + + +[metricbeat oss]: https://www.elastic.co/downloads/beats/metricbeat-oss +[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/oss/ +[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/metricbeat/examples/oss/test/goss.yaml diff --git a/metricbeat/examples/security/README.md b/metricbeat/examples/security/README.md new file mode 100644 index 000000000..8268e1e1f --- /dev/null +++ b/metricbeat/examples/security/README.md @@ -0,0 +1,28 @@ +# Security + +This example deploy Metricbeat 7.7.0-SNAPSHOT using authentication and TLS to connect to +Elasticsearch (see [values][]). + + +## Usage + +* Deploy [Elasticsearch Helm chart][]. + +* Deploy Metricbeat chart with security: `make install` + +* You can now setup a port forward to query Metricbeat indices: + + ``` + kubectl port-forward svc/security-master 9200 + curl -u elastic:changeme https://localhost:9200/_cat/indices + ``` + + +## Testing + +You can also run [goss integration tests][] using `make test` + + +[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/security/ +[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/metricbeat/examples/security/test/goss.yaml +[values]: https://github.com/elastic/helm-charts/tree/7.7/metricbeat/examples/security/values.yaml From 82d19a01c08c7adbbd985d6b16918c76b4f4ace6 Mon Sep 17 00:00:00 2001 From: Julien Mailleret Date: Tue, 28 Apr 2020 19:43:56 +0200 Subject: [PATCH 2/2] bump oss docker images --- apm-server/README.md | 2 +- elasticsearch/README.md | 2 +- filebeat/README.md | 2 +- kibana/README.md | 2 +- logstash/README.md | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apm-server/README.md b/apm-server/README.md index 359f96d45..3aa695991 100644 --- a/apm-server/README.md +++ b/apm-server/README.md @@ -150,7 +150,7 @@ about our development and testing process. [affinity]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity [annotations]: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ [apm server docker image]: https://www.elastic.co/guide/en/apm/server/7.7/running-on-docker.html -[apm server oss docker image]: https://www.docker.elastic.co/#apm-server-7-6-2-oss +[apm server oss docker image]: https://www.docker.elastic.co/#apm-server-7-7-0-oss [default elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/README.md#default [environment variables]: https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#using-environment-variables-inside-of-your-config [environment from variables]: https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#configure-all-key-value-pairs-in-a-configmap-as-container-environment-variables diff --git a/elasticsearch/README.md b/elasticsearch/README.md index 2cd44eb36..ee568fa06 100644 --- a/elasticsearch/README.md +++ b/elasticsearch/README.md @@ -407,7 +407,7 @@ about our development and testing process. [docker for mac]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/docker-for-mac [elasticsearch cluster health status params]: https://www.elastic.co/guide/en/elasticsearch/reference/7.7/cluster-health.html#request-params [elasticsearch docker image]: https://www.elastic.co/guide/en/elasticsearch/reference/7.7/docker.html -[elasticsearch oss docker image]: https://www.docker.elastic.co/#elasticsearch-7-6-2-oss +[elasticsearch oss docker image]: https://www.docker.elastic.co/#elasticsearch-7-7-0-oss [environment variables]: https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#using-environment-variables-inside-of-your-config [environment from variables]: https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#configure-all-key-value-pairs-in-a-configmap-as-container-environment-variables [examples]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/ diff --git a/filebeat/README.md b/filebeat/README.md index ff34d96f4..11ae6a9cd 100644 --- a/filebeat/README.md +++ b/filebeat/README.md @@ -176,7 +176,7 @@ about our development and testing process. [examples/oss]: https://github.com/elastic/helm-charts/tree/7.7/filebeat/examples/oss [examples/security]: https://github.com/elastic/helm-charts/tree/7.7/filebeat/examples/security [filebeat docker image]: https://www.elastic.co/guide/en/beats/filebeat/7.7/running-on-docker.html -[filebeat oss docker image]: https://www.docker.elastic.co/#filebeat-7-6-2-oss +[filebeat oss docker image]: https://www.docker.elastic.co/#filebeat-7-7-0-oss [helm]: https://helm.sh [hostNetwork]: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#host-namespaces [hostPath]: https://kubernetes.io/docs/concepts/storage/volumes/#hostpath diff --git a/kibana/README.md b/kibana/README.md index 2ad7efd63..8eaca9a39 100644 --- a/kibana/README.md +++ b/kibana/README.md @@ -223,7 +223,7 @@ about our development and testing process. [imagePullSecrets]: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-pod-that-uses-your-secret [ingress]: https://kubernetes.io/docs/concepts/services-networking/ingress/ [kibana docker image]: https://www.elastic.co/guide/en/kibana/7.7/docker.html -[kibana oss docker image]: https://www.docker.elastic.co/#kibana-7-6-2-oss +[kibana oss docker image]: https://www.docker.elastic.co/#kibana-7-7-0-oss [kubernetes secrets]: https://kubernetes.io/docs/concepts/configuration/secret/ [labels]: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ [lifecycle hooks]: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/ diff --git a/logstash/README.md b/logstash/README.md index a9369f659..d2f6686fb 100644 --- a/logstash/README.md +++ b/logstash/README.md @@ -203,7 +203,7 @@ about our development and testing process. [kubernetes secrets]: https://kubernetes.io/docs/concepts/configuration/secret/ [labels]: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ [logstash docker image]: https://www.elastic.co/guide/en/logstash/7.7/docker.html -[logstash oss docker image]: https://www.docker.elastic.co/#logstash-7-6-2-oss +[logstash oss docker image]: https://www.docker.elastic.co/#logstash-7-7-0-oss [maxUnavailable]: https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget [node affinity settings]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#node-affinity-beta-feature [nodeSelector]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector