From 92f86c68e3c80a40d65241618e99c204ece6ff96 Mon Sep 17 00:00:00 2001 From: Kshitij Gupta Date: Sat, 13 Apr 2019 20:03:06 -0700 Subject: [PATCH] Update to Elastic 7.0.1 --- .env | 2 +- .travis.yml | 44 +++-- .travis/elasticsearch-setup-passwords.exp | 25 +++ .travis/run-tests.sh | 63 ++++++ README.md | 230 ++++++++++++++-------- docker-compose.yml | 3 +- docker-stack.yml | 7 +- elasticsearch/Dockerfile | 2 +- elasticsearch/config/elasticsearch.yml | 14 +- kibana/Dockerfile | 2 +- kibana/config/kibana.yml | 10 +- logstash/Dockerfile | 2 +- logstash/config/logstash.yml | 9 +- logstash/pipeline/logstash.conf | 2 + 14 files changed, 293 insertions(+), 122 deletions(-) create mode 100755 .travis/elasticsearch-setup-passwords.exp create mode 100755 .travis/run-tests.sh diff --git a/.env b/.env index 8568cec12f..39527a4033 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -ELK_VERSION=6.7.0 +ELK_VERSION=7.0.1 diff --git a/.travis.yml b/.travis.yml index 493ad8206f..5cff355fdf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,10 @@ services: docker env: - DOCKER_COMPOSE_VERSION=1.14.0 +before_install: + - sudo apt-get update + - sudo apt-get install -y expect jq + install: # Install Docker Compose - curl -L "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)" -o "$HOME/bin/docker-compose" @@ -16,38 +20,38 @@ before_script: # Pull buildpack image (curl 7.52.0+) - docker pull buildpack-deps:artful-curl - - shopt -s expand_aliases - - alias curl='docker run --rm --net=host buildpack-deps:artful-curl curl' + + # Use built-in users + - sed -i 's/\(elasticsearch.username:\) elastic/\1 kibana/g' kibana/config/kibana.yml + - sed -i 's/\(xpack.monitoring.elasticsearch.username:\) elastic/\1 logstash_system/g' logstash/config/logstash.yml script: # Compose + - docker-compose up -d elasticsearch + - sleep 20 + - .travis/elasticsearch-setup-passwords.exp - docker-compose up -d - - sleep 60 - - curl -s --retry 10 --retry-delay 5 --retry-connrefused -D- 'http://localhost:9200/' - - curl -s --retry 10 --retry-delay 5 --retry-connrefused -D- 'http://localhost:5601/api/status' - - curl -s --retry 10 --retry-delay 5 --retry-connrefused -D- 'http://localhost:9600/_node/pipelines/main?pretty' - - echo 'dockerelk' | nc localhost 5000 - - sleep 2 - - curl -s -XPOST 'http://localhost:9200/_refresh' - - curl -s 'http://localhost:9200/_count?q=message:dockerelk' | egrep '^{"count":1,' - - curl -s 'http://localhost:9200/_search?q=message:dockerelk&pretty' + - sleep 90 + - .travis/run-tests.sh - docker-compose ps - - docker-compose logs + - docker-compose logs elasticsearch + - docker-compose logs kibana + - docker-compose logs logstash - docker-compose down -v # Swarm - docker swarm init - docker stack deploy -c ./docker-stack.yml elk + - docker service scale elk_kibana=0 --detach=false + - docker service scale elk_logstash=0 --detach=false - sleep 60 - - curl -s --retry 10 --retry-delay 5 --retry-connrefused -D- 'http://localhost:9200/' - - curl -s --retry 10 --retry-delay 5 --retry-connrefused -D- 'http://localhost:5601/api/status' - - curl -s --retry 10 --retry-delay 5 --retry-connrefused -D- 'http://localhost:9600/_node/pipelines/main?pretty' - - echo 'dockerelk' | nc localhost 5000 - - sleep 2 - - curl -s -XPOST 'http://localhost:9200/_refresh' - - curl -s 'http://localhost:9200/_count?q=message:dockerelk' | egrep '^{"count":1,' - - curl -s 'http://localhost:9200/_search?q=message:dockerelk&pretty' + - .travis/elasticsearch-setup-passwords.exp swarm + - docker service scale elk_kibana=1 --detach=false + - docker service scale elk_logstash=1 --detach=false + - sleep 90 + - .travis/run-tests.sh - docker stack services elk - docker service logs elk_elasticsearch - docker service logs elk_kibana - docker service logs elk_logstash + - docker stack rm elk diff --git a/.travis/elasticsearch-setup-passwords.exp b/.travis/elasticsearch-setup-passwords.exp new file mode 100755 index 0000000000..b5ff18ae5b --- /dev/null +++ b/.travis/elasticsearch-setup-passwords.exp @@ -0,0 +1,25 @@ +#!/usr/bin/expect -f + +# List of expected users with dummy password +set user "(elastic|apm_system|kibana|logstash_system|beats_system|remote_monitoring_user)" +set password "changeme" + +# Find elasticsearch container id +set MODE [lindex $argv 0] +if { [string match "swarm" $MODE] } { + set cid [exec docker ps -q -f label=com.docker.swarm.service.name=elk_elasticsearch] +} else { + set cid [exec docker ps -q -f label=com.docker.compose.service=elasticsearch] +} + +set cmd "docker exec -it $cid bin/elasticsearch-setup-passwords interactive -s -b" + +spawn {*}$cmd + +expect { + -re "(E|Ree)nter password for \\\[$user\\\]: " { + send "$password\r" + exp_continue + } + eof +} diff --git a/.travis/run-tests.sh b/.travis/run-tests.sh new file mode 100755 index 0000000000..018d2a631f --- /dev/null +++ b/.travis/run-tests.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +shopt -s expand_aliases +alias curl="docker run --rm --net=host buildpack-deps:artful-curl curl -s -w '\n'" + +function log { + echo -e "\n[+] $1\n" +} + +log 'Waiting for Elasticsearch readiness' +curl -D- 'http://localhost:9200/' \ + --retry 10 \ + --retry-delay 5 \ + --retry-connrefused \ + -u elastic:changeme + +log 'Waiting for Kibana readiness' +curl -D- 'http://localhost:5601/api/status' \ + --retry 10 \ + --retry-delay 5 \ + --retry-connrefused \ + -u kibana:changeme + +log 'Waiting for Logstash readiness' +curl -D- 'http://localhost:9600/_node/pipelines/main?pretty' \ + --retry 10 \ + --retry-delay 5 \ + --retry-connrefused + +log 'Creating Logstash index pattern in Kibana' +source .env +curl -X POST -D- 'http://localhost:5601/api/saved_objects/index-pattern' \ + -H 'Content-Type: application/json' \ + -H "kbn-version: ${ELK_VERSION}" \ + -u kibana:changeme \ + -d '{"attributes":{"title":"logstash-*","timeFieldName":"@timestamp"}}' + +log 'Searching index pattern via Kibana API' +response="$(curl 'http://localhost:5601/api/saved_objects/_find?type=index-pattern' -u elastic:changeme)" +echo $response +count="$(jq -rn --argjson data "${response}" '$data.total')" +if [[ $count -ne 1 ]]; then + echo "Expected 1 index pattern, got ${count}" + exit 1 +fi + +log 'Sending message to Logstash TCP input' +echo 'dockerelk' | nc localhost 5000 + +sleep 1 +curl -X POST 'http://localhost:9200/_refresh' -u elastic:changeme + +log 'Searching message in Elasticsearch' +response="$(curl 'http://localhost:9200/_count?q=message:dockerelk&pretty' -u elastic:changeme)" +echo $response +count="$(jq -rn --argjson data "${response}" '$data.count')" +if [[ $count -ne 1 ]]; then + echo "Expected 1 document, got ${count}" + exit 1 +fi diff --git a/README.md b/README.md index 2dbfd73445..ae2169bbe9 100644 --- a/README.md +++ b/README.md @@ -4,10 +4,14 @@ [![Elastic Stack version](https://img.shields.io/badge/ELK-6.7.0-blue.svg?style=flat)](https://github.com/deviantony/docker-elk/issues/376) [![Build Status](https://api.travis-ci.org/deviantony/docker-elk.svg?branch=master)](https://travis-ci.org/deviantony/docker-elk) -Run the latest version of the [Elastic stack](https://www.elastic.co/elk-stack) with Docker and Docker Compose. +Run the latest version of the [Elastic stack][elk-stack] with Docker and Docker Compose. -It will give you the ability to analyze any data set by using the searching/aggregation capabilities of Elasticsearch -and the visualization power of Kibana. +It gives you the ability to analyze any data set by using the searching/aggregation capabilities of Elasticsearch and +the visualization power of Kibana. + +> :information_source: The Docker images backing this stack include [Stack Features][stack-features] (formerly X-Pack) +with [paid features][paid-features] enabled by default (see [How to disable paid +features](#how-to-disable-paid-features) to disable them). The [trial license][trial-license] is valid for 30 days. Based on the official Docker images from Elastic: @@ -15,38 +19,41 @@ Based on the official Docker images from Elastic: * [logstash](https://github.com/elastic/logstash-docker) * [kibana](https://github.com/elastic/kibana-docker) -**Note**: Other branches in this project are available: +Other available stack variants: -* [`x-pack`](https://github.com/deviantony/docker-elk/tree/x-pack): X-Pack support * [`searchguard`](https://github.com/deviantony/docker-elk/tree/searchguard): Search Guard support -* [`vagrant`](https://github.com/deviantony/docker-elk/tree/vagrant): run Docker inside Vagrant ## Contents 1. [Requirements](#requirements) * [Host setup](#host-setup) * [SELinux](#selinux) - * [Docker for Windows](#docker-for-windows) + * [Docker for Desktop](#docker-for-desktop) + * [Windows](#windows) + * [macOS](#macos) 2. [Usage](#usage) * [Bringing up the stack](#bringing-up-the-stack) * [Initial setup](#initial-setup) + * [Setting up user authentication](#setting-up-user-authentication) + * [Default Kibana index pattern creation](#default-kibana-index-pattern-creation) 3. [Configuration](#configuration) - * [How can I tune the Kibana configuration?](#how-can-i-tune-the-kibana-configuration) - * [How can I tune the Logstash configuration?](#how-can-i-tune-the-logstash-configuration) - * [How can I tune the Elasticsearch configuration?](#how-can-i-tune-the-elasticsearch-configuration) - * [How can I scale out the Elasticsearch cluster?](#how-can-i-scale-out-the-elasticsearch-cluster) + * [How to configure Elasticsearch](#how-to-configure-elasticsearch) + * [How to configure Kibana](#how-to-configure-kibana) + * [How to configure Logstash](#how-to-configure-logstash) + * [How to disable paid features](#how-to-disable-paid-features) + * [How to scale out the Elasticsearch cluster](#how-to-scale-out-the-elasticsearch-cluster) 4. [Storage](#storage) - * [How can I persist Elasticsearch data?](#how-can-i-persist-elasticsearch-data) + * [How to persist Elasticsearch data](#how-to-persist-elasticsearch-data) 5. [Extensibility](#extensibility) - * [How can I add plugins?](#how-can-i-add-plugins) - * [How can I enable the provided extensions?](#how-can-i-enable-the-provided-extensions) + * [How to add plugins](#how-to-add-plugins) + * [How to enable the provided extensions](#how-to-enable-the-provided-extensions) 6. [JVM tuning](#jvm-tuning) - * [How can I specify the amount of memory used by a service?](#how-can-i-specify-the-amount-of-memory-used-by-a-service) - * [How can I enable a remote JMX connection to a service?](#how-can-i-enable-a-remote-jmx-connection-to-a-service) + * [How to specify the amount of memory used by a service](#how-to-specify-the-amount-of-memory-used-by-a-service) + * [How to enable a remote JMX connection to a service](#how-to-enable-a-remote-jmx-connection-to-a-service) 7. [Going further](#going-further) * [Using a newer stack version](#using-a-newer-stack-version) * [Plugins and integrations](#plugins-and-integrations) - * [Docker Swarm](#docker-swarm) + * [Swarm mode](#swarm-mode) ## Requirements @@ -56,6 +63,12 @@ Based on the official Docker images from Elastic: 2. Install [Docker Compose](https://docs.docker.com/compose/install/) version **1.6.0+** 3. Clone this repository +By default, the stack exposes the following ports: +* 5000: Logstash TCP input +* 9200: Elasticsearch HTTP +* 9300: Elasticsearch TCP transport +* 5601: Kibana + ### SELinux On distributions which have SELinux enabled out-of-the-box you will need to either re-context the files or set SELinux @@ -66,17 +79,23 @@ apply the proper context: $ chcon -R system_u:object_r:admin_home_t:s0 docker-elk/ ``` -### Docker for Windows +### Docker for Desktop + +#### Windows + +Ensure the [Shared Drives][win-shareddrives] feature is enabled for the `C:` drive. -If you're using Docker for Windows, ensure the "Shared Drives" feature is enabled for the `C:` drive (Docker for Windows > Settings > Shared Drives). See [Configuring Docker for Windows Shared Drives](https://blogs.msdn.microsoft.com/stevelasker/2016/06/14/configuring-docker-for-windows-volumes/) (MSDN Blog). +#### macOS + +The default Docker for Mac configuration allows mounting files from `/Users/`, `/Volumes/`, `/private/`, and `/tmp` +exclusively. Make sure the repository is cloned in one of those locations or follow the instructions from the +[documentation][mac-mounts] to add more locations. ## Usage ### Bringing up the stack -**Note**: In case you switched branch or updated a base image - you may need to run `docker-compose build` first - -Start the stack using `docker-compose`: +Start the stack using Docker Compose: ```console $ docker-compose up @@ -84,28 +103,50 @@ $ docker-compose up You can also run all services in the background (detached mode) by adding the `-d` flag to the above command. -Give Kibana a few seconds to initialize, then access the Kibana web UI by hitting -[http://localhost:5601](http://localhost:5601) with a web browser. +> :information_source: You must run `docker-compose build` first whenever you switch branch or update a base image. -By default, the stack exposes the following ports: -* 5000: Logstash TCP input. -* 9200: Elasticsearch HTTP -* 9300: Elasticsearch TCP transport -* 5601: Kibana +If you are starting the stack for the very first time, please read the section below attentively. -**WARNING**: If you're using `boot2docker`, you must access it via the `boot2docker` IP address instead of `localhost`. +## Initial setup -**WARNING**: If you're using *Docker Toolbox*, you must access it via the `docker-machine` IP address instead of -`localhost`. +### Setting up user authentication -Now that the stack is running, you will want to inject some log entries. The shipped Logstash configuration allows you -to send content via TCP: +> :information_source: Refer to [How to disable paid features](#how-to-disable-paid-features) to disable authentication. + +The stack is pre-configured with the following **privileged** bootstrap user: + +* user: *elastic* +* password: *changeme* + +Although all stack components work out-of-the-box with this user, we strongly recommend using the unprivileged [built-in +users][builtin-users] instead for increased security. Passwords for these users must be initialized: ```console -$ nc localhost 5000 < /path/to/logfile.log +$ docker-compose exec -T elasticsearch 'bin/elasticsearch-setup-passwords' auto --batch ``` -## Initial setup +Passwords for all 6 built-in users will be randomly generated. Take note of them and replace the `elastic` username with +`kibana` and `logstash_system` inside the Kibana and Logstash *pipeline* configuration files respectively. See the +[Configuration](#configuration) section below. + +Restart Kibana and Logstash to apply the passwords you just wrote to the configuration files. + +```console +$ docker-compose restart kibana logstash +``` + +Give Kibana a few seconds to initialize, then access the Kibana web UI by hitting +[http://localhost:5601](http://localhost:5601) with a web browser and use the following default credentials to login: + +* user: *elastic* +* password: *\* + +Now that the stack is running, you can go ahead and inject some log entries. The shipped Logstash configuration allows +you to send content via TCP: + +```console +$ nc localhost 5000 < /path/to/logfile.log +``` ### Default Kibana index pattern creation @@ -113,12 +154,11 @@ When Kibana launches for the first time, it is not configured with any index pat #### Via the Kibana web UI -**NOTE**: You need to inject data into Logstash before being able to configure a Logstash index pattern via the Kibana web -UI. Then all you have to do is hit the *Create* button. +> :information_source: You need to inject data into Logstash before being able to configure a Logstash index pattern via +the Kibana web UI. Then all you have to do is hit the *Create* button. -Refer to [Connect Kibana with -Elasticsearch](https://www.elastic.co/guide/en/kibana/current/connect-to-elasticsearch.html) for detailed instructions -about the index pattern configuration. +Refer to [Connect Kibana with Elasticsearch][connect-kibana] for detailed instructions about the index pattern +configuration. #### On the command line @@ -128,6 +168,7 @@ Create an index pattern via the Kibana API: $ curl -XPOST -D- 'http://localhost:5601/api/saved_objects/index-pattern' \ -H 'Content-Type: application/json' \ -H 'kbn-version: 6.7.0' \ + -u kibana: \ -d '{"attributes":{"title":"logstash-*","timeFieldName":"@timestamp"}}' ``` @@ -135,29 +176,14 @@ The created pattern will automatically be marked as the default index pattern as ## Configuration -**NOTE**: Configuration is not dynamically reloaded, you will need to restart the stack after any change in the -configuration of a component. - -### How can I tune the Kibana configuration? - -The Kibana default configuration is stored in `kibana/config/kibana.yml`. +> :information_source: Configuration is not dynamically reloaded, you will need to restart individual components after +any configuration change. -It is also possible to map the entire `config` directory instead of a single file. - -### How can I tune the Logstash configuration? +### How to configure Elasticsearch -The Logstash configuration is stored in `logstash/config/logstash.yml`. - -It is also possible to map the entire `config` directory instead of a single file, however you must be aware that -Logstash will be expecting a -[`log4j2.properties`](https://github.com/elastic/logstash-docker/tree/master/build/logstash/config) file for its own -logging. +The Elasticsearch configuration is stored in [`elasticsearch/config/elasticsearch.yml`][config-es]. -### How can I tune the Elasticsearch configuration? - -The Elasticsearch configuration is stored in `elasticsearch/config/elasticsearch.yml`. - -You can also specify the options you want to override directly via environment variables: +You can also specify the options you want to override by setting environment variables inside the Compose file: ```yml elasticsearch: @@ -167,14 +193,31 @@ elasticsearch: cluster.name: "my-cluster" ``` -### How can I scale out the Elasticsearch cluster? +### How to configure Kibana + +The Kibana default configuration is stored in [`kibana/config/kibana.yml`][config-kbn]. + +It is also possible to map the entire `config` directory instead of a single file. + +### How to configure Logstash + +The Logstash configuration is stored in [`logstash/config/logstash.yml`][config-ls]. -Follow the instructions from the Wiki: [Scaling out -Elasticsearch](https://github.com/deviantony/docker-elk/wiki/Elasticsearch-cluster) +It is also possible to map the entire `config` directory instead of a single file, however you must be aware that +Logstash will be expecting a [`log4j2.properties`][log4j-props] file for its own logging. + +### How to disable paid features + +Switch the value of Elasticsearch's `xpack.license.self_generated.type` option from `trial` to `basic` (see [License +settings][trial-license]). + +### How to scale out the Elasticsearch cluster + +Follow the instructions from the Wiki: [Scaling out Elasticsearch](https://github.com/deviantony/docker-elk/wiki/Elasticsearch-cluster) ## Storage -### How can I persist Elasticsearch data? +### How to persist Elasticsearch data The data stored in Elasticsearch will be persisted after container reboot but not after container removal. @@ -190,18 +233,13 @@ elasticsearch: This will store Elasticsearch data inside `/path/to/storage`. -**NOTE:** beware of these OS-specific considerations: -* **Linux:** the [unprivileged `elasticsearch` user][esuser] is used within the Elasticsearch image, therefore the - mounted data directory must be owned by the uid `1000`. -* **macOS:** the default Docker for Mac configuration allows mounting files from `/Users/`, `/Volumes/`, `/private/`, - and `/tmp` exclusively. Follow the instructions from the [documentation][macmounts] to add more locations. - -[esuser]: https://github.com/elastic/elasticsearch-docker/blob/016bcc9db1dd97ecd0ff60c1290e7fa9142f8ddd/templates/Dockerfile.j2#L22 -[macmounts]: https://docs.docker.com/docker-for-mac/osxfs/ +> :information_source: (Linux users) Beware that the Elasticsearch process runs as the [unprivileged `elasticsearch` +user][esuser] is used within the Elasticsearch image, therefore the mounted data directory must be writable by the uid +`1000`. ## Extensibility -### How can I add plugins? +### How to add plugins To add plugins to any ELK component you have to: @@ -209,7 +247,7 @@ To add plugins to any ELK component you have to: 2. Add the associated plugin code configuration to the service configuration (eg. Logstash input/output) 3. Rebuild the images using the `docker-compose build` command -### How can I enable the provided extensions? +### How to enable the provided extensions A few extensions are available inside the [`extensions`](extensions) directory. These extensions provide features which are not part of the standard Elastic stack, but can be used to enrich it with extra integrations. @@ -219,7 +257,7 @@ of them require manual changes to the default ELK configuration. ## JVM tuning -### How can I specify the amount of memory used by a service? +### How to specify the amount of memory used by a service By default, both Elasticsearch and Logstash start with [1/4 of the total host memory](https://docs.oracle.com/javase/8/docs/technotes/guides/vm/gctuning/parallel.html#default_heap_size) allocated to @@ -246,7 +284,7 @@ logstash: LS_JAVA_OPTS: "-Xmx1g -Xms1g" ``` -### How can I enable a remote JMX connection to a service? +### How to enable a remote JMX connection to a service As for the Java Heap memory (see above), you can specify JVM options to enable JMX and map the JMX port on the Docker host. @@ -274,8 +312,8 @@ $ docker-compose build $ docker-compose up ``` -**NOTE**: Always pay attention to the [upgrade instructions](https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-upgrade.html) -for each individual component before performing a stack upgrade. +> :information_source: Always pay attention to the [upgrade instructions][upgrade] for each individual component before +performing a stack upgrade. ### Plugins and integrations @@ -284,10 +322,10 @@ See the following Wiki pages: * [External applications](https://github.com/deviantony/docker-elk/wiki/External-applications) * [Popular integrations](https://github.com/deviantony/docker-elk/wiki/Popular-integrations) -### Docker Swarm +### Swarm mode -Experimental support for Docker Swarm is provided in the form of a `docker-stack.yml` file, which can be deployed in an -existing Swarm cluster using the following command: +Experimental support for Docker [Swarm mode][swarm-mode] is provided in the form of a `docker-stack.yml` file, which can +be deployed in an existing Swarm cluster using the following command: ```console $ docker stack deploy -c docker-stack.yml elk @@ -299,5 +337,29 @@ If all components get deployed without any error, the following command will sho $ docker stack services elk ``` -**NOTE:** to scale Elasticsearch in Swarm mode, configure *zen* to use the DNS name `tasks.elasticsearch` instead of -`elasticsearch`. +> :information_source: To scale Elasticsearch in Swarm mode, configure *zen* to use the DNS name `tasks.elasticsearch` +instead of `elasticsearch`. + + +[elk-stack]: https://www.elastic.co/elk-stack +[stack-features]: https://www.elastic.co/products/stack +[paid-features]: https://www.elastic.co/subscriptions +[trial-license]: https://www.elastic.co/guide/en/elasticsearch/reference/current/license-settings.html + +[win-shareddrives]: https://docs.docker.com/docker-for-windows/#shared-drives +[mac-mounts]: https://docs.docker.com/docker-for-mac/osxfs/ + +[builtin-users]: https://www.elastic.co/guide/en/x-pack/current/setting-up-authentication.html#built-in-users + +[connect-kibana]: https://www.elastic.co/guide/en/kibana/current/connect-to-elasticsearch.html + +[config-es]: ./elasticsearch/config/elasticsearch.yml +[config-kbn]: ./kibana/config/kibana.yml +[config-ls]: ./logstash/config/logstash.yml + +[log4j-props]: https://github.com/elastic/logstash-docker/tree/master/build/logstash/config +[esuser]: https://github.com/elastic/elasticsearch-docker/blob/c2877ef/.tedi/template/bin/docker-entrypoint.sh#L9-L10 + +[upgrade]: https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-upgrade.html + +[swarm-mode]: https://docs.docker.com/engine/swarm/ diff --git a/docker-compose.yml b/docker-compose.yml index aa941915df..63d8ce9250 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,7 @@ services: - "9300:9300" environment: ES_JAVA_OPTS: "-Xmx256m -Xms256m" + ELASTIC_PASSWORD: changeme networks: - elk @@ -41,7 +42,7 @@ services: args: ELK_VERSION: $ELK_VERSION volumes: - - ./kibana/config/:/usr/share/kibana/config:ro + - ./kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml:ro ports: - "5601:5601" networks: diff --git a/docker-stack.yml b/docker-stack.yml index 6eca59e1bc..af07dc401d 100644 --- a/docker-stack.yml +++ b/docker-stack.yml @@ -3,7 +3,7 @@ version: '3.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.6.1 + image: docker.elastic.co/elasticsearch/elasticsearch:6.7.0 ports: - "9200:9200" - "9300:9300" @@ -12,6 +12,7 @@ services: target: /usr/share/elasticsearch/config/elasticsearch.yml environment: ES_JAVA_OPTS: "-Xmx256m -Xms256m" + ELASTIC_PASSWORD: changeme networks: - elk deploy: @@ -19,7 +20,7 @@ services: replicas: 1 logstash: - image: docker.elastic.co/logstash/logstash-oss:6.6.1 + image: docker.elastic.co/logstash/logstash:6.7.0 ports: - "5000:5000" - "9600:9600" @@ -37,7 +38,7 @@ services: replicas: 1 kibana: - image: docker.elastic.co/kibana/kibana-oss:6.6.1 + image: docker.elastic.co/kibana/kibana:6.7.0 ports: - "5601:5601" configs: diff --git a/elasticsearch/Dockerfile b/elasticsearch/Dockerfile index f2f4f5696f..24278f89d1 100644 --- a/elasticsearch/Dockerfile +++ b/elasticsearch/Dockerfile @@ -1,7 +1,7 @@ ARG ELK_VERSION # https://github.com/elastic/elasticsearch-docker -FROM docker.elastic.co/elasticsearch/elasticsearch-oss:${ELK_VERSION} +FROM docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION} # Add your elasticsearch plugins setup here # Example: RUN elasticsearch-plugin install analysis-icu diff --git a/elasticsearch/config/elasticsearch.yml b/elasticsearch/config/elasticsearch.yml index e975770843..a736834936 100644 --- a/elasticsearch/config/elasticsearch.yml +++ b/elasticsearch/config/elasticsearch.yml @@ -1,16 +1,18 @@ --- ## Default Elasticsearch configuration from elasticsearch-docker. -## from https://github.com/elastic/elasticsearch-docker/blob/master/build/elasticsearch/elasticsearch.yml +## from https://github.com/elastic/elasticsearch-docker/blob/master/.tedi/template/elasticsearch.yml # cluster.name: "docker-cluster" network.host: 0.0.0.0 -# minimum_master_nodes need to be explicitly set when bound on a public IP -# set to 1 to allow single node clusters -# Details: https://github.com/elastic/elasticsearch/pull/17288 -discovery.zen.minimum_master_nodes: 1 - ## Use single node discovery in order to disable production mode and avoid bootstrap checks ## see https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html # discovery.type: single-node + +## X-Pack settings +## see https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-xpack.html +# +xpack.license.self_generated.type: trial +xpack.security.enabled: true +xpack.monitoring.collection.enabled: true diff --git a/kibana/Dockerfile b/kibana/Dockerfile index 25d3fa8109..202855cc92 100644 --- a/kibana/Dockerfile +++ b/kibana/Dockerfile @@ -1,7 +1,7 @@ ARG ELK_VERSION # https://github.com/elastic/kibana-docker -FROM docker.elastic.co/kibana/kibana-oss:${ELK_VERSION} +FROM docker.elastic.co/kibana/kibana:${ELK_VERSION} # Add your kibana plugins setup here # Example: RUN kibana-plugin install diff --git a/kibana/config/kibana.yml b/kibana/config/kibana.yml index 4b34a16066..db54f14fa6 100644 --- a/kibana/config/kibana.yml +++ b/kibana/config/kibana.yml @@ -1,7 +1,13 @@ --- ## Default Kibana configuration from kibana-docker. -## from https://github.com/elastic/kibana-docker/blob/master/build/kibana/config/kibana.yml +## https://github.com/elastic/kibana-docker/blob/master/.tedi/template/kibana.yml.j2 # server.name: kibana server.host: "0" -elasticsearch.url: http://elasticsearch:9200 +elasticsearch.hosts: [ "http://elasticsearch:9200" ] +xpack.monitoring.ui.container.elasticsearch.enabled: true + +## X-Pack security credentials +# +elasticsearch.username: elastic +elasticsearch.password: changeme diff --git a/logstash/Dockerfile b/logstash/Dockerfile index 715f662745..6cc8637459 100644 --- a/logstash/Dockerfile +++ b/logstash/Dockerfile @@ -1,7 +1,7 @@ ARG ELK_VERSION # https://github.com/elastic/logstash-docker -FROM docker.elastic.co/logstash/logstash-oss:${ELK_VERSION} +FROM docker.elastic.co/logstash/logstash:${ELK_VERSION} # Add your logstash plugins setup here # Example: RUN logstash-plugin install logstash-filter-json diff --git a/logstash/config/logstash.yml b/logstash/config/logstash.yml index e7e260f22b..c8e1713ca1 100644 --- a/logstash/config/logstash.yml +++ b/logstash/config/logstash.yml @@ -1,6 +1,11 @@ --- ## Default Logstash configuration from logstash-docker. -## from https://github.com/elastic/logstash-docker/blob/master/build/logstash/config/logstash-oss.yml +## from https://github.com/elastic/logstash-docker/blob/master/build/logstash/config/logstash-full.yml # http.host: "0.0.0.0" -path.config: /usr/share/logstash/pipeline +xpack.monitoring.elasticsearch.url: http://elasticsearch:9200 + +## X-Pack security credentials +# +xpack.monitoring.elasticsearch.username: elastic +xpack.monitoring.elasticsearch.password: changeme diff --git a/logstash/pipeline/logstash.conf b/logstash/pipeline/logstash.conf index 10e442ecd5..a4238897b4 100644 --- a/logstash/pipeline/logstash.conf +++ b/logstash/pipeline/logstash.conf @@ -9,5 +9,7 @@ input { output { elasticsearch { hosts => "elasticsearch:9200" + user => elastic + password => changeme } }