docker-compose.yml

---
version: '3.3'
services:


  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_STACK_VERSION}
    hostname: elasticsearch
    container_name: elasticsearch
    environment:
      - 'http.host=0.0.0.0'
      - 'transport.host=127.0.0.1'
      - 'xpack.monitoring.collection.enabled=true'
    ports:
      - '127.0.0.1:9200:9200'
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9200/_cat/health"]
      interval: 30s
      timeout: 10s
      retries: 5
    networks:
      - stack


  kibana:
    image: docker.elastic.co/kibana/kibana:${ELASTIC_STACK_VERSION}
    hostname: kibana
    container_name: kibana
    ports:
      - '127.0.0.1:5601:5601'
    networks:
      - stack
    depends_on:
      - elasticsearch


  logstash:
    image: docker.elastic.co/logstash/logstash:${ELASTIC_STACK_VERSION}
    hostname: logstash
    container_name: logstash
    environment:
      - 'xpack.monitoring.enabled=true'
    ports:
      - '127.0.0.1:4560:4560'
      - '127.0.0.1:5044:5044'
    volumes:
      # Provide a pipeline configuration for Logstash with a bind-mounted file
      - ./docker-compose/logstash.conf:/usr/share/logstash/pipeline/logstash.conf
    networks:
      - stack
    depends_on:
      - elasticsearch


  # Filebeat sending data to Logstash
  filebeat_for_logstash:
    image: docker.elastic.co/beats/filebeat:${ELASTIC_STACK_VERSION}
    hostname: filebeat_for_logstash
    container_name: filebeat_for_logstash
    volumes:
      # Bind-mount the logs/ directory, so Filebeat can read its files
      - './logs-docker/:/mnt/logs/:ro'
      # Bind-mount a custom configuration file
      - './docker-compose/filebeat-logstash.yml:/usr/share/filebeat/filebeat.yml:ro'
      # Bind-mount the config directory so it can be dynamically loaded
      - './docker-compose/filebeat-logstash/config/:/usr/share/filebeat/config/:ro'
      # Bind-mount the registry file to avoid data duplication between restarts
      - './docker-compose/filebeat-logstash/registry/:/usr/share/filebeat/data/'
    command: filebeat -e
    restart: on-failure
    networks:
      - stack
    depends_on:
      - elasticsearch
      - logstash


  # Filebeat sending data to Elasticsearch
  filebeat_for_elasticsearch:
    image: docker.elastic.co/beats/filebeat:${ELASTIC_STACK_VERSION}
    hostname: filebeat_for_elasticsearch
    container_name: filebeat_for_elasticsearch
    volumes:
      # Bind-mount the logs/ directory, so Filebeat can read its files
      - './logs-docker/:/mnt/logs/:ro'
      # Bind-mount a custom configuration file
      - './docker-compose/filebeat-elasticsearch.yml:/usr/share/filebeat/filebeat.yml:ro'
      # Bind-mount the config directory so it can be dynamically loaded
      - './docker-compose/filebeat-elasticsearch/config/:/usr/share/filebeat/config/:ro'
      # Bind-mount the registry file to avoid data duplication between restarts
      - './docker-compose/filebeat-elasticsearch/registry/:/usr/share/filebeat/data/'
    command: 'filebeat -e'
    restart: on-failure
    networks:
      - stack
    depends_on:
      - elasticsearch
      - kibana


  # Filebeat collecting from a Docker container and sending to Elasticsearch
  filebeat_docker_for_elasticsearch:
    image: docker.elastic.co/beats/filebeat:${ELASTIC_STACK_VERSION}
    hostname: filebeat_docker_for_elasticsearch
    container_name: filebeat_docker_for_elasticsearch
    user: root #To read the docker socket
    volumes:
      # Bind-mount the Docker log directory from the python container, so Filebeat can read its files
      - '/var/lib/docker/containers:/var/lib/docker/containers:ro'
      # Bind-mount a custom configuration file
      - './docker-compose/filebeat-docker.yml:/usr/share/filebeat/filebeat.yml:ro'
      # Bind-mount the registry file to avoid data duplication between restarts
      - './docker-compose/filebeat-docker/registry/:/usr/share/filebeat/data/'
      # Bind-mount the Docker daemon to enable add_docker_metadata from within the container
      - '/var/run/docker.sock:/var/run/docker.sock:ro'
    command: 'filebeat -e'
    restart: on-failure
    networks:
      - stack
    depends_on:
      - elasticsearch
      - kibana


  # Metricbeat to optionally collect metrics from the entire setup; not required for logging itself
  metricbeat:
    hostname: metricbeat
    container_name: metricbeat
    user: root #To read the docker socket
    image: docker.elastic.co/beats/metricbeat:${ELASTIC_STACK_VERSION}
    volumes:
      # Bind-mount a custom configuration file
      - './docker-compose/metricbeat.yml:/usr/share/metricbeat/metricbeat.yml:ro'
      # Bind-mount the config directory so it can be dynamically loaded
      - './docker-compose/metricbeat/config/:/usr/share/metricbeat/config/:ro'
      # Monitor the Docker host rather than the Metricbeat container; these are used by the system module
      - '/proc:/hostfs/proc:ro'
      - '/sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro'
      - '/:/hostfs:ro'
      # Bind-mount the Docker daemon to enable add_docker_metadata from within the container
      - '/var/run/docker.sock:/var/run/docker.sock:ro'
    command: 'metricbeat -e'
    restart: on-failure
    networks:
      - stack
    depends_on:
      - elasticsearch
      - kibana


  #Short lived container to configure the stack once Kibana and Elasticsearch are available
  configure_stack:
    image: docker.elastic.co/beats/filebeat:${ELASTIC_STACK_VERSION}
    hostname: configure_stack
    container_name: configure_stack
    volumes:
      - './docker-compose/setup.sh:/usr/local/bin/setup.sh:ro'
      - './docker-compose/setup_ingest-pipeline_parse-python.json:/usr/local/bin/setup_ingest-pipeline_parse-python.json:ro'
    command: '/usr/local/bin/setup.sh'
    networks:
      - stack
    depends_on:
      - elasticsearch
      - kibana

  #The python app to be monitored
  python_app:
    hostname: python_app
    container_name: python_app
    build:
      dockerfile: $PWD/dockerfile_python
      context: $PWD
      labels:
        - "app=fizzbuzz"
        - "co.elastic.logs/multiline.pattern=^\\["
        - "co.elastic.logs/multiline.negate=true"
        - "co.elastic.logs/multiline.match=after"
    volumes:
      # Bind-mount the log folder to the host so a sidecar can collect the logs
      - './logs-docker/:/logs/'
    networks:
      - stack
    depends_on:
      - elasticsearch
      - logstash


networks:
  stack: {}