logstash-8.yaml

# Logstash is a bit of a behemoth because of the way the plugin system works.
# Usually, patched plugins would be dropped on the filesystem as APK's and the
# application could just pick them up. Logstash's plugin system requires the
# plugins to be added by the logstash-plugin cli tool since it uses a generated
# Gemfile.
#
# In order to patch plugins we can do a few things
#
# 1. If the plugin is a default plugin, patch it out of the plugins-metadata.json
#    file so the upstream plugin is not installed. Add the plugin apk to the build
#    environment and use logstash-plugin to install the .gem file before packaging
#    the dist into a tarball (and eventually apk). All of this should be done in
#    the main pipeline so the changes are picked up by all subpackages.
#
# 2. If the plugin is not a default plugin, do the same as above, but in a
#    subpackage to be installed separately.
package:
  name: logstash-8
  version: 8.15.3
  epoch: 6
  description: Logstash - transport and process your logs, events, or other data
  copyright:
    - license: Apache-2.0
  dependencies:
    provides:
      - ${{package.name}}-base=${{package.full-version}}
      - logstash=${{package.full-version}}
    runtime:
      - bash # some helper scripts use bash and busybox utilities
      - busybox
      - openjdk-17-jre
      - openjdk-17-default-jvm

# Create a new major-version variable that contains only the major version
# to use in the bitnami/compat pipeline to find out the correct folder for the image.
# e.g. 25.0.2 will create a new var major-version=25
var-transforms:
  - from: ${{package.version}}
    match: ^(\d+).*
    replace: $1
    to: major-version

environment:
  contents:
    packages:
      - bash
      - busybox
      - ca-certificates-bundle
      - curl
      - gradle
      - jq
      - jruby-9.4
      # logstash plugins to load at build-time
      - logstash-filter-xml
      - logstash-integration-jdbc
      - logstash-output-opensearch
      # end logstash plugins to load at build-time
      - openjdk-17
      - openjdk-17-default-jvm
  environment:
    OSS: "true"
    LOGSTASH_SOURCE: "1"
    LANG: en_US.UTF-8
    JAVA_HOME: /usr/lib/jvm/default-jvm
    LS_JAVA_HOME: /usr/lib/jvm/default-jvm
    LS_DESTINATION: usr/share/logstash
    MAVEN_OPTS: "-Dmaven.repo.local=/var/cache/melange"
    GRADLE_USER_HOME: "/var/cache/melange"
    GRADLE_OPTS: "-Dorg.gradle.caching=true"

pipeline:
  - uses: git-checkout
    with:
      repository: https://github.com/elastic/logstash
      tag: v${{package.version}}
      expected-commit: 8364c8e89cfb113e38ec3f966df7eb1e9abe9d33

  - name: Patch sources
    runs: |
      # Disable the logstash-integration-jdbc plugin download as we build and
      # package it separately
      jq 'del(.["logstash-integration-jdbc"])' rakelib/plugins-metadata.json > /tmp/plugins-metadata.json
      mv /tmp/plugins-metadata.json rakelib/

      # Disable the logstash-filter-xml plugin download as we build and
      # package it separately
      jq 'del(.["logstash-filter-xml"])' rakelib/plugins-metadata.json > /tmp/plugins-metadata.json
      mv /tmp/plugins-metadata.json rakelib/

      # Disable -Werror which means that warnings are not treated as errors
      # during compilation.  Required for Java 21.
      sed -i '/options.compilerArgs.add("-Werror")/s/^/\/\//' build.gradle

      # A hack-fix to tackle with `Could not find method enabled() for
      # arguments [true] on Report xml` error starting after 8.11.x.
      sed -i '/enabled/d' logstash-core/build.gradle

  - name: Patch out the windows and mac tarball creation
    uses: patch
    with:
      patches: create_archive_pack.patch

  - name: Install all the default plugins
    runs: |
      ./gradlew installDefaultGems

  - name: Re-inject default plugins that have been patched
    runs: |
      for gem in \
        "logstash-integration-jdbc" \
        "logstash-filter-xml";
      do
        echo "Searching for gem search=$gem-*.gem"
        gem_path=$(find /usr/share/jruby -type f -name "$gem-*.gem")
        if [ -z "$gem_path" ]; then
          echo "Could not find gem gem_path=$gem_path gem=$gem"
          exit 1
        fi
        echo "Installing gem gem_path=$gem_path"
        bin/logstash-plugin install "$gem_path"
      done

  - name: Build tarball distribution
    runs: |
      ./gradlew assembleOssTarDistribution -Pjdk_bundle_os=linux

  - name: Package distribution
    runs: |
      tarball=$(find . -type f -name "logstash-oss-${{package.version}}-SNAPSHOT-no-jdk.tar.gz")
      if [ -z $tarball ]; then
        echo "Could not find logstash-oss tarball, nothing to unpack"
        exit 1
      fi

      mkdir -p ${{targets.contextdir}}/${LS_DESTINATION}
      tar --strip-components 1 \
        -C ${{targets.contextdir}}/${LS_DESTINATION} \
        -xf $tarball

      mkdir -p ${{targets.contextdir}}/usr/bin/$name
      for i in ${{targets.contextdir}}/${LS_DESTINATION}/bin/*; do
        name=$(basename $i)
        ln -sf /${LS_DESTINATION}/bin/$name ${{targets.contextdir}}/usr/bin/$name
      done

  - name: Patch CVEs
    runs: |
      # Hack-fix to reduce CVE count: https://github.com/elastic/logstash/pull/3855
      rm -rf ${{targets.contextdir}}/${LS_DESTINATION}/vendor/bundle/jruby/*/gems/ruby-maven-libs-*/maven-home/lib

subpackages:
  - name: ${{package.name}}-compat
    description: Compatibility with the upstream image
    dependencies:
      replaces:
        - ${{package.name}}
        - ${{package.name}}-with-output-opensearch
    pipeline:
      - runs: |
          src=docker/data/logstash
          dst="${{targets.contextdir}}/${LS_DESTINATION}"
          mkdir -p "${dst}/config" \
                   "${dst}/pipeline" \
                   "${{targets.contextdir}}/usr/bin"

          install -m644 "${src}/config/pipelines.yml"          "${dst}/config/pipelines.yml"
          install -m644 "${src}/config/logstash-oss.yml"       "${dst}/config/logstash-oss.yml"
          install -m644 "${src}/config/logstash-full.yml"      "${dst}/config/logstash-full.yml"
          install -m644 "${src}/config/log4j2.file.properties" "${dst}/config/log4j2.file.properties"
          install -m644 "${src}/config/log4j2.properties"      "${dst}/config/log4j2.properties"
          install -m644 "${src}/pipeline/default.conf"         "${dst}/pipeline/default.conf"
          install -m755 "${src}/bin/docker-entrypoint"         "${{targets.contextdir}}/usr/bin/docker-entrypoint"

  - name: ${{package.name}}-bitnami-compat
    description: "compat package with bitnami/logstash image"
    dependencies:
      runtime:
        - bash # some helper scripts use bash and busybox utilities
        - busybox
        - yq
        - posix-libc-utils # Required for getent
        - openjdk-17-jre
        - openjdk-17-default-jvm
    pipeline:
      - uses: bitnami/compat
        with:
          image: logstash
          version-path: ${{vars.major-version}}/debian-12
      - runs: |
          mkdir -p ${{targets.subpkgdir}}/opt/bitnami/logstash
          mkdir -p ${{targets.subpkgdir}}/opt/bitnami/scripts/logstash/

          chmod g+rwX ${{targets.subpkgdir}}/opt/bitnami

          src=docker/data/logstash
          dst="${{targets.contextdir}}/opt/bitnami/logstash"
          mkdir -p "${dst}/config" \
                   "${dst}/pipeline"

          tarball=$(find . -type f -name "logstash-oss-${{package.version}}-SNAPSHOT-no-jdk.tar.gz")
          if [ -z $tarball ]; then
            echo "Could not find logstash-oss tarball, nothing to unpack"
            exit 1
          fi

          tar --strip-components 1 \
            -C ${{targets.contextdir}}/opt/bitnami/logstash \
            -xf $tarball

          install -m666 config/jvm.options                     "${dst}/config/jvm.options"
          install -m644 "${src}/config/pipelines.yml"          "${dst}/config/pipelines.yml"
          install -m644 "${src}/config/logstash-oss.yml"       "${dst}/config/logstash-oss.yml"
          install -m644 "${src}/config/logstash-full.yml"      "${dst}/config/logstash-full.yml"

          # Replace the incorrect Java paths in the Bitnami scripts
          sed -i 's/export JAVA_HOME=\"${BITNAMI_ROOT_DIR}\/java\"/export JAVA_HOME="\/usr\/lib\/jvm\/java-17-openjdk"/g' ${{targets.contextdir}}/opt/bitnami/scripts/logstash-env.sh
          sed -i 's/export LS_JAVA_HOME=\/opt\/bitnami\/java/export LS_JAVA_HOME="\/usr\/lib\/jvm\/java-17-openjdk"/g' ${{targets.contextdir}}/opt/bitnami/scripts/logstash/run.sh

          # Disable some commands used in Bitnami scripts. These commands more likely fail in this since this image take non root approach
          sed -i 's/ensure_user_exists/# ensure_user_exists/g' ${{targets.contextdir}}/opt/bitnami/scripts/logstash/postunpack.sh
          sed -i 's/am_i_root/# am_i_root/g' ${{targets.contextdir}}/opt/bitnami/scripts/logstash/setup.sh

          # Use package path while unpacking
          find . -iname "*.sh" -exec sed 's#/opt/bitnami#${{targets.contextdir}}/opt/bitnami#g' -i {} \;
            ${{targets.contextdir}}/opt/bitnami/scripts/java/postunpack.sh || true
            ${{targets.contextdir}}/opt/bitnami/scripts/logstash/postunpack.sh || true
          # Restore path
          find ${{targets.contextdir}}/opt/bitnami -type f -exec sed 's#${{targets.contextdir}}##g' -i {} \;

          chmod -R u+rwX,g+rwX,o+rw ${{targets.subpkgdir}}/opt/bitnami/
          find / -perm /6000 -type f -exec chmod a-s {} \; || true

          # Link binaries used by Bitnami config
          ln -sf /opt/bitnami/scripts/logstash/entrypoint.sh ${{targets.contextdir}}/entrypoint.sh
          ln -sf /opt/bitnami/scripts/logstash/run.sh ${{targets.contextdir}}/run.sh
      - runs: |
          # mkdir these empty directories to quiet various startup messages per upstream
          mkdir -p "${{targets.subpkgdir}}"/bitnami/logstash/data
          mkdir -p "${{targets.subpkgdir}}"/bitnami/logstash/pipeline
          mkdir -p "${{targets.subpkgdir}}"/bitnami/logstash/config
      - runs: |
          # cp config to config.default because the entrypoint.sh run.sh init step wants something to copy
          cp -r "${{targets.subpkgdir}}"/opt/bitnami/logstash/config "${{targets.subpkgdir}}"/opt/bitnami/logstash/config.default
          cp -r "${{targets.subpkgdir}}"/opt/bitnami/logstash/pipeline "${{targets.subpkgdir}}"/opt/bitnami/logstash/pipeline.default
          # bitnami have a custom log4j2.properties
          cp "${{targets.subpkgdir}}"/opt/bitnami/logstash/config.default/log4j2.properties "${{targets.subpkgdir}}"/opt/bitnami/logstash/config.default/log4j2.orig.properties
          rm -f "${{targets.subpkgdir}}"/opt/bitnami/logstash/config.default/log4j2.properties
          install -Dm755 log4j2.custom.properties "${{targets.subpkgdir}}"/opt/bitnami/logstash/config.default/log4j2.properties
          # also have a custom logstash.yml
          rm "${{targets.subpkgdir}}"/opt/bitnami/logstash/config.default/logstash.yml
          install -Dm755 logstash.custom.yml "${{targets.subpkgdir}}"/opt/bitnami/logstash/config.default/logstash.yml
    test:
      environment:
        contents:
          packages:
            - bash
            - curl
            - busybox
            - openjdk-17-jre
            - openjdk-17-default-jvm
        accounts:
          groups:
            - groupname: logstash
              gid: 1001
          users:
            - username: logstash
              gid: 1001
              uid: 1001
          run-as: 1001
        environment:
          LS_JAVA_HOME: /usr/lib/jvm/java-17-openjdk
      pipeline:
        - working-directory: /tmp
          pipeline:
            - runs: |
                run-script --version
                run-script --help
            - name: "start daemon on localhost"
              uses: test/daemon-check-output
              with:
                start: |
                  env "BITNAMI_APP_NAME=logstash" \
                    "APP_VERSION=${{package.version}}" \
                    "PATH=/opt/bitnami/logstash/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \
                    /opt/bitnami/scripts/logstash/entrypoint.sh \
                    /opt/bitnami/scripts/logstash/run.sh
                timeout: 120
                expected_output: |
                  Welcome to the Bitnami logstash container
                  Starting Logstash setup
                  Starting Logstash
                  Starting http input listener
                post: |
                  #!/bin/sh -e
                  url=http://localhost:8080
                  response=$(curl -fsS --connect-timeout 5 --max-time 10 --retry 5 --retry-delay 0 --retry-max-time 40 "$url") || {
                    echo "curl ${url} failed $?"
                    exit 1
                  }
                  echo "$response" | grep -q ok || {
                    echo "response from $url did not contain \"ok\""
                    echo "response: $response"
                    exit 1
                  }
                  echo "$url had expected output: $response"

  - name: ${{package.name}}-env2yaml
    description: Merge environment variables into logstash.yml
    pipeline:
      - working-directory: docker/data/logstash/env2yaml
        pipeline:
          - uses: go/build
            with:
              packages: .
              output: env2yaml
              ldflags: -s -w
      - uses: strip

  # Due to the way logstash implements their plugin system, this is a full
  # logstash package with the opensearch plugin bundled. It is based off the
  # same artifacts in the main package, meaning it will pick up the changes to
  # the default gems that were made in the main package it just adds the
  # opensearch plugin.
  - name: ${{package.name}}-with-output-opensearch
    description: Build logstash with logstash-output-opensearch plugin
    dependencies:
      replaces:
        - ${{package.name}}
      runtime:
        - bash # some helper scripts use bash and busybox utilities
        - busybox
        - openjdk-17-jre
        - openjdk-17-default-jvm
    pipeline:
      - name: Re-inject default plugins that have been patched
        runs: |
          for gem in \
            "logstash-output-opensearch";
          do
            echo "Searching for gem search=$gem-*.gem"
            gem_path=$(find /usr/share/jruby -type f -name "$gem-*.gem")
            if [ -z "$gem_path" ]; then
              echo "Could not find gem gem_path=$gem_path gem=$gem"
              exit 1
            fi
            echo "Installing gem gem_path=$gem_path"
            bin/logstash-plugin install "$gem_path"
          done
      - name: Build tarball distribution
        runs: |
          ./gradlew assembleOssTarDistribution -Pjdk_bundle_os=linux
      - name: Package distribution
        runs: |
          mkdir -p ${{targets.contextdir}}/${LS_DESTINATION}
          tar --strip-components 1 \
            -C ${{targets.contextdir}}/${LS_DESTINATION} \
            -xf $(find . -type f -name "logstash-oss-${{package.version}}-SNAPSHOT-no-jdk.tar.gz")

          mkdir -p ${{targets.contextdir}}/usr/bin/$name
          for i in ${{targets.contextdir}}/${LS_DESTINATION}/bin/*; do
            name=$(basename $i)
            ln -sf /${LS_DESTINATION}/bin/$name ${{targets.contextdir}}/usr/bin/$name
          done
      - name: Patch CVEs
        runs: |
          # Hack-fix to reduce CVE count: https://github.com/elastic/logstash/pull/3855
          rm -rf ${{targets.contextdir}}/${LS_DESTINATION}/vendor/bundle/jruby/*/gems/ruby-maven-libs-*/maven-home/lib
    test:
      environment:
        contents:
          packages:
            - wolfi-base
        environment:
          LS_JAVA_HOME: /usr/lib/jvm/default-jvm
      pipeline:
        - name: Ensure output opensearch plugin was installed
          runs: |
            logstash-plugin list | grep logstash-output-opensearch
        - name: Ensure logstash itself works with basic function
          runs: |
            echo "hello" | \
              logstash -e 'input { stdin { } } output { stdout {} }' | \
              grep 'message.*hello'

test:
  pipeline:
    - name: Ensure default plugins were actually installed
      runs: |
        logstash-plugin list | grep logstash-integration-jdbc
        logstash-plugin list | grep logstash-filter-xml
        logstash --version
        logstash.lib.sh --version
        logstash.lib.sh --help
        pqcheck --help
        ruby --version
        ruby --help
        system-install --help
    - name: Ensure logstash itself works with basic function
      runs: |
        echo "hello" | \
          logstash -e 'input { stdin { } } output { stdout {} }' | \
          grep 'message.*hello'
    - name: Test logstash-filter-xml
      runs: |
        # Needs to be an absolute path or logstash will ignore
        logstash -f $PWD/test/logstash-filter-xml/logstash.conf

update:
  enabled: true
  github:
    identifier: elastic/logstash
    strip-prefix: v
    use-tag: true