rust.yml

# rust.yml
#
# github workflows action file for super-speedy-syslog-searcher
#
# yaml-language-server: $schema=https://raw.githubusercontent.com/SchemaStore/schemastore/master/src/schemas/json/github-workflow.json
---
name: Rust
on:
  push:
    paths:
      - .github/**
      - Cargo.toml
      - Cargo.lock
      - src/**.rs
  pull_request:
    paths:
      - .github/**
      - Cargo.toml
      - Cargo.lock
      - src/**.rs
      - benches/**
env:
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: 1
  # default rust version
  # should match first value in matrix.msrv
  MSRV_UPLOAD: "1.72.1"
  # rust version for llvm_cov job
  # should be equal or greater than $MSRV_UPLOAD
  VERSION_LLVM_COV: "1.72.1"
  # version of flamegraph which is often using latest rust
  # so a specific version must be installed for the current MSRV
  FLAMEGRAPH_VERSION: "0.6.5"
  # run options for debug
  S4_ARGSD: --blocksz 0x2 -s
  # run options for release
  S4_ARGSR: --blocksz 0x40 -s
  # run files
  S4_TEST_FILES: \
    ./logs/CentOS9/x86_64/wtmp
    ./logs/other/tests/dtf2-2.log
    ./logs/other/tests/dtf3-2a.log
    ./logs/other/tests/dtf5-6a.log.gz
    ./logs/other/tests/dtf7-20-LEVELS.log.xz
    ./logs/other/tests/gen-2-1.tar
    ./logs/other/tests/gen-20-1-faces.log
    ./logs/other/tests/gen-20-1-⚀⚁⚂⚃⚄⚅.log
    ./logs/other/tests/gen-20-2-2-faces.log
    ./logs/programs/evtx/Microsoft-Windows-Kernel-PnP%4Configuration.evtx
    ./logs/programs/journal/RHE_91_system.journal
    ./logs/standards/ctime.log
    ./logs/standards/ISO8601-Date-Extend.log
    ./logs/standards/ISO8601-Date-Ordinal.log
    ./logs/standards/ISO8601-Date-Week.log
    ./logs/standards/ISO8601-YY-MM-DD.log
    ./logs/standards/ISO8601-YYYYDDMMTHHMM.log
    ./logs/standards/ISO8601-YYYYDDMMTHHMMSS.log
    ./logs/standards/ISO8601-YYYYMM.log
    ./logs/standards/ISO8601-YYYYMMDD.log
    ./logs/standards/ISO8601-YYYY-DD-MMTHH-MM-SS.log
    ./logs/standards/ISO8601-YYYY-DDD.log
    ./logs/standards/ISO8601-YYYY-MM-DD.log
    ./logs/standards/RFC-2822.log
    ./logs/standards/RFC-3164.log
    ./logs/standards/RFC-5424-2dot-0400.log
    ./logs/standards/RFC-5424-2dotZ.log
    ./logs/standards/RFC-5424-3dotZ.log
    ./logs/standards/RFC-5424-6dot-0700.log
    ./logs/standards/Unix-ms.log
    ./logs/standards/W3C-DTF.log

jobs:

  job_rust_msrv_os:
    # this job downloads and builds dependency crates
    name: build ${{ matrix.msrv }} on ${{ matrix.os }}
    strategy:
      fail-fast: true
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
        # first `msrv` should match Cargo.toml:[package]:rust-version
        # add a few more for sanity checks (you might be surprised)
        # concise listing at https://releases.rs/
        msrv: [
          "1.72.1",
          "1.76.0",
          "1.78.0",
          "1.79.0",
        ]
    runs-on: ${{ matrix.os }}
    steps:
      # Issue #62: `git config` to workaround `git checkout` error on Windows
      - run: |
          git config --global core.protectNTFS false
      - uses: actions/checkout@v4
      - uses: moonrepo/setup-rust@v1
        with:
          channel: ${{ matrix.msrv }}
      - name: Build using rust ${{ matrix.msrv }} on ${{ matrix.os }}
        shell: bash
        run: |
          set -eux
          export CARGO_PROFILE_RELEASE_BUILD_OVERRIDE_DEBUG=true
          cargo --version
          rustc --print cfg
          cargo build --locked
          cargo build --locked --release
          ./target/release/s4 --help
      - name: Build using rust ${{ matrix.msrv }} on ${{ matrix.os }} --features mimalloc
        shell: bash
        run: |
          set -eux
          export CARGO_PROFILE_RELEASE_BUILD_OVERRIDE_DEBUG=true
          cargo build --locked --release --features mimalloc
          ./target/release/s4 --help
      - name: Build using rust ${{ matrix.msrv }} on ${{ matrix.os }} --features jemalloc
        shell: bash
        # jemalloc builds fails on github Windows
        if: ${{ matrix.os != 'windows-latest' }}
        run: |
          set -eux
          export CARGO_PROFILE_RELEASE_BUILD_OVERRIDE_DEBUG=true
          cargo build --locked --release --features jemalloc
          ./target/release/s4 --help
      - name: Upload release binary ${{ matrix.msrv }} for ${{ matrix.os }}
        if: ${{ matrix.os == 'windows-latest' && matrix.msrv == env.MSRV_UPLOAD }}
        uses: actions/upload-artifact@v4
        with:
          name: s4.exe ${{ matrix.msrv }} ${{ matrix.os }}
          path: ./target/release/s4.exe
      - name: Upload release binary ${{ matrix.msrv }} for ${{ matrix.os }}
        if: ${{ matrix.os == 'ubuntu-22.04' && matrix.msrv == env.MSRV_UPLOAD }}
        uses: actions/upload-artifact@v4
        with:
          name: s4 ${{ matrix.msrv }} ${{ matrix.os }}
          path: ./target/release/s4
      - name: Upload release binary ${{ matrix.msrv }} for ${{ matrix.os }}
        if: ${{ matrix.os == 'macos-latest' && matrix.msrv == env.MSRV_UPLOAD }}
        uses: actions/upload-artifact@v4
        with:
          name: s4 ${{ matrix.msrv }} ${{ matrix.os }}
          path: ./target/release/s4
      - name: summary
        shell: bash
        run: |
          set -eux
          cargo install default-target
          df=$(default-target)
          echo 'Built target `'"${df}"'` using rust `'"${{ matrix.msrv }}"'` on ' \
               '`'"${{ matrix.os }}"'`' >> ${GITHUB_STEP_SUMMARY}

  job_build_debug_release:
    # this job downloads and builds dependency crates
    name: build debug and release, upload
    needs: [job_rust_msrv_os]
    runs-on: ubuntu-latest
    steps:
      # build & upload debug
      - name: git checkout
        uses: actions/checkout@v4
      - uses: moonrepo/setup-rust@v1
        with:
          channel: ${{ env.MSRV_UPLOAD }}
      - name: Build Debug
        shell: bash
        run: |
          set -eux
          cargo --version
          cargo build --locked --verbose
          cp -av ./target/debug/s4 ./s4_debug
      - name: Upload debug binary for test
        uses: actions/upload-artifact@v4
        with:
          name: s4_debug
          path: ./s4_debug
      # build & upload release
      - name: Build Release
        shell: bash
        run: |
          set -eux
          cargo --version
          cargo build --locked --release --verbose
          cp -av ./target/release/s4 ./s4_release
          cargo build --locked --release --features mimalloc
          cargo build --locked --release --features jemalloc
      - name: Upload release binary for test
        uses: actions/upload-artifact@v4
        with:
          name: s4_release
          path: ./s4_release
      - name: Upload release binary (Ubuntu Linux)
        uses: actions/upload-artifact@v4
        with:
          name: s4
          path: ./target/release/s4
      # - name: Run valgrind
      #   run: |
      #     set -eux
      #     sudo apt install -y valgrind g++
      #     SCRIPT=./tools/valgrind-dhat.sh
      #     chmod -v +x -- "${PROGRAM}" "${SCRIPT}"
      #     "${SCRIPT}"

  job_cross_targets:
    name: cross ${{ matrix.target }} on ubuntu-22.04
    needs: [job_rust_msrv_os]
    strategy:
      matrix:
        target:
          # platform targets from
          # https://doc.rust-lang.org/nightly/rustc/platform-support.html
          # listings should follow the same order
          #
          # test each locally by running:
          #    cargo install cross
          #    rustup target add $TARGET
          #    cross check --lib --bins --target $TARGET
          #
          # XXX: some targets fail to missing a specific glibc required by
          #      libserde_derive, the error message reads:
          #         version `GLIBC_2.25' not found (required by /target/debug/deps/libserde_derive-917db958580e9b6d.so)
          #      see https://users.rust-lang.org/t/how-to-compile-rust-with-a-specific-glibc-version-for-gnueabihf-architecture/6680
          #      see https://github.com/japaric/rust-cross#advanced-topics
          #
          # XXX: wasm fails to compile due to missing tools
          #
          # Tier 1
          - aarch64-unknown-linux-gnu
          - i686-pc-windows-gnu
          # - i686-pc-windows-msvc # in job_cross_targets_windows
          - i686-unknown-linux-gnu
          - x86_64-pc-windows-gnu
          # - x86_64-pc-windows-msvc # in job_cross_targets_windows
          - x86_64-unknown-linux-gnu
          # Tier 2 with host tools
          # - aarch64-pc-windows-msvc  # version `GLIBC_2.25' not found
          - aarch64-unknown-linux-musl
          - arm-unknown-linux-gnueabi
          - arm-unknown-linux-gnueabihf
          - armv7-unknown-linux-gnueabihf
          # XXX: error: component 'rust-std' for target 'loongarch64-unknown-linux-gnu' is unavailable for download for channel '1.70.0'
          # - loongarch64-unknown-linux-gnu
          - powerpc-unknown-linux-gnu
          - powerpc64-unknown-linux-gnu
          # - powerpc64le-unknown-linux-gnu  # version `GLIBC_2.25' not found
          - riscv64gc-unknown-linux-gnu
          # - sparc64-unknown-linuxs-gnu # version `GLIBC_2.29' not found
          # - s390x-unknown-linux-gnu  # version `GLIBC_2.25' not found
          - x86_64-unknown-freebsd
          - x86_64-unknown-illumos
          - x86_64-unknown-linux-musl
          - x86_64-unknown-netbsd
          # Tier 2 without host tools
          # (just a few chosen)
          - aarch64-linux-android
          - i686-linux-android
          - x86_64-pc-solaris
          - x86_64-sun-solaris
          - x86_64-linux-android
          - x86_64-unknown-redox
          # - wasm32-unknown-emscripten
          # - wasm32-unknown-unknown
          # - wasm32-wasi  # error: could not compile `dlopen2`
          # Tier 3
          # - aarch64-unknown-openbsd # toolchain 1.67.1 does not support ...
          # - aarch64-unknown-netbsd # toolchain 1.67.1 does not support ...
          # - i686-unknown-netbsd  # toolchain 1.67.1 does not support ...
          # - i686-unknown-openbsd  # toolchain 1.67.1 does not support ...
          # - mips64-unknown-linux-gnuabi64
          # - x86_64-unknown-dragonfly  # toolchain 1.67.1 does not support ...
          # - x86_64-unknown-openbsd  # toolchain 1.67.1 does not support ...
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v4
      - uses: moonrepo/setup-rust@v1
        with:
          channel: ${{ env.MSRV_UPLOAD }}
          bins: cross
          targets: ${{ matrix.target }}
      - run: |
          set -eux
          uname -a
          rustup show
          cross --version
          cross check --locked --lib --bins --target ${{ matrix.target }}

  job_cross_targets_macos:
    name: cross ${{ matrix.target }} on macos-13
    needs: [job_rust_msrv_os, job_cross_targets]
    strategy:
      matrix:
        target:
          # Tier 1
          - x86_64-apple-darwin
          # Tier 2 with host tools
          - aarch64-apple-darwin
          # Tier 2 without host tools
          - aarch64-apple-ios
          - x86_64-apple-ios
    runs-on: macos-13
    steps:
      - uses: actions/checkout@v4
      # XXX: use `Swatinem/rust-cache@v2` not `moonrepo/setup-rust@v1`
      #      due to https://github.com/moonrepo/setup-rust/issues/20
      - run: rustup toolchain install ${{ env.MSRV_UPLOAD }} --profile minimal
      - run: rustup target add ${{ matrix.target }}
      - run: cargo install cross --locked
      - uses: Swatinem/rust-cache@v2
      - run: |
          set -eux
          uname -a
          rustup show
          cross --version
          cross check --locked --lib --bins --target ${{ matrix.target }}

  job_cross_targets_windows:
    name: cross ${{ matrix.target }} on windows
    needs: [job_rust_msrv_os, job_cross_targets_macos]
    strategy:
      matrix:
        target:
          # Tier 1
          - i686-pc-windows-msvc
          - x86_64-pc-windows-msvc
    runs-on: windows-latest
    steps:
      - uses: actions/checkout@v4
      - uses: moonrepo/setup-rust@v1
        with:
          channel: ${{ env.MSRV_UPLOAD }}
          bins: cross, cargo-cache
          targets: ${{ matrix.target }}
      - run: rustup target add ${{ matrix.target }}
      - name: cross check
        shell: powershell
        run: |
          Set-PSDebug -Trace 1
          rustup.exe show
          cross.exe --version
          cross.exe check --locked --lib --bins --target ${{ matrix.target }}

  # job_test_wasm:
  #  runs-on: ubuntu-latest
  #  steps:
  #    - uses: actions/checkout@v4
  #    - uses: moonrepo/setup-rust@v1
  #      with:
  #        channel: ${{ env.MSRV_UPLOAD }}
  #        bins: cargo-wasi
  #        targets: wasm32-wasi
  #    - uses: mwilliamson/setup-wasmtime-action@v2
  #      with:
  #        wasmtime-version: "12.0.1"
  #    # can not compile `filepath`
  #    # see https://github.com/evilpie/filepath/issues/6
  #    - run: cargo wasi test --color=always -- --color=always

  job_check:
    # this job downloads and builds dependency crates
    name: check
    needs: [job_rust_msrv_os]
    runs-on: ubuntu-latest
    steps:
      - name: git checkout
        uses: actions/checkout@v4
      - uses: moonrepo/setup-rust@v1
        with:
          channel: ${{ env.MSRV_UPLOAD }}
      - name: Check
        shell: bash
        # `cargo check` builds dependences and other things but skips
        # final code generation
        run: |
          set -eux
          cargo --version
          cargo check --locked --all-targets
          cargo check --locked --all-targets --release

  job_clippy:
    # this job downloads and builds dependency crates
    name: clippy
    needs: [job_rust_msrv_os]
    runs-on: ubuntu-latest
    steps:
      - name: git checkout
        uses: actions/checkout@v4
      - uses: moonrepo/setup-rust@v1
        with:
          channel: ${{ env.MSRV_UPLOAD }}
          profile: default
      - name: Clippy
        shell: bash
        run: |
          set -eux
          cargo --version
          cargo clippy --version
          cargo clippy --locked --no-deps --verbose --all-targets --all-features

  job_bench:
    # this job downloads and builds dependency crates
    name: bench
    needs: [job_rust_msrv_os]
    runs-on: ubuntu-latest
    steps:
      - name: git checkout
        uses: actions/checkout@v4
      - uses: moonrepo/setup-rust@v1
        with:
          channel: ${{ env.MSRV_UPLOAD }}
      - name: Bench Dry Run
        shell: bash
        run: |
          set -eux
          cargo --version
          cargo bench --locked --no-run --features bench_jetscii,bench_memchr,bench_stringzilla

  job_test_linux:
    # this job downloads and builds dependency crates
    name: test linux
    needs: [job_rust_msrv_os]
    runs-on: ubuntu-latest
    steps:
      - name: git checkout
        uses: actions/checkout@v4
      - uses: moonrepo/setup-rust@v1
        with:
          channel: ${{ env.MSRV_UPLOAD }}
          bins: cargo-nextest
      - name: Install libsystemd
        shell: bash
        run: |
          set -eux
          uname -a
          sudo apt update || true
          (find / -xdev \
                  \( -type f -o -type l \) \
                  -name 'libsystemd*' 2>/dev/null || true) \
              | sort
          sudo apt install --yes libsystemd0
          (find / -xdev \
                  \( -type f -o -type l \) \
                  -name 'libsystemd*' 2>/dev/null || true) \
              | sort
      - name: set log file filesystem Modified Times
        shell: bash
        run: |
          set -eux
          bash --version
          grep --version
          chmod -v +x ./tools/log-files-time-update.sh
          ./tools/log-files-time-update.sh
      - name: test (limited)
        # sanity check tests requiring libsystemd can run
        shell: bash
        run: |
          set -eux
          # can libsystemd be found?
          (find / -xdev \
                  \( -type f -o -type l \) \
                  -name 'libsystemd*' 2>/dev/null || true) \
              | sort
          # run tests that use libsystemd
          cargo test --locked journalreader_tests
      - name: nextest (all)
        shell: bash
        run: |
          set -eux
          bash --version
          cargo nextest --version
          export NEXTEST_TEST_THREADS="num-cpus"
          cargo nextest run --locked --bins --lib --no-fail-fast --final-status-level=fail

  job_test_macos:
    name: test Mac OS
    needs: [job_rust_msrv_os]
    runs-on: macos-latest
    steps:
      - name: git checkout
        uses: actions/checkout@v4
      - uses: moonrepo/setup-rust@v1
        with:
          channel: ${{ env.MSRV_UPLOAD }}
          bins: cargo-nextest
      # XXX: MacOS tools do not support `touch` arguments run in
      #      `./tools/log-files-time-update.sh`. So skip running `log-files-time-update.sh`
      #      See https://stackoverflow.com/q/78540977/471376
      - name: nextest (all)
        shell: bash
        run: |
          set -eux
          cargo nextest --version
          export NEXTEST_TEST_THREADS="num-cpus"
          cargo nextest run --locked --bins --lib --no-fail-fast --final-status-level=fail

  job_test_windows:
    name: test Windows
    needs: [job_rust_msrv_os]
    runs-on: windows-latest
    steps:
      - name: git checkout
        uses: actions/checkout@v4
      - uses: moonrepo/setup-rust@v1
        with:
          channel: ${{ env.MSRV_UPLOAD }}
          bins: cargo-nextest
      - name: set log file filesystem Modified Times
        shell: bash
        run: |
          set -eux
          uname -a
          bash --version
          grep --version
          chmod -v +x ./tools/log-files-time-update.sh
          ./tools/log-files-time-update.sh
      - name: nextest (all)
        shell: powershell
        run: |
          Set-PSDebug -Trace 1
          cargo.exe nextest --version
          ${env:NEXTEST_TEST_THREADS}="2"
          cargo.exe nextest run --locked --bins --lib --no-fail-fast --final-status-level=fail

  job_flamegraph:
    # check that `flamegraph.sh` runs
    name: flamegraph
    needs: [job_rust_msrv_os]
    runs-on: ubuntu-latest
    steps:
      - name: git checkout
        uses: actions/checkout@v4
      - uses: moonrepo/setup-rust@v1
        with:
          channel: ${{ env.MSRV_UPLOAD }}
      - uses: awalsh128/cache-apt-pkgs-action@latest
        with:
          packages: linux-tools-generic
          version: 1.0
      - name: flamegraph.sh
        shell: bash
        run: |
          set -eux
          cargo install --locked flamegraph --version=${FLAMEGRAPH_VERSION}
          RUSTFLAGS=-g cargo build --profile flamegraph
          export OUT="./flamegraph-dtf2-2.svg"
          export FREQ=1000
          # XXX: ignore failure of flamegraph.sh
          set +e
          sudo -E --preserve-env=PATH -- ./tools/flamegraph.sh ./logs/other/tests/dtf2-2.log
          ls -l "${OUT}"
          # XXX: force success
          true

  job_doc_publish:
    # this job downloads and builds dependency crates
    name: doc, publish
    needs: [job_rust_msrv_os]
    runs-on: ubuntu-latest
    steps:
      - name: git checkout
        uses: actions/checkout@v4
      - name: Build Documentation
        run: cargo doc --locked --release --no-deps -v
      - name: Publish Dry Run
        shell: bash
        run: |
          set -eux
          cargo publish --locked --dry-run
          cargo package --list --all-features
          cargo package --locked

  job_yamllint:
    # this job installs Python PIP packages
    name: yamllint
    needs: [job_rust_msrv_os]
    runs-on: ubuntu-latest
    steps:
      - name: git checkout
        uses: actions/checkout@v4
      - name: Use Python 3.10
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"
      - name: Install dependencies
        run: |
          set -eux
          python --version
          python -m pip \
            --disable-pip-version-check \
            --no-python-version-warning \
            --version
          python -m pip install \
            --disable-pip-version-check \
            --no-python-version-warning \
            --no-color \
            yamllint==1.35.1
          python -m pip list \
            --disable-pip-version-check \
            --no-python-version-warning \
            -v -v
      - name: Run yamllint.sh
        run: PYTHON=python ./tools/yamllint.sh --format github

  job_codecov_validate:
    name: codecov validate
    needs: [job_rust_msrv_os]
    runs-on: ubuntu-latest
    steps:
      - name: git checkout
        uses: actions/checkout@v4
      - name: Run codecov-validate.sh
        run: ./tools/codecov-validate.sh -v

  job_markup_link_checker:
    name: markup link checker
    needs: [job_rust_msrv_os]
    runs-on: ubuntu-latest
    steps:
      - name: Markup Link Checker (mlc)
        uses: becheran/mlc@v0.17.1
        with:
          args: ./README.md

  job_runs:
    # all of these steps need the s4 binary and the git-committed logs
    # with modified filesystem datetime
    name: run s4
    needs: job_build_debug_release
    runs-on: ubuntu-latest
    steps:
      # checkout for the log files
      - name: git checkout
        uses: actions/checkout@v4
      - uses: moonrepo/setup-rust@v1
        with:
          channel: ${{ env.MSRV_UPLOAD }}
      - name: set log file filesystem Modified Times
        shell: bash
        run: |
          set -eux
          bash --version
          grep --version
          chmod -v +x ./tools/log-files-time-update.sh
          ./tools/log-files-time-update.sh
      - name: Install libsystemd
        shell: bash
        run: |
          set -eux
          sudo apt update || true
          sudo apt install --yes libsystemd0
      # download & run debug
      - name: Download debug binary
        uses: actions/download-artifact@v4
        with:
          name: s4_debug
      - name: s4_debug --help
        shell: bash
        run: |
          set -eux
          (find / -xdev \
                  \( -type f -o -type l \) \
                  -name 'libsystemd*' 2>/dev/null || true) \
              | sort
          S4=./s4_debug
          chmod -v +x -- ${S4}
          ${S4} --version
          ${S4} --help
      - name: s4_debug --color=never S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_debug
          chmod -v +x -- ${S4}
          ${S4} --color=never ${S4_TEST_FILES} 2>/dev/null
      - name: s4_debug S4_ARGSD --color=never S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_debug
          chmod -v +x -- ${S4}
          ${S4} ${S4_ARGSD} --color=never ${S4_TEST_FILES} 2>/dev/null
      - name: s4_debug S4_ARGSD --color=never -l -n -w S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_debug
          chmod -v +x -- ${S4}
          ${S4} ${S4_ARGSD} --color=never -l -n -w ${S4_TEST_FILES} 2>/dev/null
      - name: s4_debug S4_ARGSD --color=never -l -p -w S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_debug
          chmod -v +x -- ${S4}
          ${S4} ${S4_ARGSD} --color=never -l -p -w ${S4_TEST_FILES} 2>/dev/null
      - name: s4_debug S4_ARGSD --color=never -u -n -w S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_debug
          chmod -v +x -- ${S4}
          ${S4} ${S4_ARGSD} --color=never -u -n -w ${S4_TEST_FILES} 2>/dev/null
      - name: s4_debug S4_ARGSD --color=never -u -p -w S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_debug
          chmod -v +x -- ${S4}
          ${S4} ${S4_ARGSD} --color=never -u -p -w ${S4_TEST_FILES} 2>/dev/null
      - name: s4_debug S4_ARGSD --color=never -l -n -w S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_debug
          chmod -v +x -- ${S4}
          ${S4} ${S4_ARGSD} --color=always -l -n -w ${S4_TEST_FILES} 2>/dev/null
      - name: s4_debug S4_ARGSD --color=never -l -p -w S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_debug
          chmod -v +x -- ${S4}
          ${S4} ${S4_ARGSD} --color=always -l -p -w ${S4_TEST_FILES} 2>/dev/null
      - name: s4_debug S4_ARGSD --color=never -u -n -w S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_debug
          chmod -v +x -- ${S4}
          ${S4} ${S4_ARGSD} --color=always -u -n -w ${S4_TEST_FILES} 2>/dev/null
      - name: s4_debug S4_ARGSD --color=never -u -p -w S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_debug
          chmod -v +x -- ${S4}
          ${S4} ${S4_ARGSD} --color=always -u -p -w ${S4_TEST_FILES} 2>/dev/null
      - name: s4_debug -s S4_ARGSD --color=never -l -n -w S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_debug
          chmod -v +x -- ${S4}
          ${S4} -s --color=never -l -n -w ${S4_TEST_FILES} &>/dev/null
      - name: s4_debug -s S4_ARGSD --color=never -l -p -w S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_debug
          chmod -v +x -- ${S4}
          ${S4} -s --color=never -l -p -w ${S4_TEST_FILES} &>/dev/null
      - name: s4_debug -s S4_ARGSD --color=never -u -n -w S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_debug
          chmod -v +x -- ${S4}
          ${S4} -s --color=never -u -n -w ${S4_TEST_FILES} &>/dev/null
      - name: s4_debug -s S4_ARGSD --color=never -u -p -w S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_debug
          chmod -v +x -- ${S4}
          ${S4} -s --color=never -u -p -w ${S4_TEST_FILES} &>/dev/null
      - name: s4_debug -s S4_ARGSD --color=always -l -n -w S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_debug
          chmod -v +x -- ${S4}
          ${S4} -s --color=always -l -n -w ${S4_TEST_FILES} &>/dev/null
      - name: s4_debug -s S4_ARGSD --color=always -l -p -w S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_debug
          chmod -v +x -- ${S4}
          ${S4} -s --color=always -l -p -w ${S4_TEST_FILES} &>/dev/null
      - name: s4_debug -s S4_ARGSD --color=always -u -n -w S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_debug
          chmod -v +x -- ${S4}
          ${S4} -s --color=always -u -n -w ${S4_TEST_FILES} &>/dev/null
      - name: s4_debug -s S4_ARGSD --color=always -u -p -w S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_debug
          chmod -v +x -- ${S4}
          ${S4} -s --color=always -u -p -w ${S4_TEST_FILES} &>/dev/null
      # download & run release
      - name: Download release binary
        uses: actions/download-artifact@v4
        with:
          name: s4_release
      - name: s4_release --help
        shell: bash
        run: |
          set -eux
          S4=./s4_release
          chmod -v +x -- ${S4}
          ${S4} --version
          ${S4} --help
      - name: s4_release --color=never S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_release
          chmod -v +x -- ${S4}
          ${S4} --color=never ${S4_TEST_FILES}
      - name: s4_release S4_ARGSR --color=never S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_release
          chmod -v +x -- ${S4}
          ${S4} ${S4_ARGSR} --color=never ${S4_TEST_FILES}
      - name: s4_release S4_ARGSR --color=never -l -n -w S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_release
          chmod -v +x -- ${S4}
          ${S4} ${S4_ARGSR} --color=never -l -n -w ${S4_TEST_FILES}
      - name: s4_release S4_ARGSR --color=never -l -p -w S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_release
          chmod -v +x -- ${S4}
          ${S4} ${S4_ARGSR} --color=never -l -p -w ${S4_TEST_FILES}
      - name: s4_release S4_ARGSR --color=never -u -n -w S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_release
          chmod -v +x -- ${S4}
          ${S4} ${S4_ARGSR} --color=never -u -n -w ${S4_TEST_FILES}
      - name: s4_release S4_ARGSR --color=never -u -p -w S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_release
          chmod -v +x -- ${S4}
          ${S4} ${S4_ARGSR} --color=never -u -p -w ${S4_TEST_FILES}
      - name: s4_release S4_ARGSR --color=never -l -n -w S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_release
          chmod -v +x -- ${S4}
          ${S4} ${S4_ARGSR} --color=always -l -n -w ${S4_TEST_FILES}
      - name: s4_release S4_ARGSR --color=never -l -p -w S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_release
          chmod -v +x -- ${S4}
          ${S4} ${S4_ARGSR} --color=always -l -p -w ${S4_TEST_FILES}
      - name: s4_release S4_ARGSR --color=always -u -n -w S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_release
          chmod -v +x -- ${S4}
          ${S4} ${S4_ARGSR} --color=always -u -n -w ${S4_TEST_FILES}
      - name: s4_release S4_ARGSR --color=always -u -p -w S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_release
          chmod -v +x -- ${S4}
          ${S4} ${S4_ARGSR} --color=always -u -p -w ${S4_TEST_FILES}
      - name: s4_release -s --color=always S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_release
          chmod -v +x -- ${S4}
          ${S4} -s --color=never ${S4_TEST_FILES} &>/dev/null
      - name: s4_release -s --color=never -l -n -w S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_release
          chmod -v +x -- ${S4}
          ${S4} -s --color=never -l -n -w ${S4_TEST_FILES} &>/dev/null
      - name: s4_release -s --color=never -l -p -w S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_release
          chmod -v +x -- ${S4}
          ${S4} -s --color=never -l -p -w ${S4_TEST_FILES} &>/dev/null
      - name: s4_release -s --color=never -u -n -w S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_release
          chmod -v +x -- ${S4}
          ${S4} -s --color=never -u -n -w ${S4_TEST_FILES} &>/dev/null
      - name: s4_release -s --color=never -u -p -w S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_release
          chmod -v +x -- ${S4}
          ${S4} -s --color=never -u -p -w ${S4_TEST_FILES} &>/dev/null
      - name: s4_release -s --color=always -l -n -w S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_release
          chmod -v +x -- ${S4}
          ${S4} -s --color=always -l -n -w ${S4_TEST_FILES} &>/dev/null
      - name: s4_release -s --color=always -l -p -w S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_release
          chmod -v +x -- ${S4}
          ${S4} -s --color=always -l -p -w ${S4_TEST_FILES} &>/dev/null
      - name: s4_release -s --color=always -u -n -w S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_release
          chmod -v +x -- ${S4}
          ${S4} -s --color=always -u -n -w ${S4_TEST_FILES} &>/dev/null
      - name: s4_release -s --color=always -u -p -w S4_TEST_FILES
        shell: bash
        run: |
          set -eux
          S4=./s4_release
          chmod -v +x -- ${S4}
          ${S4} -s --color=always -u -p -w ${S4_TEST_FILES} &>/dev/null
      # compare debug release
      - name: Run Script compare-debug-release.sh
        shell: bash
        run: |
          set -eux
          export PROGRAMR=./s4_release
          export PROGRAMD=./s4_debug
          SCRIPT=./tools/compare-debug-release.sh
          chmod -v +x -- "${PROGRAMR}" "${PROGRAMD}"
          "${SCRIPT}"
      # compare grep sort
      - name: Run Script compare-grep-sort.sh
        shell: bash
        run: |
          set -eux
          export PROGRAM=./s4_release
          SCRIPT=./tools/compare-grep-sort.sh
          chmod -v +x -- "${PROGRAM}" "${SCRIPT}"
          "${PROGRAM}" --version
          "${SCRIPT}"
      # compare current and expected
      - name: Run Script compare-current-and-expected
        shell: bash
        run: |
          set -eux
          export PROGRAM=./s4_release
          SCRIPT=./tools/compare-current-and-expected/compare.sh
          chmod -v +x -- "${PROGRAM}" "${SCRIPT}"
          "${PROGRAM}" --version
          "${SCRIPT}"
      # compare cat
      - name: Run Script compare-cat
        shell: bash
        run: |
          set -eux
          export PROGRAM=./s4_release
          SCRIPT=./tools/compare-cat.sh
          chmod -v +x -- "${PROGRAM}" "${SCRIPT}"
          "${PROGRAM}" --version
          "${SCRIPT}"

  # Run code coverage using cargo-llvm-cov then upload to coveralls.io
  job_code_coverage_llvm:
    name: llvm-cov
    runs-on: ubuntu-latest
    needs: job_runs
    env:
      COVERALLS_FILE: ./coveralls.lcov
    steps:
      - name: git checkout
        uses: actions/checkout@v4
      - uses: moonrepo/setup-rust@v1
        with:
          channel: ${{ env.VERSION_LLVM_COV }}
          bins: cargo-llvm-cov
          components: llvm-tools-preview
      - name: Install libsystemd
        shell: bash
        run: |
          set -eux
          sudo apt update || true
          sudo apt install --yes libsystemd0
      - name: set log file filesystem Modified Times
        shell: bash
        run: |
          set -eux
          bash --version
          grep --version
          chmod -v +x ./tools/log-files-time-update.sh
          ./tools/log-files-time-update.sh
      - name: run llvm-cov
        shell: bash
        run: |
          set -eux
          chmod -v +x ./tools/cargo-llvm-cov-run.sh
          cargo llvm-cov --version
          cargo llvm-cov show-env
          cargo llvm-cov --locked --lcov --output-path "${COVERALLS_FILE}" &>/dev/null
          ls -l "${COVERALLS_FILE}"
      - name: upload coveralls
        uses: coverallsapp/github-action@v2
        with:
          file: ${{ env.COVERALLS_FILE }}
          format: lcov
...