.gitlab-ci.yml

# Copyright (c) the JPEG XL Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# We define only two stages for development. The "build" stage only compiles the
# code and doesn't run any target code. The "test" stage runs the code compiled
# by the previous stage. Stages form dependencies in a block, all the build
# targets in one stage need to be completed before any of the next stage can
# start.
stages:
  - build
  - test
  - release
  - deploy

variables:
  # By default we don't fetch any third_party dependencies. Building requires to
  # fetch all the dependencies but many other workflows don't.
  GIT_SUBMODULE_STRATEGY: none

# A template for running in the generic cloud builders. These are tagged with
# "linux" and run on shared VMs.
.linux_host_template: &linux_host_template
  image: &jpegxl-builder gcr.io/jpegxl/jpegxl-builder@sha256:a514f2eb013a40be731e32786435a746e66f7ed5c686b86cf19d9c09b3f737e6
  tags:
    - linux
  # By default all the workflows run on master and on request. This can be
  # override by users of this template.
  only:
    - master
    - tags
    - schedules
  # Retry on system failures. This can be typically caused by the runner or VM
  # being evicted.
  retry:
    max: 2
    when:
      - unknown_failure
      - api_failure
      - stuck_or_timeout_failure
      - runner_system_failure
  # Default variables. Note that adding a "variables:" section to a template
  # using this template will completely discard this section, not just add new
  # variables.
  variables:
    GIT_SUBMODULE_STRATEGY: none
    BUILD_DIR: build
    CC: clang-7
    CXX: clang++-7

# Common artifacts for the build stage
.default_build_paths: &default_build_paths
  - build/libjxl.so*
  - build/libjxl.dll
  # Binary tools like cjxl are "cjxl.exe" in windows builds. These
  # matches will match against either one.
  - build/tools/cjxl*
  - build/tools/djxl*
  - build/tools/benchmark_xl*
  - build/tools/decode_and_encode*
  - build/tools/comparison_viewer/compare_codecs*
  - build/tools/comparison_viewer/compare_images*
  - build/tools/viewer/viewer*
  # All the test binaries in the build/ directory and .cmake files used by
  # ctest, compressed into this file together to save space when uploading.
  - build/tests.tar.xz
  # The coverage data produced at compile time.
  - build/gcno.tar.xz
  # The packed html doxygen documentation, only produced by the coverage build.
  - build/htmldoc.tar.gz
  - build/stats.json
  - build/LICENSE*

# Helper template to add the default artifacts paths to all build stages. If
# files in the "paths:" section are not present only a warning is generated.
.linux_host_build_template: &linux_host_build_template
  <<: *linux_host_template
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
    expire_in: 1 week
    paths: *default_build_paths
  variables:
    # Building always requires to fetch all the repositories.
    GIT_SUBMODULE_STRATEGY: recursive
    BUILD_DIR: build
    CC: clang-7
    CXX: clang++-7

# Build from tarball source using README.md instructions.
build:x86_64:clang:tarball:
  <<: *linux_host_build_template
  stage: build
  variables:
    GIT_SUBMODULE_STRATEGY: none
    BUILD_DIR: build
    CC: clang-7
    CXX: clang++-7
  script:
    # Simulate a tarball checkout by removing .git and not fetching any
    # submodule.
    - mv .git .git-ignore || true
    - ./deps.sh
    - apt update
    - apt install -y cmake pkg-config libbrotli-dev
        libgif-dev libjpeg-dev libopenexr-dev libpng-dev libwebp-dev
    - mkdir build
    - cd build
    - cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF ..
    - cmake --build . -- -j

# linux x86_64 default Release mode.
build:x86_64:clang:release:
  <<: *linux_host_build_template
  stage: build
  only:
    - master
    - merge_requests
    - schedules
    - tags
  script:
    # Check that the build files are up to date.
    - tools/build_cleaner.py
    - SKIP_TEST=1 PACK_TEST=1 STACK_SIZE=1
      ./ci.sh release -DCMAKE_INSTALL_PREFIX=`pwd`/prefix
    # Test that the package can be installed.
    - ninja -C build install
    - tools/build_stats.py --save build/stats.json
        cjxl djxl libjxl.so

test:x86_64:clang:release:
  <<: *linux_host_template
  stage: test
  only:
    - master
    - merge_requests
    - schedules
    - tags
  dependencies:
    - build:x86_64:clang:release
  script:
    - ./ci.sh test

# linux x86_64 with gcc in Debug mode.
build:x86_64:gcc7:debug:
  <<: *linux_host_build_template
  stage: build
  only:
    - master
    - merge_requests
    - schedules
    - tags
  script:
    - CC=gcc-7 CXX=g++-7 SKIP_TEST=1 PACK_TEST=1 ./ci.sh debug
    - tools/build_stats.py --save build/stats.json
        cjxl djxl libjxl.so

test:x86_64:gcc7:debug:
  <<: *linux_host_template
  stage: test
  dependencies:
    - build:x86_64:gcc7:debug
  only:
    - master
    - merge_requests
    - schedules
    - tags
  script:
    - TEST_STACK_LIMIT=1024 ./ci.sh test

# x86_84 test coverage build
build:x86_64:clang:coverage:
  <<: *linux_host_build_template
  stage: build
  only:
    - master
    - merge_requests
    - schedules
    - tags
  script:
    - SKIP_TEST=1 PACK_TEST=1 ./ci.sh coverage
    - tar -zcf build/htmldoc.tar.gz -C build/html .

test:x86_64:clang:coverage:
  <<: *linux_host_template
  stage: test
  only:
    - master
    - schedules
    - tags
  dependencies:
    - build:x86_64:clang:coverage
  script:
    - ./ci.sh test
    - ./ci.sh coverage_report
  variables:
    CC: clang-7
    CXX: clang++-7
    # Headers from submodules might be needed when generating the HTML reports.
    GIT_SUBMODULE_STRATEGY: recursive
    # Coverage builds use more stack for coverage tracking.
    TEST_STACK_LIMIT: 1024
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
    expire_in: 1 year
    paths:
      - build/coverage.txt
      - build/coverage*.html
      - build/coverage.xml
      - build/htmldoc.tar.gz

# Updates the "Pages" section in GitLab with the latest coverage report from
# master.
pages:
  <<: *linux_host_template
  stage: deploy
  only:
    - master
  dependencies:
    - test:x86_64:clang:coverage
  script:
    - mkdir public
    # Unpack the doxygen documentation as "pages".
    - tar -zxf build/htmldoc.tar.gz -C public
    # Link from the index to the coverage.html file.
    - sed -E "s,<!--header-->,<!--header--><a href=coverage.html>Coverage for ${CI_COMMIT_SHA}</a><br />," -i public/index.html
    # Copy the coverage files to public/coverage*
    - mv build/coverage* public/
    - sed -E "s,GCC Code Coverage Report,Coverage Report for <a href='${CI_PROJECT_URL}/commit/${CI_COMMIT_SHA}'>${CI_COMMIT_SHORT_SHA}</a>," -i public/coverage.html
  artifacts:
    paths:
      - public


# x86_64 Scalar-only release runs only on master and on request.
build:x86_64:clang:scalar:
  <<: *linux_host_build_template
  stage: build
  only:
    - master
    - merge_requests
    - schedules
    - tags
  script:
    - SKIP_TEST=1 PACK_TEST=1 CMAKE_CXX_FLAGS="-DHWY_COMPILE_ONLY_SCALAR" ./ci.sh release

test:x86_64:clang:scalar:
  <<: *linux_host_template
  stage: test
  dependencies:
    - build:x86_64:clang:scalar
  script:
    - ./ci.sh test

# i686 (x86 32-bit) builders
build:i686:clang:release:
  <<: *linux_host_build_template
  stage: build
  only:
    - master
    - merge_requests
    - schedules
    - tags
  script:
    - BUILD_TARGET=i686-linux-gnu SKIP_TEST=1 PACK_TEST=1 STACK_SIZE=1
        ./ci.sh release -DJPEGXL_ENABLE_TCMALLOC=OFF -DJPEGXL_ENABLE_FUZZERS=OFF
    - tools/build_stats.py --save build/stats.json
        --binutils=i686-linux-gnu-
        cjxl djxl libjxl.so

test:i686:clang:release:
  <<: *linux_host_template
  stage: test
  dependencies:
    - build:i686:clang:release
  script:
    - ./ci.sh test

# Windows builders. These are cross-compiled from a linux host.
build:win64:clang:release:
  <<: *linux_host_build_template
  stage: build
  only:
    - master
    - merge_requests
    - schedules
    - tags
  script:
    - BUILD_TARGET=x86_64-w64-mingw32
        SKIP_TEST=1 PACK_TEST=1
        ./ci.sh release -DJPEGXL_ENABLE_TCMALLOC=OFF -DJPEGXL_ENABLE_FUZZERS=OFF
          -DJPEGXL_ENABLE_PLUGINS=OFF

test:win64:clang:release:
  <<: *linux_host_template
  stage: test
  dependencies:
    - build:win64:clang:release
  script:
    - BUILD_TARGET=x86_64-w64-mingw32 ./ci.sh test

# aarch64 release runs only on master and on request.
build:aarch64:clang:release:
  <<: *linux_host_build_template
  stage: build
  only:
    - master
    - merge_requests
    - schedules
    - tags
    - web
  script:
    - BUILD_TARGET=aarch64-linux-gnu
        CMAKE_CXX_FLAGS="-DJXL_DISABLE_SLOW_TESTS" SKIP_TEST=1 PACK_TEST=1
        STACK_SIZE=1
        ./ci.sh release
    - tools/build_stats.py --save build/stats.json
        --binutils=aarch64-linux-gnu-
        cjxl djxl libjxl.so

test:aarch64:clang:release:
  <<: *linux_host_template
  stage: test
  dependencies:
    - build:aarch64:clang:release
  script:
    # LeakSanitizer doesn't work when running under qemu-arm.
    - ASAN_OPTIONS=detect_leaks=0 ./ci.sh test

# arm release
build:armhf:clang:release:
  <<: *linux_host_build_template
  stage: build
  only:
    - master
    - merge_requests
    - schedules
    - tags
    - web
  script:
    - BUILD_TARGET=arm-linux-gnueabihf
        CMAKE_CXX_FLAGS="-DJXL_DISABLE_SLOW_TESTS" SKIP_TEST=1 PACK_TEST=1
        STACK_SIZE=1
        ./ci.sh release
    - tools/build_stats.py --save build/stats.json
        --binutils=arm-linux-gnueabihf-
        cjxl djxl libjxl.so

test:armhf:clang:release:
  <<: *linux_host_template
  stage: test
  dependencies:
    - build:armhf:clang:release
  script:
    - ./ci.sh test

# arm debug
build:armhf:clang:debug:
  <<: *linux_host_build_template
  stage: build
  only:
    - merge_requests
  script:
    - BUILD_TARGET=arm-linux-gnueabihf SKIP_TEST=1 ./ci.sh debug

# "debug" build enabled for merge_requests. This is here to check that
# debug-only statements still compile.
build:x86_64:clang:debug:
  <<: *linux_host_build_template
  stage: build
  only:
    - merge_requests
  script:
    - SKIP_TEST=1 ./ci.sh debug

# "asan", "tsan" and "msan" builds only run on master, tags and when requested
# from the web. Only supported in x86_64 for now.
build:x86_64:clang:asan:
  <<: *linux_host_build_template
  stage: build
  script:
    - SKIP_TEST=1 PACK_TEST=1 ./ci.sh asan

test:x86_64:clang:asan:
  <<: *linux_host_template
  stage: test
  dependencies:
    - build:x86_64:clang:asan
  script:
    - ./ci.sh test

# aarch64 asan build and test.
build:aarch64:clang:asan:
  <<: *linux_host_build_template
  stage: build
  script:
    - SKIP_TEST=1 PACK_TEST=1 ./ci.sh asan

test:aarch64:clang:asan:
  <<: *linux_host_template
  stage: test
  dependencies:
    - build:aarch64:clang:asan
  # Disable asan test on arm since they timeout.
  only:
    - tags
  script:
    - ./ci.sh test

build:x86_64:clang:tsan:
  <<: *linux_host_build_template
  stage: build
  script:
    - SKIP_TEST=1 PACK_TEST=1 ./ci.sh tsan

test:x86_64:clang:tsan:
  <<: *linux_host_template
  stage: test
  dependencies:
    - build:x86_64:clang:tsan
  script:
    # tsan test runs fail with a small stack.
    - TEST_STACK_LIMIT=1024 ./ci.sh test

build:x86_64:clang:msan:
  <<: *linux_host_build_template
  stage: build
  script:
    - SKIP_TEST=1 PACK_TEST=1 ./ci.sh msan

test:x86_64:clang:msan:
  <<: *linux_host_template
  stage: test
  dependencies:
    - build:x86_64:clang:msan
  script:
    - ./ci.sh test

# Merge request linter checks.
build:lint:
  <<: *linux_host_template
  stage: build
  only:
    - merge_requests
  script:
    - LINT_OUTPUT=clang-format.patch ./ci.sh lint
  allow_failure: true
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
    expire_in: 1 month
    when: on_failure
    paths:
      - clang-format.patch

# Most clang-tidy checks are currently informational only since these are
# currently broken, however several are marked as errors.
build:tidy:
  <<: *linux_host_build_template
  stage: build
  only:
    - merge_requests
  script:
    - CMAKE_BUILD_TYPE="Release" ./ci.sh tidy
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
    expire_in: 1 month
    when: always
    paths:
      - build/clang-tidy.txt

# Faster benchmark over a smaller set of images.
.benchmark_x86_64_template: &benchmark_x86_64_template
  <<: *linux_host_template
  stage: test
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
    expire_in: 1 year
    paths:
      - build/benchmark_results/

test:fast_benchmark:release:
  <<: *benchmark_x86_64_template
  only:
    - master
    - merge_requests
    - schedules
    - tags
  dependencies:
    - build:x86_64:clang:release
  script:
    - STORE_IMAGES=0 ./ci.sh fast_benchmark

test:fast_benchmark:asan:
  <<: *benchmark_x86_64_template
  dependencies:
    - build:x86_64:clang:asan
  script:
    - STORE_IMAGES=0 ./ci.sh fast_benchmark

test:fast_benchmark.msan:
  <<: *benchmark_x86_64_template
  dependencies:
    - build:x86_64:clang:msan
  script:
    - STORE_IMAGES=0 ./ci.sh fast_benchmark

# This template runs on actual aarch64 hardware.
.benchmark_aarch64_template: &benchmark_aarch64_template
  <<: *linux_host_template
  image: gcr.io/jpegxl/jpegxl-builder-run-aarch64@sha256:27c2bb6319023ab94d66670135a971401869a3275a7e0dda177c9bb247d57e03
  stage: test
  tags:
    - aarch64
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
    expire_in: 1 year
    paths:
      - build/benchmark_results/

test:aarch64:fast_benchmark:release:
  <<: *benchmark_aarch64_template
  dependencies:
    - build:aarch64:clang:release
  script:
    # Bind to the big CPUs only.
    - ./ci.sh cpuset "${RUNNER_CPU_BIG}"
    - STORE_IMAGES=0 ./ci.sh fast_benchmark

# Benchmark test that runs separately on the big and little CPUs of the runner.
test:aarch64:arm_benchmark:release:
  <<: *benchmark_aarch64_template
  only:
    - web
    - master
  dependencies:
    - build:aarch64:clang:release
  script:
    - ./ci.sh arm_benchmark

# Benchmark ToT on nightly builds. These are scheduled at midnight UTC.
test:benchmark:release:
  image: *jpegxl-builder
  stage: test
  only:
    - schedules
  variables:
    GIT_SUBMODULE_STRATEGY: none
  dependencies:
    - build:x86_64:clang:release
  script:
    - STORE_IMAGES=0 ./ci.sh benchmark
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
    expire_in: 1 year
    paths:
      - build/benchmark_results/

build:tidy:all:
  <<: *linux_host_build_template
  stage: build
  only:
    - master
  script:
    - CMAKE_BUILD_TYPE="Release" ./ci.sh tidy all
  allow_failure: true
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
    expire_in: 1 year
    when: always
    paths:
      - build/clang-tidy.txt

# Emscripten (WASM) build.
build:ems:all:
  <<: *linux_host_build_template
  stage: build
  script:
    - export V8=/opt/.jsvu/v8
    - source /opt/emsdk/emsdk_env.sh
    - BUILD_TARGET=wasm32 SKIP_TEST=1 PACK_TEST=1 emconfigure ./ci.sh release

test:ems:all:
  <<: *linux_host_template
  stage: test
  dependencies:
    - build:ems:all
  script:
    - export V8=/opt/.jsvu/v8
    - source /opt/emsdk/emsdk_env.sh
    - BUILD_TARGET=wasm32 emconfigure ./ci.sh test

# Emscripten (WASM + SIMD) build.
build:ems_simd:all:
  <<: *linux_host_build_template
  stage: build
  only:
    - master
    - merge_requests
    - schedules
    - tags
  script:
    - export V8=/opt/.jsvu/v8
    - source /opt/emsdk/emsdk_env.sh
    - BUILD_TARGET=wasm32 ENABLE_WASM_SIMD=1 SKIP_TEST=1 PACK_TEST=1 emconfigure ./ci.sh release

test:ems_simd:all:
  <<: *linux_host_template
  stage: test
  only:
    - master
    - merge_requests
    - schedules
    - tags
  dependencies:
    - build:ems_simd:all
  script:
    - export V8=/opt/.jsvu/v8
    - source /opt/emsdk/emsdk_env.sh
    - BUILD_TARGET=wasm32 emconfigure ./ci.sh test

### Release builds

# Debian package build template.
.linux_host_debian_template: &linux_host_debian_template
  <<: *linux_host_template
  variables:
    GIT_SUBMODULE_STRATEGY: recursive
    BUILD_DIR: build
    # Prevent "apt" asking interactive questions.
    DEBIAN_FRONTEND: noninteractive
    TARGET_DIR: jpegxl-$CI_COMMIT_REF_NAME-$CI_BUILD_NAME
  stage: release
  only:
    - master
    - tags
  script:
    - apt update
    - apt install -y devscripts build-essential
    # Build and install highway.
    - apt build-dep -y ./third_party/highway
    - ./ci.sh debian_build highway
    - dpkg -i build/debs/libhwy-dev_*_amd64.deb
    # Build jpeg-xl
    - apt build-dep -y .
    - ./ci.sh debian_build jpeg-xl
    # Print package stats.
    - ./ci.sh debian_stats
    - mkdir -p ${TARGET_DIR}
    - mv -v build/debs/*.* ${TARGET_DIR}/
  artifacts:
    name: "jpegxl-$CI_COMMIT_REF_NAME-$CI_BUILD_NAME"
    expire_in: 1 week
    paths:
      - jpegxl-*/*

# Debian package build.
debian10:
  <<: *linux_host_debian_template
  image: debian:10.6

ubuntu18.04:
  <<: *linux_host_debian_template
  before_script:
    # In Ubuntu 18.04 no package installed the libgtest.a. libgtest-dev installs
    # the source files only.
    - apt update
    - apt install -y build-essential libgtest-dev cmake
    - for prj in googletest googlemock; do
        (cd /usr/src/googletest/${prj}/ &&
         cmake CMakeLists.txt -DCMAKE_INSTALL_PREFIX=/usr &&
         make all install)
      done
    # Remove libgmock-dev dependency in Ubuntu 18.04. It doesn't exist there.
    - sed '/libgmock-dev,/d' -i debian/control
  image: ubuntu:18.04

ubuntu20.04:
  <<: *linux_host_debian_template
  image: ubuntu:20.04

# Static builds.
x86_64:static:
  <<: *linux_host_build_template
  stage: release
  only:
    - master
    - tags
  script:
    # OpenEXR doesn't install the static libraries in the docker image.
    - SKIP_TEST=1 ./ci.sh release
        -DJPEGXL_DEP_LICENSE_DIR=/usr/share/doc
        -DJPEGXL_STATIC=ON
        -DBUILD_TESTING=OFF
        -DJPEGXL_ENABLE_VIEWERS=OFF
        -DJPEGXL_ENABLE_PLUGINS=OFF
        -DJPEGXL_ENABLE_OPENEXR=OFF

win64:static:
  <<: *linux_host_build_template
  stage: release
  only:
    - master
    - tags
  script:
    # OpenEXR doesn't install the static libraries in the docker image.
    - BUILD_TARGET=x86_64-w64-mingw32 SKIP_TEST=1 ./ci.sh release
        -DJPEGXL_DEP_LICENSE_DIR=/usr/share/doc
        -DJPEGXL_STATIC=ON
        -DBUILD_TESTING=OFF
        -DJPEGXL_ENABLE_VIEWERS=OFF
        -DJPEGXL_ENABLE_PLUGINS=OFF
        -DJPEGXL_ENABLE_OPENEXR=OFF
        -DJPEGXL_ENABLE_TCMALLOC=OFF
        -DJPEGXL_ENABLE_FUZZERS=OFF
    # Make sure the binary runs without any other environment variables set.
    - wine64 build/tools/cjxl.exe
        third_party/testdata/wesaturate/64px/cvo9xd_keong_macan_srgb8.png