.gitlab-ci.yml

stages:
  - .pre
  - build
  - test
  - build-conan
  - archive
  - deploy

include:
    - local: '/ci/gitlab-ci-common.yml'

variables:
  GIT_SUBMODULE_STRATEGY: recursive
  STABLE_BRANCH_NAME: master
  DO_UPLOAD: "yes" # Always upload in conan upload jobs (only run on tags)
  CONAN_PROFILE_BUILD_TYPE: Release
before_script:
  - ""


# ======================================
#
#     Docker
#
# ======================================


.build-docker-image:
  stage: .pre
  image: ${CI_REGISTRY}/traque/ont-docker-base/ont-base-docker:latest
  before_script:
    - docker login --username ${CI_REGISTRY_USER} --password ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY}
  when: manual
  retry:
    max: 2
    when: runner_system_failure
  script:
    - tag="${CI_REGISTRY_IMAGE}/${IMAGE_TAG}"
    - docker image build --pull --target "${PLATFORM}"
                         --tag "${tag}" --file ${DOCKERFILE} ci/docker
    - docker image push ${tag}

docker base aarch64:
  tags:
    - docker-builder-arm
  extends:
    - .build-docker-image
  variables:
    IMAGE_TAG: "build-arm64"
    DOCKERFILE: "ci/docker/Dockerfile.py39.arm64"

docker base x86-64:
  tags:
    - docker-builder
  extends:
    - .build-docker-image
  variables:
    IMAGE_TAG: "build-x64"
    DOCKERFILE: "ci/docker/Dockerfile.py39.x64"

docker conda:
  tags:
    - docker-builder
  extends:
    - .build-docker-image
  variables:
    IMAGE_TAG: "conda"
    DOCKERFILE: "ci/docker/Dockerfile.conda"


.docker template:
  stage: docker
  image: ${CI_REGISTRY}/traque/ont-docker-base/ont-base-docker:latest
  before_script:
    - docker login --username ${CI_REGISTRY_USER} --password ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY}
  retry:
    max: 2
    when: runner_system_failure

# ======================================
#
#     Versioning
#
# ======================================

prepare_version:
  stage: .pre
  image: ${CI_REGISTRY}/traque/ont-docker-base/ont-base-python:3.10
  script:
    - git tag -d $(git tag -l "*a*")
    - git tag -d $(git tag -l "*b*")
    - git tag -d $(git tag -l "*r*")
    - git tag -d $(git tag -l "*c*")
    - git tag -d $(git tag -l "*dev*")
    - if [[ ${CI_COMMIT_TAG/#v/} && -z $( git tag -l "${CI_COMMIT_TAG/#v/}" ) ]]; then git tag ${CI_COMMIT_TAG/#v/}; fi
    - pip install --upgrade pip setuptools_scm~=7.1
    - python -m setuptools_scm
    - cat _version.py
    # Show the version that will be used in the pod5/pyproject.toml
    - VERSION=$(grep "__version__" _version.py | awk '{print $5}' | tr -d "'" | cut -d'+' -f1)
    - echo $VERSION
    - python -m pod5_make_version
    - cat cmake/POD5Version.cmake
    - cat _version.py python/lib_pod5/src/lib_pod5/_version.py
    - cat _version.py python/pod5/src/pod5/_version.py
  artifacts:
    name: "${CI_JOB_NAME}-artifacts"
    paths:
      - "cmake/POD5Version.cmake"
      - "_version.py"
      - "python/lib_pod5/src/lib_pod5/_version.py"
      - "python/pod5/src/pod5/_version.py"


# ======================================
#
#     Pre-Flight Setup / Checks
#
# ======================================

tag_version_check:
  stage: .pre
  needs:
    - "prepare_version"
  only:
    - tags
  image: ${CI_REGISTRY}/minknow/images/build-x86_64-gcc9:latest
  script:
    - pod5_version="$(cmake -P ci/get_tag_version.cmake 2>&1)"
    - tag_version="${CI_COMMIT_TAG/#v/}"
    - if [[ "${pod5_version}" != "${tag_version}" ]]; then
        echo "Tag is for release ${tag_version}, but POD5 version is $pod5_version";
        exit 1;
      fi

api_lib_version_check:
  stage: .pre
  needs:
    - "prepare_version"
  image: ${CI_REGISTRY}/minknow/images/build-x86_64-gcc9:latest
  script:
    - cat _version.py
    - NO_DEV_VERSION=$(grep "__version__" _version.py | awk '{print $5}' | tr -d "'" | cut -d'+' -f1 | sed 's/\([0-9]\+\.[0-9]\+\.[0-9]\+\).*$/\1/')
    - echo $NO_DEV_VERSION
    - cat python/pod5/pyproject.toml
    - echo "If this jobs fails then we have forgotten to match the api and lib version in the api python/pod5/pyproject.toml"
    - grep "lib_pod5\s*==\s*$NO_DEV_VERSION" python/pod5/pyproject.toml

docs_version_check:
  stage: .pre
  needs:
    - "prepare_version"
  image: ${CI_REGISTRY}/minknow/images/build-x86_64-gcc9:latest
  script:
    - cat _version.py
    - NO_DEV_VERSION=$(grep "__version__" _version.py | awk '{print $5}' | tr -d "'" | cut -d'+' -f1 | sed 's/\([0-9]\+\.[0-9]\+\.[0-9]\+\).*$/\1/')
    - echo $NO_DEV_VERSION
    - cat docs/requirements.txt
    - echo "If this jobs fails then we have forgotten to update the pod5 version in the docs/requirements.txt"
    - grep "pod5\s*==\s*$NO_DEV_VERSION" docs/requirements.txt

pre-commit checks:
    image: ${CI_REGISTRY}/traque/ont-docker-base/ont-base-python:3.10
    stage: .pre
    tags:
        - linux
    script:
        - pip install pre-commit
          # if any of the hooks wanted to modify files, this will have a non-zero exit
        - if ! pre-commit run --all-files --verbose --color always ; then
        - rm -rf .cache
        - git checkout -f
        - pre-commit run --all-files --verbose --color always
        - fi
    variables:
        PRE_COMMIT_HOME: ${CI_PROJECT_DIR}/.cache/pre-commit
    cache:
        paths:
            - ${PRE_COMMIT_HOME}


# ======================================
#
#     Build Lib Standalone
#
# ======================================


build-standalone-ubu20:
  stage: build
  image: external-docker.artifactory.oxfordnanolabs.local/ubuntu:20.04
  needs:
    - "prepare_version"
  script:
    - export DEBIAN_FRONTEND=noninteractive
    - apt update
    - apt install -y -V ca-certificates lsb-release wget
    - wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
    - apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
    - apt update
    - apt install -y cmake build-essential libzstd-dev libzstd-dev libflatbuffers-dev libarrow-dev=12.0.1-1
    - mkdir -p build
    - cd build
    - cmake
      -D POD5_DISABLE_TESTS=OFF
      -D POD5_BUILD_EXAMPLES=ON
      -D BUILD_PYTHON_WHEEL=OFF
      ..
    - cmake --build .
    - ctest -C Release -VV


# ======================================
#
#     Build helpers
#
# ======================================


# Takes CMAKE_ARGS, AUDITWHEEL_PLATFORM, and PYTHON_VERSION.
# BUILD_PYTHON_WHEEL can optionally be disabled.
.conan-build-and-test:
  - pod5_version="$(cmake -P ci/get_tag_version.cmake 2>&1)"
  - mkdir -p build
  - cd build
  - conan install --profile ${CONAN_PROFILE} ..
  - cmake ${CMAKE_ARGS}
    -D BUILD_SHARED_LIB=ON
    -D CMAKE_BUILD_TYPE=Release
    -D POD5_DISABLE_TESTS=OFF
    -D POD5_BUILD_EXAMPLES=ON
    -D BUILD_PYTHON_WHEEL=OFF
    -D CMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake
    ..
  - cmake --build . --config Release
  - ctest -C Release -VV
  - ../ci/install.sh
  - cmake ${CMAKE_ARGS}
    -D BUILD_SHARED_LIB=OFF
    -D CMAKE_BUILD_TYPE=Release
    -D POD5_DISABLE_TESTS=OFF
    -D POD5_BUILD_EXAMPLES=ON
    -D BUILD_PYTHON_WHEEL=${BUILD_PYTHON_WHEEL:=ON}
    -D PYTHON_VERSION=${PYTHON_VERSION}
    -D CMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake
    ..
  - cmake --build . --config Release
  - ctest -C Release -VV
  - ../ci/install.sh STATIC_BUILD
  - ../ci/package.sh ${OUTPUT_SKU} ${AUDITWHEEL_PLATFORM}


# ======================================
#
#     Build Lib Linux
#
# ======================================


.build-linux:
  stage: build
  needs:
    - "prepare_version"
  retry: 1
  before_script:
    - /opt/python/cp310-cp310/bin/pip install -U pip 'conan<2' auditwheel build
    - ln -n /opt/python/cp310-cp310/bin/auditwheel /usr/bin/auditwheel
    - ln -n /opt/python/cp310-cp310/bin/conan /usr/bin/conan
    - conan config install --verify-ssl=no ${CONAN_CONFIG_URL}
  script:
    - !reference [".conan-build-and-test"]
  artifacts:
    name: "${CI_JOB_NAME}-artifacts"
    paths:
      - "lib_pod5*.tar.gz"
      - "lib_pod5*.whl"

linux-aarch64-gcc8-release-build:
  image: ${CI_REGISTRY}/ofan/ofan-docker-base/dorado/dorado-l4t-r32.4.3-deps:1.2
  extends: [".build-linux"]
  before_script:
    - conan config install --verify-ssl=no ${CONAN_CONFIG_URL}
  tags:
    - linux_aarch64
    - high-cpu
  variables:
    CONAN_PROFILE: "linux-aarch64-gcc8.jinja"
    CONAN_PROFILE_CPPSTD: 17
    OUTPUT_SKU: "linux-gcc8-arm64"
    BUILD_PYTHON_WHEEL: OFF

linux-x64-gcc9-release-build:
  image: quay.io/pypa/manylinux2014_x86_64
  extends: [".build-linux"]
  tags:
    - linux
  variables:
    CONAN_PROFILE: "linux-x86_64-gcc9.jinja"
    CONAN_PROFILE_CPPSTD: 17
    OUTPUT_SKU: "linux-x64"
    AUDITWHEEL_PLATFORM: manylinux2014_x86_64
  parallel:
    matrix:
      - PYTHON_VERSION: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]

linux-aarch64-gcc9-release-build:
  image: quay.io/pypa/manylinux2014_aarch64
  extends: [".build-linux"]
  tags:
    - linux_aarch64
    - high-cpu
  variables:
    CONAN_PROFILE: "linux-aarch64-gcc9.jinja"
    CONAN_PROFILE_CPPSTD: 17
    OUTPUT_SKU: "linux-arm64"
    AUDITWHEEL_PLATFORM: manylinux2014_aarch64
  parallel:
    matrix:
      - PYTHON_VERSION: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]


# ======================================
#
#     Build Lib OSX
#
# ======================================


.build-osx-common:
  stage: build
  needs:
    - "prepare_version"
  retry: 1
  before_script:
    - conan config install --verify-ssl=no "${CONAN_CONFIG_URL}"
    - ci/setup_python_osx.sh ${PYTHON_VERSION} ./python
    - export PATH="$(pwd)/python/Python.framework/Versions/Current/bin/:${PATH}"
    - python3 -m venv .venv/
    - source .venv/bin/activate
    - pip install --upgrade pip build
    - which python
    - python --version
  script:
    - python3 -c "import sysconfig; print(sysconfig.get_platform())"
    - !reference [".conan-build-and-test"]
  artifacts:
    name: "${CI_JOB_NAME}-artifacts"
    paths:
      - "lib_pod5*.tar.gz"
      - "lib_pod5*.whl"

osx-x64-clang14-release-build:
  extends: [".build-osx-common"]
  tags:
    - osx_x86
    - xcode-14.2
    - high-cpu
    - conan
  variables:
    CONAN_PROFILE: "macos-x86_64-appleclang-14.0.jinja"
    CONAN_PROFILE_CPPSTD: 20
  parallel:
    matrix:
      - PYTHON_VERSION: ["3.8.9", "3.9.13"]
        MACOSX_DEPLOYMENT_TARGET: "10.15"
        OUTPUT_SKU: "osx-10.15-x64"
      - PYTHON_VERSION: ["3.10.10", "3.11.2", "3.12.0", "3.13.0"]
        MACOSX_DEPLOYMENT_TARGET: "10.15"
        OUTPUT_SKU: "osx-10.15-x64"

osx-arm64-clang15-release-build:
  extends: [".build-osx-common"]
  tags:
    - osx_arm64
    - xcode-15.0.1
    - conan
  variables:
    CONAN_PROFILE: "macos-aarch64-appleclang-15.0.jinja"
    CONAN_PROFILE_CPPSTD: 20
    CMAKE_ARGS: "-DCMAKE_OSX_ARCHITECTURES=arm64"
  parallel:
    matrix:
      - PYTHON_VERSION: ["3.8.9", "3.9.13"]
        MACOSX_DEPLOYMENT_TARGET: "10.15"
        OUTPUT_SKU: "osx-10.15-arm64"
        FORCE_PYTHON_PLATFORM: macosx_11_0_arm64
      - PYTHON_VERSION: ["3.10.10", "3.11.2", "3.12.0", "3.13.0"]
        MACOSX_DEPLOYMENT_TARGET: "11.0"
        OUTPUT_SKU: "osx-11.0-arm64"
        FORCE_PYTHON_PLATFORM: macosx_11_0_arm64


# ======================================
#
#     Build Lib Windows
#
# ======================================


.build-win-common:
  stage: build
  needs:
    - "prepare_version"
  retry: 1
  before_script:
    - conan config install --verify-ssl=no "${CONAN_CONFIG_URL}"
    - c:/Python${PYTHON_VERSION}/python -m venv .venv
    - source .venv/Scripts/activate
  script:
    - pip install build
    - !reference [".conan-build-and-test"]
  after_script:
    # HACK: for some reason, pod5_unit_tests.exe is sticking around; deleting it works, but it
    # doesn't go away immediately (as though something had it open with FILE_SHARE_DELETE, although
    # the Handle utility from SysInternals couldn't find anything).
    - rm build/Release/bin/pod5_unit_tests.exe
    - date
    - while [[ -e build/Release/bin/pod5_unit_tests.exe ]]; do sleep 1; done
    - date

win-x64-msvc2019-release-build:
  extends: [".build-win-common"]
  tags:
    - windows
    - cmake
    - VS2019
    - conan
  variables:
    CONAN_PROFILE: "windows-x86_64-vs2019.jinja"
    CONAN_PROFILE_CPPSTD: 17
    OUTPUT_SKU: "win-x64"
    CMAKE_ARGS: "-A x64"
    CMAKE_GENERATOR: "Visual Studio 16 2019"
  parallel:
    matrix:
      - PYTHON_VERSION: ["38", "39", "310", "311", "312", "313"]
  artifacts:
    name: "${CI_JOB_NAME}-artifacts"
    paths:
      - "lib_pod5*.tar.gz"
      - "lib_pod5*.whl"

# ======================================
#
#     Build Python API
#
# ======================================


build-python-api:
  stage: build
  needs:
    - "prepare_version"
  image: ${CI_REGISTRY}/traque/ont-docker-base/ont-base-python:3.10
  tags:
    - linux
  retry: 1
  script:
    - git tag -d $(git tag -l "*a*")
    - git tag -d $(git tag -l "*b*")
    - git tag -d $(git tag -l "*r*")
    - git tag -d $(git tag -l "*c*")
    - git tag -d $(git tag -l "*dev*")
    - if [[ ${CI_COMMIT_TAG/#v/} && -z $( git tag -l "${CI_COMMIT_TAG/#v/}" ) ]]; then git tag ${CI_COMMIT_TAG/#v/}; fi
    - cat _version.py
    - VERSION=$(grep "__version__" _version.py | awk '{print $5}' | tr -d "'" | cut -d'+' -f1)
    - echo $VERSION
    - cd python/pod5/
    # update the lib_pod5 dependency in pod5/pyproject.toml to match
    - sed -i "s/.*lib_pod5.*/\ \ \ \ \'lib_pod5 == ${VERSION}\',/" pyproject.toml
    - cat pyproject.toml
    - pip install -U pip build
    - python -m build --outdir ../../
    - cd ../..
    - ls *.whl *.tar.gz
  artifacts:
    name: "${CI_JOB_NAME}-artifacts"
    paths:
      - "pod5*.whl"
      - "pod5*.tar.gz"


# ======================================
#
#     Test Tools
#
# ======================================


.parallel-py-minor-ver:
  parallel:
    matrix:
      - PY_MINOR_VER: ["8", "9", "10", "11", "12", "13"]

tools-linux-x64:
  extends: [".parallel-py-minor-ver"]
  stage: test
  image: ${CI_REGISTRY}/traque/ont-docker-base/ont-base-python:3.${PY_MINOR_VER}
  tags:
    - linux
  retry: 1
  before_script:
    - python3.${PY_MINOR_VER} -m venv .venv/
    - source .venv/bin/activate
  needs:
    - linux-x64-gcc9-release-build
    - build-python-api
  script:
    - pip install ./lib_pod5*cp3${PY_MINOR_VER}*.whl pod5-*.whl
    - pod5 convert fast5 ./test_data/ --output ./output_files --one-to-one ./test_data
    - python3.${PY_MINOR_VER} python/pod5/test_utils/check_pod5_files_equal.py ./output_files/multi_fast5_zip.pod5 ./test_data/multi_fast5_zip_v3.pod5
    - python3.${PY_MINOR_VER} python/pod5/test_utils/check_pod5_files_equal.py ./output_files/multi_fast5_zip.pod5 ./test_data/multi_fast5_zip_v2.pod5
    - python3.${PY_MINOR_VER} python/pod5/test_utils/check_pod5_files_equal.py ./output_files/multi_fast5_zip.pod5 ./test_data/multi_fast5_zip_v1.pod5
    - python3.${PY_MINOR_VER} python/pod5/test_utils/check_pod5_files_equal.py ./output_files/multi_fast5_zip.pod5 ./test_data/multi_fast5_zip_v0.pod5
    - pod5 convert to_fast5 ./output_files/ --output ./output_files
    - pod5 convert fast5 ./output_files/*.fast5 --output ./output_files_2 --one-to-one ./output_files/
    - python3.${PY_MINOR_VER} python/pod5/test_utils/check_pod5_files_equal.py ./output_files/multi_fast5_zip.pod5 ./output_files_2/*.pod5


# ======================================
#
#     Pytest
#
# ======================================


.pytest:
  extends: [".parallel-py-minor-ver"]
  stage: test
  retry: 1
  before_script:
    - python3.${PY_MINOR_VER} -m venv .venv/
    - source .venv/*/activate
    - python --version
    - python -m pip install --upgrade pip
  script:
    - pip install ./lib_pod5*cp3${PY_MINOR_VER}*.whl pod5-*.whl
    - pip install pytest pytest-cov pytest-mock psutil
    - pytest
    - POD5_DISABLE_MMAP_OPEN=1 pytest

pytest-linux-x64:
  extends: [".pytest"]
  image: ${CI_REGISTRY}/traque/ont-docker-base/ont-base-python:3.${PY_MINOR_VER}
  tags:
    - linux
  needs:
    - linux-x64-gcc9-release-build
    - build-python-api

pytest-linux-aarch64:
  extends: [".pytest"]
  image: ${CI_REGISTRY}/traque/ont-docker-base/ont-base-python:3.${PY_MINOR_VER}
  tags:
    - linux_aarch64
    - high-cpu
  needs:
    - linux-aarch64-gcc9-release-build
    - build-python-api

pytest-osx-x64:
  extends: [".pytest"]
  tags:
    - osx_x86
    - high-cpu
    - conan
  needs:
    - osx-x64-clang14-release-build
    - build-python-api

pytest-win-x64:
  extends: [".pytest"]
  tags:
    - windows
  needs:
    - win-x64-msvc2019-release-build
    - build-python-api


# ======================================
#
#     Conda Testing
#
# ======================================


conda_pytest:
  extends:
    - .pytest
    - .parallel-py-minor-ver
  image: ${CI_REGISTRY}/minknow/pod5-file-format/conda:latest
  tags:
    - linux
  needs:
    - linux-x64-gcc9-release-build
    - build-python-api
  before_script:
    - |
      cat > environment.yml << EOF
      name: pod5_conda_test
      channels:
        - conda-forge
        - bioconda
      dependencies:
        - python=3.${PY_MINOR_VER}
        - cmake
        - pyarrow
        - pip
      EOF
    - cat environment.yml
    - mamba --version
    - mamba env create -f environment.yml
    - conda env list
    # This is a work around for conda init in gitlab
    - eval "$(conda shell.bash hook)"
    - conda activate pod5_conda_test


# ======================================
#
#     Benchmarks
#
# ======================================


.benchmark:
  stage: test
  retry: 1
  before_script:
    - python3 -m venv .venv/
    - source .venv/bin/activate
  script:
    - pip install ./${LIB_WHEEL_GLOB} pod5-*.whl
    - pip install -r ./benchmarks/image/requirements-benchmarks.txt
    - export SLOW_5_VERSION=v1.0.0
    - ./benchmarks/image/install_slow5.sh
    - export PATH="$(pwd)/slow5tools-${SLOW_5_VERSION}/:$PATH"
    - ./benchmarks/run_benchmarks.py ./test_data/ ./benchmark-outputs

benchmark-linux-x64:
  extends: [".benchmark"]
  image: ${CI_REGISTRY}/traque/ont-docker-base/ont-base-python:3.9
  tags:
    - linux
  needs:
    - linux-x64-gcc9-release-build
    - build-python-api
  variables:
    LIB_WHEEL_GLOB: "lib_pod5*cp39*.whl"

benchmark-linux-aarch64:
  extends: [".benchmark"]
  image: ${CI_REGISTRY}/traque/ont-docker-base/ont-base-python:3.9
  tags:
    - linux_aarch64
    - high-cpu
  needs:
    - linux-aarch64-gcc9-release-build
    - build-python-api
  variables:
    LIB_WHEEL_GLOB: "lib_pod5*cp39*.whl"


# ======================================
#
#     Conan
#
# ======================================


.conan-build-common:
  retry: 1
  stage: build-conan
  dependencies:
    - "prepare_version"
  before_script:
    - conan remove "*" -f
    - conan config install --verify-ssl=no "${CONAN_CONFIG_URL}"
    - pip3 install 'conan<2'
    - cat cmake/POD5Version.cmake
    - VERSIONS="$(cmake -P ci/get_tag_version.cmake 2>&1)"
    - echo "Found versions ${VERSIONS}"
    # This is needed to enforce lf line-endings in the pybind submodule
    # otherwise conan generates different revisions for windows and unix
    - git rm -rf :/ && git checkout HEAD -- :/

.conan-upload:
  extends: .upload-package # from informatics/conan-config
  stage: deploy
  only: ["tags"]
  before_script:
    - apt update
    - apt install -y cmake
    - pip3 install 'conan<2'
    - cat cmake/POD5Version.cmake
    - VERSIONS="$(cmake -P ci/get_tag_version.cmake 2>&1)"
    - echo "Found versions ${VERSIONS}"
    # This is needed to enforce lf line-endings in the pybind submodule
    # otherwise conan generates different revisions for windows and unix
    - git rm -rf :/ && git checkout HEAD -- :/

  variables:
      EXPECTED_PACKAGE_COUNT: "4" # Expect shared and static packages

.tarball-package-without-macos: &tarball-package-without-macos
    # This is a copy of tarball-package in conan-config, except it removes macos packages first.
    # This is used because the iOS builds all copy a MacOS package into their uploads which is
    # pointless and just wastes space. Normally tarball-package is called from within build-package.
    # To change this, a caller still extends .build-package, but adds their own after_script
    # (cannot just modify the one in build-package, it must be completely replaced), and calls
    # tarball-package-without-macos instead of tarball-package. We can't call the original
    # tarball-package because it is private to conan-config.
    #
    # This also allows us to only archive the package we just built, and not any of its dependencies
    # (because we can use `conan inspect` to find the name of the right packages).
    #
    # Avoid storing things on the CI node unnecessarily
    - echo "Remove unneeded files"
    - conan remove "*" --builds --src --force
    # Remove unneeded MacOS packages
    - echo "Remove MacOS packages"
    - conan remove "flatbuffers/2.0.0@" -q "os=Macos" --force
    # Move required packages to a tarball
    - PACKAGE_DIR="${CONAN_USER_HOME#${PWD}/}/.conan/data/$(conan inspect --raw name ${CONAN_REFERENCE})"
    - echo "Packing from $PACKAGE_DIR"
    - tar -czvf "conan-${CI_JOB_ID}.tar.gz" "$PACKAGE_DIR"/*/${CONAN_USER}/${CONAN_CHANNEL}/{package,metadata.json}
    - rm -rf "${CONAN_USER_HOME}/.conan"

# Conan: build and upload packages:
build-conan:windows-x86_64-vs2019:
    extends:
        - .profile-windows-x86_64-vs2019
        - .build-package-win
        - .conan-build-common
        - .build-shared-and-static
    needs: ["prepare_version", "win-x64-msvc2019-release-build"]
upload-conan:windows-x86_64-vs2019:
    extends: .conan-upload
    dependencies: [ "prepare_version", "build-conan:windows-x86_64-vs2019" ]

build-conan:macos-x86_64-appleclang-14.0:
    extends:
        - .profile-macos-x86_64-appleclang-14.0
        - .build-package
        - .conan-build-common
        - .build-shared-and-static
    needs: ["prepare_version", "osx-x64-clang14-release-build"]
upload-conan:macos-x86_64-appleclang-14.0:
    extends: .conan-upload
    dependencies: [ "prepare_version", "build-conan:macos-x86_64-appleclang-14.0" ]

build-conan:macos-aarch64-appleclang-15.0:
    extends:
        - .profile-macos-aarch64-appleclang-15.0
        - .build-package
        - .conan-build-common
        - .build-shared-and-static
    needs: ["prepare_version", "osx-arm64-clang15-release-build"]
upload-conan:macos-aarch64-appleclang-15.0:
    extends: .conan-upload
    dependencies: [ "prepare_version", "build-conan:macos-aarch64-appleclang-15.0" ]

build-conan:ios-aarch64-appleclang-15.0:
    extends:
        - .profile-iOS-aarch64-appleclang-15.0
        - .build-package
        - .conan-build-common
        - .build-shared-and-static
    after_script:
        - *tarball-package-without-macos
    needs: ["prepare_version", "osx-arm64-clang15-release-build"] # No direct ios build stage to depend on, arm macos gives a good enough base
upload-conan:ios-aarch64-appleclang-15.0:
    extends: .conan-upload
    dependencies: [ "prepare_version", "build-conan:ios-aarch64-appleclang-15.0" ]

build-conan:iOSSimulator-aarch64-appleclang-15.0:
    extends:
        - .profile-iOSSimulator-aarch64-appleclang-15.0
        - .build-package
        - .conan-build-common
        - .build-shared-and-static
    after_script:
        - *tarball-package-without-macos
    needs: ["prepare_version", "osx-arm64-clang15-release-build"] # No direct ios build stage to depend on, arm macos gives a good enough base
upload-conan:iOSSimulator-aarch64-appleclang-15.0:
    extends: .conan-upload
    dependencies: [ "prepare_version", "build-conan:iOSSimulator-aarch64-appleclang-15.0" ]

build-conan:linux-x86_64-gcc9:
    extends:
        - .profile-linux-x86_64-gcc9
        - .build-package
        - .conan-build-common
        - .build-shared-and-static
    needs: ["prepare_version", "linux-x64-gcc9-release-build"]
upload-conan:linux-x86_64-gcc9:
    extends: .conan-upload
    dependencies: [ "prepare_version", "build-conan:linux-x86_64-gcc9" ]

build-conan:linux-x86_64-gcc11:
    extends:
        - .profile-linux-x86_64-gcc11
        - .build-package
        - .conan-build-common
        - .build-shared-and-static
    needs: ["prepare_version", "linux-x64-gcc9-release-build"]
upload-conan:linux-x86_64-gcc11:
    extends: .conan-upload
    dependencies: [ "prepare_version", "build-conan:linux-x86_64-gcc11" ]

build-conan:linux-aarch64-gcc9:
    extends:
        - .profile-linux-aarch64-gcc9
        - .build-package
        - .conan-build-common
        - .build-shared-and-static
    needs: ["prepare_version", "linux-aarch64-gcc9-release-build"]
upload-conan:linux-aarch64-gcc9:
    extends: .conan-upload
    dependencies: [ "prepare_version", "build-conan:linux-aarch64-gcc9" ]

build-conan:linux-aarch64-gcc11:
    extends:
        - .profile-linux-aarch64-gcc11
        - .build-package
        - .conan-build-common
        - .build-shared-and-static
    needs: ["prepare_version", "linux-aarch64-gcc9-release-build"]
upload-conan:linux-aarch64-gcc11:
    extends: .conan-upload
    dependencies: [ "prepare_version", "build-conan:linux-aarch64-gcc11" ]


# ======================================
#
#     Archive
#
# ======================================


build-archive:
  stage: archive
  needs:
    - linux-x64-gcc9-release-build
    - linux-aarch64-gcc9-release-build
    - linux-aarch64-gcc8-release-build
    - osx-x64-clang14-release-build
    - osx-arm64-clang15-release-build
    - win-x64-msvc2019-release-build
    - build-python-api
  script:
    - find .
  artifacts:
    name: "${CI_JOB_NAME}-artifacts"
    paths:
      - ./*.tar.gz
      - ./*.whl


# ======================================
#
#     Deploy
#
# ======================================


internal_wheel_upload:
  stage: deploy
  image: ${UPLOAD_PYTHON_IMAGE}
  needs:
    - build-archive
  script:
    - ls -lh .
    - pip install twine
    - twine upload *.whl pod5*.tar.gz
  only: ["tags"]
  when: manual

external_wheel_upload:
  stage: deploy
  image: ${UPLOAD_PYTHON_IMAGE}
  needs:
    - build-archive
  script:
    - ls -lh .
    - pip install twine
    - unset TWINE_REPOSITORY_URL
    - unset TWINE_CERT
    - twine upload lib*.whl -u __token__ -p"${EXTERNAL_LIB_POD5_PYPI_KEY}"
    - twine upload pod5*.whl pod5*.tar.gz -u __token__ -p"${EXTERNAL_POD5_PYPI_KEY}"
  only: ["tags"]
  when: manual


# ======================================
#
#     MLHub Testing
#
# ======================================

mlhub:
  stage: deploy
  image: ${MLHUB_TRIGGER_IMAGE}
  needs: ["build-archive"]
  variables:
    GIT_STRATEGY: none
  script:
    - |
      curl -i --header "Content-Type: application/json" \
      --request POST \
      --data '{
          "key": "'${MLHUB_TRIGGER_KEY}'",
          "job_name": "POD5-CI '${CI_COMMIT_REF_NAME}' - '"$CI_COMMIT_TITLE"' ",
          "script_parameters": {
                "mode":"artifact",
                "source":"'${CI_COMMIT_SHA}'"
                "python_ver":"'${PYTHON_VERSION}'"
              }
      }' \
      ${MLHUB_TRIGGER_URL}
  when: manual
  parallel:
    matrix:
      - PYTHON_VERSION:
        - "3.8"
        - "3.12"


# ======================================
#
#     Pages
#
# ======================================


pages:
  stage: deploy
  image: ${CI_REGISTRY}/traque/ont-docker-base/ont-base-python:3.10
  needs: []
  tags:
    - linux
  script:
    - make -f docs/Makefile all
    - mv _build/html public
  artifacts:
    paths:
      - public
  when: manual