diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d7a957d82..4a7a489e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,14 +3,14 @@ name: LAPIS-SILO on: push: - concurrency: group: ci-${{ github.ref }} cancel-in-progress: true env: - DOCKER_IMAGE_NAME: ghcr.io/GenSpectrum/LAPIS-SILO - DOCKER_TEST_IMAGE_NAME: silo/unittests + DOCKER_DEPENDENCY_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo-dependencies + DOCKER_LINTER_DEPENDENCY_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo-linter-dependencies + DOCKER_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo jobs: formatting-check: @@ -31,56 +31,172 @@ jobs: check-path: ${{ matrix.path['check'] }} exclude-regex: ${{ matrix.path['exclude'] }} - linter: - name: Build And Run linter + linterDependencies: + name: Build linter dependencies runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Generate dependency files hash + id: files-hash + run: | + DIR_HASH=$(echo -n ${{ hashFiles('conanfile.py', 'conanprofile.docker', '.github/workflows/ci.yml', './Dockerfile_linter_dependencies') }}) + echo "DIR_HASH=$DIR_HASH" >> $GITHUB_ENV + + - name: Docker metadata + id: dockerMetadata + uses: docker/metadata-action@v5 + with: + images: ${{ env.DOCKER_LINTER_DEPENDENCY_IMAGE_NAME }} + tags: | + type=ref,event=branch + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} + type=raw,value=${{ env.DIR_HASH }} + type=sha,prefix=commit- + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Check if image exists + id: check-image + run: | + EXISTS=$(docker manifest inspect ${{ env.DOCKER_LINTER_DEPENDENCY_IMAGE_NAME }}:${{ env.DIR_HASH }} > /dev/null 2>&1 && echo "true" || echo "false") + echo "CACHE_HIT=$EXISTS" >> $GITHUB_ENV + - name: Set up Docker Buildx + if: env.CACHE_HIT == 'false' uses: docker/setup-buildx-action@v3 - name: Build linter dependencies image + if: env.CACHE_HIT == 'false' uses: docker/build-push-action@v5 with: context: . - target: linter_dependencies - tags: linter_dependencies - file: ./Dockerfile_linter + push: true + tags: ${{ steps.dockerMetadata.outputs.tags }} + file: ./Dockerfile_linter_dependencies cache-from: type=gha,ref=linter-dependencies-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile_linter') }} cache-to: type=gha,mode=min,ref=linter-dependencies-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile_linter') }} + linter: + name: Build And Run linter + needs: linterDependencies + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Generate dependency files hash + id: files-hash + run: | + DIR_HASH=$(echo -n ${{ hashFiles('conanfile.py', 'conanprofile.docker', '.github/workflows/ci.yml', './Dockerfile_linter_dependencies') }}) + echo "DIR_HASH=$DIR_HASH" >> $GITHUB_ENV + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build linter image uses: docker/build-push-action@v5 with: context: . file: ./Dockerfile_linter - target: linter cache-from: type=gha,ref=${{ github.ref_name }}-linter-image-cache cache-to: type=gha,mode=min,ref=${{ github.ref_name }}-linter-image-cache push: false + build-args: | + DEPENDENCY_IMAGE=ghcr.io/genspectrum/lapis-silo-linter-dependencies:${{ env.DIR_HASH }} - dockerImage: - name: Build And Run Unit Tests + dependencyImage: + name: Build Docker Image Dependencies runs-on: ubuntu-latest permissions: packages: write steps: - uses: actions/checkout@v4 + - name: Generate dependency files hash + id: files-hash + run: | + DIR_HASH=$(echo -n ${{ hashFiles('conanfile.py', 'conanprofile.docker', '.github/workflows/ci.yml', './Dockerfile_dependencies') }}) + echo "DIR_HASH=$DIR_HASH" >> $GITHUB_ENV + + - name: Docker metadata + id: dockerMetadata + uses: docker/metadata-action@v5 + with: + images: ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }} + tags: | + type=ref,event=branch + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} + type=raw,value=${{ env.DIR_HASH }} + type=sha,prefix=commit- + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Check if image exists + id: check-image + run: | + EXISTS=$(docker manifest inspect ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:${{ env.DIR_HASH }} > /dev/null 2>&1 && echo "true" || echo "false") + echo "CACHE_HIT=$EXISTS" >> $GITHUB_ENV + - name: Set up Docker Buildx + if: env.CACHE_HIT == 'false' uses: docker/setup-buildx-action@v3 - - name: Build builder image + - name: Build and push image if input files changed + if: env.CACHE_HIT == 'false' uses: docker/build-push-action@v5 with: context: . - target: dep_builder - tags: dep_builder - load: true - file: ./Dockerfile + file: Dockerfile_dependencies + push: true + tags: ${{ steps.dockerMetadata.outputs.tags }} cache-from: type=gha,ref=builder-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile') }} cache-to: type=gha,mode=min,ref=builder-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile') }} + platforms: ${{ github.ref == 'refs/heads/main' && 'linux/amd64,linux/arm64' || 'linux/amd64' }} + + dockerImageUnitTests: + name: Build Docker Image and Run Unit Tests + runs-on: ubuntu-latest + needs: dependencyImage + permissions: + packages: write + steps: + - uses: actions/checkout@v4 + + - name: Generate dependency files hash + id: files-hash + run: | + DIR_HASH=$(echo -n ${{ hashFiles('conanfile.py', 'conanprofile.docker', '.github/workflows/ci.yml', './Dockerfile_dependencies') }}) + echo "DIR_HASH=$DIR_HASH" >> $GITHUB_ENV + + - name: Docker metadata + id: dockerMetadata + uses: docker/metadata-action@v5 + with: + images: ${{ env.DOCKER_IMAGE_NAME }} + tags: | + type=ref,event=branch + type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }} + type=sha,prefix=commit- + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - name: Build unit test image uses: docker/build-push-action@v5 @@ -91,6 +207,8 @@ jobs: load: true cache-from: type=gha,ref=${{ github.ref_name }}-image-cache cache-to: type=gha,mode=min,ref=${{ github.ref_name }}-image-cache + build-args: | + DEPENDENCY_IMAGE=ghcr.io/genspectrum/lapis-silo-dependencies:${{ env.DIR_HASH }} - name: Run tests uses: addnab/docker-run-action@v3 @@ -98,22 +216,6 @@ jobs: image: builder run: ./silo_test - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Docker metadata - id: dockerMetadata - uses: docker/metadata-action@v5 - with: - images: ${{ env.DOCKER_IMAGE_NAME }} - tags: | - type=ref,event=branch - type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }} - - name: Build and push production image if: ${{ github.event_name == 'push' }} uses: docker/build-push-action@v5 @@ -122,9 +224,12 @@ jobs: push: true platforms: ${{ github.ref == 'refs/heads/main' && 'linux/amd64,linux/arm64' || 'linux/amd64' }} tags: ${{ steps.dockerMetadata.outputs.tags }} + build-args: | + DEPENDENCY_IMAGE=ghcr.io/genspectrum/lapis-silo-dependencies:${{ env.DIR_HASH }} endToEndTests: name: Run End To End Tests + needs: dockerImageUnitTests runs-on: ubuntu-latest strategy: matrix: @@ -158,13 +263,6 @@ jobs: images: ${{ env.DOCKER_IMAGE_NAME }} tags: type=ref,event=branch - - name: Wait for Docker Image - uses: lewagon/wait-on-check-action@v1.3.3 - with: - ref: ${{ github.ref }} - check-name: Build And Run Unit Tests - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Start Docker Container and preprocess data run: docker compose -f ${{ matrix.preprocessing-docker-compose }} up env: diff --git a/Dockerfile b/Dockerfile index ec0369174..df7b0b3c0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,28 +1,6 @@ -FROM alpine:3.18 AS dep_builder +ARG DEPENDENCY_IMAGE=ghcr.io/genspectrum/lapis-silo-dependencies:latest -ARG TARGETPLATFORM - -RUN apk update && apk add --no-cache py3-pip \ - build-base=0.5-r3 \ - cmake=3.26.5-r0 \ - bash=5.2.15-r5 \ - linux-headers=6.3-r0 \ - boost-build=1.82.0-r0 \ - libtbb=2021.9.0-r0 - -RUN pip install conan==2.0.17 - -WORKDIR /src -COPY conanfile.py conanprofile.docker conanprofile.docker_arm ./ -RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \ - mv conanprofile.docker_arm conanprofile; \ - else \ - mv conanprofile.docker conanprofile; \ - fi - -RUN conan install . --build=missing --profile ./conanprofile --profile:build ./conanprofile --output-folder=build - -FROM dep_builder AS builder +FROM $DEPENDENCY_IMAGE AS builder COPY . ./ diff --git a/Dockerfile_dependencies b/Dockerfile_dependencies new file mode 100644 index 000000000..08a850299 --- /dev/null +++ b/Dockerfile_dependencies @@ -0,0 +1,23 @@ +FROM alpine:3.18 + +ARG TARGETPLATFORM + +RUN apk update && apk add --no-cache py3-pip \ + build-base=0.5-r3 \ + cmake=3.26.5-r0 \ + bash=5.2.15-r5 \ + linux-headers=6.3-r0 \ + boost-build=1.82.0-r0 \ + libtbb=2021.9.0-r0 + +RUN pip install conan==2.0.17 + +WORKDIR /src +COPY conanfile.py conanprofile.docker conanprofile.docker_arm ./ +RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \ + mv conanprofile.docker_arm conanprofile; \ + else \ + mv conanprofile.docker conanprofile; \ + fi + +RUN conan install . --build=missing --profile ./conanprofile --profile:build ./conanprofile --output-folder=build diff --git a/Dockerfile_linter b/Dockerfile_linter index 8045b5d8c..2c78d39e7 100644 --- a/Dockerfile_linter +++ b/Dockerfile_linter @@ -1,28 +1,6 @@ -FROM ubuntu:22.04 AS linter_dependencies +ARG DEPENDENCY_IMAGE=ghcr.io/genspectrum/lapis-silo-linter-dependencies:latest -WORKDIR /src - -RUN apt update \ - && apt install -y \ - cmake=3.22.1-1ubuntu1.22.04.2 \ - python3-pip=22.0.2+dfsg-1ubuntu0.4 \ - software-properties-common=0.99.22.9 \ - wget=1.21.2-2ubuntu1 \ - gnupg=2.2.27-3ubuntu2.1 \ - lsb-release=11.1.0ubuntu4 \ - && wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc \ - && add-apt-repository 'deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy main' \ - && apt install -y clang-tidy-19 - -RUN pip install conan==2.0.17 - -COPY conanfile.py conanprofile.docker ./ -RUN mv conanprofile.docker conanprofile - -RUN conan install . --build=missing --profile ./conanprofile --profile:build ./conanprofile --output-folder=build -s build_type=Debug - - -FROM linter_dependencies AS linter +FROM $DEPENDENCY_IMAGE AS linter WORKDIR /src diff --git a/Dockerfile_linter_dependencies b/Dockerfile_linter_dependencies new file mode 100644 index 000000000..25c183a21 --- /dev/null +++ b/Dockerfile_linter_dependencies @@ -0,0 +1,22 @@ +FROM ubuntu:22.04 + +WORKDIR /src + +RUN apt update \ + && apt install -y \ + cmake=3.22.1-1ubuntu1.22.04.2 \ + python3-pip=22.0.2+dfsg-1ubuntu0.4 \ + software-properties-common=0.99.22.9 \ + wget=1.21.2-2ubuntu1 \ + gnupg=2.2.27-3ubuntu2.1 \ + lsb-release=11.1.0ubuntu4 \ + && wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc \ + && add-apt-repository 'deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy main' \ + && apt install -y clang-tidy-19 + +RUN pip install conan==2.0.17 + +COPY conanfile.py conanprofile.docker ./ +RUN mv conanprofile.docker conanprofile + +RUN conan install . --build=missing --profile ./conanprofile --profile:build ./conanprofile --output-folder=build -s build_type=Debug