From 7577c6c0c260e98c7d156f734de49d382a49f1dd Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Mon, 9 Sep 2024 11:06:59 +0100 Subject: [PATCH] Fix multistage build --- .github/workflows/testing.yml | 14 +++- .../{Dockerfile.multistage => Dockerfile} | 2 +- tests/integration/dockerfiles/Dockerfile.sge | 75 ------------------- .../integration/dockerfiles/Dockerfile.slurm | 61 --------------- tests/integration/dockerfiles/docker-bake.hcl | 4 +- 5 files changed, 16 insertions(+), 140 deletions(-) rename tests/integration/dockerfiles/{Dockerfile.multistage => Dockerfile} (98%) delete mode 100644 tests/integration/dockerfiles/Dockerfile.sge delete mode 100644 tests/integration/dockerfiles/Dockerfile.slurm diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 949c7425..1a1154ad 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -46,7 +46,7 @@ jobs: - name: Build test bench container uses: docker/bake-action@v5 with: - load: true + load: true files: tests/integration/dockerfiles/docker-bake.hcl set: | *.cache-to=type=gha,scope=global,mode=max @@ -59,6 +59,7 @@ jobs: ports: - 27017:27017 + needs: [build-test-bench] runs-on: ubuntu-latest strategy: fail-fast: false @@ -74,6 +75,17 @@ jobs: cache: pip cache-dependency-path: pyproject.toml + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Load test bench containers + uses: docker/bake-action@v5 + with: + load: true + files: tests/integration/dockerfiles/docker-bake.hcl + set: | + *.cache-from=type=gha,scope=global + - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/tests/integration/dockerfiles/Dockerfile.multistage b/tests/integration/dockerfiles/Dockerfile similarity index 98% rename from tests/integration/dockerfiles/Dockerfile.multistage rename to tests/integration/dockerfiles/Dockerfile index 09d710df..ed86beaf 100644 --- a/tests/integration/dockerfiles/Dockerfile.multistage +++ b/tests/integration/dockerfiles/Dockerfile @@ -91,4 +91,4 @@ RUN mkdir jfr USER root RUN chmod +x /etc/startup.sh && dos2unix /etc/startup.sh USER ${USERNAME} -CMD "sudo /etc/startup.sh ; /bin/bash -l" +CMD sudo /etc/startup.sh ; /bin/bash -l diff --git a/tests/integration/dockerfiles/Dockerfile.sge b/tests/integration/dockerfiles/Dockerfile.sge deleted file mode 100644 index e4f04554..00000000 --- a/tests/integration/dockerfiles/Dockerfile.sge +++ /dev/null @@ -1,75 +0,0 @@ -# syntax=docker/dockerfile:experimental - -# -== SGE ==- -FROM ubuntu:22.04 as sge -USER root -## The SGE startup script includes the configuration of SGE, as the docker network is not ready during build -COPY ./tests/integration/dockerfiles/sge_startup.sh /etc/startup.sh - -## Following instructions from https://github.com/daimh/sge/blob/817a7fa019f0ec47425f43e843a1f58865614313/README.md -ARG SGE_USERNAME=sge -RUN apt update && apt install -y sudo build-essential curl cmake git libdb5.3-dev libhwloc-dev libmotif-dev libncurses-dev libpam0g-dev libssl-dev libsystemd-dev libtirpc-dev libxext-dev pkgconf && apt clean && rm -rf /var/lib/apt/lists/* -WORKDIR /tmp -RUN git clone https://github.com/daimh/sge --depth 1 --branch master sge -WORKDIR /tmp/sge -RUN cmake -S . -B build -DCMAKE_INSTALL_PREFIX=/opt/${SGE_USERNAME} -RUN cmake --build build -j -RUN cmake --install build -RUN useradd -rm -d /opt/${SGE_USERNAME} -s /bin/bash ${SGE_USERNAME} -RUN chown -R sge /opt/${SGE_USERNAME} -WORKDIR /opt/${SGE_USERNAME} - -ARG USERNAME=jobflow - -WORKDIR /opt -USER root - -# 3) Run an SSH server and set up Python environment and user for jobflow -# Install OpenSSH server and set it to run on startup -RUN apt update && apt install -y dos2unix openssh-server sudo python3.10-venv && apt clean && rm -rf /var/lib/apt/lists/* -RUN sed -i 's/#PasswordAuthentication no/PasswordAuthentication yes/g' /etc/ssh/sshd_config - -# Create desired user with blank password then give user access to startup script as sudo without password -# See https://github.com/nathan-hess/docker-slurm/blob/a62133d66d624d9ff0ccefbd41a0b1b2abcb9925/dockerfile_base/Dockerfile#L62C1-L65C1 -RUN useradd -rm -d /home/${USERNAME} -s /bin/bash ${USERNAME} && usermod -a -G sudo ${USERNAME} -RUN echo ${USERNAME}:${USERNAME} | chpasswd -RUN printf "${USERNAME} ALL=(root:root) NOPASSWD: /etc/startup.sh\n" >> /etc/sudoers.d/startup \ - && chmod 0440 /etc/sudoers.d/startup \ - && visudo -c - - -# Install jobflow from directory, assuming container -# is built at the root of the jobflow repo -WORKDIR /home/${USERNAME} -RUN mkdir jobflow-remote -COPY src/jobflow_remote jobflow-remote/src/jobflow_remote -COPY pyproject.toml jobflow-remote/ - -# versioningit refuses to install a package without its full git history -# so here we remove versioningit config from pyproject.toml as we don't need -# the full version number (which allows us to cache many more layers) -RUN sed -i '/\[tool.versioningit.vcs\]/,+3d' jobflow-remote/pyproject.toml - -# Annoyingly we want to use this with the Python SDK -# which does not support buildkit yet -# so cannot use --chmod in the copy directly and -# we have to become root for this step -USER root -RUN sudo chmod -R 0777 jobflow-remote -USER ${USERNAME} - -# Install jobflow in a local native virtualenv -WORKDIR /home/${USERNAME}/jobflow-remote -RUN python3 -m venv /home/${USERNAME}/.venv -RUN /home/${USERNAME}/.venv/bin/pip install -U pip -RUN /home/${USERNAME}/.venv/bin/pip install --verbose -e . - -# Make a job directory for jobflow -WORKDIR /home/${USERNAME} -RUN mkdir jfr - -# Finalize permissions on startup script -USER root -RUN chmod +x /etc/startup.sh && dos2unix /etc/startup.sh -USER ${USERNAME} -CMD "sudo /etc/startup.sh ; /bin/bash -l" diff --git a/tests/integration/dockerfiles/Dockerfile.slurm b/tests/integration/dockerfiles/Dockerfile.slurm deleted file mode 100644 index 365840ce..00000000 --- a/tests/integration/dockerfiles/Dockerfile.slurm +++ /dev/null @@ -1,61 +0,0 @@ -# syntax=docker/dockerfile:experimental -FROM nathanhess/slurm:full AS slurm -USER root -COPY ./tests/integration/dockerfiles/slurm_startup.sh /etc/startup.sh - -# 2) Add common options from ubuntu server - -ARG USERNAME=jobflow - -WORKDIR /opt -USER root - -# 3) Run an SSH server and set up Python environment and user for jobflow -# Install OpenSSH server and set it to run on startup -RUN apt update && apt install -y dos2unix openssh-server sudo python3.10-venv && apt clean && rm -rf /var/lib/apt/lists/* -RUN sed -i 's/#PasswordAuthentication no/PasswordAuthentication yes/g' /etc/ssh/sshd_config - -# Create desired user with blank password then give user access to startup script as sudo without password -# See https://github.com/nathan-hess/docker-slurm/blob/a62133d66d624d9ff0ccefbd41a0b1b2abcb9925/dockerfile_base/Dockerfile#L62C1-L65C1 -RUN useradd -rm -d /home/${USERNAME} -s /bin/bash ${USERNAME} && usermod -a -G sudo ${USERNAME} -RUN echo ${USERNAME}:${USERNAME} | chpasswd -RUN printf "${USERNAME} ALL=(root:root) NOPASSWD: /etc/startup.sh\n" >> /etc/sudoers.d/startup \ - && chmod 0440 /etc/sudoers.d/startup \ - && visudo -c - - -# Install jobflow from directory, assuming container -# is built at the root of the jobflow repo -WORKDIR /home/${USERNAME} -RUN mkdir jobflow-remote -COPY src/jobflow_remote jobflow-remote/src/jobflow_remote -COPY pyproject.toml jobflow-remote/ - -# versioningit refuses to install a package without its full git history -# so here we remove versioningit config from pyproject.toml as we don't need -# the full version number (which allows us to cache many more layers) -RUN sed -i '/\[tool.versioningit.vcs\]/,+3d' jobflow-remote/pyproject.toml - -# Annoyingly we want to use this with the Python SDK -# which does not support buildkit yet -# so cannot use --chmod in the copy directly and -# we have to become root for this step -USER root -RUN sudo chmod -R 0777 jobflow-remote -USER ${USERNAME} - -# Install jobflow in a local native virtualenv -WORKDIR /home/${USERNAME}/jobflow-remote -RUN python3 -m venv /home/${USERNAME}/.venv -RUN /home/${USERNAME}/.venv/bin/pip install -U pip -RUN /home/${USERNAME}/.venv/bin/pip install --verbose -e . - -# Make a job directory for jobflow -WORKDIR /home/${USERNAME} -RUN mkdir jfr - -# Finalize permissions on startup script -USER root -RUN chmod +x /etc/startup.sh && dos2unix /etc/startup.sh -USER ${USERNAME} -CMD sudo /etc/startup.sh ; /bin/bash -l diff --git a/tests/integration/dockerfiles/docker-bake.hcl b/tests/integration/dockerfiles/docker-bake.hcl index 4dcdffed..4ea10cef 100644 --- a/tests/integration/dockerfiles/docker-bake.hcl +++ b/tests/integration/dockerfiles/docker-bake.hcl @@ -6,7 +6,7 @@ group "default" { } target "slurm" { - dockerfile = "./tests/integration/dockerfiles/Dockerfile.slurm" + dockerfile = "./tests/integration/dockerfiles/Dockerfile" args = { QUEUE_SYSTEM = "slurm" } @@ -16,7 +16,7 @@ target "slurm" { } target "sge" { - dockerfile = "./tests/integration/dockerfiles/Dockerfile.sge" + dockerfile = "./tests/integration/dockerfiles/Dockerfile" args = { QUEUE_SYSTEM = "sge" }