Merge pull request wtsi-npg#356 from kjsanger/feature/tests-in-docker

Add simplified Docker Compose support for tests

Dockerfile

@@ -1,11 +1,11 @@
-# syntax = docker/dockerfile:1.2
 
-FROM ubuntu:bionic as builder
+FROM --platform=linux/amd64 ubuntu:bionic AS builder
 
 ARG PYTHON_VERSION=3.12
 
-RUN echo "debconf debconf/frontend select Noninteractive" | debconf-set-selections && \
-    apt-get update && \
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && \
     apt-get install -q -y --no-install-recommends \
     autoconf \
     automake \
@@ -24,14 +24,14 @@ RUN echo "debconf debconf/frontend select Noninteractive" | debconf-set-selectio
 
 WORKDIR /app
 
-COPY . .
-
-ENV PYENV_ROOT "/app/.pyenv"
+ENV PYENV_ROOT="/app/.pyenv"
 
 # Put PYENV first to ensure we use the pyenv-installed Python
-ENV PATH "${PYENV_ROOT}/shims:${PYENV_ROOT}/bin:/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin"
+ENV PATH="${PYENV_ROOT}/shims:${PYENV_ROOT}/bin:/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin"
 
-RUN ./docker/install_pyenv.sh
+COPY . /app
+
+RUN /app/docker/install_pyenv.sh
 
 RUN pyenv install "$PYTHON_VERSION"
 RUN pyenv global "$PYTHON_VERSION"
@@ -40,22 +40,23 @@ RUN pyenv global "$PYTHON_VERSION"
 # available for anything more recent than Ubuntu bionic, so that's what we use for
 # the builder (above) and for the clients. This is also the reason we resort to
 # pyenv to get a recent Python, rather than using a python-slim base image.
-FROM ghcr.io/wtsi-npg/ub-18.04-irods-clients-4.2.11
+FROM --platform=linux/amd64 ghcr.io/wtsi-npg/ub-18.04-irods-clients-4.2.11
+
+ENV DEBIAN_FRONTEND=noninteractive
 
-RUN echo "debconf debconf/frontend select Noninteractive" | debconf-set-selections && \
-    apt-get update && \
+RUN apt-get update && \
     apt-get install -q -y --no-install-recommends \
     ca-certificates \
     git \
     unattended-upgrades && \
     unattended-upgrade -d -v
 
-ENV PYENV_ROOT "/app/.pyenv"
+ENV PYENV_ROOT="/app/.pyenv"
 
 # Put PYENV first to ensure we use the pyenv-installed Python
-ENV PATH "${PYENV_ROOT}/shims:${PYENV_ROOT}/bin:/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin"
-ENV PYTHONUNBUFFERED 1
-ENV PYTHONPATH ""
+ENV PATH="${PYENV_ROOT}/shims:${PYENV_ROOT}/bin:/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin"
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONPATH=""
 
 WORKDIR /app
 
@@ -78,6 +79,6 @@ RUN apt-get remove -q -y unattended-upgrades \
 
 USER appuser
 
-ENTRYPOINT ["/app/docker/docker-entrypoint.sh"]
+ENTRYPOINT ["/app/docker/entrypoint.sh"]
 
 CMD ["check-checksums", "--version"]

Dockerfile.dev

@@ -0,0 +1,66 @@
+FROM ghcr.io/wtsi-npg/ub-18.04-baton-irods-4.2.11:latest
+
+ARG PYTHON_VERSION=3.12
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+USER root
+
+RUN apt-get update && \
+    apt-get install -q -y --no-install-recommends \
+    apt-transport-https \
+    apt-utils \
+    build-essential \
+    ca-certificates \
+    curl \
+    gcc \
+    git \
+    make \
+    libbz2-dev \
+    libncurses-dev \
+    libreadline-dev \
+    libssl-dev \
+    zlib1g-dev
+
+# Install the iRODS icommands package because it's useful for interactions with \
+# the server during development
+RUN echo "deb [arch=amd64] https://packages.irods.org/apt/ $(lsb_release -sc) main" |\
+    tee /etc/apt/sources.list.d/renci-irods.list && \
+    apt-get update && \
+    apt-get install -q -y --no-install-recommends \
+    irods-icommands="4.2.11-1~$(lsb_release -sc)"
+
+WORKDIR /app
+
+# It's more practical to build from an iRODS client image and install recent Python
+# than to build from a recent Python image and install iRODS clients.
+ENV PYENV_ROOT="/app/.pyenv"
+
+# Put PYENV first to ensure we use the pyenv-installed Python
+ENV PATH="${PYENV_ROOT}/shims:${PYENV_ROOT}/bin:/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin"
+
+COPY ./docker/install_pyenv.sh /app/docker/install_pyenv.sh
+
+RUN /app/docker/install_pyenv.sh
+
+RUN pyenv install "$PYTHON_VERSION"
+RUN pyenv global "$PYTHON_VERSION"
+
+COPY requirements.txt test-requirements.txt /app/
+
+RUN pip install --no-cache-dir -r requirements.txt && \
+    pip install --no-cache-dir -r test-requirements.txt
+
+COPY . /app/
+
+RUN pip install --no-cache-dir . && \
+    git status && \
+    ls -al
+
+RUN chown -R appuser:appuser /app
+
+USER appuser
+
+ENTRYPOINT ["/app/docker/entrypoint.sh"]
+
+CMD ["/bin/bash", "-c", "sleep infinity"]

README.md

@@ -38,16 +38,37 @@ docker pull ghcr.io/wtsi-npg/npg-irods-python:latest
 docker pull ghcr.io/wtsi-npg/npg-irods-python:1.1.0
 ```
 
-## Building and testing 
+## Running tests
 
-The tests require some services to be running (iRODS and possibly MySQL). See the
-`docker-compose.yml` file for an example of how to achieve this using our Docker images. 
+### Running directly on your local machine
 
-```commandline
-pip install -r requirements.txt
-pip install -r test-requirements.txt
-pytest --it
-```
+To run the tests locally, you will need to have the `irods` clients installed (`icommands`
+and `baton`, which means your local machine must be either be running Linux, or have
+containerised versions of these tools installed and runnable via proxy wrappers of the
+same name, to emulate the Linux environment.
+
+You will also need to have a working iRODS server to connect to.
+
+With this in place, you can run the tests with the following command:
+
+    pytest --it
+
+### Running in a container
+
+The tests can be run in a container, which requires less setup and will be less likely
+to be affected by your local environment. A Docker Compose file is provided to run the
+tests in a Linux container, against containerised iRODS and MySQL servers.
+
+To run the tests in a container, you will need to have Docker installed.
+
+With this in place, you can run the tests with the following command:
+
+    docker-compose run app pytest --it
+
+There will be a delay the first time this is run because the Docker image will be built.
+To pre-build the image, you can run:
+
+    docker-compose build
 
 ## Logging
 

docker-compose.yml

@@ -17,6 +17,7 @@ services:
       retries: 10
 
   irods-server:
+    platform: linux/amd64
     container_name: irods-server
     image: "ghcr.io/wtsi-npg/ub-16.04-irods-4.2.7:latest"
     ports:
@@ -30,18 +31,19 @@ services:
       timeout: 10s
       retries: 12
 
-  irods-clients:
-    container_name: irods-clients
-    image: "ghcr.io/wtsi-npg/ub-16.04-irods-clients-4.2.7:latest"
+  app:
+    platform: linux/amd64
+    build:
+      context: .
+      dockerfile: Dockerfile.dev
+    restart: always
     volumes:
-      - "${PWD}:${PWD}"
-      - "${PWD}/tests/.irods:${HOME}/.irods/"
+      - "./tests/.irods:/home/appuser/.irods/"
     environment:
-      CLIENT_USER: "${USER:? ERROR: The USER environment variable is unset}"
-      CLIENT_USER_ID: "${UID:? ERROR: The UID environment variable is unset}"
-      CLIENT_USER_HOME: "${HOME}"
-      IRODS_ENVIRONMENT_FILE: "${HOME}/.irods/irods_environment.json"
-    command: sleep infinity
+      IRODS_ENVIRONMENT_FILE: "/home/appuser/.irods/irods_environment.json"
+      IRODS_PASSWORD: "irods"
     depends_on:
       irods-server:
         condition: service_healthy
+      mysql-server:
+        condition: service_healthy

pyproject.toml

@@ -52,7 +52,7 @@ repository = "https://github.com/wtsi-npg/npg-irods-python.git"
 "write-html-report" = "npg_irods.cli.write_html_report:main"
 
 [build-system]
-requires = ["setuptools>=41", "wheel", "setuptools-git-versioning<2"]
+requires = ["setuptools>=41", "wheel", "setuptools-git-versioning>=2.0,<3"]
 build-backend = "setuptools.build_meta"
 
 [tool.setuptools]

pytest.ini

@@ -1,5 +1,5 @@
 [pytest]
-pythonpath = src
+pythonpath = src tests
 testpaths = tests
 python_functions = test_*
 log_cli = False

requirements.txt

@@ -1,6 +1,6 @@
 cryptography==43.0.1
 npg_id_generation@https://github.com/wtsi-npg/npg_id_generation/releases/download/5.0.1/npg_id_generation-5.0.1.tar.gz
-partisan@https://github.com/wtsi-npg/partisan/releases/download/2.10.2/partisan-2.10.2.tar.gz
+partisan@https://github.com/wtsi-npg/partisan/releases/download/2.11.1/partisan-2.11.1.tar.gz
 pymysql==1.1.1
 python-dateutil==2.9.0.post0
 rich==13.7.1

tests/__init__.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2024 Genome Research Ltd. All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from partisan.icommands import iinit
+
+# Ensure that the iRODS environment is initialised before running any tests. Calling
+# this function will create or update a local iRODS auth file ready for use by the
+# iRODS clients used in the tests.
+iinit()

tests/conftest.py

@@ -47,6 +47,7 @@
     add_rods_path,
     add_sql_test_utilities,
     add_test_groups,
+    is_running_in_github_ci,
     remove_rods_path,
     remove_sql_test_utilities,
     remove_test_groups,
@@ -67,7 +68,8 @@
 )
 
 TEST_INI = os.path.join(os.path.dirname(__file__), "testdb.ini")
-INI_SECTION = "dev"
+INI_SECTION_LOCAL = "docker"
+INI_SECTION_GITHUB = "github"
 
 
 @pytest.fixture(scope="session")
@@ -83,7 +85,9 @@ def sql_test_utilities():
 @pytest.fixture(scope="function")
 def mlwh_session() -> Session:
     """Create an empty ML warehouse database fixture."""
-    dbconfig = DBConfig.from_file(TEST_INI, INI_SECTION)
+    section = INI_SECTION_GITHUB if is_running_in_github_ci() else INI_SECTION_LOCAL
+
+    dbconfig = DBConfig.from_file(TEST_INI, section)
     engine = create_engine(dbconfig.url, echo=False)
 
     if database_exists(engine.url):

tests/helpers/__init__.py

@@ -18,6 +18,7 @@
 # @author Keith James <kdj@sanger.ac.uk>
 
 """Helper functions for testing."""
+import os
 from datetime import datetime
 from pathlib import PurePath
 
@@ -58,6 +59,14 @@
 TEST_SQL_INVALID_CHECKSUM = "setObjectChecksumInvalid"
 
 
+def is_running_in_github_ci():
+    """Return True if running in GitHub CI, False otherwise.
+
+    https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/store-information-in-variables
+    """
+    return "GITHUB_ACTIONS" in os.environ
+
+
 def add_sql_test_utilities():
     """Add to iRODS canned SQL statements required for testing."""
     if have_admin():

tests/testdb.ini

@@ -11,7 +11,15 @@
 #     -e MYSQL_PASSWORD=test \
 #     -e MYSQL_DATABASE=mlwarehouse "mysql:$MYSQL_VERSION"
 #
-[dev]
+
+[docker]
+host = mysql-server
+port = 3306
+schema = mlwarehouse
+user = test
+password = test
+
+[github]
 host = 127.0.0.1
 port = 3306
 schema = mlwarehouse