From 2266949b66586021667cce97278bf9cf647d9558 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 27 Feb 2024 18:12:50 -0500 Subject: [PATCH 1/6] Add a pipenv configuration This configuration includes a Pipfile configuration file and the generated Pipfile.lock file that pins to specific versions for the Python dependencies for this project. This will help us ensure repeatable builds. The pipenv package is added as a developmental requirement to support these files. --- requirements-dev.txt | 1 + src/Pipfile | 13 +++++++++++++ src/Pipfile.lock | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+) create mode 100644 src/Pipfile create mode 100644 src/Pipfile.lock diff --git a/requirements-dev.txt b/requirements-dev.txt index cb51627..bdc1615 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,4 @@ --requirement requirements-test.txt ipython +pipenv semver diff --git a/src/Pipfile b/src/Pipfile new file mode 100644 index 0000000..56f2fc9 --- /dev/null +++ b/src/Pipfile @@ -0,0 +1,13 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +# List any Python dependencies for the image here +[packages] +# This should match the version of the image +example = {file = "https://github.com/cisagov/skeleton-python-library/archive/v0.0.1.tar.gz"} + +# This version should match the version of Python in the image +[requires] +python_full_version = "3.12.0" diff --git a/src/Pipfile.lock b/src/Pipfile.lock new file mode 100644 index 0000000..d39d053 --- /dev/null +++ b/src/Pipfile.lock @@ -0,0 +1,38 @@ +{ + "_meta": { + "hash": { + "sha256": "654452851fea1eb2c8811649e5efe8873c8ff51f5c14dd27a4a8ebb5b15a27c4" + }, + "pipfile-spec": 6, + "requires": { + "python_full_version": "3.12.0" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "docopt": { + "hashes": [ + "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491" + ], + "version": "==0.6.2" + }, + "example": { + "file": "https://github.com/cisagov/skeleton-python-library/archive/v0.0.1.tar.gz" + }, + "setuptools": { + "hashes": [ + "sha256:02fa291a0471b3a18b2b2481ed902af520c69e8ae0919c13da936542754b4c56", + "sha256:5c0806c7d9af348e6dd3777b4f4dbb42c7ad85b190104837488eab9a7c945cf8" + ], + "markers": "python_version >= '3.8'", + "version": "==69.1.1" + } + }, + "develop": {} +} From d530d0762e521fe6987ad2981ea387f755f6ac83 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 27 Feb 2024 19:01:45 -0500 Subject: [PATCH 2/6] Install Python dependencies using pipenv Now that we have a pipenv configuration we will use it to install the Python dependencies for the image. The `build` workflow is updated to no longer pass the VERSION build argument in line with this change. --- .github/workflows/build.yml | 4 ---- Dockerfile | 34 +++++++++++++++++++++++----------- README.md | 2 -- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d8ba132..3208947 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -306,8 +306,6 @@ jobs: id: docker_build uses: docker/build-push-action@v5 with: - build-args: | - VERSION=${{ needs.prepare.outputs.source_version }} cache-from: type=local,src=${{ env.BUILDX_CACHE_DIR }} cache-to: type=local,dest=${{ env.BUILDX_CACHE_DIR }} context: . @@ -459,8 +457,6 @@ jobs: id: docker_build uses: docker/build-push-action@v5 with: - build-args: | - VERSION=${{ needs.prepare.outputs.source_version }} cache-from: type=local,src=${{ env.BUILDX_CACHE_DIR }} cache-to: type=local,dest=${{ env.BUILDX_CACHE_DIR }} context: . diff --git a/Dockerfile b/Dockerfile index 1aae8db..c36ae46 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,7 @@ -ARG VERSION=unspecified - # Official Docker images are in the form library/ while non-official # images are in the form /. FROM docker.io/library/python:3.12.0-alpine3.18 -ARG VERSION - ### # For a list of pre-defined annotation keys and value types see: # https://github.com/opencontainers/image-spec/blob/master/annotations.md @@ -33,6 +29,7 @@ ENV VIRTUAL_ENV="${CISA_HOME}/.venv" # Versions of the Python packages installed directly ENV PYTHON_PIP_VERSION=24.0 +ENV PYTHON_PIPENV_VERSION=2023.12.1 ENV PYTHON_SETUPTOOLS_VERSION=69.1.0 ENV PYTHON_WHEEL_VERSION=0.42.0 @@ -43,21 +40,36 @@ RUN addgroup --system --gid ${CISA_GID} ${CISA_GROUP} \ && adduser --system --uid ${CISA_UID} --ingroup ${CISA_GROUP} ${CISA_USER} ### -# Set up a Python virtual environment (venv); install the specified versions of pip, -# setuptools, and wheel into it; and then install the Python dependencies for -# the application. +# Install the specified version of pipenv; set up a Python virtual environment (venv); +# and install the specified versions of pip, setuptools, and wheel into the venv. # # Note that we use the --no-cache-dir flag to avoid writing to a local # cache. This results in a smaller final image, at the cost of # slightly longer install times. ### -RUN python3 -m venv ${VIRTUAL_ENV} \ +RUN python3 -m pip install --no-cache-dir --upgrade pipenv==${PYTHON_PIPENV_VERSION} \ + # Manueally create the virtual environment + && python3 -m venv ${VIRTUAL_ENV} \ + # Ensure the core Python packages are installed in the virtual environment && ${VIRTUAL_ENV}/bin/python3 -m pip install --no-cache-dir --upgrade \ pip==${PYTHON_PIP_VERSION} \ setuptools==${PYTHON_SETUPTOOLS_VERSION} \ - wheel==${PYTHON_WHEEL_VERSION} \ - && ${VIRTUAL_ENV}/bin/python3 -m pip install --no-cache-dir --upgrade \ - https://github.com/cisagov/skeleton-python-library/archive/v${VERSION}.tar.gz + wheel==${PYTHON_WHEEL_VERSION} + +### +# Check the Pipfile configuration and then install the Python dependencies into +# the virtual environment. +# +# Note that pipenv will install into a virtual environment if the VIRTUAL_ENV +# environment variable is set. We are using short flags because the rm binary +# in Alpine Linux does not support long flags. The -f instructs rm to remove +# files without prompting. +### +WORKDIR /tmp +COPY src/Pipfile src/Pipfile.lock ./ +RUN pipenv check --verbose \ + && pipenv install --clear --deploy --extra-pip-args "--no-cache-dir" --verbose \ + && rm -f Pipfile* ### # Sym-link the Python binary in the venv to the system-wide Python and add the venv to diff --git a/README.md b/README.md index 57f8c30..8d926e4 100644 --- a/README.md +++ b/README.md @@ -196,7 +196,6 @@ Build the image locally using this git repository as the [build context](https:/ ```console docker build \ - --build-arg VERSION=0.0.1 \ --tag cisagov/example:0.0.1 \ https://github.com/cisagov/example.git#develop ``` @@ -227,7 +226,6 @@ Docker: docker buildx build \ --file Dockerfile-x \ --platform linux/amd64 \ - --build-arg VERSION=0.0.1 \ --output type=docker \ --tag cisagov/example:0.0.1 . ``` From adfcfdb983af480356324d7cc4354f93cc4c9918 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 27 Feb 2024 19:09:29 -0500 Subject: [PATCH 3/6] Use a multi-stage Docker build Switch to using a multi-stage build in the Dockerfile. This reduces image size since pipenv and its dependencices are not needed in the final image. It also ensures that the system Python environment is unmodified. --- Dockerfile | 71 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 41 insertions(+), 30 deletions(-) diff --git a/Dockerfile b/Dockerfile index c36ae46..ca0742e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,29 +1,11 @@ # Official Docker images are in the form library/ while non-official # images are in the form /. -FROM docker.io/library/python:3.12.0-alpine3.18 +FROM docker.io/library/python:3.12.0-alpine3.18 as compile-stage ### -# For a list of pre-defined annotation keys and value types see: -# https://github.com/opencontainers/image-spec/blob/master/annotations.md -# -# Note: Additional labels are added by the build workflow. -### -# github@cisa.dhs.gov is a very generic email distribution, and it is -# unlikely that anyone on that distribution is familiar with the -# particulars of your repository. It is therefore *strongly* -# suggested that you use an email address here that is specific to the -# person or group that maintains this repository; for example: -# LABEL org.opencontainers.image.authors="vm-fusion-dev-group@trio.dhs.gov" -LABEL org.opencontainers.image.authors="github@cisa.dhs.gov" -LABEL org.opencontainers.image.vendor="Cybersecurity and Infrastructure Security Agency" - -### -# Unprivileged user setup variables +# Unprivileged user variables ### -ARG CISA_UID=421 -ARG CISA_GID=${CISA_UID} ARG CISA_USER="cisa" -ENV CISA_GROUP=${CISA_USER} ENV CISA_HOME="/home/${CISA_USER}" ENV VIRTUAL_ENV="${CISA_HOME}/.venv" @@ -33,12 +15,6 @@ ENV PYTHON_PIPENV_VERSION=2023.12.1 ENV PYTHON_SETUPTOOLS_VERSION=69.1.0 ENV PYTHON_WHEEL_VERSION=0.42.0 -### -# Create unprivileged user -### -RUN addgroup --system --gid ${CISA_GID} ${CISA_GROUP} \ - && adduser --system --uid ${CISA_UID} --ingroup ${CISA_GROUP} ${CISA_USER} - ### # Install the specified version of pipenv; set up a Python virtual environment (venv); # and install the specified versions of pip, setuptools, and wheel into the venv. @@ -68,18 +44,53 @@ RUN python3 -m pip install --no-cache-dir --upgrade pipenv==${PYTHON_PIPENV_VERS WORKDIR /tmp COPY src/Pipfile src/Pipfile.lock ./ RUN pipenv check --verbose \ - && pipenv install --clear --deploy --extra-pip-args "--no-cache-dir" --verbose \ - && rm -f Pipfile* + && pipenv install --clear --deploy --extra-pip-args "--no-cache-dir" --verbose + +# Official Docker images are in the form library/ while non-official +# images are in the form /. +FROM docker.io/library/python:3.12.0-alpine3.18 as build-stage + +### +# For a list of pre-defined annotation keys and value types see: +# https://github.com/opencontainers/image-spec/blob/master/annotations.md +# +# Note: Additional labels are added by the build workflow. +### +# github@cisa.dhs.gov is a very generic email distribution, and it is +# unlikely that anyone on that distribution is familiar with the +# particulars of your repository. It is therefore *strongly* +# suggested that you use an email address here that is specific to the +# person or group that maintains this repository; for example: +# LABEL org.opencontainers.image.authors="vm-fusion-dev-group@trio.dhs.gov" +LABEL org.opencontainers.image.authors="github@cisa.dhs.gov" +LABEL org.opencontainers.image.vendor="Cybersecurity and Infrastructure Security Agency" + +### +# Unprivileged user setup variables +### +ARG CISA_UID=421 +ARG CISA_GID=${CISA_UID} +ARG CISA_USER="cisa" +ENV CISA_GROUP=${CISA_USER} +ENV CISA_HOME="/home/${CISA_USER}" +ENV VIRTUAL_ENV="${CISA_HOME}/.venv" + +### +# Create unprivileged user +### +RUN addgroup --system --gid ${CISA_GID} ${CISA_GROUP} \ + && adduser --system --uid ${CISA_UID} --ingroup ${CISA_GROUP} ${CISA_USER} ### -# Sym-link the Python binary in the venv to the system-wide Python and add the venv to -# the PATH. +# Copy in the Python virtual environment created in compile-stage, Sym-link the +# Python binary in the venv to the system-wide Python and add the venv to the PATH. # # Note that we sym-link the Python binary in the venv to the system-wide Python so that # any calls to `python3` will use our virtual environment. We are using short flags # because the ln binary in Alpine Linux does not support long flags. The -f instructs # ln to remove the existing file and the -s instructs ln to create a symbolic link. ### +COPY --from=compile-stage --chown=${CISA_USER}:${CISA_GROUP} ${VIRTUAL_ENV} ${VIRTUAL_ENV} RUN ln -fs "$(command -v python3)" "${VIRTUAL_ENV}"/bin/python3 ENV PATH="${VIRTUAL_ENV}/bin:$PATH" From 8e03ad95147adeb44bd2773b146eb6f752f53b6c Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 27 Feb 2024 19:13:33 -0500 Subject: [PATCH 4/6] Install core Python packages into the system Python environment Install the core Python packages (pip, setuptools, and wheel) into the system Python environment before installing pipenv. This keeps things consistent with our usual approach to Python environments. --- Dockerfile | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index ca0742e..96a5126 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,14 +16,21 @@ ENV PYTHON_SETUPTOOLS_VERSION=69.1.0 ENV PYTHON_WHEEL_VERSION=0.42.0 ### -# Install the specified version of pipenv; set up a Python virtual environment (venv); -# and install the specified versions of pip, setuptools, and wheel into the venv. +# Install the specified versions of pip, setuptools, and wheel into the system +# Python environment; install the specified version of pipenv into the system Python +# environment; set up a Python virtual environment (venv); and install the specified +# versions of pip, setuptools, and wheel into the venv. # # Note that we use the --no-cache-dir flag to avoid writing to a local # cache. This results in a smaller final image, at the cost of # slightly longer install times. ### -RUN python3 -m pip install --no-cache-dir --upgrade pipenv==${PYTHON_PIPENV_VERSION} \ +RUN python3 -m pip install --no-cache-dir --upgrade \ + pip==${PYTHON_PIP_VERSION} \ + setuptools==${PYTHON_SETUPTOOLS_VERSION} \ + wheel==${PYTHON_WHEEL_VERSION} \ + && python3 -m pip install --no-cache-dir --upgrade \ + pipenv==${PYTHON_PIPENV_VERSION} \ # Manueally create the virtual environment && python3 -m venv ${VIRTUAL_ENV} \ # Ensure the core Python packages are installed in the virtual environment From c45345f60d9f350e59000b2f0b57075803357d5c Mon Sep 17 00:00:00 2001 From: Nick <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 28 Feb 2024 12:04:09 -0500 Subject: [PATCH 5/6] Fix outdated comment in the Dockerfile The comment references a command that is no longer being run. Co-authored-by: Shane Frasier --- Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 96a5126..38cf0a4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,9 +44,7 @@ RUN python3 -m pip install --no-cache-dir --upgrade \ # the virtual environment. # # Note that pipenv will install into a virtual environment if the VIRTUAL_ENV -# environment variable is set. We are using short flags because the rm binary -# in Alpine Linux does not support long flags. The -f instructs rm to remove -# files without prompting. +# environment variable is set. ### WORKDIR /tmp COPY src/Pipfile src/Pipfile.lock ./ From d42ae8f61b3d5cfe573ad339ce3ff3a495877c12 Mon Sep 17 00:00:00 2001 From: Nick <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 28 Feb 2024 12:04:51 -0500 Subject: [PATCH 6/6] Fix typo in Dockerfile comment Co-authored-by: dav3r --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 38cf0a4..6a00b36 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,7 +31,7 @@ RUN python3 -m pip install --no-cache-dir --upgrade \ wheel==${PYTHON_WHEEL_VERSION} \ && python3 -m pip install --no-cache-dir --upgrade \ pipenv==${PYTHON_PIPENV_VERSION} \ - # Manueally create the virtual environment + # Manually create the virtual environment && python3 -m venv ${VIRTUAL_ENV} \ # Ensure the core Python packages are installed in the virtual environment && ${VIRTUAL_ENV}/bin/python3 -m pip install --no-cache-dir --upgrade \