diff --git a/images/krte/Dockerfile b/images/krte/Dockerfile index 09b9a42f0216..34aa01c39b13 100644 --- a/images/krte/Dockerfile +++ b/images/krte/Dockerfile @@ -21,9 +21,16 @@ FROM debian:bookworm # arg that specifies the image name (for debugging) ARG IMAGE_ARG -# arg that specifies the go version to install +# arg that specifies the go version to install. +# empty value specifies the latest version. ARG GO_VERSION +# args for the user to run rootless docker (for KRTE_SYSTEMD_ROOTLESS mode) +ARG ROOTLESS_USER=rootless +ARG ROOTLESS_UID=5000 +ARG ROOTLESS_SUBUID_BEGIN=100000 +ARG ROOTLESS_SUBUID_COUNT=1048576 + # add envs: # - so we can debug with the image name:tag # - adding gsutil etc. to path (where we will install them) @@ -36,9 +43,6 @@ ENV KRTE_IMAGE=${IMAGE_ARG} \ CLOUDSDK_CORE_DISABLE_PROMPTS=1 \ CONTAINER=docker -# copy in image utility scripts -COPY wrapper.sh /usr/local/bin/ - # Install tools needed to: # - install docker # - build kind @@ -56,6 +60,8 @@ RUN echo "Installing Packages ..." \ build-essential \ ca-certificates \ curl \ + dbus \ + dbus-user-session \ file \ git \ gnupg2 \ @@ -67,10 +73,15 @@ RUN echo "Installing Packages ..." \ procps \ python3 \ rsync \ + slirp4netns \ software-properties-common \ + sudo \ + systemd \ + uidmap \ unzip \ && rm -rf /var/lib/apt/lists/* \ && echo "Installing Go ..." \ + && if [ -z "${GO_VERSION}" ]; then GO_VERSION=$(curl -fsSL https://go.dev/VERSION?m=text | grep -oP "go\K(.*)"); fi \ && export GO_TARBALL="go${GO_VERSION}.linux-amd64.tar.gz" \ && curl -fsSL "https://go.dev/dl/${GO_TARBALL}" --output "${GO_TARBALL}" \ && tar xzf "${GO_TARBALL}" -C /usr/local \ @@ -93,14 +104,30 @@ RUN echo "Installing Packages ..." \ "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ tee /etc/apt/sources.list.d/docker.list > /dev/null \ && apt-get update \ - && apt-get install -y --no-install-recommends docker-ce docker-buildx-plugin \ + && apt-get install -y --no-install-recommends docker-ce docker-buildx-plugin docker-ce-rootless-extras \ + && systemctl disable docker \ && rm -rf /var/lib/apt/lists/* \ && sed -i 's/cgroupfs_mount$/#cgroupfs_mount\n/' /etc/init.d/docker \ && echo "Ensuring Legacy Iptables ..." \ && update-alternatives --set iptables /usr/sbin/iptables-legacy \ - && update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy + && update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy \ + && echo "Setting up the systemd delegation for rootless" \ + && mkdir -p /etc/systemd/system/user@.service.d \ + && echo "[Service]" > /etc/systemd/system/user@.service.d/delegate.conf \ + && echo "Delegate=yes" >> /etc/systemd/system/user@.service.d/delegate.conf \ + && echo "Setting up the rootless user" \ + && useradd --create-home --home-dir /home/${ROOTLESS_USER} --uid ${ROOTLESS_UID} -G systemd-journal ${ROOTLESS_USER} \ + && echo "${ROOTLESS_UID}:${ROOTLESS_SUBUID_BEGIN}:${ROOTLESS_SUBUID_COUNT}" >/etc/subuid \ + && cp -f /etc/subuid /etc/subgid \ + && mkdir -p /home/${ROOTLESS_USER}/.local/share/docker \ + && chown -R ${ROOTLESS_USER} /home/${ROOTLESS_USER} + +# copy in image utility scripts +COPY containerized-systemd.sh /usr/local/bin/ +COPY wrapper.sh /usr/local/bin/ # entrypoint is our wrapper script, in Prow you will need to explicitly re-specify this ENTRYPOINT ["wrapper.sh"] # volume for docker in docker, use an emptyDir in Prow VOLUME ["/var/lib/docker"] +VOLUME ["/home/${ROOTLESS_USER}/.local/share/docker"] diff --git a/images/krte/README.md b/images/krte/README.md index 5922f283f328..3907f62f9a3d 100644 --- a/images/krte/README.md +++ b/images/krte/README.md @@ -8,3 +8,11 @@ is maintained for the sole purpose of testing Kubernetes with KIND. ## WARNING This image is _not_ supported for other use cases. Use at your own risk. + +## Build-time variables +See the `ARG` instructions in [`Dockerfile`](./Dockerfile). + +## Run-time variables +- `KRTE_SYSTEMD=true` (default: `false`): enable systemd +- `KRTE_SYSTEMD_ROOTLESS=true` (default: `false`): switch to a non-root user via systemd. + The KRTE container itself still has to be run as the root, so DO NOT specify `securityContext.runAsUser`. diff --git a/images/krte/containerized-systemd.sh b/images/krte/containerized-systemd.sh new file mode 100755 index 000000000000..677ac9bd336b --- /dev/null +++ b/images/krte/containerized-systemd.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copied from https://github.com/AkihiroSuda/containerized-systemd/tree/v0.1.1 . +# This script executes the args via systemd. +# The stdio, workdir, and environment variables are preserved. + +set -e +container=docker +export container + +if [ $# -eq 0 ]; then + echo >&2 'ERROR: No command specified. You probably want to run `journalctl -f`, or maybe `bash`?' + exit 1 +fi + +if [ ! -t 0 ]; then + echo >&2 'ERROR: TTY needs to be enabled (`docker run -t ...`).' + exit 1 +fi + +env >/etc/docker-entrypoint-env + +cat >/etc/systemd/system/docker-entrypoint.target </etc/docker-entrypoint-cmd + +cat >/etc/systemd/system/docker-entrypoint.service < /dev/null; then echo >&2 \"got signal \${EXIT_STATUS}\"; systemctl exit \$(( 128 + \$( kill -l \${EXIT_STATUS} ) )); else systemctl exit \${EXIT_STATUS}; fi" +StandardInput=tty-force +StandardOutput=inherit +StandardError=inherit +WorkingDirectory=$(pwd) +EnvironmentFile=/etc/docker-entrypoint-env + +[Install] +WantedBy=multi-user.target +EOF + +systemctl mask systemd-firstboot.service systemd-udevd.service systemd-modules-load.service +systemctl unmask systemd-logind +systemctl enable docker-entrypoint.service + +systemd= +if [ -x /lib/systemd/systemd ]; then + systemd=/lib/systemd/systemd +elif [ -x /usr/lib/systemd/systemd ]; then + systemd=/usr/lib/systemd/systemd +elif [ -x /sbin/init ]; then + systemd=/sbin/init +else + echo >&2 'ERROR: systemd is not installed' + exit 1 +fi +systemd_args="--show-status=false --unit=docker-entrypoint.target" +echo "$0: starting $systemd $systemd_args" +exec $systemd $systemd_args diff --git a/images/krte/wrapper.sh b/images/krte/wrapper.sh index 4d1a0442f616..8ea11304e13e 100755 --- a/images/krte/wrapper.sh +++ b/images/krte/wrapper.sh @@ -30,6 +30,26 @@ set -o errexit set -o pipefail set -o nounset +: "${KRTE_SYSTEMD_ROOTLESS:=false}" +: "${KRTE_SYSTEMD_ROOTLESS_USER:=rootless}" +: "${KRTE_SYSTEMD:=${KRTE_SYSTEMD_ROOTLESS}}" + +if [ "${KRTE_SYSTEMD}" = "true" ] && [ "$$" = "1" ]; then + >&2 echo "wrapper.sh] [INFO] Re-executing in systemd: \`$*\`" + exec /usr/local/bin/containerized-systemd.sh "$0" "$@" +fi +if [ "${KRTE_SYSTEMD_ROOTLESS}" = "true" ] && [ "$(id -u)" = "0" ]; then + >&2 echo "wrapper.sh] [INFO] Waiting for the systemd user session to start up" + loginctl enable-linger ${KRTE_SYSTEMD_ROOTLESS_USER} + sleep 3 + >&2 echo "wrapper.sh] [INFO] Switching to rootless: \`$*\`" + rootless_uid=$(id -u ${KRTE_SYSTEMD_ROOTLESS_USER}) + exec sudo -E -H -u ${KRTE_SYSTEMD_ROOTLESS_USER} \ + XDG_RUNTIME_DIR=/run/user/${rootless_uid} \ + DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/${rootless_uid}/bus \ + "$0" "$@" +fi + >&2 echo "wrapper.sh] [INFO] Wrapping Test Command: \`$*\`" >&2 echo "wrapper.sh] [INFO] Running in: ${KRTE_IMAGE}" >&2 echo "wrapper.sh] [INFO] See: https://github.com/kubernetes/test-infra/blob/master/images/krte/wrapper.sh" @@ -40,7 +60,11 @@ cleanup(){ if [[ "${DOCKER_IN_DOCKER_ENABLED:-false}" == "true" ]]; then >&2 echo "wrapper.sh] [CLEANUP] Cleaning up after Docker in Docker ..." docker ps -aq | xargs -r docker rm -f || true - service docker stop || true + if [ "$(id -u)" = "0" ]; then + service docker stop || true + else + systemctl --user stop docker || true + fi >&2 echo "wrapper.sh] [CLEANUP] Done cleaning up after Docker in Docker." fi } @@ -93,7 +117,11 @@ export DOCKER_IN_DOCKER_ENABLED=${DOCKER_IN_DOCKER_ENABLED:-false} if [[ "${DOCKER_IN_DOCKER_ENABLED}" == "true" ]]; then >&2 echo "wrapper.sh] [SETUP] Docker in Docker enabled, initializing ..." # If we have opted in to docker in docker, start the docker daemon, - service docker start + if [ "$(id -u)" = "0" ]; then + service docker start + else + dockerd-rootless-setuptool.sh install + fi # the service can be started but the docker socket not ready, wait for ready WAIT_N=0 while true; do