diff --git a/src/cloud-api-adaptor/docker/.gitignore b/src/cloud-api-adaptor/docker/.gitignore new file mode 100644 index 000000000..4675001d9 --- /dev/null +++ b/src/cloud-api-adaptor/docker/.gitignore @@ -0,0 +1 @@ +image/resources/* diff --git a/src/cloud-api-adaptor/docker/README.md b/src/cloud-api-adaptor/docker/README.md index f383523d0..40e2b0e10 100644 --- a/src/cloud-api-adaptor/docker/README.md +++ b/src/cloud-api-adaptor/docker/README.md @@ -13,8 +13,8 @@ The `docker` provider simulates a pod VM inside a docker container. Docker engine version 26+ supports API 1.44. Ensure you complete the [post install steps](https://docs.docker.com/engine/install/linux-postinstall/) if using non-root user - - + + - Kubernetes cluster ## Build CAA pod-VM image @@ -26,13 +26,13 @@ export CLOUD_PROVIDER=docker ``` - Build the required pod VM binaries - + ```bash cd src/cloud-api-adaptor/docker/image make ``` -This will build the required binaries inside a container and place +This will build the required binaries inside a container and place it under `resources/binaries-tree` - Build the pod VM image @@ -44,8 +44,9 @@ cd ../../ This will build the podvm docker image. By default the image is named `quay.io/confidential-containers/podvm-docker-image`. -For quick changes you can just build the binaries of podvm components and update `./resources/binaries-tree/usr/local/bin` with the new -components and run `make image` to build a new podvm image. +For quick changes you can just build the binaries of podvm components and +update `./resources/binaries-tree/usr/local/bin` with the new components and +run `make image` to build a new podvm image. You can download a ready-to-use image on your worker node. @@ -56,7 +57,6 @@ docker pull quay.io/confidential-containers/podvm-docker-image Note that before you can spin up a pod, the podvm image must be available on the K8s worker node with the docker engine installed. - ## Build CAA container image > **Note**: If you have made changes to the CAA code and you want to deploy those changes then follow [these instructions](https://github.com/confidential-containers/cloud-api-adaptor/blob/main/src/cloud-api-adaptor/install/README.md#building-custom-cloud-api-adaptor-image) to build the container image from the root of this repository. @@ -65,7 +65,6 @@ with the docker engine installed. The following [`kustomization.yaml`](../install/overlays/docker/kustomization.yaml) is used. - ### Deploy CAA on the Kubernetes cluster Run the following command to deploy CAA: @@ -80,7 +79,82 @@ For changing the CAA image to your custom built image (eg. `quay.io/myuser/cloud you can use the following: ```bash -kubectl set image ds/cloud-api-adaptor-daemonset -n confidential-containers-system cloud-api-adaptor-con=quay.io/myuser/cloud-api-adaptor +export CAA_IMAGE=quay.io/myuser/cloud-api-adaptor +kubectl set image ds/cloud-api-adaptor-daemonset -n confidential-containers-system cloud-api-adaptor-con="$CAA_IMAGE" +``` + +## Running the CAA e2e tests + +### Test Prerequisites + +To run the tests, use a test system with at least 8GB RAM and 4vCPUs. +Ubuntu 22.04 has been tested. Other Linux distros should work, but it has not +been tested. + +Following software prerequisites are needed on the test system: + +- make +- go +- yq +- kubectl +- kind +- docker + +A `prereqs.sh` helper script is available under `src/cloud-api-adaptor/docker` to install/uninstall the prerequisites. + + +> **Note:** If using the `prereqs.sh` helper script to install the +> prerequisites, then reload the shell to ensure new permissions +are in place to run `docker` and other commands. + +### Test Execution + +In order to run the tests, edit the file `src/cloud-api-adaptor/test/provisioner/docker/provision_docker.properties` +and update the `CAA_IMAGE` and `CAA_IMAGE_TAG` variables with your custom CAA image and tag. + +You can run the CAA e2e [tests/e2e/README.md](../test/e2e/README.md) by running the following command: + +```sh +make TEST_PODVM_IMAGE= TEST_PROVISION=yes CLOUD_PROVIDER=docker TEST_PROVISION_FILE=$(pwd)/test/provisioner/docker/provision_docker.properties test-e2e +``` + +This will create a two node kind cluster, automatically download the pod VM +image mentioned in the `provision_docker.properties` file and run the tests. On +completion of the test, the kind cluster will be automatically deleted. + +> **Note:** To overcome docker rate limiting issue or to download images from private registries, +create a `config.json` file under `/tmp` with your registry secrets. + +For example: +If your docker registry user is `someuser` and password is `somepassword` then create the auth string +as shown below: + +```sh +echo -n "someuser:somepassword" | base64 +c29tZXVzZXI6c29tZXBhc3N3b3Jk +``` + +This auth string needs to be used in `/tmp/config.json` as shown below: + +```sh +{ + "auths": { + "https://index.docker.io/v1/": { + "auth": "c29tZXVzZXI6c29tZXBhc3N3b3Jk" + } + } +} +``` + +If you want to use a different location for the registry secret, then remember to update the same +in the `src/cloud-api-adaptor/docker/kind-config.yaml` file. + +> **Note:** If you have executed the tests with `TEST_TEARDOWN=no`, then you'll +> need to manually delete the `kind` created cluster by running the following +> command: + +```sh +kind delete cluster --name peer-pods ``` ## Run sample application @@ -151,7 +225,7 @@ nginx-dbc79c87-jt49h 1/1 Running 1 (3m22s ago) 3m29s $ docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES e60b768b847d quay.io/confidential-containers/podvm-docker-image "/usr/local/bin/entr…" 3 minutes ago Up 3 minutes 15150/tcp podvm-nginx-dbc79c87-jt49h-b9361aef -``` +``` For debugging you can use docker commands like `docker ps`, `docker logs`, `docker exec`. @@ -161,3 +235,21 @@ For debugging you can use docker commands like `docker ps`, `docker logs`, `dock kubectl delete deployment nginx ``` +## Troubleshooting + +When using `containerd` and `nydus-snapshotter` you might encounter pod creation failure due to +issues with unpacking of image. Check the `nydus-snapshotter` troubleshooting [doc](../docs/troubleshooting/nydus-snapshotter.md). + +In order to login to the worker node you can use either of the following approaches + +```sh +kubectl debug node/peer-pods-worker -it --image=busybox + +# chroot /host +``` + +or + +```sh +docker exec -it peer-pods-worker bash +``` \ No newline at end of file diff --git a/src/cloud-api-adaptor/docker/image/Dockerfile b/src/cloud-api-adaptor/docker/image/Dockerfile index 19e54aa9e..f4702a015 100644 --- a/src/cloud-api-adaptor/docker/image/Dockerfile +++ b/src/cloud-api-adaptor/docker/image/Dockerfile @@ -4,15 +4,15 @@ ARG MINOR=11 FROM ${IMAGE}:v${VERSION}.${MINOR} -RUN apt-get update && apt-get install -y sudo && apt-get clean all +RUN apt-get update && apt-get install -y sudo libpam-systemd && apt-get clean all RUN systemctl disable kubelet RUN systemctl disable containerd -COPY ./resources/binaries-tree/etc/ etc/ -COPY ./resources/binaries-tree/usr/ usr/ -COPY ./resources/binaries-tree/pause_bundle / +COPY ./resources/binaries-tree/etc/ /etc/ +COPY ./resources/binaries-tree/usr/ /usr/ +COPY ./resources/binaries-tree/pause_bundle/ /pause_bundle/ RUN curl -LO https://raw.githubusercontent.com/confidential-containers/cloud-api-adaptor/main/src/cloud-api-adaptor/podvm/qcow2/misc-settings.sh diff --git a/src/cloud-api-adaptor/docker/image/Makefile b/src/cloud-api-adaptor/docker/image/Makefile index fe8b0123f..d7b768067 100644 --- a/src/cloud-api-adaptor/docker/image/Makefile +++ b/src/cloud-api-adaptor/docker/image/Makefile @@ -1,3 +1,5 @@ +include ../../Makefile.defaults + ARCH ?= $(subst x86_64,amd64,$(shell uname -m)) BUILDER = ubuntu-binaries-builder-$(ARCH) PODVM_IMG ?= quay.io/confidential-containers/podvm-docker-image diff --git a/src/cloud-api-adaptor/docker/kind-config.yaml b/src/cloud-api-adaptor/docker/kind-config.yaml new file mode 100644 index 000000000..3b12ea511 --- /dev/null +++ b/src/cloud-api-adaptor/docker/kind-config.yaml @@ -0,0 +1,27 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + disableDefaultCNI: true # disable kindnet + podSubnet: 192.168.0.0/16 # set to Calico's default subnet +nodes: + - role: control-plane + # Same image version as used for pod VM base image + image: kindest/node:v1.29.4 + extraMounts: + # The config.json file contains the registry secrets that you might + # need to pull images from a private registry or docker registry to avoid + # rate limiting. + - hostPath: /tmp/config.json + containerPath: /var/lib/kubelet/config.json + - role: worker + image: kindest/node:v1.29.4 + extraMounts: + - hostPath: /var/run/docker.sock + containerPath: /var/run/docker.sock + - hostPath: /var/lib/docker + containerPath: /var/lib/docker + # The config.json file contains the registry secrets that you might + # need to pull images from a private registry or docker registry to avoid + # rate limiting. + - hostPath: /tmp/config.json + containerPath: /var/lib/kubelet/config.json diff --git a/src/cloud-api-adaptor/docker/kind_cluster.sh b/src/cloud-api-adaptor/docker/kind_cluster.sh new file mode 100755 index 000000000..696750d60 --- /dev/null +++ b/src/cloud-api-adaptor/docker/kind_cluster.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# Ref: https://stackoverflow.com/questions/299728/how-do-you-use-newgrp-in-a-script-then-stay-in-that-group-when-the-script-exits +newgrp docker <&2 + exit 1 +} + +# function to check if passwordless sudo is enabled +check_sudo() { + if sudo -n true 2>/dev/null; then + echo "Passwordless sudo is enabled." + else + error_exit "Passwordless sudo is not enabled. Please enable passwordless sudo." + fi +} + +# function to compare versions +# returns true if $1 < $2 +version_lt() { + [ "$1" != "$2" ] && [ "$(printf '%s\n' "$@" | sort -V | head -n 1)" == "$1" ] +} + +# function to install OS packages +# the packages are available in the variable REQUIRED_OS_PACKAGES +# the function will install the packages using the package manager +# Following are the packages that are installed: +# make +install_os_packages() { + # Define the required OS packages + REQUIRED_OS_PACKAGES=( + "make" + ) + + # Install the required OS packages + for package in "${REQUIRED_OS_PACKAGES[@]}"; do + if [[ -x "$(command -v "${package}")" ]]; then + echo "Package ${package} is already installed. Skipping." + continue + else + echo "Installing ${package}..." + # Update a hidden file to record what was installed. This is useful for uninstallation + echo "${package}" >>"${os_packages_file}" + if [[ -x "$(command -v apt-get)" ]]; then + # shellcheck disable=SC2015 + sudo apt-get update && + sudo apt-get install -y "${package}" || + error_exit "Failed to install ${package}" + elif [[ -x "$(command -v dnf)" ]]; then + sudo dnf install -y "${package}" || + error_exit "Failed to install ${package}" + else + error_exit "Unsupported OS" + fi + fi + done + + echo "All OS packages installed successfully." + +} + +# function to download and install binary packages. +# the packages, their respective download locations and compression +# are available in the variable REQUIRED_BINARY_PACKAGES +# the function will download the packages, extract them and install them in /usr/local/bin +# Following are the packages that are installed: +# yq=https://github.com/mikefarah/yq/releases/download/v4.44.2/yq_linux_amd64 +# kubectl=https://storage.googleapis.com/kubernetes-release/release/v1.27.11/bin/linux/amd64/kubectl +# kind=https://kind.sigs.k8s.io/dl/v0.23.0/kind-linux-amd64 + +install_binary_packages() { + # Define the required binary packages + REQUIRED_BINARY_PACKAGES=( + "yq=https://github.com/mikefarah/yq/releases/download/v4.44.2/yq_linux_amd64" + "kubectl=https://storage.googleapis.com/kubernetes-release/release/v1.27.11/bin/linux/amd64/kubectl" + "kind=https://kind.sigs.k8s.io/dl/v0.23.0/kind-linux-amd64" + ) + + # Specify the installation directory + local install_dir="/usr/local/bin" + + # Install the required binary packages + for package_info in "${REQUIRED_BINARY_PACKAGES[@]}"; do + IFS='=' read -r package_name package_url <<<"${package_info}" + download_path="/tmp/${package_name}" + + if [[ -x "${install_dir}/${package_name}" ]]; then + echo "Package ${package_name} is already installed. Skipping." + continue + else + echo "Downloading ${package_name}..." + # Update a hidden file to record what was installed. This is useful for uninstallation + echo "${package_name}" >>"${bin_packages_file}" + curl -sSL "${package_url}" -o "${download_path}" || + error_exit "Failed to download ${package_name}" + + echo "Extracting ${package_name}..." + if [[ "${package_url}" == *.tar.gz ]]; then + sudo tar -xf "${download_path}" -C "${install_dir}" || + error_exit "Failed to extract ${package_name}" + # If not a tar.gz file, then it is a binary file + else + echo "${package_name} is binary file." + sudo mv "${download_path}" "${install_dir}/${package_name}" || + error_exit "Failed to move ${package_name} to ${install_dir}" + fi + + echo "Marking ${install_dir}/${package_name} executable" + sudo chmod +x "${install_dir}/${package_name}" || + error_exit "Failed to mark ${package_name} executable" + + echo "Cleaning up..." + rm -f "${download_path}" + fi + done + + echo "All binary packages installed successfully." + +} + +# function to install golang +install_golang() { + echo "Path to versions.yaml: $VERSIONS_YAML_PATH" + + # When installing, use the required min go version from the versions.yaml file + REQUIRED_GO_VERSION="$(yq '.tools.golang' "$VERSIONS_YAML_PATH")" + # Check if Go is already installed + if [[ -x "$(command -v go)" ]]; then + echo "Go is already installed" + # Check if the installed version is less than the required version + installed_go_version=$(v=$(go version | awk '{print $3}') && echo "${v#go}") + + if version_lt "$installed_go_version" "$REQUIRED_GO_VERSION"; then + echo "Warning: Found ${installed_go_version}, is lower than our required $REQUIRED_GO_VERSION" + echo "Please remove the existing go version and run this script again." + exit 1 + else + echo "Found ${installed_go_version}, good to go" + fi + else + # Install Go + echo "Installing Go" + curl -fsSL https://go.dev/dl/go${REQUIRED_GO_VERSION}.linux-amd64.tar.gz -o go.tar.gz || error_exit "Failed to download Go" + sudo tar -C /usr/local -xzf go.tar.gz || error_exit "Failed to extract Go" + touch "$go_file" + rm -f go.tar.gz + fi + +} + +# function to uninstall golang +uninstall_golang() { + # Check if Go is installed + if [[ ! -x "$(command -v go)" ]]; then + echo "Go is not installed" + return + fi + + # Uninstall Go + echo "Uninstalling Go" + + # Uninstall only if go_file exists + if [[ ! -f "$go_file" ]]; then + echo "Go was not installed using this script. Skipping uninstallation." + return + fi + + sudo rm -rf /usr/local/go + + # Remove the file that records the installed packages + rm -f "$go_file" +} + +# function to install docker +install_docker() { + # Check if Docker is already installed + if [[ -x "$(command -v docker)" ]]; then + echo "Docker is already installed" + else + # Install Docker + echo "Installing Docker" + curl -fsSL https://get.docker.com -o get-docker.sh || error_exit "Failed to download Docker installation script" + sudo sh get-docker.sh || error_exit "Failed to install Docker" + touch "$docker_file" + rm -f get-docker.sh + sudo groupadd docker + sudo usermod -aG docker "$USER" + fi +} + +#function to uninstall the binary packages +uninstall_binary_packages() { + # Remove the installed binary packages + # The packages are available under bin_packages_file file + # The function will remove the packages from install_dir + + # Check if the file exists + if [[ ! -f "${bin_packages_file}" ]]; then + echo "No binary packages to uninstall." + return + fi + + # Read the file and uninstall the packages + while IFS= read -r package_name; do + if [[ -x "${install_dir}/${package_name}" ]]; then + echo "Uninstalling ${package_name}..." + sudo rm -f "${install_dir}/${package_name}" || + error_exit "Failed to uninstall ${package_name}" + + else + echo "Package ${package_name} is not installed. Skipping." + fi + done <"${bin_packages_file}" + + # Remove the file that records the installed packages + rm -f "${bin_packages_file}" + + echo "All binary packages uninstalled successfully." +} + +# function to uninstall the OS packages +uninstall_os_packages() { + # Remove the installed Os packages + # The packages are available under os_packages_file file + # The function will remove the packages using the package manager + + # Check if the file exists + if [[ ! -f "${os_packages_file}" ]]; then + echo "No OS packages to uninstall." + return + fi + + # Read the file and uninstall the packages + while IFS= read -r package_name; do + if [[ -x "$(command -v "${package_name}")" ]]; then + echo "Uninstalling ${package_name}..." + if [[ -x "$(command -v apt-get)" ]]; then + sudo apt-get purge -y "${package_name}" || + error_exit "Failed to uninstall ${package_name}" + elif [[ -x "$(command -v dnf)" ]]; then + sudo dnf remove -y "${package_name}" || + error_exit "Failed to uninstall ${package_name}" + else + error_exit "Unsupported OS" + fi + else + echo "Package ${package_name} is not installed. Skipping." + fi + done <"${os_packages_file}" + + # Remove the file that records the installed packages + rm -f "${os_packages_file}" + + echo "All OS packages uninstalled successfully." +} + +# function to uninstall docker +uninstall_docker() { + # Check if Docker is installed + if [[ ! -x "$(command -v docker)" ]]; then + echo "Docker is not installed" + return + fi + + # Uninstall Docker + echo "Uninstalling Docker" + + # Uninstall only if docker_file exists + if [[ ! -f "$docker_file" ]]; then + echo "Docker was not installed using this script. Skipping uninstallation." + return + fi + # Check if OS is Ubuntu + if [[ -x "$(command -v apt-get)" ]]; then + sudo apt-get purge -y docker-ce docker-ce-cli containerd.io \ + docker-buildx-plugin docker-compose-plugin docker-ce-rootless-extras || error_exit "Failed to uninstall Docker" + sudo rm -rf /var/lib/docker + sudo rm -rf /var/lib/containerd + elif [[ -x "$(command -v dnf)" ]]; then + sudo dnf remove -y docker-ce docker-ce-cli containerd.io \ + docker-buildx-plugin docker-compose-plugin docker-ce-rootless-extras || error_exit "Failed to uninstall Docker" + sudo rm -rf /var/lib/docker + sudo rm -rf /var/lib/containerd + fi + + # Remove the file that records the installed packages + rm -f "$docker_file" +} + +#function to set PATH to include /usr/local/bin and /usr/local/go/bin +set_path() { + + # If bin_packages_file exists, then add /usr/local/bin to PATH + if [[ -f "${bin_packages_file}" ]]; then + # Add /usr/local/bin to PATH + if [[ ":$PATH:" == *":/usr/local/bin:"* ]]; then + echo "/usr/local/bin is already in PATH. Skipping." + else + echo "Adding /usr/local/bin to PATH..." + # shellcheck disable=SC2016 + echo 'export PATH=$PATH:/usr/local/bin' >>"$HOME"/.bashrc + fi + fi + + # If go_file exists, then add /usr/local/go/bin to PATH + if [[ -f "${go_file}" ]]; then + # Add /usr/local/go/bin to PATH + if [[ ":$PATH:" == *":/usr/local/go/bin:"* ]]; then + echo "/usr/local/go/bin is already in PATH. Skipping." + else + echo "Adding /usr/local/go/bin to PATH..." + # shellcheck disable=SC2016 + echo 'export PATH=$PATH:/usr/local/go/bin' >>"$HOME"/.bashrc + fi + fi + + # Reload the bashrc file + # shellcheck source=/dev/null + source "$HOME"/.bashrc + +} + +if [[ "$(uname)" != "Linux" || "$(uname -m)" != "x86_64" ]]; then + error_exit "This script is only for Linux x86_64 OS." +fi + +# Main function +if [[ "$1" == "install" ]]; then + check_sudo + install_os_packages + install_binary_packages + install_golang + install_docker + set_path +elif [[ "$1" == "uninstall" ]]; then + check_sudo + uninstall_binary_packages + uninstall_os_packages + uninstall_golang + uninstall_docker +else + error_exit "Invalid argument. Please provide either 'install' or 'uninstall'." +fi diff --git a/src/cloud-api-adaptor/docs/troubleshooting/nydus-snapshotter.md b/src/cloud-api-adaptor/docs/troubleshooting/nydus-snapshotter.md new file mode 100644 index 000000000..b0fc938d2 --- /dev/null +++ b/src/cloud-api-adaptor/docs/troubleshooting/nydus-snapshotter.md @@ -0,0 +1,53 @@ +# Some tips for debugging nydus-snapshotter related issues + +## Check nydus configuration + +When using `containerd` runtime, CoCo operator configures `nydus-snapshotter` to handle in-guest +container image pull. + +Login to the worker node and run the following command to verify if `nydus-snapshotter` is set up successfully. + +```sh +ctr -a /run/containerd/containerd.sock plugin ls | grep nydus +``` + +You should see an o/p like the following: + +```sh +io.containerd.snapshotter.v1 nydus - ok +``` + +## Pod creation fails with "error unpacking image" + +Sometimes when creating a pod you might encounter the following error: + +```sh +Warning Failed 8m51s (x7 over 10m) kubelet Error: failed to create containerd container: error unpacking image: failed to extract layer sha256:d51af96cf93e225825efd484ea457f867cb2b967f7415b9a3b7e65a2f803838a: failed to get reader from content store: content digest sha256:ec562eabd705d25bfea8c8d79e4610775e375524af00552fe871d3338261563c: not found +``` + +To resolve this, login to the worker node and explicitly fetch the image using the following command: + +```sh +export image=<_set_> +ctr -n k8s.io content fetch $image +``` + +Here is a concrete example describing how to download the images in case of the above error +when running e2e tests: + +```sh +images=( + "quay.io/prometheus/busybox:latest" + "quay.io/confidential-containers/test-images:testworkdir" + "docker.io/library/nginx:latest" + "docker.io/curlimages/curl:8.4.0" + "quay.io/curl/curl:latest" +) + +# Loop through each image in the list +# and download the image +for image in "${images[@]}"; do + ctr -n k8s.io content fetch $image +done +``` + diff --git a/src/cloud-api-adaptor/entrypoint.sh b/src/cloud-api-adaptor/entrypoint.sh index 1ecbe583c..113c2ca9d 100755 --- a/src/cloud-api-adaptor/entrypoint.sh +++ b/src/cloud-api-adaptor/entrypoint.sh @@ -26,7 +26,6 @@ optionals+="" [[ "${SECURE_COMMS_OUTBOUNDS}" ]] && optionals+="-secure-comms-outbounds ${SECURE_COMMS_OUTBOUNDS} " [[ "${SECURE_COMMS_KBS_ADDR}" ]] && optionals+="-secure-comms-kbs ${SECURE_COMMS_KBS_ADDR} " - test_vars() { for i in "$@"; do [ -z "${!i}" ] && echo "\$$i is NOT set" && EXT=1 @@ -166,6 +165,8 @@ docker() { [[ "${DOCKER_TLS_VERIFY}" ]] && optionals+="-docker-tls-verify ${DOCKER_TLS_VERIFY} " [[ "${DOCKER_CERT_PATH}" ]] && optionals+="-docker-cert-path ${DOCKER_CERT_PATH} " [[ "${DOCKER_API_VERSION}" ]] && optionals+="-docker-api-version ${DOCKER_API_VERSION} " + [[ "${DOCKER_PODVM_IMAGE}" ]] && optionals+="-podvm-docker-image ${DOCKER_PODVM_IMAGE} " + [[ "${DOCKER_NETWORK_NAME}" ]] && optionals+="-docker-network-name ${DOCKER_NETWORK_NAME} " set -x exec cloud-api-adaptor docker \ diff --git a/src/cloud-api-adaptor/go.mod b/src/cloud-api-adaptor/go.mod index 65346c790..69c25360f 100644 --- a/src/cloud-api-adaptor/go.mod +++ b/src/cloud-api-adaptor/go.mod @@ -50,6 +50,7 @@ require ( github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers v0.9.0-alpha1-alpha.1 github.com/confidential-containers/cloud-api-adaptor/src/peerpod-ctrl v0.9.0-alpha1-alpha.1 github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f + github.com/docker/docker v25.0.5+incompatible github.com/golang-jwt/jwt/v5 v5.2.1 github.com/moby/sys/mountinfo v0.7.1 github.com/pelletier/go-toml/v2 v2.1.0 @@ -102,7 +103,6 @@ require ( github.com/containerd/typeurl/v2 v2.1.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/distribution/reference v0.6.0 // indirect - github.com/docker/docker v25.0.5+incompatible // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect github.com/docker/go-units v0.5.0 // indirect diff --git a/src/cloud-api-adaptor/install/overlays/docker/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/docker/kustomization.yaml index d0eca0ddf..e0aed3afd 100644 --- a/src/cloud-api-adaptor/install/overlays/docker/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/docker/kustomization.yaml @@ -18,6 +18,12 @@ configMapGenerator: literals: - CLOUD_PROVIDER="docker" - CLOUD_CONFIG_VERIFY="false" # It's better set as true to enable could config verify in production env + #- DOCKER_HOST="unix:///var/run/docker.sock" # Uncomment and set if you want to use a specific docker host + #- DOCKER_API_VERSION="1.44" # Uncomment and set if you want to use a specific docker api version + #- DOCKER_TLS_VERIFY="false" # Uncomment and set if you want to use tls + #- DOCKER_CERT_PATH="" # Uncomment and set if you want to use tls + #- DOCKER_PODVM_IMAGE="quay.io/confidential-containers/podvm-docker-image" # Uncomment and set if you want to use a specific podvm image + #- DOCKER_NETWORK_NAME="bridge" # Uncomment and set if you want to use a specific docker network #- PAUSE_IMAGE="" # Uncomment and set if you want to use a specific pause image #- VXLAN_PORT="" # Uncomment and set if you want to use a specific vxlan port. Defaults to 4789 ##TLS_SETTINGS diff --git a/src/cloud-api-adaptor/test/e2e/README.md b/src/cloud-api-adaptor/test/e2e/README.md index 4cdf07b27..d61a87068 100644 --- a/src/cloud-api-adaptor/test/e2e/README.md +++ b/src/cloud-api-adaptor/test/e2e/README.md @@ -50,9 +50,10 @@ To use existing cluster which have already installed Cloud API Adaptor, you shou ## Attestation and KBS specific -We need artificats from trustee when do attestation test, to prepare trustee, do as following. +We need artifacts from the trustee repo when doing the attestation tests. +To prepare trustee, execute the following steps: -``` +```sh pushd ${cloud-api-adaptor-repo-dir}/src/cloud-api-adaptor/test git clone https://github.com/confidential-containers/trustee.git pushd trustee @@ -65,15 +66,18 @@ popd ``` We need build and use the PodVM image: -``` + +```sh pushd ${cloud-api-adaptor} make podvm-builder podvm-binaries podvm-image popd ``` + Then extract the PodVM image and use it following [extracting-the-qcow2-image](../../podvm/README.md#extracting-the-qcow2-image) -To deploy the KBS service and test attestation related cases, export following variables like: -``` +To deploy the KBS service and test attestation related cases, export the following variables like: + +```sh export DEPLOY_KBS=yes export KBS_IMAGE=$(./hack/yq-shim.sh '.oci.kbs.registry' ./versions.yaml) export KBS_IMAGE_TAG=$(./hack/yq-shim.sh '.oci.kbs.tag' ./versions.yaml) @@ -81,7 +85,7 @@ export KBS_IMAGE_TAG=$(./hack/yq-shim.sh '.oci.kbs.tag' ./versions.yaml) ## Provision file specifics -As mentioned on the previous section, a properties file can be passed to the cloud provisioner that will be used to controll the provisioning operations. The properties are specific of each cloud provider though, see on the sections below. +As mentioned on the previous section, a properties file can be passed to the cloud provisioner that will be used to control the provisioning operations. The properties are specific of each cloud provider though, see on the sections below. ### AWS provision properties diff --git a/src/cloud-api-adaptor/test/e2e/docker_common.go b/src/cloud-api-adaptor/test/e2e/docker_common.go new file mode 100644 index 000000000..217f8fbdc --- /dev/null +++ b/src/cloud-api-adaptor/test/e2e/docker_common.go @@ -0,0 +1,53 @@ +//go:build docker + +// (C) Copyright Confidential Containers Contributors +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/client" +) + +// DockerAssert implements the CloudAssert interface for Docker. +type DockerAssert struct { + // TODO: create the connection once on the initializer. + //conn client.Connect +} + +func (c DockerAssert) DefaultTimeout() time.Duration { + return 1 * time.Minute +} + +func (l DockerAssert) HasPodVM(t *testing.T, id string) { + conn, err := client.NewClientWithOpts(client.FromEnv) + if err != nil { + t.Fatal(err) + } + + // Check if the container is running + containers, err := conn.ContainerList(context.Background(), types.ContainerListOptions{}) + if err != nil { + t.Fatal(err) + } + + for _, container := range containers { + if strings.Contains(container.Names[0], id) { + return + } + } + + // It didn't find the PodVM if it reached here. + t.Error("PodVM was not created") +} + +func (l DockerAssert) GetInstanceType(t *testing.T, podName string) (string, error) { + // Get Instance Type of PodVM + return "", nil +} diff --git a/src/cloud-api-adaptor/test/e2e/docker_test.go b/src/cloud-api-adaptor/test/e2e/docker_test.go new file mode 100644 index 000000000..13423d9e8 --- /dev/null +++ b/src/cloud-api-adaptor/test/e2e/docker_test.go @@ -0,0 +1,110 @@ +//go:build docker + +// (C) Copyright Confidential Containers Contributors +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "testing" + + _ "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/test/provisioner/docker" +) + +func TestDockerCreateSimplePod(t *testing.T) { + assert := DockerAssert{} + DoTestCreateSimplePod(t, testEnv, assert) +} + +func TestDockerCreatePodWithConfigMap(t *testing.T) { + SkipTestOnCI(t) + assert := DockerAssert{} + DoTestCreatePodWithConfigMap(t, testEnv, assert) +} + +func TestDockerCreatePodWithSecret(t *testing.T) { + assert := DockerAssert{} + DoTestCreatePodWithSecret(t, testEnv, assert) +} + +func TestDockerCreatePeerPodContainerWithExternalIPAccess(t *testing.T) { + SkipTestOnCI(t) + assert := DockerAssert{} + DoTestCreatePeerPodContainerWithExternalIPAccess(t, testEnv, assert) + +} + +func TestDockerCreatePeerPodWithJob(t *testing.T) { + assert := DockerAssert{} + DoTestCreatePeerPodWithJob(t, testEnv, assert) +} + +func TestDockerCreatePeerPodAndCheckUserLogs(t *testing.T) { + assert := DockerAssert{} + DoTestCreatePeerPodAndCheckUserLogs(t, testEnv, assert) +} + +func TestDockerCreatePeerPodAndCheckWorkDirLogs(t *testing.T) { + assert := DockerAssert{} + DoTestCreatePeerPodAndCheckWorkDirLogs(t, testEnv, assert) +} + +func TestDockerCreatePeerPodAndCheckEnvVariableLogsWithImageOnly(t *testing.T) { + // This test is causing issues on CI with instability, so skip until we can resolve this. + // See https://github.com/confidential-containers/cloud-api-adaptor/issues/1831 + SkipTestOnCI(t) + assert := DockerAssert{} + DoTestCreatePeerPodAndCheckEnvVariableLogsWithImageOnly(t, testEnv, assert) +} + +func TestDockerCreatePeerPodAndCheckEnvVariableLogsWithDeploymentOnly(t *testing.T) { + assert := DockerAssert{} + DoTestCreatePeerPodAndCheckEnvVariableLogsWithDeploymentOnly(t, testEnv, assert) +} + +func TestDockerCreatePeerPodAndCheckEnvVariableLogsWithImageAndDeployment(t *testing.T) { + // This test is causing issues on CI with instability, so skip until we can resolve this. + // See https://github.com/confidential-containers/cloud-api-adaptor/issues/1831 + assert := DockerAssert{} + DoTestCreatePeerPodAndCheckEnvVariableLogsWithImageAndDeployment(t, testEnv, assert) +} + +func TestDockerCreateNginxDeployment(t *testing.T) { + assert := DockerAssert{} + DoTestNginxDeployment(t, testEnv, assert) +} + +/* +Failing due to issues will pulling image (ErrImagePull) +func TestDockerCreatePeerPodWithLargeImage(t *testing.T) { + assert := DockerAssert{} + DoTestCreatePeerPodWithLargeImage(t, testEnv, assert) +} +*/ + +func TestDockerDeletePod(t *testing.T) { + assert := DockerAssert{} + DoTestDeleteSimplePod(t, testEnv, assert) +} + +func TestDockerPodToServiceCommunication(t *testing.T) { + assert := DockerAssert{} + DoTestPodToServiceCommunication(t, testEnv, assert) +} + +func TestDockerPodsMTLSCommunication(t *testing.T) { + assert := DockerAssert{} + DoTestPodsMTLSCommunication(t, testEnv, assert) +} + +func TestDockerKbsKeyRelease(t *testing.T) { + if !isTestWithKbs() { + t.Skip("Skipping kbs related test as kbs is not deployed") + } + _ = keyBrokerService.EnableKbsCustomizedPolicy("deny_all.rego") + assert := DockerAssert{} + t.Parallel() + DoTestKbsKeyReleaseForFailure(t, testEnv, assert) + _ = keyBrokerService.EnableKbsCustomizedPolicy("allow_all.rego") + DoTestKbsKeyRelease(t, testEnv, assert) +} diff --git a/src/cloud-api-adaptor/test/provisioner/docker/provision.go b/src/cloud-api-adaptor/test/provisioner/docker/provision.go new file mode 100644 index 000000000..325a6940c --- /dev/null +++ b/src/cloud-api-adaptor/test/provisioner/docker/provision.go @@ -0,0 +1,15 @@ +//go:build docker + +// (C) Copyright Confidential Containers Contributors +// SPDX-License-Identifier: Apache-2.0 + +package docker + +import ( + pv "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/test/provisioner" +) + +func init() { + pv.NewProvisionerFunctions["docker"] = NewDockerProvisioner + pv.NewInstallOverlayFunctions["docker"] = NewDockerInstallOverlay +} diff --git a/src/cloud-api-adaptor/test/provisioner/docker/provision_common.go b/src/cloud-api-adaptor/test/provisioner/docker/provision_common.go new file mode 100644 index 000000000..676646b31 --- /dev/null +++ b/src/cloud-api-adaptor/test/provisioner/docker/provision_common.go @@ -0,0 +1,263 @@ +// (C) Copyright Confidential Containers Contributors +// SPDX-License-Identifier: Apache-2.0 + +package docker + +import ( + "context" + "fmt" + + "os" + "os/exec" + "path/filepath" + + pv "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/test/provisioner" + "github.com/containerd/containerd/reference" + "github.com/docker/docker/client" + log "github.com/sirupsen/logrus" + + "sigs.k8s.io/e2e-framework/pkg/envconf" +) + +// DockerProvisioner implements the CloudProvisioner interface for Docker. +type DockerProvisioner struct { + conn *client.Client + wd string // docker's directory path on this repository +} + +// DockerInstallOverlay implements the InstallOverlay interface +type DockerInstallOverlay struct { + Overlay *pv.KustomizeOverlay +} + +type DockerProperties struct { + DockerHost string + ApiVer string + ClusterName string + NetworkName string + PodvmImage string + CaaImage string + CaaImageTag string + KbsImage string + KbsImageTag string +} + +var DockerProps = &DockerProperties{} + +func initDockerProperties(properties map[string]string) error { + + DockerProps = &DockerProperties{ + DockerHost: properties["DOCKER_HOST"], + ApiVer: properties["DOCKER_API_VERSION"], + ClusterName: properties["CLUSTER_NAME"], + NetworkName: properties["DOCKER_NETWORK_NAME"], + PodvmImage: properties["DOCKER_PODVM_IMAGE"], + CaaImage: properties["CAA_IMAGE"], + CaaImageTag: properties["CAA_IMAGE_TAG"], + KbsImage: properties["KBS_IMAGE"], + KbsImageTag: properties["KBS_IMAGE_TAG"], + } + return nil +} + +func NewDockerProvisioner(properties map[string]string) (pv.CloudProvisioner, error) { + wd, err := filepath.Abs(filepath.Join("..", "..", "docker")) + if err != nil { + log.Errorf("Error getting the absolute path of the docker directory: %v", err) + return nil, err + } + + if err := initDockerProperties(properties); err != nil { + return nil, err + } + + // set environment variables + os.Setenv("DOCKER_HOST", DockerProps.DockerHost) + + conn, err := client.NewClientWithOpts(client.FromEnv) + if err != nil { + log.Errorf("Error creating the Docker client: %v", err) + return nil, err + } + + return &DockerProvisioner{ + conn: conn, + wd: wd, + }, nil +} + +func (l *DockerProvisioner) CreateCluster(ctx context.Context, cfg *envconf.Config) error { + + // Create kind cluster + if err := createKindCluster(l.wd); err != nil { + log.Errorf("Error creating Kind cluster: %v", err) + return err + } + + home, _ := os.UserHomeDir() + kubeconfig := filepath.Join(home, ".kube/config") + cfg.WithKubeconfigFile(kubeconfig) + + if err := pv.AddNodeRoleWorkerLabel(ctx, DockerProps.ClusterName, cfg); err != nil { + + return fmt.Errorf("labeling nodes: %w", err) + } + + return nil +} + +func (l *DockerProvisioner) DeleteCluster(ctx context.Context, cfg *envconf.Config) error { + + if err := deleteKindCluster(l.wd); err != nil { + log.Errorf("Error deleting Kind cluster: %v", err) + return err + } + return nil +} + +func (l *DockerProvisioner) CreateVPC(ctx context.Context, cfg *envconf.Config) error { + // TODO: delete the resources created on CreateVPC() that currently only checks + // the Docker's storage and network exist. + return nil +} + +func (l *DockerProvisioner) DeleteVPC(ctx context.Context, cfg *envconf.Config) error { + // TODO: delete the resources created on CreateVPC() that currently only checks + // the Docker's storage and network exist. + return nil +} + +func (l *DockerProvisioner) GetProperties(ctx context.Context, cfg *envconf.Config) map[string]string { + return map[string]string{ + "DOCKER_HOST": DockerProps.DockerHost, + "DOCKER_API_VERSION": DockerProps.ApiVer, + "CLUSTER_NAME": DockerProps.ClusterName, + "DOCKER_NETWORK_NAME": DockerProps.NetworkName, + "DOCKER_PODVM_IMAGE": DockerProps.PodvmImage, + "CAA_IMAGE": DockerProps.CaaImage, + "CAA_IMAGE_TAG": DockerProps.CaaImageTag, + "KBS_IMAGE": DockerProps.KbsImage, + "KBS_IMAGE_TAG": DockerProps.KbsImageTag, + } +} + +func (l *DockerProvisioner) UploadPodvm(imagePath string, ctx context.Context, cfg *envconf.Config) error { + + // Download the podvm image from the registry by using docker pull + cmd := exec.Command("/bin/bash", "-c", "docker pull "+DockerProps.PodvmImage) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err := cmd.Run() + if err != nil { + log.Errorf("Error pulling the podvm image: %v", err) + return err + } + + return nil +} + +func createKindCluster(workingDir string) error { + // Create kind cluster by executing the script on the node + cmd := exec.Command("/bin/bash", "-c", "./kind_cluster.sh create") + cmd.Dir = workingDir + cmd.Stdout = os.Stdout + // TODO: better handle stderr. Messages getting out of order. + cmd.Stderr = os.Stderr + cmd.Env = os.Environ() + // Set CLUSTER_NAME if available. Also unset KUBECONFIG so that the default path is used. + cmd.Env = append(cmd.Env, "CLUSTER_NAME="+DockerProps.ClusterName, "KUBECONFIG=") + err := cmd.Run() + if err != nil { + log.Errorf("Error creating Kind cluster: %v", err) + return err + } + return nil +} + +func deleteKindCluster(workingDir string) error { + // Delete kind cluster by executing the script on the node + cmd := exec.Command("/bin/bash", "-c", "./kind_cluster.sh delete") + cmd.Dir = workingDir + cmd.Stdout = os.Stdout + // TODO: better handle stderr. Messages getting out of order. + cmd.Stderr = os.Stderr + err := cmd.Run() + if err != nil { + log.Errorf("Error deleting Kind cluster: %v", err) + return err + } + return nil +} + +func NewDockerInstallOverlay(installDir, provider string) (pv.InstallOverlay, error) { + overlay, err := pv.NewKustomizeOverlay(filepath.Join(installDir, "overlays", provider)) + if err != nil { + log.Errorf("Error creating the docker provider install overlay: %v", err) + return nil, err + } + + return &DockerInstallOverlay{ + Overlay: overlay, + }, nil +} + +func isDockerKustomizeConfigMapKey(key string) bool { + switch key { + case "CLOUD_PROVIDER", "DOCKER_HOST", "DOCKER_API_VERSION", "DOCKER_PODVM_IMAGE", "DOCKER_NETWORK_NAME", "AA_KBC_PARAMS": + return true + default: + return false + } +} + +func (lio *DockerInstallOverlay) Apply(ctx context.Context, cfg *envconf.Config) error { + return lio.Overlay.Apply(ctx, cfg) +} + +func (lio *DockerInstallOverlay) Delete(ctx context.Context, cfg *envconf.Config) error { + return lio.Overlay.Delete(ctx, cfg) +} + +// Update install/overlays/docker/kustomization.yaml +func (lio *DockerInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config, properties map[string]string) error { + + // If a custom image is defined then update it in the kustomization file. + if DockerProps.CaaImage != "" { + spec, err := reference.Parse(DockerProps.CaaImage) + if err != nil { + return fmt.Errorf("parsing image: %w", err) + } + + log.Infof("Updating CAA image with %q", spec.Locator) + if err = lio.Overlay.SetKustomizeImage("cloud-api-adaptor", "newName", spec.Locator); err != nil { + return err + } + } + + if DockerProps.CaaImageTag != "" { + spec, err := reference.Parse(DockerProps.CaaImageTag) + if err != nil { + return fmt.Errorf("parsing image tag: %w", err) + } + log.Infof("Updating CAA image tag with %q", spec.Locator) + if err = lio.Overlay.SetKustomizeImage("cloud-api-adaptor", "newTag", spec.Locator); err != nil { + return err + } + } + + for k, v := range properties { + // configMapGenerator + if isDockerKustomizeConfigMapKey(k) { + if err := lio.Overlay.SetKustomizeConfigMapGeneratorLiteral("peer-pods-cm", k, v); err != nil { + return err + } + } + } + + if err := lio.Overlay.YamlReload(); err != nil { + return err + } + + return nil + +} diff --git a/src/cloud-api-adaptor/test/provisioner/docker/provision_docker.properties b/src/cloud-api-adaptor/test/provisioner/docker/provision_docker.properties new file mode 100644 index 000000000..01483011f --- /dev/null +++ b/src/cloud-api-adaptor/test/provisioner/docker/provision_docker.properties @@ -0,0 +1,11 @@ +# Docker configs +CLUSTER_NAME="peer-pods" +DOCKER_HOST="unix:///var/run/docker.sock" +DOCKER_PODVM_IMAGE="quay.io/confidential-containers/podvm-docker-image" +DOCKER_NETWORK_NAME="kind" +CAA_IMAGE="" +CAA_IMAGE_TAG="" + +# KBS configs +KBS_IMAGE="" +KBS_IMAGE_TAG="" diff --git a/src/cloud-providers/docker/docker.go b/src/cloud-providers/docker/docker.go index d8b795ad8..887c43724 100644 --- a/src/cloud-providers/docker/docker.go +++ b/src/cloud-providers/docker/docker.go @@ -13,6 +13,7 @@ import ( // eg. - https://github.com/moby/moby/blob/v25.0.5/vendor.mod "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" "github.com/docker/go-connections/nat" ) @@ -20,10 +21,14 @@ import ( // The default podvm docker image to use const defaultPodVMDockerImage = "quay.io/confidential-containers/podvm-docker-image" +// Default docker network name to connect to +const defaultDockerNetworkName = "bridge" + // Method to create and start a container // Returns the container ID and the IP address of the container func createContainer(ctx context.Context, client *client.Client, - instanceName string, volumeBinding []string) (string, string, error) { + instanceName string, volumeBinding []string, + podvmImage string, networkName string) (string, string, error) { // No need to bind the port to the host portBinding := nat.PortMap{} @@ -32,7 +37,7 @@ func createContainer(ctx context.Context, client *client.Client, resp, err := client.ContainerCreate( ctx, &container.Config{ - Image: defaultPodVMDockerImage, + Image: podvmImage, ExposedPorts: nat.PortSet{ "15150/tcp": struct{}{}, }, @@ -42,7 +47,14 @@ func createContainer(ctx context.Context, client *client.Client, Binds: volumeBinding, Privileged: true, // This line is added to create a privileged container }, - nil, nil, instanceName, + // Connect to specific network name + &network.NetworkingConfig{ + EndpointsConfig: map[string]*network.EndpointSettings{ + networkName: {}, + }, + }, + + nil, instanceName, ) if err != nil { return "", "", err @@ -61,7 +73,11 @@ func createContainer(ctx context.Context, client *client.Client, return "", "", err } - return resp.ID, inspect.NetworkSettings.IPAddress, nil + // Get the IP address of the container from the network settings + // networks: map[network-name: {IPAddress: ip-address}] + // The network name is the key in the networks map + + return resp.ID, inspect.NetworkSettings.Networks[networkName].IPAddress, nil } diff --git a/src/cloud-providers/docker/manager.go b/src/cloud-providers/docker/manager.go index 22f960d41..e5920ef80 100644 --- a/src/cloud-providers/docker/manager.go +++ b/src/cloud-providers/docker/manager.go @@ -26,6 +26,9 @@ func (_ *Manager) ParseCmd(flags *flag.FlagSet) { flags.StringVar(&dockerCfg.DockerCertPath, "docker-cert-path", "", "Path to directory with Docker TLS certificates") flags.BoolVar(&dockerCfg.DockerTLSVerify, "docker-tls-verify", false, "Use TLS and verify the remote server certificate") flags.StringVar(&dockerCfg.DataDir, "data-dir", defaultDataDir, "docker storage dir") + flags.StringVar(&dockerCfg.PodVMDockerImage, "podvm-docker-image", defaultPodVMDockerImage, "Docker image to use for podvm") + // Docker network name to connect to + flags.StringVar(&dockerCfg.NetworkName, "docker-network-name", defaultDockerNetworkName, "Docker network name to connect to") } func (m *Manager) LoadEnv() { diff --git a/src/cloud-providers/docker/provider.go b/src/cloud-providers/docker/provider.go index feea1b4e7..b7ba0e0b9 100644 --- a/src/cloud-providers/docker/provider.go +++ b/src/cloud-providers/docker/provider.go @@ -20,8 +20,10 @@ import ( var logger = log.New(log.Writer(), "[adaptor/cloud/docker] ", log.LstdFlags|log.Lmsgprefix) type dockerProvider struct { - Client *client.Client - DataDir string + Client *client.Client + DataDir string + PodVMDockerImage string + NetworkName string } const maxInstanceNameLen = 63 @@ -42,8 +44,10 @@ func NewProvider(config *Config) (*dockerProvider, error) { } return &dockerProvider{ - Client: cli, - DataDir: config.DataDir, + Client: cli, + DataDir: config.DataDir, + PodVMDockerImage: config.PodVMDockerImage, + NetworkName: config.NetworkName, }, nil } @@ -86,7 +90,8 @@ func (p *dockerProvider) CreateInstance(ctx context.Context, podName, sandboxID volumeBinding = append(volumeBinding, fmt.Sprintf("%s:%s", filepath.Join(p.DataDir, "image"), "/image")) - instanceID, ip, err := createContainer(ctx, p.Client, instanceName, volumeBinding) + instanceID, ip, err := createContainer(ctx, p.Client, instanceName, volumeBinding, + p.PodVMDockerImage, p.NetworkName) if err != nil { return nil, err } diff --git a/src/cloud-providers/docker/types.go b/src/cloud-providers/docker/types.go index 7f675f2de..b06cf3dbe 100644 --- a/src/cloud-providers/docker/types.go +++ b/src/cloud-providers/docker/types.go @@ -9,4 +9,6 @@ type Config struct { DockerCertPath string DockerTLSVerify bool DataDir string + PodVMDockerImage string + NetworkName string }