docker: Implement e2e tests for docker provider

Initial framework to run e2e tests for docker provider

The tests provisions docker (if not present), creates a 2 node kind cluster
and then runs the tests.

Signed-off-by: Pradipta Banerjee <pradipta.banerjee@gmail.com>

src/cloud-api-adaptor/docker/README.md

@@ -80,9 +80,24 @@ For changing the CAA image to your custom built image (eg. `quay.io/myuser/cloud
 you can use the following:
 
 ```bash
-kubectl set image ds/cloud-api-adaptor-daemonset -n confidential-containers-system cloud-api-adaptor-con=quay.io/myuser/cloud-api-adaptor
+export CAA_IMAGE=quay.io/myuser/cloud-api-adaptor
+kubectl set image ds/cloud-api-adaptor-daemonset -n confidential-containers-system cloud-api-adaptor-con="$CAA_IMAGE"
 ```
 
+## Running the CAA e2e tests
+
+Edit the file `src/cloud-api-adaptor/docker/provision_docker.properties` and update the `CAA_IMAGE`
+and `CAA_IMAGE_TAG` variables with your custom CAA image and tag.
+
+You can run the CAA e2e [tests/e2e/README.md](../test/e2e/README.md) by running the following command:
+
+```sh
+cd src/cloud-api-adaptor
+make TEST_PROVISION=yes CLOUD_PROVIDER=docker TEST_PROVISION_FILE=$(pwd)/docker/provision_docker.properties test-e2e
+```
+
+You can modify the variables defined in provision_docker.properties if required.
+
 ## Run sample application
 
 ### Ensure runtimeclass is present
@@ -159,5 +174,4 @@ For debugging you can use docker commands like `docker ps`, `docker logs`, `dock
 
 ```sh
 kubectl delete deployment nginx
-```
-
+```

src/cloud-api-adaptor/docker/install_docker.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+# Accept two arguments: install and uninstall
+
+# Install Docker
+if [ "$1" == "install" ]; then
+    # Check if Docker is already installed
+    if [ -x "$(command -v docker)" ]; then
+        echo "Docker is already installed"
+    else
+        # Install Docker
+        echo "Installing Docker"
+        curl -fsSL https://get.docker.com -o get-docker.sh || exit 1
+        sudo sh get-docker.sh || exit 1
+        sudo groupadd docker
+        sudo usermod -aG docker $USER
+    fi
+    exit 0
+fi
+# Uninstall Docker
+if [ "$1" == "uninstall" ]; then
+    # Check if Docker is installed
+    if [ ! -x "$(command -v docker)" ]; then
+        echo "Docker is not installed"
+        exit 0
+    fi
+
+    # Uninstall Docker
+    echo "Uninstalling Docker"
+    # Check if OS is Ubuntu
+    if [ -x "$(command -v apt-get)" ]; then
+        sudo apt-get purge -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin docker-ce-rootless-extras
+        sudo rm -rf /var/lib/docker
+        sudo rm -rf /var/lib/containerd
+        exit 0
+    elif [ -x "$(command -v dnf)" ]; then
+        sudo dnf remove -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin docker-ce-rootless-extras
+        sudo rm -rf /var/lib/docker
+        sudo rm -rf /var/lib/containerd
+        exit 0
+    fi
+
+    exit 0
+fi

src/cloud-api-adaptor/docker/kind-config.yaml

@@ -0,0 +1,16 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+networking:
+  disableDefaultCNI: true # disable kindnet
+  podSubnet: 192.168.0.0/16 # set to Calico's default subnet
+nodes:
+  - role: control-plane
+    # Same image version as used for pod VM base image
+    image: kindest/node:v1.27.11
+  - role: worker
+    image: kindest/node:v1.27.11
+    extraMounts:
+      - hostPath: /var/run/docker.sock
+        containerPath: /var/run/docker.sock
+      - hostPath: /var/lib/docker
+        containerPath: /var/lib/docker

src/cloud-api-adaptor/docker/kind_cluster.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+# Ref: https://stackoverflow.com/questions/299728/how-do-you-use-newgrp-in-a-script-then-stay-in-that-group-when-the-script-exits
+newgrp docker <<EOF
+
+# Accept two arguments: create and delete
+# create: creates a kind cluster
+# delete: deletes a kind cluster
+
+CLUSTER_NAME="${CLUSTER_NAME:-kind}"
+
+if [ "$1" == "create" ]; then
+    echo "Check if kind is already installed"
+    if [ -x "$(command -v kind)" ]; then
+        echo "kind is already installed"
+    else
+        # Install kind
+        echo "Installing kind"
+        curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.23.0/kind-linux-amd64 || exit 1
+        chmod +x ./kind
+        sudo mv ./kind /usr/local/bin/kind
+    fi
+
+    echo "Check if the cluster \$CLUSTER_NAME already exists"
+    if kind get clusters | grep -q "\$CLUSTER_NAME"; then
+        echo "Cluster \$CLUSTER_NAME already exists"
+        exit 0
+    fi
+    # Set some sysctls
+    # Ref: https://kind.sigs.k8s.io/docs/user/known-issues/#pod-errors-due-to-too-many-open-files
+    sudo sysctl fs.inotify.max_user_watches=524288
+    sudo sysctl fs.inotify.max_user_instances=512
+
+    # Create a kind cluster
+    echo "Creating a kind cluster"
+    kind create cluster --name "\$CLUSTER_NAME" --config kind-config.yaml || exit 1
+
+    # Deploy calico
+    kubectl apply -f https://raw.githubusercontent.com/projectcalico/calico/v3.28.0/manifests/calico.yaml || exit 1
+
+    exit 0
+fi
+
+if [ "$1" == "delete" ]; then
+    # Check if kind is installed
+    if [ ! -x "$(command -v kind)" ]; then
+        echo "kind is not installed"
+        exit 0
+    fi
+
+    # Delete the kind cluster
+    echo "Deleting the kind cluster"
+    kind delete cluster --name "\$CLUSTER_NAME" || exit 1
+
+    # Uninstall kind
+    echo "Uninstalling kind"
+    sudo rm -f /usr/local/bin/kind
+
+    exit 0
+fi
+EOF

src/cloud-api-adaptor/docker/provision_docker.properties

@@ -0,0 +1,11 @@
+# Docker configs
+CLUSTER_NAME="peer-pods"
+DOCKER_HOST="unix:///var/run/docker.sock"
+DOCKER_PODVM_IMAGE="quay.io/confidential-containers/podvm-docker-image"
+DOCKER_NETWORK_NAME="kind"
+CAA_IMAGE=""
+CAA_IMAGE_TAG=""
+
+# KBS configs
+KBS_IMAGE=""
+KBS_IMAGE_TAG=""

src/cloud-api-adaptor/go.mod

@@ -50,6 +50,7 @@ require (
 	github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers v0.8.2
 	github.com/confidential-containers/cloud-api-adaptor/src/peerpod-ctrl v0.8.2
 	github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f
+	github.com/docker/docker v25.0.5+incompatible
 	github.com/golang-jwt/jwt/v5 v5.0.0
 	github.com/moby/sys/mountinfo v0.7.1
 	github.com/pelletier/go-toml/v2 v2.1.0
@@ -104,7 +105,6 @@ require (
 	github.com/containerd/typeurl/v2 v2.1.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/distribution/reference v0.6.0 // indirect
-	github.com/docker/docker v25.0.5+incompatible // indirect
 	github.com/docker/go-connections v0.5.0 // indirect
 	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
 	github.com/docker/go-units v0.5.0 // indirect

src/cloud-api-adaptor/test/e2e/docker_common.go

@@ -0,0 +1,53 @@
+//go:build docker
+
+// (C) Copyright Confidential Containers Contributors
+// SPDX-License-Identifier: Apache-2.0
+
+package e2e
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/client"
+)
+
+// DockerAssert implements the CloudAssert interface for Docker.
+type DockerAssert struct {
+	// TODO: create the connection once on the initializer.
+	//conn client.Connect
+}
+
+func (c DockerAssert) DefaultTimeout() time.Duration {
+	return 1 * time.Minute
+}
+
+func (l DockerAssert) HasPodVM(t *testing.T, id string) {
+	conn, err := client.NewClientWithOpts(client.FromEnv)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Check if the container is running
+	containers, err := conn.ContainerList(context.Background(), types.ContainerListOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, container := range containers {
+		if strings.Contains(container.Names[0], id) {
+			return
+		}
+	}
+
+	// It didn't find the PodVM if it reached here.
+	t.Error("PodVM was not created")
+}
+
+func (l DockerAssert) GetInstanceType(t *testing.T, podName string) (string, error) {
+	// Get Instance Type of PodVM
+	return "", nil
+}

src/cloud-api-adaptor/test/e2e/docker_test.go

@@ -0,0 +1,110 @@
+//go:build docker
+
+// (C) Copyright Confidential Containers Contributors
+// SPDX-License-Identifier: Apache-2.0
+
+package e2e
+
+import (
+	"testing"
+
+	_ "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/test/provisioner/docker"
+)
+
+func TestDockerCreateSimplePod(t *testing.T) {
+	assert := DockerAssert{}
+	DoTestCreateSimplePod(t, testEnv, assert)
+}
+
+func TestDockerCreatePodWithConfigMap(t *testing.T) {
+	SkipTestOnCI(t)
+	assert := DockerAssert{}
+	DoTestCreatePodWithConfigMap(t, testEnv, assert)
+}
+
+func TestDockerCreatePodWithSecret(t *testing.T) {
+	assert := DockerAssert{}
+	DoTestCreatePodWithSecret(t, testEnv, assert)
+}
+
+func TestDockerCreatePeerPodContainerWithExternalIPAccess(t *testing.T) {
+	SkipTestOnCI(t)
+	assert := DockerAssert{}
+	DoTestCreatePeerPodContainerWithExternalIPAccess(t, testEnv, assert)
+
+}
+
+func TestDockerCreatePeerPodWithJob(t *testing.T) {
+	assert := DockerAssert{}
+	DoTestCreatePeerPodWithJob(t, testEnv, assert)
+}
+
+func TestDockerCreatePeerPodAndCheckUserLogs(t *testing.T) {
+	assert := DockerAssert{}
+	DoTestCreatePeerPodAndCheckUserLogs(t, testEnv, assert)
+}
+
+func TestDockerCreatePeerPodAndCheckWorkDirLogs(t *testing.T) {
+	assert := DockerAssert{}
+	DoTestCreatePeerPodAndCheckWorkDirLogs(t, testEnv, assert)
+}
+
+func TestDockerCreatePeerPodAndCheckEnvVariableLogsWithImageOnly(t *testing.T) {
+	// This test is causing issues on CI with instability, so skip until we can resolve this.
+	// See https://github.com/confidential-containers/cloud-api-adaptor/issues/1831
+	SkipTestOnCI(t)
+	assert := DockerAssert{}
+	DoTestCreatePeerPodAndCheckEnvVariableLogsWithImageOnly(t, testEnv, assert)
+}
+
+func TestDockerCreatePeerPodAndCheckEnvVariableLogsWithDeploymentOnly(t *testing.T) {
+	assert := DockerAssert{}
+	DoTestCreatePeerPodAndCheckEnvVariableLogsWithDeploymentOnly(t, testEnv, assert)
+}
+
+func TestDockerCreatePeerPodAndCheckEnvVariableLogsWithImageAndDeployment(t *testing.T) {
+	// This test is causing issues on CI with instability, so skip until we can resolve this.
+	// See https://github.com/confidential-containers/cloud-api-adaptor/issues/1831
+	assert := DockerAssert{}
+	DoTestCreatePeerPodAndCheckEnvVariableLogsWithImageAndDeployment(t, testEnv, assert)
+}
+
+func TestDockerCreateNginxDeployment(t *testing.T) {
+	assert := DockerAssert{}
+	DoTestNginxDeployment(t, testEnv, assert)
+}
+
+/*
+Failing due to issues will pulling image (ErrImagePull)
+func TestDockerCreatePeerPodWithLargeImage(t *testing.T) {
+	assert := DockerAssert{}
+	DoTestCreatePeerPodWithLargeImage(t, testEnv, assert)
+}
+*/
+
+func TestDockerDeletePod(t *testing.T) {
+	assert := DockerAssert{}
+	DoTestDeleteSimplePod(t, testEnv, assert)
+}
+
+func TestDockerPodToServiceCommunication(t *testing.T) {
+	assert := DockerAssert{}
+	DoTestPodToServiceCommunication(t, testEnv, assert)
+}
+
+func TestDockerPodsMTLSCommunication(t *testing.T) {
+	assert := DockerAssert{}
+	DoTestPodsMTLSCommunication(t, testEnv, assert)
+}
+
+func TestDockerKbsKeyRelease(t *testing.T) {
+	if !isTestWithKbs() {
+		t.Skip("Skipping kbs related test as kbs is not deployed")
+	}
+	_ = keyBrokerService.EnableKbsCustomizedPolicy("deny_all.rego")
+	assert := DockerAssert{}
+	t.Parallel()
+	DoTestKbsKeyReleaseForFailure(t, testEnv, assert)
+	_ = keyBrokerService.EnableKbsCustomizedPolicy("allow_all.rego")
+	DoTestKbsKeyRelease(t, testEnv, assert)
+}

src/cloud-api-adaptor/test/provisioner/docker/provision.go

@@ -0,0 +1,15 @@
+//go:build docker
+
+// (C) Copyright Confidential Containers Contributors
+// SPDX-License-Identifier: Apache-2.0
+
+package docker
+
+import (
+	pv "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/test/provisioner"
+)
+
+func init() {
+	pv.NewProvisionerFunctions["docker"] = NewDockerProvisioner
+	pv.NewInstallOverlayFunctions["docker"] = NewDockerInstallOverlay
+}