feat: InitContainer support for ACI Connector (#204)

Co-authored-by: suselva <101893421+suselva@users.noreply.github.com>
Co-authored-by: Arnav Arnav <fnuarnav@microsoft.com>

README.md

@@ -27,7 +27,7 @@ This document details configuring the Virtual Kubelet ACI provider.
 
 Virtual Kubelet's ACI provider relies heavily on the feature set that Azure Container Instances provide. Please check the Azure documentation accurate details on region availability, pricing and new features. The list here attempts to give an accurate reference for the features we support in ACI and the ACI provider within Virtual Kubelet.
 
-### features
+### Features
 
 * Volumes: empty dir, github repo, projection, Azure Files, Azure Files CSI drivers
 * Secure env variables, config maps
@@ -36,6 +36,7 @@ Virtual Kubelet's ACI provider relies heavily on the feature set that Azure Cont
 * Basic Azure Networking support within AKS virtual node
 * [Exec support](https://docs.microsoft.com/azure/container-instances/container-instances-exec) for container instances
 * Azure Monitor integration or formally known as OMS
+* Support for init-containers ([use init containers](#Create-pod-with-init-containers))
 
 ### Limitations
 
@@ -44,7 +45,6 @@ Virtual Kubelet's ACI provider relies heavily on the feature set that Azure Cont
 * [Limitations](https://docs.microsoft.com/azure/container-instances/container-instances-vnet) with VNet
 * VNet peering
 * Argument support for exec
-* Init containers
 * [Host aliases](https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/) support
 * downward APIs (i.e podIP)
 
@@ -650,6 +650,28 @@ Output:
 ```
 -->
 
+### Create pod with init containers
+Multiple init containers can be specified in the podspec similar to how containers are specified
+
+```yaml
+spec:
+  initContainers:
+  - image: <INIT CONTAINER IMAGE 1>
+    name: init-container-01
+    command: [ "/bin/sh" ]
+    args: [ "-c", "echo \"Hi\"" ]
+  - image: <INIT CONTAINER IMAGE 2>
+    name: init-container-02
+    command: [ "/bin/sh" ]
+    args: [ "-c", "echo \"Hi\"" ]
+  containers:
+  - image: <CONTAINER IMAGE>
+    imagePullPolicy: Always
+    name: container
+    command: [ "/bin/sh" ]
+```
+More information on init containers can be found in [Kubernetes](https://kubernetes.io/docs/concepts/workloads/pods/init-containers/) and [ACI](https://docs.microsoft.com/en-us/azure/container-instances/container-instances-init-container) documentations
+
 ## Work around for the virtual kubelet pod
 
 If your pod that's scheduled onto the Virtual Kubelet node is in a pending state please add this workaround to your Virtual Kubelet pod spec.

e2e/fixtures/initcontainers_ordertest_pod.yml

@@ -0,0 +1,51 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: vk-e2e-initcontainers-order
+  namespace: vk-test
+spec:
+  initContainers:
+  - image: alpine
+    name: init-container-01
+    command: [ "/bin/sh" ]
+    args: [ "-c", "echo Hi from init-container-01 >> /mnt/azure/newfile.txt" ]
+    volumeMounts:
+      - name: azure
+        mountPath: /mnt/azure
+  - image: alpine
+    name: init-container-02
+    command: [ "/bin/sh" ]
+    args: [ "-c", "echo Hi from init-container-02 >> /mnt/azure/newfile.txt" ]
+    volumeMounts:
+      - name: azure
+        mountPath: /mnt/azure
+  containers:
+  - image: alpine
+    imagePullPolicy: Always
+    name: container
+    command: [
+        "sh",
+        "-c",
+        "echo Hi from container >> /mnt/azure/newfile.txt; while sleep 10; do cat /mnt/azure/newfile.txt; done;"
+    ]
+    resources:
+      requests:
+        memory: 1G
+        cpu: 1
+    volumeMounts:
+      - name: azure
+        mountPath: /mnt/azure
+  nodeSelector:
+    kubernetes.io/role: agent
+    beta.kubernetes.io/os: linux
+    type: virtual-kubelet
+  tolerations:
+  - key: virtual-kubelet.io/provider
+    operator: Exists
+  volumes:
+  - name: azure
+    csi:
+      driver: file.csi.azure.com
+      volumeAttributes:
+        secretName: csidriversecret  # required
+        shareName: vncsidriversharename  # required

e2e/initcontainer_test.go

@@ -0,0 +1,113 @@
+
+package e2e
+
+import (
+	"testing"
+	"time"
+	"io/ioutil"
+	"os/exec"
+	"os"
+
+	"gotest.tools/assert"
+)
+
+func TestPodWithInitContainersOrder(t *testing.T) {
+	// delete the namespace first
+	cmd := kubectl("delete", "namespace", "vk-test", "--ignore-not-found")
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatal(string(out))
+	}
+
+	// create namespace
+	cmd = kubectl("apply", "-f", "fixtures/namespace.yml")
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatal(string(out))
+	}
+
+	testStorageAccount := os.Getenv("CSI_DRIVER_STORAGE_ACCOUNT_NAME")
+	testStorageKey := os.Getenv("CSI_DRIVER_STORAGE_ACCOUNT_KEY")
+
+	cmd = kubectl("create", "secret", "generic", "csidriversecret", "--from-literal", "azurestorageaccountname="+testStorageAccount, "--from-literal", "azurestorageaccountkey="+testStorageKey, "--namespace=vk-test")
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatal(string(out))
+	}
+
+	cmd = kubectl("apply", "-f", "fixtures/initcontainers_ordertest_pod.yml")
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatal(string(out))
+	}
+	deadline, ok := t.Deadline()
+	timeout := time.Until(deadline)
+	if !ok {
+		timeout = 300 * time.Second
+	}
+	cmd = kubectl("wait", "--for=condition=ready", "--timeout="+timeout.String(), "pod/vk-e2e-initcontainers-order", "--namespace=vk-test")
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatal(string(out))
+	}
+	t.Log("success create pod")
+
+	// query metrics
+	deadline = time.Now().Add(10 * time.Minute)
+	for {
+		t.Log("query metrics ....")
+		cmd = kubectl("get", "--raw", "/apis/metrics.k8s.io/v1beta1/namespaces/vk-test/pods/vk-e2e-initcontainers-order")
+		out, err := cmd.CombinedOutput()
+		if time.Now().After(deadline) {
+			t.Fatal("failed to query pod's stats from metrics server API")
+		}
+		if err == nil {
+			t.Logf("success query metrics %s", string(out))
+			break
+		}
+		time.Sleep(10 * time.Second)
+	}
+
+	// download file created by pod
+	cmd = exec.Command("az", "storage", "file", "download", "--account-name", testStorageAccount, "--account-key", testStorageKey, "-s", "vncsidriversharename", "-p", "newfile.txt")
+	cmd.Env = os.Environ()
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatal(string(out))
+	}
+	t.Log("file newfile.txt downloaded from storage account")
+
+	file, err := ioutil.ReadFile("newfile.txt")
+	if err != nil {
+		t.Fatal("could not read downloaded file")
+	}
+	t.Log("read file content successfully")
+
+	fileContent := string(file)
+	expectedString := "Hi from init-container-01\nHi from init-container-02\nHi from container\n"
+	assert.Equal(t, fileContent, expectedString, "file content doesn't match expected value")
+
+	// check pod status
+	t.Log("get pod status ....")
+	cmd = kubectl("get", "pod", "--field-selector=status.phase=Running", "--namespace=vk-test", "--output=jsonpath={.items..metadata.name}")
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatal(string(out))
+	}
+	if string(out) != "vk-e2e-initcontainers-order" {
+		t.Fatal("failed to get pod's status")
+	}
+	t.Logf("success query pod status %s", string(out))
+
+	// check container status
+	t.Log("get container status ....")
+	cmd = kubectl("get", "pod", "vk-e2e-initcontainers-order", "--namespace=vk-test", "--output=jsonpath={.status.containerStatuses[0].ready}")
+	out, err = cmd.CombinedOutput()
+	if err != nil {
+		t.Fatal(string(out))
+	}
+	if string(out) != "true" {
+		t.Fatal("failed to get pod's status")
+	}
+	t.Logf("success query container status %s", string(out))
+
+	t.Log("clean up pod")
+	cmd = kubectl("delete", "namespace", "vk-test", "--ignore-not-found")
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatal(string(out))
+	}
+}

e2e/pods_test.go

@@ -18,6 +18,7 @@ func TestPodLifecycle(t *testing.T) {
 	if out, err := cmd.CombinedOutput(); err != nil {
 		t.Fatal(string(out))
 	}
+
 	cmd = kubectl("apply", "-f", "fixtures/hpa.yml", "--namespace=vk-test")
 	if out, err := cmd.CombinedOutput(); err != nil {
 		t.Fatal(string(out))
@@ -99,9 +100,10 @@ func TestPodLifecycle(t *testing.T) {
 	}
 	t.Log("success query exec on the container")
 
-	t.Log("clean up pod")
+	t.Log("clean up")
 	cmd = kubectl("delete", "namespace", "vk-test", "--ignore-not-found")
 	if out, err := cmd.CombinedOutput(); err != nil {
 		t.Fatal(string(out))
 	}
 }
+

hack/e2e/aks.sh

@@ -20,7 +20,7 @@ if [ "$PR_RAND" = "" ]; then
 fi
 
 : "${RESOURCE_GROUP:=vk-aci-test-$RANDOM_NUM}"
-: "${LOCATION:=westus2}"
+: "${LOCATION:=eastus2}"
 : "${CLUSTER_NAME:=${RESOURCE_GROUP}}"
 : "${NODE_COUNT:=1}"
 : "${CHART_NAME:=vk-aci-test-aks}"

pkg/provider/aci.go

@@ -303,7 +303,15 @@ func (p *ACIProvider) CreatePod(ctx context.Context, pod *v1.Pod) error {
 		return err
 
 	}
+
+	// get initContainers
+	initContainers, err := p.getInitContainers(ctx, pod)
+	if err != nil {
+		return err
+	}
+
 	// assign all the things
+	cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.InitContainers = &initContainers
 	cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Containers = containers
 	cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Volumes = &volumes
 	cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.ImageRegistryCredentials = creds
@@ -840,6 +848,91 @@ func readDockerConfigJSONSecret(secret *v1.Secret, ips []azaci.ImageRegistryCred
 	return ips, err
 }
 
+//verify if Container is properly declared for the use on ACI
+func (p *ACIProvider) verifyContainer(container *v1.Container) error {
+	if len(container.Command) == 0 && len(container.Args) > 0 {
+		return errdefs.InvalidInput("ACI does not support providing args without specifying the command. Please supply both command and args to the pod spec.")
+	}
+	return nil
+}
+
+//this method is used for both initConainers and containers
+func (p *ACIProvider) getCommand(container *v1.Container) *[]string {
+	command := append(container.Command, container.Args...)
+	return &command
+}
+
+//get VolumeMounts declared on Container as []aci.VolumeMount
+func (p *ACIProvider) getVolumeMounts(container *v1.Container) *[]azaci.VolumeMount {
+	volumeMounts := make([]azaci.VolumeMount, 0, len(container.VolumeMounts))
+	for i := range container.VolumeMounts {
+		volumeMounts = append(volumeMounts, azaci.VolumeMount{
+			Name:      &container.VolumeMounts[i].Name,
+			MountPath: &container.VolumeMounts[i].MountPath,
+			ReadOnly:  &container.VolumeMounts[i].ReadOnly,
+		})
+	}
+	return &volumeMounts
+}
+
+//get EnvironmentVariables declared on Container as []aci.EnvironmentVariable
+func (p *ACIProvider) getEnvironmentVariables(container *v1.Container) *[]azaci.EnvironmentVariable {
+	environmentVariable := make([]azaci.EnvironmentVariable, 0, len(container.Env))
+	for i := range container.Env {
+		if container.Env[i].Value != "" {
+			envVar := getACIEnvVar(container.Env[i])
+			environmentVariable = append(environmentVariable, envVar)
+		}
+	}
+	return &environmentVariable
+}
+
+//get InitContainers defined in Pod as []aci.InitContainerDefinition
+func (p *ACIProvider) getInitContainers(ctx context.Context, pod *v1.Pod) ([]azaci.InitContainerDefinition, error) {
+	initContainers := make([]azaci.InitContainerDefinition, 0, len(pod.Spec.InitContainers))
+	for i, initContainer := range pod.Spec.InitContainers {
+		err := p.verifyContainer(&initContainer)
+		if err != nil {
+			log.G(ctx).Errorf("couldn't verify container %v", err)
+			return nil, err
+		}
+
+		if initContainer.Ports != nil {
+			log.G(ctx).Errorf("azure container instances initcontainers do not support ports")
+			return nil, errdefs.InvalidInput("azure container instances initContainers do not support ports")
+		}
+		if initContainer.Resources.Requests != nil {
+			log.G(ctx).Errorf("azure container instances initcontainers do not support resources requests")
+			return nil, errdefs.InvalidInput("azure container instances initContainers do not support resources requests")
+		}
+		if initContainer.Resources.Limits != nil {
+			log.G(ctx).Errorf("azure container instances initcontainers do not support resources limits")
+			return nil, errdefs.InvalidInput("azure container instances initContainers do not support resources limits")
+		}
+		if initContainer.LivenessProbe != nil {
+			log.G(ctx).Errorf("azure container instances initcontainers do not support livenessProbe")
+			return nil, errdefs.InvalidInput("azure container instances initContainers do not support livenessProbe")
+		}
+		if initContainer.ReadinessProbe != nil {
+			log.G(ctx).Errorf("azure container instances initcontainers do not support readinessProbe")
+			return nil, errdefs.InvalidInput("azure container instances initContainers do not support readinessProbe")
+		}
+
+		newInitContainer := azaci.InitContainerDefinition{
+			Name: &pod.Spec.InitContainers[i].Name,
+			InitContainerPropertiesDefinition: &azaci.InitContainerPropertiesDefinition {
+				Image: &pod.Spec.InitContainers[i].Image,
+				Command: p.getCommand(&pod.Spec.InitContainers[i]),
+				VolumeMounts: p.getVolumeMounts(&pod.Spec.InitContainers[i]),
+				EnvironmentVariables: p.getEnvironmentVariables(&pod.Spec.InitContainers[i]),
+			},
+		}
+
+		initContainers = append(initContainers, newInitContainer)
+	}
+	return initContainers, nil
+}
+
 func (p *ACIProvider) getContainers(pod *v1.Pod) (*[]azaci.Container, error) {
 	containers := make([]azaci.Container, 0, len(pod.Spec.Containers))