fix: Skip IPAddress check for windows containers

e2e/fixtures/win-pod.yml

@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: vk-e2e-windows
+  namespace: vk-test
+spec:
+  nodeName: TEST_WINDOWS_NODE_NAME
+  containers:
+  - image: mcr.microsoft.com/windows/nanoserver:1809
+    imagePullPolicy: Always
+    name: nanoserver
+    command: [
+      "ping",
+      "-t",
+      "localhost",
+    ]
+  nodeSelector:
+    kubernetes.io/role: agent
+    beta.kubernetes.io/os: windows
+    type: virtual-kubelet
+  tolerations:
+    - key: virtual-kubelet.io/provider
+      operator: Exists

e2e/pods_test.go

@@ -77,7 +77,7 @@ func TestPodLifecycle(t *testing.T) {
 
 	// check container logs
 	t.Log("get container logs ....")
-	cmd = kubectl("logs", "pod/vk-e2e-hpa", "-c", "hpa-example", "--namespace=vk-test")
+	cmd = kubectl("logs", "pod/vk-e2e-hpa", "-c", "hpa-example", "--namespace=vk-test", "--tail=5")
 	out, err = cmd.CombinedOutput()
 	if err != nil {
 		t.Fatal(string(out))

e2e/pods_win_test.go

@@ -0,0 +1,87 @@
+package e2e
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"testing"
+	"time"
+)
+
+func TestWindowsPodLifecycle(t *testing.T) {
+	// delete the namespace first
+	cmd := kubectl("delete", "namespace", "vk-test", "--ignore-not-found")
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatal(string(out))
+	}
+
+	// create namespace
+	cmd = kubectl("apply", "-f", "fixtures/namespace.yml")
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatal(string(out))
+	}
+
+	winNodeName := os.Getenv("TEST_WINDOWS_NODE_NAME")
+	cmd = exec.Command("sed", "-i", "-e", fmt.Sprintf("s|TEST_WINDOWS_NODE_NAME|%s|g", winNodeName), "fixtures/win-pod.yml")
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatal(string(out))
+	}
+
+	cmd = kubectl("apply", "-f", "fixtures/win-pod.yml", "--namespace=vk-test")
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatal(string(out))
+	}
+
+	deadline, ok := t.Deadline()
+	timeout := time.Until(deadline)
+	if !ok {
+		timeout = 300 * time.Second
+	}
+	cmd = kubectl("wait", "--for=condition=ready", "--timeout="+timeout.String(), "pod/vk-e2e-windows", "--namespace=vk-test")
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatal(string(out))
+	}
+	t.Log("success create pod")
+
+	// check container logs
+	t.Log("get container logs ....")
+	cmd = kubectl("logs", "pod/vk-e2e-windows", "--namespace=vk-test", "--tail=5")
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatal(string(out))
+	}
+	if string(out) == "" {
+		t.Fatal("failed to get container's logs")
+	}
+	t.Logf("success query container logs %s", string(out))
+
+	// check pod status
+	t.Log("get pod status ....")
+	cmd = kubectl("get", "pod", "--field-selector=status.phase=Running", "--namespace=vk-test", "--output=jsonpath={.items..metadata.name}")
+	out, err = cmd.CombinedOutput()
+	if err != nil {
+		t.Fatal(string(out))
+	}
+	if string(out) != "vk-e2e-windows" {
+		t.Fatal("failed to get pod's status")
+	}
+	t.Logf("success query pod status %s", string(out))
+
+	// check container status
+	t.Log("get container status ....")
+	cmd = kubectl("get", "pod", "vk-e2e-windows", "--namespace=vk-test", "--output=jsonpath={.status.containerStatuses[0].ready}")
+	out, err = cmd.CombinedOutput()
+	if err != nil {
+		t.Fatal(string(out))
+	}
+	if string(out) != "true" {
+		t.Fatal("failed to get pod's status")
+	}
+	t.Logf("success query container status %s", string(out))
+
+	t.Log("clean up pod")
+	cmd = kubectl("delete", "namespace", "vk-test", "--ignore-not-found")
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatal(string(out))
+	}
+}

hack/e2e/aks-addon.sh

@@ -24,7 +24,9 @@ fi
 : "${CLUSTER_NAME:=${RESOURCE_GROUP}}"
 : "${NODE_COUNT:=1}"
 : "${CHART_NAME:=aks-addon--test}"
+: "${WIN_CHART_NAME:=vk-aci-test-win-aks}"
 : "${TEST_NODE_NAME:=vk-aci-test-aks}"
+: "${TEST_WINDOWS_NODE_NAME:=vk-aci-test-win-aks}"
 : "${IMG_REPO:=oss/virtual-kubelet/virtual-kubelet}"
 : "${IMG_URL:=mcr.microsoft.com}"
 : "${VNET_RANGE=10.0.0.0/8}"
@@ -164,6 +166,30 @@ kubectl wait --for=condition=Ready --timeout=300s node "$TEST_NODE_NAME"
 
 export TEST_NODE_NAME
 
+## Windows VK
+helm install \
+    --kubeconfig="${KUBECONFIG}" \
+    --set nodeOsType=Windows \
+    --set "image.repository=${IMG_URL}"  \
+    --set "image.name=${IMG_REPO}" \
+    --set "image.tag=${IMG_TAG}" \
+    --set "nodeName=${TEST_WINDOWS_NODE_NAME}" \
+    --set "providers.azure.masterUri=$MASTER_URI" \
+    --set "providers.azure.managedIdentityID=$ACI_USER_IDENTITY" \
+    "$WIN_CHART_NAME" \
+    ./charts/virtual-kubelet
+
+kubectl wait --for=condition=available deploy "${TEST_WINDOWS_NODE_NAME}-virtual-kubelet-azure-aci" -n vk-azure-aci --timeout=300s
+
+while true; do
+    kubectl get node "$TEST_WINDOWS_NODE_NAME" &> /dev/null && break
+    sleep 3
+done
+
+kubectl wait --for=condition=Ready --timeout=300s node "$TEST_WINDOWS_NODE_NAME"
+
+export TEST_WINDOWS_NODE_NAME=$TEST_WINDOWS_NODE_NAME
+
 ## CSI Driver test
 az storage account create -n $CSI_DRIVER_STORAGE_ACCOUNT_NAME -g $RESOURCE_GROUP -l $LOCATION --sku Standard_LRS
 export AZURE_STORAGE_CONNECTION_STRING=$(az storage account show-connection-string -n $CSI_DRIVER_STORAGE_ACCOUNT_NAME -g "$RESOURCE_GROUP" -o tsv)

hack/e2e/aks.sh

@@ -24,7 +24,9 @@ fi
 : "${CLUSTER_NAME:=${RESOURCE_GROUP}}"
 : "${NODE_COUNT:=1}"
 : "${CHART_NAME:=vk-aci-test-aks}"
+: "${WIN_CHART_NAME:=vk-aci-test-win-aks}"
 : "${TEST_NODE_NAME:=vk-aci-test-aks}"
+: "${TEST_WINDOWS_NODE_NAME:=vk-aci-test-win-aks}"
 : "${IMG_REPO:=oss/virtual-kubelet/virtual-kubelet}"
 : "${IMG_URL:=mcr.microsoft.com}"
 
@@ -172,6 +174,7 @@ export KUBECONFIG="${TMPDIR}/kubeconfig"
 
 MASTER_URI="$(kubectl cluster-info | awk '/Kubernetes control plane/{print $7}' | sed "s,\x1B\[[0-9;]*[a-zA-Z],,g")"
 
+## Linux VK
 helm install \
     --kubeconfig="${KUBECONFIG}" \
     --set "image.repository=${IMG_URL}"  \
@@ -198,6 +201,30 @@ kubectl wait --for=condition=Ready --timeout=300s node "$TEST_NODE_NAME"
 
 export TEST_NODE_NAME
 
+## Windows VK
+helm install \
+    --kubeconfig="${KUBECONFIG}" \
+    --set nodeOsType=Windows \
+    --set "image.repository=${IMG_URL}"  \
+    --set "image.name=${IMG_REPO}" \
+    --set "image.tag=${IMG_TAG}" \
+    --set "nodeName=${TEST_WINDOWS_NODE_NAME}" \
+    --set "providers.azure.masterUri=$MASTER_URI" \
+    --set "providers.azure.managedIdentityID=$node_identity" \
+    "$WIN_CHART_NAME" \
+    ./charts/virtual-kubelet
+
+kubectl wait --for=condition=available deploy "${TEST_WINDOWS_NODE_NAME}-virtual-kubelet-azure-aci" -n vk-azure-aci --timeout=300s
+
+while true; do
+    kubectl get node "$TEST_WINDOWS_NODE_NAME" &> /dev/null && break
+    sleep 3
+done
+
+kubectl wait --for=condition=Ready --timeout=300s node "$TEST_WINDOWS_NODE_NAME"
+
+export TEST_WINDOWS_NODE_NAME
+
 ## CSI Driver test
 az storage account create -n $CSI_DRIVER_STORAGE_ACCOUNT_NAME -g $RESOURCE_GROUP -l $LOCATION --sku Standard_LRS
 export AZURE_STORAGE_CONNECTION_STRING=$(az storage account show-connection-string -n $CSI_DRIVER_STORAGE_ACCOUNT_NAME -g $RESOURCE_GROUP -o tsv)

pkg/network/aci_network.go

@@ -169,7 +169,10 @@ func (pn *ProviderNetwork) AmendVnetResources(ctx context.Context, cg client2.Co
 	subnetID := "/subscriptions/" + pn.VnetSubscriptionID + "/resourceGroups/" + pn.VnetResourceGroup + "/providers/Microsoft.Network/virtualNetworks/" + pn.VnetName + "/subnets/" + pn.SubnetName
 	cgIDList := []azaci.ContainerGroupSubnetID{{ID: &subnetID}}
 	cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.SubnetIds = &cgIDList
-	cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.DNSConfig = getDNSConfig(ctx, pod, pn.KubeDNSIP, clusterDomain)
+	// windows containers don't support DNS config
+	if cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.OsType != azaci.OperatingSystemTypesWindows {
+		cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.DNSConfig = getDNSConfig(ctx, pod, pn.KubeDNSIP, clusterDomain)
+	}
 }
 
 func getDNSConfig(ctx context.Context, pod *v1.Pod, kubeDNSIP, clusterDomain string) *azaci.DNSConfiguration {

pkg/provider/aci.go

@@ -256,9 +256,12 @@ func NewACIProvider(ctx context.Context, config string, azConfig auth.Config, az
 	}
 
 	if p.providernetwork.SubnetName != "" {
-		err = p.setACIExtensions(ctx)
-		if err != nil {
-			return nil, err
+		// windows containers don't support kube-proxy nor realtime metrics
+		if p.operatingSystem != string(azaci.OperatingSystemTypesWindows) {
+			err = p.setACIExtensions(ctx)
+			if err != nil {
+				return nil, err
+			}
 		}
 	}
 
@@ -364,7 +367,10 @@ func (p *ACIProvider) CreatePod(ctx context.Context, pod *v1.Pod) error {
 
 	p.providernetwork.AmendVnetResources(ctx, *cg, pod, p.clusterDomain)
 
-	cg.ContainerGroupPropertiesWrapper.Extensions = p.containerGroupExtensions
+	// windows containers don't support kube-proxy nor realtime metrics
+	if cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.OsType != azaci.OperatingSystemTypesWindows {
+		cg.ContainerGroupPropertiesWrapper.Extensions = p.containerGroupExtensions
+	}
 
 	log.G(ctx).Infof("start creating pod %v", pod.Name)
 	// TODO: Run in a go routine to not block workers, and use tracker.UpdatePodStatus() based on result.

pkg/provider/containergroup_to_pod.go

@@ -81,13 +81,17 @@ func (p *ACIProvider) getPodStatusFromContainerGroup(cg *azaci.ContainerGroup) (
 		return nil, err
 	}
 
+	podIp := ""
+	if cg.OsType != azaci.OperatingSystemTypesWindows {
+		podIp = *cg.IPAddress.IP
+	}
 	return &v1.PodStatus{
 		Phase:             getPodPhaseFromACIState(*aciState),
 		Conditions:        getPodConditionsFromACIState(*aciState, creationTime, lastUpdateTime, allReady),
 		Message:           "",
 		Reason:            "",
 		HostIP:            p.internalIP,
-		PodIP:             *cg.IPAddress.IP,
+		PodIP:             podIp,
 		StartTime:         &firstContainerStartTime,
 		ContainerStatuses: containerStatuses,
 	}, nil
@@ -122,6 +126,15 @@ func aciContainerStateToContainerState(cs *azaci.ContainerState) v1.ContainerSta
 				FinishedAt: metav1.NewTime(cs.FinishTime.Time),
 			},
 		}
+		// Handle windows container with no prev state
+	case "Pending":
+		return v1.ContainerState{
+			Waiting: &v1.ContainerStateWaiting{
+				Reason:  *cs.State,
+				Message: *cs.DetailStatus,
+			},
+		}
+
 	default:
 		// Handle the case where the container is pending.
 		// Which should be all other aci states.

pkg/validation/validator.go

@@ -29,11 +29,10 @@ func ValidateContainer(container containerinstance.Container) error {
 		return errors.Errorf("container %s properties CurrentState StartTime cannot be nil", *container.Name)
 	}
 	if container.InstanceView.PreviousState == nil {
-		emptyStr := ""
+		pendingState := "Pending"
 		container.InstanceView.PreviousState = &containerinstance.ContainerState{
-			State:        &emptyStr,
-			StartTime:    container.InstanceView.CurrentState.StartTime,
-			DetailStatus: &emptyStr,
+			State:        &pendingState,
+			DetailStatus: &pendingState,
 		}
 		return nil
 	}
@@ -66,16 +65,18 @@ func ValidateContainerGroup(cg *containerinstance.ContainerGroup) error {
 	if cg.Tags == nil {
 		return errors.Errorf("tags list cannot be nil for container group %s", *cg.Name)
 	}
-	if cg.IPAddress == nil {
-		return errors.Errorf("IPAddress cannot be nil for container group %s", *cg.Name)
-	} else {
-		aciState := *cg.ContainerGroupProperties.ProvisioningState
-		if cg.IPAddress.IP == nil {
-			if aciState == "Running" {
-				return errors.Errorf("podIP cannot be nil for container group %s while state is %s ", *cg.Name, aciState)
-			} else {
-				emptyIP := ""
-				cg.IPAddress.IP = &emptyIP
+	if cg.OsType != containerinstance.OperatingSystemTypesWindows {
+		if cg.IPAddress == nil {
+			return errors.Errorf("IPAddress cannot be nil for container group %s", *cg.Name)
+		} else {
+			aciState := *cg.ContainerGroupProperties.ProvisioningState
+			if cg.IPAddress.IP == nil {
+				if aciState == "Running" {
+					return errors.Errorf("podIP cannot be nil for container group %s while state is %s ", *cg.Name, aciState)
+				} else {
+					emptyIP := ""
+					cg.IPAddress.IP = &emptyIP
+				}
 			}
 		}
 	}