From b5074a873f90d2f2f1ba34e2c303a0bde8a74434 Mon Sep 17 00:00:00 2001 From: Heba Elayoty Date: Thu, 1 Dec 2022 11:43:49 -0800 Subject: [PATCH] Skip IPAddress check for windows containers Signed-off-by: Heba Elayoty --- e2e/fixtures/win-pod.yml | 23 +++++++ e2e/pods_test.go | 2 +- e2e/pods_win_test.go | 87 +++++++++++++++++++++++++++ hack/e2e/aks-addon.sh | 26 ++++++++ hack/e2e/aks.sh | 27 +++++++++ pkg/network/aci_network.go | 5 +- pkg/provider/aci.go | 14 +++-- pkg/provider/containergroup_to_pod.go | 15 ++++- pkg/validation/validator.go | 29 ++++----- 9 files changed, 207 insertions(+), 21 deletions(-) create mode 100644 e2e/fixtures/win-pod.yml create mode 100644 e2e/pods_win_test.go diff --git a/e2e/fixtures/win-pod.yml b/e2e/fixtures/win-pod.yml new file mode 100644 index 00000000..67757447 --- /dev/null +++ b/e2e/fixtures/win-pod.yml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Pod +metadata: + name: vk-e2e-windows + namespace: vk-test +spec: + nodeName: TEST_WINDOWS_NODE_NAME + containers: + - image: mcr.microsoft.com/windows/nanoserver:1809 + imagePullPolicy: Always + name: nanoserver + command: [ + "ping", + "-t", + "localhost", + ] + nodeSelector: + kubernetes.io/role: agent + beta.kubernetes.io/os: windows + type: virtual-kubelet + tolerations: + - key: virtual-kubelet.io/provider + operator: Exists diff --git a/e2e/pods_test.go b/e2e/pods_test.go index 0966236e..59ffe98e 100644 --- a/e2e/pods_test.go +++ b/e2e/pods_test.go @@ -77,7 +77,7 @@ func TestPodLifecycle(t *testing.T) { // check container logs t.Log("get container logs ....") - cmd = kubectl("logs", "pod/vk-e2e-hpa", "-c", "hpa-example", "--namespace=vk-test") + cmd = kubectl("logs", "pod/vk-e2e-hpa", "-c", "hpa-example", "--namespace=vk-test", "--tail=5") out, err = cmd.CombinedOutput() if err != nil { t.Fatal(string(out)) diff --git a/e2e/pods_win_test.go b/e2e/pods_win_test.go new file mode 100644 index 00000000..e0bec8dc --- /dev/null +++ b/e2e/pods_win_test.go @@ -0,0 +1,87 @@ +package e2e + +import ( + "fmt" + "os" + "os/exec" + "testing" + "time" +) + +func TestWindowsPodLifecycle(t *testing.T) { + // delete the namespace first + cmd := kubectl("delete", "namespace", "vk-test", "--ignore-not-found") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatal(string(out)) + } + + // create namespace + cmd = kubectl("apply", "-f", "fixtures/namespace.yml") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatal(string(out)) + } + + winNodeName := os.Getenv("TEST_WINDOWS_NODE_NAME") + cmd = exec.Command("sed", "-i", "-e", fmt.Sprintf("s|TEST_WINDOWS_NODE_NAME|%s|g", winNodeName), "fixtures/win-pod.yml") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatal(string(out)) + } + + cmd = kubectl("apply", "-f", "fixtures/win-pod.yml", "--namespace=vk-test") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatal(string(out)) + } + + deadline, ok := t.Deadline() + timeout := time.Until(deadline) + if !ok { + timeout = 300 * time.Second + } + cmd = kubectl("wait", "--for=condition=ready", "--timeout="+timeout.String(), "pod/vk-e2e-windows", "--namespace=vk-test") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatal(string(out)) + } + t.Log("success create pod") + + // check container logs + t.Log("get container logs ....") + cmd = kubectl("logs", "pod/vk-e2e-windows", "--namespace=vk-test", "--tail=5") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + if string(out) == "" { + t.Fatal("failed to get container's logs") + } + t.Logf("success query container logs %s", string(out)) + + // check pod status + t.Log("get pod status ....") + cmd = kubectl("get", "pod", "--field-selector=status.phase=Running", "--namespace=vk-test", "--output=jsonpath={.items..metadata.name}") + out, err = cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + if string(out) != "vk-e2e-windows" { + t.Fatal("failed to get pod's status") + } + t.Logf("success query pod status %s", string(out)) + + // check container status + t.Log("get container status ....") + cmd = kubectl("get", "pod", "vk-e2e-windows", "--namespace=vk-test", "--output=jsonpath={.status.containerStatuses[0].ready}") + out, err = cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + if string(out) != "true" { + t.Fatal("failed to get pod's status") + } + t.Logf("success query container status %s", string(out)) + + t.Log("clean up pod") + cmd = kubectl("delete", "namespace", "vk-test", "--ignore-not-found") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatal(string(out)) + } +} diff --git a/hack/e2e/aks-addon.sh b/hack/e2e/aks-addon.sh index da926c32..128269d5 100755 --- a/hack/e2e/aks-addon.sh +++ b/hack/e2e/aks-addon.sh @@ -24,7 +24,9 @@ fi : "${CLUSTER_NAME:=${RESOURCE_GROUP}}" : "${NODE_COUNT:=1}" : "${CHART_NAME:=aks-addon--test}" +: "${WIN_CHART_NAME:=vk-aci-test-win-aks}" : "${TEST_NODE_NAME:=vk-aci-test-aks}" +: "${TEST_WINDOWS_NODE_NAME:=vk-aci-test-win-aks}" : "${IMG_REPO:=oss/virtual-kubelet/virtual-kubelet}" : "${IMG_URL:=mcr.microsoft.com}" : "${VNET_RANGE=10.0.0.0/8}" @@ -164,6 +166,30 @@ kubectl wait --for=condition=Ready --timeout=300s node "$TEST_NODE_NAME" export TEST_NODE_NAME +## Windows VK +helm install \ + --kubeconfig="${KUBECONFIG}" \ + --set nodeOsType=Windows \ + --set "image.repository=${IMG_URL}" \ + --set "image.name=${IMG_REPO}" \ + --set "image.tag=${IMG_TAG}" \ + --set "nodeName=${TEST_WINDOWS_NODE_NAME}" \ + --set "providers.azure.masterUri=$MASTER_URI" \ + --set "providers.azure.managedIdentityID=$ACI_USER_IDENTITY" \ + "$WIN_CHART_NAME" \ + ./charts/virtual-kubelet + +kubectl wait --for=condition=available deploy "${TEST_WINDOWS_NODE_NAME}-virtual-kubelet-azure-aci" -n vk-azure-aci --timeout=300s + +while true; do + kubectl get node "$TEST_WINDOWS_NODE_NAME" &> /dev/null && break + sleep 3 +done + +kubectl wait --for=condition=Ready --timeout=300s node "$TEST_WINDOWS_NODE_NAME" + +export TEST_WINDOWS_NODE_NAME=$TEST_WINDOWS_NODE_NAME + ## CSI Driver test az storage account create -n $CSI_DRIVER_STORAGE_ACCOUNT_NAME -g $RESOURCE_GROUP -l $LOCATION --sku Standard_LRS export AZURE_STORAGE_CONNECTION_STRING=$(az storage account show-connection-string -n $CSI_DRIVER_STORAGE_ACCOUNT_NAME -g "$RESOURCE_GROUP" -o tsv) diff --git a/hack/e2e/aks.sh b/hack/e2e/aks.sh index 2a16e7d7..44d67969 100755 --- a/hack/e2e/aks.sh +++ b/hack/e2e/aks.sh @@ -24,7 +24,9 @@ fi : "${CLUSTER_NAME:=${RESOURCE_GROUP}}" : "${NODE_COUNT:=1}" : "${CHART_NAME:=vk-aci-test-aks}" +: "${WIN_CHART_NAME:=vk-aci-test-win-aks}" : "${TEST_NODE_NAME:=vk-aci-test-aks}" +: "${TEST_WINDOWS_NODE_NAME:=vk-aci-test-win-aks}" : "${IMG_REPO:=oss/virtual-kubelet/virtual-kubelet}" : "${IMG_URL:=mcr.microsoft.com}" @@ -172,6 +174,7 @@ export KUBECONFIG="${TMPDIR}/kubeconfig" MASTER_URI="$(kubectl cluster-info | awk '/Kubernetes control plane/{print $7}' | sed "s,\x1B\[[0-9;]*[a-zA-Z],,g")" +## Linux VK helm install \ --kubeconfig="${KUBECONFIG}" \ --set "image.repository=${IMG_URL}" \ @@ -198,6 +201,30 @@ kubectl wait --for=condition=Ready --timeout=300s node "$TEST_NODE_NAME" export TEST_NODE_NAME +## Windows VK +helm install \ + --kubeconfig="${KUBECONFIG}" \ + --set nodeOsType=Windows \ + --set "image.repository=${IMG_URL}" \ + --set "image.name=${IMG_REPO}" \ + --set "image.tag=${IMG_TAG}" \ + --set "nodeName=${TEST_WINDOWS_NODE_NAME}" \ + --set "providers.azure.masterUri=$MASTER_URI" \ + --set "providers.azure.managedIdentityID=$node_identity" \ + "$WIN_CHART_NAME" \ + ./charts/virtual-kubelet + +kubectl wait --for=condition=available deploy "${TEST_WINDOWS_NODE_NAME}-virtual-kubelet-azure-aci" -n vk-azure-aci --timeout=300s + +while true; do + kubectl get node "$TEST_WINDOWS_NODE_NAME" &> /dev/null && break + sleep 3 +done + +kubectl wait --for=condition=Ready --timeout=300s node "$TEST_WINDOWS_NODE_NAME" + +export TEST_WINDOWS_NODE_NAME + ## CSI Driver test az storage account create -n $CSI_DRIVER_STORAGE_ACCOUNT_NAME -g $RESOURCE_GROUP -l $LOCATION --sku Standard_LRS export AZURE_STORAGE_CONNECTION_STRING=$(az storage account show-connection-string -n $CSI_DRIVER_STORAGE_ACCOUNT_NAME -g $RESOURCE_GROUP -o tsv) diff --git a/pkg/network/aci_network.go b/pkg/network/aci_network.go index 68f4bdfa..1037e4d1 100644 --- a/pkg/network/aci_network.go +++ b/pkg/network/aci_network.go @@ -169,7 +169,10 @@ func (pn *ProviderNetwork) AmendVnetResources(ctx context.Context, cg client2.Co subnetID := "/subscriptions/" + pn.VnetSubscriptionID + "/resourceGroups/" + pn.VnetResourceGroup + "/providers/Microsoft.Network/virtualNetworks/" + pn.VnetName + "/subnets/" + pn.SubnetName cgIDList := []azaci.ContainerGroupSubnetID{{ID: &subnetID}} cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.SubnetIds = &cgIDList - cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.DNSConfig = getDNSConfig(ctx, pod, pn.KubeDNSIP, clusterDomain) + // windows containers don't support DNS config + if cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.OsType != azaci.OperatingSystemTypesWindows { + cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.DNSConfig = getDNSConfig(ctx, pod, pn.KubeDNSIP, clusterDomain) + } } func getDNSConfig(ctx context.Context, pod *v1.Pod, kubeDNSIP, clusterDomain string) *azaci.DNSConfiguration { diff --git a/pkg/provider/aci.go b/pkg/provider/aci.go index 96858c46..6720ca76 100644 --- a/pkg/provider/aci.go +++ b/pkg/provider/aci.go @@ -256,9 +256,12 @@ func NewACIProvider(ctx context.Context, config string, azConfig auth.Config, az } if p.providernetwork.SubnetName != "" { - err = p.setACIExtensions(ctx) - if err != nil { - return nil, err + // windows containers don't support kube-proxy nor realtime metrics + if p.operatingSystem != string(azaci.OperatingSystemTypesWindows) { + err = p.setACIExtensions(ctx) + if err != nil { + return nil, err + } } } @@ -364,7 +367,10 @@ func (p *ACIProvider) CreatePod(ctx context.Context, pod *v1.Pod) error { p.providernetwork.AmendVnetResources(ctx, *cg, pod, p.clusterDomain) - cg.ContainerGroupPropertiesWrapper.Extensions = p.containerGroupExtensions + // windows containers don't support kube-proxy nor realtime metrics + if cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.OsType != azaci.OperatingSystemTypesWindows { + cg.ContainerGroupPropertiesWrapper.Extensions = p.containerGroupExtensions + } log.G(ctx).Infof("start creating pod %v", pod.Name) // TODO: Run in a go routine to not block workers, and use tracker.UpdatePodStatus() based on result. diff --git a/pkg/provider/containergroup_to_pod.go b/pkg/provider/containergroup_to_pod.go index a5587ef2..1f555920 100644 --- a/pkg/provider/containergroup_to_pod.go +++ b/pkg/provider/containergroup_to_pod.go @@ -81,13 +81,17 @@ func (p *ACIProvider) getPodStatusFromContainerGroup(cg *azaci.ContainerGroup) ( return nil, err } + podIp := "" + if cg.OsType != azaci.OperatingSystemTypesWindows { + podIp = *cg.IPAddress.IP + } return &v1.PodStatus{ Phase: getPodPhaseFromACIState(*aciState), Conditions: getPodConditionsFromACIState(*aciState, creationTime, lastUpdateTime, allReady), Message: "", Reason: "", HostIP: p.internalIP, - PodIP: *cg.IPAddress.IP, + PodIP: podIp, StartTime: &firstContainerStartTime, ContainerStatuses: containerStatuses, }, nil @@ -122,6 +126,15 @@ func aciContainerStateToContainerState(cs *azaci.ContainerState) v1.ContainerSta FinishedAt: metav1.NewTime(cs.FinishTime.Time), }, } + // Handle windows container with no prev state + case "Pending": + return v1.ContainerState{ + Waiting: &v1.ContainerStateWaiting{ + Reason: *cs.State, + Message: *cs.DetailStatus, + }, + } + default: // Handle the case where the container is pending. // Which should be all other aci states. diff --git a/pkg/validation/validator.go b/pkg/validation/validator.go index e7ad0594..e39e0dd7 100644 --- a/pkg/validation/validator.go +++ b/pkg/validation/validator.go @@ -29,11 +29,10 @@ func ValidateContainer(container containerinstance.Container) error { return errors.Errorf("container %s properties CurrentState StartTime cannot be nil", *container.Name) } if container.InstanceView.PreviousState == nil { - emptyStr := "" + pendingState := "Pending" container.InstanceView.PreviousState = &containerinstance.ContainerState{ - State: &emptyStr, - StartTime: container.InstanceView.CurrentState.StartTime, - DetailStatus: &emptyStr, + State: &pendingState, + DetailStatus: &pendingState, } return nil } @@ -66,16 +65,18 @@ func ValidateContainerGroup(cg *containerinstance.ContainerGroup) error { if cg.Tags == nil { return errors.Errorf("tags list cannot be nil for container group %s", *cg.Name) } - if cg.IPAddress == nil { - return errors.Errorf("IPAddress cannot be nil for container group %s", *cg.Name) - } else { - aciState := *cg.ContainerGroupProperties.ProvisioningState - if cg.IPAddress.IP == nil { - if aciState == "Running" { - return errors.Errorf("podIP cannot be nil for container group %s while state is %s ", *cg.Name, aciState) - } else { - emptyIP := "" - cg.IPAddress.IP = &emptyIP + if cg.OsType != containerinstance.OperatingSystemTypesWindows { + if cg.IPAddress == nil { + return errors.Errorf("IPAddress cannot be nil for container group %s", *cg.Name) + } else { + aciState := *cg.ContainerGroupProperties.ProvisioningState + if cg.IPAddress.IP == nil { + if aciState == "Running" { + return errors.Errorf("podIP cannot be nil for container group %s while state is %s ", *cg.Name, aciState) + } else { + emptyIP := "" + cg.IPAddress.IP = &emptyIP + } } } }