ci: Cleanup e2e scenarios (#383)

charts/virtual-kubelet/README.md

@@ -49,6 +49,7 @@ The following table lists the configurable parameters of the azure-aci chart and
 
 | Parameter                                      | Description                                                                                                           | Default                               |
 |------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------|---------------------------------------|
+ | namespace                                      | The name of the namespace that azure-aci will be deployed in.                                                         | `vk-azure-aci`                        | 
 | image.secretName                               | The name of image secret.                                                                                             | `virtual-kubelet-aci-acr`             |
 | image.repository                               | Image repository.                                                                                                     | `mcr.microsoft.com`                   |
 | image.name                                     | Image name.                                                                                                           | `oss/virtual-kubelet/virtual-kubelet` |
@@ -77,6 +78,7 @@ The following table lists the configurable parameters of the azure-aci chart and
 | providers.azure.managedIdentityID              | Only required if `targetAKS` is false.                                                                                | ` `                                   |
 | providers.azure.aciResourceGroup               | `aciResourceGroup` and `aciRegion` are required only for non-AKS deployments.                                         | ` `                                   |
 | providers.azure.aciRegion                      | `aciResourceGroup` and `aciRegion` are required only for non-AKS deployments.                                         | ` `                                   |
+| providers.azure.enableRealTimeMetrics          | Enable Real-Time metrics.                                                                                             | `true`                                |
 | providers.azure.masterUri                      | API server URL for the AKS cluster.                                                                                   | ` `                                   |
 | providers.azure.loganalytics.enabled           | Log Analytics enabled flag.                                                                                           | `false`                               |
 | providers.azure.loganalytics.workspaceId       | Log Analytics workspace ID.                                                                                           | ` `                                   |

charts/virtual-kubelet/templates/clusterrolebinding.yaml

@@ -7,7 +7,7 @@ metadata:
 subjects:
 - kind: ServiceAccount
   name: {{ template "vk.fullname" . }}-{{ .Values.rbac.serviceAccountName }}
-  namespace: {{ .Release.Namespace }}
+  namespace: {{ .Values.namespace }}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole

charts/virtual-kubelet/templates/deployment.yaml

@@ -1,6 +1,7 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  namespace: {{ .Values.namespace }}
   name: {{ template "vk.fullname" . }}
 {{ include "vk.labels" . | indent 2 }}
     component: kubelet

charts/virtual-kubelet/templates/namespace.yml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: {{ .Values.namespace }}
+---

charts/virtual-kubelet/templates/secrets.yaml

@@ -1,6 +1,7 @@
 apiVersion: v1
 kind: Secret
 metadata:
+  namespace: {{ .Values.namespace }}
   name: {{ template "vk.fullname" . }}
 {{ include "vk.labels" . | indent 2 }}
 type: Opaque

charts/virtual-kubelet/templates/serviceaccount.yaml

@@ -2,6 +2,7 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
+  namespace: {{ .Values.namespace }}
   name: {{ template "vk.fullname" . }}-{{ .Values.rbac.serviceAccountName }}
 {{ include "vk.labels" . | indent 2 }}
 {{ end }}

charts/virtual-kubelet/values.yaml

@@ -5,6 +5,7 @@ image:
   tag: 1.4.7
   pullPolicy: Always
 
+namespace: vk-azure-aci
 useVKVersion2: true
 
 nodeName: "virtual-node-aci-linux-helm"

client/aci/client_test.go

@@ -745,6 +745,7 @@ func TestCreateContainerGroupWithVNet(t *testing.T) {
 }
 
 func TestCreateContainerGroupWithGPU(t *testing.T) {
+	t.Skip("Old custom SDK tests")
 	uid := uuid.New()
 	containerGroupName := containerGroup + "-" + uid.String()[0:6]
 

deploy/deployment.yaml

@@ -0,0 +1,114 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  namespace: kube-system
+  name: virtual-kubelet-azure-aci
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: aci-connector-linux
+  template:
+    metadata:
+      labels:
+        app: aci-connector-linux
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: kubernetes.azure.com/cluster
+                    operator: Exists
+                  - key: type
+                    operator: NotIn
+                    values:
+                      - virtual-kubelet
+                  - key: kubernetes.io/os
+                    operator: In
+                    values:
+                      - linux
+      containers:
+      - name: vk-azure-aci
+        image: TEST_IMAGE
+        imagePullPolicy: Always
+        env:
+        - name: KUBELET_PORT
+          value: "10250"
+        - name: AKS_CREDENTIAL_LOCATION
+          value: /etc/aks/azure.json
+        - name: AZURE_CLIENT_SECRET
+          valueFrom:
+            secretKeyRef:
+              name: aci-connector-linux
+              key: clientSecret
+        - name: APISERVER_CERT_LOCATION
+          value: /etc/virtual-kubelet/cert.pem
+        - name: APISERVER_KEY_LOCATION
+          value: /etc/virtual-kubelet/key.pem
+        - name: VKUBELET_POD_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
+
+        - name: ACI_EXTRA_USER_AGENT
+          value: "deploy/aks/azure-aci/e2e-test"
+        - name: ACI_SUBNET_NAME
+          valueFrom:
+            configMapKeyRef:
+              name: test-vars
+              key: aci_subnet_name
+        - name: MASTER_URI
+          valueFrom:
+            configMapKeyRef:
+              name: test-vars
+              key: master_uri
+        - name: CLUSTER_CIDR
+          valueFrom:
+            configMapKeyRef:
+              name: test-vars
+              key: cluster_subnet_cidr
+        - name: KUBE_DNS_IP
+          valueFrom:
+            configMapKeyRef:
+              name: test-vars
+              key: kube_dns_ip
+        - name: ENABLE_REAL_TIME_METRICS
+          value: "true"
+        - name: USE_VK_VERSION_2
+          value: "true"
+        - name: VIRTUALNODE_USER_IDENTITY_CLIENTID
+          valueFrom:
+            configMapKeyRef:
+              name: test-vars
+              key: aci_user_identity
+        volumeMounts:
+          - name: certificates
+            mountPath: /etc/kubernetes/certs
+            readOnly: true
+          - name: credentials
+            mountPath: "/etc/virtual-kubelet"
+          - name: aks-credential
+            mountPath: "/etc/aks/azure.json"
+        command: ["virtual-kubelet"]
+        args: [
+          "--provider", "azure",
+          "--nodename", "vk-aci-test-aks",
+          "--authentication-token-webhook=true",
+          "--client-verify-ca", "/etc/kubernetes/certs/ca.crt",
+          "--no-verify-clients=false",
+          "--os", "Linux"
+        ]
+      volumes:
+      - name: certificates
+        hostPath:
+          path: /etc/kubernetes/certs
+      - name: credentials
+        secret:
+          secretName: aci-connector-linux
+      - name: aks-credential
+        hostPath:
+          path: /etc/kubernetes/azure.json
+          type: File
+      serviceAccountName: aci-connector-linux
+      restartPolicy: Always

hack/e2e/aks-addon.sh

@@ -23,7 +23,7 @@ fi
 : "${LOCATION:=eastus2}"
 : "${CLUSTER_NAME:=${RESOURCE_GROUP}}"
 : "${NODE_COUNT:=1}"
-: "${CHART_NAME:=vk-aci-test-aks}"
+: "${CHART_NAME:=aks-addon--test}"
 : "${TEST_NODE_NAME:=vk-aci-test-aks}"
 : "${IMG_REPO:=oss/virtual-kubelet/virtual-kubelet}"
 : "${IMG_URL:=mcr.microsoft.com}"
@@ -32,9 +32,9 @@ fi
 : "${ACI_SUBNET_CIDR=10.241.0.0/16}"
 : "${VNET_NAME=aksAddonVN}"
 : "${CLUSTER_SUBNET_NAME=aksAddonsubnet}"
-: "${ACI_SUBNET_NAME=aksAddonACIsubnet}"
+: "${ACI_SUBNET_NAME=acisubnet}"
 : "${ACR_NAME=aksaddonacr$RANDOM_NUM}"
-: "${CSI_DRIVER_STORAGE_ACCOUNT_NAME=vkcsidrivers$RANDOM_NUM}"
+: "${CSI_DRIVER_STORAGE_ACCOUNT_NAME=aksaddonvk$RANDOM_NUM}"
 : "${CSI_DRIVER_SHARE_NAME=vncsidriversharename}"
 
 error() {
@@ -140,25 +140,20 @@ MASTER_URI="$(kubectl cluster-info | awk '/Kubernetes control plane/{print $7}'
 
 ACI_USER_IDENTITY="$(az aks show  -g "$RESOURCE_GROUP" -n "$CLUSTER_NAME" --query addonProfiles.aciConnectorLinux.identity.clientId -o tsv)"
 KUBE_DNS_IP="$(az aks show  -g "$RESOURCE_GROUP" -n "$CLUSTER_NAME" --query networkProfile.dnsServiceIp -o tsv)"
+CLUSTER_RESOURCE_ID="$(az aks show  -g "$RESOURCE_GROUP" -n "$CLUSTER_NAME" --query "id" -o tsv)"
+MC_RESOURCE_GROUP="$(az aks show  -g "$RESOURCE_GROUP" -n "$CLUSTER_NAME" --query "nodeResourceGroup" -o tsv)"
+SUB_ID="$(az account show --query "id" -o tsv)"
 
+kubectl create configmap test-vars -n kube-system \
+  --from-literal=master_uri="$MASTER_URI" \
+  --from-literal=aci_user_identity="$ACI_USER_IDENTITY" \
+  --from-literal=kube_dns_ip="$KUBE_DNS_IP" \
+  --from-literal=cluster_subnet_cidr="$CLUSTER_SUBNET_CIDR" \
+  --from-literal=aci_subnet_name="$ACI_SUBNET_NAME"
 
-helm install \
-    --kubeconfig="$KUBECONFIG" \
-    --set "image.repository=$IMG_URL"  \
-    --set "image.name=$IMG_REPO" \
-    --set "image.tag=$IMG_TAG" \
-    --set "nodeName=$TEST_NODE_NAME" \
-    --set providers.azure.vnet.enabled=true \
-    --set "providers.azure.vnet.subnetName=$ACI_SUBNET_NAME" \
-    --set "providers.azure.vnet.subnetCidr=$ACI_SUBNET_CIDR" \
-    --set "providers.azure.vnet.clusterCidr=$CLUSTER_SUBNET_CIDR" \
-    --set "providers.azure.vnet.kubeDnsIp=$KUBE_DNS_IP" \
-    --set "providers.azure.masterUri=$MASTER_URI" \
-    --set "providers.azure.managedIdentityID=$ACI_USER_IDENTITY" \
-    "$CHART_NAME" \
-    ./charts/virtual-kubelet
-
-kubectl wait --for=condition=available deploy "$TEST_NODE_NAME-virtual-kubelet-azure-aci" --timeout=300s
+sed -e "s|TEST_IMAGE|$ACR_NAME.azurecr.io/$IMG_REPO:$IMG_TAG|g" deploy/deployment.yaml | kubectl apply -n kube-system -f -
+
+kubectl wait --for=condition=available deploy "virtual-kubelet-azure-aci" -n kube-system --timeout=300s
 
 while true; do
     kubectl get node "$TEST_NODE_NAME" &> /dev/null && break

hack/e2e/aks.sh

@@ -187,7 +187,7 @@ helm install \
     "$CHART_NAME" \
     ./charts/virtual-kubelet
 
-kubectl wait --for=condition=available deploy "${TEST_NODE_NAME}-virtual-kubelet-azure-aci" --timeout=300s
+kubectl wait --for=condition=available deploy "${TEST_NODE_NAME}-virtual-kubelet-azure-aci" -n vk-azure-aci --timeout=300s
 
 while true; do
     kubectl get node "$TEST_NODE_NAME" &> /dev/null && break

pkg/auth/auth.go

@@ -211,7 +211,6 @@ func newAKSCredential(filePath string) (*aksCredential, error) {
 	if err := json.Unmarshal(b, &cred); err != nil {
 		return nil, err
 	}
-
 	logger.Debug("load AKS credential file successfully")
 	return &cred, nil
 }

pkg/provider/aci.go

@@ -193,7 +193,6 @@ func NewACIProvider(ctx context.Context, config string, azConfig auth.Config, az
 	if azConfig.AKSCredential != nil {
 		p.resourceGroup = azConfig.AKSCredential.ResourceGroup
 		p.region = azConfig.AKSCredential.Region
-
 		p.vnetName = azConfig.AKSCredential.VNetName
 		p.vnetResourceGroup = azConfig.AKSCredential.VNetResourceGroup
 	}
@@ -232,16 +231,14 @@ func NewACIProvider(ctx context.Context, config string, azConfig auth.Config, az
 
 	if rg := os.Getenv("ACI_RESOURCE_GROUP"); rg != "" {
 		p.resourceGroup = rg
-	}
-	if p.resourceGroup == "" {
-		return nil, errors.New("Resource group can not be empty please set ACI_RESOURCE_GROUP")
+	} else if p.resourceGroup == "" {
+		return nil, errors.New("resource group can not be empty please set ACI_RESOURCE_GROUP")
 	}
 
 	if r := os.Getenv("ACI_REGION"); r != "" {
 		p.region = r
-	}
-	if p.region == "" {
-		return nil, errors.New("Region can not be empty please set ACI_REGION")
+	} else if p.region == "" {
+		return nil, errors.New("region can not be empty please set ACI_REGION")
 	}
 
 	if r := p.region; !isValidACIRegion(r) {
@@ -920,10 +917,10 @@ func (p *ACIProvider) getInitContainers(ctx context.Context, pod *v1.Pod) ([]aza
 
 		newInitContainer := azaci.InitContainerDefinition{
 			Name: &pod.Spec.InitContainers[i].Name,
-			InitContainerPropertiesDefinition: &azaci.InitContainerPropertiesDefinition {
-				Image: &pod.Spec.InitContainers[i].Image,
-				Command: p.getCommand(&pod.Spec.InitContainers[i]),
-				VolumeMounts: p.getVolumeMounts(&pod.Spec.InitContainers[i]),
+			InitContainerPropertiesDefinition: &azaci.InitContainerPropertiesDefinition{
+				Image:                &pod.Spec.InitContainers[i].Image,
+				Command:              p.getCommand(&pod.Spec.InitContainers[i]),
+				VolumeMounts:         p.getVolumeMounts(&pod.Spec.InitContainers[i]),
 				EnvironmentVariables: p.getEnvironmentVariables(&pod.Spec.InitContainers[i]),
 			},
 		}

pkg/provider/aci_network.go

@@ -11,6 +11,7 @@ import (
 	"os"
 	"strings"
 
+	"github.com/pkg/errors"
 	utilvalidation "k8s.io/apimachinery/pkg/util/validation"
 
 	azaci "github.com/Azure/azure-sdk-for-go/services/containerinstance/mgmt/2021-10-01/containerinstance"
@@ -36,15 +37,24 @@ func (p *ACIProvider) setVNETConfig(ctx context.Context, azConfig *auth.Config)
 	if vnetSubscriptionID := os.Getenv("ACI_VNET_SUBSCRIPTION_ID"); vnetSubscriptionID != "" {
 		p.vnetSubscriptionID = vnetSubscriptionID
 	}
+
 	if vnetName := os.Getenv("ACI_VNET_NAME"); vnetName != "" {
 		p.vnetName = vnetName
+	} else if p.vnetName == "" {
+		return errors.New("vnet name can not be empty please set ACI_VNET_NAME")
 	}
+
 	if vnetResourceGroup := os.Getenv("ACI_VNET_RESOURCE_GROUP"); vnetResourceGroup != "" {
 		p.vnetResourceGroup = vnetResourceGroup
+	} else if p.vnetResourceGroup == "" {
+		return errors.New("vnet resourceGroup can not be empty please set ACI_VNET_RESOURCE_GROUP")
 	}
+
+	// Set subnet properties.
 	if subnetName := os.Getenv("ACI_SUBNET_NAME"); p.vnetName != "" && subnetName != "" {
 		p.subnetName = subnetName
 	}
+
 	if subnetCIDR := os.Getenv("ACI_SUBNET_CIDR"); subnetCIDR != "" {
 		if p.subnetName == "" {
 			return fmt.Errorf("subnet CIDR defined but no subnet name, subnet name is required to set a subnet CIDR")

pkg/provider/aci_test.go

@@ -31,6 +31,7 @@ import (
 const (
 	fakeResourceGroup = "vk-rg"
 	fakeNodeName      = "vk"
+	fakeVnetName      = "vnet"
 )
 
 var (
@@ -565,6 +566,15 @@ func createTestProvider(aciMocks *MockACIProvider, resourceManager *manager.Reso
 		}
 	}
 
+	err = os.Setenv("ACI_VNET_NAME", fakeVnetName)
+	if err != nil {
+		return nil, err
+	}
+	err = os.Setenv("ACI_VNET_RESOURCE_GROUP", fakeResourceGroup)
+	if err != nil {
+		return nil, err
+	}
+
 	err = os.Setenv("ACI_RESOURCE_GROUP", fakeResourceGroup)
 	if err != nil {
 		return nil, err