vSphere CSI Driver 2.0

.gitignore

@@ -1,6 +1,14 @@
+# Ignore unit-test coverrage report
+coverage_report.out
+
 # Ignore the generated kind-config yaml.
 /test/integration/kind-config.yaml
 
+# Ignore the unit-test conf files
+/pkg/csi/service/vanilla/test_vsphere.conf
+/pkg/csi/service/wcp/test_vsphere.conf
+/pkg/syncer/test_vsphere.conf
+
 # Ignore the build output.
 /.build
 

Makefile

@@ -18,8 +18,8 @@ export DIST_OUT ?= $(BUILD_OUT)/dist
 ################################################################################
 ##                             VERIFY GO VERSION                              ##
 ################################################################################
-# Go 1.11+ required for Go modules.
-GO_VERSION_EXP := "go1.11"
+# Go 1.13 required for Go modules.
+GO_VERSION_EXP := "go1.13"
 GO_VERSION_ACT := $(shell a="$$(go version | awk '{print $$3}')" && test $$(printf '%s\n%s' "$${a}" "$(GO_VERSION_EXP)" | sort | tail -n 1) = "$${a}" && printf '%s' "$${a}")
 ifndef GO_VERSION_ACT
 $(error Requires Go $(GO_VERSION_EXP)+ for Go module support)
@@ -235,37 +235,68 @@ endif
 TEST_FLAGS ?= -v -count=1
 .PHONY: unit build-unit-tests
 unit unit-test:
-	env -u VSPHERE_VCENTER -u VSPHERE_DATACENTER -u VSPHERE_PASSWORD -u VSPHERE_USER -u VSPHERE_DATASTORE_URL -u VSPHERE_STORAGE_POLICY_NAME -u VSPHERE_K8S_NODE -u VSPHERE_INSECURE -u KUBECONFIG go test $(TEST_FLAGS) $(PKGS_WITH_TESTS)
+	env -u VSPHERE_SERVER -u VSPHERE_DATACENTER -u VSPHERE_PASSWORD -u VSPHERE_USER -u VSPHERE_STORAGE_POLICY_NAME -u KUBECONFIG -u WCP_ENDPOINT -u WCP_PORT -u WCP_NAMESPACE -u TOKEN -u CERTIFICATE go test $(TEST_FLAGS) $(PKGS_WITH_TESTS)
 build-unit-tests:
 	$(foreach pkg,$(PKGS_WITH_TESTS),go test $(TEST_FLAGS) -c $(pkg); )
 
+INTEGRATION_TEST_PKGS ?=
 .PHONY: integration-unit-test
 integration-unit-test:
-ifndef VSPHERE_VCENTER
-	$(error Requires VSPHERE_VCENTER from a deployed testbed to run integration-unit-test)
+ifndef TYPE
+	$(error Requires TYPE from a deployed testbed to run integration-unit-test)
+else
+    ifeq ($(TYPE), guestcluster)
+        ifndef WCP_ENDPOINT
+            $(error Requires WCP_ENDPOINT from a deployed testbed to run integration-unit-test)
+        endif
+        ifndef WCP_NAMESPACE
+            $(error Requires WCP_NAMESPACE from a deployed testbed to run integration-unit-test)
+        endif
+        ifndef SUPERVISOR_STORAGE_CLASS
+            $(error Requires SUPERVISOR_STORAGE_CLASS from a deployed testbed to run integration-unit-test)
+        endif
+        ifndef TOKEN
+            $(error Requires TOKEN from a deployed testbed to run integration-unit-test)
+        endif
+        ifndef CERTIFICATE
+            $(error Requires CERTIFICATE from a deployed testbed to run integration-unit-test)
+        else
+	        $(eval INTEGRATION_TEST_PKGS += ./pkg/csi/service/wcpguest)
+        endif
+    else
+        ifndef VSPHERE_VCENTER
+            $(error Requires VSPHERE_VCENTER from a deployed testbed to run integration-unit-test)
+        endif
+        ifndef VSPHERE_USER
+            $(error Requires VSPHERE_USER from a deployed testbed to run integration-unit-test)
+        endif
+        ifndef VSPHERE_PASSWORD
+            $(error Requires VSPHERE_PASSWORD from a deployed testbed to run integration-unit-test)
+        endif
+        ifndef VSPHERE_DATACENTER
+            $(error Requires VSPHERE_DATACENTER from a deployed testbed to run integration-unit-test)
+        endif
+        ifndef VSPHERE_DATASTORE_URL
+            $(error Requires VSPHERE_DATASTORE_URL from a deployed testbed to run integration-unit-test)
+        endif
+        ifndef VSPHERE_INSECURE
+            $(error Requires VSPHERE_INSECURE from a deployed testbed to run integration-unit-test)
+        endif
+        ifeq ($(TYPE), supervisorcluster)
+	        $(eval INTEGRATION_TEST_PKGS += ./pkg/csi/service/wcp ./pkg/syncer)
+        else
+            ifndef VSPHERE_K8S_NODE
+                $(error Requires VSPHERE_K8S_NODE from a deployed testbed to run integration-unit-test)
+            endif
+            ifndef KUBECONFIG
+                $(error Requires KUBECONFIG from a deployed testbed to run integration-unit-test)
+            else
+		$(eval INTEGRATION_TEST_PKGS += ./pkg/csi/service/vanilla ./pkg/syncer)
+            endif
+        endif
+    endif
 endif
-ifndef VSPHERE_USER
-	$(error Requires VSPHERE_USER from a deployed testbed to run integration-unit-test)
-endif
-ifndef VSPHERE_PASSWORD
-	$(error Requires VSPHERE_PASSWORD from a deployed testbed to run integration-unit-test)
-endif
-ifndef VSPHERE_DATACENTER
-	$(error Requires VSPHERE_DATACENTER from a deployed testbed to run integration-unit-test)
-endif
-ifndef VSPHERE_DATASTORE_URL
-	$(error Requires VSPHERE_DATASTORE_URL from a deployed testbed to run integration-unit-test)
-endif
-ifndef VSPHERE_K8S_NODE
-	$(error Requires VSPHERE_K8S_NODE from a deployed testbed to run integration-unit-test)
-endif
-ifndef KUBECONFIG
-	$(error Requires KUBECONFIG from a deployed testbed to run integration-unit-test)
-endif
-ifndef VSPHERE_INSECURE
-	$(error Requires VSPHERE_INSECURE from a deployed testbed to run integration-unit-test)
-endif
-	    go test $(TEST_FLAGS) -tags=integration-unit ./pkg/csi/service/cns ./pkg/syncer
+	go test $(TEST_FLAGS) -tags=integration-unit $(INTEGRATION_TEST_PKGS)
 
 # The default test target.
 .PHONY: test build-tests

cmd/syncer/main.go

@@ -17,21 +17,96 @@ limitations under the License.
 package main
 
 import (
+	"context"
 	"flag"
 	"os"
 
-	"k8s.io/klog"
+	"sigs.k8s.io/vsphere-csi-driver/pkg/csi/service/logger"
 
-	metadatasyncer "sigs.k8s.io/vsphere-csi-driver/pkg/syncer"
+	"github.com/kubernetes-csi/csi-lib-utils/leaderelection"
+	cnstypes "github.com/vmware/govmomi/cns/types"
+
+	csitypes "sigs.k8s.io/vsphere-csi-driver/pkg/csi/types"
+	k8s "sigs.k8s.io/vsphere-csi-driver/pkg/kubernetes"
+	"sigs.k8s.io/vsphere-csi-driver/pkg/syncer"
+	"sigs.k8s.io/vsphere-csi-driver/pkg/syncer/cnsoperator/manager"
+	"sigs.k8s.io/vsphere-csi-driver/pkg/syncer/podlistener"
+	"sigs.k8s.io/vsphere-csi-driver/pkg/syncer/types"
+)
+
+var (
+	enableLeaderElection = flag.Bool("leader-election", false, "Enable leader election.")
 )
 
-// main is ignored when this package is built as a go plug-in.
+// main for vsphere syncer
 func main() {
-	klog.InitFlags(nil)
 	flag.Parse()
-	metadataSyncer := metadatasyncer.NewInformer()
-	if err := metadataSyncer.Init(); err != nil {
-		klog.Errorf("Error initializing Metadata Syncer")
+	// run will be executed if this instance is elected as the leader
+	// or if leader election is not enabled
+	var run func(ctx context.Context)
+	var err error
+
+	logType := logger.LogLevel(os.Getenv(logger.EnvLoggerLevel))
+	logger.SetLoggerLevel(logType)
+	ctx, log := logger.GetNewContextWithLogger()
+
+	clusterFlavor := cnstypes.CnsClusterFlavor(os.Getenv(csitypes.EnvClusterFlavor))
+	configInfo, err := types.InitConfigInfo(ctx)
+	if err != nil {
+		log.Errorf("failed to initialize the configInfo. Err: %+v", err)
 		os.Exit(1)
 	}
+
+	// Initialize PodListener for every instance of vsphere-syncer in the Supervisor
+	// Cluster, independent of whether leader election is enabled.
+	// PodListener should run on every node where csi controller can run.
+	if clusterFlavor == cnstypes.CnsClusterFlavorWorkload {
+		go func() {
+			if err := podlistener.InitPodListenerService(ctx); err != nil {
+				log.Errorf("Error initializing Pod Listener gRPC sever. Error: %+v", err)
+				os.Exit(1)
+			}
+		}()
+	}
+
+	// Initialize syncer components that are dependant on the outcome of leader election, if enabled.
+	run = initSyncerComponents(ctx, clusterFlavor, configInfo)
+
+	if !*enableLeaderElection {
+		run(context.TODO())
+	} else {
+		k8sClient, err := k8s.NewClient(ctx)
+		if err != nil {
+			log.Errorf("Creating Kubernetes client failed. Err: %v", err)
+			os.Exit(1)
+		}
+		lockName := "vsphere-syncer"
+		le := leaderelection.NewLeaderElection(k8sClient, lockName, run)
+
+		if err := le.Run(); err != nil {
+			log.Fatalf("Error initializing leader election: %v", err)
+		}
+	}
+}
+
+// initSyncerComponents initializes syncer components that are dependant on the leader election algorithm.
+// This function is only called by the leader instance of vsphere-syncer, if enabled.
+// TODO: Change name from initSyncerComponents to init<Name>Components where <Name> will be the name of this container
+func initSyncerComponents(ctx context.Context, clusterFlavor cnstypes.CnsClusterFlavor, configInfo *types.ConfigInfo) func(ctx context.Context) {
+	return func(ctx context.Context) {
+		log := logger.GetLogger(ctx)
+		// Initialize CNS Operator for Supervisor clusters
+		if clusterFlavor == cnstypes.CnsClusterFlavorWorkload {
+			go func() {
+				if err := manager.InitCnsOperator(configInfo); err != nil {
+					log.Errorf("Error initializing Cns Operator. Error: %+v", err)
+					os.Exit(1)
+				}
+			}()
+		}
+		if err := syncer.InitMetadataSyncer(ctx, clusterFlavor, configInfo); err != nil {
+			log.Errorf("Error initializing Metadata Syncer. Error: %+v", err)
+			os.Exit(1)
+		}
+	}
 }

cmd/vsphere-csi/main.go

@@ -21,20 +21,18 @@ import (
 	"flag"
 
 	"github.com/rexray/gocsi"
-	"k8s.io/klog"
 
 	"sigs.k8s.io/vsphere-csi-driver/pkg/csi/provider"
-	"sigs.k8s.io/vsphere-csi-driver/pkg/csi/service"
+	csitypes "sigs.k8s.io/vsphere-csi-driver/pkg/csi/types"
 )
 
 // main is ignored when this package is built as a go plug-in.
 func main() {
-	klog.InitFlags(nil)
 	flag.Parse()
 	gocsi.Run(
 		context.Background(),
-		service.Name,
-		"A CSI plugin for VMware vSphere storage",
+		csitypes.Name,
+		"A CSI plugin for vSphere Cloud Native Storage",
 		usage,
 		provider.New())
 }

example/vanilla-k8s-block-driver/example-pod.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: example-vanilla-block-pod
+spec:
+  containers:
+    - name: test-container
+      image: gcr.io/google_containers/busybox:1.24
+      command: ["/bin/sh", "-c", "echo 'hello' > /mnt/volume1/index.html  && chmod o+rX /mnt /mnt/volume1/index.html && while true ; do sleep 2 ; done"]
+      volumeMounts:
+        - name: test-volume
+          mountPath: /mnt/volume1
+  restartPolicy: Never
+  volumes:
+    - name: test-volume
+      persistentVolumeClaim:
+        claimName: example-vanilla-block-pvc

example/example-pvc.yaml → example/vanilla-k8s-block-driver/example-pvc.yaml

@@ -4,7 +4,7 @@ metadata:
   name: example-vanilla-block-pvc
 spec:
   accessModes:
-  - ReadWriteOnce
+    - ReadWriteOnce
   resources:
     requests:
       storage: 5Gi

example/example-sc-WaitForFirstConsumer-With-Multiple-Zone-Region.yaml → example/vanilla-k8s-block-driver/example-sc-WaitForFirstConsumer-With-Multiple-Zone-Region.yaml

@@ -13,4 +13,4 @@ allowedTopologies:
       - key: failure-domain.beta.kubernetes.io/region
         values:
           - k8s-region-eu
-          - k8s-region-us
+          - k8s-region-us

example/example-sc-multiple-zones-regions.yaml → example/vanilla-k8s-block-driver/example-sc-multiple-zones-regions.yaml

@@ -12,4 +12,4 @@ allowedTopologies:
       - key: failure-domain.beta.kubernetes.io/region
         values:
           - k8s-region-us
-          - k8s-region-eu
+          - k8s-region-eu

example/example-sc-multiple-zones.yaml → example/vanilla-k8s-block-driver/example-sc-multiple-zones.yaml

@@ -11,4 +11,4 @@ allowedTopologies:
           - k8s-zone-us-west
       - key: failure-domain.beta.kubernetes.io/region
         values:
-          - k8s-region-us
+          - k8s-region-us

example/vanilla-k8s-block-driver/example-sc-volume-expansion.yaml

@@ -0,0 +1,12 @@
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: example-vanilla-block-sc
+  namespace: kube-system
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "false"
+provisioner: csi.vsphere.vmware.com
+allowVolumeExpansion: true
+parameters:
+# DatastoreURL: "ds:///vmfs/volumes/vsan:52cdfa80721ff516-ea1e993113acfc77/" # Optional Parameter
+# StoragePolicyName: "vSAN Default Storage Policy"  # Optional Parameter

example/example-sc.yaml → example/vanilla-k8s-block-driver/example-sc.yaml

@@ -8,4 +8,4 @@ provisioner: csi.vsphere.vmware.com
 parameters:
   datastoreurl: "ds:///vmfs/volumes/vsan:52cdfa80721ff516-ea1e993113acfc77/" #Optional Parameter
   storagepolicyname: "vSAN Default Storage Policy"  #Optional Parameter
-  fstype: "ext4" #Optional Parameter
+  csi.storage.k8s.io/fstype: "ext4" #Optional Parameter

example/vanilla-k8s-block-driver/vsphere.conf

@@ -0,0 +1,9 @@
+[Global]
+cluster-id = "unique-kubernetes-cluster-id"
+
+[VirtualCenter "1.2.3.4"]
+insecure-flag = "true"
+user = "vcenter username"
+password = "vcenter password"
+port = "443"
+datacenters = "list of comma seperated datacenters where node VMs are present"

example/vanilla-k8s-file-driver/example-csi-vsphere.conf

@@ -0,0 +1,36 @@
+[Global]
+cluster-id = "unique-kubernetes-cluster-id"
+
+# This config represents default values taken when net permissions are not mentioned
+# Each Net Permission config section should have a unique identifier as a string.
+# The character `#` is reserved to store default config
+[NetPermissions "#"]
+Ips = "*"  # All Ips
+Permissions = "READ_WRITE"
+RootSquash = false
+
+[NetPermissions "B"]
+Ips = "10.20.20.0/24"
+Permissions = "READ_ONLY"
+RootSquash = true
+
+# If one of the parameters is not mentioned, its default values are assumed.
+# For example SquashRoot = false in this configuration
+[NetPermissions "C"]
+Ips = "10.30.30.0/24"
+Permissions = "NO_ACCESS"
+
+# Permissions will assume a default value of "READ_WRITE" in this configuration
+[NetPermissions "D"]
+Ips = "10.30.10.0/24"
+RootSquash = true
+
+[NetPermissions "E"]
+Ips = "10.30.1.0/24"
+
+[VirtualCenter "1.1.1.1"]
+insecure-flag = "true"
+user = "vcenter username"
+password = "vcenter password"
+port = "443"
+datacenters = "list of comma seperated datacenters where node VMs are present"