CNI Acceptance test on GKE

.circleci/config.yml

@@ -716,6 +716,75 @@ jobs:
           fail_only: true
           failure_message: "GKE acceptance tests failed. Check the logs at: ${CIRCLE_BUILD_URL}"
 
+  acceptance-gke-cni-1-20:
+    parallelism: 6
+    environment:
+      - TEST_RESULTS: /tmp/test-results
+    docker:
+      # This image is built from test/docker/Test.dockerfile
+      - image: docker.mirror.hashicorp.services/hashicorpdev/consul-helm-test:0.11.0
+
+    steps:
+      - run:
+          name: Exit if forked PR
+          command: |
+            if [ -n "$CIRCLE_PR_NUMBER" ]; then
+                echo "Skipping acceptance tests for forked PRs; marking step successful."
+                circleci step halt
+            fi
+
+      - checkout
+
+      - run:
+          name: terraform init & apply
+          working_directory: *gke-terraform-path
+          command: |
+            terraform init
+            echo "${GOOGLE_CREDENTIALS}" | gcloud auth activate-service-account --key-file=-
+
+            # On GKE, we're setting the build number instead of build URL because label values
+            # cannot contain '/'.
+            terraform apply \
+              -var project=${CLOUDSDK_CORE_PROJECT} \
+              -var init_cli=true \
+              -var cluster_count=2 \
+              -var labels="{\"build_number\": \"$CIRCLE_BUILD_NUM\"}" \
+              -auto-approve
+
+            primary_kubeconfig=$(terraform output -json | jq -r .kubeconfigs.value[0])
+            secondary_kubeconfig=$(terraform output -json | jq -r .kubeconfigs.value[1])
+
+            echo "export primary_kubeconfig=$primary_kubeconfig" >> $BASH_ENV
+            echo "export secondary_kubeconfig=$secondary_kubeconfig" >> $BASH_ENV
+
+      # Restore go module cache if there is one
+      - restore_cache:
+          keys:
+            - consul-helm-acceptance-modcache-v2-{{ checksum "acceptance/go.mod" }}
+
+      - run: mkdir -p $TEST_RESULTS
+
+      - run-acceptance-tests:
+          additional-flags: -use-gke -kubeconfig="$primary_kubeconfig" -secondary-kubeconfig="$secondary_kubeconfig" -enable-pod-security-policies -enable-transparent-proxy -enable-cni
+
+      - store_test_results:
+          path: /tmp/test-results
+      - store_artifacts:
+          path: /tmp/test-results
+
+      - run:
+          name: terraform destroy
+          working_directory: *gke-terraform-path
+          command: |
+            terraform destroy -var project=${CLOUDSDK_CORE_PROJECT} -auto-approve
+          when: always
+
+      - slack/status:
+          # temporarily sending to #cni-acceptance-tests channel
+          channel: C03V3K0040G
+          fail_only: true
+          failure_message: "GKE CNI acceptance tests failed. Check the logs at: ${CIRCLE_BUILD_URL}"
+
   acceptance-aks-1-21:
     parallelism: 6
     environment:
@@ -922,7 +991,7 @@ jobs:
           failure_message: "Acceptance tests against Kind with Kubernetes v1.23 failed. Check the logs at: ${CIRCLE_BUILD_URL}"
 
   acceptance-kind-cni-1-23:
-    parallelism: 6 
+    parallelism: 6
     environment:
       - TEST_RESULTS: /tmp/test-results
     machine:
@@ -1072,10 +1141,10 @@ workflows:
   version: 2
   test-and-build:
     jobs:
-     # Build this one control-plane binary so that acceptance and acceptance-tproxy will run
-     # The rest of these CircleCI jobs have been migrated to Github Actions. We need to wait until
-     # the summer of 2022 for larger puplic Github Action VMs be available before the acceptance tests can
-     # be moved
+      # Build this one control-plane binary so that acceptance and acceptance-tproxy will run
+      # The rest of these CircleCI jobs have been migrated to Github Actions. We need to wait until
+      # the summer of 2022 for larger puplic Github Action VMs be available before the acceptance tests can
+      # be moved
       - build-distro:
           OS: "linux"
           ARCH: "amd64 arm64"
@@ -1124,6 +1193,10 @@ workflows:
           requires:
             - cleanup-gcp-resources
             - dev-upload-docker
+      - acceptance-gke-cni-1-20:
+          requires:
+            - cleanup-gcp-resources
+            - dev-upload-docker
       - acceptance-eks-1-19:
           requires:
             - cleanup-eks-resources
@@ -1138,7 +1211,9 @@ workflows:
       - acceptance-kind-cni-1-23:
           requires:
             - dev-upload-docker
-
+      - acceptance-kind-cni-1-23:
+          requires:
+            - dev-upload-docker
 
   nightly-acceptance-tests-consul:
     triggers:

acceptance/framework/config/config.go

@@ -54,6 +54,7 @@ type TestConfig struct {
 	DebugDirectory     string
 
 	UseKind bool
+	UseGKE  bool
 
 	helmChartPath string
 }
@@ -88,6 +89,10 @@ func (t *TestConfig) HelmValuesFromConfig() (map[string]string, error) {
 
 	if t.EnableCNI {
 		setIfNotEmpty(helmValues, "connectInject.cni.enabled", "true")
+		// GKE is currently the only cloud provider that uses a different CNI bin dir.
+		if t.UseGKE {
+			setIfNotEmpty(helmValues, "connectInject.cni.cniBinDir", "/home/kubernetes/bin")
+		}
 	}
 
 	setIfNotEmpty(helmValues, "connectInject.transparentProxy.defaultEnabled", strconv.FormatBool(t.EnableTransparentProxy))

acceptance/framework/flags/flags.go

@@ -42,6 +42,7 @@ type TestFlags struct {
 	flagDebugDirectory string
 
 	flagUseKind bool
+	flagUseGKE  bool
 
 	flagDisablePeering bool
 
@@ -106,6 +107,9 @@ func (t *TestFlags) init() {
 
 	flag.BoolVar(&t.flagUseKind, "use-kind", false,
 		"If true, the tests will assume they are running against a local kind cluster(s).")
+	flag.BoolVar(&t.flagUseGKE, "use-gke", false,
+		"If true, the tests will assume they are running against a GKE cluster(s).")
+
 	flag.BoolVar(&t.flagDisablePeering, "disable-peering", false,
 		"If true, the peering tests will not run.")
 
@@ -165,5 +169,6 @@ func (t *TestFlags) TestConfigFromFlags() *config.TestConfig {
 		NoCleanupOnFailure: t.flagNoCleanupOnFailure,
 		DebugDirectory:     tempDir,
 		UseKind:            t.flagUseKind,
+		UseGKE:             t.flagUseGKE,
 	}
 }

charts/consul/templates/cni-clusterrole.yaml

@@ -20,4 +20,11 @@ rules:
   - watch
   - patch
   - update
+- apiGroups: ["policy"]
+  resources:
+  - podsecuritypolicies 
+  resourceNames:
+  - {{ template "consul.fullname" . }}-cni
+  verbs:
+  - use
 {{- end }}

charts/consul/templates/cni-podsecuritypolicy.yaml

@@ -12,8 +12,8 @@ metadata:
     component: cni 
 spec:
   privileged: true
-  # Required to prevent escalations to root.
-  allowPrivilegeEscalation: false
+  # GKE requires that allowPrivilegeEscalation:true if privileged: true.
+  allowPrivilegeEscalation: true 
   volumes:
   - hostPath
   - secret