tests/integration: Add GCP GKE tests

Add GCP as a new provider to provision infrastructure and test against.

GCP has Google Container Registry and Artifact Registry for storing
images. The test setup provisions and pushes images to both and tests
them both.

The GKE node pool is configured with OAuth Scope to provide full access
to the GCP APIs. Workload identity is not used to avoid increasing the
complixity of the setup.

Co-authored-by: Somtochi Onyekwere <somtochionyekwere@gmail.com>
Signed-off-by: Sunny <darkowlzz@protonmail.com>

tests/integration/.env.sample

@@ -5,3 +5,10 @@
 
 # Azure
 # export TF_VAR_azure_location=eastus
+
+# GCP
+# export TF_VAR_gcp_project_id=
+# export TF_VAR_gcp_region=us-central1
+# export TF_VAR_gcp_zone=us-central1-c
+# Leave GCR region empty to use gcr.io. Else set it to `us`, `eu` or `asia`.
+# export TF_VAR_gcr_region=

tests/integration/Makefile

@@ -30,3 +30,6 @@ test-aws:
 
 test-azure:
 	$(MAKE) test PROVIDER_ARG="-provider azure"
+
+test-gcp:
+	$(MAKE) test PROVIDER_ARG="-provider gcp"

tests/integration/README.md

@@ -19,6 +19,20 @@
 - Docker CLI for registry login.
 - kubectl for applying certain install manifests.
 
+### Google Cloud Platform
+
+- GCP account with project and GKE, GCR and Artifact Registry services enabled
+    in the project.
+- gcloud CLI, need to be logged in using `gcloud auth login`.
+- Docker CLI for registry login.
+- kubectl for applying certain install manifests.
+
+**NOTE:** Unlike ECR, AKS and Google Artifact Registry, Google Container
+Registry tests don't create a new registry. It pushes to an existing registry
+host in a project, for example `gcr.io`. Due to this, the test images pushed to
+GCR aren't cleaned up automatically at the end of the test and have to be
+deleted manually.
+
 ## Test setup
 
 Copy `.env.sample` to `.env`, put the respective provider configurations in the

tests/integration/gcp_test.go

@@ -0,0 +1,82 @@
+/*
+Copyright 2022 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	tfjson "github.com/hashicorp/terraform-json"
+
+	tftestenv "github.com/fluxcd/image-reflector-controller/tests/tftestenv"
+)
+
+// createKubeconfigGKE constructs kubeconfig from the terraform state output at
+// the given kubeconfig path.
+func createKubeconfigGKE(ctx context.Context, state map[string]*tfjson.StateOutput, kcPath string) error {
+	kubeconfigYaml, ok := state["gcp_kubeconfig"].Value.(string)
+	if !ok || kubeconfigYaml == "" {
+		return fmt.Errorf("failed to obtain kubeconfig from tf output")
+	}
+
+	f, err := os.Create(kcPath)
+	if err != nil {
+		return err
+	}
+	_, err = fmt.Fprint(f, kubeconfigYaml)
+	return err
+}
+
+// registryLoginGCR logs into the container/artifact registries using the
+// provider's CLI tools and returns a list of test repositories.
+func registryLoginGCR(ctx context.Context, output map[string]*tfjson.StateOutput) (map[string]string, error) {
+	// NOTE: GCR accepts dynamic repository creation by just pushing a new image
+	// with a new repository name.
+	repoURL := output["gcr_repository_url"].Value.(string)
+	if err := tftestenv.RunCommand(ctx, "./",
+		fmt.Sprintf("gcloud auth configure-docker %s", repoURL),
+		tftestenv.RunCommandOptions{},
+	); err != nil {
+		return nil, err
+	}
+
+	// NOTE: Artifact Registry calls a registry a repository. A repository can
+	// contain multiple different images, unlike ECR or ACR where a repository
+	// can contain multiple tags of only a single image.
+	// Artifact Registry also supports dynamic repository(image) creation by
+	// pushing a new image with a new image name once a new registry(repository)
+	// is created.
+	location := output["gcp_region"].Value.(string)
+	project := output["gcp_project"].Value.(string)
+	repository := output["gcp_artifact_repository"].Value.(string)
+	// Use the fixed docker formatted repository suffix with the location to
+	// create the registry address.
+	artifactRegistry := fmt.Sprintf("%s-docker.pkg.dev", location)
+	artifactURL := fmt.Sprintf("%s/%s/%s", artifactRegistry, project, repository)
+	if err := tftestenv.RunCommand(ctx, "./",
+		fmt.Sprintf("gcloud auth configure-docker %s", artifactRegistry),
+		tftestenv.RunCommandOptions{},
+	); err != nil {
+		return nil, err
+	}
+
+	return map[string]string{
+		"gcr":               repoURL + "/" + randStringRunes(5),
+		"artifact_registry": artifactURL + "/" + randStringRunes(5),
+	}, nil
+}

tests/integration/kustomization.yaml

@@ -18,3 +18,6 @@ patches:
     - op: add
       path: /spec/template/spec/containers/0/args/-
       value: --azure-autologin-for-acr
+    - op: add
+      path: /spec/template/spec/containers/0/args/-
+      value: --gcp-autologin-for-gcr

tests/integration/suite_test.go

@@ -40,6 +40,9 @@ const (
 	// terraformPathAzure is the path to the terraform working directory
 	// containing the azure terraform configurations.
 	terraformPathAzure = "./terraform/azure"
+	// terraformPathGCP is the path to the terraform working directory
+	// containing the gcp terraform configurations.
+	terraformPathGCP = "./terraform/gcp"
 	// kubeconfigPath is the path where the cluster kubeconfig is written to and
 	// used from.
 	kubeconfigPath = "./build/kubeconfig"
@@ -52,7 +55,7 @@ const (
 
 var (
 	// supportedProviders are the providers supported by the test.
-	supportedProviders = []string{"aws", "azure"}
+	supportedProviders = []string{"aws", "azure", "gcp"}
 
 	// targetProvider is the name of the kubernetes provider to test against.
 	targetProvider = flag.String("provider", "", fmt.Sprintf("name of the provider %v", supportedProviders))
@@ -204,6 +207,12 @@ func getProviderConfig(provider string) *ProviderConfig {
 			registryLogin:    registryLoginACR,
 			createKubeconfig: createKubeConfigAKS,
 		}
+	case "gcp":
+		return &ProviderConfig{
+			terraformPath:    terraformPathGCP,
+			registryLogin:    registryLoginGCR,
+			createKubeconfig: createKubeconfigGKE,
+		}
 	}
 	return nil
 }

tests/integration/terraform/gcp/gcr.tf

@@ -0,0 +1,13 @@
+data "google_container_registry_repository" "test_repo" {
+  region = var.gcr_region
+}
+
+resource "google_artifact_registry_repository" "test_repo" {
+  provider = google-beta
+
+  project       = data.google_client_config.current.project
+  location      = data.google_client_config.current.region
+  repository_id = "flux-test-repo-${random_pet.suffix.id}"
+  description   = "example docker repository"
+  format        = "DOCKER"
+}

tests/integration/terraform/gcp/gke.tf

@@ -0,0 +1,28 @@
+data "google_client_config" "current" {}
+
+resource "google_container_cluster" "primary" {
+  name               = local.name
+  location           = data.google_client_config.current.region
+  initial_node_count = 1
+  node_config {
+    machine_type = "g1-small"
+    disk_size_gb = 10
+
+    # Set the scope to grant the nodes all the API access.
+    oauth_scopes = [
+      "https://www.googleapis.com/auth/cloud-platform"
+    ]
+  }
+}
+
+# auth module to retrieve kubeconfig of the created cluster.
+module "gke_auth" {
+  source  = "terraform-google-modules/kubernetes-engine/google//modules/auth"
+  version = "~> 21"
+
+  project_id   = data.google_client_config.current.project
+  cluster_name = local.name
+  location     = data.google_client_config.current.region
+
+  depends_on = [google_container_cluster.primary]
+}

tests/integration/terraform/gcp/main.tf

@@ -0,0 +1,20 @@
+terraform {
+  required_providers {
+    kubectl = {
+      source  = "gavinbunney/kubectl"
+      version = "~> 1.14.0"
+    }
+  }
+}
+
+resource "random_pet" "suffix" {}
+
+locals {
+  name = "flux-test-${random_pet.suffix.id}"
+}
+
+provider "google" {
+  project = var.gcp_project_id
+  region  = var.gcp_region
+  zone    = var.gcp_zone
+}

tests/integration/terraform/gcp/outputs.tf

@@ -0,0 +1,20 @@
+output "gcr_repository_url" {
+  value = data.google_container_registry_repository.test_repo.repository_url
+}
+
+output "gcp_kubeconfig" {
+  value     = module.gke_auth.kubeconfig_raw
+  sensitive = true
+}
+
+output "gcp_project" {
+  value = data.google_client_config.current.project
+}
+
+output "gcp_region" {
+  value = data.google_client_config.current.region
+}
+
+output "gcp_artifact_repository" {
+  value = google_artifact_registry_repository.test_repo.repository_id
+}

tests/integration/terraform/gcp/variables.tf

@@ -0,0 +1,18 @@
+variable "gcp_project_id" {
+  type = string
+}
+
+variable "gcp_region" {
+  type    = string
+  default = "us-central1"
+}
+
+variable "gcp_zone" {
+  type    = string
+  default = "us-central1-c"
+}
+
+variable "gcr_region" {
+  type    = string
+  default = "" // Empty default to use gcr.io.
+}