Add native registry login tests for EKS, AKS and GKE

tests/integration/.env.sample

@@ -0,0 +1,14 @@
+## AWS
+# export AWS_ACCESS_KEY_ID=
+# export AWS_SECRET_ACCESS_KEY=
+# export AWS_REGION=us-east-2
+
+## Azure
+# export TF_VAR_azure_location=eastus
+
+## GCP
+# export TF_VAR_gcp_project_id=
+# export TF_VAR_gcp_region=us-central1
+# export TF_VAR_gcp_zone=us-central1-c
+## Leave GCR region empty to use gcr.io. Else set it to `us`, `eu` or `asia`.
+# export TF_VAR_gcr_region=

tests/integration/.gitignore

@@ -0,0 +1,4 @@
+.env
+build
+.terraform*
+terraform.tfstate*

tests/integration/Makefile

@@ -0,0 +1,35 @@
+GO_TEST_PATH ?= ./...
+GO_TEST_ARGS ?=
+PROVIDER_ARG ?=
+TEST_TIMEOUT ?= 30m
+FLUX_MANIFEST_URL ?= https://github.com/fluxcd/flux2/releases/latest/download/install.yaml
+INSTALL_MANIFEST_PATH ?= build/flux/install.yaml
+
+IMG ?= fluxcd/image-reflector-controller
+
+$(INSTALL_MANIFEST_PATH):
+	mkdir -p build/flux
+	curl -Lo $(INSTALL_MANIFEST_PATH) $(FLUX_MANIFEST_URL)
+
+# Build the manifests required in the test.
+build-manifests: $(INSTALL_MANIFEST_PATH)
+	cp kustomization.yaml build/flux
+	cd build/flux && kustomize edit set image fluxcd/image-reflector-controller=${IMG}
+	kustomize build build/flux > build/flux.yaml
+
+# Delete all the build files.
+distclean:
+	rm -r build/
+
+# Builds manifests and run the tests.
+test: build-manifests
+	go test -timeout $(TEST_TIMEOUT) -v $(GO_TEST_PATH) $(GO_TEST_ARGS) $(PROVIDER_ARG)
+
+test-aws:
+	$(MAKE) test PROVIDER_ARG="-provider aws"
+
+test-azure:
+	$(MAKE) test PROVIDER_ARG="-provider azure"
+
+test-gcp:
+	$(MAKE) test PROVIDER_ARG="-provider gcp"

tests/integration/README.md

@@ -0,0 +1,134 @@
+# Cloud Provider Integration Tests
+
+## Requirements
+
+### Amazon Web Services
+
+- AWS account with access key ID and secret access key with permissions to
+    create EKS cluster and ECR repository.
+- AWS CLI, does not need to be configured with the AWS account.
+- Docker CLI for registry login.
+- kubectl for applying certain install manifests.
+
+### Microsoft Azure
+
+- Azure account with an active subscription to be able to create AKS and ACR,
+    and permission to assign roles. Role assignment is required for allowing AKS
+    workloads to access ECR.
+- Azure CLI, need to be logged in using `az login`.
+- Docker CLI for registry login.
+- kubectl for applying certain install manifests.
+
+### Google Cloud Platform
+
+- GCP account with project and GKE, GCR and Artifact Registry services enabled
+    in the project.
+- gcloud CLI, need to be logged in using `gcloud auth login`.
+- Docker CLI for registry login.
+- kubectl for applying certain install manifests.
+
+**NOTE:** Unlike ECR, AKS and Google Artifact Registry, Google Container
+Registry tests don't create a new registry. It pushes to an existing registry
+host in a project, for example `gcr.io`. Due to this, the test images pushed to
+GCR aren't cleaned up automatically at the end of the test and have to be
+deleted manually.
+
+## Test setup
+
+Copy `.env.sample` to `.env`, put the respective provider configurations in the
+environment variables and source it, `source .env`.
+
+Run the test with `make test-*`:
+
+```console
+$ make test-aws
+mkdir -p build/flux
+curl -Lo build/flux/install.yaml https://github.com/fluxcd/flux2/releases/latest/download/install.yaml
+  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
+                                 Dload  Upload   Total   Spent    Left  Speed
+100  351k  100  351k    0     0   247k      0  0:00:01  0:00:01 --:--:-- 3609k
+cp kustomization.yaml build/flux
+cd build/flux && kustomize edit set image fluxcd/image-reflector-controller=fluxcd/image-reflector-controller
+kustomize build build/flux > build/flux.yaml
+go test -timeout 20m -v ./... -existing
+2022/06/15 01:55:09 Terraform binary:  /go/src/github.com/fluxcd/image-reflector-controller/tests/integration/build/terraform
+2022/06/15 01:55:09 Init Terraform
+2022/06/15 01:55:14 Applying Terraform
+2022/06/15 01:55:41 pushing test image foo111.dkr.ecr.us-east-2.amazonaws.com/flux-image-automation-test:v0.1.0
+2022/06/15 01:55:45 pushing test image foo111.dkr.ecr.us-east-2.amazonaws.com/flux-image-automation-test:v0.1.2
+2022/06/15 01:55:48 pushing test image foo111.dkr.ecr.us-east-2.amazonaws.com/flux-image-automation-test:v0.1.3
+2022/06/15 01:55:51 pushing test image foo111.dkr.ecr.us-east-2.amazonaws.com/flux-image-automation-test:v0.1.4
+2022/06/15 01:55:54 Installing flux
+=== RUN   TestImageRepositoryScan
+=== RUN   TestImageRepositoryScan/ecr
+--- PASS: TestImageRepositoryScan (2.15s)
+    --- PASS: TestImageRepositoryScan/ecr (2.15s)
+PASS
+2022/06/15 01:56:14 Destroying environment...
+ok      github.com/fluxcd/image-reflector-controller/tests/integration  1673.225s
+```
+
+In the above, the test created a build directory `build/` and downloaded the
+latest flux install manifest at `build/flux/install.yaml`. This will be used to
+install flux in the test cluster. The manifest download can be configured by
+setting the `FLUX_MANIFEST_URL` variable. Once downloaded, the file can be
+manually modified, if needed, it won't be downloaded again unless it's deleted.
+
+Then the `kustomization.yaml` is copied to `build/flux/`. This kustomization
+contains configurations to configure the flux installation by patching the
+downloaded `install.yaml`. It can also be used to set any custom images for any
+of the flux components. The image-reflector-controller image can be configured
+by setting the `IMG` variable when running the test. The kustomization is built
+and the resulting flux installation manifest is written to `build/flux.yaml`.
+This is used by the test to install flux.
+
+The go test is started with a long timeout because the infrastructure set up
+can take a long time. It can also be configured by setting the variable
+`TEST_TIMEOUT`. The test creates a new infrastructure using `tftestenv`, like
+`testenv` but helps create kubernetes cluster using terraform. It looks for any
+existing terraform binary on the current `$PATH` and downloads a new binary in
+`build/terraform` if it couldn't find one locally.
+The terraform configurations are present in `terraform/<provider>` directory.
+All the terraform state created by the test run are written in
+`terraform/<provider>` directory. The test creates a managed kubernetes cluster
+and a container registry (with optional repository in some cases). The
+repository is populated with a few randomly generated test images. The registry
+login is performed using the cloud provider CLI and docker CLI. The credentials
+are written into the default docker client config file. Flux is then installed
+using the initial `build/flux.yaml` manifest.
+
+Once the environment is ready, the individual go tests are executed. After the
+tests end, the environment is destroyed automatically.
+
+**IMPORTANT**: In case the terraform infrastructure results in a bad state,
+maybe due to a crash during the apply, the whole infrastructure can be destroyed
+by running `terraform destroy` in `terraform/<provider>` directory.
+
+## Debugging the tests
+
+For debugging environment provisioning, enable verbose output with `-verbose`
+test flag.
+
+```console
+$ make test-aws GO_TEST_ARGS="-verbose"
+```
+
+The test environment is destroyed at the end by default. Run the tests with
+`-retain` flag to retain the created test infrastructure.
+
+```console
+$ make test-aws GO_TEST_ARGS="-retain"
+```
+
+The tests require the infrastructure state to be clean. For re-running the tests
+with a retained infrastructure, set `-existing` flag.
+
+```console
+$ make test-aws GO_TEST_ARGS="-retain -existing"
+```
+
+To delete an existing infrastructure created with `-retain` flag:
+
+```console
+$ make test-aws GO_TEST_ARGS="-existing"
+```

tests/integration/aws_test.go

@@ -0,0 +1,107 @@
+/*
+Copyright 2022 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	tfjson "github.com/hashicorp/terraform-json"
+
+	tftestenv "github.com/fluxcd/image-reflector-controller/tests/tftestenv"
+)
+
+// Based on https://docs.aws.amazon.com/eks/latest/userguide/create-kubeconfig.html
+const kubeConfigTmpl = `
+apiVersion: v1
+clusters:
+- cluster:
+    certificate-authority-data: %[1]s
+    server: %[2]s
+  name: %[3]s
+contexts:
+- context:
+    cluster: %[3]s
+    user: %[4]s
+  name: %[3]s
+current-context: %[3]s
+kind: Config
+preferences: {}
+users:
+- name: %[4]s
+  user:
+    token: %[5]s
+`
+
+// kubeconfigWithClusterAuthToken returns a kubeconfig with the given cluster
+// authentication token.
+func kubeconfigWithClusterAuthToken(token, caData, endpoint, user, clusterName string) string {
+	return fmt.Sprintf(kubeConfigTmpl, caData, endpoint, clusterName, user, token)
+}
+
+// getEKSClientToken fetches the EKS cluster client token.
+func getEKSClientToken(ctx context.Context, clusterName string) ([]byte, error) {
+	err := tftestenv.RunCommand(ctx, "build",
+		fmt.Sprintf("aws eks get-token --cluster-name %s | jq -r .status.token > token", clusterName),
+		tftestenv.RunCommandOptions{},
+	)
+	if err != nil {
+		return nil, err
+	}
+	return os.ReadFile("build/token")
+}
+
+// createKubeconfigEKS constructs kubeconfig from the terraform state output at
+// the given kubeconfig path.
+func createKubeconfigEKS(ctx context.Context, state map[string]*tfjson.StateOutput, kcPath string) error {
+	clusterName := state["eks_cluster_name"].Value.(string)
+	eksHost := state["eks_cluster_endpoint"].Value.(string)
+	eksClusterArn := state["eks_cluster_endpoint"].Value.(string)
+	eksCa := state["eks_cluster_ca_certificate"].Value.(string)
+	eksToken, err := getEKSClientToken(ctx, clusterName)
+	if err != nil {
+		return fmt.Errorf("failed to obtain auth token: %w", err)
+	}
+
+	kubeconfigYaml := kubeconfigWithClusterAuthToken(string(eksToken), eksCa, eksHost, eksClusterArn, eksClusterArn)
+
+	f, err := os.Create(kcPath)
+	if err != nil {
+		return err
+	}
+	_, err = fmt.Fprint(f, kubeconfigYaml)
+	return err
+}
+
+// registryLoginECR logs into the container/artifact registries using the
+// provider's CLI tools and returns a list of test repositories.
+func registryLoginECR(ctx context.Context, output map[string]*tfjson.StateOutput) (map[string]string, error) {
+	// NOTE: ECR provides pre-existing registry per account. It requires
+	// repositories to be created explicitly using their API before pushing
+	// image.
+	repoURL := output["ecr_repository_url"].Value.(string)
+
+	if err := tftestenv.RunCommand(ctx, "./",
+		fmt.Sprintf("aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin %s", repoURL),
+		tftestenv.RunCommandOptions{},
+	); err != nil {
+		return nil, err
+	}
+
+	return map[string]string{"ecr": repoURL}, nil
+}

tests/integration/azure_test.go

@@ -0,0 +1,61 @@
+/*
+Copyright 2022 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	tfjson "github.com/hashicorp/terraform-json"
+
+	"github.com/fluxcd/image-reflector-controller/tests/tftestenv"
+)
+
+// createKubeConfigAKS constructs kubeconfig for an AKS cluster from the
+// terraform state output at the given kubeconfig path.
+func createKubeConfigAKS(ctx context.Context, state map[string]*tfjson.StateOutput, kcPath string) error {
+	kubeconfigYaml, ok := state["aks_kubeconfig"].Value.(string)
+	if !ok || kubeconfigYaml == "" {
+		return fmt.Errorf("failed to obtain kubeconfig from tf output")
+	}
+
+	f, err := os.Create(kcPath)
+	if err != nil {
+		return err
+	}
+	_, err = fmt.Fprint(f, kubeconfigYaml)
+	if err != nil {
+		return err
+	}
+	return f.Close()
+}
+
+// registryLoginACR logs into the container/artifact registries using the
+// provider's CLI tools and returns a list of test repositories.
+func registryLoginACR(ctx context.Context, output map[string]*tfjson.StateOutput) (map[string]string, error) {
+	// NOTE: ACR registry accept dynamic repository creation by just pushing a
+	// new image with a new repository name.
+	registryURL := output["acr_registry_url"].Value.(string)
+
+	if err := tftestenv.RunCommand(ctx, "./",
+		fmt.Sprintf("az acr login --name %s", registryURL), tftestenv.RunCommandOptions{},
+	); err != nil {
+		return nil, err
+	}
+	return map[string]string{"acr": registryURL + "/" + randStringRunes(5)}, nil
+}