From 8cf392e20e6179df60ca13fd4e35bb37eeda9847 Mon Sep 17 00:00:00 2001 From: Sam Stoelinga Date: Sun, 12 Jan 2020 18:26:38 -0800 Subject: [PATCH] Allow setting serviceAccount in integration test Previously it would mount .config/gcloud directory which is not recommended for systems such as CI that authenticate with Google Cloud. This commit allows you to set the path to a service account. By default previous behaviour will be as before so this shouldn't break existing systems that run the integration test. --- integration-test.sh | 4 +- integration/images.go | 93 +++++++++++++++++++-------------- integration/integration_test.go | 54 ++++++++++++------- 3 files changed, 90 insertions(+), 61 deletions(-) diff --git a/integration-test.sh b/integration-test.sh index d0a6e318f1..722a85ac41 100755 --- a/integration-test.sh +++ b/integration-test.sh @@ -1,5 +1,4 @@ #!/bin/bash - # Copyright 2018 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -36,5 +35,4 @@ fi echo "Running integration tests..." make out/executor make out/warmer -pushd integration -go test -v --bucket "${GCS_BUCKET}" --repo "${IMAGE_REPO}" --timeout 30m +go test ./integration/... -v --bucket "${GCS_BUCKET}" --repo "${IMAGE_REPO}" --timeout 30m "$@" diff --git a/integration/images.go b/integration/images.go index 8da15d6f73..0ddc8fd8f0 100644 --- a/integration/images.go +++ b/integration/images.go @@ -140,10 +140,21 @@ func NewDockerFileBuilder(dockerfiles []string) *DockerFileBuilder { return &d } +func addServiceAccountFlags(flags []string, serviceAccount string) []string { + if len(serviceAccount) > 0 { + flags = append(flags, "-e", + "GOOGLE_APPLICATION_CREDENTIALS=/secret/"+filepath.Base(serviceAccount), + "-v", filepath.Dir(serviceAccount)+":/secret/") + } else { + flags = append(flags, "-v", os.Getenv("HOME")+"/.config/gcloud:/root/.config/gcloud") + } + return flags +} + // BuildImage will build dockerfile (located at dockerfilesPath) using both kaniko and docker. // The resulting image will be tagged with imageRepo. If the dockerfile will be built with // context (i.e. it is in `buildContextTests`) the context will be pulled from gcsBucket. -func (d *DockerFileBuilder) BuildImage(imageRepo, gcsBucket, dockerfilesPath, dockerfile string) error { +func (d *DockerFileBuilder) BuildImage(imageRepo, gcsBucket, dockerfilesPath, dockerfile, serviceAccount string) error { _, ex, _, _ := runtime.Caller(0) cwd := filepath.Dir(ex) @@ -172,6 +183,7 @@ func (d *DockerFileBuilder) BuildImage(imageRepo, gcsBucket, dockerfilesPath, do if err != nil { return fmt.Errorf("Failed to build image %s with docker command \"%s\": %s %s", dockerImage, dockerCmd.Args, err, string(out)) } + fmt.Printf("Build image for Dockerfile %s as %s. docker build output: %s \n", dockerfile, dockerImage, out) contextFlag := "-c" contextPath := buildContextPath @@ -206,18 +218,22 @@ func (d *DockerFileBuilder) BuildImage(imageRepo, gcsBucket, dockerfilesPath, do // build kaniko image additionalFlags = append(buildArgs, additionalKanikoFlagsMap[dockerfile]...) kanikoImage := GetKanikoImage(imageRepo, dockerfile) - kanikoCmd := exec.Command("docker", - append([]string{"run", - "-v", os.Getenv("HOME") + "/.config/gcloud:/root/.config/gcloud", - "-v", benchmarkDir + ":/kaniko/benchmarks", - "-v", cwd + ":/workspace", - "-e", benchmarkEnv, - ExecutorImage, - "-f", path.Join(buildContextPath, dockerfilesPath, dockerfile), - "-d", kanikoImage, reproducibleFlag, - contextFlag, contextPath}, - additionalFlags...)..., - ) + fmt.Printf("Going to build image with kaniko: %s, flags: %s \n", kanikoImage, additionalFlags) + dockerRunFlags := []string{ + "run", "-e", benchmarkEnv, + "-v", cwd + ":/workspace", + "-v", benchmarkDir + ":/kaniko/benchmarks", + } + + dockerRunFlags = addServiceAccountFlags(dockerRunFlags, serviceAccount) + + dockerRunFlags = append(dockerRunFlags, ExecutorImage, + "-f", path.Join(buildContextPath, dockerfilesPath, dockerfile), + "-d", kanikoImage, reproducibleFlag, + contextFlag, contextPath) + dockerRunFlags = append(dockerRunFlags, additionalFlags...) + + kanikoCmd := exec.Command("docker", dockerRunFlags...) timer = timing.Start(dockerfile + "_kaniko") out, err = RunCommandWithoutTest(kanikoCmd) @@ -235,6 +251,7 @@ func populateVolumeCache() error { cwd := filepath.Dir(ex) warmerCmd := exec.Command("docker", append([]string{"run", + "-d", "-v", os.Getenv("HOME") + "/.config/gcloud:/root/.config/gcloud", "-v", cwd + ":/workspace", WarmerImage, @@ -251,7 +268,7 @@ func populateVolumeCache() error { } // buildCachedImages builds the images for testing caching via kaniko where version is the nth time this image has been built -func (d *DockerFileBuilder) buildCachedImages(imageRepo, cacheRepo, dockerfilesPath string, version int) error { +func (d *DockerFileBuilder) buildCachedImages(imageRepo, cacheRepo, dockerfilesPath, serviceAccount string, version int) error { _, ex, _, _ := runtime.Caller(0) cwd := filepath.Dir(ex) @@ -264,19 +281,19 @@ func (d *DockerFileBuilder) buildCachedImages(imageRepo, cacheRepo, dockerfilesP benchmarkEnv = "BENCHMARK_FILE=/workspace/benchmarks/" + dockerfile } kanikoImage := GetVersionedKanikoImage(imageRepo, dockerfile, version) - kanikoCmd := exec.Command("docker", - append([]string{"run", - "-v", os.Getenv("HOME") + "/.config/gcloud:/root/.config/gcloud", - "-v", cwd + ":/workspace", - "-e", benchmarkEnv, - ExecutorImage, - "-f", path.Join(buildContextPath, dockerfilesPath, dockerfile), - "-d", kanikoImage, - "-c", buildContextPath, - cacheFlag, - "--cache-repo", cacheRepo, - "--cache-dir", cacheDir})..., - ) + + dockerRunFlags := []string{"run", + "-v", cwd + ":/workspace", + "-e", benchmarkEnv} + dockerRunFlags = addServiceAccountFlags(dockerRunFlags, serviceAccount) + dockerRunFlags = append(dockerRunFlags, ExecutorImage, + "-f", path.Join(buildContextPath, dockerfilesPath, dockerfile), + "-d", kanikoImage, + "-c", buildContextPath, + cacheFlag, + "--cache-repo", cacheRepo, + "--cache-dir", cacheDir) + kanikoCmd := exec.Command("docker", dockerRunFlags...) timer := timing.Start(dockerfile + "_kaniko_cached_" + strconv.Itoa(version)) _, err := RunCommandWithoutTest(kanikoCmd) @@ -289,24 +306,22 @@ func (d *DockerFileBuilder) buildCachedImages(imageRepo, cacheRepo, dockerfilesP } // buildRelativePathsImage builds the images for testing passing relatives paths to Kaniko -func (d *DockerFileBuilder) buildRelativePathsImage(imageRepo, dockerfile string) error { +func (d *DockerFileBuilder) buildRelativePathsImage(imageRepo, dockerfile, serviceAccount string) error { _, ex, _, _ := runtime.Caller(0) cwd := filepath.Dir(ex) buildContextPath := "./relative-subdirectory" kanikoImage := GetKanikoImage(imageRepo, dockerfile) - kanikoCmd := exec.Command("docker", - append([]string{"run", - "-v", os.Getenv("HOME") + "/.config/gcloud:/root/.config/gcloud", - "-v", cwd + ":/workspace", - ExecutorImage, - "-f", dockerfile, - "-d", kanikoImage, - "--digest-file", "./digest", - "-c", buildContextPath, - })..., - ) + dockerRunFlags := []string{"run", "-v", cwd + ":/workspace"} + dockerRunFlags = addServiceAccountFlags(dockerRunFlags, serviceAccount) + dockerRunFlags = append(dockerRunFlags, ExecutorImage, + "-f", dockerfile, + "-d", kanikoImage, + "--digest-file", "./digest", + "-c", buildContextPath) + + kanikoCmd := exec.Command("docker", dockerRunFlags...) timer := timing.Start(dockerfile + "_kaniko_relative_paths") _, err := RunCommandWithoutTest(kanikoCmd) diff --git a/integration/integration_test.go b/integration/integration_test.go index d1dd9c8c8e..b23e37b130 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -200,13 +200,14 @@ func TestGitBuildcontext(t *testing.T) { // Build with kaniko kanikoImage := GetKanikoImage(config.imageRepo, "Dockerfile_test_git") - kanikoCmd := exec.Command("docker", - append([]string{"run", - "-v", os.Getenv("HOME") + "/.config/gcloud:/root/.config/gcloud", - ExecutorImage, - "-f", dockerfile, - "-d", kanikoImage, - "-c", fmt.Sprintf("git://%s", repo)})...) + dockerRunFlags := []string{"run"} + dockerRunFlags = addServiceAccountFlags(dockerRunFlags, config.serviceAccount) + dockerRunFlags = append(dockerRunFlags, ExecutorImage, + "-f", dockerfile, + "-d", kanikoImage, + "-c", fmt.Sprintf("git://%s", repo)) + + kanikoCmd := exec.Command("docker", dockerRunFlags...) out, err = RunCommandWithoutTest(kanikoCmd) if err != nil { @@ -243,13 +244,14 @@ func TestGitBuildContextWithBranch(t *testing.T) { // Build with kaniko kanikoImage := GetKanikoImage(config.imageRepo, "Dockerfile_test_git") - kanikoCmd := exec.Command("docker", - append([]string{"run", - "-v", os.Getenv("HOME") + "/.config/gcloud:/root/.config/gcloud", - ExecutorImage, - "-f", dockerfile, - "-d", kanikoImage, - "-c", fmt.Sprintf("git://%s", repo)})...) + dockerRunFlags := []string{"run"} + dockerRunFlags = addServiceAccountFlags(dockerRunFlags, config.serviceAccount) + dockerRunFlags = append(dockerRunFlags, ExecutorImage, + "-f", dockerfile, + "-d", kanikoImage, + "-c", fmt.Sprintf("git://%s", repo)) + + kanikoCmd := exec.Command("docker", dockerRunFlags...) out, err = RunCommandWithoutTest(kanikoCmd) if err != nil { @@ -306,7 +308,7 @@ func buildImage(t *testing.T, dockerfile string, imageBuilder *DockerFileBuilder } if err := imageBuilder.BuildImage( - config.imageRepo, config.gcsBucket, dockerfilesPath, dockerfile, + config.imageRepo, config.gcsBucket, dockerfilesPath, dockerfile, config.serviceAccount, ); err != nil { t.Errorf("Error building image: %s", err) t.FailNow() @@ -324,11 +326,11 @@ func TestCache(t *testing.T) { t.Parallel() cache := filepath.Join(config.imageRepo, "cache", fmt.Sprintf("%v", time.Now().UnixNano())) // Build the initial image which will cache layers - if err := imageBuilder.buildCachedImages(config.imageRepo, cache, dockerfilesPath, 0); err != nil { + if err := imageBuilder.buildCachedImages(config.imageRepo, cache, dockerfilesPath, config.serviceAccount, 0); err != nil { t.Fatalf("error building cached image for the first time: %v", err) } // Build the second image which should pull from the cache - if err := imageBuilder.buildCachedImages(config.imageRepo, cache, dockerfilesPath, 1); err != nil { + if err := imageBuilder.buildCachedImages(config.imageRepo, cache, dockerfilesPath, config.serviceAccount, 1); err != nil { t.Fatalf("error building cached image for the first time: %v", err) } // Make sure both images are the same @@ -359,7 +361,7 @@ func TestRelativePaths(t *testing.T) { t.Run("test_relative_"+dockerfile, func(t *testing.T) { t.Parallel() - imageBuilder.buildRelativePathsImage(config.imageRepo, dockerfile) + imageBuilder.buildRelativePathsImage(config.imageRepo, dockerfile, config.serviceAccount) dockerImage := GetDockerImage(config.imageRepo, dockerfile) kanikoImage := GetKanikoImage(config.imageRepo, dockerfile) @@ -495,6 +497,7 @@ type gcpConfig struct { imageRepo string onbuildBaseImage string hardlinkBaseImage string + serviceAccount string } type imageDetails struct { @@ -510,9 +513,22 @@ func (i imageDetails) String() string { func initGCPConfig() *gcpConfig { var c gcpConfig flag.StringVar(&c.gcsBucket, "bucket", "gs://kaniko-test-bucket", "The gcs bucket argument to uploaded the tar-ed contents of the `integration` dir to.") - flag.StringVar(&c.imageRepo, "repo", "gcr.io/kaniko-test", "The (docker) image repo to build and push images to during the test. `gcloud` must be authenticated with this repo.") + flag.StringVar(&c.imageRepo, "repo", "gcr.io/kaniko-test", "The (docker) image repo to build and push images to during the test. `gcloud` must be authenticated with this repo or serviceAccount must be set.") + flag.StringVar(&c.serviceAccount, "serviceAccount", "", "The path to the service account push images to GCR and upload/download files to GCS.") flag.Parse() + if len(c.serviceAccount) > 0 { + absPath, err := filepath.Abs("../" + c.serviceAccount) + if err != nil { + log.Fatalf("Error getting absolute path for service account: %s\n", c.serviceAccount) + } + if _, err := os.Stat(absPath); os.IsNotExist(err) { + log.Fatalf("Service account does not exist: %s\n", absPath) + } + c.serviceAccount = absPath + os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", absPath) + } + if c.gcsBucket == "" || c.imageRepo == "" { log.Fatalf("You must provide a gcs bucket (\"%s\" was provided) and a docker repo (\"%s\" was provided)", c.gcsBucket, c.imageRepo) }