Skip to content

Commit

Permalink
Allow setting serviceAccount in integration test
Browse files Browse the repository at this point in the history 
Previously it would mount .config/gcloud directory which is not
recommended for systems such as CI that authenticate with Google Cloud.
This commit allows you to set the path to a service account.

By default previous behaviour will be as before so this shouldn't break
existing systems that run the integration test.
  • Loading branch information
@samos123
samos123 committed Jan 13, 2020
1 parent 464ac13 commit 43e4d85
Show file tree
Hide file tree
Showing 3 changed files with 92 additions and 61 deletions.
4 changes: 1 addition & 3 deletions integration-test.sh
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
#!/bin/bash

# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
Expand Down Expand Up @@ -36,5 +35,4 @@ fi
echo "Running integration tests..."
make out/executor
make out/warmer
pushd integration
go test -v --bucket "${GCS_BUCKET}" --repo "${IMAGE_REPO}" --timeout 30m
go test ./integration/... -v --bucket "${GCS_BUCKET}" --repo "${IMAGE_REPO}" --timeout 30m "$@"
95 changes: 56 additions & 39 deletions integration/images.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,10 +140,21 @@ func NewDockerFileBuilder(dockerfiles []string) *DockerFileBuilder {
return &d
}

func addServiceAccountFlags(flags []string, serviceAccount string) []string {
if len(serviceAccount) > 0 {
flags = append(flags, "-e",
"GOOGLE_APPLICATION_CREDENTIALS=/secret/"+filepath.Base(serviceAccount),
"-v", filepath.Dir(serviceAccount)+":/secret/")
} else {
flags = append(flags, "-v", os.Getenv("HOME")+"/.config/gcloud:/root/.config/gcloud")
}
return flags
}

// BuildImage will build dockerfile (located at dockerfilesPath) using both kaniko and docker.
// The resulting image will be tagged with imageRepo. If the dockerfile will be built with
// context (i.e. it is in `buildContextTests`) the context will be pulled from gcsBucket.
func (d *DockerFileBuilder) BuildImage(imageRepo, gcsBucket, dockerfilesPath, dockerfile string) error {
func (d *DockerFileBuilder) BuildImage(imageRepo, gcsBucket, dockerfilesPath, dockerfile, serviceAccount string) error {
_, ex, _, _ := runtime.Caller(0)
cwd := filepath.Dir(ex)

Expand Down Expand Up @@ -172,6 +183,7 @@ func (d *DockerFileBuilder) BuildImage(imageRepo, gcsBucket, dockerfilesPath, do
if err != nil {
return fmt.Errorf("Failed to build image %s with docker command \"%s\": %s %s", dockerImage, dockerCmd.Args, err, string(out))
}
fmt.Printf("Build image for Dockerfile %s as %s. docker build output: %s \n", dockerfile, dockerImage, out)

contextFlag := "-c"
contextPath := buildContextPath
Expand Down Expand Up @@ -206,25 +218,31 @@ func (d *DockerFileBuilder) BuildImage(imageRepo, gcsBucket, dockerfilesPath, do
// build kaniko image
additionalFlags = append(buildArgs, additionalKanikoFlagsMap[dockerfile]...)
kanikoImage := GetKanikoImage(imageRepo, dockerfile)
kanikoCmd := exec.Command("docker",
append([]string{"run",
"-v", os.Getenv("HOME") + "/.config/gcloud:/root/.config/gcloud",
"-v", benchmarkDir + ":/kaniko/benchmarks",
"-v", cwd + ":/workspace",
"-e", benchmarkEnv,
ExecutorImage,
"-f", path.Join(buildContextPath, dockerfilesPath, dockerfile),
"-d", kanikoImage, reproducibleFlag,
contextFlag, contextPath},
additionalFlags...)...,
)
fmt.Printf("Going to build image with kaniko: %s, flags: %s \n", kanikoImage, additionalFlags)
dockerRunFlags := []string{
"run", "-e", benchmarkEnv,
"-v", cwd + ":/workspace",
"-v", benchmarkDir + ":/kaniko/benchmarks",
}

dockerRunFlags = addServiceAccountFlags(dockerRunFlags, serviceAccount)

dockerRunFlags = append(dockerRunFlags, ExecutorImage,
"-f", path.Join(buildContextPath, dockerfilesPath, dockerfile),
"-d", kanikoImage, reproducibleFlag,
contextFlag, contextPath)
dockerRunFlags = append(dockerRunFlags, additionalFlags...)

kanikoCmd := exec.Command("docker", dockerRunFlags...)

fmt.Printf("Starting kaniko executor: %s", kanikoCmd)
timer = timing.Start(dockerfile + "_kaniko")
out, err = RunCommandWithoutTest(kanikoCmd)
timing.DefaultRun.Stop(timer)
if err != nil {
return fmt.Errorf("Failed to build image %s with kaniko command \"%s\": %s %s", dockerImage, kanikoCmd.Args, err, string(out))
}
fmt.Printf("Successfully ran kaniko executor: %s output: %s \n", kanikoCmd, out)

d.FilesBuilt[dockerfile] = true
return nil
Expand All @@ -235,6 +253,7 @@ func populateVolumeCache() error {
cwd := filepath.Dir(ex)
warmerCmd := exec.Command("docker",
append([]string{"run",
"-d",
"-v", os.Getenv("HOME") + "/.config/gcloud:/root/.config/gcloud",
"-v", cwd + ":/workspace",
WarmerImage,
Expand All @@ -251,7 +270,7 @@ func populateVolumeCache() error {
}

// buildCachedImages builds the images for testing caching via kaniko where version is the nth time this image has been built
func (d *DockerFileBuilder) buildCachedImages(imageRepo, cacheRepo, dockerfilesPath string, version int) error {
func (d *DockerFileBuilder) buildCachedImages(imageRepo, cacheRepo, dockerfilesPath, serviceAccount string, version int) error {
_, ex, _, _ := runtime.Caller(0)
cwd := filepath.Dir(ex)

Expand All @@ -264,19 +283,19 @@ func (d *DockerFileBuilder) buildCachedImages(imageRepo, cacheRepo, dockerfilesP
benchmarkEnv = "BENCHMARK_FILE=/workspace/benchmarks/" + dockerfile
}
kanikoImage := GetVersionedKanikoImage(imageRepo, dockerfile, version)
kanikoCmd := exec.Command("docker",
append([]string{"run",
"-v", os.Getenv("HOME") + "/.config/gcloud:/root/.config/gcloud",
"-v", cwd + ":/workspace",
"-e", benchmarkEnv,
ExecutorImage,
"-f", path.Join(buildContextPath, dockerfilesPath, dockerfile),
"-d", kanikoImage,
"-c", buildContextPath,
cacheFlag,
"--cache-repo", cacheRepo,
"--cache-dir", cacheDir})...,
)

dockerRunFlags := []string{"run",
"-v", cwd + ":/workspace",
"-e", benchmarkEnv}
dockerRunFlags = addServiceAccountFlags(dockerRunFlags, serviceAccount)
dockerRunFlags = append(dockerRunFlags, ExecutorImage,
"-f", path.Join(buildContextPath, dockerfilesPath, dockerfile),
"-d", kanikoImage,
"-c", buildContextPath,
cacheFlag,
"--cache-repo", cacheRepo,
"--cache-dir", cacheDir)
kanikoCmd := exec.Command("docker", dockerRunFlags...)

timer := timing.Start(dockerfile + "_kaniko_cached_" + strconv.Itoa(version))
_, err := RunCommandWithoutTest(kanikoCmd)
Expand All @@ -289,24 +308,22 @@ func (d *DockerFileBuilder) buildCachedImages(imageRepo, cacheRepo, dockerfilesP
}

// buildRelativePathsImage builds the images for testing passing relatives paths to Kaniko
func (d *DockerFileBuilder) buildRelativePathsImage(imageRepo, dockerfile string) error {
func (d *DockerFileBuilder) buildRelativePathsImage(imageRepo, dockerfile, serviceAccount string) error {
_, ex, _, _ := runtime.Caller(0)
cwd := filepath.Dir(ex)

buildContextPath := "./relative-subdirectory"
kanikoImage := GetKanikoImage(imageRepo, dockerfile)

kanikoCmd := exec.Command("docker",
append([]string{"run",
"-v", os.Getenv("HOME") + "/.config/gcloud:/root/.config/gcloud",
"-v", cwd + ":/workspace",
ExecutorImage,
"-f", dockerfile,
"-d", kanikoImage,
"--digest-file", "./digest",
"-c", buildContextPath,
})...,
)
dockerRunFlags := []string{"run", "-v", cwd + ":/workspace"}
dockerRunFlags = addServiceAccountFlags(dockerRunFlags, serviceAccount)
dockerRunFlags = append(dockerRunFlags, ExecutorImage,
"-f", dockerfile,
"-d", kanikoImage,
"--digest-file", "./digest",
"-c", buildContextPath)

kanikoCmd := exec.Command("docker", dockerRunFlags...)

timer := timing.Start(dockerfile + "_kaniko_relative_paths")
_, err := RunCommandWithoutTest(kanikoCmd)
Expand Down
54 changes: 35 additions & 19 deletions integration/integration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,13 +200,14 @@ func TestGitBuildcontext(t *testing.T) {

// Build with kaniko
kanikoImage := GetKanikoImage(config.imageRepo, "Dockerfile_test_git")
kanikoCmd := exec.Command("docker",
append([]string{"run",
"-v", os.Getenv("HOME") + "/.config/gcloud:/root/.config/gcloud",
ExecutorImage,
"-f", dockerfile,
"-d", kanikoImage,
"-c", fmt.Sprintf("git://%s", repo)})...)
dockerRunFlags := []string{"run"}
dockerRunFlags = addServiceAccountFlags(dockerRunFlags, config.serviceAccount)
dockerRunFlags = append(dockerRunFlags, ExecutorImage,
"-f", dockerfile,
"-d", kanikoImage,
"-c", fmt.Sprintf("git://%s", repo))

kanikoCmd := exec.Command("docker", dockerRunFlags...)

out, err = RunCommandWithoutTest(kanikoCmd)
if err != nil {
Expand Down Expand Up @@ -243,13 +244,14 @@ func TestGitBuildContextWithBranch(t *testing.T) {

// Build with kaniko
kanikoImage := GetKanikoImage(config.imageRepo, "Dockerfile_test_git")
kanikoCmd := exec.Command("docker",
append([]string{"run",
"-v", os.Getenv("HOME") + "/.config/gcloud:/root/.config/gcloud",
ExecutorImage,
"-f", dockerfile,
"-d", kanikoImage,
"-c", fmt.Sprintf("git://%s", repo)})...)
dockerRunFlags := []string{"run"}
dockerRunFlags = addServiceAccountFlags(dockerRunFlags, config.serviceAccount)
dockerRunFlags = append(dockerRunFlags, ExecutorImage,
"-f", dockerfile,
"-d", kanikoImage,
"-c", fmt.Sprintf("git://%s", repo))

kanikoCmd := exec.Command("docker", dockerRunFlags...)

out, err = RunCommandWithoutTest(kanikoCmd)
if err != nil {
Expand Down Expand Up @@ -306,7 +308,7 @@ func buildImage(t *testing.T, dockerfile string, imageBuilder *DockerFileBuilder
}

if err := imageBuilder.BuildImage(
config.imageRepo, config.gcsBucket, dockerfilesPath, dockerfile,
config.imageRepo, config.gcsBucket, dockerfilesPath, dockerfile, config.serviceAccount,
); err != nil {
t.Errorf("Error building image: %s", err)
t.FailNow()
Expand All @@ -324,11 +326,11 @@ func TestCache(t *testing.T) {
t.Parallel()
cache := filepath.Join(config.imageRepo, "cache", fmt.Sprintf("%v", time.Now().UnixNano()))
// Build the initial image which will cache layers
if err := imageBuilder.buildCachedImages(config.imageRepo, cache, dockerfilesPath, 0); err != nil {
if err := imageBuilder.buildCachedImages(config.imageRepo, cache, dockerfilesPath, config.serviceAccount, 0); err != nil {
t.Fatalf("error building cached image for the first time: %v", err)
}
// Build the second image which should pull from the cache
if err := imageBuilder.buildCachedImages(config.imageRepo, cache, dockerfilesPath, 1); err != nil {
if err := imageBuilder.buildCachedImages(config.imageRepo, cache, dockerfilesPath, config.serviceAccount, 1); err != nil {
t.Fatalf("error building cached image for the first time: %v", err)
}
// Make sure both images are the same
Expand Down Expand Up @@ -359,7 +361,7 @@ func TestRelativePaths(t *testing.T) {

t.Run("test_relative_"+dockerfile, func(t *testing.T) {
t.Parallel()
imageBuilder.buildRelativePathsImage(config.imageRepo, dockerfile)
imageBuilder.buildRelativePathsImage(config.imageRepo, dockerfile, config.serviceAccount)

dockerImage := GetDockerImage(config.imageRepo, dockerfile)
kanikoImage := GetKanikoImage(config.imageRepo, dockerfile)
Expand Down Expand Up @@ -495,6 +497,7 @@ type gcpConfig struct {
imageRepo string
onbuildBaseImage string
hardlinkBaseImage string
serviceAccount string
}

type imageDetails struct {
Expand All @@ -510,9 +513,22 @@ func (i imageDetails) String() string {
func initGCPConfig() *gcpConfig {
var c gcpConfig
flag.StringVar(&c.gcsBucket, "bucket", "gs://kaniko-test-bucket", "The gcs bucket argument to uploaded the tar-ed contents of the `integration` dir to.")
flag.StringVar(&c.imageRepo, "repo", "gcr.io/kaniko-test", "The (docker) image repo to build and push images to during the test. `gcloud` must be authenticated with this repo.")
flag.StringVar(&c.imageRepo, "repo", "gcr.io/kaniko-test", "The (docker) image repo to build and push images to during the test. `gcloud` must be authenticated with this repo or serviceAccount must be set.")
flag.StringVar(&c.serviceAccount, "serviceAccount", "", "The path to the service account push images to GCR and upload/download files to GCS.")
flag.Parse()

if len(c.serviceAccount) > 0 {
absPath, err := filepath.Abs("../" + c.serviceAccount)
if err != nil {
log.Fatalf("Error getting absolute path for service account: %s\n", c.serviceAccount)
}
if _, err := os.Stat(absPath); os.IsNotExist(err) {
log.Fatalf("Service account does not exist: %s\n", absPath)
}
c.serviceAccount = absPath
os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", absPath)
}

if c.gcsBucket == "" || c.imageRepo == "" {
log.Fatalf("You must provide a gcs bucket (\"%s\" was provided) and a docker repo (\"%s\" was provided)", c.gcsBucket, c.imageRepo)
}
Expand Down

0 comments on commit 43e4d85

Please sign in to comment.