add caching copy layers back (#1518)

* add caching copy layers back

* fix test

* lint

* fix test - 2

* Add integration test

* fix lint

README.md

@@ -386,7 +386,7 @@ as a remote image destination:
 ### Caching
 
 #### Caching Layers
-kaniko can cache layers created by `RUN` commands in a remote repository.
+kaniko can cache layers created by `RUN` and `Copy` (configured by flag `--cache-copy-layers`) commands in a remote repository.
 Before executing a command, kaniko checks the cache for the layer.
 If it exists, kaniko will pull and extract the cached layer instead of executing the command.
 If not, kaniko will execute the command and then push the newly created layer to the cache.

cmd/executor/cmd/root.go

@@ -179,6 +179,7 @@ func addKanikoOptionsFlags() {
 	RootCmd.PersistentFlags().BoolVarP(&opts.SkipUnusedStages, "skip-unused-stages", "", false, "Build only used stages if defined to true. Otherwise it builds by default all stages, even the unnecessaries ones until it reaches the target stage / end of Dockerfile")
 	RootCmd.PersistentFlags().BoolVarP(&opts.RunV2, "use-new-run", "", false, "Use the experimental run implementation for detecting changes without requiring file system snapshots.")
 	RootCmd.PersistentFlags().Var(&opts.Git, "git", "Branch to clone if build context is a git repository")
+	RootCmd.PersistentFlags().BoolVarP(&opts.CacheCopyLayers, "cache-copy-layers", "", false, "Caches copy layers")
 }
 
 // addHiddenFlags marks certain flags as hidden from the executor help text

integration/images.go

@@ -308,7 +308,7 @@ func populateVolumeCache() error {
 }
 
 // buildCachedImages builds the images for testing caching via kaniko where version is the nth time this image has been built
-func (d *DockerFileBuilder) buildCachedImages(config *integrationTestConfig, cacheRepo, dockerfilesPath string, version int) error {
+func (d *DockerFileBuilder) buildCachedImages(config *integrationTestConfig, cacheRepo, dockerfilesPath string, version int, args []string) error {
 	imageRepo, serviceAccount := config.imageRepo, config.serviceAccount
 	_, ex, _, _ := runtime.Caller(0)
 	cwd := filepath.Dir(ex)
@@ -334,6 +334,9 @@ func (d *DockerFileBuilder) buildCachedImages(config *integrationTestConfig, cac
 			cacheFlag,
 			"--cache-repo", cacheRepo,
 			"--cache-dir", cacheDir)
+		for _, v := range args {
+			dockerRunFlags = append(dockerRunFlags, v)
+		}
 		kanikoCmd := exec.Command("docker", dockerRunFlags...)
 
 		_, err := RunCommandWithoutTest(kanikoCmd)

integration/integration_test.go

@@ -464,17 +464,21 @@ func buildImage(t *testing.T, dockerfile string, imageBuilder *DockerFileBuilder
 func TestCache(t *testing.T) {
 	populateVolumeCache()
 	for dockerfile := range imageBuilder.TestCacheDockerfiles {
+		args := []string{}
+		if dockerfile == "Dockerfile_test_cache_copy" {
+			args = append(args, "--cache-copy-layers=true")
+		}
 		t.Run("test_cache_"+dockerfile, func(t *testing.T) {
 			dockerfile := dockerfile
 			t.Parallel()
 
 			cache := filepath.Join(config.imageRepo, "cache", fmt.Sprintf("%v", time.Now().UnixNano()))
 			// Build the initial image which will cache layers
-			if err := imageBuilder.buildCachedImages(config, cache, dockerfilesPath, 0); err != nil {
+			if err := imageBuilder.buildCachedImages(config, cache, dockerfilesPath, 0, args); err != nil {
 				t.Fatalf("error building cached image for the first time: %v", err)
 			}
 			// Build the second image which should pull from the cache
-			if err := imageBuilder.buildCachedImages(config, cache, dockerfilesPath, 1); err != nil {
+			if err := imageBuilder.buildCachedImages(config, cache, dockerfilesPath, 1, args); err != nil {
 				t.Fatalf("error building cached image for the first time: %v", err)
 			}
 			// Make sure both images are the same

pkg/commands/commands.go

@@ -60,15 +60,15 @@ type DockerCommand interface {
 	ShouldDetectDeletedFiles() bool
 }
 
-func GetCommand(cmd instructions.Command, fileContext util.FileContext, useNewRun bool) (DockerCommand, error) {
+func GetCommand(cmd instructions.Command, fileContext util.FileContext, useNewRun bool, cacheCopy bool) (DockerCommand, error) {
 	switch c := cmd.(type) {
 	case *instructions.RunCommand:
 		if useNewRun {
 			return &RunMarkerCommand{cmd: c}, nil
 		}
 		return &RunCommand{cmd: c}, nil
 	case *instructions.CopyCommand:
-		return &CopyCommand{cmd: c, fileContext: fileContext}, nil
+		return &CopyCommand{cmd: c, fileContext: fileContext, shdCache: cacheCopy}, nil
 	case *instructions.ExposeCommand:
 		return &ExposeCommand{cmd: c}, nil
 	case *instructions.EnvCommand:

pkg/commands/copy.go

@@ -17,6 +17,7 @@ limitations under the License.
 package commands
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
@@ -41,6 +42,7 @@ type CopyCommand struct {
 	cmd           *instructions.CopyCommand
 	fileContext   util.FileContext
 	snapshotFiles []string
+	shdCache      bool
 }
 
 func (c *CopyCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error {
@@ -145,6 +147,85 @@ func (c *CopyCommand) From() string {
 	return c.cmd.From
 }
 
+func (c *CopyCommand) ShouldCacheOutput() bool {
+	return c.shdCache
+}
+
+// CacheCommand returns true since this command should be cached
+func (c *CopyCommand) CacheCommand(img v1.Image) DockerCommand {
+	return &CachingCopyCommand{
+		img:         img,
+		cmd:         c.cmd,
+		fileContext: c.fileContext,
+		extractFn:   util.ExtractFile,
+	}
+}
+
+type CachingCopyCommand struct {
+	BaseCommand
+	caching
+	img            v1.Image
+	extractedFiles []string
+	cmd            *instructions.CopyCommand
+	fileContext    util.FileContext
+	extractFn      util.ExtractFunction
+}
+
+func (cr *CachingCopyCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error {
+	logrus.Infof("Found cached layer, extracting to filesystem")
+	var err error
+
+	if cr.img == nil {
+		return errors.New(fmt.Sprintf("cached command image is nil %v", cr.String()))
+	}
+
+	layers, err := cr.img.Layers()
+	if err != nil {
+		return errors.Wrapf(err, "retrieve image layers")
+	}
+
+	if len(layers) != 1 {
+		return errors.New(fmt.Sprintf("expected %d layers but got %d", 1, len(layers)))
+	}
+
+	cr.layer = layers[0]
+	cr.extractedFiles, err = util.GetFSFromLayers(kConfig.RootDir, layers, util.ExtractFunc(cr.extractFn), util.IncludeWhiteout())
+
+	logrus.Debugf("extractedFiles: %s", cr.extractedFiles)
+	if err != nil {
+		return errors.Wrap(err, "extracting fs from image")
+	}
+
+	return nil
+}
+
+func (cr *CachingCopyCommand) FilesUsedFromContext(config *v1.Config, buildArgs *dockerfile.BuildArgs) ([]string, error) {
+	return copyCmdFilesUsedFromContext(config, buildArgs, cr.cmd, cr.fileContext)
+}
+
+func (cr *CachingCopyCommand) FilesToSnapshot() []string {
+	f := cr.extractedFiles
+	logrus.Debugf("%d files extracted by caching copy command", len(f))
+	logrus.Tracef("Extracted files: %s", f)
+
+	return f
+}
+
+func (cr *CachingCopyCommand) MetadataOnly() bool {
+	return false
+}
+
+func (cr *CachingCopyCommand) String() string {
+	if cr.cmd == nil {
+		return "nil command"
+	}
+	return cr.cmd.String()
+}
+
+func (cr *CachingCopyCommand) From() string {
+	return cr.cmd.From
+}
+
 func resolveIfSymlink(destPath string) (string, error) {
 	if !filepath.IsAbs(destPath) {
 		return "", errors.New("dest path must be abs")

pkg/commands/copy_test.go

@@ -16,6 +16,7 @@ limitations under the License.
 package commands
 
 import (
+	"archive/tar"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -105,6 +106,160 @@ func setupTestTemp() string {
 
 	return tempDir
 }
+
+func Test_CachingCopyCommand_ExecuteCommand(t *testing.T) {
+	tempDir := setupTestTemp()
+
+	tarContent, err := prepareTarFixture([]string{"foo.txt"})
+	if err != nil {
+		t.Errorf("couldn't prepare tar fixture %v", err)
+	}
+
+	config := &v1.Config{}
+	buildArgs := &dockerfile.BuildArgs{}
+
+	type testCase struct {
+		desctiption    string
+		expectLayer    bool
+		expectErr      bool
+		count          *int
+		expectedCount  int
+		command        *CachingCopyCommand
+		extractedFiles []string
+		contextFiles   []string
+	}
+	testCases := []testCase{
+		func() testCase {
+			err = ioutil.WriteFile(filepath.Join(tempDir, "foo.txt"), []byte("meow"), 0644)
+			if err != nil {
+				t.Errorf("couldn't write tempfile %v", err)
+				t.FailNow()
+			}
+
+			c := &CachingCopyCommand{
+				img: fakeImage{
+					ImageLayers: []v1.Layer{
+						fakeLayer{TarContent: tarContent},
+					},
+				},
+				fileContext: util.FileContext{Root: tempDir},
+				cmd: &instructions.CopyCommand{
+					SourcesAndDest: []string{
+						"foo.txt", "foo.txt",
+					},
+				},
+			}
+			count := 0
+			tc := testCase{
+				desctiption:    "with valid image and valid layer",
+				count:          &count,
+				expectedCount:  1,
+				expectLayer:    true,
+				extractedFiles: []string{"/foo.txt"},
+				contextFiles:   []string{"foo.txt"},
+			}
+			c.extractFn = func(_ string, _ *tar.Header, _ io.Reader) error {
+				*tc.count++
+				return nil
+			}
+			tc.command = c
+			return tc
+		}(),
+		func() testCase {
+			c := &CachingCopyCommand{}
+			tc := testCase{
+				desctiption: "with no image",
+				expectErr:   true,
+			}
+			c.extractFn = func(_ string, _ *tar.Header, _ io.Reader) error {
+				return nil
+			}
+			tc.command = c
+			return tc
+		}(),
+		func() testCase {
+			c := &CachingCopyCommand{
+				img: fakeImage{},
+			}
+			c.extractFn = func(_ string, _ *tar.Header, _ io.Reader) error {
+				return nil
+			}
+			return testCase{
+				desctiption: "with image containing no layers",
+				expectErr:   true,
+				command:     c,
+			}
+		}(),
+		func() testCase {
+			c := &CachingCopyCommand{
+				img: fakeImage{
+					ImageLayers: []v1.Layer{
+						fakeLayer{},
+					},
+				},
+			}
+			c.extractFn = func(_ string, _ *tar.Header, _ io.Reader) error {
+				return nil
+			}
+			tc := testCase{
+				desctiption: "with image one layer which has no tar content",
+				expectErr:   false, // this one probably should fail but doesn't because of how ExecuteCommand and util.GetFSFromLayers are implemented - cvgw- 2019-11-25
+				expectLayer: true,
+			}
+			tc.command = c
+			return tc
+		}(),
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.desctiption, func(t *testing.T) {
+			c := tc.command
+			err := c.ExecuteCommand(config, buildArgs)
+			if !tc.expectErr && err != nil {
+				t.Errorf("Expected err to be nil but was %v", err)
+			} else if tc.expectErr && err == nil {
+				t.Error("Expected err but was nil")
+			}
+
+			if tc.count != nil {
+				if *tc.count != tc.expectedCount {
+					t.Errorf("Expected extractFn to be called %v times but was called %v times", tc.expectedCount, *tc.count)
+				}
+				for _, file := range tc.extractedFiles {
+					match := false
+					cFiles := c.FilesToSnapshot()
+					for _, cFile := range cFiles {
+						if file == cFile {
+							match = true
+							break
+						}
+					}
+					if !match {
+						t.Errorf("Expected extracted files to include %v but did not %v", file, cFiles)
+					}
+				}
+
+				cmdFiles, err := c.FilesUsedFromContext(
+					config, buildArgs,
+				)
+				if err != nil {
+					t.Errorf("failed to get files used from context from command %v", err)
+				}
+
+				if len(cmdFiles) != len(tc.contextFiles) {
+					t.Errorf("expected files used from context to equal %v but was %v", tc.contextFiles, cmdFiles)
+				}
+			}
+
+			if c.layer == nil && tc.expectLayer {
+				t.Error("expected the command to have a layer set but instead was nil")
+			} else if c.layer != nil && !tc.expectLayer {
+				t.Error("expected the command to have no layer set but instead found a layer")
+			}
+		})
+	}
+}
+
 func TestCopyExecuteCmd(t *testing.T) {
 	tempDir := setupTestTemp()
 	defer os.RemoveAll(tempDir)

pkg/config/options.go

@@ -68,6 +68,7 @@ type KanikoOptions struct {
 	IgnoreVarRun        bool
 	SkipUnusedStages    bool
 	RunV2               bool
+	CacheCopyLayers     bool
 	Git                 KanikoGitOptions
 }
 

pkg/executor/build.go

@@ -127,7 +127,7 @@ func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage, cross
 	}
 
 	for _, cmd := range s.stage.Commands {
-		command, err := commands.GetCommand(cmd, fileContext, opts.RunV2)
+		command, err := commands.GetCommand(cmd, fileContext, opts.RunV2, opts.CacheCopyLayers)
 		if err != nil {
 			return nil, err
 		}
@@ -184,6 +184,7 @@ func (s *stageBuilder) populateCompositeKey(command fmt.Stringer, files []string
 	compositeKey.AddKey(resolvedCmd)
 	switch v := command.(type) {
 	case *commands.CopyCommand:
+	case *commands.CachingCopyCommand:
 		compositeKey = s.populateCopyCmdCompositeKey(command, v.From(), compositeKey)
 	}