Merge slice fixes into main

.github/workflows/build.yml

@@ -47,13 +47,21 @@ jobs:
           git config --global core.eol lf
           git config --global core.symlinks true
       - uses: actions/checkout@v2
+      - name: Set up go
+        uses: actions/setup-go@v2-beta
+        with:
+          go-version: '1.14'
+      - name: Install jq
+        run: |
+          choco install jq
       - name: Test
         run: |
-          make docker-run-windows DOCKER_CMD="make test"
+          make test
         shell: cmd
       - name: Build
         run: |
-          make docker-run-windows DOCKER_CMD="make build-windows package-windows"
+          make build-windows
+          make package-windows
         shell: cmd
       - uses: actions/upload-artifact@v2
         with:

Makefile

@@ -199,14 +199,16 @@ package-windows:
 	@echo "> Packaging lifecycle for $(GOOS)..."
 	$(GOCMD) run tools$/packager$/main.go -os $(GOOS) -launcherExePath $(GOOS_DIR)$/lifecycle$/launcher.exe -lifecycleExePath $(GOOS_DIR)$/lifecycle$/lifecycle.exe -lifecycleVersion $(LIFECYCLE_VERSION) -platformAPI $(PLATFORM_API) -buildpackAPI $(BUILDPACK_API) -outputPackagePath $(BUILD_DIR)$/$(ARCHIVE_NAME).tgz
 
+# Ensure workdir is clean and build image from .git
 docker-build-source-image-windows:
-	docker build -f tools/Dockerfile.windows --tag $(SOURCE_COMPILATION_IMAGE) --build-arg image_tag=$(WINDOWS_COMPILATION_IMAGE) --cache-from=$(SOURCE_COMPILATION_IMAGE) --isolation=process --quiet .
+	$(if $(shell git status --short), @echo Uncommitted changes. Refusing to run. && exit 1)
+	docker build -f tools/Dockerfile.windows --tag $(SOURCE_COMPILATION_IMAGE) --build-arg image_tag=$(WINDOWS_COMPILATION_IMAGE) --cache-from=$(SOURCE_COMPILATION_IMAGE) --isolation=process --quiet .git
 
 docker-run-windows: docker-build-source-image-windows
 docker-run-windows:
 	@echo "> Running '$(DOCKER_CMD)' in docker windows..."
 	@docker volume rm -f lifecycle-out
-	docker run -v lifecycle-out:c:/lifecycle/out -e LIFECYCLE_VERSION -e PLATFORM_API -e BUILDPACK_API -v gopathcache:c:/gopath -v '\\.\pipe\docker_engine:\\.\pipe\docker_engine' --isolation=process --rm $(SOURCE_COMPILATION_IMAGE) $(DOCKER_CMD)
+	docker run -v lifecycle-out:c:/lifecycle/out -e LIFECYCLE_VERSION -e PLATFORM_API -e BUILDPACK_API -v gopathcache:c:/gopath -v '\\.\pipe\docker_engine:\\.\pipe\docker_engine' --isolation=process --interactive --tty --rm $(SOURCE_COMPILATION_IMAGE) $(DOCKER_CMD)
 	docker run -v lifecycle-out:c:/lifecycle/out --rm $(SOURCE_COMPILATION_IMAGE) tar -cf- out | tar -xf-
 	@docker volume rm -f lifecycle-out
 

VERSION

@@ -1 +1 @@
-0.8.0
+0.8.1

archive/compress.go

@@ -0,0 +1,72 @@
+package archive
+
+import (
+	"archive/tar"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+// PathInfo associates a path with an os.FileInfo
+type PathInfo struct {
+	Path string
+	Info os.FileInfo
+}
+
+// AddFilesToArchive writes entries describing all files to the provided TarWriter
+func AddFilesToArchive(tw TarWriter, files []PathInfo) error {
+	for _, file := range files {
+		if err := AddFileToArchive(tw, file.Path, file.Info); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// AddFileToArchive writes an entry describing the file at path with the given os.FileInfo to the provided TarWriter
+func AddFileToArchive(tw TarWriter, path string, fi os.FileInfo) error {
+	if fi.Mode()&os.ModeSocket != 0 {
+		return nil
+	}
+	header, err := tar.FileInfoHeader(fi, "")
+	if err != nil {
+		return err
+	}
+	header.Name = path
+
+	if fi.Mode()&os.ModeSymlink != 0 {
+		var err error
+		target, err := os.Readlink(path)
+		if err != nil {
+			return err
+		}
+		header.Linkname = target
+	}
+	addSysAttributes(header, fi)
+	if err := tw.WriteHeader(header); err != nil {
+		return err
+	}
+	if fi.Mode().IsRegular() {
+		f, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+		if _, err := io.Copy(tw, f); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// AddDirToArchive walks dir writes entries describing dir and all of its children files to the provided TarWriter
+func AddDirToArchive(tw TarWriter, dir string) error {
+	dir = filepath.Clean(dir)
+
+	return filepath.Walk(dir, func(file string, fi os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		return AddFileToArchive(tw, file, fi)
+	})
+}

archive/compress_test.go

@@ -0,0 +1,165 @@
+package archive_test
+
+import (
+	"archive/tar"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/sclevine/spec"
+	"github.com/sclevine/spec/report"
+
+	"github.com/buildpacks/lifecycle/archive"
+	h "github.com/buildpacks/lifecycle/testhelpers"
+)
+
+func TestArchiveWrite(t *testing.T) {
+	rand.Seed(time.Now().UTC().UnixNano())
+	spec.Run(t, "tar", testWrite, spec.Report(report.Terminal{}))
+}
+
+func testWrite(t *testing.T, when spec.G, it spec.S) {
+	var tmpDir string
+
+	it.Before(func() {
+		var err error
+		tmpDir, err = ioutil.TempDir("", "archive-write-test")
+		h.AssertNil(t, err)
+	})
+
+	it.After(func() {
+		h.AssertNil(t, os.RemoveAll(tmpDir))
+	})
+
+	when("#AddDirToArchive", func() {
+		var (
+			uid  = 1234
+			gid  = 4567
+			tw   *archive.NormalizingTarWriter
+			file *os.File
+		)
+
+		it.Before(func() {
+			var err error
+			file, err = os.Create(filepath.Join(tmpDir, "tar_test-go.tar"))
+			h.AssertNil(t, err)
+			tw = &archive.NormalizingTarWriter{TarWriter: tar.NewWriter(file)}
+			tw.WithUID(uid)
+			tw.WithGID(gid)
+		})
+
+		it.After(func() {
+			file.Close()
+		})
+
+		for _, src := range []string{
+			filepath.Join("testdata", "dir-to-tar"),
+			filepath.Join("testdata", "dir-to-tar") + string(filepath.Separator),
+			filepath.Join("testdata", "dir-to-tar") + string(filepath.Separator) + ".",
+		} {
+			src := src
+			it(fmt.Sprintf("writes a tar with the src filesystem contents (%s)", src), func() {
+				h.AssertNil(t, archive.AddDirToArchive(tw, src))
+				h.AssertNil(t, file.Close())
+
+				file, err := os.Open(file.Name())
+				h.AssertNil(t, err)
+
+				defer file.Close()
+				tr := tar.NewReader(file)
+
+				tarContains(t, "directories", func() {
+					header, err := tr.Next()
+					h.AssertNil(t, err)
+					h.AssertEq(t, header.Name, "testdata/dir-to-tar")
+					assertDirectory(t, header)
+					assertModTimeNormalized(t, header)
+				})
+
+				tarContains(t, "directory symlinks", func() {
+					header, err := tr.Next()
+					h.AssertNil(t, err)
+
+					h.AssertEq(t, header.Name, "testdata/dir-to-tar/dir-link")
+					h.AssertEq(t, header.Uid, uid)
+					h.AssertEq(t, header.Gid, gid)
+					assertSymlink(t, header)
+					h.AssertEq(t, header.Linkname, filepath.FromSlash("../excluded-dir"))
+					assertModTimeNormalized(t, header)
+				})
+
+				tarContains(t, "file symlinks", func() {
+					header, err := tr.Next()
+					h.AssertNil(t, err)
+
+					h.AssertEq(t, header.Name, "testdata/dir-to-tar/file-link")
+					h.AssertEq(t, header.Uid, uid)
+					h.AssertEq(t, header.Gid, gid)
+					assertSymlink(t, header)
+					h.AssertEq(t, header.Linkname, filepath.FromSlash("../excluded-dir/excluded-file"))
+					assertModTimeNormalized(t, header)
+				})
+
+				tarContains(t, "regular files", func() {
+					header, err := tr.Next()
+					h.AssertNil(t, err)
+					h.AssertEq(t, header.Name, "testdata/dir-to-tar/some-file.txt")
+
+					fileContents := make([]byte, header.Size)
+					_, err = tr.Read(fileContents)
+					h.AssertSameInstance(t, err, io.EOF)
+					h.AssertEq(t, string(fileContents), "some-content")
+					h.AssertEq(t, header.Uid, uid)
+					h.AssertEq(t, header.Gid, gid)
+					assertModTimeNormalized(t, header)
+				})
+
+				tarContains(t, "subdir", func() {
+					header, err := tr.Next()
+					h.AssertNil(t, err)
+					h.AssertEq(t, header.Name, "testdata/dir-to-tar/sub-dir")
+					assertDirectory(t, header)
+					assertModTimeNormalized(t, header)
+				})
+
+				tarContains(t, "children of subdir", func() {
+					header, err := tr.Next()
+					h.AssertNil(t, err)
+					h.AssertEq(t, header.Name, "testdata/dir-to-tar/sub-dir/sub-file")
+					assertModTimeNormalized(t, header)
+				})
+			})
+		}
+	})
+}
+
+func tarContains(t *testing.T, m string, r func()) {
+	t.Helper()
+	r()
+}
+
+func assertDirectory(t *testing.T, header *tar.Header) {
+	t.Helper()
+	if header.Typeflag != tar.TypeDir {
+		t.Fatalf(`expected %s to be a directory`, header.Name)
+	}
+}
+
+func assertSymlink(t *testing.T, header *tar.Header) {
+	t.Helper()
+	if header.Typeflag != tar.TypeSymlink {
+		t.Fatalf(`expected %s to be a symlink`, header.Name)
+	}
+}
+
+func assertModTimeNormalized(t *testing.T, header *tar.Header) {
+	t.Helper()
+	if !header.ModTime.Equal(time.Date(1980, time.January, 1, 0, 0, 1, 0, time.UTC)) {
+		t.Fatalf(`expected %s time to be normalized, instead got: %s`, header.Name, header.ModTime.String())
+	}
+}

archive/extract.go

@@ -0,0 +1,93 @@
+package archive
+
+import (
+	"archive/tar"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+)
+
+type PathMode struct {
+	Path string
+	Mode os.FileMode
+}
+
+// Extract reads all entries from TarReader and extracts them to the filesystem
+func Extract(tr TarReader) error {
+	// Avoid umask from changing the file permissions in the tar file.
+	umask := setUmask(0)
+	defer setUmask(umask)
+
+	buf := make([]byte, 32*32*1024)
+	dirsFound := make(map[string]bool)
+
+	var pathModes []PathMode
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			for _, pathMode := range pathModes {
+				if err := os.Chmod(pathMode.Path, pathMode.Mode); err != nil {
+					return err
+				}
+			}
+			return nil
+		}
+		if err != nil {
+			return errors.Wrap(err, "error extracting from archive")
+		}
+
+		switch hdr.Typeflag {
+		case tar.TypeDir:
+			if _, err := os.Stat(hdr.Name); os.IsNotExist(err) {
+				pathMode := PathMode{hdr.Name, hdr.FileInfo().Mode()}
+				pathModes = append(pathModes, pathMode)
+			}
+			if err := os.MkdirAll(hdr.Name, os.ModePerm); err != nil {
+				return errors.Wrapf(err, "failed to create directory %q", hdr.Name)
+			}
+			dirsFound[hdr.Name] = true
+
+		case tar.TypeReg, tar.TypeRegA:
+			dirPath := filepath.Dir(hdr.Name)
+			if !dirsFound[dirPath] {
+				if _, err := os.Stat(dirPath); os.IsNotExist(err) {
+					if err := os.MkdirAll(dirPath, applyUmask(os.ModePerm, umask)); err != nil {
+						return errors.Wrapf(err, "failed to create parent dir %q for file %q", dirPath, hdr.Name)
+					}
+					dirsFound[dirPath] = true
+				}
+			}
+
+			if err := writeFile(tr, hdr.Name, hdr.FileInfo().Mode(), buf); err != nil {
+				return errors.Wrapf(err, "failed to write file %q", hdr.Name)
+			}
+		case tar.TypeSymlink:
+			if err := createSymlink(hdr); err != nil {
+				return errors.Wrapf(err, "failed to create symlink %q with target %q", hdr.Name, hdr.Linkname)
+			}
+		default:
+			return fmt.Errorf("unknown file type in tar %d", hdr.Typeflag)
+		}
+	}
+}
+
+func applyUmask(mode os.FileMode, umask int) os.FileMode {
+	return os.FileMode(int(mode) &^ umask)
+}
+
+func writeFile(in io.Reader, path string, mode os.FileMode, buf []byte) (err error) {
+	fh, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if closeErr := fh.Close(); err == nil {
+			err = closeErr
+		}
+	}()
+	_, err = io.CopyBuffer(fh, in, buf)
+	return err
+}