diff --git a/cmd/bundle/main.go b/cmd/bundle/main.go index 21a5c09399..1db85beafc 100644 --- a/cmd/bundle/main.go +++ b/cmd/bundle/main.go @@ -9,8 +9,11 @@ import ( "fmt" "log" "os" + "strconv" "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/spf13/pflag" "github.com/shipwright-io/build/pkg/bundle" @@ -18,12 +21,13 @@ import ( ) type settings struct { - help bool - image string - prune bool - target string - secretPath string - resultFileImageDigest string + help bool + image string + prune bool + target string + secretPath string + resultFileImageDigest string + resultFileSourceTimestamp string } var flagValues settings @@ -36,6 +40,7 @@ func init() { pflag.StringVar(&flagValues.image, "image", "", "Location of the bundle image (mandatory)") pflag.StringVar(&flagValues.target, "target", "/workspace/source", "The target directory to place the code") pflag.StringVar(&flagValues.resultFileImageDigest, "result-file-image-digest", "", "A file to write the image digest") + pflag.StringVar(&flagValues.resultFileSourceTimestamp, "result-file-source-timestamp", "", "A file to write the source timestamp") pflag.StringVar(&flagValues.secretPath, "secret-path", "", "A directory that contains access credentials (optional)") pflag.BoolVar(&flagValues.prune, "prune", false, "Delete bundle image from registry after it was pulled") @@ -72,10 +77,20 @@ func Do(ctx context.Context) error { } log.Printf("Pulling image %q", ref) - img, err := bundle.PullAndUnpack( - ref, - flagValues.target, - options...) + desc, err := remote.Get(ref, options...) + if err != nil { + return err + } + + img, err := desc.Image() + if err != nil { + return err + } + + rc := mutate.Extract(img) + defer rc.Close() + + unpackDetails, err := bundle.Unpack(rc, flagValues.target) if err != nil { return err } @@ -93,6 +108,17 @@ func Do(ctx context.Context) error { } } + if flagValues.resultFileSourceTimestamp != "" { + if unpackDetails.MostRecentFileTimestamp != nil { + if err = os.WriteFile(flagValues.resultFileSourceTimestamp, []byte(strconv.FormatInt(unpackDetails.MostRecentFileTimestamp.Unix(), 10)), 0644); err != nil { + return err + } + + } else { + log.Printf("Unable to determine source timestamp of content in %s\n", flagValues.target) + } + } + if flagValues.prune { // Some container registry implementations, i.e. library/registry:2 will fail to // delete the image when there is no image digest given. Use image digest from the diff --git a/cmd/bundle/main_test.go b/cmd/bundle/main_test.go index a7f8c366dc..d1cc01ae83 100644 --- a/cmd/bundle/main_test.go +++ b/cmd/bundle/main_test.go @@ -9,25 +9,31 @@ import ( "fmt" "io" "log" + "net/http/httptest" + "net/url" "os" "path/filepath" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "github.com/shipwright-io/build/cmd/bundle" - "github.com/shipwright-io/build/pkg/image" "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" containerreg "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" "k8s.io/apimachinery/pkg/util/rand" + + "github.com/shipwright-io/build/pkg/bundle" + "github.com/shipwright-io/build/pkg/image" ) var _ = Describe("Bundle Loader", func() { const exampleImage = "ghcr.io/shipwright-io/sample-go/source-bundle:latest" - var run = func(args ...string) error { + run := func(args ...string) error { // discard log output log.SetOutput(io.Discard) @@ -40,7 +46,7 @@ var _ = Describe("Bundle Loader", func() { return Do(context.Background()) } - var withTempDir = func(f func(target string)) { + withTempDir := func(f func(target string)) { path, err := os.MkdirTemp(os.TempDir(), "bundle") Expect(err).ToNot(HaveOccurred()) defer os.RemoveAll(path) @@ -56,6 +62,24 @@ var _ = Describe("Bundle Loader", func() { f(file.Name()) } + withTempRegistry := func(f func(endpoint string)) { + logLogger := log.Logger{} + logLogger.SetOutput(GinkgoWriter) + + s := httptest.NewServer( + registry.New( + registry.Logger(&logLogger), + registry.WithReferrersSupport(true), + ), + ) + defer s.Close() + + u, err := url.Parse(s.URL) + Expect(err).ToNot(HaveOccurred()) + + f(u.Host) + } + filecontent := func(path string) string { data, err := os.ReadFile(path) Expect(err).ToNot(HaveOccurred()) @@ -234,4 +258,62 @@ var _ = Describe("Bundle Loader", func() { }) }) }) + + Context("Result file checks", func() { + tmpFile := func(dir string, name string, data []byte, timestamp time.Time) { + var path = filepath.Join(dir, name) + + Expect(os.WriteFile( + path, + data, + os.FileMode(0644), + )).To(Succeed()) + + Expect(os.Chtimes( + path, + timestamp, + timestamp, + )).To(Succeed()) + } + + // Creates a controlled reference image with one file called "file" with modification + // timestamp of Friday, February 13, 2009 11:31:30 PM (unix timestamp 1234567890) + withReferenceImage := func(f func(dig name.Digest)) { + withTempRegistry(func(endpoint string) { + withTempDir(func(target string) { + timestamp := time.Unix(1234567890, 0) + + ref, err := name.ParseReference(fmt.Sprintf("%s/namespace/image:tag", endpoint)) + Expect(err).ToNot(HaveOccurred()) + Expect(ref).ToNot(BeNil()) + + tmpFile(target, "file", []byte("foobar"), timestamp) + + dig, err := bundle.PackAndPush(ref, target) + Expect(err).ToNot(HaveOccurred()) + Expect(dig).ToNot(BeNil()) + + f(dig) + }) + }) + } + + It("should store source timestamp in result file", func() { + withTempDir(func(target string) { + withTempDir(func(result string) { + withReferenceImage(func(dig name.Digest) { + resultSourceTimestamp := filepath.Join(result, "source-timestamp") + + Expect(run( + "--image", dig.String(), + "--target", target, + "--result-file-source-timestamp", resultSourceTimestamp, + )).To(Succeed()) + + Expect(filecontent(resultSourceTimestamp)).To(Equal("1234567890")) + }) + }) + }) + }) + }) }) diff --git a/pkg/bundle/bundle.go b/pkg/bundle/bundle.go index 2a32ea78ba..384ba37225 100644 --- a/pkg/bundle/bundle.go +++ b/pkg/bundle/bundle.go @@ -26,6 +26,11 @@ import ( const shpIgnoreFilename = ".shpignore" +// UnpackDetails contains details about the files that were unpacked +type UnpackDetails struct { + MostRecentFileTimestamp *time.Time +} + // PackAndPush a local directory as-is into a container image. See // remote.Option for optional options to the image push to the registry, for // example to provide the appropriate access credentials. @@ -35,12 +40,12 @@ func PackAndPush(ref name.Reference, directory string, options ...remote.Option) return name.Digest{}, err } - image, err := mutate.AppendLayers(empty.Image, bundleLayer) + image, err := mutate.Time(empty.Image, time.Unix(0, 0)) if err != nil { return name.Digest{}, err } - image, err = mutate.Time(image, time.Unix(0, 0)) + image, err = mutate.AppendLayers(image, bundleLayer) if err != nil { return name.Digest{}, err } @@ -77,7 +82,7 @@ func PullAndUnpack(ref name.Reference, targetPath string, options ...remote.Opti rc := mutate.Extract(image) defer rc.Close() - if err = Unpack(rc, targetPath); err != nil { + if _, err = Unpack(rc, targetPath); err != nil { return nil, err } @@ -224,16 +229,17 @@ func Pack(directory string) (io.ReadCloser, error) { // Unpack reads a tar stream and writes the content into the local file system // with all files and directories. -func Unpack(in io.Reader, targetPath string) error { +func Unpack(in io.Reader, targetPath string) (*UnpackDetails, error) { + var details = UnpackDetails{} var tr = tar.NewReader(in) for { header, err := tr.Next() switch { case err == io.EOF: - return nil + return &details, nil case err != nil: - return err + return nil, err case header == nil: continue @@ -241,37 +247,46 @@ func Unpack(in io.Reader, targetPath string) error { var target = filepath.Join(targetPath, header.Name) if strings.Contains(target, "/../") { - return fmt.Errorf("targetPath validation failed, path contains unexpected special elements") + return nil, fmt.Errorf("targetPath validation failed, path contains unexpected special elements") } switch header.Typeflag { case tar.TypeDir: if err := os.MkdirAll(target, os.FileMode(header.Mode)); err != nil { - return err + return nil, err } case tar.TypeReg: // Edge case in which that tarball did not have a directory entry dir, _ := filepath.Split(target) if err := os.MkdirAll(dir, os.FileMode(0755)); err != nil { - return err + return nil, err } file, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode)) if err != nil { - return err + return nil, err } if _, err := io.Copy(file, tr); err != nil { file.Close() - return err + return nil, err + } + + if err := file.Close(); err != nil { + return nil, err } - file.Close() - os.Chtimes(target, header.AccessTime, header.ModTime) + if err := os.Chtimes(target, header.AccessTime, header.ModTime); err != nil { + return nil, err + } + + if details.MostRecentFileTimestamp == nil || details.MostRecentFileTimestamp.Before(header.ModTime) { + details.MostRecentFileTimestamp = &header.ModTime + } default: - return fmt.Errorf("provided tarball contains unsupported file type, only directories and regular files are supported") + return nil, fmt.Errorf("provided tarball contains unsupported file type, only directories and regular files are supported") } } } diff --git a/pkg/bundle/bundle_test.go b/pkg/bundle/bundle_test.go index 870a3a9d4e..cd980ae5cb 100644 --- a/pkg/bundle/bundle_test.go +++ b/pkg/bundle/bundle_test.go @@ -54,7 +54,10 @@ var _ = Describe("Bundle", func() { Expect(err).ToNot(HaveOccurred()) Expect(r).ToNot(BeNil()) - Expect(Unpack(r, tempDir)).To(Succeed()) + details, err := Unpack(r, tempDir) + Expect(details).ToNot(BeNil()) + Expect(err).ToNot(HaveOccurred()) + Expect(filepath.Join(tempDir, "README.md")).To(BeAnExistingFile()) Expect(filepath.Join(tempDir, ".someToolDir", "config.json")).ToNot(BeAnExistingFile()) Expect(filepath.Join(tempDir, "somefile")).To(BeAnExistingFile())