WIP - multi-image docker archives

Signed-off-by: Valentin Rothberg <rothberg@redhat.com>

docker/archive/dest.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"os"
 
+	"github.com/containers/image/v5/docker/reference"
 	"github.com/containers/image/v5/docker/tarfile"
 	"github.com/containers/image/v5/types"
 	"github.com/pkg/errors"
@@ -17,32 +18,36 @@ type archiveImageDestination struct {
 }
 
 func newImageDestination(sys *types.SystemContext, ref archiveReference) (types.ImageDestination, error) {
+	return newArchiveImageDestination(sys, ref.path, ref.destinationRef)
+}
+
+func newArchiveImageDestination(sys *types.SystemContext, path string, ref reference.NamedTagged) (*archiveImageDestination, error) {
 	// ref.path can be either a pipe or a regular file
 	// in the case of a pipe, we require that we can open it for write
 	// in the case of a regular file, we don't want to overwrite any pre-existing file
 	// so we check for Size() == 0 below (This is racy, but using O_EXCL would also be racy,
 	// only in a different way. Either way, it’s up to the user to not have two writers to the same path.)
-	fh, err := os.OpenFile(ref.path, os.O_WRONLY|os.O_CREATE, 0644)
+	fh, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0644)
 	if err != nil {
-		return nil, errors.Wrapf(err, "error opening file %q", ref.path)
+		return nil, errors.Wrapf(err, "error opening file %q", path)
 	}
 
 	fhStat, err := fh.Stat()
 	if err != nil {
-		return nil, errors.Wrapf(err, "error statting file %q", ref.path)
+		return nil, errors.Wrapf(err, "error statting file %q", path)
 	}
 
 	if fhStat.Mode().IsRegular() && fhStat.Size() != 0 {
 		return nil, errors.New("docker-archive doesn't support modifying existing images")
 	}
 
-	tarDest := tarfile.NewDestinationWithContext(sys, fh, ref.destinationRef)
+	tarDest := tarfile.NewDestinationWithContext(sys, fh, ref)
 	if sys != nil && sys.DockerArchiveAdditionalTags != nil {
 		tarDest.AddRepoTags(sys.DockerArchiveAdditionalTags)
 	}
 	return &archiveImageDestination{
 		Destination: tarDest,
-		ref:         ref,
+		ref:         archiveReference{path, ref},
 		writer:      fh,
 	}, nil
 }
@@ -70,3 +75,61 @@ func (d *archiveImageDestination) Close() error {
 func (d *archiveImageDestination) Commit(ctx context.Context, unparsedToplevel types.UnparsedImage) error {
 	return d.Destination.Commit(ctx)
 }
+
+func (m multiImageDestinationReference)	NewImageDestination(_ context.Context, _ *types.SystemContext) (types.ImageDestination, error) {
+	return m.dest, nil
+}
+
+// MultiImageDestinations allows for creating and writing to docker archives
+// that include more than one image.
+type MultiImageDestination struct {
+	*archiveImageDestination
+	path string
+}
+
+// multiImageDestinationReference is a types.ImageReference embedding a MultiImageDestination.
+type multiImageDestinationReference struct {
+	*archiveReference
+	dest *MultiImageDestination
+}
+
+// NewMultiImageDestination returns a MultiImageDestination for the specified path.
+func NewMultiImageDestination(sys *types.SystemContext, path string) (*MultiImageDestination, error) {
+	dest, err := newArchiveImageDestination(sys, path, nil)
+	if err != nil {
+		return nil, err
+	}
+	return &MultiImageDestination{dest, path}, nil
+}
+
+// Reference returns an ImageReference embedding the MultiImageDestination.
+func (m *MultiImageDestination) Reference() types.ImageReference {
+	ref := &archiveReference{path: m.path}
+	return &multiImageDestinationReference{ref, m}
+}
+
+// Close is a NOP.  Please use Finalize() for committing the archive and
+// closing the underlying resources.
+func (m *MultiImageDestination) Close() error {
+	return nil
+}
+
+// Commit is a NOP.  Please use Finalize() for committing the archive and
+// closing the underlying resources.
+func (m *MultiImageDestination) Commit(_ context.Context, _ types.UnparsedImage) error {
+	return nil
+}
+
+// Finalize commits pending data and closes the underlying tarfile.
+func (m *MultiImageDestination) Finalize(ctx context.Context) (finalErr error) {
+	defer func() {
+		if err := m.writer.Close(); err != nil {
+			if finalErr == nil {
+				finalErr = err
+			} else {
+				finalErr = errors.Wrap(finalErr, err.Error())
+			}
+		}
+	}()
+	return m.Destination.Commit(ctx)
+}

docker/archive/src.go

@@ -4,6 +4,7 @@ import (
 	"context"
 
 	"github.com/containers/image/v5/docker/tarfile"
+	ctrImage "github.com/containers/image/v5/image"
 	"github.com/containers/image/v5/types"
 	"github.com/sirupsen/logrus"
 )
@@ -34,3 +35,83 @@ func newImageSource(ctx context.Context, sys *types.SystemContext, ref archiveRe
 func (s *archiveImageSource) Reference() types.ImageReference {
 	return s.ref
 }
+
+// MultiImageSourceItem is a reference to _one_ image in a multi-image archive.
+// Note that MultiImageSourceItem implements types.ImageReference.  It's a
+// long-lived object that can only be closed via it's parent MultiImageSource.
+type MultiImageSourceItem struct {
+	*archiveReference
+	tarSource *tarfile.Source
+}
+
+// Manifest returns the tarfile.ManifestItem.
+func (m *MultiImageSourceItem) Manifest() (*tarfile.ManifestItem, error) {
+	items, err := m.tarSource.LoadTarManifest()
+	if err != nil {
+		return nil, err
+	}
+	return &items[0], nil
+}
+
+// NewImage returns a types.ImageCloser for this reference, possibly
+// specialized for this ImageTransport.
+func (m MultiImageSourceItem) NewImage(ctx context.Context, sys *types.SystemContext) (types.ImageCloser, error) {
+	src, err := m.NewImageSource(ctx, sys)
+	if err != nil {
+		return nil, err
+	}
+	return ctrImage.FromSource(ctx, sys, src)
+}
+
+// NewImageSource returns a types.ImageSource for this reference.
+func (m MultiImageSourceItem) NewImageSource(ctx context.Context, sys *types.SystemContext) (types.ImageSource, error) {
+	return &archiveImageSource{
+		Source: m.tarSource,
+		ref:    *m.archiveReference,
+	}, nil
+}
+
+// MultiImageSource allows for reading docker archives that includes more
+// than one image.  Use Items() to extract
+type MultiImageSource struct {
+	path      string
+	tarSource *tarfile.Source
+}
+
+// NewMultiImageSource creates a MultiImageSource for the
+// specified path pointing to a docker-archive.
+func NewMultiImageSource(ctx context.Context, sys *types.SystemContext, path string) (*MultiImageSource, error) {
+	src, err := tarfile.NewSourceFromFileWithContext(sys, path)
+	if err != nil {
+		return nil, err
+	}
+	return &MultiImageSource{path: path, tarSource: src}, nil
+}
+
+// Close closes the underlying tarfile.
+func (m *MultiImageSource) Close() error {
+	return m.tarSource.Close()
+}
+
+// Items returns a MultiImageSourceItem for all manifests/images in the archive.
+// Each references embeds an ImageSource pointing to the corresponding image in
+// the archive.
+func (m *MultiImageSource) Items() ([]MultiImageSourceItem, error) {
+	items, err := m.tarSource.LoadTarManifest()
+	if err != nil {
+		return nil, err
+	}
+	references := []MultiImageSourceItem{}
+	for index := range items {
+		src, err := m.tarSource.FromManifest(index)
+		if err != nil {
+			return nil, err
+		}
+		newRef := MultiImageSourceItem{
+			&archiveReference{path: m.path},
+			src,
+		}
+		references = append(references, newRef)
+	}
+	return references, nil
+}

docker/tarfile/dest.go

@@ -24,9 +24,11 @@ import (
 
 // Destination is a partial implementation of types.ImageDestination for writing to an io.Writer.
 type Destination struct {
-	writer   io.Writer
-	tar      *tar.Writer
-	repoTags []reference.NamedTagged
+	writer       io.Writer
+	tar          *tar.Writer
+	repoTags     []reference.NamedTagged
+	manifest     []ManifestItem
+	repositories map[string]map[string]string
 	// Other state.
 	blobs  map[digest.Digest]types.BlobInfo // list of already-sent blobs
 	config []byte
@@ -46,11 +48,12 @@ func NewDestinationWithContext(sys *types.SystemContext, dest io.Writer, ref ref
 		repoTags = append(repoTags, ref)
 	}
 	return &Destination{
-		writer:   dest,
-		tar:      tar.NewWriter(dest),
-		repoTags: repoTags,
-		blobs:    make(map[digest.Digest]types.BlobInfo),
-		sysCtx:   sys,
+		writer:       dest,
+		tar:          tar.NewWriter(dest),
+		repoTags:     repoTags,
+		blobs:        make(map[digest.Digest]types.BlobInfo),
+		sysCtx:       sys,
+		repositories: map[string]map[string]string{},
 	}
 }
 
@@ -183,24 +186,14 @@ func (d *Destination) TryReusingBlob(ctx context.Context, info types.BlobInfo, c
 	return false, types.BlobInfo{}, nil
 }
 
-func (d *Destination) createRepositoriesFile(rootLayerID string) error {
-	repositories := map[string]map[string]string{}
+func (d *Destination) addRootLayerToRepositories(rootLayerID string) {
 	for _, repoTag := range d.repoTags {
-		if val, ok := repositories[repoTag.Name()]; ok {
+		if val, ok := d.repositories[repoTag.Name()]; ok {
 			val[repoTag.Tag()] = rootLayerID
 		} else {
-			repositories[repoTag.Name()] = map[string]string{repoTag.Tag(): rootLayerID}
+			d.repositories[repoTag.Name()] = map[string]string{repoTag.Tag(): rootLayerID}
 		}
 	}
-
-	b, err := json.Marshal(repositories)
-	if err != nil {
-		return errors.Wrap(err, "Error marshaling repositories")
-	}
-	if err := d.sendBytes(legacyRepositoriesFileName, b); err != nil {
-		return errors.Wrap(err, "Error writing config json file")
-	}
-	return nil
 }
 
 // PutManifest writes manifest to the destination.
@@ -229,9 +222,7 @@ func (d *Destination) PutManifest(ctx context.Context, m []byte, instanceDigest
 	}
 
 	if len(man.LayersDescriptors) > 0 {
-		if err := d.createRepositoriesFile(lastLayerID); err != nil {
-			return err
-		}
+		d.addRootLayerToRepositories(lastLayerID)
 	}
 
 	repoTags := []string{}
@@ -256,20 +247,18 @@ func (d *Destination) PutManifest(ctx context.Context, m []byte, instanceDigest
 		repoTags = append(repoTags, refString)
 	}
 
-	items := []ManifestItem{{
+	d.manifest = append(d.manifest, ManifestItem{
 		Config:       man.ConfigDescriptor.Digest.Hex() + ".json",
 		RepoTags:     repoTags,
 		Layers:       layerPaths,
 		Parent:       "",
 		LayerSources: nil,
-	}}
-	itemsBytes, err := json.Marshal(&items)
-	if err != nil {
-		return err
-	}
+	})
 
-	// FIXME? Do we also need to support the legacy format?
-	return d.sendBytes(manifestFileName, itemsBytes)
+	// Reset the repoTags to prevent them from leaking into a following
+	// image/manifest.
+	d.repoTags = []reference.NamedTagged{}
+	return nil
 }
 
 // writeLegacyLayerMetadata writes legacy VERSION and configuration files for all layers
@@ -419,6 +408,34 @@ func (d *Destination) PutSignatures(ctx context.Context, signatures [][]byte, in
 
 // Commit finishes writing data to the underlying io.Writer.
 // It is the caller's responsibility to close it, if necessary.
-func (d *Destination) Commit(ctx context.Context) error {
-	return d.tar.Close()
+func (d *Destination) Commit(ctx context.Context) (finalErr error) {
+	defer func() {
+		if err := d.tar.Close(); err != nil {
+			if finalErr == nil {
+				finalErr = err
+			} else {
+				finalErr = errors.Wrap(finalErr, err.Error())
+			}
+		}
+	}()
+	// Writing the manifest here instead of PutManifest allows for
+	// supporting multi-image archives.
+	itemsBytes, err := json.Marshal(d.manifest)
+	if err != nil {
+		return err
+	}
+
+	// FIXME? Do we also need to support the legacy format?
+	if err := d.sendBytes(manifestFileName, itemsBytes); err != nil {
+		return err
+	}
+
+	repoBytes, err := json.Marshal(d.repositories)
+	if err != nil {
+		return errors.Wrap(err, "Error marshaling repositories")
+	}
+	if err := d.sendBytes(legacyRepositoriesFileName, repoBytes); err != nil {
+		return errors.Wrap(err, "Error writing config json file")
+	}
+	return nil
 }