From 0e3c401a030bb2bab02754067f670d6fcef32afb Mon Sep 17 00:00:00 2001 From: Valentin Rothberg Date: Wed, 1 Jul 2020 14:14:40 +0200 Subject: [PATCH] WIP - multi-image docker archives Signed-off-by: Valentin Rothberg --- docker/archive/dest.go | 73 +++++++++++++++++++++++++++++++++--- docker/archive/src.go | 81 ++++++++++++++++++++++++++++++++++++++++ docker/tarfile/dest.go | 85 +++++++++++++++++++++++++----------------- docker/tarfile/src.go | 57 +++++++++++++++++----------- 4 files changed, 236 insertions(+), 60 deletions(-) diff --git a/docker/archive/dest.go b/docker/archive/dest.go index 1cf197429..487fe4a67 100644 --- a/docker/archive/dest.go +++ b/docker/archive/dest.go @@ -5,6 +5,7 @@ import ( "io" "os" + "github.com/containers/image/v5/docker/reference" "github.com/containers/image/v5/docker/tarfile" "github.com/containers/image/v5/types" "github.com/pkg/errors" @@ -17,32 +18,36 @@ type archiveImageDestination struct { } func newImageDestination(sys *types.SystemContext, ref archiveReference) (types.ImageDestination, error) { + return newArchiveImageDestination(sys, ref.path, ref.destinationRef) +} + +func newArchiveImageDestination(sys *types.SystemContext, path string, ref reference.NamedTagged) (*archiveImageDestination, error) { // ref.path can be either a pipe or a regular file // in the case of a pipe, we require that we can open it for write // in the case of a regular file, we don't want to overwrite any pre-existing file // so we check for Size() == 0 below (This is racy, but using O_EXCL would also be racy, // only in a different way. Either way, it’s up to the user to not have two writers to the same path.) - fh, err := os.OpenFile(ref.path, os.O_WRONLY|os.O_CREATE, 0644) + fh, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0644) if err != nil { - return nil, errors.Wrapf(err, "error opening file %q", ref.path) + return nil, errors.Wrapf(err, "error opening file %q", path) } fhStat, err := fh.Stat() if err != nil { - return nil, errors.Wrapf(err, "error statting file %q", ref.path) + return nil, errors.Wrapf(err, "error statting file %q", path) } if fhStat.Mode().IsRegular() && fhStat.Size() != 0 { return nil, errors.New("docker-archive doesn't support modifying existing images") } - tarDest := tarfile.NewDestinationWithContext(sys, fh, ref.destinationRef) + tarDest := tarfile.NewDestinationWithContext(sys, fh, ref) if sys != nil && sys.DockerArchiveAdditionalTags != nil { tarDest.AddRepoTags(sys.DockerArchiveAdditionalTags) } return &archiveImageDestination{ Destination: tarDest, - ref: ref, + ref: archiveReference{path, ref}, writer: fh, }, nil } @@ -70,3 +75,61 @@ func (d *archiveImageDestination) Close() error { func (d *archiveImageDestination) Commit(ctx context.Context, unparsedToplevel types.UnparsedImage) error { return d.Destination.Commit(ctx) } + +func (m multiImageDestinationReference) NewImageDestination(_ context.Context, _ *types.SystemContext) (types.ImageDestination, error) { + return m.dest, nil +} + +// MultiImageDestinations allows for creating and writing to docker archives +// that include more than one image. +type MultiImageDestination struct { + *archiveImageDestination + path string +} + +// multiImageDestinationReference is a types.ImageReference embedding a MultiImageDestination. +type multiImageDestinationReference struct { + *archiveReference + dest *MultiImageDestination +} + +// NewMultiImageDestination returns a MultiImageDestination for the specified path. +func NewMultiImageDestination(sys *types.SystemContext, path string) (*MultiImageDestination, error) { + dest, err := newArchiveImageDestination(sys, path, nil) + if err != nil { + return nil, err + } + return &MultiImageDestination{dest, path}, nil +} + +// Reference returns an ImageReference embedding the MultiImageDestination. +func (m *MultiImageDestination) Reference() types.ImageReference { + ref := &archiveReference{path: m.path} + return &multiImageDestinationReference{ref, m} +} + +// Close is a NOP. Please use Finalize() for committing the archive and +// closing the underlying resources. +func (m *MultiImageDestination) Close() error { + return nil +} + +// Commit is a NOP. Please use Finalize() for committing the archive and +// closing the underlying resources. +func (m *MultiImageDestination) Commit(_ context.Context, _ types.UnparsedImage) error { + return nil +} + +// Finalize commits pending data and closes the underlying tarfile. +func (m *MultiImageDestination) Finalize(ctx context.Context) (finalErr error) { + defer func() { + if err := m.writer.Close(); err != nil { + if finalErr == nil { + finalErr = err + } else { + finalErr = errors.Wrap(finalErr, err.Error()) + } + } + }() + return m.Destination.Commit(ctx) +} diff --git a/docker/archive/src.go b/docker/archive/src.go index 6a628508d..f37c8b8ed 100644 --- a/docker/archive/src.go +++ b/docker/archive/src.go @@ -4,6 +4,7 @@ import ( "context" "github.com/containers/image/v5/docker/tarfile" + ctrImage "github.com/containers/image/v5/image" "github.com/containers/image/v5/types" "github.com/sirupsen/logrus" ) @@ -34,3 +35,83 @@ func newImageSource(ctx context.Context, sys *types.SystemContext, ref archiveRe func (s *archiveImageSource) Reference() types.ImageReference { return s.ref } + +// MultiImageSourceItem is a reference to _one_ image in a multi-image archive. +// Note that MultiImageSourceItem implements types.ImageReference. It's a +// long-lived object that can only be closed via it's parent MultiImageSource. +type MultiImageSourceItem struct { + *archiveReference + tarSource *tarfile.Source +} + +// Manifest returns the tarfile.ManifestItem. +func (m *MultiImageSourceItem) Manifest() (*tarfile.ManifestItem, error) { + items, err := m.tarSource.LoadTarManifest() + if err != nil { + return nil, err + } + return &items[0], nil +} + +// NewImage returns a types.ImageCloser for this reference, possibly +// specialized for this ImageTransport. +func (m MultiImageSourceItem) NewImage(ctx context.Context, sys *types.SystemContext) (types.ImageCloser, error) { + src, err := m.NewImageSource(ctx, sys) + if err != nil { + return nil, err + } + return ctrImage.FromSource(ctx, sys, src) +} + +// NewImageSource returns a types.ImageSource for this reference. +func (m MultiImageSourceItem) NewImageSource(ctx context.Context, sys *types.SystemContext) (types.ImageSource, error) { + return &archiveImageSource{ + Source: m.tarSource, + ref: *m.archiveReference, + }, nil +} + +// MultiImageSource allows for reading docker archives that includes more +// than one image. Use Items() to extract +type MultiImageSource struct { + path string + tarSource *tarfile.Source +} + +// NewMultiImageSource creates a MultiImageSource for the +// specified path pointing to a docker-archive. +func NewMultiImageSource(ctx context.Context, sys *types.SystemContext, path string) (*MultiImageSource, error) { + src, err := tarfile.NewSourceFromFileWithContext(sys, path) + if err != nil { + return nil, err + } + return &MultiImageSource{path: path, tarSource: src}, nil +} + +// Close closes the underlying tarfile. +func (m *MultiImageSource) Close() error { + return m.tarSource.Close() +} + +// Items returns a MultiImageSourceItem for all manifests/images in the archive. +// Each references embeds an ImageSource pointing to the corresponding image in +// the archive. +func (m *MultiImageSource) Items() ([]MultiImageSourceItem, error) { + items, err := m.tarSource.LoadTarManifest() + if err != nil { + return nil, err + } + references := []MultiImageSourceItem{} + for index := range items { + src, err := m.tarSource.FromManifest(index) + if err != nil { + return nil, err + } + newRef := MultiImageSourceItem{ + &archiveReference{path: m.path}, + src, + } + references = append(references, newRef) + } + return references, nil +} diff --git a/docker/tarfile/dest.go b/docker/tarfile/dest.go index c171da505..1da51fbc7 100644 --- a/docker/tarfile/dest.go +++ b/docker/tarfile/dest.go @@ -24,9 +24,11 @@ import ( // Destination is a partial implementation of types.ImageDestination for writing to an io.Writer. type Destination struct { - writer io.Writer - tar *tar.Writer - repoTags []reference.NamedTagged + writer io.Writer + tar *tar.Writer + repoTags []reference.NamedTagged + manifest []ManifestItem + repositories map[string]map[string]string // Other state. blobs map[digest.Digest]types.BlobInfo // list of already-sent blobs config []byte @@ -46,11 +48,12 @@ func NewDestinationWithContext(sys *types.SystemContext, dest io.Writer, ref ref repoTags = append(repoTags, ref) } return &Destination{ - writer: dest, - tar: tar.NewWriter(dest), - repoTags: repoTags, - blobs: make(map[digest.Digest]types.BlobInfo), - sysCtx: sys, + writer: dest, + tar: tar.NewWriter(dest), + repoTags: repoTags, + blobs: make(map[digest.Digest]types.BlobInfo), + sysCtx: sys, + repositories: map[string]map[string]string{}, } } @@ -183,24 +186,14 @@ func (d *Destination) TryReusingBlob(ctx context.Context, info types.BlobInfo, c return false, types.BlobInfo{}, nil } -func (d *Destination) createRepositoriesFile(rootLayerID string) error { - repositories := map[string]map[string]string{} +func (d *Destination) addRootLayerToRepositories(rootLayerID string) { for _, repoTag := range d.repoTags { - if val, ok := repositories[repoTag.Name()]; ok { + if val, ok := d.repositories[repoTag.Name()]; ok { val[repoTag.Tag()] = rootLayerID } else { - repositories[repoTag.Name()] = map[string]string{repoTag.Tag(): rootLayerID} + d.repositories[repoTag.Name()] = map[string]string{repoTag.Tag(): rootLayerID} } } - - b, err := json.Marshal(repositories) - if err != nil { - return errors.Wrap(err, "Error marshaling repositories") - } - if err := d.sendBytes(legacyRepositoriesFileName, b); err != nil { - return errors.Wrap(err, "Error writing config json file") - } - return nil } // PutManifest writes manifest to the destination. @@ -229,9 +222,7 @@ func (d *Destination) PutManifest(ctx context.Context, m []byte, instanceDigest } if len(man.LayersDescriptors) > 0 { - if err := d.createRepositoriesFile(lastLayerID); err != nil { - return err - } + d.addRootLayerToRepositories(lastLayerID) } repoTags := []string{} @@ -256,20 +247,18 @@ func (d *Destination) PutManifest(ctx context.Context, m []byte, instanceDigest repoTags = append(repoTags, refString) } - items := []ManifestItem{{ + d.manifest = append(d.manifest, ManifestItem{ Config: man.ConfigDescriptor.Digest.Hex() + ".json", RepoTags: repoTags, Layers: layerPaths, Parent: "", LayerSources: nil, - }} - itemsBytes, err := json.Marshal(&items) - if err != nil { - return err - } + }) - // FIXME? Do we also need to support the legacy format? - return d.sendBytes(manifestFileName, itemsBytes) + // Reset the repoTags to prevent them from leaking into a following + // image/manifest. + d.repoTags = []reference.NamedTagged{} + return nil } // writeLegacyLayerMetadata writes legacy VERSION and configuration files for all layers @@ -419,6 +408,34 @@ func (d *Destination) PutSignatures(ctx context.Context, signatures [][]byte, in // Commit finishes writing data to the underlying io.Writer. // It is the caller's responsibility to close it, if necessary. -func (d *Destination) Commit(ctx context.Context) error { - return d.tar.Close() +func (d *Destination) Commit(ctx context.Context) (finalErr error) { + defer func() { + if err := d.tar.Close(); err != nil { + if finalErr == nil { + finalErr = err + } else { + finalErr = errors.Wrap(finalErr, err.Error()) + } + } + }() + // Writing the manifest here instead of PutManifest allows for + // supporting multi-image archives. + itemsBytes, err := json.Marshal(d.manifest) + if err != nil { + return err + } + + // FIXME? Do we also need to support the legacy format? + if err := d.sendBytes(manifestFileName, itemsBytes); err != nil { + return err + } + + repoBytes, err := json.Marshal(d.repositories) + if err != nil { + return errors.Wrap(err, "Error marshaling repositories") + } + if err := d.sendBytes(legacyRepositoriesFileName, repoBytes); err != nil { + return errors.Wrap(err, "Error writing config json file") + } + return nil } diff --git a/docker/tarfile/src.go b/docker/tarfile/src.go index 4d2368c70..74459c54f 100644 --- a/docker/tarfile/src.go +++ b/docker/tarfile/src.go @@ -24,6 +24,7 @@ import ( type Source struct { tarPath string removeTarPathOnClose bool // Remove temp file on close if true + manifests []ManifestItem // The following data is only available after ensureCachedDataIsPresent() succeeds tarManifest *ManifestItem // nil if not available yet. configBytes []byte @@ -76,6 +77,26 @@ func NewSourceFromFileWithContext(sys *types.SystemContext, path string) (*Sourc return NewSourceFromStreamWithSystemContext(sys, stream) } +// FromManifest returns a Source for the ManifestItem at the specified index. +// Close() is a NOP for the returned Source. +func (s *Source) FromManifest(index int) (*Source, error) { + // Note: selecting sub-sources rather than spliting one into multiple + // sub-sources entails the benefit that we can select images within an + // archive. This leaves the door open to provide tags for sources and + // copy a specific image from the archive. + items, err := s.LoadTarManifest() + if err != nil { + return nil, err + } + if index < 0 || index >= len(items) { + return nil, errors.Errorf("no manifest item for index %d", index) + } + return &Source{ + tarPath: s.tarPath, + manifests: []ManifestItem{items[index]}, + }, nil +} + // NewSourceFromStream returns a tarfile.Source for the specified inputStream, // which can be either compressed or uncompressed. The caller can close the // inputStream immediately after NewSourceFromFile returns. @@ -230,16 +251,11 @@ func (s *Source) ensureCachedDataIsPresent() error { // Call ensureCachedDataIsPresent instead. func (s *Source) ensureCachedDataIsPresentPrivate() error { // Read and parse manifest.json - tarManifest, err := s.loadTarManifest() + tarManifest, err := s.LoadTarManifest() if err != nil { return err } - // Check to make sure length is 1 - if len(tarManifest) != 1 { - return errors.Errorf("Unexpected tar manifest.json: expected 1 item, got %d", len(tarManifest)) - } - // Read and parse config. configBytes, err := s.readTarComponent(tarManifest[0].Config, iolimits.MaxConfigBodySize) if err != nil { @@ -267,20 +283,6 @@ func (s *Source) ensureCachedDataIsPresentPrivate() error { return nil } -// loadTarManifest loads and decodes the manifest.json. -func (s *Source) loadTarManifest() ([]ManifestItem, error) { - // FIXME? Do we need to deal with the legacy format? - bytes, err := s.readTarComponent(manifestFileName, iolimits.MaxTarFileManifestSize) - if err != nil { - return nil, err - } - var items []ManifestItem - if err := json.Unmarshal(bytes, &items); err != nil { - return nil, errors.Wrap(err, "Error decoding tar manifest.json") - } - return items, nil -} - // Close removes resources associated with an initialized Source, if any. func (s *Source) Close() error { if s.removeTarPathOnClose { @@ -291,7 +293,20 @@ func (s *Source) Close() error { // LoadTarManifest loads and decodes the manifest.json func (s *Source) LoadTarManifest() ([]ManifestItem, error) { - return s.loadTarManifest() + if s.manifests != nil { + return s.manifests, nil + } + // FIXME? Do we need to deal with the legacy format? + bytes, err := s.readTarComponent(manifestFileName, iolimits.MaxTarFileManifestSize) + if err != nil { + return nil, err + } + var items []ManifestItem + if err := json.Unmarshal(bytes, &items); err != nil { + return nil, errors.Wrap(err, "Error decoding tar manifest.json") + } + s.manifests = items + return s.manifests, nil } func (s *Source) prepareLayerData(tarManifest *ManifestItem, parsedConfig *manifest.Schema2Image) (map[digest.Digest]*layerInfo, error) {