WIP - multi-image archives

Signed-off-by: Valentin Rothberg <rothberg@redhat.com>

cmd/podman/images/save.go

@@ -76,11 +76,12 @@ func init() {
 }
 
 func saveFlags(flags *pflag.FlagSet) {
+	cfg := registry.PodmanConfig().Engine
 	flags.BoolVar(&saveOpts.Compress, "compress", false, "Compress tarball image layers when saving to a directory using the 'dir' transport. (default is same compression type as source)")
 	flags.StringVar(&saveOpts.Format, "format", define.V2s2Archive, "Save image to oci-archive, oci-dir (directory with oci manifest type), docker-archive, docker-dir (directory with v2s2 manifest type)")
 	flags.StringVarP(&saveOpts.Output, "output", "o", "", "Write to a specified file (default: stdout, which must be redirected)")
 	flags.BoolVarP(&saveOpts.Quiet, "quiet", "q", false, "Suppress the output")
-
+	flags.BoolVarP(&saveOpts.MultiImageArchive, "multi-image-archive", "m", cfg.MultiImageArchive, "Interpret additional arguments as images not tags and create a multi-image-archive (only for docker-archive)")
 }
 
 func save(cmd *cobra.Command, args []string) error {

docs/source/markdown/podman-save.1.md

@@ -40,6 +40,10 @@ Save image to **oci-archive, oci-dir** (directory with oci manifest type), or **
 --format docker-dir
 ```
 
+**--multi-image-archive**, **-m**
+
+Allow for creating archives with more than one image.  Additional names will be interpreted as images instead of tags.  Only supported for **docker-archive**.
+
 **--quiet**, **-q**
 
 Suppress the output

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/containers/common v0.18.0
 	github.com/containers/conmon v2.0.19+incompatible
 	github.com/containers/image/v5 v5.5.1
+	github.com/containers/libpod v1.9.3 // indirect
 	github.com/containers/psgo v1.5.1
 	github.com/containers/storage v1.21.2
 	github.com/coreos/go-systemd/v22 v22.1.0
@@ -61,10 +62,14 @@ require (
 	golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5
 	golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7
 	golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
-	golang.org/x/sys v0.0.0-20200519105757-fe76b779f299
+	golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae
 	golang.org/x/text v0.3.3 // indirect
 	gopkg.in/yaml.v2 v2.3.0
 	k8s.io/api v0.18.6
 	k8s.io/apimachinery v0.18.6
 	k8s.io/client-go v0.0.0-20190620085101-78d2af792bab
 )
+
+replace github.com/containers/image/v5 => github.com/mtrmac/image/v5 v5.0.0-20200807001926-d7c80f4c0d5b
+
+replace github.com/containers/common => github.com/vrothberg/common v0.0.3-0.20200731120403-9ff18e239651

libpod/image/image.go

@@ -16,6 +16,7 @@ import (
 
 	cp "github.com/containers/image/v5/copy"
 	"github.com/containers/image/v5/directory"
+	"github.com/containers/image/v5/docker/archive"
 	dockerarchive "github.com/containers/image/v5/docker/archive"
 	"github.com/containers/image/v5/docker/reference"
 	"github.com/containers/image/v5/image"
@@ -170,15 +171,208 @@ func (ir *Runtime) New(ctx context.Context, name, signaturePolicyPath, authfile
 	return newImage, nil
 }
 
+// SaveImages stores one more images in a multi-image archive.
+// Note that only `docker-archive` supports storing multiple
+// image.
+func (ir *Runtime) SaveImages(ctx context.Context, namesOrIDs []string, format string, outputFile string, quiet bool) (finalErr error) {
+	if format != DockerArchive {
+		return errors.Errorf("multi-image archives are only supported in in the %q format", DockerArchive)
+	}
+
+	sys := GetSystemContext("", "", false)
+
+	archWriter, err := archive.NewWriter(sys, outputFile)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		err := archWriter.Close()
+		if err == nil {
+			return
+		}
+		if finalErr == nil {
+			finalErr = err
+			return
+		}
+		finalErr = errors.Wrap(finalErr, err.Error())
+	}()
+
+	// Check whether c/image's progress bars use stderr or stdout.  Use
+	// stderr in case we need to be quiet or if the output is set to
+	// stdout.  If the output is set of stout, any log message there would
+	// corrupt the tarfile.
+	writer := os.Stdout
+	if quiet || outputFile == os.Stdout.Name() {
+		writer = os.Stderr
+	}
+
+	// extend an image with additional tags
+	type imageData struct {
+		*Image
+		tags []reference.NamedTagged
+	}
+
+	// Look up the images (and their tags) in the local storage.
+	imageMap := make(map[string]*imageData) // to group tags for an image
+	imageQueue := []string{}                // to preserve relative image order
+	for _, nameOrID := range namesOrIDs {
+		// Look up the name or ID in the local image storage.
+		localImage, err := ir.NewFromLocal(nameOrID)
+		if err != nil {
+			return err
+		}
+		id := localImage.ID()
+
+		iData, exists := imageMap[id]
+		if !exists {
+			imageQueue = append(imageQueue, id)
+			iData = &imageData{Image: localImage}
+			imageMap[id] = iData
+		}
+
+		// Unless we referred to an ID, add the input as a tag.
+		if strings.HasPrefix(id, nameOrID) {
+			continue
+		}
+		tag, err := NormalizedTag(nameOrID)
+		if err != nil {
+			return err
+		}
+		refTagged, _ := tag.(reference.NamedTagged)
+		iData.tags = append(iData.tags, refTagged)
+	}
+
+	policyContext, err := getPolicyContext(sys)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := policyContext.Destroy(); err != nil {
+			logrus.Errorf("failed to destroy policy context: %q", err)
+		}
+	}()
+
+	// Now copy the images one-by-one.
+	for _, id := range imageQueue {
+		dest, err := archWriter.NewReference(nil)
+		if err != nil {
+			return err
+		}
+
+		img := imageMap[id]
+		copyOptions := getCopyOptions(sys, writer, nil, nil, SigningOptions{}, "", img.tags)
+		copyOptions.DestinationCtx.SystemRegistriesConfPath = registries.SystemRegistriesConfPath()
+
+		// For copying, we need a source reference that we can create
+		// from the image.
+		src, err := is.Transport.ParseStoreReference(img.imageruntime.store, id)
+		if err != nil {
+			return errors.Wrapf(err, "error getting source imageReference for %q", img.InputName)
+		}
+		_, err = cp.Image(ctx, policyContext, dest, src, copyOptions)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (ir *Runtime) loadMultiImageArchive(ctx context.Context, sys *types.SystemContext, ref types.ImageReference, writer io.Writer) (loadedImages []string, finalErr error) {
+	if ref.Transport().Name() != DockerArchive {
+		return nil, errors.Errorf("wrong transport %q: only %q supports multi-image archives", ref, DockerArchive)
+	}
+
+	archReader, err := archive.NewReader(sys, ref.StringWithinTransport())
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		err := archReader.Close()
+		if err == nil {
+			return
+		}
+		if finalErr == nil {
+			finalErr = err
+			return
+		}
+		finalErr = errors.Wrap(finalErr, err.Error())
+	}()
+
+	refLists, err := archReader.List()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, refList := range refLists {
+		refPairs := []pullRefPair{}
+		destMap := make(map[string]bool)
+		for _, ref := range refList {
+			destName := ""
+			// Determine the destination. If it's a tagged item, we
+			// can use it directly, otherwise we need to determine
+			// the image digest.
+			if dkrRef := ref.DockerReference(); dkrRef != nil {
+				destName = ref.DockerReference().String()
+			} else {
+				destName, err = getImageDigest(ctx, ref, sys)
+				if err != nil {
+					return nil, err
+				}
+			}
+			// Make sure we process a destination at most once.
+			if _, exists := destMap[destName]; exists {
+				continue
+			}
+			destMap[destName] = true
+
+			pair, err := ir.getPullRefPair(ref, destName)
+			if err != nil {
+				return nil, err
+			}
+			refPairs = append(refPairs, pair)
+		}
+
+		goal := pullGoal{
+			refPairs:             refPairs,
+			pullAllPairs:         true,
+			usedSearchRegistries: false,
+			searchedRegistries:   nil,
+		}
+
+		imageNames, err := ir.doPullImage(ctx, sys, goal, writer, SigningOptions{}, &DockerRegistryOptions{}, nil)
+		if err != nil {
+			return nil, err
+		}
+
+		loadedImages = append(loadedImages, imageNames...)
+	}
+
+	return loadedImages, nil
+}
+
 // LoadFromArchiveReference creates a new image object for images pulled from a tar archive and the like (podman load)
 // This function is needed because it is possible for a tar archive to have multiple tags for one image
 func (ir *Runtime) LoadFromArchiveReference(ctx context.Context, srcRef types.ImageReference, signaturePolicyPath string, writer io.Writer) ([]*Image, error) {
 	if signaturePolicyPath == "" {
 		signaturePolicyPath = ir.SignaturePolicyPath
 	}
-	imageNames, err := ir.pullImageFromReference(ctx, srcRef, writer, "", signaturePolicyPath, SigningOptions{}, &DockerRegistryOptions{})
-	if err != nil {
-		return nil, errors.Wrapf(err, "unable to pull %s", transports.ImageName(srcRef))
+
+	var imageNames []string
+	var err error
+
+	switch srcRef.Transport().Name() {
+	case DockerArchive:
+		sys := GetSystemContext(signaturePolicyPath, "", false)
+		imageNames, err = ir.loadMultiImageArchive(ctx, sys, srcRef, writer)
+		if err != nil {
+			return nil, errors.Wrapf(err, "unable to load %s", transports.ImageName(srcRef))
+		}
+	default:
+		imageNames, err = ir.pullImageFromReference(ctx, srcRef, writer, "", signaturePolicyPath, SigningOptions{}, &DockerRegistryOptions{})
+		if err != nil {
+			return nil, errors.Wrapf(err, "unable to pull %s", transports.ImageName(srcRef))
+		}
 	}
 
 	newImages := make([]*Image, 0, len(imageNames))

pkg/domain/entities/images.go

@@ -269,11 +269,21 @@ type ImageImportReport struct {
 	Id string //nolint
 }
 
+// ImageSaveOptions provide options for saving images.
 type ImageSaveOptions struct {
+	// Compress layers when saving to a directory.
 	Compress bool
-	Format   string
-	Output   string
-	Quiet    bool
+	// Format of saving the image: oci-archive, oci-dir (directory with oci
+	// manifest type), docker-archive, docker-dir (directory with v2s2
+	// manifest type).
+	Format string
+	// MultiImageArchive denotes if the created archive shall include more
+	// than one image.
+	MultiImageArchive bool
+	// Output - write image to the specified path.
+	Output string
+	// Quiet - suppress output when copying images
+	Quiet bool
 }
 
 // ImageTreeOptions provides options for ImageEngine.Tree()

pkg/domain/infra/abi/images.go

@@ -479,6 +479,10 @@ func (ir *ImageEngine) Import(ctx context.Context, opts entities.ImageImportOpti
 }
 
 func (ir *ImageEngine) Save(ctx context.Context, nameOrID string, tags []string, options entities.ImageSaveOptions) error {
+	if options.MultiImageArchive {
+		nameOrIDs := append([]string{nameOrID}, tags...)
+		return ir.Libpod.ImageRuntime().SaveImages(ctx, nameOrIDs, options.Format, options.Output, options.Quiet)
+	}
 	newImage, err := ir.Libpod.ImageRuntime().NewFromLocal(nameOrID)
 	if err != nil {
 		return err

test/e2e/save_test.go

@@ -128,4 +128,51 @@ var _ = Describe("Podman save", func() {
 		save.WaitWithDefaultTimeout()
 		Expect(save.ExitCode()).To(Equal(0))
 	})
+
+	It("podman save --multi-image-archive (tagged images)", func() {
+		multiImageSave(podmanTest, RESTORE_IMAGES)
+	})
+
+	It("podman save --multi-image-archive (untagged images)", func() {
+		// Refer to images via ID instead of tag.
+		session := podmanTest.PodmanNoCache([]string{"images", "--format", "{{.ID}}"})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+		ids := session.OutputToStringArray()
+
+		Expect(len(RESTORE_IMAGES), len(ids))
+		multiImageSave(podmanTest, ids)
+	})
 })
+
+// Create a multi-image archive, remove all images, load it and
+// make sure that all images are (again) present.
+func multiImageSave(podmanTest *PodmanTestIntegration, images []string) {
+	// Create the archive.
+	outfile := filepath.Join(podmanTest.TempDir, "temp.tar")
+	session := podmanTest.PodmanNoCache(append([]string{"save", "-o", outfile, "--multi-image-archive"}, images...))
+	session.WaitWithDefaultTimeout()
+	Expect(session.ExitCode()).To(Equal(0))
+
+	// Remove all images.
+	session = podmanTest.PodmanNoCache([]string{"rmi", "-af"})
+	session.WaitWithDefaultTimeout()
+	Expect(session.ExitCode()).To(Equal(0))
+
+	// Now load the archive.
+	session = podmanTest.PodmanNoCache([]string{"load", "-i", outfile})
+	session.WaitWithDefaultTimeout()
+	Expect(session.ExitCode()).To(Equal(0))
+	// Grep for each image in the `podman load` output.
+	for _, image := range images {
+		found, _ := session.GrepString(image)
+		Expect(found).Should(BeTrue())
+	}
+
+	// Make sure that each image has really been loaded.
+	for _, image := range images {
+		session = podmanTest.PodmanNoCache([]string{"image", "exists", image})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+	}
+}