Exporter: support nydus image export

Dockerfile

@@ -183,6 +183,11 @@ RUN --mount=target=/root/.cache,type=cache \
   xx-verify --static /out/containerd-stargz-grpc && \
   xx-verify --static /out/ctr-remote
 
+FROM gobuild-base AS nydus-binaries
+WORKDIR /usr/bin
+RUN wget https://github.com/dragonflyoss/image-service/releases/download/v1.0.0/nydus-static-v1.0.0-x86_64.tgz
+RUN tar xzvf nydus-static-v1.0.0-x86_64.tgz
+
 FROM --platform=$BUILDPLATFORM alpine:${ALPINE_VERSION} AS fuse-overlayfs
 RUN apk add --no-cache curl
 COPY --from=xx / /
@@ -248,6 +253,7 @@ ENV BUILDKIT_INTEGRATION_CONTAINERD_EXTRA="containerd-1.3=/opt/containerd-alt/bi
 ENV BUILDKIT_INTEGRATION_SNAPSHOTTER=stargz
 ENV CGO_ENABLED=0
 COPY --from=stargz-snapshotter /out/* /usr/bin/
+COPY --from=nydus-binaries /usr/bin/nydus-static/* /usr/bin/
 COPY --from=rootlesskit /rootlesskit /usr/bin/
 COPY --from=containerd-alt /out/containerd* /opt/containerd-alt/bin/
 COPY --from=registry /bin/registry /usr/bin

client/exporters.go

@@ -1,9 +1,10 @@
 package client
 
 const (
-	ExporterImage  = "image"
-	ExporterLocal  = "local"
-	ExporterTar    = "tar"
-	ExporterOCI    = "oci"
-	ExporterDocker = "docker"
+	ExporterImage      = "image"
+	ExporterLocal      = "local"
+	ExporterTar        = "tar"
+	ExporterOCI        = "oci"
+	ExporterDocker     = "docker"
+	ExporterNydusImage = "nydus"
 )

docs/nydus.md

@@ -0,0 +1,66 @@
+# Nydus Integration With Buildkit (Experimental)
+
+## Nydus Image Overview
+
+Nydus image is a container accelerated image format provided by the [Dragonfly image-service project](https://github.com/dragonflyoss/image-service), which offer the ability to pull image data on demand, without waiting for the entire image pull to complete and then start the container. It has been put in production usage and shown vast improvements over the old OCI image format in terms of container launching speed, image space and network bandwidth efficiency, as well as data integrity. The Nydus image can also serve as an example and reference implementation for the on-going OCI image spec v2 discussion.
+
+The [benchmarking result](https://github.com/dragonflyoss/image-service/raw/master/misc/perf.jpg) shows the performance improvement compared with the OCI image for the container cold startup elapsed time on containerd. As the OCI image size increases, the container startup time of using Nydus image remains very short.
+
+## Key Features On Nydus
+
+Nydus designs and implements a user space filesystem on top of a special designed container image format that improves at all the above mentioned OCI image spec defects. Key features include:
+
+- Container images are downloaded on demand;
+- Chunk level data duplication with configurable chunk size;
+- Flatten image metadata and data to remove all intermediate layers;
+- Only usable image data is saved when building a container image;
+- Only usable image data is downloaded when running a container;
+- End-to-end image data integrity;
+- Compatible with the OCI artifacts spec and distribution spec;
+- Integrated with existing CNCF project dragonfly to support image distribution in large clusters;
+- Except for the OCI distribution spec, other image storage backends can be supported as well;
+
+## Nydus Image In A Nutshell
+
+The Nydus image format follows the OCI v1 image spec for storing and distributing image in the current OCI ecosystem. See the [Nydus Image Example](https://github.com/dragonflyoss/image-service/blob/master/contrib/nydusify/examples/manifest/manifest.json), the image manifest indexes Nydus' RAFS(Registry Acceleration File System) filesystem layer, which is a strongly verified filesystem with separate metadata (bootstrap) and data (blob). The bootstrap contains meta information about files and directories of all layers for an image and related checksum information, while the blob is the data of each layer, which is composed of many data chunks, each chunk may be shared by a layer or with other layers.
+
+With two annotation hints on the layer, [Nydus Snapshotter](https://github.com/dragonflyoss/image-service/tree/master/contrib/nydus-snapshotter) can let containerd download only the bootstrap layer of the image, but not the blob layer, so that data can be loaded on demand. At the same time, the dependency relationship between image manifest and layer is compatible with the existing image dependency tree and garbage collection mechanism of containerd and registry.
+
+The Nydus image currently requires the [Nydus Snapshotter](https://github.com/dragonflyoss/image-service/tree/master/contrib/nydus-snapshotter) to run. For better compatibility, Nydus also supports merging the Nydus manifest with the OCI manifest into a [manifest index](https://github.com/dragonflyoss/image-service/blob/master/contrib/nydusify/examples/manifest/index.json), and adding an OS Feature field to differentiate between them. This design allows Snapshotter that does not support Nydus to give preference to the OCI image to run, and maintain compatibility without changing anything.
+
+See more about Nydus image format on the [design](https://github.com/dragonflyoss/image-service/blob/master/docs/nydus-design.md) doc.
+
+## Implement Nydus Exporter In Buildkit
+
+Nydus has provided a conversion tool [Nydusify](https://github.com/dragonflyoss/image-service/blob/master/docs/nydusify.md) for converting OCI image to Nydus image, which assumes that the OCI image is already available in the registry, but a better way would be to build the Nydus images directly from the build system instead of using the conversion tool, which would increase the speed of the image export, so we experimentally integrated the Nydus exporter in Buildkit.
+
+Nydusify provides a package to do the actual image conversion. We use the same package to implement the buildkit nydus exporter. The exporter mount all layers and call the package to build to a Nydus image and push it to the remote registry. The current implementation relies on the Nydus [builder](https://github.com/dragonflyoss/image-service/blob/master/docs/nydus-image.md) (written in rust), which is used to build a layer to a RAFS layer, and in the future we can explore simpler, less dependent integrations.
+
+## Known Limitations
+
+- Exporter currently relies on the Nydus [builder](https://github.com/dragonflyoss/image-service/blob/master/docs/nydus-image.md) as the core build tool;
+- Currently only supports linux/amd64 platform image export;
+
+# Nydus Exporter Usage
+
+This section describes how to use Buildkit to export Nydus image. Nydus exporter depends on an external nydus-image binary. It can be obtained from the [Nydus Releases](https://github.com/dragonflyoss/image-service/releases) page, named `nydus-image` in tgz. We need put the binary into the directories named by the PATH environment variable.
+
+## Export with buildctl
+
+```shell
+$ buildctl build --frontend=dockerfile.v0 \
+  --local context=/tmp/hello \
+  --local dockerfile=/tmp/hello \
+  --output type=nydus,name=localhost:5000/hello
+```
+
+Keys supported by Nydus exporter:
+
+- name=[value]: Nydus image reference
+- registry.insecure=true: push to insecure HTTP registry
+- oci-mediatypes=true: use OCI mediatypes in Nydus image manifest instead of Docker's
+- merge-manifest=true: merge into manifest index if remote manifest exists
+
+## Run container with Nydus image
+
+After building with buildkit, the image should be pushed to remote registry, now we can run a container with containerd from a Nydus image, [here](https://github.com/dragonflyoss/image-service/blob/master/docs/containerd-env-setup.md) is a setup tutorial.

exporter/nydus/export.go

@@ -0,0 +1,213 @@
+package nydus
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os/exec"
+	"strconv"
+	"strings"
+
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+
+	"github.com/moby/buildkit/cache"
+	"github.com/moby/buildkit/exporter"
+	"github.com/moby/buildkit/exporter/containerimage"
+	"github.com/moby/buildkit/exporter/containerimage/exptypes"
+	"github.com/moby/buildkit/session"
+	"github.com/moby/buildkit/snapshot"
+
+	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/converter"
+	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/converter/provider"
+)
+
+const builderName = "nydus-image"
+
+const (
+	// Specify Nydus image reference.
+	keyTargetRef = "name"
+	// Merge Nydus image manifest with the OCI image manifest
+	// exists in remote registry into an manifest index.
+	keyMergeManifest = "merge-manifest"
+	// Set target manifest media types following OCI spec.
+	keyOCIMediaTypes = "oci-mediatypes"
+	// Push to insecure HTTP registry.
+	keyInsecure = "registry.insecure"
+)
+
+type Opt struct {
+	ImageOpt     containerimage.Opt
+	CacheManager cache.Manager
+}
+
+type nydusExporter struct {
+	opt Opt
+}
+
+type nydusExporterInstance struct {
+	*nydusExporter
+
+	nydusBuilder  string
+	targetRef     string
+	insecure      bool
+	mergeManifest bool
+	ociMediaTypes bool
+}
+
+func New(opt Opt) (exporter.Exporter, error) {
+	return &nydusExporter{opt: opt}, nil
+}
+
+func (exporter *nydusExporter) Resolve(ctx context.Context, opt map[string]string) (exporter.ExporterInstance, error) {
+	// Nydus exporter relies on the Nydus builder to build the
+	// image layer to a Nydus image layer.
+	if _, err := exec.LookPath(builderName); err != nil {
+		return nil, errors.Wrap(err, "not found nydus builder in PATH environment variable")
+	}
+
+	instance := &nydusExporterInstance{
+		nydusExporter: exporter,
+		nydusBuilder:  builderName,
+	}
+
+	for k, v := range opt {
+		switch k {
+		case keyTargetRef:
+			instance.targetRef = v
+		case keyInsecure:
+			if v == "" {
+				instance.insecure = true
+				continue
+			}
+			b, err := strconv.ParseBool(v)
+			if err != nil {
+				return nil, errors.Wrapf(err, "non-bool value specified for %s", k)
+			}
+			instance.insecure = b
+		case keyMergeManifest:
+			if v == "" || v == "true" {
+				instance.mergeManifest = true
+			}
+		case keyOCIMediaTypes:
+			if v == "" || v == "true" {
+				instance.ociMediaTypes = true
+			}
+		}
+	}
+
+	return instance, nil
+}
+
+// Mount a temp directory to save intermediate of Nydus image during exporting
+func (exporter *nydusExporterInstance) createTempDir(ctx context.Context, sessionGroup session.Group) (string, func() error, error) {
+	cacheManager := exporter.opt.CacheManager
+
+	mountableRef, err := cacheManager.New(
+		ctx, nil, sessionGroup, cache.WithDescription("nydus exporter intermediate"),
+	)
+	if err != nil {
+		return "", nil, errors.Wrap(err, "create mountable ref")
+	}
+	mountable, err := mountableRef.Mount(ctx, false, sessionGroup)
+	if err != nil {
+		return "", nil, errors.Wrap(err, "get mountable from mountable ref")
+	}
+
+	mounter := snapshot.LocalMounter(mountable)
+	dir, err := mounter.Mount()
+	if err != nil {
+		return "", nil, errors.Wrap(err, "mount from mountable ref")
+	}
+
+	return dir, mounter.Unmount, nil
+}
+
+// Source prepares image config and layers for Nydusify building
+func (exporter *nydusExporterInstance) getSources(inp exporter.Source, sessionID string) ([]provider.SourceProvider, error) {
+	sources := []provider.SourceProvider{}
+
+	configBytesAry := [][]byte{}
+	refs := []cache.ImmutableRef{}
+	configBytes := inp.Metadata[exptypes.ExporterImageConfigKey]
+	if configBytes != nil && inp.Ref != nil {
+		configBytesAry = append(configBytesAry, configBytes)
+		refs = append(refs, inp.Ref)
+	}
+
+	for key, data := range inp.Metadata {
+		keyAry := strings.Split(key, "/")
+		if len(keyAry) != 2 || keyAry[0] != exptypes.ExporterImageConfigKey {
+			continue
+		}
+		platform := keyAry[1]
+		if inp.Refs[platform] == nil {
+			continue
+		}
+		configBytesAry = append(configBytesAry, data)
+		refs = append(refs, inp.Refs[platform])
+	}
+
+	for idx, configBytes := range configBytesAry {
+		var config ocispec.Image
+		if err := json.Unmarshal(configBytes, &config); err != nil {
+			return nil, errors.New("unmarshal source image config")
+		}
+		source := &sourceProvider{
+			ref:       refs[idx],
+			sessionID: sessionID,
+			config:    config,
+		}
+		sources = append(sources, source)
+	}
+
+	return sources, nil
+}
+
+func (exporter *nydusExporterInstance) Name() string {
+	return fmt.Sprintf("Exporting Nydus image to %s", exporter.targetRef)
+}
+
+func (exporter *nydusExporterInstance) Export(
+	ctx context.Context, inp exporter.Source, sessionID string,
+) (map[string]string, error) {
+	sources, err := exporter.getSources(inp, sessionID)
+	if err != nil {
+		return nil, errors.Wrap(err, "get image sources")
+	}
+
+	// The remote instance communicates with registry
+	targetRemote, err := NewRemote(
+		exporter.opt.ImageOpt.SessionManager, sessionID, exporter.opt.ImageOpt.RegistryHosts, exporter.targetRef, exporter.insecure,
+	)
+	if err != nil {
+		return nil, errors.Wrap(err, "create target remote")
+	}
+
+	// Prepare temp directory for Nydus builder, the builder will output
+	// blob/bootstrap intermediate of Nydus image to this directory
+	workDir, umount, err := exporter.createTempDir(ctx, session.NewGroup(sessionID))
+	if err != nil {
+		return nil, errors.Wrap(err, "create work directory")
+	}
+	defer umount()
+
+	cvt, err := converter.New(converter.Opt{
+		Logger:          &progressLogger{},
+		SourceProviders: sources,
+		TargetRemote:    targetRemote,
+		WorkDir:         workDir,
+		NydusImagePath:  exporter.nydusBuilder,
+		MultiPlatform:   exporter.mergeManifest,
+		DockerV2Format:  !exporter.ociMediaTypes,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	if err := cvt.Convert(ctx); err != nil {
+		return nil, err
+	}
+
+	return nil, nil
+}

exporter/nydus/logger.go

@@ -0,0 +1,46 @@
+package nydus
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/converter/provider"
+
+	"github.com/moby/buildkit/util/progress"
+	"github.com/sirupsen/logrus"
+)
+
+type progressLogger struct{}
+
+// Log outputs Nydus image exporting progress log
+func (logger *progressLogger) Log(ctx context.Context, msg string, fields provider.LoggerFields) func(err error) error {
+	if fields == nil {
+		fields = make(provider.LoggerFields)
+	}
+	logrus.WithFields(fields).Info(msg)
+	if len(fields) != 0 {
+		var infos []string
+		for key, value := range fields {
+			line := fmt.Sprintf("%s=%s", key, value)
+			infos = append(infos, line)
+		}
+		sort.Strings(infos)
+		msg = msg + " [" + strings.Join(infos, " ") + "]"
+	}
+	pw, _, _ := progress.FromContext(ctx)
+	now := time.Now()
+	st := progress.Status{
+		Started: &now,
+	}
+	pw.Write(msg, st)
+	return func(err error) error {
+		now := time.Now()
+		st.Completed = &now
+		pw.Write(msg, st)
+		pw.Close()
+		return err
+	}
+}