Merge pull request #1760 from ktock/bump-ss

Bump stargz-snapshotter and partial registry logic integration

Dockerfile

@@ -11,7 +11,7 @@ ARG ROOTLESSKIT_VERSION=v0.9.5
 ARG CNI_VERSION=v0.8.6
 ARG SHADOW_VERSION=4.8.1
 ARG FUSEOVERLAYFS_VERSION=v1.1.2
-ARG STARGZ_SNAPSHOTTER_VERSION=2ee75e91f8f98f3d324290a2503269812e019fc3
+ARG STARGZ_SNAPSHOTTER_VERSION=3a04e4c2c116c85b4b66d01945cf7ebcb7a2eb5a
 
 # git stage is used for checking out remote repository sources
 FROM --platform=$BUILDPLATFORM alpine AS git

cmd/buildkitd/main_oci_worker.go

@@ -3,6 +3,7 @@
 package main
 
 import (
+	"context"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -13,13 +14,15 @@ import (
 	snapshotsapi "github.com/containerd/containerd/api/services/snapshots/v1"
 	"github.com/containerd/containerd/defaults"
 	"github.com/containerd/containerd/pkg/dialer"
+	"github.com/containerd/containerd/remotes/docker"
 	ctdsnapshot "github.com/containerd/containerd/snapshots"
 	"github.com/containerd/containerd/snapshots/native"
 	"github.com/containerd/containerd/snapshots/overlay"
 	snproxy "github.com/containerd/containerd/snapshots/proxy"
 	"github.com/containerd/containerd/sys"
 	remotesn "github.com/containerd/stargz-snapshotter/snapshot"
 	"github.com/containerd/stargz-snapshotter/stargz"
+	sgzsource "github.com/containerd/stargz-snapshotter/stargz/source"
 	"github.com/moby/buildkit/cmd/buildkitd/config"
 	"github.com/moby/buildkit/executor/oci"
 	"github.com/moby/buildkit/util/network/cniprovider"
@@ -230,7 +233,8 @@ func ociWorkerInitializer(c *cli.Context, common workerInitializerOpt) ([]worker
 		return nil, err
 	}
 
-	snFactory, err := snapshotterFactory(common.config.Root, cfg)
+	hosts := resolverFunc(common.config)
+	snFactory, err := snapshotterFactory(common.config.Root, cfg, hosts)
 	if err != nil {
 		return nil, err
 	}
@@ -267,7 +271,7 @@ func ociWorkerInitializer(c *cli.Context, common workerInitializerOpt) ([]worker
 		return nil, err
 	}
 	opt.GCPolicy = getGCPolicy(cfg.GCConfig, common.config.Root)
-	opt.RegistryHosts = resolverFunc(common.config)
+	opt.RegistryHosts = hosts
 
 	if platformsStr := cfg.Platforms; len(platformsStr) != 0 {
 		platforms, err := parsePlatforms(platformsStr)
@@ -283,7 +287,7 @@ func ociWorkerInitializer(c *cli.Context, common workerInitializerOpt) ([]worker
 	return []worker.Worker{w}, nil
 }
 
-func snapshotterFactory(commonRoot string, cfg config.OCIConfig) (runc.SnapshotterFactory, error) {
+func snapshotterFactory(commonRoot string, cfg config.OCIConfig, hosts docker.RegistryHosts) (runc.SnapshotterFactory, error) {
 	var (
 		name    = cfg.Snapshotter
 		address = cfg.ProxySnapshotterPath
@@ -348,13 +352,36 @@ func snapshotterFactory(commonRoot string, cfg config.OCIConfig) (runc.Snapshott
 			return fuseoverlayfs.NewSnapshotter(root)
 		}
 	case "stargz":
+		// Pass the registry configuration to stargz snapshotter
+		sgzhosts := func(host string) ([]docker.RegistryHost, error) {
+			base, err := hosts(host)
+			if err != nil {
+				return nil, err
+			}
+			for i := range base {
+				if base[i].Authorizer == nil {
+					// Default authorizer that don't fetch creds via session
+					// TODO(ktock): use session-based authorizer
+					base[i].Authorizer = docker.NewDockerAuthorizer(
+						docker.WithAuthClient(base[i].Client))
+				}
+			}
+			return base, nil
+		}
 		snFactory.New = func(root string) (ctdsnapshot.Snapshotter, error) {
 			fs, err := stargz.NewFilesystem(filepath.Join(root, "stargz"),
-				cfg.StargzSnapshotterConfig)
+				cfg.StargzSnapshotterConfig,
+				stargz.WithGetSources(
+					// provides source info based on the registry config and
+					// default labels.
+					sgzsource.FromDefaultLabels(sgzhosts),
+				),
+			)
 			if err != nil {
 				return nil, err
 			}
-			return remotesn.NewSnapshotter(filepath.Join(root, "snapshotter"),
+			return remotesn.NewSnapshotter(context.Background(),
+				filepath.Join(root, "snapshotter"),
 				fs, remotesn.AsynchronousRemove)
 		}
 	default:

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe
 	github.com/containerd/go-cni v1.0.1
 	github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0
-	github.com/containerd/stargz-snapshotter v0.0.0-20200903042824-2ee75e91f8f9
+	github.com/containerd/stargz-snapshotter v0.0.0-20201027054423-3a04e4c2c116
 	github.com/containerd/typeurl v1.0.1
 	github.com/coreos/go-systemd/v22 v22.1.0
 	github.com/docker/cli v0.0.0-20200227165822-2298e6a3fe24
@@ -26,7 +26,7 @@ require (
 	github.com/gogo/protobuf v1.3.1
 	// protobuf: the actual version is replaced in replace()
 	github.com/golang/protobuf v1.4.2
-	github.com/google/go-cmp v0.4.0
+	github.com/google/go-cmp v0.4.1
 	github.com/google/shlex v0.0.0-20150127133951-6f45313302b9
 	github.com/gorilla/mux v1.7.4 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.2.0
@@ -57,12 +57,12 @@ require (
 	go.etcd.io/bbolt v1.3.5
 	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
 	golang.org/x/net v0.0.0-20200707034311-ab3426394381
-	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
+	golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
 	golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f
-	golang.org/x/time v0.0.0-20191024005414-555d28b269f0
+	golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1
 	// genproto: the actual version is replaced in replace()
-	google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013
-	google.golang.org/grpc v1.28.1
+	google.golang.org/genproto v0.0.0-20200527145253-8367513e4ece
+	google.golang.org/grpc v1.29.1
 )
 
 replace (