dockerfile: support Dockerfile.pin for pinning sources

docs/build-repro.md

@@ -127,3 +127,36 @@ jq '.' metadata.json
   "containerimage.digest": "sha256:..."
 }
 ```
+
+### Reproducible build with `Dockerfile.pin`
+
+<!-- TODO: dockerfile-upstream:master -> dockerfile:1.5 after the release of 1.5 -->>
+`Dockerfile.pin` is supported in the `docker/dockerfile-upstream:master` syntax.
+```dockerfile
+# syntax=docker/dockerfile-upstream:master`
+```
+
+When `Dockerfile.pin` exists in the context, the Dockerfile builder does:
+- Pinning the digest of `docker-image` sources (`FROM ...`)
+- Pinning the digest of `http` sources (`ADD https://...`)
+- Recording the consumed entries to the exporter response `pin.consumed`
+
+The content of `Dockerfile.pin` is a subset of the build info structure:
+```json
+{
+    "sources": [
+      {
+        "type": "docker-image",
+        "ref": "docker.io/library/alpine:latest",
+        "pin": "sha256:4edbd2beb5f78b1014028f4fbb99f3237d9561100b6881aabbf5acce2c4f9454"
+      },
+      {
+        "type": "http",
+        "ref": "https://raw.githubusercontent.com/moby/buildkit/v0.10.1/README.md",
+        "pin": "sha256:6e4b94fc270e708e1068be28bd3551dc6917a4fc5a61293d51bb36e6b75c4b53"
+      }
+    ]
+}
+```
+
+In the future, Dockerfile should also support `ADD https://example.com/repo.git /dir` and pinning its commit hash.

exporter/containerimage/exptypes/types.go

@@ -13,6 +13,7 @@ const (
 	ExporterInlineCache          = "containerimage.inlinecache"
 	ExporterBuildInfo            = "containerimage.buildinfo"
 	ExporterPlatformsKey         = "refs.platforms"
+	ExporterPinConsumed          = "pin.consumed" // base64-encoded JSON of util/pin/types.Consumed
 )
 
 type Platforms struct {

frontend/dockerfile/builder/build.go

@@ -27,6 +27,8 @@ import (
 	"github.com/moby/buildkit/solver/errdefs"
 	"github.com/moby/buildkit/solver/pb"
 	binfotypes "github.com/moby/buildkit/util/buildinfo/types"
+	"github.com/moby/buildkit/util/pin"
+	pintypes "github.com/moby/buildkit/util/pin/types"
 	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
@@ -163,7 +165,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
 
 	name := "load build definition from " + filename
 
-	filenames := []string{filename, filename + ".dockerignore"}
+	filenames := []string{filename, filename + ".dockerignore", filename + ".pin"}
 
 	// dockerfile is also supported casing moby/moby#10858
 	if path.Base(filename) == defaultDockerfileName {
@@ -270,6 +272,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
 	var dtDockerfile []byte
 	var dtDockerignore []byte
 	var dtDockerignoreDefault []byte
+	var dtDockerfilePin []byte
 	eg.Go(func() error {
 		res, err := c.Solve(ctx2, client.SolveRequest{
 			Definition: def.ToPB(),
@@ -311,6 +314,17 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
 		if err == nil {
 			dtDockerignore = dt
 		}
+		dockerfilePinFilename := filename + ".pin"
+		if _, err := ref.StatFile(ctx, client.StatRequest{
+			Path: dockerfilePinFilename,
+		}); err == nil {
+			dtDockerfilePin, err = ref.ReadFile(ctx2, client.ReadRequest{
+				Filename: dockerfilePinFilename,
+			})
+			if err != nil {
+				return err
+			}
+		}
 		return nil
 	})
 	var excludes []string
@@ -427,9 +441,23 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
 					}
 				}()
 
+				var metaResolver llb.ImageMetaResolver = c
+				var pinApplier *pin.Applier
+				if dtDockerfilePin != nil {
+					var pinData pintypes.Pin
+					if err := json.Unmarshal(dtDockerfilePin, &pinData); err != nil {
+						return err
+					}
+					pinApplier = &pin.Applier{
+						Pin:               pinData,
+						ImageMetaResolver: metaResolver,
+					}
+					metaResolver = pinApplier
+				}
+
 				st, img, bi, err := dockerfile2llb.Dockerfile2LLB(ctx, dtDockerfile, dockerfile2llb.ConvertOpt{
 					Target:           opts[keyTarget],
-					MetaResolver:     c,
+					MetaResolver:     metaResolver,
 					BuildArgs:        filter(opts, buildArgPrefix),
 					Labels:           filter(opts, labelPrefix),
 					CacheIDNamespace: opts[keyCacheNSArg],
@@ -455,7 +483,8 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
 						}
 						c.Warn(ctx, defVtx, msg, warnOpts(sourceMap, location, detail, url))
 					},
-					ContextByName: contextByNameFunc(c, tp),
+					ContextByName: contextByNameFunc(c, metaResolver, tp),
+					PinApplier:    pinApplier,
 				})
 
 				if err != nil {
@@ -511,6 +540,17 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
 					return err
 				}
 
+				if len(dtDockerfilePin) != 0 {
+					pinConsumed := &pintypes.Consumed{
+						Sources: pinApplier.Consumed().Sources,
+					}
+					pinConsumedJSON, err := json.Marshal(pinConsumed)
+					if err != nil {
+						return err
+					}
+					res.AddMeta(exptypes.ExporterPinConsumed, pinConsumedJSON)
+				}
+
 				buildinfo, err := json.Marshal(bi)
 				if err != nil {
 					return errors.Wrapf(err, "failed to marshal build info")
@@ -787,7 +827,7 @@ func warnOpts(sm *llb.SourceMap, r *parser.Range, detail [][]byte, url string) c
 	return opts
 }
 
-func contextByNameFunc(c client.Client, p *ocispecs.Platform) func(context.Context, string, string) (*llb.State, *dockerfile2llb.Image, *binfotypes.BuildInfo, error) {
+func contextByNameFunc(c client.Client, metaResolver llb.ImageMetaResolver, p *ocispecs.Platform) func(context.Context, string, string) (*llb.State, *dockerfile2llb.Image, *binfotypes.BuildInfo, error) {
 	return func(ctx context.Context, name, resolveMode string) (*llb.State, *dockerfile2llb.Image, *binfotypes.BuildInfo, error) {
 		named, err := reference.ParseNormalizedNamed(name)
 		if err != nil {
@@ -801,19 +841,19 @@ func contextByNameFunc(c client.Client, p *ocispecs.Platform) func(context.Conte
 		}
 		if p != nil {
 			name := name + "::" + platforms.Format(platforms.Normalize(*p))
-			st, img, bi, err := contextByName(ctx, c, name, p, resolveMode)
+			st, img, bi, err := contextByName(ctx, c, metaResolver, name, p, resolveMode)
 			if err != nil {
 				return nil, nil, nil, err
 			}
 			if st != nil {
 				return st, img, bi, nil
 			}
 		}
-		return contextByName(ctx, c, name, p, resolveMode)
+		return contextByName(ctx, c, metaResolver, name, p, resolveMode)
 	}
 }
 
-func contextByName(ctx context.Context, c client.Client, name string, platform *ocispecs.Platform, resolveMode string) (*llb.State, *dockerfile2llb.Image, *binfotypes.BuildInfo, error) {
+func contextByName(ctx context.Context, c client.Client, metaResolver llb.ImageMetaResolver, name string, platform *ocispecs.Platform, resolveMode string) (*llb.State, *dockerfile2llb.Image, *binfotypes.BuildInfo, error) {
 	opts := c.BuildOpts().Opts
 	v, ok := opts["context:"+name]
 	if !ok {
@@ -833,6 +873,7 @@ func contextByName(ctx context.Context, c client.Client, name string, platform *
 		ref := strings.TrimPrefix(vv[1], "//")
 		imgOpt := []llb.ImageOption{
 			llb.WithCustomName("[context " + name + "] " + ref),
+			llb.WithMetaResolver(metaResolver),
 		}
 		if platform != nil {
 			imgOpt = append(imgOpt, llb.Platform(*platform))

frontend/dockerfile/dockerfile2llb/convert.go

@@ -26,9 +26,11 @@ import (
 	"github.com/moby/buildkit/solver/pb"
 	"github.com/moby/buildkit/util/apicaps"
 	binfotypes "github.com/moby/buildkit/util/buildinfo/types"
+	"github.com/moby/buildkit/util/pin"
 	"github.com/moby/buildkit/util/suggest"
 	"github.com/moby/buildkit/util/system"
 	"github.com/moby/sys/signal"
+	digest "github.com/opencontainers/go-digest"
 	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
@@ -68,6 +70,8 @@ type ConvertOpt struct {
 	Hostname         string
 	Warn             func(short, url string, detail [][]byte, location *parser.Range)
 	ContextByName    func(ctx context.Context, name, resolveMode string) (*llb.State, *Image, *binfotypes.BuildInfo, error)
+	// PinApplier must correspond to MetaResolver when PinApplier is non-nil.
+	PinApplier *pin.Applier
 }
 
 func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, *Image, *binfotypes.BuildInfo, error) {
@@ -136,6 +140,10 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
 		}
 	}
 
+	if opt.PinApplier != nil && opt.MetaResolver != nil && opt.PinApplier != opt.MetaResolver {
+		return nil, nil, nil, fmt.Errorf("opt.PinApplier must correspond to opt.MetaResolver when non-nil")
+	}
+
 	metaResolver := opt.MetaResolver
 	if metaResolver == nil {
 		metaResolver = imagemetaresolver.Default()
@@ -473,6 +481,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
 			cgroupParent:      opt.CgroupParent,
 			llbCaps:           opt.LLBCaps,
 			sourceMap:         opt.SourceMap,
+			pinApplier:        opt.PinApplier,
 		}
 
 		if err = dispatchOnBuildTriggers(d, d.image.Config.OnBuild, opt); err != nil {
@@ -597,6 +606,7 @@ type dispatchOpt struct {
 	cgroupParent      string
 	llbCaps           *apicaps.CapSet
 	sourceMap         *llb.SourceMap
+	pinApplier        *pin.Applier
 }
 
 func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error {
@@ -1019,7 +1029,14 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {
 				}
 			}
 
-			st := llb.HTTP(src, llb.Filename(f), dfCmd(cfg.params))
+			var checksum digest.Digest
+			if cfg.opt.pinApplier != nil {
+				checksum, err = cfg.opt.pinApplier.HTTPChecksum(src)
+				if err != nil {
+					return err
+				}
+			}
+			st := llb.HTTP(src, llb.Filename(f), dfCmd(cfg.params), llb.Checksum(checksum))
 
 			opts := append([]llb.CopyOption{&llb.CopyInfo{
 				Mode:           mode,

frontend/dockerfile/dockerfile_pin_test.go

@@ -0,0 +1,138 @@
+package dockerfile
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"os"
+	"testing"
+
+	"github.com/containerd/continuity/fs/fstest"
+	"github.com/moby/buildkit/client"
+	"github.com/moby/buildkit/exporter/containerimage/exptypes"
+	"github.com/moby/buildkit/frontend/dockerfile/builder"
+	pintypes "github.com/moby/buildkit/util/pin/types"
+	"github.com/moby/buildkit/util/testutil/integration"
+	"github.com/stretchr/testify/require"
+)
+
+var pinTests = integration.TestFuncs(
+	testPin,
+)
+
+func init() {
+	allTests = append(allTests, pinTests...)
+}
+
+func testPin(t *testing.T, sb integration.Sandbox) {
+	f := getFrontend(t, sb)
+	dockerfile := `
+FROM alpine
+ADD https://raw.githubusercontent.com/moby/buildkit/v0.10.1/README.md /README.md
+`
+	type testCase struct {
+		title         string
+		dockerfilePin string
+		valid         bool
+		validateResp  func(*testing.T, *client.SolveResponse)
+	}
+	testCases := []testCase{
+		{
+			title: "Valid",
+			dockerfilePin: `{
+    "sources": [
+      {
+        "type": "docker-image",
+        "ref": "docker.io/library/alpine:latest",
+        "pin": "sha256:4edbd2beb5f78b1014028f4fbb99f3237d9561100b6881aabbf5acce2c4f9454"
+      },
+      {
+        "type": "http",
+        "ref": "https://raw.githubusercontent.com/moby/buildkit/v0.10.1/README.md",
+        "pin": "sha256:6e4b94fc270e708e1068be28bd3551dc6917a4fc5a61293d51bb36e6b75c4b53"
+      }
+    ]
+}
+`,
+			valid: true,
+			validateResp: func(t *testing.T, resp *client.SolveResponse) {
+				require.Contains(t, resp.ExporterResponse, exptypes.ExporterPinConsumed)
+				dtpc, err := base64.StdEncoding.DecodeString(resp.ExporterResponse[exptypes.ExporterPinConsumed])
+				require.NoError(t, err)
+				var pc pintypes.Consumed
+				err = json.Unmarshal(dtpc, &pc)
+				require.NoError(t, err)
+				require.Len(t, pc.Sources, 2)
+				require.Equal(t, "sha256:4edbd2beb5f78b1014028f4fbb99f3237d9561100b6881aabbf5acce2c4f9454", pc.Sources[0].Pin)
+				require.Equal(t, "sha256:6e4b94fc270e708e1068be28bd3551dc6917a4fc5a61293d51bb36e6b75c4b53", pc.Sources[1].Pin)
+			},
+		},
+		{
+			title: "InvalidDockerImagePin",
+			dockerfilePin: `{
+    "sources": [
+      {
+        "type": "docker-image",
+        "ref": "docker.io/library/alpine:latest",
+        "pin": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+      },
+      {
+        "type": "http",
+        "ref": "https://raw.githubusercontent.com/moby/buildkit/v0.10.1/README.md",
+        "pin": "sha256:6e4b94fc270e708e1068be28bd3551dc6917a4fc5a61293d51bb36e6b75c4b53"
+      }
+    ]
+}
+`,
+			valid: false,
+		},
+		{
+			title: "InvalidHTTPPin",
+			dockerfilePin: `{
+    "sources": [
+      {
+        "type": "docker-image",
+        "ref": "docker.io/library/alpine:latest",
+        "pin": "sha256:4edbd2beb5f78b1014028f4fbb99f3237d9561100b6881aabbf5acce2c4f9454"
+      },
+      {
+        "type": "http",
+        "ref": "https://raw.githubusercontent.com/moby/buildkit/v0.10.1/README.md",
+        "pin": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+      }
+    ]
+}
+`,
+			valid: false,
+		},
+	}
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.title, func(t *testing.T) {
+			dir, err := tmpdir(
+				fstest.CreateFile("Dockerfile", []byte(dockerfile), 0600),
+				fstest.CreateFile("Dockerfile.pin", []byte(tc.dockerfilePin), 0600),
+			)
+			require.NoError(t, err)
+			defer os.RemoveAll(dir)
+
+			c, err := client.New(sb.Context(), sb.Address())
+			require.NoError(t, err)
+			defer c.Close()
+
+			res, err := f.Solve(sb.Context(), c, client.SolveOpt{
+				LocalDirs: map[string]string{
+					builder.DefaultLocalNameDockerfile: dir,
+					builder.DefaultLocalNameContext:    dir,
+				},
+			}, nil)
+			if !tc.valid {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+			if tc.validateResp != nil {
+				tc.validateResp(t, res)
+			}
+		})
+	}
+}

solver/llbsolver/solver.go

@@ -322,7 +322,7 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro
 		if strings.HasPrefix(k, "frontend.") {
 			exporterResponse[k] = string(v)
 		}
-		if strings.HasPrefix(k, exptypes.ExporterBuildInfo) {
+		if strings.HasPrefix(k, exptypes.ExporterBuildInfo) || strings.HasPrefix(k, exptypes.ExporterPinConsumed) {
 			exporterResponse[k] = base64.StdEncoding.EncodeToString(v)
 		}
 	}