llbsolver: support pinning sources

Alternative to PR 2816 ("dockerfile: support Dockerfile.pin for pinning sources")

This version is implemented on the llbsolver side and agnostic to the LLB frontends.
See `solver/llbsolver/ops/source.go`.

See `docs/build-repro.md` for the usage.

Signed-off-by: Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp>

api/services/control/control.proto

@@ -62,6 +62,7 @@ message SolveRequest {
 	CacheOptions Cache = 8 [(gogoproto.nullable) = false];
 	repeated string Entitlements = 9 [(gogoproto.customtype) = "github.com/moby/buildkit/util/entitlements.Entitlement" ];
 	map<string, pb.Definition> FrontendInputs = 10;
+	SourcePolicy SourcePolicy = 11;
 }
 
 message CacheOptions {
@@ -163,3 +164,14 @@ message InfoRequest {}
 message InfoResponse {
 	moby.buildkit.v1.types.BuildkitVersion buildkitVersion = 1;
 }
+
+message SourcePolicy {
+	repeated BuildInfoSource Sources = 1;
+}
+
+message BuildInfoSource {
+	string Type = 1;
+	string Ref = 2;
+	string Alias = 3;
+	string Pin = 4;
+}

client/client_test.go

@@ -46,6 +46,7 @@ import (
 	binfotypes "github.com/moby/buildkit/util/buildinfo/types"
 	"github.com/moby/buildkit/util/contentutil"
 	"github.com/moby/buildkit/util/entitlements"
+	srcpoltypes "github.com/moby/buildkit/util/sourcepolicy/types"
 	"github.com/moby/buildkit/util/testutil"
 	"github.com/moby/buildkit/util/testutil/echoserver"
 	"github.com/moby/buildkit/util/testutil/httpserver"
@@ -167,6 +168,7 @@ func TestIntegration(t *testing.T) {
 		testCallInfo,
 		testPullWithLayerLimit,
 		testExportAnnotations,
+		testSourcePolicy,
 	)
 	tests = append(tests, diffOpTestCases()...)
 	integration.Run(t, tests, mirrors)
@@ -6325,3 +6327,83 @@ func fixedWriteCloser(wc io.WriteCloser) func(map[string]string) (io.WriteCloser
 		return wc, nil
 	}
 }
+
+func testSourcePolicy(t *testing.T, sb integration.Sandbox) {
+	requiresLinux(t)
+	c, err := New(sb.Context(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	frontend := func(ctx context.Context, c gateway.Client) (*gateway.Result, error) {
+		st := llb.Image("busybox:1.34.1-uclibc").File(
+			llb.Copy(llb.HTTP("https://raw.githubusercontent.com/moby/buildkit/v0.10.1/README.md"),
+				"README.md", "README.md"))
+		def, err := st.Marshal(sb.Context())
+		if err != nil {
+			return nil, err
+		}
+		return c.Solve(ctx, gateway.SolveRequest{
+			Definition: def.ToPB(),
+		})
+	}
+
+	type testCase struct {
+		srcPol      *srcpoltypes.SourcePolicy
+		expectedErr string
+	}
+	testCases := []testCase{
+		{
+			// Valid
+			srcPol: &srcpoltypes.SourcePolicy{
+				Sources: []srcpoltypes.Source{
+					{
+						Type: "docker-image",
+						Ref:  "docker.io/library/busybox:1.34.1-uclibc",
+						Pin:  "sha256:3614ca5eacf0a3a1bcc361c939202a974b4902b9334ff36eb29ffe9011aaad83",
+					},
+					{
+						Type: "http",
+						Ref:  "https://raw.githubusercontent.com/moby/buildkit/v0.10.1/README.md",
+						Pin:  "sha256:6e4b94fc270e708e1068be28bd3551dc6917a4fc5a61293d51bb36e6b75c4b53",
+					},
+				},
+			},
+			expectedErr: "",
+		},
+		{
+			// Invalid docker-image source
+			srcPol: &srcpoltypes.SourcePolicy{
+				Sources: []srcpoltypes.Source{
+					{
+						Type: "docker-image",
+						Ref:  "docker.io/library/busybox:1.34.1-uclibc",
+						Pin:  "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", // invalid
+					},
+				},
+			},
+			expectedErr: "docker.io/library/busybox:1.34.1-uclibc@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa: not found",
+		},
+		{
+			// Invalid http source
+			srcPol: &srcpoltypes.SourcePolicy{
+				Sources: []srcpoltypes.Source{
+					{
+						Type: "http",
+						Ref:  "https://raw.githubusercontent.com/moby/buildkit/v0.10.1/README.md",
+						Pin:  "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", // invalid
+					},
+				},
+			},
+			expectedErr: "digest mismatch sha256:6e4b94fc270e708e1068be28bd3551dc6917a4fc5a61293d51bb36e6b75c4b53: sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
+		},
+	}
+	for _, tc := range testCases {
+		_, err = c.Build(sb.Context(), SolveOpt{SourcePolicy: tc.srcPol}, "", frontend, nil)
+		if tc.expectedErr == "" {
+			require.NoError(t, err)
+		} else {
+			require.Error(t, err)
+			require.Contains(t, err.Error(), tc.expectedErr)
+		}
+	}
+}

client/solve.go

@@ -22,6 +22,7 @@ import (
 	"github.com/moby/buildkit/solver/pb"
 	"github.com/moby/buildkit/util/bklog"
 	"github.com/moby/buildkit/util/entitlements"
+	srcpoltypes "github.com/moby/buildkit/util/sourcepolicy/types"
 	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/tonistiigi/fsutil"
@@ -44,6 +45,7 @@ type SolveOpt struct {
 	AllowedEntitlements   []entitlements.Entitlement
 	SharedSession         *session.Session // TODO: refactor to better session syncing
 	SessionPreInitialized bool             // TODO: refactor to better session syncing
+	SourcePolicy          *srcpoltypes.SourcePolicy
 }
 
 type ExportEntry struct {
@@ -198,6 +200,19 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
 		opt.FrontendAttrs[k] = v
 	}
 
+	var srcPol *controlapi.SourcePolicy
+	if opt.SourcePolicy != nil && len(opt.SourcePolicy.Sources) > 0 {
+		srcPol = &controlapi.SourcePolicy{}
+		for _, f := range opt.SourcePolicy.Sources {
+			srcPol.Sources = append(srcPol.Sources, &controlapi.BuildInfoSource{
+				Type:  string(f.Type),
+				Alias: f.Alias,
+				Ref:   f.Ref,
+				Pin:   f.Pin,
+			})
+		}
+	}
+
 	solveCtx, cancelSolve := context.WithCancel(ctx)
 	var res *SolveResponse
 	eg.Go(func() error {
@@ -239,6 +254,7 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
 			FrontendInputs: frontendInputs,
 			Cache:          cacheOpt.options,
 			Entitlements:   opt.AllowedEntitlements,
+			SourcePolicy:   srcPol,
 		})
 		if err != nil {
 			return errors.Wrap(err, "failed to solve")

cmd/buildctl/build.go

@@ -18,6 +18,7 @@ import (
 	"github.com/moby/buildkit/session/sshforward/sshprovider"
 	"github.com/moby/buildkit/solver/pb"
 	"github.com/moby/buildkit/util/progress/progresswriter"
+	srcpoltypes "github.com/moby/buildkit/util/sourcepolicy/types"
 	digest "github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -88,6 +89,10 @@ var buildCommand = cli.Command{
 			Name:  "metadata-file",
 			Usage: "Output build metadata (e.g., image digest) to a file as JSON",
 		},
+		cli.StringFlag{
+			Name:  "source-policy-file",
+			Usage: "Read source policy file from a JSON file",
+		},
 	},
 }
 
@@ -182,6 +187,19 @@ func buildAction(clicontext *cli.Context) error {
 		return err
 	}
 
+	var srcPol *srcpoltypes.SourcePolicy
+	if srcPolFile := clicontext.String("source-policy-file"); srcPolFile != "" {
+		b, err := os.ReadFile(srcPolFile)
+		if err != nil {
+			return err
+		}
+		var srcPolStruct srcpoltypes.SourcePolicy
+		if err := json.Unmarshal(b, &srcPolStruct); err != nil {
+			return errors.Wrapf(err, "failed to unmarshal source-policy-file %q", srcPolFile)
+		}
+		srcPol = &srcPolStruct
+	}
+
 	eg, ctx := errgroup.WithContext(bccommon.CommandContext(clicontext))
 
 	solveOpt := client.SolveOpt{
@@ -194,6 +212,7 @@ func buildAction(clicontext *cli.Context) error {
 		CacheImports:        cacheImports,
 		Session:             attachable,
 		AllowedEntitlements: allowed,
+		SourcePolicy:        srcPol,
 	}
 
 	solveOpt.FrontendAttrs, err = build.ParseOpt(clicontext.StringSlice("opt"))

control/control.go

@@ -19,7 +19,9 @@ import (
 	"github.com/moby/buildkit/solver/llbsolver"
 	"github.com/moby/buildkit/solver/pb"
 	"github.com/moby/buildkit/util/bklog"
+	binfotypes "github.com/moby/buildkit/util/buildinfo/types"
 	"github.com/moby/buildkit/util/imageutil"
+	srcpoltypes "github.com/moby/buildkit/util/sourcepolicy/types"
 	"github.com/moby/buildkit/util/throttle"
 	"github.com/moby/buildkit/util/tracing/transform"
 	"github.com/moby/buildkit/version"
@@ -306,6 +308,19 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (*
 			Attrs: im.Attrs,
 		})
 	}
+	var srcPol *srcpoltypes.SourcePolicy
+	if req.SourcePolicy != nil {
+		srcPol = &srcpoltypes.SourcePolicy{}
+		for _, f := range req.SourcePolicy.Sources {
+			s := srcpoltypes.Source{
+				Type:  binfotypes.SourceType(f.Type),
+				Ref:   f.Ref,
+				Alias: f.Alias,
+				Pin:   f.Pin,
+			}
+			srcPol.Sources = append(srcPol.Sources, s)
+		}
+	}
 
 	resp, err := c.solver.Solve(ctx, req.Ref, req.Session, frontend.SolveRequest{
 		Frontend:       req.Frontend,
@@ -317,7 +332,7 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (*
 		Exporter:        expi,
 		CacheExporter:   cacheExporter,
 		CacheExportMode: cacheExportMode,
-	}, req.Entitlements)
+	}, req.Entitlements, srcPol)
 	if err != nil {
 		return nil, err
 	}

docs/build-repro.md

@@ -127,3 +127,36 @@ jq '.' metadata.json
   "containerimage.digest": "sha256:..."
 }
 ```
+
+### Reproducing the pinned dependencies
+
+<!-- TODO: s/master/v0.11/ after the release -->
+Reproducing the pinned dependencies is supported in the master branch of BuildKit.
+
+e.g.,
+```bash
+buildctl build --frontend dockerfile.v0 --local dockerfile=. --local context=. --source-policy-file Dockerfile.pol
+```
+
+The content of the `Dockerfile.pol` is a subset of the `."containerimage.buildinfo".sources` property of the `metadata.json`:
+```json
+{
+    "sources": [
+      {
+        "type": "docker-image",
+        "ref": "docker.io/library/alpine:latest",
+        "pin": "sha256:4edbd2beb5f78b1014028f4fbb99f3237d9561100b6881aabbf5acce2c4f9454"
+      },
+      {
+        "type": "http",
+        "ref": "https://raw.githubusercontent.com/moby/buildkit/v0.10.1/README.md",
+        "pin": "sha256:6e4b94fc270e708e1068be28bd3551dc6917a4fc5a61293d51bb36e6b75c4b53"
+      }
+    ]
+}
+```
+
+Reproduction is currently supported for the following source types:
+* `docker-image`
+* `http`
+<!-- TODO: git -->

solver/llbsolver/bridge.go

@@ -24,6 +24,7 @@ import (
 	"github.com/moby/buildkit/util/buildinfo"
 	"github.com/moby/buildkit/util/flightcontrol"
 	"github.com/moby/buildkit/util/progress"
+	"github.com/moby/buildkit/util/sourcepolicy/pinner"
 	"github.com/moby/buildkit/worker"
 	digest "github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
@@ -72,6 +73,11 @@ func (b *llbBridge) loadResult(ctx context.Context, def *pb.Definition, cacheImp
 	if err != nil {
 		return nil, nil, err
 	}
+	srcPol, err := loadSourcePolicy(b.builder)
+	if err != nil {
+		return nil, nil, err
+	}
+	pinner := pinner.New(srcPol)
 	var cms []solver.CacheManager
 	for _, im := range cacheImports {
 		cmID, err := cmKey(im)
@@ -111,7 +117,7 @@ func (b *llbBridge) loadResult(ctx context.Context, def *pb.Definition, cacheImp
 	}
 	dpc := &detectPrunedCacheID{}
 
-	edge, err := Load(def, dpc.Load, ValidateEntitlements(ent), WithCacheSources(cms), NormalizeRuntimePlatforms(), WithValidateCaps())
+	edge, err := Load(def, dpc.Load, ValidateEntitlements(ent), WithCacheSources(cms), NormalizeRuntimePlatforms(), WithValidateCaps(), WithPinner(pinner))
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "failed to load LLB")
 	}

solver/llbsolver/ops/source.go

@@ -54,6 +54,11 @@ func (s *sourceOp) instance(ctx context.Context) (source.SourceInstance, error)
 	if err != nil {
 		return nil, err
 	}
+	if pinFunc := s.vtx.Options().PinFunc; pinFunc != nil {
+		if err = pinFunc(ctx, id); err != nil {
+			return nil, err
+		}
+	}
 	src, err := s.sm.Resolve(ctx, id, s.sessM, s.vtx)
 	if err != nil {
 		return nil, err

solver/llbsolver/solver.go

@@ -23,13 +23,17 @@ import (
 	"github.com/moby/buildkit/util/compression"
 	"github.com/moby/buildkit/util/entitlements"
 	"github.com/moby/buildkit/util/progress"
+	srcpoltypes "github.com/moby/buildkit/util/sourcepolicy/types"
 	"github.com/moby/buildkit/worker"
 	digest "github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
 )
 
-const keyEntitlements = "llb.entitlements"
+const (
+	keyEntitlements = "llb.entitlements"
+	keySourcePolicy = "llb.sourcepolicy"
+)
 
 type ExporterRequest struct {
 	Exporter        exporter.ExporterInstance
@@ -104,7 +108,7 @@ func (s *Solver) Bridge(b solver.Builder) frontend.FrontendLLBBridge {
 	}
 }
 
-func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req frontend.SolveRequest, exp ExporterRequest, ent []entitlements.Entitlement) (*client.SolveResponse, error) {
+func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req frontend.SolveRequest, exp ExporterRequest, ent []entitlements.Entitlement, srcPol *srcpoltypes.SourcePolicy) (*client.SolveResponse, error) {
 	j, err := s.solver.NewJob(id)
 	if err != nil {
 		return nil, err
@@ -118,6 +122,10 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro
 	}
 	j.SetValue(keyEntitlements, set)
 
+	if srcPol != nil {
+		j.SetValue(keySourcePolicy, *srcPol)
+	}
+
 	j.SessionID = sessionID
 
 	var res *frontend.Result
@@ -508,3 +516,28 @@ func loadEntitlements(b solver.Builder) (entitlements.Set, error) {
 	}
 	return ent, nil
 }
+
+func loadSourcePolicy(b solver.Builder) (*srcpoltypes.SourcePolicy, error) {
+	set := make(map[srcpoltypes.Source]struct{}, 0)
+	err := b.EachValue(context.TODO(), keySourcePolicy, func(v interface{}) error {
+		x, ok := v.(srcpoltypes.SourcePolicy)
+		if !ok {
+			return errors.Errorf("invalid source policy %T", v)
+		}
+		for _, f := range x.Sources {
+			set[f] = struct{}{}
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	var srcPol *srcpoltypes.SourcePolicy
+	if len(set) > 0 {
+		srcPol = &srcpoltypes.SourcePolicy{}
+		for k := range set {
+			srcPol.Sources = append(srcPol.Sources, k)
+		}
+	}
+	return srcPol, nil
+}