llbsolver: support pinning sources

Alternative to PR 2816 ("dockerfile: support Dockerfile.pin for pinning sources") This version is implemented on the llbsolver side and agnostic to the LLB frontends. See `solver/llbsolver/vertex.go:loadLLB()`. See `docs/build-repro.md` for the usage. Signed-off-by: Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp>
moby · Aug 3, 2022 · 6704aac · 6704aac
1 parent 6805940
commit 6704aac
Show file tree

Hide file tree

Showing 14 changed files with 1,141 additions and 159 deletions.
diff --git a/api/services/control/control.pb.go b/api/services/control/control.pb.go
diff --git a/api/services/control/control.proto b/api/services/control/control.proto
@@ -62,6 +62,7 @@ message SolveRequest {
 	CacheOptions Cache = 8 [(gogoproto.nullable) = false];
 	repeated string Entitlements = 9 [(gogoproto.customtype) = "github.com/moby/buildkit/util/entitlements.Entitlement" ];
 	map<string, pb.Definition> FrontendInputs = 10;
+	SourcePolicy SourcePolicy = 11;
 }
 
 message CacheOptions {
@@ -163,3 +164,14 @@ message InfoRequest {}
 message InfoResponse {
 	moby.buildkit.v1.types.BuildkitVersion buildkitVersion = 1;
 }
+
+message SourcePolicy {
+	repeated BuildInfoSource Sources = 1;
+}
+
+message BuildInfoSource {
+	string Type = 1;
+	string Ref = 2;
+	string Alias = 3;
+	string Pin = 4;
+}
diff --git a/client/client_test.go b/client/client_test.go
@@ -43,6 +43,7 @@ import (
 	"github.com/moby/buildkit/session/sshforward/sshprovider"
 	"github.com/moby/buildkit/solver/errdefs"
 	"github.com/moby/buildkit/solver/pb"
+	"github.com/moby/buildkit/sourcepolicy"
 	binfotypes "github.com/moby/buildkit/util/buildinfo/types"
 	"github.com/moby/buildkit/util/contentutil"
 	"github.com/moby/buildkit/util/entitlements"
@@ -167,6 +168,7 @@ func TestIntegration(t *testing.T) {
 		testCallInfo,
 		testPullWithLayerLimit,
 		testExportAnnotations,
+		testSourcePolicy,
 	)
 	tests = append(tests, diffOpTestCases()...)
 	integration.Run(t, tests, mirrors)
@@ -6325,3 +6327,83 @@ func fixedWriteCloser(wc io.WriteCloser) func(map[string]string) (io.WriteCloser
 		return wc, nil
 	}
 }
+
+func testSourcePolicy(t *testing.T, sb integration.Sandbox) {
+	requiresLinux(t)
+	c, err := New(sb.Context(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	frontend := func(ctx context.Context, c gateway.Client) (*gateway.Result, error) {
+		st := llb.Image("busybox:1.34.1-uclibc").File(
+			llb.Copy(llb.HTTP("https://raw.githubusercontent.com/moby/buildkit/v0.10.1/README.md"),
+				"README.md", "README.md"))
+		def, err := st.Marshal(sb.Context())
+		if err != nil {
+			return nil, err
+		}
+		return c.Solve(ctx, gateway.SolveRequest{
+			Definition: def.ToPB(),
+		})
+	}
+
+	type testCase struct {
+		srcPol      *sourcepolicy.SourcePolicy
+		expectedErr string
+	}
+	testCases := []testCase{
+		{
+			// Valid
+			srcPol: &sourcepolicy.SourcePolicy{
+				Sources: []sourcepolicy.Source{
+					{
+						Type: "docker-image",
+						Ref:  "docker.io/library/busybox:1.34.1-uclibc",
+						Pin:  "sha256:3614ca5eacf0a3a1bcc361c939202a974b4902b9334ff36eb29ffe9011aaad83",
+					},
+					{
+						Type: "http",
+						Ref:  "https://raw.githubusercontent.com/moby/buildkit/v0.10.1/README.md",
+						Pin:  "sha256:6e4b94fc270e708e1068be28bd3551dc6917a4fc5a61293d51bb36e6b75c4b53",
+					},
+				},
+			},
+			expectedErr: "",
+		},
+		{
+			// Invalid docker-image source
+			srcPol: &sourcepolicy.SourcePolicy{
+				Sources: []sourcepolicy.Source{
+					{
+						Type: "docker-image",
+						Ref:  "docker.io/library/busybox:1.34.1-uclibc",
+						Pin:  "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", // invalid
+					},
+				},
+			},
+			expectedErr: "docker.io/library/busybox:1.34.1-uclibc@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa: not found",
+		},
+		{
+			// Invalid http source
+			srcPol: &sourcepolicy.SourcePolicy{
+				Sources: []sourcepolicy.Source{
+					{
+						Type: "http",
+						Ref:  "https://raw.githubusercontent.com/moby/buildkit/v0.10.1/README.md",
+						Pin:  "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", // invalid
+					},
+				},
+			},
+			expectedErr: "digest mismatch sha256:6e4b94fc270e708e1068be28bd3551dc6917a4fc5a61293d51bb36e6b75c4b53: sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
+		},
+	}
+	for _, tc := range testCases {
+		_, err = c.Build(sb.Context(), SolveOpt{SourcePolicy: tc.srcPol}, "", frontend, nil)
+		if tc.expectedErr == "" {
+			require.NoError(t, err)
+		} else {
+			require.Error(t, err)
+			require.Contains(t, err.Error(), tc.expectedErr)
+		}
+	}
+}
diff --git a/client/llb/llbtest/platform_test.go b/client/llb/llbtest/platform_test.go
@@ -27,7 +27,7 @@ func TestCustomPlatform(t *testing.T) {
 	def, err := s.Marshal(context.TODO())
 	require.NoError(t, err)
 
-	e, err := llbsolver.Load(def.ToPB())
+	e, err := llbsolver.Load(context.TODO(), def.ToPB(), nil)
 	require.NoError(t, err)
 
 	require.Equal(t, depth(e), 5)
@@ -56,7 +56,7 @@ func TestDefaultPlatform(t *testing.T) {
 	def, err := s.Marshal(context.TODO())
 	require.NoError(t, err)
 
-	e, err := llbsolver.Load(def.ToPB())
+	e, err := llbsolver.Load(context.TODO(), def.ToPB(), nil)
 	require.NoError(t, err)
 
 	require.Equal(t, depth(e), 2)
@@ -80,7 +80,7 @@ func TestPlatformOnMarshal(t *testing.T) {
 	def, err := s.Marshal(context.TODO(), llb.Windows)
 	require.NoError(t, err)
 
-	e, err := llbsolver.Load(def.ToPB())
+	e, err := llbsolver.Load(context.TODO(), def.ToPB(), nil)
 	require.NoError(t, err)
 
 	expected := ocispecs.Platform{OS: "windows", Architecture: "amd64"}
@@ -100,7 +100,7 @@ func TestPlatformMixed(t *testing.T) {
 	def, err := s1.Marshal(context.TODO(), llb.LinuxAmd64)
 	require.NoError(t, err)
 
-	e, err := llbsolver.Load(def.ToPB())
+	e, err := llbsolver.Load(context.TODO(), def.ToPB(), nil)
 	require.NoError(t, err)
 
 	require.Equal(t, depth(e), 4)
@@ -129,7 +129,7 @@ func TestFallbackPath(t *testing.T) {
 	// the cap.
 	def, err := llb.Scratch().Run(llb.Shlex("cmd")).Marshal(context.TODO(), llb.LinuxAmd64)
 	require.NoError(t, err)
-	e, err := llbsolver.Load(def.ToPB())
+	e, err := llbsolver.Load(context.TODO(), def.ToPB(), nil)
 	require.NoError(t, err)
 	require.False(t, def.Metadata[e.Vertex.Digest()].Caps[pb.CapExecMetaSetsDefaultPath])
 	_, ok := getenv(e, "PATH")
@@ -141,7 +141,7 @@ func TestFallbackPath(t *testing.T) {
 	require.Error(t, cs.Supports(pb.CapExecMetaSetsDefaultPath))
 	def, err = llb.Scratch().Run(llb.Shlex("cmd")).Marshal(context.TODO(), llb.LinuxAmd64, llb.WithCaps(cs))
 	require.NoError(t, err)
-	e, err = llbsolver.Load(def.ToPB())
+	e, err = llbsolver.Load(context.TODO(), def.ToPB(), nil)
 	require.NoError(t, err)
 	require.False(t, def.Metadata[e.Vertex.Digest()].Caps[pb.CapExecMetaSetsDefaultPath])
 	v, ok := getenv(e, "PATH")
@@ -155,7 +155,7 @@ func TestFallbackPath(t *testing.T) {
 	require.NoError(t, cs.Supports(pb.CapExecMetaSetsDefaultPath))
 	def, err = llb.Scratch().Run(llb.Shlex("cmd")).Marshal(context.TODO(), llb.LinuxAmd64, llb.WithCaps(cs))
 	require.NoError(t, err)
-	e, err = llbsolver.Load(def.ToPB())
+	e, err = llbsolver.Load(context.TODO(), def.ToPB(), nil)
 	require.NoError(t, err)
 	require.True(t, def.Metadata[e.Vertex.Digest()].Caps[pb.CapExecMetaSetsDefaultPath])
 	_, ok = getenv(e, "PATH")
@@ -171,7 +171,7 @@ func TestFallbackPath(t *testing.T) {
 	} {
 		def, err = llb.Scratch().AddEnv("PATH", "foo").Run(llb.Shlex("cmd")).Marshal(context.TODO(), append(cos, llb.LinuxAmd64)...)
 		require.NoError(t, err)
-		e, err = llbsolver.Load(def.ToPB())
+		e, err = llbsolver.Load(context.TODO(), def.ToPB(), nil)
 		require.NoError(t, err)
 		// pb.CapExecMetaSetsDefaultPath setting is irrelevant (and variable).
 		v, ok = getenv(e, "PATH")

diff --git a/client/solve.go b/client/solve.go
@@ -20,6 +20,7 @@ import (
 	"github.com/moby/buildkit/session/filesync"
 	"github.com/moby/buildkit/session/grpchijack"
 	"github.com/moby/buildkit/solver/pb"
+	"github.com/moby/buildkit/sourcepolicy"
 	"github.com/moby/buildkit/util/bklog"
 	"github.com/moby/buildkit/util/entitlements"
 	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
@@ -44,6 +45,7 @@ type SolveOpt struct {
 	AllowedEntitlements   []entitlements.Entitlement
 	SharedSession         *session.Session // TODO: refactor to better session syncing
 	SessionPreInitialized bool             // TODO: refactor to better session syncing
+	SourcePolicy          *sourcepolicy.SourcePolicy
 }
 
 type ExportEntry struct {
@@ -198,6 +200,19 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
 		opt.FrontendAttrs[k] = v
 	}
 
+	var srcPol *controlapi.SourcePolicy
+	if opt.SourcePolicy != nil && len(opt.SourcePolicy.Sources) > 0 {
+		srcPol = &controlapi.SourcePolicy{}
+		for _, f := range opt.SourcePolicy.Sources {
+			srcPol.Sources = append(srcPol.Sources, &controlapi.BuildInfoSource{
+				Type:  string(f.Type),
+				Alias: f.Alias,
+				Ref:   f.Ref,
+				Pin:   f.Pin,
+			})
+		}
+	}
+
 	solveCtx, cancelSolve := context.WithCancel(ctx)
 	var res *SolveResponse
 	eg.Go(func() error {
@@ -239,6 +254,7 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
 			FrontendInputs: frontendInputs,
 			Cache:          cacheOpt.options,
 			Entitlements:   opt.AllowedEntitlements,
+			SourcePolicy:   srcPol,
 		})
 		if err != nil {
 			return errors.Wrap(err, "failed to solve")

diff --git a/cmd/buildctl/build.go b/cmd/buildctl/build.go
@@ -17,6 +17,7 @@ import (
 	"github.com/moby/buildkit/session/auth/authprovider"
 	"github.com/moby/buildkit/session/sshforward/sshprovider"
 	"github.com/moby/buildkit/solver/pb"
+	"github.com/moby/buildkit/sourcepolicy"
 	"github.com/moby/buildkit/util/progress/progresswriter"
 	digest "github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
@@ -88,6 +89,10 @@ var buildCommand = cli.Command{
 			Name:  "metadata-file",
 			Usage: "Output build metadata (e.g., image digest) to a file as JSON",
 		},
+		cli.StringFlag{
+			Name:  "source-policy-file",
+			Usage: "Read source policy file from a JSON file",
+		},
 	},
 }
 
@@ -182,6 +187,19 @@ func buildAction(clicontext *cli.Context) error {
 		return err
 	}
 
+	var srcPol *sourcepolicy.SourcePolicy
+	if srcPolFile := clicontext.String("source-policy-file"); srcPolFile != "" {
+		b, err := os.ReadFile(srcPolFile)
+		if err != nil {
+			return err
+		}
+		var srcPolStruct sourcepolicy.SourcePolicy
+		if err := json.Unmarshal(b, &srcPolStruct); err != nil {
+			return errors.Wrapf(err, "failed to unmarshal source-policy-file %q", srcPolFile)
+		}
+		srcPol = &srcPolStruct
+	}
+
 	eg, ctx := errgroup.WithContext(bccommon.CommandContext(clicontext))
 
 	solveOpt := client.SolveOpt{
@@ -194,6 +212,7 @@ func buildAction(clicontext *cli.Context) error {
 		CacheImports:        cacheImports,
 		Session:             attachable,
 		AllowedEntitlements: allowed,
+		SourcePolicy:        srcPol,
 	}
 
 	solveOpt.FrontendAttrs, err = build.ParseOpt(clicontext.StringSlice("opt"))

diff --git a/control/control.go b/control/control.go
@@ -18,7 +18,9 @@ import (
 	"github.com/moby/buildkit/solver"
 	"github.com/moby/buildkit/solver/llbsolver"
 	"github.com/moby/buildkit/solver/pb"
+	"github.com/moby/buildkit/sourcepolicy"
 	"github.com/moby/buildkit/util/bklog"
+	binfotypes "github.com/moby/buildkit/util/buildinfo/types"
 	"github.com/moby/buildkit/util/imageutil"
 	"github.com/moby/buildkit/util/throttle"
 	"github.com/moby/buildkit/util/tracing/transform"
@@ -306,6 +308,19 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (*
 			Attrs: im.Attrs,
 		})
 	}
+	var srcPol *sourcepolicy.SourcePolicy
+	if req.SourcePolicy != nil {
+		srcPol = &sourcepolicy.SourcePolicy{}
+		for _, f := range req.SourcePolicy.Sources {
+			s := sourcepolicy.Source{
+				Type:  binfotypes.SourceType(f.Type),
+				Ref:   f.Ref,
+				Alias: f.Alias,
+				Pin:   f.Pin,
+			}
+			srcPol.Sources = append(srcPol.Sources, s)
+		}
+	}
 
 	resp, err := c.solver.Solve(ctx, req.Ref, req.Session, frontend.SolveRequest{
 		Frontend:       req.Frontend,
@@ -317,7 +332,7 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (*
 		Exporter:        expi,
 		CacheExporter:   cacheExporter,
 		CacheExportMode: cacheExportMode,
-	}, req.Entitlements)
+	}, req.Entitlements, srcPol)
 	if err != nil {
 		return nil, err
 	}

diff --git a/docs/build-repro.md b/docs/build-repro.md
@@ -127,3 +127,36 @@ jq '.' metadata.json
   "containerimage.digest": "sha256:..."
 }
 ```
+
+### Reproducing the pinned dependencies
+
+<!-- TODO: s/master/v0.11/ after the release -->
+Reproducing the pinned dependencies is supported in the master branch of BuildKit.
+
+e.g.,
+```bash
+buildctl build --frontend dockerfile.v0 --local dockerfile=. --local context=. --source-policy-file Dockerfile.pol
+```
+
+The content of the `Dockerfile.pol` is a subset of the `."containerimage.buildinfo".sources` property of the `metadata.json`:
+```json
+{
+    "sources": [
+      {
+        "type": "docker-image",
+        "ref": "docker.io/library/alpine:latest",
+        "pin": "sha256:4edbd2beb5f78b1014028f4fbb99f3237d9561100b6881aabbf5acce2c4f9454"
+      },
+      {
+        "type": "http",
+        "ref": "https://raw.githubusercontent.com/moby/buildkit/v0.10.1/README.md",
+        "pin": "sha256:6e4b94fc270e708e1068be28bd3551dc6917a4fc5a61293d51bb36e6b75c4b53"
+      }
+    ]
+}
+```
+
+Reproduction is currently supported for the following source types:
+* `docker-image`
+* `http`
+<!-- TODO: git -->
diff --git a/solver/llbsolver/bridge.go b/solver/llbsolver/bridge.go
@@ -20,6 +20,7 @@ import (
 	"github.com/moby/buildkit/solver/errdefs"
 	llberrdefs "github.com/moby/buildkit/solver/llbsolver/errdefs"
 	"github.com/moby/buildkit/solver/pb"
+	"github.com/moby/buildkit/sourcepolicy/sourcepolicyllbmutator"
 	"github.com/moby/buildkit/util/bklog"
 	"github.com/moby/buildkit/util/buildinfo"
 	"github.com/moby/buildkit/util/flightcontrol"
@@ -72,6 +73,11 @@ func (b *llbBridge) loadResult(ctx context.Context, def *pb.Definition, cacheImp
 	if err != nil {
 		return nil, nil, err
 	}
+	srcPol, err := loadSourcePolicy(b.builder)
+	if err != nil {
+		return nil, nil, err
+	}
+	llbMutator := sourcepolicyllbmutator.New(srcPol)
 	var cms []solver.CacheManager
 	for _, im := range cacheImports {
 		cmID, err := cmKey(im)
@@ -111,7 +117,7 @@ func (b *llbBridge) loadResult(ctx context.Context, def *pb.Definition, cacheImp
 	}
 	dpc := &detectPrunedCacheID{}
 
-	edge, err := Load(def, dpc.Load, ValidateEntitlements(ent), WithCacheSources(cms), NormalizeRuntimePlatforms(), WithValidateCaps())
+	edge, err := Load(ctx, def, llbMutator, dpc.Load, ValidateEntitlements(ent), WithCacheSources(cms), NormalizeRuntimePlatforms(), WithValidateCaps())
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "failed to load LLB")
 	}