From 97464964b03b1fb6337f6ccfc8c89f3eb5c14323 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Mon, 11 Apr 2022 00:16:23 +0900 Subject: [PATCH] dockerfile: support Dockerfile.pin for pinning sources When Dockerfile.pin exists in the context, the dockerfile.v0 frontend does: - Pinning the digest of `docker-image` sources (`FROM ...`) - Pinning the digest of `http` sources (`ADD https://...`) - Recording the consumed Dockerfile.pin entries to the exporter response `pin.consumed` The content of Dockerfile.pin is a subset of BuildInfo: ```json { "sources": [ { "type": "docker-image", "ref": "docker.io/library/alpine:latest", "pin": "sha256:4edbd2beb5f78b1014028f4fbb99f3237d9561100b6881aabbf5acce2c4f9454" }, { "type": "http", "ref": "https://raw.githubusercontent.com/moby/buildkit/v0.10.1/README.md", "pin": "sha256:6e4b94fc270e708e1068be28bd3551dc6917a4fc5a61293d51bb36e6b75c4b53" } ] } ``` In the future, Dockerfile should also support `ADD git://...` and pinning its commit hash. (PR 2799) Closes issue 2794 "Dockerfile.pin" was originally proposed as "Dockerfile.sum" in Issue 2794. Signed-off-by: Akihiro Suda --- docs/build-repro.md | 29 ++++ exporter/containerimage/exptypes/types.go | 1 + frontend/dockerfile/builder/build.go | 55 ++++++- frontend/dockerfile/dockerfile2llb/convert.go | 19 ++- frontend/dockerfile/dockerfile_pin_test.go | 138 ++++++++++++++++++ solver/llbsolver/solver.go | 2 +- util/pin/pin.go | 116 +++++++++++++++ util/pin/types/pintypes.go | 17 +++ 8 files changed, 368 insertions(+), 9 deletions(-) create mode 100644 frontend/dockerfile/dockerfile_pin_test.go create mode 100644 util/pin/pin.go create mode 100644 util/pin/types/pintypes.go diff --git a/docs/build-repro.md b/docs/build-repro.md index 4c11bd5755a1d..3f1ab51fd4942 100644 --- a/docs/build-repro.md +++ b/docs/build-repro.md @@ -127,3 +127,32 @@ jq '.' metadata.json "containerimage.digest": "sha256:..." } ``` + +### Reproducible build with `Dockerfile.pin` + +`Dockerfile.pin` was introduced in the `docker/dockerfile:1.5.0` syntax. + +When `Dockerfile.pin` exists in the context, the Dockerfile builder does: +- Pinning the digest of `docker-image` sources (`FROM ...`) +- Pinning the digest of `http` sources (`ADD https://...`) +- Recording the consumed entries to the exporter response `pin.consumed` + +The content of `Dockerfile.pin` is a subset of the build info structure: +```json +{ + "sources": [ + { + "type": "docker-image", + "ref": "docker.io/library/alpine:latest", + "pin": "sha256:4edbd2beb5f78b1014028f4fbb99f3237d9561100b6881aabbf5acce2c4f9454" + }, + { + "type": "http", + "ref": "https://raw.githubusercontent.com/moby/buildkit/v0.10.1/README.md", + "pin": "sha256:6e4b94fc270e708e1068be28bd3551dc6917a4fc5a61293d51bb36e6b75c4b53" + } + ] +} +``` + +In the future, Dockerfile should also support `ADD git://...` and pinning its commit hash. diff --git a/exporter/containerimage/exptypes/types.go b/exporter/containerimage/exptypes/types.go index a18d660a5c4ab..df90e8e85d8c8 100644 --- a/exporter/containerimage/exptypes/types.go +++ b/exporter/containerimage/exptypes/types.go @@ -13,6 +13,7 @@ const ( ExporterInlineCache = "containerimage.inlinecache" ExporterBuildInfo = "containerimage.buildinfo" ExporterPlatformsKey = "refs.platforms" + ExporterPinConsumed = "pin.consumed" // base64-encoded JSON of util/pin/types.Consumed ) type Platforms struct { diff --git a/frontend/dockerfile/builder/build.go b/frontend/dockerfile/builder/build.go index ca16398af0fe8..174f278ebb09c 100644 --- a/frontend/dockerfile/builder/build.go +++ b/frontend/dockerfile/builder/build.go @@ -27,6 +27,8 @@ import ( "github.com/moby/buildkit/solver/errdefs" "github.com/moby/buildkit/solver/pb" binfotypes "github.com/moby/buildkit/util/buildinfo/types" + "github.com/moby/buildkit/util/pin" + pintypes "github.com/moby/buildkit/util/pin/types" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "golang.org/x/sync/errgroup" @@ -163,7 +165,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) { name := "load build definition from " + filename - filenames := []string{filename, filename + ".dockerignore"} + filenames := []string{filename, filename + ".dockerignore", filename + ".pin"} // dockerfile is also supported casing moby/moby#10858 if path.Base(filename) == defaultDockerfileName { @@ -270,6 +272,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) { var dtDockerfile []byte var dtDockerignore []byte var dtDockerignoreDefault []byte + var dtDockerfilePin []byte eg.Go(func() error { res, err := c.Solve(ctx2, client.SolveRequest{ Definition: def.ToPB(), @@ -311,6 +314,17 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) { if err == nil { dtDockerignore = dt } + dockerfilePinFilename := filename + ".pin" + if _, err := ref.StatFile(ctx, client.StatRequest{ + Path: dockerfilePinFilename, + }); err == nil { + dtDockerfilePin, err = ref.ReadFile(ctx2, client.ReadRequest{ + Filename: dockerfilePinFilename, + }) + if err != nil { + return err + } + } return nil }) var excludes []string @@ -427,9 +441,23 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) { } }() + var metaResolver llb.ImageMetaResolver = c + var pinApplier *pin.Applier + if dtDockerfilePin != nil { + var pinData pintypes.Pin + if err := json.Unmarshal(dtDockerfilePin, &pinData); err != nil { + return err + } + pinApplier = &pin.Applier{ + Pin: pinData, + ImageMetaResolver: metaResolver, + } + metaResolver = pinApplier + } + st, img, bi, err := dockerfile2llb.Dockerfile2LLB(ctx, dtDockerfile, dockerfile2llb.ConvertOpt{ Target: opts[keyTarget], - MetaResolver: c, + MetaResolver: metaResolver, BuildArgs: filter(opts, buildArgPrefix), Labels: filter(opts, labelPrefix), CacheIDNamespace: opts[keyCacheNSArg], @@ -455,7 +483,8 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) { } c.Warn(ctx, defVtx, msg, warnOpts(sourceMap, location, detail, url)) }, - ContextByName: contextByNameFunc(c, tp), + ContextByName: contextByNameFunc(c, metaResolver, tp), + PinApplier: pinApplier, }) if err != nil { @@ -511,6 +540,17 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) { return err } + if len(dtDockerfilePin) != 0 { + pinConsumed := &pintypes.Consumed{ + Sources: pinApplier.Consumed().Sources, + } + pinConsumedJSON, err := json.Marshal(pinConsumed) + if err != nil { + return err + } + res.AddMeta(exptypes.ExporterPinConsumed, pinConsumedJSON) + } + buildinfo, err := json.Marshal(bi) if err != nil { return errors.Wrapf(err, "failed to marshal build info") @@ -787,7 +827,7 @@ func warnOpts(sm *llb.SourceMap, r *parser.Range, detail [][]byte, url string) c return opts } -func contextByNameFunc(c client.Client, p *ocispecs.Platform) func(context.Context, string, string) (*llb.State, *dockerfile2llb.Image, *binfotypes.BuildInfo, error) { +func contextByNameFunc(c client.Client, metaResolver llb.ImageMetaResolver, p *ocispecs.Platform) func(context.Context, string, string) (*llb.State, *dockerfile2llb.Image, *binfotypes.BuildInfo, error) { return func(ctx context.Context, name, resolveMode string) (*llb.State, *dockerfile2llb.Image, *binfotypes.BuildInfo, error) { named, err := reference.ParseNormalizedNamed(name) if err != nil { @@ -801,7 +841,7 @@ func contextByNameFunc(c client.Client, p *ocispecs.Platform) func(context.Conte } if p != nil { name := name + "::" + platforms.Format(platforms.Normalize(*p)) - st, img, bi, err := contextByName(ctx, c, name, p, resolveMode) + st, img, bi, err := contextByName(ctx, c, metaResolver, name, p, resolveMode) if err != nil { return nil, nil, nil, err } @@ -809,11 +849,11 @@ func contextByNameFunc(c client.Client, p *ocispecs.Platform) func(context.Conte return st, img, bi, nil } } - return contextByName(ctx, c, name, p, resolveMode) + return contextByName(ctx, c, metaResolver, name, p, resolveMode) } } -func contextByName(ctx context.Context, c client.Client, name string, platform *ocispecs.Platform, resolveMode string) (*llb.State, *dockerfile2llb.Image, *binfotypes.BuildInfo, error) { +func contextByName(ctx context.Context, c client.Client, metaResolver llb.ImageMetaResolver, name string, platform *ocispecs.Platform, resolveMode string) (*llb.State, *dockerfile2llb.Image, *binfotypes.BuildInfo, error) { opts := c.BuildOpts().Opts v, ok := opts["context:"+name] if !ok { @@ -833,6 +873,7 @@ func contextByName(ctx context.Context, c client.Client, name string, platform * ref := strings.TrimPrefix(vv[1], "//") imgOpt := []llb.ImageOption{ llb.WithCustomName("[context " + name + "] " + ref), + llb.WithMetaResolver(metaResolver), } if platform != nil { imgOpt = append(imgOpt, llb.Platform(*platform)) diff --git a/frontend/dockerfile/dockerfile2llb/convert.go b/frontend/dockerfile/dockerfile2llb/convert.go index 2dddd4ea441d3..675c8fc08d96a 100644 --- a/frontend/dockerfile/dockerfile2llb/convert.go +++ b/frontend/dockerfile/dockerfile2llb/convert.go @@ -26,9 +26,11 @@ import ( "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/util/apicaps" binfotypes "github.com/moby/buildkit/util/buildinfo/types" + "github.com/moby/buildkit/util/pin" "github.com/moby/buildkit/util/suggest" "github.com/moby/buildkit/util/system" "github.com/moby/sys/signal" + digest "github.com/opencontainers/go-digest" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "golang.org/x/sync/errgroup" @@ -68,6 +70,8 @@ type ConvertOpt struct { Hostname string Warn func(short, url string, detail [][]byte, location *parser.Range) ContextByName func(ctx context.Context, name, resolveMode string) (*llb.State, *Image, *binfotypes.BuildInfo, error) + // PinApplier must correspond to MetaResolver when PinApplier is non-nil. + PinApplier *pin.Applier } func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, *Image, *binfotypes.BuildInfo, error) { @@ -136,6 +140,10 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, } } + if opt.PinApplier != nil && opt.MetaResolver != nil && opt.PinApplier != opt.MetaResolver { + return nil, nil, nil, fmt.Errorf("opt.PinApplier must correspond to opt.MetaResolver when non-nil") + } + metaResolver := opt.MetaResolver if metaResolver == nil { metaResolver = imagemetaresolver.Default() @@ -473,6 +481,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, cgroupParent: opt.CgroupParent, llbCaps: opt.LLBCaps, sourceMap: opt.SourceMap, + pinApplier: opt.PinApplier, } if err = dispatchOnBuildTriggers(d, d.image.Config.OnBuild, opt); err != nil { @@ -597,6 +606,7 @@ type dispatchOpt struct { cgroupParent string llbCaps *apicaps.CapSet sourceMap *llb.SourceMap + pinApplier *pin.Applier } func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error { @@ -1019,7 +1029,14 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error { } } - st := llb.HTTP(src, llb.Filename(f), dfCmd(cfg.params)) + var checksum digest.Digest + if cfg.opt.pinApplier != nil { + checksum, err = cfg.opt.pinApplier.HTTPChecksum(src) + if err != nil { + return err + } + } + st := llb.HTTP(src, llb.Filename(f), dfCmd(cfg.params), llb.Checksum(checksum)) opts := append([]llb.CopyOption{&llb.CopyInfo{ Mode: mode, diff --git a/frontend/dockerfile/dockerfile_pin_test.go b/frontend/dockerfile/dockerfile_pin_test.go new file mode 100644 index 0000000000000..e485b70b52546 --- /dev/null +++ b/frontend/dockerfile/dockerfile_pin_test.go @@ -0,0 +1,138 @@ +package dockerfile + +import ( + "encoding/base64" + "encoding/json" + "os" + "testing" + + "github.com/containerd/continuity/fs/fstest" + "github.com/moby/buildkit/client" + "github.com/moby/buildkit/exporter/containerimage/exptypes" + "github.com/moby/buildkit/frontend/dockerfile/builder" + pintypes "github.com/moby/buildkit/util/pin/types" + "github.com/moby/buildkit/util/testutil/integration" + "github.com/stretchr/testify/require" +) + +var pinTests = integration.TestFuncs( + testPin, +) + +func init() { + allTests = append(allTests, pinTests...) +} + +func testPin(t *testing.T, sb integration.Sandbox) { + f := getFrontend(t, sb) + dockerfile := ` +FROM alpine +ADD https://raw.githubusercontent.com/moby/buildkit/v0.10.1/README.md /README.md +` + type testCase struct { + title string + dockerfilePin string + valid bool + validateResp func(*testing.T, *client.SolveResponse) + } + testCases := []testCase{ + { + title: "Valid", + dockerfilePin: `{ + "sources": [ + { + "type": "docker-image", + "ref": "docker.io/library/alpine:latest", + "pin": "sha256:4edbd2beb5f78b1014028f4fbb99f3237d9561100b6881aabbf5acce2c4f9454" + }, + { + "type": "http", + "ref": "https://raw.githubusercontent.com/moby/buildkit/v0.10.1/README.md", + "pin": "sha256:6e4b94fc270e708e1068be28bd3551dc6917a4fc5a61293d51bb36e6b75c4b53" + } + ] +} +`, + valid: true, + validateResp: func(t *testing.T, resp *client.SolveResponse) { + require.Contains(t, resp.ExporterResponse, exptypes.ExporterPinConsumed) + dtpc, err := base64.StdEncoding.DecodeString(resp.ExporterResponse[exptypes.ExporterPinConsumed]) + require.NoError(t, err) + var pc pintypes.Consumed + err = json.Unmarshal(dtpc, &pc) + require.NoError(t, err) + require.Len(t, pc.Sources, 2) + require.Equal(t, "sha256:4edbd2beb5f78b1014028f4fbb99f3237d9561100b6881aabbf5acce2c4f9454", pc.Sources[0].Pin) + require.Equal(t, "sha256:6e4b94fc270e708e1068be28bd3551dc6917a4fc5a61293d51bb36e6b75c4b53", pc.Sources[1].Pin) + }, + }, + { + title: "InvalidDockerImagePin", + dockerfilePin: `{ + "sources": [ + { + "type": "docker-image", + "ref": "docker.io/library/alpine:latest", + "pin": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "type": "http", + "ref": "https://raw.githubusercontent.com/moby/buildkit/v0.10.1/README.md", + "pin": "sha256:6e4b94fc270e708e1068be28bd3551dc6917a4fc5a61293d51bb36e6b75c4b53" + } + ] +} +`, + valid: false, + }, + { + title: "InvalidHTTPPin", + dockerfilePin: `{ + "sources": [ + { + "type": "docker-image", + "ref": "docker.io/library/alpine:latest", + "pin": "sha256:4edbd2beb5f78b1014028f4fbb99f3237d9561100b6881aabbf5acce2c4f9454" + }, + { + "type": "http", + "ref": "https://raw.githubusercontent.com/moby/buildkit/v0.10.1/README.md", + "pin": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + } + ] +} +`, + valid: false, + }, + } + for _, tc := range testCases { + tc := tc + t.Run(tc.title, func(t *testing.T) { + dir, err := tmpdir( + fstest.CreateFile("Dockerfile", []byte(dockerfile), 0600), + fstest.CreateFile("Dockerfile.pin", []byte(tc.dockerfilePin), 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir) + + c, err := client.New(sb.Context(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + res, err := f.Solve(sb.Context(), c, client.SolveOpt{ + LocalDirs: map[string]string{ + builder.DefaultLocalNameDockerfile: dir, + builder.DefaultLocalNameContext: dir, + }, + }, nil) + if !tc.valid { + require.Error(t, err) + return + } + require.NoError(t, err) + if tc.validateResp != nil { + tc.validateResp(t, res) + } + }) + } +} diff --git a/solver/llbsolver/solver.go b/solver/llbsolver/solver.go index 19c79ee1d5dd6..871e9e8eef4be 100644 --- a/solver/llbsolver/solver.go +++ b/solver/llbsolver/solver.go @@ -322,7 +322,7 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro if strings.HasPrefix(k, "frontend.") { exporterResponse[k] = string(v) } - if strings.HasPrefix(k, exptypes.ExporterBuildInfo) { + if strings.HasPrefix(k, exptypes.ExporterBuildInfo) || strings.HasPrefix(k, exptypes.ExporterPinConsumed) { exporterResponse[k] = base64.StdEncoding.EncodeToString(v) } } diff --git a/util/pin/pin.go b/util/pin/pin.go new file mode 100644 index 0000000000000..d916f8101b1da --- /dev/null +++ b/util/pin/pin.go @@ -0,0 +1,116 @@ +package pin + +import ( + "context" + "fmt" + "sync" + + "github.com/docker/distribution/reference" + "github.com/moby/buildkit/client/llb" + binfotypes "github.com/moby/buildkit/util/buildinfo/types" + pintypes "github.com/moby/buildkit/util/pin/types" + "github.com/moby/buildkit/util/urlutil" + digest "github.com/opencontainers/go-digest" +) + +// Applier applies the pins. +// Appliers implements the llb.ImageMetaResolver interface. +type Applier struct { + Pin pintypes.Pin + ImageMetaResolver llb.ImageMetaResolver + consumed pintypes.Consumed + consumedMu sync.Mutex +} + +// ResolveImageConfig resolves "docker-image" sources. +// ImageMetaResolver falls back to a.ImageMetaResolver when the ref is not present in a.Pin . +// ResolveImageConfig implements the llb.ImageMetaResolver interface. +func (a *Applier) ResolveImageConfig(ctx context.Context, ref string, opt llb.ResolveImageConfigOpt) (digest.Digest, []byte, error) { + refParsed, err := reference.ParseNormalizedNamed(ref) + if err != nil { + return "", nil, err + } + refQuery := reference.TagNameOnly(refParsed).String() + + var ( + hit *binfotypes.Source + pinnedRef string + ) + for _, f := range a.Pin.Sources { + f := f + if f.Type != binfotypes.SourceTypeDockerImage { + continue + } + if f.Ref != refQuery { + continue + } + fRefParsed, err := reference.ParseNormalizedNamed(f.Ref) + if err != nil { + return "", nil, err + } + fDigest, err := digest.Parse(f.Pin) + if err != nil { + return "", nil, err + } + pinned, err := reference.WithDigest(fRefParsed, fDigest) + if err != nil { + return "", nil, err + } + if hit != nil { + return "", nil, fmt.Errorf("found multiple pin entries for %q", refQuery) + } + hit = &f + pinnedRef = pinned.String() + } + if hit != nil { + a.consumedMu.Lock() + a.consumed.Sources = append(a.consumed.Sources, *hit) + a.consumedMu.Unlock() + } + if pinnedRef != "" { + ref = pinnedRef + } + return a.ImageMetaResolver.ResolveImageConfig(ctx, ref, opt) +} + +// HTTPChecksum returns the checksum if url is present in a.Pin. +// Otherwise returns an empty string without an error. +func (a *Applier) HTTPChecksum(url string) (digest.Digest, error) { + urlQuery := urlutil.RedactCredentials(url) + var ( + hit *binfotypes.Source + d digest.Digest + ) + for _, f := range a.Pin.Sources { + f := f + if f.Type != binfotypes.SourceTypeHTTP { + continue + } + if f.Ref != urlQuery { + continue + } + if hit != nil { + return "", fmt.Errorf("found multiple pin entries for %q", urlQuery) + } + hit = &f + var err error + d, err = digest.Parse(f.Pin) + if err != nil { + return d, err + } + } + if hit != nil { + a.consumedMu.Lock() + a.consumed.Sources = append(a.consumed.Sources, *hit) + a.consumedMu.Unlock() + } + return d, nil +} + +func (a *Applier) Consumed() pintypes.Consumed { + a.consumedMu.Lock() + defer a.consumedMu.Unlock() + return a.consumed +} + +var _ llb.ImageMetaResolver = &Applier{} diff --git a/util/pin/types/pintypes.go b/util/pin/types/pintypes.go new file mode 100644 index 0000000000000..698075a2fbd7c --- /dev/null +++ b/util/pin/types/pintypes.go @@ -0,0 +1,17 @@ +// Package pintypes provides pin types +package pintypes + +import binfotypes "github.com/moby/buildkit/util/buildinfo/types" + +// Source corresponds to buildinfo Source. +type Source = binfotypes.Source + +// Pin provides pinning information to ensure determinism of sources. +type Pin struct { + // Sources do not need to cover all the sources. + // Sources may contain unreferenced sources too. + Sources []Source `json:"sources,omitempty"` +} + +// Consumed defines the consumed pins +type Consumed = Pin