From fd572d6e869fab2584fe9ee1f8a33a0a15c568cf Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Mon, 19 Nov 2018 22:43:21 +0000 Subject: [PATCH 1/4] builder: deprecate prune filter `unused-for` in favor of `until` This is to keep the UX consistent. `unused-for` is still accepted and a synonym. Signed-off-by: Tibor Vass (cherry picked from commit 369da264bac15769ae944cc880e66abaf158612b) Upstream-commit: 0b2d88d328ca88c8732dc11c72873b53be3bd2f8 Component: engine --- components/engine/api/swagger.yaml | 2 +- .../engine/builder/builder-next/builder.go | 48 ++++++++++++++----- 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/components/engine/api/swagger.yaml b/components/engine/api/swagger.yaml index f58a64f29ea..dfa0359fd94 100644 --- a/components/engine/api/swagger.yaml +++ b/components/engine/api/swagger.yaml @@ -6402,7 +6402,7 @@ paths: type: "string" description: | A JSON encoded value of the filters (a `map[string][]string`) to process on the list of build cache objects. Available filters: - - `unused-for=`: duration relative to daemon's time, during which build cache was not used, in Go's duration format (e.g., '24h') + - `until=`: duration relative to daemon's time, during which build cache was not used, in Go's duration format (e.g., '24h') - `id=` - `parent=` - `type=` diff --git a/components/engine/builder/builder-next/builder.go b/components/engine/builder/builder-next/builder.go index 1e3c7bda2fb..d8a526e22d3 100644 --- a/components/engine/builder/builder-next/builder.go +++ b/components/engine/builder/builder-next/builder.go @@ -2,6 +2,7 @@ package buildkit import ( "context" + "fmt" "io" "net" "strings" @@ -32,7 +33,21 @@ import ( grpcmetadata "google.golang.org/grpc/metadata" ) -var errMultipleFilterValues = errors.New("filters expect only one value") +type errMultipleFilterValues struct{} + +func (errMultipleFilterValues) Error() string { return "filters expect only one value" } + +func (errMultipleFilterValues) InvalidParameter() {} + +type errConflictFilter struct { + a, b string +} + +func (e errConflictFilter) Error() string { + return fmt.Sprintf("conflicting filters: %q and %q", e.a, e.b) +} + +func (errConflictFilter) InvalidParameter() {} var cacheFields = map[string]bool{ "id": true, @@ -130,6 +145,7 @@ func (b *Builder) Prune(ctx context.Context, opts types.BuildCachePruneOptions) validFilters := make(map[string]bool, 1+len(cacheFields)) validFilters["unused-for"] = true + validFilters["until"] = true for k, v := range cacheFields { validFilters[k] = v } @@ -504,6 +520,7 @@ func toBuildkitExtraHosts(inp []string) (string, error) { hosts := make([]string, 0, len(inp)) for _, h := range inp { parts := strings.Split(h, ":") + if len(parts) != 2 || parts[0] == "" || net.ParseIP(parts[1]) == nil { return "", errors.Errorf("invalid host %s", h) } @@ -513,21 +530,30 @@ func toBuildkitExtraHosts(inp []string) (string, error) { } func toBuildkitPruneInfo(opts types.BuildCachePruneOptions) (client.PruneInfo, error) { - var unusedFor time.Duration - unusedForValues := opts.Filters.Get("unused-for") + var until time.Duration + untilValues := opts.Filters.Get("until") // canonical + unusedForValues := opts.Filters.Get("unused-for") // deprecated synonym for "until" filter - switch len(unusedForValues) { - case 0: + if len(untilValues) > 0 && len(unusedForValues) > 0 { + return client.PruneInfo{}, errConflictFilter{"until", "unused-for"} + } + filterKey := "until" + if len(unusedForValues) > 0 { + filterKey = "unused-for" + } + untilValues = append(untilValues, unusedForValues...) + switch len(untilValues) { + case 0: + // nothing to do case 1: var err error - unusedFor, err = time.ParseDuration(unusedForValues[0]) + until, err = time.ParseDuration(untilValues[0]) if err != nil { - return client.PruneInfo{}, errors.Wrap(err, "unused-for filter expects a duration (e.g., '24h')") + return client.PruneInfo{}, errors.Wrapf(err, "%q filter expects a duration (e.g., '24h')", filterKey) } - default: - return client.PruneInfo{}, errMultipleFilterValues + return client.PruneInfo{}, errMultipleFilterValues{} } bkFilter := make([]string, 0, opts.Filters.Len()) @@ -544,13 +570,13 @@ func toBuildkitPruneInfo(opts types.BuildCachePruneOptions) (client.PruneInfo, e bkFilter = append(bkFilter, cacheField+"=="+values[0]) } default: - return client.PruneInfo{}, errMultipleFilterValues + return client.PruneInfo{}, errMultipleFilterValues{} } } } return client.PruneInfo{ All: opts.All, - KeepDuration: unusedFor, + KeepDuration: until, KeepBytes: opts.KeepStorage, Filter: []string{strings.Join(bkFilter, ",")}, }, nil From 079995778e769f500badf6faa7f39407f7aed97d Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Tue, 20 Nov 2018 23:31:50 +0000 Subject: [PATCH 2/4] builder: ignore `label` and `label!` prune filters Signed-off-by: Tibor Vass (cherry picked from commit 62923f29f514fa621e7c31011c11c610cfe33ecd) Upstream-commit: 34867646af1d1344b1f0877bb3a00a51f7700245 Component: engine --- components/engine/builder/builder-next/builder.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/engine/builder/builder-next/builder.go b/components/engine/builder/builder-next/builder.go index d8a526e22d3..463b45aba4a 100644 --- a/components/engine/builder/builder-next/builder.go +++ b/components/engine/builder/builder-next/builder.go @@ -146,6 +146,8 @@ func (b *Builder) Prune(ctx context.Context, opts types.BuildCachePruneOptions) validFilters := make(map[string]bool, 1+len(cacheFields)) validFilters["unused-for"] = true validFilters["until"] = true + validFilters["label"] = true // TODO(tiborvass): handle label + validFilters["label!"] = true // TODO(tiborvass): handle label! for k, v := range cacheFields { validFilters[k] = v } From 54654abf3a00213382f1be1d430e38a4458e4ab3 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Mon, 19 Nov 2018 17:40:08 -0800 Subject: [PATCH 3/4] builder: avoid unset credentials in containerd Signed-off-by: Tonis Tiigi (cherry picked from commit effb2bd9d23cc05305c4772338b9eb39423e92c6) Upstream-commit: 0d17f4099496ba3de583273eeebefa3cce4694ca Component: engine --- .../builder/builder-next/adapters/containerimage/pull.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/components/engine/builder/builder-next/adapters/containerimage/pull.go b/components/engine/builder/builder-next/adapters/containerimage/pull.go index f6e55f4b6b3..b1e44d95e9e 100644 --- a/components/engine/builder/builder-next/adapters/containerimage/pull.go +++ b/components/engine/builder/builder-next/adapters/containerimage/pull.go @@ -88,7 +88,10 @@ func (is *imageSource) getResolver(ctx context.Context, rfn resolver.ResolveOpti func (is *imageSource) getCredentialsFromSession(ctx context.Context) func(string) (string, string, error) { id := session.FromContext(ctx) if id == "" { - return nil + // can be removed after containerd/containerd#2812 + return func(string) (string, string, error) { + return "", "", nil + } } return func(host string) (string, string, error) { timeoutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) From 71bb7e098b3232fb4084d14aba1c876afae3c30e Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Wed, 21 Nov 2018 11:02:56 -0800 Subject: [PATCH 4/4] vendor: update buildkit to v0.3.3 Signed-off-by: Tonis Tiigi (cherry picked from commit 6204eb064561ba03ba9143d2b29d86836b2093d9) Upstream-commit: 850fff5fc7f033e76ef0eec04eb98384ddf2065d Component: engine --- components/engine/vendor.conf | 4 +- .../vendor/github.com/moby/buildkit/README.md | 28 ++++- .../buildkit/cache/contenthash/checksum.go | 2 +- .../moby/buildkit/client/llb/source.go | 25 +--- .../moby/buildkit/client/llb/state.go | 27 +++++ .../frontend/dockerfile/builder/build.go | 4 +- .../dockerfile/dockerfile2llb/convert.go | 22 +++- .../dockerfile2llb/convert_norunmount.go | 2 +- .../dockerfile2llb/convert_runmount.go | 2 +- .../dockerfile2llb/convert_secrets.go | 2 +- .../dockerfile/dockerfile2llb/convert_ssh.go | 2 +- .../instructions/commands_nosecrets.go | 2 +- .../dockerfile/instructions/commands_nossh.go | 2 +- .../instructions/commands_runmount.go | 2 +- .../instructions/commands_secrets.go | 2 +- .../dockerfile/instructions/commands_ssh.go | 2 +- .../buildkit/frontend/dockerfile/shell/lex.go | 72 +++++++---- .../moby/buildkit/session/session.go | 4 +- .../github.com/moby/buildkit/solver/jobs.go | 2 + .../moby/buildkit/util/contentutil/refs.go | 1 + .../util/rootless/specconv/specconv_linux.go | 113 ++++-------------- .../github.com/moby/buildkit/vendor.conf | 16 ++- .../github.com/tonistiigi/fsutil/walker.go | 14 ++- 23 files changed, 184 insertions(+), 168 deletions(-) diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index b822620bd39..b42132532e6 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -26,8 +26,8 @@ github.com/imdario/mergo v0.3.6 golang.org/x/sync 1d60e4601c6fd243af51cc01ddf169918a5407ca # buildkit -github.com/moby/buildkit c7bb575343df0cbfeab8b5b28149630b8153fcc6 -github.com/tonistiigi/fsutil f567071bed2416e4d87d260d3162722651182317 +github.com/moby/buildkit 8cf9bec86a7f11fe6591804aee152c8e8a7a8a0d # v0.3.3 +github.com/tonistiigi/fsutil 2862f6bc5ac9b97124e552a5c108230b38a1b0ca github.com/grpc-ecosystem/grpc-opentracing 8e809c8a86450a29b90dcc9efbf062d0fe6d9746 github.com/opentracing/opentracing-go 1361b9cd60be79c4c3a7fa9841b3c132e40066a7 github.com/google/shlex 6f45313302b9c56850fc17f99e40caebce98c716 diff --git a/components/engine/vendor/github.com/moby/buildkit/README.md b/components/engine/vendor/github.com/moby/buildkit/README.md index ae8c566f211..649835a9ebd 100644 --- a/components/engine/vendor/github.com/moby/buildkit/README.md +++ b/components/engine/vendor/github.com/moby/buildkit/README.md @@ -27,9 +27,11 @@ Read the proposal from https://github.com/moby/moby/issues/32925 Introductory blog post https://blog.mobyproject.org/introducing-buildkit-17e056cc5317 +:information_source: If you are visiting this repo for the usage of experimental Dockerfile features like `RUN --mount=type=(bind|cache|tmpfs|secret|ssh)`, please refer to [`frontend/dockerfile/docs/experimental.md`](frontend/dockerfile/docs/experimental.md). + ### Used by -[Moby](https://github.com/moby/moby/pull/37151) +[Moby & Docker](https://github.com/moby/moby/pull/37151) [img](https://github.com/genuinetools/img) @@ -37,6 +39,12 @@ Introductory blog post https://blog.mobyproject.org/introducing-buildkit-17e056c [container build interface](https://github.com/containerbuilding/cbi) +[Knative Build Templates](https://github.com/knative/build-templates) + +[boss](https://github.com/crosbymichael/boss) + +[Rio](https://github.com/rancher/rio) (on roadmap) + ### Quick start Dependencies: @@ -79,6 +87,7 @@ See [`solver/pb/ops.proto`](./solver/pb/ops.proto) for the format definition. Currently, following high-level languages has been implemented for LLB: - Dockerfile (See [Exploring Dockerfiles](#exploring-dockerfiles)) +- [Buildpacks](https://github.com/tonistiigi/buildkit-pack) - (open a PR to add your own language) For understanding the basics of LLB, `examples/buildkit*` directory contains scripts that define how to build different configurations of BuildKit itself and its dependencies using the `client` package. Running one of these scripts generates a protobuf definition of a build graph. Note that the script itself does not execute any steps of the build. @@ -145,6 +154,10 @@ buildctl build --frontend=gateway.v0 --frontend-opt=source=tonistiigi/dockerfile buildctl build --frontend gateway.v0 --frontend-opt=source=tonistiigi/dockerfile --frontend-opt=context=git://github.com/moby/moby --frontend-opt build-arg:APT_MIRROR=cdn-fastly.deb.debian.org ```` +##### Building a Dockerfile with experimental features like `RUN --mount=type=(bind|cache|tmpfs|secret|ssh)` + +See [`frontend/dockerfile/docs/experimental.md`](frontend/dockerfile/docs/experimental.md). + ### Exporters By default, the build result and intermediate cache will only remain internally in BuildKit. Exporter needs to be specified to retrieve the result. @@ -207,15 +220,22 @@ buildctl debug workers -v BuildKit can also be used by running the `buildkitd` daemon inside a Docker container and accessing it remotely. The client tool `buildctl` is also available for Mac and Windows. +We provide `buildkitd` container images as [`moby/buildkit`](https://hub.docker.com/r/moby/buildkit/tags/): + +* `moby/buildkit:latest`: built from the latest regular [release](https://github.com/moby/buildkit/releases) +* `moby/buildkit:rootless`: same as `latest` but runs as an unprivileged user, see [`docs/rootless.md`](docs/rootless.md) +* `moby/buildkit:master`: built from the master branch +* `moby/buildkit:master-rootless`: same as master but runs as an unprivileged user, see [`docs/rootless.md`](docs/rootless.md) + To run daemon in a container: ``` -docker run -d --privileged -p 1234:1234 tonistiigi/buildkit --addr tcp://0.0.0.0:1234 +docker run -d --privileged -p 1234:1234 moby/buildkit:latest --addr tcp://0.0.0.0:1234 export BUILDKIT_HOST=tcp://0.0.0.0:1234 buildctl build --help ``` -The `tonistiigi/buildkit` image can be built locally using the Dockerfile in `./hack/dockerfiles/test.Dockerfile`. +The images can be also built locally using `./hack/dockerfiles/test.Dockerfile` (or `./hack/dockerfiles/test.buildkit.Dockerfile` if you already have BuildKit). ### Opentracing support @@ -232,7 +252,7 @@ export JAEGER_TRACE=0.0.0.0:6831 ### Supported runc version -During development, BuildKit is tested with the version of runc that is being used by the containerd repository. Please refer to [runc.md](https://github.com/containerd/containerd/blob/v1.1.3/RUNC.md) for more information. +During development, BuildKit is tested with the version of runc that is being used by the containerd repository. Please refer to [runc.md](https://github.com/containerd/containerd/blob/v1.2.0-rc.1/RUNC.md) for more information. ### Running BuildKit without root privileges diff --git a/components/engine/vendor/github.com/moby/buildkit/cache/contenthash/checksum.go b/components/engine/vendor/github.com/moby/buildkit/cache/contenthash/checksum.go index 35199e02f98..8cb03359be4 100644 --- a/components/engine/vendor/github.com/moby/buildkit/cache/contenthash/checksum.go +++ b/components/engine/vendor/github.com/moby/buildkit/cache/contenthash/checksum.go @@ -628,7 +628,7 @@ func getFollowLinksWalk(root *iradix.Node, k []byte, linksWalked *int) ([]byte, if dirPath == "." || dirPath == "/" { dirPath = "" } - link := parent.Linkname + link := path.Clean(parent.Linkname) if !path.IsAbs(link) { link = path.Join("/", path.Join(path.Dir(dirPath), link)) } diff --git a/components/engine/vendor/github.com/moby/buildkit/client/llb/source.go b/components/engine/vendor/github.com/moby/buildkit/client/llb/source.go index ef03b19f1b2..52d40be8b74 100644 --- a/components/engine/vendor/github.com/moby/buildkit/client/llb/source.go +++ b/components/engine/vendor/github.com/moby/buildkit/client/llb/source.go @@ -126,30 +126,11 @@ func Image(ref string, opts ...ImageOption) State { if err != nil { src.err = err } else { - var img struct { - Config struct { - Env []string `json:"Env,omitempty"` - WorkingDir string `json:"WorkingDir,omitempty"` - User string `json:"User,omitempty"` - } `json:"config,omitempty"` - } - if err := json.Unmarshal(dt, &img); err != nil { - src.err = err - } else { - st := NewState(src.Output()) - for _, env := range img.Config.Env { - parts := strings.SplitN(env, "=", 2) - if len(parts[0]) > 0 { - var v string - if len(parts) > 1 { - v = parts[1] - } - st = st.AddEnv(parts[0], v) - } - } - st = st.Dir(img.Config.WorkingDir) + st, err := NewState(src.Output()).WithImageConfig(dt) + if err == nil { return st } + src.err = err } } return NewState(src.Output()) diff --git a/components/engine/vendor/github.com/moby/buildkit/client/llb/state.go b/components/engine/vendor/github.com/moby/buildkit/client/llb/state.go index 15639fd3eec..24e3949bf8f 100644 --- a/components/engine/vendor/github.com/moby/buildkit/client/llb/state.go +++ b/components/engine/vendor/github.com/moby/buildkit/client/llb/state.go @@ -2,8 +2,10 @@ package llb import ( "context" + "encoding/json" "fmt" "net" + "strings" "github.com/containerd/containerd/platforms" "github.com/moby/buildkit/identity" @@ -171,6 +173,31 @@ func (s State) WithOutput(o Output) State { return s } +func (s State) WithImageConfig(c []byte) (State, error) { + var img struct { + Config struct { + Env []string `json:"Env,omitempty"` + WorkingDir string `json:"WorkingDir,omitempty"` + User string `json:"User,omitempty"` + } `json:"config,omitempty"` + } + if err := json.Unmarshal(c, &img); err != nil { + return State{}, err + } + for _, env := range img.Config.Env { + parts := strings.SplitN(env, "=", 2) + if len(parts[0]) > 0 { + var v string + if len(parts) > 1 { + v = parts[1] + } + s = s.AddEnv(parts[0], v) + } + } + s = s.Dir(img.Config.WorkingDir) + return s, nil +} + func (s State) Run(ro ...RunOption) ExecState { ei := &ExecInfo{State: s} if p := s.GetPlatform(); p != nil { diff --git a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/builder/build.go b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/builder/build.go index 7e082df0824..e5df49e2e0c 100644 --- a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/builder/build.go +++ b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/builder/build.go @@ -99,7 +99,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) { name := "load build definition from " + filename src := llb.Local(LocalNameDockerfile, - llb.IncludePatterns([]string{filename}), + llb.FollowPaths([]string{filename}), llb.SessionID(c.BuildOpts().SessionID), llb.SharedKeyHint(defaultDockerfileName), dockerfile2llb.WithInternalName(name), @@ -189,7 +189,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) { if dockerignoreState == nil { st := llb.Local(LocalNameContext, llb.SessionID(c.BuildOpts().SessionID), - llb.IncludePatterns([]string{dockerignoreFilename}), + llb.FollowPaths([]string{dockerignoreFilename}), llb.SharedKeyHint(dockerignoreFilename), dockerfile2llb.WithInternalName("load "+dockerignoreFilename), ) diff --git a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert.go b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert.go index 5342046d717..c85cfe2cb67 100644 --- a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert.go +++ b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert.go @@ -35,7 +35,7 @@ const ( localNameContext = "context" historyComment = "buildkit.dockerfile.v0" - DefaultCopyImage = "tonistiigi/copy:v0.1.7@sha256:9aab7d9ab369c6daf4831bf0653f7592110ab4b7e8a33fee2b9dca546e9d3089" + DefaultCopyImage = "docker/dockerfile-copy:v0.1.9@sha256:e8f159d3f00786604b93c675ee2783f8dc194bb565e61ca5788f6a6e9d304061" ) type ConvertOpt struct { @@ -327,6 +327,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, targetPlatform: platformOpt.targetPlatform, extraHosts: opt.ExtraHosts, copyImage: opt.OverrideCopyImage, + llbCaps: opt.LLBCaps, } if opt.copyImage == "" { opt.copyImage = DefaultCopyImage @@ -441,6 +442,7 @@ type dispatchOpt struct { buildPlatforms []specs.Platform extraHosts []llb.HostIP copyImage string + llbCaps *apicaps.CapSet } func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error { @@ -467,7 +469,9 @@ func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error { err = dispatchCopy(d, c.SourcesAndDest, opt.buildContext, true, c, "", opt) if err == nil { for _, src := range c.Sources() { - d.ctxPaths[path.Join("/", filepath.ToSlash(src))] = struct{}{} + if !strings.HasPrefix(src, "http://") && !strings.HasPrefix(src, "https://") { + d.ctxPaths[path.Join("/", filepath.ToSlash(src))] = struct{}{} + } } } case *instructions.LabelCommand: @@ -627,7 +631,12 @@ func dispatchRun(d *dispatchState, c *instructions.RunCommand, proxy *llb.ProxyE return err } opt = append(opt, runMounts...) - opt = append(opt, llb.WithCustomName(prefixCommand(d, uppercaseCmd(processCmdEnv(dopt.shlex, c.String(), env)), d.prefixPlatform, d.state.GetPlatform()))) + + shlex := *dopt.shlex + shlex.RawQuotes = true + shlex.SkipUnsetEnv = true + + opt = append(opt, llb.WithCustomName(prefixCommand(d, uppercaseCmd(processCmdEnv(&shlex, c.String(), env)), d.prefixPlatform, d.state.GetPlatform()))) for _, h := range dopt.extraHosts { opt = append(opt, llb.AddExtraHost(h.Host, h.IP)) } @@ -729,6 +738,13 @@ func dispatchCopy(d *dispatchState, c instructions.SourcesAndDest, sourceState l if d.ignoreCache { runOpt = append(runOpt, llb.IgnoreCache) } + + if opt.llbCaps != nil { + if err := opt.llbCaps.Supports(pb.CapExecMetaNetwork); err == nil { + runOpt = append(runOpt, llb.Network(llb.NetModeNone)) + } + } + run := img.Run(append(runOpt, mounts...)...) d.state = run.AddMount("/dest", d.state).Platform(platform) diff --git a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_norunmount.go b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_norunmount.go index d7643405aa6..5f0cd086023 100644 --- a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_norunmount.go +++ b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_norunmount.go @@ -1,4 +1,4 @@ -// +build !dfrunmount,!dfextall +// +build !dfrunmount package dockerfile2llb diff --git a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_runmount.go b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_runmount.go index a06ab9a7cb9..8214e188e84 100644 --- a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_runmount.go +++ b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_runmount.go @@ -1,4 +1,4 @@ -// +build dfrunmount dfextall +// +build dfrunmount package dockerfile2llb diff --git a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_secrets.go b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_secrets.go index 2f6ba584f97..3e354d5b619 100644 --- a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_secrets.go +++ b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_secrets.go @@ -1,4 +1,4 @@ -// +build dfsecrets dfextall +// +build dfsecrets package dockerfile2llb diff --git a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_ssh.go b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_ssh.go index b651ad66055..e7606ff5a29 100644 --- a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_ssh.go +++ b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_ssh.go @@ -1,4 +1,4 @@ -// +build dfssh dfextall +// +build dfssh package dockerfile2llb diff --git a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands_nosecrets.go b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands_nosecrets.go index b0989b28806..58780648db8 100644 --- a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands_nosecrets.go +++ b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands_nosecrets.go @@ -1,4 +1,4 @@ -// +build !dfsecrets,!dfextall +// +build !dfsecrets package instructions diff --git a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands_nossh.go b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands_nossh.go index de809584523..a131a273c3e 100644 --- a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands_nossh.go +++ b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands_nossh.go @@ -1,4 +1,4 @@ -// +build !dfssh,!dfextall +// +build !dfssh package instructions diff --git a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands_runmount.go b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands_runmount.go index 2540a7ea7fb..c5db70feca4 100644 --- a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands_runmount.go +++ b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands_runmount.go @@ -1,4 +1,4 @@ -// +build dfrunmount dfextall +// +build dfrunmount package instructions diff --git a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands_secrets.go b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands_secrets.go index 63b3e215a02..6cce1191d73 100644 --- a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands_secrets.go +++ b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands_secrets.go @@ -1,4 +1,4 @@ -// +build dfsecrets dfextall +// +build dfsecrets package instructions diff --git a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands_ssh.go b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands_ssh.go index 8e591d3e31f..0b94a564470 100644 --- a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands_ssh.go +++ b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands_ssh.go @@ -1,4 +1,4 @@ -// +build dfssh dfextall +// +build dfssh package instructions diff --git a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/shell/lex.go b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/shell/lex.go index c242546e559..6153f50d63a 100644 --- a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/shell/lex.go +++ b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/shell/lex.go @@ -2,6 +2,7 @@ package shell import ( "bytes" + "fmt" "strings" "text/scanner" "unicode" @@ -17,7 +18,9 @@ import ( // It doesn't support all flavors of ${xx:...} formats but new ones can // be added by adding code to the "special ${} format processing" section type Lex struct { - escapeToken rune + escapeToken rune + RawQuotes bool + SkipUnsetEnv bool } // NewLex creates a new Lex which uses escapeToken to escape quotes. @@ -58,17 +61,21 @@ func (s *Lex) ProcessWordsWithMap(word string, env map[string]string) ([]string, func (s *Lex) process(word string, env map[string]string) (string, []string, error) { sw := &shellWord{ - envs: env, - escapeToken: s.escapeToken, + envs: env, + escapeToken: s.escapeToken, + skipUnsetEnv: s.SkipUnsetEnv, + rawQuotes: s.RawQuotes, } sw.scanner.Init(strings.NewReader(word)) return sw.process(word) } type shellWord struct { - scanner scanner.Scanner - envs map[string]string - escapeToken rune + scanner scanner.Scanner + envs map[string]string + escapeToken rune + rawQuotes bool + skipUnsetEnv bool } func (sw *shellWord) process(source string) (string, []string, error) { @@ -103,10 +110,8 @@ func (w *wordsStruct) addRawChar(ch rune) { } func (w *wordsStruct) addString(str string) { - var scan scanner.Scanner - scan.Init(strings.NewReader(str)) - for scan.Peek() != scanner.EOF { - w.addChar(scan.Next()) + for _, ch := range str { + w.addChar(ch) } } @@ -196,14 +201,20 @@ func (sw *shellWord) processSingleQuote() (string, error) { var result bytes.Buffer - sw.scanner.Next() + ch := sw.scanner.Next() + if sw.rawQuotes { + result.WriteRune(ch) + } for { - ch := sw.scanner.Next() + ch = sw.scanner.Next() switch ch { case scanner.EOF: return "", errors.New("unexpected end of statement while looking for matching single-quote") case '\'': + if sw.rawQuotes { + result.WriteRune(ch) + } return result.String(), nil } result.WriteRune(ch) @@ -225,14 +236,20 @@ func (sw *shellWord) processDoubleQuote() (string, error) { var result bytes.Buffer - sw.scanner.Next() + ch := sw.scanner.Next() + if sw.rawQuotes { + result.WriteRune(ch) + } for { switch sw.scanner.Peek() { case scanner.EOF: return "", errors.New("unexpected end of statement while looking for matching double-quote") case '"': - sw.scanner.Next() + ch := sw.scanner.Next() + if sw.rawQuotes { + result.WriteRune(ch) + } return result.String(), nil case '$': value, err := sw.processDollar() @@ -269,7 +286,11 @@ func (sw *shellWord) processDollar() (string, error) { if name == "" { return "$", nil } - return sw.getEnv(name), nil + value, found := sw.getEnv(name) + if !found && sw.skipUnsetEnv { + return "$" + name, nil + } + return value, nil } sw.scanner.Next() @@ -285,7 +306,11 @@ func (sw *shellWord) processDollar() (string, error) { switch ch { case '}': // Normal ${xx} case - return sw.getEnv(name), nil + value, found := sw.getEnv(name) + if !found && sw.skipUnsetEnv { + return fmt.Sprintf("${%s}", name), nil + } + return value, nil case ':': // Special ${xx:...} format processing // Yes it allows for recursive $'s in the ... spot @@ -301,19 +326,26 @@ func (sw *shellWord) processDollar() (string, error) { // Grab the current value of the variable in question so we // can use to to determine what to do based on the modifier - newValue := sw.getEnv(name) + newValue, found := sw.getEnv(name) switch modifier { case '+': if newValue != "" { newValue = word } + if !found && sw.skipUnsetEnv { + return fmt.Sprintf("${%s:%s%s}", name, string(modifier), word), nil + } return newValue, nil case '-': if newValue == "" { newValue = word } + if !found && sw.skipUnsetEnv { + return fmt.Sprintf("${%s:%s%s}", name, string(modifier), word), nil + } + return newValue, nil default: @@ -364,13 +396,13 @@ func isSpecialParam(char rune) bool { return false } -func (sw *shellWord) getEnv(name string) string { +func (sw *shellWord) getEnv(name string) (string, bool) { for key, value := range sw.envs { if EqualEnvKeys(name, key) { - return value + return value, true } } - return "" + return "", false } func BuildEnvs(env []string) map[string]string { diff --git a/components/engine/vendor/github.com/moby/buildkit/session/session.go b/components/engine/vendor/github.com/moby/buildkit/session/session.go index 47c9579633a..5d04738ef78 100644 --- a/components/engine/vendor/github.com/moby/buildkit/session/session.go +++ b/components/engine/vendor/github.com/moby/buildkit/session/session.go @@ -24,7 +24,7 @@ const ( // Dialer returns a connection that can be used by the session type Dialer func(ctx context.Context, proto string, meta map[string][]string) (net.Conn, error) -// Attachable defines a feature that can be expsed on a session +// Attachable defines a feature that can be exposed on a session type Attachable interface { Register(*grpc.Server) } @@ -66,7 +66,7 @@ func NewSession(ctx context.Context, name, sharedKey string) (*Session, error) { return s, nil } -// Allow enable a given service to be reachable through the grpc session +// Allow enables a given service to be reachable through the grpc session func (s *Session) Allow(a Attachable) { a.Register(s.grpcServer) } diff --git a/components/engine/vendor/github.com/moby/buildkit/solver/jobs.go b/components/engine/vendor/github.com/moby/buildkit/solver/jobs.go index 371f43199bf..7932bc5409a 100644 --- a/components/engine/vendor/github.com/moby/buildkit/solver/jobs.go +++ b/components/engine/vendor/github.com/moby/buildkit/solver/jobs.go @@ -171,6 +171,7 @@ func (sb *subBuilder) Build(ctx context.Context, e Edge) (CachedResult, error) { } func (sb *subBuilder) Context(ctx context.Context) context.Context { + ctx = session.NewContext(ctx, sb.state.getSessionID()) return opentracing.ContextWithSpan(progress.WithProgress(ctx, sb.mpw), sb.mspan) } @@ -475,6 +476,7 @@ func (j *Job) Discard() error { } func (j *Job) Context(ctx context.Context) context.Context { + ctx = session.NewContext(ctx, j.SessionID) return progress.WithProgress(ctx, j.pw) } diff --git a/components/engine/vendor/github.com/moby/buildkit/util/contentutil/refs.go b/components/engine/vendor/github.com/moby/buildkit/util/contentutil/refs.go index 9df84063c04..e62d7987bd3 100644 --- a/components/engine/vendor/github.com/moby/buildkit/util/contentutil/refs.go +++ b/components/engine/vendor/github.com/moby/buildkit/util/contentutil/refs.go @@ -72,6 +72,7 @@ func (w *ingester) Writer(ctx context.Context, opts ...content.WriterOpt) (conte } writer, err := w.pusher.Push(ctx, wo.Desc) if err != nil { + unlock() return nil, err } return &lockedWriter{unlock: unlock, Writer: writer}, nil diff --git a/components/engine/vendor/github.com/moby/buildkit/util/rootless/specconv/specconv_linux.go b/components/engine/vendor/github.com/moby/buildkit/util/rootless/specconv/specconv_linux.go index 1b90219f1de..12646e430ac 100644 --- a/components/engine/vendor/github.com/moby/buildkit/util/rootless/specconv/specconv_linux.go +++ b/components/engine/vendor/github.com/moby/buildkit/util/rootless/specconv/specconv_linux.go @@ -1,113 +1,40 @@ package specconv import ( - "os" - "sort" "strings" - "github.com/opencontainers/runc/libcontainer/system" - "github.com/opencontainers/runc/libcontainer/user" "github.com/opencontainers/runtime-spec/specs-go" - "github.com/pkg/errors" ) // ToRootless converts spec to be compatible with "rootless" runc. -// * Adds userns (Note: since we are already in userns, ideally we should not need to do this. runc-side issue is tracked at https://github.com/opencontainers/runc/issues/1837) -// * Fix up mount flags (same as above) -// * Replace /sys with bind-mount (FIXME: we don't need to do this if netns is unshared) +// * Remove /sys mount +// * Remove cgroups +// +// See docs/rootless.md for the supported runc revision. func ToRootless(spec *specs.Spec) error { - if !system.RunningInUserNS() { - return errors.New("needs to be in user namespace") - } - uidMap, err := user.CurrentProcessUIDMap() - if err != nil && !os.IsNotExist(err) { - return err - } - gidMap, err := user.CurrentProcessUIDMap() - if err != nil && !os.IsNotExist(err) { - return err - } - return toRootless(spec, uidMap, gidMap) -} - -// toRootless was forked from github.com/opencontainers/runc/libcontainer/specconv -func toRootless(spec *specs.Spec, uidMap, gidMap []user.IDMap) error { - if err := configureUserNS(spec, uidMap, gidMap); err != nil { - return err - } - if err := configureMounts(spec); err != nil { - return err - } - - // Remove cgroup settings. - spec.Linux.Resources = nil - spec.Linux.CgroupsPath = "" - return nil -} - -// configureUserNS add suserns and the current ID map to the spec. -// Since we are already in userns, ideally we should not need to add userns. -// However, currently rootless runc always requires userns to be added. -// https://github.com/opencontainers/runc/issues/1837 -func configureUserNS(spec *specs.Spec, uidMap, gidMap []user.IDMap) error { - spec.Linux.Namespaces = append(spec.Linux.Namespaces, specs.LinuxNamespace{ - Type: specs.UserNamespace, - }) - - sort.Slice(uidMap, func(i, j int) bool { return uidMap[i].ID < uidMap[j].ID }) - uNextContainerID := int64(0) - for _, u := range uidMap { - spec.Linux.UIDMappings = append(spec.Linux.UIDMappings, - specs.LinuxIDMapping{ - HostID: uint32(u.ID), - ContainerID: uint32(uNextContainerID), - Size: uint32(u.Count), - }) - uNextContainerID += int64(u.Count) - } - sort.Slice(gidMap, func(i, j int) bool { return gidMap[i].ID < gidMap[j].ID }) - gNextContainerID := int64(0) - for _, g := range gidMap { - spec.Linux.GIDMappings = append(spec.Linux.GIDMappings, - specs.LinuxIDMapping{ - HostID: uint32(g.ID), - ContainerID: uint32(gNextContainerID), - Size: uint32(g.Count), - }) - gNextContainerID += int64(g.Count) - } - return nil -} - -func configureMounts(spec *specs.Spec) error { + // Remove /sys mount because we can't mount /sys when the daemon netns + // is not unshared from the host. + // + // Instead, we could bind-mount /sys from the host, however, `rbind, ro` + // does not make /sys/fs/cgroup read-only (and we can't bind-mount /sys + // without rbind) + // + // PR for making /sys/fs/cgroup read-only is proposed, but it is very + // complicated: https://github.com/opencontainers/runc/pull/1869 + // + // For buildkit usecase, we suppose we don't need to provide /sys to + // containers and remove /sys mount as a workaround. var mounts []specs.Mount for _, mount := range spec.Mounts { - // Ignore all mounts that are under /sys, because we add /sys later. if strings.HasPrefix(mount.Destination, "/sys") { continue } - - // Remove all gid= and uid= mappings. - // Since we are already in userns, ideally we should not need to do this. - // https://github.com/opencontainers/runc/issues/1837 - var options []string - for _, option := range mount.Options { - if !strings.HasPrefix(option, "gid=") && !strings.HasPrefix(option, "uid=") { - options = append(options, option) - } - } - mount.Options = options mounts = append(mounts, mount) } - - // Add the sysfs mount as an rbind, because we can't mount /sys unless we have netns. - // TODO: keep original /sys mount when we have netns. - mounts = append(mounts, specs.Mount{ - Source: "/sys", - Destination: "/sys", - Type: "none", - Options: []string{"rbind", "nosuid", "noexec", "nodev", "ro"}, - }) spec.Mounts = mounts + + // Remove cgroups so as to avoid `container_linux.go:337: starting container process caused "process_linux.go:280: applying cgroup configuration for process caused \"mkdir /sys/fs/cgroup/cpuset/buildkit: permission denied\""` + spec.Linux.Resources = nil + spec.Linux.CgroupsPath = "" return nil } diff --git a/components/engine/vendor/github.com/moby/buildkit/vendor.conf b/components/engine/vendor/github.com/moby/buildkit/vendor.conf index ba9a01ed3c0..33f167b7a0a 100644 --- a/components/engine/vendor/github.com/moby/buildkit/vendor.conf +++ b/components/engine/vendor/github.com/moby/buildkit/vendor.conf @@ -6,7 +6,7 @@ github.com/davecgh/go-spew v1.1.0 github.com/pmezard/go-difflib v1.0.0 golang.org/x/sys 1b2967e3c290b7c545b3db0deeda16e9be4f98a2 -github.com/containerd/containerd d97a907f7f781c0ab8340877d8e6b53cc7f1c2f6 +github.com/containerd/containerd 1a5f9a3434ac53c0e9d27093ecc588e0c281c333 github.com/containerd/typeurl a93fcdb778cd272c6e9b3028b2f42d813e785d40 golang.org/x/sync 450f422ab23cf9881c94e2db30cac0eb1b7cf80c github.com/sirupsen/logrus v1.0.0 @@ -16,9 +16,9 @@ golang.org/x/net 0ed95abb35c445290478a5348a7b38bb154135fd github.com/gogo/protobuf v1.0.0 github.com/gogo/googleapis b23578765ee54ff6bceff57f397d833bf4ca6869 github.com/golang/protobuf v1.1.0 -github.com/containerd/continuity f44b615e492bdfb371aae2f76ec694d9da1db537 +github.com/containerd/continuity bd77b46c8352f74eb12c85bdc01f4b90f69d66b4 github.com/opencontainers/image-spec v1.0.1 -github.com/opencontainers/runc 20aff4f0488c6d4b8df4d85b4f63f1f704c11abd +github.com/opencontainers/runc a00bf0190895aa465a5fbed0268888e2c8ddfe85 github.com/Microsoft/go-winio v0.4.11 github.com/containerd/fifo 3d5202aec260678c48179c56f40e6f38a095738c github.com/opencontainers/runtime-spec eba862dc2470385a233c7507392675cbeadf7353 # v1.0.1-45-geba862d @@ -28,8 +28,9 @@ google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944 golang.org/x/text 19e51611da83d6be54ddafce4a4af510cb3e9ea4 github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9 github.com/syndtr/gocapability db04d3cc01c8b54962a58ec7e491717d06cfcc16 -github.com/Microsoft/hcsshim v0.7.3 +github.com/Microsoft/hcsshim v0.7.9 golang.org/x/crypto 0709b304e793a5edb4a2c0145f281ecdc20838a4 +github.com/containerd/cri 8506fe836677cc3bb23a16b68145128243d843b5 # release/1.2 branch github.com/urfave/cli 7bc6a0acffa589f415f88aca16cc1de5ffd66f9c github.com/morikuni/aec 39771216ff4c63d11f5e604076f9c45e8be1067b @@ -40,8 +41,8 @@ golang.org/x/time f51c12702a4d776e4c1fa9b0fabab841babae631 github.com/docker/docker 71cd53e4a197b303c6ba086bd584ffd67a884281 github.com/pkg/profile 5b67d428864e92711fcbd2f8629456121a56d91f -github.com/tonistiigi/fsutil f567071bed2416e4d87d260d3162722651182317 -github.com/hashicorp/go-immutable-radix 826af9ccf0feeee615d546d69b11f8e98da8c8f1 git://github.com/tonistiigi/go-immutable-radix.git +github.com/tonistiigi/fsutil 2862f6bc5ac9b97124e552a5c108230b38a1b0ca +github.com/hashicorp/go-immutable-radix 826af9ccf0feeee615d546d69b11f8e98da8c8f1 https://github.com/tonistiigi/go-immutable-radix github.com/hashicorp/golang-lru a0d98a5f288019575c6d1f4bb1573fef2d1fcdc4 github.com/mitchellh/hashstructure 2bca23e0e452137f789efbc8610126fd8b94f73b github.com/docker/go-connections 3ede32e2033de7505e6500d6c868c2b9ed9f169d @@ -66,6 +67,3 @@ github.com/opentracing-contrib/go-stdlib b1a47cfbdd7543e70e9ef3e73d0802ad306cc1c # used by dockerfile tests gotest.tools v2.1.0 github.com/google/go-cmp v0.2.0 - -# used by rootless spec conv test -github.com/seccomp/libseccomp-golang 32f571b70023028bd57d9288c20efbcb237f3ce0 diff --git a/components/engine/vendor/github.com/tonistiigi/fsutil/walker.go b/components/engine/vendor/github.com/tonistiigi/fsutil/walker.go index 39e6c940471..d78340dfe6b 100644 --- a/components/engine/vendor/github.com/tonistiigi/fsutil/walker.go +++ b/components/engine/vendor/github.com/tonistiigi/fsutil/walker.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "strings" + "syscall" "time" "github.com/docker/docker/pkg/fileutils" @@ -71,7 +72,7 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err return err } defer func() { - if retErr != nil && os.IsNotExist(errors.Cause(retErr)) { + if retErr != nil && isNotExist(retErr) { retErr = filepath.SkipDir } }() @@ -216,3 +217,14 @@ func trimUntilIndex(str, sep string, count int) string { } } } + +func isNotExist(err error) bool { + err = errors.Cause(err) + if os.IsNotExist(err) { + return true + } + if pe, ok := err.(*os.PathError); ok { + err = pe.Err + } + return err == syscall.ENOTDIR +}