Acb/update buildkit

.github/workflows/build.yml

@@ -13,15 +13,16 @@ on:
   pull_request:
     branches:
       - 'master'
+      - 'v*'
 
 env:
   REPO_SLUG_ORIGIN: "moby/buildkit:latest"
   REPO_SLUG_TARGET: "moby/buildkit"
   DF_REPO_SLUG_TARGET: "docker/dockerfile-upstream"
   PLATFORMS: "linux/amd64,linux/arm/v7,linux/arm64,linux/s390x,linux/ppc64le,linux/riscv64"
-  CACHEKEY_INTEGRATION_TESTS: "integration-tests"
-  CACHEKEY_BINARIES: "binaries"
-  CACHEKEY_CROSS: "cross"
+  CACHE_GHA_SCOPE_IT: "integration-tests"
+  CACHE_GHA_SCOPE_BINARIES: "binaries"
+  CACHE_GHA_SCOPE_CROSS: "cross"
 
 jobs:
   base:
@@ -31,21 +32,8 @@ jobs:
         name: Checkout
         uses: actions/checkout@v2
       -
-        name: Cache ${{ env.CACHEKEY_INTEGRATION_TESTS }}
-        uses: actions/cache@v2
-        with:
-          path: /tmp/.buildkit-cache/${{ env.CACHEKEY_INTEGRATION_TESTS }}
-          key: ${{ runner.os }}-buildkit-${{ env.CACHEKEY_INTEGRATION_TESTS }}-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildkit-${{ env.CACHEKEY_INTEGRATION_TESTS }}-
-      -
-        name: Cache ${{ env.CACHEKEY_BINARIES }}
-        uses: actions/cache@v2
-        with:
-          path: /tmp/.buildkit-cache/${{ env.CACHEKEY_BINARIES }}
-          key: ${{ runner.os }}-buildkit-${{ env.CACHEKEY_BINARIES }}-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildkit-${{ env.CACHEKEY_BINARIES }}-
+        name: Expose GitHub Runtime
+        uses: crazy-max/ghaction-github-runtime@v1
       -
         name: Set up QEMU
         uses: docker/setup-qemu-action@v1
@@ -54,28 +42,21 @@ jobs:
         uses: docker/setup-buildx-action@v1
         with:
           driver-opts: image=${{ env.REPO_SLUG_ORIGIN }}
+          buildkitd-flags: --debug
       -
-        name: Build ${{ env.CACHEKEY_BINARIES }}
+        name: Build ${{ env.CACHE_GHA_SCOPE_BINARIES }}
         run: |
           ./hack/build_ci_first_pass binaries
         env:
-          CACHEDIR_FROM: /tmp/.buildkit-cache/${{ env.CACHEKEY_BINARIES }}
-          CACHEDIR_TO: /tmp/.buildkit-cache/${{ env.CACHEKEY_BINARIES }}-new
+          CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_BINARIES }}
+          CACHE_TO: type=gha,scope=${{ env.CACHE_GHA_SCOPE_BINARIES }}
       -
-        name: Build ${{ env.CACHEKEY_INTEGRATION_TESTS }}
+        name: Build ${{ env.CACHE_GHA_SCOPE_IT }}
         run: |
           ./hack/build_ci_first_pass integration-tests
         env:
-          CACHEDIR_FROM: /tmp/.buildkit-cache/${{ env.CACHEKEY_INTEGRATION_TESTS }}
-          CACHEDIR_TO: /tmp/.buildkit-cache/${{ env.CACHEKEY_INTEGRATION_TESTS }}-new
-      -
-        # FIXME: Temp fix for https://github.com/moby/buildkit/issues/1850
-        name: Move cache
-        run: |
-          rm -rf /tmp/.buildkit-cache/${{ env.CACHEKEY_BINARIES }}
-          mv /tmp/.buildkit-cache/${{ env.CACHEKEY_BINARIES }}-new /tmp/.buildkit-cache/${{ env.CACHEKEY_BINARIES }}
-          rm -rf /tmp/.buildkit-cache/${{ env.CACHEKEY_INTEGRATION_TESTS }}
-          mv /tmp/.buildkit-cache/${{ env.CACHEKEY_INTEGRATION_TESTS }}-new /tmp/.buildkit-cache/${{ env.CACHEKEY_INTEGRATION_TESTS }}
+          CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_IT }}
+          CACHE_TO: type=gha,scope=${{ env.CACHE_GHA_SCOPE_IT }}
 
   test:
     runs-on: ubuntu-latest
@@ -108,21 +89,8 @@ jobs:
         name: Checkout
         uses: actions/checkout@v2
       -
-        name: Cache ${{ env.CACHEKEY_INTEGRATION_TESTS }}
-        uses: actions/cache@v2
-        with:
-          path: /tmp/.buildkit-cache/${{ env.CACHEKEY_INTEGRATION_TESTS }}
-          key: ${{ runner.os }}-buildkit-${{ env.CACHEKEY_INTEGRATION_TESTS }}-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildkit-${{ env.CACHEKEY_INTEGRATION_TESTS }}-
-      -
-        name: Cache ${{ env.CACHEKEY_BINARIES }}
-        uses: actions/cache@v2
-        with:
-          path: /tmp/.buildkit-cache/${{ env.CACHEKEY_BINARIES }}
-          key: ${{ runner.os }}-buildkit-${{ env.CACHEKEY_BINARIES }}-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildkit-${{ env.CACHEKEY_BINARIES }}-
+        name: Expose GitHub Runtime
+        uses: crazy-max/ghaction-github-runtime@v1
       -
         name: Set up QEMU
         uses: docker/setup-qemu-action@v1
@@ -131,6 +99,7 @@ jobs:
         uses: docker/setup-buildx-action@v1
         with:
           driver-opts: image=${{ env.REPO_SLUG_ORIGIN }}
+          buildkitd-flags: --debug
       -
         name: Test pkg=${{ matrix.pkg }} ; typ=${{ matrix.typ }} ; skipit=${{ matrix.skip-integration-tests }} ; worker=${{ matrix.worker }}
         run: |
@@ -144,7 +113,7 @@ jobs:
           TEST_COVERAGE: 1
           TESTPKGS: ${{ matrix.pkg }}
           SKIP_INTEGRATION_TESTS: ${{ matrix.skip-integration-tests }}
-          CACHEDIR_FROM: /tmp/.buildkit-cache/${{ env.CACHEKEY_INTEGRATION_TESTS }} /tmp/.buildkit-cache/${{ env.CACHEKEY_BINARIES }}
+          CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_IT }} type=gha,scope=${{ env.CACHE_GHA_SCOPE_BINARIES }}
       -
         name: Upload coverage file
         uses: actions/upload-artifact@v2
@@ -235,13 +204,8 @@ jobs:
         name: Checkout
         uses: actions/checkout@v2
       -
-        name: Cache ${{ env.CACHEKEY_CROSS }}
-        uses: actions/cache@v2
-        with:
-          path: /tmp/.buildkit-cache/${{ env.CACHEKEY_CROSS }}
-          key: ${{ runner.os }}-buildkit-${{ env.CACHEKEY_CROSS }}-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildkit-${{ env.CACHEKEY_CROSS }}-
+        name: Expose GitHub Runtime
+        uses: crazy-max/ghaction-github-runtime@v1
       -
         name: Set up QEMU
         uses: docker/setup-qemu-action@v1
@@ -250,21 +214,16 @@ jobs:
         uses: docker/setup-buildx-action@v1
         with:
           driver-opts: image=${{ env.REPO_SLUG_ORIGIN }}
+          buildkitd-flags: --debug
       -
         name: Cross
         run: |
           ./hack/cross
         env:
           PLATFORMS: ${{ env.PLATFORMS }},darwin/amd64,darwin/arm64,windows/amd64,windows/arm64
           RUNC_PLATFORMS: ${{ env.PLATFORMS }}
-          CACHEDIR_FROM: /tmp/.buildkit-cache/${{ env.CACHEKEY_CROSS }}
-          CACHEDIR_TO: /tmp/.buildkit-cache/${{ env.CACHEKEY_CROSS }}-new
-      -
-        # FIXME: Temp fix for https://github.com/moby/buildkit/issues/1850
-        name: Move cache
-        run: |
-          rm -rf /tmp/.buildkit-cache/${{ env.CACHEKEY_CROSS }}
-          mv /tmp/.buildkit-cache/${{ env.CACHEKEY_CROSS }}-new /tmp/.buildkit-cache/${{ env.CACHEKEY_CROSS }}
+          CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_CROSS }}
+          CACHE_TO: type=gha,scope=${{ env.CACHE_GHA_SCOPE_CROSS }}
 
   release-base:
     runs-on: ubuntu-latest
@@ -304,21 +263,8 @@ jobs:
         name: Checkout
         uses: actions/checkout@v2
       -
-        name: Cache image${{ matrix.target-stage }}
-        uses: actions/cache@v2
-        with:
-          path: /tmp/.buildkit-cache/image${{ matrix.target-stage }}
-          key: ${{ runner.os }}-buildkit-image${{ matrix.target-stage }}-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildkit-image${{ matrix.target-stage }}-
-      -
-        name: Cache ${{ env.CACHEKEY_CROSS }}
-        uses: actions/cache@v2
-        with:
-          path: /tmp/.buildkit-cache/${{ env.CACHEKEY_CROSS }}
-          key: ${{ runner.os }}-buildkit-${{ env.CACHEKEY_CROSS }}-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildkit-${{ env.CACHEKEY_CROSS }}-
+        name: Expose GitHub Runtime
+        uses: crazy-max/ghaction-github-runtime@v1
       -
         name: Set up QEMU
         uses: docker/setup-qemu-action@v1
@@ -327,6 +273,7 @@ jobs:
         uses: docker/setup-buildx-action@v1
         with:
           driver-opts: image=${{ env.REPO_SLUG_ORIGIN }}
+          buildkitd-flags: --debug
       -
         name: Login to DockerHub
         if: needs.release-base.outputs.push == 'push'
@@ -340,14 +287,8 @@ jobs:
           ./hack/images "${{ needs.release-base.outputs.tag }}" "$REPO_SLUG_TARGET" "${{ needs.release-base.outputs.push }}"
         env:
           TARGET: ${{ matrix.target-stage }}
-          CACHEDIR_FROM: /tmp/.buildkit-cache/${{ env.CACHEKEY_CROSS }} /tmp/.buildkit-cache/image${{ matrix.target-stage }}
-          CACHEDIR_TO: /tmp/.buildkit-cache/image${{ matrix.target-stage }}-new
-      -
-        # FIXME: Temp fix for https://github.com/moby/buildkit/issues/1850
-        name: Move cache
-        run: |
-          rm -rf /tmp/.buildkit-cache/image${{ matrix.target-stage }}
-          mv /tmp/.buildkit-cache/image${{ matrix.target-stage }}-new /tmp/.buildkit-cache/image${{ matrix.target-stage }}
+          CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_CROSS }} type=gha,scope=image${{ matrix.target-stage }}
+          CACHE_TO: type=gha,scope=image${{ matrix.target-stage }}
 
   binaries:
     runs-on: ubuntu-latest
@@ -357,21 +298,8 @@ jobs:
         name: Checkout
         uses: actions/checkout@v2
       -
-        name: Cache ${{ env.CACHEKEY_BINARIES }}
-        uses: actions/cache@v2
-        with:
-          path: /tmp/.buildkit-cache/${{ env.CACHEKEY_BINARIES }}
-          key: ${{ runner.os }}-buildkit-${{ env.CACHEKEY_BINARIES }}-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildkit-${{ env.CACHEKEY_BINARIES }}-
-      -
-        name: Cache ${{ env.CACHEKEY_CROSS }}
-        uses: actions/cache@v2
-        with:
-          path: /tmp/.buildkit-cache/${{ env.CACHEKEY_CROSS }}
-          key: ${{ runner.os }}-buildkit-${{ env.CACHEKEY_CROSS }}-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildkit-${{ env.CACHEKEY_CROSS }}-
+        name: Expose GitHub Runtime
+        uses: crazy-max/ghaction-github-runtime@v1
       -
         name: Set up QEMU
         uses: docker/setup-qemu-action@v1
@@ -380,13 +308,14 @@ jobs:
         uses: docker/setup-buildx-action@v1
         with:
           driver-opts: image=${{ env.REPO_SLUG_ORIGIN }}
+          buildkitd-flags: --debug
       -
         name: Build ${{ needs.release-base.outputs.tag }}
         run: |
           ./hack/release-tar "${{ needs.release-base.outputs.tag }}" release-out
         env:
           PLATFORMS: ${{ env.PLATFORMS }},darwin/amd64,darwin/arm64,windows/amd64,windows/arm64
-          CACHEDIR_FROM: /tmp/.buildkit-cache/${{ env.CACHEKEY_BINARIES }} /tmp/.buildkit-cache/${{ env.CACHEKEY_CROSS }}
+          CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_BINARIES }} type=gha,scope=${{ env.CACHE_GHA_SCOPE_CROSS }}
       -
         name: Move artifacts
         run: |
@@ -443,6 +372,9 @@ jobs:
       -
         name: Checkout
         uses: actions/checkout@v2
+      -
+        name: Expose GitHub Runtime
+        uses: crazy-max/ghaction-github-runtime@v1
       -
         name: Set up QEMU
         uses: docker/setup-qemu-action@v1
@@ -451,14 +383,7 @@ jobs:
         uses: docker/setup-buildx-action@v1
         with:
           driver-opts: image=${{ env.REPO_SLUG_ORIGIN }}
-      -
-        name: Cache layers
-        uses: actions/cache@v2
-        with:
-          path: /tmp/.buildkit-cache/frontend-${{ needs.frontend-base.outputs.typ }}
-          key: ${{ runner.os }}-buildkit-frontend-${{ needs.frontend-base.outputs.typ }}-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildkit-frontend-${{ needs.frontend-base.outputs.typ }}-
+          buildkitd-flags: --debug
       -
         name: Login to DockerHub
         uses: docker/login-action@v1
@@ -471,18 +396,12 @@ jobs:
         run: |
           ./frontend/dockerfile/cmd/dockerfile-frontend/hack/release "${{ needs.frontend-base.outputs.typ }}" "${{ needs.frontend-base.outputs.tag }}" "$DF_REPO_SLUG_TARGET" "${{ needs.frontend-base.outputs.push }}"
         env:
-          CACHEDIR_FROM: /tmp/.buildkit-cache/frontend-${{ needs.frontend-base.outputs.typ }}
-          CACHEDIR_TO: /tmp/.buildkit-cache/frontend-${{ needs.frontend-base.outputs.typ }}-new
+          CACHE_FROM: type=gha,scope=frontend-${{ needs.frontend-base.outputs.typ }}
+          CACHE_TO: type=gha,scope=frontend-${{ needs.frontend-base.outputs.typ }}
       -
         name: Build ${{ needs.frontend-base.outputs.typ }}/labs
         if: needs.frontend-base.outputs.typ == 'master'
         run: |
           ./frontend/dockerfile/cmd/dockerfile-frontend/hack/release "${{ needs.frontend-base.outputs.typ }}" labs "$DF_REPO_SLUG_TARGET" "${{ needs.frontend-base.outputs.push }}"
         env:
-          CACHEDIR_FROM: /tmp/.buildkit-cache/frontend-${{ needs.frontend-base.outputs.typ }}
-      -
-        # FIXME: Temp fix for https://github.com/moby/buildkit/issues/1850
-        name: Move cache
-        run: |
-          rm -rf /tmp/.buildkit-cache/frontend-${{ needs.frontend-base.outputs.typ }}
-          mv /tmp/.buildkit-cache/frontend-${{ needs.frontend-base.outputs.typ }}-new /tmp/.buildkit-cache/frontend-${{ needs.frontend-base.outputs.typ }}
+          CACHE_FROM: type=gha,scope=frontend-${{ needs.frontend-base.outputs.typ }}

.github/workflows/validate.yml

@@ -11,6 +11,7 @@ on:
   pull_request:
     branches:
       - 'master'
+      - 'v*'
 
 env:
   REPO_SLUG_ORIGIN: "moby/buildkit:latest"

README.md

@@ -59,6 +59,7 @@ You don't need to read this document unless you want to use the full-featured st
     - [Local directory](#local-directory-1)
     - [GitHub Actions cache (experimental)](#github-actions-cache-experimental)
   - [Consistent hashing](#consistent-hashing)
+- [Metadata](#metadata)
 - [Systemd socket activation](#systemd-socket-activation)
 - [Expose BuildKit as a TCP service](#expose-buildkit-as-a-tcp-service)
   - [Load balancing](#load-balancing)
@@ -230,8 +231,9 @@ Keys supported by image output:
 * `unpack=true`: unpack image after creation (for use with containerd)
 * `dangling-name-prefix=[value]`: name image with `prefix@<digest>` , used for anonymous images
 * `name-canonical=true`: add additional canonical name `name@<digest>`
-* `compression=[uncompressed,gzip,estargz]`: choose compression type for layers newly created and cached, gzip is default value. estargz should be used with `oci-mediatypes=true`.
+* `compression=[uncompressed,gzip,estargz,zstd]`: choose compression type for layers newly created and cached, gzip is default value. estargz should be used with `oci-mediatypes=true`.
 * `force-compression=true`: forcefully apply `compression` option to all layers (including already existing layers).
+* `buildinfo=[all,imageconfig,metadata,none]`: choose [build dependency](docs/build-repro.md#build-dependencies) version to export (default `all`).
 
 If credentials are required, `buildctl` will attempt to read Docker configuration file `$DOCKER_CONFIG/config.json`.
 `$DOCKER_CONFIG` defaults to `~/.docker`.
@@ -416,7 +418,7 @@ To output build metadata such as the image digest, pass the `--metadata-file` fl
 The metadata will be written as a JSON object to the specified file.
 The directory of the specified file must already exist and be writable.
 
-```
+```bash
 buildctl build ... --metadata-file metadata.json
 ```
 
@@ -545,7 +547,7 @@ Please refer to [`docs/rootless.md`](docs/rootless.md).
 
 ## Building multi-platform images
 
-See [`docker buildx` documentation](https://github.com/docker/buildx#building-multi-platform-images)
+Please refer to [`docs/multi-platform.md`](docs/multi-platform.md).
 
 ## Contributing
 

api/services/control/control.proto

@@ -121,7 +121,6 @@ message VertexStatus {
 	string name = 3;
 	int64 current = 4;
 	int64 total = 5;
-	// TODO: add started, completed
 	google.protobuf.Timestamp timestamp = 6 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
 	google.protobuf.Timestamp started = 7 [(gogoproto.stdtime) = true ];
 	google.protobuf.Timestamp completed = 8 [(gogoproto.stdtime) = true ];