[0.11 backport] azblob and s3 integration tests

.github/workflows/build.yml

@@ -182,52 +182,6 @@ jobs:
           SKIP_INTEGRATION_TESTS: ${{ matrix.skip-integration-tests }}
           CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_IT }} type=gha,scope=${{ env.CACHE_GHA_SCOPE_BINARIES }}
 
-  test-s3:
-    runs-on: ubuntu-20.04
-    needs:
-      - base
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v3
-      -
-        name: Expose GitHub Runtime
-        uses: crazy-max/ghaction-github-runtime@v2
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-        with:
-          version: ${{ env.BUILDX_VERSION }}
-          driver-opts: image=${{ env.REPO_SLUG_ORIGIN }}
-          buildkitd-flags: --debug
-      -
-        name: Test
-        run: |
-          hack/s3_test/run_test.sh
-
-  test-azblob:
-    runs-on: ubuntu-20.04
-    needs:
-      - base
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v3
-      -
-        name: Expose GitHub Runtime
-        uses: crazy-max/ghaction-github-runtime@v2
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-        with:
-          version: ${{ env.BUILDX_VERSION }}
-          driver-opts: image=${{ env.REPO_SLUG_ORIGIN }}
-          buildkitd-flags: --debug
-      -
-        name: Test
-        run: |
-          hack/azblob_test/run_test.sh
-
   test-os:
     runs-on: ${{ matrix.os }}
     strategy:

Dockerfile

@@ -11,10 +11,17 @@ ARG STARGZ_SNAPSHOTTER_VERSION=v0.13.0
 ARG NERDCTL_VERSION=v1.0.0
 ARG DNSNAME_VERSION=v1.3.1
 ARG NYDUS_VERSION=v2.1.0
+ARG MINIO_VERSION=RELEASE.2022-05-03T20-36-08Z
+ARG MINIO_MC_VERSION=RELEASE.2022-05-04T06-07-55Z
+ARG AZURITE_VERSION=3.18.0
 
 ARG GO_VERSION=1.19
 ARG ALPINE_VERSION=3.17
 
+# minio for s3 integration tests
+FROM minio/minio:${MINIO_VERSION} AS minio
+FROM minio/mc:${MINIO_MC_VERSION} AS minio-mc
+
 # alpine base for buildkit image
 # TODO: remove this when alpine image supports riscv64
 FROM alpine:${ALPINE_VERSION} AS alpine-amd64
@@ -223,16 +230,19 @@ COPY --link --from=dnsname /usr/bin/dnsname /opt/cni/bin/
 
 FROM buildkit-base AS integration-tests-base
 ENV BUILDKIT_INTEGRATION_ROOTLESS_IDPAIR="1000:1000"
-ARG NERDCTL_VERSION
 RUN apk add --no-cache shadow shadow-uidmap sudo vim iptables ip6tables dnsmasq fuse curl git-daemon \
   && useradd --create-home --home-dir /home/user --uid 1000 -s /bin/sh user \
   && echo "XDG_RUNTIME_DIR=/run/user/1000; export XDG_RUNTIME_DIR" >> /home/user/.profile \
   && mkdir -m 0700 -p /run/user/1000 \
   && chown -R user /run/user/1000 /home/user \
   && ln -s /sbin/iptables-legacy /usr/bin/iptables \
-  && xx-go --wrap \
-  && curl -Ls https://raw.githubusercontent.com/containerd/nerdctl/$NERDCTL_VERSION/extras/rootless/containerd-rootless.sh > /usr/bin/containerd-rootless.sh \
+  && xx-go --wrap
+ARG NERDCTL_VERSION
+RUN curl -Ls https://raw.githubusercontent.com/containerd/nerdctl/$NERDCTL_VERSION/extras/rootless/containerd-rootless.sh > /usr/bin/containerd-rootless.sh \
   && chmod 0755 /usr/bin/containerd-rootless.sh
+ARG AZURITE_VERSION
+RUN apk add --no-cache nodejs npm \
+  && npm install -g azurite@${AZURITE_VERSION}
 # The entrypoint script is needed for enabling nested cgroup v2 (https://github.com/moby/buildkit/issues/3265#issuecomment-1309631736)
 RUN curl -Ls https://raw.githubusercontent.com/moby/moby/v20.10.21/hack/dind > /docker-entrypoint.sh \
   && chmod 0755 /docker-entrypoint.sh
@@ -241,6 +251,8 @@ ENTRYPOINT ["/docker-entrypoint.sh"]
 ENV BUILDKIT_INTEGRATION_CONTAINERD_EXTRA="containerd-1.5=/opt/containerd-alt-15/bin"
 ENV BUILDKIT_INTEGRATION_SNAPSHOTTER=stargz
 ENV CGO_ENABLED=0
+COPY --link --from=minio /opt/bin/minio /usr/bin/
+COPY --link --from=minio-mc /usr/bin/mc /usr/bin/
 COPY --link --from=nydus /out/nydus-static/* /usr/bin/
 COPY --link --from=stargz-snapshotter /out/* /usr/bin/
 COPY --link --from=rootlesskit /rootlesskit /usr/bin/

README.md

@@ -542,6 +542,11 @@ There are 2 options supported for Azure Blob Storage authentication:
 * Any system using environment variables supported by the [Azure SDK for Go](https://docs.microsoft.com/en-us/azure/developer/go/azure-sdk-authentication). The configuration must be available for the buildkit daemon, not for the client.
 * Secret Access Key, using the `secret_access_key` attribute to specify the primary or secondary account key for your Azure Blob Storage account. [Azure Blob Storage account keys](https://docs.microsoft.com/en-us/azure/storage/common/storage-account-keys-manage)
 
+> **Note**
+>
+> Account name can also be specified with `account_name` attribute (or `$BUILDKIT_AZURE_STORAGE_ACCOUNT_NAME`)
+> if it is not part of the account URL host.
+
 `--export-cache` options:
 * `type=azblob`
 * `mode=<min|max>`: specify cache layers to export (default: `min`)

cache/remotecache/azblob/utils.go

@@ -15,6 +15,7 @@ import (
 
 const (
 	attrSecretAccessKey = "secret_access_key"
+	attrAccountName     = "account_name"
 	attrAccountURL      = "account_url"
 	attrPrefix          = "prefix"
 	attrManifestsPrefix = "manifests_prefix"
@@ -50,7 +51,16 @@ func getConfig(attrs map[string]string) (*Config, error) {
 		return &Config{}, errors.Wrap(err, "azure storage account url provided is not a valid url")
 	}
 
-	accountName := strings.Split(accountURL.Hostname(), ".")[0]
+	accountName, ok := attrs[attrAccountName]
+	if !ok {
+		accountName, ok = os.LookupEnv("BUILDKIT_AZURE_STORAGE_ACCOUNT_NAME")
+		if !ok {
+			accountName = strings.Split(accountURL.Hostname(), ".")[0]
+		}
+	}
+	if accountName == "" {
+		return &Config{}, errors.New("unable to retrieve account name from account url or ${BUILDKIT_AZURE_STORAGE_ACCOUNT_NAME} or account_name attribute for azblob cache")
+	}
 
 	container, ok := attrs[attrContainer]
 	if !ok {

client/client_test.go

@@ -112,6 +112,8 @@ func TestIntegration(t *testing.T) {
 		testReadonlyRootFS,
 		testBasicRegistryCacheImportExport,
 		testBasicLocalCacheImportExport,
+		testBasicS3CacheImportExport,
+		testBasicAzblobCacheImportExport,
 		testCachedMounts,
 		testCopyFromEmptyImage,
 		testProxyEnv,
@@ -4737,6 +4739,77 @@ func testBasicLocalCacheImportExport(t *testing.T, sb integration.Sandbox) {
 	testBasicCacheImportExport(t, sb, []CacheOptionsEntry{im}, []CacheOptionsEntry{ex})
 }
 
+func testBasicS3CacheImportExport(t *testing.T, sb integration.Sandbox) {
+	integration.CheckFeatureCompat(t, sb, integration.FeatureCacheExport)
+
+	opts := integration.MinioOpts{
+		Region:          "us-east-1",
+		AccessKeyID:     "minioadmin",
+		SecretAccessKey: "minioadmin",
+	}
+
+	s3Addr, s3Bucket, cleanup, err := integration.NewMinioServer(t, sb, opts)
+	require.NoError(t, err)
+	defer cleanup()
+
+	im := CacheOptionsEntry{
+		Type: "s3",
+		Attrs: map[string]string{
+			"region":            opts.Region,
+			"access_key_id":     opts.AccessKeyID,
+			"secret_access_key": opts.SecretAccessKey,
+			"bucket":            s3Bucket,
+			"endpoint_url":      s3Addr,
+			"use_path_style":    "true",
+		},
+	}
+	ex := CacheOptionsEntry{
+		Type: "s3",
+		Attrs: map[string]string{
+			"region":            opts.Region,
+			"access_key_id":     opts.AccessKeyID,
+			"secret_access_key": opts.SecretAccessKey,
+			"bucket":            s3Bucket,
+			"endpoint_url":      s3Addr,
+			"use_path_style":    "true",
+		},
+	}
+	testBasicCacheImportExport(t, sb, []CacheOptionsEntry{im}, []CacheOptionsEntry{ex})
+}
+
+func testBasicAzblobCacheImportExport(t *testing.T, sb integration.Sandbox) {
+	integration.CheckFeatureCompat(t, sb, integration.FeatureCacheExport)
+
+	opts := integration.AzuriteOpts{
+		AccountName: "azblobcacheaccount",
+		AccountKey:  base64.StdEncoding.EncodeToString([]byte("azblobcacheaccountkey")),
+	}
+
+	azAddr, cleanup, err := integration.NewAzuriteServer(t, sb, opts)
+	require.NoError(t, err)
+	defer cleanup()
+
+	im := CacheOptionsEntry{
+		Type: "azblob",
+		Attrs: map[string]string{
+			"account_url":       azAddr,
+			"account_name":      opts.AccountName,
+			"secret_access_key": opts.AccountKey,
+			"container":         "cachecontainer",
+		},
+	}
+	ex := CacheOptionsEntry{
+		Type: "azblob",
+		Attrs: map[string]string{
+			"account_url":       azAddr,
+			"account_name":      opts.AccountName,
+			"secret_access_key": opts.AccountKey,
+			"container":         "cachecontainer",
+		},
+	}
+	testBasicCacheImportExport(t, sb, []CacheOptionsEntry{im}, []CacheOptionsEntry{ex})
+}
+
 func testBasicInlineCacheImportExport(t *testing.T, sb integration.Sandbox) {
 	integration.CheckFeatureCompat(t, sb,
 		integration.FeatureDirectPush,