From 86c54e5a3a2a4ae9c0cc39b3af7d94c04ac689ab Mon Sep 17 00:00:00 2001 From: Oleg <70903606+dataoleg@users.noreply.github.com> Date: Mon, 6 Dec 2021 20:22:09 +0100 Subject: [PATCH] toolchain: allow to configure build_tar target (#1937) python/cpython#18080 hasn't been back-ported to python3.8, which happens to be the default interpreter version in a lot of popular base docker images. This change in python's tarfile implementation affects the behavior of https://github.com/bazelbuild/rules_docker/blob/bdea23404dd256c7ae2f8e02d346428044cc7d91/container/archive.py#L144 used by a popular container_image rule. This means that images built by bazel in environments with different python3 versions will result in different image digests. Until the hermeticity of python toolchains is guaranteed, it may be useful to have a workaround for users who can provide their own build_tar tool, akin to https://github.com/kubernetes/repo-infra/blob/72a6f5d05659f7d255b51f152e0725adfe970718/tools/build_tar/buildtar.go --- README.md | 2 ++ container/layer.bzl | 5 ++++- testing/default_toolchain/WORKSPACE | 5 +++++ toolchains/docker/BUILD.tpl | 1 + toolchains/docker/toolchain.bzl | 20 ++++++++++++++++++++ 5 files changed, 32 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cf5f85df8..a16af755b 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,8 @@ load("@io_bazel_rules_docker//toolchains/docker:toolchain.bzl", ) docker_toolchain_configure( name = "docker_config", + # OPTIONAL: Bazel target for the build_tar tool, must be compatible with build_tar.py + build_tar_target="", # OPTIONAL: Path to a directory which has a custom docker client config.json. # See https://docs.docker.com/engine/reference/commandline/cli/#configuration-files # for more details. diff --git a/container/layer.bzl b/container/layer.bzl index 38e978395..519393408 100644 --- a/container/layer.bzl +++ b/container/layer.bzl @@ -102,7 +102,10 @@ def build_layer( """ toolchain_info = ctx.toolchains["@io_bazel_rules_docker//toolchains/docker:toolchain_type"].info layer = output_layer - build_layer_exec = ctx.executable.build_layer + if toolchain_info.build_tar_target: + build_layer_exec = toolchain_info.build_tar_target.files_to_run.executable + else: + build_layer_exec = ctx.executable.build_layer args = ctx.actions.args() args.add(layer, format = "--output=%s") args.add(directory, format = "--directory=%s") diff --git a/testing/default_toolchain/WORKSPACE b/testing/default_toolchain/WORKSPACE index d6cfe107b..e3138e31b 100644 --- a/testing/default_toolchain/WORKSPACE +++ b/testing/default_toolchain/WORKSPACE @@ -26,6 +26,10 @@ load( "local_tool", ) +local_tool( + name = "build_tar", +) + local_tool( name = "xz", ) @@ -37,6 +41,7 @@ load( docker_toolchain_configure( name = "docker_config", + build_tar_target = "@build_tar//:build_tar", xz_target = "@xz//:xz", ) diff --git a/toolchains/docker/BUILD.tpl b/toolchains/docker/BUILD.tpl index dbee643a5..5c597bbde 100644 --- a/toolchains/docker/BUILD.tpl +++ b/toolchains/docker/BUILD.tpl @@ -21,6 +21,7 @@ load("@io_bazel_rules_docker//toolchains/docker:toolchain.bzl", "docker_toolchai docker_toolchain( name = "toolchain", client_config = "%{DOCKER_CONFIG}", + %{BUILD_TAR_ATTR} %{GZIP_ATTR} %{TOOL_ATTR} docker_flags = ["%{DOCKER_FLAGS}"], diff --git a/toolchains/docker/toolchain.bzl b/toolchains/docker/toolchain.bzl index 2e53efc1b..b7cd66934 100644 --- a/toolchains/docker/toolchain.bzl +++ b/toolchains/docker/toolchain.bzl @@ -18,6 +18,7 @@ This module defines docker toolchain rules DockerToolchainInfo = provider( doc = "Docker toolchain rule parameters", fields = { + "build_tar_target": "Optional Bazel target for the build_tar tool", "client_config": "A custom directory for the docker client " + "config.json. If DOCKER_CONFIG is not specified, " + "the value of the DOCKER_CONFIG environment variable " + @@ -41,6 +42,7 @@ DockerToolchainInfo = provider( def _docker_toolchain_impl(ctx): toolchain_info = platform_common.ToolchainInfo( info = DockerToolchainInfo( + build_tar_target = ctx.attr.build_tar_target, docker_flags = ctx.attr.docker_flags, client_config = ctx.attr.client_config, gzip_path = ctx.attr.gzip_path, @@ -58,6 +60,12 @@ def _docker_toolchain_impl(ctx): docker_toolchain = rule( implementation = _docker_toolchain_impl, attrs = { + "build_tar_target": attr.label( + allow_files = True, + doc = "Bazel target for the build_tar tool.", + cfg = "host", + executable = True, + ), "client_config": attr.string( default = "", doc = "A custom directory for the docker client config.json. If " + @@ -136,6 +144,10 @@ def _toolchain_configure_impl(repository_ctx): docker_flags = [] docker_flags += repository_ctx.attr.docker_flags + build_tar_attr = "" + if repository_ctx.attr.build_tar_target: + build_tar_attr = "build_tar_target = \"%s\"," % repository_ctx.attr.build_tar_target + # If client_config is not set we need to pass an empty string to the # template. client_config = repository_ctx.attr.client_config or "" @@ -143,6 +155,7 @@ def _toolchain_configure_impl(repository_ctx): "BUILD", Label("@io_bazel_rules_docker//toolchains/docker:BUILD.tpl"), { + "%{BUILD_TAR_ATTR}": "%s" % build_tar_attr, "%{DOCKER_CONFIG}": "%s" % client_config, "%{DOCKER_FLAGS}": "%s" % "\", \"".join(docker_flags), "%{TOOL_ATTR}": "%s" % tool_attr, @@ -167,6 +180,13 @@ def _toolchain_configure_impl(repository_ctx): # Repository rule to generate a docker_toolchain target toolchain_configure = repository_rule( attrs = { + "build_tar_target": attr.label( + executable = True, + cfg = "host", + allow_files = True, + mandatory = False, + doc = "The bazel target for the build_tar tool.", + ), "client_config": attr.string( mandatory = False, doc = "A custom directory for the docker client " +