From 3d1313f980bc98dedb5ee1a9ea909ee25cb46bc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Aub=C3=A9?= Date: Wed, 2 Sep 2020 14:02:37 +0000 Subject: [PATCH 1/9] For gzip operations, use a Go helper by default rather than the gzip tool. This is mostly for consistency across platforms, and to remove an external dependency. --- README.md | 2 - container/BUILD | 1 - container/go/cmd/zipper/BUILD | 34 +++++++++ container/go/cmd/zipper/zipper.go | 81 ++++++++++++++++++++ container/import.bzl | 3 +- container/layer.bzl | 3 +- skylib/zip.bzl | 112 +++++++++++++++++++--------- testing/default_toolchain/WORKSPACE | 10 --- toolchains/docker/toolchain.bzl | 9 +-- 9 files changed, 197 insertions(+), 58 deletions(-) create mode 100644 container/go/cmd/zipper/BUILD create mode 100644 container/go/cmd/zipper/zipper.go diff --git a/README.md b/README.md index 8b31dfe60..3ccfa0d99 100644 --- a/README.md +++ b/README.md @@ -142,10 +142,8 @@ docker_toolchain_configure( # Should be set explcitly for remote execution. docker_path="", # OPTIONAL: Path to the gzip binary. - # Either gzip_path or gzip_target should be set explcitly for remote execution. gzip_path="", # OPTIONAL: Bazel target for the gzip tool. - # Either gzip_path or gzip_target should be set explcitly for remote execution. gzip_target="", # OPTIONAL: Path to the xz binary. # Should be set explcitly for remote execution. diff --git a/container/BUILD b/container/BUILD index 0931ed0cc..f3f300731 100644 --- a/container/BUILD +++ b/container/BUILD @@ -179,7 +179,6 @@ bzl_library( "//skylib:filetype", "//skylib:label", "//skylib:path", - "//skylib:zip", "@bazel_tools//tools/build_defs/hash:hash.bzl", ], ) diff --git a/container/go/cmd/zipper/BUILD b/container/go/cmd/zipper/BUILD new file mode 100644 index 000000000..bcdea3c6c --- /dev/null +++ b/container/go/cmd/zipper/BUILD @@ -0,0 +1,34 @@ +# Copyright 2020 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ### +# Build file for zipper binary. + +load("@bazel_gazelle//:def.bzl", "gazelle") +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") + +# gazelle:prefix github.com/bazelbuild/rules_docker +gazelle(name = "gazelle") + +go_binary( + name = "zipper", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) + +go_library( + name = "go_default_library", + srcs = ["zipper.go"], + importpath = "github.com/bazelbuild/rules_docker", + visibility = ["//visibility:private"], +) diff --git a/container/go/cmd/zipper/zipper.go b/container/go/cmd/zipper/zipper.go new file mode 100644 index 000000000..18d249e00 --- /dev/null +++ b/container/go/cmd/zipper/zipper.go @@ -0,0 +1,81 @@ +// Copyright 2020 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//////////////////////////////////////////////// +// This package performs gzip operations. + +package main + +import ( + "bytes" + "compress/gzip" + "flag" + "io/ioutil" + "log" + "os" +) + +var ( + src = flag.String("src", "", "The source location of the file to zip/unzip.") + dst = flag.String("dst", "", "The destination location of the file, after zip/unzip.") + decompress = flag.Bool("decompress", false, "If true, perform gunzip. If false, gzip.") + fast = flag.Bool("fast", false, "If true, use the fastest compression level.") +) + +func main() { + flag.Parse() + + if *src == "" { + log.Fatalln("Required option -src was not specified.") + } + + if *dst == "" { + log.Fatalln("Required option -dst was not specified.") + } + + dat, err := ioutil.ReadFile(*src) + if err != nil { + log.Fatalf("Unable to read input file: %v", err) + } + var buf bytes.Buffer + if *decompress { + zr, err := gzip.NewReader(bytes.NewReader(dat)) + if err != nil { + log.Fatalf("Unable to read: %v", err) + } + if _, err := buf.ReadFrom(zr); err != nil { + log.Fatalf("Unable to decompress: %v", err) + } + if err := zr.Close(); err != nil { + log.Fatalf("Unable to close gzip reader: %v", err) + } + } else { + level := gzip.DefaultCompression + if *fast { + level = gzip.BestSpeed + } + zw, err := gzip.NewWriterLevel(&buf, level) + if err != nil { + log.Fatalf("Unable to create gzip writer: %v", err) + } + if _, err := zw.Write(dat); err != nil { + log.Fatalf("Unable to compress: %v", err) + } + if err := zw.Close(); err != nil { + log.Fatalf("Unable to close gzip writer: %s", err) + } + } + if err := ioutil.WriteFile(*dst, buf.Bytes(), os.ModePerm); err != nil { + log.Fatalf("Unable to write to %q: %v", *dst, err) + } +} diff --git a/container/import.bzl b/container/import.bzl index 955da9163..feb9df50c 100644 --- a/container/import.bzl +++ b/container/import.bzl @@ -39,6 +39,7 @@ load( "//skylib:zip.bzl", _gunzip = "gunzip", _gzip = "gzip", + _zip_tools = "tools", ) def _is_filetype(filename, extensions): @@ -178,7 +179,7 @@ container_import = rule( mandatory = False, ), "repository": attr.string(default = "bazel"), - }, _hash_tools, _layer_tools), + }, _hash_tools, _layer_tools, _zip_tools), executable = True, outputs = { "out": "%{name}.tar", diff --git a/container/layer.bzl b/container/layer.bzl index 46c52be50..426299be6 100644 --- a/container/layer.bzl +++ b/container/layer.bzl @@ -39,6 +39,7 @@ load( load( "//skylib:zip.bzl", _gzip = "gzip", + _zip_tools = "tools", ) _DEFAULT_MTIME = -1 @@ -288,7 +289,7 @@ _layer_attrs = dicts.add({ "portable_mtime": attr.bool(default = False), "symlinks": attr.string_dict(), "tars": attr.label_list(allow_files = tar_filetype), -}, _hash_tools, _layer_tools) +}, _hash_tools, _layer_tools, _zip_tools) _layer_outputs = { "layer": "%{name}-layer.tar", diff --git a/skylib/zip.bzl b/skylib/zip.bzl index 4644c48e2..fbf6f20d8 100644 --- a/skylib/zip.bzl +++ b/skylib/zip.bzl @@ -13,6 +13,20 @@ # limitations under the License. """Functions for producing the gzip of an artifact.""" +def _gzip_path(toolchain_info): + """Resolve the user-supplied gzip path, if any. + + Args: + toolchain_info: The DockerToolchainInfo + + Returns: + Path to gzip, or empty string. + """ + if toolchain_info.gzip_target: + return toolchain_info.gzip_target.files_to_run.executable.path + else: + return toolchain_info.gzip_path + def gzip(ctx, artifact, options = None): """Create an action to compute the gzipped artifact. @@ -26,26 +40,34 @@ def gzip(ctx, artifact, options = None): """ out = ctx.actions.declare_file(artifact.basename + ".gz") toolchain_info = ctx.toolchains["@io_bazel_rules_docker//toolchains/docker:toolchain_type"].info - input_manifests = [] - tools = [] - gzip_path = toolchain_info.gzip_path - if toolchain_info.gzip_target: - gzip_path = toolchain_info.gzip_target.files_to_run.executable.path - tools, _, input_manifests = ctx.resolve_command(tools = [toolchain_info.gzip_target]) - elif toolchain_info.gzip_path == "": - fail("gzip could not be found. Make sure it is in the path or set it " + - "explicitly in the docker_toolchain_configure") + gzip_path = _gzip_path(toolchain_info) + + if gzip_path == "": + # If the user did not specify a gzip tool, use the Go helper provided with rules_docker. + ctx.actions.run( + executable = ctx.executable._zipper, + arguments = ["-src", artifact.path, "-dst", out.path] + (options or []), + inputs = [artifact], + outputs = [out], + mnemonic = "GZIP", + tools = ctx.attr._zipper[DefaultInfo].default_runfiles.files, + ) + else: + input_manifests = [] + tools = [] + if toolchain_info.gzip_target: + tools, _, input_manifests = ctx.resolve_command(tools = [toolchain_info.gzip_target]) - opt_str = " ".join([repr(o) for o in (options or [])]) - ctx.actions.run_shell( - command = "%s -n %s < %s > %s" % (gzip_path, opt_str, artifact.path, out.path), - input_manifests = input_manifests, - inputs = [artifact], - outputs = [out], - use_default_shell_env = True, - mnemonic = "GZIP", - tools = tools, - ) + opt_str = " ".join([repr(o) for o in (options or [])]) + ctx.actions.run_shell( + command = "%s -n %s < %s > %s" % (gzip_path, opt_str, artifact.path, out.path), + input_manifests = input_manifests, + inputs = [artifact], + outputs = [out], + use_default_shell_env = True, + mnemonic = "GZIP", + tools = tools, + ) return out def gunzip(ctx, artifact): @@ -60,22 +82,38 @@ def gunzip(ctx, artifact): """ out = ctx.actions.declare_file(artifact.basename + ".nogz") toolchain_info = ctx.toolchains["@io_bazel_rules_docker//toolchains/docker:toolchain_type"].info - input_manifests = [] - tools = [] - gzip_path = toolchain_info.gzip_path - if toolchain_info.gzip_target: - gzip_path = toolchain_info.gzip_target.files_to_run.executable.path - tools, _, input_manifests = ctx.resolve_command(tools = [toolchain_info.gzip_target]) - elif toolchain_info.gzip_path == "": - fail("gzip could not be found. Make sure it is in the path or set it " + - "explicitly in the docker_toolchain_configure") - ctx.actions.run_shell( - command = "%s -d < %s > %s" % (gzip_path, artifact.path, out.path), - input_manifests = input_manifests, - inputs = [artifact], - outputs = [out], - use_default_shell_env = True, - mnemonic = "GUNZIP", - tools = tools, - ) + gzip_path = _gzip_path(toolchain_info) + + if gzip_path == "": + # If the user did not specify a gzip tool, use the Go helper provided with rules_docker. + ctx.actions.run( + executable = ctx.executable._zipper, + arguments = ["-decompress", "-src", artifact.path, "-dst", out.path], + inputs = [artifact], + outputs = [out], + mnemonic = "GUNZIP", + tools = ctx.attr._zipper[DefaultInfo].default_runfiles.files, + ) + else: + input_manifests = [] + tools = [] + if toolchain_info.gzip_target: + tools, _, input_manifests = ctx.resolve_command(tools = [toolchain_info.gzip_target]) + ctx.actions.run_shell( + command = "%s -d < %s > %s" % (gzip_path, artifact.path, out.path), + input_manifests = input_manifests, + inputs = [artifact], + outputs = [out], + use_default_shell_env = True, + mnemonic = "GUNZIP", + tools = tools, + ) return out + +tools = { + "_zipper": attr.label( + default = Label("//container/go/cmd/zipper"), + cfg = "host", + executable = True, + ), +} diff --git a/testing/default_toolchain/WORKSPACE b/testing/default_toolchain/WORKSPACE index 737dd5f0f..d94b0284b 100644 --- a/testing/default_toolchain/WORKSPACE +++ b/testing/default_toolchain/WORKSPACE @@ -21,15 +21,6 @@ local_repository( path = "replace/this/path", ) -load( - "//:local_tool.bzl", - "local_tool", -) - -local_tool( - name = "gzip", -) - load( "@io_bazel_rules_docker//toolchains/docker:toolchain.bzl", docker_toolchain_configure = "toolchain_configure", @@ -37,7 +28,6 @@ load( docker_toolchain_configure( name = "docker_config", - gzip_target = "@gzip//:gzip", ) load( diff --git a/toolchains/docker/toolchain.bzl b/toolchains/docker/toolchain.bzl index 821932e58..a4166831d 100644 --- a/toolchains/docker/toolchain.bzl +++ b/toolchains/docker/toolchain.bzl @@ -24,7 +24,7 @@ DockerToolchainInfo = provider( "will be used. DOCKER_CONFIG is not defined, the " + "home directory will be used.", "docker_flags": "Additional flags to the docker command", - "gzip_path": "Optional path to the gzip binary. If not set found via which.", + "gzip_path": "Optional path to the gzip binary.", "gzip_target": "Optional Bazel target for the gzip tool. " + "Should only be set if gzip_path is unset.", "tool_path": "Path to the docker executable", @@ -104,8 +104,6 @@ def _toolchain_configure_impl(repository_ctx): gzip_attr = "gzip_target = \"%s\"," % repository_ctx.attr.gzip_target elif repository_ctx.attr.gzip_path: gzip_attr = "gzip_path = \"%s\"," % repository_ctx.attr.gzip_path - elif repository_ctx.which("gzip"): - gzip_attr = "gzip_path = \"%s\"," % repository_ctx.which("gzip") docker_flags = [] docker_flags += repository_ctx.attr.docker_flags @@ -161,9 +159,8 @@ toolchain_configure = repository_rule( ), "gzip_path": attr.string( mandatory = False, - doc = "The full path to the gzip binary. If not specified, it will " + - "be searched for in the path. If not available, running commands " + - "that use gzip will fail.", + doc = "The full path to the gzip binary. If not specified, a tool will " + + "be compiled and used.", ), "gzip_target": attr.label( executable = True, From c15777ac3857bdb0e9721297c9fa6a90c10f7b34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Aub=C3=A9?= Date: Wed, 2 Sep 2020 15:34:23 +0000 Subject: [PATCH 2/9] Update some test digests. --- tests/container/BUILD | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/container/BUILD b/tests/container/BUILD index 8da4f20ba..855f135e6 100644 --- a/tests/container/BUILD +++ b/tests/container/BUILD @@ -238,13 +238,13 @@ container_push( # BEGIN_DO_NOT_IMPORT file_test( name = "test_digest_output1", - content = "sha256:18e27a73e3faabc819f903fc8e603de2bfd49813e231c2e8ce98b533577450e2", + content = "sha256:4ec78686f1edd4f30cca6222f17ace5a2b9f8db9708727c3a140c954af0a534d", file = ":deb_image_with_dpkgs.digest", ) file_test( name = "test_digest_output2", - content = "sha256:8da7a9421c68be888336492184d54b42db91a0b1f2096d85f4eaf3a25b7f234e", + content = "sha256:cf3bc4bfee0707fc15ee50b27af6dcb0fe6c0585a6ba89870736ac8f72c702e9", file = ":null_cmd_and_entrypoint_none.digest", ) # END_DO_NOT_IMPORT @@ -585,7 +585,7 @@ container_push( # BEGIN_DO_NOT_IMPORT file_test( name = "test_push_digest_output", - content = "sha256:0149d475d8e25903446311d3da4a4aa54f348103b84c43b6ad3351484fdaff67", + content = "sha256:ea10d597d15d62e3a33ea378f75b08d9afc332f685495900724b857a8b9d2d8a", file = ":push_test.digest", ) @@ -610,13 +610,13 @@ file_test( file_test( name = "alpine_custom_attr_digest_test", - content = "sha256:6a7bd5d833271be520664af1391e399b2fe10ae7647ee091ee24e2e2774d38d6", + content = "sha256:2f3bf2d061a1bab130cdd98f10b73dfceb08bfbf94a610c62f20c787073e5178", file = ":new_push_test_legacy_from_container_img.digest", ) file_test( name = "new_alpine_linux_armv6_image_tar_digest_test", - content = "sha256:7b2554a0a34e7ba53f5e70f4b1ed1f8515d9f9d34b0dd75e33fe8fc6384a04c5", + content = "sha256:249804db68a05bb88cc353d4aff090b084d8b8312665638d2f8d7bdaeaa423b3", file = ":new_alpine_linux_armv6_image_tar.digest", ) From 1da202ae559e26634ac03ac4530591b4a79115e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Aub=C3=A9?= Date: Wed, 2 Sep 2020 15:59:18 +0000 Subject: [PATCH 3/9] Run buildifier. --- skylib/zip.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/skylib/zip.bzl b/skylib/zip.bzl index fbf6f20d8..c8c248268 100644 --- a/skylib/zip.bzl +++ b/skylib/zip.bzl @@ -41,7 +41,7 @@ def gzip(ctx, artifact, options = None): out = ctx.actions.declare_file(artifact.basename + ".gz") toolchain_info = ctx.toolchains["@io_bazel_rules_docker//toolchains/docker:toolchain_type"].info gzip_path = _gzip_path(toolchain_info) - + if gzip_path == "": # If the user did not specify a gzip tool, use the Go helper provided with rules_docker. ctx.actions.run( @@ -83,7 +83,7 @@ def gunzip(ctx, artifact): out = ctx.actions.declare_file(artifact.basename + ".nogz") toolchain_info = ctx.toolchains["@io_bazel_rules_docker//toolchains/docker:toolchain_type"].info gzip_path = _gzip_path(toolchain_info) - + if gzip_path == "": # If the user did not specify a gzip tool, use the Go helper provided with rules_docker. ctx.actions.run( From 88f50bb27a02043a4c9f5b22d1deeabe8f52835b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Aub=C3=A9?= Date: Thu, 3 Sep 2020 12:30:07 +0000 Subject: [PATCH 4/9] gazelle() calls are unnecessary. --- container/go/cmd/zipper/BUILD | 4 ---- 1 file changed, 4 deletions(-) diff --git a/container/go/cmd/zipper/BUILD b/container/go/cmd/zipper/BUILD index bcdea3c6c..c6effcf73 100644 --- a/container/go/cmd/zipper/BUILD +++ b/container/go/cmd/zipper/BUILD @@ -14,12 +14,8 @@ # ### # Build file for zipper binary. -load("@bazel_gazelle//:def.bzl", "gazelle") load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") -# gazelle:prefix github.com/bazelbuild/rules_docker -gazelle(name = "gazelle") - go_binary( name = "zipper", embed = [":go_default_library"], From 365fec3b98f51553b9634d696b0affa7bc5603dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Aub=C3=A9?= Date: Thu, 3 Sep 2020 13:48:10 +0000 Subject: [PATCH 5/9] Use io.Copy to avoid loading the entire file into memory. --- container/go/cmd/zipper/zipper.go | 48 ++++++++++++++++++------------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/container/go/cmd/zipper/zipper.go b/container/go/cmd/zipper/zipper.go index 18d249e00..6f09f1a60 100644 --- a/container/go/cmd/zipper/zipper.go +++ b/container/go/cmd/zipper/zipper.go @@ -17,10 +17,9 @@ package main import ( - "bytes" "compress/gzip" "flag" - "io/ioutil" + "io" "log" "os" ) @@ -43,39 +42,48 @@ func main() { log.Fatalln("Required option -dst was not specified.") } - dat, err := ioutil.ReadFile(*src) + var copy_from io.Reader + var copy_to io.Writer + + f, err := os.Open(*src) if err != nil { log.Fatalf("Unable to read input file: %v", err) } - var buf bytes.Buffer + t, err := os.Create(*dst) + if err != nil { + log.Fatalf("Unable to create output file: %v", err) + } + if *decompress { - zr, err := gzip.NewReader(bytes.NewReader(dat)) + zr, err := gzip.NewReader(f) if err != nil { log.Fatalf("Unable to read: %v", err) } - if _, err := buf.ReadFrom(zr); err != nil { - log.Fatalf("Unable to decompress: %v", err) - } - if err := zr.Close(); err != nil { - log.Fatalf("Unable to close gzip reader: %v", err) - } + defer func() { + if err := zr.Close(); err != nil { + log.Fatalf("Unable to close gzip reader: %v", err) + } + }() + copy_from = zr + copy_to = t } else { level := gzip.DefaultCompression if *fast { level = gzip.BestSpeed } - zw, err := gzip.NewWriterLevel(&buf, level) + zw, err := gzip.NewWriterLevel(t, level) if err != nil { log.Fatalf("Unable to create gzip writer: %v", err) } - if _, err := zw.Write(dat); err != nil { - log.Fatalf("Unable to compress: %v", err) - } - if err := zw.Close(); err != nil { - log.Fatalf("Unable to close gzip writer: %s", err) - } + defer func() { + if err := zw.Close(); err != nil { + log.Fatalf("Unable to close gzip writer: %v", err) + } + }() + copy_from = f + copy_to = zw } - if err := ioutil.WriteFile(*dst, buf.Bytes(), os.ModePerm); err != nil { - log.Fatalf("Unable to write to %q: %v", *dst, err) + if _, err := io.Copy(copy_to, copy_from); err != nil { + log.Fatalf("Unable to perform the gzip operation from %q to %q: %v", *src, *dst, err) } } From 8c4f01e18266e2b69a8d95ecbe667f1839666ce8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Aub=C3=A9?= Date: Thu, 3 Sep 2020 18:32:30 +0000 Subject: [PATCH 6/9] Merge gzip() and gunzip() since they have more in common than not. --- skylib/zip.bzl | 73 +++++++++++++++++++++++--------------------------- 1 file changed, 34 insertions(+), 39 deletions(-) diff --git a/skylib/zip.bzl b/skylib/zip.bzl index c8c248268..9cc5cd438 100644 --- a/skylib/zip.bzl +++ b/skylib/zip.bzl @@ -27,47 +27,68 @@ def _gzip_path(toolchain_info): else: return toolchain_info.gzip_path -def gzip(ctx, artifact, options = None): - """Create an action to compute the gzipped artifact. +def _gzip(ctx, artifact, out, decompress, options, mnemonic): + """A helper that calls either the compiled zipper, or the gzip tool. Args: ctx: The context - artifact: The artifact to zip - options: str list, Command-line options to pass to gzip. + artifact: The artifact to zip/unzip + out: The output file. + decompress: Whether to decompress (True) or compress (False) + options: Additional options for compression only. Returns: - the gzipped artifact. + Path to gzip, or empty string. """ - out = ctx.actions.declare_file(artifact.basename + ".gz") toolchain_info = ctx.toolchains["@io_bazel_rules_docker//toolchains/docker:toolchain_type"].info gzip_path = _gzip_path(toolchain_info) - if gzip_path == "": - # If the user did not specify a gzip tool, use the Go helper provided with rules_docker. + if not gzip_path: + # The user did not specify a gzip tool; use the Go helper provided with rules_docker. ctx.actions.run( executable = ctx.executable._zipper, - arguments = ["-src", artifact.path, "-dst", out.path] + (options or []), + arguments = ["-src", artifact.path, "-dst", out.path] + ( + ["-decompress"] if decompress else [] + ) + (options or []), inputs = [artifact], outputs = [out], - mnemonic = "GZIP", + mnemonic = mnemonic, tools = ctx.attr._zipper[DefaultInfo].default_runfiles.files, ) else: + # Call the gzip path or target supplied by the user. input_manifests = [] tools = [] if toolchain_info.gzip_target: tools, _, input_manifests = ctx.resolve_command(tools = [toolchain_info.gzip_target]) opt_str = " ".join([repr(o) for o in (options or [])]) + command = "%s -d %s < %s > %s" if decompress else "%s -n %s < %s > %s" + command = command % (gzip_path, opt_str, artifact.path, out.path) + ctx.actions.run_shell( - command = "%s -n %s < %s > %s" % (gzip_path, opt_str, artifact.path, out.path), + command = command, input_manifests = input_manifests, inputs = [artifact], outputs = [out], use_default_shell_env = True, - mnemonic = "GZIP", + mnemonic = mnemonic, tools = tools, ) + +def gzip(ctx, artifact, options = None): + """Create an action to compute the gzipped artifact. + + Args: + ctx: The context + artifact: The artifact to zip + options: str list, Command-line options to pass to gzip. + + Returns: + the gzipped artifact. + """ + out = ctx.actions.declare_file(artifact.basename + ".gz") + _gzip(ctx, artifact, out, False, options, "GZIP") return out def gunzip(ctx, artifact): @@ -81,33 +102,7 @@ def gunzip(ctx, artifact): the gunzipped artifact. """ out = ctx.actions.declare_file(artifact.basename + ".nogz") - toolchain_info = ctx.toolchains["@io_bazel_rules_docker//toolchains/docker:toolchain_type"].info - gzip_path = _gzip_path(toolchain_info) - - if gzip_path == "": - # If the user did not specify a gzip tool, use the Go helper provided with rules_docker. - ctx.actions.run( - executable = ctx.executable._zipper, - arguments = ["-decompress", "-src", artifact.path, "-dst", out.path], - inputs = [artifact], - outputs = [out], - mnemonic = "GUNZIP", - tools = ctx.attr._zipper[DefaultInfo].default_runfiles.files, - ) - else: - input_manifests = [] - tools = [] - if toolchain_info.gzip_target: - tools, _, input_manifests = ctx.resolve_command(tools = [toolchain_info.gzip_target]) - ctx.actions.run_shell( - command = "%s -d < %s > %s" % (gzip_path, artifact.path, out.path), - input_manifests = input_manifests, - inputs = [artifact], - outputs = [out], - use_default_shell_env = True, - mnemonic = "GUNZIP", - tools = tools, - ) + _gzip(ctx, artifact, out, True, None, "GUNZIP") return out tools = { From 36a9d4e160e8717622253d3c97df30f527478aae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Aub=C3=A9?= Date: Thu, 3 Sep 2020 18:39:42 +0000 Subject: [PATCH 7/9] Minor documentation fixes. --- skylib/zip.bzl | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/skylib/zip.bzl b/skylib/zip.bzl index 9cc5cd438..a05b6e0cf 100644 --- a/skylib/zip.bzl +++ b/skylib/zip.bzl @@ -35,10 +35,7 @@ def _gzip(ctx, artifact, out, decompress, options, mnemonic): artifact: The artifact to zip/unzip out: The output file. decompress: Whether to decompress (True) or compress (False) - options: Additional options for compression only. - - Returns: - Path to gzip, or empty string. + options: str list, Command-line options. """ toolchain_info = ctx.toolchains["@io_bazel_rules_docker//toolchains/docker:toolchain_type"].info gzip_path = _gzip_path(toolchain_info) From a2150305035a310e24bfe3e34f49bdab8878cb4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Aub=C3=A9?= Date: Thu, 3 Sep 2020 21:13:31 +0000 Subject: [PATCH 8/9] Make arguments more explicit. --- skylib/zip.bzl | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/skylib/zip.bzl b/skylib/zip.bzl index a05b6e0cf..0bff9a493 100644 --- a/skylib/zip.bzl +++ b/skylib/zip.bzl @@ -85,7 +85,14 @@ def gzip(ctx, artifact, options = None): the gzipped artifact. """ out = ctx.actions.declare_file(artifact.basename + ".gz") - _gzip(ctx, artifact, out, False, options, "GZIP") + _gzip( + ctx = ctx, + artifact = artifact, + out = out, + decompress = False, + options = options, + mnemonic = "GZIP", + ) return out def gunzip(ctx, artifact): @@ -99,7 +106,14 @@ def gunzip(ctx, artifact): the gunzipped artifact. """ out = ctx.actions.declare_file(artifact.basename + ".nogz") - _gzip(ctx, artifact, out, True, None, "GUNZIP") + _gzip( + ctx = ctx, + artifact = artifact, + out = out, + decompress = True, + options = None, + mnemonic = "GUNZIP", + ) return out tools = { From 4f0e32012562e286bf1894502f62111dd5ec76fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Aub=C3=A9?= Date: Tue, 8 Sep 2020 17:29:11 +0000 Subject: [PATCH 9/9] Fix a lint warning with the function docstring --- skylib/zip.bzl | 1 + 1 file changed, 1 insertion(+) diff --git a/skylib/zip.bzl b/skylib/zip.bzl index 0bff9a493..b8a7d6d1a 100644 --- a/skylib/zip.bzl +++ b/skylib/zip.bzl @@ -36,6 +36,7 @@ def _gzip(ctx, artifact, out, decompress, options, mnemonic): out: The output file. decompress: Whether to decompress (True) or compress (False) options: str list, Command-line options. + mnemonic: A one-word description of the action """ toolchain_info = ctx.toolchains["@io_bazel_rules_docker//toolchains/docker:toolchain_type"].info gzip_path = _gzip_path(toolchain_info)