Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update SHA for go-containerregistry #1815

Merged
merged 5 commits into from
Apr 24, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion .bazelci/presubmit.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
---
buildifier:
version: 0.22.0
version: 4.0.1
# Check for issues with the format of our bazel config files.
# All warnings from https://github.com/bazelbuild/buildtools/blob/master/WARNINGS.md
# are enabled except:
Expand Down Expand Up @@ -205,6 +205,10 @@ platforms:
- "..."
# tests/docker/security require gcloud setup to access asci-toolchains images
- "-//tests/docker/security/..."
# Failing with
# Unable to load docker image from bazel-out/k8-fastbuild/bin/tests/docker/util/test_container_commit.build: archive/tar: invalid tar header
# TODO(alexeagle): re-enable after investigating
- "-//tests/docker/util:all"
# contrib targets are not compatible with remote exec
- "-//tests/contrib:derivative_with_volume_repro_test"
- "-//tests/contrib:random_file_img_non_repro_test"
Expand All @@ -227,6 +231,10 @@ platforms:
- "//tests/..."
# tests/docker/security require gcloud setup to access asci-toolchains images
- "-//tests/docker/security/..."
# Failing with
# Unable to load docker image from bazel-out/k8-fastbuild/bin/tests/docker/util/test_container_commit.build: archive/tar: invalid tar header
# TODO(alexeagle): re-enable after investigating
- "-//tests/docker/util:all"
# contrib tests are not compatible with remote exec
- "-//tests/contrib:derivative_with_volume_repro_test"
- "-//tests/contrib:random_file_img_non_repro_test"
Expand Down
6 changes: 0 additions & 6 deletions WORKSPACE
Original file line number Diff line number Diff line change
Expand Up @@ -112,12 +112,6 @@ http_file(
urls = ["https://bazel.build/bazel-release.pub.gpg"],
)

http_file(
name = "launchpad_openjdk_gpg",
sha256 = "e9a596d0c194a562be9fd2c2a0994d7885505a1145fed0fbd5ae4c11d56220a0",
urls = ["http://keyserver.ubuntu.com/pks/lookup?op=get&fingerprint=on&search=0xEB9B1D8886F44E2A"],
)

container_load(
name = "pause_tar",
file = "//testdata:pause.tar",
Expand Down
6 changes: 3 additions & 3 deletions container/bundle.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -54,12 +54,12 @@ def _container_bundle_impl(ctx):

layer = _get_layers(ctx, ctx.label.name, image_target_dict[target])
images[tag] = layer
runfiles += [layer.get("config")]
runfiles += [layer.get("config_digest")]
runfiles.append(layer.get("config"))
runfiles.append(layer.get("config_digest"))
runfiles += layer.get("unzipped_layer", [])
runfiles += layer.get("diff_id", [])
if layer.get("legacy"):
runfiles += [layer.get("legacy")]
runfiles.append(layer.get("legacy"))

_incr_load(
ctx,
Expand Down
4 changes: 2 additions & 2 deletions container/image.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -164,11 +164,11 @@ def _add_create_image_config_args(

if base_config:
args.add("-baseConfig", base_config)
inputs += [base_config]
inputs.append(base_config)

if base_manifest:
args.add("-baseManifest", base_manifest)
inputs += [base_manifest]
inputs.append(base_manifest)

if architecture:
args.add("-architecture", architecture)
Expand Down
8 changes: 4 additions & 4 deletions container/import.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -76,10 +76,10 @@ def _container_import_impl(ctx):
diff_ids = []
for layer in ctx.files.layers:
zipped, unzipped = _layer_pair(ctx, layer)
zipped_layers += [zipped]
unzipped_layers += [unzipped]
blobsums += [_sha256(ctx, zipped)]
diff_ids += [_sha256(ctx, unzipped)]
zipped_layers.append(zipped)
unzipped_layers.append(unzipped)
blobsums.append(_sha256(ctx, zipped))
diff_ids.append(_sha256(ctx, unzipped))

manifest = None
manifest_digest = None
Expand Down
22 changes: 11 additions & 11 deletions container/layer_tools.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -129,11 +129,11 @@ def _add_join_layers_args(args, inputs, images):
for tag in images:
image = images[tag]
args.add(image["config"], format = "--tag=" + tag + "=%s")
inputs += [image["config"]]
inputs.append(image["config"])

if image.get("manifest"):
args.add(image["manifest"], format = "--basemanifest=" + tag + "=%s")
inputs += [image["manifest"]]
inputs.append(image["manifest"])

for i in range(0, len(image["diff_id"])):
# There's no way to do this with attrs w/o resolving paths here afaik
Expand All @@ -152,7 +152,7 @@ def _add_join_layers_args(args, inputs, images):

if image.get("legacy"):
args.add("--tarball", image["legacy"])
inputs += [image["legacy"]]
inputs.append(image["legacy"])

def assemble(
ctx,
Expand Down Expand Up @@ -229,15 +229,15 @@ def incremental_load(

# First load the legacy base image, if it exists.
if image.get("legacy"):
load_statements += [
load_statements.append(
"load_legacy '%s'" % _get_runfile_path(ctx, image["legacy"]),
]
)

pairs = zip(image["diff_id"], image["unzipped_layer"])

# Import the config and the subset of layers not present
# in the daemon.
load_statements += [
load_statements.append(
"import_config '%s' %s" % (
_get_runfile_path(ctx, image["config"]),
" ".join([
Expand All @@ -248,24 +248,24 @@ def incremental_load(
for (diff_id, unzipped_layer) in pairs
]),
),
]
)

# Now tag the imported config with the specified tag.
tag_reference = tag if not stamp else tag.replace("{", "${")
tag_statements += [
tag_statements.append(
"tag_layer \"%s\" '%s'" % (
# Turn stamp variable references into bash variables.
# It is notable that the only legal use of '{' in a
# tag would be for stamp variables, '$' is not allowed.
tag_reference,
_get_runfile_path(ctx, image["config_digest"]),
),
]
)
if run:
# Args are embedded into the image, so omitted here.
run_statements += [
run_statements.append(
"\"${DOCKER}\" ${DOCKER_FLAGS} run %s %s" % (run_flags, tag_reference),
]
)

ctx.actions.expand_template(
template = ctx.file.incremental_load_template,
Expand Down
2 changes: 1 addition & 1 deletion container/push.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ def _impl(ctx):
))

if ctx.attr.skip_unchanged_digest:
pusher_args += ["-skip-unchanged-digest"]
pusher_args.append("-skip-unchanged-digest")
digester_args += ["--dst", str(ctx.outputs.digest.path), "--format", str(ctx.attr.format)]
ctx.actions.run(
inputs = digester_input,
Expand Down
2 changes: 1 addition & 1 deletion contrib/compare_ids_test.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ def _compare_ids_test_impl(ctx):
tar_files = []
for file in ctx.files.images:
if file.short_path.endswith("tar"):
tar_files += [file]
tar_files.append(file)

if (len(tar_files) == 0):
fail("No images provided for test.")
Expand Down
4 changes: 2 additions & 2 deletions contrib/push-all.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,8 @@ def _impl(ctx):
is_executable = True,
)

scripts += [out]
runfiles += [out]
scripts.append(out)
runfiles.append(out)
runfiles += pusher_inputs

ctx.actions.expand_template(
Expand Down
2 changes: 1 addition & 1 deletion docker/package_managers/install_pkgs.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -196,8 +196,8 @@ _attrs = {
}

_outputs = {
"out": "%{name}.tar",
"build_script": "%{name}.build",
"out": "%{name}.tar",
}

# Export install_pkgs rule for other bazel rules to depend on.
Expand Down
2 changes: 1 addition & 1 deletion docker/util/run.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -267,8 +267,8 @@ _commit_attrs = {
),
}
_commit_outputs = {
"out": "%{name}_commit.tar",
"build": "%{name}.build",
"out": "%{name}_commit.tar",
}

container_run_and_commit = rule(
Expand Down
6 changes: 3 additions & 3 deletions java/image.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,8 @@ jar_app_layer = rule(
# https://github.com/bazelbuild/bazel/issues/2176
"data_path": attr.string(default = "."),

# The rest of the dependencies.
"deps": attr.label_list(),
# Override the defaults.
"directory": attr.string(default = "/app"),
# The full list of dependencies that have their own layers
Expand All @@ -239,10 +241,8 @@ jar_app_layer = rule(
"legacy_run_behavior": attr.bool(default = False),
# The main class to invoke on startup.
"main_class": attr.string(mandatory = False),
"workdir": attr.string(default = ""),
"runtime_deps": attr.label_list(),
# The rest of the dependencies.
"deps": attr.label_list(),
"workdir": attr.string(default = ""),

# Whether the classpath should be passed as a file.
"_classpath_as_file": attr.bool(default = False),
Expand Down
2 changes: 1 addition & 1 deletion repositories/go_repositories.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ def go_deps():
name = "com_github_google_go_containerregistry",
urls = ["https://api.github.com/repos/google/go-containerregistry/tarball/8a2841911ffee4f6892ca0083e89752fb46c48dd"], # v0.1.4
strip_prefix = "google-go-containerregistry-8a28419",
sha256 = "60b9a600affa5667bd444019a4e218b7752d8500cfa923c1ac54ce2f88f773e2",
sha256 = "cadb09cb5bcbe00688c73d716d1c9e774d6e4959abec4c425a1b995faf33e964",

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not opposed to updating the sha, but is there nay concern the artifact was tampered with?

I'm not sure if this is relevant, but neither sha matches the shas of the artifacts for that version in the regular non-api project: #1814 (comment)

cc @alexeagle

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@alexeagle @umeshkumhar thoughts on this? Do we know why the sha suddenly changed? And why it differs from those in the repo's release page?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@SrodriguezO Not 100% sure, but this issue is raised with this commit ID. In this PR, you can see ko version is changed.

There are some reported issues where ko on version change has changed the tar hashes, which impacted SHA checks..More info here:
kubernetes/kubernetes#99376

One another possibility can be that the maintainer has re-published/ updated tarball, which caused SHA change, which I don't think have happened.

I have not explored to the root cause. Let me know if that helps

Copy link
Collaborator

@uhthomas uhthomas Apr 24, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi,

I'm facing an issue where using rules_docker v0.16.0 results in:

Checksum was cadb09cb5bcbe00688c73d716d1c9e774d6e4959abec4c425a1b995faf33e964 but wanted 60b9a600affa5667bd444019a4e218b7752d8500cfa923c1ac54ce2f88f773e2

The root cause is explained in ko-build/ko#315 and kubernetes/kubernetes#99376.

The TL;DR is that Kubernetes was using short hashes for versions, which may change in length over time. The hash is injected into the source through git archive.

This SHA256 change should fix things for now, but will eventually break again. The real solution will be to update the dependency to a commit after kubernetes/kubernetes@ab221b5.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Further, my solution to this problem, without updating from rules_docker v0.16.0, has been to explicitly pull in github.com/google/go-containerregistry v0.4.1 via rules_go.

See https://github.com/uhthomas/6f.io/compare/8548f56df648693c0e518454cfccad0792e48eb8..3915a86d6bd9c6a376198b1d1d79979a7d4c542f

importpath = "github.com/google/go-containerregistry",
type = "tar.gz",
)
Expand Down
2 changes: 1 addition & 1 deletion skylib/path.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ def canonicalize(path):

# Strip ./ from the beginning if specified.
# There is no way to handle .// correctly (no function that would make
# that possible and Skylark is not turing complete) so just consider it
# that possible and Starlark is not turing complete) so just consider it
# as an absolute path. A path of / should preserve the entire
# path up to the repository root.

Expand Down
4 changes: 2 additions & 2 deletions tests/contrib/automatic_container_release/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.

licenses(["notice"]) # Apache 2.0

load(
"@bazel_tools//tools/build_rules:test_rules.bzl",
"file_test",
Expand All @@ -25,6 +23,8 @@ load(
load("//contrib/automatic_container_release:metadata_merge.bzl", "metadata_merge")
load("//contrib/automatic_container_release:packages_metadata.bzl", "packages_metadata")

licenses(["notice"]) # Apache 2.0

configs_test(
name = "configs_test",
dependency_update_specs = ["deps_spec.yaml"],
Expand Down
10 changes: 10 additions & 0 deletions tests/docker/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -15,3 +15,13 @@
package(default_visibility = ["//visibility:public"])

licenses(["notice"]) # Apache 2.0

# This file fetched from
# wget "http://keyserver.ubuntu.com/pks/lookup?op=get&fingerprint=on&search=0xEB9B1D8886F44E2A"
# because it was being served with unstable contents, causing our CI
# to fail 50% of the time.
filegroup(
name = "launchpad_openjdk_gpg",
srcs = ["launchpad_openjdk_gpg.pub"],
visibility = ["//tests/docker:__subpackages__"],
)
14 changes: 14 additions & 0 deletions tests/docker/launchpad_openjdk_gpg.pub
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
-----BEGIN PGP PUBLIC KEY BLOCK-----
Comment: Hostname:
Version: Hockeypuck ~unreleased

xo0ES8OmlQEEALy8ttT3KgmjoqCkeU7S04/j615RDivV0CHv+5mdJFQY10wo6v6k
rmXxK757eIoRDN1B3ztl8vjqLHxi8oA+f4inZcrrIZYjW0MghO+IcKEbKCrPUPjD
fQBSIvZD5ZaiibMNwZJ6TidS2r6nSAH+aoMRgXYycQPGkAHDcaPKY54zABEBAAHN
JExhdW5jaHBhZCBPcGVuSkRLIGJ1aWxkcyAoYWxsIGFyY2hzKcK2BBMBAgAgBQJL
w6aVAhsDBgsJCAcDAgQVAggDBBYCAwECHgECF4AACgkQ65sdiIb0TiraSgQAr+WD
+qGcW5yrkS2QBMq+g+Ew2tLqeDLw8Y2xZ4BhnRzB3WljBfwj7wt/KOugjJWaEWuY
GiuLBf6vy+At5VtXs0FSk6oIZR0XZvUVamFPXeTkZqUiw4oTNlBTeOhAxrevRWzn
mLuRAIWlr0dARJyLCK5MMVVhesouoG8WpOrniXs=
=JbYw
-----END PGP PUBLIC KEY BLOCK-----
6 changes: 3 additions & 3 deletions tests/docker/package_managers/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.

package(default_visibility = ["//visibility:public"])

load(
"@bazel_tools//tools/build_rules:test_rules.bzl",
"file_test",
Expand All @@ -25,6 +23,8 @@ load("//docker/package_managers:apt_key.bzl", "add_apt_key")
load("//docker/package_managers:download_pkgs.bzl", "download_pkgs")
load("//docker/package_managers:install_pkgs.bzl", "install_pkgs")

package(default_visibility = ["//visibility:public"])

download_pkgs(
name = "test_download_pkgs",
image_tar = "@ubuntu1604//:ubuntu1604_vanilla.tar",
Expand Down Expand Up @@ -106,7 +106,7 @@ add_apt_key(
image = "@debian9//:builder.tar",
keys = [
"@bazel_gpg//file",
"@launchpad_openjdk_gpg//file",
"//tests/docker:launchpad_openjdk_gpg",
],
)

Expand Down
2 changes: 1 addition & 1 deletion tests/docker/toolchain_container/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ container_test(
pkg_tar(
name = "test_tar",
files = [
"@launchpad_openjdk_gpg//file",
"//tests/docker:launchpad_openjdk_gpg",
],
)

Expand Down