From 69df93d16c5e31e31ec665132556be0c94de1aea Mon Sep 17 00:00:00 2001 From: Fabian Meumertzheim Date: Fri, 12 Apr 2024 12:33:27 +0200 Subject: [PATCH] Add support for `--incompatible_enable_proto_toolchain_resolution` --- .bazelci/presubmit.yml | 16 ++++++++++++ .bazelrc | 3 ++- MODULE.bazel | 2 +- WORKSPACE | 51 +++++++++++++++++++++++++++++++++++++ proto/BUILD.bazel | 1 + proto/compiler.bzl | 43 +++++++++++++++++++++++-------- proto/private/BUILD.bazel | 6 +++++ proto/private/toolchain.bzl | 47 ++++++++++++++++++++++++++++++++++ tests/bcr/.bazelversion | 2 +- tests/bcr/MODULE.bazel | 4 +++ 10 files changed, 162 insertions(+), 13 deletions(-) create mode 100644 proto/private/BUILD.bazel create mode 100644 proto/private/toolchain.bzl diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index f27efbb689..195e07bbcc 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -73,6 +73,22 @@ tasks: - "@go_default_sdk//..." test_targets: - "//..." + bcr_tests_proto: + name: BCR test module (--incompatible_enable_proto_toolchain_resolution) + platform: ${{ platform }} + bazel: 7.1.1 + working_directory: tests/bcr + build_flags: + - "--allow_yanked_versions=all" + - "--incompatible_enable_proto_toolchain_resolution" + test_flags: + - "--allow_yanked_versions=all" + - "--incompatible_enable_proto_toolchain_resolution" + build_targets: + - "//..." + - "@go_default_sdk//..." + test_targets: + - "//..." macos: shell_commands: - tests/core/cgo/generate_imported_dylib.sh diff --git a/.bazelrc b/.bazelrc index 2cc995f10c..931cacc20a 100644 --- a/.bazelrc +++ b/.bazelrc @@ -33,7 +33,8 @@ build:incompatible --incompatible_enforce_config_setting_visibility build:incompatible --incompatible_disallow_empty_glob build:incompatible --incompatible_disable_starlark_host_transitions build:incompatible --nolegacy_external_runfiles +build:incompatible --incompatible_enable_proto_toolchain_resolution # Also enable all incompatible flags in go_bazel_test by default. # TODO: Add --incompatible_disallow_empty_glob once # https://github.com/bazelbuild/bazel-gazelle/pull/1405 has been released. -test:incompatible --test_env=GO_BAZEL_TEST_BAZELFLAGS='--incompatible_load_proto_rules_from_bzl --incompatible_enable_cc_toolchain_resolution --incompatible_config_setting_private_default_visibility --incompatible_enforce_config_setting_visibility --incompatible_disable_starlark_host_transitions --nolegacy_external_runfiles' +test:incompatible --test_env=GO_BAZEL_TEST_BAZELFLAGS='--incompatible_load_proto_rules_from_bzl --incompatible_enable_cc_toolchain_resolution --incompatible_config_setting_private_default_visibility --incompatible_enforce_config_setting_visibility --incompatible_disable_starlark_host_transitions --nolegacy_external_runfiles --incompatible_enable_proto_toolchain_resolution' diff --git a/MODULE.bazel b/MODULE.bazel index 6329272c2d..25a9ccf319 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -10,7 +10,7 @@ module( bazel_dep(name = "bazel_features", version = "1.9.1", repo_name = "io_bazel_rules_go_bazel_features") bazel_dep(name = "bazel_skylib", version = "1.2.0") bazel_dep(name = "platforms", version = "0.0.4") -bazel_dep(name = "rules_proto", version = "4.0.0") +bazel_dep(name = "rules_proto", version = "6.0.0-rc2") bazel_dep(name = "protobuf", version = "3.19.2", repo_name = "com_google_protobuf") go_sdk = use_extension("//go:extensions.bzl", "go_sdk") diff --git a/WORKSPACE b/WORKSPACE index 849414cc35..352cfc800b 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -3,10 +3,61 @@ workspace(name = "io_bazel_rules_go") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies") +# Required by toolchains_protoc. +http_archive( + name = "platforms", + sha256 = "5eda539c841265031c2f82d8ae7a3a6490bd62176e0c038fc469eabf91f6149b", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/platforms/releases/download/0.0.9/platforms-0.0.9.tar.gz", + "https://github.com/bazelbuild/platforms/releases/download/0.0.9/platforms-0.0.9.tar.gz", + ], +) + +# The non-polyfill version of this is needed by rules_proto below. +http_archive( + name = "bazel_features", + sha256 = "d7787da289a7fb497352211ad200ec9f698822a9e0757a4976fd9f713ff372b3", + strip_prefix = "bazel_features-1.9.1", + url = "https://github.com/bazel-contrib/bazel_features/releases/download/v1.9.1/bazel_features-v1.9.1.tar.gz", +) + +load("@bazel_features//:deps.bzl", "bazel_features_deps") + +bazel_features_deps() + go_rules_dependencies() go_register_toolchains(version = "1.21.8") +http_archive( + name = "rules_proto", + sha256 = "71fdbed00a0709521ad212058c60d13997b922a5d01dbfd997f0d57d689e7b67", + strip_prefix = "rules_proto-6.0.0-rc2", + url = "https://github.com/bazelbuild/rules_proto/releases/download/6.0.0-rc2/rules_proto-6.0.0-rc2.tar.gz", +) + +load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies") + +rules_proto_dependencies() + +load("@rules_proto//proto:toolchains.bzl", "rules_proto_toolchains") + +rules_proto_toolchains() + +http_archive( + name = "toolchains_protoc", + sha256 = "b312e6de6485e01f753cb87fa09b4193f1762593141790dd0a90abf5e520b1d7", + strip_prefix = "toolchains_protoc-0.2.1", + url = "https://github.com/alexeagle/toolchains_protoc/releases/download/v0.2.1/toolchains_protoc-v0.2.1.tar.gz", +) + +load("@toolchains_protoc//protoc:toolchain.bzl", "protoc_toolchains") + +protoc_toolchains( + name = "protoc_toolchains", + version = "v25.3", +) + http_archive( name = "com_google_protobuf", sha256 = "75be42bd736f4df6d702a0e4e4d30de9ee40eac024c4b845d17ae4cc831fe4ae", diff --git a/proto/BUILD.bazel b/proto/BUILD.bazel index 4c47c31cc7..38be92355f 100644 --- a/proto/BUILD.bazel +++ b/proto/BUILD.bazel @@ -127,6 +127,7 @@ go_proto_compiler( non_go_reset_target( name = "protoc", dep = "@com_google_protobuf//:protoc", + deprecation = "No longer used by rules_go, will be removed in a future release.", visibility = ["//visibility:public"], ) diff --git a/proto/compiler.bzl b/proto/compiler.bzl index b51242271b..d1486dd54a 100644 --- a/proto/compiler.bzl +++ b/proto/compiler.bzl @@ -16,6 +16,11 @@ load( "@bazel_skylib//lib:paths.bzl", "paths", ) +load( + "@rules_proto//proto:proto_common.bzl", + "ProtoLangToolchainInfo", + proto_toolchains = "toolchains", +) load( "//go:def.bzl", "GoLibrary", @@ -31,6 +36,15 @@ load( "go_reset_target", ) +# This is actually a misuse of Proto toolchains: The proper way to use `protoc` would be to go +# through a Go-specific `proto_lang_toolchain` and use the methods on `proto_common` to interact +# with `protoc`. Since rules_go has a very bespoke setup with customizable compilers and the need +# to apply reset transitions in case `protoc` *is* built from source, this would require major +# changes. +# TODO: Revisit this after --incompatible_enable_proto_toolchain_resolution has been enabled by +# default. +_PROTO_TOOLCHAIN_TYPE = "@rules_proto//proto:toolchain_type" + GoProtoCompiler = provider( doc = "Information and dependencies needed to generate Go code from protos", fields = { @@ -104,7 +118,7 @@ def go_proto_compile(go, compiler, protos, imports, importpath): transitive_descriptor_sets = depset(direct = [], transitive = desc_sets) args = go.actions.args() - args.add("-protoc", compiler.internal.protoc) + args.add("-protoc", compiler.internal.protoc.executable) args.add("-importpath", importpath) args.add("-out_path", outpath) args.add("-plugin", compiler.internal.plugin) @@ -122,7 +136,6 @@ def go_proto_compile(go, compiler, protos, imports, importpath): inputs = depset( direct = [ compiler.internal.go_protoc, - compiler.internal.protoc, compiler.internal.plugin, ], transitive = [transitive_descriptor_sets], @@ -132,6 +145,7 @@ def go_proto_compile(go, compiler, protos, imports, importpath): mnemonic = "GoProtocGen", executable = compiler.internal.go_protoc, toolchain = GO_TOOLCHAIN_LABEL, + tools = [compiler.internal.protoc], arguments = [args], env = go.env, # We may need the shell environment (potentially augmented with --action_env) @@ -172,6 +186,11 @@ def _go_proto_compiler_impl(ctx): go = go_context(ctx) library = go.new_library(go) source = go.library_to_source(go, ctx.attr, library, ctx.coverage_instrumented()) + proto_toolchain = proto_toolchains.find_toolchain( + ctx, + legacy_attr = "_legacy_proto_toolchain", + toolchain_type = _PROTO_TOOLCHAIN_TYPE, + ) return [ GoProtoCompiler( deps = ctx.attr.deps, @@ -181,7 +200,7 @@ def _go_proto_compiler_impl(ctx): options = ctx.attr.options, suffix = ctx.attr.suffix, suffixes = ctx.attr.suffixes, - protoc = ctx.executable._protoc, + protoc = proto_toolchain.proto_compiler, go_protoc = ctx.executable._go_protoc, plugin = ctx.executable.plugin, import_path_option = ctx.attr.import_path_option, @@ -210,16 +229,20 @@ _go_proto_compiler = rule( cfg = "exec", default = "//go/tools/builders:go-protoc", ), - "_protoc": attr.label( - executable = True, - cfg = "exec", - default = "//proto:protoc", - ), "_go_context_data": attr.label( default = "//:go_context_data", ), - }, - toolchains = [GO_TOOLCHAIN], + } | proto_toolchains.if_legacy_toolchain({ + "_legacy_proto_toolchain": attr.label( + # Setting cfg = "exec" here as the legacy_proto_toolchain target + # already needs to apply the non_go_tool_transition. Flipping the + # two would be more idiomatic, but proto_toolchains.find_toolchain + # doesn't support split transitions. + cfg = "exec", + default = "//proto/private:legacy_proto_toolchain", + ), + }), + toolchains = [GO_TOOLCHAIN] + proto_toolchains.use_toolchain(_PROTO_TOOLCHAIN_TYPE), ) def go_proto_compiler(name, **kwargs): diff --git a/proto/private/BUILD.bazel b/proto/private/BUILD.bazel new file mode 100644 index 0000000000..ecbbf29f63 --- /dev/null +++ b/proto/private/BUILD.bazel @@ -0,0 +1,6 @@ +load(":toolchain.bzl", "legacy_proto_toolchain") + +legacy_proto_toolchain( + name = "legacy_proto_toolchain", + visibility = ["//proto:__pkg__"], +) diff --git a/proto/private/toolchain.bzl b/proto/private/toolchain.bzl new file mode 100644 index 0000000000..25b59ff0f2 --- /dev/null +++ b/proto/private/toolchain.bzl @@ -0,0 +1,47 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Helper that wraps --proto_compiler into a ProtoLangToolchainInfo for backwards +# compatibility with --noincompatible_enable_proto_toolchain_resolution. + +load( + "@rules_proto//proto:proto_common.bzl", + "ProtoLangToolchainInfo", +) +load( + "//go/private/rules:transition.bzl", + "go_reset_target", + "non_go_tool_transition", +) + +def _legacy_proto_toolchain_impl(ctx): + return [ + ProtoLangToolchainInfo( + proto_compiler = ctx.attr._protoc.files_to_run, + ), + ] + +legacy_proto_toolchain = rule( + implementation = _legacy_proto_toolchain_impl, + cfg = non_go_tool_transition, + attrs = { + "_protoc": attr.label( + default = configuration_field(fragment = "proto", name = "proto_compiler"), + ), + "_allowlist_function_transition": attr.label( + default = "@bazel_tools//tools/allowlists/function_transition_allowlist", + ), + }, + fragments = ["proto"], +) diff --git a/tests/bcr/.bazelversion b/tests/bcr/.bazelversion index 09b254e90c..798e38995c 100644 --- a/tests/bcr/.bazelversion +++ b/tests/bcr/.bazelversion @@ -1 +1 @@ -6.0.0 +6.3.0 diff --git a/tests/bcr/MODULE.bazel b/tests/bcr/MODULE.bazel index edf60dbfda..b24c39e31f 100644 --- a/tests/bcr/MODULE.bazel +++ b/tests/bcr/MODULE.bazel @@ -21,6 +21,10 @@ local_path_override( bazel_dep(name = "gazelle", version = "0.33.0") bazel_dep(name = "protobuf", version = "3.19.6") +# Required with --incompatible_enable_proto_toolchain_resolution. +# Avoids building protoc from source, which speeds up CI runs. +bazel_dep(name = "toolchains_protoc", version = "0.2.1") + go_sdk = use_extension("@my_rules_go//go:extensions.bzl", "go_sdk") go_sdk.download( patch_strip = 1,