From a0bd064c61060af92b6e25648bd3737dc9e88dca Mon Sep 17 00:00:00 2001 From: Sri Krishna Paritala Date: Mon, 7 Feb 2022 14:58:32 +0530 Subject: [PATCH 1/4] make buf version selectable --- buf/BUILD.bazel | 2 +- buf/internal/BUILD.bazel | 10 ++ buf/internal/cmd/generate_deps/BUILD.bazel | 14 -- buf/internal/cmd/generate_deps/main.go | 73 --------- buf/internal/dependencies.bzl | 93 ----------- buf/internal/toolchain.bzl | 164 ++++++++++++++++++++ buf/repositories.bzl | 16 +- tools/BUILD.bazel | 9 -- tools/buf/BUILD.bazel | 18 +-- tools/buf/toolchain.bzl | 21 --- tools/protoc-gen-buf-breaking/BUILD.bazel | 15 +- tools/protoc-gen-buf-breaking/toolchain.bzl | 21 --- tools/protoc-gen-buf-lint/BUILD.bazel | 15 +- tools/protoc-gen-buf-lint/toolchain.bzl | 21 --- tools/toolchain.bzl | 51 ------ 15 files changed, 187 insertions(+), 356 deletions(-) delete mode 100644 buf/internal/cmd/generate_deps/BUILD.bazel delete mode 100644 buf/internal/cmd/generate_deps/main.go create mode 100644 buf/internal/toolchain.bzl delete mode 100644 tools/BUILD.bazel delete mode 100644 tools/buf/toolchain.bzl delete mode 100644 tools/protoc-gen-buf-breaking/toolchain.bzl delete mode 100644 tools/protoc-gen-buf-lint/toolchain.bzl delete mode 100644 tools/toolchain.bzl diff --git a/buf/BUILD.bazel b/buf/BUILD.bazel index e163497..391c373 100644 --- a/buf/BUILD.bazel +++ b/buf/BUILD.bazel @@ -16,7 +16,7 @@ bzl_library( srcs = ["repositories.bzl"], deps = [ "//buf/internal:dependencies", - "//tools:toolchain", + "//buf/internal:toolchain", "@bazel_tools//tools/build_defs/repo:http.bzl", "@bazel_tools//tools/build_defs/repo:utils.bzl", ], diff --git a/buf/internal/BUILD.bazel b/buf/internal/BUILD.bazel index ee050fa..6d2496f 100644 --- a/buf/internal/BUILD.bazel +++ b/buf/internal/BUILD.bazel @@ -29,3 +29,13 @@ bzl_library( name = "dependencies", srcs = ["dependencies.bzl"], ) + +bzl_library( + name = "toolchain", + srcs = [ + "toolchain.bzl", + ], + deps = [ + "@bazel_tools//tools/build_defs/repo:utils.bzl", + ], +) diff --git a/buf/internal/cmd/generate_deps/BUILD.bazel b/buf/internal/cmd/generate_deps/BUILD.bazel deleted file mode 100644 index dfa0954..0000000 --- a/buf/internal/cmd/generate_deps/BUILD.bazel +++ /dev/null @@ -1,14 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") - -go_library( - name = "generate_deps_lib", - srcs = ["main.go"], - importpath = "github.com/bufbuild/rules_buf/buf/internal/cmd/generate_deps", - visibility = ["//visibility:private"], -) - -go_binary( - name = "generate_deps", - embed = [":generate_deps_lib"], - visibility = ["//:__subpackages__"], -) diff --git a/buf/internal/cmd/generate_deps/main.go b/buf/internal/cmd/generate_deps/main.go deleted file mode 100644 index 937f49b..0000000 --- a/buf/internal/cmd/generate_deps/main.go +++ /dev/null @@ -1,73 +0,0 @@ -package main - -import ( - "bufio" - "flag" - "fmt" - "log" - "net/http" - "os" - "strings" -) - -var ( - bufVersion = flag.String("buf-version", "", "Buf version [required]") -) - -const dependecyTemplate = ` - %q: { - "sha256": %q, - "urls": [%q], - "executable": True, - }, -` - -func main() { - log.SetFlags(0) - log.SetOutput(os.Stderr) - - flag.Parse() - - if *bufVersion == "" { - log.Fatalln("buf-version is missing and is required") - } - - res, err := http.Get(fmt.Sprintf("https://github.com/bufbuild/buf/releases/download/%s/sha256.txt", *bufVersion)) - if err != nil { - log.Fatalf("unable to read buf release sum file for version: %s, err: %v", *bufVersion, err) - } - defer res.Body.Close() - - sumScanner := bufio.NewScanner(res.Body) - - var buffer strings.Builder - fmt.Fprintln(&buffer, "buf_toolchains_dependencies = {") - for sumScanner.Scan() { - line := sumScanner.Text() - if strings.HasSuffix(line, ".tar.gz") { - continue - } - - var sum, bin string - fmt.Sscanf(line, "%s %s", &sum, &bin) - - url := fmt.Sprintf("https://github.com/bufbuild/buf/releases/download/%s/%s", *bufVersion, bin) - - bin = strings.ToLower(bin) - // Bazel defines macOS as osx - bin = strings.ReplaceAll(bin, "darwin", "osx") - // Windows binaries are suffixed with .exe. This only effects the targets name and the binary will continue to have the suffix. - bin = strings.TrimSuffix(bin, ".exe") - - fmt.Fprintf( - &buffer, - dependecyTemplate, - bin, - sum, - url, - ) - } - fmt.Fprintln(&buffer, "}") - - fmt.Print(buffer.String()) -} diff --git a/buf/internal/dependencies.bzl b/buf/internal/dependencies.bzl index b657783..17a1f91 100644 --- a/buf/internal/dependencies.bzl +++ b/buf/internal/dependencies.bzl @@ -1,98 +1,5 @@ """rules_buf dependencies""" -buf_toolchains_dependencies = { - "buf-osx-arm64": { - "sha256": "af48c127fac69c443d29db423735e74da7c36237d6054d980582e10f1fe1c258", - "urls": ["https://github.com/bufbuild/buf/releases/download/v1.0.0-rc11/buf-Darwin-arm64"], - "executable": True, - }, - "buf-osx-x86_64": { - "sha256": "53cf0d2e175d2556948b6dddf9f5c2a76893926513252165690b581279c5821f", - "urls": ["https://github.com/bufbuild/buf/releases/download/v1.0.0-rc11/buf-Darwin-x86_64"], - "executable": True, - }, - "buf-linux-aarch64": { - "sha256": "62b9a21769971ff38a25583da3282cb82d8dbe0021eee7e2b0c062abafd22bcc", - "urls": ["https://github.com/bufbuild/buf/releases/download/v1.0.0-rc11/buf-Linux-aarch64"], - "executable": True, - }, - "buf-linux-x86_64": { - "sha256": "50c1c65222c55e84251180d9c038962865fb0049653125a2f4fb522f043117b2", - "urls": ["https://github.com/bufbuild/buf/releases/download/v1.0.0-rc11/buf-Linux-x86_64"], - "executable": True, - }, - "buf-windows-arm64": { - "sha256": "2008148e20b4025d1b9ea43b523026cc0d6cdec25d4eeb3f8fcd958be049a74f", - "urls": ["https://github.com/bufbuild/buf/releases/download/v1.0.0-rc11/buf-Windows-arm64.exe"], - "executable": True, - }, - "buf-windows-x86_64": { - "sha256": "0d41c6b062e3dc1f96a649114c2076e763dd6275661e40a66caf6d4316f21706", - "urls": ["https://github.com/bufbuild/buf/releases/download/v1.0.0-rc11/buf-Windows-x86_64.exe"], - "executable": True, - }, - "protoc-gen-buf-breaking-osx-arm64": { - "sha256": "ae7b256062abb59904c0a5f6ee627e32307c36e328d247d1836087d67fdcf906", - "urls": ["https://github.com/bufbuild/buf/releases/download/v1.0.0-rc11/protoc-gen-buf-breaking-Darwin-arm64"], - "executable": True, - }, - "protoc-gen-buf-breaking-osx-x86_64": { - "sha256": "597630e7519894fe24a976685455437ca3850019d41d9c973e21ffaea96cb944", - "urls": ["https://github.com/bufbuild/buf/releases/download/v1.0.0-rc11/protoc-gen-buf-breaking-Darwin-x86_64"], - "executable": True, - }, - "protoc-gen-buf-breaking-linux-aarch64": { - "sha256": "297aa090e3c2aa335dd70e0a30517236dc0d43f88f2d2c2aa2f83c4d2b2fb9d5", - "urls": ["https://github.com/bufbuild/buf/releases/download/v1.0.0-rc11/protoc-gen-buf-breaking-Linux-aarch64"], - "executable": True, - }, - "protoc-gen-buf-breaking-linux-x86_64": { - "sha256": "a845d8acc57a800ce166344050436b12695f87c8750283eea91ea7e339a502e2", - "urls": ["https://github.com/bufbuild/buf/releases/download/v1.0.0-rc11/protoc-gen-buf-breaking-Linux-x86_64"], - "executable": True, - }, - "protoc-gen-buf-breaking-windows-arm64": { - "sha256": "6e372f4d609a53f47e6ad8883c4fbf1c31e74b83122de45a38d43abbecc7c748", - "urls": ["https://github.com/bufbuild/buf/releases/download/v1.0.0-rc11/protoc-gen-buf-breaking-Windows-arm64.exe"], - "executable": True, - }, - "protoc-gen-buf-breaking-windows-x86_64": { - "sha256": "f7e6139932e28b3e3fd5837f2f94c13678b978ebfdc1c84d72dc94fdf9dc29c4", - "urls": ["https://github.com/bufbuild/buf/releases/download/v1.0.0-rc11/protoc-gen-buf-breaking-Windows-x86_64.exe"], - "executable": True, - }, - "protoc-gen-buf-lint-osx-arm64": { - "sha256": "612c1f3967b10f34f9bb112331e473ecbcd44458bcf438a01bc0826cfbb66086", - "urls": ["https://github.com/bufbuild/buf/releases/download/v1.0.0-rc11/protoc-gen-buf-lint-Darwin-arm64"], - "executable": True, - }, - "protoc-gen-buf-lint-osx-x86_64": { - "sha256": "3960ba4111f05e85c5c1d772fdb00da1aa14eac16f2bfdeede6db2c66805b9a9", - "urls": ["https://github.com/bufbuild/buf/releases/download/v1.0.0-rc11/protoc-gen-buf-lint-Darwin-x86_64"], - "executable": True, - }, - "protoc-gen-buf-lint-linux-aarch64": { - "sha256": "45165de7e24c3d2298b343e868eefde8f5f25435477e0645caacf464221982c7", - "urls": ["https://github.com/bufbuild/buf/releases/download/v1.0.0-rc11/protoc-gen-buf-lint-Linux-aarch64"], - "executable": True, - }, - "protoc-gen-buf-lint-linux-x86_64": { - "sha256": "9e8a92491477fec00c6414231b35f78099f3f5b6e674a3812f59461af8af0b3c", - "urls": ["https://github.com/bufbuild/buf/releases/download/v1.0.0-rc11/protoc-gen-buf-lint-Linux-x86_64"], - "executable": True, - }, - "protoc-gen-buf-lint-windows-arm64": { - "sha256": "f758dfddd12dd9124d4b70c63f8ad1b841a1c28866af6f9be258a00b5063a211", - "urls": ["https://github.com/bufbuild/buf/releases/download/v1.0.0-rc11/protoc-gen-buf-lint-Windows-arm64.exe"], - "executable": True, - }, - "protoc-gen-buf-lint-windows-x86_64": { - "sha256": "8cfb5a8d19571738b20a83a067f6b44e63c6747413b6a4d6b408118e079fbfe0", - "urls": ["https://github.com/bufbuild/buf/releases/download/v1.0.0-rc11/protoc-gen-buf-lint-Windows-x86_64.exe"], - "executable": True, - }, -} - bazel_dependencies = { "bazel_skylib": { "sha256": "c6966ec828da198c5d9adbaa94c05e3a1c7f21bd012a0b29ba8ddbccb2c93b0d", diff --git a/buf/internal/toolchain.bzl b/buf/internal/toolchain.bzl new file mode 100644 index 0000000..fa568ab --- /dev/null +++ b/buf/internal/toolchain.bzl @@ -0,0 +1,164 @@ +"""Buf toolchains macros to declare and register toolchains""" + +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "update_attrs") + +_BUF_RELEASE_PLATFORMS = [ + { + "os": "osx", + "cpu": "arm64", + }, + { + "os": "osx", + "cpu": "x86_64", + }, + { + "os": "linux", + "cpu": "aarch64", + }, + { + "os": "linux", + "cpu": "x86_64", + }, + { + "os": "windows", + "cpu": "arm64", + }, + { + "os": "windows", + "cpu": "x86_64", + }, +] + +_BUF_RELEASES_REPO = "buf_releases" + +def _register_toolchains(name): + labels = [ + "@rules_buf//tools/{name}:{name}-{os}-{cpu}_toolchain".format( + name = name, + os = p["os"], + cpu = p["cpu"], + ) + for p in _BUF_RELEASE_PLATFORMS + ] + native.register_toolchains( + *labels + ) + +def _buf_toolchain_impl(ctx): + toolchain_info = platform_common.ToolchainInfo( + cli = ctx.executable.cli, + ) + return [toolchain_info] + +_buf_toolchain = rule( + implementation = _buf_toolchain_impl, + attrs = { + "cli": attr.label( + doc = "The buf cli", + executable = True, + cfg = "exec", + mandatory = True, + ), + }, +) + +_BUF_RELEASE_BUILD_FILE = """ +package(default_visibility = ["//visibility:public"]) +filegroup( + name = "{name}", + srcs = ["{bin}"], +) +""" + +def _buf_download_releases_impl(ctx): + version = ctx.attr.version + if not version: + ctx.report_progress("Finding latest buf version") + + # Get the latest version from github. Refer: https://docs.github.com/en/rest/reference/releases + # + # TODO: Change this to use https://api.github.com/repos/bufbuild/buf/releases/latest once we hit v1. + ctx.download( + url = "https://api.github.com/repos/bufbuild/buf/releases?per_page=1", + output = "versions.json", + ) + versions_data = ctx.read("versions.json") + versions = json.decode(versions_data) + version = versions[0]["name"] + + ctx.report_progress("Downloading buf release hash") + ctx.download( + url = [ + "https://github.com/bufbuild/buf/releases/download/{}/sha256.txt".format(version), + ], + output = "sha256.txt", + ) + sha_list = ctx.read("sha256.txt").splitlines() + for sha_line in sha_list: + if sha_line.strip(" ").endswith(".tar.gz"): + continue + (sum, _, bin) = sha_line.partition(" ") + bin = bin.strip(" ") + + # Bazel defines macOS as osx + # Windows binaries are suffixed with .exe. This only effects the targets name and the binary will continue to have the suffix. + target = bin.lower().replace("darwin", "osx").rstrip(".exe") + ctx.download( + url = "https://github.com/bufbuild/buf/releases/download/{}/{}".format(version, bin), + sha256 = sum, + executable = True, + output = "{}/{}".format(target, bin), + ) + ctx.file("{}/BUILD".format(target), _BUF_RELEASE_BUILD_FILE.format(name = target, bin = bin)) + ctx.file("WORKSPACE", "workspace(name = \"{name}\")".format(name = ctx.name)) + return update_attrs(ctx.attr, ["version"], {"version": version}) + +_buf_download_releases = repository_rule( + implementation = _buf_download_releases_impl, + attrs = { + "version": attr.string( + doc = "Buf release version", + ), + }, +) + +def declare_buf_toolchains(name, cmd): + """declare_buf_toolchains macro declares toolchains based on public releases of [buf](github.com/bufbuild/buf/releases) + + Args: + name: Macro name, can be anything, + cmd: Must be one of [buf, protoc-gen-bug-lint, protoc-gen-buf-breaking] + """ + native.toolchain_type(name = "toolchain_type") + for p in _BUF_RELEASE_PLATFORMS: + os = p["os"] + cpu = p["cpu"] + exec_name = "{}-{}-{}".format(cmd, os, cpu) + _buf_toolchain( + name = exec_name, + cli = "@{}//{}".format(_BUF_RELEASES_REPO, exec_name), + ) + + native.toolchain( + name = "{}_toolchain".format(exec_name), + toolchain = ":{}".format(exec_name), + toolchain_type = ":toolchain_type", + exec_compatible_with = [ + "@platforms//os:{}".format(os), + "@platforms//cpu:{}".format(cpu), + ], + ) + +# buildifier: disable=unnamed-macro +def rules_buf_toolchains(version = None): + """rules_buf_toolchains downloads buf, protoc-gen-buf-lint, and protoc-gen-buf-breaking from github releases of buf: https://github.com/bufbuild/buf/releases + + Args: + version: Release version, eg: `v.1.0.0-rc12`. If `None` defaults to latest + """ + + _buf_download_releases(name = _BUF_RELEASES_REPO, version = version) + + _register_toolchains("buf") + _register_toolchains("protoc-gen-buf-breaking") + _register_toolchains("protoc-gen-buf-lint") diff --git a/buf/repositories.bzl b/buf/repositories.bzl index e2aa185..a593bbd 100644 --- a/buf/repositories.bzl +++ b/buf/repositories.bzl @@ -1,21 +1,15 @@ """Dependencies and toolchains required to use rules_buf.""" -load("//buf/internal:dependencies.bzl", "bazel_dependencies", "buf_toolchains_dependencies") +load("//buf/internal:dependencies.bzl", "bazel_dependencies") load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file") -load("//tools:toolchain.bzl", "register_toolchains") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("//buf/internal:toolchain.bzl", _rules_buf_toolchains = "rules_buf_toolchains") def rules_buf_dependencies(): """Utility method to load all dependencies of `rules_buf`.""" - - for name in buf_toolchains_dependencies: - maybe(http_file, name, **buf_toolchains_dependencies[name]) for name in bazel_dependencies: maybe(http_archive, name, **bazel_dependencies[name]) -def rules_buf_toolchains(): +def rules_buf_toolchains(version = None): """Utility method to load all buf toolchains.""" - - register_toolchains("buf") - register_toolchains("protoc-gen-buf-breaking") - register_toolchains("protoc-gen-buf-lint") \ No newline at end of file + _rules_buf_toolchains(version) diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel deleted file mode 100644 index 1aa0496..0000000 --- a/tools/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -load("@bazel_skylib//:bzl_library.bzl", "bzl_library") - -bzl_library( - name = "toolchain", - srcs = [ - "toolchain.bzl", - ], - visibility = ["//:__subpackages__"], -) diff --git a/tools/buf/BUILD.bazel b/tools/buf/BUILD.bazel index 072fbaf..8121555 100644 --- a/tools/buf/BUILD.bazel +++ b/tools/buf/BUILD.bazel @@ -1,18 +1,6 @@ -load("@bazel_skylib//:bzl_library.bzl", "bzl_library") -load("//tools:toolchain.bzl", "declare_toolchains") -load(":toolchain.bzl", "buf_toolchain") +load("//buf/internal:toolchain.bzl", "declare_buf_toolchains") -bzl_library( - name = "toolchain", - srcs = ["toolchain.bzl"], - visibility = ["//visibility:public"], - deps = [ - ], -) - -toolchain_type(name = "toolchain_type") - -declare_toolchains( +declare_buf_toolchains( name = "buf_toolchain_declaration", - toolchain = buf_toolchain, + cmd = "buf", ) diff --git a/tools/buf/toolchain.bzl b/tools/buf/toolchain.bzl deleted file mode 100644 index 701e249..0000000 --- a/tools/buf/toolchain.bzl +++ /dev/null @@ -1,21 +0,0 @@ -""" -Declares the buf toolchain -""" - -def _buf_toolchain_impl(ctx): - toolchain_info = platform_common.ToolchainInfo( - cli = ctx.executable.cli, - ) - return [toolchain_info] - -buf_toolchain = rule( - implementation = _buf_toolchain_impl, - attrs = { - "cli": attr.label( - doc = "The buf cli", - executable = True, - cfg = "exec", - mandatory = True, - ), - }, -) diff --git a/tools/protoc-gen-buf-breaking/BUILD.bazel b/tools/protoc-gen-buf-breaking/BUILD.bazel index 6ee6408..302f123 100644 --- a/tools/protoc-gen-buf-breaking/BUILD.bazel +++ b/tools/protoc-gen-buf-breaking/BUILD.bazel @@ -1,17 +1,6 @@ -load("@bazel_skylib//:bzl_library.bzl", "bzl_library") -load("//tools:toolchain.bzl", "declare_toolchains") -load(":toolchain.bzl", "protoc_gen_buf_breaking_toolchain") +load("//buf/internal:toolchain.bzl", "declare_buf_toolchains") -bzl_library( - name = "toolchain", - srcs = ["toolchain.bzl"], - visibility = ["//visibility:public"], -) - -toolchain_type(name = "toolchain_type") - -declare_toolchains( +declare_buf_toolchains( name = "protoc_gen_buf_breaking_toolchain_declaration", cmd = "protoc-gen-buf-breaking", - toolchain = protoc_gen_buf_breaking_toolchain, ) diff --git a/tools/protoc-gen-buf-breaking/toolchain.bzl b/tools/protoc-gen-buf-breaking/toolchain.bzl deleted file mode 100644 index 1e3051d..0000000 --- a/tools/protoc-gen-buf-breaking/toolchain.bzl +++ /dev/null @@ -1,21 +0,0 @@ -""" -Declares the protoc-gen-buf-breaking toolchain -""" - -def _toolchain_impl(ctx): - toolchain_info = platform_common.ToolchainInfo( - cli = ctx.executable.cli, - ) - return [toolchain_info] - -protoc_gen_buf_breaking_toolchain = rule( - implementation = _toolchain_impl, - attrs = { - "cli": attr.label( - doc = "The protoc-gen-buf-breaking cli", - executable = True, - cfg = "exec", - mandatory = True, - ), - }, -) diff --git a/tools/protoc-gen-buf-lint/BUILD.bazel b/tools/protoc-gen-buf-lint/BUILD.bazel index 4dd6907..7ea0f68 100644 --- a/tools/protoc-gen-buf-lint/BUILD.bazel +++ b/tools/protoc-gen-buf-lint/BUILD.bazel @@ -1,17 +1,6 @@ -load("@bazel_skylib//:bzl_library.bzl", "bzl_library") -load("//tools:toolchain.bzl", "declare_toolchains") -load(":toolchain.bzl", "protoc_gen_buf_lint_toolchain") +load("//buf/internal:toolchain.bzl", "declare_buf_toolchains") -bzl_library( - name = "toolchain", - srcs = ["toolchain.bzl"], - visibility = ["//visibility:public"], -) - -toolchain_type(name = "toolchain_type") - -declare_toolchains( +declare_buf_toolchains( name = "protoc_gen_buf_lint_toolchain_declaration", cmd = "protoc-gen-buf-lint", - toolchain = protoc_gen_buf_lint_toolchain, ) diff --git a/tools/protoc-gen-buf-lint/toolchain.bzl b/tools/protoc-gen-buf-lint/toolchain.bzl deleted file mode 100644 index 946a569..0000000 --- a/tools/protoc-gen-buf-lint/toolchain.bzl +++ /dev/null @@ -1,21 +0,0 @@ -""" -Declares the protoc-gen-buf-lint toolchain -""" - -def _toolchain_impl(ctx): - toolchain_info = platform_common.ToolchainInfo( - cli = ctx.executable.cli, - ) - return [toolchain_info] - -protoc_gen_buf_lint_toolchain = rule( - implementation = _toolchain_impl, - attrs = { - "cli": attr.label( - doc = "The protoc-gen-buf-lint cli", - executable = True, - cfg = "exec", - mandatory = True, - ), - }, -) diff --git a/tools/toolchain.bzl b/tools/toolchain.bzl deleted file mode 100644 index 912e8f8..0000000 --- a/tools/toolchain.bzl +++ /dev/null @@ -1,51 +0,0 @@ -"""Buf toolchains macros to declare and register toolchains""" - -_BUF_RELEASE_PLATFORMS = [ - { - "os": "osx", - "cpu": "arm64", - }, - { - "os": "osx", - "cpu": "x86_64", - }, - { - "os": "linux", - "cpu": "aarch64", - }, - { - "os": "linux", - "cpu": "x86_64", - }, - { - "os": "windows", - "cpu": "arm64", - }, - { - "os": "windows", - "cpu": "x86_64", - }, -] - -def declare_toolchains(name, toolchain, cmd = "buf"): - for p in _BUF_RELEASE_PLATFORMS: - exec_name = "{}-{}-{}".format(cmd, p["os"], p["cpu"]) - toolchain( - name = exec_name, - cli = "@{}//file".format(exec_name), - ) - - native.toolchain( - name = "{}_toolchain".format(exec_name), - toolchain = ":{}".format(exec_name), - toolchain_type = ":toolchain_type", - ) - -def register_toolchains(name): - labels = [ - "@rules_buf//tools/{}:{}-{}-{}_toolchain".format(name, name, p["os"], p["cpu"]) - for p in _BUF_RELEASE_PLATFORMS - ] - native.register_toolchains( - *labels - ) From 2ec475ec5276e6fb187f6f0ffb671bc9f782dac8 Mon Sep 17 00:00:00 2001 From: Sri Krishna Paritala Date: Mon, 7 Feb 2022 18:39:21 +0530 Subject: [PATCH 2/4] remove alias --- BUILD.bazel | 5 ----- 1 file changed, 5 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index 3b985c3..2ead7c4 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -30,11 +30,6 @@ stardoc( ], ) -alias( - name = "generate_deps", - actual = "//buf/internal/cmd/generate_deps:generate_deps", -) - buildifier( name = "buildifier", ) From 5a7479027695ced25c142d828be258788acdc47e Mon Sep 17 00:00:00 2001 From: Sri Krishna Paritala Date: Tue, 8 Feb 2022 20:20:46 +0530 Subject: [PATCH 3/4] selective download --- WORKSPACE | 2 +- buf/internal/dependencies.bzl | 8 + buf/internal/toolchain.bzl | 217 +++++++++++++--------- buf/repositories.bzl | 14 +- tools/buf/BUILD.bazel | 7 +- tools/protoc-gen-buf-breaking/BUILD.bazel | 7 +- tools/protoc-gen-buf-lint/BUILD.bazel | 7 +- 7 files changed, 149 insertions(+), 113 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index eaa9eef..ace98a5 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -4,7 +4,7 @@ load("//buf:repositories.bzl", "rules_buf_dependencies", "rules_buf_toolchains") rules_buf_dependencies() -rules_buf_toolchains() +rules_buf_toolchains(version = "v1.0.0-rc12") load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") diff --git a/buf/internal/dependencies.bzl b/buf/internal/dependencies.bzl index 17a1f91..1d57561 100644 --- a/buf/internal/dependencies.bzl +++ b/buf/internal/dependencies.bzl @@ -1,5 +1,8 @@ """rules_buf dependencies""" +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + bazel_dependencies = { "bazel_skylib": { "sha256": "c6966ec828da198c5d9adbaa94c05e3a1c7f21bd012a0b29ba8ddbccb2c93b0d", @@ -17,3 +20,8 @@ bazel_dependencies = { ], }, } + +def rules_buf_dependencies(): + """Utility method to load all dependencies of `rules_buf`.""" + for name in bazel_dependencies: + maybe(http_archive, name, **bazel_dependencies[name]) diff --git a/buf/internal/toolchain.bzl b/buf/internal/toolchain.bzl index fa568ab..1bbab4b 100644 --- a/buf/internal/toolchain.bzl +++ b/buf/internal/toolchain.bzl @@ -2,48 +2,21 @@ load("@bazel_tools//tools/build_defs/repo:utils.bzl", "update_attrs") -_BUF_RELEASE_PLATFORMS = [ - { - "os": "osx", - "cpu": "arm64", - }, - { - "os": "osx", - "cpu": "x86_64", - }, - { - "os": "linux", - "cpu": "aarch64", - }, - { - "os": "linux", - "cpu": "x86_64", - }, - { - "os": "windows", - "cpu": "arm64", - }, - { - "os": "windows", - "cpu": "x86_64", - }, -] +_TOOLCHAINS_REPO = "rules_buf_toolchains" -_BUF_RELEASES_REPO = "buf_releases" +_BUILD_FILE = """ +load(":toolchain.bzl", "declare_buf_toolchains") -def _register_toolchains(name): - labels = [ - "@rules_buf//tools/{name}:{name}-{os}-{cpu}_toolchain".format( - name = name, - os = p["os"], - cpu = p["cpu"], - ) - for p in _BUF_RELEASE_PLATFORMS - ] - native.register_toolchains( - *labels - ) +package(default_visibility = ["//visibility:public"]) +declare_buf_toolchains( + os = "{os}", + cpu = "{cpu}", + rules_buf_repo_name = "{rules_buf_repo_name}", + ) +""" + +_TOOLCHAIN_FILE = """ def _buf_toolchain_impl(ctx): toolchain_info = platform_common.ToolchainInfo( cli = ctx.executable.cli, @@ -56,20 +29,93 @@ _buf_toolchain = rule( "cli": attr.label( doc = "The buf cli", executable = True, - cfg = "exec", + allow_single_file = True, mandatory = True, + cfg = "exec", ), }, ) -_BUF_RELEASE_BUILD_FILE = """ -package(default_visibility = ["//visibility:public"]) -filegroup( - name = "{name}", - srcs = ["{bin}"], -) +def declare_buf_toolchains(os, cpu, rules_buf_repo_name): + for cmd in ["buf", "protoc-gen-buf-lint", "protoc-gen-buf-breaking"]: + ext = "" + if os == "windows": + ext = ".exe" + toolchain_impl = cmd + "_toolchain_impl" + _buf_toolchain( + name = toolchain_impl, + cli = str(Label("//:"+ cmd)), + ) + native.toolchain( + name = cmd + "_toolchain", + toolchain = ":" + toolchain_impl, + toolchain_type = "@{}//tools/{}:toolchain_type".format(rules_buf_repo_name, cmd), + exec_compatible_with = [ + "@platforms//os:" + os, + "@platforms//cpu:" + cpu, + ], + ) + """ +def _register_toolchains(repo, cmd): + native.register_toolchains( + "@{repo}//:{cmd}_toolchain".format( + repo = repo, + cmd = cmd, + ), + ) + +# Copied from rules_go: https://github.com/bazelbuild/rules_go/blob/bd44f4242b46e73fb2a81fc87ea4b52173bde84e/go/private/sdk.bzl#L240 +# +# NOTE: This doesn't check for windows/arm64 +def _detect_host_platform(ctx): + if ctx.os.name == "linux": + goos, goarch = "linux", "amd64" + res = ctx.execute(["uname", "-p"]) + if res.return_code == 0: + uname = res.stdout.strip() + if uname == "s390x": + goarch = "s390x" + elif uname == "i686": + goarch = "386" + + # uname -p is not working on Aarch64 boards + # or for ppc64le on some distros + res = ctx.execute(["uname", "-m"]) + if res.return_code == 0: + uname = res.stdout.strip() + if uname == "aarch64": + goarch = "arm64" + elif uname == "armv6l": + goarch = "arm" + elif uname == "armv7l": + goarch = "arm" + elif uname == "ppc64le": + goarch = "ppc64le" + + # Default to amd64 when uname doesn't return a known value. + + elif ctx.os.name == "mac os x": + goos, goarch = "darwin", "amd64" + + res = ctx.execute(["uname", "-m"]) + if res.return_code == 0: + uname = res.stdout.strip() + if uname == "arm64": + goarch = "arm64" + + # Default to amd64 when uname doesn't return a known value. + + elif ctx.os.name.startswith("windows"): + goos, goarch = "windows", "amd64" + elif ctx.os.name == "freebsd": + goos, goarch = "freebsd", "amd64" + else: + fail("Unsupported operating system: " + ctx.os.name) + + return goos, goarch + def _buf_download_releases_impl(ctx): version = ctx.attr.version if not version: @@ -86,6 +132,14 @@ def _buf_download_releases_impl(ctx): versions = json.decode(versions_data) version = versions[0]["name"] + os, cpu = _detect_host_platform(ctx) + if os not in ["linux", "darwin", "windows"] or cpu not in ["arm64", "amd64"]: + fail("Unsupported operating system or cpu architecture ") + if os == "linux" and cpu == "arm64": + cpu = "aarch64" + if cpu == "amd64": + cpu = "x86_64" + ctx.report_progress("Downloading buf release hash") ctx.download( url = [ @@ -93,24 +147,41 @@ def _buf_download_releases_impl(ctx): ], output = "sha256.txt", ) + ctx.file("WORKSPACE", "workspace(name = \"{name}\")".format(name = ctx.name)) + ctx.file("toolchain.bzl", _TOOLCHAIN_FILE) sha_list = ctx.read("sha256.txt").splitlines() for sha_line in sha_list: if sha_line.strip(" ").endswith(".tar.gz"): continue (sum, _, bin) = sha_line.partition(" ") bin = bin.strip(" ") + lower_case_bin = bin.lower() + if lower_case_bin.find(os) == -1 or lower_case_bin.find(cpu) == -1: + continue - # Bazel defines macOS as osx - # Windows binaries are suffixed with .exe. This only effects the targets name and the binary will continue to have the suffix. - target = bin.lower().replace("darwin", "osx").rstrip(".exe") - ctx.download( + output = lower_case_bin[:lower_case_bin.find(os) - 1] + if os == "windows": + output += ".exe" + + ctx.report_progress("Downloading " + bin) + download_info = ctx.download( url = "https://github.com/bufbuild/buf/releases/download/{}/{}".format(version, bin), sha256 = sum, executable = True, - output = "{}/{}".format(target, bin), + output = output, ) - ctx.file("{}/BUILD".format(target), _BUF_RELEASE_BUILD_FILE.format(name = target, bin = bin)) - ctx.file("WORKSPACE", "workspace(name = \"{name}\")".format(name = ctx.name)) + + if os == "darwin": + os = "osx" + + ctx.file( + "BUILD", + _BUILD_FILE.format( + os = os, + cpu = cpu, + rules_buf_repo_name = Label("//buf/repositories.bzl").workspace_name, + ), + ) return update_attrs(ctx.attr, ["version"], {"version": version}) _buf_download_releases = repository_rule( @@ -122,43 +193,17 @@ _buf_download_releases = repository_rule( }, ) -def declare_buf_toolchains(name, cmd): - """declare_buf_toolchains macro declares toolchains based on public releases of [buf](github.com/bufbuild/buf/releases) - - Args: - name: Macro name, can be anything, - cmd: Must be one of [buf, protoc-gen-bug-lint, protoc-gen-buf-breaking] - """ - native.toolchain_type(name = "toolchain_type") - for p in _BUF_RELEASE_PLATFORMS: - os = p["os"] - cpu = p["cpu"] - exec_name = "{}-{}-{}".format(cmd, os, cpu) - _buf_toolchain( - name = exec_name, - cli = "@{}//{}".format(_BUF_RELEASES_REPO, exec_name), - ) - - native.toolchain( - name = "{}_toolchain".format(exec_name), - toolchain = ":{}".format(exec_name), - toolchain_type = ":toolchain_type", - exec_compatible_with = [ - "@platforms//os:{}".format(os), - "@platforms//cpu:{}".format(cpu), - ], - ) - # buildifier: disable=unnamed-macro -def rules_buf_toolchains(version = None): - """rules_buf_toolchains downloads buf, protoc-gen-buf-lint, and protoc-gen-buf-breaking from github releases of buf: https://github.com/bufbuild/buf/releases +def rules_buf_toolchains(name = _TOOLCHAINS_REPO, version = None): + """rules_buf_toolchains sets up toolchains for buf, protoc-gen-buf-lint, and protoc-gen-buf-breaking Args: + name: The name of the toolchains repository. Defaults to "buf_toolchains" version: Release version, eg: `v.1.0.0-rc12`. If `None` defaults to latest """ - _buf_download_releases(name = _BUF_RELEASES_REPO, version = version) + _buf_download_releases(name = name, version = version) - _register_toolchains("buf") - _register_toolchains("protoc-gen-buf-breaking") - _register_toolchains("protoc-gen-buf-lint") + _register_toolchains(name, "buf") + _register_toolchains(name, "protoc-gen-buf-breaking") + _register_toolchains(name, "protoc-gen-buf-lint") diff --git a/buf/repositories.bzl b/buf/repositories.bzl index a593bbd..eade69c 100644 --- a/buf/repositories.bzl +++ b/buf/repositories.bzl @@ -1,15 +1,7 @@ """Dependencies and toolchains required to use rules_buf.""" -load("//buf/internal:dependencies.bzl", "bazel_dependencies") -load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("//buf/internal:dependencies.bzl", _rules_buf_dependencies = "rules_buf_dependencies") load("//buf/internal:toolchain.bzl", _rules_buf_toolchains = "rules_buf_toolchains") -def rules_buf_dependencies(): - """Utility method to load all dependencies of `rules_buf`.""" - for name in bazel_dependencies: - maybe(http_archive, name, **bazel_dependencies[name]) - -def rules_buf_toolchains(version = None): - """Utility method to load all buf toolchains.""" - _rules_buf_toolchains(version) +rules_buf_dependencies = _rules_buf_dependencies +rules_buf_toolchains = _rules_buf_toolchains diff --git a/tools/buf/BUILD.bazel b/tools/buf/BUILD.bazel index 8121555..ba1f967 100644 --- a/tools/buf/BUILD.bazel +++ b/tools/buf/BUILD.bazel @@ -1,6 +1,3 @@ -load("//buf/internal:toolchain.bzl", "declare_buf_toolchains") +package(default_visibility = ["//visibility:public"]) -declare_buf_toolchains( - name = "buf_toolchain_declaration", - cmd = "buf", -) +toolchain_type(name = "toolchain_type") diff --git a/tools/protoc-gen-buf-breaking/BUILD.bazel b/tools/protoc-gen-buf-breaking/BUILD.bazel index 302f123..ba1f967 100644 --- a/tools/protoc-gen-buf-breaking/BUILD.bazel +++ b/tools/protoc-gen-buf-breaking/BUILD.bazel @@ -1,6 +1,3 @@ -load("//buf/internal:toolchain.bzl", "declare_buf_toolchains") +package(default_visibility = ["//visibility:public"]) -declare_buf_toolchains( - name = "protoc_gen_buf_breaking_toolchain_declaration", - cmd = "protoc-gen-buf-breaking", -) +toolchain_type(name = "toolchain_type") diff --git a/tools/protoc-gen-buf-lint/BUILD.bazel b/tools/protoc-gen-buf-lint/BUILD.bazel index 7ea0f68..ba1f967 100644 --- a/tools/protoc-gen-buf-lint/BUILD.bazel +++ b/tools/protoc-gen-buf-lint/BUILD.bazel @@ -1,6 +1,3 @@ -load("//buf/internal:toolchain.bzl", "declare_buf_toolchains") +package(default_visibility = ["//visibility:public"]) -declare_buf_toolchains( - name = "protoc_gen_buf_lint_toolchain_declaration", - cmd = "protoc-gen-buf-lint", -) +toolchain_type(name = "toolchain_type") From 21ab56626409f4ea5eb02318a278ccf279419372 Mon Sep 17 00:00:00 2001 From: Sri Krishna Paritala Date: Tue, 8 Feb 2022 22:19:03 +0530 Subject: [PATCH 4/4] add note regarding repo os and arch --- buf/internal/toolchain.bzl | 1 + 1 file changed, 1 insertion(+) diff --git a/buf/internal/toolchain.bzl b/buf/internal/toolchain.bzl index 1bbab4b..a70c9b8 100644 --- a/buf/internal/toolchain.bzl +++ b/buf/internal/toolchain.bzl @@ -69,6 +69,7 @@ def _register_toolchains(repo, cmd): # Copied from rules_go: https://github.com/bazelbuild/rules_go/blob/bd44f4242b46e73fb2a81fc87ea4b52173bde84e/go/private/sdk.bzl#L240 # # NOTE: This doesn't check for windows/arm64 +# We can upgrade to repository_ctx.os.name and repository_ctx.os.arch once bazel 5.1 releases bazelbuild/bazel#14685 def _detect_host_platform(ctx): if ctx.os.name == "linux": goos, goarch = "linux", "amd64"