From 46710cabc4448ce85dca3ae907acbe89da726b56 Mon Sep 17 00:00:00 2001 From: "David L. Jones" Date: Wed, 20 Apr 2022 16:26:44 -0700 Subject: [PATCH] Move rules_pkg targets into //pkg, and add experimental C++ library rules (#9823) This change moves the `pkg_*` rules into the `//pkg` package, which cleans up the root package. It also adds an experimental `cc_dist_library` rule, which is similar to Bazel's `cc_import` rule. The goal of `cc_dist_library` is to produce output libraries from several targets. For example, splitting `//:protobuf` into multiple targets means that `bazel-bin/libprotobuf.a` won't contain all of the objects. The `cc_dist_library` creates a single library from several different `cc_library` targets. This may be useful for future packaging targets. --- BUILD | 95 +++------------- pkg/BUILD | 117 ++++++++++++++++++++ pkg/cc_dist_library.bzl | 232 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 366 insertions(+), 78 deletions(-) create mode 100644 pkg/BUILD create mode 100644 pkg/cc_dist_library.bzl diff --git a/BUILD b/BUILD index 5de4aa3f2ffc..e43762651420 100644 --- a/BUILD +++ b/BUILD @@ -2,13 +2,12 @@ load("@bazel_skylib//rules:common_settings.bzl", "string_flag") load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test", "objc_library", native_cc_proto_library = "cc_proto_library") -load("@rules_pkg//:pkg.bzl", "pkg_zip") -load("@rules_pkg//:mappings.bzl", "pkg_attributes", "pkg_files") load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain", "proto_library") load("@rules_python//python:defs.bzl", "py_library") load("@rules_java//java:defs.bzl", "java_binary", "java_lite_proto_library", "java_proto_library") load(":cc_proto_blacklist_test.bzl", "cc_proto_blacklist_test") load(":protobuf_release.bzl", "package_naming") + licenses(["notice"]) exports_files(["LICENSE"]) @@ -353,6 +352,11 @@ filegroup( visibility = ["//visibility:public"], ) +exports_files( + srcs = WELL_KNOWN_PROTOS, + visibility = ["//pkg:__pkg__"], +) + filegroup( name = "lite_well_known_protos", srcs = LITE_WELL_KNOWN_PROTOS, @@ -519,70 +523,6 @@ cc_binary( deps = [":protoc_lib"], ) - -################################################################################ -# Generates protoc release artifacts. -################################################################################ - -genrule( - name = "protoc_readme", - visibility = ["//visibility:private"], - cmd = """ -echo "Protocol Buffers - Google's data interchange format -Copyright 2008 Google Inc. -https://developers.google.com/protocol-buffers/ -This package contains a precompiled binary version of the protocol buffer -compiler (protoc). This binary is intended for users who want to use Protocol -Buffers in languages other than C++ but do not want to compile protoc -themselves. To install, simply place this binary somewhere in your PATH. -If you intend to use the included well known types then don't forget to -copy the contents of the 'include' directory somewhere as well, for example -into '/usr/local/include/'. -Please refer to our official github site for more installation instructions: - https://github.com/protocolbuffers/protobuf" > $@ - """, - outs = ["readme.txt"], -) - -# plugin.proto is excluded from this list because it belongs in a nested folder (protobuf/compiler/plugin.proto) -pkg_files( - name = "wkt_protos_files", - srcs = [value[0] for value in WELL_KNOWN_PROTO_MAP.values() if not value[0].endswith("plugin.proto")], - visibility = ["//visibility:private"], - prefix = "include/google/protobuf", -) - -pkg_files( - name = "compiler_plugin_protos_files", - srcs = ["src/google/protobuf/compiler/plugin.proto"], - visibility = ["//visibility:private"], - prefix = "include/google/protobuf/compiler", -) - -pkg_files( - name = "protoc_files", - srcs = [":protoc"], - attributes = pkg_attributes(mode = "0555"), - visibility = ["//visibility:private"], - prefix = "bin/", -) - -package_naming( - name = "protoc_pkg_naming", -) - -pkg_zip( - name = "protoc_release", - package_file_name = "protoc-{version}-{platform}.zip", - package_variables = ":protoc_pkg_naming", - srcs = [ - ":protoc_files", - ":wkt_protos_files", - ":compiler_plugin_protos_files", - "readme.txt", - ], -) - ################################################################################ # Tests ################################################################################ @@ -905,10 +845,10 @@ internal_gen_kt_protos( internal_gen_kt_protos( name = "gen_well_known_protos_kotlinlite", + lite = True, visibility = [ "//java:__subpackages__", ], - lite = True, deps = [proto + "_proto" for proto in LITE_WELL_KNOWN_PROTO_MAP.keys()], ) @@ -1340,17 +1280,16 @@ cc_binary( # ], # ) - java_proto_library( name = "java_test_protos", - deps = [":generic_test_protos"], visibility = ["//java:__subpackages__"], + deps = [":generic_test_protos"], ) java_lite_proto_library( name = "java_lite_test_protos", - deps = [":generic_test_protos"], visibility = ["//java:__subpackages__"], + deps = [":generic_test_protos"], ) java_proto_library( @@ -1452,57 +1391,57 @@ filegroup( proto_library( name = "kt_unittest_lite", srcs = [ - "src/google/protobuf/unittest_lite.proto", + "src/google/protobuf/map_lite_unittest.proto", "src/google/protobuf/unittest_import_lite.proto", "src/google/protobuf/unittest_import_public_lite.proto", - "src/google/protobuf/map_lite_unittest.proto", + "src/google/protobuf/unittest_lite.proto", ], strip_import_prefix = "src", ) internal_gen_kt_protos( name = "gen_kotlin_unittest_lite", - deps = [":kt_unittest_lite"], lite = True, visibility = ["//java:__subpackages__"], + deps = [":kt_unittest_lite"], ) proto_library( name = "kt_unittest", srcs = [ + "src/google/protobuf/map_proto2_unittest.proto", "src/google/protobuf/unittest.proto", "src/google/protobuf/unittest_import.proto", "src/google/protobuf/unittest_import_public.proto", - "src/google/protobuf/map_proto2_unittest.proto", ], strip_import_prefix = "src", ) internal_gen_kt_protos( name = "gen_kotlin_unittest", - deps = [":kt_unittest"], visibility = ["//java:__subpackages__"], + deps = [":kt_unittest"], ) proto_library( name = "kt_proto3_unittest", srcs = [ - "src/google/protobuf/unittest_proto3.proto", "src/google/protobuf/unittest_import.proto", "src/google/protobuf/unittest_import_public.proto", + "src/google/protobuf/unittest_proto3.proto", ], strip_import_prefix = "src", ) internal_gen_kt_protos( name = "gen_kotlin_proto3_unittest_lite", - deps = [":kt_proto3_unittest"], lite = True, visibility = ["//java:__subpackages__"], + deps = [":kt_proto3_unittest"], ) internal_gen_kt_protos( name = "gen_kotlin_proto3_unittest", - deps = [":kt_proto3_unittest"], visibility = ["//java:__subpackages__"], + deps = [":kt_proto3_unittest"], ) diff --git a/pkg/BUILD b/pkg/BUILD new file mode 100644 index 000000000000..3e5f5d1d75c5 --- /dev/null +++ b/pkg/BUILD @@ -0,0 +1,117 @@ +load("@rules_pkg//:pkg.bzl", "pkg_zip") +load("@rules_pkg//:mappings.bzl", "pkg_attributes", "pkg_files") +load("//:protobuf_release.bzl", "package_naming") +load(":cc_dist_library.bzl", "cc_dist_library") + +pkg_files( + name = "wkt_protos_files", + srcs = [ + "//:any_proto", + "//:api_proto", + "//:duration_proto", + "//:empty_proto", + "//:field_mask_proto", + "//:source_context_proto", + "//:struct_proto", + "//:timestamp_proto", + "//:type_proto", + "//:wrappers_proto", + ], + prefix = "include/google/protobuf", + visibility = ["//visibility:private"], +) + +pkg_files( + name = "descriptor_protos_files", + srcs = [ + "//:descriptor_proto", + ], + prefix = "include/google/protobuf", + visibility = ["//visibility:private"], +) + +pkg_files( + name = "compiler_plugin_protos_files", + srcs = ["//:compiler_plugin_proto"], + prefix = "include/google/protobuf/compiler", + visibility = ["//visibility:private"], +) + +################################################################################ +# Generates protoc release artifacts. +################################################################################ + +genrule( + name = "protoc_readme", + outs = ["readme.txt"], + cmd = """ +echo "Protocol Buffers - Google's data interchange format +Copyright 2008 Google Inc. +https://developers.google.com/protocol-buffers/ +This package contains a precompiled binary version of the protocol buffer +compiler (protoc). This binary is intended for users who want to use Protocol +Buffers in languages other than C++ but do not want to compile protoc +themselves. To install, simply place this binary somewhere in your PATH. +If you intend to use the included well known types then don't forget to +copy the contents of the 'include' directory somewhere as well, for example +into '/usr/local/include/'. +Please refer to our official github site for more installation instructions: + https://github.com/protocolbuffers/protobuf" > $@ + """, + visibility = ["//visibility:private"], +) + +pkg_files( + name = "protoc_files", + srcs = ["//:protoc"], + attributes = pkg_attributes(mode = "0555"), + prefix = "bin/", + visibility = ["//visibility:private"], +) + +package_naming( + name = "protoc_pkg_naming", +) + +pkg_zip( + name = "protoc_release", + srcs = [ + ":compiler_plugin_protos_files", + ":descriptor_protos_files", + ":protoc_files", + ":protoc_readme", + ":wkt_protos_files", + ], + package_file_name = "protoc-{version}-{platform}.zip", + package_variables = ":protoc_pkg_naming", +) + +################################################################################ +# Protobuf runtime libraries. +################################################################################ + +cc_dist_library( + name = "protobuf_lite", + linkopts = select({ + "//:msvc": [], + "//conditions:default": ["-lpthread"], + }), + deps = [ + "//:protobuf_lite", + ], +) + +cc_dist_library( + name = "protobuf", + linkopts = select({ + "//:msvc": [], + "//conditions:default": [ + "-lz", + "-lpthread", + ], + }), + deps = [ + "//:protobuf", + "//:protobuf_lite", + ], +) diff --git a/pkg/cc_dist_library.bzl b/pkg/cc_dist_library.bzl new file mode 100644 index 000000000000..654fb73e35f5 --- /dev/null +++ b/pkg/cc_dist_library.bzl @@ -0,0 +1,232 @@ +# Rules for distributable C++ libraries + +load("@rules_cc//cc:action_names.bzl", cc_action_names = "ACTION_NAMES") +load("@rules_cc//cc:find_cc_toolchain.bzl", "find_cc_toolchain") + +# Creates an action to build the `output_file` static library (archive) +# using `object_files`. +def _create_archive_action( + ctx, + feature_configuration, + cc_toolchain, + output_file, + object_files): + # Based on Bazel's src/main/starlark/builtins_bzl/common/cc/cc_import.bzl: + + # Build the command line and add args for all of the input files: + archiver_variables = cc_common.create_link_variables( + feature_configuration = feature_configuration, + cc_toolchain = cc_toolchain, + output_file = output_file.path, + is_using_linker = False, + ) + command_line = cc_common.get_memory_inefficient_command_line( + feature_configuration = feature_configuration, + action_name = cc_action_names.cpp_link_static_library, + variables = archiver_variables, + ) + args = ctx.actions.args() + args.add_all(command_line) + args.add_all(object_files) + args.use_param_file("@%s", use_always = True) + + archiver_path = cc_common.get_tool_for_action( + feature_configuration = feature_configuration, + action_name = cc_action_names.cpp_link_static_library, + ) + + env = cc_common.get_environment_variables( + feature_configuration = feature_configuration, + action_name = cc_action_names.cpp_link_static_library, + variables = archiver_variables, + ) + + ctx.actions.run( + executable = archiver_path, + arguments = [args], + env = env, + inputs = depset( + direct = object_files, + transitive = [ + cc_toolchain.all_files, + ], + ), + use_default_shell_env = True, + outputs = [output_file], + mnemonic = "CppArchiveDist", + ) + +# Implementation for cc_dist_library rule. +def _cc_dist_library_impl(ctx): + cc_toolchain_info = find_cc_toolchain(ctx) + if cc_toolchain_info.ar_executable == None: + return [] + + feature_configuration = cc_common.configure_features( + ctx = ctx, + cc_toolchain = cc_toolchain_info, + ) + + # Collect the set of object files from the immediate deps. + + objs = [] + pic_objs = [] + for dep in ctx.attr.deps: + if CcInfo not in dep: + continue + + link_ctx = dep[CcInfo].linking_context + if link_ctx == None: + continue + + linker_inputs = link_ctx.linker_inputs.to_list() + for link_input in linker_inputs: + if link_input.owner != dep.label: + # This is a transitive dep: skip it. + continue + + for lib in link_input.libraries: + objs.extend(lib.objects or []) + pic_objs.extend(lib.pic_objects or []) + + # For static libraries, build separately with and without pic. + + stemname = "lib" + ctx.label.name + outputs = [] + + if len(objs) > 0: + archive_out = ctx.actions.declare_file(stemname + ".a") + _create_archive_action( + ctx, + feature_configuration, + cc_toolchain_info, + archive_out, + objs, + ) + outputs.append(archive_out) + + if len(pic_objs) > 0: + pic_archive_out = ctx.actions.declare_file(stemname + ".pic.a") + _create_archive_action( + ctx, + feature_configuration, + cc_toolchain_info, + pic_archive_out, + pic_objs, + ) + outputs.append(pic_archive_out) + + # For dynamic libraries, use the `cc_common.link` command to ensure + # everything gets built correctly according to toolchain definitions. + + compilation_outputs = cc_common.create_compilation_outputs( + objects = depset(objs), + pic_objects = depset(pic_objs), + ) + link_output = cc_common.link( + actions = ctx.actions, + feature_configuration = feature_configuration, + cc_toolchain = cc_toolchain_info, + compilation_outputs = compilation_outputs, + name = ctx.label.name, + output_type = "dynamic_library", + user_link_flags = ctx.attr.linkopts, + ) + library_to_link = link_output.library_to_link + + # Note: library_to_link.dynamic_library and interface_library are often + # symlinks in the solib directory. For DefaultInfo, prefer reporting + # the resolved artifact paths. + if library_to_link.resolved_symlink_dynamic_library != None: + outputs.append(library_to_link.resolved_symlink_dynamic_library) + elif library_to_link.dynamic_library != None: + outputs.append(library_to_link.dynamic_library) + + if library_to_link.resolved_symlink_interface_library != None: + outputs.append(library_to_link.resolved_symlink_interface_library) + elif library_to_link.interface_library != None: + outputs.append(library_to_link.interface_library) + + # We could expose the libraries for use from cc rules: + # + # linking_context = cc_common.create_linking_context( + # linker_inputs = depset([ + # cc_common.create_linker_input( + # owner = ctx.label, + # libraries = depset([library_to_link]), + # ), + # ]), + # ) + # cc_info = CcInfo(linking_context = linking_context) # and return this + # + # However, if the goal is to force a protobuf dependency to use the + # DSO, then `cc_import` is a better-supported way to do so. + # + # If we wanted to expose CcInfo from this rule (and make it usable as a + # C++ dependency), then we would probably want to include the static + # archive and headers as well. exposing headers would probably require + # an additional aspect to extract CcInfos with just the deps' headers. + + return [ + DefaultInfo(files = depset(outputs)), + ] + +cc_dist_library = rule( + implementation = _cc_dist_library_impl, + doc = """ +Create libraries suitable for distribution. + +This rule creates static and dynamic libraries from the libraries listed in +'deps'. The resulting libraries are suitable for distributing all of 'deps' +in a single logical library, for example, in an installable binary package. +Only the targets listed in 'deps' are included in the result (i.e., the +output does not include transitive dependencies), allowing precise control +over the library boundary. + +The outputs of this rule are a dynamic library and a static library. (If +the build produces both PIC and non-PIC object files, then there is also a +second static library.) The example below illustrates additional details. + +This rule is different from Bazel's experimental `shared_cc_library` in +several ways. First, this rule ignores transitive dependencies, which means +that dynamic library dependencies generally need to be specified via +'linkopts'. Second, this rule produces a static archive library in addition +to the dynamic shared library. Third, this rule is not directly usable as a +C++ dependency (although the outputs could be used, e.g., by `cc_import`). + +Example: + + cc_library(name = "a", srcs = ["a.cc"], hdrs = ["a.h"]) + cc_library(name = "b", srcs = ["b.cc"], hdrs = ["b.h"], deps = [":a"]) + cc_library(name = "c", srcs = ["c.cc"], hdrs = ["c.h"], deps = [":b"]) + + # Creates libdist.so and (typically) libdist.pic.a: + # (This may also produce libdist.a if the build produces non-PIC objects.) + cc_dist_library( + name = "dist", + linkopts = ["-la"], # libdist.so dynamically links against liba.so. + deps = [":b", ":c"], # Output contains b.o and c.o, but not a.o. + ) +""", + attrs = { + "deps": attr.label_list( + doc = ("The list of libraries to be included in the outputs. " + + "Only these targets' compilation outputs will be " + + "included (i.e., the transitive dependencies are not " + + "included in the output)."), + ), + "linkopts": attr.string_list( + doc = ("Add these flags to the C++ linker command when creating " + + "the dynamic library."), + ), + # C++ toolchain before https://github.com/bazelbuild/bazel/issues/7260: + "_cc_toolchain": attr.label( + default = Label("@rules_cc//cc:current_cc_toolchain"), + ), + }, + toolchains = [ + # C++ toolchain after https://github.com/bazelbuild/bazel/issues/7260: + "@bazel_tools//tools/cpp:toolchain_type", + ], + fragments = ["cpp"], +)