diff --git a/api/bazel/envoy_http_archive.bzl b/api/bazel/envoy_http_archive.bzl index 13b98f770619..15fd65b2af27 100644 --- a/api/bazel/envoy_http_archive.bzl +++ b/api/bazel/envoy_http_archive.bzl @@ -10,8 +10,7 @@ def envoy_http_archive(name, locations, **kwargs): # This repository has already been defined, probably because the user # wants to override the version. Do nothing. return - loc_key = kwargs.pop("repository_key", name) - location = locations[loc_key] + location = locations[name] # HTTP tarball at a given URL. Add a BUILD file if requested. http_archive( diff --git a/api/bazel/external_deps.bzl b/api/bazel/external_deps.bzl new file mode 100644 index 000000000000..cd9b6759f98a --- /dev/null +++ b/api/bazel/external_deps.bzl @@ -0,0 +1,116 @@ +load("@envoy_api//bazel:repository_locations_utils.bzl", "load_repository_locations_spec") + +# Envoy dependencies may be annotated with the following attributes: +DEPENDENCY_ANNOTATIONS = [ + # List of the categories describing how the dependency is being used. This attribute is used + # for automatic tracking of security posture of Envoy's dependencies. + # Possible values are documented in the USE_CATEGORIES list below. + # This attribute is mandatory for each dependecy. + "use_category", + + # Attribute specifying CPE (Common Platform Enumeration, see https://nvd.nist.gov/products/cpe) ID + # of the dependency. The ID may be in v2.3 or v2.2 format, although v2.3 is prefferred. See + # https://nvd.nist.gov/products/cpe for CPE format. Use single wildcard '*' for version and vector elements + # i.e. 'cpe:2.3:a:nghttp2:nghttp2:*'. Use "N/A" for dependencies without CPE assigned. + # This attribute is optional for components with use categories listed in the + # USE_CATEGORIES_WITH_CPE_OPTIONAL + "cpe", +] + +# NOTE: If a dependency use case is either dataplane or controlplane, the other uses are not needed +# to be declared. +USE_CATEGORIES = [ + # This dependency is used in API protos. + "api", + # This dependency is used in build process. + "build", + # This dependency is used to process xDS requests. + "controlplane", + # This dependency is used in processing downstream or upstream requests (core). + "dataplane_core", + # This dependency is used in processing downstream or upstream requests (extensions). + "dataplane_ext", + # This dependecy is used for logging, metrics or tracing (core). It may process unstrusted input. + "observability_core", + # This dependecy is used for logging, metrics or tracing (extensions). It may process unstrusted input. + "observability_ext", + # This dependency does not handle untrusted data and is used for various utility purposes. + "other", + # This dependency is used only in tests. + "test_only", +] + +# Components with these use categories are not required to specify the 'cpe' +# and 'last_updated' annotation. +USE_CATEGORIES_WITH_CPE_OPTIONAL = ["build", "other", "test_only", "api"] + +def _fail_missing_attribute(attr, key): + fail("The '%s' attribute must be defined for external dependecy " % attr + key) + +# Method for verifying content of the repository location specifications. +# +# We also remove repository metadata attributes so that further consumers, e.g. +# http_archive, are not confused by them. +def load_repository_locations(repository_locations_spec): + locations = {} + for key, location in load_repository_locations_spec(repository_locations_spec).items(): + mutable_location = dict(location) + locations[key] = mutable_location + + if "sha256" not in location or len(location["sha256"]) == 0: + _fail_missing_attribute("sha256", key) + + if "project_name" not in location: + _fail_missing_attribute("project_name", key) + mutable_location.pop("project_name") + + if "project_desc" not in location: + _fail_missing_attribute("project_desc", key) + mutable_location.pop("project_desc") + + if "project_url" not in location: + _fail_missing_attribute("project_url", key) + project_url = mutable_location.pop("project_url") + if not project_url.startswith("https://") and not project_url.startswith("http://"): + fail("project_url must start with https:// or http://: " + project_url) + + if "version" not in location: + _fail_missing_attribute("version", key) + mutable_location.pop("version") + + if "use_category" not in location: + _fail_missing_attribute("use_category", key) + use_category = mutable_location.pop("use_category") + + if "dataplane_ext" in use_category or "observability_ext" in use_category: + if "extensions" not in location: + _fail_missing_attribute("extensions", key) + mutable_location.pop("extensions") + + if "last_updated" not in location: + _fail_missing_attribute("last_updated", key) + last_updated = mutable_location.pop("last_updated") + + # Starlark doesn't have regexes. + if len(last_updated) != 10 or last_updated[4] != "-" or last_updated[7] != "-": + fail("last_updated must match YYYY-DD-MM: " + last_updated) + + if "cpe" in location: + cpe = mutable_location.pop("cpe") + + # Starlark doesn't have regexes. + cpe_components = len(cpe.split(":")) + + # We allow cpe:2.3:a:foo:* and cpe:2.3.:a:foo:bar:* only. + cpe_components_valid = cpe_components in [5, 6] + cpe_matches = (cpe == "N/A" or (cpe.startswith("cpe:2.3:a:") and cpe.endswith(":*") and cpe_components_valid)) + if not cpe_matches: + fail("CPE must match cpe:2.3:a:::*: " + cpe) + elif not [category for category in USE_CATEGORIES_WITH_CPE_OPTIONAL if category in location["use_category"]]: + _fail_missing_attribute("cpe", key) + + for category in location["use_category"]: + if category not in USE_CATEGORIES: + fail("Unknown use_category value '" + category + "' for dependecy " + key) + + return locations diff --git a/api/bazel/repositories.bzl b/api/bazel/repositories.bzl index a64e733cf74a..a12a0ea98b3a 100644 --- a/api/bazel/repositories.bzl +++ b/api/bazel/repositories.bzl @@ -1,40 +1,43 @@ load(":envoy_http_archive.bzl", "envoy_http_archive") -load(":repository_locations.bzl", "REPOSITORY_LOCATIONS") +load(":external_deps.bzl", "load_repository_locations") +load(":repository_locations.bzl", "REPOSITORY_LOCATIONS_SPEC") -def api_dependencies(): +REPOSITORY_LOCATIONS = load_repository_locations(REPOSITORY_LOCATIONS_SPEC) + +# Use this macro to reference any HTTP archive from bazel/repository_locations.bzl. +def external_http_archive(name, **kwargs): envoy_http_archive( - "bazel_skylib", + name, locations = REPOSITORY_LOCATIONS, + **kwargs ) - envoy_http_archive( - "com_envoyproxy_protoc_gen_validate", - locations = REPOSITORY_LOCATIONS, + +def api_dependencies(): + external_http_archive( + name = "bazel_skylib", ) - envoy_http_archive( + external_http_archive( + name = "com_envoyproxy_protoc_gen_validate", + ) + external_http_archive( name = "com_google_googleapis", - locations = REPOSITORY_LOCATIONS, ) - envoy_http_archive( + external_http_archive( name = "com_github_cncf_udpa", - locations = REPOSITORY_LOCATIONS, ) - envoy_http_archive( + external_http_archive( name = "prometheus_metrics_model", - locations = REPOSITORY_LOCATIONS, build_file_content = PROMETHEUSMETRICS_BUILD_CONTENT, ) - envoy_http_archive( + external_http_archive( name = "opencensus_proto", - locations = REPOSITORY_LOCATIONS, ) - envoy_http_archive( + external_http_archive( name = "rules_proto", - locations = REPOSITORY_LOCATIONS, ) - envoy_http_archive( + external_http_archive( name = "com_github_openzipkin_zipkinapi", - locations = REPOSITORY_LOCATIONS, build_file_content = ZIPKINAPI_BUILD_CONTENT, ) diff --git a/api/bazel/repository_locations.bzl b/api/bazel/repository_locations.bzl index 152d4be5ee0d..bdcf31e867d2 100644 --- a/api/bazel/repository_locations.bzl +++ b/api/bazel/repository_locations.bzl @@ -1,4 +1,5 @@ -DEPENDENCY_REPOSITORIES_SPEC = dict( +# This should match the schema defined in external_deps.bzl. +REPOSITORY_LOCATIONS_SPEC = dict( bazel_skylib = dict( project_name = "bazel-skylib", project_desc = "Common useful functions and rules for Bazel", @@ -88,23 +89,3 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( use_category = ["api"], ), ) - -def _format_version(s, version): - return s.format(version = version, dash_version = version.replace(".", "-"), underscore_version = version.replace(".", "_")) - -# Interpolate {version} in the above dependency specs. This code should be capable of running in both Python -# and Starlark. -def _dependency_repositories(): - locations = {} - for key, location in DEPENDENCY_REPOSITORIES_SPEC.items(): - mutable_location = dict(location) - locations[key] = mutable_location - - # Fixup with version information. - if "version" in location: - if "strip_prefix" in location: - mutable_location["strip_prefix"] = _format_version(location["strip_prefix"], location["version"]) - mutable_location["urls"] = [_format_version(url, location["version"]) for url in location["urls"]] - return locations - -REPOSITORY_LOCATIONS = _dependency_repositories() diff --git a/api/bazel/repository_locations_utils.bzl b/api/bazel/repository_locations_utils.bzl new file mode 100644 index 000000000000..3b984e1bc580 --- /dev/null +++ b/api/bazel/repository_locations_utils.bzl @@ -0,0 +1,20 @@ +def _format_version(s, version): + return s.format(version = version, dash_version = version.replace(".", "-"), underscore_version = version.replace(".", "_")) + +# Generate a "repository location specification" from raw repository +# specification. The information should match the format required by +# external_deps.bzl. This function mostly does interpolation of {version} in +# the repository info fields. This code should be capable of running in both +# Python and Starlark. +def load_repository_locations_spec(repository_locations_spec): + locations = {} + for key, location in repository_locations_spec.items(): + mutable_location = dict(location) + locations[key] = mutable_location + + # Fixup with version information. + if "version" in location: + if "strip_prefix" in location: + mutable_location["strip_prefix"] = _format_version(location["strip_prefix"], location["version"]) + mutable_location["urls"] = [_format_version(url, location["version"]) for url in location["urls"]] + return locations diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index a8115a021ced..12825602848c 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -1,8 +1,8 @@ -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load(":dev_binding.bzl", "envoy_dev_binding") load(":genrule_repository.bzl", "genrule_repository") load("@envoy_api//bazel:envoy_http_archive.bzl", "envoy_http_archive") -load(":repository_locations.bzl", "DEPENDENCY_ANNOTATIONS", "DEPENDENCY_REPOSITORIES", "USE_CATEGORIES", "USE_CATEGORIES_WITH_CPE_OPTIONAL") +load("@envoy_api//bazel:external_deps.bzl", "load_repository_locations") +load(":repository_locations.bzl", "REPOSITORY_LOCATIONS_SPEC") load("@com_google_googleapis//:repository_rules.bzl", "switched_rules_by_language") load(":crates.bzl", "raze_fetch_remote_crates") @@ -18,97 +18,29 @@ WINDOWS_SKIP_TARGETS = [ # Make all contents of an external repository accessible under a filegroup. Used for external HTTP # archives, e.g. cares. -BUILD_ALL_CONTENT = """filegroup(name = "all", srcs = glob(["**"]), visibility = ["//visibility:public"])""" - def _build_all_content(exclude = []): return """filegroup(name = "all", srcs = glob(["**"], exclude={}), visibility = ["//visibility:public"])""".format(repr(exclude)) -def _fail_missing_attribute(attr, key): - fail("The '%s' attribute must be defined for external dependecy " % attr + key) - -# Method for verifying content of the DEPENDENCY_REPOSITORIES defined in bazel/repository_locations.bzl -# Verification is here so that bazel/repository_locations.bzl can be loaded into other tools written in Python, -# and as such needs to be free of bazel specific constructs. -# -# We also remove the attributes for further consumption in this file, since rules such as http_archive -# don't recognize them. -def _repository_locations(): - locations = {} - for key, location in DEPENDENCY_REPOSITORIES.items(): - mutable_location = dict(location) - locations[key] = mutable_location - - if "sha256" not in location or len(location["sha256"]) == 0: - _fail_missing_attribute("sha256", key) - - if "project_name" not in location: - _fail_missing_attribute("project_name", key) - mutable_location.pop("project_name") - - if "project_desc" not in location: - _fail_missing_attribute("project_desc", key) - mutable_location.pop("project_desc") - - if "project_url" not in location: - _fail_missing_attribute("project_url", key) - project_url = mutable_location.pop("project_url") - if not project_url.startswith("https://") and not project_url.startswith("http://"): - fail("project_url must start with https:// or http://: " + project_url) - - if "version" not in location: - _fail_missing_attribute("version", key) - mutable_location.pop("version") - - if "use_category" not in location: - _fail_missing_attribute("use_category", key) - use_category = mutable_location.pop("use_category") - - if "dataplane_ext" in use_category or "observability_ext" in use_category: - if "extensions" not in location: - _fail_missing_attribute("extensions", key) - mutable_location.pop("extensions") - - if "last_updated" not in location: - _fail_missing_attribute("last_updated", key) - last_updated = mutable_location.pop("last_updated") - - # Starlark doesn't have regexes. - if len(last_updated) != 10 or last_updated[4] != "-" or last_updated[7] != "-": - fail("last_updated must match YYYY-DD-MM: " + last_updated) - - if "cpe" in location: - cpe = mutable_location.pop("cpe") - - # Starlark doesn't have regexes. - cpe_matches = (cpe != "N/A" and (not cpe.startswith("cpe:2.3:a:") or not cpe.endswith(":*") and len(cpe.split(":")) != 6)) - if cpe_matches: - fail("CPE must match cpe:2.3:a:::*: " + cpe) - elif not [category for category in USE_CATEGORIES_WITH_CPE_OPTIONAL if category in location["use_category"]]: - _fail_missing_attribute("cpe", key) - - for category in location["use_category"]: - if category not in USE_CATEGORIES: - fail("Unknown use_category value '" + category + "' for dependecy " + key) - - return locations - -REPOSITORY_LOCATIONS = _repository_locations() - -# To initialize http_archive REPOSITORY_LOCATIONS dictionaries must be stripped of annotations. -# See repository_locations.bzl for the list of annotation attributes. -def _get_location(dependency): - stripped = dict(REPOSITORY_LOCATIONS[dependency]) - for attribute in DEPENDENCY_ANNOTATIONS: - stripped.pop(attribute, None) - return stripped - -def _repository_impl(name, **kwargs): +BUILD_ALL_CONTENT = _build_all_content() + +REPOSITORY_LOCATIONS = load_repository_locations(REPOSITORY_LOCATIONS_SPEC) + +# Use this macro to reference any HTTP archive from bazel/repository_locations.bzl. +def external_http_archive(name, **kwargs): envoy_http_archive( name, locations = REPOSITORY_LOCATIONS, **kwargs ) +# Use this macro to reference any genrule_repository sourced from bazel/repository_locations.bzl. +def external_genrule_repository(name, **kwargs): + location = REPOSITORY_LOCATIONS[name] + genrule_repository( + name = name, + **dict(location, **kwargs) + ) + def _default_envoy_build_config_impl(ctx): ctx.file("WORKSPACE", "") ctx.file("BUILD.bazel", "") @@ -124,26 +56,26 @@ _default_envoy_build_config = repository_rule( # Python dependencies. def _python_deps(): # TODO(htuch): convert these to pip3_import. - _repository_impl( + external_http_archive( name = "com_github_twitter_common_lang", build_file = "@envoy//bazel/external:twitter_common_lang.BUILD", ) - _repository_impl( + external_http_archive( name = "com_github_twitter_common_rpc", build_file = "@envoy//bazel/external:twitter_common_rpc.BUILD", ) - _repository_impl( + external_http_archive( name = "com_github_twitter_common_finagle_thrift", build_file = "@envoy//bazel/external:twitter_common_finagle_thrift.BUILD", ) - _repository_impl( + external_http_archive( name = "six", build_file = "@com_google_protobuf//third_party:six.BUILD", ) # Bazel native C++ dependencies. For the dependencies that doesn't provide autoconf/automake builds. def _cc_deps(): - _repository_impl("grpc_httpjson_transcoding") + external_http_archive("grpc_httpjson_transcoding") native.bind( name = "path_matcher", actual = "@grpc_httpjson_transcoding//src:path_matcher", @@ -157,16 +89,16 @@ def _go_deps(skip_targets): # Keep the skip_targets check around until Istio Proxy has stopped using # it to exclude the Go rules. if "io_bazel_rules_go" not in skip_targets: - _repository_impl( + external_http_archive( name = "io_bazel_rules_go", # TODO(wrowe, sunjayBhatia): remove when Windows RBE supports batch file invocation patch_args = ["-p1"], patches = ["@envoy//bazel:rules_go.patch"], ) - _repository_impl("bazel_gazelle") + external_http_archive("bazel_gazelle") def _rust_deps(): - _repository_impl("io_bazel_rules_rust") + external_http_archive("io_bazel_rules_rust") raze_fetch_remote_crates() def envoy_dependencies(skip_targets = []): @@ -232,13 +164,13 @@ def envoy_dependencies(skip_targets = []): _proxy_wasm_cpp_sdk() _proxy_wasm_cpp_host() _emscripten_toolchain() - _repository_impl("com_googlesource_code_re2") + external_http_archive("com_googlesource_code_re2") _com_google_cel_cpp() - _repository_impl("com_github_google_flatbuffers") - _repository_impl("bazel_toolchains") - _repository_impl("bazel_compdb") - _repository_impl("envoy_build_tools") - _repository_impl("rules_cc") + external_http_archive("com_github_google_flatbuffers") + external_http_archive("bazel_toolchains") + external_http_archive("bazel_compdb") + external_http_archive("envoy_build_tools") + external_http_archive("rules_cc") # Unconditional, since we use this only for compiler-agnostic fuzzing utils. _org_llvm_releases_compiler_rt() @@ -267,25 +199,22 @@ def envoy_dependencies(skip_targets = []): ) def _boringssl(): - _repository_impl( + external_http_archive( name = "boringssl", patch_args = ["-p1"], patches = ["@envoy//bazel:boringssl_static.patch"], ) def _boringssl_fips(): - location = REPOSITORY_LOCATIONS["boringssl_fips"] - genrule_repository( + external_genrule_repository( name = "boringssl_fips", - urls = location["urls"], - sha256 = location["sha256"], genrule_cmd_file = "@envoy//bazel/external:boringssl_fips.genrule_cmd", build_file = "@envoy//bazel/external:boringssl_fips.BUILD", patches = ["@envoy//bazel/external:boringssl_fips.patch"], ) def _com_github_circonus_labs_libcircllhist(): - _repository_impl( + external_http_archive( name = "com_github_circonus_labs_libcircllhist", build_file = "@envoy//bazel/external:libcircllhist.BUILD", ) @@ -295,11 +224,9 @@ def _com_github_circonus_labs_libcircllhist(): ) def _com_github_c_ares_c_ares(): - location = _get_location("com_github_c_ares_c_ares") - http_archive( + external_http_archive( name = "com_github_c_ares_c_ares", build_file_content = BUILD_ALL_CONTENT, - **location ) native.bind( name = "ares", @@ -307,7 +234,7 @@ def _com_github_c_ares_c_ares(): ) def _com_github_cyan4973_xxhash(): - _repository_impl( + external_http_archive( name = "com_github_cyan4973_xxhash", build_file = "@envoy//bazel/external:xxhash.BUILD", ) @@ -317,7 +244,7 @@ def _com_github_cyan4973_xxhash(): ) def _com_github_envoyproxy_sqlparser(): - _repository_impl( + external_http_archive( name = "com_github_envoyproxy_sqlparser", build_file = "@envoy//bazel/external:sqlparser.BUILD", ) @@ -327,7 +254,7 @@ def _com_github_envoyproxy_sqlparser(): ) def _com_github_mirror_tclap(): - _repository_impl( + external_http_archive( name = "com_github_mirror_tclap", build_file = "@envoy//bazel/external:tclap.BUILD", patch_args = ["-p1"], @@ -343,7 +270,7 @@ def _com_github_mirror_tclap(): ) def _com_github_fmtlib_fmt(): - _repository_impl( + external_http_archive( name = "com_github_fmtlib_fmt", build_file = "@envoy//bazel/external:fmtlib.BUILD", ) @@ -353,7 +280,7 @@ def _com_github_fmtlib_fmt(): ) def _com_github_gabime_spdlog(): - _repository_impl( + external_http_archive( name = "com_github_gabime_spdlog", build_file = "@envoy//bazel/external:spdlog.BUILD", ) @@ -363,10 +290,8 @@ def _com_github_gabime_spdlog(): ) def _com_github_google_benchmark(): - location = _get_location("com_github_google_benchmark") - http_archive( + external_http_archive( name = "com_github_google_benchmark", - **location ) native.bind( name = "benchmark", @@ -374,13 +299,13 @@ def _com_github_google_benchmark(): ) def _com_github_google_libprotobuf_mutator(): - _repository_impl( + external_http_archive( name = "com_github_google_libprotobuf_mutator", build_file = "@envoy//bazel/external:libprotobuf_mutator.BUILD", ) def _com_github_jbeder_yaml_cpp(): - _repository_impl( + external_http_archive( name = "com_github_jbeder_yaml_cpp", ) native.bind( @@ -389,11 +314,9 @@ def _com_github_jbeder_yaml_cpp(): ) def _com_github_libevent_libevent(): - location = _get_location("com_github_libevent_libevent") - http_archive( + external_http_archive( name = "com_github_libevent_libevent", build_file_content = BUILD_ALL_CONTENT, - **location ) native.bind( name = "event", @@ -401,7 +324,7 @@ def _com_github_libevent_libevent(): ) def _net_zlib(): - _repository_impl( + external_http_archive( name = "net_zlib", build_file_content = BUILD_ALL_CONTENT, patch_args = ["-p1"], @@ -420,16 +343,15 @@ def _net_zlib(): ) def _com_github_zlib_ng_zlib_ng(): - _repository_impl( + external_http_archive( name = "com_github_zlib_ng_zlib_ng", build_file_content = BUILD_ALL_CONTENT, ) def _com_google_cel_cpp(): - _repository_impl("com_google_cel_cpp") - _repository_impl("rules_antlr") - location = _get_location("antlr4_runtimes") - http_archive( + external_http_archive("com_google_cel_cpp") + external_http_archive("rules_antlr") + external_http_archive( name = "antlr4_runtimes", build_file_content = """ package(default_visibility = ["//visibility:public"]) @@ -443,19 +365,18 @@ cc_library( patch_args = ["-p1"], # Patches ASAN violation of initialization fiasco patches = ["@envoy//bazel:antlr.patch"], - **location ) # Parser dependencies # TODO: upgrade this when cel is upgraded to use the latest version - http_archive( + external_http_archive( name = "rules_antlr", sha256 = "7249d1569293d9b239e23c65f6b4c81a07da921738bde0dfeb231ed98be40429", strip_prefix = "rules_antlr-3cc2f9502a54ceb7b79b37383316b23c4da66f9a", urls = ["https://github.com/marcohu/rules_antlr/archive/3cc2f9502a54ceb7b79b37383316b23c4da66f9a.tar.gz"], ) - http_archive( + external_http_archive( name = "antlr4_runtimes", build_file_content = """ package(default_visibility = ["//visibility:public"]) @@ -475,8 +396,7 @@ cc_library( ) def _com_github_nghttp2_nghttp2(): - location = _get_location("com_github_nghttp2_nghttp2") - http_archive( + external_http_archive( name = "com_github_nghttp2_nghttp2", build_file_content = BUILD_ALL_CONTENT, patch_args = ["-p1"], @@ -485,7 +405,6 @@ def _com_github_nghttp2_nghttp2(): # https://github.com/nghttp2/nghttp2/pull/1395 # https://github.com/envoyproxy/envoy/pull/8572#discussion_r334067786 patches = ["@envoy//bazel/foreign_cc:nghttp2.patch"], - **location ) native.bind( name = "nghttp2", @@ -493,7 +412,7 @@ def _com_github_nghttp2_nghttp2(): ) def _io_opentracing_cpp(): - _repository_impl( + external_http_archive( name = "io_opentracing_cpp", patch_args = ["-p1"], # Workaround for LSAN false positive in https://github.com/envoyproxy/envoy/issues/7647 @@ -505,15 +424,15 @@ def _io_opentracing_cpp(): ) def _com_lightstep_tracer_cpp(): - _repository_impl("com_lightstep_tracer_cpp") + external_http_archive("com_lightstep_tracer_cpp") native.bind( name = "lightstep", actual = "@com_lightstep_tracer_cpp//:manual_tracer_lib", ) def _com_github_datadog_dd_opentracing_cpp(): - _repository_impl("com_github_datadog_dd_opentracing_cpp") - _repository_impl( + external_http_archive("com_github_datadog_dd_opentracing_cpp") + external_http_archive( name = "com_github_msgpack_msgpack_c", build_file = "@com_github_datadog_dd_opentracing_cpp//:bazel/external/msgpack.BUILD", ) @@ -523,7 +442,7 @@ def _com_github_datadog_dd_opentracing_cpp(): ) def _com_github_tencent_rapidjson(): - _repository_impl( + external_http_archive( name = "com_github_tencent_rapidjson", build_file = "@envoy//bazel/external:rapidjson.BUILD", ) @@ -533,7 +452,7 @@ def _com_github_tencent_rapidjson(): ) def _com_github_nodejs_http_parser(): - _repository_impl( + external_http_archive( name = "com_github_nodejs_http_parser", build_file = "@envoy//bazel/external:http-parser.BUILD", ) @@ -543,7 +462,7 @@ def _com_github_nodejs_http_parser(): ) def _com_google_googletest(): - _repository_impl("com_google_googletest") + external_http_archive("com_google_googletest") native.bind( name = "googletest", actual = "@com_google_googletest//:gtest", @@ -554,7 +473,7 @@ def _com_google_googletest(): # pull in more bits of abseil as needed, and is now the preferred # method for pure Bazel deps. def _com_google_absl(): - _repository_impl("com_google_absl") + external_http_archive("com_google_absl") native.bind( name = "abseil_any", actual = "@com_google_absl//absl/types:any", @@ -657,8 +576,8 @@ def _com_google_absl(): ) def _com_google_protobuf(): - _repository_impl("rules_python") - _repository_impl( + external_http_archive("rules_python") + external_http_archive( "com_google_protobuf", patches = ["@envoy//bazel:protobuf.patch"], patch_args = ["-p1"], @@ -689,10 +608,8 @@ def _com_google_protobuf(): ) def _io_opencensus_cpp(): - location = _get_location("io_opencensus_cpp") - http_archive( + external_http_archive( name = "io_opencensus_cpp", - **location ) native.bind( name = "opencensus_trace", @@ -733,8 +650,7 @@ def _io_opencensus_cpp(): def _com_github_curl(): # Used by OpenCensus Zipkin exporter. - location = _get_location("com_github_curl") - http_archive( + external_http_archive( name = "com_github_curl", build_file_content = BUILD_ALL_CONTENT + """ cc_library(name = "curl", visibility = ["//visibility:public"], deps = ["@envoy//bazel/foreign_cc:curl"]) @@ -749,7 +665,6 @@ cc_library(name = "curl", visibility = ["//visibility:public"], deps = ["@envoy/ # by elimination of the curl dependency. patches = ["@envoy//bazel/foreign_cc:curl.patch"], patch_args = ["-p1"], - **location ) native.bind( name = "curl", @@ -757,13 +672,11 @@ cc_library(name = "curl", visibility = ["//visibility:public"], deps = ["@envoy/ ) def _com_googlesource_chromium_v8(): - location = _get_location("com_googlesource_chromium_v8") - genrule_repository( + external_genrule_repository( name = "com_googlesource_chromium_v8", genrule_cmd_file = "@envoy//bazel/external:wee8.genrule_cmd", build_file = "@envoy//bazel/external:wee8.BUILD", patches = ["@envoy//bazel/external:wee8.patch"], - **location ) native.bind( name = "wee8", @@ -771,11 +684,8 @@ def _com_googlesource_chromium_v8(): ) def _com_googlesource_quiche(): - location = REPOSITORY_LOCATIONS["com_googlesource_quiche"] - genrule_repository( + external_genrule_repository( name = "com_googlesource_quiche", - urls = location["urls"], - sha256 = location["sha256"], genrule_cmd_file = "@envoy//bazel/external:quiche.genrule_cmd", build_file = "@envoy//bazel/external:quiche.BUILD", ) @@ -801,7 +711,7 @@ def _com_googlesource_quiche(): ) def _com_googlesource_googleurl(): - _repository_impl( + external_http_archive( name = "com_googlesource_googleurl", ) native.bind( @@ -810,14 +720,14 @@ def _com_googlesource_googleurl(): ) def _org_llvm_releases_compiler_rt(): - _repository_impl( + external_http_archive( name = "org_llvm_releases_compiler_rt", build_file = "@envoy//bazel/external:compiler_rt.BUILD", ) def _com_github_grpc_grpc(): - _repository_impl("com_github_grpc_grpc") - _repository_impl("build_bazel_rules_apple") + external_http_archive("com_github_grpc_grpc") + external_http_archive("build_bazel_rules_apple") # Rebind some stuff to match what the gRPC Bazel is expecting. native.bind( @@ -859,7 +769,7 @@ def _com_github_grpc_grpc(): ) def _upb(): - _repository_impl( + external_http_archive( name = "upb", patches = ["@envoy//bazel:upb.patch"], patch_args = ["-p1"], @@ -871,26 +781,28 @@ def _upb(): ) def _proxy_wasm_cpp_sdk(): - _repository_impl(name = "proxy_wasm_cpp_sdk") + external_http_archive(name = "proxy_wasm_cpp_sdk") def _proxy_wasm_cpp_host(): - _repository_impl( + external_http_archive( name = "proxy_wasm_cpp_host", build_file = "@envoy//bazel/external:proxy_wasm_cpp_host.BUILD", ) def _emscripten_toolchain(): - _repository_impl( + external_http_archive( name = "emscripten_toolchain", build_file_content = _build_all_content(exclude = [ "upstream/emscripten/cache/is_vanilla.txt", ".emscripten_sanity", ]), - patch_cmds = REPOSITORY_LOCATIONS["emscripten_toolchain"]["patch_cmds"], + patch_cmds = [ + "[[ \"$(uname -m)\" == \"x86_64\" ]] && ./emsdk install 1.39.6-upstream && ./emsdk activate --embedded 1.39.6-upstream || true", + ], ) def _com_github_google_jwt_verify(): - _repository_impl("com_github_google_jwt_verify") + external_http_archive("com_github_google_jwt_verify") native.bind( name = "jwt_verify_lib", @@ -898,14 +810,12 @@ def _com_github_google_jwt_verify(): ) def _com_github_luajit_luajit(): - location = _get_location("com_github_luajit_luajit") - http_archive( + external_http_archive( name = "com_github_luajit_luajit", build_file_content = BUILD_ALL_CONTENT, patches = ["@envoy//bazel/foreign_cc:luajit.patch"], patch_args = ["-p1"], patch_cmds = ["chmod u+x build.py"], - **location ) native.bind( @@ -914,14 +824,12 @@ def _com_github_luajit_luajit(): ) def _com_github_moonjit_moonjit(): - location = _get_location("com_github_moonjit_moonjit") - http_archive( + external_http_archive( name = "com_github_moonjit_moonjit", build_file_content = BUILD_ALL_CONTENT, patches = ["@envoy//bazel/foreign_cc:moonjit.patch"], patch_args = ["-p1"], patch_cmds = ["chmod u+x build.py"], - **location ) native.bind( @@ -930,7 +838,7 @@ def _com_github_moonjit_moonjit(): ) def _com_github_google_tcmalloc(): - _repository_impl( + external_http_archive( name = "com_github_google_tcmalloc", ) @@ -940,26 +848,21 @@ def _com_github_google_tcmalloc(): ) def _com_github_gperftools_gperftools(): - location = _get_location("com_github_gperftools_gperftools") - http_archive( + external_http_archive( name = "com_github_gperftools_gperftools", build_file_content = BUILD_ALL_CONTENT, - **location ) - native.bind( name = "gperftools", actual = "@envoy//bazel/foreign_cc:gperftools", ) def _org_llvm_llvm(): - location = _get_location("org_llvm_llvm") - http_archive( + external_http_archive( name = "org_llvm_llvm", build_file_content = BUILD_ALL_CONTENT, patch_args = ["-p1"], patches = ["@envoy//bazel/foreign_cc:llvm.patch"], - **location ) native.bind( name = "llvm", @@ -967,11 +870,9 @@ def _org_llvm_llvm(): ) def _com_github_wavm_wavm(): - location = _get_location("com_github_wavm_wavm") - http_archive( + external_http_archive( name = "com_github_wavm_wavm", build_file_content = BUILD_ALL_CONTENT, - **location ) native.bind( name = "wavm", @@ -993,31 +894,28 @@ filegroup( visibility = ["//visibility:public"], ) """ - http_archive( + external_http_archive( name = "kafka_source", build_file_content = KAFKASOURCE_BUILD_CONTENT, patches = ["@envoy//bazel/external:kafka_int32.patch"], - **_get_location("kafka_source") ) # This archive provides Kafka (and Zookeeper) binaries, that are used during Kafka integration # tests. - http_archive( + external_http_archive( name = "kafka_server_binary", build_file_content = BUILD_ALL_CONTENT, - **_get_location("kafka_server_binary") ) # This archive provides Kafka client in Python, so we can use it to interact with Kafka server # during interation tests. - http_archive( + external_http_archive( name = "kafka_python_client", build_file_content = BUILD_ALL_CONTENT, - **_get_location("kafka_python_client") ) def _foreign_cc_dependencies(): - _repository_impl("rules_foreign_cc") + external_http_archive("rules_foreign_cc") def _is_linux(ctxt): return ctxt.os.name == "linux" diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 07038daa5a72..7ab570248cd2 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1,51 +1,5 @@ -# Validation of content in this file is done on the bazel/repositories.bzl file to make it free of bazel -# constructs. This is to allow this file to be loaded into Python based build and maintenance tools. - -# Envoy dependencies may be annotated with the following attributes: -DEPENDENCY_ANNOTATIONS = [ - # List of the categories describing how the dependency is being used. This attribute is used - # for automatic tracking of security posture of Envoy's dependencies. - # Possible values are documented in the USE_CATEGORIES list below. - # This attribute is mandatory for each dependecy. - "use_category", - - # Attribute specifying CPE (Common Platform Enumeration, see https://nvd.nist.gov/products/cpe) ID - # of the dependency. The ID may be in v2.3 or v2.2 format, although v2.3 is prefferred. See - # https://nvd.nist.gov/products/cpe for CPE format. Use single wildcard '*' for version and vector elements - # i.e. 'cpe:2.3:a:nghttp2:nghttp2:*'. Use "N/A" for dependencies without CPE assigned. - # This attribute is optional for components with use categories listed in the - # USE_CATEGORIES_WITH_CPE_OPTIONAL - "cpe", -] - -# NOTE: If a dependency use case is either dataplane or controlplane, the other uses are not needed -# to be declared. -USE_CATEGORIES = [ - # This dependency is used in API protos. - "api", - # This dependency is used in build process. - "build", - # This dependency is used to process xDS requests. - "controlplane", - # This dependency is used in processing downstream or upstream requests (core). - "dataplane_core", - # This dependency is used in processing downstream or upstream requests (extensions). - "dataplane_ext", - # This dependecy is used for logging, metrics or tracing (core). It may process unstrusted input. - "observability_core", - # This dependecy is used for logging, metrics or tracing (extensions). It may process unstrusted input. - "observability_ext", - # This dependency does not handle untrusted data and is used for various utility purposes. - "other", - # This dependency is used only in tests. - "test_only", -] - -# Components with these use categories are not required to specify the 'cpe' -# and 'last_updated' annotation. -USE_CATEGORIES_WITH_CPE_OPTIONAL = ["build", "other", "test_only"] - -DEPENDENCY_REPOSITORIES_SPEC = dict( +# This should match the schema defined in external_deps.bzl. +REPOSITORY_LOCATIONS_SPEC = dict( bazel_compdb = dict( project_name = "bazel-compilation-database", project_desc = "Clang JSON compilation database support for Bazel", @@ -619,11 +573,11 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( project_name = "LLVM", project_desc = "LLVM Compiler Infrastructure", project_url = "https://llvm.org", - version = "10.0", + version = "10.0.0", sha256 = "df83a44b3a9a71029049ec101fb0077ecbbdf5fe41e395215025779099a98fdf", - strip_prefix = "llvm-{version}.0.src", - urls = ["https://github.com/llvm/llvm-project/releases/download/llvmorg-{version}.0/llvm-{version}.0.src.tar.xz"], - last_updated = "2020-03-24", + strip_prefix = "llvm-{version}.src", + urls = ["https://github.com/llvm/llvm-project/releases/download/llvmorg-{version}/llvm-{version}.src.tar.xz"], + last_updated = "2020-10-09", use_category = ["dataplane_ext"], extensions = [ "envoy.access_loggers.wasm", @@ -632,7 +586,7 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( "envoy.filters.network.wasm", "envoy.stat_sinks.wasm", ], - cpe = "N/A", + cpe = "cpe:2.3:a:llvm:*", ), com_github_wavm_wavm = dict( project_name = "WAVM", @@ -642,7 +596,7 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( sha256 = "cc3fcaf05d57010c9cf8eb920234679dede6c780137b55001fd34e4d14806f7c", strip_prefix = "WAVM-{version}", urls = ["https://github.com/WAVM/WAVM/archive/{version}.tar.gz"], - last_updated = "2020-07-06", + last_updated = "2020-10-09", use_category = ["dataplane_ext"], extensions = [ "envoy.access_loggers.wasm", @@ -651,7 +605,7 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( "envoy.filters.network.wasm", "envoy.stat_sinks.wasm", ], - cpe = "N/A", + cpe = "cpe:2.3:a:webassembly_virtual_machine_project:webassembly_virtual_machine:*", ), io_opencensus_cpp = dict( project_name = "OpenCensus C++", @@ -852,7 +806,6 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( version = "7afb39d868a973caa6216a535c24e37fb666b6f3", sha256 = "213d0b441bcc3df2c87933b24a593b5fd482fa8f4db158b707c60005b9e70040", strip_prefix = "proxy-wasm-cpp-sdk-{version}", - # 2020-09-10 urls = ["https://github.com/proxy-wasm/proxy-wasm-cpp-sdk/archive/{version}.tar.gz"], use_category = ["dataplane_ext"], extensions = [ @@ -862,14 +815,13 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( "envoy.filters.network.wasm", "envoy.stat_sinks.wasm", ], - last_updated = "2020-07-29", + last_updated = "2020-10-09", cpe = "N/A", ), proxy_wasm_cpp_host = dict( project_name = "WebAssembly for Proxies (C++ host implementation)", project_desc = "WebAssembly for Proxies (C++ host implementation)", project_url = "https://github.com/proxy-wasm/proxy-wasm-cpp-host", - # 2020-09-10 version = "49ed20e895b728aae6b811950a2939ecbaf76f7c", sha256 = "fa03293d01450b9164f8f56ef9227301f7d1af4f373f996400f75c93f6ebc822", strip_prefix = "proxy-wasm-cpp-host-{version}", @@ -882,7 +834,7 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( "envoy.filters.network.wasm", "envoy.stat_sinks.wasm", ], - last_updated = "2020-07-29", + last_updated = "2020-10-09", cpe = "N/A", ), # TODO: upgrade to the latest version (1.41 currently fails tests) @@ -890,15 +842,12 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( project_name = "Emscripten SDK", project_desc = "Emscripten SDK (use by Wasm)", project_url = "https://github.com/emscripten-core/emsdk", - version = "1.39", + version = "1.39.6", sha256 = "4ac0f1f3de8b3f1373d435cd7e58bd94de4146e751f099732167749a229b443b", - patch_cmds = [ - "[[ \"$(uname -m)\" == \"x86_64\" ]] && ./emsdk install 1.39.6-upstream && ./emsdk activate --embedded 1.39.6-upstream || true", - ], - strip_prefix = "emsdk-{version}.6", - urls = ["https://github.com/emscripten-core/emsdk/archive/{version}.6.tar.gz"], + strip_prefix = "emsdk-{version}", + urls = ["https://github.com/emscripten-core/emsdk/archive/{version}.tar.gz"], use_category = ["build"], - last_updated = "2020-07-29", + last_updated = "2020-10-09", ), io_bazel_rules_rust = dict( project_name = "Bazel rust rules", @@ -911,7 +860,7 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( strip_prefix = "rules_rust-{version}", urls = ["https://github.com/bazelbuild/rules_rust/archive/{version}.tar.gz"], use_category = ["build"], - last_updated = "2020-07-29", + last_updated = "2020-10-09", ), rules_antlr = dict( project_name = "ANTLR Rules for Bazel", @@ -921,7 +870,7 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( sha256 = "7249d1569293d9b239e23c65f6b4c81a07da921738bde0dfeb231ed98be40429", strip_prefix = "rules_antlr-{version}", urls = ["https://github.com/marcohu/rules_antlr/archive/{version}.tar.gz"], - # This should be "build", but that trips the verification in the docs. + # ANTLR has a runtime component, so is not purely build. use_category = ["dataplane_ext"], extensions = [ "envoy.access_loggers.wasm", @@ -953,23 +902,3 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( cpe = "N/A", ), ) - -def _format_version(s, version): - return s.format(version = version, dash_version = version.replace(".", "-"), underscore_version = version.replace(".", "_")) - -# Interpolate {version} in the above dependency specs. This code should be capable of running in both Python -# and Starlark. -def _dependency_repositories(): - locations = {} - for key, location in DEPENDENCY_REPOSITORIES_SPEC.items(): - mutable_location = dict(location) - locations[key] = mutable_location - - # Fixup with version information. - if "version" in location: - if "strip_prefix" in location: - mutable_location["strip_prefix"] = _format_version(location["strip_prefix"], location["version"]) - mutable_location["urls"] = [_format_version(url, location["version"]) for url in location["urls"]] - return locations - -DEPENDENCY_REPOSITORIES = _dependency_repositories() diff --git a/docs/generate_external_dep_rst.py b/docs/generate_external_dep_rst.py index 7fd4be73824b..3348df538837 100755 --- a/docs/generate_external_dep_rst.py +++ b/docs/generate_external_dep_rst.py @@ -10,13 +10,22 @@ from importlib.util import spec_from_loader, module_from_spec from importlib.machinery import SourceFileLoader -# bazel/repository_locations.bzl must have a .bzl suffix for Starlark import, so + +# Shared Starlark/Python files must have a .bzl suffix for Starlark import, so # we are forced to do this workaround. -_repository_locations_spec = spec_from_loader( - 'repository_locations', - SourceFileLoader('repository_locations', 'bazel/repository_locations.bzl')) -repository_locations = module_from_spec(_repository_locations_spec) -_repository_locations_spec.loader.exec_module(repository_locations) +def LoadModule(name, path): + spec = spec_from_loader(name, SourceFileLoader(name, path)) + module = module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +envoy_repository_locations = LoadModule('envoy_repository_locations', + 'bazel/repository_locations.bzl') +api_repository_locations = LoadModule('api_repository_locations', + 'api/bazel/repository_locations.bzl') +repository_locations_utils = LoadModule('repository_locations_utils', + 'api/bazel/repository_locations_utils.bzl') # Render a CSV table given a list of table headers, widths and list of rows @@ -101,7 +110,10 @@ def GetVersionUrl(metadata): Dep = namedtuple('Dep', ['name', 'sort_name', 'version', 'cpe', 'last_updated']) use_categories = defaultdict(lambda: defaultdict(list)) # Bin rendered dependencies into per-use category lists. - for k, v in repository_locations.DEPENDENCY_REPOSITORIES.items(): + spec_loader = repository_locations_utils.load_repository_locations_spec + spec = spec_loader(envoy_repository_locations.REPOSITORY_LOCATIONS_SPEC) + spec.update(spec_loader(api_repository_locations.REPOSITORY_LOCATIONS_SPEC)) + for k, v in spec.items(): cpe = v.get('cpe', '') if cpe == 'N/A': cpe = '' diff --git a/docs/root/intro/arch_overview/security/external_deps.rst b/docs/root/intro/arch_overview/security/external_deps.rst index e033bb2fc51e..5a0fb0ff5d78 100644 --- a/docs/root/intro/arch_overview/security/external_deps.rst +++ b/docs/root/intro/arch_overview/security/external_deps.rst @@ -21,6 +21,11 @@ Control plane .. include:: external_dep_controlplane.rst +API +--- + +.. include:: external_dep_api.rst + Observability (core) -------------------- diff --git a/generated_api_shadow/bazel/envoy_http_archive.bzl b/generated_api_shadow/bazel/envoy_http_archive.bzl index 13b98f770619..15fd65b2af27 100644 --- a/generated_api_shadow/bazel/envoy_http_archive.bzl +++ b/generated_api_shadow/bazel/envoy_http_archive.bzl @@ -10,8 +10,7 @@ def envoy_http_archive(name, locations, **kwargs): # This repository has already been defined, probably because the user # wants to override the version. Do nothing. return - loc_key = kwargs.pop("repository_key", name) - location = locations[loc_key] + location = locations[name] # HTTP tarball at a given URL. Add a BUILD file if requested. http_archive( diff --git a/generated_api_shadow/bazel/external_deps.bzl b/generated_api_shadow/bazel/external_deps.bzl new file mode 100644 index 000000000000..cd9b6759f98a --- /dev/null +++ b/generated_api_shadow/bazel/external_deps.bzl @@ -0,0 +1,116 @@ +load("@envoy_api//bazel:repository_locations_utils.bzl", "load_repository_locations_spec") + +# Envoy dependencies may be annotated with the following attributes: +DEPENDENCY_ANNOTATIONS = [ + # List of the categories describing how the dependency is being used. This attribute is used + # for automatic tracking of security posture of Envoy's dependencies. + # Possible values are documented in the USE_CATEGORIES list below. + # This attribute is mandatory for each dependecy. + "use_category", + + # Attribute specifying CPE (Common Platform Enumeration, see https://nvd.nist.gov/products/cpe) ID + # of the dependency. The ID may be in v2.3 or v2.2 format, although v2.3 is prefferred. See + # https://nvd.nist.gov/products/cpe for CPE format. Use single wildcard '*' for version and vector elements + # i.e. 'cpe:2.3:a:nghttp2:nghttp2:*'. Use "N/A" for dependencies without CPE assigned. + # This attribute is optional for components with use categories listed in the + # USE_CATEGORIES_WITH_CPE_OPTIONAL + "cpe", +] + +# NOTE: If a dependency use case is either dataplane or controlplane, the other uses are not needed +# to be declared. +USE_CATEGORIES = [ + # This dependency is used in API protos. + "api", + # This dependency is used in build process. + "build", + # This dependency is used to process xDS requests. + "controlplane", + # This dependency is used in processing downstream or upstream requests (core). + "dataplane_core", + # This dependency is used in processing downstream or upstream requests (extensions). + "dataplane_ext", + # This dependecy is used for logging, metrics or tracing (core). It may process unstrusted input. + "observability_core", + # This dependecy is used for logging, metrics or tracing (extensions). It may process unstrusted input. + "observability_ext", + # This dependency does not handle untrusted data and is used for various utility purposes. + "other", + # This dependency is used only in tests. + "test_only", +] + +# Components with these use categories are not required to specify the 'cpe' +# and 'last_updated' annotation. +USE_CATEGORIES_WITH_CPE_OPTIONAL = ["build", "other", "test_only", "api"] + +def _fail_missing_attribute(attr, key): + fail("The '%s' attribute must be defined for external dependecy " % attr + key) + +# Method for verifying content of the repository location specifications. +# +# We also remove repository metadata attributes so that further consumers, e.g. +# http_archive, are not confused by them. +def load_repository_locations(repository_locations_spec): + locations = {} + for key, location in load_repository_locations_spec(repository_locations_spec).items(): + mutable_location = dict(location) + locations[key] = mutable_location + + if "sha256" not in location or len(location["sha256"]) == 0: + _fail_missing_attribute("sha256", key) + + if "project_name" not in location: + _fail_missing_attribute("project_name", key) + mutable_location.pop("project_name") + + if "project_desc" not in location: + _fail_missing_attribute("project_desc", key) + mutable_location.pop("project_desc") + + if "project_url" not in location: + _fail_missing_attribute("project_url", key) + project_url = mutable_location.pop("project_url") + if not project_url.startswith("https://") and not project_url.startswith("http://"): + fail("project_url must start with https:// or http://: " + project_url) + + if "version" not in location: + _fail_missing_attribute("version", key) + mutable_location.pop("version") + + if "use_category" not in location: + _fail_missing_attribute("use_category", key) + use_category = mutable_location.pop("use_category") + + if "dataplane_ext" in use_category or "observability_ext" in use_category: + if "extensions" not in location: + _fail_missing_attribute("extensions", key) + mutable_location.pop("extensions") + + if "last_updated" not in location: + _fail_missing_attribute("last_updated", key) + last_updated = mutable_location.pop("last_updated") + + # Starlark doesn't have regexes. + if len(last_updated) != 10 or last_updated[4] != "-" or last_updated[7] != "-": + fail("last_updated must match YYYY-DD-MM: " + last_updated) + + if "cpe" in location: + cpe = mutable_location.pop("cpe") + + # Starlark doesn't have regexes. + cpe_components = len(cpe.split(":")) + + # We allow cpe:2.3:a:foo:* and cpe:2.3.:a:foo:bar:* only. + cpe_components_valid = cpe_components in [5, 6] + cpe_matches = (cpe == "N/A" or (cpe.startswith("cpe:2.3:a:") and cpe.endswith(":*") and cpe_components_valid)) + if not cpe_matches: + fail("CPE must match cpe:2.3:a:::*: " + cpe) + elif not [category for category in USE_CATEGORIES_WITH_CPE_OPTIONAL if category in location["use_category"]]: + _fail_missing_attribute("cpe", key) + + for category in location["use_category"]: + if category not in USE_CATEGORIES: + fail("Unknown use_category value '" + category + "' for dependecy " + key) + + return locations diff --git a/generated_api_shadow/bazel/repositories.bzl b/generated_api_shadow/bazel/repositories.bzl index a64e733cf74a..a12a0ea98b3a 100644 --- a/generated_api_shadow/bazel/repositories.bzl +++ b/generated_api_shadow/bazel/repositories.bzl @@ -1,40 +1,43 @@ load(":envoy_http_archive.bzl", "envoy_http_archive") -load(":repository_locations.bzl", "REPOSITORY_LOCATIONS") +load(":external_deps.bzl", "load_repository_locations") +load(":repository_locations.bzl", "REPOSITORY_LOCATIONS_SPEC") -def api_dependencies(): +REPOSITORY_LOCATIONS = load_repository_locations(REPOSITORY_LOCATIONS_SPEC) + +# Use this macro to reference any HTTP archive from bazel/repository_locations.bzl. +def external_http_archive(name, **kwargs): envoy_http_archive( - "bazel_skylib", + name, locations = REPOSITORY_LOCATIONS, + **kwargs ) - envoy_http_archive( - "com_envoyproxy_protoc_gen_validate", - locations = REPOSITORY_LOCATIONS, + +def api_dependencies(): + external_http_archive( + name = "bazel_skylib", ) - envoy_http_archive( + external_http_archive( + name = "com_envoyproxy_protoc_gen_validate", + ) + external_http_archive( name = "com_google_googleapis", - locations = REPOSITORY_LOCATIONS, ) - envoy_http_archive( + external_http_archive( name = "com_github_cncf_udpa", - locations = REPOSITORY_LOCATIONS, ) - envoy_http_archive( + external_http_archive( name = "prometheus_metrics_model", - locations = REPOSITORY_LOCATIONS, build_file_content = PROMETHEUSMETRICS_BUILD_CONTENT, ) - envoy_http_archive( + external_http_archive( name = "opencensus_proto", - locations = REPOSITORY_LOCATIONS, ) - envoy_http_archive( + external_http_archive( name = "rules_proto", - locations = REPOSITORY_LOCATIONS, ) - envoy_http_archive( + external_http_archive( name = "com_github_openzipkin_zipkinapi", - locations = REPOSITORY_LOCATIONS, build_file_content = ZIPKINAPI_BUILD_CONTENT, ) diff --git a/generated_api_shadow/bazel/repository_locations.bzl b/generated_api_shadow/bazel/repository_locations.bzl index 152d4be5ee0d..bdcf31e867d2 100644 --- a/generated_api_shadow/bazel/repository_locations.bzl +++ b/generated_api_shadow/bazel/repository_locations.bzl @@ -1,4 +1,5 @@ -DEPENDENCY_REPOSITORIES_SPEC = dict( +# This should match the schema defined in external_deps.bzl. +REPOSITORY_LOCATIONS_SPEC = dict( bazel_skylib = dict( project_name = "bazel-skylib", project_desc = "Common useful functions and rules for Bazel", @@ -88,23 +89,3 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( use_category = ["api"], ), ) - -def _format_version(s, version): - return s.format(version = version, dash_version = version.replace(".", "-"), underscore_version = version.replace(".", "_")) - -# Interpolate {version} in the above dependency specs. This code should be capable of running in both Python -# and Starlark. -def _dependency_repositories(): - locations = {} - for key, location in DEPENDENCY_REPOSITORIES_SPEC.items(): - mutable_location = dict(location) - locations[key] = mutable_location - - # Fixup with version information. - if "version" in location: - if "strip_prefix" in location: - mutable_location["strip_prefix"] = _format_version(location["strip_prefix"], location["version"]) - mutable_location["urls"] = [_format_version(url, location["version"]) for url in location["urls"]] - return locations - -REPOSITORY_LOCATIONS = _dependency_repositories() diff --git a/generated_api_shadow/bazel/repository_locations_utils.bzl b/generated_api_shadow/bazel/repository_locations_utils.bzl new file mode 100644 index 000000000000..3b984e1bc580 --- /dev/null +++ b/generated_api_shadow/bazel/repository_locations_utils.bzl @@ -0,0 +1,20 @@ +def _format_version(s, version): + return s.format(version = version, dash_version = version.replace(".", "-"), underscore_version = version.replace(".", "_")) + +# Generate a "repository location specification" from raw repository +# specification. The information should match the format required by +# external_deps.bzl. This function mostly does interpolation of {version} in +# the repository info fields. This code should be capable of running in both +# Python and Starlark. +def load_repository_locations_spec(repository_locations_spec): + locations = {} + for key, location in repository_locations_spec.items(): + mutable_location = dict(location) + locations[key] = mutable_location + + # Fixup with version information. + if "version" in location: + if "strip_prefix" in location: + mutable_location["strip_prefix"] = _format_version(location["strip_prefix"], location["version"]) + mutable_location["urls"] = [_format_version(url, location["version"]) for url in location["urls"]] + return locations diff --git a/tools/dependency/validate.py b/tools/dependency/validate.py index 273c3117918c..813d9bd90787 100755 --- a/tools/dependency/validate.py +++ b/tools/dependency/validate.py @@ -59,7 +59,7 @@ def DepsByUseCategory(self, use_category): Returns: Set of dependency identifiers that match use_category. """ - return set(name for name, metadata in repository_locations.DEPENDENCY_REPOSITORIES.items() + return set(name for name, metadata in repository_locations.REPOSITORY_LOCATIONS_SPEC.items() if use_category in metadata['use_category']) def GetMetadata(self, dependency): @@ -72,7 +72,7 @@ def GetMetadata(self, dependency): A dictionary with the repository metadata as defined in bazel/repository_locations.bzl. """ - return repository_locations.DEPENDENCY_REPOSITORIES.get(dependency) + return repository_locations.REPOSITORY_LOCATIONS_SPEC.get(dependency) class BuildGraph(object):