dependencies: refactor repository location schema utils, cleanups. (#…

…13452)

- Refactor code responsible for processing repository location specs, i.e. checking for the presence of fields like last_updated and interpolation of version. The same code is now usable by both API repository_locations.bzl and bazel/repository_locations.bzl.

- Cleanup reference to repo locations in repository_locations.bzl, now using a consistent set of macros.

- Add API dependencies to dependency dashboard.

Risk level: Low
Testing: Docs build.

Part of #12673

Signed-off-by: Harvey Tuch <htuch@google.com>

api/bazel/envoy_http_archive.bzl

@@ -10,8 +10,7 @@ def envoy_http_archive(name, locations, **kwargs):
         # This repository has already been defined, probably because the user
         # wants to override the version. Do nothing.
         return
-    loc_key = kwargs.pop("repository_key", name)
-    location = locations[loc_key]
+    location = locations[name]
 
     # HTTP tarball at a given URL. Add a BUILD file if requested.
     http_archive(

api/bazel/external_deps.bzl

@@ -0,0 +1,116 @@
+load("@envoy_api//bazel:repository_locations_utils.bzl", "load_repository_locations_spec")
+
+# Envoy dependencies may be annotated with the following attributes:
+DEPENDENCY_ANNOTATIONS = [
+    # List of the categories describing how the dependency is being used. This attribute is used
+    # for automatic tracking of security posture of Envoy's dependencies.
+    # Possible values are documented in the USE_CATEGORIES list below.
+    # This attribute is mandatory for each dependecy.
+    "use_category",
+
+    # Attribute specifying CPE (Common Platform Enumeration, see https://nvd.nist.gov/products/cpe) ID
+    # of the dependency. The ID may be in v2.3 or v2.2 format, although v2.3 is prefferred. See
+    # https://nvd.nist.gov/products/cpe for CPE format. Use single wildcard '*' for version and vector elements
+    # i.e. 'cpe:2.3:a:nghttp2:nghttp2:*'. Use "N/A" for dependencies without CPE assigned.
+    # This attribute is optional for components with use categories listed in the
+    # USE_CATEGORIES_WITH_CPE_OPTIONAL
+    "cpe",
+]
+
+# NOTE: If a dependency use case is either dataplane or controlplane, the other uses are not needed
+# to be declared.
+USE_CATEGORIES = [
+    # This dependency is used in API protos.
+    "api",
+    # This dependency is used in build process.
+    "build",
+    # This dependency is used to process xDS requests.
+    "controlplane",
+    # This dependency is used in processing downstream or upstream requests (core).
+    "dataplane_core",
+    # This dependency is used in processing downstream or upstream requests (extensions).
+    "dataplane_ext",
+    # This dependecy is used for logging, metrics or tracing (core). It may process unstrusted input.
+    "observability_core",
+    # This dependecy is used for logging, metrics or tracing (extensions). It may process unstrusted input.
+    "observability_ext",
+    # This dependency does not handle untrusted data and is used for various utility purposes.
+    "other",
+    # This dependency is used only in tests.
+    "test_only",
+]
+
+# Components with these use categories are not required to specify the 'cpe'
+# and 'last_updated' annotation.
+USE_CATEGORIES_WITH_CPE_OPTIONAL = ["build", "other", "test_only", "api"]
+
+def _fail_missing_attribute(attr, key):
+    fail("The '%s' attribute must be defined for external dependecy " % attr + key)
+
+# Method for verifying content of the repository location specifications.
+#
+# We also remove repository metadata attributes so that further consumers, e.g.
+# http_archive, are not confused by them.
+def load_repository_locations(repository_locations_spec):
+    locations = {}
+    for key, location in load_repository_locations_spec(repository_locations_spec).items():
+        mutable_location = dict(location)
+        locations[key] = mutable_location
+
+        if "sha256" not in location or len(location["sha256"]) == 0:
+            _fail_missing_attribute("sha256", key)
+
+        if "project_name" not in location:
+            _fail_missing_attribute("project_name", key)
+        mutable_location.pop("project_name")
+
+        if "project_desc" not in location:
+            _fail_missing_attribute("project_desc", key)
+        mutable_location.pop("project_desc")
+
+        if "project_url" not in location:
+            _fail_missing_attribute("project_url", key)
+        project_url = mutable_location.pop("project_url")
+        if not project_url.startswith("https://") and not project_url.startswith("http://"):
+            fail("project_url must start with https:// or http://: " + project_url)
+
+        if "version" not in location:
+            _fail_missing_attribute("version", key)
+        mutable_location.pop("version")
+
+        if "use_category" not in location:
+            _fail_missing_attribute("use_category", key)
+        use_category = mutable_location.pop("use_category")
+
+        if "dataplane_ext" in use_category or "observability_ext" in use_category:
+            if "extensions" not in location:
+                _fail_missing_attribute("extensions", key)
+            mutable_location.pop("extensions")
+
+        if "last_updated" not in location:
+            _fail_missing_attribute("last_updated", key)
+        last_updated = mutable_location.pop("last_updated")
+
+        # Starlark doesn't have regexes.
+        if len(last_updated) != 10 or last_updated[4] != "-" or last_updated[7] != "-":
+            fail("last_updated must match YYYY-DD-MM: " + last_updated)
+
+        if "cpe" in location:
+            cpe = mutable_location.pop("cpe")
+
+            # Starlark doesn't have regexes.
+            cpe_components = len(cpe.split(":"))
+
+            # We allow cpe:2.3:a:foo:* and cpe:2.3.:a:foo:bar:* only.
+            cpe_components_valid = cpe_components in [5, 6]
+            cpe_matches = (cpe == "N/A" or (cpe.startswith("cpe:2.3:a:") and cpe.endswith(":*") and cpe_components_valid))
+            if not cpe_matches:
+                fail("CPE must match cpe:2.3:a:<facet>:<facet>:*: " + cpe)
+        elif not [category for category in USE_CATEGORIES_WITH_CPE_OPTIONAL if category in location["use_category"]]:
+            _fail_missing_attribute("cpe", key)
+
+        for category in location["use_category"]:
+            if category not in USE_CATEGORIES:
+                fail("Unknown use_category value '" + category + "' for dependecy " + key)
+
+    return locations

api/bazel/repositories.bzl

@@ -1,40 +1,43 @@
 load(":envoy_http_archive.bzl", "envoy_http_archive")
-load(":repository_locations.bzl", "REPOSITORY_LOCATIONS")
+load(":external_deps.bzl", "load_repository_locations")
+load(":repository_locations.bzl", "REPOSITORY_LOCATIONS_SPEC")
 
-def api_dependencies():
+REPOSITORY_LOCATIONS = load_repository_locations(REPOSITORY_LOCATIONS_SPEC)
+
+# Use this macro to reference any HTTP archive from bazel/repository_locations.bzl.
+def external_http_archive(name, **kwargs):
     envoy_http_archive(
-        "bazel_skylib",
+        name,
         locations = REPOSITORY_LOCATIONS,
+        **kwargs
     )
-    envoy_http_archive(
-        "com_envoyproxy_protoc_gen_validate",
-        locations = REPOSITORY_LOCATIONS,
+
+def api_dependencies():
+    external_http_archive(
+        name = "bazel_skylib",
     )
-    envoy_http_archive(
+    external_http_archive(
+        name = "com_envoyproxy_protoc_gen_validate",
+    )
+    external_http_archive(
         name = "com_google_googleapis",
-        locations = REPOSITORY_LOCATIONS,
     )
-    envoy_http_archive(
+    external_http_archive(
         name = "com_github_cncf_udpa",
-        locations = REPOSITORY_LOCATIONS,
     )
 
-    envoy_http_archive(
+    external_http_archive(
         name = "prometheus_metrics_model",
-        locations = REPOSITORY_LOCATIONS,
         build_file_content = PROMETHEUSMETRICS_BUILD_CONTENT,
     )
-    envoy_http_archive(
+    external_http_archive(
         name = "opencensus_proto",
-        locations = REPOSITORY_LOCATIONS,
     )
-    envoy_http_archive(
+    external_http_archive(
         name = "rules_proto",
-        locations = REPOSITORY_LOCATIONS,
     )
-    envoy_http_archive(
+    external_http_archive(
         name = "com_github_openzipkin_zipkinapi",
-        locations = REPOSITORY_LOCATIONS,
         build_file_content = ZIPKINAPI_BUILD_CONTENT,
     )
 

api/bazel/repository_locations.bzl

@@ -1,4 +1,5 @@
-DEPENDENCY_REPOSITORIES_SPEC = dict(
+# This should match the schema defined in external_deps.bzl.
+REPOSITORY_LOCATIONS_SPEC = dict(
     bazel_skylib = dict(
         project_name = "bazel-skylib",
         project_desc = "Common useful functions and rules for Bazel",
@@ -88,23 +89,3 @@ DEPENDENCY_REPOSITORIES_SPEC = dict(
         use_category = ["api"],
     ),
 )
-
-def _format_version(s, version):
-    return s.format(version = version, dash_version = version.replace(".", "-"), underscore_version = version.replace(".", "_"))
-
-# Interpolate {version} in the above dependency specs. This code should be capable of running in both Python
-# and Starlark.
-def _dependency_repositories():
-    locations = {}
-    for key, location in DEPENDENCY_REPOSITORIES_SPEC.items():
-        mutable_location = dict(location)
-        locations[key] = mutable_location
-
-        # Fixup with version information.
-        if "version" in location:
-            if "strip_prefix" in location:
-                mutable_location["strip_prefix"] = _format_version(location["strip_prefix"], location["version"])
-            mutable_location["urls"] = [_format_version(url, location["version"]) for url in location["urls"]]
-    return locations
-
-REPOSITORY_LOCATIONS = _dependency_repositories()

api/bazel/repository_locations_utils.bzl

@@ -0,0 +1,20 @@
+def _format_version(s, version):
+    return s.format(version = version, dash_version = version.replace(".", "-"), underscore_version = version.replace(".", "_"))
+
+# Generate a "repository location specification" from raw repository
+# specification. The information should match the format required by
+# external_deps.bzl. This function mostly does interpolation of {version} in
+# the repository info fields. This code should be capable of running in both
+# Python and Starlark.
+def load_repository_locations_spec(repository_locations_spec):
+    locations = {}
+    for key, location in repository_locations_spec.items():
+        mutable_location = dict(location)
+        locations[key] = mutable_location
+
+        # Fixup with version information.
+        if "version" in location:
+            if "strip_prefix" in location:
+                mutable_location["strip_prefix"] = _format_version(location["strip_prefix"], location["version"])
+            mutable_location["urls"] = [_format_version(url, location["version"]) for url in location["urls"]]
+    return locations