From 73abe76983314c8abb5a486b4b9e71c600ca4a42 Mon Sep 17 00:00:00 2001 From: Harvey Tuch Date: Wed, 14 Oct 2020 17:59:07 -0400 Subject: [PATCH 1/3] [WiP] dependencies: compute release dates via GH API. The use of last_updated was ambiguous (is it when an Envoy contributor creates a PR, merges a commit, or when the dependency is released?). We really are after the release date as a measure of how stale the dependency is. This patch introduces a tool, tools/dependency/release_dates.py, that uses the GitHub API to compute release date. Co-authored-by: Michael Payne Signed-off-by: Harvey Tuch --- DEPENDENCY_POLICY.md | 7 +- api/bazel/external_deps.bzl | 13 +- api/bazel/repository_locations.bzl | 16 +-- bazel/repositories_extra.bzl | 18 +-- bazel/repository_locations.bzl | 132 +++++++++--------- docs/generate_external_dep_rst.py | 10 +- generated_api_shadow/bazel/external_deps.bzl | 13 +- .../bazel/repository_locations.bzl | 16 +-- tools/dependency/release_dates.py | 121 ++++++++++++++++ tools/dependency/release_dates.sh | 7 + tools/dependency/requirements.txt | 26 ++++ 11 files changed, 267 insertions(+), 112 deletions(-) create mode 100644 tools/dependency/release_dates.py create mode 100755 tools/dependency/release_dates.sh create mode 100644 tools/dependency/requirements.txt diff --git a/DEPENDENCY_POLICY.md b/DEPENDENCY_POLICY.md index 0944ad59030b..777d7e64adf4 100644 --- a/DEPENDENCY_POLICY.md +++ b/DEPENDENCY_POLICY.md @@ -45,8 +45,11 @@ Dependency declarations must: version is being used. * Provide accurate entries for `use_category`. Please think carefully about whether there are data or control plane implications of the dependency. -* Reflect the date (YYYY-MM-DD) at which they were last updated in the `last_updated` field. This - date is preferably the date at which the PR is created. +* Reflect the UTC date (YYYY-MM-DD format) for the dependency release. This is when + the dependency was updated in its repository. For dependencies that have + releases, this is the date of the release. For dependencies without releases + or for scenarios where we temporarily need to use a commit, this date should + be the date of the commit in UTC. * CPEs are compulsory for all dependencies that are not purely build/test. [CPEs](https://en.wikipedia.org/wiki/Common_Platform_Enumeration) provide metadata that allow us to correlate with related CVEs in dashboards and other tooling, and also provide a machine diff --git a/api/bazel/external_deps.bzl b/api/bazel/external_deps.bzl index cd9b6759f98a..3d35f4587426 100644 --- a/api/bazel/external_deps.bzl +++ b/api/bazel/external_deps.bzl @@ -40,8 +40,7 @@ USE_CATEGORIES = [ "test_only", ] -# Components with these use categories are not required to specify the 'cpe' -# and 'last_updated' annotation. +# Components with these use categories are not required to specify the 'cpe'. USE_CATEGORIES_WITH_CPE_OPTIONAL = ["build", "other", "test_only", "api"] def _fail_missing_attribute(attr, key): @@ -87,13 +86,13 @@ def load_repository_locations(repository_locations_spec): _fail_missing_attribute("extensions", key) mutable_location.pop("extensions") - if "last_updated" not in location: - _fail_missing_attribute("last_updated", key) - last_updated = mutable_location.pop("last_updated") + if "release_date" not in location: + _fail_missing_attribute("release_date", key) + release_date = mutable_location.pop("release_date") # Starlark doesn't have regexes. - if len(last_updated) != 10 or last_updated[4] != "-" or last_updated[7] != "-": - fail("last_updated must match YYYY-DD-MM: " + last_updated) + if len(release_date) != 10 or release_date[4] != "-" or release_date[7] != "-": + fail("release_date must match YYYY-DD-MM: " + release_date) if "cpe" in location: cpe = mutable_location.pop("cpe") diff --git a/api/bazel/repository_locations.bzl b/api/bazel/repository_locations.bzl index bdcf31e867d2..e46f7d77f8e5 100644 --- a/api/bazel/repository_locations.bzl +++ b/api/bazel/repository_locations.bzl @@ -7,7 +7,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( version = "1.0.3", sha256 = "1c531376ac7e5a180e0237938a2536de0c54d93f5c278634818e0efc952dd56c", urls = ["https://github.com/bazelbuild/bazel-skylib/releases/download/{version}/bazel-skylib-{version}.tar.gz"], - last_updated = "2020-08-27", + release_date = "2020-08-27", use_category = ["api"], ), com_envoyproxy_protoc_gen_validate = dict( @@ -18,7 +18,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "e368733c9fb7f8489591ffaf269170d7658cc0cd1ee322b601512b769446d3c8", strip_prefix = "protoc-gen-validate-{version}", urls = ["https://github.com/envoyproxy/protoc-gen-validate/archive/{version}.tar.gz"], - last_updated = "2020-06-09", + release_date = "2020-06-08", use_category = ["api"], ), com_github_cncf_udpa = dict( @@ -29,7 +29,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "83a7dcc316d741031f34c0409021432b74a39c4811845a177133f02f948fe2d8", strip_prefix = "udpa-{version}", urls = ["https://github.com/cncf/udpa/archive/v{version}.tar.gz"], - last_updated = "2020-09-23", + release_date = "2020-06-29", use_category = ["api"], ), com_github_openzipkin_zipkinapi = dict( @@ -40,7 +40,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "688c4fe170821dd589f36ec45aaadc03a618a40283bc1f97da8fa11686fc816b", strip_prefix = "zipkin-api-{version}", urls = ["https://github.com/openzipkin/zipkin-api/archive/{version}.tar.gz"], - last_updated = "2020-09-23", + release_date = "2019-08-23", use_category = ["api"], ), com_google_googleapis = dict( @@ -52,7 +52,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "a45019af4d3290f02eaeb1ce10990166978c807cb33a9692141a076ba46d1405", strip_prefix = "googleapis-{version}", urls = ["https://github.com/googleapis/googleapis/archive/{version}.tar.gz"], - last_updated = "2019-12-02", + release_date = "2019-12-02", use_category = ["api"], ), opencensus_proto = dict( @@ -63,7 +63,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "b7e13f0b4259e80c3070b583c2f39e53153085a6918718b1c710caf7037572b0", strip_prefix = "opencensus-proto-{version}/src", urls = ["https://github.com/census-instrumentation/opencensus-proto/archive/v{version}.tar.gz"], - last_updated = "2020-06-20", + release_date = "2020-07-21", use_category = ["api"], ), prometheus_metrics_model = dict( @@ -74,7 +74,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "6748b42f6879ad4d045c71019d2512c94be3dd86f60965e9e31e44a3f464323e", strip_prefix = "client_model-{version}", urls = ["https://github.com/prometheus/client_model/archive/{version}.tar.gz"], - last_updated = "2020-06-23", + release_date = "2020-06-23", use_category = ["api"], ), rules_proto = dict( @@ -85,7 +85,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "aa1ee19226f707d44bee44c720915199c20c84a23318bb0597ed4e5c873ccbd5", strip_prefix = "rules_proto-{version}", urls = ["https://github.com/bazelbuild/rules_proto/archive/{version}.tar.gz"], - last_updated = "2020-08-17", + release_date = "2020-08-17", use_category = ["api"], ), ) diff --git a/bazel/repositories_extra.bzl b/bazel/repositories_extra.bzl index 3c76127ea3eb..3c68e7d415c3 100644 --- a/bazel/repositories_extra.bzl +++ b/bazel/repositories_extra.bzl @@ -14,7 +14,7 @@ def _python_deps(): # project_name = "PyYAML", # project_url = "https://github.com/yaml/pyyaml", # version = "5.3.1", - # last_update = "2020-03-18" + # release_date = "2020-03-18" # use_category = ["other"], # cpe = "cpe:2.3:a:pyyaml:pyyaml:*", ) @@ -26,14 +26,14 @@ def _python_deps(): # project_name = "Jinja", # project_url = "http://palletsprojects.com/p/jinja", # version = "2.11.2", - # last_update = "2020-04-13" + # release_date = "2020-04-13" # use_category = ["test"], # cpe = "cpe:2.3:a:palletsprojects:jinja:*", # project_name = "MarkupSafe", # project_url = "https://markupsafe.palletsprojects.com/en/1.1.x/", # version = "1.1.1", - # last_update = "2019-02-23" + # release_date = "2019-02-23" # use_category = ["test"], ) pip3_import( @@ -44,14 +44,14 @@ def _python_deps(): # project_name = "Jinja", # project_url = "http://palletsprojects.com/p/jinja", # version = "2.11.2", - # last_update = "2020-04-13" + # release_date = "2020-04-13" # use_category = ["test"], # cpe = "cpe:2.3:a:palletsprojects:jinja:*", # project_name = "MarkupSafe", # project_url = "https://markupsafe.palletsprojects.com/en/1.1.x/", # version = "1.1.1", - # last_update = "2019-02-23" + # release_date = "2019-02-23" # use_category = ["test"], ) pip3_import( @@ -62,7 +62,7 @@ def _python_deps(): # project_name = "Clang", # project_url = "https://clang.llvm.org/", # version = "10.0.1", - # last_update = "2020-07-21" + # release_date = "2020-07-21" # use_category = ["other"], # cpe = "cpe:2.3:a:llvm:clang:*", ) @@ -74,7 +74,7 @@ def _python_deps(): # project_name = "PyYAML", # project_url = "https://github.com/yaml/pyyaml", # version = "5.3.1", - # last_update = "2020-03-18" + # release_date = "2020-03-18" # use_category = ["other"], # cpe = "cpe:2.3:a:pyyaml:pyyaml:*", ) @@ -86,14 +86,14 @@ def _python_deps(): # project_name = "Apache Thrift", # project_url = "http://thrift.apache.org/", # version = "0.11.0", - # last_update = "2017-12-07" + # release_date = "2017-12-07" # use_category = ["dataplane"], # cpe = "cpe:2.3:a:apache:thrift:*", # project_name = "Six: Python 2 and 3 Compatibility Library", # project_url = "https://six.readthedocs.io/", # version = "1.15.0", - # last_update = "2020-05-21" + # release_date = "2020-05-21" # use_category = ["dataplane"], ) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 3e44753ab382..e0e5927d1829 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -8,7 +8,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "bcecfd622c4ef272fd4ba42726a52e140b961c4eac23025f18b346c968a8cfb4", strip_prefix = "bazel-compilation-database-{version}", urls = ["https://github.com/grailbio/bazel-compilation-database/archive/{version}.tar.gz"], - last_updated = "2020-08-01", + release_date = "2020-08-01", use_category = ["build"], ), bazel_gazelle = dict( @@ -18,7 +18,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( version = "0.21.1", sha256 = "cdb02a887a7187ea4d5a27452311a75ed8637379a1287d8eeb952138ea485f7d", urls = ["https://github.com/bazelbuild/bazel-gazelle/releases/download/v{version}/bazel-gazelle-v{version}.tar.gz"], - last_updated = "2020-05-28", + release_date = "2020-05-28", use_category = ["build"], ), bazel_toolchains = dict( @@ -32,7 +32,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( "https://github.com/bazelbuild/bazel-toolchains/releases/download/{version}/bazel-toolchains-{version}.tar.gz", "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/{version}.tar.gz", ], - last_updated = "2020-08-10", + release_date = "2020-08-10", use_category = ["build"], ), build_bazel_rules_apple = dict( @@ -42,7 +42,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( version = "0.19.0", sha256 = "7a7afdd4869bb201c9352eed2daf37294d42b093579b70423490c1b4d4f6ce42", urls = ["https://github.com/bazelbuild/rules_apple/releases/download/{version}/rules_apple.{version}.tar.gz"], - last_updated = "2020-10-10", + release_date = "2019-10-10", use_category = ["build"], ), envoy_build_tools = dict( @@ -53,7 +53,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "dc3881d16e7b0c855a7279f5757d55e4aa55fe2befbd9e34215b971818622f9e", strip_prefix = "envoy-build-tools-{version}", urls = ["https://github.com/envoyproxy/envoy-build-tools/archive/{version}.tar.gz"], - last_updated = "2020-10-01", + release_date = "2020-10-01", use_category = ["build"], ), boringssl = dict( @@ -71,7 +71,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( # chromium-85.0.4183.83 urls = ["https://github.com/google/boringssl/archive/{version}.tar.gz"], use_category = ["controlplane", "dataplane_core"], - last_updated = "2020-06-23", + release_date = "2020-06-23", cpe = "cpe:2.3:a:google:boringssl:*", ), boringssl_fips = dict( @@ -82,7 +82,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "3b5fdf23274d4179c2077b5e8fa625d9debd7a390aac1d165b7e47234f648bb8", urls = ["https://commondatastorage.googleapis.com/chromium-boringssl-fips/boringssl-ae223d6138807a13006342edfeef32e813246b39.tar.xz"], use_category = ["controlplane", "dataplane_core"], - last_updated = "2019-08-08", + release_date = "2019-08-08", cpe = "cpe:2.3:a:google:boringssl:*", ), com_google_absl = dict( @@ -94,7 +94,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( strip_prefix = "abseil-cpp-{version}", urls = ["https://github.com/abseil/abseil-cpp/archive/{version}.tar.gz"], use_category = ["dataplane_core", "controlplane"], - last_updated = "2020-10-01", + release_date = "2020-10-01", cpe = "N/A", ), com_github_c_ares_c_ares = dict( @@ -106,7 +106,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( strip_prefix = "c-ares-{version}", urls = ["https://github.com/c-ares/c-ares/releases/download/cares-{underscore_version}/c-ares-{version}.tar.gz"], use_category = ["dataplane_core", "controlplane"], - last_updated = "2020-05-11", + release_date = "2020-05-11", cpe = "cpe:2.3:a:c-ares_project:c-ares:*", ), com_github_circonus_labs_libcircllhist = dict( @@ -118,7 +118,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( strip_prefix = "libcircllhist-{version}", urls = ["https://github.com/circonus-labs/libcircllhist/archive/{version}.tar.gz"], use_category = ["controlplane", "observability_core", "dataplane_core"], - last_updated = "2019-02-11", + release_date = "2019-02-11", cpe = "N/A", ), com_github_cyan4973_xxhash = dict( @@ -130,7 +130,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( strip_prefix = "xxHash-{version}", urls = ["https://github.com/Cyan4973/xxHash/archive/v{version}.tar.gz"], use_category = ["dataplane_core", "controlplane"], - last_updated = "2020-03-04", + release_date = "2020-03-05", cpe = "N/A", ), com_github_envoyproxy_sqlparser = dict( @@ -146,7 +146,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( "envoy.filters.network.mysql_proxy", "envoy.filters.network.postgres_proxy", ], - last_updated = "2020-06-10", + release_date = "2020-06-10", cpe = "N/A", ), com_github_mirror_tclap = dict( @@ -157,7 +157,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "f0ede0721dddbb5eba3a47385a6e8681b14f155e1129dd39d1a959411935098f", strip_prefix = "tclap-tclap-{version}-release-final", urls = ["https://github.com/mirror/tclap/archive/tclap-{version}-release-final.tar.gz"], - last_updated = "2017-11-10", + release_date = "2011-04-16", use_category = ["other"], ), com_github_fmtlib_fmt = dict( @@ -169,7 +169,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( strip_prefix = "fmt-{version}", urls = ["https://github.com/fmtlib/fmt/releases/download/{version}/fmt-{version}.zip"], use_category = ["dataplane_core", "controlplane"], - last_updated = "2020-08-07", + release_date = "2020-08-06", cpe = "cpe:2.3:a:fmt:fmt:*", ), com_github_gabime_spdlog = dict( @@ -181,7 +181,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( strip_prefix = "spdlog-{version}", urls = ["https://github.com/gabime/spdlog/archive/v{version}.tar.gz"], use_category = ["dataplane_core", "controlplane"], - last_updated = "2020-07-09", + release_date = "2020-07-09", cpe = "N/A", ), com_github_google_libprotobuf_mutator = dict( @@ -192,7 +192,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "49a26dbe77c75f2eca1dd8a9fbdb31c4496d9af42df027ff57569c5a7a5d980d", strip_prefix = "libprotobuf-mutator-{version}", urls = ["https://github.com/google/libprotobuf-mutator/archive/{version}.tar.gz"], - last_updated = "2020-08-18", + release_date = "2020-08-18", use_category = ["test_only"], ), com_github_google_tcmalloc = dict( @@ -204,7 +204,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( strip_prefix = "tcmalloc-{version}", urls = ["https://github.com/google/tcmalloc/archive/{version}.tar.gz"], use_category = ["dataplane_core", "controlplane"], - last_updated = "2020-09-16", + release_date = "2020-09-16", cpe = "N/A", ), com_github_gperftools_gperftools = dict( @@ -215,7 +215,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "240deacdd628b6459671b83eb0c4db8e97baadf659f25b92e9a078d536bd513e", strip_prefix = "gperftools-{version}", urls = ["https://github.com/gperftools/gperftools/releases/download/gperftools-{version}/gperftools-{version}.tar.gz"], - last_updated = "2020-07-06", + release_date = "2020-07-06", use_category = ["dataplane_core", "controlplane"], cpe = "cpe:2.3:a:gperftools_project:gperftools:*", ), @@ -230,7 +230,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( strip_prefix = "grpc-{version}", urls = ["https://github.com/grpc/grpc/archive/{version}.tar.gz"], use_category = ["dataplane_core", "controlplane"], - last_updated = "2020-02-11", + release_date = "2020-02-11", cpe = "cpe:2.3:a:grpc:grpc:*", ), com_github_luajit_luajit = dict( @@ -243,7 +243,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( urls = ["https://github.com/LuaJIT/LuaJIT/archive/v{version}.tar.gz"], use_category = ["dataplane_ext"], extensions = ["envoy.filters.http.lua"], - last_updated = "2017-11-07", + release_date = "2017-05-01", cpe = "cpe:2.3:a:luajit:luajit:*", ), com_github_moonjit_moonjit = dict( @@ -256,7 +256,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( urls = ["https://github.com/moonjit/moonjit/archive/{version}.tar.gz"], use_category = ["dataplane_ext"], extensions = ["envoy.filters.http.lua"], - last_updated = "2020-01-14", + release_date = "2020-01-14", cpe = "cpe:2.3:a:moonjit_project:moonjit:*", ), com_github_nghttp2_nghttp2 = dict( @@ -268,7 +268,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( strip_prefix = "nghttp2-{version}", urls = ["https://github.com/nghttp2/nghttp2/releases/download/v{version}/nghttp2-{version}.tar.gz"], use_category = ["controlplane", "dataplane_core"], - last_updated = "2020-06-02", + release_date = "2020-06-02", cpe = "cpe:2.3:a:nghttp2:nghttp2:*", ), io_opentracing_cpp = dict( @@ -285,7 +285,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( "envoy.tracers.dynamic_ot", "envoy.tracers.lightstep", ], - last_updated = "2019-01-16", + release_date = "2019-01-16", cpe = "N/A", ), com_lightstep_tracer_cpp = dict( @@ -298,7 +298,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( urls = ["https://github.com/lightstep/lightstep-tracer-cpp/archive/{version}.tar.gz"], use_category = ["observability_ext"], extensions = ["envoy.tracers.lightstep"], - last_updated = "2020-08-24", + release_date = "2020-08-25", cpe = "N/A", ), com_github_datadog_dd_opentracing_cpp = dict( @@ -311,7 +311,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( urls = ["https://github.com/DataDog/dd-opentracing-cpp/archive/v{version}.tar.gz"], use_category = ["observability_ext"], extensions = ["envoy.tracers.datadog"], - last_updated = "2020-05-15", + release_date = "2020-05-15", cpe = "N/A", ), com_github_google_benchmark = dict( @@ -323,7 +323,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( strip_prefix = "benchmark-{version}", urls = ["https://github.com/google/benchmark/archive/v{version}.tar.gz"], use_category = ["test_only"], - last_updated = "2020-06-09", + release_date = "2020-06-09", ), com_github_libevent_libevent = dict( project_name = "libevent", @@ -344,7 +344,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( strip_prefix = "libevent-{version}", urls = ["https://github.com/libevent/libevent/archive/{version}.tar.gz"], use_category = ["dataplane_core", "controlplane"], - last_updated = "2020-07-31", + release_date = "2020-07-28", cpe = "cpe:2.3:a:libevent_project:libevent:*", ), # This should be removed, see https://github.com/envoyproxy/envoy/issues/13261. @@ -360,7 +360,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( strip_prefix = "zlib-{version}", urls = ["https://github.com/madler/zlib/archive/{version}.tar.gz"], use_category = ["controlplane", "dataplane_core"], - last_updated = "2019-04-14", + release_date = "2019-04-14", cpe = "cpe:2.3:a:gnu:zlib:*", ), com_github_zlib_ng_zlib_ng = dict( @@ -372,7 +372,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( strip_prefix = "zlib-ng-{version}", urls = ["https://github.com/zlib-ng/zlib-ng/archive/{version}.tar.gz"], use_category = ["controlplane", "dataplane_core"], - last_updated = "2020-08-16", + release_date = "2020-08-16", cpe = "N/A", ), com_github_jbeder_yaml_cpp = dict( @@ -386,7 +386,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( # YAML is also used for runtime as well as controlplane. It shouldn't appear on the # dataplane but we can't verify this automatically due to code structure today. use_category = ["controlplane", "dataplane_core"], - last_updated = "2020-07-28", + release_date = "2020-07-27", cpe = "cpe:2.3:a:yaml-cpp_project:yaml-cpp:*", ), com_github_msgpack_msgpack_c = dict( @@ -399,7 +399,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( urls = ["https://github.com/msgpack/msgpack-c/releases/download/cpp-{version}/msgpack-{version}.tar.gz"], use_category = ["observability_ext"], extensions = ["envoy.tracers.datadog"], - last_updated = "2020-06-05", + release_date = "2020-06-05", cpe = "N/A", ), com_github_google_jwt_verify = dict( @@ -412,7 +412,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( urls = ["https://github.com/google/jwt_verify_lib/archive/{version}.tar.gz"], use_category = ["dataplane_ext"], extensions = ["envoy.filters.http.jwt_authn"], - last_updated = "2020-07-09", + release_date = "2020-07-10", cpe = "N/A", ), com_github_nodejs_http_parser = dict( @@ -426,7 +426,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( strip_prefix = "http-parser-{version}", urls = ["https://github.com/nodejs/http-parser/archive/{version}.tar.gz"], use_category = ["controlplane", "dataplane_core"], - last_updated = "2020-07-10", + release_date = "2020-07-10", cpe = "cpe:2.3:a:nodejs:node.js:*", ), com_github_tencent_rapidjson = dict( @@ -440,7 +440,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( # We're mostly using com_google_protobuf for JSON, but there are some extensions and hard to # disentangle uses on the dataplane, e.g. header_formatter, Squash filter. use_category = ["controlplane", "dataplane_core"], - last_updated = "2019-12-02", + release_date = "2019-12-03", cpe = "cpe:2.3:a:tencent:rapidjson:*", ), com_github_twitter_common_lang = dict( @@ -451,7 +451,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "56d1d266fd4767941d11c27061a57bc1266a3342e551bde3780f9e9eb5ad0ed1", strip_prefix = "twitter.common.lang-{version}/src", urls = ["https://files.pythonhosted.org/packages/08/bc/d6409a813a9dccd4920a6262eb6e5889e90381453a5f58938ba4cf1d9420/twitter.common.lang-{version}.tar.gz"], - last_updated = "2018-06-26", + release_date = "2016-10-17", use_category = ["test_only"], ), com_github_twitter_common_rpc = dict( @@ -462,7 +462,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "0792b63fb2fb32d970c2e9a409d3d00633190a22eb185145fe3d9067fdaa4514", strip_prefix = "twitter.common.rpc-{version}/src", urls = ["https://files.pythonhosted.org/packages/be/97/f5f701b703d0f25fbf148992cd58d55b4d08d3db785aad209255ee67e2d0/twitter.common.rpc-{version}.tar.gz"], - last_updated = "2018-06-26", + release_date = "2016-10-17", use_category = ["test_only"], ), com_github_twitter_common_finagle_thrift = dict( @@ -473,7 +473,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "1e3a57d11f94f58745e6b83348ecd4fa74194618704f45444a15bc391fde497a", strip_prefix = "twitter.common.finagle-thrift-{version}/src", urls = ["https://files.pythonhosted.org/packages/f9/e7/4f80d582578f8489226370762d2cf6bc9381175d1929eba1754e03f70708/twitter.common.finagle-thrift-{version}.tar.gz"], - last_updated = "2018-06-26", + release_date = "2016-10-17", use_category = ["test_only"], ), com_google_googletest = dict( @@ -486,7 +486,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "7897bfaa5ad39a479177cfb5c3ce010184dbaee22a7c3727b212282871918751", strip_prefix = "googletest-{version}", urls = ["https://github.com/google/googletest/archive/{version}.tar.gz"], - last_updated = "2020-09-10", + release_date = "2020-09-10", use_category = ["test_only"], ), com_google_protobuf = dict( @@ -498,7 +498,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( strip_prefix = "protobuf-{version}", urls = ["https://github.com/protocolbuffers/protobuf/releases/download/v{version}/protobuf-all-{version}.tar.gz"], use_category = ["dataplane_core", "controlplane"], - last_updated = "2020-10-24", + release_date = "2019-10-24", cpe = "cpe:2.3:a:google:protobuf:*", ), grpc_httpjson_transcoding = dict( @@ -511,7 +511,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( urls = ["https://github.com/grpc-ecosystem/grpc-httpjson-transcoding/archive/{version}.tar.gz"], use_category = ["dataplane_ext"], extensions = ["envoy.filters.http.grpc_json_transcoder"], - last_updated = "2020-03-02", + release_date = "2020-03-02", cpe = "N/A", ), io_bazel_rules_go = dict( @@ -522,7 +522,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "0310e837aed522875791750de44408ec91046c630374990edd51827cb169f616", urls = ["https://github.com/bazelbuild/rules_go/releases/download/v{version}/rules_go-v{version}.tar.gz"], use_category = ["build"], - last_updated = "2020-08-06", + release_date = "2020-08-06", ), rules_cc = dict( project_name = "C++ rules for Bazel", @@ -533,7 +533,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "9d48151ea71b3e225adfb6867e6d2c7d0dce46cbdc8710d9a9a628574dfd40a0", strip_prefix = "rules_cc-{version}", urls = ["https://github.com/bazelbuild/rules_cc/archive/{version}.tar.gz"], - last_updated = "2020-05-13", + release_date = "2020-05-13", use_category = ["build"], ), rules_foreign_cc = dict( @@ -544,7 +544,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "2b1cf88de0b6e0195f6571cfde3a5bd406d11b42117d6adef2395c9525a1902e", strip_prefix = "rules_foreign_cc-{version}", urls = ["https://github.com/bazelbuild/rules_foreign_cc/archive/{version}.tar.gz"], - last_updated = "2020-08-21", + release_date = "2020-08-21", use_category = ["build"], ), rules_python = dict( @@ -556,7 +556,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "76a8fd4e7eca2a3590f816958faa0d83c9b2ce9c32634c5c375bcccf161d3bb5", strip_prefix = "rules_python-{version}", urls = ["https://github.com/bazelbuild/rules_python/archive/{version}.tar.gz"], - last_updated = "2020-04-09", + release_date = "2020-04-09", use_category = ["build"], ), six = dict( @@ -566,7 +566,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( version = "1.12.0", sha256 = "d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73", urls = ["https://files.pythonhosted.org/packages/dd/bf/4138e7bfb757de47d1f4b6994648ec67a51efe58fa907c1e11e350cddfca/six-{version}.tar.gz"], - last_updated = "2019-11-17", + release_date = "2018-12-09", use_category = ["other"], ), org_llvm_llvm = dict( @@ -577,7 +577,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "df83a44b3a9a71029049ec101fb0077ecbbdf5fe41e395215025779099a98fdf", strip_prefix = "llvm-{version}.src", urls = ["https://github.com/llvm/llvm-project/releases/download/llvmorg-{version}/llvm-{version}.src.tar.xz"], - last_updated = "2020-10-09", + release_date = "2020-03-23", use_category = ["dataplane_ext"], extensions = [ "envoy.access_loggers.wasm", @@ -596,7 +596,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "cc3fcaf05d57010c9cf8eb920234679dede6c780137b55001fd34e4d14806f7c", strip_prefix = "WAVM-{version}", urls = ["https://github.com/WAVM/WAVM/archive/{version}.tar.gz"], - last_updated = "2020-10-09", + release_date = "2020-07-07", use_category = ["dataplane_ext"], extensions = [ "envoy.access_loggers.wasm", @@ -617,7 +617,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( urls = ["https://github.com/census-instrumentation/opencensus-cpp/archive/{version}.tar.gz"], use_category = ["observability_ext"], extensions = ["envoy.tracers.opencensus"], - last_updated = "2020-10-13", + release_date = "2020-10-08", cpe = "N/A", ), # This should be removed, see https://github.com/envoyproxy/envoy/issues/11816. @@ -636,7 +636,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( "envoy.grpc_credentials.aws_iam", "envoy.tracers.opencensus", ], - last_updated = "2020-08-19", + release_date = "2020-08-19", cpe = "cpe:2.3:a:haxx:curl:*", ), com_googlesource_chromium_v8 = dict( @@ -656,7 +656,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( "envoy.filters.network.wasm", "envoy.stat_sinks.wasm", ], - last_updated = "2020-08-31", + release_date = "2020-08-31", cpe = "cpe:2.3:a:google:v8:*", ), com_googlesource_quiche = dict( @@ -669,7 +669,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( urls = ["https://storage.googleapis.com/quiche-envoy-integration/{version}.tar.gz"], use_category = ["dataplane_ext"], extensions = ["envoy.transport_sockets.quic"], - last_updated = "2020-09-18", + release_date = "2020-09-18", cpe = "N/A", ), com_googlesource_googleurl = dict( @@ -682,7 +682,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( urls = ["https://storage.googleapis.com/quiche-envoy-integration/googleurl_{version}.tar.gz"], use_category = ["dataplane_ext"], extensions = [], - last_updated = "2020-08-05", + release_date = "2020-08-05", cpe = "N/A", ), com_google_cel_cpp = dict( @@ -703,7 +703,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( "envoy.filters.network.wasm", "envoy.stat_sinks.wasm", ], - last_updated = "2020-07-14", + release_date = "2020-07-14", cpe = "N/A", ), com_github_google_flatbuffers = dict( @@ -722,7 +722,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( "envoy.filters.network.wasm", "envoy.stat_sinks.wasm", ], - last_updated = "2020-07-29", + release_date = "2020-04-02", cpe = "N/A", ), com_googlesource_code_re2 = dict( @@ -734,7 +734,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( strip_prefix = "re2-{version}", urls = ["https://github.com/google/re2/archive/{version}.tar.gz"], use_category = ["controlplane", "dataplane_core"], - last_updated = "2020-07-06", + release_date = "2020-07-06", cpe = "N/A", ), # Included to access FuzzedDataProvider.h. This is compiler agnostic but @@ -749,7 +749,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( # Only allow peeking at fuzzer related files for now. strip_prefix = "compiler-rt-{version}.src", urls = ["https://github.com/llvm/llvm-project/releases/download/llvmorg-{version}/compiler-rt-{version}.src.tar.xz"], - last_updated = "2020-03-24", + release_date = "2020-03-23", use_category = ["test_only"], ), upb = dict( @@ -761,7 +761,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( strip_prefix = "upb-{version}", urls = ["https://github.com/protocolbuffers/upb/archive/{version}.tar.gz"], use_category = ["controlplane"], - last_updated = "2019-11-19", + release_date = "2019-11-19", cpe = "N/A", ), kafka_source = dict( @@ -774,7 +774,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( urls = ["https://github.com/apache/kafka/archive/{version}.zip"], use_category = ["dataplane_ext"], extensions = ["envoy.filters.network.kafka_broker"], - last_updated = "2020-08-26", + release_date = "2020-03-03", cpe = "cpe:2.3:a:apache:kafka:*", ), kafka_server_binary = dict( @@ -785,7 +785,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "2177cbd14118999e1d76fec628ca78ace7e6f841219dbc6035027c796bbe1a2a", strip_prefix = "kafka_2.12-{version}", urls = ["https://mirrors.gigenet.com/apache/kafka/{version}/kafka_2.12-{version}.tgz"], - last_updated = "2020-08-26", + release_date = "2020-03-12", use_category = ["test_only"], ), kafka_python_client = dict( @@ -796,7 +796,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "05f7c6eecb402f11fcb7e524c903f1ba1c38d3bdc9bf42bc8ec3cf7567b9f979", strip_prefix = "kafka-python-{version}", urls = ["https://github.com/dpkp/kafka-python/archive/{version}.tar.gz"], - last_updated = "2020-08-26", + release_date = "2020-02-20", use_category = ["test_only"], ), proxy_wasm_cpp_sdk = dict( @@ -815,7 +815,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( "envoy.filters.network.wasm", "envoy.stat_sinks.wasm", ], - last_updated = "2020-10-09", + release_date = "2020-09-11", cpe = "N/A", ), proxy_wasm_cpp_host = dict( @@ -834,7 +834,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( "envoy.filters.network.wasm", "envoy.stat_sinks.wasm", ], - last_updated = "2020-10-09", + release_date = "2020-09-11", cpe = "N/A", ), # TODO: upgrade to the latest version (1.41 currently fails tests) @@ -847,7 +847,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( strip_prefix = "emsdk-{version}", urls = ["https://github.com/emscripten-core/emsdk/archive/{version}.tar.gz"], use_category = ["build"], - last_updated = "2020-10-09", + release_date = "2020-01-15", ), io_bazel_rules_rust = dict( project_name = "Bazel rust rules", @@ -860,7 +860,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( strip_prefix = "rules_rust-{version}", urls = ["https://github.com/bazelbuild/rules_rust/archive/{version}.tar.gz"], use_category = ["build"], - last_updated = "2020-10-09", + release_date = "2020-06-30", ), rules_antlr = dict( project_name = "ANTLR Rules for Bazel", @@ -879,7 +879,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( "envoy.filters.network.wasm", "envoy.stat_sinks.wasm", ], - last_updated = "2020-07-29", + release_date = "2019-06-21", cpe = "N/A", ), antlr4_runtimes = dict( @@ -898,7 +898,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( "envoy.filters.network.wasm", "envoy.stat_sinks.wasm", ], - last_updated = "2020-07-29", + release_date = "2017-12-09", cpe = "N/A", ), ) diff --git a/docs/generate_external_dep_rst.py b/docs/generate_external_dep_rst.py index 3348df538837..eb3525b9ce03 100755 --- a/docs/generate_external_dep_rst.py +++ b/docs/generate_external_dep_rst.py @@ -107,7 +107,7 @@ def GetVersionUrl(metadata): if __name__ == '__main__': security_rst_root = sys.argv[1] - Dep = namedtuple('Dep', ['name', 'sort_name', 'version', 'cpe', 'last_updated']) + Dep = namedtuple('Dep', ['name', 'sort_name', 'version', 'cpe', 'release_date']) use_categories = defaultdict(lambda: defaultdict(list)) # Bin rendered dependencies into per-use category lists. spec_loader = repository_locations_utils.load_repository_locations_spec @@ -123,14 +123,14 @@ def GetVersionUrl(metadata): project_url = v['project_url'] name = RstLink(project_name, project_url) version = RstLink(RenderVersion(v['version']), GetVersionUrl(v)) - last_updated = v['last_updated'] - dep = Dep(name, project_name.lower(), version, cpe, last_updated) + release_date = v['release_date'] + dep = Dep(name, project_name.lower(), version, cpe, release_date) for category in v['use_category']: for ext in v.get('extensions', ['core']): use_categories[category][ext].append(dep) def CsvRow(dep): - return [dep.name, dep.version, dep.last_updated, dep.cpe] + return [dep.name, dep.version, dep.release_date, dep.cpe] # Generate per-use category RST with CSV tables. for category, exts in use_categories.items(): @@ -139,6 +139,6 @@ def CsvRow(dep): if ext_name != 'core': content += RenderTitle(ext_name) output_path = pathlib.Path(security_rst_root, f'external_dep_{category}.rst') - content += CsvTable(['Name', 'Version', 'Last updated', 'CPE'], [2, 1, 1, 2], + content += CsvTable(['Name', 'Version', 'Release date', 'CPE'], [2, 1, 1, 2], [CsvRow(dep) for dep in sorted(deps, key=lambda d: d.sort_name)]) output_path.write_text(content) diff --git a/generated_api_shadow/bazel/external_deps.bzl b/generated_api_shadow/bazel/external_deps.bzl index cd9b6759f98a..3d35f4587426 100644 --- a/generated_api_shadow/bazel/external_deps.bzl +++ b/generated_api_shadow/bazel/external_deps.bzl @@ -40,8 +40,7 @@ USE_CATEGORIES = [ "test_only", ] -# Components with these use categories are not required to specify the 'cpe' -# and 'last_updated' annotation. +# Components with these use categories are not required to specify the 'cpe'. USE_CATEGORIES_WITH_CPE_OPTIONAL = ["build", "other", "test_only", "api"] def _fail_missing_attribute(attr, key): @@ -87,13 +86,13 @@ def load_repository_locations(repository_locations_spec): _fail_missing_attribute("extensions", key) mutable_location.pop("extensions") - if "last_updated" not in location: - _fail_missing_attribute("last_updated", key) - last_updated = mutable_location.pop("last_updated") + if "release_date" not in location: + _fail_missing_attribute("release_date", key) + release_date = mutable_location.pop("release_date") # Starlark doesn't have regexes. - if len(last_updated) != 10 or last_updated[4] != "-" or last_updated[7] != "-": - fail("last_updated must match YYYY-DD-MM: " + last_updated) + if len(release_date) != 10 or release_date[4] != "-" or release_date[7] != "-": + fail("release_date must match YYYY-DD-MM: " + release_date) if "cpe" in location: cpe = mutable_location.pop("cpe") diff --git a/generated_api_shadow/bazel/repository_locations.bzl b/generated_api_shadow/bazel/repository_locations.bzl index bdcf31e867d2..e46f7d77f8e5 100644 --- a/generated_api_shadow/bazel/repository_locations.bzl +++ b/generated_api_shadow/bazel/repository_locations.bzl @@ -7,7 +7,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( version = "1.0.3", sha256 = "1c531376ac7e5a180e0237938a2536de0c54d93f5c278634818e0efc952dd56c", urls = ["https://github.com/bazelbuild/bazel-skylib/releases/download/{version}/bazel-skylib-{version}.tar.gz"], - last_updated = "2020-08-27", + release_date = "2020-08-27", use_category = ["api"], ), com_envoyproxy_protoc_gen_validate = dict( @@ -18,7 +18,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "e368733c9fb7f8489591ffaf269170d7658cc0cd1ee322b601512b769446d3c8", strip_prefix = "protoc-gen-validate-{version}", urls = ["https://github.com/envoyproxy/protoc-gen-validate/archive/{version}.tar.gz"], - last_updated = "2020-06-09", + release_date = "2020-06-08", use_category = ["api"], ), com_github_cncf_udpa = dict( @@ -29,7 +29,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "83a7dcc316d741031f34c0409021432b74a39c4811845a177133f02f948fe2d8", strip_prefix = "udpa-{version}", urls = ["https://github.com/cncf/udpa/archive/v{version}.tar.gz"], - last_updated = "2020-09-23", + release_date = "2020-06-29", use_category = ["api"], ), com_github_openzipkin_zipkinapi = dict( @@ -40,7 +40,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "688c4fe170821dd589f36ec45aaadc03a618a40283bc1f97da8fa11686fc816b", strip_prefix = "zipkin-api-{version}", urls = ["https://github.com/openzipkin/zipkin-api/archive/{version}.tar.gz"], - last_updated = "2020-09-23", + release_date = "2019-08-23", use_category = ["api"], ), com_google_googleapis = dict( @@ -52,7 +52,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "a45019af4d3290f02eaeb1ce10990166978c807cb33a9692141a076ba46d1405", strip_prefix = "googleapis-{version}", urls = ["https://github.com/googleapis/googleapis/archive/{version}.tar.gz"], - last_updated = "2019-12-02", + release_date = "2019-12-02", use_category = ["api"], ), opencensus_proto = dict( @@ -63,7 +63,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "b7e13f0b4259e80c3070b583c2f39e53153085a6918718b1c710caf7037572b0", strip_prefix = "opencensus-proto-{version}/src", urls = ["https://github.com/census-instrumentation/opencensus-proto/archive/v{version}.tar.gz"], - last_updated = "2020-06-20", + release_date = "2020-07-21", use_category = ["api"], ), prometheus_metrics_model = dict( @@ -74,7 +74,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "6748b42f6879ad4d045c71019d2512c94be3dd86f60965e9e31e44a3f464323e", strip_prefix = "client_model-{version}", urls = ["https://github.com/prometheus/client_model/archive/{version}.tar.gz"], - last_updated = "2020-06-23", + release_date = "2020-06-23", use_category = ["api"], ), rules_proto = dict( @@ -85,7 +85,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "aa1ee19226f707d44bee44c720915199c20c84a23318bb0597ed4e5c873ccbd5", strip_prefix = "rules_proto-{version}", urls = ["https://github.com/bazelbuild/rules_proto/archive/{version}.tar.gz"], - last_updated = "2020-08-17", + release_date = "2020-08-17", use_category = ["api"], ), ) diff --git a/tools/dependency/release_dates.py b/tools/dependency/release_dates.py new file mode 100644 index 000000000000..b2a50c7649e8 --- /dev/null +++ b/tools/dependency/release_dates.py @@ -0,0 +1,121 @@ +# CLI tool to query GitHub API and compare with repository_locations.bzl. It +# will: +# - Compute the release date of dependencies and warn if there is a mismatch +# with the metdata release date. +# - Look at the latest release tag and warn if this is later than the dependency +# version in the .bzl. +# +# Usage: +# tools/dependency/release_dates.sh +# +# You will need to set a GitHub access token in the GH_ACCESS_TOKEN environment +# variable. You can general personal access tokens under developer settings on +# GitHub. You should restrict the scope of the token to "repo: public_repo". + +from importlib.util import spec_from_loader, module_from_spec +from importlib.machinery import SourceFileLoader +import os +import sys + +import github + + +# TODO(htuch): refactor with docs/generate_external_dep_rst.py and validate.py. +# Shared Starlark/Python files must have a .bzl suffix for Starlark import, so +# we are forced to do this workaround. +def LoadModule(name, path): + spec = spec_from_loader(name, SourceFileLoader(name, path)) + module = module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +repository_locations_utils = LoadModule('repository_locations_utils', + 'api/bazel/repository_locations_utils.bzl') + + +# TODO(htuch): refactor with docs/generate_external_dep_rst.py +def GetVersionFromGitHubUrl(github_url): + components = github_url.split('/') + if components[5] == 'archive': + # Only support .tar.gz, .zip today. Figure out the release tag from this + # filename. + if components[6].endswith('.tar.gz'): + github_version = components[6][:-len('.tar.gz')] + else: + assert (components[6].endswith('.zip')) + github_version = components[6][:-len('.zip')] + else: + # Release tag is a path component. + assert (components[5] == 'releases') + github_version = components[7] + return github_version + + +def FormatUtcDate(date): + # We only handle naive datetime objects right now, which is what PyGithub + # appears to be handing us. + assert (date.tzinfo is None) + return date.date().isoformat() + + +def PrintReleaseDates(repository_locations, github_instance): + # TODO(htuch): This would benefit from some more decomposition and + # restructuring. + for dep, metadata in sorted(repository_locations.items()): + release_date = None + warning = None + is_github_repository = False + for url in metadata['urls']: + if url.startswith('https://github.com/'): + is_github_repository = True + organization, project = url.split('/')[3:5] + repo = github_instance.get_repo(f'{organization}/{project}') + github_version = GetVersionFromGitHubUrl(url) + metadata_version = metadata['version'] + tagged_release = len(metadata_version) != 40 + if tagged_release: + tags = repo.get_tags() + for tag in tags: + if tag.name == github_version: + release_date = tag.commit.commit.committer.date + if not release_date: + warning = 'No matching tag' + else: + assert (metadata_version == github_version) + commit = repo.get_commit(github_version) + release_date = commit.commit.committer.date + if warning: + print(f'*WARNING* {dep}: {warning}') + if release_date: + try: + latest_release = repo.get_latest_release() + if latest_release.created_at > release_date and latest_release.tag_name != github_version: + print( + f'*WARNING* {dep} has a newer release than {github_version}@<{release_date}>: {latest_release.tag_name}@<{latest_release.created_at}>' + ) + except github.UnknownObjectException: + pass + mismatch = '' + iso_release_date = FormatUtcDate(release_date) + if iso_release_date != metadata['release_date']: + mismatch = ' [MISMATCH]' + print(f'{dep} has a release date {iso_release_date}{mismatch}') + elif not is_github_repository: + print(f'{dep} is not a GitHub repository') + else: + print(f'{dep} is a GitHub repository with no no inferrable release date') + + +if __name__ == '__main__': + if len(sys.argv) != 2: + print('Usage: %s ' % sys.argv[0]) + sys.exit(1) + access_token = os.getenv('GH_ACCESS_TOKEN') + if not access_token: + print('Missing GH_ACCESS_TOKEN') + sys.exit(1) + path = sys.argv[1] + spec_loader = repository_locations_utils.load_repository_locations_spec + path_module = LoadModule('repository_locations', path) + PrintReleaseDates(spec_loader(path_module.REPOSITORY_LOCATIONS_SPEC), github.Github(access_token)) diff --git a/tools/dependency/release_dates.sh b/tools/dependency/release_dates.sh new file mode 100755 index 000000000000..08496d63eee5 --- /dev/null +++ b/tools/dependency/release_dates.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +. tools/shell_utils.sh + +set -e + +python_venv release_dates "$1" diff --git a/tools/dependency/requirements.txt b/tools/dependency/requirements.txt new file mode 100644 index 000000000000..8f25e6588e09 --- /dev/null +++ b/tools/dependency/requirements.txt @@ -0,0 +1,26 @@ +PyGithub==1.53 \ + --hash=sha256:776befaddab9d8fddd525d52a6ca1ac228cf62b5b1e271836d766f4925e1452e \ + --hash=sha256:8ad656bf79958e775ec59f7f5a3dbcbadac12147ae3dc42708b951064096af15 +requests==2.24.0 \ + --hash=sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b \ + --hash=sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898 +Deprecated==1.2.10 \ + --hash=sha256:525ba66fb5f90b07169fdd48b6373c18f1ee12728ca277ca44567a367d9d7f74 \ + --hash=sha256:a766c1dccb30c5f6eb2b203f87edd1d8588847709c78589e1521d769addc8218 +PyJWT==1.7.1 \ + --hash=sha256:5c6eca3c2940464d106b99ba83b00c6add741c9becaec087fb7ccdefea71350e \ + --hash=sha256:8d59a976fb773f3e6a39c85636357c4f0e242707394cadadd9814f5cbaa20e96 +certifi==2020.6.20 \ + --hash=sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3 \ + --hash=sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41 +chardet==3.0.4 \ + --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \ + --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 +idna==2.10 \ + --hash=sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6 \ + --hash=sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0 +wrapt==1.12.1 \ + --hash=sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7 +urllib3==1.25.10 \ + --hash=sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a \ + --hash=sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461 From f5041a90495b420148540ab752fb5b04490cd5cb Mon Sep 17 00:00:00 2001 From: Harvey Tuch Date: Mon, 19 Oct 2020 07:05:26 -0400 Subject: [PATCH 2/3] Ready to review. Signed-off-by: Harvey Tuch --- .azure-pipelines/pipelines.yml | 15 ++ api/bazel/external_deps.bzl | 10 +- bazel/repository_locations.bzl | 12 +- ci/check_repository_locations.sh | 13 ++ ci/do_ci.sh | 14 +- ci/run_envoy_docker.sh | 1 + docs/generate_external_dep_rst.py | 36 +---- generated_api_shadow/bazel/external_deps.bzl | 10 +- tools/dependency/release_dates.py | 150 +++++++++---------- tools/dependency/utils.py | 35 +++++ tools/shell_utils.sh | 2 +- 11 files changed, 170 insertions(+), 128 deletions(-) create mode 100755 ci/check_repository_locations.sh diff --git a/.azure-pipelines/pipelines.yml b/.azure-pipelines/pipelines.yml index 4efd687e546c..e8841f1e8f82 100644 --- a/.azure-pipelines/pipelines.yml +++ b/.azure-pipelines/pipelines.yml @@ -87,6 +87,21 @@ stages: AZP_BRANCH: $(Build.SourceBranch) AZP_SHA1: $(Build.SourceVersion) condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['PostSubmit'], true)) + - job: dependencies + dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel. + pool: + vmImage: "ubuntu-18.04" + steps: + - script: ci/run_envoy_docker.sh 'ci/do_ci.sh deps' + workingDirectory: $(Build.SourcesDirectory) + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + GH_ACCESS_TOKEN: $(GitHubPublicRepoOnlyAccessToken) + displayName: "Verify dependency information" + - stage: sync condition: and(succeeded(), eq(variables['PostSubmit'], true)) diff --git a/api/bazel/external_deps.bzl b/api/bazel/external_deps.bzl index 9d95724e7a8d..e8283e4fee10 100644 --- a/api/bazel/external_deps.bzl +++ b/api/bazel/external_deps.bzl @@ -17,14 +17,18 @@ DEPENDENCY_ANNOTATIONS = [ # Envoy (see the external dependency at the given version for information). "implied_untracked_deps", - # When the dependency was last updated in Envoy. - "last_updated", - # Project metadata. "project_desc", "project_name", "project_url", + # Reflects the UTC date (YYYY-MM-DD format) for the dependency release. This + # is when the dependency was updated in its repository. For dependencies + # that have releases, this is the date of the release. For dependencies + # without releases or for scenarios where we temporarily need to use a + # commit, this date should be the date of the commit in UTC. + "release_date", + # List of the categories describing how the dependency is being used. This attribute is used # for automatic tracking of security posture of Envoy's dependencies. # Possible values are documented in the USE_CATEGORIES list below. diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index eb4f210ef37d..8d60741072d4 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -243,7 +243,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "20a159c38a98ecdb6368e8d655343b6036622a29a1621da9dc303f7ed9bf37f3", strip_prefix = "LuaJIT-{version}", urls = ["https://github.com/LuaJIT/LuaJIT/archive/{version}.tar.gz"], - release_date = "2020-10-13", + release_date = "2020-10-12", use_category = ["dataplane_ext"], extensions = ["envoy.filters.http.lua"], cpe = "cpe:2.3:a:luajit:luajit:*", @@ -270,7 +270,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( strip_prefix = "nghttp2-{version}", urls = ["https://github.com/nghttp2/nghttp2/releases/download/v{version}/nghttp2-{version}.tar.gz"], use_category = ["controlplane", "dataplane_core"], - release_date = "2020-06-03", + release_date = "2020-06-02", cpe = "cpe:2.3:a:nghttp2:nghttp2:*", ), io_opentracing_cpp = dict( @@ -664,7 +664,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( "envoy.filters.network.wasm", "envoy.stat_sinks.wasm", ], - release_date = "2020-08-31", + release_date = "2020-08-17", cpe = "cpe:2.3:a:google:v8:*", ), com_googlesource_quiche = dict( @@ -677,7 +677,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( urls = ["https://storage.googleapis.com/quiche-envoy-integration/{version}.tar.gz"], use_category = ["dataplane_ext"], extensions = ["envoy.transport_sockets.quic"], - release_date = "2020-09-18", + release_date = "2020-09-17", cpe = "N/A", ), com_googlesource_googleurl = dict( @@ -690,7 +690,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( urls = ["https://storage.googleapis.com/quiche-envoy-integration/googleurl_{version}.tar.gz"], use_category = ["dataplane_ext"], extensions = [], - release_date = "2020-08-05", + release_date = "2020-07-30", cpe = "N/A", ), com_google_cel_cpp = dict( @@ -906,7 +906,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( "envoy.filters.network.wasm", "envoy.stat_sinks.wasm", ], - release_date = "2020-10-09", + release_date = "2018-12-18", cpe = "N/A", ), ) diff --git a/ci/check_repository_locations.sh b/ci/check_repository_locations.sh new file mode 100755 index 000000000000..135808f3ee59 --- /dev/null +++ b/ci/check_repository_locations.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -e + +function no_change { + echo "No change to **/repository_locations.bzl" + exit 0 +} + +(./tools/git/modified_since_last_github_commit.sh . bzl | grep repository_locations) || no_change + +./tools/dependency/release_dates.sh ./bazel/repository_locations.bzl +./tools/dependency/release_dates.sh ./api/bazel/repository_locations.bzl diff --git a/ci/do_ci.sh b/ci/do_ci.sh index c9c268e70e04..0a1faa8b8b24 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -411,14 +411,18 @@ elif [[ "$CI_TARGET" == "fix_spelling_pedantic" ]]; then exit 0 elif [[ "$CI_TARGET" == "docs" ]]; then echo "generating docs..." - # Validate dependency relationships between core/extensions and external deps. - tools/dependency/validate_test.py - tools/dependency/validate.py - # Validate the CVE scanner works. TODO(htuch): create a dedicated tools CI target. - python3.8 tools/dependency/cve_scan_test.py # Build docs. BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS[*]}" docs/build.sh exit 0 +elif [[ "$CI_TARGET" == "deps" ]]; then + echo "verifying dependencies..." + # Validate dependency relationships between core/extensions and external deps. + ./tools/dependency/validate_test.py + ./tools/dependency/validate.py + # Validate the CVE scanner works. + python3.8 tools/dependency/cve_scan_test.py + ./ci/check_repository_locations.sh + exit 0 elif [[ "$CI_TARGET" == "verify_examples" ]]; then echo "verify examples..." docker load < "$ENVOY_DOCKER_BUILD_DIR/docker/envoy-docker-images.tar.xz" diff --git a/ci/run_envoy_docker.sh b/ci/run_envoy_docker.sh index 842b51b6ce89..bc8e6dc070e7 100755 --- a/ci/run_envoy_docker.sh +++ b/ci/run_envoy_docker.sh @@ -86,6 +86,7 @@ docker run --rm \ -e SYSTEM_PULLREQUEST_TARGETBRANCH \ -e SYSTEM_PULLREQUEST_PULLREQUESTNUMBER \ -e GCS_ARTIFACT_BUCKET \ + -e GH_ACCESS_TOKEN \ -e BUILD_SOURCEBRANCHNAME \ -e BAZELISK_BASE_URL \ -e ENVOY_BUILD_ARCH \ diff --git a/docs/generate_external_dep_rst.py b/docs/generate_external_dep_rst.py index c324f46371b2..4c4008e64dd0 100755 --- a/docs/generate_external_dep_rst.py +++ b/docs/generate_external_dep_rst.py @@ -53,37 +53,17 @@ def RenderTitle(title): # SHA. Otherwise, return the tarball download. def GetVersionUrl(metadata): # Figure out if it's a GitHub repo. - github_repo = None - github_version = None - for url in metadata['urls']: - if url.startswith('https://github.com/'): - components = url.split('/') - github_repo = f'https://github.com/{components[3]}/{components[4]}' - if components[5] == 'archive': - # Only support .tar.gz, .zip today. Figure out the release tag from this - # filename. - if components[6].endswith('.tar.gz'): - github_version = components[6][:-len('.tar.gz')] - else: - assert (components[6].endswith('.zip')) - github_version = components[6][:-len('.zip')] - else: - # Release tag is a path component. - assert (components[5] == 'releases') - github_version = components[7] - break + github_release = dep_utils.GetGitHubReleaseFromUrls(metadata['urls']) # If not, direct download link for tarball - download_url = metadata['urls'][0] - if not github_repo: - return download_url - # If it's not a GH hash, it's a tagged release. - tagged_release = len(metadata['version']) != 40 - if tagged_release: + if not github_release: + return metadata['urls'][0] + github_repo = f'https://github.com/{github_release.organization}/{github_release.project}' + if github_release.tagged: # The GitHub version should look like the metadata version, but might have # something like a "v" prefix. - return f'{github_repo}/releases/tag/{github_version}' - assert (metadata['version'] == github_version) - return f'{github_repo}/tree/{github_version}' + return f'{github_repo}/releases/tag/{github_release.version}' + assert (metadata['version'] == github_release.version) + return f'{github_repo}/tree/{github_release.version}' if __name__ == '__main__': diff --git a/generated_api_shadow/bazel/external_deps.bzl b/generated_api_shadow/bazel/external_deps.bzl index 9d95724e7a8d..e8283e4fee10 100644 --- a/generated_api_shadow/bazel/external_deps.bzl +++ b/generated_api_shadow/bazel/external_deps.bzl @@ -17,14 +17,18 @@ DEPENDENCY_ANNOTATIONS = [ # Envoy (see the external dependency at the given version for information). "implied_untracked_deps", - # When the dependency was last updated in Envoy. - "last_updated", - # Project metadata. "project_desc", "project_name", "project_url", + # Reflects the UTC date (YYYY-MM-DD format) for the dependency release. This + # is when the dependency was updated in its repository. For dependencies + # that have releases, this is the date of the release. For dependencies + # without releases or for scenarios where we temporarily need to use a + # commit, this date should be the date of the commit in UTC. + "release_date", + # List of the categories describing how the dependency is being used. This attribute is used # for automatic tracking of security posture of Envoy's dependencies. # Possible values are documented in the USE_CATEGORIES list below. diff --git a/tools/dependency/release_dates.py b/tools/dependency/release_dates.py index b2a50c7649e8..a523db72d189 100644 --- a/tools/dependency/release_dates.py +++ b/tools/dependency/release_dates.py @@ -1,57 +1,30 @@ -# CLI tool to query GitHub API and compare with repository_locations.bzl. It -# will: -# - Compute the release date of dependencies and warn if there is a mismatch +# CLI tool to query GitHub API and compare with repository_locations.bzl. It: +# - Computes the release date of dependencies and fails if there is a mismatch # with the metdata release date. -# - Look at the latest release tag and warn if this is later than the dependency -# version in the .bzl. +# - Looks up the latest release tag on GitHub and warns if this is later than the +# dependency version in the .bzl. # # Usage: # tools/dependency/release_dates.sh # # You will need to set a GitHub access token in the GH_ACCESS_TOKEN environment -# variable. You can general personal access tokens under developer settings on +# variable. You can generate personal access tokens under developer settings on # GitHub. You should restrict the scope of the token to "repo: public_repo". -from importlib.util import spec_from_loader, module_from_spec -from importlib.machinery import SourceFileLoader import os import sys import github +import utils -# TODO(htuch): refactor with docs/generate_external_dep_rst.py and validate.py. -# Shared Starlark/Python files must have a .bzl suffix for Starlark import, so -# we are forced to do this workaround. -def LoadModule(name, path): - spec = spec_from_loader(name, SourceFileLoader(name, path)) - module = module_from_spec(spec) - spec.loader.exec_module(module) - return module - -repository_locations_utils = LoadModule('repository_locations_utils', - 'api/bazel/repository_locations_utils.bzl') - - -# TODO(htuch): refactor with docs/generate_external_dep_rst.py -def GetVersionFromGitHubUrl(github_url): - components = github_url.split('/') - if components[5] == 'archive': - # Only support .tar.gz, .zip today. Figure out the release tag from this - # filename. - if components[6].endswith('.tar.gz'): - github_version = components[6][:-len('.tar.gz')] - else: - assert (components[6].endswith('.zip')) - github_version = components[6][:-len('.zip')] - else: - # Release tag is a path component. - assert (components[5] == 'releases') - github_version = components[7] - return github_version +# Thrown on errors related to release date. +class ReleaseDateError(Exception): + pass +# Format a datetime object as UTC YYYY-MM-DD. def FormatUtcDate(date): # We only handle naive datetime objects right now, which is what PyGithub # appears to be handing us. @@ -59,52 +32,60 @@ def FormatUtcDate(date): return date.date().isoformat() -def PrintReleaseDates(repository_locations, github_instance): - # TODO(htuch): This would benefit from some more decomposition and - # restructuring. +# Obtain latest release version and compare against metadata version, warn on +# mismatch. +def VerifyAndPrintLatestRelease(dep, repo, metadata_version, release_date): + try: + latest_release = repo.get_latest_release() + if latest_release.created_at > release_date and latest_release.tag_name != metadata_version: + print(f'*WARNING* {dep} has a newer release than {metadata_version}@<{release_date}>: ' + f'{latest_release.tag_name}@<{latest_release.created_at}>') + except github.UnknownObjectException: + pass + + +# Print GitHub release date, throw ReleaseDateError on mismatch with metadata release date. +def VerifyAndPrintReleaseDate(dep, github_release_date, metadata_release_date): + mismatch = '' + iso_release_date = FormatUtcDate(github_release_date) + print(f'{dep} has a GitHub release date {iso_release_date}') + if iso_release_date != metadata_release_date: + raise ReleaseDateError(f'Mismatch with metadata release date of {metadata_release_date}') + + +# Extract release date from GitHub API. +def GetReleaseDate(repo, metadata_version, github_release): + if github_release.tagged: + tags = repo.get_tags() + for tag in tags: + if tag.name == github_release.version: + return tag.commit.commit.committer.date + return None + else: + assert (metadata_version == github_release.version) + commit = repo.get_commit(github_release.version) + return commit.commit.committer.date + + +# Verify release dates in metadata against GitHub API. +def VerifyAndPrintReleaseDates(repository_locations, github_instance): for dep, metadata in sorted(repository_locations.items()): release_date = None - warning = None - is_github_repository = False - for url in metadata['urls']: - if url.startswith('https://github.com/'): - is_github_repository = True - organization, project = url.split('/')[3:5] - repo = github_instance.get_repo(f'{organization}/{project}') - github_version = GetVersionFromGitHubUrl(url) - metadata_version = metadata['version'] - tagged_release = len(metadata_version) != 40 - if tagged_release: - tags = repo.get_tags() - for tag in tags: - if tag.name == github_version: - release_date = tag.commit.commit.committer.date - if not release_date: - warning = 'No matching tag' - else: - assert (metadata_version == github_version) - commit = repo.get_commit(github_version) - release_date = commit.commit.committer.date - if warning: - print(f'*WARNING* {dep}: {warning}') - if release_date: - try: - latest_release = repo.get_latest_release() - if latest_release.created_at > release_date and latest_release.tag_name != github_version: - print( - f'*WARNING* {dep} has a newer release than {github_version}@<{release_date}>: {latest_release.tag_name}@<{latest_release.created_at}>' - ) - except github.UnknownObjectException: - pass - mismatch = '' - iso_release_date = FormatUtcDate(release_date) - if iso_release_date != metadata['release_date']: - mismatch = ' [MISMATCH]' - print(f'{dep} has a release date {iso_release_date}{mismatch}') - elif not is_github_repository: + # Obtain release information from GitHub API. + github_release = utils.GetGitHubReleaseFromUrls(metadata['urls']) + if not github_release: print(f'{dep} is not a GitHub repository') + continue + repo = github_instance.get_repo(f'{github_release.organization}/{github_release.project}') + release_date = GetReleaseDate(repo, metadata['version'], github_release) + if release_date: + # Check whether there is a more recent version and warn if necessary. + VerifyAndPrintLatestRelease(dep, repo, github_release.version, release_date) + # Verify that the release date in metadata and GitHub correspond, + # otherwise throw ReleaseDateError. + VerifyAndPrintReleaseDate(dep, release_date, metadata['release_date']) else: - print(f'{dep} is a GitHub repository with no no inferrable release date') + raise ReleaseDateError(f'{dep} is a GitHub repository with no no inferrable release date') if __name__ == '__main__': @@ -116,6 +97,11 @@ def PrintReleaseDates(repository_locations, github_instance): print('Missing GH_ACCESS_TOKEN') sys.exit(1) path = sys.argv[1] - spec_loader = repository_locations_utils.load_repository_locations_spec - path_module = LoadModule('repository_locations', path) - PrintReleaseDates(spec_loader(path_module.REPOSITORY_LOCATIONS_SPEC), github.Github(access_token)) + spec_loader = utils.repository_locations_utils.load_repository_locations_spec + path_module = utils.LoadModule('repository_locations', path) + try: + VerifyAndPrintReleaseDates(spec_loader(path_module.REPOSITORY_LOCATIONS_SPEC), + github.Github(access_token)) + except ReleaseDateError as e: + print(f'An error occurred while processing {path}, please verify the correctness of the ' + f'metadata: {e}') diff --git a/tools/dependency/utils.py b/tools/dependency/utils.py index df1930d5cd32..15deb07576a4 100644 --- a/tools/dependency/utils.py +++ b/tools/dependency/utils.py @@ -1,5 +1,6 @@ # Utilities for reasoning about dependencies. +from collections import namedtuple from importlib.util import spec_from_loader, module_from_spec from importlib.machinery import SourceFileLoader @@ -21,8 +22,42 @@ def LoadModule(name, path): 'api/bazel/repository_locations_utils.bzl') +# All repository location metadata in the Envoy repository. def RepositoryLocations(): spec_loader = repository_locations_utils.load_repository_locations_spec locations = spec_loader(envoy_repository_locations.REPOSITORY_LOCATIONS_SPEC) locations.update(spec_loader(api_repository_locations.REPOSITORY_LOCATIONS_SPEC)) return locations + + +# Information releated to a GitHub release version. +GitHubRelease = namedtuple('GitHubRelease', ['organization', 'project', 'version', 'tagged']) + + +# Search through a list of URLs and determine if any contain a GitHub URL. If +# so, use heuristics to extract the release version and repo details, return +# this, otherwise return None. +def GetGitHubReleaseFromUrls(urls): + for url in urls: + if not url.startswith('https://github.com/'): + continue + components = url.split('/') + if components[5] == 'archive': + # Only support .tar.gz, .zip today. Figure out the release tag from this + # filename. + if components[6].endswith('.tar.gz'): + github_version = components[6][:-len('.tar.gz')] + else: + assert (components[6].endswith('.zip')) + github_version = components[6][:-len('.zip')] + else: + # Release tag is a path component. + assert (components[5] == 'releases') + github_version = components[7] + # If it's not a GH hash, it's a tagged release. + tagged_release = len(github_version) != 40 + return GitHubRelease(organization=components[3], + project=components[4], + version=github_version, + tagged=tagged_release) + return None diff --git a/tools/shell_utils.sh b/tools/shell_utils.sh index 470b9c6fc078..e32c3c95056b 100644 --- a/tools/shell_utils.sh +++ b/tools/shell_utils.sh @@ -22,7 +22,7 @@ python_venv() { VENV_DIR="${BUILD_DIR}/${PY_NAME}" source_venv "${VENV_DIR}" - pip install -r "${SCRIPT_DIR}"/requirements.txt + pip3 install -r "${SCRIPT_DIR}"/requirements.txt shift python3 "${SCRIPT_DIR}/${PY_NAME}.py" "$*" From 67fbd4069ad77f467343e96302b36b658f3fd987 Mon Sep 17 00:00:00 2001 From: Harvey Tuch Date: Tue, 20 Oct 2020 09:32:30 -0400 Subject: [PATCH 3/3] Review feedback. Signed-off-by: Harvey Tuch --- .azure-pipelines/pipelines.yml | 2 +- ci/check_repository_locations.sh | 4 ++-- ci/run_envoy_docker.sh | 2 +- tools/dependency/release_dates.py | 6 +++--- tools/deprecate_version/deprecate_version.py | 7 +++---- tools/github/sync_assignable.py | 4 ++-- 6 files changed, 12 insertions(+), 13 deletions(-) diff --git a/.azure-pipelines/pipelines.yml b/.azure-pipelines/pipelines.yml index e8841f1e8f82..1fbb4e56e1b2 100644 --- a/.azure-pipelines/pipelines.yml +++ b/.azure-pipelines/pipelines.yml @@ -99,7 +99,7 @@ stages: BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - GH_ACCESS_TOKEN: $(GitHubPublicRepoOnlyAccessToken) + GITHUB_TOKEN: $(GitHubPublicRepoOnlyAccessToken) displayName: "Verify dependency information" diff --git a/ci/check_repository_locations.sh b/ci/check_repository_locations.sh index 135808f3ee59..bd799f1b05d9 100755 --- a/ci/check_repository_locations.sh +++ b/ci/check_repository_locations.sh @@ -9,5 +9,5 @@ function no_change { (./tools/git/modified_since_last_github_commit.sh . bzl | grep repository_locations) || no_change -./tools/dependency/release_dates.sh ./bazel/repository_locations.bzl -./tools/dependency/release_dates.sh ./api/bazel/repository_locations.bzl +./tools/dependency/release_dates.sh ./bazel/repository_locations.bzl +./tools/dependency/release_dates.sh ./api/bazel/repository_locations.bzl diff --git a/ci/run_envoy_docker.sh b/ci/run_envoy_docker.sh index bc8e6dc070e7..e0f204e67fcf 100755 --- a/ci/run_envoy_docker.sh +++ b/ci/run_envoy_docker.sh @@ -86,7 +86,7 @@ docker run --rm \ -e SYSTEM_PULLREQUEST_TARGETBRANCH \ -e SYSTEM_PULLREQUEST_PULLREQUESTNUMBER \ -e GCS_ARTIFACT_BUCKET \ - -e GH_ACCESS_TOKEN \ + -e GITHUB_TOKEN \ -e BUILD_SOURCEBRANCHNAME \ -e BAZELISK_BASE_URL \ -e ENVOY_BUILD_ARCH \ diff --git a/tools/dependency/release_dates.py b/tools/dependency/release_dates.py index a523db72d189..7780ff45f928 100644 --- a/tools/dependency/release_dates.py +++ b/tools/dependency/release_dates.py @@ -7,7 +7,7 @@ # Usage: # tools/dependency/release_dates.sh # -# You will need to set a GitHub access token in the GH_ACCESS_TOKEN environment +# You will need to set a GitHub access token in the GITHUB_TOKEN environment # variable. You can generate personal access tokens under developer settings on # GitHub. You should restrict the scope of the token to "repo: public_repo". @@ -92,9 +92,9 @@ def VerifyAndPrintReleaseDates(repository_locations, github_instance): if len(sys.argv) != 2: print('Usage: %s ' % sys.argv[0]) sys.exit(1) - access_token = os.getenv('GH_ACCESS_TOKEN') + access_token = os.getenv('GITHUB_TOKEN') if not access_token: - print('Missing GH_ACCESS_TOKEN') + print('Missing GITHUB_TOKEN') sys.exit(1) path = sys.argv[1] spec_loader = utils.repository_locations_utils.load_repository_locations_spec diff --git a/tools/deprecate_version/deprecate_version.py b/tools/deprecate_version/deprecate_version.py index 9cbde123785e..70091e979124 100644 --- a/tools/deprecate_version/deprecate_version.py +++ b/tools/deprecate_version/deprecate_version.py @@ -10,7 +10,7 @@ # # python tools/deprecate_version/deprecate_version.py # -# A GitHub access token must be set in GH_ACCESS_TOKEN. To create one, go to +# A GitHub access token must be set in GITHUB_TOKEN. To create one, go to # Settings -> Developer settings -> Personal access tokens in GitHub and create # a token with public_repo scope. Keep this safe, it's broader than it needs to # be thanks to GH permission model @@ -183,10 +183,9 @@ def GetRuntimeAndPr(): print('No code is deprecated.') sys.exit(0) - access_token = os.getenv('GH_ACCESS_TOKEN') + access_token = os.getenv('GITHUB_TOKEN') if not access_token: - print( - 'Missing GH_ACCESS_TOKEN: see instructions in tools/deprecate_version/deprecate_version.py') + print('Missing GITHUB_TOKEN: see instructions in tools/deprecate_version/deprecate_version.py') sys.exit(1) CreateIssues(access_token, runtime_and_pr) diff --git a/tools/github/sync_assignable.py b/tools/github/sync_assignable.py index 3a437fa8d35f..910dc1b74225 100644 --- a/tools/github/sync_assignable.py +++ b/tools/github/sync_assignable.py @@ -45,9 +45,9 @@ def SyncAssignable(access_token): if __name__ == '__main__': - access_token = os.getenv('GH_ACCESS_TOKEN') + access_token = os.getenv('GITHUB_TOKEN') if not access_token: - print('Missing GH_ACCESS_TOKEN') + print('Missing GITHUB_TOKEN') sys.exit(1) sys.exit(SyncAssignable(access_token))