From 8de1c01d5810e6fb2a713672ef0174364744f7cb Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Thu, 11 Jul 2024 13:26:19 +0000 Subject: [PATCH 1/6] chore: bake the grpc plugin in the hermetic library generation image --- .../library_generation.Dockerfile | 12 +++++- library_generation/generate_library.sh | 4 +- .../test/generate_library_unit_tests.sh | 40 +++++++++++++++++++ library_generation/utils/utilities.sh | 27 ++++++++++--- 4 files changed, 75 insertions(+), 8 deletions(-) diff --git a/.cloudbuild/library_generation/library_generation.Dockerfile b/.cloudbuild/library_generation/library_generation.Dockerfile index 2629c092aa..63385f136a 100644 --- a/.cloudbuild/library_generation/library_generation.Dockerfile +++ b/.cloudbuild/library_generation/library_generation.Dockerfile @@ -19,7 +19,9 @@ SHELL [ "/bin/bash", "-c" ] ARG OWLBOT_CLI_COMMITTISH=ac84fa5c423a0069bbce3d2d869c9730c8fdf550 ARG PROTOC_VERSION=25.3 +ARG GRPC_VERSION=1.62.2 ENV HOME=/home +ENV OS_ARCHITECTURE="linux-x86_64" # install OS tools RUN apt-get update && apt-get install -y \ @@ -32,11 +34,19 @@ COPY library_generation /src # install protoc WORKDIR /protoc RUN source /src/utils/utilities.sh \ - && download_protoc "${PROTOC_VERSION}" "linux-x86_64" + && download_protoc "${PROTOC_VERSION}" "${OS_ARCHITECTURE}" # we indicate protoc is available in the container via env vars ENV DOCKER_PROTOC_LOCATION=/protoc ENV DOCKER_PROTOC_VERSION="${PROTOC_VERSION}" +# install grpc +WORKDIR /grpc +RUN source /src/utils/utilities.sh \ + && download_grpc_plugin "${GRPC_VERSION}" "${OS_ARCHITECTURE}" +# similar to protoc, we indicate grpc is available in the container via env vars +ENV DOCKER_GRPC_LOCATION="/grpc/protoc-gen-grpc-java-${GRPC_VERSION}-${OS_ARCHITECTURE}.exe" +ENV DOCKER_GRPC_VERSION="${GRPC_VERSION}" + # use python 3.11 (the base image has several python versions; here we define the default one) RUN rm $(which python3) RUN ln -s $(which python3.11) /usr/local/bin/python diff --git a/library_generation/generate_library.sh b/library_generation/generate_library.sh index ecf8c64a50..8988c57b50 100755 --- a/library_generation/generate_library.sh +++ b/library_generation/generate_library.sh @@ -183,13 +183,15 @@ case "${proto_path}" in ;; esac # download gapic-generator-java, protobuf and grpc plugin. +# this function will create the environment variables "protoc_path" and +# "grpc_path", to be used in the protoc calls below download_tools "${gapic_generator_version}" "${protoc_version}" "${grpc_version}" "${os_architecture}" ##################### Section 1 ##################### # generate grpc-*/ ##################################################### if [[ ! "${transport}" == "rest" ]]; then # do not need to generate grpc-* if the transport is `rest`. - "${protoc_path}"/protoc "--plugin=protoc-gen-rpc-plugin=protoc-gen-grpc-java-${grpc_version}-${os_architecture}.exe" \ + "${protoc_path}"/protoc "--plugin=protoc-gen-rpc-plugin=${grpc_path}" \ "--rpc-plugin_out=:${temp_destination_path}/java_grpc.jar" \ ${proto_files} # Do not quote because this variable should not be treated as one long string. # unzip java_grpc.jar to grpc-*/src/main/java diff --git a/library_generation/test/generate_library_unit_tests.sh b/library_generation/test/generate_library_unit_tests.sh index 5384d6982c..b14865033a 100755 --- a/library_generation/test/generate_library_unit_tests.sh +++ b/library_generation/test/generate_library_unit_tests.sh @@ -28,10 +28,21 @@ get_grpc_version_failed_with_invalid_generator_version_test() { assertEquals 1 $((res)) } +get_grpc_version_succeed_docker_env_var_test() { + local version_with_docker + local version_without_docker + export DOCKER_GRPC_VERSION="9.9.9" + # get_grpc_version should prioritize DOCKER_GRPC_VERSION + version_with_docker=$(get_grpc_version "2.24.0") + assertEquals "${DOCKER_GRPC_VERSION}" "${version_with_docker}" + unset DOCKER_GRPC_VERSION +} + get_protoc_version_succeed_docker_env_var_test() { local version_with_docker local version_without_docker export DOCKER_PROTOC_VERSION="9.9.9" + # get_protoc_version should prioritize DOCKER_PROTOC_VERSION version_with_docker=$(get_protoc_version "2.24.0") assertEquals "${DOCKER_PROTOC_VERSION}" "${version_with_docker}" unset DOCKER_PROTOC_VERSION @@ -162,6 +173,8 @@ download_tools_succeed_with_baked_protoc() { local test_ggj_version="2.40.0" local test_grpc_version="1.64.0" + # we expect download_tools to decide to use DOCKER_PROTOC_LOCATION because + # the protoc version we want to download is the same as DOCKER_PROTOC_VERSION download_tools "${test_ggj_version}" "99.99" "${test_grpc_version}" "linux-x86_64" assertEquals "${protoc_bin_folder}" "${protoc_path}" @@ -169,6 +182,31 @@ download_tools_succeed_with_baked_protoc() { unset DOCKER_PROTOC_LOCATION unset DOCKER_PROTOC_VERSION unset output_folder + unset protoc_path +} + +download_tools_succeed_with_baked_grpc() { + # This test has the same structure as + # download_tools_succeed_with_baked_protoc, but meant to test grpc. + local test_dir=$(mktemp -d) + pushd "${test_dir}" + export DOCKER_GRPC_LOCATION=$(mktemp -d) + export DOCKER_GRPC_VERSION="99.99" + export output_folder=$(get_output_folder) + mkdir "${output_folder}" + + local test_ggj_version="2.40.0" + local test_protoc_version="1.64.0" + # we expect download_tools to decide to use DOCKER_GRPC_LOCATION because + # the protoc version we want to download is the same as DOCKER_GRPC_VERSION + download_tools "${test_ggj_version}" "${test_protoc_version}" "99.99" "linux-x86_64" + assertEquals "${DOCKER_GRPC_LOCATION}" "${grpc_path}" + + rm -rdf "${output_folder}" + unset DOCKER_GRPC_LOCATION + unset DOCKER_GRPC_VERSION + unset output_folder + unset grpc_path } download_grpc_plugin_succeed_with_valid_version_linux_test() { @@ -293,6 +331,7 @@ test_list=( extract_folder_name_test get_grpc_version_succeed_with_valid_generator_version_test get_grpc_version_failed_with_invalid_generator_version_test + get_grpc_version_succeed_docker_env_var_test get_protoc_version_succeed_docker_env_var_test get_protoc_version_succeed_with_valid_generator_version_test get_protoc_version_failed_with_invalid_generator_version_test @@ -307,6 +346,7 @@ test_list=( download_protoc_failed_with_invalid_version_linux_test download_protoc_failed_with_invalid_arch_test download_tools_succeed_with_baked_protoc + download_tools_succeed_with_baked_grpc download_grpc_plugin_succeed_with_valid_version_linux_test download_grpc_plugin_succeed_with_valid_version_macos_test download_grpc_plugin_failed_with_invalid_version_linux_test diff --git a/library_generation/utils/utilities.sh b/library_generation/utils/utilities.sh index 2e26ed6027..12ea9ac7e3 100755 --- a/library_generation/utils/utilities.sh +++ b/library_generation/utils/utilities.sh @@ -105,6 +105,11 @@ download_gapic_generator_pom_parent() { get_grpc_version() { local gapic_generator_version=$1 local grpc_version + if [[ -n "${DOCKER_GRPC_VERSION}" ]]; then + >&2 echo "Using grpc version baked into the container: ${DOCKER_GRPC_VERSION}" + echo "${DOCKER_GRPC_VERSION}" + return + fi pushd "${output_folder}" > /dev/null # get grpc version from gapic-generator-java-pom-parent/pom.xml download_gapic_generator_pom_parent "${gapic_generator_version}" @@ -147,7 +152,15 @@ download_tools() { export protoc_path=$(download_protoc "${protoc_version}" "${os_architecture}") fi - download_grpc_plugin "${grpc_version}" "${os_architecture}" + # similar case with grpc + if [[ "${DOCKER_GRPC_VERSION}" == "${grpc_version}" ]]; then + # if the specified protoc_version matches the one baked in the docker + # container, we just point grpc_path to its location. + export grpc_path="${DOCKER_GRPC_LOCATION}" + else + export grpc_path=$(download_grpc_plugin "${grpc_version}" "${os_architecture}") + fi + popd } @@ -198,13 +211,15 @@ download_protoc() { download_grpc_plugin() { local grpc_version=$1 local os_architecture=$2 - if [ ! -f "protoc-gen-grpc-java-${grpc_version}-${os_architecture}.exe" ]; then + grpc_filename="protoc-gen-grpc-java-${grpc_version}-${os_architecture}.exe" + if [ ! -f "${grpc_filename}" ]; then # download protoc-gen-grpc-java plugin from Google maven central mirror. download_from \ - "https://maven-central.storage-download.googleapis.com/maven2/io/grpc/protoc-gen-grpc-java/${grpc_version}/protoc-gen-grpc-java-${grpc_version}-${os_architecture}.exe" \ - "protoc-gen-grpc-java-${grpc_version}-${os_architecture}.exe" - chmod +x "protoc-gen-grpc-java-${grpc_version}-${os_architecture}.exe" + "https://maven-central.storage-download.googleapis.com/maven2/io/grpc/protoc-gen-grpc-java/${grpc_version}/${grpc_filename}" \ + "${grpc_filename}" + chmod +x "${grpc_filename}" fi + echo "$(pwd)/${grpc_filename}" } download_from() { @@ -282,7 +297,7 @@ get_proto_path_from_preprocessed_sources() { pushd "${sources}" > /dev/null local proto_library=$(find . -maxdepth 1 -type d -name 'proto-*' | sed 's/\.\///') local found_libraries=$(echo "${proto_library}" | wc -l) - if [ -z ${proto_library} ]; then + if [[ -z ${proto_library} ]]; then echo "no proto libraries found in the supplied sources path" exit 1 elif [ ${found_libraries} -gt 1 ]; then From 15b2fc51ab724e6f3e39857ada3b9842eff1e306 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Thu, 11 Jul 2024 13:40:36 +0000 Subject: [PATCH 2/6] improve comments --- library_generation/generate_library.sh | 4 ++-- .../test/generate_library_unit_tests.sh | 7 +++++-- library_generation/utils/utilities.sh | 18 ++++++++++++++++++ 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/library_generation/generate_library.sh b/library_generation/generate_library.sh index 8988c57b50..1b113ba11b 100755 --- a/library_generation/generate_library.sh +++ b/library_generation/generate_library.sh @@ -183,8 +183,8 @@ case "${proto_path}" in ;; esac # download gapic-generator-java, protobuf and grpc plugin. -# this function will create the environment variables "protoc_path" and -# "grpc_path", to be used in the protoc calls below +# the download_tools function will create the environment variables "protoc_path" +# and "grpc_path", to be used in the protoc calls below. download_tools "${gapic_generator_version}" "${protoc_version}" "${grpc_version}" "${os_architecture}" ##################### Section 1 ##################### # generate grpc-*/ diff --git a/library_generation/test/generate_library_unit_tests.sh b/library_generation/test/generate_library_unit_tests.sh index b14865033a..0261a041c3 100755 --- a/library_generation/test/generate_library_unit_tests.sh +++ b/library_generation/test/generate_library_unit_tests.sh @@ -174,7 +174,10 @@ download_tools_succeed_with_baked_protoc() { local test_ggj_version="2.40.0" local test_grpc_version="1.64.0" # we expect download_tools to decide to use DOCKER_PROTOC_LOCATION because - # the protoc version we want to download is the same as DOCKER_PROTOC_VERSION + # the protoc version we want to download is the same as DOCKER_PROTOC_VERSION. + # Note that `protoc_bin_folder` is just the expected formatted value that + # download_tools will format using DOCKER_PROTOC_VERSION (via + # download_protoc). download_tools "${test_ggj_version}" "99.99" "${test_grpc_version}" "linux-x86_64" assertEquals "${protoc_bin_folder}" "${protoc_path}" @@ -187,7 +190,7 @@ download_tools_succeed_with_baked_protoc() { download_tools_succeed_with_baked_grpc() { # This test has the same structure as - # download_tools_succeed_with_baked_protoc, but meant to test grpc. + # download_tools_succeed_with_baked_protoc, but meant for the grpc plugin. local test_dir=$(mktemp -d) pushd "${test_dir}" export DOCKER_GRPC_LOCATION=$(mktemp -d) diff --git a/library_generation/utils/utilities.sh b/library_generation/utils/utilities.sh index 12ea9ac7e3..a93df59b42 100755 --- a/library_generation/utils/utilities.sh +++ b/library_generation/utils/utilities.sh @@ -102,6 +102,10 @@ download_gapic_generator_pom_parent() { download_generator_artifact "${gapic_generator_version}" "gapic-generator-java-pom-parent-${gapic_generator_version}.pom" "gapic-generator-java-pom-parent" } +# This function returns the version of the grpc plugin to generate the libraries. If +# DOCKER_GRPC_VERSION is set, this will be the version. Otherwise, it will be +# computed from the gapic-generator-pom-parent artifact at the specified +# gapic_generator_version. get_grpc_version() { local gapic_generator_version=$1 local grpc_version @@ -118,6 +122,10 @@ get_grpc_version() { echo "${grpc_version}" } +# This function returns the version of protoc to generate the libraries. If +# DOCKER_PROTOC_VERSION is set, this will be the version. Otherwise, it will be +# computed from the gapic-generator-pom-parent artifact at the specified +# gapic_generator_version. get_protoc_version() { local gapic_generator_version=$1 local protoc_version @@ -134,6 +142,16 @@ get_protoc_version() { echo "${protoc_version}" } +# Given the versions of the gapic generator, protoc and the protoc-grpc plugin, +# this function will download each one of the tools and create the environment +# variables "protoc_path" and "grpc_path" which are expected upstream. Note that +# if the specified versions of protoc and grpc match DOCKER_PROTOC_VERSION and +# DOCKER_GRPC_VERSION respectively, this function will instead set "protoc_path" +# and "grpc_path" to DOCKER_PROTOC_PATH and DOCKER_GRPC_PATH respectively (no +# download), since the docker image will have downloaded these tools beforehand. +# For the case of gapic-generator-java, no env var will be exported for the +# upstream flow, but instead it will be assigned a default filename that will be +# referenced by the file `library_generation/gapic-generator-java-wrapper`. download_tools() { local gapic_generator_version=$1 local protoc_version=$2 From 0a95554b4a4ecb2d2b5ca9c968aac82ba27e9887 Mon Sep 17 00:00:00 2001 From: Diego Marquez Date: Mon, 22 Jul 2024 16:02:12 -0400 Subject: [PATCH 3/6] Update library_generation/utils/utilities.sh Co-authored-by: Blake Li --- library_generation/utils/utilities.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library_generation/utils/utilities.sh b/library_generation/utils/utilities.sh index a93df59b42..72fefec76a 100755 --- a/library_generation/utils/utilities.sh +++ b/library_generation/utils/utilities.sh @@ -172,7 +172,7 @@ download_tools() { # similar case with grpc if [[ "${DOCKER_GRPC_VERSION}" == "${grpc_version}" ]]; then - # if the specified protoc_version matches the one baked in the docker + # if the specified grpc_version matches the one baked in the docker # container, we just point grpc_path to its location. export grpc_path="${DOCKER_GRPC_LOCATION}" else From 034edf985b1e7fc6076984712f4ffb2de01732db Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Mon, 22 Jul 2024 20:17:48 +0000 Subject: [PATCH 4/6] add grpc to renovate.json --- renovate.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/renovate.json b/renovate.json index 473aa6f717..1ea28fa0fe 100644 --- a/renovate.json +++ b/renovate.json @@ -41,6 +41,18 @@ "depNameTemplate": "protocolbuffers/protobuf", "extractVersionTemplate": "^v(?.*)$" }, + { + "customType": "regex", + "fileMatch": [ + "^\\.cloudbuild/library_generation/library_generation\\.Dockerfile$" + ], + "matchStrings": [ + "ARG GRPC_VERSION=[\"']?(?.+?)[\"']?\\s+" + ], + "datasourceTemplate": "github-releases", + "depNameTemplate": "grpc/grpc", + "extractVersionTemplate": "^v(?.*)$" + }, { "customType": "regex", "fileMatch": [ From 9795e23af1aeab77d6c9557000c311e21f972978 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Mon, 22 Jul 2024 20:30:17 +0000 Subject: [PATCH 5/6] add GRPC_VERSION to renovate config --- renovate.json | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/renovate.json b/renovate.json index 1ea28fa0fe..f0ad762025 100644 --- a/renovate.json +++ b/renovate.json @@ -32,7 +32,7 @@ { "customType": "regex", "fileMatch": [ - "^\\.cloudbuild/library_generation/library_generation\\.Dockerfile$" + "^\\.cloudbuild/library_generation/library_generation\\.Dockerfile$" ], "matchStrings": [ "ARG PROTOC_VERSION=[\"']?(?.+?)[\"']?\\s+" @@ -41,18 +41,6 @@ "depNameTemplate": "protocolbuffers/protobuf", "extractVersionTemplate": "^v(?.*)$" }, - { - "customType": "regex", - "fileMatch": [ - "^\\.cloudbuild/library_generation/library_generation\\.Dockerfile$" - ], - "matchStrings": [ - "ARG GRPC_VERSION=[\"']?(?.+?)[\"']?\\s+" - ], - "datasourceTemplate": "github-releases", - "depNameTemplate": "grpc/grpc", - "extractVersionTemplate": "^v(?.*)$" - }, { "customType": "regex", "fileMatch": [ @@ -67,10 +55,12 @@ { "customType": "regex", "fileMatch": [ - "^gax-java/dependencies\\.properties$" + "^gax-java/dependencies\\.properties$", + "^\\.cloudbuild/library_generation/library_generation\\.Dockerfile$" ], "matchStrings": [ - "version\\.io_grpc=(?.+?)\\n" + "version\\.io_grpc=(?.+?)\\n", + "ARG GRPC_VERSION=[\"']?(?.+?)[\"']?\\s+" ], "depNameTemplate": "io.grpc:grpc-core", "datasourceTemplate": "maven" From bfac93f955c83609d36c922807cb2128d2431ca3 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Thu, 25 Jul 2024 15:59:44 +0000 Subject: [PATCH 6/6] update grpc to 1.65.1 --- .cloudbuild/library_generation/library_generation.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cloudbuild/library_generation/library_generation.Dockerfile b/.cloudbuild/library_generation/library_generation.Dockerfile index 63385f136a..fd10cd7bee 100644 --- a/.cloudbuild/library_generation/library_generation.Dockerfile +++ b/.cloudbuild/library_generation/library_generation.Dockerfile @@ -19,7 +19,7 @@ SHELL [ "/bin/bash", "-c" ] ARG OWLBOT_CLI_COMMITTISH=ac84fa5c423a0069bbce3d2d869c9730c8fdf550 ARG PROTOC_VERSION=25.3 -ARG GRPC_VERSION=1.62.2 +ARG GRPC_VERSION=1.65.1 ENV HOME=/home ENV OS_ARCHITECTURE="linux-x86_64"