From 4bbb8ac916fa9737fe5ab8e00b6f4953d780c2cb Mon Sep 17 00:00:00 2001 From: Diego Marquez Date: Thu, 25 Jul 2024 17:26:03 -0400 Subject: [PATCH] chore: bake the grpc plugin in the hermetic library generation image (#3045) This PR bakes the gRPC plugin in the hermetic library generation image. The approach is very similar to https://github.com/googleapis/sdk-platform-java/pull/2707 * The docker image will have a configurable `ARG` for the gRPC plugin * The docker image will download the gRPC plugin at the version specified in the previous point * The docker image will set the variables DOCKER_GRPC_VERSION and DOCKER_GRPC_LOCATION * `generate_library.sh` will now be able to avoid downloading the gRPC plugin if the computed plugin version is the same as DOCKER_GRPC_VERSION (all this via `utilities.sh`). --------- Co-authored-by: Blake Li --- .../library_generation.Dockerfile | 12 ++++- library_generation/generate_library.sh | 4 +- .../test/generate_library_unit_tests.sh | 43 ++++++++++++++++++ library_generation/utils/utilities.sh | 45 ++++++++++++++++--- renovate.json | 8 ++-- 5 files changed, 101 insertions(+), 11 deletions(-) diff --git a/.cloudbuild/library_generation/library_generation.Dockerfile b/.cloudbuild/library_generation/library_generation.Dockerfile index 2629c092aa..fd10cd7bee 100644 --- a/.cloudbuild/library_generation/library_generation.Dockerfile +++ b/.cloudbuild/library_generation/library_generation.Dockerfile @@ -19,7 +19,9 @@ SHELL [ "/bin/bash", "-c" ] ARG OWLBOT_CLI_COMMITTISH=ac84fa5c423a0069bbce3d2d869c9730c8fdf550 ARG PROTOC_VERSION=25.3 +ARG GRPC_VERSION=1.65.1 ENV HOME=/home +ENV OS_ARCHITECTURE="linux-x86_64" # install OS tools RUN apt-get update && apt-get install -y \ @@ -32,11 +34,19 @@ COPY library_generation /src # install protoc WORKDIR /protoc RUN source /src/utils/utilities.sh \ - && download_protoc "${PROTOC_VERSION}" "linux-x86_64" + && download_protoc "${PROTOC_VERSION}" "${OS_ARCHITECTURE}" # we indicate protoc is available in the container via env vars ENV DOCKER_PROTOC_LOCATION=/protoc ENV DOCKER_PROTOC_VERSION="${PROTOC_VERSION}" +# install grpc +WORKDIR /grpc +RUN source /src/utils/utilities.sh \ + && download_grpc_plugin "${GRPC_VERSION}" "${OS_ARCHITECTURE}" +# similar to protoc, we indicate grpc is available in the container via env vars +ENV DOCKER_GRPC_LOCATION="/grpc/protoc-gen-grpc-java-${GRPC_VERSION}-${OS_ARCHITECTURE}.exe" +ENV DOCKER_GRPC_VERSION="${GRPC_VERSION}" + # use python 3.11 (the base image has several python versions; here we define the default one) RUN rm $(which python3) RUN ln -s $(which python3.11) /usr/local/bin/python diff --git a/library_generation/generate_library.sh b/library_generation/generate_library.sh index ecf8c64a50..1b113ba11b 100755 --- a/library_generation/generate_library.sh +++ b/library_generation/generate_library.sh @@ -183,13 +183,15 @@ case "${proto_path}" in ;; esac # download gapic-generator-java, protobuf and grpc plugin. +# the download_tools function will create the environment variables "protoc_path" +# and "grpc_path", to be used in the protoc calls below. download_tools "${gapic_generator_version}" "${protoc_version}" "${grpc_version}" "${os_architecture}" ##################### Section 1 ##################### # generate grpc-*/ ##################################################### if [[ ! "${transport}" == "rest" ]]; then # do not need to generate grpc-* if the transport is `rest`. - "${protoc_path}"/protoc "--plugin=protoc-gen-rpc-plugin=protoc-gen-grpc-java-${grpc_version}-${os_architecture}.exe" \ + "${protoc_path}"/protoc "--plugin=protoc-gen-rpc-plugin=${grpc_path}" \ "--rpc-plugin_out=:${temp_destination_path}/java_grpc.jar" \ ${proto_files} # Do not quote because this variable should not be treated as one long string. # unzip java_grpc.jar to grpc-*/src/main/java diff --git a/library_generation/test/generate_library_unit_tests.sh b/library_generation/test/generate_library_unit_tests.sh index 5384d6982c..0261a041c3 100755 --- a/library_generation/test/generate_library_unit_tests.sh +++ b/library_generation/test/generate_library_unit_tests.sh @@ -28,10 +28,21 @@ get_grpc_version_failed_with_invalid_generator_version_test() { assertEquals 1 $((res)) } +get_grpc_version_succeed_docker_env_var_test() { + local version_with_docker + local version_without_docker + export DOCKER_GRPC_VERSION="9.9.9" + # get_grpc_version should prioritize DOCKER_GRPC_VERSION + version_with_docker=$(get_grpc_version "2.24.0") + assertEquals "${DOCKER_GRPC_VERSION}" "${version_with_docker}" + unset DOCKER_GRPC_VERSION +} + get_protoc_version_succeed_docker_env_var_test() { local version_with_docker local version_without_docker export DOCKER_PROTOC_VERSION="9.9.9" + # get_protoc_version should prioritize DOCKER_PROTOC_VERSION version_with_docker=$(get_protoc_version "2.24.0") assertEquals "${DOCKER_PROTOC_VERSION}" "${version_with_docker}" unset DOCKER_PROTOC_VERSION @@ -162,6 +173,11 @@ download_tools_succeed_with_baked_protoc() { local test_ggj_version="2.40.0" local test_grpc_version="1.64.0" + # we expect download_tools to decide to use DOCKER_PROTOC_LOCATION because + # the protoc version we want to download is the same as DOCKER_PROTOC_VERSION. + # Note that `protoc_bin_folder` is just the expected formatted value that + # download_tools will format using DOCKER_PROTOC_VERSION (via + # download_protoc). download_tools "${test_ggj_version}" "99.99" "${test_grpc_version}" "linux-x86_64" assertEquals "${protoc_bin_folder}" "${protoc_path}" @@ -169,6 +185,31 @@ download_tools_succeed_with_baked_protoc() { unset DOCKER_PROTOC_LOCATION unset DOCKER_PROTOC_VERSION unset output_folder + unset protoc_path +} + +download_tools_succeed_with_baked_grpc() { + # This test has the same structure as + # download_tools_succeed_with_baked_protoc, but meant for the grpc plugin. + local test_dir=$(mktemp -d) + pushd "${test_dir}" + export DOCKER_GRPC_LOCATION=$(mktemp -d) + export DOCKER_GRPC_VERSION="99.99" + export output_folder=$(get_output_folder) + mkdir "${output_folder}" + + local test_ggj_version="2.40.0" + local test_protoc_version="1.64.0" + # we expect download_tools to decide to use DOCKER_GRPC_LOCATION because + # the protoc version we want to download is the same as DOCKER_GRPC_VERSION + download_tools "${test_ggj_version}" "${test_protoc_version}" "99.99" "linux-x86_64" + assertEquals "${DOCKER_GRPC_LOCATION}" "${grpc_path}" + + rm -rdf "${output_folder}" + unset DOCKER_GRPC_LOCATION + unset DOCKER_GRPC_VERSION + unset output_folder + unset grpc_path } download_grpc_plugin_succeed_with_valid_version_linux_test() { @@ -293,6 +334,7 @@ test_list=( extract_folder_name_test get_grpc_version_succeed_with_valid_generator_version_test get_grpc_version_failed_with_invalid_generator_version_test + get_grpc_version_succeed_docker_env_var_test get_protoc_version_succeed_docker_env_var_test get_protoc_version_succeed_with_valid_generator_version_test get_protoc_version_failed_with_invalid_generator_version_test @@ -307,6 +349,7 @@ test_list=( download_protoc_failed_with_invalid_version_linux_test download_protoc_failed_with_invalid_arch_test download_tools_succeed_with_baked_protoc + download_tools_succeed_with_baked_grpc download_grpc_plugin_succeed_with_valid_version_linux_test download_grpc_plugin_succeed_with_valid_version_macos_test download_grpc_plugin_failed_with_invalid_version_linux_test diff --git a/library_generation/utils/utilities.sh b/library_generation/utils/utilities.sh index 2e26ed6027..72fefec76a 100755 --- a/library_generation/utils/utilities.sh +++ b/library_generation/utils/utilities.sh @@ -102,9 +102,18 @@ download_gapic_generator_pom_parent() { download_generator_artifact "${gapic_generator_version}" "gapic-generator-java-pom-parent-${gapic_generator_version}.pom" "gapic-generator-java-pom-parent" } +# This function returns the version of the grpc plugin to generate the libraries. If +# DOCKER_GRPC_VERSION is set, this will be the version. Otherwise, it will be +# computed from the gapic-generator-pom-parent artifact at the specified +# gapic_generator_version. get_grpc_version() { local gapic_generator_version=$1 local grpc_version + if [[ -n "${DOCKER_GRPC_VERSION}" ]]; then + >&2 echo "Using grpc version baked into the container: ${DOCKER_GRPC_VERSION}" + echo "${DOCKER_GRPC_VERSION}" + return + fi pushd "${output_folder}" > /dev/null # get grpc version from gapic-generator-java-pom-parent/pom.xml download_gapic_generator_pom_parent "${gapic_generator_version}" @@ -113,6 +122,10 @@ get_grpc_version() { echo "${grpc_version}" } +# This function returns the version of protoc to generate the libraries. If +# DOCKER_PROTOC_VERSION is set, this will be the version. Otherwise, it will be +# computed from the gapic-generator-pom-parent artifact at the specified +# gapic_generator_version. get_protoc_version() { local gapic_generator_version=$1 local protoc_version @@ -129,6 +142,16 @@ get_protoc_version() { echo "${protoc_version}" } +# Given the versions of the gapic generator, protoc and the protoc-grpc plugin, +# this function will download each one of the tools and create the environment +# variables "protoc_path" and "grpc_path" which are expected upstream. Note that +# if the specified versions of protoc and grpc match DOCKER_PROTOC_VERSION and +# DOCKER_GRPC_VERSION respectively, this function will instead set "protoc_path" +# and "grpc_path" to DOCKER_PROTOC_PATH and DOCKER_GRPC_PATH respectively (no +# download), since the docker image will have downloaded these tools beforehand. +# For the case of gapic-generator-java, no env var will be exported for the +# upstream flow, but instead it will be assigned a default filename that will be +# referenced by the file `library_generation/gapic-generator-java-wrapper`. download_tools() { local gapic_generator_version=$1 local protoc_version=$2 @@ -147,7 +170,15 @@ download_tools() { export protoc_path=$(download_protoc "${protoc_version}" "${os_architecture}") fi - download_grpc_plugin "${grpc_version}" "${os_architecture}" + # similar case with grpc + if [[ "${DOCKER_GRPC_VERSION}" == "${grpc_version}" ]]; then + # if the specified grpc_version matches the one baked in the docker + # container, we just point grpc_path to its location. + export grpc_path="${DOCKER_GRPC_LOCATION}" + else + export grpc_path=$(download_grpc_plugin "${grpc_version}" "${os_architecture}") + fi + popd } @@ -198,13 +229,15 @@ download_protoc() { download_grpc_plugin() { local grpc_version=$1 local os_architecture=$2 - if [ ! -f "protoc-gen-grpc-java-${grpc_version}-${os_architecture}.exe" ]; then + grpc_filename="protoc-gen-grpc-java-${grpc_version}-${os_architecture}.exe" + if [ ! -f "${grpc_filename}" ]; then # download protoc-gen-grpc-java plugin from Google maven central mirror. download_from \ - "https://maven-central.storage-download.googleapis.com/maven2/io/grpc/protoc-gen-grpc-java/${grpc_version}/protoc-gen-grpc-java-${grpc_version}-${os_architecture}.exe" \ - "protoc-gen-grpc-java-${grpc_version}-${os_architecture}.exe" - chmod +x "protoc-gen-grpc-java-${grpc_version}-${os_architecture}.exe" + "https://maven-central.storage-download.googleapis.com/maven2/io/grpc/protoc-gen-grpc-java/${grpc_version}/${grpc_filename}" \ + "${grpc_filename}" + chmod +x "${grpc_filename}" fi + echo "$(pwd)/${grpc_filename}" } download_from() { @@ -282,7 +315,7 @@ get_proto_path_from_preprocessed_sources() { pushd "${sources}" > /dev/null local proto_library=$(find . -maxdepth 1 -type d -name 'proto-*' | sed 's/\.\///') local found_libraries=$(echo "${proto_library}" | wc -l) - if [ -z ${proto_library} ]; then + if [[ -z ${proto_library} ]]; then echo "no proto libraries found in the supplied sources path" exit 1 elif [ ${found_libraries} -gt 1 ]; then diff --git a/renovate.json b/renovate.json index 473aa6f717..f0ad762025 100644 --- a/renovate.json +++ b/renovate.json @@ -32,7 +32,7 @@ { "customType": "regex", "fileMatch": [ - "^\\.cloudbuild/library_generation/library_generation\\.Dockerfile$" + "^\\.cloudbuild/library_generation/library_generation\\.Dockerfile$" ], "matchStrings": [ "ARG PROTOC_VERSION=[\"']?(?.+?)[\"']?\\s+" @@ -55,10 +55,12 @@ { "customType": "regex", "fileMatch": [ - "^gax-java/dependencies\\.properties$" + "^gax-java/dependencies\\.properties$", + "^\\.cloudbuild/library_generation/library_generation\\.Dockerfile$" ], "matchStrings": [ - "version\\.io_grpc=(?.+?)\\n" + "version\\.io_grpc=(?.+?)\\n", + "ARG GRPC_VERSION=[\"']?(?.+?)[\"']?\\s+" ], "depNameTemplate": "io.grpc:grpc-core", "datasourceTemplate": "maven"