From a94c2f0e8a99f0ddf17106cbc8117cefe6b0e127 Mon Sep 17 00:00:00 2001 From: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> Date: Thu, 21 Mar 2024 00:56:54 +0000 Subject: [PATCH] feat: add generation config comparator (#2587) In this PR: - Add a generation config comparator to compare the change type and which libraries are affected by the change. - Refactor `LibraryConfig`. - Add unit tests (no integration tests because the comparator has not been used). The comparator makes assumptions that the following library level parameters will not change: - googleapis commit (no use case) This is the first step to [improve](https://docs.google.com/document/d/1JiCcG3X7lnxaJErKe0ES_JkyU7ECb40nf2Xez3gWvuo/edit?tab=t.g3vua2kd06gx#heading=h.pygigzqg78jp) performance of hermetic code generation. --- library_generation/model/library_config.py | 68 +++ library_generation/test/integration_tests.py | 10 +- library_generation/test/model/__init__.py | 0 .../test/model/library_config_unit_tests.py | 39 ++ .../test/utilities_unit_tests.py | 6 - ...generation_config_comparator_unit_tests.py | 457 ++++++++++++++++++ library_generation/utilities.py | 15 +- .../utils/generation_config_comparator.py | 297 ++++++++++++ 8 files changed, 868 insertions(+), 24 deletions(-) create mode 100644 library_generation/test/model/__init__.py create mode 100644 library_generation/test/model/library_config_unit_tests.py create mode 100644 library_generation/test/utils/generation_config_comparator_unit_tests.py create mode 100644 library_generation/utils/generation_config_comparator.py diff --git a/library_generation/model/library_config.py b/library_generation/model/library_config.py index b6f8ae1b48..30d923da81 100644 --- a/library_generation/model/library_config.py +++ b/library_generation/model/library_config.py @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from hashlib import sha1 from typing import List, Optional from library_generation.model.gapic_config import GapicConfig @@ -71,3 +72,70 @@ def __init__( self.cloud_api = cloud_api self.requires_billing = requires_billing self.extra_versioned_modules = extra_versioned_modules + + def get_library_name(self) -> str: + """ + Return the library name of a given LibraryConfig object + :return: the library name + """ + return self.library_name if self.library_name else self.api_shortname + + def __eq__(self, other): + return ( + self.api_shortname == other.api_shortname + and self.api_description == other.api_description + and self.name_pretty == other.name_pretty + and self.product_documentation == other.product_documentation + and self.gapic_configs == other.gapic_configs + and self.library_type == other.library_type + and self.release_level == other.release_level + and self.api_id == other.api_id + and self.api_reference == other.api_reference + and self.codeowner_team == other.codeowner_team + and self.excluded_dependencies == other.excluded_dependencies + and self.excluded_poms == other.excluded_poms + and self.client_documentation == other.client_documentation + and self.distribution_name == other.distribution_name + and self.googleapis_commitish == other.googleapis_commitish + and self.group_id == other.group_id + and self.issue_tracker == other.issue_tracker + and self.library_name == other.library_name + and self.rest_documentation == other.rest_documentation + and self.rpc_documentation == other.rpc_documentation + and self.cloud_api == other.cloud_api + and self.requires_billing == other.requires_billing + and self.extra_versioned_modules == other.extra_versioned_modules + ) + + def __hash__(self): + m = sha1() + m.update( + str( + [ + self.api_shortname, + self.api_description, + self.name_pretty, + self.product_documentation, + self.library_type, + self.release_level, + self.api_id, + self.api_reference, + self.codeowner_team, + self.excluded_dependencies, + self.excluded_poms, + self.client_documentation, + self.distribution_name, + self.googleapis_commitish, + self.group_id, + self.issue_tracker, + self.library_name, + self.rest_documentation, + self.rpc_documentation, + self.cloud_api, + self.requires_billing, + self.extra_versioned_modules, + ] + + [config.proto_path for config in self.gapic_configs] + ).encode("utf-8") + ) + return int(m.hexdigest(), 16) diff --git a/library_generation/test/integration_tests.py b/library_generation/test/integration_tests.py index 3bf50d0394..35cbf565ac 100755 --- a/library_generation/test/integration_tests.py +++ b/library_generation/test/integration_tests.py @@ -29,7 +29,6 @@ from library_generation.model.generation_config import from_yaml, GenerationConfig from library_generation.test.compare_poms import compare_xml from library_generation.utilities import ( - get_library_name, sh_util as shell_call, run_process_and_print_output, ) @@ -214,7 +213,7 @@ def __pull_repo_to(cls, default_dest: Path, repo: str, committish: str) -> str: def __get_library_names_from_config(cls, config: GenerationConfig) -> List[str]: library_names = [] for library in config.libraries: - library_names.append(f"java-{get_library_name(library)}") + library_names.append(f"java-{library.get_library_name()}") return library_names @@ -248,7 +247,7 @@ def __load_json_to_sorted_list(cls, path: str) -> List[tuple]: @classmethod def __recursive_diff_files( - self, + cls, dcmp: dircmp, diff_files: List[str], left_only: List[str], @@ -256,13 +255,14 @@ def __recursive_diff_files( dirname: str = "", ): """ - recursively compares two subdirectories. The found differences are passed to three expected list references + Recursively compares two subdirectories. The found differences are + passed to three expected list references. """ append_dirname = lambda d: dirname + d diff_files.extend(map(append_dirname, dcmp.diff_files)) left_only.extend(map(append_dirname, dcmp.left_only)) right_only.extend(map(append_dirname, dcmp.right_only)) for sub_dirname, sub_dcmp in dcmp.subdirs.items(): - self.__recursive_diff_files( + cls.__recursive_diff_files( sub_dcmp, diff_files, left_only, right_only, dirname + sub_dirname + "/" ) diff --git a/library_generation/test/model/__init__.py b/library_generation/test/model/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/library_generation/test/model/library_config_unit_tests.py b/library_generation/test/model/library_config_unit_tests.py new file mode 100644 index 0000000000..1c0b0d0690 --- /dev/null +++ b/library_generation/test/model/library_config_unit_tests.py @@ -0,0 +1,39 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest + +from library_generation.model.library_config import LibraryConfig + + +class LibraryConfigTest(unittest.TestCase): + def test_get_library_returns_library_name(self): + library = LibraryConfig( + api_shortname="secret", + name_pretty="", + product_documentation="", + api_description="", + gapic_configs=list(), + library_name="secretmanager", + ) + self.assertEqual("secretmanager", library.get_library_name()) + + def test_get_library_returns_api_shortname(self): + library = LibraryConfig( + api_shortname="secret", + name_pretty="", + product_documentation="", + api_description="", + gapic_configs=list(), + ) + self.assertEqual("secret", library.get_library_name()) diff --git a/library_generation/test/utilities_unit_tests.py b/library_generation/test/utilities_unit_tests.py index aa6d99ac24..3d114f977f 100644 --- a/library_generation/test/utilities_unit_tests.py +++ b/library_generation/test/utilities_unit_tests.py @@ -391,12 +391,6 @@ def test_remove_version_from_returns_self(self): "google/cloud/aiplatform", util.remove_version_from(proto_path) ) - def test_get_library_returns_library_name(self): - self.assertEqual("bare-metal-solution", util.get_library_name(library_1)) - - def test_get_library_returns_api_shortname(self): - self.assertEqual("secretmanager", util.get_library_name(library_2)) - def test_generate_prerequisite_files_non_monorepo_success(self): library_path = self.__setup_prerequisite_files( num_libraries=1, library_type="GAPIC_COMBO" diff --git a/library_generation/test/utils/generation_config_comparator_unit_tests.py b/library_generation/test/utils/generation_config_comparator_unit_tests.py new file mode 100644 index 0000000000..c2c025c579 --- /dev/null +++ b/library_generation/test/utils/generation_config_comparator_unit_tests.py @@ -0,0 +1,457 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest + +from library_generation.model.gapic_config import GapicConfig +from library_generation.model.generation_config import GenerationConfig +from library_generation.model.library_config import LibraryConfig +from library_generation.utils.generation_config_comparator import ChangeType +from library_generation.utils.generation_config_comparator import compare_config + + +class GenerationConfigComparatorTest(unittest.TestCase): + def setUp(self) -> None: + self.baseline_library = LibraryConfig( + api_shortname="existing_library", + api_description="", + name_pretty="", + product_documentation="", + gapic_configs=[], + ) + self.latest_library = LibraryConfig( + api_shortname="existing_library", + api_description="", + name_pretty="", + product_documentation="", + gapic_configs=[], + ) + self.baseline_config = GenerationConfig( + gapic_generator_version="", + googleapis_commitish="", + owlbot_cli_image="", + synthtool_commitish="", + template_excludes=[], + path_to_yaml="", + grpc_version="", + protobuf_version="", + libraries=[self.baseline_library], + ) + self.latest_config = GenerationConfig( + gapic_generator_version="", + googleapis_commitish="", + owlbot_cli_image="", + synthtool_commitish="", + template_excludes=[], + path_to_yaml="", + grpc_version="", + protobuf_version="", + libraries=[self.latest_library], + ) + + def test_compare_config_not_change(self): + result = compare_config( + baseline_config=self.baseline_config, + latest_config=self.latest_config, + ) + self.assertTrue(len(result) == 0) + + def test_compare_config_googleapis_update(self): + self.baseline_config.googleapis_commitish = ( + "1a45bf7393b52407188c82e63101db7dc9c72026" + ) + self.latest_config.googleapis_commitish = ( + "1e6517ef4f949191c9e471857cf5811c8abcab84" + ) + result = compare_config( + baseline_config=self.baseline_config, + latest_config=self.latest_config, + ) + self.assertEqual({ChangeType.GOOGLEAPIS_COMMIT: []}, result) + + def test_compare_config_generator_update(self): + self.baseline_config.gapic_generator_version = "1.2.3" + self.latest_config.gapic_generator_version = "1.2.4" + result = compare_config( + baseline_config=self.baseline_config, + latest_config=self.latest_config, + ) + self.assertTrue(len(result[ChangeType.REPO_LEVEL_CHANGE]) == 1) + config_change = result[ChangeType.REPO_LEVEL_CHANGE][0] + self.assertEqual("gapic_generator_version", config_change.changed_param) + self.assertEqual("1.2.4", config_change.latest_value) + + def test_compare_config_owlbot_cli_update(self): + self.baseline_config.owlbot_cli_image = "image_version_123" + self.latest_config.owlbot_cli_image = "image_version_456" + result = compare_config( + baseline_config=self.baseline_config, + latest_config=self.latest_config, + ) + self.assertTrue(len(result[ChangeType.REPO_LEVEL_CHANGE]) == 1) + config_change = result[ChangeType.REPO_LEVEL_CHANGE][0] + self.assertEqual("owlbot_cli_image", config_change.changed_param) + self.assertEqual("image_version_456", config_change.latest_value) + + def test_compare_config_synthtool_update(self): + self.baseline_config.synthtool_commitish = "commit123" + self.latest_config.synthtool_commitish = "commit456" + result = compare_config( + baseline_config=self.baseline_config, + latest_config=self.latest_config, + ) + self.assertTrue(len(result[ChangeType.REPO_LEVEL_CHANGE]) == 1) + config_change = result[ChangeType.REPO_LEVEL_CHANGE][0] + self.assertEqual("synthtool_commitish", config_change.changed_param) + self.assertEqual("commit456", config_change.latest_value) + + def test_compare_protobuf_update(self): + self.baseline_config.protobuf_version = "3.25.2" + self.latest_config.protobuf_version = "3.27.0" + result = compare_config( + baseline_config=self.baseline_config, + latest_config=self.latest_config, + ) + self.assertTrue(len(result[ChangeType.REPO_LEVEL_CHANGE]) == 1) + config_change = result[ChangeType.REPO_LEVEL_CHANGE][0] + self.assertEqual("protobuf_version", config_change.changed_param) + self.assertEqual("3.27.0", config_change.latest_value) + + def test_compare_config_grpc_update(self): + self.baseline_config.grpc_version = "1.60.0" + self.latest_config.grpc_version = "1.61.0" + result = compare_config( + baseline_config=self.baseline_config, + latest_config=self.latest_config, + ) + self.assertTrue(len(result[ChangeType.REPO_LEVEL_CHANGE]) == 1) + config_change = result[ChangeType.REPO_LEVEL_CHANGE][0] + self.assertEqual("grpc_version", config_change.changed_param) + self.assertEqual("1.61.0", config_change.latest_value) + + def test_compare_config_template_excludes_update(self): + self.baseline_config.template_excludes = [".github/*", ".kokoro/*"] + self.latest_config.template_excludes = [ + ".github/*", + ".kokoro/*", + "samples/*", + "CODE_OF_CONDUCT.md", + ] + result = compare_config( + baseline_config=self.baseline_config, + latest_config=self.latest_config, + ) + self.assertTrue(len(result[ChangeType.REPO_LEVEL_CHANGE]) == 1) + config_change = result[ChangeType.REPO_LEVEL_CHANGE][0] + self.assertEqual("template_excludes", config_change.changed_param) + self.assertEqual( + [ + ".github/*", + ".kokoro/*", + "samples/*", + "CODE_OF_CONDUCT.md", + ], + config_change.latest_value, + ) + + def test_compare_config_library_addition(self): + self.latest_config.libraries.append( + LibraryConfig( + api_shortname="new_library", + api_description="", + name_pretty="", + product_documentation="", + gapic_configs=[], + ) + ) + result = compare_config( + baseline_config=self.baseline_config, + latest_config=self.latest_config, + ) + self.assertTrue(len(result[ChangeType.LIBRARIES_ADDITION]) == 1) + config_change = result[ChangeType.LIBRARIES_ADDITION][0] + self.assertEqual("new_library", config_change.library_name) + + def test_compare_config_api_shortname_update_without_library_name(self): + self.latest_config.libraries[0].api_shortname = "new_api_shortname" + result = compare_config( + baseline_config=self.baseline_config, + latest_config=self.latest_config, + ) + self.assertTrue(len(result[ChangeType.LIBRARIES_ADDITION]) == 1) + config_change = result[ChangeType.LIBRARIES_ADDITION][0] + self.assertEqual("new_api_shortname", config_change.library_name) + + def test_compare_config_api_shortname_update_with_library_name_raise_error(self): + self.baseline_config.libraries[0].library_name = "old_library_name" + self.latest_config.libraries[0].library_name = "old_library_name" + self.latest_config.libraries[0].api_shortname = "new_api_shortname" + self.assertRaisesRegex( + ValueError, + r"api_shortname.*library_name", + compare_config, + self.baseline_config, + self.latest_config, + ) + + def test_compare_config_library_name_update(self): + self.latest_config.libraries[0].library_name = "new_library_name" + result = compare_config( + baseline_config=self.baseline_config, + latest_config=self.latest_config, + ) + self.assertTrue(len(result[ChangeType.LIBRARIES_ADDITION]) == 1) + config_change = result[ChangeType.LIBRARIES_ADDITION][0] + self.assertEqual("new_library_name", config_change.library_name) + + def test_compare_config_api_description_update(self): + self.latest_config.libraries[0].api_description = "updated description" + result = compare_config( + baseline_config=self.baseline_config, + latest_config=self.latest_config, + ) + self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) + config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("api_description", config_change.changed_param) + self.assertEqual("updated description", config_change.latest_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_name_pretty_update(self): + self.latest_config.libraries[0].name_pretty = "new name" + result = compare_config( + baseline_config=self.baseline_config, + latest_config=self.latest_config, + ) + self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) + config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("name_pretty", config_change.changed_param) + self.assertEqual("new name", config_change.latest_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_product_docs_update(self): + self.latest_config.libraries[0].product_documentation = "new docs" + result = compare_config( + baseline_config=self.baseline_config, + latest_config=self.latest_config, + ) + self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) + config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("product_documentation", config_change.changed_param) + self.assertEqual("new docs", config_change.latest_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_library_type_update(self): + self.latest_config.libraries[0].library_type = "GAPIC_COMBO" + result = compare_config( + baseline_config=self.baseline_config, + latest_config=self.latest_config, + ) + self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) + config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("library_type", config_change.changed_param) + self.assertEqual("GAPIC_COMBO", config_change.latest_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_release_level_update(self): + self.latest_config.libraries[0].release_level = "STABLE" + result = compare_config( + baseline_config=self.baseline_config, + latest_config=self.latest_config, + ) + self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) + config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("release_level", config_change.changed_param) + self.assertEqual("STABLE", config_change.latest_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_api_id_update(self): + self.latest_config.libraries[0].api_id = "new_id" + result = compare_config( + baseline_config=self.baseline_config, + latest_config=self.latest_config, + ) + self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) + config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("api_id", config_change.changed_param) + self.assertEqual("new_id", config_change.latest_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_api_reference_update(self): + self.latest_config.libraries[0].api_reference = "new api_reference" + result = compare_config( + baseline_config=self.baseline_config, + latest_config=self.latest_config, + ) + self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) + config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("api_reference", config_change.changed_param) + self.assertEqual("new api_reference", config_change.latest_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_code_owner_team_update(self): + self.latest_config.libraries[0].codeowner_team = "new team" + result = compare_config( + baseline_config=self.baseline_config, + latest_config=self.latest_config, + ) + self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) + config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("codeowner_team", config_change.changed_param) + self.assertEqual("new team", config_change.latest_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_excluded_deps_update(self): + self.latest_config.libraries[0].excluded_dependencies = "group:artifact" + result = compare_config( + baseline_config=self.baseline_config, + latest_config=self.latest_config, + ) + self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) + config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("excluded_dependencies", config_change.changed_param) + self.assertEqual("group:artifact", config_change.latest_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_excluded_poms_update(self): + self.latest_config.libraries[0].excluded_poms = "pom.xml" + result = compare_config( + baseline_config=self.baseline_config, + latest_config=self.latest_config, + ) + self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) + config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("excluded_poms", config_change.changed_param) + self.assertEqual("pom.xml", config_change.latest_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_client_docs_update(self): + self.latest_config.libraries[0].client_documentation = "new client docs" + result = compare_config( + baseline_config=self.baseline_config, + latest_config=self.latest_config, + ) + self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) + config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("client_documentation", config_change.changed_param) + self.assertEqual("new client docs", config_change.latest_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_distribution_name_update(self): + self.latest_config.libraries[0].distribution_name = "new_group:new_artifact" + result = compare_config( + baseline_config=self.baseline_config, + latest_config=self.latest_config, + ) + self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) + config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("distribution_name", config_change.changed_param) + self.assertEqual("new_group:new_artifact", config_change.latest_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_group_id_update(self): + self.latest_config.libraries[0].group_id = "new_group" + result = compare_config( + baseline_config=self.baseline_config, + latest_config=self.latest_config, + ) + self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) + config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("group_id", config_change.changed_param) + self.assertEqual("new_group", config_change.latest_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_issue_tracker_update(self): + self.latest_config.libraries[0].issue_tracker = "new issue tracker" + result = compare_config( + baseline_config=self.baseline_config, + latest_config=self.latest_config, + ) + self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) + config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("issue_tracker", config_change.changed_param) + self.assertEqual("new issue tracker", config_change.latest_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_rest_docs_update(self): + self.latest_config.libraries[0].rest_documentation = "new rest docs" + result = compare_config( + baseline_config=self.baseline_config, + latest_config=self.latest_config, + ) + self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) + config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("rest_documentation", config_change.changed_param) + self.assertEqual("new rest docs", config_change.latest_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_rpc_docs_update(self): + self.latest_config.libraries[0].rpc_documentation = "new rpc docs" + result = compare_config( + baseline_config=self.baseline_config, + latest_config=self.latest_config, + ) + self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) + config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("rpc_documentation", config_change.changed_param) + self.assertEqual("new rpc docs", config_change.latest_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_cloud_api_update(self): + self.latest_config.libraries[0].cloud_api = False + result = compare_config( + baseline_config=self.baseline_config, + latest_config=self.latest_config, + ) + self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) + config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("cloud_api", config_change.changed_param) + self.assertEqual(False, config_change.latest_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_requires_billing_update(self): + self.latest_config.libraries[0].requires_billing = False + result = compare_config( + baseline_config=self.baseline_config, + latest_config=self.latest_config, + ) + self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) + config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("requires_billing", config_change.changed_param) + self.assertEqual(False, config_change.latest_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_extra_versioned_mod_update(self): + self.latest_config.libraries[0].extra_versioned_modules = "extra module" + result = compare_config( + baseline_config=self.baseline_config, + latest_config=self.latest_config, + ) + self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) + config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("extra_versioned_modules", config_change.changed_param) + self.assertEqual("extra module", config_change.latest_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_version_addition(self): + self.latest_config.libraries[0].gapic_configs = [ + GapicConfig(proto_path="google/new/library/v1") + ] + result = compare_config( + baseline_config=self.baseline_config, + latest_config=self.latest_config, + ) + self.assertTrue(len(result[ChangeType.GAPIC_ADDITION]) == 1) + config_change = result[ChangeType.GAPIC_ADDITION][0] + self.assertEqual("", config_change.changed_param) + self.assertEqual("google/new/library/v1", config_change.latest_value) + self.assertEqual("existing_library", config_change.library_name) diff --git a/library_generation/utilities.py b/library_generation/utilities.py index 29e175c1b8..de3dd6ce44 100755 --- a/library_generation/utilities.py +++ b/library_generation/utilities.py @@ -39,17 +39,6 @@ def create_argument(arg_key: str, arg_container: object) -> List[str]: return [] -def get_library_name( - library: LibraryConfig, -) -> str: - """ - Return the library name of a given LibraryConfig object - :param library: an object of LibraryConfig - :return: the library name - """ - return library.library_name if library.library_name else library.api_shortname - - def run_process_and_print_output(arguments: List[str], job_name: str = "Job"): """ Runs a process with the given "arguments" list and prints its output. @@ -231,7 +220,7 @@ def generate_prerequisite_files( :return: None """ cloud_prefix = "cloud-" if library.cloud_api else "" - library_name = get_library_name(library) + library_name = library.get_library_name() distribution_name = ( library.distribution_name if library.distribution_name @@ -341,7 +330,7 @@ def get_file_paths(config: GenerationConfig) -> Dict[str, str]: paths = {} for library in config.libraries: for gapic_config in library.gapic_configs: - paths[gapic_config.proto_path] = get_library_name(library) + paths[gapic_config.proto_path] = library.get_library_name() return paths diff --git a/library_generation/utils/generation_config_comparator.py b/library_generation/utils/generation_config_comparator.py new file mode 100644 index 0000000000..4fe8358230 --- /dev/null +++ b/library_generation/utils/generation_config_comparator.py @@ -0,0 +1,297 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from collections import defaultdict +from enum import Enum +from typing import Any + +from typing import Dict +from typing import List + +from library_generation.model.gapic_config import GapicConfig +from library_generation.model.generation_config import GenerationConfig +from library_generation.model.library_config import LibraryConfig + + +class ChangeType(Enum): + GOOGLEAPIS_COMMIT = 1 + REPO_LEVEL_CHANGE = 2 + LIBRARIES_ADDITION = 3 + # As of Mar. 2024, we decide not to produce this type of change because we + # still need to manually remove the libray. + # LIBRARIES_REMOVAL = 4 + LIBRARY_LEVEL_CHANGE = 5 + GAPIC_ADDITION = 6 + # As of Mar. 2024, we decide not to produce this type of change because we + # still need to manually remove the libray. + # GAPIC_REMOVAL = 7 + + +class HashLibrary: + """ + Data class to group a LibraryConfig object and its hash value together. + """ + + def __init__(self, hash_value: int, library: LibraryConfig): + self.hash_value = hash_value + self.library = library + + +class ConfigChange: + def __init__(self, changed_param: str, latest_value: str, library_name: str = ""): + self.changed_param = changed_param + self.latest_value = latest_value + self.library_name = library_name + + +def compare_config( + baseline_config: GenerationConfig, latest_config: GenerationConfig +) -> Dict[ChangeType, list[ConfigChange]]: + """ + Compare two GenerationConfig object and output a mapping from ConfigChange + to a list of ConfigChange objects. + All libraries in the latest configuration will be affected if one of the + repository level parameters is changed. + + :param baseline_config: the baseline GenerationConfig object + :param latest_config: the latest GenerationConfig object + :return: a mapping from ConfigChange to a list of ConfigChange objects. + """ + diff = defaultdict(list[ConfigChange]) + baseline_params = __convert_params_to_sorted_list(baseline_config) + latest_params = __convert_params_to_sorted_list(latest_config) + for baseline_param, latest_param in zip(baseline_params, latest_params): + if baseline_param == latest_param: + continue + if baseline_param[0] == "googleapis_commitish": + diff[ChangeType.GOOGLEAPIS_COMMIT] = [] + else: + config_change = ConfigChange( + changed_param=latest_param[0], + latest_value=latest_param[1], + ) + diff[ChangeType.REPO_LEVEL_CHANGE].append(config_change) + + __compare_libraries( + diff=diff, + baseline_library_configs=baseline_config.libraries, + latest_library_configs=latest_config.libraries, + ) + return diff + + +def __compare_libraries( + diff: Dict[ChangeType, list[ConfigChange]], + baseline_library_configs: List[LibraryConfig], + latest_library_configs: List[LibraryConfig], +) -> None: + """ + Compare two lists of LibraryConfig and put the difference into a + given Dict. + + :param diff: a mapping from ConfigChange to a list of ConfigChange objects. + :param baseline_library_configs: a list of LibraryConfig object. + :param latest_library_configs: a list of LibraryConfig object. + """ + baseline_libraries = __convert_to_hashed_library_dict(baseline_library_configs) + latest_libraries = __convert_to_hashed_library_dict(latest_library_configs) + changed_libraries = [] + # 1st round comparison. + for library_name, hash_library in baseline_libraries.items(): + # 1. find any library removed from baseline_libraries. + # a library is removed from baseline_libraries if the library_name + # is not in latest_libraries. + # please see the reason of comment out these lines of code in the + # comment of ChangeType.LIBRARIES_REMOVAL. + # if library_name not in latest_libraries: + # config_change = ConfigChange( + # changed_param="", latest_value="", library_name=library_name + # ) + # diff[ChangeType.LIBRARIES_REMOVAL].append(config_change) + + # 2. find any library that exists in both configs but at least one + # parameter is changed, which means the hash value is different. + if ( + library_name in latest_libraries + and hash_library.hash_value != latest_libraries[library_name].hash_value + ): + changed_libraries.append(library_name) + # 2nd round comparison. + for library_name in latest_libraries: + # find any library added to latest_libraries. + # a library is added to latest_libraries if the library_name + # is not in baseline_libraries. + if library_name not in baseline_libraries: + config_change = ConfigChange( + changed_param="", latest_value="", library_name=library_name + ) + diff[ChangeType.LIBRARIES_ADDITION].append(config_change) + # 3rd round comparison. + __compare_changed_libraries( + diff=diff, + baseline_libraries=baseline_libraries, + latest_libraries=latest_libraries, + changed_libraries=changed_libraries, + ) + + +def __convert_to_hashed_library_dict( + libraries: List[LibraryConfig], +) -> Dict[str, HashLibrary]: + """ + Convert a list of LibraryConfig objects to a Dict. + For each library object, the key is the library_name of the object, the + value is a HashLibrary object. + + :param libraries: a list of LibraryConfig object. + :return: a mapping from library_name to HashLibrary object. + """ + return { + library.get_library_name(): HashLibrary(hash(library), library) + for library in libraries + } + + +def __compare_changed_libraries( + diff: Dict[ChangeType, list[ConfigChange]], + baseline_libraries: Dict[str, HashLibrary], + latest_libraries: Dict[str, HashLibrary], + changed_libraries: List[str], +) -> None: + """ + Compare each library with the same library_name to find what parameters are + changed. + + :param diff: a mapping from ConfigChange to a list of ConfigChange objects. + :param baseline_libraries: a mapping from library_name to HashLibrary + object. + :param latest_libraries: a mapping from library_name to HashLibrary object. + :param changed_libraries: a list of library_name of changed libraries. + :raise ValueError: if api_shortname of a library is changed but library_name + remains the same. + """ + for library_name in changed_libraries: + baseline_library = baseline_libraries[library_name].library + latest_library = latest_libraries[library_name].library + baseline_params = __convert_params_to_sorted_list(baseline_library) + latest_params = __convert_params_to_sorted_list(latest_library) + for baseline_param, latest_param in zip(baseline_params, latest_params): + if baseline_param == latest_param: + continue + if baseline_param[0] == "api_shortname": + raise ValueError( + f"{library_name}: api_shortname must not change when library_name remains the same." + ) + else: + config_change = ConfigChange( + changed_param=latest_param[0], + latest_value=latest_param[1], + library_name=library_name, + ) + diff[ChangeType.LIBRARY_LEVEL_CHANGE].append(config_change) + + # compare gapic_configs + baseline_gapic_configs = baseline_library.gapic_configs + latest_gapic_configs = latest_library.gapic_configs + __compare_gapic_configs( + diff=diff, + library_name=library_name, + baseline_gapic_configs=baseline_gapic_configs, + latest_gapic_configs=latest_gapic_configs, + ) + + +def __compare_gapic_configs( + diff: Dict[ChangeType, list[ConfigChange]], + library_name: str, + baseline_gapic_configs: List[GapicConfig], + latest_gapic_configs: List[GapicConfig], +) -> None: + baseline_proto_paths = {config.proto_path for config in baseline_gapic_configs} + latest_proto_paths = {config.proto_path for config in latest_gapic_configs} + # 1st round of comparison, find any versioned proto_path is removed + # from baseline gapic configs. + # please see the reason of comment out these lines of code in the + # comment of ChangeType.GAPIC_REMOVAL. + # for proto_path in baseline_proto_paths: + # if proto_path in latest_proto_paths: + # continue + # config_change = ConfigChange( + # changed_param="", latest_value=proto_path, library_name=library_name + # ) + # diff[ChangeType.GAPIC_REMOVAL].append(config_change) + + # 2nd round of comparison, find any versioned proto_path is added + # to latest gapic configs. + for proto_path in latest_proto_paths: + if proto_path in baseline_proto_paths: + continue + config_change = ConfigChange( + changed_param="", latest_value=proto_path, library_name=library_name + ) + diff[ChangeType.GAPIC_ADDITION].append(config_change) + + +def __convert_params_to_sorted_list(obj: Any) -> List[tuple]: + """ + Convert the parameter and its value of a given object to a sorted list of + tuples. + + Only the following types of parameters will be considered: + + - str + - bool + - list[str] + - None + + Note that built-in params, e.g., __str__, and methods will be skipped. + + :param obj: an object + :return: a sorted list of tuples. + """ + param_and_values = [] + for param, value in vars(obj).items(): + if ( + param.startswith("__") # skip built-in params + or callable(getattr(obj, param)) # skip methods + # skip if the type of param is not one of the following types + # 1. str + # 2. bool + # 3. list[str] + # 4. None + or not ( + isinstance(getattr(obj, param), str) + or isinstance(getattr(obj, param), bool) + or __is_list_of_str(obj=obj, param=param) + or getattr(obj, param) is None + ) + ): + continue + param_and_values.append((param, value)) + return sorted(param_and_values) + + +def __is_list_of_str(obj: Any, param: str) -> bool: + """ + Returns True if the type of param of a given object is a list of str; False + otherwise. + + This method is a workaround of https://bugs.python.org/issue28339. + """ + value = getattr(obj, param) + if not isinstance(value, list): + return False + for v in value: + if not isinstance(v, str): + return False + return True