Skip to content

Commit

Permalink
feat: add libraries_bom_version to generation configuration (#2639)
Browse files Browse the repository at this point in the history
In this PR:
- Add `libraries_bom_version` as a required parameter in generation
config.

The integration test is verified using a feature branch of synthtool:
googleapis/synthtool#1956.

The synthtool commit in integration test is changed to
`63cc541da2c45fcfca2136c43e638da1fbae174d` after the above feature
branch is merged.
  • Loading branch information
JoeWang1127 authored Apr 16, 2024
1 parent e542af8 commit 56c7ca5
Show file tree
Hide file tree
Showing 16 changed files with 113 additions and 63 deletions.
10 changes: 8 additions & 2 deletions library_generation/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ They are shared by library level parameters.
| protobuf_version | No | inferred from the generator if not specified |
| grpc_version | No | inferred from the generator if not specified |
| googleapis-commitish | Yes | |
| libraries_bom_version | Yes | |
| owlbot-cli-image | Yes | |
| synthtool-commitish | Yes | |
| template_excludes | Yes | |
Expand Down Expand Up @@ -147,6 +148,7 @@ The GAPIC level parameters define how to generate a GAPIC library.
gapic_generator_version: 2.34.0
protobuf_version: 25.2
googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026
libraries_bom_version: 26.37.0
owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409
synthtool_commitish: 6612ab8f3afcd5e292aecd647f0fa68812c9f5b5
destination_path: google-cloud-java
Expand Down Expand Up @@ -192,11 +194,15 @@ libraries:
# Local Environment Setup before running `entry_point.py`

1. Assuming Python 3 is installed, follow official guide from [Python.org](https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#create-and-use-virtual-environments) to create a virtual environment. The virtual environment can be installed to any folder, usually it is recommended to be installed under the root folder of the project(`sdk-platform-java` in this case).
2. Assuming the virtual environment is installed under `sdk-platform-java`. Run the following command under the root folder of `sdk-platform-java` to install the dependencies of `library_generation`
1. Assuming Python 3 is installed, follow official guide from [Python.org](https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#create-and-use-virtual-environments) to create a virtual environment.
The virtual environment can be installed to any folder, usually it is recommended to be installed under the root folder of the project(`sdk-platform-java` in this case).
2. Assuming the virtual environment is installed under `sdk-platform-java`.
Run the following command under the root folder of `sdk-platform-java` to install the dependencies of `library_generation`

```bash
python -m pip install -r library_generation/requirements.txt
```

3. Run the following command to install `library_generation` as a module, which allows the `library_generation` module to be imported from anywhere
```bash
python -m pip install library_generation/
Expand Down
3 changes: 3 additions & 0 deletions library_generation/model/generation_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ def __init__(
self,
gapic_generator_version: str,
googleapis_commitish: str,
libraries_bom_version: str,
owlbot_cli_image: str,
synthtool_commitish: str,
template_excludes: List[str],
Expand All @@ -38,6 +39,7 @@ def __init__(
):
self.gapic_generator_version = gapic_generator_version
self.googleapis_commitish = googleapis_commitish
self.libraris_bom_version = libraries_bom_version
self.owlbot_cli_image = owlbot_cli_image
self.synthtool_commitish = synthtool_commitish
self.template_excludes = template_excludes
Expand Down Expand Up @@ -116,6 +118,7 @@ def from_yaml(path_to_yaml: str) -> GenerationConfig:
grpc_version=__optional(config, "grpc_version", None),
protobuf_version=__optional(config, "protobuf_version", None),
googleapis_commitish=__required(config, "googleapis_commitish"),
libraries_bom_version=__required(config, "libraries_bom_version"),
owlbot_cli_image=__required(config, "owlbot_cli_image"),
synthtool_commitish=__required(config, "synthtool_commitish"),
template_excludes=__required(config, "template_excludes"),
Expand Down
1 change: 1 addition & 0 deletions library_generation/owlbot/src/apply_repo_templates.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ def apply_repo_templates(configuration_yaml_path: str, monorepo: bool) -> None:
excludes=config.template_excludes,
template_path=Path(repo_templates_path),
monorepo=monorepo,
libraries_bom_version=config.libraris_bom_version,
)


Expand Down
2 changes: 1 addition & 1 deletion library_generation/owlbot/templates/java_library/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file:
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>libraries-bom</artifactId>
<version>{{ metadata['latest_bom_version'] }}</version>
<version>{{ metadata['libraries_bom_version'] }}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
Expand Down
1 change: 1 addition & 0 deletions library_generation/test/generate_repo_unit_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ def __get_an_empty_generation_config() -> GenerationConfig:
return GenerationConfig(
gapic_generator_version="",
googleapis_commitish="",
libraries_bom_version="",
synthtool_commitish="",
owlbot_cli_image="",
template_excludes=[],
Expand Down
1 change: 1 addition & 0 deletions library_generation/test/model/config_change_unit_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,7 @@ def __get_a_gen_config(
return GenerationConfig(
gapic_generator_version="",
googleapis_commitish=googleapis_commitish,
libraries_bom_version="",
owlbot_cli_image="",
synthtool_commitish="",
template_excludes=[],
Expand Down
60 changes: 59 additions & 1 deletion library_generation/test/model/generation_config_unit_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
import os
import unittest
from pathlib import Path

from library_generation.model.generation_config import from_yaml, GenerationConfig
from library_generation.model.library_config import LibraryConfig

Expand All @@ -39,6 +38,63 @@


class GenerationConfigTest(unittest.TestCase):
def test_from_yaml_succeeds(self):
config = from_yaml(f"{test_config_dir}/generation_config.yaml")
self.assertEqual("2.34.0", config.gapic_generator_version)
self.assertEqual(25.2, config.protobuf_version)
self.assertEqual(
"1a45bf7393b52407188c82e63101db7dc9c72026", config.googleapis_commitish
)
self.assertEqual("26.37.0", config.libraris_bom_version)
self.assertEqual(
"sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409",
config.owlbot_cli_image,
)
self.assertEqual(
"6612ab8f3afcd5e292aecd647f0fa68812c9f5b5", config.synthtool_commitish
)
self.assertEqual(
[
".github/*",
".kokoro/*",
"samples/*",
"CODE_OF_CONDUCT.md",
"CONTRIBUTING.md",
"LICENSE",
"SECURITY.md",
"java.header",
"license-checks.xml",
"renovate.json",
".gitignore",
],
config.template_excludes,
)
library = config.libraries[0]
self.assertEqual("cloudasset", library.api_shortname)
self.assertEqual("Cloud Asset Inventory", library.name_pretty)
self.assertEqual(
"https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview",
library.product_documentation,
)
self.assertEqual(
"provides inventory services based on a time series database.",
library.api_description,
)
self.assertEqual("asset", library.library_name)
self.assertEqual("@googleapis/analytics-dpe", library.codeowner_team)
self.assertEqual(
"proto-google-iam-v1-bom,google-iam-policy,proto-google-iam-v1",
library.excluded_poms,
)
self.assertEqual("google-iam-policy", library.excluded_dependencies)
gapics = library.gapic_configs
self.assertEqual(5, len(gapics))
self.assertEqual("google/cloud/asset/v1", gapics[0].proto_path)
self.assertEqual("google/cloud/asset/v1p1beta1", gapics[1].proto_path)
self.assertEqual("google/cloud/asset/v1p2beta1", gapics[2].proto_path)
self.assertEqual("google/cloud/asset/v1p5beta1", gapics[3].proto_path)
self.assertEqual("google/cloud/asset/v1p7beta1", gapics[4].proto_path)

def test_get_proto_path_to_library_name_success(self):
paths = from_yaml(
f"{test_config_dir}/generation_config.yaml"
Expand All @@ -58,6 +114,7 @@ def test_is_monorepo_with_one_library_returns_false(self):
config = GenerationConfig(
gapic_generator_version="",
googleapis_commitish="",
libraries_bom_version="",
owlbot_cli_image="",
synthtool_commitish="",
template_excludes=[],
Expand All @@ -69,6 +126,7 @@ def test_is_monorepo_with_two_libraries_returns_true(self):
config = GenerationConfig(
gapic_generator_version="",
googleapis_commitish="",
libraries_bom_version="",
owlbot_cli_image="",
synthtool_commitish="",
template_excludes=[],
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
gapic_generator_version: 2.37.0
gapic_generator_version: 2.38.1
protobuf_version: 25.2
googleapis_commitish: a17d4caf184b050d50cacf2b0d579ce72c31ce74
libraries_bom_version: 26.37.0
owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409
synthtool_commitish: 5e1fb2032fa44bc170677b38713023b4fec51a4e
synthtool_commitish: 63cc541da2c45fcfca2136c43e638da1fbae174d
template_excludes:
- ".github/*"
- ".kokoro/*"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
gapic_generator_version: 2.38.1
protobuf_version: 25.2
googleapis_commitish: 4ce0ff67a3d4509be641cbe47a35844ddc1268fc
libraries_bom_version: 26.37.0
owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409
synthtool_commitish: 6612ab8f3afcd5e292aecd647f0fa68812c9f5b5
synthtool_commitish: 63cc541da2c45fcfca2136c43e638da1fbae174d
template_excludes:
- ".github/*"
- ".kokoro/*"
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
gapic_generator_version: 2.34.0
googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026
libraries:
- api_shortname: apigeeconnect
name_pretty: Apigee Connect
api_description: "allows the Apigee hybrid management"
product_documentation: "https://cloud.google.com/apigee/docs/hybrid/v1.3/apigee-connect/"
GAPICs:
- proto_path: google/cloud/apigeeconnect/v1
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
gapic_generator_version: 2.34.0
googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026
libraries_bom_version: 26.37.0
libraries:
- api_shortname: apigeeconnect
name_pretty: Apigee Connect
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
gapic_generator_version: 2.34.0
googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026
libraries_bom_version: 26.37.0
owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409
libraries:
- api_shortname: apigeeconnect
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
gapic_generator_version: 2.34.0
googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026
libraries_bom_version: 26.37.0
owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409
synthtool_commitish: 6612ab8f3afcd5e292aecd647f0fa68812c9f5b5
libraries:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
gapic_generator_version: 2.34.0
protobuf_version: 25.2
googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026
libraries_bom_version: 26.37.0
owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409
synthtool_commitish: 6612ab8f3afcd5e292aecd647f0fa68812c9f5b5
template_excludes:
Expand Down
61 changes: 5 additions & 56 deletions library_generation/test/utilities_unit_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,10 @@ def test_eprint_valid_input_succeeds(self):
"googleapis_commitish",
f"{test_config_dir}/config_without_googleapis.yaml",
),
(
"libraries_bom_version",
f"{test_config_dir}/config_without_libraries_bom_version.yaml",
),
("owlbot_cli_image", f"{test_config_dir}/config_without_owlbot.yaml"),
("synthtool_commitish", f"{test_config_dir}/config_without_synthtool.yaml"),
(
Expand All @@ -159,62 +163,6 @@ def test_from_yaml_without_key_fails(self, error_message_contains, path_to_yaml)
path_to_yaml,
)

def test_from_yaml_succeeds(self):
config = from_yaml(f"{test_config_dir}/generation_config.yaml")
self.assertEqual("2.34.0", config.gapic_generator_version)
self.assertEqual(25.2, config.protobuf_version)
self.assertEqual(
"1a45bf7393b52407188c82e63101db7dc9c72026", config.googleapis_commitish
)
self.assertEqual(
"sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409",
config.owlbot_cli_image,
)
self.assertEqual(
"6612ab8f3afcd5e292aecd647f0fa68812c9f5b5", config.synthtool_commitish
)
self.assertEqual(
[
".github/*",
".kokoro/*",
"samples/*",
"CODE_OF_CONDUCT.md",
"CONTRIBUTING.md",
"LICENSE",
"SECURITY.md",
"java.header",
"license-checks.xml",
"renovate.json",
".gitignore",
],
config.template_excludes,
)
library = config.libraries[0]
self.assertEqual("cloudasset", library.api_shortname)
self.assertEqual("Cloud Asset Inventory", library.name_pretty)
self.assertEqual(
"https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview",
library.product_documentation,
)
self.assertEqual(
"provides inventory services based on a time series database.",
library.api_description,
)
self.assertEqual("asset", library.library_name)
self.assertEqual("@googleapis/analytics-dpe", library.codeowner_team)
self.assertEqual(
"proto-google-iam-v1-bom,google-iam-policy,proto-google-iam-v1",
library.excluded_poms,
)
self.assertEqual("google-iam-policy", library.excluded_dependencies)
gapics = library.gapic_configs
self.assertEqual(5, len(gapics))
self.assertEqual("google/cloud/asset/v1", gapics[0].proto_path)
self.assertEqual("google/cloud/asset/v1p1beta1", gapics[1].proto_path)
self.assertEqual("google/cloud/asset/v1p2beta1", gapics[2].proto_path)
self.assertEqual("google/cloud/asset/v1p5beta1", gapics[3].proto_path)
self.assertEqual("google/cloud/asset/v1p7beta1", gapics[4].proto_path)

@parameterized.expand(
[
("BUILD_no_additional_protos.bazel", " "),
Expand Down Expand Up @@ -481,6 +429,7 @@ def __get_a_gen_config(
return GenerationConfig(
gapic_generator_version="",
googleapis_commitish="",
libraries_bom_version="",
owlbot_cli_image="",
synthtool_commitish="",
template_excludes=[
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ def setUp(self) -> None:
self.baseline_config = GenerationConfig(
gapic_generator_version="",
googleapis_commitish="",
libraries_bom_version="",
owlbot_cli_image="",
synthtool_commitish="",
template_excludes=[],
Expand All @@ -49,6 +50,7 @@ def setUp(self) -> None:
self.current_config = GenerationConfig(
gapic_generator_version="",
googleapis_commitish="",
libraries_bom_version="",
owlbot_cli_image="",
synthtool_commitish="",
template_excludes=[],
Expand Down Expand Up @@ -91,6 +93,20 @@ def test_compare_config_generator_update(self):
self.assertEqual("gapic_generator_version", config_change.changed_param)
self.assertEqual("1.2.4", config_change.current_value)

def test_compare_config_libraries_bom_update(self):
self.baseline_config.libraris_bom_version = "26.36.0"
self.current_config.libraris_bom_version = "26.37.0"
result = compare_config(
baseline_config=self.baseline_config,
current_config=self.current_config,
)
self.assertTrue(
len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1
)
config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0]
self.assertEqual("libraris_bom_version", config_change.changed_param)
self.assertEqual("26.37.0", config_change.current_value)

def test_compare_config_owlbot_cli_update(self):
self.baseline_config.owlbot_cli_image = "image_version_123"
self.current_config.owlbot_cli_image = "image_version_456"
Expand Down

0 comments on commit 56c7ca5

Please sign in to comment.