From 56c7ca51d4ca99629251e0aa9cac316eb349c974 Mon Sep 17 00:00:00 2001 From: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> Date: Tue, 16 Apr 2024 14:15:31 -0400 Subject: [PATCH] feat: add `libraries_bom_version` to generation configuration (#2639) In this PR: - Add `libraries_bom_version` as a required parameter in generation config. The integration test is verified using a feature branch of synthtool: https://github.com/googleapis/synthtool/pull/1956. The synthtool commit in integration test is changed to `63cc541da2c45fcfca2136c43e638da1fbae174d` after the above feature branch is merged. --- library_generation/README.md | 10 ++- library_generation/model/generation_config.py | 3 + .../owlbot/src/apply_repo_templates.py | 1 + .../owlbot/templates/java_library/README.md | 2 +- .../test/generate_repo_unit_tests.py | 1 + .../test/model/config_change_unit_tests.py | 1 + .../test/model/generation_config_unit_test.py | 60 +++++++++++++++++- .../baseline_generation_config.yaml | 5 +- .../current_generation_config.yaml | 3 +- .../config_without_libraries_bom_version.yaml | 9 +++ .../test-config/config_without_owlbot.yaml | 1 + .../test-config/config_without_synthtool.yaml | 1 + .../config_without_temp_excludes.yaml | 1 + .../test-config/generation_config.yaml | 1 + .../test/utilities_unit_tests.py | 61 ++----------------- ...generation_config_comparator_unit_tests.py | 16 +++++ 16 files changed, 113 insertions(+), 63 deletions(-) create mode 100644 library_generation/test/resources/test-config/config_without_libraries_bom_version.yaml diff --git a/library_generation/README.md b/library_generation/README.md index dfa013e3d0..50ea1afe9c 100644 --- a/library_generation/README.md +++ b/library_generation/README.md @@ -97,6 +97,7 @@ They are shared by library level parameters. | protobuf_version | No | inferred from the generator if not specified | | grpc_version | No | inferred from the generator if not specified | | googleapis-commitish | Yes | | +| libraries_bom_version | Yes | | | owlbot-cli-image | Yes | | | synthtool-commitish | Yes | | | template_excludes | Yes | | @@ -147,6 +148,7 @@ The GAPIC level parameters define how to generate a GAPIC library. gapic_generator_version: 2.34.0 protobuf_version: 25.2 googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 +libraries_bom_version: 26.37.0 owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 synthtool_commitish: 6612ab8f3afcd5e292aecd647f0fa68812c9f5b5 destination_path: google-cloud-java @@ -192,11 +194,15 @@ libraries: # Local Environment Setup before running `entry_point.py` -1. Assuming Python 3 is installed, follow official guide from [Python.org](https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#create-and-use-virtual-environments) to create a virtual environment. The virtual environment can be installed to any folder, usually it is recommended to be installed under the root folder of the project(`sdk-platform-java` in this case). -2. Assuming the virtual environment is installed under `sdk-platform-java`. Run the following command under the root folder of `sdk-platform-java` to install the dependencies of `library_generation` +1. Assuming Python 3 is installed, follow official guide from [Python.org](https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#create-and-use-virtual-environments) to create a virtual environment. +The virtual environment can be installed to any folder, usually it is recommended to be installed under the root folder of the project(`sdk-platform-java` in this case). +2. Assuming the virtual environment is installed under `sdk-platform-java`. +Run the following command under the root folder of `sdk-platform-java` to install the dependencies of `library_generation` + ```bash python -m pip install -r library_generation/requirements.txt ``` + 3. Run the following command to install `library_generation` as a module, which allows the `library_generation` module to be imported from anywhere ```bash python -m pip install library_generation/ diff --git a/library_generation/model/generation_config.py b/library_generation/model/generation_config.py index e0c27cd7d0..97eebe6788 100644 --- a/library_generation/model/generation_config.py +++ b/library_generation/model/generation_config.py @@ -29,6 +29,7 @@ def __init__( self, gapic_generator_version: str, googleapis_commitish: str, + libraries_bom_version: str, owlbot_cli_image: str, synthtool_commitish: str, template_excludes: List[str], @@ -38,6 +39,7 @@ def __init__( ): self.gapic_generator_version = gapic_generator_version self.googleapis_commitish = googleapis_commitish + self.libraris_bom_version = libraries_bom_version self.owlbot_cli_image = owlbot_cli_image self.synthtool_commitish = synthtool_commitish self.template_excludes = template_excludes @@ -116,6 +118,7 @@ def from_yaml(path_to_yaml: str) -> GenerationConfig: grpc_version=__optional(config, "grpc_version", None), protobuf_version=__optional(config, "protobuf_version", None), googleapis_commitish=__required(config, "googleapis_commitish"), + libraries_bom_version=__required(config, "libraries_bom_version"), owlbot_cli_image=__required(config, "owlbot_cli_image"), synthtool_commitish=__required(config, "synthtool_commitish"), template_excludes=__required(config, "template_excludes"), diff --git a/library_generation/owlbot/src/apply_repo_templates.py b/library_generation/owlbot/src/apply_repo_templates.py index 26e9c3509d..8860586640 100644 --- a/library_generation/owlbot/src/apply_repo_templates.py +++ b/library_generation/owlbot/src/apply_repo_templates.py @@ -24,6 +24,7 @@ def apply_repo_templates(configuration_yaml_path: str, monorepo: bool) -> None: excludes=config.template_excludes, template_path=Path(repo_templates_path), monorepo=monorepo, + libraries_bom_version=config.libraris_bom_version, ) diff --git a/library_generation/owlbot/templates/java_library/README.md b/library_generation/owlbot/templates/java_library/README.md index e849a97147..04f1f4e834 100644 --- a/library_generation/owlbot/templates/java_library/README.md +++ b/library_generation/owlbot/templates/java_library/README.md @@ -44,7 +44,7 @@ If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: com.google.cloud libraries-bom - {{ metadata['latest_bom_version'] }} + {{ metadata['libraries_bom_version'] }} pom import diff --git a/library_generation/test/generate_repo_unit_tests.py b/library_generation/test/generate_repo_unit_tests.py index 3696e9f524..a947e29f5e 100644 --- a/library_generation/test/generate_repo_unit_tests.py +++ b/library_generation/test/generate_repo_unit_tests.py @@ -45,6 +45,7 @@ def __get_an_empty_generation_config() -> GenerationConfig: return GenerationConfig( gapic_generator_version="", googleapis_commitish="", + libraries_bom_version="", synthtool_commitish="", owlbot_cli_image="", template_excludes=[], diff --git a/library_generation/test/model/config_change_unit_tests.py b/library_generation/test/model/config_change_unit_tests.py index 94e62ab902..388dd1f614 100644 --- a/library_generation/test/model/config_change_unit_tests.py +++ b/library_generation/test/model/config_change_unit_tests.py @@ -239,6 +239,7 @@ def __get_a_gen_config( return GenerationConfig( gapic_generator_version="", googleapis_commitish=googleapis_commitish, + libraries_bom_version="", owlbot_cli_image="", synthtool_commitish="", template_excludes=[], diff --git a/library_generation/test/model/generation_config_unit_test.py b/library_generation/test/model/generation_config_unit_test.py index 0edb0f9159..08fea21523 100644 --- a/library_generation/test/model/generation_config_unit_test.py +++ b/library_generation/test/model/generation_config_unit_test.py @@ -14,7 +14,6 @@ import os import unittest from pathlib import Path - from library_generation.model.generation_config import from_yaml, GenerationConfig from library_generation.model.library_config import LibraryConfig @@ -39,6 +38,63 @@ class GenerationConfigTest(unittest.TestCase): + def test_from_yaml_succeeds(self): + config = from_yaml(f"{test_config_dir}/generation_config.yaml") + self.assertEqual("2.34.0", config.gapic_generator_version) + self.assertEqual(25.2, config.protobuf_version) + self.assertEqual( + "1a45bf7393b52407188c82e63101db7dc9c72026", config.googleapis_commitish + ) + self.assertEqual("26.37.0", config.libraris_bom_version) + self.assertEqual( + "sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409", + config.owlbot_cli_image, + ) + self.assertEqual( + "6612ab8f3afcd5e292aecd647f0fa68812c9f5b5", config.synthtool_commitish + ) + self.assertEqual( + [ + ".github/*", + ".kokoro/*", + "samples/*", + "CODE_OF_CONDUCT.md", + "CONTRIBUTING.md", + "LICENSE", + "SECURITY.md", + "java.header", + "license-checks.xml", + "renovate.json", + ".gitignore", + ], + config.template_excludes, + ) + library = config.libraries[0] + self.assertEqual("cloudasset", library.api_shortname) + self.assertEqual("Cloud Asset Inventory", library.name_pretty) + self.assertEqual( + "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview", + library.product_documentation, + ) + self.assertEqual( + "provides inventory services based on a time series database.", + library.api_description, + ) + self.assertEqual("asset", library.library_name) + self.assertEqual("@googleapis/analytics-dpe", library.codeowner_team) + self.assertEqual( + "proto-google-iam-v1-bom,google-iam-policy,proto-google-iam-v1", + library.excluded_poms, + ) + self.assertEqual("google-iam-policy", library.excluded_dependencies) + gapics = library.gapic_configs + self.assertEqual(5, len(gapics)) + self.assertEqual("google/cloud/asset/v1", gapics[0].proto_path) + self.assertEqual("google/cloud/asset/v1p1beta1", gapics[1].proto_path) + self.assertEqual("google/cloud/asset/v1p2beta1", gapics[2].proto_path) + self.assertEqual("google/cloud/asset/v1p5beta1", gapics[3].proto_path) + self.assertEqual("google/cloud/asset/v1p7beta1", gapics[4].proto_path) + def test_get_proto_path_to_library_name_success(self): paths = from_yaml( f"{test_config_dir}/generation_config.yaml" @@ -58,6 +114,7 @@ def test_is_monorepo_with_one_library_returns_false(self): config = GenerationConfig( gapic_generator_version="", googleapis_commitish="", + libraries_bom_version="", owlbot_cli_image="", synthtool_commitish="", template_excludes=[], @@ -69,6 +126,7 @@ def test_is_monorepo_with_two_libraries_returns_true(self): config = GenerationConfig( gapic_generator_version="", googleapis_commitish="", + libraries_bom_version="", owlbot_cli_image="", synthtool_commitish="", template_excludes=[], diff --git a/library_generation/test/resources/integration/google-cloud-java/baseline_generation_config.yaml b/library_generation/test/resources/integration/google-cloud-java/baseline_generation_config.yaml index ad9eafb31e..9bfa47a7c5 100644 --- a/library_generation/test/resources/integration/google-cloud-java/baseline_generation_config.yaml +++ b/library_generation/test/resources/integration/google-cloud-java/baseline_generation_config.yaml @@ -1,8 +1,9 @@ -gapic_generator_version: 2.37.0 +gapic_generator_version: 2.38.1 protobuf_version: 25.2 googleapis_commitish: a17d4caf184b050d50cacf2b0d579ce72c31ce74 +libraries_bom_version: 26.37.0 owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 -synthtool_commitish: 5e1fb2032fa44bc170677b38713023b4fec51a4e +synthtool_commitish: 63cc541da2c45fcfca2136c43e638da1fbae174d template_excludes: - ".github/*" - ".kokoro/*" diff --git a/library_generation/test/resources/integration/google-cloud-java/current_generation_config.yaml b/library_generation/test/resources/integration/google-cloud-java/current_generation_config.yaml index 96f1a67292..447c525424 100644 --- a/library_generation/test/resources/integration/google-cloud-java/current_generation_config.yaml +++ b/library_generation/test/resources/integration/google-cloud-java/current_generation_config.yaml @@ -1,8 +1,9 @@ gapic_generator_version: 2.38.1 protobuf_version: 25.2 googleapis_commitish: 4ce0ff67a3d4509be641cbe47a35844ddc1268fc +libraries_bom_version: 26.37.0 owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 -synthtool_commitish: 6612ab8f3afcd5e292aecd647f0fa68812c9f5b5 +synthtool_commitish: 63cc541da2c45fcfca2136c43e638da1fbae174d template_excludes: - ".github/*" - ".kokoro/*" diff --git a/library_generation/test/resources/test-config/config_without_libraries_bom_version.yaml b/library_generation/test/resources/test-config/config_without_libraries_bom_version.yaml new file mode 100644 index 0000000000..7921f68bd2 --- /dev/null +++ b/library_generation/test/resources/test-config/config_without_libraries_bom_version.yaml @@ -0,0 +1,9 @@ +gapic_generator_version: 2.34.0 +googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 +libraries: + - api_shortname: apigeeconnect + name_pretty: Apigee Connect + api_description: "allows the Apigee hybrid management" + product_documentation: "https://cloud.google.com/apigee/docs/hybrid/v1.3/apigee-connect/" + GAPICs: + - proto_path: google/cloud/apigeeconnect/v1 diff --git a/library_generation/test/resources/test-config/config_without_owlbot.yaml b/library_generation/test/resources/test-config/config_without_owlbot.yaml index 7921f68bd2..0d1bb7deea 100644 --- a/library_generation/test/resources/test-config/config_without_owlbot.yaml +++ b/library_generation/test/resources/test-config/config_without_owlbot.yaml @@ -1,5 +1,6 @@ gapic_generator_version: 2.34.0 googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 +libraries_bom_version: 26.37.0 libraries: - api_shortname: apigeeconnect name_pretty: Apigee Connect diff --git a/library_generation/test/resources/test-config/config_without_synthtool.yaml b/library_generation/test/resources/test-config/config_without_synthtool.yaml index 8907f96bf7..820c53032e 100644 --- a/library_generation/test/resources/test-config/config_without_synthtool.yaml +++ b/library_generation/test/resources/test-config/config_without_synthtool.yaml @@ -1,5 +1,6 @@ gapic_generator_version: 2.34.0 googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 +libraries_bom_version: 26.37.0 owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 libraries: - api_shortname: apigeeconnect diff --git a/library_generation/test/resources/test-config/config_without_temp_excludes.yaml b/library_generation/test/resources/test-config/config_without_temp_excludes.yaml index 9def2f3be6..54dab6449d 100644 --- a/library_generation/test/resources/test-config/config_without_temp_excludes.yaml +++ b/library_generation/test/resources/test-config/config_without_temp_excludes.yaml @@ -1,5 +1,6 @@ gapic_generator_version: 2.34.0 googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 +libraries_bom_version: 26.37.0 owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 synthtool_commitish: 6612ab8f3afcd5e292aecd647f0fa68812c9f5b5 libraries: diff --git a/library_generation/test/resources/test-config/generation_config.yaml b/library_generation/test/resources/test-config/generation_config.yaml index d84ed3afd2..a6deecfeeb 100644 --- a/library_generation/test/resources/test-config/generation_config.yaml +++ b/library_generation/test/resources/test-config/generation_config.yaml @@ -1,6 +1,7 @@ gapic_generator_version: 2.34.0 protobuf_version: 25.2 googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 +libraries_bom_version: 26.37.0 owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 synthtool_commitish: 6612ab8f3afcd5e292aecd647f0fa68812c9f5b5 template_excludes: diff --git a/library_generation/test/utilities_unit_tests.py b/library_generation/test/utilities_unit_tests.py index fae52e023b..bc8b965c2c 100644 --- a/library_generation/test/utilities_unit_tests.py +++ b/library_generation/test/utilities_unit_tests.py @@ -143,6 +143,10 @@ def test_eprint_valid_input_succeeds(self): "googleapis_commitish", f"{test_config_dir}/config_without_googleapis.yaml", ), + ( + "libraries_bom_version", + f"{test_config_dir}/config_without_libraries_bom_version.yaml", + ), ("owlbot_cli_image", f"{test_config_dir}/config_without_owlbot.yaml"), ("synthtool_commitish", f"{test_config_dir}/config_without_synthtool.yaml"), ( @@ -159,62 +163,6 @@ def test_from_yaml_without_key_fails(self, error_message_contains, path_to_yaml) path_to_yaml, ) - def test_from_yaml_succeeds(self): - config = from_yaml(f"{test_config_dir}/generation_config.yaml") - self.assertEqual("2.34.0", config.gapic_generator_version) - self.assertEqual(25.2, config.protobuf_version) - self.assertEqual( - "1a45bf7393b52407188c82e63101db7dc9c72026", config.googleapis_commitish - ) - self.assertEqual( - "sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409", - config.owlbot_cli_image, - ) - self.assertEqual( - "6612ab8f3afcd5e292aecd647f0fa68812c9f5b5", config.synthtool_commitish - ) - self.assertEqual( - [ - ".github/*", - ".kokoro/*", - "samples/*", - "CODE_OF_CONDUCT.md", - "CONTRIBUTING.md", - "LICENSE", - "SECURITY.md", - "java.header", - "license-checks.xml", - "renovate.json", - ".gitignore", - ], - config.template_excludes, - ) - library = config.libraries[0] - self.assertEqual("cloudasset", library.api_shortname) - self.assertEqual("Cloud Asset Inventory", library.name_pretty) - self.assertEqual( - "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview", - library.product_documentation, - ) - self.assertEqual( - "provides inventory services based on a time series database.", - library.api_description, - ) - self.assertEqual("asset", library.library_name) - self.assertEqual("@googleapis/analytics-dpe", library.codeowner_team) - self.assertEqual( - "proto-google-iam-v1-bom,google-iam-policy,proto-google-iam-v1", - library.excluded_poms, - ) - self.assertEqual("google-iam-policy", library.excluded_dependencies) - gapics = library.gapic_configs - self.assertEqual(5, len(gapics)) - self.assertEqual("google/cloud/asset/v1", gapics[0].proto_path) - self.assertEqual("google/cloud/asset/v1p1beta1", gapics[1].proto_path) - self.assertEqual("google/cloud/asset/v1p2beta1", gapics[2].proto_path) - self.assertEqual("google/cloud/asset/v1p5beta1", gapics[3].proto_path) - self.assertEqual("google/cloud/asset/v1p7beta1", gapics[4].proto_path) - @parameterized.expand( [ ("BUILD_no_additional_protos.bazel", " "), @@ -481,6 +429,7 @@ def __get_a_gen_config( return GenerationConfig( gapic_generator_version="", googleapis_commitish="", + libraries_bom_version="", owlbot_cli_image="", synthtool_commitish="", template_excludes=[ diff --git a/library_generation/test/utils/generation_config_comparator_unit_tests.py b/library_generation/test/utils/generation_config_comparator_unit_tests.py index bb0e8f83a2..a782cebe21 100644 --- a/library_generation/test/utils/generation_config_comparator_unit_tests.py +++ b/library_generation/test/utils/generation_config_comparator_unit_tests.py @@ -39,6 +39,7 @@ def setUp(self) -> None: self.baseline_config = GenerationConfig( gapic_generator_version="", googleapis_commitish="", + libraries_bom_version="", owlbot_cli_image="", synthtool_commitish="", template_excludes=[], @@ -49,6 +50,7 @@ def setUp(self) -> None: self.current_config = GenerationConfig( gapic_generator_version="", googleapis_commitish="", + libraries_bom_version="", owlbot_cli_image="", synthtool_commitish="", template_excludes=[], @@ -91,6 +93,20 @@ def test_compare_config_generator_update(self): self.assertEqual("gapic_generator_version", config_change.changed_param) self.assertEqual("1.2.4", config_change.current_value) + def test_compare_config_libraries_bom_update(self): + self.baseline_config.libraris_bom_version = "26.36.0" + self.current_config.libraris_bom_version = "26.37.0" + result = compare_config( + baseline_config=self.baseline_config, + current_config=self.current_config, + ) + self.assertTrue( + len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] + self.assertEqual("libraris_bom_version", config_change.changed_param) + self.assertEqual("26.37.0", config_change.current_value) + def test_compare_config_owlbot_cli_update(self): self.baseline_config.owlbot_cli_image = "image_version_123" self.current_config.owlbot_cli_image = "image_version_456"