Skip to content

Commit

Permalink
feat: add config change (#2604)
Browse files Browse the repository at this point in the history
In this PR:
- Add `config_change.py` to get:
- Changed libraries, which will be passed to `generate_repo.py` to
generate libraries selectedly.
- Qualified commits, which will be passed to
`generate_pr_description.py` to generate PR description.
- Refactor part of utility methods to `proto_path_utils.py`.
- Refactor `generation_config_comparator.py` to move data class to
`config_change.py`.
- Refactor `utilities.py` and `utilities.sh` to `utils/utilities.py` and
`utils/utilities.sh`.
- Add unit tests.

Follow up of #2587

For the goal of series of PRs, please refer to [improvement
proposal](https://docs.google.com/document/d/1JiCcG3X7lnxaJErKe0ES_JkyU7ECb40nf2Xez3gWvuo/edit?tab=t.g3vua2kd06gx#bookmark=id.72s3ukwwzevo).
  • Loading branch information
JoeWang1127 authored and lqiu96 committed Apr 2, 2024
1 parent 38afb9a commit ccc0b62
Show file tree
Hide file tree
Showing 22 changed files with 726 additions and 213 deletions.
2 changes: 1 addition & 1 deletion library_generation/generate_composed_library.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
import os
from pathlib import Path
from typing import List
import library_generation.utilities as util
import library_generation.utils.utilities as util
from library_generation.model.generation_config import GenerationConfig
from library_generation.model.gapic_config import GapicConfig
from library_generation.model.gapic_inputs import GapicInputs
Expand Down
2 changes: 1 addition & 1 deletion library_generation/generate_library.sh
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ done

script_dir=$(dirname "$(readlink -f "$0")")
# source utility functions
source "${script_dir}"/utilities.sh
source "${script_dir}"/utils/utilities.sh
output_folder="$(get_output_folder)"

if [ -z "${gapic_generator_version}" ]; then
Expand Down
6 changes: 2 additions & 4 deletions library_generation/generate_pr_description.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,8 @@
import click
from git import Commit, Repo
from library_generation.model.generation_config import from_yaml
from library_generation.utilities import find_versioned_proto_path
from library_generation.utils.proto_path_utils import find_versioned_proto_path
from library_generation.utils.commit_message_formatter import format_commit_message
from library_generation.utilities import get_file_paths
from library_generation.utils.commit_message_formatter import wrap_nested_commit
from library_generation.utils.commit_message_formatter import wrap_override_commit


Expand Down Expand Up @@ -87,7 +85,7 @@ def generate_pr_descriptions(
baseline_commit: str,
) -> str:
config = from_yaml(generation_config_yaml)
paths = get_file_paths(config)
paths = config.get_proto_path_to_library_name()
return __get_commit_messages(
repo_url=repo_url,
latest_commit=config.googleapis_commitish,
Expand Down
2 changes: 1 addition & 1 deletion library_generation/generate_repo.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

import library_generation.utilities as util
import library_generation.utils.utilities as util
import click
import os
from library_generation.generate_composed_library import generate_composed_library
Expand Down
158 changes: 158 additions & 0 deletions library_generation/model/config_change.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import shutil
from enum import Enum
from typing import Optional
from git import Commit, Repo
from library_generation.model.generation_config import GenerationConfig
from library_generation.model.library_config import LibraryConfig
from library_generation.utils.utilities import sh_util
from library_generation.utils.proto_path_utils import find_versioned_proto_path


class ChangeType(Enum):
GOOGLEAPIS_COMMIT = 1
REPO_LEVEL_CHANGE = 2
LIBRARIES_ADDITION = 3
# As of Mar. 2024, we decide not to produce this type of change because we
# still need to manually remove the libray.
# LIBRARIES_REMOVAL = 4
LIBRARY_LEVEL_CHANGE = 5
GAPIC_ADDITION = 6
# As of Mar. 2024, we decide not to produce this type of change because we
# still need to manually remove the libray.
# GAPIC_REMOVAL = 7


class HashLibrary:
"""
Data class to group a LibraryConfig object and its hash value together.
"""

def __init__(self, hash_value: int, library: LibraryConfig):
self.hash_value = hash_value
self.library = library


class LibraryChange:
def __init__(self, changed_param: str, latest_value: str, library_name: str = ""):
self.changed_param = changed_param
self.latest_value = latest_value
self.library_name = library_name


class QualifiedCommit:
def __init__(self, commit: Commit, libraries: set[str]):
self.commit = commit
self.libraries = libraries


class ConfigChange:
ALL_LIBRARIES_CHANGED = None

def __init__(
self,
change_to_libraries: dict[ChangeType, list[LibraryChange]],
baseline_config: GenerationConfig,
latest_config: GenerationConfig,
):
self.change_to_libraries = change_to_libraries
self.baseline_config = baseline_config
self.latest_config = latest_config

def get_changed_libraries(self) -> Optional[list[str]]:
"""
Returns a unique, sorted list of library name of changed libraries.
None if there is a repository level change, which means all libraries
in the latest_config will be generated.
:return: library names of change libraries.
"""
if ChangeType.REPO_LEVEL_CHANGE in self.change_to_libraries:
return ConfigChange.ALL_LIBRARIES_CHANGED
library_names = set()
for change_type, library_changes in self.change_to_libraries.items():
if change_type == ChangeType.GOOGLEAPIS_COMMIT:
library_names.update(self.__get_library_names_from_qualified_commits())
else:
library_names.update(
[library_change.library_name for library_change in library_changes]
)
return sorted(list(library_names))

def get_qualified_commits(
self,
repo_url: str = "https://github.com/googleapis/googleapis.git",
) -> list[QualifiedCommit]:
"""
Returns qualified commits from configuration change.
A qualified commit is a commit that changes at least one file (excluding
BUILD.bazel) within a versioned proto path in the given proto_paths.
:param repo_url: the repository contains the commit history.
:return: QualifiedCommit objects.
"""
tmp_dir = sh_util("get_output_folder")
shutil.rmtree(tmp_dir, ignore_errors=True)
os.mkdir(tmp_dir)
# we only need commit history, thus shadow clone is enough.
repo = Repo.clone_from(url=repo_url, to_path=tmp_dir, filter=["blob:none"])
commit = repo.commit(self.latest_config.googleapis_commitish)
proto_paths = self.latest_config.get_proto_path_to_library_name()
qualified_commits = []
while str(commit.hexsha) != self.baseline_config.googleapis_commitish:
qualified_commit = ConfigChange.__create_qualified_commit(
proto_paths=proto_paths, commit=commit
)
if qualified_commit is not None:
qualified_commits.append(qualified_commit)
commit_parents = commit.parents
if len(commit_parents) == 0:
break
commit = commit_parents[0]
shutil.rmtree(tmp_dir, ignore_errors=True)
return qualified_commits

def __get_library_names_from_qualified_commits(self) -> list[str]:
qualified_commits = self.get_qualified_commits()
library_names = []
for qualified_commit in qualified_commits:
library_names.extend(qualified_commit.libraries)
return library_names

@staticmethod
def __create_qualified_commit(
proto_paths: dict[str, str], commit: Commit
) -> Optional[QualifiedCommit]:
"""
Returns a qualified commit from the given Commit object; otherwise None.
:param proto_paths: a mapping from versioned proto_path to library_name
:param commit: a GitHub commit object.
:return: qualified commits.
"""
libraries = set()
for file in commit.stats.files.keys():
if file.endswith("BUILD.bazel"):
continue
versioned_proto_path = find_versioned_proto_path(file)
if versioned_proto_path in proto_paths:
# Even though a commit usually only changes one
# library, we don't want to miss generating a
# library because the commit may change multiple
# libraries.
libraries.add(proto_paths[versioned_proto_path])
if len(libraries) == 0:
return None
return QualifiedCommit(commit=commit, libraries=libraries)
12 changes: 12 additions & 0 deletions library_generation/model/generation_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,18 @@ def __init__(
# monorepos have more than one library defined in the config yaml
self.is_monorepo = len(libraries) > 1

def get_proto_path_to_library_name(self) -> dict[str, str]:
"""
Get versioned proto_path to library_name mapping from configuration.
:return: versioned proto_path to library_name mapping
"""
paths = {}
for library in self.libraries:
for gapic_config in library.gapic_configs:
paths[gapic_config.proto_path] = library.get_library_name()
return paths


def from_yaml(path_to_yaml: str) -> GenerationConfig:
"""
Expand Down
2 changes: 1 addition & 1 deletion library_generation/postprocess_library.sh
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ synthtool_commitish=$6
is_monorepo=$7
configuration_yaml_path=$8

source "${scripts_root}"/utilities.sh
source "${scripts_root}"/utils/utilities.sh

declare -a required_inputs=("postprocessing_target" "versions_file" "owlbot_cli_image_sha" "synthtool_commitish" "is_monorepo")
for required_input in "${required_inputs[@]}"; do
Expand Down
4 changes: 3 additions & 1 deletion library_generation/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,9 @@
},
package_data={
"library_generation": [
"*.sh",
"generate_library.sh",
"postprocess_library.sh",
"utils/utilities.sh",
"templates/*.j2",
"gapic-generator-java-wrapper",
"requirements.*",
Expand Down
2 changes: 1 addition & 1 deletion library_generation/test/compare_poms.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
The only comparison points are: element path (e.g. project/dependencies) and element text
There is a special case for `dependency`, where the maven coordinates are prepared as well
"""
from library_generation.utilities import eprint
from library_generation.utils.utilities import eprint
import xml.etree.ElementTree as et
from collections import Counter
import sys
Expand Down
2 changes: 1 addition & 1 deletion library_generation/test/generate_library_unit_tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ set -xeo pipefail
# Unit tests against ../utilities.sh
script_dir=$(dirname "$(readlink -f "$0")")
source "${script_dir}"/test_utilities.sh
source "${script_dir}"/../utilities.sh
source "${script_dir}"/../utils/utilities.sh

# Unit tests
extract_folder_name_test() {
Expand Down
2 changes: 1 addition & 1 deletion library_generation/test/integration_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
from library_generation.generate_repo import generate_from_yaml
from library_generation.model.generation_config import from_yaml, GenerationConfig
from library_generation.test.compare_poms import compare_xml
from library_generation.utilities import (
from library_generation.utils.utilities import (
sh_util as shell_call,
run_process_and_print_output,
)
Expand Down
Loading

0 comments on commit ccc0b62

Please sign in to comment.