feat: Move release note generation to a sub module

.cloudbuild/library_generation/library_generation.Dockerfile

@@ -45,19 +45,20 @@ RUN apt-get update && apt-get install -y \
 	&& apt-get clean
 
 # copy source code
-COPY library_generation /src
+COPY hermetic_build/common /src/common
+COPY hermetic_build/library_generation /src/library_generation
 
 # install protoc
 WORKDIR /protoc
-RUN source /src/utils/utilities.sh \
+RUN source /src/library_generation/utils/utilities.sh \
 	&& download_protoc "${PROTOC_VERSION}" "${OS_ARCHITECTURE}"
 # we indicate protoc is available in the container via env vars
 ENV DOCKER_PROTOC_LOCATION=/protoc
 ENV DOCKER_PROTOC_VERSION="${PROTOC_VERSION}"
 
 # install grpc
 WORKDIR /grpc
-RUN source /src/utils/utilities.sh \
+RUN source /src/library_generation/utils/utilities.sh \
 	&& download_grpc_plugin "${GRPC_VERSION}" "${OS_ARCHITECTURE}"
 # similar to protoc, we indicate grpc is available in the container via env vars
 ENV DOCKER_GRPC_LOCATION="/grpc/protoc-gen-grpc-java-${GRPC_VERSION}-${OS_ARCHITECTURE}.exe"
@@ -71,16 +72,16 @@ ENV DOCKER_GRPC_VERSION="${GRPC_VERSION}"
 COPY --from=ggj-build "/sdk-platform-java/gapic-generator-java.jar" "${HOME}/.library_generation/gapic-generator-java.jar"
 RUN chmod 755 "${HOME}/.library_generation/gapic-generator-java.jar"
 
-#  use python 3.11 (the base image has several python versions; here we define the default one)
+#  use python 3.12 (the base image has several python versions; here we define the default one)
 RUN rm $(which python3)
-RUN ln -s $(which python3.11) /usr/local/bin/python
-RUN ln -s $(which python3.11) /usr/local/bin/python3
+RUN ln -s $(which python3.12) /usr/local/bin/python
+RUN ln -s $(which python3.12) /usr/local/bin/python3
 RUN python -m pip install --upgrade pip
 
 # install main scripts as a python package
-WORKDIR /src
-RUN python -m pip install -r requirements.txt
-RUN python -m pip install .
+WORKDIR /
+RUN python -m pip install src/common
+RUN python -m pip install src/library_generation
 
 # Install nvm with node and npm
 ENV NODE_VERSION 20.12.0
@@ -120,4 +121,4 @@ RUN chmod -R a+rw /home
 RUN chmod -R a+rx /home/.nvm
 
 WORKDIR /workspace
-ENTRYPOINT [ "python", "/src/cli/entry_point.py", "generate" ]
+ENTRYPOINT [ "python", "/src/library_generation/cli/entry_point.py", "generate" ]

.github/scripts/action.yaml

@@ -37,11 +37,30 @@ inputs:
 runs:
   using: "composite"
   steps:
+  - uses: actions/setup-python@v5
+    with:
+      python-version: 3.12
+  - name: Copy python script
+    shell: bash
+    run: |
+      set -x
+      # repository root
+      cd ${{ github.action_path }}/../../
+      rsync -rv \
+        --exclude=tests \
+        hermetic_build "${GITHUB_WORKSPACE}"
   - name: Copy shell script
     shell: bash
     run: |
       cd ${{ github.action_path }}
-      cp hermetic_library_generation.sh $GITHUB_WORKSPACE
+      cp hermetic_library_generation.sh "${GITHUB_WORKSPACE}"
+  - name: Install python packages
+    shell: bash
+    run: |
+      cd "${GITHUB_WORKSPACE}"
+      pip install hermetic_build/common
+      pip install hermetic_build/library_generation
+      pip install hermetic_build/release_note_generation
   - name: Generate changed libraries
     shell: bash
     run: |

.github/scripts/hermetic_library_generation.sh

@@ -106,12 +106,16 @@ docker run \
   --current-generation-config-path="${workspace_name}/${generation_config}" \
   --api-definitions-path="${workspace_name}/googleapis"
 
+python hermetic_build/release_note_generation/cli/generate_release_note.py generate \
+  --baseline-generation-config-path="${baseline_generation_config}" \
+  --current-generation-config-path="${generation_config}"
+
 # remove api definitions after generation
 rm -rf "${api_def_dir}"
 
 # commit the change to the pull request.
 rm -rdf output googleapis "${baseline_generation_config}"
-git add --all -- ':!pr_description.txt' ':!hermetic_library_generation.sh'
+git add --all -- ':!pr_description.txt' ':!hermetic_library_generation.sh' ':!hermetic_build'
 changed_files=$(git diff --cached --name-only)
 if [[ "${changed_files}" != "" ]]; then
     echo "Commit changes..."

.github/workflows/verify_library_generation.yaml

@@ -27,7 +27,7 @@ jobs:
           git checkout -b "${head_ref}" fork/${head_ref}
           changed_directories="$(git diff --name-only "fork/${head_ref}" "origin/${base_ref}")"
         fi
-        if [[ ${changed_directories} =~ "library_generation/" ]]; then
+        if [[ ${changed_directories} =~ "hermetic_build/" ]]; then
           echo "should_run=true" >> $GITHUB_OUTPUT
         else
           echo "should_run=false" >> $GITHUB_OUTPUT
@@ -46,32 +46,19 @@ jobs:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5
       with:
-        python-version: 3.11
-    - name: install pyenv
-      shell: bash
-      run: |
-        set -ex
-        curl https://pyenv.run | bash
-        # setup environment
-        export PYENV_ROOT="$HOME/.pyenv"
-        export PATH="$PYENV_ROOT/bin:$PATH"
-        echo "PYENV_ROOT=${PYENV_ROOT}" >> $GITHUB_ENV
-        echo "PATH=${PATH}" >> $GITHUB_ENV
-
-        set +ex
+        python-version: 3.12
     - name: install python dependencies
       shell: bash
       run: |
         set -ex
-        pushd library_generation
-        pip install -r requirements.txt
-        pip install .
-        popd
+        pip install --upgrade pip
+        pip install hermetic_build/common
+        pip install hermetic_build/library_generation
     - name: Run integration tests
       shell: bash
       run: |
         set -x
-        python -m unittest library_generation/test/integration_tests.py
+        python -m unittest hermetic_build/library_generation/tests/integration_tests.py
   library-generation-unit-tests:
     runs-on: ubuntu-22.04
     needs: should-run-library-generation-tests
@@ -80,23 +67,23 @@ jobs:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5
       with:
-        python-version: 3.11
+        python-version: 3.12
     - name: install python dependencies
       shell: bash
       run: |
         set -ex
-        pushd library_generation
-        pip install -r requirements.txt
-        pip install .
-        popd
+        pip install --upgrade pip
+        pip install hermetic_build/common
+        pip install hermetic_build/library_generation
+        pip install hermetic_build/release_note_generation
     - name: Run shell unit tests
       run: |
         set -x
-        library_generation/test/generate_library_unit_tests.sh
+        hermetic_build/library_generation/tests/generate_library_unit_tests.sh
     - name: Run python unit tests
       run: |
         set -x
-        python -m unittest discover -s library_generation/test/ -p "*unit_tests.py"
+        python -m unittest discover -s hermetic_build -p "*unit_tests.py"
   library-generation-lint-shell:
     runs-on: ubuntu-22.04
     needs: should-run-library-generation-tests
@@ -106,7 +93,7 @@ jobs:
     - name: Run ShellCheck
       uses: ludeeus/action-shellcheck@2.0.0
       with:
-        scandir: 'library_generation'
+        scandir: 'hermetic_build'
         format: tty
         severity: error
   library-generation-lint-python:
@@ -115,16 +102,20 @@ jobs:
     if: needs.should-run-library-generation-tests.outputs.should_run == 'true'
     steps:
     - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: 3.12
     - name: install python dependencies
       shell: bash
       run: |
         set -ex
-        pushd library_generation
-        pip install -r requirements.txt
-        popd
+        pip install --upgrade pip
+        pip install hermetic_build/common
+        pip install hermetic_build/library_generation
+        pip install hermetic_build/release_note_generation
     - name: Lint
       shell: bash
       run: |
         # exclude generated golden files
         # exclude owlbot until further refaction
-        black --check library_generation --exclude "(library_generation/test/resources/goldens)"
+        black --check hermetic_build --exclude "(library_generation/tests/resources/goldens)"

library_generation/model/generation_config.py → hermetic_build/common/model/generation_config.py

@@ -16,8 +16,8 @@
 
 import yaml
 from typing import Optional
-from library_generation.model.library_config import LibraryConfig
-from library_generation.model.gapic_config import GapicConfig
+from common.model.library_config import LibraryConfig
+from common.model.gapic_config import GapicConfig
 
 REPO_LEVEL_PARAMETER = "Repo level parameter"
 LIBRARY_LEVEL_PARAMETER = "Library level parameter"

library_generation/model/library_config.py → hermetic_build/common/model/library_config.py

@@ -14,8 +14,8 @@
 # limitations under the License.
 from hashlib import sha256
 from typing import Optional
-from library_generation.model.gapic_config import GapicConfig
-from library_generation.model.gapic_inputs import GapicInputs
+from common.model.gapic_config import GapicConfig
+from common.model.gapic_inputs import GapicInputs
 
 
 MAVEN_COORDINATE_SEPARATOR = ":"

hermetic_build/common/setup.py

@@ -0,0 +1,19 @@
+"""
+Package information of library_generation python scripts
+"""
+
+from setuptools import setup
+
+setup(
+    name="common",
+    version="0.1",
+    python_requires=">=3.12",
+    package_dir={
+        "common": ".",
+    },
+    install_requires=[
+        "black==24.8.0",
+        "parameterized==0.9.0",
+        "PyYAML==6.0.2",
+    ],
+)

library_generation/test/model/gapic_config_unit_tests.py → hermetic_build/common/tests/model/gapic_config_unit_tests.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 import unittest
 
-from library_generation.model.gapic_config import GapicConfig
+from common.model.gapic_config import GapicConfig
 
 
 class GapicConfigTest(unittest.TestCase):

library_generation/test/model/gapic_inputs_unit_tests.py → hermetic_build/common/tests/model/gapic_inputs_unit_tests.py

@@ -3,7 +3,7 @@
 from pathlib import Path
 
 from parameterized import parameterized
-from library_generation.model.gapic_inputs import parse
+from common.model.gapic_inputs import parse
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
 resources_dir = os.path.join(script_dir, "..", "resources")

library_generation/test/model/generation_config_unit_tests.py → hermetic_build/common/tests/model/generation_config_unit_tests.py

@@ -14,8 +14,8 @@
 import os
 import unittest
 from pathlib import Path
-from library_generation.model.generation_config import from_yaml, GenerationConfig
-from library_generation.model.library_config import LibraryConfig
+from common.model.generation_config import from_yaml, GenerationConfig
+from common.model.library_config import LibraryConfig
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
 resources_dir = os.path.join(script_dir, "..", "resources")

library_generation/test/model/library_config_unit_tests.py → hermetic_build/common/tests/model/library_config_unit_tests.py

@@ -13,8 +13,8 @@
 # limitations under the License.
 import unittest
 
-from library_generation.model.gapic_config import GapicConfig
-from library_generation.model.library_config import LibraryConfig
+from common.model.gapic_config import GapicConfig
+from common.model.library_config import LibraryConfig
 
 
 class LibraryConfigTest(unittest.TestCase):

library_generation/DEVELOPMENT.md → hermetic_build/library_generation/DEVELOPMENT.md

@@ -22,8 +22,7 @@ It requires docker and python 3.x to be installed.
 
 ```
 python -m pip install .
-python -m pip install -r requirements.txt
-python -m unittest test/integration_tests.py
+python -m unittest tests/integration_tests.py
 ```
 
 # Running the unit tests
@@ -34,7 +33,7 @@ Every unit test script ends with `unit_tests.py`.
 To avoid them specifying them individually, we can use the following command:
 
 ```bash
-python -m unittest discover -s test/ -p "*unit_tests.py"
+python -m unittest discover -s tests/ -p "*unit_tests.py"
 ```
 
 > [!NOTE]