From ff7a0e0ac0f03cfb7e0fa489f32902f9b2040d7e Mon Sep 17 00:00:00 2001 From: Jeff Ching Date: Fri, 5 Oct 2018 11:08:15 -0700 Subject: [PATCH 1/5] Add discogapic generator --- synthtool/gcp/__init__.py | 2 + synthtool/gcp/discogapic_generator.py | 220 ++++++++++++++++++++++++++ 2 files changed, 222 insertions(+) create mode 100644 synthtool/gcp/discogapic_generator.py diff --git a/synthtool/gcp/__init__.py b/synthtool/gcp/__init__.py index b8748a253..4be803783 100644 --- a/synthtool/gcp/__init__.py +++ b/synthtool/gcp/__init__.py @@ -13,7 +13,9 @@ # limitations under the License. from . import gapic_generator +from . import discogapic_generator from . import common +DiscoGAPICGenerator = discogapic_generator.DiscoGAPICGenerator GAPICGenerator = gapic_generator.GAPICGenerator CommonTemplates = common.CommonTemplates diff --git a/synthtool/gcp/discogapic_generator.py b/synthtool/gcp/discogapic_generator.py new file mode 100644 index 000000000..7b4178e16 --- /dev/null +++ b/synthtool/gcp/discogapic_generator.py @@ -0,0 +1,220 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from pathlib import Path +import tempfile +import platform +import subprocess +import venv + +from synthtool import _tracked_paths +from synthtool import cache +from synthtool import log +from synthtool import shell +from synthtool.sources import git + +ARTMAN_VERSION = os.environ.get("SYNTHTOOL_ARTMAN_VERSION", "latest") +ARTMAN_VENV = cache.get_cache_dir() / "artman_venv" +DISCOVERY_ARTIFACT_MANAGER_URL: str = "git@github.com:googleapis/discovery-artifact-manager.git" + + +# Docker on mac by default cannot use the default temp file location +# instead use the more standard *nix /tmp location\ +if platform.system() == "Darwin": + tempfile.tempdir = "/tmp" + + +def _run_artman(image, root_dir, config, *args): + """Executes artman command in the artman container. + Args: + root_dir: The input directory that will be mounted to artman docker + container as local googleapis directory. + Returns: + The output directory with artman-generated files. + """ + container_name = "artman-docker" + output_dir = root_dir / "artman-genfiles" + + docker_cmd = [ + "docker", + "run", + "--name", + container_name, + "--rm", + "-i", + "-e", + f"HOST_USER_ID={os.getuid()}", + "-e", + f"HOST_GROUP_ID={os.getgid()}", + "-e", + "RUNNING_IN_ARTMAN_DOCKER=True", + "-v", + f"{root_dir}:{root_dir}", + "-v", + f"{output_dir}:{output_dir}", + "-w", + root_dir, + image, + "/bin/bash", + "-c", + ] + + artman_command = " ".join( + map(str, ["artman", "--local", "--config", config, "generate"] + list(args)) + ) + + cmd = docker_cmd + [artman_command] + + shell.run(cmd, cwd=root_dir) + + return output_dir + + +class DiscoGAPICGenerator: + def __init__(self): + + self._ensure_dependencies_installed() + self._install_artman() + self._clone_discovery_artifact_manager() + + def py_library(self, service: str, version: str, **kwargs) -> Path: + """ + Generates the Python Library files using artman/GAPIC + returns a `Path` object + library: path to library. 'google/cloud/speech' + version: version of lib. 'v1' + """ + return self._generate_code(service, version, "python", **kwargs) + + def node_library(self, service: str, version: str, **kwargs) -> Path: + return self._generate_code(service, version, "nodejs", **kwargs) + + nodejs_library = node_library + + def ruby_library(self, service: str, version: str, **kwargs) -> Path: + return self._generate_code(service, version, "ruby", **kwargs) + + def php_library(self, service: str, version: str, **kwargs) -> Path: + return self._generate_code(service, version, "php", **kwargs) + + def java_library(self, service: str, version: str, **kwargs) -> Path: + return self._generate_code(service, version, "java", **kwargs) + + def _generate_code( + self, + service, + version, + language, + config_path=None, + artman_output_name=None + ): + # map the language to the artman argument and subdir of genfiles + GENERATE_FLAG_LANGUAGE = { + "python": ("python_gapic", "python"), + "nodejs": ("nodejs_gapic", "js"), + "ruby": ("ruby_gapic", "ruby"), + "php": ("php_gapic", "php"), + "java": ("java_discogapic", "java"), + } + + if language not in GENERATE_FLAG_LANGUAGE: + raise ValueError("provided language unsupported") + + gapic_language_arg, gen_language = GENERATE_FLAG_LANGUAGE[language] + + if self.discovery_artifact_manager is None: + raise RuntimeError( + f"Unable to generate {config_path}, the googleapis repository" + "is unavailable." + ) + + # Run the code generator. + # $ artman --config path/to/artman_api.yaml generate python_gapic + if config_path is None: + config_path = ( + Path("gapic/google") / service / f"artman_{service}_{version}.yaml" + ) + elif Path(config_path).is_absolute(): + config_path = Path(config_path).relative_to("/") + else: + config_path = Path("gapic/google") / service / Path(config_path) + + if not (self.discovery_artifact_manager / config_path).exists(): + raise FileNotFoundError( + f"Unable to find configuration yaml file: {config_path}." + ) + + log.debug(f"Running generator for {config_path}.") + output_root = _run_artman( + f"googleapis/artman:{ARTMAN_VERSION}", + self.discovery_artifact_manager, + config_path, + gapic_language_arg, + ) + + # Expect the output to be in the artman-genfiles directory. + # example: /artman-genfiles/python/speech-v1 + if artman_output_name is None: + artman_output_name = f"{service}-{version}" + genfiles = output_root / gen_language / artman_output_name + + if not genfiles.exists(): + raise FileNotFoundError( + f"Unable to find generated output of artman: {genfiles}." + ) + + log.success(f"Generated code into {genfiles}.") + + _tracked_paths.add(genfiles) + return genfiles + + def _ensure_dependencies_installed(self): + log.debug("Ensuring dependencies.") + + dependencies = ["docker", "git"] + failed_dependencies = [] + for dependency in dependencies: + return_code = shell.run(["which", dependency], check=False).returncode + if return_code: + failed_dependencies.append(dependency) + + if failed_dependencies: + raise EnvironmentError( + f"Dependencies missing: {', '.join(failed_dependencies)}" + ) + + def _install_artman(self): + if not ARTMAN_VENV.exists(): + venv.main([str(ARTMAN_VENV)]) + + if ARTMAN_VERSION != "latest": + version_specifier = f"=={ARTMAN_VERSION}" + else: + version_specifier = "" + + shell.run( + [ + ARTMAN_VENV / "bin" / "pip", + "install", + "--upgrade", + f"googleapis-artman{version_specifier}", + ] + ) + log.debug("Pulling artman image.") + shell.run(["docker", "pull", f"googleapis/artman:{ARTMAN_VERSION}"]) + + def _clone_discovery_artifact_manager(self): + log.debug("Cloning discovery-artifact-manager.") + self.discovery_artifact_manager = git.clone(DISCOVERY_ARTIFACT_MANAGER_URL, depth=1) From da181a3b541e508f6023c3d7a89925d1c4a44f20 Mon Sep 17 00:00:00 2001 From: Jeff Ching Date: Fri, 5 Oct 2018 11:19:07 -0700 Subject: [PATCH 2/5] blacken --- synthtool/gcp/discogapic_generator.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/synthtool/gcp/discogapic_generator.py b/synthtool/gcp/discogapic_generator.py index 7b4178e16..92880c408 100644 --- a/synthtool/gcp/discogapic_generator.py +++ b/synthtool/gcp/discogapic_generator.py @@ -113,12 +113,7 @@ def java_library(self, service: str, version: str, **kwargs) -> Path: return self._generate_code(service, version, "java", **kwargs) def _generate_code( - self, - service, - version, - language, - config_path=None, - artman_output_name=None + self, service, version, language, config_path=None, artman_output_name=None ): # map the language to the artman argument and subdir of genfiles GENERATE_FLAG_LANGUAGE = { @@ -217,4 +212,6 @@ def _install_artman(self): def _clone_discovery_artifact_manager(self): log.debug("Cloning discovery-artifact-manager.") - self.discovery_artifact_manager = git.clone(DISCOVERY_ARTIFACT_MANAGER_URL, depth=1) + self.discovery_artifact_manager = git.clone( + DISCOVERY_ARTIFACT_MANAGER_URL, depth=1 + ) From 4aa36e38afc15928f44dce26d2cdc1c6f1afe62b Mon Sep 17 00:00:00 2001 From: Jeff Ching Date: Fri, 5 Oct 2018 11:19:20 -0700 Subject: [PATCH 3/5] fix lint --- synthtool/gcp/discogapic_generator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synthtool/gcp/discogapic_generator.py b/synthtool/gcp/discogapic_generator.py index 92880c408..6aaede72d 100644 --- a/synthtool/gcp/discogapic_generator.py +++ b/synthtool/gcp/discogapic_generator.py @@ -16,7 +16,6 @@ from pathlib import Path import tempfile import platform -import subprocess import venv from synthtool import _tracked_paths From 675a936528e6c2ee284c4a14b92d45f4de57b155 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Tue, 9 Oct 2018 13:15:50 -0700 Subject: [PATCH 4/5] Create Artman class --- synthtool/gcp/artman.py | 115 ++++++++++++++++++++++++++ synthtool/gcp/discogapic_generator.py | 104 +---------------------- synthtool/gcp/gapic_generator.py | 104 +---------------------- 3 files changed, 122 insertions(+), 201 deletions(-) create mode 100644 synthtool/gcp/artman.py diff --git a/synthtool/gcp/artman.py b/synthtool/gcp/artman.py new file mode 100644 index 000000000..00661087e --- /dev/null +++ b/synthtool/gcp/artman.py @@ -0,0 +1,115 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import tempfile +import platform +import venv + +from synthtool import cache +from synthtool import log +from synthtool import shell + +ARTMAN_VERSION = os.environ.get("SYNTHTOOL_ARTMAN_VERSION", "latest") +ARTMAN_VENV = cache.get_cache_dir() / "artman_venv" + + +class Artman: + def __init__(self): + # Docker on mac by default cannot use the default temp file location + # # instead use the more standard *nix /tmp location\ + if platform.system() == "Darwin": + tempfile.tempdir = "/tmp" + self._ensure_dependencies_installed() + self._install_artman() + + def run(self, image, root_dir, config, *args): + """Executes artman command in the artman container. + Args: + root_dir: The input directory that will be mounted to artman docker + container as local googleapis directory. + Returns: + The output directory with artman-generated files. + """ + container_name = "artman-docker" + output_dir = root_dir / "artman-genfiles" + + docker_cmd = [ + "docker", + "run", + "--name", + container_name, + "--rm", + "-i", + "-e", + f"HOST_USER_ID={os.getuid()}", + "-e", + f"HOST_GROUP_ID={os.getgid()}", + "-e", + "RUNNING_IN_ARTMAN_DOCKER=True", + "-v", + f"{root_dir}:{root_dir}", + "-v", + f"{output_dir}:{output_dir}", + "-w", + root_dir, + image, + "/bin/bash", + "-c", + ] + + artman_command = " ".join( + map(str, ["artman", "--local", "--config", config, "generate"] + list(args)) + ) + + cmd = docker_cmd + [artman_command] + + shell.run(cmd, cwd=root_dir) + + return output_dir + + def _ensure_dependencies_installed(self): + log.debug("Ensuring dependencies.") + + dependencies = ["docker", "git"] + failed_dependencies = [] + for dependency in dependencies: + return_code = shell.run(["which", dependency], check=False).returncode + if return_code: + failed_dependencies.append(dependency) + + if failed_dependencies: + raise EnvironmentError( + f"Dependencies missing: {', '.join(failed_dependencies)}" + ) + + def _install_artman(self): + if not ARTMAN_VENV.exists(): + venv.main([str(ARTMAN_VENV)]) + + if ARTMAN_VERSION != "latest": + version_specifier = f"=={ARTMAN_VERSION}" + else: + version_specifier = "" + + shell.run( + [ + ARTMAN_VENV / "bin" / "pip", + "install", + "--upgrade", + f"googleapis-artman{version_specifier}", + ] + ) + log.debug("Pulling artman image.") + shell.run(["docker", "pull", f"googleapis/artman:{ARTMAN_VERSION}"]) diff --git a/synthtool/gcp/discogapic_generator.py b/synthtool/gcp/discogapic_generator.py index 6aaede72d..48c224838 100644 --- a/synthtool/gcp/discogapic_generator.py +++ b/synthtool/gcp/discogapic_generator.py @@ -12,80 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os from pathlib import Path -import tempfile -import platform -import venv from synthtool import _tracked_paths -from synthtool import cache from synthtool import log -from synthtool import shell -from synthtool.sources import git +from synthtool.gcp import artman -ARTMAN_VERSION = os.environ.get("SYNTHTOOL_ARTMAN_VERSION", "latest") -ARTMAN_VENV = cache.get_cache_dir() / "artman_venv" DISCOVERY_ARTIFACT_MANAGER_URL: str = "git@github.com:googleapis/discovery-artifact-manager.git" -# Docker on mac by default cannot use the default temp file location -# instead use the more standard *nix /tmp location\ -if platform.system() == "Darwin": - tempfile.tempdir = "/tmp" - - -def _run_artman(image, root_dir, config, *args): - """Executes artman command in the artman container. - Args: - root_dir: The input directory that will be mounted to artman docker - container as local googleapis directory. - Returns: - The output directory with artman-generated files. - """ - container_name = "artman-docker" - output_dir = root_dir / "artman-genfiles" - - docker_cmd = [ - "docker", - "run", - "--name", - container_name, - "--rm", - "-i", - "-e", - f"HOST_USER_ID={os.getuid()}", - "-e", - f"HOST_GROUP_ID={os.getgid()}", - "-e", - "RUNNING_IN_ARTMAN_DOCKER=True", - "-v", - f"{root_dir}:{root_dir}", - "-v", - f"{output_dir}:{output_dir}", - "-w", - root_dir, - image, - "/bin/bash", - "-c", - ] - - artman_command = " ".join( - map(str, ["artman", "--local", "--config", config, "generate"] + list(args)) - ) - - cmd = docker_cmd + [artman_command] - - shell.run(cmd, cwd=root_dir) - - return output_dir - - class DiscoGAPICGenerator: def __init__(self): - - self._ensure_dependencies_installed() - self._install_artman() self._clone_discovery_artifact_manager() def py_library(self, service: str, version: str, **kwargs) -> Path: @@ -151,8 +88,8 @@ def _generate_code( ) log.debug(f"Running generator for {config_path}.") - output_root = _run_artman( - f"googleapis/artman:{ARTMAN_VERSION}", + output_root = artman.Artman().run( + f"googleapis/artman:{artman.ARTMAN_VERSION}", self.discovery_artifact_manager, config_path, gapic_language_arg, @@ -174,41 +111,6 @@ def _generate_code( _tracked_paths.add(genfiles) return genfiles - def _ensure_dependencies_installed(self): - log.debug("Ensuring dependencies.") - - dependencies = ["docker", "git"] - failed_dependencies = [] - for dependency in dependencies: - return_code = shell.run(["which", dependency], check=False).returncode - if return_code: - failed_dependencies.append(dependency) - - if failed_dependencies: - raise EnvironmentError( - f"Dependencies missing: {', '.join(failed_dependencies)}" - ) - - def _install_artman(self): - if not ARTMAN_VENV.exists(): - venv.main([str(ARTMAN_VENV)]) - - if ARTMAN_VERSION != "latest": - version_specifier = f"=={ARTMAN_VERSION}" - else: - version_specifier = "" - - shell.run( - [ - ARTMAN_VENV / "bin" / "pip", - "install", - "--upgrade", - f"googleapis-artman{version_specifier}", - ] - ) - log.debug("Pulling artman image.") - shell.run(["docker", "pull", f"googleapis/artman:{ARTMAN_VERSION}"]) - def _clone_discovery_artifact_manager(self): log.debug("Cloning discovery-artifact-manager.") self.discovery_artifact_manager = git.clone( diff --git a/synthtool/gcp/gapic_generator.py b/synthtool/gcp/gapic_generator.py index 35fc9bce9..6d596272e 100644 --- a/synthtool/gcp/gapic_generator.py +++ b/synthtool/gcp/gapic_generator.py @@ -12,82 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os from pathlib import Path -import tempfile -import platform import subprocess -import venv from synthtool import _tracked_paths -from synthtool import cache from synthtool import log -from synthtool import shell +from synthtool.gcp import artman from synthtool.sources import git -ARTMAN_VERSION = os.environ.get("SYNTHTOOL_ARTMAN_VERSION", "latest") -ARTMAN_VENV = cache.get_cache_dir() / "artman_venv" GOOGLEAPIS_URL: str = "git@github.com:googleapis/googleapis.git" GOOGLEAPIS_PRIVATE_URL: str = ("git@github.com:googleapis/googleapis-private.git") -# Docker on mac by default cannot use the default temp file location -# instead use the more standard *nix /tmp location\ -if platform.system() == "Darwin": - tempfile.tempdir = "/tmp" - - -def _run_artman(image, root_dir, config, *args): - """Executes artman command in the artman container. - Args: - root_dir: The input directory that will be mounted to artman docker - container as local googleapis directory. - Returns: - The output directory with artman-generated files. - """ - container_name = "artman-docker" - output_dir = root_dir / "artman-genfiles" - - docker_cmd = [ - "docker", - "run", - "--name", - container_name, - "--rm", - "-i", - "-e", - f"HOST_USER_ID={os.getuid()}", - "-e", - f"HOST_GROUP_ID={os.getgid()}", - "-e", - "RUNNING_IN_ARTMAN_DOCKER=True", - "-v", - f"{root_dir}:{root_dir}", - "-v", - f"{output_dir}:{output_dir}", - "-w", - root_dir, - image, - "/bin/bash", - "-c", - ] - - artman_command = " ".join( - map(str, ["artman", "--local", "--config", config, "generate"] + list(args)) - ) - - cmd = docker_cmd + [artman_command] - - shell.run(cmd, cwd=root_dir) - - return output_dir - - class GAPICGenerator: def __init__(self): - - self._ensure_dependencies_installed() - self._install_artman() self._clone_googleapis() def py_library(self, service: str, version: str, **kwargs) -> Path: @@ -165,8 +103,9 @@ def _generate_code( ) log.debug(f"Running generator for {config_path}.") - output_root = _run_artman( - f"googleapis/artman:{ARTMAN_VERSION}", + + output_root = artman.Artman().run( + f"googleapis/artman:{artman.ARTMAN_VERSION}", googleapis, config_path, gapic_language_arg, @@ -188,41 +127,6 @@ def _generate_code( _tracked_paths.add(genfiles) return genfiles - def _ensure_dependencies_installed(self): - log.debug("Ensuring dependencies.") - - dependencies = ["docker", "git"] - failed_dependencies = [] - for dependency in dependencies: - return_code = shell.run(["which", dependency], check=False).returncode - if return_code: - failed_dependencies.append(dependency) - - if failed_dependencies: - raise EnvironmentError( - f"Dependencies missing: {', '.join(failed_dependencies)}" - ) - - def _install_artman(self): - if not ARTMAN_VENV.exists(): - venv.main([str(ARTMAN_VENV)]) - - if ARTMAN_VERSION != "latest": - version_specifier = f"=={ARTMAN_VERSION}" - else: - version_specifier = "" - - shell.run( - [ - ARTMAN_VENV / "bin" / "pip", - "install", - "--upgrade", - f"googleapis-artman{version_specifier}", - ] - ) - log.debug("Pulling artman image.") - shell.run(["docker", "pull", f"googleapis/artman:{ARTMAN_VERSION}"]) - def _clone_googleapis(self): log.debug("Cloning googleapis.") self.googleapis = git.clone(GOOGLEAPIS_URL, depth=1) From 8ae81583b9e5fd59860342157c14b13bbef1fd86 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Thu, 11 Oct 2018 09:46:54 -0700 Subject: [PATCH 5/5] Fix lint --- synthtool/gcp/discogapic_generator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synthtool/gcp/discogapic_generator.py b/synthtool/gcp/discogapic_generator.py index 48c224838..b4a4d58c9 100644 --- a/synthtool/gcp/discogapic_generator.py +++ b/synthtool/gcp/discogapic_generator.py @@ -17,6 +17,7 @@ from synthtool import _tracked_paths from synthtool import log from synthtool.gcp import artman +from synthtool.sources import git DISCOVERY_ARTIFACT_MANAGER_URL: str = "git@github.com:googleapis/discovery-artifact-manager.git"