pw_build: Update Python GN templates

This CL adds new targets to pw_create_python_source_tree to match
existing targets created by pw_python_package.

- .install
  pip installs a merged Python package
- .wheel
  Creates a distributable wheel

python_package.py changes here are used for gathering Python
dependency info in the absense of package setup information. For
example: a Python library that is not a pip installable Python
package.

- Make setup.cfg (setup sources in general) optional for saved GN
  python package metadata.
- Add top_level_source_dir property to find the root folder of a
  Python dependency.
- Update package_dir to fall back on top_level_source_dir if no
  setup.cfg files are present.
- Update package_name to fall back on top_level_source_dir or the gn
  target name.

Change-Id: I2b46daf4c7d09fe6b16190b9a0c4ef6d4e3d86ea
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/92141
Reviewed-by: Armando Montanez <amontanez@google.com>
Commit-Queue: Anthony DiGirolamo <tonymd@google.com>

pw_build/py/pw_build/create_python_tree.py

@@ -24,7 +24,11 @@
 import tempfile
 from typing import Iterable
 
-from pw_build.python_package import PythonPackage, load_packages
+try:
+    from pw_build.python_package import PythonPackage, load_packages
+except ImportError:
+    # Load from python_package from this directory if pw_build is not available.
+    from python_package import PythonPackage, load_packages  # type: ignore
 
 
 def _parse_args():
@@ -84,7 +88,7 @@ def get_current_git_sha() -> str:
 
 
 def get_current_date() -> str:
-    return datetime.now().strftime('%Y%m%d%H%M')
+    return datetime.now().strftime('%Y%m%d%H%M%S')
 
 
 class UnexpectedConfigSection(Exception):
@@ -145,7 +149,9 @@ def update_config_with_packages(
     included_packages = [pkg.package_name for pkg in python_packages]
 
     for pkg in python_packages:
-        assert pkg.config
+        # Skip this package if no setup.cfg is defined.
+        if not pkg.config:
+            continue
 
         # Collect install_requires
         if pkg.config.has_option('options', 'install_requires'):

pw_build/py/pw_build/python_package.py

@@ -54,6 +54,17 @@ def change_working_dir(directory: Path):
         os.chdir(original_dir)
 
 
+class UnknownPythonPackageName(Exception):
+    """Exception thrown when a Python package_name cannot be determined."""
+
+
+class MissingSetupSources(Exception):
+    """Exception thrown when a Python package is missing setup source files.
+
+    For example: setup.cfg and pyproject.toml.i
+    """
+
+
 @dataclass
 class PythonPackage:
     """Class to hold a single Python package's metadata."""
@@ -62,7 +73,7 @@ class PythonPackage:
     setup_sources: List[Path]
     tests: List[Path]
     inputs: List[Path]
-    gn_target_name: Optional[str] = None
+    gn_target_name: str = ''
     generate_setup: Optional[Dict] = None
     config: Optional[configparser.ConfigParser] = None
 
@@ -85,8 +96,9 @@ def __post_init__(self):
             self.config = self._load_config()
 
     @property
-    def setup_dir(self) -> Path:
-        assert len(self.setup_sources) > 0
+    def setup_dir(self) -> Optional[Path]:
+        if not self.setup_sources:
+            return None
         # Assuming all setup_source files live in the same parent directory.
         return self.setup_sources[0].parent
 
@@ -101,22 +113,53 @@ def setup_py(self) -> Path:
         return setup_py[0]
 
     @property
-    def setup_cfg(self) -> Path:
+    def setup_cfg(self) -> Optional[Path]:
         setup_cfg = [
             setup_file for setup_file in self.setup_sources
             if str(setup_file).endswith('setup.cfg')
         ]
-        assert len(setup_cfg) == 1
+        if len(setup_cfg) < 1:
+            return None
         return setup_cfg[0]
 
     @property
     def package_name(self) -> str:
-        assert self.config
-        return self.config['metadata']['name']
+        if self.config:
+            return self.config['metadata']['name']
+        top_level_source_dir = self.top_level_source_dir
+        if top_level_source_dir:
+            return top_level_source_dir.name
+
+        actual_gn_target_name = self.gn_target_name.split(':')
+        if len(actual_gn_target_name) < 2:
+            raise UnknownPythonPackageName(
+                'Cannot determine the package_name for the Python '
+                f'library/package: {self}')
+
+        return actual_gn_target_name[-1]
 
     @property
     def package_dir(self) -> Path:
-        return self.setup_cfg.parent / self.package_name
+        if self.setup_cfg:
+            return self.setup_cfg.parent / self.package_name
+        root_source_dir = self.top_level_source_dir
+        if root_source_dir:
+            return root_source_dir
+        return self.sources[0].parent
+
+    @property
+    def top_level_source_dir(self) -> Optional[Path]:
+        source_dir_paths = sorted(set(
+            (len(sfile.parts), sfile.parent) for sfile in self.sources),
+                                  key=lambda s: s[1])
+        if not source_dir_paths:
+            return None
+
+        top_level_source_dir = source_dir_paths[0][1]
+        if not top_level_source_dir.is_dir():
+            return None
+
+        return top_level_source_dir
 
     def _load_config(self) -> Optional[configparser.ConfigParser]:
         config = configparser.ConfigParser()
@@ -127,9 +170,21 @@ def _load_config(self) -> Optional[configparser.ConfigParser]:
             return config
         return None
 
+    def copy_sources_to(self, destination: Path) -> None:
+        """Copy this PythonPackage source files to another path."""
+        new_destination = destination / self.package_dir.name
+        new_destination.mkdir(parents=True, exist_ok=True)
+        shutil.copytree(self.package_dir, new_destination, dirs_exist_ok=True)
+
     def setuptools_build_with_base(self,
                                    build_base: Path,
                                    include_tests: bool = False) -> Path:
+        """Run setuptools build for this package."""
+        # If there is no setup_dir or setup_sources, just copy this packages
+        # source files.
+        if not self.setup_dir:
+            self.copy_sources_to(build_base)
+            return build_base
         # Create the lib install dir in case it doesn't exist.
         lib_dir_path = build_base / 'lib'
         lib_dir_path.mkdir(parents=True, exist_ok=True)
@@ -158,11 +213,24 @@ def setuptools_build_with_base(self,
 
         return lib_dir_path
 
-    def setuptools_develop(self) -> None:
+    def setuptools_develop(self, no_deps=False) -> None:
+        if not self.setup_dir:
+            raise MissingSetupSources(
+                'Cannot find setup source file root folder (the location of '
+                f'setup.cfg) for the Python library/package: {self}')
+
         with change_working_dir(self.setup_dir):
-            setuptools.setup(script_args=['develop'])
+            develop_args = ['develop']
+            if no_deps:
+                develop_args.append('--no-deps')
+            setuptools.setup(script_args=develop_args)
 
     def setuptools_install(self) -> None:
+        if not self.setup_dir:
+            raise MissingSetupSources(
+                'Cannot find setup source file root folder (the location of '
+                f'setup.cfg) for the Python library/package: {self}')
+
         with change_working_dir(self.setup_dir):
             setuptools.setup(script_args=['install'])
 
@@ -197,12 +265,14 @@ def install_requires_entries(self) -> List[str]:
         return this_requires
 
 
-def load_packages(input_list_files: Iterable[Path]) -> List[PythonPackage]:
+def load_packages(input_list_files: Iterable[Path],
+                  ignore_missing=False) -> List[PythonPackage]:
     """Load Python package metadata and configs."""
 
     packages = []
     for input_path in input_list_files:
-
+        if ignore_missing and not input_path.is_file():
+            continue
         with input_path.open() as input_file:
             # Each line contains the path to a json file.
             for json_file in input_file.readlines():

pw_build/python.rst

@@ -289,6 +289,17 @@ Arguments
        append_date_to_version = true
      }
 
+Using this template will create additional targets for installing and building a
+Python wheel. For example if you define ``pw_create_python_source_tree("awesome")``
+the 3 resulting targets that get created will be:
+
+- ``awesome`` - This will create the merged package with all source files in
+  place in the out directory under ``out/obj/awesome/``.
+- ``awesome.wheel`` - This builds a Python wheel from the above source files
+  under ``out/obj/awesome._build_wheel/awesome*.whl``.
+- ``awesome.install`` - This pip installs the merged package into the user's
+  development environment.
+
 Example
 -------
 

pw_build/python_dist.gni

@@ -154,9 +154,11 @@ template("pw_python_zip_with_setup") {
 #     format of each item in this list follows this convention:
 #       //some/nested/source_file > nested/destination_file
 template("pw_create_python_source_tree") {
+  _metadata_path_list_suffix =
+      "_pw_create_python_source_tree_metadata_path_list.txt"
   _output_dir = "${target_out_dir}/${target_name}/"
   _metadata_json_file_list =
-      "${target_gen_dir}/${target_name}_metadata_path_list.txt"
+      "${target_gen_dir}/${target_name}${_metadata_path_list_suffix}"
 
   # If generating a setup.cfg file a common base file must be provided.
   if (defined(invoker.generate_setup_cfg)) {
@@ -200,9 +202,14 @@ template("pw_create_python_source_tree") {
 
   _include_tests = defined(invoker.include_tests) && invoker.include_tests
 
+  _public_deps = []
+  if (defined(invoker.public_deps)) {
+    _public_deps += invoker.public_deps
+  }
+
   # Build a list of relative paths containing all the python
   # package_metadata.json files we depend on.
-  generated_file("${target_name}._metadata_path_list.txt") {
+  generated_file("${target_name}.${_metadata_path_list_suffix}") {
     data_keys = [ "pw_python_package_metadata_json" ]
     rebase = root_build_dir
     deps = invoker.packages
@@ -211,10 +218,12 @@ template("pw_create_python_source_tree") {
 
   # Run the python action on the metadata_path_list.txt file
   pw_python_action(target_name) {
-    deps =
-        invoker.packages + [ ":${invoker.target_name}._metadata_path_list.txt" ]
+    deps = invoker.packages +
+           [ ":${invoker.target_name}.${_metadata_path_list_suffix}" ]
+
     script = "$dir_pw_build/py/pw_build/create_python_tree.py"
     inputs = _extra_file_inputs
+    public_deps = _public_deps
 
     args = [
       "--tree-destination-dir",
@@ -254,4 +263,72 @@ template("pw_create_python_source_tree") {
       args += [ "--include-tests" ]
     }
   }
+
+  # Template to install a bundled Python package.
+  pw_python_action("$target_name.install") {
+    module = "pip"
+    public_deps = []
+    if (defined(invoker.public_deps)) {
+      public_deps += invoker.public_deps
+    }
+
+    args = [
+      "install",
+
+      # This speeds up pip installs. At this point in the gn build the
+      # virtualenv is already activated so build isolation isn't required.
+      # This requires that pip, setuptools, and wheel packages are
+      # installed.
+      "--no-build-isolation",
+    ]
+    public_deps += [ ":${invoker.target_name}" ]
+
+    inputs = pw_build_PIP_CONSTRAINTS
+    foreach(_constraints_file, pw_build_PIP_CONSTRAINTS) {
+      args += [
+        "--constraint",
+        rebase_path(_constraints_file, root_build_dir),
+      ]
+    }
+    stamp = true
+    args += [ rebase_path(_output_dir, root_build_dir) ]
+  }
+
+  # Template to build a bundled Python package wheel.
+  pw_python_action("$target_name._build_wheel") {
+    metadata = {
+      pw_python_package_wheels = [ "$target_out_dir/$target_name" ]
+    }
+    module = "build"
+    args = [
+             rebase_path(_output_dir, root_build_dir),
+             "--wheel",
+             "--no-isolation",
+             "--outdir",
+           ] + rebase_path(metadata.pw_python_package_wheels, root_build_dir)
+
+    public_deps = []
+    if (defined(invoker.public_deps)) {
+      public_deps += invoker.public_deps
+    }
+    public_deps += [ ":${invoker.target_name}" ]
+
+    stamp = true
+  }
+  group("$target_name.wheel") {
+    public_deps = [ ":${invoker.target_name}._build_wheel" ]
+  }
+
+  # Stub target groups to match a pw_python_package. This lets $target_name be
+  # used as a python_dep in pw_python_group.
+  group("$target_name._run_pip_install") {
+  }
+  group("$target_name.lint") {
+  }
+  group("$target_name.lint.mypy") {
+  }
+  group("$target_name.lint.pylint") {
+  }
+  group("$target_name.tests") {
+  }
 }

pw_env_setup/BUILD.gn

@@ -94,6 +94,7 @@ pw_python_requirements("renode_requirements") {
   ]
 }
 
+# This target creates a Python source tree with some included bazel build files.
 pw_create_python_source_tree("build_pigweed_python_source_tree") {
   packages = _pigweed_python_deps
   include_tests = true
@@ -108,6 +109,8 @@ pw_create_python_source_tree("build_pigweed_python_source_tree") {
   ]
 }
 
+# This target is responsible for building the Python source uploaded to PyPI:
+# https://pypi.org/project/pigweed/
 pw_create_python_source_tree("pypi_pigweed_python_source_tree") {
   packages = _pigweed_python_deps
   generate_setup_cfg = {