Merge pull request #12 from cidrblock/feature_all_deps

Install all python deps using builder

.config/dictionary.txt

@@ -5,4 +5,5 @@ fileh
 levelname
 levelno
 pip4a
+reqs
 uninstallation

.config/requirements.in

@@ -1 +1,3 @@
+ansible-builder
+subprocess-tee
 pyyaml

.pre-commit-config.yaml

@@ -77,15 +77,17 @@ repos:
           - --output-format=colorized
         additional_dependencies:
           - pytest
-          - tox
           - pyyaml
+          - subprocess_tee
+          - tox
 
   - repo: https://github.com/pre-commit/mirrors-mypy.git
     rev: v1.5.1
     hooks:
       - id: mypy
         additional_dependencies:
           - pytest
+          - subprocess_tee
           - tox
           - types-PyYAML
           - types-setuptools

requirements.txt

@@ -4,20 +4,33 @@
 #
 #    pip-compile --extra=docs --extra=test --no-annotate --output-file=requirements.txt --strip-extras
 #
+ansible-builder==3.0.0
+attrs==23.1.0
+bindep==2.11.0
 cachetools==5.3.1
 chardet==5.2.0
 colorama==0.4.6
 coverage==7.3.0
 distlib==0.3.7
+distro==1.8.0
 execnet==2.0.2
 filelock==3.12.2
 iniconfig==2.0.0
+jsonschema==4.19.0
+jsonschema-specifications==2023.7.1
 packaging==23.1
+parsley==1.3
+pbr==5.11.1
 platformdirs==3.10.0
 pluggy==1.2.0
 pyproject-api==1.5.3
 pytest==7.4.0
 pytest-xdist==3.3.1
 pyyaml==6.0.1
+referencing==0.30.2
+requirements-parser==0.5.0
+rpds-py==0.10.2
+subprocess-tee==0.4.1
 tox==4.9.0
+types-setuptools==68.1.0.1
 virtualenv==20.24.3

src/pip4a/base.py

@@ -32,6 +32,7 @@ def __init__(self: Base, app: App) -> None:
         self.interpreter: Path
         self.site_pkg_path: Path
         self.python_path: Path
+        self._pip_include_test: bool = False
 
     def _set_interpreter(  # noqa: PLR0912
         self: Base,

src/pip4a/constants.py

@@ -7,5 +7,8 @@ class Constants:
     """Constants, for now, for pip4a."""
 
     TEST_REQUIREMENTS_PY = Path("./test-requirements.txt").resolve()
-    REQUIREMENTS_PY = Path("./requirements.txt").resolve()
-    COLLECTION_BUILD_DIR = Path("./build").resolve()
+    WORKING_DIR = Path("./.pip4a_cache").resolve()
+    COLLECTION_BUILD_DIR = WORKING_DIR / "src"
+    DISCOVERED_PYTHON_REQS = WORKING_DIR / "discovered_requirements.txt"
+    DISCOVERED_BINDEP_REQS = WORKING_DIR / "discovered_bindep.txt"
+    INSTALLED_COLLECTIONS = WORKING_DIR / "installed_collections.txt"

src/pip4a/installer.py

@@ -4,13 +4,15 @@
 
 import logging
 import os
+import re
 import shutil
 import subprocess
 
 from pathlib import Path
 
 from .base import Base
 from .constants import Constants as C  # noqa: N817
+from .utils import oxford_join, subprocess_run
 
 
 logger = logging.getLogger(__name__)
@@ -29,13 +31,16 @@ def run(self: Installer) -> None:
         self._set_bindir()
         self._set_site_pkg_path()
 
-        self._pip_install(C.REQUIREMENTS_PY)
-        if "[test]" in self.app.args.collection_specifier:
-            self._pip_install(C.TEST_REQUIREMENTS_PY)
         self._install_core()
         self._install_collection()
         if self.app.args.editable:
             self._swap_editable_collection()
+
+        if "[test]" in self.app.args.collection_specifier:
+            self._pip_include_test = True
+
+        self._discover_deps()
+        self._pip_install()
         self._check_bindep()
 
         if self.app.args.venv and (self.python_path != self.interpreter):
@@ -53,7 +58,7 @@ def _init_build_dir(self: Installer) -> None:
         logger.info(msg)
         if C.COLLECTION_BUILD_DIR.exists():
             shutil.rmtree(C.COLLECTION_BUILD_DIR)
-        C.COLLECTION_BUILD_DIR.mkdir()
+        C.COLLECTION_BUILD_DIR.mkdir(parents=True)
 
     def _install_core(self: Installer) -> None:
         """Install ansible-core if not installed already."""
@@ -63,35 +68,41 @@ def _install_core(self: Installer) -> None:
         msg = "Installing ansible-core."
         logger.info(msg)
         command = f"{self.interpreter} -m pip install ansible-core"
-        msg = f"Running command: {command}"
-        logger.debug(msg)
         try:
-            subprocess.run(
-                command,
-                check=True,
-                capture_output=not self.app.args.verbose,
-                shell=True,  # noqa: S602
-            )
+            subprocess_run(command=command, verbose=self.app.args.verbose)
         except subprocess.CalledProcessError as exc:
             err = f"Failed to install ansible-core: {exc}"
             logger.critical(err)
 
+    def _discover_deps(self: Installer) -> None:
+        """Discover the dependencies."""
+        command = (
+            f"ansible-builder introspect {self.site_pkg_path}"
+            f" --write-pip {C.DISCOVERED_PYTHON_REQS}"
+            f" --write-bindep {C.DISCOVERED_BINDEP_REQS}"
+            " --sanitize"
+        )
+        if self._pip_include_test and C.TEST_REQUIREMENTS_PY.exists():
+            command += f" --user-pip {C.TEST_REQUIREMENTS_PY}"
+        msg = f"Writing discovered python requirements to: {C.DISCOVERED_PYTHON_REQS}"
+        logger.info(msg)
+        msg = f"Writing discovered system package requirements to: {C.DISCOVERED_BINDEP_REQS}"
+        logger.info(msg)
+        try:
+            subprocess_run(command=command, verbose=self.app.args.verbose)
+        except subprocess.CalledProcessError as exc:
+            err = f"Failed to discover requirements: {exc}"
+            logger.critical(err)
+
     def _install_collection(self: Installer) -> None:
         """Install the collection from the build directory."""
         self._init_build_dir()
 
         command = f"cp -r --parents $(git ls-files 2> /dev/null || ls) {C.COLLECTION_BUILD_DIR}"
         msg = "Copying collection to build directory using git ls-files."
         logger.info(msg)
-        msg = f"Running command: {command}"
-        logger.debug(msg)
         try:
-            subprocess.run(
-                command,
-                check=True,
-                capture_output=not self.app.args.verbose,
-                shell=True,  # noqa: S602
-            )
+            subprocess_run(command=command, verbose=self.app.args.verbose)
         except subprocess.CalledProcessError as exc:
             err = f"Failed to copy collection to build directory: {exc}"
             logger.critical(err)
@@ -100,20 +111,14 @@ def _install_collection(self: Installer) -> None:
             f"cd {C.COLLECTION_BUILD_DIR} &&"
             f" {self.bindir / 'ansible-galaxy'} collection build"
             f" --output-path {C.COLLECTION_BUILD_DIR}"
+            " --force"
         )
 
         msg = "Running ansible-galaxy to build collection."
         logger.info(msg)
-        msg = f"Running command: {command}"
-        logger.debug(msg)
 
         try:
-            subprocess.run(
-                command,
-                check=True,
-                capture_output=not self.app.args.verbose,
-                shell=True,  # noqa: S602
-            )
+            subprocess_run(command=command, verbose=self.app.args.verbose)
         except subprocess.CalledProcessError as exc:
             err = f"Failed to build collection: {exc} {exc.stderr}"
             logger.critical(err)
@@ -131,28 +136,42 @@ def _install_collection(self: Installer) -> None:
             raise RuntimeError(err)
         tarball = built[0]
 
+        # Remove installed collection if it exists
+        site_pkg_collection_path = (
+            self.site_pkg_path
+            / "ansible_collections"
+            / self.app.collection_name.split(".")[0]
+            / self.app.collection_name.split(".")[1]
+        )
+        if site_pkg_collection_path.exists():
+            msg = f"Removing installed {site_pkg_collection_path}"
+            logger.debug(msg)
+            if site_pkg_collection_path.is_symlink():
+                site_pkg_collection_path.unlink()
+            else:
+                shutil.rmtree(site_pkg_collection_path)
+
         command = (
             f"{self.bindir / 'ansible-galaxy'} collection"
             f" install {tarball} -p {self.site_pkg_path}"
+            " --force"
         )
         env = os.environ
         if not self.app.args.verbose:
             env["ANSIBLE_GALAXY_COLLECTIONS_PATH_WARNING"] = "false"
         msg = "Running ansible-galaxy to install collection and it's dependencies."
         logger.info(msg)
-        msg = f"Running command: {command}"
-        logger.debug(msg)
         try:
-            subprocess.run(
-                command,
-                check=True,
-                capture_output=not self.app.args.verbose,
-                env=env,
-                shell=True,  # noqa: S602
-            )
+            proc = subprocess_run(command=command, verbose=self.app.args.verbose)
         except subprocess.CalledProcessError as exc:
             err = f"Failed to install collection: {exc} {exc.stderr}"
             logger.critical(err)
+            return
+        installed = re.findall(r"(\w+\.\w+):.*installed", proc.stdout)
+        msg = f"Installed collections: {oxford_join(installed)}"
+        logger.info(msg)
+        with C.INSTALLED_COLLECTIONS.open(mode="w") as f:
+            f.write("\n".join(installed))
 
     def _swap_editable_collection(self: Installer) -> None:
         """Swap the installed collection with the current working directory."""
@@ -176,98 +195,37 @@ def _swap_editable_collection(self: Installer) -> None:
         logger.info(msg)
         site_pkg_collection_path.symlink_to(cwd)
 
-    def _pip_install(self: Installer, requirements_file: Path) -> None:
+    def _pip_install(self: Installer) -> None:
         """Install the dependencies."""
-        if not requirements_file.exists():
-            msg = f"Requirements file {requirements_file} does not exist, skipping"
-            logger.info(msg)
-            return
+        command = f"{self.interpreter} -m pip install -r {C.DISCOVERED_PYTHON_REQS}"
 
-        if requirements_file.stat().st_size == 0:
-            msg = f"Requirements file {requirements_file} is empty, skipping"
-            logger.info(msg)
-            return
-
-        command = f"{self.interpreter} -m pip install -r {requirements_file}"
-
-        msg = f"Installing python requirements from {requirements_file}"
+        msg = f"Installing python requirements from {C.DISCOVERED_PYTHON_REQS}"
         logger.info(msg)
-        msg = f"Running command: {command}"
-
-        logger.debug(msg)
         try:
-            subprocess.run(
-                command,
-                check=True,
-                capture_output=not self.app.args.verbose,
-                shell=True,  # noqa: S602
-            )
+            subprocess_run(command=command, verbose=self.app.args.verbose)
         except subprocess.CalledProcessError as exc:
-            err = f"Failed to install requirements from {requirements_file}: {exc}"
+            err = (
+                f"Failed to install requirements from {C.DISCOVERED_PYTHON_REQS}: {exc}"
+            )
             raise RuntimeError(err) from exc
 
     def _check_bindep(self: Installer) -> None:
         """Check the bindep file."""
-        bindep = Path("./bindep.txt").resolve()
-        if not bindep.exists():
-            msg = f"System package requirements file {bindep} does not exist, skipping"
-            logger.info(msg)
-            return
-        msg = f"bindep file found: {bindep}"
-        logger.debug(msg)
-
-        command = f"{self.interpreter} -m pip show bindep"
-        msg = f"Running command: {command}"
-        logger.debug(msg)
+        command = f"{self.bindir / 'bindep'} -b -f {C.DISCOVERED_BINDEP_REQS}"
         try:
-            subprocess.run(
-                command,
-                check=True,
-                shell=True,  # noqa: S602
+            subprocess_run(command=command, verbose=self.app.args.verbose)
+        except subprocess.CalledProcessError as exc:
+            lines = exc.stdout.splitlines()
+            msg = (
+                "Required system packages are missing."
+                " Please use the system package manager to install them."
             )
-            bindep_found = True
-        except subprocess.CalledProcessError:
-            bindep_found = False
-
-        msg = f"bindep found: {bindep_found}"
-        logger.debug(msg)
-
-        if not bindep_found:
-            msg = f"Installing bindep for: {bindep}"
-            logger.info(msg)
-            command = f"{self.interpreter} -m pip install bindep"
-            try:
-                subprocess.run(
-                    command,
-                    check=True,
-                    capture_output=not self.app.args.verbose,
-                    shell=True,  # noqa: S602
-                )
-            except subprocess.CalledProcessError as exc:
-                err = f"Failed to install bindep: {exc}"
-                logger.critical(err)
-
-        command = f"{self.bindir / 'bindep'} -b -f {bindep}"
-        msg = f"Running command: {command}"
-        logger.debug(msg)
-        proc = subprocess.run(
-            command,
-            check=False,
-            capture_output=True,
-            shell=True,  # noqa: S602
-            text=True,
-        )
-        if proc.returncode == 0:
+            logger.warning(msg)
+            for line in lines:
+                msg = f"Missing: {line}"
+                logger.warning(msg)
+                pass
+        else:
             msg = "All required system packages are installed."
             logger.debug(msg)
             return
-
-        lines = proc.stdout.splitlines()
-        msg = (
-            "Required system packages are missing."
-            " Please use the system package manager to install them."
-        )
-        logger.warning(msg)
-        for line in lines:
-            msg = f"Missing: {line}"
-            logger.warning(msg)