Make collection location cache_dir aware (#384)

src/ansible_compat/config.py

@@ -8,14 +8,17 @@
 import re
 import subprocess
 from collections import UserDict
-from typing import Literal
+from typing import TYPE_CHECKING, Literal
 
 from packaging.version import Version
 
 from ansible_compat.constants import ANSIBLE_MIN_VERSION
 from ansible_compat.errors import InvalidPrerequisiteError, MissingAnsibleError
 from ansible_compat.ports import cache
 
+if TYPE_CHECKING:
+    from pathlib import Path
+
 
 # do not use lru_cache here, as environment can change between calls
 def ansible_collections_path() -> str:
@@ -397,35 +400,49 @@ def __init__(
         self,
         config_dump: str | None = None,
         data: dict[str, object] | None = None,
+        cache_dir: Path | None = None,
     ) -> None:
         """Load config dictionary."""
         super().__init__()
 
+        self.cache_dir = cache_dir
         if data:
             self.data = copy.deepcopy(data)
-            return
-
-        if not config_dump:
-            env = os.environ.copy()
-            # Avoid possible ANSI garbage
-            env["ANSIBLE_FORCE_COLOR"] = "0"
-            config_dump = subprocess.check_output(
-                ["ansible-config", "dump"],  # noqa: S603
-                universal_newlines=True,
-                env=env,
-            )
+        else:
+            if not config_dump:
+                env = os.environ.copy()
+                # Avoid possible ANSI garbage
+                env["ANSIBLE_FORCE_COLOR"] = "0"
+                config_dump = subprocess.check_output(
+                    ["ansible-config", "dump"],  # noqa: S603
+                    universal_newlines=True,
+                    env=env,
+                )
 
-        for match in re.finditer(
-            r"^(?P<key>[A-Za-z0-9_]+).* = (?P<value>.*)$",
-            config_dump,
-            re.MULTILINE,
-        ):
-            key = match.groupdict()["key"]
-            value = match.groupdict()["value"]
-            try:
-                self[key] = ast.literal_eval(value)
-            except (NameError, SyntaxError, ValueError):
-                self[key] = value
+            for match in re.finditer(
+                r"^(?P<key>[A-Za-z0-9_]+).* = (?P<value>.*)$",
+                config_dump,
+                re.MULTILINE,
+            ):
+                key = match.groupdict()["key"]
+                value = match.groupdict()["value"]
+                try:
+                    self[key] = ast.literal_eval(value)
+                except (NameError, SyntaxError, ValueError):
+                    self[key] = value
+        # inject isolation collections paths into the config
+        if self.cache_dir:
+            cpaths = self.data["COLLECTIONS_PATHS"]
+            if cpaths and isinstance(cpaths, list):
+                cpaths.insert(
+                    0,
+                    f"{self.cache_dir}/collections",
+                )
+            else:  # pragma: no cover
+                msg = f"Unexpected data type for COLLECTIONS_PATHS: {cpaths}"
+                raise RuntimeError(msg)
+        if data:
+            return
 
     def __getattribute__(self, attr_name: str) -> object:
         """Allow access of config options as attributes."""

src/ansible_compat/runtime.py

@@ -209,7 +209,7 @@ def __init__(
 
         if isolated:
             self.cache_dir = get_cache_dir(self.project_dir)
-        self.config = AnsibleConfig()
+        self.config = AnsibleConfig(cache_dir=self.cache_dir)
 
         # Add the sys.path to the collection paths if not isolated
         self._add_sys_path_to_collection_paths()
@@ -273,13 +273,13 @@ def load_collections(self) -> None:
         self.collections = OrderedDict()
         no_collections_msg = "None of the provided paths were usable"
 
+        # do not use --path because it does not allow multiple values
         proc = self.run(
             [
                 "ansible-galaxy",
                 "collection",
                 "list",
                 "--format=json",
-                f"-p={':'.join(self.config.collections_paths)}",
             ],
         )
         if proc.returncode == RC_ANSIBLE_OPTIONS_ERROR and (
@@ -392,6 +392,8 @@ def run(  # ruff: disable=PLR0913
         # https://github.com/ansible/ansible-lint/issues/3522
         env["ANSIBLE_VERBOSE_TO_STDERR"] = "True"
 
+        env["ANSIBLE_COLLECTIONS_PATH"] = ":".join(self.config.collections_paths)
+
         for _ in range(self.max_retries + 1 if retry else 1):
             result = run_func(
                 args,
@@ -520,7 +522,7 @@ def install_collection(
             env={**self.environ, ansible_collections_path(): ":".join(cpaths)},
         )
         if process.returncode != 0:
-            msg = f"Command returned {process.returncode} code:\n{process.stdout}\n{process.stderr}"
+            msg = f"Command {' '.join(cmd)}, returned {process.returncode} code:\n{process.stdout}\n{process.stderr}"
             _logger.error(msg)
             raise InvalidPrerequisiteError(msg)
 
@@ -608,19 +610,10 @@ def install_requirements(  # noqa: C901
                 )
             else:
                 cmd.extend(["-r", str(requirement)])
-                cpaths = self.config.collections_paths
-                if self.cache_dir:
-                    # we cannot use '-p' because it breaks galaxy ability to ignore already installed collections, so
-                    # we hack ansible_collections_path instead and inject our own path there.
-                    dest_path = f"{self.cache_dir}/collections"
-                    if dest_path not in cpaths:
-                        # pylint: disable=no-member
-                        cpaths.insert(0, dest_path)
                 _logger.info("Running %s", " ".join(cmd))
                 result = self.run(
                     cmd,
                     retry=retry,
-                    env={**os.environ, "ANSIBLE_COLLECTIONS_PATH": ":".join(cpaths)},
                 )
                 _logger.debug(result.stdout)
                 if result.returncode != 0:
@@ -757,12 +750,6 @@ def require_collection(
                 msg,
             )
 
-        if self.cache_dir:
-            # if we have a cache dir, we want to be use that would be preferred
-            # destination when installing a missing collection
-            # https://github.com/PyCQA/pylint/issues/4667
-            paths.insert(0, f"{self.cache_dir}/collections")  # pylint: disable=E1101
-
         for path in paths:
             collpath = Path(path) / "ansible_collections" / ns / coll
             if collpath.exists():

test/test_version.py

@@ -6,7 +6,7 @@ def test_version_module() -> None:
     # import kept here to allow mypy/pylint to run when module is not installed
     # and the generated _version.py is missing.
     # pylint: disable=no-name-in-module,no-member
-    import ansible_compat._version  # type: ignore[import-not-found]
+    import ansible_compat._version  # type: ignore[import-not-found,unused-ignore]
 
     assert ansible_compat._version.__version__
     assert ansible_compat._version.__version_tuple__