Add an integration test with ansible-lint (#250)

* integration test with ansible-lint

* add lint to test reqs

* Update tests/integration/test_lint.py

Co-authored-by: Sviatoslav Sydorenko (Святослав Сидоренко) <wk.cvs.github@sydorenko.org.ua>

* more hacking

* chore: auto fixes from pre-commit.com hooks

* Update tests/integration/test_lint.py

Co-authored-by: Sviatoslav Sydorenko (Святослав Сидоренко) <wk.cvs.github@sydorenko.org.ua>

* rm ansi code cleanup hack

* improve regex in assert

* chore: auto fixes from pre-commit.com hooks

* update cspell dictionary

* fix mypy lint errors

* different names for fixtures and functions

* add shell equals true for ruff S603

* Revert "add shell equals true for ruff S603"

This reverts commit b1046a7.

* add noqa comments for ruff S603

* address ruff S607 using shutil which

* ignore ruff S603 in config

* remove pointless assert

* chore: auto fixes from pre-commit.com hooks

* avoid shutil which

* add ansiblelint to cspell dictionary

* add comment about ignorning ruff s603

* some more changes

* ignore ruff s105 rule

* Add playbook test

* rm blank line

---------

Co-authored-by: Sviatoslav Sydorenko (Святослав Сидоренко) <wk.cvs.github@sydorenko.org.ua>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Bradley A. Thornton <bthornto@redhat.com>

.config/requirements-test.in

@@ -1,4 +1,5 @@
 argcomplete
+ansible-lint
 black
 coverage[toml]
 mypy

tests/conftest.py

@@ -7,24 +7,20 @@
 
 from pathlib import Path
 from subprocess import CalledProcessError, CompletedProcess
-from typing import TYPE_CHECKING, Any
+from typing import Protocol
 
 import pytest
 
 from ansible_creator.output import Output
 from ansible_creator.utils import TermFeatures
 
 
-if TYPE_CHECKING:
-    from collections.abc import Callable
-
-
 os.environ["HOME"] = str(Path.home())
 os.environ["DEV_WORKSPACE"] = "collections/ansible_collections"
 
 
 @pytest.fixture()
-def cli() -> Callable[[Any], CompletedProcess[str] | CalledProcessError]:
+def cli() -> CliRunCallable:
     """Fixture to run CLI commands.
 
     Returns:
@@ -63,11 +59,35 @@ def home_path() -> Path:
     return Path.home()
 
 
-def cli_run(args: list[str]) -> CompletedProcess[str] | CalledProcessError:
-    """Execute a command using subprocess.
+class CliRunCallable(Protocol):
+    """Callable protocol for cli_run function."""
+
+    def __call__(
+        self,
+        args: str,
+        env: dict[str, str] | None = None,
+    ) -> CompletedProcess[str] | CalledProcessError:
+        """Run a command using subprocess.
+
+        Args:
+            args: Command to run.
+            env: Supplemental environment variables.
+
+        Returns:
+            CompletedProcess: CompletedProcess object.
+            CalledProcessError: CalledProcessError object.
+        """
+
+
+def cli_run(
+    args: str,
+    env: dict[str, str] | None = None,
+) -> CompletedProcess[str] | CalledProcessError:
+    """Run a command using subprocess.
 
     Args:
         args: Command to run.
+        env: Supplemental environment variables.
 
     Returns:
         CompletedProcess: CompletedProcess object.
@@ -76,6 +96,8 @@ def cli_run(args: list[str]) -> CompletedProcess[str] | CalledProcessError:
     updated_env = os.environ.copy()
     # this helps asserting stdout/stderr
     updated_env.update({"LINES": "40", "COLUMNS": "300", "TERM": "xterm-256color"})
+    if env:
+        updated_env.update(env)
     try:
         result = subprocess.run(
             args,

tests/integration/test_init.py

@@ -5,20 +5,20 @@
 import re
 import sys
 
-from collections.abc import Callable
 from pathlib import Path
-from subprocess import CalledProcessError, CompletedProcess
-from typing import Any
+from typing import TYPE_CHECKING
 
 import pytest
 
 
-cli_type = Callable[[Any], CompletedProcess[str] | CalledProcessError]
+if TYPE_CHECKING:
+    from tests.conftest import CliRunCallable
+
 
 CREATOR_BIN = Path(sys.executable).parent / "ansible-creator"
 
 
-def test_run_help(cli: cli_type) -> None:
+def test_run_help(cli: CliRunCallable) -> None:
     """Test running ansible-creator --help.
 
     Args:
@@ -36,7 +36,7 @@ def test_run_help(cli: cli_type) -> None:
     assert "Initialize a new Ansible project." in result.stdout
 
 
-def test_run_no_subcommand(cli: cli_type) -> None:
+def test_run_no_subcommand(cli: CliRunCallable) -> None:
     """Test running ansible-creator without subcommand.
 
     Args:
@@ -47,7 +47,7 @@ def test_run_no_subcommand(cli: cli_type) -> None:
     assert "the following arguments are required: command" in result.stderr
 
 
-def test_run_init_no_input(cli: cli_type) -> None:
+def test_run_init_no_input(cli: CliRunCallable) -> None:
     """Test running ansible-creator init without any input.
 
     Args:
@@ -64,7 +64,7 @@ def test_run_init_no_input(cli: cli_type) -> None:
     argvalues=["init --project ansible-project", "init --init-path /tmp"],
     ids=["project_no_scm", "collection_no_name"],
 )
-def test_run_deprecated_failure(command: str, cli: cli_type) -> None:
+def test_run_deprecated_failure(command: str, cli: CliRunCallable) -> None:
     """Test running ansible-creator init with deprecated options.
 
     Args:
@@ -87,7 +87,7 @@ def test_run_deprecated_failure(command: str, cli: cli_type) -> None:
     ids=("short", "underscore", "no_dot"),
 )
 @pytest.mark.parametrize("command", ("collection", "playbook"))
-def test_run_init_invalid_name(command: str, args: str, expected: str, cli: cli_type) -> None:
+def test_run_init_invalid_name(command: str, args: str, expected: str, cli: CliRunCallable) -> None:
     """Test running ansible-creator init with invalid collection name.
 
     Args:
@@ -102,7 +102,7 @@ def test_run_init_invalid_name(command: str, args: str, expected: str, cli: cli_
     assert expected in result.stderr
 
 
-def test_run_init_basic(cli: cli_type, tmp_path: Path) -> None:
+def test_run_init_basic(cli: CliRunCallable, tmp_path: Path) -> None:
     """Test running ansible-creator init with empty/non-empty/force.
 
     Args:

tests/integration/test_lint.py

@@ -0,0 +1,105 @@
+"""Check fixture content integration with ansible-lint.
+
+The fixture content is compared to the output of ansible-creator in the
+test_run_success_for_collection and test_run_success_ansible_project
+tests.
+"""
+
+from __future__ import annotations
+
+import re
+import sys
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from tests.defaults import FIXTURES_DIR
+
+
+if TYPE_CHECKING:
+    import pytest
+
+    from tests.conftest import CliRunCallable
+
+GALAXY_BIN = Path(sys.executable).parent / "ansible-galaxy"
+LINT_BIN = Path(sys.executable).parent / "ansible-lint"
+
+LINT_RE = re.compile(
+    r"Passed: (?P<failures>\d+) failure\(s\),"
+    r" (?P<warnings>\d+) warning\(s\) on (?P<files>\d+) files.",
+)
+LINT_PROFILE_RE = re.compile(
+    r"Last profile that met the validation criteria was '(?P<profile>\w+)'.",
+)
+
+
+def test_lint_collection(
+    cli: CliRunCallable,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Lint the scaffolded collection with ansible-lint.
+
+    Args:
+        cli: CLI callable.
+        monkeypatch: Monkeypatch fixture.
+    """
+    project_path = FIXTURES_DIR / "collection"
+    monkeypatch.chdir(project_path)
+
+    args = str(LINT_BIN)
+    env = {"NO_COLOR": "1"}
+    result = cli(args=args, env=env)
+
+    assert result.returncode == 0
+
+    match = LINT_RE.search(result.stderr)
+    assert match is not None
+    assert int(match.group("failures")) == 0
+    assert int(match.group("warnings")) == 0
+    assert int(match.group("files")) > 0
+
+    match = LINT_PROFILE_RE.search(result.stderr)
+    assert match is not None
+    assert match.group("profile") == "production"
+
+
+def test_lint_playbook_project(
+    tmp_path: Path,
+    cli: CliRunCallable,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Lint the scaffolded playbook project with ansible-lint.
+
+    This is an expensive test as it installs collections from the requirements.yml file.
+    If it becomes necessary again, consider using a session fixture to install the collections.
+
+    Args:
+        tmp_path: Temporary path.
+        cli: CLI callable.
+        monkeypatch: Monkeypatch fixture.
+    """
+    req_path = str(
+        FIXTURES_DIR / "project" / "playbook_project" / "collections" / "requirements.yml",
+    )
+    dest_path = tmp_path / "collections"
+    galaxy_cmd = f"{GALAXY_BIN} collection install -r {req_path} -p {dest_path}"
+    result = cli(args=galaxy_cmd)
+    assert result.returncode == 0
+
+    project_path = FIXTURES_DIR / "project" / "playbook_project"
+    monkeypatch.chdir(project_path)
+    args = str(LINT_BIN)
+    env = {"NO_COLOR": "1", "ANSIBLE_COLLECTIONS_PATH": str(dest_path)}
+    result = cli(args=args, env=env)
+
+    assert result.returncode == 0
+
+    match = LINT_RE.search(result.stderr)
+    assert match is not None
+    assert int(match.group("failures")) == 0
+    assert int(match.group("warnings")) == 0
+    assert int(match.group("files")) > 0
+
+    match = LINT_PROFILE_RE.search(result.stderr)
+    assert match is not None
+    assert match.group("profile") == "production"