feat: updates skip_items to accept glob patterns

Signed-off-by: Jennifer Power <barnabei.jennifer@gmail.com>

action.yml

@@ -12,20 +12,20 @@ inputs:
   check_only: 
     description: "Runs tasks and exits with an error if there is a diff. Defaults to false"
     required: false
-    default: false
+    default: "false"
   github_token:
     description: "GitHub token used to make authenticated API requests"
     required: false
   skip_assemble:
     description: "Skip assembly task. Defaults to false"
     required: false
-    default: false
+    default: "false"
   skip_regenerate:
     description: "Skip regenerate task. Defaults to false."
     required: false
-    default: false
+    default: "false"
   skip_items:
-    description: "Comma-separated list of content by Trestle name to skip during task execution. For example `profile_x,profile_y`."
+    description: "Comma-separated glob patterns list of content by Trestle name to skip during task execution. For example `profile_x,profile_y*,`."
     required: false
   ssp_index_path:
     description: Path relative to the repository path where the ssp index is located. See project README.md for information about the ssp index.

tests/trestlebot/tasks/test_assemble_task.py

@@ -18,6 +18,7 @@
 
 import os
 import pathlib
+from typing import List
 from unittest.mock import Mock, patch
 
 import pytest
@@ -70,6 +71,41 @@ def test_assemble_task_isolated(tmp_trestle_dir: str) -> None:
         mock.assemble.assert_called_once_with(markdown_path=md_path)
 
 
+@pytest.mark.parametrize(
+    "skip_list",
+    [
+        ["simplified_nist_catalog"],
+        ["*nist*"],
+        ["simplified_nist_catalog", "simplified_nist_profile"],
+        ["*catalog", "*profile"],
+    ],
+)
+def test_assemble_task_with_skip(tmp_trestle_dir: str, skip_list: List[str]) -> None:
+    """Test ssp assemble with a skip list"""
+    trestle_root = pathlib.Path(tmp_trestle_dir)
+    md_path = os.path.join(cat_md_dir, test_cat)
+    args = testutils.setup_for_catalog(trestle_root, test_cat, md_path)
+    cat_generate = CatalogGenerate()
+    assert cat_generate._run(args) == 0
+
+    mock = Mock(spec=AuthorObjectBase)
+
+    assemble_task = AssembleTask(
+        working_dir=tmp_trestle_dir,
+        authored_model=AuthoredType.CATALOG.value,
+        markdown_dir=cat_md_dir,
+        skip_model_list=skip_list,
+    )
+
+    with patch(
+        "trestlebot.tasks.authored.types.get_authored_object"
+    ) as mock_get_authored_object:
+        mock_get_authored_object.return_value = mock
+
+        assert assemble_task.execute() == 0
+        mock.assemble.assert_not_called()
+
+
 def test_catalog_assemble_task(tmp_trestle_dir: str) -> None:
     """Test catalog assemble at the task level"""
     trestle_root = pathlib.Path(tmp_trestle_dir)

tests/trestlebot/tasks/test_regenerate_task .py

@@ -19,6 +19,7 @@
 import argparse
 import os
 import pathlib
+from typing import List
 from unittest.mock import Mock, patch
 
 import pytest
@@ -68,6 +69,40 @@ def test_regenerate_task_isolated(tmp_trestle_dir: str) -> None:
         )
 
 
+@pytest.mark.parametrize(
+    "skip_list",
+    [
+        ["simplified_nist_catalog"],
+        ["*nist*"],
+        ["simplified_nist_catalog", "simplified_nist_profile"],
+        ["*catalog", "*profile"],
+    ],
+)
+def test_regenerate_task_with_skip(tmp_trestle_dir: str, skip_list: List[str]) -> None:
+    """Test the regenerate task isolated from AuthoredObject implementation"""
+    trestle_root = pathlib.Path(tmp_trestle_dir)
+    md_path = os.path.join(cat_md_dir, test_cat)
+    _ = testutils.setup_for_catalog(trestle_root, test_cat, md_path)
+
+    mock = Mock(spec=AuthorObjectBase)
+
+    regenerate_task = RegenerateTask(
+        working_dir=tmp_trestle_dir,
+        authored_model=AuthoredType.CATALOG.value,
+        markdown_dir=cat_md_dir,
+        skip_model_list=skip_list,
+    )
+
+    with patch(
+        "trestlebot.tasks.authored.types.get_authored_object"
+    ) as mock_get_authored_object:
+        mock_get_authored_object.return_value = mock
+
+        assert regenerate_task.execute() == 0
+
+        mock.regenerate.assert_not_called()
+
+
 def test_catalog_regenerate_task(tmp_trestle_dir: str) -> None:
     """Test catalog regenerate at the task level"""
     trestle_root = pathlib.Path(tmp_trestle_dir)

trestlebot/cli.py

@@ -69,7 +69,7 @@ def _parse_cli_arguments() -> argparse.Namespace:
         "--skip-items",
         type=str,
         required=False,
-        help="Comma-separated list of items of the chosen model type to skip when running tasks",
+        help="Comma-separated list of glob patterns of the chosen model type to skip when running tasks",
     )
     parser.add_argument(
         "--skip-assemble",

trestlebot/tasks/assemble_task.py

@@ -16,6 +16,7 @@
 
 """Trestle Bot Assembly Tasks"""
 
+import pathlib
 import os
 from typing import List
 
@@ -55,8 +56,7 @@ def __init__(
         self._authored_model = authored_model
         self._markdown_dir = markdown_dir
         self._ssp_index_path = ssp_index_path
-        self._skip_model_list = skip_model_list
-        super().__init__(working_dir)
+        super().__init__(working_dir, skip_model_list)
 
     def execute(self) -> int:
         """Execute task"""
@@ -73,13 +73,11 @@ def _assemble(self) -> int:
             self._authored_model, self.get_working_dir(), self._ssp_index_path
         )
         search_path = os.path.join(self.get_working_dir(), self._markdown_dir)
-        for model in os.listdir(search_path):
+        for model in self.iterate_models(pathlib.Path(search_path)):
             # Construct model path from markdown path. AuthoredObject already has
             # the working dir data as part of object construction.
-            if model in self._skip_model_list or model == ".keep":
-                continue
-            model_path = os.path.join(self._markdown_dir, model)
-
+            model_base_name = os.path.basename(model)
+            model_path = os.path.join(self._markdown_dir, model_base_name)
             try:
                 authored_object.assemble(markdown_path=model_path)
             except AuthoredObjectException as e:

trestlebot/tasks/base_task.py

@@ -16,10 +16,13 @@
 
 """Trestle Bot base task for extendable bot pre-tasks"""
 
+import fnmatch
+import pathlib
 from abc import ABC, abstractmethod
+from typing import Iterable, List
 
-
-# TODO: Incorporate a configuration file for model specific tasks
+from trestle.common import const
+from trestle.common.file_utils import is_hidden
 
 
 class TaskException(Exception):
@@ -31,19 +34,40 @@ class TaskBase(ABC):
     Abstract base class for tasks with a work directory.
     """
 
-    def __init__(self, working_dir: str) -> None:
+    def __init__(self, working_dir: str, skip_list: List[str]) -> None:
         """
         Initialize base task.
 
         Args:
             working_dir: Working directory to complete operations in.
+            skip_list: List of glob pattern to be skipped during processing.
         """
         self._working_dir = working_dir
+        self._skip_model_list = skip_list
+        self._skip_model_list.append(const.TRESTLE_KEEP_FILE)
 
     def get_working_dir(self) -> str:
         """Return the working directory"""
         return self._working_dir
 
+    def iterate_models(self, directory_path: pathlib.Path) -> Iterable[pathlib.Path]:
+        """Iterate over the models in the working directory"""
+        filtered_paths = list(
+            filter(
+                lambda p: not self._is_skipped(p.name)
+                and (not is_hidden(p) or p.is_dir()),
+                pathlib.Path.iterdir(directory_path),
+            )
+        )
+
+        return filtered_paths.__iter__()
+
+    def _is_skipped(self, model_name: str) -> bool:
+        """Return True if the model is in the skip list"""
+        return any(
+            fnmatch.fnmatch(model_name, pattern) for pattern in self._skip_model_list
+        )
+
     @abstractmethod
     def execute(self) -> int:
         """Execute the task and return the exit code"""

trestlebot/tasks/regenerate_task.py

@@ -17,6 +17,7 @@
 """Trestle Bot Regenerate Tasks"""
 
 import os
+import pathlib
 from typing import List
 
 from trestlebot import const
@@ -55,8 +56,7 @@ def __init__(
         self._authored_model = authored_model
         self._markdown_dir = markdown_dir
         self._ssp_index_path = ssp_index_path
-        self._skip_model_list = skip_model_list
-        super().__init__(working_dir)
+        super().__init__(working_dir, skip_model_list)
 
     def execute(self) -> int:
         """Execute task"""
@@ -76,10 +76,9 @@ def _regenerate(self) -> int:
         model_dir = types.get_trestle_model_dir(self._authored_model)
 
         search_path = os.path.join(self.get_working_dir(), model_dir)
-        for model in os.listdir(search_path):
-            if model in self._skip_model_list or model == ".keep":
-                continue
-            model_path = os.path.join(model_dir, model)
+        for model in self.iterate_models(pathlib.Path(search_path)):
+            model_base_name = os.path.basename(model)
+            model_path = os.path.join(model_dir, model_base_name)
 
             try:
                 authored_object.regenerate(