PSCE-237 - Adds Rule YAML to CSV transformation

pyproject.toml

@@ -24,6 +24,7 @@ gitpython = "^3.1.35"
 compliance-trestle = {git = "https://github.com/IBM/compliance-trestle.git", rev = "725f6980f83b040fec4145cfebd4c80af2f7196f"}
 github3-py = "^4.0.1"
 python-gitlab = "^3.15.0"
+ruamel-yaml = "^0.17.32"
 
 [tool.poetry.group.dev.dependencies]
 flake8 = "^6.0.0"
@@ -61,6 +62,22 @@ testpaths = [
 
 [tool.mypy]
 
+plugins = [
+  "pydantic.mypy"
+]
+
+follow_imports = "skip"
+warn_redundant_casts = true
+disallow_any_generics = true
+check_untyped_defs = true
+no_implicit_reexport = true
+disallow_untyped_defs = true
+
+[tool.pydantic-mypy]
+init_forbid_extra = true
+init_typed = true
+warn_required_dynamic_aliases = true
+
 [[tool.mypy.overrides]]
 module = "github3.*"
 ignore_missing_imports = true

tests/data/yaml/test_complete_rule.yaml

@@ -0,0 +1,17 @@
+x-trestle-rule-info:
+  name: example_rule_1
+  description: My rule description for example rule 1
+  parameter:
+    name: prm_1
+    description: prm_1 description
+    alternative-values: {'default': '5%', '5pc': '5%', '10pc': '10%', '15pc': '15%', '20pc': '20%'}
+    default-value: '5%'
+  profile:
+    description: Simple NIST Profile
+    href: profiles/simplified_nist_profile/profile.json
+    include-controls:
+      - id: ac-1
+x-trestle-component-info:
+  name: Component 1
+  description: Component 1 description
+  type: service

tests/data/yaml/test_complete_rule_no_params.yaml

@@ -0,0 +1,12 @@
+x-trestle-rule-info:
+  name: example_rule_2
+  description: My rule description for example rule 2
+  profile:
+    description: Simple NIST Profile
+    href: profiles/simplified_nist_profile/profile.json
+    include-controls:
+      - id: ac-1
+x-trestle-component-info:
+  name: Component 1
+  description: Component 1 description
+  type: service

tests/data/yaml/test_incomplete_rule.yaml

@@ -0,0 +1,16 @@
+x-trestle-rule-info:
+  description: My rule description for example rule 1
+  parameter:
+    name: prm_1
+    description: prm_1 description
+    alternative-values: {'default': '5%', '5pc': '5%', '10pc': '10%', '15pc': '15%', '20pc': '20%'}
+    default-value: '5%'
+  profile:
+    description: Simple NIST Profile
+    href: profiles/simplified_nist_profile/profile.json
+    include-controls:
+      - id: ac-2
+x-trestle-component-info:
+  name: Component 1
+  description: Component 1 description
+  type: service

tests/testutils.py

@@ -30,10 +30,16 @@
 from trestle.oscal import component as comp
 from trestle.oscal import profile as prof
 
-from trestlebot.const import COMPDEF_KEY_NAME, LEVERAGED_SSP_KEY_NAME, PROFILE_KEY_NAME
+from trestlebot.const import (
+    COMPDEF_KEY_NAME,
+    LEVERAGED_SSP_KEY_NAME,
+    PROFILE_KEY_NAME,
+    YAML_EXTENSION,
+)
 
 
 JSON_TEST_DATA_PATH = pathlib.Path("tests/data/json/").resolve()
+YAML_TEST_DATA_PATH = pathlib.Path("tests/data/yaml/").resolve()
 
 
 def clean(repo_path: str, repo: Optional[Repo]) -> None:
@@ -55,7 +61,21 @@ def load_from_json(
         tmp_trestle_dir, model_name, model_type, FileContentType.JSON  # type: ignore
     )
     dst_path.parent.mkdir(parents=True, exist_ok=True)
-    shutil.copy2(src_path, dst_path)  # type: ignore
+    shutil.copy2(src_path, dst_path)
+
+
+def load_from_yaml(
+    rules_path: pathlib.Path,
+    file_prefix: str,
+    dst_name: str = "",
+) -> None:
+    """Load rule from YAML test dir."""
+    if not dst_name:
+        dst_name = file_prefix
+    src_path = YAML_TEST_DATA_PATH / f"{file_prefix}{YAML_EXTENSION}"
+    rules_path = rules_path.joinpath(f"{dst_name}{YAML_EXTENSION}")
+    rules_path.parent.mkdir(parents=True, exist_ok=True)
+    shutil.copy2(src_path, rules_path)
 
 
 def setup_for_ssp(
@@ -170,6 +190,44 @@ def setup_for_compdef(
     return args
 
 
+def setup_rules_view(
+    tmp_trestle_dir: pathlib.Path,
+    output_name: str,
+    rules_dir: str = "rules",
+    comp_name: str = "test_comp",
+    incomplete_rule: bool = False,
+    skip_rules: bool = False,
+) -> None:
+    """Prepare rules view for testing with a single component definition and test component."""
+    load_from_json(
+        tmp_trestle_dir,
+        "simplified_nist_profile",
+        "simplified_nist_profile",
+        prof.Profile,  # type: ignore
+    )
+    load_from_json(
+        tmp_trestle_dir,
+        "simplified_nist_catalog",
+        "simplified_nist_catalog",
+        cat.Catalog,  # type: ignore
+    )
+    rules_path = tmp_trestle_dir.joinpath(rules_dir)
+    compdef_dir = rules_path.joinpath(output_name)
+    comp_dir = compdef_dir.joinpath(comp_name)
+    comp_dir.mkdir(parents=True, exist_ok=True)
+
+    if skip_rules:
+        return
+
+    if incomplete_rule:
+        load_from_yaml(comp_dir, "test_incomplete_rule")
+    else:
+        # Load a complete rule with optional fields
+        load_from_yaml(comp_dir, "test_complete_rule")
+        # Load a complete rule with only required fields
+        load_from_yaml(comp_dir, "test_complete_rule_no_params")
+
+
 def write_index_json(
     file_path: str,
     ssp_name: str,

tests/trestlebot/tasks/test_rule_transform_task.py

@@ -0,0 +1,114 @@
+#!/usr/bin/python
+
+#    Copyright 2023 Red Hat, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Test for Trestle Bot rule transform task"""
+
+import pathlib
+
+import pytest
+import trestle.oscal.component as osc_comp
+from trestle.common.err import TrestleNotFoundError
+from trestle.common.model_utils import ModelUtils
+from trestle.core.models.file_content_type import FileContentType
+from trestle.tasks.csv_to_oscal_cd import RULE_DESCRIPTION, RULE_ID
+
+from tests.testutils import setup_rules_view
+from trestlebot.tasks.base_task import TaskException
+from trestlebot.tasks.rule_transform_task import RuleTransformTask
+from trestlebot.transformers.yaml_to_csv import RulesYAMLTransformer
+
+
+test_comp = "test_comp"
+test_rules_dir = "test_rules_dir"
+
+
+def test_rule_transform_task(tmp_trestle_dir: str) -> None:
+    """Test rule transform task."""
+    trestle_root = pathlib.Path(tmp_trestle_dir)
+    setup_rules_view(trestle_root, test_comp, test_rules_dir)
+    transformer = RulesYAMLTransformer()
+    rule_transform_task = RuleTransformTask(
+        tmp_trestle_dir, test_rules_dir, transformer
+    )
+    return_code = rule_transform_task.execute()
+    assert return_code == 0
+
+    # Check that the compdef model is unchanged
+    orig_comp, _ = ModelUtils.load_model_for_class(
+        trestle_root, test_comp, osc_comp.ComponentDefinition, FileContentType.JSON
+    )
+
+    assert orig_comp is not None
+    assert orig_comp.metadata.title == "Component definition for test_comp"
+    assert orig_comp.components is not None
+    assert len(orig_comp.components) == 1
+
+    component = orig_comp.components[0]
+
+    assert component.props is not None
+    assert len(component.props) == 5
+    assert component.props[0].name == RULE_ID
+    assert component.props[0].value == "example_rule_1"
+    assert component.props[1].name == RULE_DESCRIPTION
+    assert component.props[1].value == "My rule description for example rule 1"
+
+
+def test_rule_transform_task_with_no_rules(tmp_trestle_dir: str) -> None:
+    """Test rule transform task with no rules."""
+    trestle_root = pathlib.Path(tmp_trestle_dir)
+    setup_rules_view(trestle_root, test_comp, test_rules_dir, skip_rules=True)
+    transformer = RulesYAMLTransformer()
+    rule_transform_task = RuleTransformTask(
+        tmp_trestle_dir, test_rules_dir, transformer
+    )
+
+    with pytest.raises(
+        TaskException, match="No rules found for component definition test_comp"
+    ):
+        rule_transform_task.execute()
+
+
+def test_rule_transform_task_with_invalid_rule(tmp_trestle_dir: str) -> None:
+    """Test rule transform task with invalid rule."""
+    trestle_root = pathlib.Path(tmp_trestle_dir)
+    setup_rules_view(trestle_root, test_comp, test_rules_dir, incomplete_rule=True)
+    transformer = RulesYAMLTransformer()
+    rule_transform_task = RuleTransformTask(
+        tmp_trestle_dir, test_rules_dir, transformer
+    )
+
+    with pytest.raises(
+        TaskException, match="Failed to transform rule .*: Missing key in YAML file: .*"
+    ):
+        rule_transform_task.execute()
+
+
+def test_rule_transform_task_with_skip(tmp_trestle_dir: str) -> None:
+    """Test rule transform task with skip."""
+    trestle_root = pathlib.Path(tmp_trestle_dir)
+    setup_rules_view(trestle_root, test_comp, test_rules_dir)
+    transformer = RulesYAMLTransformer()
+    rule_transform_task = RuleTransformTask(
+        tmp_trestle_dir, test_rules_dir, transformer, skip_model_list=[test_comp]
+    )
+    return_code = rule_transform_task.execute()
+    assert return_code == 0
+
+    # Check that the compdef model is not present
+    with pytest.raises(TrestleNotFoundError):
+        ModelUtils.load_model_for_class(
+            trestle_root, test_comp, osc_comp.ComponentDefinition, FileContentType.JSON
+        )

tests/trestlebot/test_cli.py

@@ -18,7 +18,7 @@
 
 import logging
 import sys
-from typing import List
+from typing import Any, Dict, List
 from unittest.mock import patch
 
 import pytest
@@ -27,7 +27,7 @@
 
 
 @pytest.fixture
-def valid_args_dict() -> dict:
+def valid_args_dict() -> Dict[str, str]:
     return {
         "branch": "main",
         "markdown-path": "/my/path",
@@ -39,7 +39,7 @@ def valid_args_dict() -> dict:
     }
 
 
-def args_dict_to_list(args_dict: dict) -> List[str]:
+def args_dict_to_list(args_dict: Dict[str, str]) -> List[str]:
     args = []
     for k, v in args_dict.items():
         args.append(f"--{k}")
@@ -48,7 +48,9 @@ def args_dict_to_list(args_dict: dict) -> List[str]:
     return args
 
 
-def test_invalid_oscal_model(monkeypatch, valid_args_dict, caplog):
+def test_invalid_oscal_model(
+    monkeypatch: Any, valid_args_dict: Dict[str, str], caplog: Any
+) -> None:
     """Test invalid oscal model"""
     args_dict = valid_args_dict
     args_dict["oscal-model"] = "fake"
@@ -65,7 +67,9 @@ def test_invalid_oscal_model(monkeypatch, valid_args_dict, caplog):
     )
 
 
-def test_no_ssp_index(monkeypatch, valid_args_dict, caplog):
+def test_no_ssp_index(
+    monkeypatch: Any, valid_args_dict: Dict[str, str], caplog: Any
+) -> None:
     """Test missing index file for ssp"""
     args_dict = valid_args_dict
     args_dict["oscal-model"] = "ssp"
@@ -82,7 +86,9 @@ def test_no_ssp_index(monkeypatch, valid_args_dict, caplog):
     )
 
 
-def test_no_markdown_path(monkeypatch, valid_args_dict, caplog):
+def test_no_markdown_path(
+    monkeypatch: Any, valid_args_dict: Dict[str, str], caplog: Any
+) -> None:
     """Test without a markdown file passed as a flag"""
     args_dict = valid_args_dict
     args_dict["markdown-path"] = ""
@@ -98,7 +104,9 @@ def test_no_markdown_path(monkeypatch, valid_args_dict, caplog):
     )
 
 
-def test_with_target_branch(monkeypatch, valid_args_dict, caplog):
+def test_with_target_branch(
+    monkeypatch: Any, valid_args_dict: Dict[str, str], caplog: Any
+) -> None:
     """Test with target branch set an an unsupported Git provider"""
     args_dict = valid_args_dict
     args_dict["target-branch"] = "main"

tests/trestlebot/transformers/__init__.py

@@ -0,0 +1,16 @@
+#!/usr/bin/python
+
+#    Copyright 2023 Red Hat, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+"""Test transformers package."""