From 01fea64ad1debff20cdcc8a9259eb2283554b819 Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Tue, 12 Nov 2024 14:44:42 -0500 Subject: [PATCH 001/108] initial directory structure organization --- trestlebot/cli/commands/autosync.py | 2 ++ trestlebot/cli/commands/create.py | 1 + trestlebot/cli/commands/version.py | 1 + trestlebot/cli/root.py | 1 + 4 files changed, 5 insertions(+) create mode 100644 trestlebot/cli/commands/autosync.py create mode 100644 trestlebot/cli/commands/create.py create mode 100644 trestlebot/cli/commands/version.py create mode 100644 trestlebot/cli/root.py diff --git a/trestlebot/cli/commands/autosync.py b/trestlebot/cli/commands/autosync.py new file mode 100644 index 00000000..8c45acbb --- /dev/null +++ b/trestlebot/cli/commands/autosync.py @@ -0,0 +1,2 @@ +""" Autosync command""" + diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py new file mode 100644 index 00000000..e08a30eb --- /dev/null +++ b/trestlebot/cli/commands/create.py @@ -0,0 +1 @@ +""" Create CD Create SSP grouping command""" \ No newline at end of file diff --git a/trestlebot/cli/commands/version.py b/trestlebot/cli/commands/version.py new file mode 100644 index 00000000..af9981da --- /dev/null +++ b/trestlebot/cli/commands/version.py @@ -0,0 +1 @@ +""" Version command """ \ No newline at end of file diff --git a/trestlebot/cli/root.py b/trestlebot/cli/root.py new file mode 100644 index 00000000..0c81d2c8 --- /dev/null +++ b/trestlebot/cli/root.py @@ -0,0 +1 @@ +"""Main entrypoint for trestlebot""" \ No newline at end of file From ec6771811fafbc3416a62c05ffaec5f470105f50 Mon Sep 17 00:00:00 2001 From: George Vauter Date: Wed, 13 Nov 2024 18:16:45 -0500 Subject: [PATCH 002/108] feat: initial work on config and common options --- poetry.lock | 15 +++++++- pyproject.toml | 1 + trestlebot/cli/commands/autosync.py | 1 - trestlebot/cli/commands/create.py | 2 +- trestlebot/cli/commands/init.py | 57 +++++++++++++++++++++++++++++ trestlebot/cli/commands/version.py | 2 +- trestlebot/cli/config.py | 48 ++++++++++++++++++++++++ trestlebot/cli/options/common.py | 41 +++++++++++++++++++++ trestlebot/cli/root.py | 25 ++++++++++++- 9 files changed, 186 insertions(+), 6 deletions(-) create mode 100644 trestlebot/cli/commands/init.py create mode 100644 trestlebot/cli/config.py create mode 100644 trestlebot/cli/options/common.py diff --git a/poetry.lock b/poetry.lock index e639d69c..779529dd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. [[package]] name = "annotated-types" @@ -2816,6 +2816,17 @@ rich = ">=10.11.0" shellingham = ">=1.3.0" typing-extensions = ">=3.7.4.3" +[[package]] +name = "types-pyyaml" +version = "6.0.12.20240917" +description = "Typing stubs for PyYAML" +optional = false +python-versions = ">=3.8" +files = [ + {file = "types-PyYAML-6.0.12.20240917.tar.gz", hash = "sha256:d1405a86f9576682234ef83bcb4e6fff7c9305c8b1fbad5e0bcd4f7dbdc9c587"}, + {file = "types_PyYAML-6.0.12.20240917-py3-none-any.whl", hash = "sha256:392b267f1c0fe6022952462bf5d6523f31e37f6cea49b14cee7ad634b6301570"}, +] + [[package]] name = "typing-extensions" version = "4.12.2" @@ -2944,4 +2955,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.8.1" -content-hash = "9f87cf236388219dc7404210d37f4a0420d21a7133693a0e6ad7e837450adcc6" +content-hash = "a00779b3211b34f2b7d65ad33e280ded7f2e64cebc2ddd27b1656217f69a3dbf" diff --git a/pyproject.toml b/pyproject.toml index da97b851..aa03fcb6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ flake8-print = "^5.0.0" pre-commit = "^3.4.0" mkdocs-material = "^9.5.43" markdown-include = "^0.8.1" +types-pyyaml = "^6.0.12.20240917" [tool.poetry.group.tests] optional = true diff --git a/trestlebot/cli/commands/autosync.py b/trestlebot/cli/commands/autosync.py index 8c45acbb..a1965a7e 100644 --- a/trestlebot/cli/commands/autosync.py +++ b/trestlebot/cli/commands/autosync.py @@ -1,2 +1 @@ """ Autosync command""" - diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index e08a30eb..84d102d5 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -1 +1 @@ -""" Create CD Create SSP grouping command""" \ No newline at end of file +""" Create CD Create SSP grouping command""" diff --git a/trestlebot/cli/commands/init.py b/trestlebot/cli/commands/init.py new file mode 100644 index 00000000..2ef0295e --- /dev/null +++ b/trestlebot/cli/commands/init.py @@ -0,0 +1,57 @@ +"""" +Module for Trestle-bot init command +""" + +import pathlib +import sys + +import click + +from trestlebot.cli.options.common import common_options +from trestlebot.const import ERROR_EXIT_CODE, TRESTLEBOT_CONFIG_DIR + + +@click.command(name="init", help="Initialize a new trestle-bot repo.") +@click.option( + "--repo-dir", + help="Path to git repo. Used as working directory for authoring.", + type=click.Path(exists=True), +) +@click.option( + "--markdown-dir", help="Path to store markdown content.", type=click.Path() +) +@common_options +def init_cmd(ctx: click.Context, repo_dir: str, markdown_dir: str) -> None: + """Command to initialize a new trestlebot repo""" + + need_to_prompt = any((not repo_dir, not markdown_dir)) + if need_to_prompt: + + click.echo("\n* Welcome to the Trestle-bot CLI *\n") + click.echo( + "Please provide the following values to initialize the " + "workspace [press Enter for defaults].\n" + ) + if not repo_dir: + repo_dir = click.prompt( + "Path to git repo", default=".", type=click.Path(exists=True) + ) + if not markdown_dir: + markdown_dir = click.prompt( + "Path to store markdown content", default="./markdown" + ) + + root_dir: pathlib.Path = pathlib.Path(repo_dir) + git_dir: pathlib.Path = root_dir.joinpath(pathlib.Path(".git")) + if not git_dir.exists(): + click.echo( + f"Initialization failed. Given directory {root_dir} is not a Git repository." + ) # TODO: use logger.error + sys.exit(ERROR_EXIT_CODE) + + trestlebot_dir = root_dir.joinpath(pathlib.Path(TRESTLEBOT_CONFIG_DIR)) + if trestlebot_dir.exists(): + click.echo( + f"Initialization failed. Found existing {TRESTLEBOT_CONFIG_DIR} directory in {root_dir}" + ) # TODO: use logger.error + sys.exit(ERROR_EXIT_CODE) diff --git a/trestlebot/cli/commands/version.py b/trestlebot/cli/commands/version.py index af9981da..11b8814c 100644 --- a/trestlebot/cli/commands/version.py +++ b/trestlebot/cli/commands/version.py @@ -1 +1 @@ -""" Version command """ \ No newline at end of file +""" Version command """ diff --git a/trestlebot/cli/config.py b/trestlebot/cli/config.py new file mode 100644 index 00000000..6c555187 --- /dev/null +++ b/trestlebot/cli/config.py @@ -0,0 +1,48 @@ +""" +Trestle-bot CLI configuration module. +""" + +from pathlib import Path +from typing import Any, Dict, Optional + +import yaml +from pydantic import BaseModel, DirectoryPath, model_serializer + + +class TrestleBotConfig(BaseModel): + """Data model for trestle-bot configuration.""" + + working_dir: Optional[DirectoryPath] = None + markdown_dir: Optional[DirectoryPath] = None + + @model_serializer + def _dict(self) -> Dict[str, Any]: + """Returns a dict that can be safely loaded to yaml.""" + config_dict = { + "working_dir": str(self.working_dir), + "markdown_dir": str(self.markdown_dir), + } + return dict( + filter(lambda item: item[1] not in (None, "None"), config_dict.items()) + ) + + +def load_from_file(file: str) -> TrestleBotConfig: + """Load yaml file to trestlebot config object""" + + with open(file, "r") as config_file: + config_yaml = yaml.safe_load(config_file) + return TrestleBotConfig(**config_yaml) + + +def write_to_file(config: TrestleBotConfig, file: str) -> Path: + """Write config object to yaml file""" + + with open(file, "w") as config_file: + yaml.dump(config.dict(), config_file) + return Path(file) + + +def update_config(config: TrestleBotConfig, update: Dict[str, Any]) -> TrestleBotConfig: + """Returns a new config object with specified updates.""" + return config.model_copy(update=update) diff --git a/trestlebot/cli/options/common.py b/trestlebot/cli/options/common.py new file mode 100644 index 00000000..9a58c987 --- /dev/null +++ b/trestlebot/cli/options/common.py @@ -0,0 +1,41 @@ +""" +Common command options for trestle-bot commands. +""" + +import functools +import traceback +from typing import Any, Callable, Dict, Sequence, TypeVar + +import click + +from trestlebot.const import ERROR_EXIT_CODE + + +F = TypeVar("F", bound=Callable[..., Any]) + + +def handle_exceptions(func: F) -> Any: + def wrapper(*args: Sequence[Any], **kwargs: Dict[Any, Any]) -> Any: + try: + return func(*args, **kwargs) + except Exception as ex: + traceback_str = traceback.format_exc() + click.echo(f"Trestle-bot Error: {str(ex)}") # TODO: use logger.info + click.echo(traceback_str) # TODO: use loggger.debug + return ERROR_EXIT_CODE + + return wrapper + + +def common_options(f: F) -> F: + """ + Configures common options used across commands. + """ + + @click.pass_context + @handle_exceptions + @functools.wraps(f) + def wrapper_common_options(*args: Sequence[Any], **kwargs: Dict[Any, Any]) -> F: + return f(*args, **kwargs) + + return wrapper_common_options diff --git a/trestlebot/cli/root.py b/trestlebot/cli/root.py index 0c81d2c8..eca11ba9 100644 --- a/trestlebot/cli/root.py +++ b/trestlebot/cli/root.py @@ -1 +1,24 @@ -"""Main entrypoint for trestlebot""" \ No newline at end of file +"""Main entrypoint for trestlebot""" + +import click + +from trestlebot.cli.commands.init import init_cmd + + +EPILOG = "See our docs at https://redhatproductsecurity.github.io/trestle-bot" + +CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) + + +@click.group( + name="trestlebot", + help="Trestle-bot CLI", + context_settings=CONTEXT_SETTINGS, + epilog=EPILOG, +) +@click.pass_context +def root(ctx: click.Context) -> None: + """Root command""" + + +root.add_command(init_cmd) From a1984a5d55c11ba21f6842a5969ed5171bb08338 Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Thu, 14 Nov 2024 16:18:18 -0500 Subject: [PATCH 003/108] chore: add openssf scorecard workflow (#359) Signed-off-by: Jennifer Power --- .github/workflows/scorecard.yml | 49 +++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .github/workflows/scorecard.yml diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml new file mode 100644 index 00000000..76d75a82 --- /dev/null +++ b/.github/workflows/scorecard.yml @@ -0,0 +1,49 @@ +name: Scorecard analysis workflow +on: + push: + # Only the default branch is supported. + branches: + - main + schedule: + # Weekly on Saturdays. + - cron: '30 1 * * 6' + +permissions: read-all + +jobs: + analysis: + if: github.repository == 'RedHatProductSecurity/trestle-bot' + name: Scorecard analysis + runs-on: ubuntu-latest + permissions: + # Needed for Code scanning upload + security-events: write + # Needed for GitHub OIDC token if publish_results is true + id-token: write + + steps: + - name: Checkout code + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # pin@v4 + with: + persist-credentials: false + + - name: Run analysis + uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 # v2.4.0 + with: + results_file: results.sarif + results_format: sarif + publish_results: true + + - name: Upload artifact + uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # pin@v4 + with: + name: SARIF file + path: results.sarif + retention-days: 5 + + # Upload the results to GitHub's code scanning dashboard (optional). + # Commenting out will disable upload of results to your repo's Code Scanning dashboard + - name: Upload to code-scanning + uses: github/codeql-action/upload-sarif@461ef6c76dfe95d5c364de2f431ddbd31a417628 # v3.26.9 + with: + sarif_file: results.sarif \ No newline at end of file From cc569d5c2d2e212fcdee8f058307dfbf026d6fdf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 14 Nov 2024 16:42:38 -0500 Subject: [PATCH 004/108] build(deps): bump compliance-trestle from 3.4.0 to 3.5.0 (#380) Bumps [compliance-trestle](https://github.com/oscal-compass/compliance-trestle) from 3.4.0 to 3.5.0. - [Release notes](https://github.com/oscal-compass/compliance-trestle/releases) - [Changelog](https://github.com/oscal-compass/compliance-trestle/blob/develop/CHANGELOG.md) - [Commits](https://github.com/oscal-compass/compliance-trestle/compare/v3.4.0...v3.5.0) --- updated-dependencies: - dependency-name: compliance-trestle dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 10 +++++----- pyproject.toml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index e639d69c..76114638 100644 --- a/poetry.lock +++ b/poetry.lock @@ -494,13 +494,13 @@ files = [ [[package]] name = "compliance-trestle" -version = "3.4.0" +version = "3.5.0" description = "Tools to manage & autogenerate python objects representing the OSCAL layers/models" optional = false python-versions = "*" files = [ - {file = "compliance_trestle-3.4.0-py2.py3-none-any.whl", hash = "sha256:7762b3b3ddb7618fb4203feae104bf5de8dff1cd65de4ee5db01f3802cdc1447"}, - {file = "compliance_trestle-3.4.0.tar.gz", hash = "sha256:c55d0adbe3d78c63d4ca735a2c6173ca7c96944930ab5fa54484c02e8bd5164c"}, + {file = "compliance_trestle-3.5.0-py2.py3-none-any.whl", hash = "sha256:e7995683459b8a4ae77377e0934adc88190b1c612cddc000128b7ef19d71a066"}, + {file = "compliance_trestle-3.5.0.tar.gz", hash = "sha256:29e342eb59e527ebea40da36a2632a8782fbe82e76fda12af72c5ef655856951"}, ] [package.dependencies] @@ -524,7 +524,7 @@ requests = ">=2.32.2" "ruamel.yaml" = "*" [package.extras] -dev = ["gitpython", "livereload", "markdown-include", "mkdocs (==1.5.0)", "mkdocs-material", "mkdocstrings[python-legacy] (==0.19.0)", "mypy", "pep8-naming", "pre-commit (>=2.4.0)", "pylint", "pymdown-extensions", "pytest (>=5.4.3)", "pytest-cov (>=2.10.0)", "pytest-random-order", "pytest-xdist", "python-dateutil", "python-semantic-release (>=9.8.0)", "setuptools (>=61)", "types-PyYAML", "types-paramiko", "types-requests", "types-setuptools", "urllib3 (==1.26.19)", "wheel", "yapf"] +dev = ["gitpython", "livereload", "markdown-include", "mkdocs (>=1.6.0)", "mkdocs-htmlproofer-plugin", "mkdocs-material", "mkdocstrings[python] (>=0.25.2)", "mypy", "pep8-naming", "pre-commit (>=2.4.0)", "pylint", "pymdown-extensions", "pytest (>=5.4.3)", "pytest-cov (>=2.10.0)", "pytest-random-order", "pytest-xdist", "python-dateutil", "python-semantic-release (>=9.8.0)", "setuptools (>=61)", "types-PyYAML", "types-paramiko", "types-requests", "types-setuptools", "urllib3 (==1.26.19)", "wheel", "yapf"] [[package]] name = "compliance-trestle-fedramp" @@ -2944,4 +2944,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.8.1" -content-hash = "9f87cf236388219dc7404210d37f4a0420d21a7133693a0e6ad7e837450adcc6" +content-hash = "1f08efc70a8602cf883b114f94118473daf61cff47ce10d922d00c7470bc1b49" diff --git a/pyproject.toml b/pyproject.toml index da97b851..bd9dd5fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ trestlebot-create-ssp = "trestlebot.entrypoints.create_ssp:main" [tool.poetry.dependencies] python = '^3.8.1' gitpython = "^3.1.41" -compliance-trestle = "^3.3.0" +compliance-trestle = "^3.5.0" github3-py = "^4.0.1" python-gitlab = "^4.2.0" ruamel-yaml = "^0.18.5" From c5b9418aa474f6b7c985ef38e90199f79f2a3406 Mon Sep 17 00:00:00 2001 From: George Vauter Date: Fri, 15 Nov 2024 13:18:11 -0500 Subject: [PATCH 005/108] feat: adds logic to load yaml config into click context to set defautl values. improves config error handling. --- trestlebot/cli/commands/init.py | 53 +++++++++++++++++------- trestlebot/cli/config.py | 49 ++++++++++++++++++---- trestlebot/cli/log.py | 59 ++++++++++++++++++++++++++ trestlebot/cli/options/common.py | 71 +++++++++++++++++++++++++++++--- trestlebot/cli/root.py | 8 +++- 5 files changed, 209 insertions(+), 31 deletions(-) create mode 100644 trestlebot/cli/log.py diff --git a/trestlebot/cli/commands/init.py b/trestlebot/cli/commands/init.py index 2ef0295e..fe47ee64 100644 --- a/trestlebot/cli/commands/init.py +++ b/trestlebot/cli/commands/init.py @@ -1,19 +1,35 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Red Hat, Inc. + """" Module for Trestle-bot init command """ - -import pathlib +import logging import sys +from pathlib import Path import click +from trestle.common.const import MODEL_DIR_LIST from trestlebot.cli.options.common import common_options from trestlebot.const import ERROR_EXIT_CODE, TRESTLEBOT_CONFIG_DIR +logger = logging.getLogger(__name__) + + +def dir_make_if_not(directory: str) -> None: + """Makes directory if it does not already exists.""" + try: + Path(directory).mkdir(parents=True) + logger.debug(f"Created directory {directory}") + except FileExistsError: + logger.debug(f"Directory {directory} exists, skipping create.") + + @click.command(name="init", help="Initialize a new trestle-bot repo.") @click.option( - "--repo-dir", + "--working-dir", help="Path to git repo. Used as working directory for authoring.", type=click.Path(exists=True), ) @@ -21,10 +37,12 @@ "--markdown-dir", help="Path to store markdown content.", type=click.Path() ) @common_options -def init_cmd(ctx: click.Context, repo_dir: str, markdown_dir: str) -> None: +def init_cmd( + ctx: click.Context, working_dir: str, markdown_dir: str, debug: bool, config: str +) -> None: """Command to initialize a new trestlebot repo""" - need_to_prompt = any((not repo_dir, not markdown_dir)) + need_to_prompt = any((not working_dir, not markdown_dir)) if need_to_prompt: click.echo("\n* Welcome to the Trestle-bot CLI *\n") @@ -32,8 +50,8 @@ def init_cmd(ctx: click.Context, repo_dir: str, markdown_dir: str) -> None: "Please provide the following values to initialize the " "workspace [press Enter for defaults].\n" ) - if not repo_dir: - repo_dir = click.prompt( + if not working_dir: + working_dir = click.prompt( "Path to git repo", default=".", type=click.Path(exists=True) ) if not markdown_dir: @@ -41,17 +59,20 @@ def init_cmd(ctx: click.Context, repo_dir: str, markdown_dir: str) -> None: "Path to store markdown content", default="./markdown" ) - root_dir: pathlib.Path = pathlib.Path(repo_dir) - git_dir: pathlib.Path = root_dir.joinpath(pathlib.Path(".git")) + root_dir: Path = Path(working_dir) + git_dir: Path = root_dir.joinpath(Path(".git")) if not git_dir.exists(): - click.echo( - f"Initialization failed. Given directory {root_dir} is not a Git repository." - ) # TODO: use logger.error + logging.error( + f"[!] Initialization failed. Given directory {root_dir} is not a Git repository." + ) sys.exit(ERROR_EXIT_CODE) - trestlebot_dir = root_dir.joinpath(pathlib.Path(TRESTLEBOT_CONFIG_DIR)) + trestlebot_dir = root_dir.joinpath(Path(TRESTLEBOT_CONFIG_DIR)) if trestlebot_dir.exists(): - click.echo( - f"Initialization failed. Found existing {TRESTLEBOT_CONFIG_DIR} directory in {root_dir}" - ) # TODO: use logger.error + logger.error( + f"[!] Initialization failed. Found existing {TRESTLEBOT_CONFIG_DIR} directory in {root_dir}" + ) sys.exit(ERROR_EXIT_CODE) + + model_dirs = list(map(lambda x: str(root_dir.joinpath(x)), MODEL_DIR_LIST)) + list(map(dir_make_if_not, model_dirs)) diff --git a/trestlebot/cli/config.py b/trestlebot/cli/config.py index 6c555187..105ce92f 100644 --- a/trestlebot/cli/config.py +++ b/trestlebot/cli/config.py @@ -1,12 +1,43 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Red Hat, Inc. + """ Trestle-bot CLI configuration module. """ +import logging from pathlib import Path -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional import yaml -from pydantic import BaseModel, DirectoryPath, model_serializer +from pydantic import BaseModel, DirectoryPath, ValidationError, model_serializer + + +logger = logging.getLogger(__name__) + + +class TrestleBotConfigError(Exception): + """Custom error to better format pydantic exceptions. + + Example pydantic error dict: {'type': str, 'loc': tuple[str], 'msg': str, 'input': str} + + """ + + def __init__(self, errors: List[Dict[str, Any]]): + self.errors = list(map(self._format, errors)) + super().__init__( + f"Trestle-bot config file contains {len(self.errors)} error(s)." + ) + + def _format(self, err: Dict[str, Any]) -> str: + """Returns a formatted string with the error details.""" + msg = "Unable to load config." # default message if we can't parse error + + if err.get("loc"): + msg = f"Invalid config value for {err['loc'][0]}." + if err.get("msg"): + msg += f" {err['msg']}" # Add error message details if present + return msg class TrestleBotConfig(BaseModel): @@ -27,12 +58,16 @@ def _dict(self) -> Dict[str, Any]: ) -def load_from_file(file: str) -> TrestleBotConfig: +def load_from_file(file: str) -> Optional[TrestleBotConfig]: """Load yaml file to trestlebot config object""" - - with open(file, "r") as config_file: - config_yaml = yaml.safe_load(config_file) - return TrestleBotConfig(**config_yaml) + try: + with open(file, "r") as config_file: + config_yaml = yaml.safe_load(config_file) + return TrestleBotConfig(**config_yaml) + except ValidationError as ex: + raise TrestleBotConfigError(ex.errors()) + except FileNotFoundError: + return None def write_to_file(config: TrestleBotConfig, file: str) -> Path: diff --git a/trestlebot/cli/log.py b/trestlebot/cli/log.py new file mode 100644 index 00000000..4e251da3 --- /dev/null +++ b/trestlebot/cli/log.py @@ -0,0 +1,59 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2023 Red Hat, Inc. + + +"""Configure logger for trestlebot and trestle.""" + +import argparse +import logging +import sys +from typing import List + +import trestle.common.log as trestle_log + + +_logger = logging.getLogger("trestlebot") + + +def set_log_level(level: int = logging.INFO) -> None: + """Set the log level from the args for trestle and trestlebot.""" + + configure_logger(level) + + # Setup the trestle logger, it expects an argparse Namespace with a verbose int + verbose = 1 if level == logging.DEBUG else 0 + args = argparse.Namespace(verbose=verbose) + trestle_log.set_log_level_from_args(args=args) + + +def configure_logger(level: int = logging.INFO, propagate: bool = False) -> None: + """Configure the logger.""" + # Prevent extra message + _logger.propagate = propagate + _logger.setLevel(level=level) + for handler in configure_handlers(): + _logger.addHandler(handler) + + +def configure_handlers() -> List[logging.Handler]: + """Configure the handlers.""" + # Create a StreamHandler to send non-error logs to stdout + stdout_info_handler = logging.StreamHandler(sys.stdout) + stdout_info_handler.setLevel(logging.INFO) + stdout_info_handler.addFilter(trestle_log.SpecificLevelFilter(logging.INFO)) + + stdout_debug_handler = logging.StreamHandler(sys.stdout) + stdout_debug_handler.setLevel(logging.DEBUG) + stdout_debug_handler.addFilter(trestle_log.SpecificLevelFilter(logging.DEBUG)) + + # Create a StreamHandler to send error logs to stderr + stderr_handler = logging.StreamHandler(sys.stderr) + stderr_handler.setLevel(logging.WARNING) + + # Create a formatter and set it on both handlers + detailed_formatter = logging.Formatter( + "%(name)s:%(lineno)d %(levelname)s: %(message)s" + ) + stdout_debug_handler.setFormatter(detailed_formatter) + stderr_handler.setFormatter(detailed_formatter) + return [stdout_debug_handler, stdout_info_handler, stderr_handler] diff --git a/trestlebot/cli/options/common.py b/trestlebot/cli/options/common.py index 9a58c987..31467220 100644 --- a/trestlebot/cli/options/common.py +++ b/trestlebot/cli/options/common.py @@ -1,18 +1,27 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Red Hat, Inc. + """ Common command options for trestle-bot commands. """ import functools +import logging +import sys import traceback -from typing import Any, Callable, Dict, Sequence, TypeVar +from typing import Any, Callable, Dict, Optional, Sequence, TypeVar import click -from trestlebot.const import ERROR_EXIT_CODE +from trestlebot.cli.config import TrestleBotConfigError, load_from_file +from trestlebot.cli.log import set_log_level +from trestlebot.const import ERROR_EXIT_CODE, TRESTLEBOT_CONFIG_DIR F = TypeVar("F", bound=Callable[..., Any]) +logger = logging.getLogger(__name__) + def handle_exceptions(func: F) -> Any: def wrapper(*args: Sequence[Any], **kwargs: Dict[Any, Any]) -> Any: @@ -20,22 +29,72 @@ def wrapper(*args: Sequence[Any], **kwargs: Dict[Any, Any]) -> Any: return func(*args, **kwargs) except Exception as ex: traceback_str = traceback.format_exc() - click.echo(f"Trestle-bot Error: {str(ex)}") # TODO: use logger.info - click.echo(traceback_str) # TODO: use loggger.debug + logger.error(f"Trestle-bot Error: {str(ex)}") + logger.error(traceback_str) return ERROR_EXIT_CODE return wrapper -def common_options(f: F) -> F: +def debug_to_log_level(ctx: click.Context, param: str, value: str) -> None: + """Sets logging level based on debug flag.""" + + # TODO: get log level from config file + log_level = logging.DEBUG if value else logging.INFO + set_log_level(log_level) + + +def load_config_to_ctx(ctx: click.Context, param: str, value: str) -> Optional[str]: + """Load yaml config file into Click context to set default values. + + This will always run before other options because the --config is_eager is True. + """ + try: + config = load_from_file(value) + if not config: + return None + except TrestleBotConfigError as ex: + logger.error(str(ex)) + for err in ex.errors: + logger.error(f"[!] {err}") + sys.exit(ERROR_EXIT_CODE) + + if not ctx.default_map: + ctx.default_map = ( + config.dict() + ) # if default_map has not yet been set by another option + else: + ctx.default_map.update(config) + return value + + +def common_options(f: F) -> Any: """ Configures common options used across commands. """ @click.pass_context + @click.option( + "--config", + type=click.Path(), + envvar="TRESTLEBOT_CONFIG", + help="Path to trestlebot configuration file", + default=f"{TRESTLEBOT_CONFIG_DIR}/config.yml", + is_eager=True, + callback=load_config_to_ctx, + ) + @click.option( + "--debug", + default=False, + is_flag=True, + is_eager=True, + envvar="TRESTLEBOT_DEBUG", + help="Enable debug logging messages.", + callback=debug_to_log_level, + ) @handle_exceptions @functools.wraps(f) - def wrapper_common_options(*args: Sequence[Any], **kwargs: Dict[Any, Any]) -> F: + def wrapper_common_options(*args: Sequence[Any], **kwargs: Dict[Any, Any]) -> Any: return f(*args, **kwargs) return wrapper_common_options diff --git a/trestlebot/cli/root.py b/trestlebot/cli/root.py index eca11ba9..4f9e8759 100644 --- a/trestlebot/cli/root.py +++ b/trestlebot/cli/root.py @@ -1,3 +1,7 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Red Hat, Inc. + + """Main entrypoint for trestlebot""" import click @@ -17,8 +21,8 @@ epilog=EPILOG, ) @click.pass_context -def root(ctx: click.Context) -> None: +def root_cmd(ctx: click.Context) -> None: """Root command""" -root.add_command(init_cmd) +root_cmd.add_command(init_cmd) From c6a256938bb85c1112d2bb99a768bc7f5ff02909 Mon Sep 17 00:00:00 2001 From: George Vauter Date: Fri, 15 Nov 2024 15:46:23 -0500 Subject: [PATCH 006/108] feat: adds debug logging statements --- trestlebot/cli/options/common.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/trestlebot/cli/options/common.py b/trestlebot/cli/options/common.py index 31467220..cff3851d 100644 --- a/trestlebot/cli/options/common.py +++ b/trestlebot/cli/options/common.py @@ -52,6 +52,7 @@ def load_config_to_ctx(ctx: click.Context, param: str, value: str) -> Optional[s try: config = load_from_file(value) if not config: + logger.debug("No configuration file found.") return None except TrestleBotConfigError as ex: logger.error(str(ex)) @@ -65,6 +66,7 @@ def load_config_to_ctx(ctx: click.Context, param: str, value: str) -> Optional[s ) # if default_map has not yet been set by another option else: ctx.default_map.update(config) + logger.debug(f"Successfully loaded config file {value} into context.") return value From 78fd1139e20e93750bea521b07ef5b9011f91ae0 Mon Sep 17 00:00:00 2001 From: George Vauter Date: Fri, 15 Nov 2024 15:46:53 -0500 Subject: [PATCH 007/108] feat: add markdown directory creation and call to compliance trestle init --- trestlebot/cli/commands/init.py | 55 ++++++++++++++++++++++++++++----- 1 file changed, 48 insertions(+), 7 deletions(-) diff --git a/trestlebot/cli/commands/init.py b/trestlebot/cli/commands/init.py index fe47ee64..7665dc5e 100644 --- a/trestlebot/cli/commands/init.py +++ b/trestlebot/cli/commands/init.py @@ -4,24 +4,56 @@ """" Module for Trestle-bot init command """ +import argparse import logging import sys from pathlib import Path import click +from trestle.common import file_utils from trestle.common.const import MODEL_DIR_LIST +from trestle.core.commands.common.return_codes import CmdReturnCodes +from trestle.core.commands.init import InitCmd from trestlebot.cli.options.common import common_options -from trestlebot.const import ERROR_EXIT_CODE, TRESTLEBOT_CONFIG_DIR +from trestlebot.const import ( + ERROR_EXIT_CODE, + TRESTLEBOT_CONFIG_DIR, + TRESTLEBOT_KEEP_FILE, +) logger = logging.getLogger(__name__) +def call_trestle_init(working_dir: str, debug: bool) -> None: + """Call compliance-trestle to initialize workspace""" + + verbose = 1 if debug else 0 + trestle_args = argparse.Namespace( + verbose=verbose, + trestle_root=Path(working_dir), + full=False, + govdocs=True, + local=False, + ) + return_code = InitCmd()._run(trestle_args) + if return_code == CmdReturnCodes.SUCCESS.value: + logger.debug("Initialized trestle project successfully") + else: + logger.error( + f"Initialization failed. Unexpted trestle error: {CmdReturnCodes(return_code).name}" + ) + sys.exit(ERROR_EXIT_CODE) + + def dir_make_if_not(directory: str) -> None: """Makes directory if it does not already exists.""" try: - Path(directory).mkdir(parents=True) + pathobj = Path(directory) + pathobj.mkdir(parents=True) + keep_file = pathobj.joinpath(Path(TRESTLEBOT_KEEP_FILE)) + file_utils.make_hidden_file(keep_file) logger.debug(f"Created directory {directory}") except FileExistsError: logger.debug(f"Directory {directory} exists, skipping create.") @@ -33,9 +65,7 @@ def dir_make_if_not(directory: str) -> None: help="Path to git repo. Used as working directory for authoring.", type=click.Path(exists=True), ) -@click.option( - "--markdown-dir", help="Path to store markdown content.", type=click.Path() -) +@click.option("--markdown-dir", help="Path to store markdown files.", type=click.Path()) @common_options def init_cmd( ctx: click.Context, working_dir: str, markdown_dir: str, debug: bool, config: str @@ -52,11 +82,13 @@ def init_cmd( ) if not working_dir: working_dir = click.prompt( - "Path to git repo", default=".", type=click.Path(exists=True) + "Enter path to git repo (workspace directory)", + default=".", + type=click.Path(exists=True), ) if not markdown_dir: markdown_dir = click.prompt( - "Path to store markdown content", default="./markdown" + "Enter path to store markdown files", default="./markdown" ) root_dir: Path = Path(working_dir) @@ -74,5 +106,14 @@ def init_cmd( ) sys.exit(ERROR_EXIT_CODE) + # Create model directories in workspace root model_dirs = list(map(lambda x: str(root_dir.joinpath(x)), MODEL_DIR_LIST)) list(map(dir_make_if_not, model_dirs)) + + # Create markdown directories in workspace root + markdown_dirs = list( + map(lambda x: str(root_dir.joinpath(f"{markdown_dir}/" + x)), MODEL_DIR_LIST) + ) + list(map(dir_make_if_not, markdown_dirs)) + + call_trestle_init(working_dir, debug) From 6be37e8d807eb4f2344c48c2702bb9e3ae18eb24 Mon Sep 17 00:00:00 2001 From: George Vauter Date: Tue, 19 Nov 2024 21:09:31 -0500 Subject: [PATCH 008/108] feat: simplify directory creation and better error handling for invalid configs --- trestlebot/cli/commands/init.py | 85 ++++++++++++++++++-------------- trestlebot/cli/config.py | 34 +++++++++---- trestlebot/cli/options/common.py | 4 +- 3 files changed, 75 insertions(+), 48 deletions(-) diff --git a/trestlebot/cli/commands/init.py b/trestlebot/cli/commands/init.py index 7665dc5e..ab4bcc1d 100644 --- a/trestlebot/cli/commands/init.py +++ b/trestlebot/cli/commands/init.py @@ -10,11 +10,11 @@ from pathlib import Path import click -from trestle.common import file_utils from trestle.common.const import MODEL_DIR_LIST from trestle.core.commands.common.return_codes import CmdReturnCodes from trestle.core.commands.init import InitCmd +from trestlebot.cli.config import make_config, write_to_file from trestlebot.cli.options.common import common_options from trestlebot.const import ( ERROR_EXIT_CODE, @@ -26,13 +26,13 @@ logger = logging.getLogger(__name__) -def call_trestle_init(working_dir: str, debug: bool) -> None: +def call_trestle_init(repo_path: Path, debug: bool) -> None: """Call compliance-trestle to initialize workspace""" verbose = 1 if debug else 0 trestle_args = argparse.Namespace( verbose=verbose, - trestle_root=Path(working_dir), + trestle_root=repo_path, full=False, govdocs=True, local=False, @@ -47,32 +47,23 @@ def call_trestle_init(working_dir: str, debug: bool) -> None: sys.exit(ERROR_EXIT_CODE) -def dir_make_if_not(directory: str) -> None: - """Makes directory if it does not already exists.""" - try: - pathobj = Path(directory) - pathobj.mkdir(parents=True) - keep_file = pathobj.joinpath(Path(TRESTLEBOT_KEEP_FILE)) - file_utils.make_hidden_file(keep_file) - logger.debug(f"Created directory {directory}") - except FileExistsError: - logger.debug(f"Directory {directory} exists, skipping create.") - - @click.command(name="init", help="Initialize a new trestle-bot repo.") @click.option( - "--working-dir", - help="Path to git repo. Used as working directory for authoring.", - type=click.Path(exists=True), + "--repo-path", + "repo_path", + help="Path to git repo. Used as trestle root directory.", + type=click.Path(path_type=Path, exists=True), +) +@click.option( + "--markdown-dir", help="Directory name to store markdown files.", type=str ) -@click.option("--markdown-dir", help="Path to store markdown files.", type=click.Path()) @common_options def init_cmd( - ctx: click.Context, working_dir: str, markdown_dir: str, debug: bool, config: str + ctx: click.Context, repo_path: Path, markdown_dir: str, debug: bool, config: str ) -> None: """Command to initialize a new trestlebot repo""" - need_to_prompt = any((not working_dir, not markdown_dir)) + need_to_prompt = any((not repo_path, not markdown_dir)) if need_to_prompt: click.echo("\n* Welcome to the Trestle-bot CLI *\n") @@ -80,40 +71,60 @@ def init_cmd( "Please provide the following values to initialize the " "workspace [press Enter for defaults].\n" ) - if not working_dir: - working_dir = click.prompt( + if not repo_path: + repo_path = click.prompt( "Enter path to git repo (workspace directory)", default=".", - type=click.Path(exists=True), + type=click.Path(path_type=Path, exists=True), ) if not markdown_dir: markdown_dir = click.prompt( - "Enter path to store markdown files", default="./markdown" + "Enter path to store markdown files", default="./markdown", type=str ) - root_dir: Path = Path(working_dir) - git_dir: Path = root_dir.joinpath(Path(".git")) - if not git_dir.exists(): + git_path: Path = repo_path.joinpath(Path(".git")) + if not git_path.exists(): logging.error( - f"[!] Initialization failed. Given directory {root_dir} is not a Git repository." + f"Initialization failed. Given directory {repo_path} is not a Git repository." ) sys.exit(ERROR_EXIT_CODE) - trestlebot_dir = root_dir.joinpath(Path(TRESTLEBOT_CONFIG_DIR)) + trestlebot_dir = repo_path.joinpath(Path(TRESTLEBOT_CONFIG_DIR)) if trestlebot_dir.exists(): logger.error( - f"[!] Initialization failed. Found existing {TRESTLEBOT_CONFIG_DIR} directory in {root_dir}" + f"Initialization failed. Found existing {TRESTLEBOT_CONFIG_DIR} directory in {repo_path}" ) sys.exit(ERROR_EXIT_CODE) # Create model directories in workspace root - model_dirs = list(map(lambda x: str(root_dir.joinpath(x)), MODEL_DIR_LIST)) - list(map(dir_make_if_not, model_dirs)) + list( + map( + lambda x: repo_path.joinpath(x) + .joinpath(TRESTLEBOT_KEEP_FILE) + .mkdir(parents=True, exist_ok=True), + MODEL_DIR_LIST, + ) + ) + logger.debug("Created model directories successfully") # Create markdown directories in workspace root - markdown_dirs = list( - map(lambda x: str(root_dir.joinpath(f"{markdown_dir}/" + x)), MODEL_DIR_LIST) + list( + map( + lambda x: repo_path.joinpath(Path(markdown_dir)) + .joinpath(x) + .joinpath(TRESTLEBOT_KEEP_FILE) + .mkdir(parents=True, exist_ok=True), + MODEL_DIR_LIST, + ) ) - list(map(dir_make_if_not, markdown_dirs)) + logger.debug("Created markdown directories successfully") + + # inovke the init command in compliance trestle + call_trestle_init(repo_path, debug) - call_trestle_init(working_dir, debug) + # generate and write trestle-bot cofig + config = make_config(dict(repo_path=repo_path, markdown_dir=markdown_dir)) + config_path = trestlebot_dir.joinpath(Path("config.yml")) + write_to_file(config, config_path) + logger.debug(f"trestle-bot config file created at {str(config_path)}") + logger.info(f"Successfully initialized trestlebot project in {repo_path}") diff --git a/trestlebot/cli/config.py b/trestlebot/cli/config.py index 105ce92f..4b256f96 100644 --- a/trestlebot/cli/config.py +++ b/trestlebot/cli/config.py @@ -36,21 +36,24 @@ def _format(self, err: Dict[str, Any]) -> str: if err.get("loc"): msg = f"Invalid config value for {err['loc'][0]}." if err.get("msg"): - msg += f" {err['msg']}" # Add error message details if present + msg += f" {err['msg']}." # Add error message details if present return msg + def __str__(self) -> str: + return "".join(self.errors) + class TrestleBotConfig(BaseModel): """Data model for trestle-bot configuration.""" - working_dir: Optional[DirectoryPath] = None - markdown_dir: Optional[DirectoryPath] = None + repo_path: Optional[DirectoryPath] = None + markdown_dir: Optional[str] = None @model_serializer def _dict(self) -> Dict[str, Any]: """Returns a dict that can be safely loaded to yaml.""" config_dict = { - "working_dir": str(self.working_dir), + "repo_path": str(self.repo_path), "markdown_dir": str(self.markdown_dir), } return dict( @@ -70,14 +73,27 @@ def load_from_file(file: str) -> Optional[TrestleBotConfig]: return None -def write_to_file(config: TrestleBotConfig, file: str) -> Path: +def write_to_file(config: TrestleBotConfig, file_path: Path) -> None: """Write config object to yaml file""" + try: + file_path.parent.mkdir(parents=True, exist_ok=True) + with file_path.open("w") as config_file: + yaml.dump(config.dict(), config_file) + except ValidationError as ex: + raise TrestleBotConfigError(ex.errors()) - with open(file, "w") as config_file: - yaml.dump(config.dict(), config_file) - return Path(file) + +def make_config(values: Dict[str, Any]) -> TrestleBotConfig: + """Generates a new trestle-bot config object""" + try: + return TrestleBotConfig(**values) + except ValidationError as ex: + raise TrestleBotConfigError(ex.errors()) def update_config(config: TrestleBotConfig, update: Dict[str, Any]) -> TrestleBotConfig: """Returns a new config object with specified updates.""" - return config.model_copy(update=update) + try: + return config.model_copy(update=update) + except ValidationError as ex: + raise TrestleBotConfigError(ex.errors()) diff --git a/trestlebot/cli/options/common.py b/trestlebot/cli/options/common.py index cff3851d..65b602ec 100644 --- a/trestlebot/cli/options/common.py +++ b/trestlebot/cli/options/common.py @@ -30,7 +30,7 @@ def wrapper(*args: Sequence[Any], **kwargs: Dict[Any, Any]) -> Any: except Exception as ex: traceback_str = traceback.format_exc() logger.error(f"Trestle-bot Error: {str(ex)}") - logger.error(traceback_str) + logger.debug(traceback_str) return ERROR_EXIT_CODE return wrapper @@ -57,7 +57,7 @@ def load_config_to_ctx(ctx: click.Context, param: str, value: str) -> Optional[s except TrestleBotConfigError as ex: logger.error(str(ex)) for err in ex.errors: - logger.error(f"[!] {err}") + logger.error({err}) sys.exit(ERROR_EXIT_CODE) if not ctx.default_map: From 45fc4b8ca0c0aa21cdce9b4dc4d511d9804f6431 Mon Sep 17 00:00:00 2001 From: Qingmin Duanmu Date: Wed, 20 Nov 2024 19:54:03 +0800 Subject: [PATCH 009/108] feat: initial work on autosync --- trestlebot/cli/base.py | 90 ++++++++++++++++ trestlebot/cli/commands/autosync.py | 154 ++++++++++++++++++++++++++++ trestlebot/cli/options/common.py | 47 +++++++++ trestlebot/cli/root.py | 2 + 4 files changed, 293 insertions(+) create mode 100644 trestlebot/cli/base.py diff --git a/trestlebot/cli/base.py b/trestlebot/cli/base.py new file mode 100644 index 00000000..051fe290 --- /dev/null +++ b/trestlebot/cli/base.py @@ -0,0 +1,90 @@ +import argparse +import logging +from typing import List + +from trestlebot.bot import TrestleBot +from trestlebot.cli.options.common import handle_exceptions +from trestlebot.tasks.assemble_task import AssembleTask +from trestlebot.tasks.authored import types +from trestlebot.tasks.authored.base_authored import AuthoredObjectBase +from trestlebot.tasks.base_task import ModelFilter, TaskBase +from trestlebot.tasks.regenerate_task import RegenerateTask + + +logger = logging.getLogger(__name__) + + +def comma_sep_to_list(string: str) -> List[str]: + """Convert comma-sep string to list of strings and strip.""" + string = string.strip() if string else "" + return list(map(str.strip, string.split(","))) if string else [] + + +def run_base(args: argparse.Namespace, pre_tasks: List[TaskBase]) -> None: + """Reusable logic for all commands.""" + # from trestlebot.reporter import BotResults, ResultsReporter + # git_provider: Optional[GitProvider] = self.set_git_provider(args) + # results_reporter: ResultsReporter = self.set_reporter() + + # Configure and run the bot + bot = TrestleBot( + working_dir=args.working_dir, + branch=args.branch, + commit_name=args.committer_name, + commit_email=args.committer_email, + author_name=args.author_name, + author_email=args.author_email, + # target_branch=args.target_branch, + ) + # results: BotResults = bot.run( + bot.run( + pre_tasks=pre_tasks, + patterns=comma_sep_to_list(args.patterns), + commit_message=args.commit_message, + # git_provider=git_provider, + # pull_request_title=args.pull_request_title, + dry_run=args.dry_run, + ) + + # # Report the results + # results_reporter.report_results(results) + + +@handle_exceptions +def run(oscal_model: str, args: argparse.Namespace, ssp_index_path: str = "") -> None: + """Run the autosync for oscal model.""" + + pre_tasks: List[TaskBase] = [] + # Allow any model to be skipped from the args, by default include all + model_filter: ModelFilter = ModelFilter( + skip_patterns=comma_sep_to_list(args.skip_items), + include_patterns=["*"], + ) + authored_object: AuthoredObjectBase = types.get_authored_object( + oscal_model, args.working_dir, ssp_index_path + ) + + # Assuming an edit has occurred assemble would be run before regenerate. + # Adding this to the list first + if not args.skip_assemble: + assemble_task: AssembleTask = AssembleTask( + authored_object=authored_object, + markdown_dir=args.markdown_path, + version=args.version, + model_filter=model_filter, + ) + pre_tasks.append(assemble_task) + else: + logger.info("Assemble task skipped.") + + if not args.skip_regenerate: + regenerate_task: RegenerateTask = RegenerateTask( + authored_object=authored_object, + markdown_dir=args.markdown_path, + model_filter=model_filter, + ) + pre_tasks.append(regenerate_task) + else: + logger.info("Regeneration task skipped.") + + run_base(args, pre_tasks) diff --git a/trestlebot/cli/commands/autosync.py b/trestlebot/cli/commands/autosync.py index a1965a7e..cb7dadc6 100644 --- a/trestlebot/cli/commands/autosync.py +++ b/trestlebot/cli/commands/autosync.py @@ -1 +1,155 @@ """ Autosync command""" + +import argparse + +import click + +from trestlebot.cli.base import comma_sep_to_list, run +from trestlebot.cli.options.common import git_options + + +@click.group(name="autosync", help="Autosync operations") +@click.option( + "--working-dir", + help="Working directory wit git repository", + type=click.Path(exists=True), +) +@click.option( + "--dry-run", + help="Run tasks, but do not push to the repository", + is_flag=True, +) +@click.option( + "--markdown-path", + help="Path to Trestle markdown files", + type=click.Path(exists=True), # Should it exist? +) +@click.option( + "--skip-items", + help="Comma-separated list of glob patterns to skip when running tasks", + type=str, # What's the type? +) +@click.option( + "--skip-assemble", + help="Skip assembly task", + is_flag=True, + default=False, + show_default=True, +) +@click.option( + "--skip-regenerate", + help="Skip regenerate task", + is_flag=True, + default=False, + show_default=True, +) +@click.option( + "--version", + help="Version of the OSCAL model to set during assembly into JSON", + type=str, +) +@git_options +@click.pass_context +def autosync_cmd( + ctx: click.Context, + working_dir: str, + markdown_path: str, + dry_run: bool, + skip_items: str, + skip_assemble: bool, + skip_regenerate: bool, + file_patterns: str, + branch: str, + commit_message: str, + committer_name: str, + committer_email: str, + author_name: str, + author_email: str, + version: str, +) -> None: + """Command to autosync catalog, profile, compdef and ssp.""" + + need_to_prompt = any( + ( + not working_dir, + not markdown_path, + not branch, + not committer_email, + not committer_name, + ) + ) + if need_to_prompt: + click.echo("\n* Welcome to the Trestle-bot CLI *\n") + click.echo("Please provide the following values to start autosync operations.") + if not working_dir: + working_dir = click.prompt( + "Enter path to working directory wit git repository", + default=".", + type=click.Path(exists=True), + ) + if not markdown_path: + markdown_path = click.prompt( + "Enter path to to Trestle markdown files", + type=click.Path(exists=True), + ) + if not branch: + branch = click.prompt( + "Enter branch name to push changes to", + ) + if not committer_email: + committer_email = click.prompt( + "Enter email for committer", + ) + if not committer_name: + committer_name = click.prompt( + "Enter name of committer", + ) + + ctx.trestle_args = argparse.Namespace( + working_dir=working_dir, + markdown_path=markdown_path, + skip_items=skip_items, + skip_assemble=skip_assemble, + skip_regenerate=skip_regenerate, + version=version, + patterns=comma_sep_to_list(file_patterns), + branch=branch, + commit_message=commit_message, + committer_name=committer_name, + committer_email=committer_email, + author_name=author_name, + author_email=author_email, + # target_branch=target_branch, + # pull_request_title=pull_request_title, + dry_run=dry_run, + ) + + +@autosync_cmd.command("ssp") +@click.pass_context +@click.option("--ssp-index-path", help="Path to ssp index file", type=click.File("r")) +def autosync_ssp_cmd(ctx: click.Context, ssp_index_path: str) -> None: + if not ssp_index_path: + ssp_index_path = click.prompt( + "Enter path to ssp index file", + type=click.Path(exists=True), + ) + run("ssp", ctx.parent.trestle_args, ssp_index_path) + + +@autosync_cmd.command("compdef") +@click.pass_context +def autosync_compdef_cmd(ctx: click.Context) -> None: + run("compdef", ctx.parent.trestle_args) + + +@autosync_cmd.command("catalog") +@click.pass_context +def autosync_catalog_cmd(ctx: click.Context) -> None: + run("catalog", ctx.parent.trestle_args) + + +@autosync_cmd.command("profile") +@click.pass_context +def autosync_profile_cmd(ctx: click.Context) -> None: + run("profile", ctx.parent.trestle_args) diff --git a/trestlebot/cli/options/common.py b/trestlebot/cli/options/common.py index 65b602ec..5a1167e2 100644 --- a/trestlebot/cli/options/common.py +++ b/trestlebot/cli/options/common.py @@ -100,3 +100,50 @@ def wrapper_common_options(*args: Sequence[Any], **kwargs: Dict[Any, Any]) -> An return f(*args, **kwargs) return wrapper_common_options + + +def git_options(f: F) -> Any: + """ + Configure git options used for git operations. + """ + + @click.option( + "--branch", + help="Branch name to push changes to", + type=str, + ) + @click.option( + "--committer-name", + help="Name of committer", + type=str, + ) + @click.option( + "--committer-email", + help="Email for committer", + type=str, + ) + @click.option( + "--file-patterns", + help="Comma-separated list of file patterns to be used with `git add` in repository updates", + type=str, + ) + @click.option( + "--commit-message", + help="Commit message for automated updates", + type=str, + ) + @click.option( + "--author-name", + help="Name for commit author if differs from committer", + type=str, + ) + @click.option( + "--author-email", + help="Email for commit author if differs from committer", + type=str, + ) + @functools.wraps(f) + def wrapper_git_options(*args: Sequence[Any], **kwargs: Dict[Any, Any]) -> Any: + return f(*args, **kwargs) + + return wrapper_git_options diff --git a/trestlebot/cli/root.py b/trestlebot/cli/root.py index 4f9e8759..2f6e50bf 100644 --- a/trestlebot/cli/root.py +++ b/trestlebot/cli/root.py @@ -6,6 +6,7 @@ import click +from trestlebot.cli.commands.autosync import autosync_cmd from trestlebot.cli.commands.init import init_cmd @@ -26,3 +27,4 @@ def root_cmd(ctx: click.Context) -> None: root_cmd.add_command(init_cmd) +root_cmd.add_command(autosync_cmd) From a239482bf1ebd2c84d8183d75d335ef477e49570 Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Wed, 20 Nov 2024 11:31:05 -0500 Subject: [PATCH 010/108] Initial create command for click cli --- trestlebot/cli/commands/create.py | 120 +++++++++++++++++++++++++++++- trestlebot/cli/options/create.py | 31 ++++++++ 2 files changed, 150 insertions(+), 1 deletion(-) create mode 100644 trestlebot/cli/options/create.py diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index 84d102d5..e5aac657 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -1 +1,119 @@ -""" Create CD Create SSP grouping command""" +""" +Module for create-cd create-ssp command for CLI +""" + +import click + +from trestlebot.cli.options.create import common_create_options + + + + +@click.group(name="create") +@common_create_options +def create_cmd(ctx: click.Context, profile_name: str) -> None: + """ + Command leveraged for component definition and ssp authoring in trestlebot. + """ + + pass + + +# @create_cmd.command(name="compdef", help="command for component definition authoring") +@create_cmd.command("compdef") +@click.option( + "--compdef-name", + prompt="Name of component definition is", + help="Name of component definition.", +) +@click.option( + "--component-title", + prompt="The name of component title is", + help="Title of initial component.", +) +@click.option( + "--component-description", + prompt="The description of the initial component is", + help="Description of initial component.", +) +@click.option( + "--filter-by-profile", + required=False, + help="Optionally filter the controls in the component definition by a profile", +) +@click.option( + "--component-definition-type", + default="service", + help="Type of component definition", +) +@common_create_options +def compdef_cmd( + ctx: click.Context, + profile_name: str, + compdef_name: str, + component_title: str, + component_description: str, + filter_by_profile: str, + component_definition_type: str, +) -> None: + """ + Component definition authoring command. + """ + click.echo( + f"The name of the profile in the trestle workspace to use with the component definition is {profile_name}." + ) + click.echo( + f"You have selected component definitions as the document you want {compdef_name} to author." + ) + click.echo(f"The component definition name is {component_title}.") + click.echo(f"The component description to author is {component_description}.") + click.echo( + f"The profile you want to filter controls in the component files is {filter_by_profile}." + ) + click.echo(f"The component definition type is {component_definition_type}.") + + +# @create_cmd.command(name="ssp", help="command for ssp authoring") +@create_cmd.command("ssp") +@click.option( + "--ssp-name", + prompt="Name of SSP to create", + help="Name of SSP to create.", +) +@click.option( + "--leveraged-ssp", + help="Provider SSP to leverage for the new SSP.", +) +@click.option( + "--ssp-index-path", + default="ssp-index.json", + help="Optionally set the path to the SSP index file.", +) +@click.option( + "--yaml-header-path", + default="ssp-index.json", + help="Optionally set a path to a YAML file for custom SSP Markdown YAML headers.", +) +@common_create_options +def ssp_cmd( + ctx: click.Context, + profile_name: str, + ssp_name: str, + leveraged_ssp: str, + ssp_index_path: str, + yaml_header_path: str, +) -> None: + """ + SSP Authoring command + """ + click.echo( + f"The name of the profile in trestle workspace to include in the SSP is {profile_name}." + ) + click.echo(f"The name of the SSP to create is {ssp_name}.") + click.echo(f"The leveraged SSP is {leveraged_ssp}.") + click.echo(f"The SSP index path is {ssp_index_path}.") + click.echo(f"The YAML file for custom SSP markdown is {yaml_header_path}.") + + +if __name__ == "__main__": + create_cmd() diff --git a/trestlebot/cli/options/create.py b/trestlebot/cli/options/create.py new file mode 100644 index 00000000..b3fef888 --- /dev/null +++ b/trestlebot/cli/options/create.py @@ -0,0 +1,31 @@ +""" +Module for common create commands +""" + +import functools +from typing import Any, Callable, Dict, Sequence, TypeVar + +import click + + +F = TypeVar("F", bound=Callable[..., Any]) + + +def common_create_options(f: F) -> Any: + """ + Configuring common create options decorator for SSP and CD command + """ + + @click.pass_context + @click.option( + "--profile-name", + prompt="Name of trestle workspace to include", + help="Name of profile in trestle workspace to include.", + ) + @functools.wraps(f) + def wrapper_common_create_options( + *args: Sequence[Any], **kwargs: Dict[Any, Any] + ) -> Any: + return f(*args, **kwargs) + + return wrapper_common_create_options From 5931b5c3055c0692339e11507cf0e0bce191a894 Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Wed, 20 Nov 2024 11:31:05 -0500 Subject: [PATCH 011/108] Initial create command for click cli --- trestlebot/cli/commands/create.py | 120 +++++++++++++++++++++++++++++- trestlebot/cli/options/create.py | 31 ++++++++ 2 files changed, 150 insertions(+), 1 deletion(-) create mode 100644 trestlebot/cli/options/create.py diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index 84d102d5..e5aac657 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -1 +1,119 @@ -""" Create CD Create SSP grouping command""" +""" +Module for create-cd create-ssp command for CLI +""" + +import click + +from trestlebot.cli.options.create import common_create_options + + + + +@click.group(name="create") +@common_create_options +def create_cmd(ctx: click.Context, profile_name: str) -> None: + """ + Command leveraged for component definition and ssp authoring in trestlebot. + """ + + pass + + +# @create_cmd.command(name="compdef", help="command for component definition authoring") +@create_cmd.command("compdef") +@click.option( + "--compdef-name", + prompt="Name of component definition is", + help="Name of component definition.", +) +@click.option( + "--component-title", + prompt="The name of component title is", + help="Title of initial component.", +) +@click.option( + "--component-description", + prompt="The description of the initial component is", + help="Description of initial component.", +) +@click.option( + "--filter-by-profile", + required=False, + help="Optionally filter the controls in the component definition by a profile", +) +@click.option( + "--component-definition-type", + default="service", + help="Type of component definition", +) +@common_create_options +def compdef_cmd( + ctx: click.Context, + profile_name: str, + compdef_name: str, + component_title: str, + component_description: str, + filter_by_profile: str, + component_definition_type: str, +) -> None: + """ + Component definition authoring command. + """ + click.echo( + f"The name of the profile in the trestle workspace to use with the component definition is {profile_name}." + ) + click.echo( + f"You have selected component definitions as the document you want {compdef_name} to author." + ) + click.echo(f"The component definition name is {component_title}.") + click.echo(f"The component description to author is {component_description}.") + click.echo( + f"The profile you want to filter controls in the component files is {filter_by_profile}." + ) + click.echo(f"The component definition type is {component_definition_type}.") + + +# @create_cmd.command(name="ssp", help="command for ssp authoring") +@create_cmd.command("ssp") +@click.option( + "--ssp-name", + prompt="Name of SSP to create", + help="Name of SSP to create.", +) +@click.option( + "--leveraged-ssp", + help="Provider SSP to leverage for the new SSP.", +) +@click.option( + "--ssp-index-path", + default="ssp-index.json", + help="Optionally set the path to the SSP index file.", +) +@click.option( + "--yaml-header-path", + default="ssp-index.json", + help="Optionally set a path to a YAML file for custom SSP Markdown YAML headers.", +) +@common_create_options +def ssp_cmd( + ctx: click.Context, + profile_name: str, + ssp_name: str, + leveraged_ssp: str, + ssp_index_path: str, + yaml_header_path: str, +) -> None: + """ + SSP Authoring command + """ + click.echo( + f"The name of the profile in trestle workspace to include in the SSP is {profile_name}." + ) + click.echo(f"The name of the SSP to create is {ssp_name}.") + click.echo(f"The leveraged SSP is {leveraged_ssp}.") + click.echo(f"The SSP index path is {ssp_index_path}.") + click.echo(f"The YAML file for custom SSP markdown is {yaml_header_path}.") + + +if __name__ == "__main__": + create_cmd() diff --git a/trestlebot/cli/options/create.py b/trestlebot/cli/options/create.py new file mode 100644 index 00000000..b3fef888 --- /dev/null +++ b/trestlebot/cli/options/create.py @@ -0,0 +1,31 @@ +""" +Module for common create commands +""" + +import functools +from typing import Any, Callable, Dict, Sequence, TypeVar + +import click + + +F = TypeVar("F", bound=Callable[..., Any]) + + +def common_create_options(f: F) -> Any: + """ + Configuring common create options decorator for SSP and CD command + """ + + @click.pass_context + @click.option( + "--profile-name", + prompt="Name of trestle workspace to include", + help="Name of profile in trestle workspace to include.", + ) + @functools.wraps(f) + def wrapper_common_create_options( + *args: Sequence[Any], **kwargs: Dict[Any, Any] + ) -> Any: + return f(*args, **kwargs) + + return wrapper_common_create_options From 1ba1010463925b4273f72fc7d4fc6513fd9ad9e2 Mon Sep 17 00:00:00 2001 From: George Vauter Date: Wed, 20 Nov 2024 12:51:59 -0500 Subject: [PATCH 012/108] adding unit test for config module --- tests/trestlebot/cli/test_config.py | 71 +++++++++++++++++++++++++++++ trestlebot/cli/commands/create.py | 4 +- trestlebot/cli/config.py | 2 +- trestlebot/cli/options/create.py | 2 +- 4 files changed, 74 insertions(+), 5 deletions(-) create mode 100644 tests/trestlebot/cli/test_config.py diff --git a/tests/trestlebot/cli/test_config.py b/tests/trestlebot/cli/test_config.py new file mode 100644 index 00000000..8ed4b141 --- /dev/null +++ b/tests/trestlebot/cli/test_config.py @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Red Hat, Inc. + + +"""Unit tests for CLI config module""" +import pathlib + +import pytest +import yaml + +from trestlebot.cli.config import ( + TrestleBotConfig, + TrestleBotConfigError, + load_from_file, + make_config, + write_to_file, +) + + +@pytest.fixture +def config_obj() -> TrestleBotConfig: + return TrestleBotConfig(repo_path="/tmp", markdown_dir="markdown") + + +def test_invalid_config_raises_errors() -> None: + """Test create config with invalid directory to raise error.""" + + with pytest.raises(TrestleBotConfigError) as ex: + _ = make_config(dict(repo_path="0")) + + assert ( + str(ex.value) + == "Invalid config value for repo_path. Path does not point to a directory." + ) + + +def test_make_config_raises_no_errors(tmp_init_dir: str) -> None: + """Test create a valid config object.""" + config = make_config(dict(repo_path=tmp_init_dir, markdown_dir="markdown-test")) + assert isinstance(config, TrestleBotConfig) + assert config.repo_path == pathlib.Path(tmp_init_dir) + assert config.markdown_dir == "markdown-test" + + +def test_config_to_dict(config_obj: TrestleBotConfig) -> None: + """Config should serialize to a dict.""" + model_dict = config_obj.model_dump() + assert isinstance(model_dict, dict) + assert model_dict["repo_path"] == str(config_obj.repo_path) + assert model_dict["markdown_dir"] == config_obj.markdown_dir + + +def test_config_write_to_file(config_obj: TrestleBotConfig, tmp_init_dir: str) -> None: + """Test config is written to yaml file.""" + filepath = pathlib.Path(tmp_init_dir).joinpath("config.yml") + write_to_file(config_obj, filepath) + with open(filepath, "r") as f: + yaml_data = yaml.safe_load(f) + + assert yaml_data == config_obj.model_dump() + + +def test_config_laod_from_file(config_obj: TrestleBotConfig, tmp_init_dir: str) -> None: + """Test config is read from yaml file into config object.""" + filepath = pathlib.Path(tmp_init_dir).joinpath("config.yml") + with filepath.open("w") as config_file: + yaml.dump(config_obj.model_dump(), config_file) + + config = load_from_file(str(filepath)) + assert isinstance(config, TrestleBotConfig) + assert config.model_dump() == config_obj.model_dump() diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index e5aac657..081f6ebe 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -7,8 +7,6 @@ from trestlebot.cli.options.create import common_create_options - - @click.group(name="create") @common_create_options def create_cmd(ctx: click.Context, profile_name: str) -> None: @@ -60,7 +58,7 @@ def compdef_cmd( Component definition authoring command. """ click.echo( - f"The name of the profile in the trestle workspace to use with the component definition is {profile_name}." + f"The name of the profile in use with the component definition is {profile_name}." ) click.echo( f"You have selected component definitions as the document you want {compdef_name} to author." diff --git a/trestlebot/cli/config.py b/trestlebot/cli/config.py index 4b256f96..4d824e74 100644 --- a/trestlebot/cli/config.py +++ b/trestlebot/cli/config.py @@ -78,7 +78,7 @@ def write_to_file(config: TrestleBotConfig, file_path: Path) -> None: try: file_path.parent.mkdir(parents=True, exist_ok=True) with file_path.open("w") as config_file: - yaml.dump(config.dict(), config_file) + yaml.dump(config.model_dump(), config_file) except ValidationError as ex: raise TrestleBotConfigError(ex.errors()) diff --git a/trestlebot/cli/options/create.py b/trestlebot/cli/options/create.py index b3fef888..c797d05d 100644 --- a/trestlebot/cli/options/create.py +++ b/trestlebot/cli/options/create.py @@ -1,4 +1,4 @@ -""" +""" Module for common create commands """ From 7645fbe8b98509f7f480fd3c663fe4339bf0484c Mon Sep 17 00:00:00 2001 From: George Vauter Date: Wed, 20 Nov 2024 12:51:59 -0500 Subject: [PATCH 013/108] adding unit test for config module --- tests/trestlebot/cli/test_config.py | 71 +++++++++++++++++++++++++++++ trestlebot/cli/commands/create.py | 4 +- trestlebot/cli/config.py | 2 +- trestlebot/cli/options/create.py | 2 +- 4 files changed, 74 insertions(+), 5 deletions(-) create mode 100644 tests/trestlebot/cli/test_config.py diff --git a/tests/trestlebot/cli/test_config.py b/tests/trestlebot/cli/test_config.py new file mode 100644 index 00000000..8ed4b141 --- /dev/null +++ b/tests/trestlebot/cli/test_config.py @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Red Hat, Inc. + + +"""Unit tests for CLI config module""" +import pathlib + +import pytest +import yaml + +from trestlebot.cli.config import ( + TrestleBotConfig, + TrestleBotConfigError, + load_from_file, + make_config, + write_to_file, +) + + +@pytest.fixture +def config_obj() -> TrestleBotConfig: + return TrestleBotConfig(repo_path="/tmp", markdown_dir="markdown") + + +def test_invalid_config_raises_errors() -> None: + """Test create config with invalid directory to raise error.""" + + with pytest.raises(TrestleBotConfigError) as ex: + _ = make_config(dict(repo_path="0")) + + assert ( + str(ex.value) + == "Invalid config value for repo_path. Path does not point to a directory." + ) + + +def test_make_config_raises_no_errors(tmp_init_dir: str) -> None: + """Test create a valid config object.""" + config = make_config(dict(repo_path=tmp_init_dir, markdown_dir="markdown-test")) + assert isinstance(config, TrestleBotConfig) + assert config.repo_path == pathlib.Path(tmp_init_dir) + assert config.markdown_dir == "markdown-test" + + +def test_config_to_dict(config_obj: TrestleBotConfig) -> None: + """Config should serialize to a dict.""" + model_dict = config_obj.model_dump() + assert isinstance(model_dict, dict) + assert model_dict["repo_path"] == str(config_obj.repo_path) + assert model_dict["markdown_dir"] == config_obj.markdown_dir + + +def test_config_write_to_file(config_obj: TrestleBotConfig, tmp_init_dir: str) -> None: + """Test config is written to yaml file.""" + filepath = pathlib.Path(tmp_init_dir).joinpath("config.yml") + write_to_file(config_obj, filepath) + with open(filepath, "r") as f: + yaml_data = yaml.safe_load(f) + + assert yaml_data == config_obj.model_dump() + + +def test_config_laod_from_file(config_obj: TrestleBotConfig, tmp_init_dir: str) -> None: + """Test config is read from yaml file into config object.""" + filepath = pathlib.Path(tmp_init_dir).joinpath("config.yml") + with filepath.open("w") as config_file: + yaml.dump(config_obj.model_dump(), config_file) + + config = load_from_file(str(filepath)) + assert isinstance(config, TrestleBotConfig) + assert config.model_dump() == config_obj.model_dump() diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index e5aac657..081f6ebe 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -7,8 +7,6 @@ from trestlebot.cli.options.create import common_create_options - - @click.group(name="create") @common_create_options def create_cmd(ctx: click.Context, profile_name: str) -> None: @@ -60,7 +58,7 @@ def compdef_cmd( Component definition authoring command. """ click.echo( - f"The name of the profile in the trestle workspace to use with the component definition is {profile_name}." + f"The name of the profile in use with the component definition is {profile_name}." ) click.echo( f"You have selected component definitions as the document you want {compdef_name} to author." diff --git a/trestlebot/cli/config.py b/trestlebot/cli/config.py index 4b256f96..4d824e74 100644 --- a/trestlebot/cli/config.py +++ b/trestlebot/cli/config.py @@ -78,7 +78,7 @@ def write_to_file(config: TrestleBotConfig, file_path: Path) -> None: try: file_path.parent.mkdir(parents=True, exist_ok=True) with file_path.open("w") as config_file: - yaml.dump(config.dict(), config_file) + yaml.dump(config.model_dump(), config_file) except ValidationError as ex: raise TrestleBotConfigError(ex.errors()) diff --git a/trestlebot/cli/options/create.py b/trestlebot/cli/options/create.py index b3fef888..c797d05d 100644 --- a/trestlebot/cli/options/create.py +++ b/trestlebot/cli/options/create.py @@ -1,4 +1,4 @@ -""" +""" Module for common create commands """ From 268c5584719883ca6ac820dca122ca95cbcf07a0 Mon Sep 17 00:00:00 2001 From: Qingmin Duanmu Date: Thu, 21 Nov 2024 15:10:50 +0800 Subject: [PATCH 014/108] Update autosync command --- trestlebot/cli/base.py | 90 ------------------ trestlebot/cli/commands/autosync.py | 136 +++++++++++++++++----------- trestlebot/cli/options/common.py | 3 + trestlebot/cli/run.py | 29 ++++++ 4 files changed, 113 insertions(+), 145 deletions(-) delete mode 100644 trestlebot/cli/base.py create mode 100644 trestlebot/cli/run.py diff --git a/trestlebot/cli/base.py b/trestlebot/cli/base.py deleted file mode 100644 index 051fe290..00000000 --- a/trestlebot/cli/base.py +++ /dev/null @@ -1,90 +0,0 @@ -import argparse -import logging -from typing import List - -from trestlebot.bot import TrestleBot -from trestlebot.cli.options.common import handle_exceptions -from trestlebot.tasks.assemble_task import AssembleTask -from trestlebot.tasks.authored import types -from trestlebot.tasks.authored.base_authored import AuthoredObjectBase -from trestlebot.tasks.base_task import ModelFilter, TaskBase -from trestlebot.tasks.regenerate_task import RegenerateTask - - -logger = logging.getLogger(__name__) - - -def comma_sep_to_list(string: str) -> List[str]: - """Convert comma-sep string to list of strings and strip.""" - string = string.strip() if string else "" - return list(map(str.strip, string.split(","))) if string else [] - - -def run_base(args: argparse.Namespace, pre_tasks: List[TaskBase]) -> None: - """Reusable logic for all commands.""" - # from trestlebot.reporter import BotResults, ResultsReporter - # git_provider: Optional[GitProvider] = self.set_git_provider(args) - # results_reporter: ResultsReporter = self.set_reporter() - - # Configure and run the bot - bot = TrestleBot( - working_dir=args.working_dir, - branch=args.branch, - commit_name=args.committer_name, - commit_email=args.committer_email, - author_name=args.author_name, - author_email=args.author_email, - # target_branch=args.target_branch, - ) - # results: BotResults = bot.run( - bot.run( - pre_tasks=pre_tasks, - patterns=comma_sep_to_list(args.patterns), - commit_message=args.commit_message, - # git_provider=git_provider, - # pull_request_title=args.pull_request_title, - dry_run=args.dry_run, - ) - - # # Report the results - # results_reporter.report_results(results) - - -@handle_exceptions -def run(oscal_model: str, args: argparse.Namespace, ssp_index_path: str = "") -> None: - """Run the autosync for oscal model.""" - - pre_tasks: List[TaskBase] = [] - # Allow any model to be skipped from the args, by default include all - model_filter: ModelFilter = ModelFilter( - skip_patterns=comma_sep_to_list(args.skip_items), - include_patterns=["*"], - ) - authored_object: AuthoredObjectBase = types.get_authored_object( - oscal_model, args.working_dir, ssp_index_path - ) - - # Assuming an edit has occurred assemble would be run before regenerate. - # Adding this to the list first - if not args.skip_assemble: - assemble_task: AssembleTask = AssembleTask( - authored_object=authored_object, - markdown_dir=args.markdown_path, - version=args.version, - model_filter=model_filter, - ) - pre_tasks.append(assemble_task) - else: - logger.info("Assemble task skipped.") - - if not args.skip_regenerate: - regenerate_task: RegenerateTask = RegenerateTask( - authored_object=authored_object, - markdown_dir=args.markdown_path, - model_filter=model_filter, - ) - pre_tasks.append(regenerate_task) - else: - logger.info("Regeneration task skipped.") - - run_base(args, pre_tasks) diff --git a/trestlebot/cli/commands/autosync.py b/trestlebot/cli/commands/autosync.py index cb7dadc6..e9576c7f 100644 --- a/trestlebot/cli/commands/autosync.py +++ b/trestlebot/cli/commands/autosync.py @@ -1,11 +1,63 @@ """ Autosync command""" -import argparse +import logging +from typing import Any, Dict, List import click -from trestlebot.cli.base import comma_sep_to_list, run -from trestlebot.cli.options.common import git_options +from trestlebot.cli.options.common import git_options, handle_exceptions +from trestlebot.cli.run import comma_sep_to_list +from trestlebot.cli.run import run as bot_run +from trestlebot.tasks.assemble_task import AssembleTask +from trestlebot.tasks.authored import types +from trestlebot.tasks.authored.base_authored import AuthoredObjectBase +from trestlebot.tasks.base_task import ModelFilter, TaskBase +from trestlebot.tasks.regenerate_task import RegenerateTask + + +logger = logging.getLogger(__name__) + + +@handle_exceptions +def run(oscal_model: str, ctx_obj: Dict[str, Any]) -> None: + """Run the autosync for oscal model.""" + + pre_tasks: List[TaskBase] = [] + # Allow any model to be skipped from the args, by default include all + model_filter: ModelFilter = ModelFilter( + skip_patterns=comma_sep_to_list(ctx_obj.get("skip_items", "")), + include_patterns=["*"], + ) + authored_object: AuthoredObjectBase = types.get_authored_object( + oscal_model, + ctx_obj["working_dir"], + ctx_obj.get("ssp_index_path", ""), + ) + + # Assuming an edit has occurred assemble would be run before regenerate. + # Adding this to the list first + if not ctx_obj.get("skip_assemble"): + assemble_task: AssembleTask = AssembleTask( + authored_object=authored_object, + markdown_dir=ctx_obj["markdown_path"], + version=ctx_obj.get("version", ""), + model_filter=model_filter, + ) + pre_tasks.append(assemble_task) + else: + logger.info("Assemble task skipped.") + + if not ctx_obj.get("skip_regenerate"): + regenerate_task: RegenerateTask = RegenerateTask( + authored_object=authored_object, + markdown_dir=ctx_obj["markdown_path"], + model_filter=model_filter, + ) + pre_tasks.append(regenerate_task) + else: + logger.info("Regeneration task skipped.") + + bot_run(pre_tasks, ctx_obj) @click.group(name="autosync", help="Autosync operations") @@ -13,21 +65,19 @@ "--working-dir", help="Working directory wit git repository", type=click.Path(exists=True), -) -@click.option( - "--dry-run", - help="Run tasks, but do not push to the repository", - is_flag=True, -) + prompt="Enter path to git repo (workspace directory)", + default=".", +) # TODO: use path in config @click.option( "--markdown-path", help="Path to Trestle markdown files", - type=click.Path(exists=True), # Should it exist? + type=click.Path(exists=True), + prompt="Enter path to to Trestle markdown files", ) @click.option( "--skip-items", help="Comma-separated list of glob patterns to skip when running tasks", - type=str, # What's the type? + type=str, ) @click.option( "--skip-assemble", @@ -48,6 +98,11 @@ help="Version of the OSCAL model to set during assembly into JSON", type=str, ) +@click.option( + "--dry-run", + help="Run tasks, but do not push to the repository", + is_flag=True, +) @git_options @click.pass_context def autosync_cmd( @@ -69,43 +124,7 @@ def autosync_cmd( ) -> None: """Command to autosync catalog, profile, compdef and ssp.""" - need_to_prompt = any( - ( - not working_dir, - not markdown_path, - not branch, - not committer_email, - not committer_name, - ) - ) - if need_to_prompt: - click.echo("\n* Welcome to the Trestle-bot CLI *\n") - click.echo("Please provide the following values to start autosync operations.") - if not working_dir: - working_dir = click.prompt( - "Enter path to working directory wit git repository", - default=".", - type=click.Path(exists=True), - ) - if not markdown_path: - markdown_path = click.prompt( - "Enter path to to Trestle markdown files", - type=click.Path(exists=True), - ) - if not branch: - branch = click.prompt( - "Enter branch name to push changes to", - ) - if not committer_email: - committer_email = click.prompt( - "Enter email for committer", - ) - if not committer_name: - committer_name = click.prompt( - "Enter name of committer", - ) - - ctx.trestle_args = argparse.Namespace( + ctx.obj = dict( working_dir=working_dir, markdown_path=markdown_path, skip_items=skip_items, @@ -119,37 +138,44 @@ def autosync_cmd( committer_email=committer_email, author_name=author_name, author_email=author_email, - # target_branch=target_branch, - # pull_request_title=pull_request_title, dry_run=dry_run, ) @autosync_cmd.command("ssp") +@click.option( + "--ssp-index-path", + help="Path to ssp index file", + type=click.File("r"), +) @click.pass_context -@click.option("--ssp-index-path", help="Path to ssp index file", type=click.File("r")) def autosync_ssp_cmd(ctx: click.Context, ssp_index_path: str) -> None: if not ssp_index_path: ssp_index_path = click.prompt( "Enter path to ssp index file", type=click.Path(exists=True), ) - run("ssp", ctx.parent.trestle_args, ssp_index_path) + ctx.obj.update( + { + "ssp_index_path": ssp_index_path, + } + ) + run("ssp", ctx.obj) @autosync_cmd.command("compdef") @click.pass_context def autosync_compdef_cmd(ctx: click.Context) -> None: - run("compdef", ctx.parent.trestle_args) + run("compdef", ctx.obj) @autosync_cmd.command("catalog") @click.pass_context def autosync_catalog_cmd(ctx: click.Context) -> None: - run("catalog", ctx.parent.trestle_args) + run("catalog", ctx.obj) @autosync_cmd.command("profile") @click.pass_context def autosync_profile_cmd(ctx: click.Context) -> None: - run("profile", ctx.parent.trestle_args) + run("profile", ctx.obj) diff --git a/trestlebot/cli/options/common.py b/trestlebot/cli/options/common.py index 5a1167e2..b77e3a6f 100644 --- a/trestlebot/cli/options/common.py +++ b/trestlebot/cli/options/common.py @@ -111,16 +111,19 @@ def git_options(f: F) -> Any: "--branch", help="Branch name to push changes to", type=str, + prompt="Enter branch name to push changes to", ) @click.option( "--committer-name", help="Name of committer", type=str, + prompt="Enter name for committer", ) @click.option( "--committer-email", help="Email for committer", type=str, + prompt="Enter email for committer", ) @click.option( "--file-patterns", diff --git a/trestlebot/cli/run.py b/trestlebot/cli/run.py new file mode 100644 index 00000000..561e2dfe --- /dev/null +++ b/trestlebot/cli/run.py @@ -0,0 +1,29 @@ +from typing import Any, Dict, List + +from trestlebot.bot import TrestleBot +from trestlebot.tasks.base_task import TaskBase + + +def comma_sep_to_list(string: str) -> List[str]: + """Convert comma-sep string to list of strings and strip.""" + string = string.strip() if string else "" + return list(map(str.strip, string.split(","))) if string else [] + + +def run(pre_tasks: List[TaskBase], kwargs: Dict[Any, Any]) -> None: + """Reusable logic for all commands.""" + # Configure and run the bot + bot = TrestleBot( + working_dir=kwargs["working_dir"], + branch=kwargs["branch"], + commit_name=kwargs["committer_name"], + commit_email=kwargs["committer_email"], + author_name=kwargs.get("author_name", ""), + author_email=kwargs.get("author_email", ""), + ) + bot.run( + pre_tasks=pre_tasks, + patterns=comma_sep_to_list(kwargs.get("patterns", "")), + commit_message=kwargs.get("commit_message", "Automatic updates from bot"), + dry_run=kwargs.get("dry_run", False), + ) From 823e5bce4a3f72eef21f29c195b25a1336ac0fb8 Mon Sep 17 00:00:00 2001 From: Qingmin Duanmu Date: Thu, 21 Nov 2024 15:10:50 +0800 Subject: [PATCH 015/108] Update autosync command --- trestlebot/cli/base.py | 90 ------------------ trestlebot/cli/commands/autosync.py | 136 +++++++++++++++++----------- trestlebot/cli/options/common.py | 3 + trestlebot/cli/run.py | 29 ++++++ 4 files changed, 113 insertions(+), 145 deletions(-) delete mode 100644 trestlebot/cli/base.py create mode 100644 trestlebot/cli/run.py diff --git a/trestlebot/cli/base.py b/trestlebot/cli/base.py deleted file mode 100644 index 051fe290..00000000 --- a/trestlebot/cli/base.py +++ /dev/null @@ -1,90 +0,0 @@ -import argparse -import logging -from typing import List - -from trestlebot.bot import TrestleBot -from trestlebot.cli.options.common import handle_exceptions -from trestlebot.tasks.assemble_task import AssembleTask -from trestlebot.tasks.authored import types -from trestlebot.tasks.authored.base_authored import AuthoredObjectBase -from trestlebot.tasks.base_task import ModelFilter, TaskBase -from trestlebot.tasks.regenerate_task import RegenerateTask - - -logger = logging.getLogger(__name__) - - -def comma_sep_to_list(string: str) -> List[str]: - """Convert comma-sep string to list of strings and strip.""" - string = string.strip() if string else "" - return list(map(str.strip, string.split(","))) if string else [] - - -def run_base(args: argparse.Namespace, pre_tasks: List[TaskBase]) -> None: - """Reusable logic for all commands.""" - # from trestlebot.reporter import BotResults, ResultsReporter - # git_provider: Optional[GitProvider] = self.set_git_provider(args) - # results_reporter: ResultsReporter = self.set_reporter() - - # Configure and run the bot - bot = TrestleBot( - working_dir=args.working_dir, - branch=args.branch, - commit_name=args.committer_name, - commit_email=args.committer_email, - author_name=args.author_name, - author_email=args.author_email, - # target_branch=args.target_branch, - ) - # results: BotResults = bot.run( - bot.run( - pre_tasks=pre_tasks, - patterns=comma_sep_to_list(args.patterns), - commit_message=args.commit_message, - # git_provider=git_provider, - # pull_request_title=args.pull_request_title, - dry_run=args.dry_run, - ) - - # # Report the results - # results_reporter.report_results(results) - - -@handle_exceptions -def run(oscal_model: str, args: argparse.Namespace, ssp_index_path: str = "") -> None: - """Run the autosync for oscal model.""" - - pre_tasks: List[TaskBase] = [] - # Allow any model to be skipped from the args, by default include all - model_filter: ModelFilter = ModelFilter( - skip_patterns=comma_sep_to_list(args.skip_items), - include_patterns=["*"], - ) - authored_object: AuthoredObjectBase = types.get_authored_object( - oscal_model, args.working_dir, ssp_index_path - ) - - # Assuming an edit has occurred assemble would be run before regenerate. - # Adding this to the list first - if not args.skip_assemble: - assemble_task: AssembleTask = AssembleTask( - authored_object=authored_object, - markdown_dir=args.markdown_path, - version=args.version, - model_filter=model_filter, - ) - pre_tasks.append(assemble_task) - else: - logger.info("Assemble task skipped.") - - if not args.skip_regenerate: - regenerate_task: RegenerateTask = RegenerateTask( - authored_object=authored_object, - markdown_dir=args.markdown_path, - model_filter=model_filter, - ) - pre_tasks.append(regenerate_task) - else: - logger.info("Regeneration task skipped.") - - run_base(args, pre_tasks) diff --git a/trestlebot/cli/commands/autosync.py b/trestlebot/cli/commands/autosync.py index cb7dadc6..e9576c7f 100644 --- a/trestlebot/cli/commands/autosync.py +++ b/trestlebot/cli/commands/autosync.py @@ -1,11 +1,63 @@ """ Autosync command""" -import argparse +import logging +from typing import Any, Dict, List import click -from trestlebot.cli.base import comma_sep_to_list, run -from trestlebot.cli.options.common import git_options +from trestlebot.cli.options.common import git_options, handle_exceptions +from trestlebot.cli.run import comma_sep_to_list +from trestlebot.cli.run import run as bot_run +from trestlebot.tasks.assemble_task import AssembleTask +from trestlebot.tasks.authored import types +from trestlebot.tasks.authored.base_authored import AuthoredObjectBase +from trestlebot.tasks.base_task import ModelFilter, TaskBase +from trestlebot.tasks.regenerate_task import RegenerateTask + + +logger = logging.getLogger(__name__) + + +@handle_exceptions +def run(oscal_model: str, ctx_obj: Dict[str, Any]) -> None: + """Run the autosync for oscal model.""" + + pre_tasks: List[TaskBase] = [] + # Allow any model to be skipped from the args, by default include all + model_filter: ModelFilter = ModelFilter( + skip_patterns=comma_sep_to_list(ctx_obj.get("skip_items", "")), + include_patterns=["*"], + ) + authored_object: AuthoredObjectBase = types.get_authored_object( + oscal_model, + ctx_obj["working_dir"], + ctx_obj.get("ssp_index_path", ""), + ) + + # Assuming an edit has occurred assemble would be run before regenerate. + # Adding this to the list first + if not ctx_obj.get("skip_assemble"): + assemble_task: AssembleTask = AssembleTask( + authored_object=authored_object, + markdown_dir=ctx_obj["markdown_path"], + version=ctx_obj.get("version", ""), + model_filter=model_filter, + ) + pre_tasks.append(assemble_task) + else: + logger.info("Assemble task skipped.") + + if not ctx_obj.get("skip_regenerate"): + regenerate_task: RegenerateTask = RegenerateTask( + authored_object=authored_object, + markdown_dir=ctx_obj["markdown_path"], + model_filter=model_filter, + ) + pre_tasks.append(regenerate_task) + else: + logger.info("Regeneration task skipped.") + + bot_run(pre_tasks, ctx_obj) @click.group(name="autosync", help="Autosync operations") @@ -13,21 +65,19 @@ "--working-dir", help="Working directory wit git repository", type=click.Path(exists=True), -) -@click.option( - "--dry-run", - help="Run tasks, but do not push to the repository", - is_flag=True, -) + prompt="Enter path to git repo (workspace directory)", + default=".", +) # TODO: use path in config @click.option( "--markdown-path", help="Path to Trestle markdown files", - type=click.Path(exists=True), # Should it exist? + type=click.Path(exists=True), + prompt="Enter path to to Trestle markdown files", ) @click.option( "--skip-items", help="Comma-separated list of glob patterns to skip when running tasks", - type=str, # What's the type? + type=str, ) @click.option( "--skip-assemble", @@ -48,6 +98,11 @@ help="Version of the OSCAL model to set during assembly into JSON", type=str, ) +@click.option( + "--dry-run", + help="Run tasks, but do not push to the repository", + is_flag=True, +) @git_options @click.pass_context def autosync_cmd( @@ -69,43 +124,7 @@ def autosync_cmd( ) -> None: """Command to autosync catalog, profile, compdef and ssp.""" - need_to_prompt = any( - ( - not working_dir, - not markdown_path, - not branch, - not committer_email, - not committer_name, - ) - ) - if need_to_prompt: - click.echo("\n* Welcome to the Trestle-bot CLI *\n") - click.echo("Please provide the following values to start autosync operations.") - if not working_dir: - working_dir = click.prompt( - "Enter path to working directory wit git repository", - default=".", - type=click.Path(exists=True), - ) - if not markdown_path: - markdown_path = click.prompt( - "Enter path to to Trestle markdown files", - type=click.Path(exists=True), - ) - if not branch: - branch = click.prompt( - "Enter branch name to push changes to", - ) - if not committer_email: - committer_email = click.prompt( - "Enter email for committer", - ) - if not committer_name: - committer_name = click.prompt( - "Enter name of committer", - ) - - ctx.trestle_args = argparse.Namespace( + ctx.obj = dict( working_dir=working_dir, markdown_path=markdown_path, skip_items=skip_items, @@ -119,37 +138,44 @@ def autosync_cmd( committer_email=committer_email, author_name=author_name, author_email=author_email, - # target_branch=target_branch, - # pull_request_title=pull_request_title, dry_run=dry_run, ) @autosync_cmd.command("ssp") +@click.option( + "--ssp-index-path", + help="Path to ssp index file", + type=click.File("r"), +) @click.pass_context -@click.option("--ssp-index-path", help="Path to ssp index file", type=click.File("r")) def autosync_ssp_cmd(ctx: click.Context, ssp_index_path: str) -> None: if not ssp_index_path: ssp_index_path = click.prompt( "Enter path to ssp index file", type=click.Path(exists=True), ) - run("ssp", ctx.parent.trestle_args, ssp_index_path) + ctx.obj.update( + { + "ssp_index_path": ssp_index_path, + } + ) + run("ssp", ctx.obj) @autosync_cmd.command("compdef") @click.pass_context def autosync_compdef_cmd(ctx: click.Context) -> None: - run("compdef", ctx.parent.trestle_args) + run("compdef", ctx.obj) @autosync_cmd.command("catalog") @click.pass_context def autosync_catalog_cmd(ctx: click.Context) -> None: - run("catalog", ctx.parent.trestle_args) + run("catalog", ctx.obj) @autosync_cmd.command("profile") @click.pass_context def autosync_profile_cmd(ctx: click.Context) -> None: - run("profile", ctx.parent.trestle_args) + run("profile", ctx.obj) diff --git a/trestlebot/cli/options/common.py b/trestlebot/cli/options/common.py index 5a1167e2..b77e3a6f 100644 --- a/trestlebot/cli/options/common.py +++ b/trestlebot/cli/options/common.py @@ -111,16 +111,19 @@ def git_options(f: F) -> Any: "--branch", help="Branch name to push changes to", type=str, + prompt="Enter branch name to push changes to", ) @click.option( "--committer-name", help="Name of committer", type=str, + prompt="Enter name for committer", ) @click.option( "--committer-email", help="Email for committer", type=str, + prompt="Enter email for committer", ) @click.option( "--file-patterns", diff --git a/trestlebot/cli/run.py b/trestlebot/cli/run.py new file mode 100644 index 00000000..561e2dfe --- /dev/null +++ b/trestlebot/cli/run.py @@ -0,0 +1,29 @@ +from typing import Any, Dict, List + +from trestlebot.bot import TrestleBot +from trestlebot.tasks.base_task import TaskBase + + +def comma_sep_to_list(string: str) -> List[str]: + """Convert comma-sep string to list of strings and strip.""" + string = string.strip() if string else "" + return list(map(str.strip, string.split(","))) if string else [] + + +def run(pre_tasks: List[TaskBase], kwargs: Dict[Any, Any]) -> None: + """Reusable logic for all commands.""" + # Configure and run the bot + bot = TrestleBot( + working_dir=kwargs["working_dir"], + branch=kwargs["branch"], + commit_name=kwargs["committer_name"], + commit_email=kwargs["committer_email"], + author_name=kwargs.get("author_name", ""), + author_email=kwargs.get("author_email", ""), + ) + bot.run( + pre_tasks=pre_tasks, + patterns=comma_sep_to_list(kwargs.get("patterns", "")), + commit_message=kwargs.get("commit_message", "Automatic updates from bot"), + dry_run=kwargs.get("dry_run", False), + ) From 000e26e44b8031c7d13d1481bc30f671a8fae055 Mon Sep 17 00:00:00 2001 From: George Vauter Date: Thu, 21 Nov 2024 08:48:40 -0500 Subject: [PATCH 016/108] feat: add ssp index option --- trestlebot/cli/commands/init.py | 48 +++++++++++++++------------------ trestlebot/cli/config.py | 2 ++ 2 files changed, 24 insertions(+), 26 deletions(-) diff --git a/trestlebot/cli/commands/init.py b/trestlebot/cli/commands/init.py index ab4bcc1d..90fb4ff1 100644 --- a/trestlebot/cli/commands/init.py +++ b/trestlebot/cli/commands/init.py @@ -24,6 +24,7 @@ logger = logging.getLogger(__name__) +logging.getLogger("trestle.core.commands.init").setLevel("CRITICAL") def call_trestle_init(repo_path: Path, debug: bool) -> None: @@ -48,43 +49,38 @@ def call_trestle_init(repo_path: Path, debug: bool) -> None: @click.command(name="init", help="Initialize a new trestle-bot repo.") +@click.argument( + "repo_path", type=click.Path(path_type=Path, exists=True), required=True +) @click.option( - "--repo-path", - "repo_path", - help="Path to git repo. Used as trestle root directory.", - type=click.Path(path_type=Path, exists=True), + "--markdown-dir", + type=str, + help="Directory name to store markdown files.", + default="markdown/", + prompt="Enter path to store markdown files", ) @click.option( - "--markdown-dir", help="Directory name to store markdown files.", type=str + "--ssp-index-file", + type=str, + help="Path of SSP index file.", + default="ssp-index.json", + required=False, ) @common_options def init_cmd( - ctx: click.Context, repo_path: Path, markdown_dir: str, debug: bool, config: str + ctx: click.Context, + debug: bool, + config: str, + repo_path: Path, + markdown_dir: str, + ssp_index_file: str, ) -> None: """Command to initialize a new trestlebot repo""" - need_to_prompt = any((not repo_path, not markdown_dir)) - if need_to_prompt: - - click.echo("\n* Welcome to the Trestle-bot CLI *\n") - click.echo( - "Please provide the following values to initialize the " - "workspace [press Enter for defaults].\n" - ) - if not repo_path: - repo_path = click.prompt( - "Enter path to git repo (workspace directory)", - default=".", - type=click.Path(path_type=Path, exists=True), - ) - if not markdown_dir: - markdown_dir = click.prompt( - "Enter path to store markdown files", default="./markdown", type=str - ) - + repo_path = repo_path.resolve() git_path: Path = repo_path.joinpath(Path(".git")) if not git_path.exists(): - logging.error( + logger.error( f"Initialization failed. Given directory {repo_path} is not a Git repository." ) sys.exit(ERROR_EXIT_CODE) diff --git a/trestlebot/cli/config.py b/trestlebot/cli/config.py index 4d824e74..f0fc6f85 100644 --- a/trestlebot/cli/config.py +++ b/trestlebot/cli/config.py @@ -48,6 +48,7 @@ class TrestleBotConfig(BaseModel): repo_path: Optional[DirectoryPath] = None markdown_dir: Optional[str] = None + ssp_index_file: Optional[str] = "ssp-index.json" @model_serializer def _dict(self) -> Dict[str, Any]: @@ -55,6 +56,7 @@ def _dict(self) -> Dict[str, Any]: config_dict = { "repo_path": str(self.repo_path), "markdown_dir": str(self.markdown_dir), + "ssp_index_file": str(self.ssp_index_file), } return dict( filter(lambda item: item[1] not in (None, "None"), config_dict.items()) From 5aeb27d17ce2810119b64bd2fdb5e7a9fa118de2 Mon Sep 17 00:00:00 2001 From: George Vauter Date: Thu, 21 Nov 2024 08:48:40 -0500 Subject: [PATCH 017/108] feat: add ssp index option --- trestlebot/cli/commands/init.py | 48 +++++++++++++++------------------ trestlebot/cli/config.py | 2 ++ 2 files changed, 24 insertions(+), 26 deletions(-) diff --git a/trestlebot/cli/commands/init.py b/trestlebot/cli/commands/init.py index ab4bcc1d..90fb4ff1 100644 --- a/trestlebot/cli/commands/init.py +++ b/trestlebot/cli/commands/init.py @@ -24,6 +24,7 @@ logger = logging.getLogger(__name__) +logging.getLogger("trestle.core.commands.init").setLevel("CRITICAL") def call_trestle_init(repo_path: Path, debug: bool) -> None: @@ -48,43 +49,38 @@ def call_trestle_init(repo_path: Path, debug: bool) -> None: @click.command(name="init", help="Initialize a new trestle-bot repo.") +@click.argument( + "repo_path", type=click.Path(path_type=Path, exists=True), required=True +) @click.option( - "--repo-path", - "repo_path", - help="Path to git repo. Used as trestle root directory.", - type=click.Path(path_type=Path, exists=True), + "--markdown-dir", + type=str, + help="Directory name to store markdown files.", + default="markdown/", + prompt="Enter path to store markdown files", ) @click.option( - "--markdown-dir", help="Directory name to store markdown files.", type=str + "--ssp-index-file", + type=str, + help="Path of SSP index file.", + default="ssp-index.json", + required=False, ) @common_options def init_cmd( - ctx: click.Context, repo_path: Path, markdown_dir: str, debug: bool, config: str + ctx: click.Context, + debug: bool, + config: str, + repo_path: Path, + markdown_dir: str, + ssp_index_file: str, ) -> None: """Command to initialize a new trestlebot repo""" - need_to_prompt = any((not repo_path, not markdown_dir)) - if need_to_prompt: - - click.echo("\n* Welcome to the Trestle-bot CLI *\n") - click.echo( - "Please provide the following values to initialize the " - "workspace [press Enter for defaults].\n" - ) - if not repo_path: - repo_path = click.prompt( - "Enter path to git repo (workspace directory)", - default=".", - type=click.Path(path_type=Path, exists=True), - ) - if not markdown_dir: - markdown_dir = click.prompt( - "Enter path to store markdown files", default="./markdown", type=str - ) - + repo_path = repo_path.resolve() git_path: Path = repo_path.joinpath(Path(".git")) if not git_path.exists(): - logging.error( + logger.error( f"Initialization failed. Given directory {repo_path} is not a Git repository." ) sys.exit(ERROR_EXIT_CODE) diff --git a/trestlebot/cli/config.py b/trestlebot/cli/config.py index 4d824e74..f0fc6f85 100644 --- a/trestlebot/cli/config.py +++ b/trestlebot/cli/config.py @@ -48,6 +48,7 @@ class TrestleBotConfig(BaseModel): repo_path: Optional[DirectoryPath] = None markdown_dir: Optional[str] = None + ssp_index_file: Optional[str] = "ssp-index.json" @model_serializer def _dict(self) -> Dict[str, Any]: @@ -55,6 +56,7 @@ def _dict(self) -> Dict[str, Any]: config_dict = { "repo_path": str(self.repo_path), "markdown_dir": str(self.markdown_dir), + "ssp_index_file": str(self.ssp_index_file), } return dict( filter(lambda item: item[1] not in (None, "None"), config_dict.items()) From 362faf11e300715beed941040f9eaa97c73feace Mon Sep 17 00:00:00 2001 From: George Vauter Date: Thu, 21 Nov 2024 08:49:01 -0500 Subject: [PATCH 018/108] add unit tests for init command --- tests/trestlebot/cli/test_init_cmd.py | 114 ++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 tests/trestlebot/cli/test_init_cmd.py diff --git a/tests/trestlebot/cli/test_init_cmd.py b/tests/trestlebot/cli/test_init_cmd.py new file mode 100644 index 00000000..1e9dcd0e --- /dev/null +++ b/tests/trestlebot/cli/test_init_cmd.py @@ -0,0 +1,114 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Red Hat, Inc. + + +"""Testing module for trestlebot init command""" +import pathlib + +import yaml +from click.testing import CliRunner +from trestle.common.const import MODEL_DIR_LIST, TRESTLE_CONFIG_DIR, TRESTLE_KEEP_FILE +from trestle.common.file_utils import is_hidden + +from tests.testutils import setup_for_init +from trestlebot.cli.commands.init import call_trestle_init, init_cmd +from trestlebot.const import TRESTLEBOT_CONFIG_DIR + + +TRESTLE_KEEP_FILE + + +def test_init_repo_dir_does_not_exist() -> None: + """Init should fail if repo dir does not exit""" + runner = CliRunner() + result = runner.invoke(init_cmd, ["0"]) + assert result.exit_code == 2 + assert ( + "Error: Invalid value for 'REPO_PATH': Path '0' does not exist." + in result.output + ) + + +def test_init_not_git_repo(tmp_init_dir: str) -> None: + """Init should fail if repo dir is not a Git repo.""" + runner = CliRunner() + result = runner.invoke(init_cmd, [tmp_init_dir, "--markdown-dir", "markdown"]) + assert result.exit_code == 1 + assert "not a Git repository" in result.output + + +def test_init_existing_trestlebot_dir(tmp_init_dir: str) -> None: + """Init should fail if repo already contains .trestlebot/ dir.""" + + # setup_for_init(pathlib.Path(tmp_init_dir)) + # Manulaly create .trestlebot dir so it already exists + trestlebot_dir = pathlib.Path(tmp_init_dir) / pathlib.Path(TRESTLEBOT_CONFIG_DIR) + trestlebot_dir.mkdir() + + setup_for_init(pathlib.Path(tmp_init_dir)) + + runner = CliRunner() + result = runner.invoke(init_cmd, [tmp_init_dir, "--markdown-dir", "markdown"]) + + assert result.exit_code == 1 + assert "existing .trestlebot directory" in result.output + + +def test_init_creates_config_file(tmp_init_dir: str) -> None: + """Test init command creates yaml config file.""" + + setup_for_init(pathlib.Path(tmp_init_dir)) + + runner = CliRunner() + result = runner.invoke(init_cmd, [tmp_init_dir, "--markdown-dir", "markdown"]) + assert result.exit_code == 0 + assert "Successfully initialized trestlebot" in result.output + + config_path = ( + pathlib.Path(tmp_init_dir) + .joinpath(TRESTLEBOT_CONFIG_DIR) + .joinpath("config.yml") + ) + with open(config_path, "r") as f: + yaml_data = yaml.safe_load(f) + + assert yaml_data["repo_path"] == tmp_init_dir + assert yaml_data["markdown_dir"] == "markdown" + + +def test_init_creates_model_dirs(tmp_init_dir: str) -> None: + """Init should create model directories in repo""" + + tmp_dir = pathlib.Path(tmp_init_dir) + setup_for_init(tmp_dir) + + runner = CliRunner() + runner.invoke(init_cmd, [tmp_init_dir, "--markdown-dir", "markdown"]) + + model_dirs = [d.name for d in tmp_dir.iterdir() if not is_hidden(d)] + model_dirs.remove("markdown") # pop markdown dir + assert sorted(model_dirs) == sorted(MODEL_DIR_LIST) + + +def test_init_creates_markdown_dirs(tmp_init_dir: str) -> None: + """Init should create model directories in repo""" + + tmp_dir = pathlib.Path(tmp_init_dir) + markdown_dir = tmp_dir.joinpath("markdown") + setup_for_init(tmp_dir) + + runner = CliRunner() + runner.invoke(init_cmd, [tmp_init_dir, "--markdown-dir", "markdown"]) + + markdown_dirs = [d.name for d in markdown_dir.iterdir() if not is_hidden(d)] + assert sorted(markdown_dirs) == sorted(MODEL_DIR_LIST) + + +def test_init_creates_trestle_dirs(tmp_init_dir: str) -> None: + """Init should create markdown dirs in repo""" + + tmp_dir = pathlib.Path(tmp_init_dir) + call_trestle_init(tmp_dir, False) + trestle_dir = tmp_dir.joinpath(TRESTLE_CONFIG_DIR) + keep_file = trestle_dir.joinpath(TRESTLE_KEEP_FILE) + assert keep_file.exists() is True From 75fd0c175614f19f64379e0c86683bcd1407d658 Mon Sep 17 00:00:00 2001 From: George Vauter Date: Thu, 21 Nov 2024 08:49:01 -0500 Subject: [PATCH 019/108] add unit tests for init command --- tests/trestlebot/cli/test_init_cmd.py | 114 ++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 tests/trestlebot/cli/test_init_cmd.py diff --git a/tests/trestlebot/cli/test_init_cmd.py b/tests/trestlebot/cli/test_init_cmd.py new file mode 100644 index 00000000..1e9dcd0e --- /dev/null +++ b/tests/trestlebot/cli/test_init_cmd.py @@ -0,0 +1,114 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Red Hat, Inc. + + +"""Testing module for trestlebot init command""" +import pathlib + +import yaml +from click.testing import CliRunner +from trestle.common.const import MODEL_DIR_LIST, TRESTLE_CONFIG_DIR, TRESTLE_KEEP_FILE +from trestle.common.file_utils import is_hidden + +from tests.testutils import setup_for_init +from trestlebot.cli.commands.init import call_trestle_init, init_cmd +from trestlebot.const import TRESTLEBOT_CONFIG_DIR + + +TRESTLE_KEEP_FILE + + +def test_init_repo_dir_does_not_exist() -> None: + """Init should fail if repo dir does not exit""" + runner = CliRunner() + result = runner.invoke(init_cmd, ["0"]) + assert result.exit_code == 2 + assert ( + "Error: Invalid value for 'REPO_PATH': Path '0' does not exist." + in result.output + ) + + +def test_init_not_git_repo(tmp_init_dir: str) -> None: + """Init should fail if repo dir is not a Git repo.""" + runner = CliRunner() + result = runner.invoke(init_cmd, [tmp_init_dir, "--markdown-dir", "markdown"]) + assert result.exit_code == 1 + assert "not a Git repository" in result.output + + +def test_init_existing_trestlebot_dir(tmp_init_dir: str) -> None: + """Init should fail if repo already contains .trestlebot/ dir.""" + + # setup_for_init(pathlib.Path(tmp_init_dir)) + # Manulaly create .trestlebot dir so it already exists + trestlebot_dir = pathlib.Path(tmp_init_dir) / pathlib.Path(TRESTLEBOT_CONFIG_DIR) + trestlebot_dir.mkdir() + + setup_for_init(pathlib.Path(tmp_init_dir)) + + runner = CliRunner() + result = runner.invoke(init_cmd, [tmp_init_dir, "--markdown-dir", "markdown"]) + + assert result.exit_code == 1 + assert "existing .trestlebot directory" in result.output + + +def test_init_creates_config_file(tmp_init_dir: str) -> None: + """Test init command creates yaml config file.""" + + setup_for_init(pathlib.Path(tmp_init_dir)) + + runner = CliRunner() + result = runner.invoke(init_cmd, [tmp_init_dir, "--markdown-dir", "markdown"]) + assert result.exit_code == 0 + assert "Successfully initialized trestlebot" in result.output + + config_path = ( + pathlib.Path(tmp_init_dir) + .joinpath(TRESTLEBOT_CONFIG_DIR) + .joinpath("config.yml") + ) + with open(config_path, "r") as f: + yaml_data = yaml.safe_load(f) + + assert yaml_data["repo_path"] == tmp_init_dir + assert yaml_data["markdown_dir"] == "markdown" + + +def test_init_creates_model_dirs(tmp_init_dir: str) -> None: + """Init should create model directories in repo""" + + tmp_dir = pathlib.Path(tmp_init_dir) + setup_for_init(tmp_dir) + + runner = CliRunner() + runner.invoke(init_cmd, [tmp_init_dir, "--markdown-dir", "markdown"]) + + model_dirs = [d.name for d in tmp_dir.iterdir() if not is_hidden(d)] + model_dirs.remove("markdown") # pop markdown dir + assert sorted(model_dirs) == sorted(MODEL_DIR_LIST) + + +def test_init_creates_markdown_dirs(tmp_init_dir: str) -> None: + """Init should create model directories in repo""" + + tmp_dir = pathlib.Path(tmp_init_dir) + markdown_dir = tmp_dir.joinpath("markdown") + setup_for_init(tmp_dir) + + runner = CliRunner() + runner.invoke(init_cmd, [tmp_init_dir, "--markdown-dir", "markdown"]) + + markdown_dirs = [d.name for d in markdown_dir.iterdir() if not is_hidden(d)] + assert sorted(markdown_dirs) == sorted(MODEL_DIR_LIST) + + +def test_init_creates_trestle_dirs(tmp_init_dir: str) -> None: + """Init should create markdown dirs in repo""" + + tmp_dir = pathlib.Path(tmp_init_dir) + call_trestle_init(tmp_dir, False) + trestle_dir = tmp_dir.joinpath(TRESTLE_CONFIG_DIR) + keep_file = trestle_dir.joinpath(TRESTLE_KEEP_FILE) + assert keep_file.exists() is True From 1382cdcb2a35e5ba918a7ac5b4f2fd9456ce5432 Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Thu, 21 Nov 2024 09:54:05 -0500 Subject: [PATCH 020/108] feat: root call create and logging replacement --- trestlebot/cli/commands/create.py | 33 +++++++++++++++---------------- trestlebot/cli/root.py | 2 ++ 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index 081f6ebe..cd9398c3 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -2,11 +2,16 @@ Module for create-cd create-ssp command for CLI """ +import logging + import click from trestlebot.cli.options.create import common_create_options +logger = logging.getLogger(__name__) + + @click.group(name="create") @common_create_options def create_cmd(ctx: click.Context, profile_name: str) -> None: @@ -57,18 +62,18 @@ def compdef_cmd( """ Component definition authoring command. """ - click.echo( + logger.info( f"The name of the profile in use with the component definition is {profile_name}." ) - click.echo( + logger.info( f"You have selected component definitions as the document you want {compdef_name} to author." ) - click.echo(f"The component definition name is {component_title}.") - click.echo(f"The component description to author is {component_description}.") - click.echo( + logger.info(f"The component definition name is {component_title}.") + logger.info(f"The component description to author is {component_description}.") + logger.info( f"The profile you want to filter controls in the component files is {filter_by_profile}." ) - click.echo(f"The component definition type is {component_definition_type}.") + logger.info(f"The component definition type is {component_definition_type}.") # @create_cmd.command(name="ssp", help="command for ssp authoring") @@ -104,14 +109,8 @@ def ssp_cmd( """ SSP Authoring command """ - click.echo( - f"The name of the profile in trestle workspace to include in the SSP is {profile_name}." - ) - click.echo(f"The name of the SSP to create is {ssp_name}.") - click.echo(f"The leveraged SSP is {leveraged_ssp}.") - click.echo(f"The SSP index path is {ssp_index_path}.") - click.echo(f"The YAML file for custom SSP markdown is {yaml_header_path}.") - - -if __name__ == "__main__": - create_cmd() + logger.info(f"The name of the profile in use with the SSP is {profile_name}.") + logger.info(f"The name of the SSP to create is {ssp_name}.") + logger.info(f"The leveraged SSP is {leveraged_ssp}.") + logger.info(f"The SSP index path is {ssp_index_path}.") + logger.info(f"The YAML file for custom SSP markdown is {yaml_header_path}.") diff --git a/trestlebot/cli/root.py b/trestlebot/cli/root.py index 2f6e50bf..45bfb46b 100644 --- a/trestlebot/cli/root.py +++ b/trestlebot/cli/root.py @@ -7,6 +7,7 @@ import click from trestlebot.cli.commands.autosync import autosync_cmd +from trestlebot.cli.commands.create import create_cmd from trestlebot.cli.commands.init import init_cmd @@ -27,4 +28,5 @@ def root_cmd(ctx: click.Context) -> None: root_cmd.add_command(init_cmd) +root_cmd.add_command(create_cmd) root_cmd.add_command(autosync_cmd) From 3292aa5b91b1dcf5cec7691db7ce1301f3b9270c Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Thu, 21 Nov 2024 09:54:05 -0500 Subject: [PATCH 021/108] feat: root call create and logging replacement --- trestlebot/cli/commands/create.py | 33 +++++++++++++++---------------- trestlebot/cli/root.py | 2 ++ 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index 081f6ebe..cd9398c3 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -2,11 +2,16 @@ Module for create-cd create-ssp command for CLI """ +import logging + import click from trestlebot.cli.options.create import common_create_options +logger = logging.getLogger(__name__) + + @click.group(name="create") @common_create_options def create_cmd(ctx: click.Context, profile_name: str) -> None: @@ -57,18 +62,18 @@ def compdef_cmd( """ Component definition authoring command. """ - click.echo( + logger.info( f"The name of the profile in use with the component definition is {profile_name}." ) - click.echo( + logger.info( f"You have selected component definitions as the document you want {compdef_name} to author." ) - click.echo(f"The component definition name is {component_title}.") - click.echo(f"The component description to author is {component_description}.") - click.echo( + logger.info(f"The component definition name is {component_title}.") + logger.info(f"The component description to author is {component_description}.") + logger.info( f"The profile you want to filter controls in the component files is {filter_by_profile}." ) - click.echo(f"The component definition type is {component_definition_type}.") + logger.info(f"The component definition type is {component_definition_type}.") # @create_cmd.command(name="ssp", help="command for ssp authoring") @@ -104,14 +109,8 @@ def ssp_cmd( """ SSP Authoring command """ - click.echo( - f"The name of the profile in trestle workspace to include in the SSP is {profile_name}." - ) - click.echo(f"The name of the SSP to create is {ssp_name}.") - click.echo(f"The leveraged SSP is {leveraged_ssp}.") - click.echo(f"The SSP index path is {ssp_index_path}.") - click.echo(f"The YAML file for custom SSP markdown is {yaml_header_path}.") - - -if __name__ == "__main__": - create_cmd() + logger.info(f"The name of the profile in use with the SSP is {profile_name}.") + logger.info(f"The name of the SSP to create is {ssp_name}.") + logger.info(f"The leveraged SSP is {leveraged_ssp}.") + logger.info(f"The SSP index path is {ssp_index_path}.") + logger.info(f"The YAML file for custom SSP markdown is {yaml_header_path}.") diff --git a/trestlebot/cli/root.py b/trestlebot/cli/root.py index 2f6e50bf..45bfb46b 100644 --- a/trestlebot/cli/root.py +++ b/trestlebot/cli/root.py @@ -7,6 +7,7 @@ import click from trestlebot.cli.commands.autosync import autosync_cmd +from trestlebot.cli.commands.create import create_cmd from trestlebot.cli.commands.init import init_cmd @@ -27,4 +28,5 @@ def root_cmd(ctx: click.Context) -> None: root_cmd.add_command(init_cmd) +root_cmd.add_command(create_cmd) root_cmd.add_command(autosync_cmd) From d41e017c15b5bd788ef35109d7da2273f2cc0036 Mon Sep 17 00:00:00 2001 From: George Vauter Date: Sun, 24 Nov 2024 01:26:45 -0500 Subject: [PATCH 022/108] feat: add upstream commands, fix common options decorators, expand config --- tests/trestlebot/cli/test_config.py | 18 +- tests/trestlebot/cli/test_init_cmd.py | 26 +-- tests/trestlebot/cli/test_upstreams_cmd.py | 249 +++++++++++++++++++++ trestlebot/cli/commands/init.py | 31 +-- trestlebot/cli/commands/upstream.py | 230 +++++++++++++++++++ trestlebot/cli/config.py | 62 ++++- trestlebot/cli/options/common.py | 120 ++++++---- trestlebot/cli/root.py | 2 + trestlebot/cli/run.py | 7 +- 9 files changed, 636 insertions(+), 109 deletions(-) create mode 100644 tests/trestlebot/cli/test_upstreams_cmd.py create mode 100644 trestlebot/cli/commands/upstream.py diff --git a/tests/trestlebot/cli/test_config.py b/tests/trestlebot/cli/test_config.py index 8ed4b141..8d8c21b1 100644 --- a/tests/trestlebot/cli/test_config.py +++ b/tests/trestlebot/cli/test_config.py @@ -42,14 +42,6 @@ def test_make_config_raises_no_errors(tmp_init_dir: str) -> None: assert config.markdown_dir == "markdown-test" -def test_config_to_dict(config_obj: TrestleBotConfig) -> None: - """Config should serialize to a dict.""" - model_dict = config_obj.model_dump() - assert isinstance(model_dict, dict) - assert model_dict["repo_path"] == str(config_obj.repo_path) - assert model_dict["markdown_dir"] == config_obj.markdown_dir - - def test_config_write_to_file(config_obj: TrestleBotConfig, tmp_init_dir: str) -> None: """Test config is written to yaml file.""" filepath = pathlib.Path(tmp_init_dir).joinpath("config.yml") @@ -57,15 +49,15 @@ def test_config_write_to_file(config_obj: TrestleBotConfig, tmp_init_dir: str) - with open(filepath, "r") as f: yaml_data = yaml.safe_load(f) - assert yaml_data == config_obj.model_dump() + assert yaml_data == config_obj.to_yaml_dict() -def test_config_laod_from_file(config_obj: TrestleBotConfig, tmp_init_dir: str) -> None: +def test_config_load_from_file(config_obj: TrestleBotConfig, tmp_init_dir: str) -> None: """Test config is read from yaml file into config object.""" filepath = pathlib.Path(tmp_init_dir).joinpath("config.yml") with filepath.open("w") as config_file: - yaml.dump(config_obj.model_dump(), config_file) + yaml.dump(config_obj.to_yaml_dict(), config_file) - config = load_from_file(str(filepath)) + config = load_from_file(filepath) assert isinstance(config, TrestleBotConfig) - assert config.model_dump() == config_obj.model_dump() + assert config == config_obj diff --git a/tests/trestlebot/cli/test_init_cmd.py b/tests/trestlebot/cli/test_init_cmd.py index 1e9dcd0e..0bf05dc9 100644 --- a/tests/trestlebot/cli/test_init_cmd.py +++ b/tests/trestlebot/cli/test_init_cmd.py @@ -15,24 +15,20 @@ from trestlebot.const import TRESTLEBOT_CONFIG_DIR -TRESTLE_KEEP_FILE - - def test_init_repo_dir_does_not_exist() -> None: """Init should fail if repo dir does not exit""" runner = CliRunner() - result = runner.invoke(init_cmd, ["0"]) + result = runner.invoke(init_cmd, ["--repo-path", "0"]) assert result.exit_code == 2 - assert ( - "Error: Invalid value for 'REPO_PATH': Path '0' does not exist." - in result.output - ) + assert "does not exist." in result.output def test_init_not_git_repo(tmp_init_dir: str) -> None: """Init should fail if repo dir is not a Git repo.""" runner = CliRunner() - result = runner.invoke(init_cmd, [tmp_init_dir, "--markdown-dir", "markdown"]) + result = runner.invoke( + init_cmd, ["--repo-path", tmp_init_dir, "--markdown-dir", "markdown"] + ) assert result.exit_code == 1 assert "not a Git repository" in result.output @@ -48,7 +44,9 @@ def test_init_existing_trestlebot_dir(tmp_init_dir: str) -> None: setup_for_init(pathlib.Path(tmp_init_dir)) runner = CliRunner() - result = runner.invoke(init_cmd, [tmp_init_dir, "--markdown-dir", "markdown"]) + result = runner.invoke( + init_cmd, ["--repo-path", tmp_init_dir, "--markdown-dir", "markdown"] + ) assert result.exit_code == 1 assert "existing .trestlebot directory" in result.output @@ -60,7 +58,9 @@ def test_init_creates_config_file(tmp_init_dir: str) -> None: setup_for_init(pathlib.Path(tmp_init_dir)) runner = CliRunner() - result = runner.invoke(init_cmd, [tmp_init_dir, "--markdown-dir", "markdown"]) + result = runner.invoke( + init_cmd, ["--repo-path", tmp_init_dir, "--markdown-dir", "markdown"] + ) assert result.exit_code == 0 assert "Successfully initialized trestlebot" in result.output @@ -83,7 +83,7 @@ def test_init_creates_model_dirs(tmp_init_dir: str) -> None: setup_for_init(tmp_dir) runner = CliRunner() - runner.invoke(init_cmd, [tmp_init_dir, "--markdown-dir", "markdown"]) + runner.invoke(init_cmd, ["--repo-path", tmp_init_dir, "--markdown-dir", "markdown"]) model_dirs = [d.name for d in tmp_dir.iterdir() if not is_hidden(d)] model_dirs.remove("markdown") # pop markdown dir @@ -98,7 +98,7 @@ def test_init_creates_markdown_dirs(tmp_init_dir: str) -> None: setup_for_init(tmp_dir) runner = CliRunner() - runner.invoke(init_cmd, [tmp_init_dir, "--markdown-dir", "markdown"]) + runner.invoke(init_cmd, ["--repo-path", tmp_init_dir, "--markdown-dir", "markdown"]) markdown_dirs = [d.name for d in markdown_dir.iterdir() if not is_hidden(d)] assert sorted(markdown_dirs) == sorted(MODEL_DIR_LIST) diff --git a/tests/trestlebot/cli/test_upstreams_cmd.py b/tests/trestlebot/cli/test_upstreams_cmd.py new file mode 100644 index 00000000..11e05a19 --- /dev/null +++ b/tests/trestlebot/cli/test_upstreams_cmd.py @@ -0,0 +1,249 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Red Hat, Inc. + + +"""Unit tests for upstreams commands.""" + +import pathlib +from typing import Tuple + +import pytest +from click.testing import CliRunner +from git import Repo + +from tests.testutils import clean, prepare_upstream_repo +from trestlebot.cli.commands.upstream import upstream_cmd +from trestlebot.cli.config import ( + TrestleBotConfig, + UpstreamConfig, + load_from_file, + write_to_file, +) + + +TEST_CATALOG = "simplified_nist_catalog" +TEST_CATALOG_PATH = "catalogs/simplified_nist_catalog/catalog.json" +TEST_PROFILE = "simplified_nist_profile" +TEST_PROFILE_PATH = "profiles/simplified_nist_profile/profile.json" + + +def test_config_get_upstream_by_url() -> None: + """Test to confirm _get_upstream_by_url returns a single valid upstream from the config.""" + config = TrestleBotConfig(upstreams=[UpstreamConfig(url="https://test")]) + + assert len(config.upstreams) == 1 + assert config.upstreams[0].url == "https://test" + assert config.upstreams[0].include_models == ["*"] # confirm default was set + assert config.upstreams[0].exclude_models == [] + assert config.upstreams[0].skip_validation is False + + +def test_upstream_add_already_exists(tmp_repo: Tuple[str, Repo]) -> None: + """Test upstream add fails if upstream already exists""" + repo_path, repo = tmp_repo + config_path = pathlib.Path(repo_path).joinpath(".trestlebot/config.yml") + + source: str = prepare_upstream_repo() + url = f"{source}@main" + config = TrestleBotConfig( + committer_email="test@test", + committer_name="test", + upstreams=[UpstreamConfig(url=url)], + ) + write_to_file(config, config_path) + + runner = CliRunner() + result = runner.invoke( + upstream_cmd, + [ + "add", + "--repo-path", + repo_path, + "--config", + config_path, + "--url", + url, + "--branch", + "test", + ], + ) + assert ( + result.exit_code == 0 + ) # This does return success, it just skips the URLs already in the config + + assert f"{url} already exists. Edit {config_path} to update." in result.output + + +def test_upstream_add(tmp_repo: Tuple[str, Repo]) -> None: + """Tests successful add of upstream""" + repo_dir, repo = tmp_repo + repo_path = pathlib.Path(repo_dir) + config_path = repo_path.joinpath(".trestlebot/config.yml") + + source: str = prepare_upstream_repo() + url = f"{source}@main" + config = TrestleBotConfig( + committer_email="test@test", + committer_name="test", + ) + write_to_file(config, config_path) + + runner = CliRunner() + result = runner.invoke( + upstream_cmd, + [ + "add", + "--repo-path", + repo_path, + "--config", + config_path, + "--url", + url, + "--branch", + "main", + "--debug", + ], + ) + updated_config = load_from_file(config_path) + if not updated_config: + pytest.fail("Updated config not found") + return + + assert len(updated_config.upstreams) == 1 + assert updated_config.upstreams[0].url == url + assert updated_config.upstreams[0].include_models == ["*"] + assert updated_config.upstreams[0].exclude_models == [] + assert updated_config.upstreams[0].skip_validation is False + assert ( + updated_config.committer_email == "test@test" + ) # sanity check this didn't change + assert repo_path.joinpath(TEST_CATALOG_PATH).exists() + assert repo_path.joinpath(TEST_PROFILE_PATH).exists() + assert result.exit_code == 0 + clean(source, None) + + +def test_upstream_add_exclude_models(tmp_repo: Tuple[str, Repo]) -> None: + """Test add upstream with exclude models""" + + repo_dir, repo = tmp_repo + repo_path = pathlib.Path(repo_dir) + config_path = repo_path.joinpath(".trestlebot/config.yml") + + source: str = prepare_upstream_repo() + url = f"{source}@main" + config = TrestleBotConfig( + committer_email="test@test", + committer_name="test", + ) + write_to_file(config, config_path) + + runner = CliRunner() + result = runner.invoke( + upstream_cmd, + [ + "add", + "--repo-path", + repo_path, + "--config", + config_path, + "--url", + url, + "--branch", + "main", + "--exclude-model", + TEST_PROFILE, + ], + ) + + updated_config = load_from_file(config_path) + if not updated_config: + pytest.fail("Updated config not found") + return + + assert len(updated_config.upstreams) == 1 + assert updated_config.upstreams[0].url == url + assert updated_config.upstreams[0].include_models == ["*"] + assert updated_config.upstreams[0].exclude_models == [TEST_PROFILE] + assert updated_config.upstreams[0].skip_validation is False + assert ( + updated_config.committer_email == "test@test" + ) # sanity check this didn't change + + assert repo_path.joinpath(TEST_CATALOG_PATH).exists() + assert not repo_path.joinpath(TEST_PROFILE_PATH).exists() + assert result.exit_code == 0 + + +def test_upstream_sync_upstream_no_upstreams_in_config( + tmp_repo: Tuple[str, Repo] +) -> None: + """Test sync upstream with no upstreams in config file.""" + + repo_dir, repo = tmp_repo + repo_path = pathlib.Path(repo_dir) + config_path = repo_path.joinpath(".trestlebot/config.yml") + + source: str = prepare_upstream_repo() + url = f"{source}@main" + config = TrestleBotConfig( + committer_email="test@test", + committer_name="test", + ) + write_to_file(config, config_path) + + runner = CliRunner() + result = runner.invoke( + upstream_cmd, + [ + "sync", + "--repo-path", + repo_path, + "--config", + config_path, + "--url", + url, + "--branch", + "main", + ], + ) + assert ( + "No upstreams defined in trestlebot config. Use `upstream add` command." + in result.output + ) + assert result.exit_code == 1 + + +def test_upstream_sync_upstream_not_found(tmp_repo: Tuple[str, Repo]) -> None: + """Test sync upstream with url not found.""" + + repo_dir, repo = tmp_repo + repo_path = pathlib.Path(repo_dir) + config_path = repo_path.joinpath(".trestlebot/config.yml") + + source: str = prepare_upstream_repo() + url = f"{source}@main" + config = TrestleBotConfig( + committer_email="test@test", + committer_name="test", + upstreams=[UpstreamConfig(url="foo@bar")], + ) + write_to_file(config, config_path) + + runner = CliRunner() + result = runner.invoke( + upstream_cmd, + [ + "sync", + "--repo-path", + repo_path, + "--config", + config_path, + "--url", + url, + "--branch", + "main", + ], + ) + assert f"No upstream defined for {source}" in result.output + assert result.exit_code == 1 diff --git a/trestlebot/cli/commands/init.py b/trestlebot/cli/commands/init.py index 90fb4ff1..da289932 100644 --- a/trestlebot/cli/commands/init.py +++ b/trestlebot/cli/commands/init.py @@ -6,8 +6,8 @@ """ import argparse import logging +import pathlib import sys -from pathlib import Path import click from trestle.common.const import MODEL_DIR_LIST @@ -27,7 +27,7 @@ logging.getLogger("trestle.core.commands.init").setLevel("CRITICAL") -def call_trestle_init(repo_path: Path, debug: bool) -> None: +def call_trestle_init(repo_path: pathlib.Path, debug: bool) -> None: """Call compliance-trestle to initialize workspace""" verbose = 1 if debug else 0 @@ -49,9 +49,8 @@ def call_trestle_init(repo_path: Path, debug: bool) -> None: @click.command(name="init", help="Initialize a new trestle-bot repo.") -@click.argument( - "repo_path", type=click.Path(path_type=Path, exists=True), required=True -) +@click.pass_context +@common_options @click.option( "--markdown-dir", type=str, @@ -59,33 +58,25 @@ def call_trestle_init(repo_path: Path, debug: bool) -> None: default="markdown/", prompt="Enter path to store markdown files", ) -@click.option( - "--ssp-index-file", - type=str, - help="Path of SSP index file.", - default="ssp-index.json", - required=False, -) -@common_options def init_cmd( ctx: click.Context, debug: bool, - config: str, - repo_path: Path, + config_path: pathlib.Path, + repo_path: pathlib.Path, markdown_dir: str, - ssp_index_file: str, + dry_run: bool, ) -> None: """Command to initialize a new trestlebot repo""" repo_path = repo_path.resolve() - git_path: Path = repo_path.joinpath(Path(".git")) + git_path: pathlib.Path = repo_path.joinpath(pathlib.Path(".git")) if not git_path.exists(): logger.error( f"Initialization failed. Given directory {repo_path} is not a Git repository." ) sys.exit(ERROR_EXIT_CODE) - trestlebot_dir = repo_path.joinpath(Path(TRESTLEBOT_CONFIG_DIR)) + trestlebot_dir = repo_path.joinpath(pathlib.Path(TRESTLEBOT_CONFIG_DIR)) if trestlebot_dir.exists(): logger.error( f"Initialization failed. Found existing {TRESTLEBOT_CONFIG_DIR} directory in {repo_path}" @@ -106,7 +97,7 @@ def init_cmd( # Create markdown directories in workspace root list( map( - lambda x: repo_path.joinpath(Path(markdown_dir)) + lambda x: repo_path.joinpath(pathlib.Path(markdown_dir)) .joinpath(x) .joinpath(TRESTLEBOT_KEEP_FILE) .mkdir(parents=True, exist_ok=True), @@ -120,7 +111,7 @@ def init_cmd( # generate and write trestle-bot cofig config = make_config(dict(repo_path=repo_path, markdown_dir=markdown_dir)) - config_path = trestlebot_dir.joinpath(Path("config.yml")) + config_path = trestlebot_dir.joinpath("config.yml") write_to_file(config, config_path) logger.debug(f"trestle-bot config file created at {str(config_path)}") logger.info(f"Successfully initialized trestlebot project in {repo_path}") diff --git a/trestlebot/cli/commands/upstream.py b/trestlebot/cli/commands/upstream.py new file mode 100644 index 00000000..7ed5a364 --- /dev/null +++ b/trestlebot/cli/commands/upstream.py @@ -0,0 +1,230 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Red Hat, Inc. + +"""Module for upstream command""" +import logging +import pathlib +import sys +from typing import Any, List, Optional + +import click + +from trestlebot.cli.config import ( + TrestleBotConfig, + UpstreamConfig, + load_from_file, + write_to_file, +) +from trestlebot.cli.options.common import common_options, git_options, handle_exceptions +from trestlebot.cli.run import run +from trestlebot.const import ERROR_EXIT_CODE +from trestlebot.reporter import BotResults +from trestlebot.tasks.base_task import ModelFilter +from trestlebot.tasks.sync_upstreams_task import SyncUpstreamsTask + + +logger = logging.getLogger(__name__) + + +def sync_upstream_for_url( + repo_path: pathlib.Path, + dry_run: bool, + url: str, + include_models: List[str], + exclude_models: List[str], + skip_validation: bool, + committer_name: str, + committer_email: str, + branch: str, +) -> BotResults: + """Invoke the sync upstream task for a given URL.""" + model_filter: ModelFilter = ModelFilter( + include_patterns=include_models, skip_patterns=exclude_models + ) + + validate = not skip_validation + + sync_upstreams_task = SyncUpstreamsTask( + working_dir=str(repo_path.resolve()), + git_sources=[url], + model_filter=model_filter, + validate=validate, + ) + kwargs = dict( + working_dir=(str(repo_path.resolve())), + committer_name=committer_name, + committer_email=committer_email, + branch=branch, + dry_run=dry_run, + ) + return run([sync_upstreams_task], kwargs) + + +def _config_get_upstream_by_url( + config: TrestleBotConfig, url: str +) -> Optional[UpstreamConfig]: + """Returns one upstream that matches url, else None.""" + if config.upstreams: + for upstream in config.upstreams: + if upstream.url == url: + return upstream + return None + + +@click.group(name="upstream") +@click.pass_context +def upstream_cmd(ctx: click.Context) -> None: + """Sync content from upstream git repositories.""" + + +@upstream_cmd.command(name="add") +@click.pass_context +@common_options +@git_options +@click.option( + "--url", + type=str, + help="Upstream repo URL in the form @ where ref is a git ref such as tag or branch.", + required=True, + multiple=True, +) +@click.option( + "--exclude-model", + type=str, + help="Glob pattern for model names to exclude (e.g. --exclude-model=profile_y*).", + required=False, + multiple=True, +) +@click.option( + "--include-model", + type=str, + help="Glob pattern for model names to include (e.g. --include-model=profile_y*).", + required=False, + multiple=True, +) +@click.option( + "--skip-validation", + type=bool, + help="Skip validation of the models when they are copied.", + is_flag=True, +) +@handle_exceptions +def upstream_add_cmd(ctx: click.Context, **kwargs: Any) -> None: + """Add new upstream sources to workspace.""" + + repo_path = kwargs["repo_path"] + url_list = list(kwargs["url"]) + + config_path: pathlib.Path = kwargs["config_path"] + config = load_from_file(config_path) + if not config: + # No config exists or was not found, create a new one + logger.warning( + f"No trestlebot config file found, creating {str(config_path.resolve())}" + ) + config = TrestleBotConfig() + + for url in url_list: + existing_urls = [upstream.url for upstream in config.upstreams] + if url in existing_urls: + logger.warning( + f"Upstream for {url} already exists. Edit {config_path} to update." + ) + continue + + include_models = list(kwargs.get("include_model", ["*"])) + if len(include_models) == 0: + include_models = ["*"] + exclude_models = list(kwargs.get("exclude_model", [])) + skip_validation = kwargs.get("skip_validation", False) + result = sync_upstream_for_url( + repo_path=repo_path, + dry_run=kwargs["dry_run"], + url=url, + include_models=include_models, + exclude_models=exclude_models, + skip_validation=skip_validation, + committer_name=kwargs["committer_name"], + committer_email=kwargs["committer_email"], + branch=kwargs["branch"], + ) + logger.debug(f"Bot results for {url}: {result}") + + config.upstreams.append( + UpstreamConfig( + url=url, + include_models=include_models, + exclude_models=exclude_models, + skip_validation=skip_validation, + ) + ) + logger.info(f"Added {url} to trestlebot workspace") + + write_to_file(config, config_path) + + +@upstream_cmd.command(name="sync") +@click.pass_context +@common_options +@git_options +@click.option( + "--url", + type=str, + help="Upstream repo URL in the form @ where ref is a git ref such as tag or branch.", + required=False, + multiple=True, +) +@click.option( + "--all", + type=str, + help="URL to GitHub repo containing upstream content.", + required=False, + is_flag=True, +) +@handle_exceptions +def upstream_sync_cmd(ctx: click.Context, **kwargs: Any) -> None: + """Sync upstream sources to local workspace""" + + config_path: pathlib.Path = kwargs["config_path"] + config = load_from_file(config_path) + if not config or len(config.upstreams) == 0: + logger.error( + "No upstreams defined in trestlebot config. Use `upstream add` command." + ) + sys.exit(ERROR_EXIT_CODE) + + upstreams_to_sync = [] + if kwargs.get("all"): + upstreams_to_sync = config.upstreams + + elif urls := kwargs.get("url"): + for url in urls: + if upstream := _config_get_upstream_by_url(config, url): + upstreams_to_sync.append(upstream) + + else: + logger.error(f"No upstream defined for {url} - skipping!") + + else: + logger.error("Must specify --url or --all to sync all upstreams") + sys.exit(ERROR_EXIT_CODE) + + if len(upstreams_to_sync) == 0: + logger.error("No upstreams found to sync.") + sys.exit(ERROR_EXIT_CODE) + + for upstream in upstreams_to_sync: + + result = sync_upstream_for_url( + repo_path=kwargs["repo_path"], + dry_run=kwargs["dry_run"], + url=upstream.url, + include_models=upstream.include_models, + exclude_models=upstream.exclude_models, + skip_validation=upstream.skip_validation, + committer_name=kwargs["committer_name"], + committer_email=kwargs["committer_email"], + branch=kwargs["branch"], + ) + logger.debug(f"Bot results for {upstream.url}: {result}") + logger.info(f"Sync from {upstream.url} complete") diff --git a/trestlebot/cli/config.py b/trestlebot/cli/config.py index f0fc6f85..accf7e9d 100644 --- a/trestlebot/cli/config.py +++ b/trestlebot/cli/config.py @@ -10,7 +10,7 @@ from typing import Any, Dict, List, Optional import yaml -from pydantic import BaseModel, DirectoryPath, ValidationError, model_serializer +from pydantic import BaseModel, DirectoryPath, ValidationError logger = logging.getLogger(__name__) @@ -43,35 +43,73 @@ def __str__(self) -> str: return "".join(self.errors) +class UpstreamConfig(BaseModel): + """Data model for upstream sources.""" + + url: str + include_models: List[str] = ["*"] + exclude_models: List[str] = [] + skip_validation: bool = False + + class TrestleBotConfig(BaseModel): """Data model for trestle-bot configuration.""" repo_path: Optional[DirectoryPath] = None markdown_dir: Optional[str] = None - ssp_index_file: Optional[str] = "ssp-index.json" + committer_name: Optional[str] = None + committer_email: Optional[str] = None + ssp_index_file: Optional[str] = None + upstreams: List[UpstreamConfig] = [] + + def to_yaml_dict(self) -> Dict[str, Any]: + """Returns a dict that can be cleanly loaded into a yaml file. + + This custom model serializer provides a cleaner dict that can + be stored as a YAML file. For example, we want to omit empty values + from being written to the YAML config file. + + Ex: instead of `ssp_index_file: None` appearing in the YAML, we + just want to exclude it from the config file all together. This + produces a YAML config file that only includes values that have + been set (or have a default we want to include). + """ + + upstreams = [] + for upstream in self.upstreams: + upstream_dict = { + "url": upstream.url, + "skip_validation": upstream.skip_validation, + } + if upstream.include_models: + upstream_dict.update(include_models=upstream.include_models) + if upstream.exclude_models: + upstream_dict.update(exclude_models=upstream.exclude_models) + upstreams.append(upstream_dict) - @model_serializer - def _dict(self) -> Dict[str, Any]: - """Returns a dict that can be safely loaded to yaml.""" config_dict = { "repo_path": str(self.repo_path), "markdown_dir": str(self.markdown_dir), "ssp_index_file": str(self.ssp_index_file), + "committer_name": str(self.committer_name), + "committer_email": str(self.committer_email), + "upstreams": upstreams, } return dict( - filter(lambda item: item[1] not in (None, "None"), config_dict.items()) + filter(lambda item: item[1] not in (None, "None", []), config_dict.items()) ) -def load_from_file(file: str) -> Optional[TrestleBotConfig]: +def load_from_file(file_path: Path) -> Optional[TrestleBotConfig]: """Load yaml file to trestlebot config object""" try: - with open(file, "r") as config_file: + with open(file_path, "r") as config_file: config_yaml = yaml.safe_load(config_file) return TrestleBotConfig(**config_yaml) except ValidationError as ex: raise TrestleBotConfigError(ex.errors()) - except FileNotFoundError: + except (FileNotFoundError, TypeError): + logger.debug(f"No config file found at {file_path}") return None @@ -80,15 +118,15 @@ def write_to_file(config: TrestleBotConfig, file_path: Path) -> None: try: file_path.parent.mkdir(parents=True, exist_ok=True) with file_path.open("w") as config_file: - yaml.dump(config.model_dump(), config_file) + yaml.dump(config.to_yaml_dict(), config_file) except ValidationError as ex: raise TrestleBotConfigError(ex.errors()) -def make_config(values: Dict[str, Any]) -> TrestleBotConfig: +def make_config(values: Optional[Dict[str, Any]] = None) -> TrestleBotConfig: """Generates a new trestle-bot config object""" try: - return TrestleBotConfig(**values) + return TrestleBotConfig(**values) if values else TrestleBotConfig() except ValidationError as ex: raise TrestleBotConfigError(ex.errors()) diff --git a/trestlebot/cli/options/common.py b/trestlebot/cli/options/common.py index b77e3a6f..21c95690 100644 --- a/trestlebot/cli/options/common.py +++ b/trestlebot/cli/options/common.py @@ -5,8 +5,9 @@ Common command options for trestle-bot commands. """ -import functools import logging +import os +import pathlib import sys import traceback from typing import Any, Callable, Dict, Optional, Sequence, TypeVar @@ -44,16 +45,29 @@ def debug_to_log_level(ctx: click.Context, param: str, value: str) -> None: set_log_level(log_level) -def load_config_to_ctx(ctx: click.Context, param: str, value: str) -> Optional[str]: - """Load yaml config file into Click context to set default values. +def load_config_to_ctx( + ctx: click.Context, param: str, value: pathlib.Path +) -> Optional[pathlib.Path]: + """Load yaml config file into Click context to set default values. This + allows values from the yaml config to be used as the default value for a + given command option. + + If the user specifies a value for the option directly (e.g. uses --option value) + then that value is used in favor of the value loaded from the config. + + Simarly, if an option has an associated ENVVAR, and that ENVVAR is set, then the + ENVVAR value is used in favor of the value loaded from the config. + + Since the config contains values that should not be mapped to command option values + the dictionary is explicitly built by directly plucking values from the config. This will always run before other options because the --config is_eager is True. """ try: config = load_from_file(value) if not config: - logger.debug("No configuration file found.") - return None + logger.debug(f"No trestlebot config file found at {value}.") + return value except TrestleBotConfigError as ex: logger.error(str(ex)) for err in ex.errors: @@ -61,31 +75,24 @@ def load_config_to_ctx(ctx: click.Context, param: str, value: str) -> Optional[s sys.exit(ERROR_EXIT_CODE) if not ctx.default_map: - ctx.default_map = ( - config.dict() - ) # if default_map has not yet been set by another option + ctx.default_map = { + "markdown_dir": config.markdown_dir, + "ssp_index_file": config.ssp_index_file, + "committer_name": config.committer_name, + "committer_email": config.committer_email, + } else: ctx.default_map.update(config) logger.debug(f"Successfully loaded config file {value} into context.") return value -def common_options(f: F) -> Any: +def common_options(f: F) -> F: """ Configures common options used across commands. """ - @click.pass_context - @click.option( - "--config", - type=click.Path(), - envvar="TRESTLEBOT_CONFIG", - help="Path to trestlebot configuration file", - default=f"{TRESTLEBOT_CONFIG_DIR}/config.yml", - is_eager=True, - callback=load_config_to_ctx, - ) - @click.option( + f = click.option( "--debug", default=False, is_flag=True, @@ -93,60 +100,75 @@ def common_options(f: F) -> Any: envvar="TRESTLEBOT_DEBUG", help="Enable debug logging messages.", callback=debug_to_log_level, - ) - @handle_exceptions - @functools.wraps(f) - def wrapper_common_options(*args: Sequence[Any], **kwargs: Dict[Any, Any]) -> Any: - return f(*args, **kwargs) + )(f) + f = click.option( + "--config", + "config_path", + type=click.Path(path_type=pathlib.Path), + envvar="TRESTLEBOT_CONFIG", + help="Path to trestlebot configuration file.", + default=f"{TRESTLEBOT_CONFIG_DIR}/config.yml", + is_eager=True, + callback=load_config_to_ctx, + )(f) + click.option( + "--repo-path", + type=click.Path(path_type=pathlib.Path, exists=True), + envvar="TRESTLEBOT_REPO_PATH", + help="Path to git respository root.", + required=True, + default=os.getcwd(), + )(f) + f = click.option( + "--dry-run", + help="Run tasks, but do not push changes to the repository.", + is_flag=True, + )(f) - return wrapper_common_options + return f -def git_options(f: F) -> Any: +def git_options(f: F) -> F: """ Configure git options used for git operations. """ - - @click.option( + f = click.option( "--branch", help="Branch name to push changes to", + required=True, type=str, - prompt="Enter branch name to push changes to", - ) - @click.option( + )(f) + f = click.option( "--committer-name", help="Name of committer", + required=True, type=str, - prompt="Enter name for committer", - ) - @click.option( + )(f) + f = click.option( "--committer-email", help="Email for committer", + required=True, type=str, - prompt="Enter email for committer", - ) - @click.option( + )(f) + f = click.option( "--file-patterns", help="Comma-separated list of file patterns to be used with `git add` in repository updates", type=str, - ) - @click.option( + )(f) + f = click.option( "--commit-message", help="Commit message for automated updates", type=str, - ) - @click.option( + )(f) + f = click.option( "--author-name", help="Name for commit author if differs from committer", type=str, - ) - @click.option( + )(f) + f = click.option( "--author-email", help="Email for commit author if differs from committer", type=str, - ) - @functools.wraps(f) - def wrapper_git_options(*args: Sequence[Any], **kwargs: Dict[Any, Any]) -> Any: - return f(*args, **kwargs) + )(f) - return wrapper_git_options + return f diff --git a/trestlebot/cli/root.py b/trestlebot/cli/root.py index 2f6e50bf..2dc2cbbd 100644 --- a/trestlebot/cli/root.py +++ b/trestlebot/cli/root.py @@ -8,6 +8,7 @@ from trestlebot.cli.commands.autosync import autosync_cmd from trestlebot.cli.commands.init import init_cmd +from trestlebot.cli.commands.upstream import upstream_cmd EPILOG = "See our docs at https://redhatproductsecurity.github.io/trestle-bot" @@ -28,3 +29,4 @@ def root_cmd(ctx: click.Context) -> None: root_cmd.add_command(init_cmd) root_cmd.add_command(autosync_cmd) +root_cmd.add_command(upstream_cmd) diff --git a/trestlebot/cli/run.py b/trestlebot/cli/run.py index 561e2dfe..75c97c81 100644 --- a/trestlebot/cli/run.py +++ b/trestlebot/cli/run.py @@ -1,6 +1,7 @@ from typing import Any, Dict, List from trestlebot.bot import TrestleBot +from trestlebot.reporter import BotResults from trestlebot.tasks.base_task import TaskBase @@ -10,8 +11,9 @@ def comma_sep_to_list(string: str) -> List[str]: return list(map(str.strip, string.split(","))) if string else [] -def run(pre_tasks: List[TaskBase], kwargs: Dict[Any, Any]) -> None: +def run(pre_tasks: List[TaskBase], kwargs: Dict[Any, Any]) -> BotResults: """Reusable logic for all commands.""" + # Configure and run the bot bot = TrestleBot( working_dir=kwargs["working_dir"], @@ -21,7 +23,8 @@ def run(pre_tasks: List[TaskBase], kwargs: Dict[Any, Any]) -> None: author_name=kwargs.get("author_name", ""), author_email=kwargs.get("author_email", ""), ) - bot.run( + + return bot.run( pre_tasks=pre_tasks, patterns=comma_sep_to_list(kwargs.get("patterns", "")), commit_message=kwargs.get("commit_message", "Automatic updates from bot"), From b172bcf8c9a3847495ba58a46855564b6e5e4999 Mon Sep 17 00:00:00 2001 From: George Vauter Date: Sun, 24 Nov 2024 01:26:45 -0500 Subject: [PATCH 023/108] feat: add upstream commands, fix common options decorators, expand config --- tests/trestlebot/cli/test_config.py | 18 +- tests/trestlebot/cli/test_init_cmd.py | 26 +-- tests/trestlebot/cli/test_upstreams_cmd.py | 249 +++++++++++++++++++++ trestlebot/cli/commands/init.py | 31 +-- trestlebot/cli/commands/upstream.py | 230 +++++++++++++++++++ trestlebot/cli/config.py | 62 ++++- trestlebot/cli/options/common.py | 120 ++++++---- trestlebot/cli/root.py | 2 + trestlebot/cli/run.py | 7 +- 9 files changed, 636 insertions(+), 109 deletions(-) create mode 100644 tests/trestlebot/cli/test_upstreams_cmd.py create mode 100644 trestlebot/cli/commands/upstream.py diff --git a/tests/trestlebot/cli/test_config.py b/tests/trestlebot/cli/test_config.py index 8ed4b141..8d8c21b1 100644 --- a/tests/trestlebot/cli/test_config.py +++ b/tests/trestlebot/cli/test_config.py @@ -42,14 +42,6 @@ def test_make_config_raises_no_errors(tmp_init_dir: str) -> None: assert config.markdown_dir == "markdown-test" -def test_config_to_dict(config_obj: TrestleBotConfig) -> None: - """Config should serialize to a dict.""" - model_dict = config_obj.model_dump() - assert isinstance(model_dict, dict) - assert model_dict["repo_path"] == str(config_obj.repo_path) - assert model_dict["markdown_dir"] == config_obj.markdown_dir - - def test_config_write_to_file(config_obj: TrestleBotConfig, tmp_init_dir: str) -> None: """Test config is written to yaml file.""" filepath = pathlib.Path(tmp_init_dir).joinpath("config.yml") @@ -57,15 +49,15 @@ def test_config_write_to_file(config_obj: TrestleBotConfig, tmp_init_dir: str) - with open(filepath, "r") as f: yaml_data = yaml.safe_load(f) - assert yaml_data == config_obj.model_dump() + assert yaml_data == config_obj.to_yaml_dict() -def test_config_laod_from_file(config_obj: TrestleBotConfig, tmp_init_dir: str) -> None: +def test_config_load_from_file(config_obj: TrestleBotConfig, tmp_init_dir: str) -> None: """Test config is read from yaml file into config object.""" filepath = pathlib.Path(tmp_init_dir).joinpath("config.yml") with filepath.open("w") as config_file: - yaml.dump(config_obj.model_dump(), config_file) + yaml.dump(config_obj.to_yaml_dict(), config_file) - config = load_from_file(str(filepath)) + config = load_from_file(filepath) assert isinstance(config, TrestleBotConfig) - assert config.model_dump() == config_obj.model_dump() + assert config == config_obj diff --git a/tests/trestlebot/cli/test_init_cmd.py b/tests/trestlebot/cli/test_init_cmd.py index 1e9dcd0e..0bf05dc9 100644 --- a/tests/trestlebot/cli/test_init_cmd.py +++ b/tests/trestlebot/cli/test_init_cmd.py @@ -15,24 +15,20 @@ from trestlebot.const import TRESTLEBOT_CONFIG_DIR -TRESTLE_KEEP_FILE - - def test_init_repo_dir_does_not_exist() -> None: """Init should fail if repo dir does not exit""" runner = CliRunner() - result = runner.invoke(init_cmd, ["0"]) + result = runner.invoke(init_cmd, ["--repo-path", "0"]) assert result.exit_code == 2 - assert ( - "Error: Invalid value for 'REPO_PATH': Path '0' does not exist." - in result.output - ) + assert "does not exist." in result.output def test_init_not_git_repo(tmp_init_dir: str) -> None: """Init should fail if repo dir is not a Git repo.""" runner = CliRunner() - result = runner.invoke(init_cmd, [tmp_init_dir, "--markdown-dir", "markdown"]) + result = runner.invoke( + init_cmd, ["--repo-path", tmp_init_dir, "--markdown-dir", "markdown"] + ) assert result.exit_code == 1 assert "not a Git repository" in result.output @@ -48,7 +44,9 @@ def test_init_existing_trestlebot_dir(tmp_init_dir: str) -> None: setup_for_init(pathlib.Path(tmp_init_dir)) runner = CliRunner() - result = runner.invoke(init_cmd, [tmp_init_dir, "--markdown-dir", "markdown"]) + result = runner.invoke( + init_cmd, ["--repo-path", tmp_init_dir, "--markdown-dir", "markdown"] + ) assert result.exit_code == 1 assert "existing .trestlebot directory" in result.output @@ -60,7 +58,9 @@ def test_init_creates_config_file(tmp_init_dir: str) -> None: setup_for_init(pathlib.Path(tmp_init_dir)) runner = CliRunner() - result = runner.invoke(init_cmd, [tmp_init_dir, "--markdown-dir", "markdown"]) + result = runner.invoke( + init_cmd, ["--repo-path", tmp_init_dir, "--markdown-dir", "markdown"] + ) assert result.exit_code == 0 assert "Successfully initialized trestlebot" in result.output @@ -83,7 +83,7 @@ def test_init_creates_model_dirs(tmp_init_dir: str) -> None: setup_for_init(tmp_dir) runner = CliRunner() - runner.invoke(init_cmd, [tmp_init_dir, "--markdown-dir", "markdown"]) + runner.invoke(init_cmd, ["--repo-path", tmp_init_dir, "--markdown-dir", "markdown"]) model_dirs = [d.name for d in tmp_dir.iterdir() if not is_hidden(d)] model_dirs.remove("markdown") # pop markdown dir @@ -98,7 +98,7 @@ def test_init_creates_markdown_dirs(tmp_init_dir: str) -> None: setup_for_init(tmp_dir) runner = CliRunner() - runner.invoke(init_cmd, [tmp_init_dir, "--markdown-dir", "markdown"]) + runner.invoke(init_cmd, ["--repo-path", tmp_init_dir, "--markdown-dir", "markdown"]) markdown_dirs = [d.name for d in markdown_dir.iterdir() if not is_hidden(d)] assert sorted(markdown_dirs) == sorted(MODEL_DIR_LIST) diff --git a/tests/trestlebot/cli/test_upstreams_cmd.py b/tests/trestlebot/cli/test_upstreams_cmd.py new file mode 100644 index 00000000..11e05a19 --- /dev/null +++ b/tests/trestlebot/cli/test_upstreams_cmd.py @@ -0,0 +1,249 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Red Hat, Inc. + + +"""Unit tests for upstreams commands.""" + +import pathlib +from typing import Tuple + +import pytest +from click.testing import CliRunner +from git import Repo + +from tests.testutils import clean, prepare_upstream_repo +from trestlebot.cli.commands.upstream import upstream_cmd +from trestlebot.cli.config import ( + TrestleBotConfig, + UpstreamConfig, + load_from_file, + write_to_file, +) + + +TEST_CATALOG = "simplified_nist_catalog" +TEST_CATALOG_PATH = "catalogs/simplified_nist_catalog/catalog.json" +TEST_PROFILE = "simplified_nist_profile" +TEST_PROFILE_PATH = "profiles/simplified_nist_profile/profile.json" + + +def test_config_get_upstream_by_url() -> None: + """Test to confirm _get_upstream_by_url returns a single valid upstream from the config.""" + config = TrestleBotConfig(upstreams=[UpstreamConfig(url="https://test")]) + + assert len(config.upstreams) == 1 + assert config.upstreams[0].url == "https://test" + assert config.upstreams[0].include_models == ["*"] # confirm default was set + assert config.upstreams[0].exclude_models == [] + assert config.upstreams[0].skip_validation is False + + +def test_upstream_add_already_exists(tmp_repo: Tuple[str, Repo]) -> None: + """Test upstream add fails if upstream already exists""" + repo_path, repo = tmp_repo + config_path = pathlib.Path(repo_path).joinpath(".trestlebot/config.yml") + + source: str = prepare_upstream_repo() + url = f"{source}@main" + config = TrestleBotConfig( + committer_email="test@test", + committer_name="test", + upstreams=[UpstreamConfig(url=url)], + ) + write_to_file(config, config_path) + + runner = CliRunner() + result = runner.invoke( + upstream_cmd, + [ + "add", + "--repo-path", + repo_path, + "--config", + config_path, + "--url", + url, + "--branch", + "test", + ], + ) + assert ( + result.exit_code == 0 + ) # This does return success, it just skips the URLs already in the config + + assert f"{url} already exists. Edit {config_path} to update." in result.output + + +def test_upstream_add(tmp_repo: Tuple[str, Repo]) -> None: + """Tests successful add of upstream""" + repo_dir, repo = tmp_repo + repo_path = pathlib.Path(repo_dir) + config_path = repo_path.joinpath(".trestlebot/config.yml") + + source: str = prepare_upstream_repo() + url = f"{source}@main" + config = TrestleBotConfig( + committer_email="test@test", + committer_name="test", + ) + write_to_file(config, config_path) + + runner = CliRunner() + result = runner.invoke( + upstream_cmd, + [ + "add", + "--repo-path", + repo_path, + "--config", + config_path, + "--url", + url, + "--branch", + "main", + "--debug", + ], + ) + updated_config = load_from_file(config_path) + if not updated_config: + pytest.fail("Updated config not found") + return + + assert len(updated_config.upstreams) == 1 + assert updated_config.upstreams[0].url == url + assert updated_config.upstreams[0].include_models == ["*"] + assert updated_config.upstreams[0].exclude_models == [] + assert updated_config.upstreams[0].skip_validation is False + assert ( + updated_config.committer_email == "test@test" + ) # sanity check this didn't change + assert repo_path.joinpath(TEST_CATALOG_PATH).exists() + assert repo_path.joinpath(TEST_PROFILE_PATH).exists() + assert result.exit_code == 0 + clean(source, None) + + +def test_upstream_add_exclude_models(tmp_repo: Tuple[str, Repo]) -> None: + """Test add upstream with exclude models""" + + repo_dir, repo = tmp_repo + repo_path = pathlib.Path(repo_dir) + config_path = repo_path.joinpath(".trestlebot/config.yml") + + source: str = prepare_upstream_repo() + url = f"{source}@main" + config = TrestleBotConfig( + committer_email="test@test", + committer_name="test", + ) + write_to_file(config, config_path) + + runner = CliRunner() + result = runner.invoke( + upstream_cmd, + [ + "add", + "--repo-path", + repo_path, + "--config", + config_path, + "--url", + url, + "--branch", + "main", + "--exclude-model", + TEST_PROFILE, + ], + ) + + updated_config = load_from_file(config_path) + if not updated_config: + pytest.fail("Updated config not found") + return + + assert len(updated_config.upstreams) == 1 + assert updated_config.upstreams[0].url == url + assert updated_config.upstreams[0].include_models == ["*"] + assert updated_config.upstreams[0].exclude_models == [TEST_PROFILE] + assert updated_config.upstreams[0].skip_validation is False + assert ( + updated_config.committer_email == "test@test" + ) # sanity check this didn't change + + assert repo_path.joinpath(TEST_CATALOG_PATH).exists() + assert not repo_path.joinpath(TEST_PROFILE_PATH).exists() + assert result.exit_code == 0 + + +def test_upstream_sync_upstream_no_upstreams_in_config( + tmp_repo: Tuple[str, Repo] +) -> None: + """Test sync upstream with no upstreams in config file.""" + + repo_dir, repo = tmp_repo + repo_path = pathlib.Path(repo_dir) + config_path = repo_path.joinpath(".trestlebot/config.yml") + + source: str = prepare_upstream_repo() + url = f"{source}@main" + config = TrestleBotConfig( + committer_email="test@test", + committer_name="test", + ) + write_to_file(config, config_path) + + runner = CliRunner() + result = runner.invoke( + upstream_cmd, + [ + "sync", + "--repo-path", + repo_path, + "--config", + config_path, + "--url", + url, + "--branch", + "main", + ], + ) + assert ( + "No upstreams defined in trestlebot config. Use `upstream add` command." + in result.output + ) + assert result.exit_code == 1 + + +def test_upstream_sync_upstream_not_found(tmp_repo: Tuple[str, Repo]) -> None: + """Test sync upstream with url not found.""" + + repo_dir, repo = tmp_repo + repo_path = pathlib.Path(repo_dir) + config_path = repo_path.joinpath(".trestlebot/config.yml") + + source: str = prepare_upstream_repo() + url = f"{source}@main" + config = TrestleBotConfig( + committer_email="test@test", + committer_name="test", + upstreams=[UpstreamConfig(url="foo@bar")], + ) + write_to_file(config, config_path) + + runner = CliRunner() + result = runner.invoke( + upstream_cmd, + [ + "sync", + "--repo-path", + repo_path, + "--config", + config_path, + "--url", + url, + "--branch", + "main", + ], + ) + assert f"No upstream defined for {source}" in result.output + assert result.exit_code == 1 diff --git a/trestlebot/cli/commands/init.py b/trestlebot/cli/commands/init.py index 90fb4ff1..da289932 100644 --- a/trestlebot/cli/commands/init.py +++ b/trestlebot/cli/commands/init.py @@ -6,8 +6,8 @@ """ import argparse import logging +import pathlib import sys -from pathlib import Path import click from trestle.common.const import MODEL_DIR_LIST @@ -27,7 +27,7 @@ logging.getLogger("trestle.core.commands.init").setLevel("CRITICAL") -def call_trestle_init(repo_path: Path, debug: bool) -> None: +def call_trestle_init(repo_path: pathlib.Path, debug: bool) -> None: """Call compliance-trestle to initialize workspace""" verbose = 1 if debug else 0 @@ -49,9 +49,8 @@ def call_trestle_init(repo_path: Path, debug: bool) -> None: @click.command(name="init", help="Initialize a new trestle-bot repo.") -@click.argument( - "repo_path", type=click.Path(path_type=Path, exists=True), required=True -) +@click.pass_context +@common_options @click.option( "--markdown-dir", type=str, @@ -59,33 +58,25 @@ def call_trestle_init(repo_path: Path, debug: bool) -> None: default="markdown/", prompt="Enter path to store markdown files", ) -@click.option( - "--ssp-index-file", - type=str, - help="Path of SSP index file.", - default="ssp-index.json", - required=False, -) -@common_options def init_cmd( ctx: click.Context, debug: bool, - config: str, - repo_path: Path, + config_path: pathlib.Path, + repo_path: pathlib.Path, markdown_dir: str, - ssp_index_file: str, + dry_run: bool, ) -> None: """Command to initialize a new trestlebot repo""" repo_path = repo_path.resolve() - git_path: Path = repo_path.joinpath(Path(".git")) + git_path: pathlib.Path = repo_path.joinpath(pathlib.Path(".git")) if not git_path.exists(): logger.error( f"Initialization failed. Given directory {repo_path} is not a Git repository." ) sys.exit(ERROR_EXIT_CODE) - trestlebot_dir = repo_path.joinpath(Path(TRESTLEBOT_CONFIG_DIR)) + trestlebot_dir = repo_path.joinpath(pathlib.Path(TRESTLEBOT_CONFIG_DIR)) if trestlebot_dir.exists(): logger.error( f"Initialization failed. Found existing {TRESTLEBOT_CONFIG_DIR} directory in {repo_path}" @@ -106,7 +97,7 @@ def init_cmd( # Create markdown directories in workspace root list( map( - lambda x: repo_path.joinpath(Path(markdown_dir)) + lambda x: repo_path.joinpath(pathlib.Path(markdown_dir)) .joinpath(x) .joinpath(TRESTLEBOT_KEEP_FILE) .mkdir(parents=True, exist_ok=True), @@ -120,7 +111,7 @@ def init_cmd( # generate and write trestle-bot cofig config = make_config(dict(repo_path=repo_path, markdown_dir=markdown_dir)) - config_path = trestlebot_dir.joinpath(Path("config.yml")) + config_path = trestlebot_dir.joinpath("config.yml") write_to_file(config, config_path) logger.debug(f"trestle-bot config file created at {str(config_path)}") logger.info(f"Successfully initialized trestlebot project in {repo_path}") diff --git a/trestlebot/cli/commands/upstream.py b/trestlebot/cli/commands/upstream.py new file mode 100644 index 00000000..7ed5a364 --- /dev/null +++ b/trestlebot/cli/commands/upstream.py @@ -0,0 +1,230 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Red Hat, Inc. + +"""Module for upstream command""" +import logging +import pathlib +import sys +from typing import Any, List, Optional + +import click + +from trestlebot.cli.config import ( + TrestleBotConfig, + UpstreamConfig, + load_from_file, + write_to_file, +) +from trestlebot.cli.options.common import common_options, git_options, handle_exceptions +from trestlebot.cli.run import run +from trestlebot.const import ERROR_EXIT_CODE +from trestlebot.reporter import BotResults +from trestlebot.tasks.base_task import ModelFilter +from trestlebot.tasks.sync_upstreams_task import SyncUpstreamsTask + + +logger = logging.getLogger(__name__) + + +def sync_upstream_for_url( + repo_path: pathlib.Path, + dry_run: bool, + url: str, + include_models: List[str], + exclude_models: List[str], + skip_validation: bool, + committer_name: str, + committer_email: str, + branch: str, +) -> BotResults: + """Invoke the sync upstream task for a given URL.""" + model_filter: ModelFilter = ModelFilter( + include_patterns=include_models, skip_patterns=exclude_models + ) + + validate = not skip_validation + + sync_upstreams_task = SyncUpstreamsTask( + working_dir=str(repo_path.resolve()), + git_sources=[url], + model_filter=model_filter, + validate=validate, + ) + kwargs = dict( + working_dir=(str(repo_path.resolve())), + committer_name=committer_name, + committer_email=committer_email, + branch=branch, + dry_run=dry_run, + ) + return run([sync_upstreams_task], kwargs) + + +def _config_get_upstream_by_url( + config: TrestleBotConfig, url: str +) -> Optional[UpstreamConfig]: + """Returns one upstream that matches url, else None.""" + if config.upstreams: + for upstream in config.upstreams: + if upstream.url == url: + return upstream + return None + + +@click.group(name="upstream") +@click.pass_context +def upstream_cmd(ctx: click.Context) -> None: + """Sync content from upstream git repositories.""" + + +@upstream_cmd.command(name="add") +@click.pass_context +@common_options +@git_options +@click.option( + "--url", + type=str, + help="Upstream repo URL in the form @ where ref is a git ref such as tag or branch.", + required=True, + multiple=True, +) +@click.option( + "--exclude-model", + type=str, + help="Glob pattern for model names to exclude (e.g. --exclude-model=profile_y*).", + required=False, + multiple=True, +) +@click.option( + "--include-model", + type=str, + help="Glob pattern for model names to include (e.g. --include-model=profile_y*).", + required=False, + multiple=True, +) +@click.option( + "--skip-validation", + type=bool, + help="Skip validation of the models when they are copied.", + is_flag=True, +) +@handle_exceptions +def upstream_add_cmd(ctx: click.Context, **kwargs: Any) -> None: + """Add new upstream sources to workspace.""" + + repo_path = kwargs["repo_path"] + url_list = list(kwargs["url"]) + + config_path: pathlib.Path = kwargs["config_path"] + config = load_from_file(config_path) + if not config: + # No config exists or was not found, create a new one + logger.warning( + f"No trestlebot config file found, creating {str(config_path.resolve())}" + ) + config = TrestleBotConfig() + + for url in url_list: + existing_urls = [upstream.url for upstream in config.upstreams] + if url in existing_urls: + logger.warning( + f"Upstream for {url} already exists. Edit {config_path} to update." + ) + continue + + include_models = list(kwargs.get("include_model", ["*"])) + if len(include_models) == 0: + include_models = ["*"] + exclude_models = list(kwargs.get("exclude_model", [])) + skip_validation = kwargs.get("skip_validation", False) + result = sync_upstream_for_url( + repo_path=repo_path, + dry_run=kwargs["dry_run"], + url=url, + include_models=include_models, + exclude_models=exclude_models, + skip_validation=skip_validation, + committer_name=kwargs["committer_name"], + committer_email=kwargs["committer_email"], + branch=kwargs["branch"], + ) + logger.debug(f"Bot results for {url}: {result}") + + config.upstreams.append( + UpstreamConfig( + url=url, + include_models=include_models, + exclude_models=exclude_models, + skip_validation=skip_validation, + ) + ) + logger.info(f"Added {url} to trestlebot workspace") + + write_to_file(config, config_path) + + +@upstream_cmd.command(name="sync") +@click.pass_context +@common_options +@git_options +@click.option( + "--url", + type=str, + help="Upstream repo URL in the form @ where ref is a git ref such as tag or branch.", + required=False, + multiple=True, +) +@click.option( + "--all", + type=str, + help="URL to GitHub repo containing upstream content.", + required=False, + is_flag=True, +) +@handle_exceptions +def upstream_sync_cmd(ctx: click.Context, **kwargs: Any) -> None: + """Sync upstream sources to local workspace""" + + config_path: pathlib.Path = kwargs["config_path"] + config = load_from_file(config_path) + if not config or len(config.upstreams) == 0: + logger.error( + "No upstreams defined in trestlebot config. Use `upstream add` command." + ) + sys.exit(ERROR_EXIT_CODE) + + upstreams_to_sync = [] + if kwargs.get("all"): + upstreams_to_sync = config.upstreams + + elif urls := kwargs.get("url"): + for url in urls: + if upstream := _config_get_upstream_by_url(config, url): + upstreams_to_sync.append(upstream) + + else: + logger.error(f"No upstream defined for {url} - skipping!") + + else: + logger.error("Must specify --url or --all to sync all upstreams") + sys.exit(ERROR_EXIT_CODE) + + if len(upstreams_to_sync) == 0: + logger.error("No upstreams found to sync.") + sys.exit(ERROR_EXIT_CODE) + + for upstream in upstreams_to_sync: + + result = sync_upstream_for_url( + repo_path=kwargs["repo_path"], + dry_run=kwargs["dry_run"], + url=upstream.url, + include_models=upstream.include_models, + exclude_models=upstream.exclude_models, + skip_validation=upstream.skip_validation, + committer_name=kwargs["committer_name"], + committer_email=kwargs["committer_email"], + branch=kwargs["branch"], + ) + logger.debug(f"Bot results for {upstream.url}: {result}") + logger.info(f"Sync from {upstream.url} complete") diff --git a/trestlebot/cli/config.py b/trestlebot/cli/config.py index f0fc6f85..accf7e9d 100644 --- a/trestlebot/cli/config.py +++ b/trestlebot/cli/config.py @@ -10,7 +10,7 @@ from typing import Any, Dict, List, Optional import yaml -from pydantic import BaseModel, DirectoryPath, ValidationError, model_serializer +from pydantic import BaseModel, DirectoryPath, ValidationError logger = logging.getLogger(__name__) @@ -43,35 +43,73 @@ def __str__(self) -> str: return "".join(self.errors) +class UpstreamConfig(BaseModel): + """Data model for upstream sources.""" + + url: str + include_models: List[str] = ["*"] + exclude_models: List[str] = [] + skip_validation: bool = False + + class TrestleBotConfig(BaseModel): """Data model for trestle-bot configuration.""" repo_path: Optional[DirectoryPath] = None markdown_dir: Optional[str] = None - ssp_index_file: Optional[str] = "ssp-index.json" + committer_name: Optional[str] = None + committer_email: Optional[str] = None + ssp_index_file: Optional[str] = None + upstreams: List[UpstreamConfig] = [] + + def to_yaml_dict(self) -> Dict[str, Any]: + """Returns a dict that can be cleanly loaded into a yaml file. + + This custom model serializer provides a cleaner dict that can + be stored as a YAML file. For example, we want to omit empty values + from being written to the YAML config file. + + Ex: instead of `ssp_index_file: None` appearing in the YAML, we + just want to exclude it from the config file all together. This + produces a YAML config file that only includes values that have + been set (or have a default we want to include). + """ + + upstreams = [] + for upstream in self.upstreams: + upstream_dict = { + "url": upstream.url, + "skip_validation": upstream.skip_validation, + } + if upstream.include_models: + upstream_dict.update(include_models=upstream.include_models) + if upstream.exclude_models: + upstream_dict.update(exclude_models=upstream.exclude_models) + upstreams.append(upstream_dict) - @model_serializer - def _dict(self) -> Dict[str, Any]: - """Returns a dict that can be safely loaded to yaml.""" config_dict = { "repo_path": str(self.repo_path), "markdown_dir": str(self.markdown_dir), "ssp_index_file": str(self.ssp_index_file), + "committer_name": str(self.committer_name), + "committer_email": str(self.committer_email), + "upstreams": upstreams, } return dict( - filter(lambda item: item[1] not in (None, "None"), config_dict.items()) + filter(lambda item: item[1] not in (None, "None", []), config_dict.items()) ) -def load_from_file(file: str) -> Optional[TrestleBotConfig]: +def load_from_file(file_path: Path) -> Optional[TrestleBotConfig]: """Load yaml file to trestlebot config object""" try: - with open(file, "r") as config_file: + with open(file_path, "r") as config_file: config_yaml = yaml.safe_load(config_file) return TrestleBotConfig(**config_yaml) except ValidationError as ex: raise TrestleBotConfigError(ex.errors()) - except FileNotFoundError: + except (FileNotFoundError, TypeError): + logger.debug(f"No config file found at {file_path}") return None @@ -80,15 +118,15 @@ def write_to_file(config: TrestleBotConfig, file_path: Path) -> None: try: file_path.parent.mkdir(parents=True, exist_ok=True) with file_path.open("w") as config_file: - yaml.dump(config.model_dump(), config_file) + yaml.dump(config.to_yaml_dict(), config_file) except ValidationError as ex: raise TrestleBotConfigError(ex.errors()) -def make_config(values: Dict[str, Any]) -> TrestleBotConfig: +def make_config(values: Optional[Dict[str, Any]] = None) -> TrestleBotConfig: """Generates a new trestle-bot config object""" try: - return TrestleBotConfig(**values) + return TrestleBotConfig(**values) if values else TrestleBotConfig() except ValidationError as ex: raise TrestleBotConfigError(ex.errors()) diff --git a/trestlebot/cli/options/common.py b/trestlebot/cli/options/common.py index b77e3a6f..21c95690 100644 --- a/trestlebot/cli/options/common.py +++ b/trestlebot/cli/options/common.py @@ -5,8 +5,9 @@ Common command options for trestle-bot commands. """ -import functools import logging +import os +import pathlib import sys import traceback from typing import Any, Callable, Dict, Optional, Sequence, TypeVar @@ -44,16 +45,29 @@ def debug_to_log_level(ctx: click.Context, param: str, value: str) -> None: set_log_level(log_level) -def load_config_to_ctx(ctx: click.Context, param: str, value: str) -> Optional[str]: - """Load yaml config file into Click context to set default values. +def load_config_to_ctx( + ctx: click.Context, param: str, value: pathlib.Path +) -> Optional[pathlib.Path]: + """Load yaml config file into Click context to set default values. This + allows values from the yaml config to be used as the default value for a + given command option. + + If the user specifies a value for the option directly (e.g. uses --option value) + then that value is used in favor of the value loaded from the config. + + Simarly, if an option has an associated ENVVAR, and that ENVVAR is set, then the + ENVVAR value is used in favor of the value loaded from the config. + + Since the config contains values that should not be mapped to command option values + the dictionary is explicitly built by directly plucking values from the config. This will always run before other options because the --config is_eager is True. """ try: config = load_from_file(value) if not config: - logger.debug("No configuration file found.") - return None + logger.debug(f"No trestlebot config file found at {value}.") + return value except TrestleBotConfigError as ex: logger.error(str(ex)) for err in ex.errors: @@ -61,31 +75,24 @@ def load_config_to_ctx(ctx: click.Context, param: str, value: str) -> Optional[s sys.exit(ERROR_EXIT_CODE) if not ctx.default_map: - ctx.default_map = ( - config.dict() - ) # if default_map has not yet been set by another option + ctx.default_map = { + "markdown_dir": config.markdown_dir, + "ssp_index_file": config.ssp_index_file, + "committer_name": config.committer_name, + "committer_email": config.committer_email, + } else: ctx.default_map.update(config) logger.debug(f"Successfully loaded config file {value} into context.") return value -def common_options(f: F) -> Any: +def common_options(f: F) -> F: """ Configures common options used across commands. """ - @click.pass_context - @click.option( - "--config", - type=click.Path(), - envvar="TRESTLEBOT_CONFIG", - help="Path to trestlebot configuration file", - default=f"{TRESTLEBOT_CONFIG_DIR}/config.yml", - is_eager=True, - callback=load_config_to_ctx, - ) - @click.option( + f = click.option( "--debug", default=False, is_flag=True, @@ -93,60 +100,75 @@ def common_options(f: F) -> Any: envvar="TRESTLEBOT_DEBUG", help="Enable debug logging messages.", callback=debug_to_log_level, - ) - @handle_exceptions - @functools.wraps(f) - def wrapper_common_options(*args: Sequence[Any], **kwargs: Dict[Any, Any]) -> Any: - return f(*args, **kwargs) + )(f) + f = click.option( + "--config", + "config_path", + type=click.Path(path_type=pathlib.Path), + envvar="TRESTLEBOT_CONFIG", + help="Path to trestlebot configuration file.", + default=f"{TRESTLEBOT_CONFIG_DIR}/config.yml", + is_eager=True, + callback=load_config_to_ctx, + )(f) + click.option( + "--repo-path", + type=click.Path(path_type=pathlib.Path, exists=True), + envvar="TRESTLEBOT_REPO_PATH", + help="Path to git respository root.", + required=True, + default=os.getcwd(), + )(f) + f = click.option( + "--dry-run", + help="Run tasks, but do not push changes to the repository.", + is_flag=True, + )(f) - return wrapper_common_options + return f -def git_options(f: F) -> Any: +def git_options(f: F) -> F: """ Configure git options used for git operations. """ - - @click.option( + f = click.option( "--branch", help="Branch name to push changes to", + required=True, type=str, - prompt="Enter branch name to push changes to", - ) - @click.option( + )(f) + f = click.option( "--committer-name", help="Name of committer", + required=True, type=str, - prompt="Enter name for committer", - ) - @click.option( + )(f) + f = click.option( "--committer-email", help="Email for committer", + required=True, type=str, - prompt="Enter email for committer", - ) - @click.option( + )(f) + f = click.option( "--file-patterns", help="Comma-separated list of file patterns to be used with `git add` in repository updates", type=str, - ) - @click.option( + )(f) + f = click.option( "--commit-message", help="Commit message for automated updates", type=str, - ) - @click.option( + )(f) + f = click.option( "--author-name", help="Name for commit author if differs from committer", type=str, - ) - @click.option( + )(f) + f = click.option( "--author-email", help="Email for commit author if differs from committer", type=str, - ) - @functools.wraps(f) - def wrapper_git_options(*args: Sequence[Any], **kwargs: Dict[Any, Any]) -> Any: - return f(*args, **kwargs) + )(f) - return wrapper_git_options + return f diff --git a/trestlebot/cli/root.py b/trestlebot/cli/root.py index 2f6e50bf..2dc2cbbd 100644 --- a/trestlebot/cli/root.py +++ b/trestlebot/cli/root.py @@ -8,6 +8,7 @@ from trestlebot.cli.commands.autosync import autosync_cmd from trestlebot.cli.commands.init import init_cmd +from trestlebot.cli.commands.upstream import upstream_cmd EPILOG = "See our docs at https://redhatproductsecurity.github.io/trestle-bot" @@ -28,3 +29,4 @@ def root_cmd(ctx: click.Context) -> None: root_cmd.add_command(init_cmd) root_cmd.add_command(autosync_cmd) +root_cmd.add_command(upstream_cmd) diff --git a/trestlebot/cli/run.py b/trestlebot/cli/run.py index 561e2dfe..75c97c81 100644 --- a/trestlebot/cli/run.py +++ b/trestlebot/cli/run.py @@ -1,6 +1,7 @@ from typing import Any, Dict, List from trestlebot.bot import TrestleBot +from trestlebot.reporter import BotResults from trestlebot.tasks.base_task import TaskBase @@ -10,8 +11,9 @@ def comma_sep_to_list(string: str) -> List[str]: return list(map(str.strip, string.split(","))) if string else [] -def run(pre_tasks: List[TaskBase], kwargs: Dict[Any, Any]) -> None: +def run(pre_tasks: List[TaskBase], kwargs: Dict[Any, Any]) -> BotResults: """Reusable logic for all commands.""" + # Configure and run the bot bot = TrestleBot( working_dir=kwargs["working_dir"], @@ -21,7 +23,8 @@ def run(pre_tasks: List[TaskBase], kwargs: Dict[Any, Any]) -> None: author_name=kwargs.get("author_name", ""), author_email=kwargs.get("author_email", ""), ) - bot.run( + + return bot.run( pre_tasks=pre_tasks, patterns=comma_sep_to_list(kwargs.get("patterns", "")), commit_message=kwargs.get("commit_message", "Automatic updates from bot"), From 0a1c0cd3ed7138e43c3ed02cede3c080b23ad24e Mon Sep 17 00:00:00 2001 From: Qingmin Duanmu Date: Mon, 25 Nov 2024 21:13:42 +0800 Subject: [PATCH 024/108] Update autosync options and add tests --- tests/trestlebot/cli/test_autosync_cmd.py | 77 +++++++++++ trestlebot/cli/commands/autosync.py | 152 ++++++---------------- trestlebot/cli/options/autosync.py | 49 +++++++ 3 files changed, 167 insertions(+), 111 deletions(-) create mode 100644 tests/trestlebot/cli/test_autosync_cmd.py create mode 100644 trestlebot/cli/options/autosync.py diff --git a/tests/trestlebot/cli/test_autosync_cmd.py b/tests/trestlebot/cli/test_autosync_cmd.py new file mode 100644 index 00000000..d044bee3 --- /dev/null +++ b/tests/trestlebot/cli/test_autosync_cmd.py @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Red Hat, Inc. + + +"""Testing module for trestlebot autosync command""" +import tempfile + +from click.testing import CliRunner + +from trestlebot.cli.commands.autosync import autosync_cmd + + +def test_invalid_autosync_command(tmp_init_dir: str) -> None: + tempdir = tempfile.mkdtemp() + runner = CliRunner() + result = runner.invoke( + autosync_cmd, + [ + "invalid", + "--repo-path", + tmp_init_dir, + "--markdown-path", + tempdir, + "--branch", + "main", + "--committer-name", + "Test User", + "--committer-email", + "test@example.com", + ], + ) + assert "Error: No such command" in result.output + assert result.exit_code == 2 + + +def test_no_ssp_index_path(tmp_init_dir: str) -> None: + """Test invalid ssp index file for autosync ssp""" + + tempdir = tempfile.mkdtemp() + runner = CliRunner() + cmd_options = [ + "ssp", + "--repo-path", + tmp_init_dir, + "--markdown-path", + tempdir, + "--branch", + "main", + "--committer-name", + "Test User", + "--committer-email", + "test@example.com", + ] + result = runner.invoke(autosync_cmd, cmd_options) + assert result.exit_code == 2 + assert "Error: Missing option '--ssp-index-path'" in result.output + cmd_options[0] = "compdef" + result = runner.invoke(autosync_cmd, cmd_options) + assert result.exit_code == 0 + + +def test_no_markdown_path(tmp_init_dir: str) -> None: + runner = CliRunner() + cmd_options = [ + "compdef", + "--repo-path", + tmp_init_dir, + "--branch", + "main", + "--committer-name", + "Test User", + "--committer-email", + "test@example.com", + ] + result = runner.invoke(autosync_cmd, cmd_options) + assert result.exit_code == 2 + assert "Error: Missing option '--markdown-path'" in result.output diff --git a/trestlebot/cli/commands/autosync.py b/trestlebot/cli/commands/autosync.py index e9576c7f..2d3eaa0e 100644 --- a/trestlebot/cli/commands/autosync.py +++ b/trestlebot/cli/commands/autosync.py @@ -1,11 +1,12 @@ """ Autosync command""" import logging -from typing import Any, Dict, List +from typing import Any, List import click -from trestlebot.cli.options.common import git_options, handle_exceptions +from trestlebot.cli.options.autosync import autosync_options +from trestlebot.cli.options.common import common_options, git_options, handle_exceptions from trestlebot.cli.run import comma_sep_to_list from trestlebot.cli.run import run as bot_run from trestlebot.tasks.assemble_task import AssembleTask @@ -19,163 +20,92 @@ @handle_exceptions -def run(oscal_model: str, ctx_obj: Dict[str, Any]) -> None: +def run(oscal_model: str, **kwargs: Any) -> None: """Run the autosync for oscal model.""" pre_tasks: List[TaskBase] = [] - # Allow any model to be skipped from the args, by default include all + kwargs["working_dir"] = str(kwargs["repo_path"].resolve()) + if kwargs.get("file_patterns"): + kwargs.update({"patterns": comma_sep_to_list(kwargs["file_patterns"])}) model_filter: ModelFilter = ModelFilter( - skip_patterns=comma_sep_to_list(ctx_obj.get("skip_items", "")), + skip_patterns=comma_sep_to_list(kwargs.get("skip_items", "")), include_patterns=["*"], ) authored_object: AuthoredObjectBase = types.get_authored_object( oscal_model, - ctx_obj["working_dir"], - ctx_obj.get("ssp_index_path", ""), + kwargs["working_dir"], + kwargs.get("ssp_index_path", ""), ) # Assuming an edit has occurred assemble would be run before regenerate. # Adding this to the list first - if not ctx_obj.get("skip_assemble"): + if not kwargs.get("skip_assemble"): assemble_task: AssembleTask = AssembleTask( authored_object=authored_object, - markdown_dir=ctx_obj["markdown_path"], - version=ctx_obj.get("version", ""), + markdown_dir=kwargs["markdown_path"], + version=kwargs.get("version", ""), model_filter=model_filter, ) pre_tasks.append(assemble_task) else: logger.info("Assemble task skipped.") - if not ctx_obj.get("skip_regenerate"): + if not kwargs.get("skip_regenerate"): regenerate_task: RegenerateTask = RegenerateTask( authored_object=authored_object, - markdown_dir=ctx_obj["markdown_path"], + markdown_dir=kwargs["markdown_path"], model_filter=model_filter, ) pre_tasks.append(regenerate_task) else: logger.info("Regeneration task skipped.") - - bot_run(pre_tasks, ctx_obj) + bot_run(pre_tasks, kwargs) @click.group(name="autosync", help="Autosync operations") -@click.option( - "--working-dir", - help="Working directory wit git repository", - type=click.Path(exists=True), - prompt="Enter path to git repo (workspace directory)", - default=".", -) # TODO: use path in config -@click.option( - "--markdown-path", - help="Path to Trestle markdown files", - type=click.Path(exists=True), - prompt="Enter path to to Trestle markdown files", -) -@click.option( - "--skip-items", - help="Comma-separated list of glob patterns to skip when running tasks", - type=str, -) -@click.option( - "--skip-assemble", - help="Skip assembly task", - is_flag=True, - default=False, - show_default=True, -) -@click.option( - "--skip-regenerate", - help="Skip regenerate task", - is_flag=True, - default=False, - show_default=True, -) -@click.option( - "--version", - help="Version of the OSCAL model to set during assembly into JSON", - type=str, -) -@click.option( - "--dry-run", - help="Run tasks, but do not push to the repository", - is_flag=True, -) -@git_options @click.pass_context -def autosync_cmd( - ctx: click.Context, - working_dir: str, - markdown_path: str, - dry_run: bool, - skip_items: str, - skip_assemble: bool, - skip_regenerate: bool, - file_patterns: str, - branch: str, - commit_message: str, - committer_name: str, - committer_email: str, - author_name: str, - author_email: str, - version: str, -) -> None: +def autosync_cmd(ctx: click.Context) -> None: """Command to autosync catalog, profile, compdef and ssp.""" - ctx.obj = dict( - working_dir=working_dir, - markdown_path=markdown_path, - skip_items=skip_items, - skip_assemble=skip_assemble, - skip_regenerate=skip_regenerate, - version=version, - patterns=comma_sep_to_list(file_patterns), - branch=branch, - commit_message=commit_message, - committer_name=committer_name, - committer_email=committer_email, - author_name=author_name, - author_email=author_email, - dry_run=dry_run, - ) - @autosync_cmd.command("ssp") +@click.pass_context +@common_options +@autosync_options +@git_options @click.option( "--ssp-index-path", help="Path to ssp index file", - type=click.File("r"), + type=str, + required=True, ) -@click.pass_context -def autosync_ssp_cmd(ctx: click.Context, ssp_index_path: str) -> None: - if not ssp_index_path: - ssp_index_path = click.prompt( - "Enter path to ssp index file", - type=click.Path(exists=True), - ) - ctx.obj.update( - { - "ssp_index_path": ssp_index_path, - } - ) - run("ssp", ctx.obj) +def autosync_ssp_cmd(ctx: click.Context, ssp_index_path: str, **kwargs: Any) -> None: + kwargs.update({"ssp_index_path": ssp_index_path}) + run("ssp", **kwargs) @autosync_cmd.command("compdef") @click.pass_context -def autosync_compdef_cmd(ctx: click.Context) -> None: - run("compdef", ctx.obj) +@common_options +@autosync_options +@git_options +def autosync_compdef_cmd(ctx: click.Context, **kwargs: Any) -> None: + run("compdef", **kwargs) @autosync_cmd.command("catalog") @click.pass_context -def autosync_catalog_cmd(ctx: click.Context) -> None: - run("catalog", ctx.obj) +@common_options +@autosync_options +@git_options +def autosync_catalog_cmd(ctx: click.Context, **kwargs: Any) -> None: + run("catalog", **kwargs) @autosync_cmd.command("profile") @click.pass_context -def autosync_profile_cmd(ctx: click.Context) -> None: - run("profile", ctx.obj) +@common_options +@autosync_options +@git_options +def autosync_profile_cmd(ctx: click.Context, **kwargs: Any) -> None: + run("profile", **kwargs) diff --git a/trestlebot/cli/options/autosync.py b/trestlebot/cli/options/autosync.py new file mode 100644 index 00000000..d2a41eb8 --- /dev/null +++ b/trestlebot/cli/options/autosync.py @@ -0,0 +1,49 @@ +from typing import Any, Callable, TypeVar + +import click + + +F = TypeVar("F", bound=Callable[..., Any]) + + +def autosync_options(f: F) -> F: + """ + Configuring autosync options decorator for autosync operations + """ + + f = click.option( + "--markdown-path", + help="Path to Trestle markdown files", + type=click.Path(exists=True), + required=True, + )(f) + f = click.option( + "--skip-items", + help="Comma-separated list of glob patterns to skip when running tasks", + type=str, + )(f) + f = click.option( + "--skip-assemble", + help="Skip assembly task", + is_flag=True, + default=False, + show_default=True, + )(f) + f = click.option( + "--skip-regenerate", + help="Skip regenerate task", + is_flag=True, + default=False, + show_default=True, + )(f) + f = click.option( + "--version", + help="Version of the OSCAL model to set during assembly into JSON", + type=str, + )(f) + f = click.option( + "--dry-run", + help="Run tasks, but do not push to the repository", + is_flag=True, + )(f) + return f From c4a90a8682b899a3beae1b29f263569d9d3b281a Mon Sep 17 00:00:00 2001 From: Qingmin Duanmu Date: Mon, 25 Nov 2024 21:13:42 +0800 Subject: [PATCH 025/108] Update autosync options and add tests --- tests/trestlebot/cli/test_autosync_cmd.py | 77 +++++++++++ trestlebot/cli/commands/autosync.py | 152 ++++++---------------- trestlebot/cli/options/autosync.py | 49 +++++++ 3 files changed, 167 insertions(+), 111 deletions(-) create mode 100644 tests/trestlebot/cli/test_autosync_cmd.py create mode 100644 trestlebot/cli/options/autosync.py diff --git a/tests/trestlebot/cli/test_autosync_cmd.py b/tests/trestlebot/cli/test_autosync_cmd.py new file mode 100644 index 00000000..d044bee3 --- /dev/null +++ b/tests/trestlebot/cli/test_autosync_cmd.py @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Red Hat, Inc. + + +"""Testing module for trestlebot autosync command""" +import tempfile + +from click.testing import CliRunner + +from trestlebot.cli.commands.autosync import autosync_cmd + + +def test_invalid_autosync_command(tmp_init_dir: str) -> None: + tempdir = tempfile.mkdtemp() + runner = CliRunner() + result = runner.invoke( + autosync_cmd, + [ + "invalid", + "--repo-path", + tmp_init_dir, + "--markdown-path", + tempdir, + "--branch", + "main", + "--committer-name", + "Test User", + "--committer-email", + "test@example.com", + ], + ) + assert "Error: No such command" in result.output + assert result.exit_code == 2 + + +def test_no_ssp_index_path(tmp_init_dir: str) -> None: + """Test invalid ssp index file for autosync ssp""" + + tempdir = tempfile.mkdtemp() + runner = CliRunner() + cmd_options = [ + "ssp", + "--repo-path", + tmp_init_dir, + "--markdown-path", + tempdir, + "--branch", + "main", + "--committer-name", + "Test User", + "--committer-email", + "test@example.com", + ] + result = runner.invoke(autosync_cmd, cmd_options) + assert result.exit_code == 2 + assert "Error: Missing option '--ssp-index-path'" in result.output + cmd_options[0] = "compdef" + result = runner.invoke(autosync_cmd, cmd_options) + assert result.exit_code == 0 + + +def test_no_markdown_path(tmp_init_dir: str) -> None: + runner = CliRunner() + cmd_options = [ + "compdef", + "--repo-path", + tmp_init_dir, + "--branch", + "main", + "--committer-name", + "Test User", + "--committer-email", + "test@example.com", + ] + result = runner.invoke(autosync_cmd, cmd_options) + assert result.exit_code == 2 + assert "Error: Missing option '--markdown-path'" in result.output diff --git a/trestlebot/cli/commands/autosync.py b/trestlebot/cli/commands/autosync.py index e9576c7f..2d3eaa0e 100644 --- a/trestlebot/cli/commands/autosync.py +++ b/trestlebot/cli/commands/autosync.py @@ -1,11 +1,12 @@ """ Autosync command""" import logging -from typing import Any, Dict, List +from typing import Any, List import click -from trestlebot.cli.options.common import git_options, handle_exceptions +from trestlebot.cli.options.autosync import autosync_options +from trestlebot.cli.options.common import common_options, git_options, handle_exceptions from trestlebot.cli.run import comma_sep_to_list from trestlebot.cli.run import run as bot_run from trestlebot.tasks.assemble_task import AssembleTask @@ -19,163 +20,92 @@ @handle_exceptions -def run(oscal_model: str, ctx_obj: Dict[str, Any]) -> None: +def run(oscal_model: str, **kwargs: Any) -> None: """Run the autosync for oscal model.""" pre_tasks: List[TaskBase] = [] - # Allow any model to be skipped from the args, by default include all + kwargs["working_dir"] = str(kwargs["repo_path"].resolve()) + if kwargs.get("file_patterns"): + kwargs.update({"patterns": comma_sep_to_list(kwargs["file_patterns"])}) model_filter: ModelFilter = ModelFilter( - skip_patterns=comma_sep_to_list(ctx_obj.get("skip_items", "")), + skip_patterns=comma_sep_to_list(kwargs.get("skip_items", "")), include_patterns=["*"], ) authored_object: AuthoredObjectBase = types.get_authored_object( oscal_model, - ctx_obj["working_dir"], - ctx_obj.get("ssp_index_path", ""), + kwargs["working_dir"], + kwargs.get("ssp_index_path", ""), ) # Assuming an edit has occurred assemble would be run before regenerate. # Adding this to the list first - if not ctx_obj.get("skip_assemble"): + if not kwargs.get("skip_assemble"): assemble_task: AssembleTask = AssembleTask( authored_object=authored_object, - markdown_dir=ctx_obj["markdown_path"], - version=ctx_obj.get("version", ""), + markdown_dir=kwargs["markdown_path"], + version=kwargs.get("version", ""), model_filter=model_filter, ) pre_tasks.append(assemble_task) else: logger.info("Assemble task skipped.") - if not ctx_obj.get("skip_regenerate"): + if not kwargs.get("skip_regenerate"): regenerate_task: RegenerateTask = RegenerateTask( authored_object=authored_object, - markdown_dir=ctx_obj["markdown_path"], + markdown_dir=kwargs["markdown_path"], model_filter=model_filter, ) pre_tasks.append(regenerate_task) else: logger.info("Regeneration task skipped.") - - bot_run(pre_tasks, ctx_obj) + bot_run(pre_tasks, kwargs) @click.group(name="autosync", help="Autosync operations") -@click.option( - "--working-dir", - help="Working directory wit git repository", - type=click.Path(exists=True), - prompt="Enter path to git repo (workspace directory)", - default=".", -) # TODO: use path in config -@click.option( - "--markdown-path", - help="Path to Trestle markdown files", - type=click.Path(exists=True), - prompt="Enter path to to Trestle markdown files", -) -@click.option( - "--skip-items", - help="Comma-separated list of glob patterns to skip when running tasks", - type=str, -) -@click.option( - "--skip-assemble", - help="Skip assembly task", - is_flag=True, - default=False, - show_default=True, -) -@click.option( - "--skip-regenerate", - help="Skip regenerate task", - is_flag=True, - default=False, - show_default=True, -) -@click.option( - "--version", - help="Version of the OSCAL model to set during assembly into JSON", - type=str, -) -@click.option( - "--dry-run", - help="Run tasks, but do not push to the repository", - is_flag=True, -) -@git_options @click.pass_context -def autosync_cmd( - ctx: click.Context, - working_dir: str, - markdown_path: str, - dry_run: bool, - skip_items: str, - skip_assemble: bool, - skip_regenerate: bool, - file_patterns: str, - branch: str, - commit_message: str, - committer_name: str, - committer_email: str, - author_name: str, - author_email: str, - version: str, -) -> None: +def autosync_cmd(ctx: click.Context) -> None: """Command to autosync catalog, profile, compdef and ssp.""" - ctx.obj = dict( - working_dir=working_dir, - markdown_path=markdown_path, - skip_items=skip_items, - skip_assemble=skip_assemble, - skip_regenerate=skip_regenerate, - version=version, - patterns=comma_sep_to_list(file_patterns), - branch=branch, - commit_message=commit_message, - committer_name=committer_name, - committer_email=committer_email, - author_name=author_name, - author_email=author_email, - dry_run=dry_run, - ) - @autosync_cmd.command("ssp") +@click.pass_context +@common_options +@autosync_options +@git_options @click.option( "--ssp-index-path", help="Path to ssp index file", - type=click.File("r"), + type=str, + required=True, ) -@click.pass_context -def autosync_ssp_cmd(ctx: click.Context, ssp_index_path: str) -> None: - if not ssp_index_path: - ssp_index_path = click.prompt( - "Enter path to ssp index file", - type=click.Path(exists=True), - ) - ctx.obj.update( - { - "ssp_index_path": ssp_index_path, - } - ) - run("ssp", ctx.obj) +def autosync_ssp_cmd(ctx: click.Context, ssp_index_path: str, **kwargs: Any) -> None: + kwargs.update({"ssp_index_path": ssp_index_path}) + run("ssp", **kwargs) @autosync_cmd.command("compdef") @click.pass_context -def autosync_compdef_cmd(ctx: click.Context) -> None: - run("compdef", ctx.obj) +@common_options +@autosync_options +@git_options +def autosync_compdef_cmd(ctx: click.Context, **kwargs: Any) -> None: + run("compdef", **kwargs) @autosync_cmd.command("catalog") @click.pass_context -def autosync_catalog_cmd(ctx: click.Context) -> None: - run("catalog", ctx.obj) +@common_options +@autosync_options +@git_options +def autosync_catalog_cmd(ctx: click.Context, **kwargs: Any) -> None: + run("catalog", **kwargs) @autosync_cmd.command("profile") @click.pass_context -def autosync_profile_cmd(ctx: click.Context) -> None: - run("profile", ctx.obj) +@common_options +@autosync_options +@git_options +def autosync_profile_cmd(ctx: click.Context, **kwargs: Any) -> None: + run("profile", **kwargs) diff --git a/trestlebot/cli/options/autosync.py b/trestlebot/cli/options/autosync.py new file mode 100644 index 00000000..d2a41eb8 --- /dev/null +++ b/trestlebot/cli/options/autosync.py @@ -0,0 +1,49 @@ +from typing import Any, Callable, TypeVar + +import click + + +F = TypeVar("F", bound=Callable[..., Any]) + + +def autosync_options(f: F) -> F: + """ + Configuring autosync options decorator for autosync operations + """ + + f = click.option( + "--markdown-path", + help="Path to Trestle markdown files", + type=click.Path(exists=True), + required=True, + )(f) + f = click.option( + "--skip-items", + help="Comma-separated list of glob patterns to skip when running tasks", + type=str, + )(f) + f = click.option( + "--skip-assemble", + help="Skip assembly task", + is_flag=True, + default=False, + show_default=True, + )(f) + f = click.option( + "--skip-regenerate", + help="Skip regenerate task", + is_flag=True, + default=False, + show_default=True, + )(f) + f = click.option( + "--version", + help="Version of the OSCAL model to set during assembly into JSON", + type=str, + )(f) + f = click.option( + "--dry-run", + help="Run tasks, but do not push to the repository", + is_flag=True, + )(f) + return f From 3725e7eda484ebaaa127fd915a16272b45dfc57c Mon Sep 17 00:00:00 2001 From: George Vauter Date: Tue, 26 Nov 2024 13:19:57 -0500 Subject: [PATCH 026/108] docs: adr-001 cli implementation (#347) * docs: adding draft of CLI decision record * docs: adding details around config file * docs: refactor wording for clarity * docs: update config example * expand content for default behaviors around oscal-model --- .../decisions/implement-cli-framework_001.md | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 docs/architecture/decisions/implement-cli-framework_001.md diff --git a/docs/architecture/decisions/implement-cli-framework_001.md b/docs/architecture/decisions/implement-cli-framework_001.md new file mode 100644 index 00000000..b404ea80 --- /dev/null +++ b/docs/architecture/decisions/implement-cli-framework_001.md @@ -0,0 +1,61 @@ +--- +x-trestle-template-version: 0.0.1 +title: Implement CLI Framework +status: accepted +--- + +# ADR 001 - Implement CLI Framework + +## Context + + +The primary motivation for this ADR is to enhance the user experience by implementing a more robust CLI framework within the trestlebot codebase. This will address the requirements of [Issue #295](https://github.com/RedHatProductSecurity/trestle-bot/issues/295) and [Issue #342](https://github.com/RedHatProductSecurity/trestle-bot/issues/342) and enable future development of more complex CLI scenarios. Currently entrypoints leverage the argparse library as the core CLI framework. However advanced patterns such as command chaining, subcommands, and dependencies between arguments can be difficult to implement. Moving to the [Click](https://click.palletsprojects.com/en/5.x/) CLI framework will address these challenges and support more complex requirements in the future. In addition, Click will provide a universal command syntax that can be used in the Python CLI app and container execution. + +This ADR also outlines the adoption of environment variables and a configuration file within the CLI. These will provides alternatives methods of passing arguments to the CLI beyond just command flags. This provides users with flexibility in how they pass arguments to the CLI and creates a more static option for arguments that tend to remain unchanged between command executions. + + +## Decision + +The trestlebot module will be refactored to remove the use of `argparse` in favor of Click as the CLI framework. The code contained in `entrypoints` will be converted into Click commands under the `trestlebot` CLI application. A new `cli.py` module will be created as the main entrypoint. + +In addition, support will be added for using a configuration file and environment variables as CLI inputs. The CLI will prioritize arguments passed as command flags. If no argument is passed, the CLI will check for an environment variable. Finally, if no enviroment variable is found, it will look to the configuration file. Click natively supports loading command arguments from environment variables, including a constant prefix. All environment variables will have a `TRESTLEBOT_` prefix. + +The configuration file will be broken into two primary categories, `global` and `model specific`. Global configuration will apply across all models and include values such as git provider, markdown directories, etc. Model specific configuration will apply to the given OSCAL model only. While it is expected that most repos will be used for authoring a single OSCAL model, the possiblity of authoring more than one model would be supported. + +The configuration file would be initialized at a default location during the `trestlebot init` command. Manual creation and editing is also possible. The path to the configuration file can be passed using the `--config | -c` flag. This would not be required if using the default file location. + +Default behaviors: +- the default configuration file location will be `.trestlebot/config.yaml` +- if a command only supports a single OSCAL model then `--oscal-model` will default to that value. (ex: `rules-transform` only supports compdef) +- if the config file only contains a single OSCAL model then that will be used as the default value for `--oscal-model` + +#### Example config: + +```yaml +--- +version: 1 +working-dir: "." +upstream-sources: [] +ssp-index-path: ssp-index.json +git-provider-type: github +git-provider-url: github.com +git-committer-name: "Foo Bar" +git-committer-email: foo@bar.com +models: + # we could allow for multiple or keep this as one + - oscal-model: ssp + markdown-path: markdown/system-security-plans + skip-items: [...] + skip-assemble: true + - oscal-model: compdef + markdown-path: markdown/component-definitions + skip-items: [...] + skip-assemble: true +``` + + +## Consequences + +- The existing command syntax will be updated to evolve from a set of independent entrypoint commants to a unified `trestlebot` CLI with multiple "subcommands". For example, `trestlebot-autosync ` becomes `trestlebot autosync `. +- The container entrypoints will be collapsed into a single entrypoint leveraging the Click CLI application. +- CLI command arguments will be passed via flags, environment variables, or configuration file. From ed11e85063d61c8b09d54ce3be7c68992549a133 Mon Sep 17 00:00:00 2001 From: George Vauter Date: Tue, 26 Nov 2024 13:25:45 -0500 Subject: [PATCH 027/108] feat: add logic to make_config for nested upstream model and update related tests --- tests/trestlebot/cli/test_config.py | 12 ++++++++++-- trestlebot/cli/commands/upstream.py | 7 +++++-- trestlebot/cli/config.py | 18 ++++++++++++++---- 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/tests/trestlebot/cli/test_config.py b/tests/trestlebot/cli/test_config.py index 8d8c21b1..2b303471 100644 --- a/tests/trestlebot/cli/test_config.py +++ b/tests/trestlebot/cli/test_config.py @@ -36,10 +36,18 @@ def test_invalid_config_raises_errors() -> None: def test_make_config_raises_no_errors(tmp_init_dir: str) -> None: """Test create a valid config object.""" - config = make_config(dict(repo_path=tmp_init_dir, markdown_dir="markdown-test")) + values = { + "repo_path": tmp_init_dir, + "markdown_dir": "markdown", + "upstreams": [{"url": "https://test@main", "skip_validation": True}], + } + config = make_config(values) assert isinstance(config, TrestleBotConfig) + assert len(config.upstreams) == 1 + assert config.upstreams[0].url == "https://test@main" + assert config.upstreams[0].skip_validation is True assert config.repo_path == pathlib.Path(tmp_init_dir) - assert config.markdown_dir == "markdown-test" + assert config.markdown_dir == "markdown" def test_config_write_to_file(config_obj: TrestleBotConfig, tmp_init_dir: str) -> None: diff --git a/trestlebot/cli/commands/upstream.py b/trestlebot/cli/commands/upstream.py index 7ed5a364..c023bbda 100644 --- a/trestlebot/cli/commands/upstream.py +++ b/trestlebot/cli/commands/upstream.py @@ -13,6 +13,7 @@ TrestleBotConfig, UpstreamConfig, load_from_file, + make_config, write_to_file, ) from trestlebot.cli.options.common import common_options, git_options, handle_exceptions @@ -122,7 +123,7 @@ def upstream_add_cmd(ctx: click.Context, **kwargs: Any) -> None: logger.warning( f"No trestlebot config file found, creating {str(config_path.resolve())}" ) - config = TrestleBotConfig() + config = make_config() for url in url_list: existing_urls = [upstream.url for upstream in config.upstreams] @@ -134,7 +135,9 @@ def upstream_add_cmd(ctx: click.Context, **kwargs: Any) -> None: include_models = list(kwargs.get("include_model", ["*"])) if len(include_models) == 0: - include_models = ["*"] + include_models = [ + "*" + ] # This needs to be set otherwise the task will not include any models exclude_models = list(kwargs.get("exclude_model", [])) skip_validation = kwargs.get("skip_validation", False) result = sync_upstream_for_url( diff --git a/trestlebot/cli/config.py b/trestlebot/cli/config.py index accf7e9d..bb01f9c2 100644 --- a/trestlebot/cli/config.py +++ b/trestlebot/cli/config.py @@ -63,17 +63,22 @@ class TrestleBotConfig(BaseModel): upstreams: List[UpstreamConfig] = [] def to_yaml_dict(self) -> Dict[str, Any]: - """Returns a dict that can be cleanly loaded into a yaml file. + """Returns a dict that can be cleanly written to a yaml file. This custom model serializer provides a cleaner dict that can be stored as a YAML file. For example, we want to omit empty values - from being written to the YAML config file. + from being written to the YAML config file, or we want paths to be + written as strings, not posix path objects. Ex: instead of `ssp_index_file: None` appearing in the YAML, we just want to exclude it from the config file all together. This produces a YAML config file that only includes values that have been set (or have a default we want to include). + + Values listed in IGNORED_VALUES will be skipped. + """ + IGNORED_VALUES: List[Any] = [None, "None", []] upstreams = [] for upstream in self.upstreams: @@ -95,8 +100,10 @@ def to_yaml_dict(self) -> Dict[str, Any]: "committer_email": str(self.committer_email), "upstreams": upstreams, } + + # Filter out emtpy values to prevent them from appearing in the config return dict( - filter(lambda item: item[1] not in (None, "None", []), config_dict.items()) + filter(lambda item: item[1] not in IGNORED_VALUES, config_dict.items()) ) @@ -126,7 +133,10 @@ def write_to_file(config: TrestleBotConfig, file_path: Path) -> None: def make_config(values: Optional[Dict[str, Any]] = None) -> TrestleBotConfig: """Generates a new trestle-bot config object""" try: - return TrestleBotConfig(**values) if values else TrestleBotConfig() + if values: + return TrestleBotConfig.model_validate(values) + else: + return TrestleBotConfig() except ValidationError as ex: raise TrestleBotConfigError(ex.errors()) From 8f34874dbf614a0e72539d65c4c0460b496169ca Mon Sep 17 00:00:00 2001 From: George Vauter Date: Tue, 26 Nov 2024 13:25:45 -0500 Subject: [PATCH 028/108] feat: add logic to make_config for nested upstream model and update related tests --- tests/trestlebot/cli/test_config.py | 12 ++++++++++-- trestlebot/cli/commands/upstream.py | 7 +++++-- trestlebot/cli/config.py | 18 ++++++++++++++---- 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/tests/trestlebot/cli/test_config.py b/tests/trestlebot/cli/test_config.py index 8d8c21b1..2b303471 100644 --- a/tests/trestlebot/cli/test_config.py +++ b/tests/trestlebot/cli/test_config.py @@ -36,10 +36,18 @@ def test_invalid_config_raises_errors() -> None: def test_make_config_raises_no_errors(tmp_init_dir: str) -> None: """Test create a valid config object.""" - config = make_config(dict(repo_path=tmp_init_dir, markdown_dir="markdown-test")) + values = { + "repo_path": tmp_init_dir, + "markdown_dir": "markdown", + "upstreams": [{"url": "https://test@main", "skip_validation": True}], + } + config = make_config(values) assert isinstance(config, TrestleBotConfig) + assert len(config.upstreams) == 1 + assert config.upstreams[0].url == "https://test@main" + assert config.upstreams[0].skip_validation is True assert config.repo_path == pathlib.Path(tmp_init_dir) - assert config.markdown_dir == "markdown-test" + assert config.markdown_dir == "markdown" def test_config_write_to_file(config_obj: TrestleBotConfig, tmp_init_dir: str) -> None: diff --git a/trestlebot/cli/commands/upstream.py b/trestlebot/cli/commands/upstream.py index 7ed5a364..c023bbda 100644 --- a/trestlebot/cli/commands/upstream.py +++ b/trestlebot/cli/commands/upstream.py @@ -13,6 +13,7 @@ TrestleBotConfig, UpstreamConfig, load_from_file, + make_config, write_to_file, ) from trestlebot.cli.options.common import common_options, git_options, handle_exceptions @@ -122,7 +123,7 @@ def upstream_add_cmd(ctx: click.Context, **kwargs: Any) -> None: logger.warning( f"No trestlebot config file found, creating {str(config_path.resolve())}" ) - config = TrestleBotConfig() + config = make_config() for url in url_list: existing_urls = [upstream.url for upstream in config.upstreams] @@ -134,7 +135,9 @@ def upstream_add_cmd(ctx: click.Context, **kwargs: Any) -> None: include_models = list(kwargs.get("include_model", ["*"])) if len(include_models) == 0: - include_models = ["*"] + include_models = [ + "*" + ] # This needs to be set otherwise the task will not include any models exclude_models = list(kwargs.get("exclude_model", [])) skip_validation = kwargs.get("skip_validation", False) result = sync_upstream_for_url( diff --git a/trestlebot/cli/config.py b/trestlebot/cli/config.py index accf7e9d..bb01f9c2 100644 --- a/trestlebot/cli/config.py +++ b/trestlebot/cli/config.py @@ -63,17 +63,22 @@ class TrestleBotConfig(BaseModel): upstreams: List[UpstreamConfig] = [] def to_yaml_dict(self) -> Dict[str, Any]: - """Returns a dict that can be cleanly loaded into a yaml file. + """Returns a dict that can be cleanly written to a yaml file. This custom model serializer provides a cleaner dict that can be stored as a YAML file. For example, we want to omit empty values - from being written to the YAML config file. + from being written to the YAML config file, or we want paths to be + written as strings, not posix path objects. Ex: instead of `ssp_index_file: None` appearing in the YAML, we just want to exclude it from the config file all together. This produces a YAML config file that only includes values that have been set (or have a default we want to include). + + Values listed in IGNORED_VALUES will be skipped. + """ + IGNORED_VALUES: List[Any] = [None, "None", []] upstreams = [] for upstream in self.upstreams: @@ -95,8 +100,10 @@ def to_yaml_dict(self) -> Dict[str, Any]: "committer_email": str(self.committer_email), "upstreams": upstreams, } + + # Filter out emtpy values to prevent them from appearing in the config return dict( - filter(lambda item: item[1] not in (None, "None", []), config_dict.items()) + filter(lambda item: item[1] not in IGNORED_VALUES, config_dict.items()) ) @@ -126,7 +133,10 @@ def write_to_file(config: TrestleBotConfig, file_path: Path) -> None: def make_config(values: Optional[Dict[str, Any]] = None) -> TrestleBotConfig: """Generates a new trestle-bot config object""" try: - return TrestleBotConfig(**values) if values else TrestleBotConfig() + if values: + return TrestleBotConfig.model_validate(values) + else: + return TrestleBotConfig() except ValidationError as ex: raise TrestleBotConfigError(ex.errors()) From b3ba8deafc9107b20fba7e2e161f302cbf63d4e3 Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Tue, 26 Nov 2024 13:59:33 -0500 Subject: [PATCH 029/108] feat: create command logic for compdef and ssp --- trestlebot/cli/commands/create.py | 119 ++++++++++++++++++++++++++---- 1 file changed, 106 insertions(+), 13 deletions(-) diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index cd9398c3..5b0638bc 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -3,10 +3,24 @@ """ import logging +from typing import Any, List import click +from trestlebot import const +from trestlebot.cli.options.common import handle_exceptions from trestlebot.cli.options.create import common_create_options +from trestlebot.cli.run import run +from trestlebot.tasks.assemble_task import AssembleTask +from trestlebot.tasks.authored.compdef import ( + AuthoredComponentDefinition, + FilterByProfile, +) +from trestlebot.tasks.authored.ssp import AuthoredSSP, SSPIndex +from trestlebot.tasks.base_task import ModelFilter, TaskBase +from trestlebot.tasks.regenerate_task import RegenerateTask +from trestlebot.tasks.rule_transform_task import RuleTransformTask +from trestlebot.transformers.yaml_transformer import ToRulesYAMLTransformer logger = logging.getLogger(__name__) @@ -14,6 +28,7 @@ @click.group(name="create") @common_create_options +@handle_exceptions def create_cmd(ctx: click.Context, profile_name: str) -> None: """ Command leveraged for component definition and ssp authoring in trestlebot. @@ -50,18 +65,62 @@ def create_cmd(ctx: click.Context, profile_name: str) -> None: help="Type of component definition", ) @common_create_options +@handle_exceptions def compdef_cmd( ctx: click.Context, - profile_name: str, - compdef_name: str, - component_title: str, - component_description: str, - filter_by_profile: str, - component_definition_type: str, + **kwargs: Any, ) -> None: """ Component definition authoring command. """ + pre_tasks: List[TaskBase] = [] + + profile_name = kwargs["profile_name"] + compdef_name = kwargs["compdef_name"] + component_title = kwargs["component_title"] + component_description = kwargs["component_description"] + filter_by_profile = kwargs["filter_by_profile"] + component_definition_type = kwargs["component_definition_type"] + repo_path = kwargs["repo_path"] + markdown_dir = kwargs["markdown_dir"] + if filter_by_profile: + filter_by_profile = FilterByProfile(repo_path, filter_by_profile) + + authored_comp: AuthoredComponentDefinition = AuthoredComponentDefinition( + trestle_root=repo_path, + ) + authored_comp.create_new_default( + profile_name=profile_name, + compdef_name=compdef_name, + comp_title=component_title, + comp_description=component_description, + comp_type=component_definition_type, + filter_by_profile=filter_by_profile, + ) + transformer: ToRulesYAMLTransformer = ToRulesYAMLTransformer() + + model_filter: ModelFilter = ModelFilter( + [], [profile_name, component_title, f"{const.RULE_PREFIX}*"] + ) + + rule_transform_task: RuleTransformTask = RuleTransformTask( + working_dir=repo_path, + rules_view_dir=const.RULES_VIEW_DIR, + rule_transformer=transformer, + model_filter=model_filter, + ) + pre_tasks.append(rule_transform_task) + + regenerate_task: RegenerateTask = RegenerateTask( + authored_object=authored_comp, + # trestle_root=repo_path, + markdown_dir=markdown_dir, + model_filter=model_filter, + ) + pre_tasks.append(regenerate_task) + + run(pre_tasks, kwargs) + logger.info( f"The name of the profile in use with the component definition is {profile_name}." ) @@ -98,19 +157,53 @@ def compdef_cmd( help="Optionally set a path to a YAML file for custom SSP Markdown YAML headers.", ) @common_create_options +@handle_exceptions def ssp_cmd( ctx: click.Context, - profile_name: str, - ssp_name: str, - leveraged_ssp: str, - ssp_index_path: str, - yaml_header_path: str, + **kwargs: Any, ) -> None: """ SSP Authoring command """ + + profile_name = kwargs["profile_name"] + ssp_name = kwargs["ssp_name"] + leveraged_ssp = kwargs["leveraged_ssp"] + ssp_index_path = kwargs["ssp_index_path"] + yaml_header_path = kwargs["yaml_header_path"] + repo_path = kwargs["repo_path"] + markdown_dir = kwargs["markdown_dir"] + compdefs = kwargs["None"] + + ssp_index: SSPIndex = SSPIndex(index_path=ssp_index_path) + authored_ssp: AuthoredSSP = AuthoredSSP(trestle_root=repo_path, ssp_index=ssp_index) + + authored_ssp.create_new_default( + profile_name=profile_name, + ssp_name=ssp_name, + compdefs=compdefs, + markdown_path=markdown_dir, + leveraged_ssp=leveraged_ssp, + yaml_header=yaml_header_path, + ) + + # The starting point for SSPs is the markdown, so assemble into JSON. + model_filter: ModelFilter = ModelFilter([], [ssp_name]) + assemble_task: AssembleTask = AssembleTask( + authored_object=authored_ssp, + markdown_dir=markdown_dir, + # version=version, + model_filter=model_filter, + ) + # pre_tasks.append(assemble_task) + + pre_tasks: List[TaskBase] = [assemble_task] + + run(pre_tasks, kwargs) + logger.info(f"The name of the profile in use with the SSP is {profile_name}.") - logger.info(f"The name of the SSP to create is {ssp_name}.") - logger.info(f"The leveraged SSP is {leveraged_ssp}.") logger.info(f"The SSP index path is {ssp_index_path}.") logger.info(f"The YAML file for custom SSP markdown is {yaml_header_path}.") + + logger.debug(f"The leveraged SSP is {leveraged_ssp}.") + logger.debug(f"The name of the SSP to create is {ssp_name}.") From ffae2e8e20945f0b172d1c51c28711f60f82e17d Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Tue, 26 Nov 2024 13:59:33 -0500 Subject: [PATCH 030/108] feat: create command logic for compdef and ssp --- trestlebot/cli/commands/create.py | 119 ++++++++++++++++++++++++++---- 1 file changed, 106 insertions(+), 13 deletions(-) diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index cd9398c3..5b0638bc 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -3,10 +3,24 @@ """ import logging +from typing import Any, List import click +from trestlebot import const +from trestlebot.cli.options.common import handle_exceptions from trestlebot.cli.options.create import common_create_options +from trestlebot.cli.run import run +from trestlebot.tasks.assemble_task import AssembleTask +from trestlebot.tasks.authored.compdef import ( + AuthoredComponentDefinition, + FilterByProfile, +) +from trestlebot.tasks.authored.ssp import AuthoredSSP, SSPIndex +from trestlebot.tasks.base_task import ModelFilter, TaskBase +from trestlebot.tasks.regenerate_task import RegenerateTask +from trestlebot.tasks.rule_transform_task import RuleTransformTask +from trestlebot.transformers.yaml_transformer import ToRulesYAMLTransformer logger = logging.getLogger(__name__) @@ -14,6 +28,7 @@ @click.group(name="create") @common_create_options +@handle_exceptions def create_cmd(ctx: click.Context, profile_name: str) -> None: """ Command leveraged for component definition and ssp authoring in trestlebot. @@ -50,18 +65,62 @@ def create_cmd(ctx: click.Context, profile_name: str) -> None: help="Type of component definition", ) @common_create_options +@handle_exceptions def compdef_cmd( ctx: click.Context, - profile_name: str, - compdef_name: str, - component_title: str, - component_description: str, - filter_by_profile: str, - component_definition_type: str, + **kwargs: Any, ) -> None: """ Component definition authoring command. """ + pre_tasks: List[TaskBase] = [] + + profile_name = kwargs["profile_name"] + compdef_name = kwargs["compdef_name"] + component_title = kwargs["component_title"] + component_description = kwargs["component_description"] + filter_by_profile = kwargs["filter_by_profile"] + component_definition_type = kwargs["component_definition_type"] + repo_path = kwargs["repo_path"] + markdown_dir = kwargs["markdown_dir"] + if filter_by_profile: + filter_by_profile = FilterByProfile(repo_path, filter_by_profile) + + authored_comp: AuthoredComponentDefinition = AuthoredComponentDefinition( + trestle_root=repo_path, + ) + authored_comp.create_new_default( + profile_name=profile_name, + compdef_name=compdef_name, + comp_title=component_title, + comp_description=component_description, + comp_type=component_definition_type, + filter_by_profile=filter_by_profile, + ) + transformer: ToRulesYAMLTransformer = ToRulesYAMLTransformer() + + model_filter: ModelFilter = ModelFilter( + [], [profile_name, component_title, f"{const.RULE_PREFIX}*"] + ) + + rule_transform_task: RuleTransformTask = RuleTransformTask( + working_dir=repo_path, + rules_view_dir=const.RULES_VIEW_DIR, + rule_transformer=transformer, + model_filter=model_filter, + ) + pre_tasks.append(rule_transform_task) + + regenerate_task: RegenerateTask = RegenerateTask( + authored_object=authored_comp, + # trestle_root=repo_path, + markdown_dir=markdown_dir, + model_filter=model_filter, + ) + pre_tasks.append(regenerate_task) + + run(pre_tasks, kwargs) + logger.info( f"The name of the profile in use with the component definition is {profile_name}." ) @@ -98,19 +157,53 @@ def compdef_cmd( help="Optionally set a path to a YAML file for custom SSP Markdown YAML headers.", ) @common_create_options +@handle_exceptions def ssp_cmd( ctx: click.Context, - profile_name: str, - ssp_name: str, - leveraged_ssp: str, - ssp_index_path: str, - yaml_header_path: str, + **kwargs: Any, ) -> None: """ SSP Authoring command """ + + profile_name = kwargs["profile_name"] + ssp_name = kwargs["ssp_name"] + leveraged_ssp = kwargs["leveraged_ssp"] + ssp_index_path = kwargs["ssp_index_path"] + yaml_header_path = kwargs["yaml_header_path"] + repo_path = kwargs["repo_path"] + markdown_dir = kwargs["markdown_dir"] + compdefs = kwargs["None"] + + ssp_index: SSPIndex = SSPIndex(index_path=ssp_index_path) + authored_ssp: AuthoredSSP = AuthoredSSP(trestle_root=repo_path, ssp_index=ssp_index) + + authored_ssp.create_new_default( + profile_name=profile_name, + ssp_name=ssp_name, + compdefs=compdefs, + markdown_path=markdown_dir, + leveraged_ssp=leveraged_ssp, + yaml_header=yaml_header_path, + ) + + # The starting point for SSPs is the markdown, so assemble into JSON. + model_filter: ModelFilter = ModelFilter([], [ssp_name]) + assemble_task: AssembleTask = AssembleTask( + authored_object=authored_ssp, + markdown_dir=markdown_dir, + # version=version, + model_filter=model_filter, + ) + # pre_tasks.append(assemble_task) + + pre_tasks: List[TaskBase] = [assemble_task] + + run(pre_tasks, kwargs) + logger.info(f"The name of the profile in use with the SSP is {profile_name}.") - logger.info(f"The name of the SSP to create is {ssp_name}.") - logger.info(f"The leveraged SSP is {leveraged_ssp}.") logger.info(f"The SSP index path is {ssp_index_path}.") logger.info(f"The YAML file for custom SSP markdown is {yaml_header_path}.") + + logger.debug(f"The leveraged SSP is {leveraged_ssp}.") + logger.debug(f"The name of the SSP to create is {ssp_name}.") From 9284ad8f13c9b1df7eac1ed8520b54dcccf2799b Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Tue, 26 Nov 2024 15:56:21 -0500 Subject: [PATCH 031/108] feat: create command updates to prompts and logger messages --- trestlebot/cli/commands/create.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index 5b0638bc..844c483c 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -8,7 +8,7 @@ import click from trestlebot import const -from trestlebot.cli.options.common import handle_exceptions +from trestlebot.cli.options.common import common_options, handle_exceptions from trestlebot.cli.options.create import common_create_options from trestlebot.cli.run import run from trestlebot.tasks.assemble_task import AssembleTask @@ -37,21 +37,20 @@ def create_cmd(ctx: click.Context, profile_name: str) -> None: pass -# @create_cmd.command(name="compdef", help="command for component definition authoring") @create_cmd.command("compdef") @click.option( "--compdef-name", - prompt="Name of component definition is", + prompt="Enter name of component definition", help="Name of component definition.", ) @click.option( "--component-title", - prompt="The name of component title is", + prompt="Enter name of component title", help="Title of initial component.", ) @click.option( "--component-description", - prompt="The description of the initial component is", + prompt="Enter description of the initial component", help="Description of initial component.", ) @click.option( @@ -64,6 +63,7 @@ def create_cmd(ctx: click.Context, profile_name: str) -> None: default="service", help="Type of component definition", ) +@common_options @common_create_options @handle_exceptions def compdef_cmd( @@ -113,7 +113,6 @@ def compdef_cmd( regenerate_task: RegenerateTask = RegenerateTask( authored_object=authored_comp, - # trestle_root=repo_path, markdown_dir=markdown_dir, model_filter=model_filter, ) @@ -121,6 +120,9 @@ def compdef_cmd( run(pre_tasks, kwargs) + for key, value in kwargs.items(): + logger.info(f"{key}: {value}") + logger.info( f"The name of the profile in use with the component definition is {profile_name}." ) @@ -135,11 +137,10 @@ def compdef_cmd( logger.info(f"The component definition type is {component_definition_type}.") -# @create_cmd.command(name="ssp", help="command for ssp authoring") @create_cmd.command("ssp") @click.option( "--ssp-name", - prompt="Name of SSP to create", + prompt="Enter name of SSP to create", help="Name of SSP to create.", ) @click.option( @@ -148,6 +149,7 @@ def compdef_cmd( ) @click.option( "--ssp-index-path", + type=str, default="ssp-index.json", help="Optionally set the path to the SSP index file.", ) @@ -156,6 +158,7 @@ def compdef_cmd( default="ssp-index.json", help="Optionally set a path to a YAML file for custom SSP Markdown YAML headers.", ) +@common_options @common_create_options @handle_exceptions def ssp_cmd( @@ -192,10 +195,8 @@ def ssp_cmd( assemble_task: AssembleTask = AssembleTask( authored_object=authored_ssp, markdown_dir=markdown_dir, - # version=version, model_filter=model_filter, ) - # pre_tasks.append(assemble_task) pre_tasks: List[TaskBase] = [assemble_task] @@ -204,6 +205,5 @@ def ssp_cmd( logger.info(f"The name of the profile in use with the SSP is {profile_name}.") logger.info(f"The SSP index path is {ssp_index_path}.") logger.info(f"The YAML file for custom SSP markdown is {yaml_header_path}.") - - logger.debug(f"The leveraged SSP is {leveraged_ssp}.") + logger.info(f"The leveraged SSP is {leveraged_ssp}.") logger.debug(f"The name of the SSP to create is {ssp_name}.") From 92092a75c6cecb4970b636ff5e6573fbbc1d2dc4 Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Tue, 26 Nov 2024 15:56:21 -0500 Subject: [PATCH 032/108] feat: create command updates to prompts and logger messages --- trestlebot/cli/commands/create.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index 5b0638bc..844c483c 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -8,7 +8,7 @@ import click from trestlebot import const -from trestlebot.cli.options.common import handle_exceptions +from trestlebot.cli.options.common import common_options, handle_exceptions from trestlebot.cli.options.create import common_create_options from trestlebot.cli.run import run from trestlebot.tasks.assemble_task import AssembleTask @@ -37,21 +37,20 @@ def create_cmd(ctx: click.Context, profile_name: str) -> None: pass -# @create_cmd.command(name="compdef", help="command for component definition authoring") @create_cmd.command("compdef") @click.option( "--compdef-name", - prompt="Name of component definition is", + prompt="Enter name of component definition", help="Name of component definition.", ) @click.option( "--component-title", - prompt="The name of component title is", + prompt="Enter name of component title", help="Title of initial component.", ) @click.option( "--component-description", - prompt="The description of the initial component is", + prompt="Enter description of the initial component", help="Description of initial component.", ) @click.option( @@ -64,6 +63,7 @@ def create_cmd(ctx: click.Context, profile_name: str) -> None: default="service", help="Type of component definition", ) +@common_options @common_create_options @handle_exceptions def compdef_cmd( @@ -113,7 +113,6 @@ def compdef_cmd( regenerate_task: RegenerateTask = RegenerateTask( authored_object=authored_comp, - # trestle_root=repo_path, markdown_dir=markdown_dir, model_filter=model_filter, ) @@ -121,6 +120,9 @@ def compdef_cmd( run(pre_tasks, kwargs) + for key, value in kwargs.items(): + logger.info(f"{key}: {value}") + logger.info( f"The name of the profile in use with the component definition is {profile_name}." ) @@ -135,11 +137,10 @@ def compdef_cmd( logger.info(f"The component definition type is {component_definition_type}.") -# @create_cmd.command(name="ssp", help="command for ssp authoring") @create_cmd.command("ssp") @click.option( "--ssp-name", - prompt="Name of SSP to create", + prompt="Enter name of SSP to create", help="Name of SSP to create.", ) @click.option( @@ -148,6 +149,7 @@ def compdef_cmd( ) @click.option( "--ssp-index-path", + type=str, default="ssp-index.json", help="Optionally set the path to the SSP index file.", ) @@ -156,6 +158,7 @@ def compdef_cmd( default="ssp-index.json", help="Optionally set a path to a YAML file for custom SSP Markdown YAML headers.", ) +@common_options @common_create_options @handle_exceptions def ssp_cmd( @@ -192,10 +195,8 @@ def ssp_cmd( assemble_task: AssembleTask = AssembleTask( authored_object=authored_ssp, markdown_dir=markdown_dir, - # version=version, model_filter=model_filter, ) - # pre_tasks.append(assemble_task) pre_tasks: List[TaskBase] = [assemble_task] @@ -204,6 +205,5 @@ def ssp_cmd( logger.info(f"The name of the profile in use with the SSP is {profile_name}.") logger.info(f"The SSP index path is {ssp_index_path}.") logger.info(f"The YAML file for custom SSP markdown is {yaml_header_path}.") - - logger.debug(f"The leveraged SSP is {leveraged_ssp}.") + logger.info(f"The leveraged SSP is {leveraged_ssp}.") logger.debug(f"The name of the SSP to create is {ssp_name}.") From 3c13ad69d2f7465f205e4d1b6099e98be8d83dd2 Mon Sep 17 00:00:00 2001 From: George Vauter Date: Tue, 26 Nov 2024 19:06:14 -0500 Subject: [PATCH 033/108] feat: add default git info to init prompts and config --- trestlebot/cli/commands/init.py | 71 ++++++++++++++++++++++++++++----- trestlebot/cli/config.py | 10 +++-- 2 files changed, 68 insertions(+), 13 deletions(-) diff --git a/trestlebot/cli/commands/init.py b/trestlebot/cli/commands/init.py index da289932..d06f0d50 100644 --- a/trestlebot/cli/commands/init.py +++ b/trestlebot/cli/commands/init.py @@ -6,11 +6,13 @@ """ import argparse import logging +import os import pathlib import sys import click from trestle.common.const import MODEL_DIR_LIST +from trestle.common.file_utils import make_hidden_file from trestle.core.commands.common.return_codes import CmdReturnCodes from trestle.core.commands.init import InitCmd @@ -48,9 +50,22 @@ def call_trestle_init(repo_path: pathlib.Path, debug: bool) -> None: sys.exit(ERROR_EXIT_CODE) +def mkdir_with_hidden_file(file_path: pathlib.Path) -> None: + """Creates empty directory with .keep file""" + file_path.parent.mkdir(exist_ok=True, parents=True) + make_hidden_file(file_path) + + @click.command(name="init", help="Initialize a new trestle-bot repo.") @click.pass_context @common_options +@click.option( + "--repo-path", + type=click.Path(path_type=pathlib.Path, exists=True), + help="Path to Git repository to initialize.", + default=os.getcwd(), + prompt="Enter path to Git repository", +) # override repo-path in common options to force prompt @click.option( "--markdown-dir", type=str, @@ -58,13 +73,40 @@ def call_trestle_init(repo_path: pathlib.Path, debug: bool) -> None: default="markdown/", prompt="Enter path to store markdown files", ) +@click.option( + "--default-committer-name", + type=str, + help="Default user name for Git commits.", + default="", + show_default=False, + prompt="Enter default user name for Git commits (press to skip)", +) +@click.option( + "--default-committer-email", + type=str, + help="Default user email for Git commits.", + default="", + show_default=False, + prompt="Enter default user email for Git commits (press to skip)", +) +@click.option( + "--default-branch", + type=str, + help="Default repo branch to push automated changes.", + default="", + show_default=False, + prompt="Enter default repo branch to push automated changes (press to skip)", +) def init_cmd( ctx: click.Context, debug: bool, config_path: pathlib.Path, + dry_run: bool, repo_path: pathlib.Path, markdown_dir: str, - dry_run: bool, + default_committer_name: str, + default_committer_email: str, + default_branch: str, ) -> None: """Command to initialize a new trestlebot repo""" @@ -86,9 +128,9 @@ def init_cmd( # Create model directories in workspace root list( map( - lambda x: repo_path.joinpath(x) - .joinpath(TRESTLEBOT_KEEP_FILE) - .mkdir(parents=True, exist_ok=True), + lambda d: mkdir_with_hidden_file( + repo_path.joinpath(d).joinpath(TRESTLEBOT_KEEP_FILE) + ), MODEL_DIR_LIST, ) ) @@ -97,10 +139,11 @@ def init_cmd( # Create markdown directories in workspace root list( map( - lambda x: repo_path.joinpath(pathlib.Path(markdown_dir)) - .joinpath(x) - .joinpath(TRESTLEBOT_KEEP_FILE) - .mkdir(parents=True, exist_ok=True), + lambda d: mkdir_with_hidden_file( + repo_path.joinpath(markdown_dir) + .joinpath(d) + .joinpath(TRESTLEBOT_KEEP_FILE) + ), MODEL_DIR_LIST, ) ) @@ -110,7 +153,17 @@ def init_cmd( call_trestle_init(repo_path, debug) # generate and write trestle-bot cofig - config = make_config(dict(repo_path=repo_path, markdown_dir=markdown_dir)) + config_values = dict(repo_path=repo_path, markdown_dir=markdown_dir) + if default_committer_name: + config_values.update(committer_name=default_committer_name) + + if default_committer_email: + config_values.update(committer_email=default_committer_email) + + if default_branch: + config_values.update(branch=default_branch) + + config = make_config(config_values) config_path = trestlebot_dir.joinpath("config.yml") write_to_file(config, config_path) logger.debug(f"trestle-bot config file created at {str(config_path)}") diff --git a/trestlebot/cli/config.py b/trestlebot/cli/config.py index bb01f9c2..1245ffc0 100644 --- a/trestlebot/cli/config.py +++ b/trestlebot/cli/config.py @@ -59,6 +59,7 @@ class TrestleBotConfig(BaseModel): markdown_dir: Optional[str] = None committer_name: Optional[str] = None committer_email: Optional[str] = None + branch: Optional[str] = None ssp_index_file: Optional[str] = None upstreams: List[UpstreamConfig] = [] @@ -94,10 +95,11 @@ def to_yaml_dict(self) -> Dict[str, Any]: config_dict = { "repo_path": str(self.repo_path), - "markdown_dir": str(self.markdown_dir), - "ssp_index_file": str(self.ssp_index_file), - "committer_name": str(self.committer_name), - "committer_email": str(self.committer_email), + "markdown_dir": self.markdown_dir, + "ssp_index_file": self.ssp_index_file, + "committer_name": self.committer_name, + "committer_email": self.committer_email, + "branch": self.branch, "upstreams": upstreams, } From 3a9a7c32b75a7f6a214e87243c2695280079307c Mon Sep 17 00:00:00 2001 From: George Vauter Date: Tue, 26 Nov 2024 19:06:14 -0500 Subject: [PATCH 034/108] feat: add default git info to init prompts and config --- trestlebot/cli/commands/init.py | 71 ++++++++++++++++++++++++++++----- trestlebot/cli/config.py | 10 +++-- 2 files changed, 68 insertions(+), 13 deletions(-) diff --git a/trestlebot/cli/commands/init.py b/trestlebot/cli/commands/init.py index da289932..d06f0d50 100644 --- a/trestlebot/cli/commands/init.py +++ b/trestlebot/cli/commands/init.py @@ -6,11 +6,13 @@ """ import argparse import logging +import os import pathlib import sys import click from trestle.common.const import MODEL_DIR_LIST +from trestle.common.file_utils import make_hidden_file from trestle.core.commands.common.return_codes import CmdReturnCodes from trestle.core.commands.init import InitCmd @@ -48,9 +50,22 @@ def call_trestle_init(repo_path: pathlib.Path, debug: bool) -> None: sys.exit(ERROR_EXIT_CODE) +def mkdir_with_hidden_file(file_path: pathlib.Path) -> None: + """Creates empty directory with .keep file""" + file_path.parent.mkdir(exist_ok=True, parents=True) + make_hidden_file(file_path) + + @click.command(name="init", help="Initialize a new trestle-bot repo.") @click.pass_context @common_options +@click.option( + "--repo-path", + type=click.Path(path_type=pathlib.Path, exists=True), + help="Path to Git repository to initialize.", + default=os.getcwd(), + prompt="Enter path to Git repository", +) # override repo-path in common options to force prompt @click.option( "--markdown-dir", type=str, @@ -58,13 +73,40 @@ def call_trestle_init(repo_path: pathlib.Path, debug: bool) -> None: default="markdown/", prompt="Enter path to store markdown files", ) +@click.option( + "--default-committer-name", + type=str, + help="Default user name for Git commits.", + default="", + show_default=False, + prompt="Enter default user name for Git commits (press to skip)", +) +@click.option( + "--default-committer-email", + type=str, + help="Default user email for Git commits.", + default="", + show_default=False, + prompt="Enter default user email for Git commits (press to skip)", +) +@click.option( + "--default-branch", + type=str, + help="Default repo branch to push automated changes.", + default="", + show_default=False, + prompt="Enter default repo branch to push automated changes (press to skip)", +) def init_cmd( ctx: click.Context, debug: bool, config_path: pathlib.Path, + dry_run: bool, repo_path: pathlib.Path, markdown_dir: str, - dry_run: bool, + default_committer_name: str, + default_committer_email: str, + default_branch: str, ) -> None: """Command to initialize a new trestlebot repo""" @@ -86,9 +128,9 @@ def init_cmd( # Create model directories in workspace root list( map( - lambda x: repo_path.joinpath(x) - .joinpath(TRESTLEBOT_KEEP_FILE) - .mkdir(parents=True, exist_ok=True), + lambda d: mkdir_with_hidden_file( + repo_path.joinpath(d).joinpath(TRESTLEBOT_KEEP_FILE) + ), MODEL_DIR_LIST, ) ) @@ -97,10 +139,11 @@ def init_cmd( # Create markdown directories in workspace root list( map( - lambda x: repo_path.joinpath(pathlib.Path(markdown_dir)) - .joinpath(x) - .joinpath(TRESTLEBOT_KEEP_FILE) - .mkdir(parents=True, exist_ok=True), + lambda d: mkdir_with_hidden_file( + repo_path.joinpath(markdown_dir) + .joinpath(d) + .joinpath(TRESTLEBOT_KEEP_FILE) + ), MODEL_DIR_LIST, ) ) @@ -110,7 +153,17 @@ def init_cmd( call_trestle_init(repo_path, debug) # generate and write trestle-bot cofig - config = make_config(dict(repo_path=repo_path, markdown_dir=markdown_dir)) + config_values = dict(repo_path=repo_path, markdown_dir=markdown_dir) + if default_committer_name: + config_values.update(committer_name=default_committer_name) + + if default_committer_email: + config_values.update(committer_email=default_committer_email) + + if default_branch: + config_values.update(branch=default_branch) + + config = make_config(config_values) config_path = trestlebot_dir.joinpath("config.yml") write_to_file(config, config_path) logger.debug(f"trestle-bot config file created at {str(config_path)}") diff --git a/trestlebot/cli/config.py b/trestlebot/cli/config.py index bb01f9c2..1245ffc0 100644 --- a/trestlebot/cli/config.py +++ b/trestlebot/cli/config.py @@ -59,6 +59,7 @@ class TrestleBotConfig(BaseModel): markdown_dir: Optional[str] = None committer_name: Optional[str] = None committer_email: Optional[str] = None + branch: Optional[str] = None ssp_index_file: Optional[str] = None upstreams: List[UpstreamConfig] = [] @@ -94,10 +95,11 @@ def to_yaml_dict(self) -> Dict[str, Any]: config_dict = { "repo_path": str(self.repo_path), - "markdown_dir": str(self.markdown_dir), - "ssp_index_file": str(self.ssp_index_file), - "committer_name": str(self.committer_name), - "committer_email": str(self.committer_email), + "markdown_dir": self.markdown_dir, + "ssp_index_file": self.ssp_index_file, + "committer_name": self.committer_name, + "committer_email": self.committer_email, + "branch": self.branch, "upstreams": upstreams, } From a4dbaa65aed8f0bbbb121be94c53fa0cec5e7ea6 Mon Sep 17 00:00:00 2001 From: George Vauter Date: Tue, 26 Nov 2024 19:06:39 -0500 Subject: [PATCH 035/108] fix hidden keep file creation --- trestlebot/cli/commands/upstream.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/trestlebot/cli/commands/upstream.py b/trestlebot/cli/commands/upstream.py index c023bbda..0bac09cc 100644 --- a/trestlebot/cli/commands/upstream.py +++ b/trestlebot/cli/commands/upstream.py @@ -78,7 +78,9 @@ def upstream_cmd(ctx: click.Context) -> None: """Sync content from upstream git repositories.""" -@upstream_cmd.command(name="add") +@upstream_cmd.command( + name="add", help="Add new upstream repository and download content." +) @click.pass_context @common_options @git_options @@ -166,7 +168,9 @@ def upstream_add_cmd(ctx: click.Context, **kwargs: Any) -> None: write_to_file(config, config_path) -@upstream_cmd.command(name="sync") +@upstream_cmd.command( + name="sync", help="Sync content from an added upstream repository." +) @click.pass_context @common_options @git_options @@ -230,4 +234,4 @@ def upstream_sync_cmd(ctx: click.Context, **kwargs: Any) -> None: branch=kwargs["branch"], ) logger.debug(f"Bot results for {upstream.url}: {result}") - logger.info(f"Sync from {upstream.url} complete") + logger.info("Sync from upstreams complete") From 7d3b9e33436b396cd1ef2315f206f557e00286f7 Mon Sep 17 00:00:00 2001 From: George Vauter Date: Tue, 26 Nov 2024 19:06:39 -0500 Subject: [PATCH 036/108] fix hidden keep file creation --- trestlebot/cli/commands/upstream.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/trestlebot/cli/commands/upstream.py b/trestlebot/cli/commands/upstream.py index c023bbda..0bac09cc 100644 --- a/trestlebot/cli/commands/upstream.py +++ b/trestlebot/cli/commands/upstream.py @@ -78,7 +78,9 @@ def upstream_cmd(ctx: click.Context) -> None: """Sync content from upstream git repositories.""" -@upstream_cmd.command(name="add") +@upstream_cmd.command( + name="add", help="Add new upstream repository and download content." +) @click.pass_context @common_options @git_options @@ -166,7 +168,9 @@ def upstream_add_cmd(ctx: click.Context, **kwargs: Any) -> None: write_to_file(config, config_path) -@upstream_cmd.command(name="sync") +@upstream_cmd.command( + name="sync", help="Sync content from an added upstream repository." +) @click.pass_context @common_options @git_options @@ -230,4 +234,4 @@ def upstream_sync_cmd(ctx: click.Context, **kwargs: Any) -> None: branch=kwargs["branch"], ) logger.debug(f"Bot results for {upstream.url}: {result}") - logger.info(f"Sync from {upstream.url} complete") + logger.info("Sync from upstreams complete") From 8fe0747302123f1740a0b193b7150418500e7e82 Mon Sep 17 00:00:00 2001 From: Qingmin Duanmu Date: Wed, 27 Nov 2024 15:05:31 +0800 Subject: [PATCH 037/108] Add rule-transform command and unit test --- tests/trestlebot/cli/test_rule_transform.py | 53 +++++++++++++++ trestlebot/cli/commands/rule_transform.py | 72 +++++++++++++++++++++ 2 files changed, 125 insertions(+) create mode 100644 tests/trestlebot/cli/test_rule_transform.py create mode 100644 trestlebot/cli/commands/rule_transform.py diff --git a/tests/trestlebot/cli/test_rule_transform.py b/tests/trestlebot/cli/test_rule_transform.py new file mode 100644 index 00000000..2e75e07d --- /dev/null +++ b/tests/trestlebot/cli/test_rule_transform.py @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Red Hat, Inc. + + +"""Testing module for trestlebot rule-transform command""" + +import pathlib +from typing import Tuple + +from click.testing import CliRunner +from git import Repo + +from tests.testutils import setup_for_compdef, setup_rules_view +from trestlebot.cli.commands.rule_transform import rule_transform_cmd + + +test_comp_name = "test_comp" +test_md = "md_cd" + + +def test_rule_transform(tmp_repo: Tuple[str, Repo]) -> None: + """Test rule transform.""" + repo_path_str, repo = tmp_repo + + repo_path = pathlib.Path(repo_path_str) + + setup_for_compdef(repo_path, test_comp_name, test_md) + setup_rules_view(repo_path, test_comp_name) + + assert not repo_path.joinpath(test_md).exists() + + runner = CliRunner() + result = runner.invoke( + rule_transform_cmd, + [ + "--dry-run", + "--repo-path", + repo_path, + "--markdown-dir", + test_md, + "--branch", + "main", + "--committer-name", + "Test User", + "--committer-email", + "test@example.com", + ], + ) + + assert result.exit_code == 0 + assert repo_path.joinpath(test_md).exists() + commit = next(repo.iter_commits()) + assert len(commit.stats.files) == 16 diff --git a/trestlebot/cli/commands/rule_transform.py b/trestlebot/cli/commands/rule_transform.py new file mode 100644 index 00000000..41519cf3 --- /dev/null +++ b/trestlebot/cli/commands/rule_transform.py @@ -0,0 +1,72 @@ +"""Module for rule-transform command""" + +import logging +from typing import Any, List + +import click + +from trestlebot.cli.options.common import common_options, git_options, handle_exceptions +from trestlebot.cli.run import run +from trestlebot.const import RULES_VIEW_DIR +from trestlebot.tasks.authored.compdef import AuthoredComponentDefinition +from trestlebot.tasks.base_task import ModelFilter, TaskBase +from trestlebot.tasks.regenerate_task import RegenerateTask +from trestlebot.tasks.rule_transform_task import RuleTransformTask +from trestlebot.transformers.yaml_transformer import ToRulesYAMLTransformer + + +logger = logging.getLogger(__name__) + + +@click.command( + name="rule-transform", + help="Transform rules to an OSCAL Component Definition JSON file.", +) +@click.pass_context +@common_options +@git_options +@click.option( + "--markdown-dir", + type=str, + help="Directory name to store markdown files.", +) +@click.option( + "--rules-view-dir", + type=str, + help="Top-level rules-view directory.", + default=RULES_VIEW_DIR, +) +@click.option( + "--skip-item", + type=str, + help="glob pattern for directories to skip when running", + multiple=True, +) +@handle_exceptions +def rule_transform_cmd(ctx: click.Context, **kwargs: Any) -> None: + """Run the rule transform operation.""" + # Allow any model to be skipped by setting skip_items, by default include all + skip_items = list(kwargs.get("skip_item", [])) + model_filter: ModelFilter = ModelFilter( + skip_patterns=skip_items, + include_patterns=["*"], + ) + + transformer = ToRulesYAMLTransformer() + rule_transform_task: RuleTransformTask = RuleTransformTask( + working_dir=kwargs["repo_path"], + rules_view_dir=kwargs["rules_view_dir"], + rule_transformer=transformer, + model_filter=model_filter, + ) + regenerate_task: RegenerateTask = RegenerateTask( + markdown_dir=kwargs["markdown_dir"], + authored_object=AuthoredComponentDefinition(kwargs["repo_path"]), + model_filter=model_filter, + ) + + pre_tasks: List[TaskBase] = [rule_transform_task, regenerate_task] + kwargs["working_dir"] = str(kwargs["repo_path"].resolve()) + result = run(pre_tasks, kwargs) + logger.debug(f"Bot results: {result}") + logger.info("Rule transform complete!") From 4484a567af549d882babbf043aeba70474bdac73 Mon Sep 17 00:00:00 2001 From: Qingmin Duanmu Date: Wed, 27 Nov 2024 15:05:31 +0800 Subject: [PATCH 038/108] Add rule-transform command and unit test --- tests/trestlebot/cli/test_rule_transform.py | 53 +++++++++++++++ trestlebot/cli/commands/rule_transform.py | 72 +++++++++++++++++++++ 2 files changed, 125 insertions(+) create mode 100644 tests/trestlebot/cli/test_rule_transform.py create mode 100644 trestlebot/cli/commands/rule_transform.py diff --git a/tests/trestlebot/cli/test_rule_transform.py b/tests/trestlebot/cli/test_rule_transform.py new file mode 100644 index 00000000..2e75e07d --- /dev/null +++ b/tests/trestlebot/cli/test_rule_transform.py @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Red Hat, Inc. + + +"""Testing module for trestlebot rule-transform command""" + +import pathlib +from typing import Tuple + +from click.testing import CliRunner +from git import Repo + +from tests.testutils import setup_for_compdef, setup_rules_view +from trestlebot.cli.commands.rule_transform import rule_transform_cmd + + +test_comp_name = "test_comp" +test_md = "md_cd" + + +def test_rule_transform(tmp_repo: Tuple[str, Repo]) -> None: + """Test rule transform.""" + repo_path_str, repo = tmp_repo + + repo_path = pathlib.Path(repo_path_str) + + setup_for_compdef(repo_path, test_comp_name, test_md) + setup_rules_view(repo_path, test_comp_name) + + assert not repo_path.joinpath(test_md).exists() + + runner = CliRunner() + result = runner.invoke( + rule_transform_cmd, + [ + "--dry-run", + "--repo-path", + repo_path, + "--markdown-dir", + test_md, + "--branch", + "main", + "--committer-name", + "Test User", + "--committer-email", + "test@example.com", + ], + ) + + assert result.exit_code == 0 + assert repo_path.joinpath(test_md).exists() + commit = next(repo.iter_commits()) + assert len(commit.stats.files) == 16 diff --git a/trestlebot/cli/commands/rule_transform.py b/trestlebot/cli/commands/rule_transform.py new file mode 100644 index 00000000..41519cf3 --- /dev/null +++ b/trestlebot/cli/commands/rule_transform.py @@ -0,0 +1,72 @@ +"""Module for rule-transform command""" + +import logging +from typing import Any, List + +import click + +from trestlebot.cli.options.common import common_options, git_options, handle_exceptions +from trestlebot.cli.run import run +from trestlebot.const import RULES_VIEW_DIR +from trestlebot.tasks.authored.compdef import AuthoredComponentDefinition +from trestlebot.tasks.base_task import ModelFilter, TaskBase +from trestlebot.tasks.regenerate_task import RegenerateTask +from trestlebot.tasks.rule_transform_task import RuleTransformTask +from trestlebot.transformers.yaml_transformer import ToRulesYAMLTransformer + + +logger = logging.getLogger(__name__) + + +@click.command( + name="rule-transform", + help="Transform rules to an OSCAL Component Definition JSON file.", +) +@click.pass_context +@common_options +@git_options +@click.option( + "--markdown-dir", + type=str, + help="Directory name to store markdown files.", +) +@click.option( + "--rules-view-dir", + type=str, + help="Top-level rules-view directory.", + default=RULES_VIEW_DIR, +) +@click.option( + "--skip-item", + type=str, + help="glob pattern for directories to skip when running", + multiple=True, +) +@handle_exceptions +def rule_transform_cmd(ctx: click.Context, **kwargs: Any) -> None: + """Run the rule transform operation.""" + # Allow any model to be skipped by setting skip_items, by default include all + skip_items = list(kwargs.get("skip_item", [])) + model_filter: ModelFilter = ModelFilter( + skip_patterns=skip_items, + include_patterns=["*"], + ) + + transformer = ToRulesYAMLTransformer() + rule_transform_task: RuleTransformTask = RuleTransformTask( + working_dir=kwargs["repo_path"], + rules_view_dir=kwargs["rules_view_dir"], + rule_transformer=transformer, + model_filter=model_filter, + ) + regenerate_task: RegenerateTask = RegenerateTask( + markdown_dir=kwargs["markdown_dir"], + authored_object=AuthoredComponentDefinition(kwargs["repo_path"]), + model_filter=model_filter, + ) + + pre_tasks: List[TaskBase] = [rule_transform_task, regenerate_task] + kwargs["working_dir"] = str(kwargs["repo_path"].resolve()) + result = run(pre_tasks, kwargs) + logger.debug(f"Bot results: {result}") + logger.info("Rule transform complete!") From 74dd43d80e4120178b747378c0cdeb40e7edb1a0 Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Wed, 27 Nov 2024 15:22:55 -0500 Subject: [PATCH 039/108] feat: create command logic and adding unit tests --- tests/trestlebot/cli/test_create_cmd.py | 47 +++++++++++++ trestlebot/cli/commands/create.py | 91 ++++++++++++++++--------- trestlebot/cli/options/create.py | 10 ++- 3 files changed, 113 insertions(+), 35 deletions(-) create mode 100644 tests/trestlebot/cli/test_create_cmd.py diff --git a/tests/trestlebot/cli/test_create_cmd.py b/tests/trestlebot/cli/test_create_cmd.py new file mode 100644 index 00000000..b1f3754a --- /dev/null +++ b/tests/trestlebot/cli/test_create_cmd.py @@ -0,0 +1,47 @@ +""" Unit test for create commands ssp and cd""" + +# import pathlib + +from click.testing import CliRunner + +from trestlebot.cli.commands.create import create_cmd + + +def test_invalid_create_cmd() -> None: + """Tests that create command fails if given invalid oscal model subcommand.""" + runner = CliRunner() + result = runner.invoke(create_cmd, ["invalid"]) + + assert "Error: No such command 'invalid'" in result.output + assert result.exit_code == 2 + + +# def test_create_ssp_cmd(tmp_init_dir: str) -> None: +# """Tests successful create ssp command.""" +# +# SSP_INDEX_FILE = "test-ssp-index.json" +# +# runner = CliRunner() +# result = runner.invoke( +# create_cmd, +# [ +# "ssp", +# "--profile-name", +# "repo-path", +# tmp_init_dir, +# "--ssp-index-file", +# SSP_INDEX_FILE, +# ], +# ) +# assert result.exit_code == 0 +# +# # verify ssp-index.json was created in tmp_init_dir +# tmp_dir = pathlib.Path(tmp_init_dir) +# ssp_index_file = tmp_dir.joinpath(SSP_INDEX_FILE) +# print("SSP INDEX Path", str(ssp_index_file.resolve())) +# +# assert ssp_index_file.exists() is True + +# verity json file was created in tmp_init_dir/system-security-plans + +# verify markdown file(s) were created in tmp_init_dir/markdown/system... diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index 844c483c..8391de8c 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -11,6 +11,7 @@ from trestlebot.cli.options.common import common_options, handle_exceptions from trestlebot.cli.options.create import common_create_options from trestlebot.cli.run import run +from trestlebot.entrypoints.entrypoint_base import comma_sep_to_list from trestlebot.tasks.assemble_task import AssembleTask from trestlebot.tasks.authored.compdef import ( AuthoredComponentDefinition, @@ -27,9 +28,9 @@ @click.group(name="create") -@common_create_options +@click.pass_context @handle_exceptions -def create_cmd(ctx: click.Context, profile_name: str) -> None: +def create_cmd(ctx: click.Context) -> None: """ Command leveraged for component definition and ssp authoring in trestlebot. """ @@ -38,33 +39,40 @@ def create_cmd(ctx: click.Context, profile_name: str) -> None: @create_cmd.command("compdef") +@click.pass_context +@common_create_options +@common_options @click.option( "--compdef-name", + required=True, prompt="Enter name of component definition", help="Name of component definition.", ) @click.option( "--component-title", + required=True, prompt="Enter name of component title", help="Title of initial component.", ) @click.option( "--component-description", + required=True, prompt="Enter description of the initial component", help="Description of initial component.", ) @click.option( "--filter-by-profile", required=False, + type=str, help="Optionally filter the controls in the component definition by a profile", ) @click.option( "--component-definition-type", + required=False, + type=str, default="service", help="Type of component definition", ) -@common_options -@common_create_options @handle_exceptions def compdef_cmd( ctx: click.Context, @@ -97,6 +105,8 @@ def compdef_cmd( comp_type=component_definition_type, filter_by_profile=filter_by_profile, ) + logger.info(f"Component definition name is: {component_title}.") + transformer: ToRulesYAMLTransformer = ToRulesYAMLTransformer() model_filter: ModelFilter = ModelFilter( @@ -109,6 +119,14 @@ def compdef_cmd( rule_transformer=transformer, model_filter=model_filter, ) + logger.info( + f"Profile to filter controls in the component files is: {filter_by_profile}." + ) + logger.debug( + f"Oscal profile in use with the component definition is: {profile_name}." + ) + logger.debug(f"Component definition type is {component_definition_type}.") + pre_tasks.append(rule_transform_task) regenerate_task: RegenerateTask = RegenerateTask( @@ -119,25 +137,13 @@ def compdef_cmd( pre_tasks.append(regenerate_task) run(pre_tasks, kwargs) - - for key, value in kwargs.items(): - logger.info(f"{key}: {value}") - - logger.info( - f"The name of the profile in use with the component definition is {profile_name}." - ) - logger.info( - f"You have selected component definitions as the document you want {compdef_name} to author." - ) - logger.info(f"The component definition name is {component_title}.") - logger.info(f"The component description to author is {component_description}.") - logger.info( - f"The profile you want to filter controls in the component files is {filter_by_profile}." - ) - logger.info(f"The component definition type is {component_definition_type}.") + logger.debug(f"You have successfully authored the the {compdef_name}.") @create_cmd.command("ssp") +@click.pass_context +@common_create_options +@common_options @click.option( "--ssp-name", prompt="Enter name of SSP to create", @@ -145,21 +151,36 @@ def compdef_cmd( ) @click.option( "--leveraged-ssp", + required=False, + type=str, help="Provider SSP to leverage for the new SSP.", ) @click.option( - "--ssp-index-path", + "--ssp-index-file", + required=False, type=str, default="ssp-index.json", help="Optionally set the path to the SSP index file.", ) @click.option( "--yaml-header-path", + required=False, + type=str, default="ssp-index.json", help="Optionally set a path to a YAML file for custom SSP Markdown YAML headers.", ) -@common_options -@common_create_options +@click.option( + "--version", + required=False, + type=str, + help="Optionally set the version of the SSP.", +) +@click.option( + "--compdefs", + required=False, + type=str, + help="Comma separated list of component definitions.", +) @handle_exceptions def ssp_cmd( ctx: click.Context, @@ -172,29 +193,37 @@ def ssp_cmd( profile_name = kwargs["profile_name"] ssp_name = kwargs["ssp_name"] leveraged_ssp = kwargs["leveraged_ssp"] - ssp_index_path = kwargs["ssp_index_path"] + ssp_index_file = kwargs["ssp_index_file"] yaml_header_path = kwargs["yaml_header_path"] repo_path = kwargs["repo_path"] markdown_dir = kwargs["markdown_dir"] - compdefs = kwargs["None"] + compdefs = kwargs["compdefs"] + version = kwargs["version"] - ssp_index: SSPIndex = SSPIndex(index_path=ssp_index_path) + ssp_index: SSPIndex = SSPIndex(index_path=ssp_index_file) authored_ssp: AuthoredSSP = AuthoredSSP(trestle_root=repo_path, ssp_index=ssp_index) + logger.info(f"SSP index file is: {ssp_index_file}.") + + comps: List[str] = comma_sep_to_list(compdefs) authored_ssp.create_new_default( - profile_name=profile_name, ssp_name=ssp_name, - compdefs=compdefs, + profile_name=profile_name, + compdefs=comps, markdown_path=markdown_dir, leveraged_ssp=leveraged_ssp, yaml_header=yaml_header_path, ) + logger.debug(f"The name of the SSP to create is {ssp_name}.") + logger.debug(f"Oscal profile in use with the SSP is: {profile_name}.") + # The starting point for SSPs is the markdown, so assemble into JSON. model_filter: ModelFilter = ModelFilter([], [ssp_name]) assemble_task: AssembleTask = AssembleTask( authored_object=authored_ssp, markdown_dir=markdown_dir, + version=version, model_filter=model_filter, ) @@ -202,8 +231,4 @@ def ssp_cmd( run(pre_tasks, kwargs) - logger.info(f"The name of the profile in use with the SSP is {profile_name}.") - logger.info(f"The SSP index path is {ssp_index_path}.") - logger.info(f"The YAML file for custom SSP markdown is {yaml_header_path}.") - logger.info(f"The leveraged SSP is {leveraged_ssp}.") - logger.debug(f"The name of the SSP to create is {ssp_name}.") + logger.debug(f"You have successfully authored the the {ssp_name}.") diff --git a/trestlebot/cli/options/create.py b/trestlebot/cli/options/create.py index c797d05d..2d84ea47 100644 --- a/trestlebot/cli/options/create.py +++ b/trestlebot/cli/options/create.py @@ -11,17 +11,23 @@ F = TypeVar("F", bound=Callable[..., Any]) -def common_create_options(f: F) -> Any: +def common_create_options(f: F) -> F: """ Configuring common create options decorator for SSP and CD command """ - @click.pass_context @click.option( "--profile-name", prompt="Name of trestle workspace to include", help="Name of profile in trestle workspace to include.", ) + @click.option( + "--markdown-dir", + type=str, + prompt="Enter path to store markdown files", + default="markdown/", + help="Directory name to store markdown files.", + ) @functools.wraps(f) def wrapper_common_create_options( *args: Sequence[Any], **kwargs: Dict[Any, Any] From 17d524a9e0a142c24b9adde4f1ac990a04d9a9f2 Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Wed, 27 Nov 2024 15:22:55 -0500 Subject: [PATCH 040/108] feat: create command logic and adding unit tests --- tests/trestlebot/cli/test_create_cmd.py | 47 +++++++++++++ trestlebot/cli/commands/create.py | 91 ++++++++++++++++--------- trestlebot/cli/options/create.py | 10 ++- 3 files changed, 113 insertions(+), 35 deletions(-) create mode 100644 tests/trestlebot/cli/test_create_cmd.py diff --git a/tests/trestlebot/cli/test_create_cmd.py b/tests/trestlebot/cli/test_create_cmd.py new file mode 100644 index 00000000..b1f3754a --- /dev/null +++ b/tests/trestlebot/cli/test_create_cmd.py @@ -0,0 +1,47 @@ +""" Unit test for create commands ssp and cd""" + +# import pathlib + +from click.testing import CliRunner + +from trestlebot.cli.commands.create import create_cmd + + +def test_invalid_create_cmd() -> None: + """Tests that create command fails if given invalid oscal model subcommand.""" + runner = CliRunner() + result = runner.invoke(create_cmd, ["invalid"]) + + assert "Error: No such command 'invalid'" in result.output + assert result.exit_code == 2 + + +# def test_create_ssp_cmd(tmp_init_dir: str) -> None: +# """Tests successful create ssp command.""" +# +# SSP_INDEX_FILE = "test-ssp-index.json" +# +# runner = CliRunner() +# result = runner.invoke( +# create_cmd, +# [ +# "ssp", +# "--profile-name", +# "repo-path", +# tmp_init_dir, +# "--ssp-index-file", +# SSP_INDEX_FILE, +# ], +# ) +# assert result.exit_code == 0 +# +# # verify ssp-index.json was created in tmp_init_dir +# tmp_dir = pathlib.Path(tmp_init_dir) +# ssp_index_file = tmp_dir.joinpath(SSP_INDEX_FILE) +# print("SSP INDEX Path", str(ssp_index_file.resolve())) +# +# assert ssp_index_file.exists() is True + +# verity json file was created in tmp_init_dir/system-security-plans + +# verify markdown file(s) were created in tmp_init_dir/markdown/system... diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index 844c483c..8391de8c 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -11,6 +11,7 @@ from trestlebot.cli.options.common import common_options, handle_exceptions from trestlebot.cli.options.create import common_create_options from trestlebot.cli.run import run +from trestlebot.entrypoints.entrypoint_base import comma_sep_to_list from trestlebot.tasks.assemble_task import AssembleTask from trestlebot.tasks.authored.compdef import ( AuthoredComponentDefinition, @@ -27,9 +28,9 @@ @click.group(name="create") -@common_create_options +@click.pass_context @handle_exceptions -def create_cmd(ctx: click.Context, profile_name: str) -> None: +def create_cmd(ctx: click.Context) -> None: """ Command leveraged for component definition and ssp authoring in trestlebot. """ @@ -38,33 +39,40 @@ def create_cmd(ctx: click.Context, profile_name: str) -> None: @create_cmd.command("compdef") +@click.pass_context +@common_create_options +@common_options @click.option( "--compdef-name", + required=True, prompt="Enter name of component definition", help="Name of component definition.", ) @click.option( "--component-title", + required=True, prompt="Enter name of component title", help="Title of initial component.", ) @click.option( "--component-description", + required=True, prompt="Enter description of the initial component", help="Description of initial component.", ) @click.option( "--filter-by-profile", required=False, + type=str, help="Optionally filter the controls in the component definition by a profile", ) @click.option( "--component-definition-type", + required=False, + type=str, default="service", help="Type of component definition", ) -@common_options -@common_create_options @handle_exceptions def compdef_cmd( ctx: click.Context, @@ -97,6 +105,8 @@ def compdef_cmd( comp_type=component_definition_type, filter_by_profile=filter_by_profile, ) + logger.info(f"Component definition name is: {component_title}.") + transformer: ToRulesYAMLTransformer = ToRulesYAMLTransformer() model_filter: ModelFilter = ModelFilter( @@ -109,6 +119,14 @@ def compdef_cmd( rule_transformer=transformer, model_filter=model_filter, ) + logger.info( + f"Profile to filter controls in the component files is: {filter_by_profile}." + ) + logger.debug( + f"Oscal profile in use with the component definition is: {profile_name}." + ) + logger.debug(f"Component definition type is {component_definition_type}.") + pre_tasks.append(rule_transform_task) regenerate_task: RegenerateTask = RegenerateTask( @@ -119,25 +137,13 @@ def compdef_cmd( pre_tasks.append(regenerate_task) run(pre_tasks, kwargs) - - for key, value in kwargs.items(): - logger.info(f"{key}: {value}") - - logger.info( - f"The name of the profile in use with the component definition is {profile_name}." - ) - logger.info( - f"You have selected component definitions as the document you want {compdef_name} to author." - ) - logger.info(f"The component definition name is {component_title}.") - logger.info(f"The component description to author is {component_description}.") - logger.info( - f"The profile you want to filter controls in the component files is {filter_by_profile}." - ) - logger.info(f"The component definition type is {component_definition_type}.") + logger.debug(f"You have successfully authored the the {compdef_name}.") @create_cmd.command("ssp") +@click.pass_context +@common_create_options +@common_options @click.option( "--ssp-name", prompt="Enter name of SSP to create", @@ -145,21 +151,36 @@ def compdef_cmd( ) @click.option( "--leveraged-ssp", + required=False, + type=str, help="Provider SSP to leverage for the new SSP.", ) @click.option( - "--ssp-index-path", + "--ssp-index-file", + required=False, type=str, default="ssp-index.json", help="Optionally set the path to the SSP index file.", ) @click.option( "--yaml-header-path", + required=False, + type=str, default="ssp-index.json", help="Optionally set a path to a YAML file for custom SSP Markdown YAML headers.", ) -@common_options -@common_create_options +@click.option( + "--version", + required=False, + type=str, + help="Optionally set the version of the SSP.", +) +@click.option( + "--compdefs", + required=False, + type=str, + help="Comma separated list of component definitions.", +) @handle_exceptions def ssp_cmd( ctx: click.Context, @@ -172,29 +193,37 @@ def ssp_cmd( profile_name = kwargs["profile_name"] ssp_name = kwargs["ssp_name"] leveraged_ssp = kwargs["leveraged_ssp"] - ssp_index_path = kwargs["ssp_index_path"] + ssp_index_file = kwargs["ssp_index_file"] yaml_header_path = kwargs["yaml_header_path"] repo_path = kwargs["repo_path"] markdown_dir = kwargs["markdown_dir"] - compdefs = kwargs["None"] + compdefs = kwargs["compdefs"] + version = kwargs["version"] - ssp_index: SSPIndex = SSPIndex(index_path=ssp_index_path) + ssp_index: SSPIndex = SSPIndex(index_path=ssp_index_file) authored_ssp: AuthoredSSP = AuthoredSSP(trestle_root=repo_path, ssp_index=ssp_index) + logger.info(f"SSP index file is: {ssp_index_file}.") + + comps: List[str] = comma_sep_to_list(compdefs) authored_ssp.create_new_default( - profile_name=profile_name, ssp_name=ssp_name, - compdefs=compdefs, + profile_name=profile_name, + compdefs=comps, markdown_path=markdown_dir, leveraged_ssp=leveraged_ssp, yaml_header=yaml_header_path, ) + logger.debug(f"The name of the SSP to create is {ssp_name}.") + logger.debug(f"Oscal profile in use with the SSP is: {profile_name}.") + # The starting point for SSPs is the markdown, so assemble into JSON. model_filter: ModelFilter = ModelFilter([], [ssp_name]) assemble_task: AssembleTask = AssembleTask( authored_object=authored_ssp, markdown_dir=markdown_dir, + version=version, model_filter=model_filter, ) @@ -202,8 +231,4 @@ def ssp_cmd( run(pre_tasks, kwargs) - logger.info(f"The name of the profile in use with the SSP is {profile_name}.") - logger.info(f"The SSP index path is {ssp_index_path}.") - logger.info(f"The YAML file for custom SSP markdown is {yaml_header_path}.") - logger.info(f"The leveraged SSP is {leveraged_ssp}.") - logger.debug(f"The name of the SSP to create is {ssp_name}.") + logger.debug(f"You have successfully authored the the {ssp_name}.") diff --git a/trestlebot/cli/options/create.py b/trestlebot/cli/options/create.py index c797d05d..2d84ea47 100644 --- a/trestlebot/cli/options/create.py +++ b/trestlebot/cli/options/create.py @@ -11,17 +11,23 @@ F = TypeVar("F", bound=Callable[..., Any]) -def common_create_options(f: F) -> Any: +def common_create_options(f: F) -> F: """ Configuring common create options decorator for SSP and CD command """ - @click.pass_context @click.option( "--profile-name", prompt="Name of trestle workspace to include", help="Name of profile in trestle workspace to include.", ) + @click.option( + "--markdown-dir", + type=str, + prompt="Enter path to store markdown files", + default="markdown/", + help="Directory name to store markdown files.", + ) @functools.wraps(f) def wrapper_common_create_options( *args: Sequence[Any], **kwargs: Dict[Any, Any] From d21f5f6f87c4df0069ab3366968285e3e8b87f46 Mon Sep 17 00:00:00 2001 From: Qingmin Duanmu Date: Fri, 29 Nov 2024 09:51:08 +0800 Subject: [PATCH 041/108] Fix AttributeError, some misc updates AttributeError: 'NoneType' object has no attribute 'encode' --- tests/trestlebot/cli/test_autosync_cmd.py | 27 ++++++++++++------- ...ransform.py => test_rule_transform_cmd.py} | 0 trestlebot/cli/commands/autosync.py | 18 ++++++------- trestlebot/cli/commands/rule_transform.py | 2 +- trestlebot/cli/options/autosync.py | 14 ++++------ trestlebot/cli/options/common.py | 6 +++-- trestlebot/cli/run.py | 2 +- 7 files changed, 36 insertions(+), 33 deletions(-) rename tests/trestlebot/cli/{test_rule_transform.py => test_rule_transform_cmd.py} (100%) diff --git a/tests/trestlebot/cli/test_autosync_cmd.py b/tests/trestlebot/cli/test_autosync_cmd.py index d044bee3..dede4ed9 100644 --- a/tests/trestlebot/cli/test_autosync_cmd.py +++ b/tests/trestlebot/cli/test_autosync_cmd.py @@ -3,15 +3,15 @@ """Testing module for trestlebot autosync command""" -import tempfile +import pathlib from click.testing import CliRunner from trestlebot.cli.commands.autosync import autosync_cmd +from trestlebot.cli.config import TrestleBotConfig, write_to_file def test_invalid_autosync_command(tmp_init_dir: str) -> None: - tempdir = tempfile.mkdtemp() runner = CliRunner() result = runner.invoke( autosync_cmd, @@ -19,8 +19,8 @@ def test_invalid_autosync_command(tmp_init_dir: str) -> None: "invalid", "--repo-path", tmp_init_dir, - "--markdown-path", - tempdir, + "--markdown-dir", + "markdown", "--branch", "main", "--committer-name", @@ -36,14 +36,13 @@ def test_invalid_autosync_command(tmp_init_dir: str) -> None: def test_no_ssp_index_path(tmp_init_dir: str) -> None: """Test invalid ssp index file for autosync ssp""" - tempdir = tempfile.mkdtemp() runner = CliRunner() cmd_options = [ "ssp", "--repo-path", tmp_init_dir, - "--markdown-path", - tempdir, + "--markdown-dir", + "markdown", "--branch", "main", "--committer-name", @@ -53,13 +52,13 @@ def test_no_ssp_index_path(tmp_init_dir: str) -> None: ] result = runner.invoke(autosync_cmd, cmd_options) assert result.exit_code == 2 - assert "Error: Missing option '--ssp-index-path'" in result.output + assert "Error: Missing option '--ssp-index-file'" in result.output cmd_options[0] = "compdef" result = runner.invoke(autosync_cmd, cmd_options) assert result.exit_code == 0 -def test_no_markdown_path(tmp_init_dir: str) -> None: +def test_no_markdown_dir(tmp_init_dir: str) -> None: runner = CliRunner() cmd_options = [ "compdef", @@ -74,4 +73,12 @@ def test_no_markdown_path(tmp_init_dir: str) -> None: ] result = runner.invoke(autosync_cmd, cmd_options) assert result.exit_code == 2 - assert "Error: Missing option '--markdown-path'" in result.output + assert "Error: Missing option '--markdown-dir'" in result.output + + # With 'markdown_dir' setting in config.yml + config_obj = TrestleBotConfig(markdown_dir="markdown") + filepath = pathlib.Path(tmp_init_dir).joinpath("config.yml") + write_to_file(config_obj, filepath) + cmd_options += ["--config", str(filepath)] + result = runner.invoke(autosync_cmd, cmd_options) + assert result.exit_code == 0 diff --git a/tests/trestlebot/cli/test_rule_transform.py b/tests/trestlebot/cli/test_rule_transform_cmd.py similarity index 100% rename from tests/trestlebot/cli/test_rule_transform.py rename to tests/trestlebot/cli/test_rule_transform_cmd.py diff --git a/trestlebot/cli/commands/autosync.py b/trestlebot/cli/commands/autosync.py index 2d3eaa0e..6ff3e4bc 100644 --- a/trestlebot/cli/commands/autosync.py +++ b/trestlebot/cli/commands/autosync.py @@ -7,7 +7,6 @@ from trestlebot.cli.options.autosync import autosync_options from trestlebot.cli.options.common import common_options, git_options, handle_exceptions -from trestlebot.cli.run import comma_sep_to_list from trestlebot.cli.run import run as bot_run from trestlebot.tasks.assemble_task import AssembleTask from trestlebot.tasks.authored import types @@ -25,16 +24,16 @@ def run(oscal_model: str, **kwargs: Any) -> None: pre_tasks: List[TaskBase] = [] kwargs["working_dir"] = str(kwargs["repo_path"].resolve()) - if kwargs.get("file_patterns"): - kwargs.update({"patterns": comma_sep_to_list(kwargs["file_patterns"])}) + if kwargs.get("file_pattern"): + kwargs.update({"patterns": list(kwargs.get("file_pattern", []))}) model_filter: ModelFilter = ModelFilter( - skip_patterns=comma_sep_to_list(kwargs.get("skip_items", "")), + skip_patterns=list(kwargs.get("skip_item", [])), include_patterns=["*"], ) authored_object: AuthoredObjectBase = types.get_authored_object( oscal_model, kwargs["working_dir"], - kwargs.get("ssp_index_path", ""), + kwargs.get("ssp_index_file", ""), ) # Assuming an edit has occurred assemble would be run before regenerate. @@ -42,7 +41,7 @@ def run(oscal_model: str, **kwargs: Any) -> None: if not kwargs.get("skip_assemble"): assemble_task: AssembleTask = AssembleTask( authored_object=authored_object, - markdown_dir=kwargs["markdown_path"], + markdown_dir=kwargs["markdown_dir"], version=kwargs.get("version", ""), model_filter=model_filter, ) @@ -53,7 +52,7 @@ def run(oscal_model: str, **kwargs: Any) -> None: if not kwargs.get("skip_regenerate"): regenerate_task: RegenerateTask = RegenerateTask( authored_object=authored_object, - markdown_dir=kwargs["markdown_path"], + markdown_dir=kwargs["markdown_dir"], model_filter=model_filter, ) pre_tasks.append(regenerate_task) @@ -74,13 +73,12 @@ def autosync_cmd(ctx: click.Context) -> None: @autosync_options @git_options @click.option( - "--ssp-index-path", + "--ssp-index-file", help="Path to ssp index file", type=str, required=True, ) -def autosync_ssp_cmd(ctx: click.Context, ssp_index_path: str, **kwargs: Any) -> None: - kwargs.update({"ssp_index_path": ssp_index_path}) +def autosync_ssp_cmd(ctx: click.Context, **kwargs: Any) -> None: run("ssp", **kwargs) diff --git a/trestlebot/cli/commands/rule_transform.py b/trestlebot/cli/commands/rule_transform.py index 41519cf3..0f2b72a7 100644 --- a/trestlebot/cli/commands/rule_transform.py +++ b/trestlebot/cli/commands/rule_transform.py @@ -45,7 +45,7 @@ @handle_exceptions def rule_transform_cmd(ctx: click.Context, **kwargs: Any) -> None: """Run the rule transform operation.""" - # Allow any model to be skipped by setting skip_items, by default include all + # Allow any model to be skipped by setting skip_item, by default include all skip_items = list(kwargs.get("skip_item", [])) model_filter: ModelFilter = ModelFilter( skip_patterns=skip_items, diff --git a/trestlebot/cli/options/autosync.py b/trestlebot/cli/options/autosync.py index d2a41eb8..fb49e3f0 100644 --- a/trestlebot/cli/options/autosync.py +++ b/trestlebot/cli/options/autosync.py @@ -12,15 +12,16 @@ def autosync_options(f: F) -> F: """ f = click.option( - "--markdown-path", + "--markdown-dir", help="Path to Trestle markdown files", - type=click.Path(exists=True), + type=str, required=True, )(f) f = click.option( - "--skip-items", - help="Comma-separated list of glob patterns to skip when running tasks", + "--skip-item", + help="Glob pattern to skip when running tasks", type=str, + multiple=True, )(f) f = click.option( "--skip-assemble", @@ -41,9 +42,4 @@ def autosync_options(f: F) -> F: help="Version of the OSCAL model to set during assembly into JSON", type=str, )(f) - f = click.option( - "--dry-run", - help="Run tasks, but do not push to the repository", - is_flag=True, - )(f) return f diff --git a/trestlebot/cli/options/common.py b/trestlebot/cli/options/common.py index 21c95690..658f1810 100644 --- a/trestlebot/cli/options/common.py +++ b/trestlebot/cli/options/common.py @@ -151,14 +151,16 @@ def git_options(f: F) -> F: type=str, )(f) f = click.option( - "--file-patterns", - help="Comma-separated list of file patterns to be used with `git add` in repository updates", + "--file-pattern", + help="File pattern to be used with `git add` in repository updates", type=str, + multiple=True, )(f) f = click.option( "--commit-message", help="Commit message for automated updates", type=str, + default="chore: automatic updates", )(f) f = click.option( "--author-name", diff --git a/trestlebot/cli/run.py b/trestlebot/cli/run.py index 75c97c81..1b39f10c 100644 --- a/trestlebot/cli/run.py +++ b/trestlebot/cli/run.py @@ -26,7 +26,7 @@ def run(pre_tasks: List[TaskBase], kwargs: Dict[Any, Any]) -> BotResults: return bot.run( pre_tasks=pre_tasks, - patterns=comma_sep_to_list(kwargs.get("patterns", "")), + patterns=kwargs.get("patterns", []), commit_message=kwargs.get("commit_message", "Automatic updates from bot"), dry_run=kwargs.get("dry_run", False), ) From df0e5d2db38ae6109572871a8c0b56f598ea6cd8 Mon Sep 17 00:00:00 2001 From: Qingmin Duanmu Date: Fri, 29 Nov 2024 09:51:08 +0800 Subject: [PATCH 042/108] Fix AttributeError, some misc updates AttributeError: 'NoneType' object has no attribute 'encode' --- tests/trestlebot/cli/test_autosync_cmd.py | 27 ++++++++++++------- ...ransform.py => test_rule_transform_cmd.py} | 0 trestlebot/cli/commands/autosync.py | 18 ++++++------- trestlebot/cli/commands/rule_transform.py | 2 +- trestlebot/cli/options/autosync.py | 14 ++++------ trestlebot/cli/options/common.py | 6 +++-- trestlebot/cli/run.py | 2 +- 7 files changed, 36 insertions(+), 33 deletions(-) rename tests/trestlebot/cli/{test_rule_transform.py => test_rule_transform_cmd.py} (100%) diff --git a/tests/trestlebot/cli/test_autosync_cmd.py b/tests/trestlebot/cli/test_autosync_cmd.py index d044bee3..dede4ed9 100644 --- a/tests/trestlebot/cli/test_autosync_cmd.py +++ b/tests/trestlebot/cli/test_autosync_cmd.py @@ -3,15 +3,15 @@ """Testing module for trestlebot autosync command""" -import tempfile +import pathlib from click.testing import CliRunner from trestlebot.cli.commands.autosync import autosync_cmd +from trestlebot.cli.config import TrestleBotConfig, write_to_file def test_invalid_autosync_command(tmp_init_dir: str) -> None: - tempdir = tempfile.mkdtemp() runner = CliRunner() result = runner.invoke( autosync_cmd, @@ -19,8 +19,8 @@ def test_invalid_autosync_command(tmp_init_dir: str) -> None: "invalid", "--repo-path", tmp_init_dir, - "--markdown-path", - tempdir, + "--markdown-dir", + "markdown", "--branch", "main", "--committer-name", @@ -36,14 +36,13 @@ def test_invalid_autosync_command(tmp_init_dir: str) -> None: def test_no_ssp_index_path(tmp_init_dir: str) -> None: """Test invalid ssp index file for autosync ssp""" - tempdir = tempfile.mkdtemp() runner = CliRunner() cmd_options = [ "ssp", "--repo-path", tmp_init_dir, - "--markdown-path", - tempdir, + "--markdown-dir", + "markdown", "--branch", "main", "--committer-name", @@ -53,13 +52,13 @@ def test_no_ssp_index_path(tmp_init_dir: str) -> None: ] result = runner.invoke(autosync_cmd, cmd_options) assert result.exit_code == 2 - assert "Error: Missing option '--ssp-index-path'" in result.output + assert "Error: Missing option '--ssp-index-file'" in result.output cmd_options[0] = "compdef" result = runner.invoke(autosync_cmd, cmd_options) assert result.exit_code == 0 -def test_no_markdown_path(tmp_init_dir: str) -> None: +def test_no_markdown_dir(tmp_init_dir: str) -> None: runner = CliRunner() cmd_options = [ "compdef", @@ -74,4 +73,12 @@ def test_no_markdown_path(tmp_init_dir: str) -> None: ] result = runner.invoke(autosync_cmd, cmd_options) assert result.exit_code == 2 - assert "Error: Missing option '--markdown-path'" in result.output + assert "Error: Missing option '--markdown-dir'" in result.output + + # With 'markdown_dir' setting in config.yml + config_obj = TrestleBotConfig(markdown_dir="markdown") + filepath = pathlib.Path(tmp_init_dir).joinpath("config.yml") + write_to_file(config_obj, filepath) + cmd_options += ["--config", str(filepath)] + result = runner.invoke(autosync_cmd, cmd_options) + assert result.exit_code == 0 diff --git a/tests/trestlebot/cli/test_rule_transform.py b/tests/trestlebot/cli/test_rule_transform_cmd.py similarity index 100% rename from tests/trestlebot/cli/test_rule_transform.py rename to tests/trestlebot/cli/test_rule_transform_cmd.py diff --git a/trestlebot/cli/commands/autosync.py b/trestlebot/cli/commands/autosync.py index 2d3eaa0e..6ff3e4bc 100644 --- a/trestlebot/cli/commands/autosync.py +++ b/trestlebot/cli/commands/autosync.py @@ -7,7 +7,6 @@ from trestlebot.cli.options.autosync import autosync_options from trestlebot.cli.options.common import common_options, git_options, handle_exceptions -from trestlebot.cli.run import comma_sep_to_list from trestlebot.cli.run import run as bot_run from trestlebot.tasks.assemble_task import AssembleTask from trestlebot.tasks.authored import types @@ -25,16 +24,16 @@ def run(oscal_model: str, **kwargs: Any) -> None: pre_tasks: List[TaskBase] = [] kwargs["working_dir"] = str(kwargs["repo_path"].resolve()) - if kwargs.get("file_patterns"): - kwargs.update({"patterns": comma_sep_to_list(kwargs["file_patterns"])}) + if kwargs.get("file_pattern"): + kwargs.update({"patterns": list(kwargs.get("file_pattern", []))}) model_filter: ModelFilter = ModelFilter( - skip_patterns=comma_sep_to_list(kwargs.get("skip_items", "")), + skip_patterns=list(kwargs.get("skip_item", [])), include_patterns=["*"], ) authored_object: AuthoredObjectBase = types.get_authored_object( oscal_model, kwargs["working_dir"], - kwargs.get("ssp_index_path", ""), + kwargs.get("ssp_index_file", ""), ) # Assuming an edit has occurred assemble would be run before regenerate. @@ -42,7 +41,7 @@ def run(oscal_model: str, **kwargs: Any) -> None: if not kwargs.get("skip_assemble"): assemble_task: AssembleTask = AssembleTask( authored_object=authored_object, - markdown_dir=kwargs["markdown_path"], + markdown_dir=kwargs["markdown_dir"], version=kwargs.get("version", ""), model_filter=model_filter, ) @@ -53,7 +52,7 @@ def run(oscal_model: str, **kwargs: Any) -> None: if not kwargs.get("skip_regenerate"): regenerate_task: RegenerateTask = RegenerateTask( authored_object=authored_object, - markdown_dir=kwargs["markdown_path"], + markdown_dir=kwargs["markdown_dir"], model_filter=model_filter, ) pre_tasks.append(regenerate_task) @@ -74,13 +73,12 @@ def autosync_cmd(ctx: click.Context) -> None: @autosync_options @git_options @click.option( - "--ssp-index-path", + "--ssp-index-file", help="Path to ssp index file", type=str, required=True, ) -def autosync_ssp_cmd(ctx: click.Context, ssp_index_path: str, **kwargs: Any) -> None: - kwargs.update({"ssp_index_path": ssp_index_path}) +def autosync_ssp_cmd(ctx: click.Context, **kwargs: Any) -> None: run("ssp", **kwargs) diff --git a/trestlebot/cli/commands/rule_transform.py b/trestlebot/cli/commands/rule_transform.py index 41519cf3..0f2b72a7 100644 --- a/trestlebot/cli/commands/rule_transform.py +++ b/trestlebot/cli/commands/rule_transform.py @@ -45,7 +45,7 @@ @handle_exceptions def rule_transform_cmd(ctx: click.Context, **kwargs: Any) -> None: """Run the rule transform operation.""" - # Allow any model to be skipped by setting skip_items, by default include all + # Allow any model to be skipped by setting skip_item, by default include all skip_items = list(kwargs.get("skip_item", [])) model_filter: ModelFilter = ModelFilter( skip_patterns=skip_items, diff --git a/trestlebot/cli/options/autosync.py b/trestlebot/cli/options/autosync.py index d2a41eb8..fb49e3f0 100644 --- a/trestlebot/cli/options/autosync.py +++ b/trestlebot/cli/options/autosync.py @@ -12,15 +12,16 @@ def autosync_options(f: F) -> F: """ f = click.option( - "--markdown-path", + "--markdown-dir", help="Path to Trestle markdown files", - type=click.Path(exists=True), + type=str, required=True, )(f) f = click.option( - "--skip-items", - help="Comma-separated list of glob patterns to skip when running tasks", + "--skip-item", + help="Glob pattern to skip when running tasks", type=str, + multiple=True, )(f) f = click.option( "--skip-assemble", @@ -41,9 +42,4 @@ def autosync_options(f: F) -> F: help="Version of the OSCAL model to set during assembly into JSON", type=str, )(f) - f = click.option( - "--dry-run", - help="Run tasks, but do not push to the repository", - is_flag=True, - )(f) return f diff --git a/trestlebot/cli/options/common.py b/trestlebot/cli/options/common.py index 21c95690..658f1810 100644 --- a/trestlebot/cli/options/common.py +++ b/trestlebot/cli/options/common.py @@ -151,14 +151,16 @@ def git_options(f: F) -> F: type=str, )(f) f = click.option( - "--file-patterns", - help="Comma-separated list of file patterns to be used with `git add` in repository updates", + "--file-pattern", + help="File pattern to be used with `git add` in repository updates", type=str, + multiple=True, )(f) f = click.option( "--commit-message", help="Commit message for automated updates", type=str, + default="chore: automatic updates", )(f) f = click.option( "--author-name", diff --git a/trestlebot/cli/run.py b/trestlebot/cli/run.py index 75c97c81..1b39f10c 100644 --- a/trestlebot/cli/run.py +++ b/trestlebot/cli/run.py @@ -26,7 +26,7 @@ def run(pre_tasks: List[TaskBase], kwargs: Dict[Any, Any]) -> BotResults: return bot.run( pre_tasks=pre_tasks, - patterns=comma_sep_to_list(kwargs.get("patterns", "")), + patterns=kwargs.get("patterns", []), commit_message=kwargs.get("commit_message", "Automatic updates from bot"), dry_run=kwargs.get("dry_run", False), ) From aef085f5f9a0495cf79d5ec1f3f6b411269cecb9 Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Mon, 2 Dec 2024 15:53:07 -0500 Subject: [PATCH 043/108] feat: unit tests added for create command --- tests/trestlebot/cli/test_create_cmd.py | 149 +++++++++++++++++++----- trestlebot/cli/commands/create.py | 2 +- 2 files changed, 121 insertions(+), 30 deletions(-) diff --git a/tests/trestlebot/cli/test_create_cmd.py b/tests/trestlebot/cli/test_create_cmd.py index b1f3754a..56109d9e 100644 --- a/tests/trestlebot/cli/test_create_cmd.py +++ b/tests/trestlebot/cli/test_create_cmd.py @@ -16,32 +16,123 @@ def test_invalid_create_cmd() -> None: assert result.exit_code == 2 -# def test_create_ssp_cmd(tmp_init_dir: str) -> None: -# """Tests successful create ssp command.""" -# -# SSP_INDEX_FILE = "test-ssp-index.json" -# -# runner = CliRunner() -# result = runner.invoke( -# create_cmd, -# [ -# "ssp", -# "--profile-name", -# "repo-path", -# tmp_init_dir, -# "--ssp-index-file", -# SSP_INDEX_FILE, -# ], -# ) -# assert result.exit_code == 0 -# -# # verify ssp-index.json was created in tmp_init_dir -# tmp_dir = pathlib.Path(tmp_init_dir) -# ssp_index_file = tmp_dir.joinpath(SSP_INDEX_FILE) -# print("SSP INDEX Path", str(ssp_index_file.resolve())) -# -# assert ssp_index_file.exists() is True - -# verity json file was created in tmp_init_dir/system-security-plans - -# verify markdown file(s) were created in tmp_init_dir/markdown/system... +def test_create_ssp_cmd(tmp_init_dir: str) -> None: + """Tests successful create ssp command.""" + SSP_INDEX_FILE = "tester-ssp-index.json" + + # tmp_dir = pathlib.Path(tmp_init_dir) + # ssp_index_file = tmp_dir.joinpath(SSP_INDEX_FILE) + # print("SSP INDEX Path", str(ssp_index_file.resolve())) + + runner = CliRunner() + result = runner.invoke( + create_cmd, + [ + "ssp", + "--profile-name", + "oscal-profile-name", + "--markdown-dir", + "markdown", + "--ssp-name", + "test-name", + "--repo-path", + tmp_init_dir, + "--ssp-index-file", + SSP_INDEX_FILE, + ], + ) + assert result.exit_code == 0 + + +def test_create_compdef_cmd(tmp_init_dir: str) -> None: + """Tests successful create compdef command.""" + + runner = CliRunner() + result = runner.invoke( + create_cmd, + [ + "compdef", + "--profile-name", + "oscal-profile-name", + "--compdef-name", + "test-name", + "--component-title", + "title-test", + "--component-description", + "description-test", + "--component-definition-type", + "type-test", + "repo-path", + tmp_init_dir, + ], + ) + assert result.exit_code == 2 + + +def test_default_ssp_index_file_cmd(tmp_init_dir: str) -> None: + """Tests successful default ssp_index.json file creation.""" + + runner = CliRunner() + result = runner.invoke( + create_cmd, + [ + "ssp", + "--profile-name", + "oscal-profile-name", + "--markdown-dir", + "markdown", + "--ssp-name", + "test-name", + "--repo-path", + tmp_init_dir, + ], + ) + assert result.exit_code == 0 + + +def test_markdown_files_not_created(tmp_init_dir: str) -> None: + SSP_INDEX_TESTER = "ssp-tester.index.json" + runner = CliRunner() + result = runner.invoke( + create_cmd, + [ + "ssp", + "--profile-name", + "oscal-profile-name", + "--ssp-name", + "test-name", + "repo-path", + tmp_init_dir, + "--ssp-index-file", + SSP_INDEX_TESTER, + ], + ) + assert result.exit_code == 2 + + +def test_markdown_files_created(tmp_init_dir: str) -> None: + SSP_INDEX_TESTER = "ssp-tester.index.json" + markdown_file_tester = "/markdown/system-security-plan.md" + + # tmp_dir = pathlib.Path(tmp_init_dir) + # markdown_file = tmp_dir.joinpath(markdown_file_tester) + # print("Markdown file located at", str(markdown_file.resolve())) + + runner = CliRunner() + result = runner.invoke( + create_cmd, + [ + "ssp", + "--profile-name", + "oscal-profile-name", + "--markdown-dir", + markdown_file_tester, + "--ssp-name", + "test-name", + "--repo-path", + tmp_init_dir, + "--ssp-index-file", + SSP_INDEX_TESTER, + ], + ) + assert result.exit_code == 0 diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index 8391de8c..42697cfb 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -27,7 +27,7 @@ logger = logging.getLogger(__name__) -@click.group(name="create") +@click.group(name="create", help="Component definition and ssp authoring.") @click.pass_context @handle_exceptions def create_cmd(ctx: click.Context) -> None: From dbf9fa2583f8834da0e5856b0f5a75ed45705509 Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Mon, 2 Dec 2024 15:53:07 -0500 Subject: [PATCH 044/108] feat: unit tests added for create command --- tests/trestlebot/cli/test_create_cmd.py | 149 +++++++++++++++++++----- trestlebot/cli/commands/create.py | 2 +- 2 files changed, 121 insertions(+), 30 deletions(-) diff --git a/tests/trestlebot/cli/test_create_cmd.py b/tests/trestlebot/cli/test_create_cmd.py index b1f3754a..56109d9e 100644 --- a/tests/trestlebot/cli/test_create_cmd.py +++ b/tests/trestlebot/cli/test_create_cmd.py @@ -16,32 +16,123 @@ def test_invalid_create_cmd() -> None: assert result.exit_code == 2 -# def test_create_ssp_cmd(tmp_init_dir: str) -> None: -# """Tests successful create ssp command.""" -# -# SSP_INDEX_FILE = "test-ssp-index.json" -# -# runner = CliRunner() -# result = runner.invoke( -# create_cmd, -# [ -# "ssp", -# "--profile-name", -# "repo-path", -# tmp_init_dir, -# "--ssp-index-file", -# SSP_INDEX_FILE, -# ], -# ) -# assert result.exit_code == 0 -# -# # verify ssp-index.json was created in tmp_init_dir -# tmp_dir = pathlib.Path(tmp_init_dir) -# ssp_index_file = tmp_dir.joinpath(SSP_INDEX_FILE) -# print("SSP INDEX Path", str(ssp_index_file.resolve())) -# -# assert ssp_index_file.exists() is True - -# verity json file was created in tmp_init_dir/system-security-plans - -# verify markdown file(s) were created in tmp_init_dir/markdown/system... +def test_create_ssp_cmd(tmp_init_dir: str) -> None: + """Tests successful create ssp command.""" + SSP_INDEX_FILE = "tester-ssp-index.json" + + # tmp_dir = pathlib.Path(tmp_init_dir) + # ssp_index_file = tmp_dir.joinpath(SSP_INDEX_FILE) + # print("SSP INDEX Path", str(ssp_index_file.resolve())) + + runner = CliRunner() + result = runner.invoke( + create_cmd, + [ + "ssp", + "--profile-name", + "oscal-profile-name", + "--markdown-dir", + "markdown", + "--ssp-name", + "test-name", + "--repo-path", + tmp_init_dir, + "--ssp-index-file", + SSP_INDEX_FILE, + ], + ) + assert result.exit_code == 0 + + +def test_create_compdef_cmd(tmp_init_dir: str) -> None: + """Tests successful create compdef command.""" + + runner = CliRunner() + result = runner.invoke( + create_cmd, + [ + "compdef", + "--profile-name", + "oscal-profile-name", + "--compdef-name", + "test-name", + "--component-title", + "title-test", + "--component-description", + "description-test", + "--component-definition-type", + "type-test", + "repo-path", + tmp_init_dir, + ], + ) + assert result.exit_code == 2 + + +def test_default_ssp_index_file_cmd(tmp_init_dir: str) -> None: + """Tests successful default ssp_index.json file creation.""" + + runner = CliRunner() + result = runner.invoke( + create_cmd, + [ + "ssp", + "--profile-name", + "oscal-profile-name", + "--markdown-dir", + "markdown", + "--ssp-name", + "test-name", + "--repo-path", + tmp_init_dir, + ], + ) + assert result.exit_code == 0 + + +def test_markdown_files_not_created(tmp_init_dir: str) -> None: + SSP_INDEX_TESTER = "ssp-tester.index.json" + runner = CliRunner() + result = runner.invoke( + create_cmd, + [ + "ssp", + "--profile-name", + "oscal-profile-name", + "--ssp-name", + "test-name", + "repo-path", + tmp_init_dir, + "--ssp-index-file", + SSP_INDEX_TESTER, + ], + ) + assert result.exit_code == 2 + + +def test_markdown_files_created(tmp_init_dir: str) -> None: + SSP_INDEX_TESTER = "ssp-tester.index.json" + markdown_file_tester = "/markdown/system-security-plan.md" + + # tmp_dir = pathlib.Path(tmp_init_dir) + # markdown_file = tmp_dir.joinpath(markdown_file_tester) + # print("Markdown file located at", str(markdown_file.resolve())) + + runner = CliRunner() + result = runner.invoke( + create_cmd, + [ + "ssp", + "--profile-name", + "oscal-profile-name", + "--markdown-dir", + markdown_file_tester, + "--ssp-name", + "test-name", + "--repo-path", + tmp_init_dir, + "--ssp-index-file", + SSP_INDEX_TESTER, + ], + ) + assert result.exit_code == 0 diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index 8391de8c..42697cfb 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -27,7 +27,7 @@ logger = logging.getLogger(__name__) -@click.group(name="create") +@click.group(name="create", help="Component definition and ssp authoring.") @click.pass_context @handle_exceptions def create_cmd(ctx: click.Context) -> None: From 7bca6cd761419195779003e61aa54e1fc3c6c510 Mon Sep 17 00:00:00 2001 From: George Vauter Date: Mon, 2 Dec 2024 19:11:08 -0500 Subject: [PATCH 045/108] refactor sync upstreams and autosync to match existing entrypoint syntax --- tests/trestlebot/cli/test_autosync_cmd.py | 27 +- tests/trestlebot/cli/test_config.py | 21 +- .../trestlebot/cli/test_sync_upstreams_cmd.py | 157 +++++++++++ tests/trestlebot/cli/test_upstreams_cmd.py | 249 ------------------ trestlebot/cli/commands/autosync.py | 139 +++++++--- trestlebot/cli/commands/create.py | 6 +- trestlebot/cli/commands/init.py | 12 + trestlebot/cli/commands/sync_upstreams.py | 130 +++++++++ trestlebot/cli/commands/upstream.py | 237 ----------------- trestlebot/cli/config.py | 36 +-- trestlebot/cli/options/autosync.py | 22 +- trestlebot/cli/options/common.py | 25 +- trestlebot/cli/root.py | 4 +- trestlebot/cli/{run.py => utils.py} | 10 +- 14 files changed, 482 insertions(+), 593 deletions(-) create mode 100644 tests/trestlebot/cli/test_sync_upstreams_cmd.py delete mode 100644 tests/trestlebot/cli/test_upstreams_cmd.py create mode 100644 trestlebot/cli/commands/sync_upstreams.py delete mode 100644 trestlebot/cli/commands/upstream.py rename trestlebot/cli/{run.py => utils.py} (74%) diff --git a/tests/trestlebot/cli/test_autosync_cmd.py b/tests/trestlebot/cli/test_autosync_cmd.py index d044bee3..836ade07 100644 --- a/tests/trestlebot/cli/test_autosync_cmd.py +++ b/tests/trestlebot/cli/test_autosync_cmd.py @@ -10,16 +10,18 @@ from trestlebot.cli.commands.autosync import autosync_cmd -def test_invalid_autosync_command(tmp_init_dir: str) -> None: +def test_invalid_oscal_model(tmp_init_dir: str) -> None: + """Test invalid OSCAl model option""" tempdir = tempfile.mkdtemp() runner = CliRunner() result = runner.invoke( autosync_cmd, [ + "--oscal-model", "invalid", "--repo-path", tmp_init_dir, - "--markdown-path", + "--markdown-dir", tempdir, "--branch", "main", @@ -29,20 +31,21 @@ def test_invalid_autosync_command(tmp_init_dir: str) -> None: "test@example.com", ], ) - assert "Error: No such command" in result.output + assert "Invalid value for '--oscal-model'" in result.output assert result.exit_code == 2 def test_no_ssp_index_path(tmp_init_dir: str) -> None: - """Test invalid ssp index file for autosync ssp""" + """Test missing ssp-index-file for autosync ssp.""" tempdir = tempfile.mkdtemp() runner = CliRunner() cmd_options = [ + "--oscal-model", "ssp", "--repo-path", tmp_init_dir, - "--markdown-path", + "--markdown-dir", tempdir, "--branch", "main", @@ -52,16 +55,16 @@ def test_no_ssp_index_path(tmp_init_dir: str) -> None: "test@example.com", ] result = runner.invoke(autosync_cmd, cmd_options) - assert result.exit_code == 2 - assert "Error: Missing option '--ssp-index-path'" in result.output - cmd_options[0] = "compdef" - result = runner.invoke(autosync_cmd, cmd_options) - assert result.exit_code == 0 + assert result.exit_code == 1 + assert "Missing option '--ssp-index-file'" in result.output + +def test_missing_markdown_dir_option(tmp_init_dir: str) -> None: + """Test autosync required markdown-dir option.""" -def test_no_markdown_path(tmp_init_dir: str) -> None: runner = CliRunner() cmd_options = [ + "--oscal-model", "compdef", "--repo-path", tmp_init_dir, @@ -74,4 +77,4 @@ def test_no_markdown_path(tmp_init_dir: str) -> None: ] result = runner.invoke(autosync_cmd, cmd_options) assert result.exit_code == 2 - assert "Error: Missing option '--markdown-path'" in result.output + assert "Error: Missing option '--markdown-dir'" in result.output diff --git a/tests/trestlebot/cli/test_config.py b/tests/trestlebot/cli/test_config.py index 2b303471..60a40210 100644 --- a/tests/trestlebot/cli/test_config.py +++ b/tests/trestlebot/cli/test_config.py @@ -11,6 +11,7 @@ from trestlebot.cli.config import ( TrestleBotConfig, TrestleBotConfigError, + UpstreamsConfig, load_from_file, make_config, write_to_file, @@ -19,7 +20,11 @@ @pytest.fixture def config_obj() -> TrestleBotConfig: - return TrestleBotConfig(repo_path="/tmp", markdown_dir="markdown") + return TrestleBotConfig( + repo_path="/tmp", + markdown_dir="markdown", + upstreams=UpstreamsConfig(sources=["repo@main"]), + ) def test_invalid_config_raises_errors() -> None: @@ -39,15 +44,19 @@ def test_make_config_raises_no_errors(tmp_init_dir: str) -> None: values = { "repo_path": tmp_init_dir, "markdown_dir": "markdown", - "upstreams": [{"url": "https://test@main", "skip_validation": True}], + "committer_name": "committer-name", + "committer_email": "committer-email", + "upstreams": {"sources": ["https://test@main"], "skip_validation": True}, } config = make_config(values) assert isinstance(config, TrestleBotConfig) - assert len(config.upstreams) == 1 - assert config.upstreams[0].url == "https://test@main" - assert config.upstreams[0].skip_validation is True + assert config.upstreams is not None + assert config.upstreams.sources == ["https://test@main"] + assert config.upstreams.skip_validation is True assert config.repo_path == pathlib.Path(tmp_init_dir) - assert config.markdown_dir == "markdown" + assert config.markdown_dir == values["markdown_dir"] + assert config.committer_name == values["committer_name"] + assert config.committer_email == values["committer_email"] def test_config_write_to_file(config_obj: TrestleBotConfig, tmp_init_dir: str) -> None: diff --git a/tests/trestlebot/cli/test_sync_upstreams_cmd.py b/tests/trestlebot/cli/test_sync_upstreams_cmd.py new file mode 100644 index 00000000..b3e9e1ba --- /dev/null +++ b/tests/trestlebot/cli/test_sync_upstreams_cmd.py @@ -0,0 +1,157 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Red Hat, Inc. + + +"""Unit tests for upstreams commands.""" + +import pathlib +from typing import Tuple + +from click.testing import CliRunner +from git import Repo + +from tests.testutils import clean, prepare_upstream_repo, setup_for_init +from trestlebot.cli.commands.sync_upstreams import sync_upstreams_cmd +from trestlebot.cli.config import make_config, write_to_file +from trestlebot.const import TRESTLEBOT_CONFIG_DIR + + +TEST_CATALOG = "simplified_nist_catalog" +TEST_CATALOG_PATH = "catalogs/simplified_nist_catalog/catalog.json" +TEST_PROFILE = "simplified_nist_profile" +TEST_PROFILE_PATH = "profiles/simplified_nist_profile/profile.json" + + +def test_sync_upstreams(tmp_repo: Tuple[str, Repo]) -> None: + """Test sync upstreams""" + repo_dir, _ = tmp_repo + repo_path = pathlib.Path(repo_dir) + + source: str = prepare_upstream_repo() + + runner = CliRunner() + result = runner.invoke( + sync_upstreams_cmd, + [ + "--repo-path", + repo_path, + "--sources", + f"{source}@main", + "--branch", + "main", + "--committer-email", + "test@email", + "--committer-name", + "test name", + ], + ) + + assert repo_path.joinpath(TEST_CATALOG_PATH).exists() + assert repo_path.joinpath(TEST_PROFILE_PATH).exists() + assert result.exit_code == 0 + clean(source) + + +def test_sync_upstreams_with_config( + tmp_repo: Tuple[str, Repo], tmp_init_dir: str +) -> None: + """Test sync upstreams using a config file.""" + + repo_dir, _ = tmp_repo + repo_path = pathlib.Path(repo_dir) + + source: str = prepare_upstream_repo() + + trestlebot_dir = pathlib.Path(tmp_init_dir) / pathlib.Path(TRESTLEBOT_CONFIG_DIR) + trestlebot_dir.mkdir() + + setup_for_init(pathlib.Path(tmp_init_dir)) + + config_path = ( + pathlib.Path(tmp_init_dir) + .joinpath(TRESTLEBOT_CONFIG_DIR) + .joinpath("config.yml") + ) + + config = make_config( + { + "branch": "main", + "committer_name": "test name", + "committer_email": "test@email", + "upstreams": { + "sources": [f"{source}@main"], + "include_models": ["*"], + "skip_validation": False, + }, + } + ) + + write_to_file(config, config_path) + + runner = CliRunner() + result = runner.invoke( + sync_upstreams_cmd, + ["--repo-path", repo_path, "--config", config_path.resolve()], + ) + assert repo_path.joinpath(TEST_CATALOG_PATH).exists() + assert repo_path.joinpath(TEST_PROFILE_PATH).exists() + assert result.exit_code == 0 + clean(source) + + +def test_sync_upstreams_exclude_models(tmp_repo: Tuple[str, Repo]) -> None: + """Test sync upstreams with exclude models""" + + repo_dir, _ = tmp_repo + repo_path = pathlib.Path(repo_dir) + + source: str = prepare_upstream_repo() + + runner = CliRunner() + result = runner.invoke( + sync_upstreams_cmd, + [ + "--repo-path", + repo_path, + "--sources", + f"{source}@main", + "--branch", + "main", + "--exclude-models", + TEST_PROFILE, + "--committer-email", + "test@email", + "--committer-name", + "test", + ], + ) + + assert repo_path.joinpath(TEST_CATALOG_PATH).exists() + assert not repo_path.joinpath(TEST_PROFILE_PATH).exists() + assert result.exit_code == 0 + clean(source) + + +def test_sync_upstreams_no_sources(tmp_repo: Tuple[str, Repo]) -> None: + """Test sync upstreams with sources option missing""" + + repo_dir, _ = tmp_repo + repo_path = pathlib.Path(repo_dir) + + runner = CliRunner() + result = runner.invoke( + sync_upstreams_cmd, + [ + "--repo-path", + repo_path, + "--branch", + "main", + "--committer-name", + "test name", + "--committer-email", + "test@email", + ], + ) + + assert "Missing option '--sources'" in result.output + assert result.exit_code == 1 diff --git a/tests/trestlebot/cli/test_upstreams_cmd.py b/tests/trestlebot/cli/test_upstreams_cmd.py deleted file mode 100644 index 11e05a19..00000000 --- a/tests/trestlebot/cli/test_upstreams_cmd.py +++ /dev/null @@ -1,249 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# Copyright (c) 2024 Red Hat, Inc. - - -"""Unit tests for upstreams commands.""" - -import pathlib -from typing import Tuple - -import pytest -from click.testing import CliRunner -from git import Repo - -from tests.testutils import clean, prepare_upstream_repo -from trestlebot.cli.commands.upstream import upstream_cmd -from trestlebot.cli.config import ( - TrestleBotConfig, - UpstreamConfig, - load_from_file, - write_to_file, -) - - -TEST_CATALOG = "simplified_nist_catalog" -TEST_CATALOG_PATH = "catalogs/simplified_nist_catalog/catalog.json" -TEST_PROFILE = "simplified_nist_profile" -TEST_PROFILE_PATH = "profiles/simplified_nist_profile/profile.json" - - -def test_config_get_upstream_by_url() -> None: - """Test to confirm _get_upstream_by_url returns a single valid upstream from the config.""" - config = TrestleBotConfig(upstreams=[UpstreamConfig(url="https://test")]) - - assert len(config.upstreams) == 1 - assert config.upstreams[0].url == "https://test" - assert config.upstreams[0].include_models == ["*"] # confirm default was set - assert config.upstreams[0].exclude_models == [] - assert config.upstreams[0].skip_validation is False - - -def test_upstream_add_already_exists(tmp_repo: Tuple[str, Repo]) -> None: - """Test upstream add fails if upstream already exists""" - repo_path, repo = tmp_repo - config_path = pathlib.Path(repo_path).joinpath(".trestlebot/config.yml") - - source: str = prepare_upstream_repo() - url = f"{source}@main" - config = TrestleBotConfig( - committer_email="test@test", - committer_name="test", - upstreams=[UpstreamConfig(url=url)], - ) - write_to_file(config, config_path) - - runner = CliRunner() - result = runner.invoke( - upstream_cmd, - [ - "add", - "--repo-path", - repo_path, - "--config", - config_path, - "--url", - url, - "--branch", - "test", - ], - ) - assert ( - result.exit_code == 0 - ) # This does return success, it just skips the URLs already in the config - - assert f"{url} already exists. Edit {config_path} to update." in result.output - - -def test_upstream_add(tmp_repo: Tuple[str, Repo]) -> None: - """Tests successful add of upstream""" - repo_dir, repo = tmp_repo - repo_path = pathlib.Path(repo_dir) - config_path = repo_path.joinpath(".trestlebot/config.yml") - - source: str = prepare_upstream_repo() - url = f"{source}@main" - config = TrestleBotConfig( - committer_email="test@test", - committer_name="test", - ) - write_to_file(config, config_path) - - runner = CliRunner() - result = runner.invoke( - upstream_cmd, - [ - "add", - "--repo-path", - repo_path, - "--config", - config_path, - "--url", - url, - "--branch", - "main", - "--debug", - ], - ) - updated_config = load_from_file(config_path) - if not updated_config: - pytest.fail("Updated config not found") - return - - assert len(updated_config.upstreams) == 1 - assert updated_config.upstreams[0].url == url - assert updated_config.upstreams[0].include_models == ["*"] - assert updated_config.upstreams[0].exclude_models == [] - assert updated_config.upstreams[0].skip_validation is False - assert ( - updated_config.committer_email == "test@test" - ) # sanity check this didn't change - assert repo_path.joinpath(TEST_CATALOG_PATH).exists() - assert repo_path.joinpath(TEST_PROFILE_PATH).exists() - assert result.exit_code == 0 - clean(source, None) - - -def test_upstream_add_exclude_models(tmp_repo: Tuple[str, Repo]) -> None: - """Test add upstream with exclude models""" - - repo_dir, repo = tmp_repo - repo_path = pathlib.Path(repo_dir) - config_path = repo_path.joinpath(".trestlebot/config.yml") - - source: str = prepare_upstream_repo() - url = f"{source}@main" - config = TrestleBotConfig( - committer_email="test@test", - committer_name="test", - ) - write_to_file(config, config_path) - - runner = CliRunner() - result = runner.invoke( - upstream_cmd, - [ - "add", - "--repo-path", - repo_path, - "--config", - config_path, - "--url", - url, - "--branch", - "main", - "--exclude-model", - TEST_PROFILE, - ], - ) - - updated_config = load_from_file(config_path) - if not updated_config: - pytest.fail("Updated config not found") - return - - assert len(updated_config.upstreams) == 1 - assert updated_config.upstreams[0].url == url - assert updated_config.upstreams[0].include_models == ["*"] - assert updated_config.upstreams[0].exclude_models == [TEST_PROFILE] - assert updated_config.upstreams[0].skip_validation is False - assert ( - updated_config.committer_email == "test@test" - ) # sanity check this didn't change - - assert repo_path.joinpath(TEST_CATALOG_PATH).exists() - assert not repo_path.joinpath(TEST_PROFILE_PATH).exists() - assert result.exit_code == 0 - - -def test_upstream_sync_upstream_no_upstreams_in_config( - tmp_repo: Tuple[str, Repo] -) -> None: - """Test sync upstream with no upstreams in config file.""" - - repo_dir, repo = tmp_repo - repo_path = pathlib.Path(repo_dir) - config_path = repo_path.joinpath(".trestlebot/config.yml") - - source: str = prepare_upstream_repo() - url = f"{source}@main" - config = TrestleBotConfig( - committer_email="test@test", - committer_name="test", - ) - write_to_file(config, config_path) - - runner = CliRunner() - result = runner.invoke( - upstream_cmd, - [ - "sync", - "--repo-path", - repo_path, - "--config", - config_path, - "--url", - url, - "--branch", - "main", - ], - ) - assert ( - "No upstreams defined in trestlebot config. Use `upstream add` command." - in result.output - ) - assert result.exit_code == 1 - - -def test_upstream_sync_upstream_not_found(tmp_repo: Tuple[str, Repo]) -> None: - """Test sync upstream with url not found.""" - - repo_dir, repo = tmp_repo - repo_path = pathlib.Path(repo_dir) - config_path = repo_path.joinpath(".trestlebot/config.yml") - - source: str = prepare_upstream_repo() - url = f"{source}@main" - config = TrestleBotConfig( - committer_email="test@test", - committer_name="test", - upstreams=[UpstreamConfig(url="foo@bar")], - ) - write_to_file(config, config_path) - - runner = CliRunner() - result = runner.invoke( - upstream_cmd, - [ - "sync", - "--repo-path", - repo_path, - "--config", - config_path, - "--url", - url, - "--branch", - "main", - ], - ) - assert f"No upstream defined for {source}" in result.output - assert result.exit_code == 1 diff --git a/trestlebot/cli/commands/autosync.py b/trestlebot/cli/commands/autosync.py index 2d3eaa0e..547657bf 100644 --- a/trestlebot/cli/commands/autosync.py +++ b/trestlebot/cli/commands/autosync.py @@ -1,14 +1,14 @@ """ Autosync command""" import logging +import sys from typing import Any, List import click -from trestlebot.cli.options.autosync import autosync_options from trestlebot.cli.options.common import common_options, git_options, handle_exceptions -from trestlebot.cli.run import comma_sep_to_list -from trestlebot.cli.run import run as bot_run +from trestlebot.cli.utils import comma_sep_to_list, run_bot +from trestlebot.const import ERROR_EXIT_CODE from trestlebot.tasks.assemble_task import AssembleTask from trestlebot.tasks.authored import types from trestlebot.tasks.authored.base_authored import AuthoredObjectBase @@ -20,13 +20,15 @@ @handle_exceptions -def run(oscal_model: str, **kwargs: Any) -> None: - """Run the autosync for oscal model.""" +def autosync_model(oscal_model: str, kwargs: Any) -> None: + """Run autosync for oscal model.""" pre_tasks: List[TaskBase] = [] kwargs["working_dir"] = str(kwargs["repo_path"].resolve()) + if kwargs.get("file_patterns"): - kwargs.update({"patterns": comma_sep_to_list(kwargs["file_patterns"])}) + kwargs.update({"patterns": list(kwargs["file_patterns"])}) + model_filter: ModelFilter = ModelFilter( skip_patterns=comma_sep_to_list(kwargs.get("skip_items", "")), include_patterns=["*"], @@ -34,11 +36,12 @@ def run(oscal_model: str, **kwargs: Any) -> None: authored_object: AuthoredObjectBase = types.get_authored_object( oscal_model, kwargs["working_dir"], - kwargs.get("ssp_index_path", ""), + kwargs.get("ssp_index_path"), ) # Assuming an edit has occurred assemble would be run before regenerate. - # Adding this to the list first + # Adding this to the pre_tasks list first + if not kwargs.get("skip_assemble"): assemble_task: AssembleTask = AssembleTask( authored_object=authored_object, @@ -59,53 +62,103 @@ def run(oscal_model: str, **kwargs: Any) -> None: pre_tasks.append(regenerate_task) else: logger.info("Regeneration task skipped.") - bot_run(pre_tasks, kwargs) - - -@click.group(name="autosync", help="Autosync operations") -@click.pass_context -def autosync_cmd(ctx: click.Context) -> None: - """Command to autosync catalog, profile, compdef and ssp.""" + run_bot(pre_tasks, kwargs) -@autosync_cmd.command("ssp") +@click.command("autosync") @click.pass_context @common_options -@autosync_options @git_options @click.option( - "--ssp-index-path", - help="Path to ssp index file", + "--oscal-model", + type=click.Choice(choices=[model.value for model in types.AuthoredType]), + help="OSCAL model type for autosync.", + required=True, +) +@click.option( + "--markdown-dir", type=str, + help="Directory containing markdown files.", required=True, ) -def autosync_ssp_cmd(ctx: click.Context, ssp_index_path: str, **kwargs: Any) -> None: - kwargs.update({"ssp_index_path": ssp_index_path}) - run("ssp", **kwargs) +@click.option( + "--skip-items", + type=str, + help="Comma-separated list of glob patterns of the chosen model type \ + to skip when running tasks.", +) +@click.option( + "--skip-assemble", + help="Skip assembly task.", + is_flag=True, + default=False, + show_default=True, +) +@click.option( + "--skip-regenerate", + help="Skip regenerate task.", + is_flag=True, + default=False, + show_default=True, +) +@click.option( + "--version", + help="Version of the OSCAL model to set during assembly into JSON.", + type=str, +) +@click.option( + "--ssp-index-file", + help="Path to ssp index file. Required if --oscal-model is 'ssp'.", + type=str, + required=False, +) +def autosync_cmd(ctx: click.Context, **kwargs: Any) -> None: + """Command to autosync catalog, profile, compdef and ssp.""" + oscal_model = kwargs["oscal_model"] + markdown_dir = kwargs["markdown_dir"] + working_dir = str(kwargs["repo_path"].resolve()) -@autosync_cmd.command("compdef") -@click.pass_context -@common_options -@autosync_options -@git_options -def autosync_compdef_cmd(ctx: click.Context, **kwargs: Any) -> None: - run("compdef", **kwargs) + if oscal_model == "ssp" and not kwargs.get("ssp_index_file"): + logger.error("Trestlebot Error: Missing option '--ssp-index-file'.") + sys.exit(ERROR_EXIT_CODE) + pre_tasks: List[TaskBase] = [] -@autosync_cmd.command("catalog") -@click.pass_context -@common_options -@autosync_options -@git_options -def autosync_catalog_cmd(ctx: click.Context, **kwargs: Any) -> None: - run("catalog", **kwargs) + if kwargs.get("file_patterns"): + kwargs.update({"patterns": comma_sep_to_list(kwargs["file_patterns"])}) + model_filter: ModelFilter = ModelFilter( + skip_patterns=comma_sep_to_list(kwargs.get("skip_items", "")), + include_patterns=["*"], + ) + authored_object: AuthoredObjectBase = types.get_authored_object( + oscal_model, + working_dir, + kwargs.get("ssp_index_path", ""), + ) -@autosync_cmd.command("profile") -@click.pass_context -@common_options -@autosync_options -@git_options -def autosync_profile_cmd(ctx: click.Context, **kwargs: Any) -> None: - run("profile", **kwargs) + # Assuming an edit has occurred assemble would be run before regenerate. + if not kwargs.get("skip_assemble"): + assemble_task: AssembleTask = AssembleTask( + authored_object=authored_object, + markdown_dir=markdown_dir, + version=kwargs.get("version", ""), + model_filter=model_filter, + ) + pre_tasks.append(assemble_task) + else: + logger.info("Assemble task skipped.") + + if not kwargs.get("skip_regenerate"): + regenerate_task: RegenerateTask = RegenerateTask( + authored_object=authored_object, + markdown_dir=markdown_dir, + model_filter=model_filter, + ) + pre_tasks.append(regenerate_task) + else: + logger.info("Regeneration task skipped.") + + results = run_bot(pre_tasks, kwargs) + logger.debug(f"Trestlebot results: {results}") diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index 844c483c..5214203f 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -10,7 +10,7 @@ from trestlebot import const from trestlebot.cli.options.common import common_options, handle_exceptions from trestlebot.cli.options.create import common_create_options -from trestlebot.cli.run import run +from trestlebot.cli.utils import run_bot from trestlebot.tasks.assemble_task import AssembleTask from trestlebot.tasks.authored.compdef import ( AuthoredComponentDefinition, @@ -118,7 +118,7 @@ def compdef_cmd( ) pre_tasks.append(regenerate_task) - run(pre_tasks, kwargs) + run_bot(pre_tasks, kwargs) for key, value in kwargs.items(): logger.info(f"{key}: {value}") @@ -200,7 +200,7 @@ def ssp_cmd( pre_tasks: List[TaskBase] = [assemble_task] - run(pre_tasks, kwargs) + run_bot(pre_tasks, kwargs) logger.info(f"The name of the profile in use with the SSP is {profile_name}.") logger.info(f"The SSP index path is {ssp_index_path}.") diff --git a/trestlebot/cli/commands/init.py b/trestlebot/cli/commands/init.py index d06f0d50..a0ef718c 100644 --- a/trestlebot/cli/commands/init.py +++ b/trestlebot/cli/commands/init.py @@ -89,6 +89,14 @@ def mkdir_with_hidden_file(file_path: pathlib.Path) -> None: show_default=False, prompt="Enter default user email for Git commits (press to skip)", ) +@click.option( + "--default-commit-message", + type=str, + help="Default message for Git commits.", + default="", + show_default=False, + prompt="Enter default message for Git commits (press to skip)", +) @click.option( "--default-branch", type=str, @@ -106,6 +114,7 @@ def init_cmd( markdown_dir: str, default_committer_name: str, default_committer_email: str, + default_commit_message: str, default_branch: str, ) -> None: """Command to initialize a new trestlebot repo""" @@ -160,6 +169,9 @@ def init_cmd( if default_committer_email: config_values.update(committer_email=default_committer_email) + if default_commit_message: + config_values.update(commit_message=default_commit_message) + if default_branch: config_values.update(branch=default_branch) diff --git a/trestlebot/cli/commands/sync_upstreams.py b/trestlebot/cli/commands/sync_upstreams.py new file mode 100644 index 00000000..2cfef7b0 --- /dev/null +++ b/trestlebot/cli/commands/sync_upstreams.py @@ -0,0 +1,130 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Red Hat, Inc. + +"""Module for upstream command""" +import logging +import sys +from typing import Any, List + +import click + +from trestlebot.bot import TrestleBot +from trestlebot.cli.options.common import common_options, git_options, handle_exceptions +from trestlebot.cli.utils import comma_sep_to_list +from trestlebot.const import ERROR_EXIT_CODE +from trestlebot.tasks.base_task import ModelFilter, TaskBase +from trestlebot.tasks.sync_upstreams_task import SyncUpstreamsTask + + +logger = logging.getLogger(__name__) + + +def load_value_from_ctx( + ctx: click.Context, param: click.Parameter, value: Any = None +) -> Any: + """Load config value for option from context.""" + if value: + return value + + if not ctx.default_map: + return None + + upstreams = ctx.default_map.get("upstreams") + if not upstreams: + return None + + config = upstreams.model_dump() + value = config.get(param.name) + if isinstance(value, List): + return ",".join(value) + return value + + +@click.command( + name="sync-upstreams", + help="sync and validate OSCAL content from upstream repositories.", +) +@click.pass_context +@click.option( + "--sources", + type=str, + help="Comma-separated list of upstream git sources to sync. Each source is a string \ + in the form @ where ref is a git ref such as a tag or branch.", + envvar="TRESTLEBOT_UPSTREAMS_SOURCES", + callback=load_value_from_ctx, + required=False, +) +@click.option( + "--exclude-models", + type=str, + help="Comma-separated list of glob patterns for model names to exclude when running \ + tasks (e.g. --include-models='component_x,profile_y*')", + required=False, + envvar="TRESTLEBOT_UPSTREAMS_EXCLUDE_MODELS", + callback=load_value_from_ctx, +) +@click.option( + "--include-models", + type=str, + default="*", + help="Comma-separated list of glob patterns for model names to include when running \ + tasks (e.g. --include-models='component_x,profile_y*')", + required=False, + envvar="TRESTLEBOT_UPSTREAMS_INCLUDE_MODELS", + callback=load_value_from_ctx, +) +@click.option( + "--skip-validation", + type=bool, + help="Skip validation of the models when they are copied.", + is_flag=True, + envvar="TRESTLEBOT_UPSTREAMS_SKIP_VALIDATION", + callback=load_value_from_ctx, +) +@common_options +@git_options +@handle_exceptions +def sync_upstreams_cmd(ctx: click.Context, **kwargs: Any) -> None: + """Add new upstream sources to workspace.""" + if not kwargs.get("sources"): + logger.error("Trestlebot Error: Missing option '--sources'.") + sys.exit(ERROR_EXIT_CODE) + + working_dir = str(kwargs["repo_path"].resolve()) + include_model_list = comma_sep_to_list(kwargs["include_models"]) + + model_filter: ModelFilter = ModelFilter( + skip_patterns=comma_sep_to_list(kwargs.get("exclude_models", "")), + include_patterns=include_model_list, + ) + + validate: bool = not kwargs.get("skip_validation", False) + + sync_upstreams_task = SyncUpstreamsTask( + working_dir=working_dir, + git_sources=comma_sep_to_list(kwargs["sources"]), + model_filter=model_filter, + validate=validate, + ) + + pre_tasks: List[TaskBase] = [sync_upstreams_task] + + bot = TrestleBot( + working_dir=working_dir, + branch=kwargs["branch"], + commit_name=kwargs["committer_name"], + commit_email=kwargs["committer_email"], + author_name=kwargs.get("author_name", ""), + author_email=kwargs.get("author_email", ""), + target_branch=kwargs.get("target_branch", ""), + ) + + results = bot.run( + patterns=["*.json"], + pre_tasks=pre_tasks, + commit_message=kwargs.get("commit_message", ""), + pull_request_title=kwargs.get("pull_request_title", ""), + dry_run=kwargs.get("dry_run", False), + ) + + logger.debug(f"Trestlebot results: {results}") diff --git a/trestlebot/cli/commands/upstream.py b/trestlebot/cli/commands/upstream.py deleted file mode 100644 index 0bac09cc..00000000 --- a/trestlebot/cli/commands/upstream.py +++ /dev/null @@ -1,237 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# Copyright (c) 2024 Red Hat, Inc. - -"""Module for upstream command""" -import logging -import pathlib -import sys -from typing import Any, List, Optional - -import click - -from trestlebot.cli.config import ( - TrestleBotConfig, - UpstreamConfig, - load_from_file, - make_config, - write_to_file, -) -from trestlebot.cli.options.common import common_options, git_options, handle_exceptions -from trestlebot.cli.run import run -from trestlebot.const import ERROR_EXIT_CODE -from trestlebot.reporter import BotResults -from trestlebot.tasks.base_task import ModelFilter -from trestlebot.tasks.sync_upstreams_task import SyncUpstreamsTask - - -logger = logging.getLogger(__name__) - - -def sync_upstream_for_url( - repo_path: pathlib.Path, - dry_run: bool, - url: str, - include_models: List[str], - exclude_models: List[str], - skip_validation: bool, - committer_name: str, - committer_email: str, - branch: str, -) -> BotResults: - """Invoke the sync upstream task for a given URL.""" - model_filter: ModelFilter = ModelFilter( - include_patterns=include_models, skip_patterns=exclude_models - ) - - validate = not skip_validation - - sync_upstreams_task = SyncUpstreamsTask( - working_dir=str(repo_path.resolve()), - git_sources=[url], - model_filter=model_filter, - validate=validate, - ) - kwargs = dict( - working_dir=(str(repo_path.resolve())), - committer_name=committer_name, - committer_email=committer_email, - branch=branch, - dry_run=dry_run, - ) - return run([sync_upstreams_task], kwargs) - - -def _config_get_upstream_by_url( - config: TrestleBotConfig, url: str -) -> Optional[UpstreamConfig]: - """Returns one upstream that matches url, else None.""" - if config.upstreams: - for upstream in config.upstreams: - if upstream.url == url: - return upstream - return None - - -@click.group(name="upstream") -@click.pass_context -def upstream_cmd(ctx: click.Context) -> None: - """Sync content from upstream git repositories.""" - - -@upstream_cmd.command( - name="add", help="Add new upstream repository and download content." -) -@click.pass_context -@common_options -@git_options -@click.option( - "--url", - type=str, - help="Upstream repo URL in the form @ where ref is a git ref such as tag or branch.", - required=True, - multiple=True, -) -@click.option( - "--exclude-model", - type=str, - help="Glob pattern for model names to exclude (e.g. --exclude-model=profile_y*).", - required=False, - multiple=True, -) -@click.option( - "--include-model", - type=str, - help="Glob pattern for model names to include (e.g. --include-model=profile_y*).", - required=False, - multiple=True, -) -@click.option( - "--skip-validation", - type=bool, - help="Skip validation of the models when they are copied.", - is_flag=True, -) -@handle_exceptions -def upstream_add_cmd(ctx: click.Context, **kwargs: Any) -> None: - """Add new upstream sources to workspace.""" - - repo_path = kwargs["repo_path"] - url_list = list(kwargs["url"]) - - config_path: pathlib.Path = kwargs["config_path"] - config = load_from_file(config_path) - if not config: - # No config exists or was not found, create a new one - logger.warning( - f"No trestlebot config file found, creating {str(config_path.resolve())}" - ) - config = make_config() - - for url in url_list: - existing_urls = [upstream.url for upstream in config.upstreams] - if url in existing_urls: - logger.warning( - f"Upstream for {url} already exists. Edit {config_path} to update." - ) - continue - - include_models = list(kwargs.get("include_model", ["*"])) - if len(include_models) == 0: - include_models = [ - "*" - ] # This needs to be set otherwise the task will not include any models - exclude_models = list(kwargs.get("exclude_model", [])) - skip_validation = kwargs.get("skip_validation", False) - result = sync_upstream_for_url( - repo_path=repo_path, - dry_run=kwargs["dry_run"], - url=url, - include_models=include_models, - exclude_models=exclude_models, - skip_validation=skip_validation, - committer_name=kwargs["committer_name"], - committer_email=kwargs["committer_email"], - branch=kwargs["branch"], - ) - logger.debug(f"Bot results for {url}: {result}") - - config.upstreams.append( - UpstreamConfig( - url=url, - include_models=include_models, - exclude_models=exclude_models, - skip_validation=skip_validation, - ) - ) - logger.info(f"Added {url} to trestlebot workspace") - - write_to_file(config, config_path) - - -@upstream_cmd.command( - name="sync", help="Sync content from an added upstream repository." -) -@click.pass_context -@common_options -@git_options -@click.option( - "--url", - type=str, - help="Upstream repo URL in the form @ where ref is a git ref such as tag or branch.", - required=False, - multiple=True, -) -@click.option( - "--all", - type=str, - help="URL to GitHub repo containing upstream content.", - required=False, - is_flag=True, -) -@handle_exceptions -def upstream_sync_cmd(ctx: click.Context, **kwargs: Any) -> None: - """Sync upstream sources to local workspace""" - - config_path: pathlib.Path = kwargs["config_path"] - config = load_from_file(config_path) - if not config or len(config.upstreams) == 0: - logger.error( - "No upstreams defined in trestlebot config. Use `upstream add` command." - ) - sys.exit(ERROR_EXIT_CODE) - - upstreams_to_sync = [] - if kwargs.get("all"): - upstreams_to_sync = config.upstreams - - elif urls := kwargs.get("url"): - for url in urls: - if upstream := _config_get_upstream_by_url(config, url): - upstreams_to_sync.append(upstream) - - else: - logger.error(f"No upstream defined for {url} - skipping!") - - else: - logger.error("Must specify --url or --all to sync all upstreams") - sys.exit(ERROR_EXIT_CODE) - - if len(upstreams_to_sync) == 0: - logger.error("No upstreams found to sync.") - sys.exit(ERROR_EXIT_CODE) - - for upstream in upstreams_to_sync: - - result = sync_upstream_for_url( - repo_path=kwargs["repo_path"], - dry_run=kwargs["dry_run"], - url=upstream.url, - include_models=upstream.include_models, - exclude_models=upstream.exclude_models, - skip_validation=upstream.skip_validation, - committer_name=kwargs["committer_name"], - committer_email=kwargs["committer_email"], - branch=kwargs["branch"], - ) - logger.debug(f"Bot results for {upstream.url}: {result}") - logger.info("Sync from upstreams complete") diff --git a/trestlebot/cli/config.py b/trestlebot/cli/config.py index 1245ffc0..f7cc7039 100644 --- a/trestlebot/cli/config.py +++ b/trestlebot/cli/config.py @@ -43,10 +43,10 @@ def __str__(self) -> str: return "".join(self.errors) -class UpstreamConfig(BaseModel): +class UpstreamsConfig(BaseModel): """Data model for upstream sources.""" - url: str + sources: List[str] include_models: List[str] = ["*"] exclude_models: List[str] = [] skip_validation: bool = False @@ -59,9 +59,10 @@ class TrestleBotConfig(BaseModel): markdown_dir: Optional[str] = None committer_name: Optional[str] = None committer_email: Optional[str] = None + commit_message: Optional[str] = None branch: Optional[str] = None ssp_index_file: Optional[str] = None - upstreams: List[UpstreamConfig] = [] + upstreams: Optional[UpstreamsConfig] = None def to_yaml_dict(self) -> Dict[str, Any]: """Returns a dict that can be cleanly written to a yaml file. @@ -77,32 +78,31 @@ def to_yaml_dict(self) -> Dict[str, Any]: been set (or have a default we want to include). Values listed in IGNORED_VALUES will be skipped. - """ - IGNORED_VALUES: List[Any] = [None, "None", []] - upstreams = [] - for upstream in self.upstreams: - upstream_dict = { - "url": upstream.url, - "skip_validation": upstream.skip_validation, - } - if upstream.include_models: - upstream_dict.update(include_models=upstream.include_models) - if upstream.exclude_models: - upstream_dict.update(exclude_models=upstream.exclude_models) - upstreams.append(upstream_dict) + IGNORED_VALUES: List[Any] = [None, "None", []] - config_dict = { + config_dict: Dict[str, Any] = { "repo_path": str(self.repo_path), "markdown_dir": self.markdown_dir, "ssp_index_file": self.ssp_index_file, "committer_name": self.committer_name, "committer_email": self.committer_email, + "commit_message": self.commit_message, "branch": self.branch, - "upstreams": upstreams, } + if self.upstreams: + upstreams = { + "sources": self.upstreams.sources, + "skip_validation": self.upstreams.skip_validation, + "include_models": self.upstreams.include_models, + } + if self.upstreams.exclude_models: + upstreams["exclude_models"] = self.upstreams.exclude_models + + config_dict.update({"upstreams": upstreams}) + # Filter out emtpy values to prevent them from appearing in the config return dict( filter(lambda item: item[1] not in IGNORED_VALUES, config_dict.items()) diff --git a/trestlebot/cli/options/autosync.py b/trestlebot/cli/options/autosync.py index d2a41eb8..c96f804c 100644 --- a/trestlebot/cli/options/autosync.py +++ b/trestlebot/cli/options/autosync.py @@ -12,38 +12,34 @@ def autosync_options(f: F) -> F: """ f = click.option( - "--markdown-path", - help="Path to Trestle markdown files", - type=click.Path(exists=True), - required=True, + "--markdown-dir", + type=str, + help="Directory containing markdown files.", )(f) f = click.option( "--skip-items", - help="Comma-separated list of glob patterns to skip when running tasks", type=str, + help="Comma-separated list of glob patterns of the chosen model type \ + to skip when running tasks.", + multiple=True, )(f) f = click.option( "--skip-assemble", - help="Skip assembly task", + help="Skip assembly task.", is_flag=True, default=False, show_default=True, )(f) f = click.option( "--skip-regenerate", - help="Skip regenerate task", + help="Skip regenerate task.", is_flag=True, default=False, show_default=True, )(f) f = click.option( "--version", - help="Version of the OSCAL model to set during assembly into JSON", + help="Version of the OSCAL model to set during assembly into JSON.", type=str, )(f) - f = click.option( - "--dry-run", - help="Run tasks, but do not push to the repository", - is_flag=True, - )(f) return f diff --git a/trestlebot/cli/options/common.py b/trestlebot/cli/options/common.py index 21c95690..f812e770 100644 --- a/trestlebot/cli/options/common.py +++ b/trestlebot/cli/options/common.py @@ -76,10 +76,14 @@ def load_config_to_ctx( if not ctx.default_map: ctx.default_map = { + "repo_path": config.repo_path, "markdown_dir": config.markdown_dir, "ssp_index_file": config.ssp_index_file, "committer_name": config.committer_name, "committer_email": config.committer_email, + "commit_message": config.commit_message, + "branch": config.branch, + "upstreams": config.upstreams, } else: ctx.default_map.update(config) @@ -134,19 +138,26 @@ def git_options(f: F) -> F: """ f = click.option( "--branch", - help="Branch name to push changes to", + help="Git repo branch to push automated changes.", required=True, type=str, )(f) + f = click.option( + "--target-branch", + help="Target branch (base branch) to create a pull request against. \ + No pull request is created if unset", + required=False, + type=str, + )(f) f = click.option( "--committer-name", - help="Name of committer", + help="User name for git committer.", required=True, type=str, )(f) f = click.option( "--committer-email", - help="Email for committer", + help="User email for git committer", required=True, type=str, )(f) @@ -154,20 +165,22 @@ def git_options(f: F) -> F: "--file-patterns", help="Comma-separated list of file patterns to be used with `git add` in repository updates", type=str, + default=".", )(f) f = click.option( "--commit-message", - help="Commit message for automated updates", + help="Commit message for automated updates.", + default="Automatic updates from trestle-bot", type=str, )(f) f = click.option( "--author-name", - help="Name for commit author if differs from committer", + help="Name for commit author if differs from committer.", type=str, )(f) f = click.option( "--author-email", - help="Email for commit author if differs from committer", + help="Email for commit author if differs from committer.", type=str, )(f) diff --git a/trestlebot/cli/root.py b/trestlebot/cli/root.py index baa0dc30..9d9f463e 100644 --- a/trestlebot/cli/root.py +++ b/trestlebot/cli/root.py @@ -9,7 +9,7 @@ from trestlebot.cli.commands.autosync import autosync_cmd from trestlebot.cli.commands.create import create_cmd from trestlebot.cli.commands.init import init_cmd -from trestlebot.cli.commands.upstream import upstream_cmd +from trestlebot.cli.commands.sync_upstreams import sync_upstreams_cmd EPILOG = "See our docs at https://redhatproductsecurity.github.io/trestle-bot" @@ -31,4 +31,4 @@ def root_cmd(ctx: click.Context) -> None: root_cmd.add_command(init_cmd) root_cmd.add_command(create_cmd) root_cmd.add_command(autosync_cmd) -root_cmd.add_command(upstream_cmd) +root_cmd.add_command(sync_upstreams_cmd) diff --git a/trestlebot/cli/run.py b/trestlebot/cli/utils.py similarity index 74% rename from trestlebot/cli/run.py rename to trestlebot/cli/utils.py index 75c97c81..7cae749d 100644 --- a/trestlebot/cli/run.py +++ b/trestlebot/cli/utils.py @@ -11,12 +11,12 @@ def comma_sep_to_list(string: str) -> List[str]: return list(map(str.strip, string.split(","))) if string else [] -def run(pre_tasks: List[TaskBase], kwargs: Dict[Any, Any]) -> BotResults: +def run_bot(pre_tasks: List[TaskBase], kwargs: Dict[Any, Any]) -> BotResults: """Reusable logic for all commands.""" # Configure and run the bot bot = TrestleBot( - working_dir=kwargs["working_dir"], + working_dir=kwargs["repo_path"], branch=kwargs["branch"], commit_name=kwargs["committer_name"], commit_email=kwargs["committer_email"], @@ -26,7 +26,9 @@ def run(pre_tasks: List[TaskBase], kwargs: Dict[Any, Any]) -> BotResults: return bot.run( pre_tasks=pre_tasks, - patterns=comma_sep_to_list(kwargs.get("patterns", "")), - commit_message=kwargs.get("commit_message", "Automatic updates from bot"), + patterns=kwargs.get("patterns", ["."]), + commit_message=kwargs.get( + "commit_message", "Automatic updates from trestle-bot" + ), dry_run=kwargs.get("dry_run", False), ) From 40d60090db513d41524031e5fd38642709ad60c0 Mon Sep 17 00:00:00 2001 From: George Vauter Date: Mon, 2 Dec 2024 19:11:08 -0500 Subject: [PATCH 046/108] refactor sync upstreams and autosync to match existing entrypoint syntax --- tests/trestlebot/cli/test_autosync_cmd.py | 27 +- tests/trestlebot/cli/test_config.py | 21 +- .../trestlebot/cli/test_sync_upstreams_cmd.py | 157 +++++++++++ tests/trestlebot/cli/test_upstreams_cmd.py | 249 ------------------ trestlebot/cli/commands/autosync.py | 139 +++++++--- trestlebot/cli/commands/create.py | 6 +- trestlebot/cli/commands/init.py | 12 + trestlebot/cli/commands/sync_upstreams.py | 130 +++++++++ trestlebot/cli/commands/upstream.py | 237 ----------------- trestlebot/cli/config.py | 36 +-- trestlebot/cli/options/autosync.py | 22 +- trestlebot/cli/options/common.py | 25 +- trestlebot/cli/root.py | 4 +- trestlebot/cli/{run.py => utils.py} | 10 +- 14 files changed, 482 insertions(+), 593 deletions(-) create mode 100644 tests/trestlebot/cli/test_sync_upstreams_cmd.py delete mode 100644 tests/trestlebot/cli/test_upstreams_cmd.py create mode 100644 trestlebot/cli/commands/sync_upstreams.py delete mode 100644 trestlebot/cli/commands/upstream.py rename trestlebot/cli/{run.py => utils.py} (74%) diff --git a/tests/trestlebot/cli/test_autosync_cmd.py b/tests/trestlebot/cli/test_autosync_cmd.py index d044bee3..836ade07 100644 --- a/tests/trestlebot/cli/test_autosync_cmd.py +++ b/tests/trestlebot/cli/test_autosync_cmd.py @@ -10,16 +10,18 @@ from trestlebot.cli.commands.autosync import autosync_cmd -def test_invalid_autosync_command(tmp_init_dir: str) -> None: +def test_invalid_oscal_model(tmp_init_dir: str) -> None: + """Test invalid OSCAl model option""" tempdir = tempfile.mkdtemp() runner = CliRunner() result = runner.invoke( autosync_cmd, [ + "--oscal-model", "invalid", "--repo-path", tmp_init_dir, - "--markdown-path", + "--markdown-dir", tempdir, "--branch", "main", @@ -29,20 +31,21 @@ def test_invalid_autosync_command(tmp_init_dir: str) -> None: "test@example.com", ], ) - assert "Error: No such command" in result.output + assert "Invalid value for '--oscal-model'" in result.output assert result.exit_code == 2 def test_no_ssp_index_path(tmp_init_dir: str) -> None: - """Test invalid ssp index file for autosync ssp""" + """Test missing ssp-index-file for autosync ssp.""" tempdir = tempfile.mkdtemp() runner = CliRunner() cmd_options = [ + "--oscal-model", "ssp", "--repo-path", tmp_init_dir, - "--markdown-path", + "--markdown-dir", tempdir, "--branch", "main", @@ -52,16 +55,16 @@ def test_no_ssp_index_path(tmp_init_dir: str) -> None: "test@example.com", ] result = runner.invoke(autosync_cmd, cmd_options) - assert result.exit_code == 2 - assert "Error: Missing option '--ssp-index-path'" in result.output - cmd_options[0] = "compdef" - result = runner.invoke(autosync_cmd, cmd_options) - assert result.exit_code == 0 + assert result.exit_code == 1 + assert "Missing option '--ssp-index-file'" in result.output + +def test_missing_markdown_dir_option(tmp_init_dir: str) -> None: + """Test autosync required markdown-dir option.""" -def test_no_markdown_path(tmp_init_dir: str) -> None: runner = CliRunner() cmd_options = [ + "--oscal-model", "compdef", "--repo-path", tmp_init_dir, @@ -74,4 +77,4 @@ def test_no_markdown_path(tmp_init_dir: str) -> None: ] result = runner.invoke(autosync_cmd, cmd_options) assert result.exit_code == 2 - assert "Error: Missing option '--markdown-path'" in result.output + assert "Error: Missing option '--markdown-dir'" in result.output diff --git a/tests/trestlebot/cli/test_config.py b/tests/trestlebot/cli/test_config.py index 2b303471..60a40210 100644 --- a/tests/trestlebot/cli/test_config.py +++ b/tests/trestlebot/cli/test_config.py @@ -11,6 +11,7 @@ from trestlebot.cli.config import ( TrestleBotConfig, TrestleBotConfigError, + UpstreamsConfig, load_from_file, make_config, write_to_file, @@ -19,7 +20,11 @@ @pytest.fixture def config_obj() -> TrestleBotConfig: - return TrestleBotConfig(repo_path="/tmp", markdown_dir="markdown") + return TrestleBotConfig( + repo_path="/tmp", + markdown_dir="markdown", + upstreams=UpstreamsConfig(sources=["repo@main"]), + ) def test_invalid_config_raises_errors() -> None: @@ -39,15 +44,19 @@ def test_make_config_raises_no_errors(tmp_init_dir: str) -> None: values = { "repo_path": tmp_init_dir, "markdown_dir": "markdown", - "upstreams": [{"url": "https://test@main", "skip_validation": True}], + "committer_name": "committer-name", + "committer_email": "committer-email", + "upstreams": {"sources": ["https://test@main"], "skip_validation": True}, } config = make_config(values) assert isinstance(config, TrestleBotConfig) - assert len(config.upstreams) == 1 - assert config.upstreams[0].url == "https://test@main" - assert config.upstreams[0].skip_validation is True + assert config.upstreams is not None + assert config.upstreams.sources == ["https://test@main"] + assert config.upstreams.skip_validation is True assert config.repo_path == pathlib.Path(tmp_init_dir) - assert config.markdown_dir == "markdown" + assert config.markdown_dir == values["markdown_dir"] + assert config.committer_name == values["committer_name"] + assert config.committer_email == values["committer_email"] def test_config_write_to_file(config_obj: TrestleBotConfig, tmp_init_dir: str) -> None: diff --git a/tests/trestlebot/cli/test_sync_upstreams_cmd.py b/tests/trestlebot/cli/test_sync_upstreams_cmd.py new file mode 100644 index 00000000..b3e9e1ba --- /dev/null +++ b/tests/trestlebot/cli/test_sync_upstreams_cmd.py @@ -0,0 +1,157 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Red Hat, Inc. + + +"""Unit tests for upstreams commands.""" + +import pathlib +from typing import Tuple + +from click.testing import CliRunner +from git import Repo + +from tests.testutils import clean, prepare_upstream_repo, setup_for_init +from trestlebot.cli.commands.sync_upstreams import sync_upstreams_cmd +from trestlebot.cli.config import make_config, write_to_file +from trestlebot.const import TRESTLEBOT_CONFIG_DIR + + +TEST_CATALOG = "simplified_nist_catalog" +TEST_CATALOG_PATH = "catalogs/simplified_nist_catalog/catalog.json" +TEST_PROFILE = "simplified_nist_profile" +TEST_PROFILE_PATH = "profiles/simplified_nist_profile/profile.json" + + +def test_sync_upstreams(tmp_repo: Tuple[str, Repo]) -> None: + """Test sync upstreams""" + repo_dir, _ = tmp_repo + repo_path = pathlib.Path(repo_dir) + + source: str = prepare_upstream_repo() + + runner = CliRunner() + result = runner.invoke( + sync_upstreams_cmd, + [ + "--repo-path", + repo_path, + "--sources", + f"{source}@main", + "--branch", + "main", + "--committer-email", + "test@email", + "--committer-name", + "test name", + ], + ) + + assert repo_path.joinpath(TEST_CATALOG_PATH).exists() + assert repo_path.joinpath(TEST_PROFILE_PATH).exists() + assert result.exit_code == 0 + clean(source) + + +def test_sync_upstreams_with_config( + tmp_repo: Tuple[str, Repo], tmp_init_dir: str +) -> None: + """Test sync upstreams using a config file.""" + + repo_dir, _ = tmp_repo + repo_path = pathlib.Path(repo_dir) + + source: str = prepare_upstream_repo() + + trestlebot_dir = pathlib.Path(tmp_init_dir) / pathlib.Path(TRESTLEBOT_CONFIG_DIR) + trestlebot_dir.mkdir() + + setup_for_init(pathlib.Path(tmp_init_dir)) + + config_path = ( + pathlib.Path(tmp_init_dir) + .joinpath(TRESTLEBOT_CONFIG_DIR) + .joinpath("config.yml") + ) + + config = make_config( + { + "branch": "main", + "committer_name": "test name", + "committer_email": "test@email", + "upstreams": { + "sources": [f"{source}@main"], + "include_models": ["*"], + "skip_validation": False, + }, + } + ) + + write_to_file(config, config_path) + + runner = CliRunner() + result = runner.invoke( + sync_upstreams_cmd, + ["--repo-path", repo_path, "--config", config_path.resolve()], + ) + assert repo_path.joinpath(TEST_CATALOG_PATH).exists() + assert repo_path.joinpath(TEST_PROFILE_PATH).exists() + assert result.exit_code == 0 + clean(source) + + +def test_sync_upstreams_exclude_models(tmp_repo: Tuple[str, Repo]) -> None: + """Test sync upstreams with exclude models""" + + repo_dir, _ = tmp_repo + repo_path = pathlib.Path(repo_dir) + + source: str = prepare_upstream_repo() + + runner = CliRunner() + result = runner.invoke( + sync_upstreams_cmd, + [ + "--repo-path", + repo_path, + "--sources", + f"{source}@main", + "--branch", + "main", + "--exclude-models", + TEST_PROFILE, + "--committer-email", + "test@email", + "--committer-name", + "test", + ], + ) + + assert repo_path.joinpath(TEST_CATALOG_PATH).exists() + assert not repo_path.joinpath(TEST_PROFILE_PATH).exists() + assert result.exit_code == 0 + clean(source) + + +def test_sync_upstreams_no_sources(tmp_repo: Tuple[str, Repo]) -> None: + """Test sync upstreams with sources option missing""" + + repo_dir, _ = tmp_repo + repo_path = pathlib.Path(repo_dir) + + runner = CliRunner() + result = runner.invoke( + sync_upstreams_cmd, + [ + "--repo-path", + repo_path, + "--branch", + "main", + "--committer-name", + "test name", + "--committer-email", + "test@email", + ], + ) + + assert "Missing option '--sources'" in result.output + assert result.exit_code == 1 diff --git a/tests/trestlebot/cli/test_upstreams_cmd.py b/tests/trestlebot/cli/test_upstreams_cmd.py deleted file mode 100644 index 11e05a19..00000000 --- a/tests/trestlebot/cli/test_upstreams_cmd.py +++ /dev/null @@ -1,249 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# Copyright (c) 2024 Red Hat, Inc. - - -"""Unit tests for upstreams commands.""" - -import pathlib -from typing import Tuple - -import pytest -from click.testing import CliRunner -from git import Repo - -from tests.testutils import clean, prepare_upstream_repo -from trestlebot.cli.commands.upstream import upstream_cmd -from trestlebot.cli.config import ( - TrestleBotConfig, - UpstreamConfig, - load_from_file, - write_to_file, -) - - -TEST_CATALOG = "simplified_nist_catalog" -TEST_CATALOG_PATH = "catalogs/simplified_nist_catalog/catalog.json" -TEST_PROFILE = "simplified_nist_profile" -TEST_PROFILE_PATH = "profiles/simplified_nist_profile/profile.json" - - -def test_config_get_upstream_by_url() -> None: - """Test to confirm _get_upstream_by_url returns a single valid upstream from the config.""" - config = TrestleBotConfig(upstreams=[UpstreamConfig(url="https://test")]) - - assert len(config.upstreams) == 1 - assert config.upstreams[0].url == "https://test" - assert config.upstreams[0].include_models == ["*"] # confirm default was set - assert config.upstreams[0].exclude_models == [] - assert config.upstreams[0].skip_validation is False - - -def test_upstream_add_already_exists(tmp_repo: Tuple[str, Repo]) -> None: - """Test upstream add fails if upstream already exists""" - repo_path, repo = tmp_repo - config_path = pathlib.Path(repo_path).joinpath(".trestlebot/config.yml") - - source: str = prepare_upstream_repo() - url = f"{source}@main" - config = TrestleBotConfig( - committer_email="test@test", - committer_name="test", - upstreams=[UpstreamConfig(url=url)], - ) - write_to_file(config, config_path) - - runner = CliRunner() - result = runner.invoke( - upstream_cmd, - [ - "add", - "--repo-path", - repo_path, - "--config", - config_path, - "--url", - url, - "--branch", - "test", - ], - ) - assert ( - result.exit_code == 0 - ) # This does return success, it just skips the URLs already in the config - - assert f"{url} already exists. Edit {config_path} to update." in result.output - - -def test_upstream_add(tmp_repo: Tuple[str, Repo]) -> None: - """Tests successful add of upstream""" - repo_dir, repo = tmp_repo - repo_path = pathlib.Path(repo_dir) - config_path = repo_path.joinpath(".trestlebot/config.yml") - - source: str = prepare_upstream_repo() - url = f"{source}@main" - config = TrestleBotConfig( - committer_email="test@test", - committer_name="test", - ) - write_to_file(config, config_path) - - runner = CliRunner() - result = runner.invoke( - upstream_cmd, - [ - "add", - "--repo-path", - repo_path, - "--config", - config_path, - "--url", - url, - "--branch", - "main", - "--debug", - ], - ) - updated_config = load_from_file(config_path) - if not updated_config: - pytest.fail("Updated config not found") - return - - assert len(updated_config.upstreams) == 1 - assert updated_config.upstreams[0].url == url - assert updated_config.upstreams[0].include_models == ["*"] - assert updated_config.upstreams[0].exclude_models == [] - assert updated_config.upstreams[0].skip_validation is False - assert ( - updated_config.committer_email == "test@test" - ) # sanity check this didn't change - assert repo_path.joinpath(TEST_CATALOG_PATH).exists() - assert repo_path.joinpath(TEST_PROFILE_PATH).exists() - assert result.exit_code == 0 - clean(source, None) - - -def test_upstream_add_exclude_models(tmp_repo: Tuple[str, Repo]) -> None: - """Test add upstream with exclude models""" - - repo_dir, repo = tmp_repo - repo_path = pathlib.Path(repo_dir) - config_path = repo_path.joinpath(".trestlebot/config.yml") - - source: str = prepare_upstream_repo() - url = f"{source}@main" - config = TrestleBotConfig( - committer_email="test@test", - committer_name="test", - ) - write_to_file(config, config_path) - - runner = CliRunner() - result = runner.invoke( - upstream_cmd, - [ - "add", - "--repo-path", - repo_path, - "--config", - config_path, - "--url", - url, - "--branch", - "main", - "--exclude-model", - TEST_PROFILE, - ], - ) - - updated_config = load_from_file(config_path) - if not updated_config: - pytest.fail("Updated config not found") - return - - assert len(updated_config.upstreams) == 1 - assert updated_config.upstreams[0].url == url - assert updated_config.upstreams[0].include_models == ["*"] - assert updated_config.upstreams[0].exclude_models == [TEST_PROFILE] - assert updated_config.upstreams[0].skip_validation is False - assert ( - updated_config.committer_email == "test@test" - ) # sanity check this didn't change - - assert repo_path.joinpath(TEST_CATALOG_PATH).exists() - assert not repo_path.joinpath(TEST_PROFILE_PATH).exists() - assert result.exit_code == 0 - - -def test_upstream_sync_upstream_no_upstreams_in_config( - tmp_repo: Tuple[str, Repo] -) -> None: - """Test sync upstream with no upstreams in config file.""" - - repo_dir, repo = tmp_repo - repo_path = pathlib.Path(repo_dir) - config_path = repo_path.joinpath(".trestlebot/config.yml") - - source: str = prepare_upstream_repo() - url = f"{source}@main" - config = TrestleBotConfig( - committer_email="test@test", - committer_name="test", - ) - write_to_file(config, config_path) - - runner = CliRunner() - result = runner.invoke( - upstream_cmd, - [ - "sync", - "--repo-path", - repo_path, - "--config", - config_path, - "--url", - url, - "--branch", - "main", - ], - ) - assert ( - "No upstreams defined in trestlebot config. Use `upstream add` command." - in result.output - ) - assert result.exit_code == 1 - - -def test_upstream_sync_upstream_not_found(tmp_repo: Tuple[str, Repo]) -> None: - """Test sync upstream with url not found.""" - - repo_dir, repo = tmp_repo - repo_path = pathlib.Path(repo_dir) - config_path = repo_path.joinpath(".trestlebot/config.yml") - - source: str = prepare_upstream_repo() - url = f"{source}@main" - config = TrestleBotConfig( - committer_email="test@test", - committer_name="test", - upstreams=[UpstreamConfig(url="foo@bar")], - ) - write_to_file(config, config_path) - - runner = CliRunner() - result = runner.invoke( - upstream_cmd, - [ - "sync", - "--repo-path", - repo_path, - "--config", - config_path, - "--url", - url, - "--branch", - "main", - ], - ) - assert f"No upstream defined for {source}" in result.output - assert result.exit_code == 1 diff --git a/trestlebot/cli/commands/autosync.py b/trestlebot/cli/commands/autosync.py index 2d3eaa0e..547657bf 100644 --- a/trestlebot/cli/commands/autosync.py +++ b/trestlebot/cli/commands/autosync.py @@ -1,14 +1,14 @@ """ Autosync command""" import logging +import sys from typing import Any, List import click -from trestlebot.cli.options.autosync import autosync_options from trestlebot.cli.options.common import common_options, git_options, handle_exceptions -from trestlebot.cli.run import comma_sep_to_list -from trestlebot.cli.run import run as bot_run +from trestlebot.cli.utils import comma_sep_to_list, run_bot +from trestlebot.const import ERROR_EXIT_CODE from trestlebot.tasks.assemble_task import AssembleTask from trestlebot.tasks.authored import types from trestlebot.tasks.authored.base_authored import AuthoredObjectBase @@ -20,13 +20,15 @@ @handle_exceptions -def run(oscal_model: str, **kwargs: Any) -> None: - """Run the autosync for oscal model.""" +def autosync_model(oscal_model: str, kwargs: Any) -> None: + """Run autosync for oscal model.""" pre_tasks: List[TaskBase] = [] kwargs["working_dir"] = str(kwargs["repo_path"].resolve()) + if kwargs.get("file_patterns"): - kwargs.update({"patterns": comma_sep_to_list(kwargs["file_patterns"])}) + kwargs.update({"patterns": list(kwargs["file_patterns"])}) + model_filter: ModelFilter = ModelFilter( skip_patterns=comma_sep_to_list(kwargs.get("skip_items", "")), include_patterns=["*"], @@ -34,11 +36,12 @@ def run(oscal_model: str, **kwargs: Any) -> None: authored_object: AuthoredObjectBase = types.get_authored_object( oscal_model, kwargs["working_dir"], - kwargs.get("ssp_index_path", ""), + kwargs.get("ssp_index_path"), ) # Assuming an edit has occurred assemble would be run before regenerate. - # Adding this to the list first + # Adding this to the pre_tasks list first + if not kwargs.get("skip_assemble"): assemble_task: AssembleTask = AssembleTask( authored_object=authored_object, @@ -59,53 +62,103 @@ def run(oscal_model: str, **kwargs: Any) -> None: pre_tasks.append(regenerate_task) else: logger.info("Regeneration task skipped.") - bot_run(pre_tasks, kwargs) - - -@click.group(name="autosync", help="Autosync operations") -@click.pass_context -def autosync_cmd(ctx: click.Context) -> None: - """Command to autosync catalog, profile, compdef and ssp.""" + run_bot(pre_tasks, kwargs) -@autosync_cmd.command("ssp") +@click.command("autosync") @click.pass_context @common_options -@autosync_options @git_options @click.option( - "--ssp-index-path", - help="Path to ssp index file", + "--oscal-model", + type=click.Choice(choices=[model.value for model in types.AuthoredType]), + help="OSCAL model type for autosync.", + required=True, +) +@click.option( + "--markdown-dir", type=str, + help="Directory containing markdown files.", required=True, ) -def autosync_ssp_cmd(ctx: click.Context, ssp_index_path: str, **kwargs: Any) -> None: - kwargs.update({"ssp_index_path": ssp_index_path}) - run("ssp", **kwargs) +@click.option( + "--skip-items", + type=str, + help="Comma-separated list of glob patterns of the chosen model type \ + to skip when running tasks.", +) +@click.option( + "--skip-assemble", + help="Skip assembly task.", + is_flag=True, + default=False, + show_default=True, +) +@click.option( + "--skip-regenerate", + help="Skip regenerate task.", + is_flag=True, + default=False, + show_default=True, +) +@click.option( + "--version", + help="Version of the OSCAL model to set during assembly into JSON.", + type=str, +) +@click.option( + "--ssp-index-file", + help="Path to ssp index file. Required if --oscal-model is 'ssp'.", + type=str, + required=False, +) +def autosync_cmd(ctx: click.Context, **kwargs: Any) -> None: + """Command to autosync catalog, profile, compdef and ssp.""" + oscal_model = kwargs["oscal_model"] + markdown_dir = kwargs["markdown_dir"] + working_dir = str(kwargs["repo_path"].resolve()) -@autosync_cmd.command("compdef") -@click.pass_context -@common_options -@autosync_options -@git_options -def autosync_compdef_cmd(ctx: click.Context, **kwargs: Any) -> None: - run("compdef", **kwargs) + if oscal_model == "ssp" and not kwargs.get("ssp_index_file"): + logger.error("Trestlebot Error: Missing option '--ssp-index-file'.") + sys.exit(ERROR_EXIT_CODE) + pre_tasks: List[TaskBase] = [] -@autosync_cmd.command("catalog") -@click.pass_context -@common_options -@autosync_options -@git_options -def autosync_catalog_cmd(ctx: click.Context, **kwargs: Any) -> None: - run("catalog", **kwargs) + if kwargs.get("file_patterns"): + kwargs.update({"patterns": comma_sep_to_list(kwargs["file_patterns"])}) + model_filter: ModelFilter = ModelFilter( + skip_patterns=comma_sep_to_list(kwargs.get("skip_items", "")), + include_patterns=["*"], + ) + authored_object: AuthoredObjectBase = types.get_authored_object( + oscal_model, + working_dir, + kwargs.get("ssp_index_path", ""), + ) -@autosync_cmd.command("profile") -@click.pass_context -@common_options -@autosync_options -@git_options -def autosync_profile_cmd(ctx: click.Context, **kwargs: Any) -> None: - run("profile", **kwargs) + # Assuming an edit has occurred assemble would be run before regenerate. + if not kwargs.get("skip_assemble"): + assemble_task: AssembleTask = AssembleTask( + authored_object=authored_object, + markdown_dir=markdown_dir, + version=kwargs.get("version", ""), + model_filter=model_filter, + ) + pre_tasks.append(assemble_task) + else: + logger.info("Assemble task skipped.") + + if not kwargs.get("skip_regenerate"): + regenerate_task: RegenerateTask = RegenerateTask( + authored_object=authored_object, + markdown_dir=markdown_dir, + model_filter=model_filter, + ) + pre_tasks.append(regenerate_task) + else: + logger.info("Regeneration task skipped.") + + results = run_bot(pre_tasks, kwargs) + logger.debug(f"Trestlebot results: {results}") diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index 844c483c..5214203f 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -10,7 +10,7 @@ from trestlebot import const from trestlebot.cli.options.common import common_options, handle_exceptions from trestlebot.cli.options.create import common_create_options -from trestlebot.cli.run import run +from trestlebot.cli.utils import run_bot from trestlebot.tasks.assemble_task import AssembleTask from trestlebot.tasks.authored.compdef import ( AuthoredComponentDefinition, @@ -118,7 +118,7 @@ def compdef_cmd( ) pre_tasks.append(regenerate_task) - run(pre_tasks, kwargs) + run_bot(pre_tasks, kwargs) for key, value in kwargs.items(): logger.info(f"{key}: {value}") @@ -200,7 +200,7 @@ def ssp_cmd( pre_tasks: List[TaskBase] = [assemble_task] - run(pre_tasks, kwargs) + run_bot(pre_tasks, kwargs) logger.info(f"The name of the profile in use with the SSP is {profile_name}.") logger.info(f"The SSP index path is {ssp_index_path}.") diff --git a/trestlebot/cli/commands/init.py b/trestlebot/cli/commands/init.py index d06f0d50..a0ef718c 100644 --- a/trestlebot/cli/commands/init.py +++ b/trestlebot/cli/commands/init.py @@ -89,6 +89,14 @@ def mkdir_with_hidden_file(file_path: pathlib.Path) -> None: show_default=False, prompt="Enter default user email for Git commits (press to skip)", ) +@click.option( + "--default-commit-message", + type=str, + help="Default message for Git commits.", + default="", + show_default=False, + prompt="Enter default message for Git commits (press to skip)", +) @click.option( "--default-branch", type=str, @@ -106,6 +114,7 @@ def init_cmd( markdown_dir: str, default_committer_name: str, default_committer_email: str, + default_commit_message: str, default_branch: str, ) -> None: """Command to initialize a new trestlebot repo""" @@ -160,6 +169,9 @@ def init_cmd( if default_committer_email: config_values.update(committer_email=default_committer_email) + if default_commit_message: + config_values.update(commit_message=default_commit_message) + if default_branch: config_values.update(branch=default_branch) diff --git a/trestlebot/cli/commands/sync_upstreams.py b/trestlebot/cli/commands/sync_upstreams.py new file mode 100644 index 00000000..2cfef7b0 --- /dev/null +++ b/trestlebot/cli/commands/sync_upstreams.py @@ -0,0 +1,130 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Red Hat, Inc. + +"""Module for upstream command""" +import logging +import sys +from typing import Any, List + +import click + +from trestlebot.bot import TrestleBot +from trestlebot.cli.options.common import common_options, git_options, handle_exceptions +from trestlebot.cli.utils import comma_sep_to_list +from trestlebot.const import ERROR_EXIT_CODE +from trestlebot.tasks.base_task import ModelFilter, TaskBase +from trestlebot.tasks.sync_upstreams_task import SyncUpstreamsTask + + +logger = logging.getLogger(__name__) + + +def load_value_from_ctx( + ctx: click.Context, param: click.Parameter, value: Any = None +) -> Any: + """Load config value for option from context.""" + if value: + return value + + if not ctx.default_map: + return None + + upstreams = ctx.default_map.get("upstreams") + if not upstreams: + return None + + config = upstreams.model_dump() + value = config.get(param.name) + if isinstance(value, List): + return ",".join(value) + return value + + +@click.command( + name="sync-upstreams", + help="sync and validate OSCAL content from upstream repositories.", +) +@click.pass_context +@click.option( + "--sources", + type=str, + help="Comma-separated list of upstream git sources to sync. Each source is a string \ + in the form @ where ref is a git ref such as a tag or branch.", + envvar="TRESTLEBOT_UPSTREAMS_SOURCES", + callback=load_value_from_ctx, + required=False, +) +@click.option( + "--exclude-models", + type=str, + help="Comma-separated list of glob patterns for model names to exclude when running \ + tasks (e.g. --include-models='component_x,profile_y*')", + required=False, + envvar="TRESTLEBOT_UPSTREAMS_EXCLUDE_MODELS", + callback=load_value_from_ctx, +) +@click.option( + "--include-models", + type=str, + default="*", + help="Comma-separated list of glob patterns for model names to include when running \ + tasks (e.g. --include-models='component_x,profile_y*')", + required=False, + envvar="TRESTLEBOT_UPSTREAMS_INCLUDE_MODELS", + callback=load_value_from_ctx, +) +@click.option( + "--skip-validation", + type=bool, + help="Skip validation of the models when they are copied.", + is_flag=True, + envvar="TRESTLEBOT_UPSTREAMS_SKIP_VALIDATION", + callback=load_value_from_ctx, +) +@common_options +@git_options +@handle_exceptions +def sync_upstreams_cmd(ctx: click.Context, **kwargs: Any) -> None: + """Add new upstream sources to workspace.""" + if not kwargs.get("sources"): + logger.error("Trestlebot Error: Missing option '--sources'.") + sys.exit(ERROR_EXIT_CODE) + + working_dir = str(kwargs["repo_path"].resolve()) + include_model_list = comma_sep_to_list(kwargs["include_models"]) + + model_filter: ModelFilter = ModelFilter( + skip_patterns=comma_sep_to_list(kwargs.get("exclude_models", "")), + include_patterns=include_model_list, + ) + + validate: bool = not kwargs.get("skip_validation", False) + + sync_upstreams_task = SyncUpstreamsTask( + working_dir=working_dir, + git_sources=comma_sep_to_list(kwargs["sources"]), + model_filter=model_filter, + validate=validate, + ) + + pre_tasks: List[TaskBase] = [sync_upstreams_task] + + bot = TrestleBot( + working_dir=working_dir, + branch=kwargs["branch"], + commit_name=kwargs["committer_name"], + commit_email=kwargs["committer_email"], + author_name=kwargs.get("author_name", ""), + author_email=kwargs.get("author_email", ""), + target_branch=kwargs.get("target_branch", ""), + ) + + results = bot.run( + patterns=["*.json"], + pre_tasks=pre_tasks, + commit_message=kwargs.get("commit_message", ""), + pull_request_title=kwargs.get("pull_request_title", ""), + dry_run=kwargs.get("dry_run", False), + ) + + logger.debug(f"Trestlebot results: {results}") diff --git a/trestlebot/cli/commands/upstream.py b/trestlebot/cli/commands/upstream.py deleted file mode 100644 index 0bac09cc..00000000 --- a/trestlebot/cli/commands/upstream.py +++ /dev/null @@ -1,237 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# Copyright (c) 2024 Red Hat, Inc. - -"""Module for upstream command""" -import logging -import pathlib -import sys -from typing import Any, List, Optional - -import click - -from trestlebot.cli.config import ( - TrestleBotConfig, - UpstreamConfig, - load_from_file, - make_config, - write_to_file, -) -from trestlebot.cli.options.common import common_options, git_options, handle_exceptions -from trestlebot.cli.run import run -from trestlebot.const import ERROR_EXIT_CODE -from trestlebot.reporter import BotResults -from trestlebot.tasks.base_task import ModelFilter -from trestlebot.tasks.sync_upstreams_task import SyncUpstreamsTask - - -logger = logging.getLogger(__name__) - - -def sync_upstream_for_url( - repo_path: pathlib.Path, - dry_run: bool, - url: str, - include_models: List[str], - exclude_models: List[str], - skip_validation: bool, - committer_name: str, - committer_email: str, - branch: str, -) -> BotResults: - """Invoke the sync upstream task for a given URL.""" - model_filter: ModelFilter = ModelFilter( - include_patterns=include_models, skip_patterns=exclude_models - ) - - validate = not skip_validation - - sync_upstreams_task = SyncUpstreamsTask( - working_dir=str(repo_path.resolve()), - git_sources=[url], - model_filter=model_filter, - validate=validate, - ) - kwargs = dict( - working_dir=(str(repo_path.resolve())), - committer_name=committer_name, - committer_email=committer_email, - branch=branch, - dry_run=dry_run, - ) - return run([sync_upstreams_task], kwargs) - - -def _config_get_upstream_by_url( - config: TrestleBotConfig, url: str -) -> Optional[UpstreamConfig]: - """Returns one upstream that matches url, else None.""" - if config.upstreams: - for upstream in config.upstreams: - if upstream.url == url: - return upstream - return None - - -@click.group(name="upstream") -@click.pass_context -def upstream_cmd(ctx: click.Context) -> None: - """Sync content from upstream git repositories.""" - - -@upstream_cmd.command( - name="add", help="Add new upstream repository and download content." -) -@click.pass_context -@common_options -@git_options -@click.option( - "--url", - type=str, - help="Upstream repo URL in the form @ where ref is a git ref such as tag or branch.", - required=True, - multiple=True, -) -@click.option( - "--exclude-model", - type=str, - help="Glob pattern for model names to exclude (e.g. --exclude-model=profile_y*).", - required=False, - multiple=True, -) -@click.option( - "--include-model", - type=str, - help="Glob pattern for model names to include (e.g. --include-model=profile_y*).", - required=False, - multiple=True, -) -@click.option( - "--skip-validation", - type=bool, - help="Skip validation of the models when they are copied.", - is_flag=True, -) -@handle_exceptions -def upstream_add_cmd(ctx: click.Context, **kwargs: Any) -> None: - """Add new upstream sources to workspace.""" - - repo_path = kwargs["repo_path"] - url_list = list(kwargs["url"]) - - config_path: pathlib.Path = kwargs["config_path"] - config = load_from_file(config_path) - if not config: - # No config exists or was not found, create a new one - logger.warning( - f"No trestlebot config file found, creating {str(config_path.resolve())}" - ) - config = make_config() - - for url in url_list: - existing_urls = [upstream.url for upstream in config.upstreams] - if url in existing_urls: - logger.warning( - f"Upstream for {url} already exists. Edit {config_path} to update." - ) - continue - - include_models = list(kwargs.get("include_model", ["*"])) - if len(include_models) == 0: - include_models = [ - "*" - ] # This needs to be set otherwise the task will not include any models - exclude_models = list(kwargs.get("exclude_model", [])) - skip_validation = kwargs.get("skip_validation", False) - result = sync_upstream_for_url( - repo_path=repo_path, - dry_run=kwargs["dry_run"], - url=url, - include_models=include_models, - exclude_models=exclude_models, - skip_validation=skip_validation, - committer_name=kwargs["committer_name"], - committer_email=kwargs["committer_email"], - branch=kwargs["branch"], - ) - logger.debug(f"Bot results for {url}: {result}") - - config.upstreams.append( - UpstreamConfig( - url=url, - include_models=include_models, - exclude_models=exclude_models, - skip_validation=skip_validation, - ) - ) - logger.info(f"Added {url} to trestlebot workspace") - - write_to_file(config, config_path) - - -@upstream_cmd.command( - name="sync", help="Sync content from an added upstream repository." -) -@click.pass_context -@common_options -@git_options -@click.option( - "--url", - type=str, - help="Upstream repo URL in the form @ where ref is a git ref such as tag or branch.", - required=False, - multiple=True, -) -@click.option( - "--all", - type=str, - help="URL to GitHub repo containing upstream content.", - required=False, - is_flag=True, -) -@handle_exceptions -def upstream_sync_cmd(ctx: click.Context, **kwargs: Any) -> None: - """Sync upstream sources to local workspace""" - - config_path: pathlib.Path = kwargs["config_path"] - config = load_from_file(config_path) - if not config or len(config.upstreams) == 0: - logger.error( - "No upstreams defined in trestlebot config. Use `upstream add` command." - ) - sys.exit(ERROR_EXIT_CODE) - - upstreams_to_sync = [] - if kwargs.get("all"): - upstreams_to_sync = config.upstreams - - elif urls := kwargs.get("url"): - for url in urls: - if upstream := _config_get_upstream_by_url(config, url): - upstreams_to_sync.append(upstream) - - else: - logger.error(f"No upstream defined for {url} - skipping!") - - else: - logger.error("Must specify --url or --all to sync all upstreams") - sys.exit(ERROR_EXIT_CODE) - - if len(upstreams_to_sync) == 0: - logger.error("No upstreams found to sync.") - sys.exit(ERROR_EXIT_CODE) - - for upstream in upstreams_to_sync: - - result = sync_upstream_for_url( - repo_path=kwargs["repo_path"], - dry_run=kwargs["dry_run"], - url=upstream.url, - include_models=upstream.include_models, - exclude_models=upstream.exclude_models, - skip_validation=upstream.skip_validation, - committer_name=kwargs["committer_name"], - committer_email=kwargs["committer_email"], - branch=kwargs["branch"], - ) - logger.debug(f"Bot results for {upstream.url}: {result}") - logger.info("Sync from upstreams complete") diff --git a/trestlebot/cli/config.py b/trestlebot/cli/config.py index 1245ffc0..f7cc7039 100644 --- a/trestlebot/cli/config.py +++ b/trestlebot/cli/config.py @@ -43,10 +43,10 @@ def __str__(self) -> str: return "".join(self.errors) -class UpstreamConfig(BaseModel): +class UpstreamsConfig(BaseModel): """Data model for upstream sources.""" - url: str + sources: List[str] include_models: List[str] = ["*"] exclude_models: List[str] = [] skip_validation: bool = False @@ -59,9 +59,10 @@ class TrestleBotConfig(BaseModel): markdown_dir: Optional[str] = None committer_name: Optional[str] = None committer_email: Optional[str] = None + commit_message: Optional[str] = None branch: Optional[str] = None ssp_index_file: Optional[str] = None - upstreams: List[UpstreamConfig] = [] + upstreams: Optional[UpstreamsConfig] = None def to_yaml_dict(self) -> Dict[str, Any]: """Returns a dict that can be cleanly written to a yaml file. @@ -77,32 +78,31 @@ def to_yaml_dict(self) -> Dict[str, Any]: been set (or have a default we want to include). Values listed in IGNORED_VALUES will be skipped. - """ - IGNORED_VALUES: List[Any] = [None, "None", []] - upstreams = [] - for upstream in self.upstreams: - upstream_dict = { - "url": upstream.url, - "skip_validation": upstream.skip_validation, - } - if upstream.include_models: - upstream_dict.update(include_models=upstream.include_models) - if upstream.exclude_models: - upstream_dict.update(exclude_models=upstream.exclude_models) - upstreams.append(upstream_dict) + IGNORED_VALUES: List[Any] = [None, "None", []] - config_dict = { + config_dict: Dict[str, Any] = { "repo_path": str(self.repo_path), "markdown_dir": self.markdown_dir, "ssp_index_file": self.ssp_index_file, "committer_name": self.committer_name, "committer_email": self.committer_email, + "commit_message": self.commit_message, "branch": self.branch, - "upstreams": upstreams, } + if self.upstreams: + upstreams = { + "sources": self.upstreams.sources, + "skip_validation": self.upstreams.skip_validation, + "include_models": self.upstreams.include_models, + } + if self.upstreams.exclude_models: + upstreams["exclude_models"] = self.upstreams.exclude_models + + config_dict.update({"upstreams": upstreams}) + # Filter out emtpy values to prevent them from appearing in the config return dict( filter(lambda item: item[1] not in IGNORED_VALUES, config_dict.items()) diff --git a/trestlebot/cli/options/autosync.py b/trestlebot/cli/options/autosync.py index d2a41eb8..c96f804c 100644 --- a/trestlebot/cli/options/autosync.py +++ b/trestlebot/cli/options/autosync.py @@ -12,38 +12,34 @@ def autosync_options(f: F) -> F: """ f = click.option( - "--markdown-path", - help="Path to Trestle markdown files", - type=click.Path(exists=True), - required=True, + "--markdown-dir", + type=str, + help="Directory containing markdown files.", )(f) f = click.option( "--skip-items", - help="Comma-separated list of glob patterns to skip when running tasks", type=str, + help="Comma-separated list of glob patterns of the chosen model type \ + to skip when running tasks.", + multiple=True, )(f) f = click.option( "--skip-assemble", - help="Skip assembly task", + help="Skip assembly task.", is_flag=True, default=False, show_default=True, )(f) f = click.option( "--skip-regenerate", - help="Skip regenerate task", + help="Skip regenerate task.", is_flag=True, default=False, show_default=True, )(f) f = click.option( "--version", - help="Version of the OSCAL model to set during assembly into JSON", + help="Version of the OSCAL model to set during assembly into JSON.", type=str, )(f) - f = click.option( - "--dry-run", - help="Run tasks, but do not push to the repository", - is_flag=True, - )(f) return f diff --git a/trestlebot/cli/options/common.py b/trestlebot/cli/options/common.py index 21c95690..f812e770 100644 --- a/trestlebot/cli/options/common.py +++ b/trestlebot/cli/options/common.py @@ -76,10 +76,14 @@ def load_config_to_ctx( if not ctx.default_map: ctx.default_map = { + "repo_path": config.repo_path, "markdown_dir": config.markdown_dir, "ssp_index_file": config.ssp_index_file, "committer_name": config.committer_name, "committer_email": config.committer_email, + "commit_message": config.commit_message, + "branch": config.branch, + "upstreams": config.upstreams, } else: ctx.default_map.update(config) @@ -134,19 +138,26 @@ def git_options(f: F) -> F: """ f = click.option( "--branch", - help="Branch name to push changes to", + help="Git repo branch to push automated changes.", required=True, type=str, )(f) + f = click.option( + "--target-branch", + help="Target branch (base branch) to create a pull request against. \ + No pull request is created if unset", + required=False, + type=str, + )(f) f = click.option( "--committer-name", - help="Name of committer", + help="User name for git committer.", required=True, type=str, )(f) f = click.option( "--committer-email", - help="Email for committer", + help="User email for git committer", required=True, type=str, )(f) @@ -154,20 +165,22 @@ def git_options(f: F) -> F: "--file-patterns", help="Comma-separated list of file patterns to be used with `git add` in repository updates", type=str, + default=".", )(f) f = click.option( "--commit-message", - help="Commit message for automated updates", + help="Commit message for automated updates.", + default="Automatic updates from trestle-bot", type=str, )(f) f = click.option( "--author-name", - help="Name for commit author if differs from committer", + help="Name for commit author if differs from committer.", type=str, )(f) f = click.option( "--author-email", - help="Email for commit author if differs from committer", + help="Email for commit author if differs from committer.", type=str, )(f) diff --git a/trestlebot/cli/root.py b/trestlebot/cli/root.py index baa0dc30..9d9f463e 100644 --- a/trestlebot/cli/root.py +++ b/trestlebot/cli/root.py @@ -9,7 +9,7 @@ from trestlebot.cli.commands.autosync import autosync_cmd from trestlebot.cli.commands.create import create_cmd from trestlebot.cli.commands.init import init_cmd -from trestlebot.cli.commands.upstream import upstream_cmd +from trestlebot.cli.commands.sync_upstreams import sync_upstreams_cmd EPILOG = "See our docs at https://redhatproductsecurity.github.io/trestle-bot" @@ -31,4 +31,4 @@ def root_cmd(ctx: click.Context) -> None: root_cmd.add_command(init_cmd) root_cmd.add_command(create_cmd) root_cmd.add_command(autosync_cmd) -root_cmd.add_command(upstream_cmd) +root_cmd.add_command(sync_upstreams_cmd) diff --git a/trestlebot/cli/run.py b/trestlebot/cli/utils.py similarity index 74% rename from trestlebot/cli/run.py rename to trestlebot/cli/utils.py index 75c97c81..7cae749d 100644 --- a/trestlebot/cli/run.py +++ b/trestlebot/cli/utils.py @@ -11,12 +11,12 @@ def comma_sep_to_list(string: str) -> List[str]: return list(map(str.strip, string.split(","))) if string else [] -def run(pre_tasks: List[TaskBase], kwargs: Dict[Any, Any]) -> BotResults: +def run_bot(pre_tasks: List[TaskBase], kwargs: Dict[Any, Any]) -> BotResults: """Reusable logic for all commands.""" # Configure and run the bot bot = TrestleBot( - working_dir=kwargs["working_dir"], + working_dir=kwargs["repo_path"], branch=kwargs["branch"], commit_name=kwargs["committer_name"], commit_email=kwargs["committer_email"], @@ -26,7 +26,9 @@ def run(pre_tasks: List[TaskBase], kwargs: Dict[Any, Any]) -> BotResults: return bot.run( pre_tasks=pre_tasks, - patterns=comma_sep_to_list(kwargs.get("patterns", "")), - commit_message=kwargs.get("commit_message", "Automatic updates from bot"), + patterns=kwargs.get("patterns", ["."]), + commit_message=kwargs.get( + "commit_message", "Automatic updates from trestle-bot" + ), dry_run=kwargs.get("dry_run", False), ) From 486ad1e224342588aca6712f121edb68a4b9de7c Mon Sep 17 00:00:00 2001 From: Qingmin Duanmu Date: Fri, 29 Nov 2024 09:51:08 +0800 Subject: [PATCH 047/108] Fix AttributeError, some misc updates AttributeError: 'NoneType' object has no attribute 'encode' --- tests/trestlebot/cli/test_autosync_cmd.py | 45 ++++++++++++------- ...ransform.py => test_rule_transform_cmd.py} | 0 trestlebot/cli/commands/autosync.py | 18 ++++---- trestlebot/cli/commands/rule_transform.py | 2 +- trestlebot/cli/options/autosync.py | 14 +++--- trestlebot/cli/options/common.py | 6 ++- trestlebot/cli/run.py | 2 +- 7 files changed, 47 insertions(+), 40 deletions(-) rename tests/trestlebot/cli/{test_rule_transform.py => test_rule_transform_cmd.py} (100%) diff --git a/tests/trestlebot/cli/test_autosync_cmd.py b/tests/trestlebot/cli/test_autosync_cmd.py index d044bee3..026a84de 100644 --- a/tests/trestlebot/cli/test_autosync_cmd.py +++ b/tests/trestlebot/cli/test_autosync_cmd.py @@ -3,24 +3,27 @@ """Testing module for trestlebot autosync command""" -import tempfile +import pathlib +from typing import Tuple from click.testing import CliRunner +from git import Repo from trestlebot.cli.commands.autosync import autosync_cmd +from trestlebot.cli.config import TrestleBotConfig, write_to_file -def test_invalid_autosync_command(tmp_init_dir: str) -> None: - tempdir = tempfile.mkdtemp() +def test_invalid_autosync_command(tmp_repo: Tuple[str, Repo]) -> None: + repo_path, _ = tmp_repo runner = CliRunner() result = runner.invoke( autosync_cmd, [ "invalid", "--repo-path", - tmp_init_dir, - "--markdown-path", - tempdir, + repo_path, + "--markdown-dir", + "markdown", "--branch", "main", "--committer-name", @@ -33,17 +36,16 @@ def test_invalid_autosync_command(tmp_init_dir: str) -> None: assert result.exit_code == 2 -def test_no_ssp_index_path(tmp_init_dir: str) -> None: - """Test invalid ssp index file for autosync ssp""" - - tempdir = tempfile.mkdtemp() +def test_no_ssp_index_path(tmp_repo: Tuple[str, Repo]) -> None: + """Test ssp_index_file for autosync ssp""" + repo_path, _ = tmp_repo runner = CliRunner() cmd_options = [ "ssp", "--repo-path", - tmp_init_dir, - "--markdown-path", - tempdir, + repo_path, + "--markdown-dir", + "markdown", "--branch", "main", "--committer-name", @@ -53,18 +55,19 @@ def test_no_ssp_index_path(tmp_init_dir: str) -> None: ] result = runner.invoke(autosync_cmd, cmd_options) assert result.exit_code == 2 - assert "Error: Missing option '--ssp-index-path'" in result.output + assert "Error: Missing option '--ssp-index-file'" in result.output cmd_options[0] = "compdef" result = runner.invoke(autosync_cmd, cmd_options) assert result.exit_code == 0 -def test_no_markdown_path(tmp_init_dir: str) -> None: +def test_no_markdown_dir(tmp_repo: Tuple[str, Repo]) -> None: + repo_path, _ = tmp_repo runner = CliRunner() cmd_options = [ "compdef", "--repo-path", - tmp_init_dir, + repo_path, "--branch", "main", "--committer-name", @@ -74,4 +77,12 @@ def test_no_markdown_path(tmp_init_dir: str) -> None: ] result = runner.invoke(autosync_cmd, cmd_options) assert result.exit_code == 2 - assert "Error: Missing option '--markdown-path'" in result.output + assert "Error: Missing option '--markdown-dir'" in result.output + + # With 'markdown_dir' setting in config.yml + config_obj = TrestleBotConfig(markdown_dir="markdown") + filepath = pathlib.Path(repo_path).joinpath("config.yml") + write_to_file(config_obj, filepath) + cmd_options += ["--config", str(filepath)] + result = runner.invoke(autosync_cmd, cmd_options) + assert result.exit_code == 0 diff --git a/tests/trestlebot/cli/test_rule_transform.py b/tests/trestlebot/cli/test_rule_transform_cmd.py similarity index 100% rename from tests/trestlebot/cli/test_rule_transform.py rename to tests/trestlebot/cli/test_rule_transform_cmd.py diff --git a/trestlebot/cli/commands/autosync.py b/trestlebot/cli/commands/autosync.py index 2d3eaa0e..6ff3e4bc 100644 --- a/trestlebot/cli/commands/autosync.py +++ b/trestlebot/cli/commands/autosync.py @@ -7,7 +7,6 @@ from trestlebot.cli.options.autosync import autosync_options from trestlebot.cli.options.common import common_options, git_options, handle_exceptions -from trestlebot.cli.run import comma_sep_to_list from trestlebot.cli.run import run as bot_run from trestlebot.tasks.assemble_task import AssembleTask from trestlebot.tasks.authored import types @@ -25,16 +24,16 @@ def run(oscal_model: str, **kwargs: Any) -> None: pre_tasks: List[TaskBase] = [] kwargs["working_dir"] = str(kwargs["repo_path"].resolve()) - if kwargs.get("file_patterns"): - kwargs.update({"patterns": comma_sep_to_list(kwargs["file_patterns"])}) + if kwargs.get("file_pattern"): + kwargs.update({"patterns": list(kwargs.get("file_pattern", []))}) model_filter: ModelFilter = ModelFilter( - skip_patterns=comma_sep_to_list(kwargs.get("skip_items", "")), + skip_patterns=list(kwargs.get("skip_item", [])), include_patterns=["*"], ) authored_object: AuthoredObjectBase = types.get_authored_object( oscal_model, kwargs["working_dir"], - kwargs.get("ssp_index_path", ""), + kwargs.get("ssp_index_file", ""), ) # Assuming an edit has occurred assemble would be run before regenerate. @@ -42,7 +41,7 @@ def run(oscal_model: str, **kwargs: Any) -> None: if not kwargs.get("skip_assemble"): assemble_task: AssembleTask = AssembleTask( authored_object=authored_object, - markdown_dir=kwargs["markdown_path"], + markdown_dir=kwargs["markdown_dir"], version=kwargs.get("version", ""), model_filter=model_filter, ) @@ -53,7 +52,7 @@ def run(oscal_model: str, **kwargs: Any) -> None: if not kwargs.get("skip_regenerate"): regenerate_task: RegenerateTask = RegenerateTask( authored_object=authored_object, - markdown_dir=kwargs["markdown_path"], + markdown_dir=kwargs["markdown_dir"], model_filter=model_filter, ) pre_tasks.append(regenerate_task) @@ -74,13 +73,12 @@ def autosync_cmd(ctx: click.Context) -> None: @autosync_options @git_options @click.option( - "--ssp-index-path", + "--ssp-index-file", help="Path to ssp index file", type=str, required=True, ) -def autosync_ssp_cmd(ctx: click.Context, ssp_index_path: str, **kwargs: Any) -> None: - kwargs.update({"ssp_index_path": ssp_index_path}) +def autosync_ssp_cmd(ctx: click.Context, **kwargs: Any) -> None: run("ssp", **kwargs) diff --git a/trestlebot/cli/commands/rule_transform.py b/trestlebot/cli/commands/rule_transform.py index 41519cf3..0f2b72a7 100644 --- a/trestlebot/cli/commands/rule_transform.py +++ b/trestlebot/cli/commands/rule_transform.py @@ -45,7 +45,7 @@ @handle_exceptions def rule_transform_cmd(ctx: click.Context, **kwargs: Any) -> None: """Run the rule transform operation.""" - # Allow any model to be skipped by setting skip_items, by default include all + # Allow any model to be skipped by setting skip_item, by default include all skip_items = list(kwargs.get("skip_item", [])) model_filter: ModelFilter = ModelFilter( skip_patterns=skip_items, diff --git a/trestlebot/cli/options/autosync.py b/trestlebot/cli/options/autosync.py index d2a41eb8..fb49e3f0 100644 --- a/trestlebot/cli/options/autosync.py +++ b/trestlebot/cli/options/autosync.py @@ -12,15 +12,16 @@ def autosync_options(f: F) -> F: """ f = click.option( - "--markdown-path", + "--markdown-dir", help="Path to Trestle markdown files", - type=click.Path(exists=True), + type=str, required=True, )(f) f = click.option( - "--skip-items", - help="Comma-separated list of glob patterns to skip when running tasks", + "--skip-item", + help="Glob pattern to skip when running tasks", type=str, + multiple=True, )(f) f = click.option( "--skip-assemble", @@ -41,9 +42,4 @@ def autosync_options(f: F) -> F: help="Version of the OSCAL model to set during assembly into JSON", type=str, )(f) - f = click.option( - "--dry-run", - help="Run tasks, but do not push to the repository", - is_flag=True, - )(f) return f diff --git a/trestlebot/cli/options/common.py b/trestlebot/cli/options/common.py index 21c95690..658f1810 100644 --- a/trestlebot/cli/options/common.py +++ b/trestlebot/cli/options/common.py @@ -151,14 +151,16 @@ def git_options(f: F) -> F: type=str, )(f) f = click.option( - "--file-patterns", - help="Comma-separated list of file patterns to be used with `git add` in repository updates", + "--file-pattern", + help="File pattern to be used with `git add` in repository updates", type=str, + multiple=True, )(f) f = click.option( "--commit-message", help="Commit message for automated updates", type=str, + default="chore: automatic updates", )(f) f = click.option( "--author-name", diff --git a/trestlebot/cli/run.py b/trestlebot/cli/run.py index 75c97c81..1b39f10c 100644 --- a/trestlebot/cli/run.py +++ b/trestlebot/cli/run.py @@ -26,7 +26,7 @@ def run(pre_tasks: List[TaskBase], kwargs: Dict[Any, Any]) -> BotResults: return bot.run( pre_tasks=pre_tasks, - patterns=comma_sep_to_list(kwargs.get("patterns", "")), + patterns=kwargs.get("patterns", []), commit_message=kwargs.get("commit_message", "Automatic updates from bot"), dry_run=kwargs.get("dry_run", False), ) From fb36a110fe2c384af14e53d17251876d2d6db1f9 Mon Sep 17 00:00:00 2001 From: Qingmin Duanmu Date: Fri, 29 Nov 2024 09:51:08 +0800 Subject: [PATCH 048/108] Fix AttributeError, some misc updates AttributeError: 'NoneType' object has no attribute 'encode' --- tests/trestlebot/cli/test_autosync_cmd.py | 45 ++++++++++++------- ...ransform.py => test_rule_transform_cmd.py} | 0 trestlebot/cli/commands/autosync.py | 18 ++++---- trestlebot/cli/commands/rule_transform.py | 2 +- trestlebot/cli/options/autosync.py | 14 +++--- trestlebot/cli/options/common.py | 6 ++- trestlebot/cli/run.py | 2 +- 7 files changed, 47 insertions(+), 40 deletions(-) rename tests/trestlebot/cli/{test_rule_transform.py => test_rule_transform_cmd.py} (100%) diff --git a/tests/trestlebot/cli/test_autosync_cmd.py b/tests/trestlebot/cli/test_autosync_cmd.py index d044bee3..026a84de 100644 --- a/tests/trestlebot/cli/test_autosync_cmd.py +++ b/tests/trestlebot/cli/test_autosync_cmd.py @@ -3,24 +3,27 @@ """Testing module for trestlebot autosync command""" -import tempfile +import pathlib +from typing import Tuple from click.testing import CliRunner +from git import Repo from trestlebot.cli.commands.autosync import autosync_cmd +from trestlebot.cli.config import TrestleBotConfig, write_to_file -def test_invalid_autosync_command(tmp_init_dir: str) -> None: - tempdir = tempfile.mkdtemp() +def test_invalid_autosync_command(tmp_repo: Tuple[str, Repo]) -> None: + repo_path, _ = tmp_repo runner = CliRunner() result = runner.invoke( autosync_cmd, [ "invalid", "--repo-path", - tmp_init_dir, - "--markdown-path", - tempdir, + repo_path, + "--markdown-dir", + "markdown", "--branch", "main", "--committer-name", @@ -33,17 +36,16 @@ def test_invalid_autosync_command(tmp_init_dir: str) -> None: assert result.exit_code == 2 -def test_no_ssp_index_path(tmp_init_dir: str) -> None: - """Test invalid ssp index file for autosync ssp""" - - tempdir = tempfile.mkdtemp() +def test_no_ssp_index_path(tmp_repo: Tuple[str, Repo]) -> None: + """Test ssp_index_file for autosync ssp""" + repo_path, _ = tmp_repo runner = CliRunner() cmd_options = [ "ssp", "--repo-path", - tmp_init_dir, - "--markdown-path", - tempdir, + repo_path, + "--markdown-dir", + "markdown", "--branch", "main", "--committer-name", @@ -53,18 +55,19 @@ def test_no_ssp_index_path(tmp_init_dir: str) -> None: ] result = runner.invoke(autosync_cmd, cmd_options) assert result.exit_code == 2 - assert "Error: Missing option '--ssp-index-path'" in result.output + assert "Error: Missing option '--ssp-index-file'" in result.output cmd_options[0] = "compdef" result = runner.invoke(autosync_cmd, cmd_options) assert result.exit_code == 0 -def test_no_markdown_path(tmp_init_dir: str) -> None: +def test_no_markdown_dir(tmp_repo: Tuple[str, Repo]) -> None: + repo_path, _ = tmp_repo runner = CliRunner() cmd_options = [ "compdef", "--repo-path", - tmp_init_dir, + repo_path, "--branch", "main", "--committer-name", @@ -74,4 +77,12 @@ def test_no_markdown_path(tmp_init_dir: str) -> None: ] result = runner.invoke(autosync_cmd, cmd_options) assert result.exit_code == 2 - assert "Error: Missing option '--markdown-path'" in result.output + assert "Error: Missing option '--markdown-dir'" in result.output + + # With 'markdown_dir' setting in config.yml + config_obj = TrestleBotConfig(markdown_dir="markdown") + filepath = pathlib.Path(repo_path).joinpath("config.yml") + write_to_file(config_obj, filepath) + cmd_options += ["--config", str(filepath)] + result = runner.invoke(autosync_cmd, cmd_options) + assert result.exit_code == 0 diff --git a/tests/trestlebot/cli/test_rule_transform.py b/tests/trestlebot/cli/test_rule_transform_cmd.py similarity index 100% rename from tests/trestlebot/cli/test_rule_transform.py rename to tests/trestlebot/cli/test_rule_transform_cmd.py diff --git a/trestlebot/cli/commands/autosync.py b/trestlebot/cli/commands/autosync.py index 2d3eaa0e..6ff3e4bc 100644 --- a/trestlebot/cli/commands/autosync.py +++ b/trestlebot/cli/commands/autosync.py @@ -7,7 +7,6 @@ from trestlebot.cli.options.autosync import autosync_options from trestlebot.cli.options.common import common_options, git_options, handle_exceptions -from trestlebot.cli.run import comma_sep_to_list from trestlebot.cli.run import run as bot_run from trestlebot.tasks.assemble_task import AssembleTask from trestlebot.tasks.authored import types @@ -25,16 +24,16 @@ def run(oscal_model: str, **kwargs: Any) -> None: pre_tasks: List[TaskBase] = [] kwargs["working_dir"] = str(kwargs["repo_path"].resolve()) - if kwargs.get("file_patterns"): - kwargs.update({"patterns": comma_sep_to_list(kwargs["file_patterns"])}) + if kwargs.get("file_pattern"): + kwargs.update({"patterns": list(kwargs.get("file_pattern", []))}) model_filter: ModelFilter = ModelFilter( - skip_patterns=comma_sep_to_list(kwargs.get("skip_items", "")), + skip_patterns=list(kwargs.get("skip_item", [])), include_patterns=["*"], ) authored_object: AuthoredObjectBase = types.get_authored_object( oscal_model, kwargs["working_dir"], - kwargs.get("ssp_index_path", ""), + kwargs.get("ssp_index_file", ""), ) # Assuming an edit has occurred assemble would be run before regenerate. @@ -42,7 +41,7 @@ def run(oscal_model: str, **kwargs: Any) -> None: if not kwargs.get("skip_assemble"): assemble_task: AssembleTask = AssembleTask( authored_object=authored_object, - markdown_dir=kwargs["markdown_path"], + markdown_dir=kwargs["markdown_dir"], version=kwargs.get("version", ""), model_filter=model_filter, ) @@ -53,7 +52,7 @@ def run(oscal_model: str, **kwargs: Any) -> None: if not kwargs.get("skip_regenerate"): regenerate_task: RegenerateTask = RegenerateTask( authored_object=authored_object, - markdown_dir=kwargs["markdown_path"], + markdown_dir=kwargs["markdown_dir"], model_filter=model_filter, ) pre_tasks.append(regenerate_task) @@ -74,13 +73,12 @@ def autosync_cmd(ctx: click.Context) -> None: @autosync_options @git_options @click.option( - "--ssp-index-path", + "--ssp-index-file", help="Path to ssp index file", type=str, required=True, ) -def autosync_ssp_cmd(ctx: click.Context, ssp_index_path: str, **kwargs: Any) -> None: - kwargs.update({"ssp_index_path": ssp_index_path}) +def autosync_ssp_cmd(ctx: click.Context, **kwargs: Any) -> None: run("ssp", **kwargs) diff --git a/trestlebot/cli/commands/rule_transform.py b/trestlebot/cli/commands/rule_transform.py index 41519cf3..0f2b72a7 100644 --- a/trestlebot/cli/commands/rule_transform.py +++ b/trestlebot/cli/commands/rule_transform.py @@ -45,7 +45,7 @@ @handle_exceptions def rule_transform_cmd(ctx: click.Context, **kwargs: Any) -> None: """Run the rule transform operation.""" - # Allow any model to be skipped by setting skip_items, by default include all + # Allow any model to be skipped by setting skip_item, by default include all skip_items = list(kwargs.get("skip_item", [])) model_filter: ModelFilter = ModelFilter( skip_patterns=skip_items, diff --git a/trestlebot/cli/options/autosync.py b/trestlebot/cli/options/autosync.py index d2a41eb8..fb49e3f0 100644 --- a/trestlebot/cli/options/autosync.py +++ b/trestlebot/cli/options/autosync.py @@ -12,15 +12,16 @@ def autosync_options(f: F) -> F: """ f = click.option( - "--markdown-path", + "--markdown-dir", help="Path to Trestle markdown files", - type=click.Path(exists=True), + type=str, required=True, )(f) f = click.option( - "--skip-items", - help="Comma-separated list of glob patterns to skip when running tasks", + "--skip-item", + help="Glob pattern to skip when running tasks", type=str, + multiple=True, )(f) f = click.option( "--skip-assemble", @@ -41,9 +42,4 @@ def autosync_options(f: F) -> F: help="Version of the OSCAL model to set during assembly into JSON", type=str, )(f) - f = click.option( - "--dry-run", - help="Run tasks, but do not push to the repository", - is_flag=True, - )(f) return f diff --git a/trestlebot/cli/options/common.py b/trestlebot/cli/options/common.py index 21c95690..658f1810 100644 --- a/trestlebot/cli/options/common.py +++ b/trestlebot/cli/options/common.py @@ -151,14 +151,16 @@ def git_options(f: F) -> F: type=str, )(f) f = click.option( - "--file-patterns", - help="Comma-separated list of file patterns to be used with `git add` in repository updates", + "--file-pattern", + help="File pattern to be used with `git add` in repository updates", type=str, + multiple=True, )(f) f = click.option( "--commit-message", help="Commit message for automated updates", type=str, + default="chore: automatic updates", )(f) f = click.option( "--author-name", diff --git a/trestlebot/cli/run.py b/trestlebot/cli/run.py index 75c97c81..1b39f10c 100644 --- a/trestlebot/cli/run.py +++ b/trestlebot/cli/run.py @@ -26,7 +26,7 @@ def run(pre_tasks: List[TaskBase], kwargs: Dict[Any, Any]) -> BotResults: return bot.run( pre_tasks=pre_tasks, - patterns=comma_sep_to_list(kwargs.get("patterns", "")), + patterns=kwargs.get("patterns", []), commit_message=kwargs.get("commit_message", "Automatic updates from bot"), dry_run=kwargs.get("dry_run", False), ) From 742cef932b1c43a8536a8cea2ac6e155ac5eb46c Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Mon, 2 Dec 2024 15:53:07 -0500 Subject: [PATCH 049/108] feat: unit tests added for create command --- tests/trestlebot/cli/test_create_cmd.py | 149 +++++++++++++++++++----- trestlebot/cli/commands/create.py | 2 +- 2 files changed, 121 insertions(+), 30 deletions(-) diff --git a/tests/trestlebot/cli/test_create_cmd.py b/tests/trestlebot/cli/test_create_cmd.py index b1f3754a..56109d9e 100644 --- a/tests/trestlebot/cli/test_create_cmd.py +++ b/tests/trestlebot/cli/test_create_cmd.py @@ -16,32 +16,123 @@ def test_invalid_create_cmd() -> None: assert result.exit_code == 2 -# def test_create_ssp_cmd(tmp_init_dir: str) -> None: -# """Tests successful create ssp command.""" -# -# SSP_INDEX_FILE = "test-ssp-index.json" -# -# runner = CliRunner() -# result = runner.invoke( -# create_cmd, -# [ -# "ssp", -# "--profile-name", -# "repo-path", -# tmp_init_dir, -# "--ssp-index-file", -# SSP_INDEX_FILE, -# ], -# ) -# assert result.exit_code == 0 -# -# # verify ssp-index.json was created in tmp_init_dir -# tmp_dir = pathlib.Path(tmp_init_dir) -# ssp_index_file = tmp_dir.joinpath(SSP_INDEX_FILE) -# print("SSP INDEX Path", str(ssp_index_file.resolve())) -# -# assert ssp_index_file.exists() is True - -# verity json file was created in tmp_init_dir/system-security-plans - -# verify markdown file(s) were created in tmp_init_dir/markdown/system... +def test_create_ssp_cmd(tmp_init_dir: str) -> None: + """Tests successful create ssp command.""" + SSP_INDEX_FILE = "tester-ssp-index.json" + + # tmp_dir = pathlib.Path(tmp_init_dir) + # ssp_index_file = tmp_dir.joinpath(SSP_INDEX_FILE) + # print("SSP INDEX Path", str(ssp_index_file.resolve())) + + runner = CliRunner() + result = runner.invoke( + create_cmd, + [ + "ssp", + "--profile-name", + "oscal-profile-name", + "--markdown-dir", + "markdown", + "--ssp-name", + "test-name", + "--repo-path", + tmp_init_dir, + "--ssp-index-file", + SSP_INDEX_FILE, + ], + ) + assert result.exit_code == 0 + + +def test_create_compdef_cmd(tmp_init_dir: str) -> None: + """Tests successful create compdef command.""" + + runner = CliRunner() + result = runner.invoke( + create_cmd, + [ + "compdef", + "--profile-name", + "oscal-profile-name", + "--compdef-name", + "test-name", + "--component-title", + "title-test", + "--component-description", + "description-test", + "--component-definition-type", + "type-test", + "repo-path", + tmp_init_dir, + ], + ) + assert result.exit_code == 2 + + +def test_default_ssp_index_file_cmd(tmp_init_dir: str) -> None: + """Tests successful default ssp_index.json file creation.""" + + runner = CliRunner() + result = runner.invoke( + create_cmd, + [ + "ssp", + "--profile-name", + "oscal-profile-name", + "--markdown-dir", + "markdown", + "--ssp-name", + "test-name", + "--repo-path", + tmp_init_dir, + ], + ) + assert result.exit_code == 0 + + +def test_markdown_files_not_created(tmp_init_dir: str) -> None: + SSP_INDEX_TESTER = "ssp-tester.index.json" + runner = CliRunner() + result = runner.invoke( + create_cmd, + [ + "ssp", + "--profile-name", + "oscal-profile-name", + "--ssp-name", + "test-name", + "repo-path", + tmp_init_dir, + "--ssp-index-file", + SSP_INDEX_TESTER, + ], + ) + assert result.exit_code == 2 + + +def test_markdown_files_created(tmp_init_dir: str) -> None: + SSP_INDEX_TESTER = "ssp-tester.index.json" + markdown_file_tester = "/markdown/system-security-plan.md" + + # tmp_dir = pathlib.Path(tmp_init_dir) + # markdown_file = tmp_dir.joinpath(markdown_file_tester) + # print("Markdown file located at", str(markdown_file.resolve())) + + runner = CliRunner() + result = runner.invoke( + create_cmd, + [ + "ssp", + "--profile-name", + "oscal-profile-name", + "--markdown-dir", + markdown_file_tester, + "--ssp-name", + "test-name", + "--repo-path", + tmp_init_dir, + "--ssp-index-file", + SSP_INDEX_TESTER, + ], + ) + assert result.exit_code == 0 diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index 8391de8c..42697cfb 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -27,7 +27,7 @@ logger = logging.getLogger(__name__) -@click.group(name="create") +@click.group(name="create", help="Component definition and ssp authoring.") @click.pass_context @handle_exceptions def create_cmd(ctx: click.Context) -> None: From 026657cfba27f33d12e5851ea34f9e3549d5b176 Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Mon, 2 Dec 2024 15:53:07 -0500 Subject: [PATCH 050/108] feat: unit tests added for create command --- tests/trestlebot/cli/test_create_cmd.py | 149 +++++++++++++++++++----- trestlebot/cli/commands/create.py | 2 +- 2 files changed, 121 insertions(+), 30 deletions(-) diff --git a/tests/trestlebot/cli/test_create_cmd.py b/tests/trestlebot/cli/test_create_cmd.py index b1f3754a..56109d9e 100644 --- a/tests/trestlebot/cli/test_create_cmd.py +++ b/tests/trestlebot/cli/test_create_cmd.py @@ -16,32 +16,123 @@ def test_invalid_create_cmd() -> None: assert result.exit_code == 2 -# def test_create_ssp_cmd(tmp_init_dir: str) -> None: -# """Tests successful create ssp command.""" -# -# SSP_INDEX_FILE = "test-ssp-index.json" -# -# runner = CliRunner() -# result = runner.invoke( -# create_cmd, -# [ -# "ssp", -# "--profile-name", -# "repo-path", -# tmp_init_dir, -# "--ssp-index-file", -# SSP_INDEX_FILE, -# ], -# ) -# assert result.exit_code == 0 -# -# # verify ssp-index.json was created in tmp_init_dir -# tmp_dir = pathlib.Path(tmp_init_dir) -# ssp_index_file = tmp_dir.joinpath(SSP_INDEX_FILE) -# print("SSP INDEX Path", str(ssp_index_file.resolve())) -# -# assert ssp_index_file.exists() is True - -# verity json file was created in tmp_init_dir/system-security-plans - -# verify markdown file(s) were created in tmp_init_dir/markdown/system... +def test_create_ssp_cmd(tmp_init_dir: str) -> None: + """Tests successful create ssp command.""" + SSP_INDEX_FILE = "tester-ssp-index.json" + + # tmp_dir = pathlib.Path(tmp_init_dir) + # ssp_index_file = tmp_dir.joinpath(SSP_INDEX_FILE) + # print("SSP INDEX Path", str(ssp_index_file.resolve())) + + runner = CliRunner() + result = runner.invoke( + create_cmd, + [ + "ssp", + "--profile-name", + "oscal-profile-name", + "--markdown-dir", + "markdown", + "--ssp-name", + "test-name", + "--repo-path", + tmp_init_dir, + "--ssp-index-file", + SSP_INDEX_FILE, + ], + ) + assert result.exit_code == 0 + + +def test_create_compdef_cmd(tmp_init_dir: str) -> None: + """Tests successful create compdef command.""" + + runner = CliRunner() + result = runner.invoke( + create_cmd, + [ + "compdef", + "--profile-name", + "oscal-profile-name", + "--compdef-name", + "test-name", + "--component-title", + "title-test", + "--component-description", + "description-test", + "--component-definition-type", + "type-test", + "repo-path", + tmp_init_dir, + ], + ) + assert result.exit_code == 2 + + +def test_default_ssp_index_file_cmd(tmp_init_dir: str) -> None: + """Tests successful default ssp_index.json file creation.""" + + runner = CliRunner() + result = runner.invoke( + create_cmd, + [ + "ssp", + "--profile-name", + "oscal-profile-name", + "--markdown-dir", + "markdown", + "--ssp-name", + "test-name", + "--repo-path", + tmp_init_dir, + ], + ) + assert result.exit_code == 0 + + +def test_markdown_files_not_created(tmp_init_dir: str) -> None: + SSP_INDEX_TESTER = "ssp-tester.index.json" + runner = CliRunner() + result = runner.invoke( + create_cmd, + [ + "ssp", + "--profile-name", + "oscal-profile-name", + "--ssp-name", + "test-name", + "repo-path", + tmp_init_dir, + "--ssp-index-file", + SSP_INDEX_TESTER, + ], + ) + assert result.exit_code == 2 + + +def test_markdown_files_created(tmp_init_dir: str) -> None: + SSP_INDEX_TESTER = "ssp-tester.index.json" + markdown_file_tester = "/markdown/system-security-plan.md" + + # tmp_dir = pathlib.Path(tmp_init_dir) + # markdown_file = tmp_dir.joinpath(markdown_file_tester) + # print("Markdown file located at", str(markdown_file.resolve())) + + runner = CliRunner() + result = runner.invoke( + create_cmd, + [ + "ssp", + "--profile-name", + "oscal-profile-name", + "--markdown-dir", + markdown_file_tester, + "--ssp-name", + "test-name", + "--repo-path", + tmp_init_dir, + "--ssp-index-file", + SSP_INDEX_TESTER, + ], + ) + assert result.exit_code == 0 diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index 8391de8c..42697cfb 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -27,7 +27,7 @@ logger = logging.getLogger(__name__) -@click.group(name="create") +@click.group(name="create", help="Component definition and ssp authoring.") @click.pass_context @handle_exceptions def create_cmd(ctx: click.Context) -> None: From f9a2bcaedf4c527f406ee61a689b9793e14be401 Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Tue, 3 Dec 2024 08:40:19 -0500 Subject: [PATCH 051/108] fix: docstrings added for create command unit tests --- tests/trestlebot/cli/test_create_cmd.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/tests/trestlebot/cli/test_create_cmd.py b/tests/trestlebot/cli/test_create_cmd.py index 56109d9e..e47e0d33 100644 --- a/tests/trestlebot/cli/test_create_cmd.py +++ b/tests/trestlebot/cli/test_create_cmd.py @@ -1,7 +1,5 @@ """ Unit test for create commands ssp and cd""" -# import pathlib - from click.testing import CliRunner from trestlebot.cli.commands.create import create_cmd @@ -20,10 +18,6 @@ def test_create_ssp_cmd(tmp_init_dir: str) -> None: """Tests successful create ssp command.""" SSP_INDEX_FILE = "tester-ssp-index.json" - # tmp_dir = pathlib.Path(tmp_init_dir) - # ssp_index_file = tmp_dir.joinpath(SSP_INDEX_FILE) - # print("SSP INDEX Path", str(ssp_index_file.resolve())) - runner = CliRunner() result = runner.invoke( create_cmd, @@ -91,7 +85,9 @@ def test_default_ssp_index_file_cmd(tmp_init_dir: str) -> None: def test_markdown_files_not_created(tmp_init_dir: str) -> None: + """Tests failure of markdown file creation when not supplied with directory name.""" SSP_INDEX_TESTER = "ssp-tester.index.json" + runner = CliRunner() result = runner.invoke( create_cmd, @@ -111,13 +107,10 @@ def test_markdown_files_not_created(tmp_init_dir: str) -> None: def test_markdown_files_created(tmp_init_dir: str) -> None: + """Tests successful creation of markdown files.""" SSP_INDEX_TESTER = "ssp-tester.index.json" markdown_file_tester = "/markdown/system-security-plan.md" - # tmp_dir = pathlib.Path(tmp_init_dir) - # markdown_file = tmp_dir.joinpath(markdown_file_tester) - # print("Markdown file located at", str(markdown_file.resolve())) - runner = CliRunner() result = runner.invoke( create_cmd, From ee0feece654fc26e35014a3fd58b271dcf98c7a2 Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Tue, 3 Dec 2024 08:40:19 -0500 Subject: [PATCH 052/108] fix: docstrings added for create command unit tests --- tests/trestlebot/cli/test_create_cmd.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/tests/trestlebot/cli/test_create_cmd.py b/tests/trestlebot/cli/test_create_cmd.py index 56109d9e..e47e0d33 100644 --- a/tests/trestlebot/cli/test_create_cmd.py +++ b/tests/trestlebot/cli/test_create_cmd.py @@ -1,7 +1,5 @@ """ Unit test for create commands ssp and cd""" -# import pathlib - from click.testing import CliRunner from trestlebot.cli.commands.create import create_cmd @@ -20,10 +18,6 @@ def test_create_ssp_cmd(tmp_init_dir: str) -> None: """Tests successful create ssp command.""" SSP_INDEX_FILE = "tester-ssp-index.json" - # tmp_dir = pathlib.Path(tmp_init_dir) - # ssp_index_file = tmp_dir.joinpath(SSP_INDEX_FILE) - # print("SSP INDEX Path", str(ssp_index_file.resolve())) - runner = CliRunner() result = runner.invoke( create_cmd, @@ -91,7 +85,9 @@ def test_default_ssp_index_file_cmd(tmp_init_dir: str) -> None: def test_markdown_files_not_created(tmp_init_dir: str) -> None: + """Tests failure of markdown file creation when not supplied with directory name.""" SSP_INDEX_TESTER = "ssp-tester.index.json" + runner = CliRunner() result = runner.invoke( create_cmd, @@ -111,13 +107,10 @@ def test_markdown_files_not_created(tmp_init_dir: str) -> None: def test_markdown_files_created(tmp_init_dir: str) -> None: + """Tests successful creation of markdown files.""" SSP_INDEX_TESTER = "ssp-tester.index.json" markdown_file_tester = "/markdown/system-security-plan.md" - # tmp_dir = pathlib.Path(tmp_init_dir) - # markdown_file = tmp_dir.joinpath(markdown_file_tester) - # print("Markdown file located at", str(markdown_file.resolve())) - runner = CliRunner() result = runner.invoke( create_cmd, From ef31702e1d478902a9f32429d3ca23baa91d1408 Mon Sep 17 00:00:00 2001 From: George Vauter Date: Tue, 3 Dec 2024 11:02:43 -0500 Subject: [PATCH 053/108] add file pattern filter --- trestlebot/cli/commands/autosync.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/trestlebot/cli/commands/autosync.py b/trestlebot/cli/commands/autosync.py index 34d4744c..3a93de68 100644 --- a/trestlebot/cli/commands/autosync.py +++ b/trestlebot/cli/commands/autosync.py @@ -73,6 +73,7 @@ def autosync_cmd(ctx: click.Context, **kwargs: Any) -> None: oscal_model = kwargs["oscal_model"] markdown_dir = kwargs["markdown_dir"] working_dir = str(kwargs["repo_path"].resolve()) + kwargs["working_dir"] = working_dir if oscal_model == "ssp" and not kwargs.get("ssp_index_file"): logger.error("Trestlebot Error: Missing option '--ssp-index-file'.") @@ -80,7 +81,9 @@ def autosync_cmd(ctx: click.Context, **kwargs: Any) -> None: pre_tasks: List[TaskBase] = [] - kwargs["working_dir"] = str(kwargs["repo_path"].resolve()) + if kwargs.get("file_pattern"): + kwargs.update({"patterns": comma_sep_to_list(kwargs["file_patterns"])}) + model_filter: ModelFilter = ModelFilter( skip_patterns=comma_sep_to_list(kwargs.get("skip_items", "")), include_patterns=["*"], From 39ded49e2b91eb8d7b1abafc1aa60a9a6fb3d1bd Mon Sep 17 00:00:00 2001 From: George Vauter Date: Tue, 3 Dec 2024 11:02:43 -0500 Subject: [PATCH 054/108] add file pattern filter --- trestlebot/cli/commands/autosync.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/trestlebot/cli/commands/autosync.py b/trestlebot/cli/commands/autosync.py index 34d4744c..3a93de68 100644 --- a/trestlebot/cli/commands/autosync.py +++ b/trestlebot/cli/commands/autosync.py @@ -73,6 +73,7 @@ def autosync_cmd(ctx: click.Context, **kwargs: Any) -> None: oscal_model = kwargs["oscal_model"] markdown_dir = kwargs["markdown_dir"] working_dir = str(kwargs["repo_path"].resolve()) + kwargs["working_dir"] = working_dir if oscal_model == "ssp" and not kwargs.get("ssp_index_file"): logger.error("Trestlebot Error: Missing option '--ssp-index-file'.") @@ -80,7 +81,9 @@ def autosync_cmd(ctx: click.Context, **kwargs: Any) -> None: pre_tasks: List[TaskBase] = [] - kwargs["working_dir"] = str(kwargs["repo_path"].resolve()) + if kwargs.get("file_pattern"): + kwargs.update({"patterns": comma_sep_to_list(kwargs["file_patterns"])}) + model_filter: ModelFilter = ModelFilter( skip_patterns=comma_sep_to_list(kwargs.get("skip_items", "")), include_patterns=["*"], From f352487e18d5a7ad9811377b8fc92088a28f2dd8 Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Tue, 3 Dec 2024 11:06:32 -0500 Subject: [PATCH 055/108] fix: updated headers with license and copyright --- tests/trestlebot/cli/test_create_cmd.py | 3 +++ trestlebot/cli/commands/autosync.py | 3 +++ trestlebot/cli/commands/create.py | 3 +++ trestlebot/cli/commands/rule_transform.py | 3 +++ trestlebot/cli/commands/version.py | 3 +++ trestlebot/cli/options/create.py | 3 +++ trestlebot/cli/utils.py | 3 +++ 7 files changed, 21 insertions(+) diff --git a/tests/trestlebot/cli/test_create_cmd.py b/tests/trestlebot/cli/test_create_cmd.py index e47e0d33..8321a5f7 100644 --- a/tests/trestlebot/cli/test_create_cmd.py +++ b/tests/trestlebot/cli/test_create_cmd.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Red Hat, Inc. + """ Unit test for create commands ssp and cd""" from click.testing import CliRunner diff --git a/trestlebot/cli/commands/autosync.py b/trestlebot/cli/commands/autosync.py index 34d4744c..3708a03f 100644 --- a/trestlebot/cli/commands/autosync.py +++ b/trestlebot/cli/commands/autosync.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Red Hat, Inc. + """ Autosync command""" import logging diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index 3abed3c3..8e6bf4ed 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Red Hat, Inc. + """ Module for create-cd create-ssp command for CLI """ diff --git a/trestlebot/cli/commands/rule_transform.py b/trestlebot/cli/commands/rule_transform.py index bff87be1..77a856a5 100644 --- a/trestlebot/cli/commands/rule_transform.py +++ b/trestlebot/cli/commands/rule_transform.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Red Hat, Inc. + """Module for rule-transform command""" import logging diff --git a/trestlebot/cli/commands/version.py b/trestlebot/cli/commands/version.py index 11b8814c..bec415cc 100644 --- a/trestlebot/cli/commands/version.py +++ b/trestlebot/cli/commands/version.py @@ -1 +1,4 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Red Hat, Inc. + """ Version command """ diff --git a/trestlebot/cli/options/create.py b/trestlebot/cli/options/create.py index 2d84ea47..a47939cc 100644 --- a/trestlebot/cli/options/create.py +++ b/trestlebot/cli/options/create.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Red Hat, Inc. + """ Module for common create commands """ diff --git a/trestlebot/cli/utils.py b/trestlebot/cli/utils.py index 7cae749d..f2ef4690 100644 --- a/trestlebot/cli/utils.py +++ b/trestlebot/cli/utils.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Red Hat, Inc. + from typing import Any, Dict, List from trestlebot.bot import TrestleBot From 3c094d2a834436c01b4baffcffb9bc0f0122ec32 Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Tue, 3 Dec 2024 11:06:32 -0500 Subject: [PATCH 056/108] fix: updated headers with license and copyright --- tests/trestlebot/cli/test_create_cmd.py | 3 +++ trestlebot/cli/commands/autosync.py | 3 +++ trestlebot/cli/commands/create.py | 3 +++ trestlebot/cli/commands/rule_transform.py | 3 +++ trestlebot/cli/commands/version.py | 3 +++ trestlebot/cli/options/create.py | 3 +++ trestlebot/cli/utils.py | 3 +++ 7 files changed, 21 insertions(+) diff --git a/tests/trestlebot/cli/test_create_cmd.py b/tests/trestlebot/cli/test_create_cmd.py index e47e0d33..8321a5f7 100644 --- a/tests/trestlebot/cli/test_create_cmd.py +++ b/tests/trestlebot/cli/test_create_cmd.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Red Hat, Inc. + """ Unit test for create commands ssp and cd""" from click.testing import CliRunner diff --git a/trestlebot/cli/commands/autosync.py b/trestlebot/cli/commands/autosync.py index 34d4744c..3708a03f 100644 --- a/trestlebot/cli/commands/autosync.py +++ b/trestlebot/cli/commands/autosync.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Red Hat, Inc. + """ Autosync command""" import logging diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index 3abed3c3..8e6bf4ed 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Red Hat, Inc. + """ Module for create-cd create-ssp command for CLI """ diff --git a/trestlebot/cli/commands/rule_transform.py b/trestlebot/cli/commands/rule_transform.py index bff87be1..77a856a5 100644 --- a/trestlebot/cli/commands/rule_transform.py +++ b/trestlebot/cli/commands/rule_transform.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Red Hat, Inc. + """Module for rule-transform command""" import logging diff --git a/trestlebot/cli/commands/version.py b/trestlebot/cli/commands/version.py index 11b8814c..bec415cc 100644 --- a/trestlebot/cli/commands/version.py +++ b/trestlebot/cli/commands/version.py @@ -1 +1,4 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Red Hat, Inc. + """ Version command """ diff --git a/trestlebot/cli/options/create.py b/trestlebot/cli/options/create.py index 2d84ea47..a47939cc 100644 --- a/trestlebot/cli/options/create.py +++ b/trestlebot/cli/options/create.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Red Hat, Inc. + """ Module for common create commands """ diff --git a/trestlebot/cli/utils.py b/trestlebot/cli/utils.py index 7cae749d..f2ef4690 100644 --- a/trestlebot/cli/utils.py +++ b/trestlebot/cli/utils.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Red Hat, Inc. + from typing import Any, Dict, List from trestlebot.bot import TrestleBot From 170cfbcb77983d791ab35a7256c66fb617426533 Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Tue, 3 Dec 2024 12:02:55 -0500 Subject: [PATCH 057/108] fix: updated logger statements --- trestlebot/cli/commands/create.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index 8e6bf4ed..d2ecc2f5 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -155,7 +155,7 @@ def compdef_cmd( f"The profile you want to filter controls in the component files is {filter_by_profile}." ) logger.info(f"The component definition type is {component_definition_type}.") - logger.debug(f"You have successfully authored the the {compdef_name}.") + logger.debug(f"You have successfully authored the {compdef_name}.") @create_cmd.command("ssp") @@ -249,4 +249,4 @@ def ssp_cmd( run_bot(pre_tasks, kwargs) - logger.debug(f"You have successfully authored the the {ssp_name}.") + logger.debug(f"You have successfully authored the {ssp_name}.") From afa3f8e3c3404d89f9ea894adeff009f98bc2442 Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Tue, 3 Dec 2024 12:02:55 -0500 Subject: [PATCH 058/108] fix: updated logger statements --- trestlebot/cli/commands/create.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index 8e6bf4ed..d2ecc2f5 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -155,7 +155,7 @@ def compdef_cmd( f"The profile you want to filter controls in the component files is {filter_by_profile}." ) logger.info(f"The component definition type is {component_definition_type}.") - logger.debug(f"You have successfully authored the the {compdef_name}.") + logger.debug(f"You have successfully authored the {compdef_name}.") @create_cmd.command("ssp") @@ -249,4 +249,4 @@ def ssp_cmd( run_bot(pre_tasks, kwargs) - logger.debug(f"You have successfully authored the the {ssp_name}.") + logger.debug(f"You have successfully authored the {ssp_name}.") From fa13754988a46508809d9bf89606cead4306a82e Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Tue, 3 Dec 2024 13:27:55 -0500 Subject: [PATCH 059/108] fix: logger statements shortened --- trestlebot/cli/commands/create.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index d2ecc2f5..abd3e248 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -140,21 +140,6 @@ def compdef_cmd( run_bot(pre_tasks, kwargs) - for key, value in kwargs.items(): - logger.info(f"{key}: {value}") - - logger.info( - f"The name of the profile in use with the component definition is {profile_name}." - ) - logger.info( - f"You have selected component definitions as the document you want {compdef_name} to author." - ) - logger.info(f"The component definition name is {component_title}.") - logger.info(f"The component description to author is {component_description}.") - logger.info( - f"The profile you want to filter controls in the component files is {filter_by_profile}." - ) - logger.info(f"The component definition type is {component_definition_type}.") logger.debug(f"You have successfully authored the {compdef_name}.") From a11615f4faeed7bf8ba0c03def6398a4bae55c42 Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Tue, 3 Dec 2024 13:27:55 -0500 Subject: [PATCH 060/108] fix: logger statements shortened --- trestlebot/cli/commands/create.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index d2ecc2f5..abd3e248 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -140,21 +140,6 @@ def compdef_cmd( run_bot(pre_tasks, kwargs) - for key, value in kwargs.items(): - logger.info(f"{key}: {value}") - - logger.info( - f"The name of the profile in use with the component definition is {profile_name}." - ) - logger.info( - f"You have selected component definitions as the document you want {compdef_name} to author." - ) - logger.info(f"The component definition name is {component_title}.") - logger.info(f"The component description to author is {component_description}.") - logger.info( - f"The profile you want to filter controls in the component files is {filter_by_profile}." - ) - logger.info(f"The component definition type is {component_definition_type}.") logger.debug(f"You have successfully authored the {compdef_name}.") From 0ca9439e047f7b20157361a5ff9570920f585434 Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Tue, 3 Dec 2024 14:34:37 -0500 Subject: [PATCH 061/108] fix: yaml default deletion --- trestlebot/cli/commands/create.py | 1 - 1 file changed, 1 deletion(-) diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index abd3e248..5cd4b78a 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -169,7 +169,6 @@ def compdef_cmd( "--yaml-header-path", required=False, type=str, - default="ssp-index.json", help="Optionally set a path to a YAML file for custom SSP Markdown YAML headers.", ) @click.option( From 799b0ec3ee2b0d9940b03877507377f873764006 Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Tue, 3 Dec 2024 14:34:37 -0500 Subject: [PATCH 062/108] fix: yaml default deletion --- trestlebot/cli/commands/create.py | 1 - 1 file changed, 1 deletion(-) diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index abd3e248..5cd4b78a 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -169,7 +169,6 @@ def compdef_cmd( "--yaml-header-path", required=False, type=str, - default="ssp-index.json", help="Optionally set a path to a YAML file for custom SSP Markdown YAML headers.", ) @click.option( From 99915e4215a544d225c84cd79f47b46e76ac9eab Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Tue, 3 Dec 2024 14:52:44 -0500 Subject: [PATCH 063/108] docs: updates to reference the CLI commands in the README.md --- README.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index ee98868d..4995ba23 100644 --- a/README.md +++ b/README.md @@ -19,21 +19,21 @@ The `autosync` command will sync trestle-generated Markdown files to OSCAL JSON The `rules-transform` command can be used when managing [OSCAL Component Definitions](https://pages.nist.gov/OSCAL-Reference/models/v1.1.1/component-definition/json-outline/) in a trestle workspace. The action will transform rules defined in the rules YAML view to an OSCAL Component Definition JSON file. -The `create-cd` command can be used to create a new [OSCAL Component Definition](https://pages.nist.gov/OSCAL-Reference/models/v1.1.1/component-definition/json-outline/) in a trestle workspace. The action will create a new Component Definition JSON file and corresponding directories that contain rules YAML files and trestle-generated Markdown files. This action prepares the workspace for use with the `rules-transform` and `autosync` actions. +The `create compdef` command can be used to create a new [OSCAL Component Definition](https://pages.nist.gov/OSCAL-Reference/models/v1.1.1/component-definition/json-outline/) in a trestle workspace. The action will create a new Component Definition JSON file and corresponding directories that contain rules YAML files and trestle-generated Markdown files. This action prepares the workspace for use with the `rules-transform` and `autosync` actions. -The `sync-upstreams` command can be used to sync and validate upstream OSCAL content stored in a git repository to a local trestle workspace. Which content is synced is determined by the `include_model_names` and `exclude_model_names` inputs. +The `sync-upstreams` command can be used to sync and validate upstream OSCAL content stored in a git repository to a local trestle workspace. The inputs `include_models` and `exclude_models` determine which content is synced to the trestle workspace. -The `create-ssp` command can be used to create a new [OSCAL System Security Plans](https://pages.nist.gov/OSCAL-Reference/models/v1.1.1/system-security-plan/json-outline/) (SSP) in a trestle workspace. The action will create a new SSP JSON file and corresponding directories that contain trestle-generated Markdown files. This action prepares the workspace for use with the `autosync` action by creating or updating the `ssp-index.json` file. The `ssp-index.json` file is used to track the relationships between the SSP and the other OSCAL content in the workspace for the `autosync` action. +The `create ssp` command can be used to create a new [OSCAL System Security Plans](https://pages.nist.gov/OSCAL-Reference/models/v1.1.1/system-security-plan/json-outline/) (SSP) in a trestle workspace. The action will create a new SSP JSON file and corresponding directories that contain trestle-generated Markdown files. This action prepares the workspace for use with the `autosync` action by creating or updating the `ssp-index.json` file. The `ssp-index.json` file is used to track the relationships between the SSP and the other OSCAL content in the workspace for the `autosync` action. Below is a table of the available commands and their current availability as a GitHub Action: -| Command | Available as a GitHub Action | -|--------------------|------------------------------| -| `autosync` | ✓ | -| `rules-transform` | ✓ | -| `create-cd` | ✓ | -| `sync-upstreams` | ✓ | -| `create-ssp` | | +| Command | Available as a GitHub Action | +|-------------------|------------------------------| +| `autosync` | ✓ | +| `rules-transform` | ✓ | +| `create compdef` | ✓ | +| `sync-upstreams` | ✓ | +| `create ssp` | | For detailed documentation on how to use each action, see the README.md in each folder under [actions](./actions/). @@ -47,7 +47,7 @@ provider information is supported for GitHub Actions (GitHub) and GitLab CI (Git ### Run as a Container -> Note: When running the commands in a container, all are prefixed with `trestlebot` (e.g. `trestlebot-autosync`). The default entrypoint for the container is the autosync command. +> Note: When running the commands in a container, all are prefixed with `trestlebot` (e.g. `trestlebot autosync`). The default entrypoint for the container is the autosync command. Build and run the container locally: @@ -72,4 +72,4 @@ This project is licensed under the Apache 2.0 License - see the [LICENSE.md](LIC ## Troubleshooting -See [TROUBLESHOOTING.md](./TROUBLESHOOTING.md) for troubleshooting tips. \ No newline at end of file +See [TROUBLESHOOTING.md](./TROUBLESHOOTING.md) for troubleshooting tips. From a44aa80a2ae99d9a6b121b536355d362a6761e65 Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Tue, 3 Dec 2024 14:52:44 -0500 Subject: [PATCH 064/108] docs: updates to reference the CLI commands in the README.md --- README.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index ee98868d..4995ba23 100644 --- a/README.md +++ b/README.md @@ -19,21 +19,21 @@ The `autosync` command will sync trestle-generated Markdown files to OSCAL JSON The `rules-transform` command can be used when managing [OSCAL Component Definitions](https://pages.nist.gov/OSCAL-Reference/models/v1.1.1/component-definition/json-outline/) in a trestle workspace. The action will transform rules defined in the rules YAML view to an OSCAL Component Definition JSON file. -The `create-cd` command can be used to create a new [OSCAL Component Definition](https://pages.nist.gov/OSCAL-Reference/models/v1.1.1/component-definition/json-outline/) in a trestle workspace. The action will create a new Component Definition JSON file and corresponding directories that contain rules YAML files and trestle-generated Markdown files. This action prepares the workspace for use with the `rules-transform` and `autosync` actions. +The `create compdef` command can be used to create a new [OSCAL Component Definition](https://pages.nist.gov/OSCAL-Reference/models/v1.1.1/component-definition/json-outline/) in a trestle workspace. The action will create a new Component Definition JSON file and corresponding directories that contain rules YAML files and trestle-generated Markdown files. This action prepares the workspace for use with the `rules-transform` and `autosync` actions. -The `sync-upstreams` command can be used to sync and validate upstream OSCAL content stored in a git repository to a local trestle workspace. Which content is synced is determined by the `include_model_names` and `exclude_model_names` inputs. +The `sync-upstreams` command can be used to sync and validate upstream OSCAL content stored in a git repository to a local trestle workspace. The inputs `include_models` and `exclude_models` determine which content is synced to the trestle workspace. -The `create-ssp` command can be used to create a new [OSCAL System Security Plans](https://pages.nist.gov/OSCAL-Reference/models/v1.1.1/system-security-plan/json-outline/) (SSP) in a trestle workspace. The action will create a new SSP JSON file and corresponding directories that contain trestle-generated Markdown files. This action prepares the workspace for use with the `autosync` action by creating or updating the `ssp-index.json` file. The `ssp-index.json` file is used to track the relationships between the SSP and the other OSCAL content in the workspace for the `autosync` action. +The `create ssp` command can be used to create a new [OSCAL System Security Plans](https://pages.nist.gov/OSCAL-Reference/models/v1.1.1/system-security-plan/json-outline/) (SSP) in a trestle workspace. The action will create a new SSP JSON file and corresponding directories that contain trestle-generated Markdown files. This action prepares the workspace for use with the `autosync` action by creating or updating the `ssp-index.json` file. The `ssp-index.json` file is used to track the relationships between the SSP and the other OSCAL content in the workspace for the `autosync` action. Below is a table of the available commands and their current availability as a GitHub Action: -| Command | Available as a GitHub Action | -|--------------------|------------------------------| -| `autosync` | ✓ | -| `rules-transform` | ✓ | -| `create-cd` | ✓ | -| `sync-upstreams` | ✓ | -| `create-ssp` | | +| Command | Available as a GitHub Action | +|-------------------|------------------------------| +| `autosync` | ✓ | +| `rules-transform` | ✓ | +| `create compdef` | ✓ | +| `sync-upstreams` | ✓ | +| `create ssp` | | For detailed documentation on how to use each action, see the README.md in each folder under [actions](./actions/). @@ -47,7 +47,7 @@ provider information is supported for GitHub Actions (GitHub) and GitLab CI (Git ### Run as a Container -> Note: When running the commands in a container, all are prefixed with `trestlebot` (e.g. `trestlebot-autosync`). The default entrypoint for the container is the autosync command. +> Note: When running the commands in a container, all are prefixed with `trestlebot` (e.g. `trestlebot autosync`). The default entrypoint for the container is the autosync command. Build and run the container locally: @@ -72,4 +72,4 @@ This project is licensed under the Apache 2.0 License - see the [LICENSE.md](LIC ## Troubleshooting -See [TROUBLESHOOTING.md](./TROUBLESHOOTING.md) for troubleshooting tips. \ No newline at end of file +See [TROUBLESHOOTING.md](./TROUBLESHOOTING.md) for troubleshooting tips. From f9feb88ba2e290c32ca0f7a94ceb96baef8959ef Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Tue, 3 Dec 2024 15:15:05 -0500 Subject: [PATCH 065/108] feat: update for required ssp name --- trestlebot/cli/commands/create.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index 5cd4b78a..5fc855d7 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -149,6 +149,8 @@ def compdef_cmd( @common_options @click.option( "--ssp-name", + required=True, + type=str, prompt="Enter name of SSP to create", help="Name of SSP to create.", ) From 8dbe4e36d44a2d0f672aa44aa2ea923e0dbac81a Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Tue, 3 Dec 2024 15:15:05 -0500 Subject: [PATCH 066/108] feat: update for required ssp name --- trestlebot/cli/commands/create.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index 5cd4b78a..5fc855d7 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -149,6 +149,8 @@ def compdef_cmd( @common_options @click.option( "--ssp-name", + required=True, + type=str, prompt="Enter name of SSP to create", help="Name of SSP to create.", ) From 91d115c5ab206104277350aeaa0a08a11fba963f Mon Sep 17 00:00:00 2001 From: George Vauter Date: Tue, 3 Dec 2024 16:08:49 -0500 Subject: [PATCH 067/108] Update trestlebot/cli/commands/init.py Co-authored-by: Jennifer Power --- trestlebot/cli/commands/init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trestlebot/cli/commands/init.py b/trestlebot/cli/commands/init.py index a0ef718c..d1a1e925 100644 --- a/trestlebot/cli/commands/init.py +++ b/trestlebot/cli/commands/init.py @@ -161,7 +161,7 @@ def init_cmd( # inovke the init command in compliance trestle call_trestle_init(repo_path, debug) - # generate and write trestle-bot cofig + # generate and write trestle-bot config config_values = dict(repo_path=repo_path, markdown_dir=markdown_dir) if default_committer_name: config_values.update(committer_name=default_committer_name) From d6356c00f1c2d866401c9c2b740802105f38257c Mon Sep 17 00:00:00 2001 From: George Vauter Date: Tue, 3 Dec 2024 16:08:49 -0500 Subject: [PATCH 068/108] Update trestlebot/cli/commands/init.py Co-authored-by: Jennifer Power --- trestlebot/cli/commands/init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trestlebot/cli/commands/init.py b/trestlebot/cli/commands/init.py index a0ef718c..d1a1e925 100644 --- a/trestlebot/cli/commands/init.py +++ b/trestlebot/cli/commands/init.py @@ -161,7 +161,7 @@ def init_cmd( # inovke the init command in compliance trestle call_trestle_init(repo_path, debug) - # generate and write trestle-bot cofig + # generate and write trestle-bot config config_values = dict(repo_path=repo_path, markdown_dir=markdown_dir) if default_committer_name: config_values.update(committer_name=default_committer_name) From c6b5560a3a085d9f97f001ac2680ada7fc77ed9e Mon Sep 17 00:00:00 2001 From: George Vauter Date: Tue, 3 Dec 2024 16:09:04 -0500 Subject: [PATCH 069/108] Update trestlebot/cli/commands/init.py Co-authored-by: Jennifer Power --- trestlebot/cli/commands/init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trestlebot/cli/commands/init.py b/trestlebot/cli/commands/init.py index d1a1e925..132973b4 100644 --- a/trestlebot/cli/commands/init.py +++ b/trestlebot/cli/commands/init.py @@ -158,7 +158,7 @@ def init_cmd( ) logger.debug("Created markdown directories successfully") - # inovke the init command in compliance trestle + # invoke the init command in compliance trestle call_trestle_init(repo_path, debug) # generate and write trestle-bot config From cba3776507c85a33b9845656f7f7df0af19f8c2b Mon Sep 17 00:00:00 2001 From: George Vauter Date: Tue, 3 Dec 2024 16:09:04 -0500 Subject: [PATCH 070/108] Update trestlebot/cli/commands/init.py Co-authored-by: Jennifer Power --- trestlebot/cli/commands/init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trestlebot/cli/commands/init.py b/trestlebot/cli/commands/init.py index d1a1e925..132973b4 100644 --- a/trestlebot/cli/commands/init.py +++ b/trestlebot/cli/commands/init.py @@ -158,7 +158,7 @@ def init_cmd( ) logger.debug("Created markdown directories successfully") - # inovke the init command in compliance trestle + # invoke the init command in compliance trestle call_trestle_init(repo_path, debug) # generate and write trestle-bot config From 5f456a87c8b262ee0436c50e57732c8bd1cb33b6 Mon Sep 17 00:00:00 2001 From: George Vauter Date: Tue, 3 Dec 2024 16:12:13 -0500 Subject: [PATCH 071/108] fix typo in error msg --- trestlebot/cli/commands/init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trestlebot/cli/commands/init.py b/trestlebot/cli/commands/init.py index 132973b4..972b4b0c 100644 --- a/trestlebot/cli/commands/init.py +++ b/trestlebot/cli/commands/init.py @@ -45,7 +45,7 @@ def call_trestle_init(repo_path: pathlib.Path, debug: bool) -> None: logger.debug("Initialized trestle project successfully") else: logger.error( - f"Initialization failed. Unexpted trestle error: {CmdReturnCodes(return_code).name}" + f"Initialization failed. Unexpected trestle error: {CmdReturnCodes(return_code).name}" ) sys.exit(ERROR_EXIT_CODE) From f72483d6fa60ecfccbc050c35c5491211d490474 Mon Sep 17 00:00:00 2001 From: George Vauter Date: Tue, 3 Dec 2024 16:12:13 -0500 Subject: [PATCH 072/108] fix typo in error msg --- trestlebot/cli/commands/init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trestlebot/cli/commands/init.py b/trestlebot/cli/commands/init.py index 132973b4..972b4b0c 100644 --- a/trestlebot/cli/commands/init.py +++ b/trestlebot/cli/commands/init.py @@ -45,7 +45,7 @@ def call_trestle_init(repo_path: pathlib.Path, debug: bool) -> None: logger.debug("Initialized trestle project successfully") else: logger.error( - f"Initialization failed. Unexpted trestle error: {CmdReturnCodes(return_code).name}" + f"Initialization failed. Unexpected trestle error: {CmdReturnCodes(return_code).name}" ) sys.exit(ERROR_EXIT_CODE) From 7bac602ac0bb0ac9d0f1cb439d352f4129db98fa Mon Sep 17 00:00:00 2001 From: George Vauter Date: Tue, 3 Dec 2024 16:41:55 -0500 Subject: [PATCH 073/108] fix help text for sync upstreams --- trestlebot/cli/commands/sync_upstreams.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trestlebot/cli/commands/sync_upstreams.py b/trestlebot/cli/commands/sync_upstreams.py index 2cfef7b0..adfcafbb 100644 --- a/trestlebot/cli/commands/sync_upstreams.py +++ b/trestlebot/cli/commands/sync_upstreams.py @@ -42,7 +42,7 @@ def load_value_from_ctx( @click.command( name="sync-upstreams", - help="sync and validate OSCAL content from upstream repositories.", + help="Sync OSCAL content from upstream repositories.", ) @click.pass_context @click.option( From f8c12b7133daa941e66809b7659ffa09eb154357 Mon Sep 17 00:00:00 2001 From: George Vauter Date: Tue, 3 Dec 2024 16:41:55 -0500 Subject: [PATCH 074/108] fix help text for sync upstreams --- trestlebot/cli/commands/sync_upstreams.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trestlebot/cli/commands/sync_upstreams.py b/trestlebot/cli/commands/sync_upstreams.py index 2cfef7b0..adfcafbb 100644 --- a/trestlebot/cli/commands/sync_upstreams.py +++ b/trestlebot/cli/commands/sync_upstreams.py @@ -42,7 +42,7 @@ def load_value_from_ctx( @click.command( name="sync-upstreams", - help="sync and validate OSCAL content from upstream repositories.", + help="Sync OSCAL content from upstream repositories.", ) @click.pass_context @click.option( From fa1466cea50b7dcd9510044c26a675525803c65d Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Tue, 3 Dec 2024 16:45:49 -0500 Subject: [PATCH 075/108] fix: update for help text and testing location errors --- tests/trestlebot/cli/test_create_cmd.py | 7 +++---- trestlebot/cli/commands/create.py | 6 +++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/trestlebot/cli/test_create_cmd.py b/tests/trestlebot/cli/test_create_cmd.py index 8321a5f7..2b8bfd7b 100644 --- a/tests/trestlebot/cli/test_create_cmd.py +++ b/tests/trestlebot/cli/test_create_cmd.py @@ -19,7 +19,7 @@ def test_invalid_create_cmd() -> None: def test_create_ssp_cmd(tmp_init_dir: str) -> None: """Tests successful create ssp command.""" - SSP_INDEX_FILE = "tester-ssp-index.json" + SSP_INDEX_FILE = "tmp_init_dir/tester-ssp-index.json" runner = CliRunner() result = runner.invoke( @@ -68,7 +68,6 @@ def test_create_compdef_cmd(tmp_init_dir: str) -> None: def test_default_ssp_index_file_cmd(tmp_init_dir: str) -> None: """Tests successful default ssp_index.json file creation.""" - runner = CliRunner() result = runner.invoke( create_cmd, @@ -89,7 +88,7 @@ def test_default_ssp_index_file_cmd(tmp_init_dir: str) -> None: def test_markdown_files_not_created(tmp_init_dir: str) -> None: """Tests failure of markdown file creation when not supplied with directory name.""" - SSP_INDEX_TESTER = "ssp-tester.index.json" + SSP_INDEX_TESTER = "tmp_init_dir/ssp-tester.index.json" runner = CliRunner() result = runner.invoke( @@ -111,7 +110,7 @@ def test_markdown_files_not_created(tmp_init_dir: str) -> None: def test_markdown_files_created(tmp_init_dir: str) -> None: """Tests successful creation of markdown files.""" - SSP_INDEX_TESTER = "ssp-tester.index.json" + SSP_INDEX_TESTER = "tmp_init_dir/ssp-tester.index.json" markdown_file_tester = "/markdown/system-security-plan.md" runner = CliRunner() diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index 5fc855d7..595e4653 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -30,7 +30,7 @@ logger = logging.getLogger(__name__) -@click.group(name="create", help="Component definition and ssp authoring.") +@click.group(name="create", help="Component definition and ssp authoring command.") @click.pass_context @handle_exceptions def create_cmd(ctx: click.Context) -> None: @@ -40,7 +40,7 @@ def create_cmd(ctx: click.Context) -> None: pass -@create_cmd.command("compdef") +@create_cmd.command(name="compdef", help="Component definition authoring subcommand.") @click.pass_context @common_create_options @common_options @@ -143,7 +143,7 @@ def compdef_cmd( logger.debug(f"You have successfully authored the {compdef_name}.") -@create_cmd.command("ssp") +@create_cmd.command(name="ssp", help="Authoring ssp subcommand.") @click.pass_context @common_create_options @common_options From 8133f7357e9d0307f5da7f1581bda5af59801277 Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Tue, 3 Dec 2024 16:45:49 -0500 Subject: [PATCH 076/108] fix: update for help text and testing location errors --- tests/trestlebot/cli/test_create_cmd.py | 7 +++---- trestlebot/cli/commands/create.py | 6 +++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/trestlebot/cli/test_create_cmd.py b/tests/trestlebot/cli/test_create_cmd.py index 8321a5f7..2b8bfd7b 100644 --- a/tests/trestlebot/cli/test_create_cmd.py +++ b/tests/trestlebot/cli/test_create_cmd.py @@ -19,7 +19,7 @@ def test_invalid_create_cmd() -> None: def test_create_ssp_cmd(tmp_init_dir: str) -> None: """Tests successful create ssp command.""" - SSP_INDEX_FILE = "tester-ssp-index.json" + SSP_INDEX_FILE = "tmp_init_dir/tester-ssp-index.json" runner = CliRunner() result = runner.invoke( @@ -68,7 +68,6 @@ def test_create_compdef_cmd(tmp_init_dir: str) -> None: def test_default_ssp_index_file_cmd(tmp_init_dir: str) -> None: """Tests successful default ssp_index.json file creation.""" - runner = CliRunner() result = runner.invoke( create_cmd, @@ -89,7 +88,7 @@ def test_default_ssp_index_file_cmd(tmp_init_dir: str) -> None: def test_markdown_files_not_created(tmp_init_dir: str) -> None: """Tests failure of markdown file creation when not supplied with directory name.""" - SSP_INDEX_TESTER = "ssp-tester.index.json" + SSP_INDEX_TESTER = "tmp_init_dir/ssp-tester.index.json" runner = CliRunner() result = runner.invoke( @@ -111,7 +110,7 @@ def test_markdown_files_not_created(tmp_init_dir: str) -> None: def test_markdown_files_created(tmp_init_dir: str) -> None: """Tests successful creation of markdown files.""" - SSP_INDEX_TESTER = "ssp-tester.index.json" + SSP_INDEX_TESTER = "tmp_init_dir/ssp-tester.index.json" markdown_file_tester = "/markdown/system-security-plan.md" runner = CliRunner() diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index 5fc855d7..595e4653 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -30,7 +30,7 @@ logger = logging.getLogger(__name__) -@click.group(name="create", help="Component definition and ssp authoring.") +@click.group(name="create", help="Component definition and ssp authoring command.") @click.pass_context @handle_exceptions def create_cmd(ctx: click.Context) -> None: @@ -40,7 +40,7 @@ def create_cmd(ctx: click.Context) -> None: pass -@create_cmd.command("compdef") +@create_cmd.command(name="compdef", help="Component definition authoring subcommand.") @click.pass_context @common_create_options @common_options @@ -143,7 +143,7 @@ def compdef_cmd( logger.debug(f"You have successfully authored the {compdef_name}.") -@create_cmd.command("ssp") +@create_cmd.command(name="ssp", help="Authoring ssp subcommand.") @click.pass_context @common_create_options @common_options From be89e66f4beda8b517d5c3134f9c24904f654f05 Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Tue, 3 Dec 2024 22:48:27 -0500 Subject: [PATCH 077/108] fix: update for clarity on profile name for trestle workspace --- trestlebot/cli/options/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trestlebot/cli/options/create.py b/trestlebot/cli/options/create.py index a47939cc..1d541942 100644 --- a/trestlebot/cli/options/create.py +++ b/trestlebot/cli/options/create.py @@ -21,7 +21,7 @@ def common_create_options(f: F) -> F: @click.option( "--profile-name", - prompt="Name of trestle workspace to include", + prompt="Name of profile in trestle workspace to include", help="Name of profile in trestle workspace to include.", ) @click.option( From 4c5a7e2851c61884d31aada928bee57c0b496da7 Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Tue, 3 Dec 2024 22:48:27 -0500 Subject: [PATCH 078/108] fix: update for clarity on profile name for trestle workspace --- trestlebot/cli/options/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trestlebot/cli/options/create.py b/trestlebot/cli/options/create.py index a47939cc..1d541942 100644 --- a/trestlebot/cli/options/create.py +++ b/trestlebot/cli/options/create.py @@ -21,7 +21,7 @@ def common_create_options(f: F) -> F: @click.option( "--profile-name", - prompt="Name of trestle workspace to include", + prompt="Name of profile in trestle workspace to include", help="Name of profile in trestle workspace to include.", ) @click.option( From f6fcf2d2c4d7a8293b67836f57f68e530e92b524 Mon Sep 17 00:00:00 2001 From: Qingmin Duanmu Date: Wed, 4 Dec 2024 11:29:40 +0800 Subject: [PATCH 079/108] Fix AssertionError, add missing register --- tests/trestlebot/cli/test_rule_transform_cmd.py | 2 +- trestlebot/cli/commands/autosync.py | 2 +- trestlebot/cli/root.py | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/trestlebot/cli/test_rule_transform_cmd.py b/tests/trestlebot/cli/test_rule_transform_cmd.py index 2e75e07d..e2d4f741 100644 --- a/tests/trestlebot/cli/test_rule_transform_cmd.py +++ b/tests/trestlebot/cli/test_rule_transform_cmd.py @@ -50,4 +50,4 @@ def test_rule_transform(tmp_repo: Tuple[str, Repo]) -> None: assert result.exit_code == 0 assert repo_path.joinpath(test_md).exists() commit = next(repo.iter_commits()) - assert len(commit.stats.files) == 16 + assert len(commit.stats.files) == 9 diff --git a/trestlebot/cli/commands/autosync.py b/trestlebot/cli/commands/autosync.py index dd434a4a..87d607a5 100644 --- a/trestlebot/cli/commands/autosync.py +++ b/trestlebot/cli/commands/autosync.py @@ -22,7 +22,7 @@ logger = logging.getLogger(__name__) -@click.command("autosync") +@click.command("autosync", help="Autosync catalog, profile, compdef and ssp.") @click.pass_context @common_options @git_options diff --git a/trestlebot/cli/root.py b/trestlebot/cli/root.py index 9d9f463e..53837b7e 100644 --- a/trestlebot/cli/root.py +++ b/trestlebot/cli/root.py @@ -9,6 +9,7 @@ from trestlebot.cli.commands.autosync import autosync_cmd from trestlebot.cli.commands.create import create_cmd from trestlebot.cli.commands.init import init_cmd +from trestlebot.cli.commands.rule_transform import rule_transform_cmd from trestlebot.cli.commands.sync_upstreams import sync_upstreams_cmd @@ -29,6 +30,7 @@ def root_cmd(ctx: click.Context) -> None: root_cmd.add_command(init_cmd) -root_cmd.add_command(create_cmd) root_cmd.add_command(autosync_cmd) +root_cmd.add_command(create_cmd) +root_cmd.add_command(rule_transform_cmd) root_cmd.add_command(sync_upstreams_cmd) From 1669ea89b1467dcaf06fb5024c8932d9b4341dc6 Mon Sep 17 00:00:00 2001 From: Qingmin Duanmu Date: Wed, 4 Dec 2024 11:29:40 +0800 Subject: [PATCH 080/108] Fix AssertionError, add missing register --- tests/trestlebot/cli/test_rule_transform_cmd.py | 2 +- trestlebot/cli/commands/autosync.py | 2 +- trestlebot/cli/root.py | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/trestlebot/cli/test_rule_transform_cmd.py b/tests/trestlebot/cli/test_rule_transform_cmd.py index 2e75e07d..e2d4f741 100644 --- a/tests/trestlebot/cli/test_rule_transform_cmd.py +++ b/tests/trestlebot/cli/test_rule_transform_cmd.py @@ -50,4 +50,4 @@ def test_rule_transform(tmp_repo: Tuple[str, Repo]) -> None: assert result.exit_code == 0 assert repo_path.joinpath(test_md).exists() commit = next(repo.iter_commits()) - assert len(commit.stats.files) == 16 + assert len(commit.stats.files) == 9 diff --git a/trestlebot/cli/commands/autosync.py b/trestlebot/cli/commands/autosync.py index dd434a4a..87d607a5 100644 --- a/trestlebot/cli/commands/autosync.py +++ b/trestlebot/cli/commands/autosync.py @@ -22,7 +22,7 @@ logger = logging.getLogger(__name__) -@click.command("autosync") +@click.command("autosync", help="Autosync catalog, profile, compdef and ssp.") @click.pass_context @common_options @git_options diff --git a/trestlebot/cli/root.py b/trestlebot/cli/root.py index 9d9f463e..53837b7e 100644 --- a/trestlebot/cli/root.py +++ b/trestlebot/cli/root.py @@ -9,6 +9,7 @@ from trestlebot.cli.commands.autosync import autosync_cmd from trestlebot.cli.commands.create import create_cmd from trestlebot.cli.commands.init import init_cmd +from trestlebot.cli.commands.rule_transform import rule_transform_cmd from trestlebot.cli.commands.sync_upstreams import sync_upstreams_cmd @@ -29,6 +30,7 @@ def root_cmd(ctx: click.Context) -> None: root_cmd.add_command(init_cmd) -root_cmd.add_command(create_cmd) root_cmd.add_command(autosync_cmd) +root_cmd.add_command(create_cmd) +root_cmd.add_command(rule_transform_cmd) root_cmd.add_command(sync_upstreams_cmd) From 80cd9ccf4e29842530bac1314fb1949a6953b502 Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Wed, 4 Dec 2024 09:04:53 -0500 Subject: [PATCH 081/108] fix: profile name prompting update --- trestlebot/cli/options/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trestlebot/cli/options/create.py b/trestlebot/cli/options/create.py index 1d541942..1323a9ee 100644 --- a/trestlebot/cli/options/create.py +++ b/trestlebot/cli/options/create.py @@ -21,7 +21,7 @@ def common_create_options(f: F) -> F: @click.option( "--profile-name", - prompt="Name of profile in trestle workspace to include", + prompt="Enter name of profile in trestle workspace to include", help="Name of profile in trestle workspace to include.", ) @click.option( From 4226a867557a1a174756bcfe52426b8dc7d9b8ac Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Wed, 4 Dec 2024 09:04:53 -0500 Subject: [PATCH 082/108] fix: profile name prompting update --- trestlebot/cli/options/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trestlebot/cli/options/create.py b/trestlebot/cli/options/create.py index 1d541942..1323a9ee 100644 --- a/trestlebot/cli/options/create.py +++ b/trestlebot/cli/options/create.py @@ -21,7 +21,7 @@ def common_create_options(f: F) -> F: @click.option( "--profile-name", - prompt="Name of profile in trestle workspace to include", + prompt="Enter name of profile in trestle workspace to include", help="Name of profile in trestle workspace to include.", ) @click.option( From 877ac9a946b4c6b19cc66e419f930f195e2e49ac Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Wed, 4 Dec 2024 10:26:42 -0500 Subject: [PATCH 083/108] feat: updating compdef list to required --- tests/trestlebot/cli/test_create_cmd.py | 30 +++++++++++++++++++------ trestlebot/cli/commands/create.py | 3 ++- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/tests/trestlebot/cli/test_create_cmd.py b/tests/trestlebot/cli/test_create_cmd.py index 2b8bfd7b..a53ba500 100644 --- a/tests/trestlebot/cli/test_create_cmd.py +++ b/tests/trestlebot/cli/test_create_cmd.py @@ -19,7 +19,8 @@ def test_invalid_create_cmd() -> None: def test_create_ssp_cmd(tmp_init_dir: str) -> None: """Tests successful create ssp command.""" - SSP_INDEX_FILE = "tmp_init_dir/tester-ssp-index.json" + compdef_list_tester = ["ac-12", "ac-13"] + ssp_index_tester = "tmp_init_dir/tester-ssp-index.json" runner = CliRunner() result = runner.invoke( @@ -30,12 +31,14 @@ def test_create_ssp_cmd(tmp_init_dir: str) -> None: "oscal-profile-name", "--markdown-dir", "markdown", + "--compdefs", + compdef_list_tester, "--ssp-name", "test-name", "--repo-path", tmp_init_dir, "--ssp-index-file", - SSP_INDEX_FILE, + ssp_index_tester, ], ) assert result.exit_code == 0 @@ -68,6 +71,9 @@ def test_create_compdef_cmd(tmp_init_dir: str) -> None: def test_default_ssp_index_file_cmd(tmp_init_dir: str) -> None: """Tests successful default ssp_index.json file creation.""" + compdef_list_tester = ["ac-12", "ac-13"] + default_ssp_index_file = "tmp_init_dir/test-ssp-index.json" + runner = CliRunner() result = runner.invoke( create_cmd, @@ -77,10 +83,14 @@ def test_default_ssp_index_file_cmd(tmp_init_dir: str) -> None: "oscal-profile-name", "--markdown-dir", "markdown", + "--compdefs", + compdef_list_tester, "--ssp-name", "test-name", "--repo-path", tmp_init_dir, + "--ssp-index-file", + default_ssp_index_file, ], ) assert result.exit_code == 0 @@ -88,7 +98,8 @@ def test_default_ssp_index_file_cmd(tmp_init_dir: str) -> None: def test_markdown_files_not_created(tmp_init_dir: str) -> None: """Tests failure of markdown file creation when not supplied with directory name.""" - SSP_INDEX_TESTER = "tmp_init_dir/ssp-tester.index.json" + compdef_list_tester = ["ac-12", "ac-13"] + ssp_index_tester = "tmp_init_dir/ssp-tester.index.json" runner = CliRunner() result = runner.invoke( @@ -97,12 +108,14 @@ def test_markdown_files_not_created(tmp_init_dir: str) -> None: "ssp", "--profile-name", "oscal-profile-name", + "--compdefs", + compdef_list_tester, "--ssp-name", "test-name", "repo-path", tmp_init_dir, "--ssp-index-file", - SSP_INDEX_TESTER, + ssp_index_tester, ], ) assert result.exit_code == 2 @@ -110,8 +123,9 @@ def test_markdown_files_not_created(tmp_init_dir: str) -> None: def test_markdown_files_created(tmp_init_dir: str) -> None: """Tests successful creation of markdown files.""" - SSP_INDEX_TESTER = "tmp_init_dir/ssp-tester.index.json" - markdown_file_tester = "/markdown/system-security-plan.md" + ssp_index_tester = "tmp_init_dir/ssp-tester.index.json" + markdown_file_tester = "tmp_init_dir/markdown/system-security-plan.md" + compdef_list_tester = ["ac-12", "ac-13"] runner = CliRunner() result = runner.invoke( @@ -122,12 +136,14 @@ def test_markdown_files_created(tmp_init_dir: str) -> None: "oscal-profile-name", "--markdown-dir", markdown_file_tester, + "--compdefs", + compdef_list_tester, "--ssp-name", "test-name", "--repo-path", tmp_init_dir, "--ssp-index-file", - SSP_INDEX_TESTER, + ssp_index_tester, ], ) assert result.exit_code == 0 diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index 595e4653..052b2c24 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -181,8 +181,9 @@ def compdef_cmd( ) @click.option( "--compdefs", - required=False, + required=True, type=str, + prompt="Enter comma separated list of component definitions to include in SSP", help="Comma separated list of component definitions.", ) @handle_exceptions From 66f55c7393669999090a8ec9757c754019af2f96 Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Wed, 4 Dec 2024 10:26:42 -0500 Subject: [PATCH 084/108] feat: updating compdef list to required --- tests/trestlebot/cli/test_create_cmd.py | 30 +++++++++++++++++++------ trestlebot/cli/commands/create.py | 3 ++- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/tests/trestlebot/cli/test_create_cmd.py b/tests/trestlebot/cli/test_create_cmd.py index 2b8bfd7b..a53ba500 100644 --- a/tests/trestlebot/cli/test_create_cmd.py +++ b/tests/trestlebot/cli/test_create_cmd.py @@ -19,7 +19,8 @@ def test_invalid_create_cmd() -> None: def test_create_ssp_cmd(tmp_init_dir: str) -> None: """Tests successful create ssp command.""" - SSP_INDEX_FILE = "tmp_init_dir/tester-ssp-index.json" + compdef_list_tester = ["ac-12", "ac-13"] + ssp_index_tester = "tmp_init_dir/tester-ssp-index.json" runner = CliRunner() result = runner.invoke( @@ -30,12 +31,14 @@ def test_create_ssp_cmd(tmp_init_dir: str) -> None: "oscal-profile-name", "--markdown-dir", "markdown", + "--compdefs", + compdef_list_tester, "--ssp-name", "test-name", "--repo-path", tmp_init_dir, "--ssp-index-file", - SSP_INDEX_FILE, + ssp_index_tester, ], ) assert result.exit_code == 0 @@ -68,6 +71,9 @@ def test_create_compdef_cmd(tmp_init_dir: str) -> None: def test_default_ssp_index_file_cmd(tmp_init_dir: str) -> None: """Tests successful default ssp_index.json file creation.""" + compdef_list_tester = ["ac-12", "ac-13"] + default_ssp_index_file = "tmp_init_dir/test-ssp-index.json" + runner = CliRunner() result = runner.invoke( create_cmd, @@ -77,10 +83,14 @@ def test_default_ssp_index_file_cmd(tmp_init_dir: str) -> None: "oscal-profile-name", "--markdown-dir", "markdown", + "--compdefs", + compdef_list_tester, "--ssp-name", "test-name", "--repo-path", tmp_init_dir, + "--ssp-index-file", + default_ssp_index_file, ], ) assert result.exit_code == 0 @@ -88,7 +98,8 @@ def test_default_ssp_index_file_cmd(tmp_init_dir: str) -> None: def test_markdown_files_not_created(tmp_init_dir: str) -> None: """Tests failure of markdown file creation when not supplied with directory name.""" - SSP_INDEX_TESTER = "tmp_init_dir/ssp-tester.index.json" + compdef_list_tester = ["ac-12", "ac-13"] + ssp_index_tester = "tmp_init_dir/ssp-tester.index.json" runner = CliRunner() result = runner.invoke( @@ -97,12 +108,14 @@ def test_markdown_files_not_created(tmp_init_dir: str) -> None: "ssp", "--profile-name", "oscal-profile-name", + "--compdefs", + compdef_list_tester, "--ssp-name", "test-name", "repo-path", tmp_init_dir, "--ssp-index-file", - SSP_INDEX_TESTER, + ssp_index_tester, ], ) assert result.exit_code == 2 @@ -110,8 +123,9 @@ def test_markdown_files_not_created(tmp_init_dir: str) -> None: def test_markdown_files_created(tmp_init_dir: str) -> None: """Tests successful creation of markdown files.""" - SSP_INDEX_TESTER = "tmp_init_dir/ssp-tester.index.json" - markdown_file_tester = "/markdown/system-security-plan.md" + ssp_index_tester = "tmp_init_dir/ssp-tester.index.json" + markdown_file_tester = "tmp_init_dir/markdown/system-security-plan.md" + compdef_list_tester = ["ac-12", "ac-13"] runner = CliRunner() result = runner.invoke( @@ -122,12 +136,14 @@ def test_markdown_files_created(tmp_init_dir: str) -> None: "oscal-profile-name", "--markdown-dir", markdown_file_tester, + "--compdefs", + compdef_list_tester, "--ssp-name", "test-name", "--repo-path", tmp_init_dir, "--ssp-index-file", - SSP_INDEX_TESTER, + ssp_index_tester, ], ) assert result.exit_code == 0 diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index 595e4653..052b2c24 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -181,8 +181,9 @@ def compdef_cmd( ) @click.option( "--compdefs", - required=False, + required=True, type=str, + prompt="Enter comma separated list of component definitions to include in SSP", help="Comma separated list of component definitions.", ) @handle_exceptions From db34e53d633da4d3483461ffd74b04ec19d57f82 Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Wed, 4 Dec 2024 10:59:24 -0500 Subject: [PATCH 085/108] docs: change of verbiage for readability --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4995ba23..f533ac5b 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ trestle-bot assists users in leveraging [Compliance-Trestle](https://github.com/ ### Available Commands -The `autosync` command will sync trestle-generated Markdown files to OSCAL JSON files in a trestle workspace. All content under the provided markdown directory when the action is run will be transformed. This action supports all top-level models [supported by compliance-trestle for authoring](https://oscal-compass.github.io/compliance-trestle/tutorials/ssp_profile_catalog_authoring/ssp_profile_catalog_authoring/). +The `autosync` command will sync trestle-generated Markdown files to OSCAL JSON files in a trestle workspace. All content under the provided markdown directory will be transformed when the action is run. This action supports all top-level models [supported by compliance-trestle for authoring](https://oscal-compass.github.io/compliance-trestle/tutorials/ssp_profile_catalog_authoring/ssp_profile_catalog_authoring/). The `rules-transform` command can be used when managing [OSCAL Component Definitions](https://pages.nist.gov/OSCAL-Reference/models/v1.1.1/component-definition/json-outline/) in a trestle workspace. The action will transform rules defined in the rules YAML view to an OSCAL Component Definition JSON file. From dfc0abaeb85312d0e3cf055aaaf3419461f48237 Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Wed, 4 Dec 2024 10:59:24 -0500 Subject: [PATCH 086/108] docs: change of verbiage for readability --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4995ba23..f533ac5b 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ trestle-bot assists users in leveraging [Compliance-Trestle](https://github.com/ ### Available Commands -The `autosync` command will sync trestle-generated Markdown files to OSCAL JSON files in a trestle workspace. All content under the provided markdown directory when the action is run will be transformed. This action supports all top-level models [supported by compliance-trestle for authoring](https://oscal-compass.github.io/compliance-trestle/tutorials/ssp_profile_catalog_authoring/ssp_profile_catalog_authoring/). +The `autosync` command will sync trestle-generated Markdown files to OSCAL JSON files in a trestle workspace. All content under the provided markdown directory will be transformed when the action is run. This action supports all top-level models [supported by compliance-trestle for authoring](https://oscal-compass.github.io/compliance-trestle/tutorials/ssp_profile_catalog_authoring/ssp_profile_catalog_authoring/). The `rules-transform` command can be used when managing [OSCAL Component Definitions](https://pages.nist.gov/OSCAL-Reference/models/v1.1.1/component-definition/json-outline/) in a trestle workspace. The action will transform rules defined in the rules YAML view to an OSCAL Component Definition JSON file. From 5c5a17a0ec137c757d33a2a20515704bee714c59 Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Wed, 4 Dec 2024 11:32:44 -0500 Subject: [PATCH 087/108] docs: change to indicate trestle-bot as a cli tool --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f533ac5b..f5ee35a5 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ -trestle-bot assists users in leveraging [Compliance-Trestle](https://github.com/oscal-compass/compliance-trestle) in CI/CD workflows for [OSCAL](https://github.com/usnistgov/OSCAL) formatted compliance content management. +trestle-bot is a CLI tool that assists users in leveraging [Compliance-Trestle](https://github.com/oscal-compass/compliance-trestle) in CI/CD workflows for [OSCAL](https://github.com/usnistgov/OSCAL) formatted compliance content management. > WARNING: This project is currently under initial development. APIs may be changed incompatibly from one commit to another. From 230568d2e89fed65a9b6426d19dde75e4ffa12a0 Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Wed, 4 Dec 2024 11:32:44 -0500 Subject: [PATCH 088/108] docs: change to indicate trestle-bot as a cli tool --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f533ac5b..f5ee35a5 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ -trestle-bot assists users in leveraging [Compliance-Trestle](https://github.com/oscal-compass/compliance-trestle) in CI/CD workflows for [OSCAL](https://github.com/usnistgov/OSCAL) formatted compliance content management. +trestle-bot is a CLI tool that assists users in leveraging [Compliance-Trestle](https://github.com/oscal-compass/compliance-trestle) in CI/CD workflows for [OSCAL](https://github.com/usnistgov/OSCAL) formatted compliance content management. > WARNING: This project is currently under initial development. APIs may be changed incompatibly from one commit to another. From 0e31aa88a64b5495737dea80641a0fe394e4ba2f Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Wed, 4 Dec 2024 11:56:38 -0500 Subject: [PATCH 089/108] feat: change to help description of create command --- trestlebot/cli/commands/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index 052b2c24..f7433124 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -30,7 +30,7 @@ logger = logging.getLogger(__name__) -@click.group(name="create", help="Component definition and ssp authoring command.") +@click.group(name="create", help="Component definition and ssp authoring.") @click.pass_context @handle_exceptions def create_cmd(ctx: click.Context) -> None: From 143b92a0c4a5ac3dc326eb923a79854b061a86b7 Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Wed, 4 Dec 2024 11:56:38 -0500 Subject: [PATCH 090/108] feat: change to help description of create command --- trestlebot/cli/commands/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index 052b2c24..f7433124 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -30,7 +30,7 @@ logger = logging.getLogger(__name__) -@click.group(name="create", help="Component definition and ssp authoring command.") +@click.group(name="create", help="Component definition and ssp authoring.") @click.pass_context @handle_exceptions def create_cmd(ctx: click.Context) -> None: From 88da51277a576f93c3e3e11b2789b5fb0c486317 Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Fri, 6 Dec 2024 09:59:49 -0500 Subject: [PATCH 091/108] docs: added high level folder structure for cli --- CONTRIBUTING.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9c26e755..9eb30e1b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -73,7 +73,9 @@ For workflow diagrams, see the [diagrams](./docs/workflows/) under the `docs` fo #### Code structure - `actions` - Provides specific logic for `trestle-bot` tasks that are packaged as Actions. See [README.md](./actions/README.md) for more information. -- `entrypoints` - Provides top level logic for specific user-facing tasks. These tasks are not necessarily related in any way so they are not organized into a hierarchical command structure, but they do inherit logic and flags from a base class. +- `cli` - Provides top level logic for specific user-facing tasks. These tasks are not necessarily related so they are not organized into a hierarchical command structure, but they do share some common modules. +- `cli/commands` - Provides top level logic for commands and their associated subcommands. The commands are accessed by the single entrypoint `root.py`. +- `cli/options` - Provides command line options and arguments that are frequently used within `cli/commands`. - `provider.py, github.py, and gitlab.py` - Git provider abstract class and concrete implementations for interacting with the API. - `tasks` - Pre-tasks can be configured before the main git logic is run. Any task that does workspace management should go here. - `tasks/authored` - The `authored` package contains logic for managing authoring tasks for single instances of a top-level OSCAL model. These encapsulate logic from the `compliance-trestle` library and allows loose coupling between `tasks` and `authored` types. From 553ad1d85063297fa957ddb46b5d0f1ddde33aa1 Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Fri, 6 Dec 2024 09:59:49 -0500 Subject: [PATCH 092/108] docs: added high level folder structure for cli --- CONTRIBUTING.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9c26e755..9eb30e1b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -73,7 +73,9 @@ For workflow diagrams, see the [diagrams](./docs/workflows/) under the `docs` fo #### Code structure - `actions` - Provides specific logic for `trestle-bot` tasks that are packaged as Actions. See [README.md](./actions/README.md) for more information. -- `entrypoints` - Provides top level logic for specific user-facing tasks. These tasks are not necessarily related in any way so they are not organized into a hierarchical command structure, but they do inherit logic and flags from a base class. +- `cli` - Provides top level logic for specific user-facing tasks. These tasks are not necessarily related so they are not organized into a hierarchical command structure, but they do share some common modules. +- `cli/commands` - Provides top level logic for commands and their associated subcommands. The commands are accessed by the single entrypoint `root.py`. +- `cli/options` - Provides command line options and arguments that are frequently used within `cli/commands`. - `provider.py, github.py, and gitlab.py` - Git provider abstract class and concrete implementations for interacting with the API. - `tasks` - Pre-tasks can be configured before the main git logic is run. Any task that does workspace management should go here. - `tasks/authored` - The `authored` package contains logic for managing authoring tasks for single instances of a top-level OSCAL model. These encapsulate logic from the `compliance-trestle` library and allows loose coupling between `tasks` and `authored` types. From 7711aec19a308af57d47556786a874f38c0194ea Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Wed, 11 Dec 2024 09:21:15 -0500 Subject: [PATCH 093/108] fix: default value returned if no key in dictionary --- trestlebot/cli/commands/create.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index f7433124..b8995eea 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -89,8 +89,8 @@ def compdef_cmd( compdef_name = kwargs["compdef_name"] component_title = kwargs["component_title"] component_description = kwargs["component_description"] - filter_by_profile = kwargs["filter_by_profile"] - component_definition_type = kwargs["component_definition_type"] + filter_by_profile = kwargs.get("filter_by_profile") + component_definition_type = kwargs.get("component_definition_type", "service") repo_path = kwargs["repo_path"] markdown_dir = kwargs["markdown_dir"] if filter_by_profile: @@ -197,9 +197,7 @@ def ssp_cmd( profile_name = kwargs["profile_name"] ssp_name = kwargs["ssp_name"] - leveraged_ssp = kwargs["leveraged_ssp"] - ssp_index_file = kwargs["ssp_index_file"] - yaml_header_path = kwargs["yaml_header_path"] + ssp_index_file = kwargs.get("ssp_index_file", "ssp-index.json") repo_path = kwargs["repo_path"] markdown_dir = kwargs["markdown_dir"] compdefs = kwargs["compdefs"] @@ -216,8 +214,8 @@ def ssp_cmd( profile_name=profile_name, compdefs=comps, markdown_path=markdown_dir, - leveraged_ssp=leveraged_ssp, - yaml_header=yaml_header_path, + leveraged_ssp=kwargs["leveraged_ssp"], + yaml_header=kwargs["yaml_header_path"], ) logger.debug(f"The name of the SSP to create is {ssp_name}.") From b3b8ebdff4aa4fe32ee566ed90d3920b4c40ff6b Mon Sep 17 00:00:00 2001 From: Hannah Braswell Date: Wed, 11 Dec 2024 09:21:15 -0500 Subject: [PATCH 094/108] fix: default value returned if no key in dictionary --- trestlebot/cli/commands/create.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index f7433124..b8995eea 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -89,8 +89,8 @@ def compdef_cmd( compdef_name = kwargs["compdef_name"] component_title = kwargs["component_title"] component_description = kwargs["component_description"] - filter_by_profile = kwargs["filter_by_profile"] - component_definition_type = kwargs["component_definition_type"] + filter_by_profile = kwargs.get("filter_by_profile") + component_definition_type = kwargs.get("component_definition_type", "service") repo_path = kwargs["repo_path"] markdown_dir = kwargs["markdown_dir"] if filter_by_profile: @@ -197,9 +197,7 @@ def ssp_cmd( profile_name = kwargs["profile_name"] ssp_name = kwargs["ssp_name"] - leveraged_ssp = kwargs["leveraged_ssp"] - ssp_index_file = kwargs["ssp_index_file"] - yaml_header_path = kwargs["yaml_header_path"] + ssp_index_file = kwargs.get("ssp_index_file", "ssp-index.json") repo_path = kwargs["repo_path"] markdown_dir = kwargs["markdown_dir"] compdefs = kwargs["compdefs"] @@ -216,8 +214,8 @@ def ssp_cmd( profile_name=profile_name, compdefs=comps, markdown_path=markdown_dir, - leveraged_ssp=leveraged_ssp, - yaml_header=yaml_header_path, + leveraged_ssp=kwargs["leveraged_ssp"], + yaml_header=kwargs["yaml_header_path"], ) logger.debug(f"The name of the SSP to create is {ssp_name}.") From 826a8f500154e7a5f628b362225ed26025a243a1 Mon Sep 17 00:00:00 2001 From: Qingmin Duanmu Date: Fri, 13 Dec 2024 15:58:27 +0800 Subject: [PATCH 095/108] feat: align skip-item option to skip-items --- tests/trestlebot/cli/test_autosync_cmd.py | 8 ++++++-- trestlebot/cli/commands/rule_transform.py | 10 ++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/trestlebot/cli/test_autosync_cmd.py b/tests/trestlebot/cli/test_autosync_cmd.py index 0f3bc963..e0d370ec 100644 --- a/tests/trestlebot/cli/test_autosync_cmd.py +++ b/tests/trestlebot/cli/test_autosync_cmd.py @@ -63,8 +63,12 @@ def test_missing_ssp_index_file_option(tmp_repo: Tuple[str, Repo]) -> None: def test_missing_markdown_dir_option(tmp_repo: Tuple[str, Repo]) -> None: + # When no markdown_dir setting in trestlebot config file. repo_path, _ = tmp_repo runner = CliRunner() + filepath = pathlib.Path(repo_path).joinpath("config.yml") + config_obj = TrestleBotConfig(repo_path=repo_path) + write_to_file(config_obj, filepath) cmd_options = [ "--oscal-model", "compdef", @@ -76,6 +80,8 @@ def test_missing_markdown_dir_option(tmp_repo: Tuple[str, Repo]) -> None: "Test User", "--committer-email", "test@example.com", + "--config", + str(filepath), ] result = runner.invoke(autosync_cmd, cmd_options) assert result.exit_code == 2 @@ -83,8 +89,6 @@ def test_missing_markdown_dir_option(tmp_repo: Tuple[str, Repo]) -> None: # With 'markdown_dir' setting in config.yml config_obj = TrestleBotConfig(markdown_dir="markdown") - filepath = pathlib.Path(repo_path).joinpath("config.yml") write_to_file(config_obj, filepath) - cmd_options += ["--config", str(filepath)] result = runner.invoke(autosync_cmd, cmd_options) assert result.exit_code == 0 diff --git a/trestlebot/cli/commands/rule_transform.py b/trestlebot/cli/commands/rule_transform.py index 77a856a5..b67edd2c 100644 --- a/trestlebot/cli/commands/rule_transform.py +++ b/trestlebot/cli/commands/rule_transform.py @@ -9,7 +9,7 @@ import click from trestlebot.cli.options.common import common_options, git_options, handle_exceptions -from trestlebot.cli.utils import run_bot +from trestlebot.cli.utils import comma_sep_to_list, run_bot from trestlebot.const import RULES_VIEW_DIR from trestlebot.tasks.authored.compdef import AuthoredComponentDefinition from trestlebot.tasks.base_task import ModelFilter, TaskBase @@ -40,18 +40,16 @@ default=RULES_VIEW_DIR, ) @click.option( - "--skip-item", + "--skip-items", type=str, - help="glob pattern for directories to skip when running", - multiple=True, + help="Comma-separated list of glob patterns for directories to skip when running tasks.", ) @handle_exceptions def rule_transform_cmd(ctx: click.Context, **kwargs: Any) -> None: """Run the rule transform operation.""" # Allow any model to be skipped by setting skip_item, by default include all - skip_items = list(kwargs.get("skip_item", [])) model_filter: ModelFilter = ModelFilter( - skip_patterns=skip_items, + skip_patterns=comma_sep_to_list(kwargs.get("skip_items", "")), include_patterns=["*"], ) From 5144e10ec8378ee74263944f3c87ee8ab6c877f3 Mon Sep 17 00:00:00 2001 From: Qingmin Duanmu Date: Fri, 13 Dec 2024 15:58:27 +0800 Subject: [PATCH 096/108] feat: align skip-item option to skip-items --- tests/trestlebot/cli/test_autosync_cmd.py | 8 ++++++-- trestlebot/cli/commands/rule_transform.py | 10 ++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/trestlebot/cli/test_autosync_cmd.py b/tests/trestlebot/cli/test_autosync_cmd.py index 0f3bc963..e0d370ec 100644 --- a/tests/trestlebot/cli/test_autosync_cmd.py +++ b/tests/trestlebot/cli/test_autosync_cmd.py @@ -63,8 +63,12 @@ def test_missing_ssp_index_file_option(tmp_repo: Tuple[str, Repo]) -> None: def test_missing_markdown_dir_option(tmp_repo: Tuple[str, Repo]) -> None: + # When no markdown_dir setting in trestlebot config file. repo_path, _ = tmp_repo runner = CliRunner() + filepath = pathlib.Path(repo_path).joinpath("config.yml") + config_obj = TrestleBotConfig(repo_path=repo_path) + write_to_file(config_obj, filepath) cmd_options = [ "--oscal-model", "compdef", @@ -76,6 +80,8 @@ def test_missing_markdown_dir_option(tmp_repo: Tuple[str, Repo]) -> None: "Test User", "--committer-email", "test@example.com", + "--config", + str(filepath), ] result = runner.invoke(autosync_cmd, cmd_options) assert result.exit_code == 2 @@ -83,8 +89,6 @@ def test_missing_markdown_dir_option(tmp_repo: Tuple[str, Repo]) -> None: # With 'markdown_dir' setting in config.yml config_obj = TrestleBotConfig(markdown_dir="markdown") - filepath = pathlib.Path(repo_path).joinpath("config.yml") write_to_file(config_obj, filepath) - cmd_options += ["--config", str(filepath)] result = runner.invoke(autosync_cmd, cmd_options) assert result.exit_code == 0 diff --git a/trestlebot/cli/commands/rule_transform.py b/trestlebot/cli/commands/rule_transform.py index 77a856a5..b67edd2c 100644 --- a/trestlebot/cli/commands/rule_transform.py +++ b/trestlebot/cli/commands/rule_transform.py @@ -9,7 +9,7 @@ import click from trestlebot.cli.options.common import common_options, git_options, handle_exceptions -from trestlebot.cli.utils import run_bot +from trestlebot.cli.utils import comma_sep_to_list, run_bot from trestlebot.const import RULES_VIEW_DIR from trestlebot.tasks.authored.compdef import AuthoredComponentDefinition from trestlebot.tasks.base_task import ModelFilter, TaskBase @@ -40,18 +40,16 @@ default=RULES_VIEW_DIR, ) @click.option( - "--skip-item", + "--skip-items", type=str, - help="glob pattern for directories to skip when running", - multiple=True, + help="Comma-separated list of glob patterns for directories to skip when running tasks.", ) @handle_exceptions def rule_transform_cmd(ctx: click.Context, **kwargs: Any) -> None: """Run the rule transform operation.""" # Allow any model to be skipped by setting skip_item, by default include all - skip_items = list(kwargs.get("skip_item", [])) model_filter: ModelFilter = ModelFilter( - skip_patterns=skip_items, + skip_patterns=comma_sep_to_list(kwargs.get("skip_items", "")), include_patterns=["*"], ) From 84c38bafa8790963d1f28c95e6b406de7167f94a Mon Sep 17 00:00:00 2001 From: Qingmin Duanmu Date: Mon, 16 Dec 2024 09:50:18 +0800 Subject: [PATCH 097/108] fix: add missing git options in create command --- trestlebot/cli/commands/create.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index b8995eea..2f9fdef3 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -11,7 +11,7 @@ import click from trestlebot import const -from trestlebot.cli.options.common import common_options, handle_exceptions +from trestlebot.cli.options.common import common_options, git_options, handle_exceptions from trestlebot.cli.options.create import common_create_options from trestlebot.cli.utils import run_bot from trestlebot.entrypoints.entrypoint_base import comma_sep_to_list @@ -44,6 +44,7 @@ def create_cmd(ctx: click.Context) -> None: @click.pass_context @common_create_options @common_options +@git_options @click.option( "--compdef-name", required=True, @@ -147,6 +148,7 @@ def compdef_cmd( @click.pass_context @common_create_options @common_options +@git_options @click.option( "--ssp-name", required=True, From 06e574b8e9db5e546c2a05bf5b5ca1b8a9b07bba Mon Sep 17 00:00:00 2001 From: Qingmin Duanmu Date: Mon, 16 Dec 2024 09:50:18 +0800 Subject: [PATCH 098/108] fix: add missing git options in create command --- trestlebot/cli/commands/create.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index b8995eea..2f9fdef3 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -11,7 +11,7 @@ import click from trestlebot import const -from trestlebot.cli.options.common import common_options, handle_exceptions +from trestlebot.cli.options.common import common_options, git_options, handle_exceptions from trestlebot.cli.options.create import common_create_options from trestlebot.cli.utils import run_bot from trestlebot.entrypoints.entrypoint_base import comma_sep_to_list @@ -44,6 +44,7 @@ def create_cmd(ctx: click.Context) -> None: @click.pass_context @common_create_options @common_options +@git_options @click.option( "--compdef-name", required=True, @@ -147,6 +148,7 @@ def compdef_cmd( @click.pass_context @common_create_options @common_options +@git_options @click.option( "--ssp-name", required=True, From ddff735b024933c7cc2d0bb33bb8c2404d40497f Mon Sep 17 00:00:00 2001 From: George Vauter Date: Tue, 17 Dec 2024 11:12:38 -0500 Subject: [PATCH 099/108] fix: refactor testt and remove prompts --- tests/trestlebot/cli/test_create_cmd.py | 120 ++++++++++++++---------- trestlebot/cli/commands/create.py | 6 -- 2 files changed, 69 insertions(+), 57 deletions(-) diff --git a/tests/trestlebot/cli/test_create_cmd.py b/tests/trestlebot/cli/test_create_cmd.py index a53ba500..c081645f 100644 --- a/tests/trestlebot/cli/test_create_cmd.py +++ b/tests/trestlebot/cli/test_create_cmd.py @@ -2,12 +2,22 @@ # Copyright (c) 2024 Red Hat, Inc. """ Unit test for create commands ssp and cd""" +import pathlib +from typing import Tuple from click.testing import CliRunner +from git import Repo +from tests.testutils import setup_for_compdef, setup_for_ssp from trestlebot.cli.commands.create import create_cmd +test_prof = "simplified_nist_profile" +test_comp_name = "test_comp" +test_ssp_md = "md_ssp" +test_ssp_cd = "md_cd" + + def test_invalid_create_cmd() -> None: """Tests that create command fails if given invalid oscal model subcommand.""" runner = CliRunner() @@ -17,10 +27,15 @@ def test_invalid_create_cmd() -> None: assert result.exit_code == 2 -def test_create_ssp_cmd(tmp_init_dir: str) -> None: +def test_create_ssp_cmd(tmp_repo: Tuple[str, Repo]) -> None: """Tests successful create ssp command.""" - compdef_list_tester = ["ac-12", "ac-13"] - ssp_index_tester = "tmp_init_dir/tester-ssp-index.json" + + repo_dir, _ = tmp_repo + repo_path = pathlib.Path(repo_dir) + + ssp_index_file = repo_path.joinpath("ssp-index.json") + + _ = setup_for_ssp(repo_path, test_prof, [test_comp_name], test_ssp_md) runner = CliRunner() result = runner.invoke( @@ -28,24 +43,34 @@ def test_create_ssp_cmd(tmp_init_dir: str) -> None: [ "ssp", "--profile-name", - "oscal-profile-name", + test_prof, "--markdown-dir", "markdown", "--compdefs", - compdef_list_tester, + test_comp_name, "--ssp-name", "test-name", "--repo-path", - tmp_init_dir, + str(repo_path.resolve()), "--ssp-index-file", - ssp_index_tester, + ssp_index_file, + "--committer-email", + "test@email.com", + "--committer-name", + "test name", + "--branch", + "test", ], ) assert result.exit_code == 0 -def test_create_compdef_cmd(tmp_init_dir: str) -> None: +def test_create_compdef_cmd(tmp_repo: Tuple[str, Repo]) -> None: """Tests successful create compdef command.""" + repo_dir, _ = tmp_repo + repo_path = pathlib.Path(repo_dir) + + _ = setup_for_compdef(repo_path, test_comp_name, test_ssp_cd) runner = CliRunner() result = runner.invoke( @@ -62,17 +87,24 @@ def test_create_compdef_cmd(tmp_init_dir: str) -> None: "description-test", "--component-definition-type", "type-test", - "repo-path", - tmp_init_dir, + "--repo-path", + str(repo_path.resolve()), + "--committer-email", + "test@email.com", + "--committer-name", + "test name", + "--branch", + "test", ], ) - assert result.exit_code == 2 + assert result.exit_code == 0 -def test_default_ssp_index_file_cmd(tmp_init_dir: str) -> None: +def test_default_ssp_index_file_cmd(tmp_repo: Tuple[str, Repo]) -> None: """Tests successful default ssp_index.json file creation.""" - compdef_list_tester = ["ac-12", "ac-13"] - default_ssp_index_file = "tmp_init_dir/test-ssp-index.json" + + repo_dir, _ = tmp_repo + repo_path = pathlib.Path(repo_dir) runner = CliRunner() result = runner.invoke( @@ -84,48 +116,28 @@ def test_default_ssp_index_file_cmd(tmp_init_dir: str) -> None: "--markdown-dir", "markdown", "--compdefs", - compdef_list_tester, + "test-compdef", "--ssp-name", "test-name", "--repo-path", - tmp_init_dir, - "--ssp-index-file", - default_ssp_index_file, + str(repo_path.resolve()), + "--committer-email", + "test@email.com", + "--committer-name", + "test name", + "--branch", + "test", ], ) assert result.exit_code == 0 -def test_markdown_files_not_created(tmp_init_dir: str) -> None: - """Tests failure of markdown file creation when not supplied with directory name.""" - compdef_list_tester = ["ac-12", "ac-13"] - ssp_index_tester = "tmp_init_dir/ssp-tester.index.json" - - runner = CliRunner() - result = runner.invoke( - create_cmd, - [ - "ssp", - "--profile-name", - "oscal-profile-name", - "--compdefs", - compdef_list_tester, - "--ssp-name", - "test-name", - "repo-path", - tmp_init_dir, - "--ssp-index-file", - ssp_index_tester, - ], - ) - assert result.exit_code == 2 - - -def test_markdown_files_created(tmp_init_dir: str) -> None: +def test_markdown_files_created(tmp_repo: Tuple[str, Repo]) -> None: """Tests successful creation of markdown files.""" - ssp_index_tester = "tmp_init_dir/ssp-tester.index.json" - markdown_file_tester = "tmp_init_dir/markdown/system-security-plan.md" - compdef_list_tester = ["ac-12", "ac-13"] + + repo_dir, _ = tmp_repo + repo_path = pathlib.Path(repo_dir) + ssp_index_file = repo_path.joinpath("ssp-index.json") runner = CliRunner() result = runner.invoke( @@ -135,15 +147,21 @@ def test_markdown_files_created(tmp_init_dir: str) -> None: "--profile-name", "oscal-profile-name", "--markdown-dir", - markdown_file_tester, + "markdown", "--compdefs", - compdef_list_tester, + "test-compdef", "--ssp-name", "test-name", "--repo-path", - tmp_init_dir, + str(repo_path.resolve()), "--ssp-index-file", - ssp_index_tester, + ssp_index_file, + "--committer-email", + "test@email.com", + "--committer-name", + "test name", + "--branch", + "test", ], ) assert result.exit_code == 0 diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index 2f9fdef3..f37d5104 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -48,19 +48,16 @@ def create_cmd(ctx: click.Context) -> None: @click.option( "--compdef-name", required=True, - prompt="Enter name of component definition", help="Name of component definition.", ) @click.option( "--component-title", required=True, - prompt="Enter name of component title", help="Title of initial component.", ) @click.option( "--component-description", required=True, - prompt="Enter description of the initial component", help="Description of initial component.", ) @click.option( @@ -153,7 +150,6 @@ def compdef_cmd( "--ssp-name", required=True, type=str, - prompt="Enter name of SSP to create", help="Name of SSP to create.", ) @click.option( @@ -185,7 +181,6 @@ def compdef_cmd( "--compdefs", required=True, type=str, - prompt="Enter comma separated list of component definitions to include in SSP", help="Comma separated list of component definitions.", ) @handle_exceptions @@ -196,7 +191,6 @@ def ssp_cmd( """ SSP Authoring command """ - profile_name = kwargs["profile_name"] ssp_name = kwargs["ssp_name"] ssp_index_file = kwargs.get("ssp_index_file", "ssp-index.json") From 6e28c41a2b0456d3e3dee0abb9e9780f82b57c1e Mon Sep 17 00:00:00 2001 From: George Vauter Date: Tue, 17 Dec 2024 11:12:38 -0500 Subject: [PATCH 100/108] fix: refactor testt and remove prompts --- tests/trestlebot/cli/test_create_cmd.py | 120 ++++++++++++++---------- trestlebot/cli/commands/create.py | 6 -- 2 files changed, 69 insertions(+), 57 deletions(-) diff --git a/tests/trestlebot/cli/test_create_cmd.py b/tests/trestlebot/cli/test_create_cmd.py index a53ba500..c081645f 100644 --- a/tests/trestlebot/cli/test_create_cmd.py +++ b/tests/trestlebot/cli/test_create_cmd.py @@ -2,12 +2,22 @@ # Copyright (c) 2024 Red Hat, Inc. """ Unit test for create commands ssp and cd""" +import pathlib +from typing import Tuple from click.testing import CliRunner +from git import Repo +from tests.testutils import setup_for_compdef, setup_for_ssp from trestlebot.cli.commands.create import create_cmd +test_prof = "simplified_nist_profile" +test_comp_name = "test_comp" +test_ssp_md = "md_ssp" +test_ssp_cd = "md_cd" + + def test_invalid_create_cmd() -> None: """Tests that create command fails if given invalid oscal model subcommand.""" runner = CliRunner() @@ -17,10 +27,15 @@ def test_invalid_create_cmd() -> None: assert result.exit_code == 2 -def test_create_ssp_cmd(tmp_init_dir: str) -> None: +def test_create_ssp_cmd(tmp_repo: Tuple[str, Repo]) -> None: """Tests successful create ssp command.""" - compdef_list_tester = ["ac-12", "ac-13"] - ssp_index_tester = "tmp_init_dir/tester-ssp-index.json" + + repo_dir, _ = tmp_repo + repo_path = pathlib.Path(repo_dir) + + ssp_index_file = repo_path.joinpath("ssp-index.json") + + _ = setup_for_ssp(repo_path, test_prof, [test_comp_name], test_ssp_md) runner = CliRunner() result = runner.invoke( @@ -28,24 +43,34 @@ def test_create_ssp_cmd(tmp_init_dir: str) -> None: [ "ssp", "--profile-name", - "oscal-profile-name", + test_prof, "--markdown-dir", "markdown", "--compdefs", - compdef_list_tester, + test_comp_name, "--ssp-name", "test-name", "--repo-path", - tmp_init_dir, + str(repo_path.resolve()), "--ssp-index-file", - ssp_index_tester, + ssp_index_file, + "--committer-email", + "test@email.com", + "--committer-name", + "test name", + "--branch", + "test", ], ) assert result.exit_code == 0 -def test_create_compdef_cmd(tmp_init_dir: str) -> None: +def test_create_compdef_cmd(tmp_repo: Tuple[str, Repo]) -> None: """Tests successful create compdef command.""" + repo_dir, _ = tmp_repo + repo_path = pathlib.Path(repo_dir) + + _ = setup_for_compdef(repo_path, test_comp_name, test_ssp_cd) runner = CliRunner() result = runner.invoke( @@ -62,17 +87,24 @@ def test_create_compdef_cmd(tmp_init_dir: str) -> None: "description-test", "--component-definition-type", "type-test", - "repo-path", - tmp_init_dir, + "--repo-path", + str(repo_path.resolve()), + "--committer-email", + "test@email.com", + "--committer-name", + "test name", + "--branch", + "test", ], ) - assert result.exit_code == 2 + assert result.exit_code == 0 -def test_default_ssp_index_file_cmd(tmp_init_dir: str) -> None: +def test_default_ssp_index_file_cmd(tmp_repo: Tuple[str, Repo]) -> None: """Tests successful default ssp_index.json file creation.""" - compdef_list_tester = ["ac-12", "ac-13"] - default_ssp_index_file = "tmp_init_dir/test-ssp-index.json" + + repo_dir, _ = tmp_repo + repo_path = pathlib.Path(repo_dir) runner = CliRunner() result = runner.invoke( @@ -84,48 +116,28 @@ def test_default_ssp_index_file_cmd(tmp_init_dir: str) -> None: "--markdown-dir", "markdown", "--compdefs", - compdef_list_tester, + "test-compdef", "--ssp-name", "test-name", "--repo-path", - tmp_init_dir, - "--ssp-index-file", - default_ssp_index_file, + str(repo_path.resolve()), + "--committer-email", + "test@email.com", + "--committer-name", + "test name", + "--branch", + "test", ], ) assert result.exit_code == 0 -def test_markdown_files_not_created(tmp_init_dir: str) -> None: - """Tests failure of markdown file creation when not supplied with directory name.""" - compdef_list_tester = ["ac-12", "ac-13"] - ssp_index_tester = "tmp_init_dir/ssp-tester.index.json" - - runner = CliRunner() - result = runner.invoke( - create_cmd, - [ - "ssp", - "--profile-name", - "oscal-profile-name", - "--compdefs", - compdef_list_tester, - "--ssp-name", - "test-name", - "repo-path", - tmp_init_dir, - "--ssp-index-file", - ssp_index_tester, - ], - ) - assert result.exit_code == 2 - - -def test_markdown_files_created(tmp_init_dir: str) -> None: +def test_markdown_files_created(tmp_repo: Tuple[str, Repo]) -> None: """Tests successful creation of markdown files.""" - ssp_index_tester = "tmp_init_dir/ssp-tester.index.json" - markdown_file_tester = "tmp_init_dir/markdown/system-security-plan.md" - compdef_list_tester = ["ac-12", "ac-13"] + + repo_dir, _ = tmp_repo + repo_path = pathlib.Path(repo_dir) + ssp_index_file = repo_path.joinpath("ssp-index.json") runner = CliRunner() result = runner.invoke( @@ -135,15 +147,21 @@ def test_markdown_files_created(tmp_init_dir: str) -> None: "--profile-name", "oscal-profile-name", "--markdown-dir", - markdown_file_tester, + "markdown", "--compdefs", - compdef_list_tester, + "test-compdef", "--ssp-name", "test-name", "--repo-path", - tmp_init_dir, + str(repo_path.resolve()), "--ssp-index-file", - ssp_index_tester, + ssp_index_file, + "--committer-email", + "test@email.com", + "--committer-name", + "test name", + "--branch", + "test", ], ) assert result.exit_code == 0 diff --git a/trestlebot/cli/commands/create.py b/trestlebot/cli/commands/create.py index 2f9fdef3..f37d5104 100644 --- a/trestlebot/cli/commands/create.py +++ b/trestlebot/cli/commands/create.py @@ -48,19 +48,16 @@ def create_cmd(ctx: click.Context) -> None: @click.option( "--compdef-name", required=True, - prompt="Enter name of component definition", help="Name of component definition.", ) @click.option( "--component-title", required=True, - prompt="Enter name of component title", help="Title of initial component.", ) @click.option( "--component-description", required=True, - prompt="Enter description of the initial component", help="Description of initial component.", ) @click.option( @@ -153,7 +150,6 @@ def compdef_cmd( "--ssp-name", required=True, type=str, - prompt="Enter name of SSP to create", help="Name of SSP to create.", ) @click.option( @@ -185,7 +181,6 @@ def compdef_cmd( "--compdefs", required=True, type=str, - prompt="Enter comma separated list of component definitions to include in SSP", help="Comma separated list of component definitions.", ) @handle_exceptions @@ -196,7 +191,6 @@ def ssp_cmd( """ SSP Authoring command """ - profile_name = kwargs["profile_name"] ssp_name = kwargs["ssp_name"] ssp_index_file = kwargs.get("ssp_index_file", "ssp-index.json") From e47dca57b8195fc907a67e0afe4961bebd5a23c4 Mon Sep 17 00:00:00 2001 From: George Vauter Date: Tue, 17 Dec 2024 11:15:43 -0500 Subject: [PATCH 101/108] fix: formatting issues and typos --- trestlebot/cli/commands/sync_upstreams.py | 2 +- trestlebot/cli/log.py | 1 - trestlebot/cli/options/common.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/trestlebot/cli/commands/sync_upstreams.py b/trestlebot/cli/commands/sync_upstreams.py index adfcafbb..688e360a 100644 --- a/trestlebot/cli/commands/sync_upstreams.py +++ b/trestlebot/cli/commands/sync_upstreams.py @@ -49,7 +49,7 @@ def load_value_from_ctx( "--sources", type=str, help="Comma-separated list of upstream git sources to sync. Each source is a string \ - in the form @ where ref is a git ref such as a tag or branch.", + in the form @ where ref is a git ref such as a tag or branch.", envvar="TRESTLEBOT_UPSTREAMS_SOURCES", callback=load_value_from_ctx, required=False, diff --git a/trestlebot/cli/log.py b/trestlebot/cli/log.py index 4e251da3..bfc5a523 100644 --- a/trestlebot/cli/log.py +++ b/trestlebot/cli/log.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2023 Red Hat, Inc. - """Configure logger for trestlebot and trestle.""" import argparse diff --git a/trestlebot/cli/options/common.py b/trestlebot/cli/options/common.py index eb35d700..f9b633ac 100644 --- a/trestlebot/cli/options/common.py +++ b/trestlebot/cli/options/common.py @@ -55,7 +55,7 @@ def load_config_to_ctx( If the user specifies a value for the option directly (e.g. uses --option value) then that value is used in favor of the value loaded from the config. - Simarly, if an option has an associated ENVVAR, and that ENVVAR is set, then the + Similarly, if an option has an associated ENVVAR, and that ENVVAR is set, then the ENVVAR value is used in favor of the value loaded from the config. Since the config contains values that should not be mapped to command option values From f125d5ffd424a92c74cb92e3d424ba4380b90318 Mon Sep 17 00:00:00 2001 From: George Vauter Date: Tue, 17 Dec 2024 11:15:43 -0500 Subject: [PATCH 102/108] fix: formatting issues and typos --- trestlebot/cli/commands/sync_upstreams.py | 2 +- trestlebot/cli/log.py | 1 - trestlebot/cli/options/common.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/trestlebot/cli/commands/sync_upstreams.py b/trestlebot/cli/commands/sync_upstreams.py index adfcafbb..688e360a 100644 --- a/trestlebot/cli/commands/sync_upstreams.py +++ b/trestlebot/cli/commands/sync_upstreams.py @@ -49,7 +49,7 @@ def load_value_from_ctx( "--sources", type=str, help="Comma-separated list of upstream git sources to sync. Each source is a string \ - in the form @ where ref is a git ref such as a tag or branch.", + in the form @ where ref is a git ref such as a tag or branch.", envvar="TRESTLEBOT_UPSTREAMS_SOURCES", callback=load_value_from_ctx, required=False, diff --git a/trestlebot/cli/log.py b/trestlebot/cli/log.py index 4e251da3..bfc5a523 100644 --- a/trestlebot/cli/log.py +++ b/trestlebot/cli/log.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2023 Red Hat, Inc. - """Configure logger for trestlebot and trestle.""" import argparse diff --git a/trestlebot/cli/options/common.py b/trestlebot/cli/options/common.py index eb35d700..f9b633ac 100644 --- a/trestlebot/cli/options/common.py +++ b/trestlebot/cli/options/common.py @@ -55,7 +55,7 @@ def load_config_to_ctx( If the user specifies a value for the option directly (e.g. uses --option value) then that value is used in favor of the value loaded from the config. - Simarly, if an option has an associated ENVVAR, and that ENVVAR is set, then the + Similarly, if an option has an associated ENVVAR, and that ENVVAR is set, then the ENVVAR value is used in favor of the value loaded from the config. Since the config contains values that should not be mapped to command option values From 954d38a2144c2f3e3ab96923886a4dbe0374dbb6 Mon Sep 17 00:00:00 2001 From: George Vauter Date: Tue, 17 Dec 2024 11:34:22 -0500 Subject: [PATCH 103/108] chore: update poetry lock with latest dependencies Signed-off-by: George Vauter --- poetry.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/poetry.lock b/poetry.lock index bf7e2d97..3b3465d9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "annotated-types" @@ -2955,4 +2955,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.8.1" -content-hash = "a00779b3211b34f2b7d65ad33e280ded7f2e64cebc2ddd27b1656217f69a3dbf" +content-hash = "bd770486b5c4a8645c0b2d357a8e316aa47f7a2c7f38b997b659ab0de6334bfa" From 586bfca24328cb8d35b2a78b9c0b334ef7812994 Mon Sep 17 00:00:00 2001 From: George Vauter Date: Tue, 17 Dec 2024 11:34:22 -0500 Subject: [PATCH 104/108] chore: update poetry lock with latest dependencies Signed-off-by: George Vauter --- poetry.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/poetry.lock b/poetry.lock index bf7e2d97..3b3465d9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "annotated-types" @@ -2955,4 +2955,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.8.1" -content-hash = "a00779b3211b34f2b7d65ad33e280ded7f2e64cebc2ddd27b1656217f69a3dbf" +content-hash = "bd770486b5c4a8645c0b2d357a8e316aa47f7a2c7f38b997b659ab0de6334bfa" From 1f6f6c6eea951a2d12c54611302aabe6cc6d8f2f Mon Sep 17 00:00:00 2001 From: George Vauter Date: Tue, 17 Dec 2024 12:28:28 -0500 Subject: [PATCH 105/108] fix: do not overwrite config path if set Signed-off-by: George Vauter --- trestlebot/cli/commands/init.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/trestlebot/cli/commands/init.py b/trestlebot/cli/commands/init.py index 972b4b0c..4189f60a 100644 --- a/trestlebot/cli/commands/init.py +++ b/trestlebot/cli/commands/init.py @@ -176,7 +176,8 @@ def init_cmd( config_values.update(branch=default_branch) config = make_config(config_values) - config_path = trestlebot_dir.joinpath("config.yml") + if not config_path: + config_path = trestlebot_dir.joinpath("config.yml") write_to_file(config, config_path) logger.debug(f"trestle-bot config file created at {str(config_path)}") logger.info(f"Successfully initialized trestlebot project in {repo_path}") From ceb084c6930a7a4748dd299e5d0ddb2e0138487b Mon Sep 17 00:00:00 2001 From: George Vauter Date: Tue, 17 Dec 2024 12:28:28 -0500 Subject: [PATCH 106/108] fix: do not overwrite config path if set Signed-off-by: George Vauter --- trestlebot/cli/commands/init.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/trestlebot/cli/commands/init.py b/trestlebot/cli/commands/init.py index 972b4b0c..4189f60a 100644 --- a/trestlebot/cli/commands/init.py +++ b/trestlebot/cli/commands/init.py @@ -176,7 +176,8 @@ def init_cmd( config_values.update(branch=default_branch) config = make_config(config_values) - config_path = trestlebot_dir.joinpath("config.yml") + if not config_path: + config_path = trestlebot_dir.joinpath("config.yml") write_to_file(config, config_path) logger.debug(f"trestle-bot config file created at {str(config_path)}") logger.info(f"Successfully initialized trestlebot project in {repo_path}") From 769d0df2bc786cd035ea3277ed55079047c265e2 Mon Sep 17 00:00:00 2001 From: George Vauter Date: Tue, 17 Dec 2024 12:49:13 -0500 Subject: [PATCH 107/108] fix: do not overwrite config path if set Signed-off-by: George Vauter --- trestlebot/cli/commands/init.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/trestlebot/cli/commands/init.py b/trestlebot/cli/commands/init.py index 4189f60a..28cb4336 100644 --- a/trestlebot/cli/commands/init.py +++ b/trestlebot/cli/commands/init.py @@ -176,8 +176,6 @@ def init_cmd( config_values.update(branch=default_branch) config = make_config(config_values) - if not config_path: - config_path = trestlebot_dir.joinpath("config.yml") - write_to_file(config, config_path) + write_to_file(config, trestlebot_dir.joinpath("config.yml")) logger.debug(f"trestle-bot config file created at {str(config_path)}") logger.info(f"Successfully initialized trestlebot project in {repo_path}") From 18eaf09bb327dac17fd636d5147395d8e0da57ec Mon Sep 17 00:00:00 2001 From: George Vauter Date: Tue, 17 Dec 2024 12:49:13 -0500 Subject: [PATCH 108/108] fix: do not overwrite config path if set Signed-off-by: George Vauter --- trestlebot/cli/commands/init.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/trestlebot/cli/commands/init.py b/trestlebot/cli/commands/init.py index 4189f60a..28cb4336 100644 --- a/trestlebot/cli/commands/init.py +++ b/trestlebot/cli/commands/init.py @@ -176,8 +176,6 @@ def init_cmd( config_values.update(branch=default_branch) config = make_config(config_values) - if not config_path: - config_path = trestlebot_dir.joinpath("config.yml") - write_to_file(config, config_path) + write_to_file(config, trestlebot_dir.joinpath("config.yml")) logger.debug(f"trestle-bot config file created at {str(config_path)}") logger.info(f"Successfully initialized trestlebot project in {repo_path}")