diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d6e552bf..39e66a35 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,10 +15,14 @@ Before you start contributing, please take a moment to read through the guide be - [How It Works](#how-it-works) - [Components](#components) - [Code structure](#code-structure) - - [Format and Styling](#format-and-styling) - - [Type Hints and Static Type Checking](#type-hints-and-static-type-checking) - - [Analysis Tools](#analysis-tools) + - [Documentation](#documentation) + - [Update the `actions` files](#update-the-actions-files) + - [Tools](#tools) + - [Format and Styling](#format-and-styling) + - [Type Hints and Static Type Checking](#type-hints-and-static-type-checking) + - [Analysis Tools](#analysis-tools) - [Running tests](#running-tests) + - [Running tests with make](#running-tests-with-make) - [Run with poetry](#run-with-poetry) - [Local testing](#local-testing) @@ -72,7 +76,19 @@ For workflow diagrams, see the [diagrams](./docs/diagrams/) under the `docs` fol - `transformers` - This contains data transformation logic; specifically for rules. -### Format and Styling +### Documentation + +#### Update the `actions` files + +Each `README.md` under the `actions` directory have an Actions Inputs and Action Outputs section. These sections are generated from the `action.yml` file in the directory. To update the `README.md` files, run the following command: + +```bash +make update-action-readmes +``` + +### Tools + +#### Format and Styling This project uses `black` and `isort` for formatting and `flake8` for linting. You can run these commands to format and lint your code. Linting checks can be run as a pre-commit hook and are verified in CI. @@ -84,11 +100,11 @@ make lint For non-Python files, we use [Megalinter](https://github.com/oxsecurity/megalinter) to lint in a CI task. See [megalinter.yaml](https://github.com/RedHatProductSecurity/trestle-bot/blob/main/.mega-linter.yml) for more information. -### Type Hints and Static Type Checking +#### Type Hints and Static Type Checking We encourage the use of type hints in Python code to enhance readability, maintainability, and robustness of the codebase. Type hints serve as documentation and aid in catching potential errors during development. For static type analysis, we utilize `mypy`. Running `make lint` will run `mypy` checks on the codebase. -### Analysis Tools +#### Analysis Tools - [SonarCloud](https://sonarcloud.io/dashboard?id=rh-psce_trestle-bot) - We use SonarCloud to analyze code quality, coverage, and security. To not break GitHub security model, this will not run on a forked repository. - [Semgrep](https://semgrep.dev/docs/extensions/overview/#pre-commit) - Identify issues in the local development environment before committing code. These checks are also run in CI. @@ -98,6 +114,7 @@ We encourage the use of type hints in Python code to enhance readability, mainta Run all tests with `make test` or `make test-slow` to run all tests including end-to-end. For information on end-to-end tests, see [README.md](./tests/e2e/README.md). +#### Running tests with make ```bash # Run all tests make test diff --git a/Makefile b/Makefile index 0893f2a1..86033b9a 100644 --- a/Makefile +++ b/Makefile @@ -72,4 +72,8 @@ publish: .PHONY: publish build-and-publish: build publish -.PHONY: build-and-publish \ No newline at end of file +.PHONY: build-and-publish + +update-action-readmes: + @poetry run python scripts/update_action_readmes.py +.PHONY: update-action-readmes diff --git a/actions/autosync/README.md b/actions/autosync/README.md index fa0a9b45..d9bc700d 100644 --- a/actions/autosync/README.md +++ b/actions/autosync/README.md @@ -18,9 +18,43 @@ name: Example Workflow oscal_model: "profile" ``` -## Inputs and Outputs - -Checkout [`action.yml`](./action.yml) for a full list of supported inputs and outputs. +## Action Inputs + + +| Name | Description | Default | Required | +| --- | --- | --- | --- | +| markdown_path | Path relative to the repository path where the Trestle markdown files are located. See action README.md for more information. | None | True | +| oscal_model | OSCAL Model type to assemble. Values can be catalog, profile, compdef, or ssp. | None | True | +| check_only | Runs tasks and exits with an error if there is a diff. Defaults to false | false | False | +| github_token | GitHub token used to make authenticated API requests | None | False | +| skip_assemble | Skip assembly task. Defaults to false | false | False | +| skip_regenerate | Skip regenerate task. Defaults to false. | false | False | +| skip_items | Comma-separated glob patterns list of content by Trestle name to skip during task execution. For example `profile_x,profile_y*,`. | None | False | +| ssp_index_path | Path relative to the repository path where the ssp index is located. See action README.md for information about the ssp index. | ssp-index.json | False | +| commit_message | Commit message | Sync automatic updates | False | +| pull_request_title | Custom pull request title | Automatic updates from trestlebot | False | +| branch | Name of the Git branch to which modifications should be pushed. Required if Action is used on the `pull_request` event. | ${{ github.ref_name }} | False | +| target_branch | Target branch (or base branch) to create a pull request against. If unset, no pull request will be created. If set, a pull request will be created using the `branch` field as the head branch. | None | False | +| file_pattern | Comma separated file pattern list used for `git add`. For example `component-definitions/*,*json`. Defaults to (`.`) | . | False | +| repository | Local file path to the git repository. Defaults to the current directory (`.`) | . | False | +| commit_user_name | Name used for the commit user | github-actions[bot] | False | +| commit_user_email | Email address used for the commit user | 41898282+github-actions[bot]@users.noreply.github.com | False | +| commit_author_name | Name used for the commit author. Defaults to the username of whoever triggered this workflow run. | ${{ github.actor }} | False | +| commit_author_email | Email address used for the commit author. Defaults to the email of whoever triggered this workflow run. | ${{ github.actor }}@users.noreply.github.com | False | +| verbose | Enable verbose logging | false | False | + + + +## Action Outputs + + +| Name | Description | +| --- | --- | +| changes | Value is "true" if changes were committed back to the repository. | +| commit | Full hash of the created commit. Only present if the "changes" output is "true". | +| pr_number | Number of the submitted pull request. Only present if a pull request is submitted. | + + ### Additional information on workflow inputs diff --git a/actions/create-cd/README.md b/actions/create-cd/README.md index 73d45ea2..56fb88c1 100644 --- a/actions/create-cd/README.md +++ b/actions/create-cd/README.md @@ -20,9 +20,43 @@ name: Example Workflow component_description: "My Component Description" ``` -## Inputs and Outputs +## Action Inputs -Checkout [`action.yml`](./action.yml) for a full list of supported inputs and outputs. + +| Name | Description | Default | Required | +| --- | --- | --- | --- | +| markdown_path | Path relative to the repository path to create markdown files. See action README.md for more information. | None | True | +| profile_name | Name of the Trestle profile to use for the component definition | None | True | +| component_definition_name | Name of the component definition to create | None | True | +| component_title | Name of the component to create | None | True | +| component_type | Type of the component to create | service | False | +| component_description | Description of the component to create | None | True | +| filter_by_profile | Name of the profile in the workspace to filter controls by | None | False | +| github_token | GitHub token used to make authenticated API requests | None | False | +| commit_message | Commit message | Sync automatic updates | False | +| pull_request_title | Custom pull request title | Automatic updates from trestlebot | False | +| branch | Name of the Git branch to which modifications should be pushed. Required if Action is used on the `pull_request` event. | ${{ github.ref_name }} | False | +| target_branch | Target branch (or base branch) to create a pull request against. If unset, no pull request will be created. If set, a pull request will be created using the `branch` field as the head branch. | None | False | +| file_pattern | Comma separated file pattern list used for `git add`. For example `component-definitions/*,*json`. Defaults to (`.`) | . | False | +| repository | Local file path to the git repository. Defaults to the current directory (`.`) | . | False | +| commit_user_name | Name used for the commit user | github-actions[bot] | False | +| commit_user_email | Email address used for the commit user | 41898282+github-actions[bot]@users.noreply.github.com | False | +| commit_author_name | Name used for the commit author. Defaults to the username of whoever triggered this workflow run. | ${{ github.actor }} | False | +| commit_author_email | Email address used for the commit author. Defaults to the email of whoever triggered this workflow run. | ${{ github.actor }}@users.noreply.github.com | False | +| verbose | Enable verbose logging | false | False | + + + +## Action Outputs + + +| Name | Description | +| --- | --- | +| changes | Value is "true" if changes were committed back to the repository. | +| commit | Full hash of the created commit. Only present if the "changes" output is "true". | +| pr_number | Number of the submitted pull request. Only present if a pull request is submitted. | + + ### Additional information on workflow inputs diff --git a/actions/rules-transform/README.md b/actions/rules-transform/README.md index 67d29b00..c6668d82 100644 --- a/actions/rules-transform/README.md +++ b/actions/rules-transform/README.md @@ -27,6 +27,39 @@ With custom rules directory: rules_view_path: "custom-rules-dir/" ``` +## Action Inputs + + +| Name | Description | Default | Required | +| --- | --- | --- | --- | +| rules_view_path | Path relative to the repository path where the Trestle rules view files are located. Defaults to `rules/`. | rules/ | False | +| github_token | GitHub token used to make authenticated API requests | None | False | +| skip_items | Comma-separated glob patterns list of content by Trestle name to skip during task execution. For example `compdef_x,compdef_y*,`. | None | False | +| commit_message | Commit message | Sync automatic updates | False | +| pull_request_title | Custom pull request title | Automatic updates from trestlebot | False | +| branch | Name of the Git branch to which modifications should be pushed. Required if Action is used on the `pull_request` event. | ${{ github.ref_name }} | False | +| target_branch | Target branch (or base branch) to create a pull request against. If unset, no pull request will be created. If set, a pull request will be created using the `branch` field as the head branch. | None | False | +| file_pattern | Comma separated file pattern list used for `git add`. For example `component-definitions/*,*json`. Defaults to (`.`) | . | False | +| repository | Local file path to the git repository. Defaults to the current directory (`.`) | . | False | +| commit_user_name | Name used for the commit user | github-actions[bot] | False | +| commit_user_email | Email address used for the commit user | 41898282+github-actions[bot]@users.noreply.github.com | False | +| commit_author_name | Name used for the commit author. Defaults to the username of whoever triggered this workflow run. | ${{ github.actor }} | False | +| commit_author_email | Email address used for the commit author. Defaults to the email of whoever triggered this workflow run. | ${{ github.actor }}@users.noreply.github.com | False | +| verbose | Enable verbose logging | false | False | + + + +## Action Outputs + + +| Name | Description | +| --- | --- | +| changes | Value is "true" if changes were committed back to the repository. | +| commit | Full hash of the created commit. Only present if the "changes" output is "true". | +| pr_number | Number of the submitted pull request. Only present if a pull request is submitted. | + + + ## Action Behavior The purpose of this action is to sync the rules view data in YAML to OSCAL with `compliance-trestle` and commit changes back to the branch or submit a pull request (if desired). Below are the main use-cases/workflows available: diff --git a/scripts/update_action_readmes.py b/scripts/update_action_readmes.py new file mode 100755 index 00000000..ff6cbf54 --- /dev/null +++ b/scripts/update_action_readmes.py @@ -0,0 +1,154 @@ +#!/usr/bin/python + +# Copyright 2023 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +""" +Find every action.yml file in the repo and README.md next to it +and update the Action Inputs and Action Outputs sections in the README.md +""" + +import logging +import os +import sys +from typing import Any, List, Dict + +from ruamel.yaml import YAML, YAMLError + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + +ACTION_INPUTS_START = "" +ACTION_INPUTS_END = "" +ACTION_OUTPUTS_START = "" +ACTION_OUTPUTS_END = "" + + +class InvalidReadmeFile(Exception): + """Exception raised for invalid README.md file""" + + def __init__(self, start_marker: str, end_marker: str) -> None: + super().__init__( + f"Missing start marker '{start_marker}' and/or " + f"end marker '{end_marker}' in README.md file" + ) + + +def load_action_yml(action_yml_file: str) -> Dict[str, Any]: + """Load the action.yml file as a dictionary""" + with open(action_yml_file, "r") as stream: + try: + yaml = YAML(typ="safe") + action_yml = yaml.load(stream) + except YAMLError as e: + raise RuntimeError(f"Error loading YAML file '{action_yml_file}': {e}") + return action_yml + + +def generate_inputs_markdown_table(inputs: Dict[str, Any]) -> str: + """Generate the Action Inputs markdown table""" + table = "| Name | Description | Default | Required |\n| --- | --- | --- | --- |\n" + for name, input in inputs.items(): + table += f"| {name} | {input.get('description', None)} | {input.get('default', None)} | {input.get('required', None)} |\n" # noqa E501 + return table + + +def generate_outputs_markdown_table(outputs: Dict[str, Any]) -> str: + """Generate the Action Outputs markdown table""" + table = "| Name | Description |\n| --- | --- |\n" + for name, output in outputs.items(): + table += f"| {name} | {output.get('description', None)} |\n" + return table + + +def replace(all_content: str, start: str, end: str, new_content: str) -> str: + """Replace the content between start (plus a new line) and end with new_content""" + start_line = all_content.find(start) + end_line = all_content.find(end) + if start_line == -1 or end_line == -1: + raise InvalidReadmeFile(start, end) + + lines: List[str] = all_content.split("\n") + + start_marker_index = lines.index(start) + end_marker_index = lines.index(end) + + # Replace content between markers excluding marker lines + lines = lines[: start_marker_index + 1] + [new_content] + lines[end_marker_index:] + updated_content = "\n".join(lines) + return updated_content + + +def replace_readme_sections(content: str, action_yml: Dict[str, Any]) -> str: + """Replace the Action Inputs and Action Outputs sections in the README.md file""" + inputs_table = generate_inputs_markdown_table(action_yml["inputs"]) + outputs_table = generate_outputs_markdown_table(action_yml["outputs"]) + replaced_content = replace( + content, ACTION_INPUTS_START, ACTION_INPUTS_END, inputs_table + ) + replaced_content = replace( + replaced_content, ACTION_OUTPUTS_START, ACTION_OUTPUTS_END, outputs_table + ) + return replaced_content + + +def update_readme_file(readme_file: str, action_yml: Dict[str, Any]) -> None: + """Updates the README.md file with action inputs and outputs""" + with open(readme_file, "r") as stream: + existing_content = stream.read() + try: + updated_content = replace_readme_sections(existing_content, action_yml) + except InvalidReadmeFile as e: + logging.warning(f"Skipping README file {readme_file}: {e}") + return # Don't continue if the readme file is invalid + + if updated_content != existing_content: + logging.info(f"Updated README.md file: {readme_file}") + with open(readme_file, "w") as stream: + stream.write(updated_content) + else: + logging.info(f"README.md file is up to date: {readme_file}") + + +def find_actions_files() -> List[str]: + """Find every action.yml file in the repo""" + action_yml_files: List[str] = [] + for root, _, files in os.walk("actions"): + for file in files: + if file.endswith("action.yml"): + action_yml_files.append(os.path.join(root, file)) + return action_yml_files + + +def main() -> None: + try: + action_yml_files: List[str] = find_actions_files() + for action_yml_file in action_yml_files: + # find the README.md file next to the action.yml file + readme_file = action_yml_file.replace("action.yml", "README.md") + if not os.path.exists(readme_file): + logging.warning( + f"README.md not found for action.yml file: {action_yml_file}" + ) + continue + + action_yml: Dict[str, Any] = load_action_yml(action_yml_file) + update_readme_file(readme_file, action_yml) + except Exception as e: + logging.exception(f"Unexpected error during README.md updates: {e}", exc_info=True) + sys.exit(1) + + +if __name__ == "__main__": + main()