diff --git a/.cookiecutter.json b/.cookiecutter.json index 38bb860..1731ded 100644 --- a/.cookiecutter.json +++ b/.cookiecutter.json @@ -7,7 +7,6 @@ "app_name": "nautobot_plugin_nornir", "verbose_name": "Nautobot Nornir Plugin", "app_slug": "nautobot-plugin-nornir", - "_comment": "app_slug should match the folder structure, but project_slug should match the repo name.", "project_slug": "nautobot-app-nornir", "repo_url": "https://github.com/nautobot/nautobot-app-nornir", "base_url": "plugin-nornir", @@ -22,7 +21,7 @@ "_drift_manager": { "template": "https://github.com/nautobot/cookiecutter-nautobot-app.git", "template_dir": "nautobot-app", - "template_ref": "develop", + "template_ref": "nautobot-app-v2.3.0", "cookie_dir": "", "branch_prefix": "drift-manager", "pull_request_strategy": "create", @@ -30,7 +29,7 @@ "black" ], "draft": true, - "baked_commit_ref": "7cd62373450b4459a5fa3397a2d27214f7217a9c" + "baked_commit_ref": "ea85a24b83f594fc5b27c38f8c39d4bb0b90bed2" } } } diff --git a/.dockerignore b/.dockerignore index 2270f49..a0bf06f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -19,7 +19,6 @@ FAQ.md .git/ .gitignore .github -tasks.py LICENSE **/*.log **/.vscode/ diff --git a/.flake8 b/.flake8 deleted file mode 100644 index c9f5e84..0000000 --- a/.flake8 +++ /dev/null @@ -1,10 +0,0 @@ -[flake8] -ignore = - E501, # Line length is enforced by Black, so flake8 doesn't need to check it - W503 # Black disagrees with this rule, as does PEP 8; Black wins -exclude = - migrations, - __pycache__, - manage.py, - settings.py, - .venv diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 649e9c0..9e72099 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,3 +1,3 @@ -# Default owners for all files in this repository +# Default owner(s) of all files in this repository * @itdependsnetworks -* @jeffkala \ No newline at end of file +* @jeffkala diff --git a/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md b/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md index 3950227..06c1d5f 100644 --- a/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md +++ b/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md @@ -1,5 +1,5 @@ " +issue_format = "[#{issue}](https://github.com/nautobot/nautobot-app-nornir/issues/{issue})" + +[[tool.towncrier.type]] +directory = "security" +name = "Security" +showcontent = true + +[[tool.towncrier.type]] +directory = "added" +name = "Added" +showcontent = true + +[[tool.towncrier.type]] +directory = "changed" +name = "Changed" +showcontent = true + +[[tool.towncrier.type]] +directory = "deprecated" +name = "Deprecated" +showcontent = true + +[[tool.towncrier.type]] +directory = "removed" +name = "Removed" +showcontent = true + +[[tool.towncrier.type]] +directory = "fixed" +name = "Fixed" +showcontent = true + +[[tool.towncrier.type]] +directory = "dependencies" +name = "Dependencies" +showcontent = true + +[[tool.towncrier.type]] +directory = "documentation" +name = "Documentation" +showcontent = true + +[[tool.towncrier.type]] +directory = "housekeeping" +name = "Housekeeping" +showcontent = true diff --git a/tasks.py b/tasks.py index d8af7a0..19d78e1 100644 --- a/tasks.py +++ b/tasks.py @@ -13,6 +13,8 @@ """ import os +from pathlib import Path +from time import sleep from invoke.collection import Collection from invoke.tasks import task as invoke_task @@ -67,6 +69,25 @@ def _is_compose_included(context, name): return f"docker-compose.{name}.yml" in context.nautobot_plugin_nornir.compose_files +def _await_healthy_service(context, service): + container_id = docker_compose(context, f"ps -q -- {service}", pty=False, echo=False, hide=True).stdout.strip() + _await_healthy_container(context, container_id) + + +def _await_healthy_container(context, container_id): + while True: + result = context.run( + "docker inspect --format='{{.State.Health.Status}}' " + container_id, + pty=False, + echo=False, + hide=True, + ) + if result.stdout.strip() == "healthy": + break + print(f"Waiting for `{container_id}` container to become healthy ...") + sleep(1) + + def task(function=None, *args, **kwargs): """Task decorator to override the default Invoke task decorator and add each task to the invoke namespace.""" @@ -128,15 +149,27 @@ def docker_compose(context, command, **kwargs): def run_command(context, command, **kwargs): """Wrapper to run a command locally or inside the nautobot container.""" if is_truthy(context.nautobot_plugin_nornir.local): + if "command_env" in kwargs: + kwargs["env"] = { + **kwargs.get("env", {}), + **kwargs.pop("command_env"), + } context.run(command, **kwargs) else: # Check if nautobot is running, no need to start another nautobot container to run a command docker_compose_status = "ps --services --filter status=running" results = docker_compose(context, docker_compose_status, hide="out") + + command_env_args = "" + if "command_env" in kwargs: + command_env = kwargs.pop("command_env") + for key, value in command_env.items(): + command_env_args += f' --env="{key}={value}"' + if "nautobot" in results.stdout: - compose_command = f"exec nautobot {command}" + compose_command = f"exec{command_env_args} nautobot {command}" else: - compose_command = f"run --rm --entrypoint '{command}' nautobot" + compose_command = f"run{command_env_args} --rm --entrypoint='{command}' nautobot" pty = kwargs.pop("pty", True) @@ -216,11 +249,46 @@ def stop(context, service=""): docker_compose(context, "stop" if service else "down --remove-orphans", service=service) -@task -def destroy(context): +@task( + aliases=("down",), + help={ + "volumes": "Remove Docker compose volumes (default: True)", + "import-db-file": "Import database from `import-db-file` file into the fresh environment (default: empty)", + }, +) +def destroy(context, volumes=True, import_db_file=""): """Destroy all containers and volumes.""" print("Destroying Nautobot...") - docker_compose(context, "down --remove-orphans --volumes") + docker_compose(context, f"down --remove-orphans {'--volumes' if volumes else ''}") + + if not import_db_file: + return + + if not volumes: + raise ValueError("Cannot specify `--no-volumes` and `--import-db-file` arguments at the same time.") + + print(f"Importing database file: {import_db_file}...") + + input_path = Path(import_db_file).absolute() + if not input_path.is_file(): + raise ValueError(f"File not found: {input_path}") + + command = [ + "run", + "--rm", + "--detach", + f"--volume='{input_path}:/docker-entrypoint-initdb.d/dump.sql'", + "--", + "db", + ] + + container_id = docker_compose(context, " ".join(command), pty=False, echo=False, hide=True).stdout.strip() + _await_healthy_container(context, container_id) + print("Stopping database container...") + context.run(f"docker stop {container_id}", pty=False, echo=False, hide=True) + + print("Database import complete, you can start Nautobot with the following command:") + print("invoke start") @task @@ -271,15 +339,22 @@ def logs(context, service="", follow=False, tail=0): # ------------------------------------------------------------------------------ # ACTIONS # ------------------------------------------------------------------------------ -@task(help={"file": "Python file to execute"}) -def nbshell(context, file=""): +@task( + help={ + "file": "Python file to execute", + "env": "Environment variables to pass to the command", + "plain": "Flag to run nbshell in plain mode (default: False)", + }, +) +def nbshell(context, file="", env={}, plain=False): """Launch an interactive nbshell session.""" command = [ "nautobot-server", "nbshell", + "--plain" if plain else "", f"< '{file}'" if file else "", ] - run_command(context, " ".join(command), pty=not bool(file)) + run_command(context, " ".join(command), pty=not bool(file), command_env=env) @task @@ -419,32 +494,55 @@ def dbshell(context, db_name="", input_file="", output_file="", query=""): f"> '{output_file}'" if output_file else "", ] - docker_compose(context, " ".join(command), env=env, pty=not (input_file or output_file or query)) + docker_compose( + context, + " ".join(command), + env=env, + pty=not (input_file or output_file or query), + ) @task( help={ + "db-name": "Database name to create (default: Nautobot database)", "input-file": "SQL dump file to replace the existing database with. This can be generated using `invoke backup-db` (default: `dump.sql`).", } ) -def import_db(context, input_file="dump.sql"): - """Stop Nautobot containers and replace the current database with the dump into the running `db` container.""" - docker_compose(context, "stop -- nautobot worker") +def import_db(context, db_name="", input_file="dump.sql"): + """Stop Nautobot containers and replace the current database with the dump into `db` container.""" + docker_compose(context, "stop -- nautobot worker beat") + start(context, "db") + _await_healthy_service(context, "db") command = ["exec -- db sh -c '"] if _is_compose_included(context, "mysql"): + if not db_name: + db_name = "$MYSQL_DATABASE" command += [ + "mysql --user root --password=$MYSQL_ROOT_PASSWORD", + '--execute="', + f"DROP DATABASE IF EXISTS {db_name};", + f"CREATE DATABASE {db_name};", + ( + "" + if db_name == "$MYSQL_DATABASE" + else f"GRANT ALL PRIVILEGES ON {db_name}.* TO $MYSQL_USER; FLUSH PRIVILEGES;" + ), + '"', + "&&", "mysql", - "--database=$MYSQL_DATABASE", + f"--database={db_name}", "--user=$MYSQL_USER", "--password=$MYSQL_PASSWORD", ] elif _is_compose_included(context, "postgres"): + if not db_name: + db_name = "$POSTGRES_DB" command += [ - "psql", - "--username=$POSTGRES_USER", - "postgres", + f"dropdb --if-exists --user=$POSTGRES_USER {db_name} &&", + f"createdb --user=$POSTGRES_USER {db_name} &&", + f"psql --user=$POSTGRES_USER --dbname={db_name}", ] else: raise ValueError("Unsupported database backend.") @@ -467,7 +565,10 @@ def import_db(context, input_file="dump.sql"): } ) def backup_db(context, db_name="", output_file="dump.sql", readable=True): - """Dump database into `output_file` file from running `db` container.""" + """Dump database into `output_file` file from `db` container.""" + start(context, "db") + _await_healthy_service(context, "db") + command = ["exec -- db sh -c '"] if _is_compose_included(context, "mysql"): @@ -475,17 +576,12 @@ def backup_db(context, db_name="", output_file="dump.sql", readable=True): "mysqldump", "--user=root", "--password=$MYSQL_ROOT_PASSWORD", - "--add-drop-database", "--skip-extended-insert" if readable else "", - "--databases", db_name if db_name else "$MYSQL_DATABASE", ] elif _is_compose_included(context, "postgres"): command += [ "pg_dump", - "--clean", - "--create", - "--if-exists", "--username=$POSTGRES_USER", f"--dbname={db_name or '$POSTGRES_DB'}", "--inserts" if readable else "", @@ -542,31 +638,22 @@ def help_task(context): context.run(f"invoke {task_name} --help") -# ------------------------------------------------------------------------------ -# TESTS -# ------------------------------------------------------------------------------ @task( help={ - "autoformat": "Apply formatting recommendations automatically, rather than failing if formatting is incorrect.", + "version": "Version of Nautobot Nornir Plugin to generate the release notes for.", } ) -def black(context, autoformat=False): - """Check Python code style with Black.""" - if autoformat: - black_command = "black" - else: - black_command = "black --check --diff" - - command = f"{black_command} ." - +def generate_release_notes(context, version=""): + """Generate Release Notes using Towncrier.""" + command = "env DJANGO_SETTINGS_MODULE=nautobot.core.settings towncrier build" + if version: + command += f" --version {version}" run_command(context, command) -@task -def flake8(context): - """Check for PEP8 compliance and other style issues.""" - command = "flake8 . --config .flake8" - run_command(context, command) +# ------------------------------------------------------------------------------ +# TESTS +# ------------------------------------------------------------------------------ @task @@ -583,24 +670,47 @@ def pylint(context): run_command(context, command) -@task -def pydocstyle(context): - """Run pydocstyle to validate docstring formatting adheres to NTC defined standards.""" - # We exclude the /migrations/ directory since it is autogenerated code - command = "pydocstyle ." - run_command(context, command) +@task(aliases=("a",)) +def autoformat(context): + """Run code autoformatting.""" + ruff(context, action=["format"], fix=True) -@task -def bandit(context): - """Run bandit to validate basic static code security analysis.""" - command = "bandit --recursive . --configfile .bandit.yml" - run_command(context, command) +@task( + help={ + "action": "Available values are `['lint', 'format']`. Can be used multiple times. (default: `['lint', 'format']`)", + "target": "File or directory to inspect, repeatable (default: all files in the project will be inspected)", + "fix": "Automatically fix selected actions. May not be able to fix all issues found. (default: False)", + "output_format": "See https://docs.astral.sh/ruff/settings/#output-format for details. (default: `concise`)", + }, + iterable=["action", "target"], +) +def ruff(context, action=None, target=None, fix=False, output_format="concise"): + """Run ruff to perform code formatting and/or linting.""" + if not action: + action = ["lint", "format"] + if not target: + target = ["."] + + if "format" in action: + command = "ruff format " + if not fix: + command += "--check " + command += " ".join(target) + run_command(context, command, warn=True) + + if "lint" in action: + command = "ruff check " + if fix: + command += "--fix " + command += f"--output-format {output_format} " + command += " ".join(target) + run_command(context, command, warn=True) @task def yamllint(context): - """Run yamllint to validate formating adheres to NTC defined YAML standards. + """Run yamllint to validate formatting adheres to NTC defined YAML standards. Args: context (obj): Used to run specific commands @@ -627,7 +737,7 @@ def check_migrations(context): "verbose": "Enable verbose test output.", } ) -def unittest( +def unittest( # noqa: PLR0913 context, keepdb=False, label="nautobot_plugin_nornir", @@ -669,20 +779,14 @@ def unittest_coverage(context): } ) def tests(context, failfast=False, keepdb=False, lint_only=False): - """Run all tests for this plugin.""" + """Run all tests for this app.""" # If we are not running locally, start the docker containers so we don't have to for each test if not is_truthy(context.nautobot_plugin_nornir.local): print("Starting Docker Containers...") start(context) # Sorted loosely from fastest to slowest - print("Running black...") - black(context) - print("Running flake8...") - flake8(context) - print("Running bandit...") - bandit(context) - print("Running pydocstyle...") - pydocstyle(context) + print("Running ruff...") + ruff(context) print("Running yamllint...") yamllint(context) print("Running poetry check...") @@ -693,8 +797,42 @@ def tests(context, failfast=False, keepdb=False, lint_only=False): pylint(context) print("Running mkdocs...") build_and_check_docs(context) + print("Checking app config schema...") + validate_app_config(context) if not lint_only: print("Running unit tests...") unittest(context, failfast=failfast, keepdb=keepdb) unittest_coverage(context) print("All tests have passed!") + + +@task +def generate_app_config_schema(context): + """Generate the app config schema from the current app config. + + WARNING: Review and edit the generated file before committing. + + Its content is inferred from: + + - The current configuration in `PLUGINS_CONFIG` + - `NautobotAppConfig.default_settings` + - `NautobotAppConfig.required_settings` + """ + start(context, service="nautobot") + nbshell( + context, + file="development/app_config_schema.py", + env={"APP_CONFIG_SCHEMA_COMMAND": "generate"}, + ) + + +@task +def validate_app_config(context): + """Validate the app config based on the app config schema.""" + start(context, service="nautobot") + nbshell( + context, + plain=True, + file="development/app_config_schema.py", + env={"APP_CONFIG_SCHEMA_COMMAND": "validate"}, + )