diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4124695..0a8ce6b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,3 +12,7 @@ repos: args: - --in-place - --remove-all-unused-imports + - repo: https://github.com/Lucas-C/pre-commit-hooks-nodejs + rev: v1.1.2 + hooks: + - id: markdown-toc diff --git a/CHANGELOG.md b/CHANGELOG.md index e4a8d33..ff72430 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,31 @@ # Changelog +## 1.0.0 +- [BREAKING CHANGE] The version does not support python configuration anymore, use YAML `artifactory-cleanup.yaml` config file to specify policies and rules. +- [BREAKING CHANGE] No `under_score` names for rules, only `CamelCase` +- Support YAML configuration! #54 🎉 +- Introduce new and stable (from this point) API for Rules #33 +- Fix `KeepLatestNVersionImagesByProperty` wrong behaviour #60 +- Rename `DeleteEmptyFolder` to `DeleteEmptyFolders` +- Rename `--policy-name` flag to `--policy` + +### Backward incompatible changes +Keep these in mind if you create your own Rules and going to update from `0.4.1` to `1.0.0`. + +In order to simplify API for Rule and CleanupPolicy and support some feature we have to introduce backward incompatible changes. + +#### Rules API +- Methods have been changed: + - `_aql_add_filter(aql_query_list)` => `aql_add_filter(filters)` + - `_aql_add_text(aql_text)` => `aql_add_text(aql)` + - `_filter_result(result_artifact)` => `filter(artifacts)` +- `filter(artifacts)` must return `ArtifactsList` instance, not just a list +- Removed `artifactory_server`. Read below about new `self.session`, probably you don't need it anymore +- Renamed `self.artifactory_session` to `self.session`. + - Call `self.session.get('/relative/path)` - it adds Artifactory URL at the start of `/relative/path` and calls `http://artifactory.example.com/relaive/path`. + - If you still need to get Artifactory URL in rules - get it with `self.session.base_url`. +- Instead of `self.filter_result(self, result_artifacts)` use `artifacts.remove(artifacts_to_keep)` method in `self.filter` + ## 0.4.2 - Fix: Failed to run artifactory-cleanup-0.4.1 command #64 diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..f415c8a --- /dev/null +++ b/Makefile @@ -0,0 +1,7 @@ +copy-examples: + cp tests/data/cleanup.yaml examples/artifactory-cleanup.yaml + cp tests/data/myrule.py examples/myrule.py + cp tests/data/policies.py examples/python_style_config.py + +build: + docker build . --file docker/Dockerfile --tag devopshq/artifactory-cleanup diff --git a/README.md b/README.md index bc506d5..53cc49d 100644 --- a/README.md +++ b/README.md @@ -1,210 +1,520 @@ # Artifactory cleanup # -`artifactory-cleanup` is a tool for cleaning artifacts in Jfrog Artifactory. +`artifactory-cleanup` is an extended and flexible cleanup tool for JFrog Artifactory. + +The tool has simple YAML-defined cleanup configuration and can be extended with your own rules on Python. +Everything must be as a code, even cleanup policies! # Tables of Contents -- [Artifactory cleanup](#artifactory-cleanup) -- [Tables of Contents](#tables-of-contents) - [Installation](#installation) - [Usage](#usage) - - [Commands](#commands) - - [Available Rules](#available-rules) - - [Artifact cleanup policies](#artifact-cleanup-policies) - - [Docker Container Usage](#docker-container-usage) -- [FAQ](#faq) + * [Notes](#notes) + * [Commands](#commands) +- [Rules](#rules) + * [Common](#common) + * [Delete](#delete) + * [Keep](#keep) + * [Docker](#docker) + * [Filters](#filters) + * [Create your own rule](#create-your-own-rule) +- [How to](#how-to) + * [How to connect self-signed certificates for docker?](#how-to-connect-self-signed-certificates-for-docker) + * [How to clean up Conan repository?](#how-to-clean-up-conan-repository) + * [How to keep latest N docker images?](#how-to-keep-latest-n-docker-images) - [Release](#release) - + # Installation -Upgrade/install to the newest available version: -```bash -python3 -mpip install artifactory-cleanup -# Directly from git -python3 -mpip install git+https://github.com/devopshq/artifactory-cleanup.git +As simple as one command! -# To be able to change files -git clone https://github.com/devopshq/artifactory-cleanup.git -cd artifactory-cleanup -python3 -mpip install -e . +```bash +# docker +docker pull devopshq/artifactory-cleanup +docker run -rm devopshq/artifactory-cleanup artifactory-cleanup --help + +# python (later we call it 'cli') +python3 -mpip install artifactory-cleanup +artifactory-cleanup --help ``` # Usage -Suppose you want to remove all artifacts older than N days from 'reponame'. +Suppose you want to remove **all artifacts older than N days** from **reponame** repository. You should take the following steps: -1. Install `artifactory-cleanup` -2. Сreate a python file, for example, `reponame.py` with the following contents: +1. Install `artifactory-cleanup` (see above) +2. Create a configuration file `artifactory-cleanup.yaml`. variables. + +```yaml +# artifactory-cleanup.yaml +artifactory-cleanup: + server: https://repo.example.com/artifactory + # $VAR is auto populated from environment variables + user: $ARTIFACTORY_USERNAME + password: $ARTIFACTORY_PASSWORD + + policies: + - name: Remove all files from repo-name-here older then 7 days + rules: + - rule: Repo + name: "reponame" + - rule: DeleteOlderThan + days: 7 +``` -```python -from artifactory_cleanup import rules -from artifactory_cleanup import CleanupPolicy +3. Run the command **TO SHOW** (not remove) artifacts that will be deleted. By default `artifacory-cleanup` uses "dry + mode". + +```bash +# Set the credentials with delete permissions +export ARTIFACTORY_USERNAME=usernamehere +export ARTIFACTORY_PASSWORD=password -RULES = [ +# docker +docker run -rm -v "$(pwd)":/app -e ARTIFACTORY_USERNAME -e ARTIFACTORY_PASSWORD devopshq/artifactory-cleanup artifactory-cleanup - # ------ ALL REPOS -------- - CleanupPolicy( - 'Delete files older than 30 days', - rules.Repo('reponame'), - rules.DeleteOlderThan(days=30), - ), -] +# cli +artifactory-cleanup ``` -3. Run the command to SHOW (not remove) artifacts that will be deleted: + +4. Verify that right artifacts will be removed and add `--destroy` flag **TO REMOVE** artifacts: + ```bash -artifactory-cleanup --user user --password password --artifactory-server https://repo.example.com/artifactory --config reponame.py +# docker +docker run -rm -v "$(pwd)":/app -e ARTIFACTORY_USERNAME -e ARTIFACTORY_PASSWORD devopshq/artifactory-cleanup artifactory-cleanup --destroy + +# cli +artifactory-cleanup --destroy ``` -4. Add `--destroy` flag to REMOVE artifacts + +Looking for more examples? Check [examples](./examples) folder! + +## Notes + +- **Always** specify version of `artifactory-cleanup` when using it in the production. Do not use `1.0.0`, use the + latest in pypi: https://pypi.org/project/artifactory-cleanup/ + ```bash -artifactory-cleanup --destroy --user user --password password --artifactory-server https://repo.example.com/artifactory --config reponame.py +# docker +docker pull devopshq/artifactory-cleanup:1.0.0 +docker run -rm devopshq/artifactory-cleanup:1.0.0 artifactory-cleanup --version + +# python (later we call it 'cli') +python3 -mpip install artifactory-cleanup==1.0.0 +artifactory-cleanup --help ``` +- Use CI servers or cron-like utilities to run `artifactory-cleanup` every day (or every hour). TeamCity and GitHub have + built-in support and show additional logs format +- Do not save credentials in the configuration file, use environment variables. + ## Commands ## ```bash -# Debug -# debug run - only print founded artifacts. it do not delete -artifactory-cleanup --user user --password password --artifactory-server https://repo.example.com/artifactory --config reponame.py +# Debug - "dry run" mode by default +# debug run - only print artifacts. it does not delete any artifacts +artifactory-cleanup -# Debug run only for policytestname. Find any *policytestname* -# debug run - only print founded artifacts. it do not delete -artifactory-cleanup --policy-name policytestname --user user --password password --artifactory-server https://repo.example.com/artifactory --config reponame.py +# Debug run only for policytestname. +artifactory-cleanup --policy-name policytestname # REMOVE # For remove artifacts use --destroy -artifactory-cleanup --destroy --user user --password password --artifactory-server https://repo.example.com/artifactory --config reponame.py +artifactory-cleanup --destroy + +# Specify config filename +artifactory-cleanup --config artifactory-cleanup.yaml + +# Look in the future - shows what the tool WILL remove after 10 days +artifactory-cleanup --days-in-future=10 + +# Not satisfied with built-in rules? Write your own rules in python and connect them! +artifactory-cleanup --load-rules=myrule.py +docker run -v "$(pwd)":/app devopshq/artifactory-cleanup artifactory-cleanup --load-rules=myrule.py +``` + +# Rules + +## Common + +- `Repo` - Apply the rule to one repository. If no name is specified, it is taken from the rule name (in `CleanupPolicy` + definition) + +```yaml +- rule: Repo + name: reponame +``` + +```yaml +# OR - if you have a single policy for the repo - you can name the policy as reponame +# Both configurations are equal +policies: + - name: reponame + rules: + - rule: Repo +``` + +- `RepoByMask` - Apply rule to repositories matching by mask + +```yaml +- rule: RepoByMask + mask: "*.banned" +``` + +- `PropertyEq`- Delete repository artifacts only with a specific property value (property_name is the name of the + parameter, property_value is the value) + +```yaml +- rule: PropertyEq + property_key: key-name + property_value: 1 +``` + +- `PropertyNeq`- Delete repository artifacts only if the value != specified. If there is no value, delete it anyway. + Allows you to specify the deletion flag `do_not_delete = 1` + +```yaml +- rule: PropertyNeq + property_key: key-name + property_value: 1 +``` + +## Delete + +- `DeleteOlderThan` - deletes artifacts that are older than N days + +```yaml +- rule: DeleteOlderThan + days: 1 +``` + +- `DeleteWithoutDownloads` - deletes artifacts that have never been downloaded (DownloadCount=0). Better to use + with `DeleteOlderThan` rule + +```yaml +- rule: DeleteWithoutDownloads +``` + +- `DeleteOlderThanNDaysWithoutDownloads` - deletes artifacts that are older than N days and have not been + downloaded + +```yaml +- rule: DeleteOlderThanNDaysWithoutDownloads + days: 1 +``` + +- `DeleteNotUsedSince` - delete artifacts that were downloaded, but for a long time. N days passed. Or not + downloaded at all from the moment of creation and it's been N days + +```yaml +- rule: DeleteNotUsedSince + days: 1 +``` + +- `DeleteEmptyFolders` - Clean up empty folders in given repository list + +```yaml +- rule: DeleteEmptyFolders +``` + +## Keep + +- `KeepLatestNFiles` - Leaves the last (by creation time) files in the amount of N pieces. WITHOUT accounting + subfolders + +```yaml +- rule: KeepLatestNFiles + count: 1 +``` + +- `KeepLatestNFilesInFolder` - Leaves the last (by creation time) files in the number of N pieces in each + folder + +```yaml +- rule: KeepLatestNFilesInFolder + count: 1 +``` + +- `KeepLatestVersionNFilesInFolder` - Leaves the latest N (by version) files in each + folder. The definition of the version is using regexp. By default `[^\d][\._]((\d+\.)+\d+)` + +```yaml +- rule: KeepLatestVersionNFilesInFolder + count: 1 + custom_regexp: "[^\\d][\\._]((\\d+\\.)+\\d+)" +``` + +- `KeepLatestNupkgNVersions` - Leaves N nupkg (adds `*.nupkg` filter) in release feature builds + +```yaml +- rule: KeepLatestNupkgNVersions + count: 1 +``` + +## Docker + +- `DeleteDockerImagesOlderThan` - Delete docker images that are older than N days + +```yaml +- rule: DeleteDockerImagesOlderThan + days: 1 +``` + +- `DeleteDockerImagesOlderThanNDaysWithoutDownloads` - Deletes docker images that are older than N days and have + not been downloaded + +```yaml +- rule: DeleteDockerImagesOlderThanNDaysWithoutDownloads + days: 1 +``` + +- `DeleteDockerImagesNotUsed` - Removes Docker image not downloaded since N days + +```yaml +- rule: DeleteDockerImagesNotUsed + days: 1 +``` + +- `IncludeDockerImages` - Apply to docker images with the specified names and tags + +```yaml +- rule: IncludeDockerImages + masks: "*singlemask*" +- rule: IncludeDockerImages + masks: + - "*production*" + - "*release*" +``` + +- `ExcludeDockerImages` - Exclude Docker images by name and tags. + +```yaml +- rule: ExcludeDockerImages + masks: + - "*production*" + - "*release*" +``` + +- `KeepLatestNVersionImagesByProperty(count=N, custom_regexp='some-regexp', number_of_digits_in_version=X)` - Leaves N + Docker images with the same major. `(^\d+\.\d+\.\d+$)` is the default regexp how to determine version which matches semver `1.1.1`. If you + need to add minor then set `number_of_digits_in_version` to 2 or if patch then set to 3 (by default we match major, which 1) + +- `DeleteDockerImageIfNotContainedInProperties(docker_repo='docker-local', properties_prefix='my-prop', image_prefix=None, full_docker_repo_name=None)` + - Remove Docker image, if it is not found in the properties of the artifact repository. +- `DeleteDockerImageIfNotContainedInPropertiesValue(docker_repo='docker-local', properties_prefix='my-prop', image_prefix=None, full_docker_repo_name=None)` + - Remove Docker image, if it is not found in the properties of the artifact repository. + +```yaml +- rule: KeepLatestNVersionImagesByProperty + count: 1 + custom_regexp: "[^\\d][\\._]((\\d+\\.)+\\d+)" +``` + +## Filters + +- `IncludePath` - Apply to artifacts by path / mask. + +```yaml +- rule: IncludePath + masks: "*production*" +- rule: IncludePath + masks: + - "*production*" + - "*develop*" ``` -## Available Rules ## +- `IncludeFilename` - Apply to artifacts by name/mask -All rules are imported from the `rules` module. -See also [List of available cleaning rules](docs/RULES) +```yaml +- rule: IncludeFilename + masks: + - "*production*" + - "*develop*" +``` -## Artifact cleanup policies ## +- `ExcludePath` - Exclude artifacts by path/mask -To add a cleaning policy you need: +```yaml +- rule: ExcludePath + masks: + - "*production*" + - "*develop*" +``` -- Create a python file, for example, `reponame.py`. `artifacroty-cleanup` imports the variable `RULES`, so you can make a python package. -- Add a cleanup rule from the [available cleanup rules](docs/RULES). +- `ExcludeFilename` - Exclude artifacts by name/mask + +```yaml +- rule: ExcludeFilename + masks: + - "*.tag.gz" + - "*.zip" +``` + +## Create your own rule + +If you want to create your own rule, you can do it! + +The basic flow how the tool calls Rules: + +1. `Rule.check(*args, **kwargs)` - verify that the Rule configured right. Call other services to get more information. +2. `Rule.aql_add_filter(filters)` - add Artifactory Query Language expressions +3. `Rule.aql_add_text(aql)` - add text to the result aql query +4. `artifactory-cleanup` calls Artifactory with AQL and pass the result to the next step +5. `Rule.filter(artifacts)` - filter out artifacts. The method returns **artifacts that will be removed!**. + - To keep artifacts use `artifacts.keep(artifact)` method + +Create `myrule.py` file at the same folder as `artifactory-cleanup.yaml`: ```python -from artifactory_cleanup import rules -from artifactory_cleanup import CleanupPolicy +# myrule.py +from typing import List + +from artifactory_cleanup import register +from artifactory_cleanup.rules import Rule, ArtifactsList -RULES = [ - CleanupPolicy( - 'Delete all * .tmp repositories older than 7 days', - rules.RepoByMask('*. tmp'), - rules.DeleteOlderThan(days=7), - ), - CleanupPolicy( - 'Delete all images older than 30 days from docker-registry exclude latest, release', - rules.Repo('docker-registry'), - rules.ExcludeDockerImages(['*:latest', '*:release*']), - rules.DeleteDockerImagesNotUsed(days=30), - ), -] +class MySimpleRule(Rule): + """For more methods look at Rule source code""" + + def __init__(self, my_param: str, value: int): + self.my_param = my_param + self.value = value + + def aql_add_filter(self, filters: List) -> List: + print(f"Today is {self.today}") + print(self.my_param) + print(self.value) + return filters + + def filter(self, artifacts: ArtifactsList) -> ArtifactsList: + """I'm here just to print the list""" + print(self.my_param) + print(self.value) + # You can make requests to artifactory by using self.session: + # url = f"/api/storage/{self.repo}" + # r = self.session.get(url) + # r.raise_for_status() + return artifacts + + +# Register your rule in the system +register(MySimpleRule) ``` -## Docker Container Usage ## -The below command assumes you to have your rules configuration file `rules.py` in the current working directory. +Use `rule: MySimpleRule` in configuration: + +```yaml +# artifactory-cleanup.yaml +- rule: MySimpleRule + my_param: "Hello, world!" + value: 42 +``` -To run the container use the following command: +Specify `--load-rules` to the command: ```bash -# Dry mode - log artifacts that will be removed -docker run \ - --mount type=bind,source=./rules.py,target=/tmp/rules.py \ - -e ARTIFACTORY_USER= \ - -e ARTIFACTORY_PASSWORD= \ - -e ARTIFACTORY_URL= \ - -e ARTIFACTORY_RULES_CONFIG=/tmp/rules.py \ - devopshq/artifactory-cleanup:latest +# docker +docker run -v "$(pwd)":/app devopshq/artifactory-cleanup artifactory-cleanup --load-rules=myrule.py -# Destroy mode - remove artifacts -docker run \ - --mount type=bind,source=./rules.py,target=/tmp/rules.py \ - -e ARTIFACTORY_USER= \ - -e ARTIFACTORY_PASSWORD= \ - -e ARTIFACTORY_URL= \ - -e ARTIFACTORY_RULES_CONFIG=/tmp/rules.py \ - -e ARTIFACTORY_DESTROY_MODE_ENABLED="true" \ - devopshq/artifactory-cleanup:latest +# cli +artifactory-cleanup --load-rules=myrule.py ``` -The environment variables specify the necessary `artifactory-cleanup` arguments. +# How to -In case you have set up your Artifactory self-signed certificates, place all certificates of the chain of trust into the `docker/certificates/` folder and add an additional argument `--mount type=bind,source=./certificates/,target=/mnt/self-signed-certs/` to a command. - -To build the container image locally run the following command in the folder of the `Dockerfile`. +## How to connect self-signed certificates for docker? +In case you have set up your Artifactory self-signed certificates, place all certificates of the chain of trust into +the `certificates` folder and add additional argument to the command: ```bash -docker build . --tag artifactory-cleanup +docker run -v "$(pwd)":/app -v "$(pwd)/certificates":/mnt/self-signed-certs/ devopshq/artifactory-cleanup artifactory-cleanup ``` -# FAQ ## How to clean up Conan repository? + We can handle conan's metadata by creating two policies: + 1. First one removes files but keep all metadata. -2. Second one look at folders and if it contains only medata files - removes it (because there's no associated with metadata files) +2. Second one look at folders and if it contains only medata files - removes it (because there's no associated with + metadata files) The idea came from https://github.com/devopshq/artifactory-cleanup/issues/47 -```python -from artifactory_cleanup import rules -from artifactory_cleanup import CleanupPolicy -RULES = [ - CleanupPolicy( - 'Delete files older than 60 days', - rules.repo('conan-testing'), - rules.delete_not_used_since(days=60), - # Keep conan metadata - rules.exclude_filename(['.timestamp', 'index.json']), - ), - CleanupPolicy( - 'Delete empty folders', - rules.repo('conan-testing'), - rules.delete_empty_folder(), - # Exclude metadata files - # If a folder only contains these files, consider it as empty - rules.exclude_filename(['.timestamp', 'index.json']), - ), -] +```yaml +# artifactory-cleanup.yaml +artifactory-cleanup: + server: https://repo.example.com/artifactory + user: $ARTIFACTORY_USERNAME + password: $ARTIFACTORY_PASSWORD + + policies: + - name: Conan - delete files older than 60 days + rules: + - rule: Repo + name: "conan-testing" + - rule: DeleteNotUsedSince + days: 60 + - rule: ExcludeFilename + masks: + - ".timestamp" + - "index.json" + - name: Conan - delete empty folders (to fix the index) + rules: + - rule: Repo + name: "conan-testing" + - rule: DeleteEmptyFolders + - rule: ExcludeFilename + masks: + - ".timestamp" + - "index.json" ``` ## How to keep latest N docker images? + We can combine docker rules with usual "files" rules! The idea came from https://github.com/devopshq/artifactory-cleanup/issues/61 -```python -CleanupPolicy( - 'docker-demo-cleanup', - # Select repo - rules.repo('docker-demo'), - # Delete docker images older than 30 days - rules.DeleteDockerImagesOlderThan(days=30), - # Keep these tags for all images - rules.ExcludeDockerImages(['*:latest', '*:release*']), - # Exclude these docker tags - rules.ExcludePath("base-tools*"), - # Keep 3 docker tags for all images - rules.KeepLatestNFilesInFolder(count=3), -) +```yaml +# artifactory-cleanup.yaml +artifactory-cleanup: + server: https://repo.example.com/artifactory + user: $ARTIFACTORY_USERNAME + password: $ARTIFACTORY_PASSWORD + + policies: + - name: Remove docker images, but keep last 3 + rules: + # Select repo + - rule: Repo + name: docker-demo + # Delete docker images older than 30 days + - rule: DeleteDockerImagesOlderThan + days: 30 + # Keep these tags for all images + - rule: ExcludeDockerImages + masks: + - "*:latest" + - "*:release*" + # Exclude these docker tags + - rule: ExcludePath + masks: "*base-tools*" + # Keep 3 docker tags for all images + - rule: KeepLatestNFilesInFolder + count: 3 ``` - # Release In order to provide a new release of `artifactory-cleanup`, there are two steps involved. 1. Bump the version in the [setup.py](setup.py) -2. Create a Git release tag (e.g. `v0.3.3`) by creating a release on Github +2. Bump the version in the [__init__.py](./artifactory_cleanup/__init__.py) +3. Create a Git release tag (e.g. `v0.3.3`) by creating a release on GitHub diff --git a/artifactory_cleanup/__init__.py b/artifactory_cleanup/__init__.py index 9309883..7a31dba 100644 --- a/artifactory_cleanup/__init__.py +++ b/artifactory_cleanup/__init__.py @@ -1,2 +1,10 @@ from artifactory_cleanup.cli import ArtifactoryCleanupCLI # noqa +from artifactory_cleanup.loaders import registry from artifactory_cleanup.rules.base import CleanupPolicy # noqa + + +def register(rule): + registry.register(rule) + + +__version__ = "1.0.0" diff --git a/artifactory_cleanup/artifactorycleanup.py b/artifactory_cleanup/artifactorycleanup.py index 5423f3e..16650ce 100644 --- a/artifactory_cleanup/artifactorycleanup.py +++ b/artifactory_cleanup/artifactorycleanup.py @@ -4,7 +4,8 @@ from attr import dataclass from requests import Session -from artifactory_cleanup.rules.base import CleanupPolicy +from artifactory_cleanup.errors import ArtifactoryCleanupException +from artifactory_cleanup.rules.base import CleanupPolicy, ArtifactDict @dataclass @@ -14,20 +15,14 @@ class CleanupSummary: artifacts_size: int -class ArtifactoryCleanupException(Exception): - pass - - class ArtifactoryCleanup: def __init__( self, - server: str, session: Session, policies: List[CleanupPolicy], destroy: bool, today: date, ): - self.server = server self.session = session self.policies = policies self.destroy = destroy @@ -36,35 +31,34 @@ def __init__( def _init_policies(self, today): for policy in self.policies: - policy.init(self.session, self.server, today) + policy.init(self.session, today) def cleanup(self, block_ctx_mgr, test_ctx_mgr) -> Iterator[CleanupSummary]: for policy in self.policies: with block_ctx_mgr(policy.name): - # prepare + # Prepare + with block_ctx_mgr("Check"): + policy.check() + with block_ctx_mgr("AQL filter"): - policy.aql_filter() + policy.build_aql_query() # Get artifacts with block_ctx_mgr("Get artifacts"): - print("*" * 80) - print("AQL Query:") - print(policy.aql_text) - print("*" * 80) artifacts = policy.get_artifacts() print("Found {} artifacts".format(len(artifacts))) - # Filter + # Filter artifacts with block_ctx_mgr("Filter results"): artifacts_to_remove = policy.filter(artifacts) print(f"Found {len(artifacts_to_remove)} artifacts AFTER filtering") - # Delete or debug + # Delete artifacts for artifact in artifacts_to_remove: with test_ctx_mgr(get_name_for_ci(artifact)): policy.delete(artifact, destroy=self.destroy) - # Info + # Show summary print(f"Deleted artifacts count: {len(artifacts_to_remove)}") try: artifacts_size = sum([x["size"] for x in artifacts_to_remove]) @@ -86,12 +80,13 @@ def only(self, policy_name: str): """ policies = [policy for policy in self.policies if policy_name in policy.name] if not policies: - raise ArtifactoryCleanupException( - f"Rule with name '{policy_name}' not found" - ) + msg = f"Rule with name '{policy_name}' not found" + raise ArtifactoryCleanupException(msg) + + self.policies = policies -def get_name_for_ci(artifact): +def get_name_for_ci(artifact: ArtifactDict) -> str: return "cleanup.{}.{}_{}".format( escape(artifact["repo"]), escape(artifact["path"]), @@ -99,7 +94,7 @@ def get_name_for_ci(artifact): ) -def escape(name): +def escape(name: str) -> str: """ Escape name for some CI servers """ diff --git a/artifactory_cleanup/base_url_session.py b/artifactory_cleanup/base_url_session.py new file mode 100644 index 0000000..78c92d4 --- /dev/null +++ b/artifactory_cleanup/base_url_session.py @@ -0,0 +1,22 @@ +from urllib.parse import urljoin + +import requests + + +class BaseUrlSession(requests.Session): + """ + Perform all queries based on the base URL + + If base url is "http://example.com/" then session.get("/api/version/") queries "http://example.com/api/version/" + """ + + def __init__(self, base_url=None): + super(BaseUrlSession, self).__init__() + self.base_url = base_url.rstrip("/") + "/" + + def request(self, method, url, *args, **kwargs): + if not url.startswith("http://") or url.startswith("https://"): + url = url.lstrip("/") + url = urljoin(self.base_url, url) + + return super(BaseUrlSession, self).request(method, url, *args, **kwargs) diff --git a/artifactory_cleanup/cli.py b/artifactory_cleanup/cli.py index b4bcd60..312c999 100644 --- a/artifactory_cleanup/cli.py +++ b/artifactory_cleanup/cli.py @@ -11,7 +11,12 @@ from artifactory_cleanup.artifactorycleanup import ( ArtifactoryCleanup, ) -from artifactory_cleanup.loaders import ConfigLoaderPython, ConfigLoaderCLI +from artifactory_cleanup.base_url_session import BaseUrlSession +from artifactory_cleanup.errors import InvalidConfigError +from artifactory_cleanup.loaders import ( + PythonLoader, + YamlConfigLoader, +) from artifactory_cleanup.context_managers import get_context_managers requests.packages.urllib3.disable_warnings() @@ -25,37 +30,17 @@ def init_logging(): class ArtifactoryCleanupCLI(cli.Application): - _artifactory_server = cli.SwitchAttr( - ["--artifactory-server"], - help="URL to artifactory, e.g: https://arti.example.com/artifactory", - mandatory=True, - envname="ARTIFACTORY_SERVER", - ) - - _user = cli.SwitchAttr( - ["--user"], - help="Login to access to the artifactory", - mandatory=True, - envname="ARTIFACTORY_USER", - ) - - _password = cli.SwitchAttr( - ["--password"], - help="Password to access to the artifactory", - mandatory=True, - envname="ARTIFACTORY_PASSWORD", - ) - - _policy_name = cli.SwitchAttr( - ["--policy-name"], - help="Name for a rule", - mandatory=False, - ) - _config = cli.SwitchAttr( ["--config"], help="Name of config with list of policies", - mandatory=True, + mandatory=False, + default="artifactory-cleanup.yaml", + ) + + _policy = cli.SwitchAttr( + ["--policy"], + help="Name for a policy to execute", + mandatory=False, ) _destroy = cli.Flag( @@ -71,6 +56,19 @@ class ArtifactoryCleanupCLI(cli.Application): excludes=["--destroy"], ) + _load_rules = cli.SwitchAttr( + "--load-rules", + help="Load rules from python file", + mandatory=False, + ) + + @property + def VERSION(self): + # To prevent circular imports + from artifactory_cleanup import __version__ + + return __version__ + def _destroy_or_verbose(self): print("*" * 80) if self._destroy: @@ -82,19 +80,28 @@ def _get_today(self): today = date.today() if self._days_in_future: today = today + timedelta(days=int(self._days_in_future)) - print(f"Simulating cleanup actions that will occur on {today}") + print(f"Simulating cleanup actions that will occur on {today}") return today def main(self): - self._destroy_or_verbose() today = self._get_today() - - server, user, password = ConfigLoaderCLI(self).get_connection() - session = requests.Session() + if self._load_rules: + PythonLoader.import_module(self._load_rules) + + loader = YamlConfigLoader(self._config) + try: + policies = loader.get_policies() + except InvalidConfigError as err: + print("Failed to load config file") + print(str(err), file=sys.stderr) + sys.exit(1) + + server, user, password = loader.get_connection() + session = BaseUrlSession(server) session.auth = HTTPBasicAuth(user, password) - policies = ConfigLoaderPython.load(self._config) + + self._destroy_or_verbose() cleanup = ArtifactoryCleanup( - server=server, session=session, policies=policies, destroy=self._destroy, @@ -102,8 +109,8 @@ def main(self): ) # Filter policies by name - if self._policy_name: - cleanup.only(self._policy_name) + if self._policy: + cleanup.only(self._policy) table = PrettyTable() table.field_names = ["Cleanup Policy", "Files count", "Size"] diff --git a/artifactory_cleanup/errors.py b/artifactory_cleanup/errors.py new file mode 100644 index 0000000..01c4619 --- /dev/null +++ b/artifactory_cleanup/errors.py @@ -0,0 +1,6 @@ +class ArtifactoryCleanupException(Exception): + pass + + +class InvalidConfigError(ArtifactoryCleanupException): + pass diff --git a/artifactory_cleanup/loaders.py b/artifactory_cleanup/loaders.py index eadfd79..147be77 100644 --- a/artifactory_cleanup/loaders.py +++ b/artifactory_cleanup/loaders.py @@ -1,42 +1,210 @@ import importlib +import inspect +import logging +import os.path import sys +from copy import deepcopy from pathlib import Path -from typing import List, Tuple +from typing import List, Tuple, Type, Dict, Union -from artifactory_cleanup.rules.base import CleanupPolicy +import cfgv +import yaml +from artifactory_cleanup import rules +from artifactory_cleanup.errors import InvalidConfigError +from artifactory_cleanup.rules import Repo +from artifactory_cleanup.rules.base import CleanupPolicy, Rule -class ConfigLoaderPython: +logger = logging.getLogger("artifactory-cleanup") + +RULE_SCHEMA = cfgv.Map( + "Rule", + "rule", + cfgv.Required("rule", cfgv.check_string), +) + + +def _get_check_fn(annotation): + if annotation is int: + return cfgv.check_int + if annotation is str: + return cfgv.check_string + return cfgv.check_any + + +class SchemaBuilder: + def _get_rule_conditionals(self, name, rule) -> List[cfgv.Conditional]: + if rule.schema is not None: + return rule.schema + conditionals = [] + params = list(inspect.signature(rule.__init__).parameters.values()) + ignore = {"self", "args", "kwargs"} + for param in params: + if param.name in ignore: + continue + if param.annotation is param.empty: + check_fn = cfgv.check_any + else: + check_fn = _get_check_fn(param.annotation) + + if param.default is not param.empty: + cond = cfgv.ConditionalOptional( + param.name, + check_fn, + param.default, + "rule", + cfgv.In(name), + ensure_absent=False, + ) + else: + cond = cfgv.Conditional( + param.name, + check_fn, + "rule", + cfgv.In(name), + ensure_absent=False, + ) + conditionals.append(cond) + return conditionals + + def get_rules_conditionals(self, rules) -> List[cfgv.Conditional]: + conditionals = [] + for name, rule in rules.items(): + conditionals.extend(self._get_rule_conditionals(name, rule)) + return conditionals + + def get_root_schema(self, rules): + conditionals = self.get_rules_conditionals(rules) + rules_names = list(rules.keys()) + rule_schema = cfgv.Map( + "Rule", + "rule", + cfgv.Required("rule", cfgv.check_string), + cfgv.Required("rule", cfgv.check_one_of(rules_names)), + *conditionals, + ) + policy_schema = cfgv.Map( + "Policy", + "name", + cfgv.NoAdditionalKeys(["name", "rules"]), + cfgv.Required("name", cfgv.check_string), + cfgv.RequiredRecurse("rules", cfgv.Array(rule_schema)), + ) + + config_schema = cfgv.Map( + "Config", + None, + cfgv.NoAdditionalKeys(["server", "user", "password", "policies"]), + cfgv.Required("server", cfgv.check_string), + cfgv.Required("user", cfgv.check_string), + cfgv.Required("password", cfgv.check_string), + cfgv.RequiredRecurse("policies", cfgv.Array(policy_schema)), + ) + + root_schema = cfgv.Map( + "Artifactory Cleanup", + None, + cfgv.NoAdditionalKeys(["artifactory-cleanup"]), + cfgv.RequiredRecurse("artifactory-cleanup", config_schema), + ) + return root_schema + + +class RuleRegistry: + def __init__(self): + self.rules: Dict[str, Type[Rule]] = {} + + def get(self, name: str) -> Type[Rule]: + return self.rules[name] + + def register(self, rule: Type[Rule], name=None, warning=True): + name = name or rule.name() + if name in self.rules and warning: + logger.warning(f"Rule with a name '{name}' has been registered before.") + return + self.rules[name] = rule + + def register_builtin_rules(self): + for name, obj in vars(rules).items(): + if inspect.isclass(obj) and issubclass(obj, Rule): + self.register(obj, warning=False) + + +registry = RuleRegistry() +registry.register_builtin_rules() + + +class YamlConfigLoader: """ - Load policies and rules from python file + Load configuration and policies from yaml file """ + _rules = {} + + def __init__(self, filepath): + self.filepath = Path(filepath) + + def get_policies(self) -> List[CleanupPolicy]: + config = self.load(self.filepath) + policies = [] + + for policy_data in config["artifactory-cleanup"]["policies"]: + policy_name = policy_data["name"] + rules = [] + for rule_data in policy_data["rules"]: + try: + rule = self._build_rule(rule_data) + except Exception as exc: + print( + f"Failed to initialize rule '{rule_data['rule']}' in policy '{policy_name}'.", + file=sys.stdout, + ) + print(exc, file=sys.stdout) + sys.exit(1) + + rules.append(rule) + policy = CleanupPolicy(policy_name, *rules) + policies.append(policy) + return policies + + def _build_rule(self, rule_data: Dict) -> Union[Rule, Type[Rule]]: + kwargs = deepcopy(rule_data) + rule_cls = registry.get(kwargs.pop("rule")) + + # For Repo rule, CleanupPolicy initialize it later with the name of the policy + if rule_cls == Repo and not kwargs: + return rule_cls + + return rule_cls(**kwargs) + @staticmethod - def load(filename) -> List[CleanupPolicy]: - try: - filepath = Path(filename) - policies_directory = filepath.parent - # Get module name without the py suffix: policies.py => policies - module_name = filepath.stem - sys.path.append(str(policies_directory)) - policies = getattr(importlib.import_module(module_name), "RULES") - - # Validate that all policies is CleanupPolicy - for policy in policies: - if not isinstance(policy, CleanupPolicy): - sys.exit(f"Rule '{policy}' is not CleanupPolicy, check it please") - - return policies - except ImportError as error: - print("Error: {}".format(error)) - sys.exit(1) - - -class ConfigLoaderCLI: - def __init__(self, cli): - self.cli = cli + def load(filename): + schema = SchemaBuilder().get_root_schema(registry.rules) + return cfgv.load_from_filename( + filename, schema, yaml.safe_load, InvalidConfigError + ) def get_connection(self) -> Tuple[str, str, str]: - # remove trailing slash - server = self.cli._artifactory_server.rstrip("/") - return server, self.cli._user, self.cli._password + config = self.load(self.filepath) + server = config["artifactory-cleanup"]["server"] + user = config["artifactory-cleanup"]["user"] + password = config["artifactory-cleanup"]["password"] + + user = os.path.expandvars(user) + password = os.path.expandvars(password) + return server, user, password + + +class PythonLoader: + """ + Load rules from a python file + """ + + @staticmethod + def import_module(filename): + filepath = Path(filename) + directory = filepath.parent + sys.path.append(str(directory)) + # Get module name without the py suffix: policies.py => policies + module_name = filepath.stem + return importlib.import_module(module_name) diff --git a/artifactory_cleanup/rules/base.py b/artifactory_cleanup/rules/base.py index b3f280b..44ed48d 100644 --- a/artifactory_cleanup/rules/base.py +++ b/artifactory_cleanup/rules/base.py @@ -1,7 +1,74 @@ import inspect import json +import sys +from copy import deepcopy +from datetime import date +from typing import Optional, Union, List, Dict from urllib.parse import quote +import cfgv +from hurry.filesize import size + +from artifactory_cleanup.base_url_session import BaseUrlSession + +if sys.version_info >= (3, 8): + from typing import TypedDict +else: + from typing_extensions import TypedDict + + +class ArtifactDict(TypedDict): + repo: str + path: str + name: str + properties: dict + stats: dict + size: int + + +class ArtifactsList(List[ArtifactDict]): + def keep(self, artifacts): + """Just a shortcut for better readability""" + return self.remove(artifacts) + + def remove(self, artifacts: Union[ArtifactDict, List[ArtifactDict]]) -> None: + """ + Remove artifacts (or one artifact) and log that. + """ + if not isinstance(artifacts, list): + artifacts = [artifacts] + + for artifact in artifacts: + print(f"Filter package {artifact['path']}/{artifact['name']}") + super().remove(artifact) + + @classmethod + def from_response(cls, artifacts: List[Dict]) -> "ArtifactsList": + """ + :param artifacts: Pure AQL response + """ + return ArtifactsList(cls.prepare(artifact) for artifact in artifacts) + + @classmethod + def prepare(cls, artifact: Dict) -> ArtifactDict: + """ + Convert properties, stat from the list format to the dict format + """ + if "properties" in artifact: + if not isinstance(artifact["properties"], dict): + artifact["properties"] = { + x["key"]: x.get("value") for x in artifact["properties"] + } + else: + artifact["properties"] = {} + + if "stats" in artifact: + artifact["stats"] = artifact["stats"][0] + else: + artifact["stats"] = {} + + return artifact + class Rule(object): """ @@ -11,133 +78,72 @@ class Rule(object): Make it small and then combine them in CleanupPolicy in more complicated entities. """ - def __init__(self): - self.artifactory_session = None - self.artifactory_server = None - self.today = None + session: Optional[BaseUrlSession] = None + today: date = None + + # You can overwrite checks for config file + schema: Optional[List[cfgv.Conditional]] = None - def init(self, artifactory_session, artifactory_server, today): + @classmethod + def name(cls) -> str: + return cls.__name__ + + @classmethod + def title(cls) -> str: + """Cut the docstring to show only the very first important line""" + docs = [x.strip() for x in cls.__doc__.splitlines() if x][0] + return docs + + def init(self, session, today, *args, **kwargs) -> None: """ - Init the rule after got all information + Init the rule after got all information. + + Please make sure to add *args, **kwargs for future extension """ - self.artifactory_session = artifactory_session - self.artifactory_server = artifactory_server + self.session = session self.today = today - def _aql_add_filter(self, aql_query_list): + def check(self, *args, **kwargs): + """ + Checks that Rule is configured right. + Make sure to add args, kwargs, because we can add more options there in the future """ - Add filter to `items.find` AQL part. - Here you can filter artifacts on Artifactory side with AQL: - https://www.jfrog.com/confluence/display/JFROG/Artifactory+Query+Language + def aql_add_filter(self, filters: List) -> List: + """ + Add one or more filters to `.find()` AQL part. + It's called in the documentation: .find() + https://www.jfrog.com/confluence/display/JFROG/Artifactory+Query+Language#usage + The artifacts will be filtered on Artifactory side. It's better to filter out as much as possible with that filter rather than get all artifacts and filter out them in memory because it leads to a heavy response from Artifactory - Also, you can find any conflicts with others rules here, if they conflict on AQL level + You can detect any conflicts with others rules here, if they conflict on AQL level. """ - return aql_query_list + return filters - def _aql_add_text(self, aql_text): + def aql_add_text(self, aql: str) -> str: """ - Change AQL text after applying all rules filters + You can change AQL text after applying all rules filters. + + You can apply sort rules and other AQL filter here. + https://www.jfrog.com/confluence/display/JFROG/Artifactory+Query+Language """ - return aql_text + return aql - def _filter_result(self, result_artifact): + def filter(self, artifacts: ArtifactsList) -> ArtifactsList: """ Filter artifacts after performing AQL query. - To remove artifacts from the list - please use Rule.remove_artifact method in order to log the action as well. + To keep artifacts - use `artifacts.remove(artifacts_to_keep)` method. If you have your own logic - please overwrite the method in your Rule class. Here you can filter artifacts in memory, make additional calls to Artifactory or even call other services! - :param result_artifact: Filtered artifacts list that we get after filter them with AQL + :param artifacts: Filtered artifacts list that we get after filter them with AQL :return List of artifacts that you are going to remove """ - return result_artifact - - @staticmethod - def remove_artifact(artifacts, result_artifact): - """ - Remove and log artifacts - :param artifacts: Artifacts to remove - :param result_artifact: Artifacts remove from it - """ - if not isinstance(artifacts, list): - artifacts = [artifacts] - for artifact in artifacts: - print(f"Filter package {artifact['path']}/{artifact['name']}") - result_artifact.remove(artifact) - - def aql_add_filter(self, aql_query_list): - """ - Add filters to `items.find` AQL part - """ - print(f"Add AQL Filter - rule: {self.__class__.__name__} - {self.little_doc}") - new_aql_query_list = self._aql_add_filter(aql_query_list) - if aql_query_list != new_aql_query_list: - print("Before AQL query: {}".format(aql_query_list)) - print("After AQL query: {}".format(new_aql_query_list)) - print() - return new_aql_query_list - - def aql_add_text(self, aql_text): - """ - Adds some expression to AQL query - """ - print(f"Add AQL Text - rule: {self.__class__.__name__} - {self.little_doc}") - new_aql_text = self._aql_add_text(aql_text) - if new_aql_text != aql_text: - print("Before AQL text: {}".format(aql_text)) - print("After AQL text: {}".format(new_aql_text)) - print() - return new_aql_text - - def filter_result(self, result_artifacts): - """ - Filter artifacts after performing AQL query - - It's a high level function, if you want to specify your own logic - please overwrite in your Rule class `_filter_result` method - """ - print(f"Filter artifacts - rule: {self.__class__.__name__} - {self.little_doc}") - new_result = self._filter_result(result_artifacts) - if len(new_result) != len(result_artifacts): - print("Before count: {}".format(len(result_artifacts))) - print("After count: {}".format(len(new_result))) - print() - - return new_result - - @property - def little_doc(self): - """ - Cut the docstring to show only the very first important line - """ - docs = [x.strip() for x in self.__doc__.splitlines() if x][0] - return docs - - @classmethod - def prepare_artifact(cls, artifacts): - """properties, stat are given in list format, convert them to dict format""" - all_artifacts = [] - for artifact in artifacts: - if "properties" in artifact: - artifact["properties"] = { - x["key"]: x.get("value") for x in artifact["properties"] - } - else: - artifact["properties"] = {} - - if "stats" in artifact: - artifact["stats"] = artifact["stats"][0] - else: - artifact["stats"] = {} - - all_artifacts.append(artifact) - - return all_artifacts + return artifacts class CleanupPolicy(object): @@ -150,99 +156,157 @@ class CleanupPolicy(object): rules.repo, rules.DeleteOlderThan(days=7), ) - """ - def __init__(self, name, *rules): + # domain_query in https://www.jfrog.com/confluence/display/JFROG/Artifactory+Query+Language#usage + DOMAIN = "items" + + session: Optional[BaseUrlSession] = None + today: date = None + + def __init__(self, name: str, *rules: Rule): if not isinstance(name, str): - raise Exception( + raise ValueError( "Bad CleanupPolicy, first argument must be name.\n" - "CleanupPolicy argument: name={}, *args={}".format(name, rules) + f"You called: CleanupPolicy(name={name}, *rules={rules})" ) + self.name = name self.rules = list(rules) + self.aql_text = None # init object if passed not initialized class - # for `rules.repo` rule + # for `rules.repo` rule, see above in the docstring for i, rule in enumerate(self.rules): if inspect.isclass(rule): self.rules[i] = rule(self.name) - # Assigned in self.init() function - self.artifactory_session = None - self.artifactory_url = None - self.today = None - - # Defined in aql_filter - self.aql_query_list = [] + def check(self, *args, **kwargs) -> None: + """ + Check that we're ready to run the policy. + Here you can call additional APIs to check they're available or check that rules are consistent, + like have no contradictory rules in one set + """ + for rule in self.rules: + self._check_rules_are_updated(rule) + try: + rule.check(*args, **kwargs) + except Exception as exc: + print( + f"Check failed for rule '{rule.name()}' in policy '{self.name}':", + file=sys.stdout, + ) + print(exc, file=sys.stdout) + sys.exit(1) + + def _check_rules_are_updated(self, rule): + # Make sure people update their own rules to the latest interface + # 0.4 => 1.0.0 + old_attributes = ("_aql_add_filter", "_aql_add_text", "_filter_result") + old_rule = any(hasattr(rule, attr) for attr in old_attributes) + if old_rule: + raise ValueError( + f"Please update the Rule '{rule.name()()}' to the new Rule API.\n" + "- Read CHANGELOG.md https://github.com/devopshq/artifactory-cleanup/blob/master/CHANGELOG.md\n" + "- Read README.md https://github.com/devopshq/artifactory-cleanup#readme" + ) - def init(self, artifactory_session, artifactory_url, today): + def init(self, session, today) -> None: """ Set properties and apply them to all rules """ - self.artifactory_session = artifactory_session - self.artifactory_url = artifactory_url + self.session = session self.today = today for rule in self.rules: - rule.init(artifactory_session, artifactory_url, today) + rule.init(session, today) - def aql_filter(self): + def build_aql_query(self) -> None: """ Collect all aql queries into a single list so that the rules check for conflicts among themselves """ + aql_find_filters = self._get_aql_find_filters() + self.aql_text = self._get_aql_text(aql_find_filters) + print("*" * 80) + print("Result AQL Query:") + print(self.aql_text) + print("*" * 80) + + def _get_aql_find_filters(self) -> Dict: + """Go over all rules and get .find filters""" + filters = [] for rule in self.rules: - self.aql_query_list = rule.aql_add_filter(self.aql_query_list) - - @property - def aql_text(self): + before_query_list = deepcopy(filters) + print(f"Add AQL Filter - rule: {rule.name()} - {rule.title()}") + filters = rule.aql_add_filter(filters) + if before_query_list != filters: + print("Before AQL query: {}".format(before_query_list)) + print("After AQL query: {}".format(filters)) + print() + return {"$and": filters} + + def _get_aql_text(self, find_filters: Dict) -> str: """ Collect from all rules additional texts of requests """ - aql_query_dict = {"$and": self.aql_query_list} - aql_text = 'items.find({query_dict}).include("*", "property", "stat")'.format( - query_dict=json.dumps(aql_query_dict) - ) + filters_text = json.dumps(find_filters) + aql = f'{self.DOMAIN}.find({filters_text}).include("*", "property", "stat")' for rule in self.rules: - aql_text = rule.aql_add_text(aql_text) - return aql_text - - def get_artifacts(self): - aql_url = "{}/api/search/aql".format(self.artifactory_url) - r = self.artifactory_session.post(aql_url, data=self.aql_text) + before_aql = aql + print(f"Add AQL Text - rule: {rule.name()} - {rule.title()}") + aql = rule.aql_add_text(aql) + if before_aql != aql: + print("Before AQL text: {}".format(before_aql)) + print("After AQL text: {}".format(aql)) + print() + return aql + + def get_artifacts(self) -> ArtifactsList: + """ + Get artifacts from Artifactory by AQL filters that we collect from all rules in the policy + :return list of artifacts + """ + assert self.aql_text, "Call build_aql_query before calling get_artifacts" + r = self.session.post("/api/search/aql", data=self.aql_text) r.raise_for_status() content = r.json() artifacts = content["results"] - artifacts = Rule.prepare_artifact(artifacts) - artifacts = sorted(artifacts, key=lambda x: x["path"]) - return artifacts + return ArtifactsList.from_response(artifacts) - def filter(self, artifacts): + def filter(self, artifacts: ArtifactsList) -> ArtifactsList: """ Filter artifacts again all rules """ for rule in self.rules: - artifacts = rule.filter_result(artifacts) + before = len(artifacts) + print(f"Filter artifacts - rule: {rule.name()} - {rule.title()}") + artifacts = rule.filter(artifacts) + + if not isinstance(artifacts, ArtifactsList): + raise ValueError(f"`{rule.name()}` rule must return ArtifactsList") + + if before != len(artifacts): + print(f"Before count: {before}") + print(f"After count: {len(artifacts)}") + print() return artifacts - def delete(self, artifact, destroy): + def delete(self, artifact: ArtifactDict, destroy: bool) -> None: """ Delete the artifact :param artifact: artifact to remove :param destroy: if False - just log the action, do not actually remove the artifact - :return: """ - if artifact["path"] == ".": - artifact_path = "{repo}/{name}".format(**artifact) - else: - artifact_path = "{repo}/{path}/{name}".format(**artifact) - + path = "{repo}/{name}" if artifact["path"] == "." else "{repo}/{path}/{name}" + artifact_path = path.format(**artifact) artifact_path = quote(artifact_path) - if destroy: - print("DESTROY MODE - delete {}".format(artifact_path)) - delete_url = "{}/{}".format(self.artifactory_url, artifact_path) - r = self.artifactory_session.delete(delete_url) - r.raise_for_status() - else: - print("DEBUG - delete {}".format(artifact_path)) + artifact_size = artifact.get("size", 0) or 0 + + if not destroy: + print(f"DEBUG - we would delete '{artifact_path}' - {size(artifact_size)}") + return + + print(f"DESTROY MODE - delete '{artifact_path} - {size(artifact_size)}'") + r = self.session.delete(artifact_path) + r.raise_for_status() diff --git a/artifactory_cleanup/rules/delete.py b/artifactory_cleanup/rules/delete.py index 5081d05..4c48989 100644 --- a/artifactory_cleanup/rules/delete.py +++ b/artifactory_cleanup/rules/delete.py @@ -7,20 +7,20 @@ class DeleteOlderThan(Rule): """Deletes artifacts older than `` days`` days""" - def __init__(self, *, days): + def __init__(self, *, days: int): self.days = timedelta(days=days) - def _aql_add_filter(self, aql_query_list): + def aql_add_filter(self, filters): older_than_date = self.today - self.days older_than_date_txt = older_than_date.isoformat() print("Delete artifacts older than {}".format(older_than_date_txt)) - update_dict = { + filter_ = { "created": { "$lt": older_than_date_txt, } } - aql_query_list.append(update_dict) - return aql_query_list + filters.append(filter_) + return filters class DeleteWithoutDownloads(Rule): @@ -29,10 +29,10 @@ class DeleteWithoutDownloads(Rule): Better to use with :class:`DeleteOlderThan` """ - def _aql_add_filter(self, aql_query_list): - update_dict = {"stat.downloads": {"$eq": None}} - aql_query_list.append(update_dict) - return aql_query_list + def aql_add_filter(self, filters): + filter_ = {"stat.downloads": {"$eq": None}} + filters.append(filter_) + return filters class DeleteOlderThanNDaysWithoutDownloads(Rule): @@ -40,19 +40,19 @@ class DeleteOlderThanNDaysWithoutDownloads(Rule): Deletes artifacts that are older than n days and have not been downloaded. """ - def __init__(self, *, days): + def __init__(self, *, days: int): self.days = timedelta(days=days) - def _aql_add_filter(self, aql_query_list): + def aql_add_filter(self, filters): last_day = self.today - self.days - update_dict = { + filter_ = { "$and": [ {"stat.downloads": {"$eq": None}}, {"created": {"$lte": last_day.isoformat()}}, ], } - aql_query_list.append(update_dict) - return aql_query_list + filters.append(filter_) + return filters class DeleteNotUsedSince(Rule): @@ -61,13 +61,13 @@ class DeleteNotUsedSince(Rule): Or not downloaded at all from the moment of creation and it's been N days. """ - def __init__(self, days): + def __init__(self, days: int): self.days = timedelta(days=days) - def _aql_add_filter(self, aql_query_list): + def aql_add_filter(self, filters): last_day = self.today - self.days - update_dict = { + filter_ = { "$or": [ {"stat.downloaded": {"$lte": str(last_day)}}, { @@ -79,12 +79,12 @@ def _aql_add_filter(self, aql_query_list): ] } - aql_query_list.append(update_dict) + filters.append(filter_) - return aql_query_list + return filters -class DeleteEmptyFolder(Rule): +class DeleteEmptyFolders(Rule): """ Remove empty folders. @@ -92,22 +92,13 @@ class DeleteEmptyFolder(Rule): We use the rule to help with some specific cases - look at README.md "FAQ: How to clean up Conan repository" """ - def _aql_add_filter(self, aql_query_list): + def aql_add_filter(self, filters): # Get list of all files and folders all_files_dict = {"path": {"$match": "**"}, "type": {"$eq": "any"}} - aql_query_list.append(all_files_dict) - return aql_query_list + filters.append(all_files_dict) + return filters - def _filter_result(self, result_artifacts): - repositories = utils.build_repositories(result_artifacts) + def filter(self, artifacts): + repositories = utils.build_repositories(artifacts) folders = utils.get_empty_folders(repositories) return folders - - -# under_score - old style of naming -# Keep it for backward compatibility -delete_older_than = DeleteOlderThan -delete_without_downloads = DeleteWithoutDownloads -delete_older_than_n_days_without_downloads = DeleteOlderThanNDaysWithoutDownloads -delete_not_used_since = DeleteNotUsedSince -delete_empty_folder = DeleteEmptyFolder diff --git a/artifactory_cleanup/rules/docker.py b/artifactory_cleanup/rules/docker.py index fc6b43d..1625d6a 100644 --- a/artifactory_cleanup/rules/docker.py +++ b/artifactory_cleanup/rules/docker.py @@ -1,10 +1,14 @@ import re from collections import defaultdict from datetime import timedelta +from typing import Tuple +import pydash from artifactory import ArtifactoryPath from artifactory_cleanup.context_managers import get_context_managers -from artifactory_cleanup.rules.base import Rule +from artifactory_cleanup.rules import Rule +from artifactory_cleanup.rules.base import ArtifactsList +from artifactory_cleanup.rules.utils import to_masks ctx_mgr_block, ctx_mgr_test = get_context_managers() @@ -15,78 +19,85 @@ class RuleForDocker(Rule): """ def get_docker_images_list(self, docker_repo): - _href = "{}/api/docker/{}/v2/_catalog".format( - self.artifactory_server, docker_repo - ) - r = self.artifactory_session.get(_href) + url = f"/api/docker/{docker_repo}/v2/_catalog" + r = self.session.get(url) r.raise_for_status() content = r.json() return content["repositories"] def get_docker_tags_list(self, docker_repo, docker_image): - _href = "{}/api/docker/{}/v2/{}/tags/list".format( - self.artifactory_server, docker_repo, docker_image - ) - r = self.artifactory_session.get(_href) + url = f"/api/docker/{docker_repo}/v2/{docker_image}/tags/list" + r = self.session.get(url) r.raise_for_status() content = r.json() - return content["tags"] - def _collect_docker_size(self, new_result): - docker_repos = list(set(x["repo"] for x in new_result)) + def _manifest_to_docker_images(self, artifacts: ArtifactsList): + """ + Convert manifest.json path to folder path + Docker rules get path to "manifest.json" file, + in order to remove the whole image we have to "up" one leve + """ + for artifact in artifacts: + # already done it or it's just a folder + if "name" not in artifact or artifact["name"] != "manifest.json": + continue + artifact["path"], docker_tag = artifact["path"].rsplit("/", 1) + artifact["name"] = docker_tag + # We're going to collect docker size later + if "size" in artifact: + del artifact["size"] + return artifacts + + def _collect_docker_size(self, artifacts): + # skip if already get the size + sizes_collected = all("size" in artifact for artifact in artifacts) + if sizes_collected: + return + + docker_repos = list(set(x["repo"] for x in artifacts)) if docker_repos: - aql = ArtifactoryPath( - self.artifactory_server, session=self.artifactory_session - ) + aql = ArtifactoryPath(self.session.base_url, session=self.session) args = ["items.find", {"$or": [{"repo": repo} for repo in docker_repos]}] artifacts_list = aql.aql(*args) - images_dict = defaultdict(int) + images_sizes = defaultdict(int) for docker_layer in artifacts_list: - images_dict[docker_layer["path"]] += docker_layer["size"] + image_key = (docker_layer["repo"], docker_layer["path"]) + images_sizes[image_key] += docker_layer["size"] - for artifact in new_result: + for artifact in artifacts: image = f"{artifact['path']}/{artifact['name']}" - artifact["size"] = images_dict[image] + image_key = (artifact["repo"], image) + artifact["size"] = images_sizes.get(image_key, 0) - def filter_result(self, result_artifacts): - """Determines the size of deleted images""" - new_result = super(RuleForDocker, self).filter_result(result_artifacts) - self._collect_docker_size(new_result) + def aql_add_filter(self, filters): + filters.append({"name": {"$match": "manifest.json"}}) + return filters - return new_result + def filter(self, artifacts): + """Determines the size of deleted images""" + artifacts = self._manifest_to_docker_images(artifacts) + artifacts = super(RuleForDocker, self).filter(artifacts) + self._collect_docker_size(artifacts) + return artifacts class DeleteDockerImagesOlderThan(RuleForDocker): """Removes Docker image older than ``days`` days""" - def __init__(self, *, days): + def __init__(self, *, days: int): self.days = timedelta(days=days) - def _aql_add_filter(self, aql_query_list): + def aql_add_filter(self, filters): older_than_date = self.today - self.days older_than_date_txt = older_than_date.isoformat() print("Delete docker images older than {}".format(older_than_date_txt)) - update_dict = { - "modified": { - "$lt": older_than_date_txt, - }, - "name": { - "$match": "manifest.json", - }, - } - aql_query_list.append(update_dict) - return aql_query_list - - def _filter_result(self, result_artifact): - for artifact in result_artifact: - artifact["path"], docker_tag = artifact["path"].rsplit("/", 1) - artifact["name"] = docker_tag - - return result_artifact + filter_ = {"modified": {"$lt": older_than_date_txt}} + filters.append(filter_) + return super().aql_add_filter(filters) class DeleteDockerImagesOlderThanNDaysWithoutDownloads(RuleForDocker): @@ -94,44 +105,29 @@ class DeleteDockerImagesOlderThanNDaysWithoutDownloads(RuleForDocker): Deletes images that are older than n days and have not been downloaded. """ - def __init__(self, *, days): + def __init__(self, *, days: int): self.days = timedelta(days=days) - def _aql_add_filter(self, aql_query_list): + def aql_add_filter(self, filters): last_day = self.today - self.days - update_dict = { - "name": { - "$match": "manifest.json", - }, - "$and": [ - {"stat.downloads": {"$eq": None}}, - {"created": {"$lte": last_day.isoformat()}}, - ], + filter_ = { + {"stat.downloads": {"$eq": None}}, + {"created": {"$lte": last_day.isoformat()}}, } - aql_query_list.append(update_dict) - return aql_query_list - - def _filter_result(self, result_artifact): - for artifact in result_artifact: - artifact["path"], docker_tag = artifact["path"].rsplit("/", 1) - artifact["name"] = docker_tag - - return result_artifact + filters.append(filter_) + return super().aql_add_filter(filters) class DeleteDockerImagesNotUsed(RuleForDocker): """Removes Docker image not downloaded ``days`` days""" - def __init__(self, *, days): + def __init__(self, *, days: int): self.days = timedelta(days=days) - def _aql_add_filter(self, aql_query_list): + def aql_add_filter(self, filters): last_day = self.today - self.days print("Delete docker images not used from {}".format(last_day.isoformat())) - update_dict = { - "name": { - "$match": "manifest.json", - }, + filter_ = { "$or": [ {"stat.downloaded": {"$lte": last_day.isoformat()}}, { @@ -142,18 +138,63 @@ def _aql_add_filter(self, aql_query_list): }, ], } - aql_query_list.append(update_dict) - return aql_query_list + filters.append(filter_) + return super().aql_add_filter(filters) - def _filter_result(self, result_artifact): - for artifact in result_artifact: - artifact["path"], docker_tag = artifact["path"].rsplit("/", 1) - artifact["name"] = docker_tag - return result_artifact +class FilterDockerImages(RuleForDocker): + operator = None + boolean_operator = None + + def __init__(self, masks): + if not self.operator: + raise AttributeError("Attribute 'operator' must be specified") + if not self.boolean_operator: + raise AttributeError("Attribute 'boolean_operator' must be specified") + self.masks = to_masks(masks) -class KeepLatestNVersionImagesByProperty(Rule): + def get_masks(self): + # alpine:2.4 => alpine/2.4 + return [mask.replace(":", "/") for mask in self.masks] + + def check(self, masks): + for mask in self.masks: + if ":" not in mask: + raise AttributeError(f"Mask '{mask}' must contain ':' in docker rules") + + def aql_add_filter(self, filters): + rule_list = [] + for mask in self.get_masks(): + filter_ = { + "path": { + self.operator: mask, + } + } + rule_list.append(filter_) + filters.append({self.boolean_operator: rule_list}) + return super().aql_add_filter(filters) + + +class IncludeDockerImages(FilterDockerImages): + """ + Apply to docker images with the specified names and tags. + """ + + operator = "$match" + boolean_operator = "$or" + + +class ExcludeDockerImages(FilterDockerImages): + """ + Exclude Docker images by name and tags. + """ + + operator = "$nmatch" + boolean_operator = "$and" + + +class KeepLatestNVersionImagesByProperty(RuleForDocker): r""" Leaves ``count`` Docker images with the same major. If you need to add minor then put 2 or if patch then put 3. @@ -164,40 +205,42 @@ class KeepLatestNVersionImagesByProperty(Rule): def __init__( self, - count, - custom_regexp=r"(^\d*\.\d*\.\d*.\d+$)", - number_of_digits_in_version=1, + count: int, + custom_regexp=r"(^\d+\.\d+\.\d+$)", + number_of_digits_in_version: int = 1, ): self.count = count self.custom_regexp = custom_regexp self.property = r"docker.manifest" self.number_of_digits_in_version = number_of_digits_in_version - def _filter_result(self, result_artifact): - artifacts_by_path_and_name = defaultdict(list) - for artifact in result_artifact[:]: - property = artifact["properties"][self.property] - version = re.findall(self.custom_regexp, property) - if len(version) == 1: - version_splitted = version[0].split(".") - key = artifact["path"] + "/" + version_splitted[0] - key += ".".join(version_splitted[: self.number_of_digits_in_version]) - artifacts_by_path_and_name[key].append([version_splitted[0], artifact]) - - for artifactory_with_version in artifacts_by_path_and_name.values(): - artifactory_with_version.sort( - key=lambda x: [int(x) for x in x[0].split(".")] - ) + def get_version(self, artifact) -> Tuple: + """Parse property and get version from it""" + value = artifact["properties"][self.property] + match = re.match(self.custom_regexp, value) + if not match: + raise ValueError(f"Can not find version in '{artifact}'") + version_str = match.group() + version = tuple(map(int, version_str.split("."))) + return version - good_artifact_count = len(artifactory_with_version) - self.count - if good_artifact_count < 0: - good_artifact_count = 0 + def filter(self, artifacts): + artifacts.sort(key=lambda x: x["path"]) - good_artifacts = artifactory_with_version[good_artifact_count:] - for artifact in good_artifacts: - self.remove_artifact(artifact[1], result_artifact) + def _groupby(artifact): + """Group by major/minor/patch version""" + return self.get_version(artifact)[: self.number_of_digits_in_version] - return result_artifact + # Group artifacts by major/minor or patch + grouped = pydash.group_by(artifacts, iteratee=_groupby) + + for main_version, artifacts_ in grouped.items(): + artifacts_ = list(artifacts_) + artifacts_.sort(key=self.get_version, reverse=True) + # Keep latest N artifacts + artifacts.keep(artifacts_[: self.count]) + + return super().filter(artifacts) class DeleteDockerImageIfNotContainedInProperties(RuleForDocker): @@ -217,10 +260,10 @@ def __init__( self.image_prefix = image_prefix self.full_docker_repo_name = full_docker_repo_name - def get_properties_dict(self, result_artifact): + def get_properties_dict(self, artifacts): properties_dict = defaultdict(dict) - for artifact in result_artifact: + for artifact in artifacts: if artifact.get("properties"): properties_with_image = [ x @@ -237,9 +280,9 @@ def get_properties_dict(self, result_artifact): return properties_dict - def _filter_result(self, result_artifact): + def filter(self, artifacts): images = self.get_docker_images_list(self.docker_repo) - properties_dict = self.get_properties_dict(result_artifact) + properties_dict = self.get_properties_dict(artifacts) result_docker_images = [] for image in images: @@ -280,7 +323,7 @@ def _filter_result(self, result_artifact): } ) - return result_docker_images + return super().filter(artifacts) class DeleteDockerImageIfNotContainedInPropertiesValue(RuleForDocker): @@ -300,10 +343,10 @@ def __init__( self.image_prefix = image_prefix self.full_docker_repo_name = full_docker_repo_name - def get_properties_values(self, result_artifact): + def get_properties_values(self, artifacts): """Creates a list of artifact property values if the value starts with self.properties_prefix""" properties_values = set() - for artifact in result_artifact: + for artifact in artifacts: properties_values |= set( ( artifact["properties"].get(x) @@ -314,9 +357,9 @@ def get_properties_values(self, result_artifact): return properties_values - def _filter_result(self, result_artifact): + def filter(self, artifacts): images = self.get_docker_images_list(self.docker_repo) - properties_values = self.get_properties_values(result_artifact) + properties_values = self.get_properties_values(artifacts) result_docker_images = [] for image in images: @@ -349,19 +392,3 @@ def _filter_result(self, result_artifact): ) return result_docker_images - - -# under_score - old style of naming -# Keep it for backward compatibility -delete_docker_images_older_than = DeleteDockerImagesOlderThan -delete_docker_images_older_than_n_days_without_downloads = ( - DeleteDockerImagesOlderThanNDaysWithoutDownloads -) -delete_docker_images_not_used = DeleteDockerImagesNotUsed -keep_latest_n_version_images_by_property = KeepLatestNVersionImagesByProperty -delete_docker_image_if_not_contained_in_properties = ( - DeleteDockerImageIfNotContainedInProperties -) -delete_docker_image_if_not_contained_in_properties_value = ( - DeleteDockerImageIfNotContainedInPropertiesValue -) diff --git a/artifactory_cleanup/rules/exception.py b/artifactory_cleanup/rules/exception.py deleted file mode 100644 index 6d58828..0000000 --- a/artifactory_cleanup/rules/exception.py +++ /dev/null @@ -1,2 +0,0 @@ -class PolicyException(Exception): - pass diff --git a/artifactory_cleanup/rules/filters.py b/artifactory_cleanup/rules/filters.py index b27af51..acb44bb 100644 --- a/artifactory_cleanup/rules/filters.py +++ b/artifactory_cleanup/rules/filters.py @@ -1,182 +1,70 @@ +from artifactory_cleanup.rules.utils import to_masks from artifactory_cleanup.rules.base import Rule -class IncludePath(Rule): - """ - Apply to artifacts by path / mask. - - You can specify multiple paths:: - - IncludePath('*production*'), - IncludePath(['*release*', '*master*']), - - """ - - def __init__(self, mask): - self.mask = mask - - def _aql_add_filter(self, aql_query_list): - update_dict = { - "path": { - "$match": self.mask, - } - } - aql_query_list.append(update_dict) - return aql_query_list - - -class __FilterDockerImages(Rule): +class FilterRule(Rule): + attribute_name = None operator = None boolean_operator = None def __init__(self, masks): - if isinstance(masks, str): - self.masks = [masks] - elif isinstance(masks, list): - self.masks = masks - else: - raise AttributeError("Mask must by str|list") - - def _aql_add_filter(self, aql_query_list): + if not self.attribute_name: + raise AttributeError("Attribute 'attribute_name' must be specified") if not self.operator: raise AttributeError("Attribute 'operator' must be specified") - if not self.boolean_operator: raise AttributeError("Attribute 'boolean_operator' must be specified") + self.masks = to_masks(masks) + + def aql_add_filter(self, filters): rule_list = [] for mask in self.masks: - if ":" not in mask: - raise AttributeError("Mask '{}' must contain ':'".format(mask)) - # alpine:2.4 => alpine/2.4 - mask = mask.replace(":", "/") - update_dict = { - "path": { + filter_ = { + self.attribute_name: { self.operator: mask, } } - rule_list.append(update_dict) - - aql_query_list.append({self.boolean_operator: rule_list}) - return aql_query_list + rule_list.append(filter_) + filters.append({self.boolean_operator: rule_list}) + return super().aql_add_filter(filters) -class IncludeDockerImages(__FilterDockerImages): +class IncludePath(FilterRule): """ - Apply to docker images with the specified names and tags. - - You can specify multiple names and tags:: - - IncludeDockerImages('*:production*'), - IncludeDockerImages(['ubuntu:*', 'debian:9']), - + Apply to artifacts by path / mask. """ + attribute_name = "path" operator = "$match" boolean_operator = "$or" -class ExcludeDockerImages(__FilterDockerImages): - """ - Exclude Docker images by name and tags. - - You can specify multiple names and tags:: - - ExcludePath('*:production*'), - ExcludePath(['ubuntu:*', 'debian:9']), - - """ - - operator = "$nmatch" - boolean_operator = "$and" - - -class IncludeFilename(Rule): +class IncludeFilename(FilterRule): """ Apply to artifacts by name/mask. - - You can specify multiple paths:: - - IncludeFilename('*-*'), # feature-branches - IncludeFilename(['*tar.gz', '*.nupkg']), - """ - def __init__(self, mask): - self.mask = mask - - def _aql_add_filter(self, aql_query_list): - update_dict = { - "name": { - "$match": self.mask, - } - } - aql_query_list.append(update_dict) - return aql_query_list - - -class _ExcludeMask(Rule): - attribute_name = None - - def __init__( - self, - masks, - ): - if isinstance(masks, str): - self.masks = [masks] - elif isinstance(masks, list): - self.masks = masks - else: - raise AttributeError("Mask must by str|list") - - def _aql_add_filter(self, aql_query_list): - rule_list = [] - for mask in self.masks: - update_dict = { - self.attribute_name: { - "$nmatch": mask, - } - } - rule_list.append(update_dict) - and_list = {"$and": rule_list} - - aql_query_list.append(and_list) - return aql_query_list + attribute_name = "name" + operator = "$match" + boolean_operator = "$or" -class ExcludePath(_ExcludeMask): +class ExcludePath(FilterRule): """ - Exclude artifacts by path/mask. - - You can specify multiple paths:: - - ExcludePath('*production*'), - ExcludePath(['*release*', '*master*']), - + Exclude artifacts by path. """ attribute_name = "path" + operator = "$nmatch" + boolean_operator = "$and" -class ExcludeFilename(_ExcludeMask): +class ExcludeFilename(FilterRule): """ - Exclude artifacts by name/mask. - - You can specify multiple paths:: - - ExcludeFilename('*-*'), # feature-branch - ExcludeFilename(['*tar.gz', '*.nupkg']), - + Exclude artifacts by filename. """ attribute_name = "name" - - -# under_score - old style of naming -# Keep it for backward compatibility -include_path = IncludePath -include_docker_images = IncludeDockerImages -exclude_docker_images = ExcludeDockerImages -include_filename = IncludeFilename -exclude_path = ExcludePath -exclude_filename = ExcludeFilename + operator = "$nmatch" + boolean_operator = "$and" diff --git a/artifactory_cleanup/rules/keep.py b/artifactory_cleanup/rules/keep.py index c4205e3..b6bdd61 100644 --- a/artifactory_cleanup/rules/keep.py +++ b/artifactory_cleanup/rules/keep.py @@ -2,23 +2,24 @@ from collections import defaultdict from itertools import groupby + from artifactory_cleanup.rules.base import Rule class KeepLatestNupkgNVersions(Rule): r"""Leaves ``count`` nupkg (adds * .nupkg filter) in release \ feature builds""" - def __init__(self, count): + def __init__(self, count: int): self.count = count - def _filter_result(self, result_artifact): + def filter(self, artifacts): artifact_grouped = defaultdict(lambda: defaultdict(lambda: defaultdict(list))) # Groupby: # - Nuget package name # - Nuget Feature # - Nuget MajorMinor version - for artifact in result_artifact: + for artifact in artifacts: if not artifact["name"].endswith(".nupkg"): continue @@ -37,11 +38,9 @@ def _filter_result(self, result_artifact): artifact_grouped = self.good_artifacts(artifact_grouped) - result_artifact = self.remove_founded_artifacts( - artifact_grouped, result_artifact - ) + artifacts = self.remove_founded_artifacts(artifact_grouped, artifacts) - return result_artifact + return artifacts def good_artifacts(self, artifact_grouped): for package, features in artifact_grouped.items(): @@ -65,7 +64,7 @@ def good_artifacts(self, artifact_grouped): ] return artifact_grouped - def remove_founded_artifacts(self, artifact_grouped, result_artifact): + def remove_founded_artifacts(self, artifact_grouped, artifacts): # Remove found artifact for package, features in artifact_grouped.items(): for feature, versions in features.items(): @@ -78,8 +77,8 @@ def remove_founded_artifacts(self, artifact_grouped, result_artifact): **locals() ) ) - result_artifact.remove(artifact) - return result_artifact + artifacts.keep(artifact) + return artifacts @staticmethod def keyfunc(s): @@ -97,41 +96,32 @@ def keyfunc(s): class KeepLatestNFiles(Rule): """Leaves the last (by creation time) files in the amount of N pieces. WITHOUT accounting subfolders""" - def __init__(self, count): + def __init__(self, count: int): self.count = count - def _aql_add_text(self, aql_text): - aql_text = "{}.sort({})".format(aql_text, r'{"$asc" : ["created"]}') - return aql_text - - def _filter_result(self, result_artifact): - artifact_count = len(result_artifact) + def filter(self, artifacts): + artifacts.sort(key=lambda x: x["created"], reverse=True) + artifact_count = len(artifacts) good_artifact_count = artifact_count - self.count if good_artifact_count < 0: good_artifact_count = 0 - good_artifacts = result_artifact[good_artifact_count:] - for artifact in good_artifacts: - print("Filter package {path}/{name}".format(**artifact)) - result_artifact.remove(artifact) - - return result_artifact + good_artifacts = artifacts[good_artifact_count:] + artifacts.keep(good_artifacts) + return artifacts class KeepLatestNFilesInFolder(Rule): """Leaves the last (by creation time) files in the number of ``count`` pieces in each folder""" - def __init__(self, count): + def __init__(self, count: int): self.count = count - def _aql_add_text(self, aql_text): - aql_text = "{}.sort({})".format(aql_text, r'{"$asc" : ["created"]}') - return aql_text - - def _filter_result(self, result_artifact): + def filter(self, artifacts): + artifacts.sort(key=lambda x: x["created"], reverse=True) artifacts_by_path = defaultdict(list) - for artifact in result_artifact: + for artifact in artifacts: path = artifact["path"] artifacts_by_path[path].append(artifact) @@ -142,11 +132,9 @@ def _filter_result(self, result_artifact): good_artifact_count = 0 good_artifacts = _artifacts[good_artifact_count:] - for artifact in good_artifacts: - print("Filter package {path}/{name}".format(**artifact)) - result_artifact.remove(artifact) + artifacts.keep(good_artifacts) - return result_artifact + return artifacts class KeepLatestVersionNFilesInFolder(Rule): @@ -159,10 +147,10 @@ def __init__(self, count, custom_regexp=r"[^\d][\._]((\d+\.)+\d+)"): self.count = count self.custom_regexp = custom_regexp - def _filter_result(self, result_artifact): + def filter(self, artifacts): artifacts_by_path_and_name = defaultdict(list) - for artifact in result_artifact[:]: + for artifact in artifacts[:]: path = artifact["path"] version = re.findall(self.custom_regexp, artifact["name"]) # save the version only if it was possible to uniquely determine it @@ -177,7 +165,7 @@ def _filter_result(self, result_artifact): key = path + "/" + name_without_version artifacts_by_path_and_name[key].append(artifactory_with_version) else: - self.remove_artifact(artifact, result_artifact) + artifacts.keep(artifact) for artifactory_with_version in artifacts_by_path_and_name.values(): artifactory_with_version.sort( @@ -190,15 +178,6 @@ def _filter_result(self, result_artifact): good_artifact_count = 0 good_artifacts = artifactory_with_version[good_artifact_count:] - for artifact in good_artifacts: - self.remove_artifact(artifact[1], result_artifact) - - return result_artifact - + artifacts.keep(good_artifacts) -# under_score - old style of naming -# Keep it for backward compatibility -keep_latest_nupkg_n_version = KeepLatestNupkgNVersions -keep_latest_n_file = KeepLatestNFiles -keep_latest_n_file_in_folder = KeepLatestNFilesInFolder -keep_latest_version_n_file_in_folder = KeepLatestVersionNFilesInFolder + return artifacts diff --git a/artifactory_cleanup/rules/repo.py b/artifactory_cleanup/rules/repo.py index d1ba171..7220d6b 100644 --- a/artifactory_cleanup/rules/repo.py +++ b/artifactory_cleanup/rules/repo.py @@ -2,55 +2,51 @@ from requests import HTTPError -from artifactory_cleanup.rules.base import Rule -from artifactory_cleanup.rules.exception import PolicyException +from artifactory_cleanup.errors import InvalidConfigError +from artifactory_cleanup.rules.base import Rule, ArtifactsList class Repo(Rule): """ - Apply the rule to one repository. - If no name is specified, it is taken from the rule name:: - - CleanupPolicy( - 'myrepo.snapshot', - # if the rule is one for all repositories - you can skip duplicate name - rules.repo, - ... - ), - + Apply the policy to one repository. """ + schema = [] + def __init__(self, name: str): bad_sym = set("*/[]") if set(name) & bad_sym: - raise PolicyException( + raise InvalidConfigError( "Bad name for repo: {name}, contains bad symbols: {bad_sym}\n" "Check that your have repo() correct".format( name=name, bad_sym="".join(bad_sym) ) ) - self.name = name + self.repo = name - def _aql_add_filter(self, aql_query_list): - print("Get from {}".format(self.name)) - request_url = "{}/api/storage/{}".format(self.artifactory_server, self.name) + def check(self, *args, **kwargs): + print(f"Checking '{self.repo}' repository exists.") try: - print("Checking the existence of the {} repository".format(self.name)) - r = self.artifactory_session.get(request_url) + url = f"/api/storage/{self.repo}" + r = self.session.get(url) r.raise_for_status() - print("The {} repository exists".format(self.name)) + print(f"The {self.repo} repository exists.") except HTTPError as e: - stderr.write("The {} repository does not exist".format(self.name)) + stderr.write(f"The {self.repo} repository does not exist!") print(e) exit(1) - update_dict = { + def aql_add_filter(self, filters): + filter_ = { "repo": { - "$eq": self.name, + "$eq": self.repo, } } - aql_query_list.append(update_dict) - return aql_query_list + filters.append(filter_) + return filters + + def filter(self, artifacts: ArtifactsList): + return artifacts class RepoByMask(Rule): @@ -58,18 +54,18 @@ class RepoByMask(Rule): Apply rule to repositories matching by mask """ - def __init__(self, mask): + def __init__(self, mask: str): self.mask = mask - def _aql_add_filter(self, aql_query_list): + def aql_add_filter(self, filters): print("Get from {}".format(self.mask)) - update_dict = { + filter_ = { "repo": { "$match": self.mask, } } - aql_query_list.append(update_dict) - return aql_query_list + filters.append(filter_) + return filters class PropertyEq(Rule): @@ -79,15 +75,15 @@ def __init__(self, property_key, property_value): self.property_key = property_key self.property_value = property_value - def _aql_add_filter(self, aql_query_list): - update_dict = { + def aql_add_filter(self, filters): + filter_ = { "$and": [ {"property.key": {"$eq": self.property_key}}, {"property.value": {"$eq": self.property_value}}, ] } - aql_query_list.append(update_dict) - return aql_query_list + filters.append(filter_) + return filters class PropertyNeq(Rule): @@ -104,19 +100,11 @@ def __init__(self, property_key, property_value): self.property_key = property_key self.property_value = str(property_value) - def _filter_result(self, result_artifact): + def filter(self, artifacts): good_artifact = [ x - for x in result_artifact + for x in artifacts if x["properties"].get(self.property_key) == self.property_value ] - self.remove_artifact(good_artifact, result_artifact) - return result_artifact - - -# under_score - old style of naming -# Keep it for backward compatibility -repo = Repo -repo_by_mask = RepoByMask -property_eq = PropertyEq -property_neq = PropertyNeq + artifacts.remove(good_artifact) + return artifacts diff --git a/artifactory_cleanup/rules/utils.py b/artifactory_cleanup/rules/utils.py index 037452a..d87b1e7 100644 --- a/artifactory_cleanup/rules/utils.py +++ b/artifactory_cleanup/rules/utils.py @@ -1,8 +1,10 @@ from collections import defaultdict -from typing import Dict, List, Tuple, Optional +from typing import Dict, List, Tuple, Optional, Union from treelib import Node, Tree +from artifactory_cleanup.rules.base import ArtifactsList + def is_repository(data): return data["path"] == "." and data["name"] == "." @@ -161,7 +163,7 @@ def build_repositories(artifacts: List[Dict]) -> List[RepositoryTree]: return list(repositories.values()) -def get_empty_folders(repositories: List[RepositoryTree]) -> List[Dict]: +def get_empty_folders(repositories: List[RepositoryTree]) -> ArtifactsList: folders = [] for repo in repositories: repo.count_files() @@ -170,9 +172,19 @@ def get_empty_folders(repositories: List[RepositoryTree]) -> List[Dict]: folders.extend(_folders) # Convert to raw data, similar to JSON Artifactory response - artifacts = [folder.get_raw_data() for folder in folders] + artifacts = ArtifactsList(folder.get_raw_data() for folder in folders) for data in artifacts: if is_repository(data): raise ValueError("Can not remove repository root") return artifacts + + +def to_masks(masks: Union[str, List[str]]): + """Ensure masks passed as string OR List""" + if isinstance(masks, str): + return [masks] + elif isinstance(masks, list): + return masks + else: + raise AttributeError("'masks' argument must by list of string OR string") diff --git a/docker/Dockerfile b/docker/Dockerfile index 96dba8d..34a801b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,8 +1,5 @@ FROM python:3.9.12-slim-buster -COPY . /app -WORKDIR /app - # set CERT paths for python libraries, necessary for self-signed certificates # - Requests Library # -> https://docs.python-requests.org/en/master/user/advanced/#ssl-cert-verification @@ -11,6 +8,8 @@ ENV REQUESTS_CA_BUNDLE /etc/ssl/certs/ca-certificates.crt # -> https://www.openssl.org/docs/man1.1.1/man3/SSL_CTX_set_default_verify_paths.html ENV SSL_CERT_FILE /etc/ssl/certs/ca-certificates.crt -RUN pip install . +COPY . /src +RUN pip install /src && rm -rf /src +WORKDIR /app CMD ["bash", "/app/docker/run.sh"] diff --git a/docker/run.sh b/docker/run.sh index 1cb7d8b..fda72bc 100755 --- a/docker/run.sh +++ b/docker/run.sh @@ -1,25 +1,4 @@ #!/bin/bash - -if [[ -z "$ARTIFACTORY_USER" ]];then - echo "mandatory ARTIFACTORY_USER environment variable not set!" - exit 3 -fi -if [[ -z "$ARTIFACTORY_URL" ]];then - echo "mandatory ARTIFACTORY_URL environment variable not set!" - exit 3 -fi -if [[ -z "$ARTIFACTORY_PASSWORD" ]];then - echo "mandatory ARTIFACTORY_PASSWORD environment variable not set!" - exit 3 -fi -if [[ -z "$ARTIFACTORY_RULES_CONFIG" ]];then - echo "mandatory ARTIFACTORY_RULES_CONFIG environment variable not set!" - exit 3 -fi - -# check if /tmp/rules.py exists -[ ! -f "$ARTIFACTORY_RULES_CONFIG" ] && echo "$ARTIFACTORY_RULES_CONFIG not found" && exit 3 - # install/trust self-signed certificates for Artifactory instances # with self-signed CA # further reading: https://askubuntu.com/a/649463 @@ -29,14 +8,5 @@ if (( ${#self_signed_certificates} )); then update-ca-certificates fi -# move to rules config parent directory -cd $( dirname $ARTIFACTORY_RULES_CONFIG) - -DESTROY="" -if [[ -v ARTIFACTORY_DESTROY_MODE_ENABLED ]]; then - DESTROY="--destroy" -fi - -# execute artifactory cleanup -echo "artifactory-cleanup $DESTROY --user $ARTIFACTORY_USER --password $ARTIFACTORY_PASSWORD --artifactory-server $ARTIFACTORY_URL --config $( basename $ARTIFACTORY_RULES_CONFIG)" -artifactory-cleanup $DESTROY --user $ARTIFACTORY_USER --password $ARTIFACTORY_PASSWORD --artifactory-server $ARTIFACTORY_URL --config $( basename $ARTIFACTORY_RULES_CONFIG) \ No newline at end of file +echo "Command to execute: $*" +exec "$@" diff --git a/docs/RULES/README.md b/docs/RULES/README.md deleted file mode 100644 index 41de99d..0000000 --- a/docs/RULES/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# Available Rules - -## common - rules that apply to all repositories - -| Name | Description | -| --- | --- | -| `DeleteOlderThan(days=N)` | Deletes artifacts that are older than N days | -| `DeleteWithoutDownloads()` | Deletes artifacts that have never been downloaded (DownloadCount=0). Better to use with `DeleteOlderThan` rule | -| `DeleteOlderThanNDaysWithoutDownloads(days=N)` | Deletes artifacts that are older than N days and have not been downloaded | -| `DeleteNotUsedSince(days=N)` | Delete artifacts that were downloaded, but for a long time. N days passed. Or not downloaded at all from the moment of creation and it's been N days | -| `DeleteEmptyFolder()` | Clean up empty folders in given repository list | -| `KeepLatestNupkgNVersions(count=N)` | Leaves N nupkg (adds `*.nupkg` filter) in release feature builds | -| `KeepLatestNFiles(count=N)` | Leaves the last (by creation time) files in the amount of N pieces. WITHOUT accounting subfolders | -| `KeepLatestNFilesInFolder(count=N)` | Leaves the last (by creation time) files in the number of N pieces in each folder | -| `KeepLatestVersionNFilesInFolder(count, custom_regexp='some-regexp')` | Leaves the latest N (by version) files in each folder. The definition of the version is using regexp. By default `[^\d][\._]((\d+\.)+\d+)` | -| `Repo('reponame')` | Apply the rule to one repository. If no name is specified, it is taken from the rule name (in `CleanupPolicy` definition) | -| `RepoByMask('*.banned')` | Apply rule to repositories matching by mask | -| `PropertyEq(property_key, property_value)`| Delete repository artifacts only with a specific property value (property_name is the name of the parameter, property_value is the value) | -| `PropertyNeq(property_key, property_value)`| Delete repository artifacts only if the value != specified. If there is no value, delete it anyway. Allows you to specify the deletion flag `do_not_delete = 1`| - -## docker - cleanup rules for docker images - -| Name | Description | -| --- | --- | -| `DeleteDockerImagesOlderThan(days=N)` | Delete docker images that are older than N days | -| `DeleteDockerImagesOlderThanNDaysWithoutDownloads(days=N)` | Deletes docker images that are older than N days and have not been downloaded | -| `DeleteDockerImagesNotUsed(days=N)` | Removes Docker image not downloaded since N days | -| `DeleteDockerImageIfNotContainedInProperties(docker_repo='docker-local', properties_prefix='my-prop', image_prefix=None, full_docker_repo_name=None)` | Remove Docker image, if it is not found in the properties of the artifact repository. | -| `DeleteDockerImageIfNotContainedInPropertiesValue(docker_repo='docker-local', properties_prefix='my-prop', image_prefix=None, full_docker_repo_name=None)` | Remove Docker image, if it is not found in the properties of the artifact repository. | -| `KeepLatestNVersionImagesByProperty(count=N, custom_regexp='some-regexp', number_of_digits_in_version=X)` | Leaves N Docker images with the same major. `(^ \d*\.\d*\.\d*.\d+$)` is the default regexp how to determine version. If you need to add minor then put 2 or if patch then put 3 (By default `1`) | - - -## filters - rules with different filters - -| Name | Description | -| --- | --- | -| `IncludePath('my-path/**')` | Apply to artifacts by path / mask. You can specify multiple paths: `IncludePath('*production*'), IncludePath(['*release*', '*master*'])` | -| `IncludeFilename('*.zip')` | Apply to artifacts by name/mask. You can specify multiple paths: `IncludeFilename('*-*'), IncludeFilename(['*tar.gz', '*.nupkg'])` | -| `IncludeDockerImages('*:latest*')` | Apply to docker images with the specified names and tags. You can specify multiple names and tags: `IncludeDockerImages('*:production*'), IncludeDockerImages(['ubuntu:*', 'debian:9'])` | -| `ExcludePath('my-path/**')` | Exclude artifacts by path/mask. You can specify multiple paths: `ExcludePath('*production*'), ExcludePath(['*release*', '*master*'])` | -| `ExcludeFilename('*.backup')` | Exclude artifacts by name/mask. You can specify multiple paths: `ExcludeFilename('*-*'), ExcludeFilename(['*tar.gz', '*.nupkg'])` | -| `ExcludeDockerImages('*:tag-*')` | Exclude Docker images by name and tags. You can specify multiple names and tags: `ExcludePath('*:production*'), ExcludePath(['ubuntu:*', 'debian:9'])` | diff --git a/examples/artifactory-cleanup.yaml b/examples/artifactory-cleanup.yaml index 70783d0..4bba8ab 100644 --- a/examples/artifactory-cleanup.yaml +++ b/examples/artifactory-cleanup.yaml @@ -1,29 +1,20 @@ -# Proposal: https://github.com/devopshq/artifactory-cleanup/issues/54 -# Please leave a comment if you'd enjoy using this format! - artifactory-cleanup: server: https://repo.example.com/artifactory # $VAR is auto populated from environment variables user: $ARTIFACTORY_USERNAME - password: $ARTIFACTORY_USERNAME + password: $ARTIFACTORY_PASSWORD policies: - - name: Remove all .tmp files older than 7 days - rules: - - RepoByMask: "*.tmp" - - DeleteOlderThan: 7 - - - name: My Docker Cleanup Policies + - name: Remove all files from repo-name-here older then 7 days rules: - - RepoByMask: "docker-*-tmp" - - DeleteOlderThan: - days: 7 + - rule: Repo + name: "repo-name-here" + - rule: DeleteOlderThan + days: 7 - - name: My Docker Cleanup Policies + - name: Use your own rules! rules: - - RepoByMask: "docker-*-tmp" - - ExcludeDockerImages: - - '*:latest' - - '*:release*' - - DeleteOlderThan: - days: 7 + - rule: Repo + name: "repo-name-here" + - rule: MySimpleRule + my_param: "Hello, world!" diff --git a/examples/example-policies.py b/examples/example-policies.py deleted file mode 100644 index ae38bee..0000000 --- a/examples/example-policies.py +++ /dev/null @@ -1,19 +0,0 @@ -from artifactory_cleanup import rules -from artifactory_cleanup.rules import CleanupPolicy - -RULES = [ - CleanupPolicy( - "Remove all files from *.tmp repositories older then 7 days", - rules.RepoByMask("*.tmp"), - rules.DeleteOlderThan(days=7), - ), - CleanupPolicy( - "docker-tmp", - rules.RepoByMask("docker*-tmp"), - rules.DeleteDockerImagesOlderThan(days=1), - ), - CleanupPolicy( - "reponame.snapshot", - rules.DeleteOlderThan(days=7), - ), -] diff --git a/examples/myrule.py b/examples/myrule.py new file mode 100644 index 0000000..7431265 --- /dev/null +++ b/examples/myrule.py @@ -0,0 +1,27 @@ +""" +Simple example to show how we can create our own rules in python +""" +from typing import List + +from artifactory_cleanup import register +from artifactory_cleanup.rules import Rule, ArtifactsList + + +class MySimpleRule(Rule): + """For more methods look at Rule source code""" + + def __init__(self, my_param): + self.my_param = my_param + + def aql_add_filter(self, filters: List) -> List: + print(self.my_param) + return filters + + def filter(self, artifacts: ArtifactsList) -> ArtifactsList: + """I'm here just to print the list""" + print(self.my_param) + return artifacts + + +# Register your rule in the system +register(MySimpleRule) diff --git a/setup.py b/setup.py index 0e386a1..93697a1 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ setup( name="artifactory-cleanup", - version="0.4.2", + version="1.0.0", description="Rules and cleanup policies for Artifactory", long_description=long_description, long_description_content_type="text/markdown", @@ -35,6 +35,10 @@ "teamcity-messages", "treelib", "attrs", + "pydash", + "pyyaml", + "cfgv~=3.3", + 'typing-extensions; python_version < "3.8.0"', ], classifiers=[ "Development Status :: 5 - Production/Stable", diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py index ed69f13..480d83c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,11 +9,11 @@ def requests_mock_all(requests_mock): """ -@pytest.fixture() -def requests_repo_name_here(requests_mock): - requests_mock.get("http://example.com/api/storage/repo-name-here") - requests_mock.post( - "http://example.com/api/search/aql", +def attach_requests_mock_to(mock, server): + server = server.rstrip("/") + mock.get(f"{server}/api/storage/repo-name-here") + mock.post( + f"{server}/api/search/aql", json={ "results": [ { @@ -43,7 +43,12 @@ def requests_repo_name_here(requests_mock): "range": {"start_pos": 0, "end_pos": 12, "total": 12}, }, ) - requests_mock.delete( - "http://example.com/repo-name-here/path/to/file/filename1.json" - ) + mock.delete(f"{server}/repo-name-here/path/to/file/filename1.json") + return mock + + +@pytest.fixture() +def requests_repo_name_here(requests_mock): + attach_requests_mock_to(requests_mock, "http://example.com/") + attach_requests_mock_to(requests_mock, "https://repo.example.com/artifactory") return requests_mock diff --git a/tests/data/__init__.py b/tests/data/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/data/all-built-in-rules.yaml b/tests/data/all-built-in-rules.yaml new file mode 100644 index 0000000..4caa706 --- /dev/null +++ b/tests/data/all-built-in-rules.yaml @@ -0,0 +1,96 @@ +artifactory-cleanup: + server: https://repo.example.com/artifactory + user: $ARTIFACTORY_USERNAME + password: $ARTIFACTORY_PASSWORD + + policies: + - name: repo-without-name + rules: + - rule: Repo + + - name: repo + rules: + - rule: Repo + name: "repo-name-here" + - rule: RepoByMask + mask: "*-tmp" + - rule: PropertyEq + property_key: key-name + property_value: 1 + - rule: PropertyNeq + property_key: key-name + property_value: 1 + + - name: delete + rules: + - rule: DeleteOlderThan + days: 1 + - rule: DeleteWithoutDownloads + - rule: DeleteOlderThanNDaysWithoutDownloads + days: 1 + - rule: DeleteNotUsedSince + days: 1 + - rule: DeleteEmptyFolders + + - name: keep + rules: + - rule: KeepLatestNupkgNVersions + count: 1 + - rule: KeepLatestNFiles + count: 1 + - rule: KeepLatestNFilesInFolder + count: 1 + - rule: KeepLatestVersionNFilesInFolder + count: 1 + custom_regexp: "[^\\d][\\._]((\\d+\\.)+\\d+)" + + - name: filters + rules: + - rule: IncludePath + masks: "*production*" + - rule: IncludePath + masks: + - "*production*" + - "*test*" + - rule: IncludeFilename + masks: "*-*" + - rule: IncludeFilename + masks: + - "*production*" + - "*release*" + - rule: ExcludePath + masks: "*singlemask*" + - rule: ExcludePath + masks: + - "*production*" + - "*release*" + - rule: ExcludeFilename + masks: "*filename*" + - rule: ExcludeFilename + masks: + - "*production*" + - "*release*" + + - name: docker + rules: + - rule: DeleteDockerImagesOlderThan + days: 1 + - rule: DeleteDockerImagesOlderThanNDaysWithoutDownloads + days: 1 + - rule: DeleteDockerImagesNotUsed + days: 1 + - rule: IncludeDockerImages + masks: "*singlemask*" + - rule: IncludeDockerImages + masks: + - "*production*" + - "*release*" + - rule: ExcludeDockerImages + masks: "*singlemask*" + - rule: ExcludeDockerImages + masks: + - "*production*" + - "*release*" + - rule: KeepLatestNVersionImagesByProperty + count: 1 + custom_regexp: "[^\\d][\\._]((\\d+\\.)+\\d+)" diff --git a/tests/data/cleanup.yaml b/tests/data/cleanup.yaml new file mode 100644 index 0000000..4bba8ab --- /dev/null +++ b/tests/data/cleanup.yaml @@ -0,0 +1,20 @@ +artifactory-cleanup: + server: https://repo.example.com/artifactory + # $VAR is auto populated from environment variables + user: $ARTIFACTORY_USERNAME + password: $ARTIFACTORY_PASSWORD + + policies: + - name: Remove all files from repo-name-here older then 7 days + rules: + - rule: Repo + name: "repo-name-here" + - rule: DeleteOlderThan + days: 7 + + - name: Use your own rules! + rules: + - rule: Repo + name: "repo-name-here" + - rule: MySimpleRule + my_param: "Hello, world!" diff --git a/tests/data/myrule.py b/tests/data/myrule.py new file mode 100644 index 0000000..7431265 --- /dev/null +++ b/tests/data/myrule.py @@ -0,0 +1,27 @@ +""" +Simple example to show how we can create our own rules in python +""" +from typing import List + +from artifactory_cleanup import register +from artifactory_cleanup.rules import Rule, ArtifactsList + + +class MySimpleRule(Rule): + """For more methods look at Rule source code""" + + def __init__(self, my_param): + self.my_param = my_param + + def aql_add_filter(self, filters: List) -> List: + print(self.my_param) + return filters + + def filter(self, artifacts: ArtifactsList) -> ArtifactsList: + """I'm here just to print the list""" + print(self.my_param) + return artifacts + + +# Register your rule in the system +register(MySimpleRule) diff --git a/tests/data/policies.py b/tests/data/policies.py deleted file mode 100644 index e257401..0000000 --- a/tests/data/policies.py +++ /dev/null @@ -1,9 +0,0 @@ -from artifactory_cleanup import CleanupPolicy, rules - -RULES = [ - CleanupPolicy( - "Remove all files from repo-name-here older then 7 days", - rules.Repo("repo-name-here"), - rules.DeleteOlderThan(days=7), - ), -] diff --git a/tests/test_cli.py b/tests/test_cli.py index 3a0b8b0..e91a137 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -19,25 +19,23 @@ def test_dry_mode(capsys, shared_datadir, requests_mock): _, code = ArtifactoryCleanupCLI.run( [ "ArtifactoryCleanupCLI", - "--user", - "user", - "--password", - "password", - "--artifactory-server", - "http://example.com/", "--config", - str(shared_datadir / "policies.py"), + str(shared_datadir / "cleanup.yaml"), + "--load-rules", + str(shared_datadir / "myrule.py"), ], exit=False, ) stdout, stderr = capsys.readouterr() assert code == 0, stdout assert "Verbose MODE" in stdout - assert "DEBUG - delete repo-name-here/path/to/file/filename1.json" in stdout + assert ( + "DEBUG - we would delete 'repo-name-here/path/to/file/filename1.json'" in stdout + ) assert ( - requests_mock.call_count == 2 - ), "Requests: check repository exists, AQL, NO DELETE" + requests_mock.call_count == 4 + ), "Requests: check repository exists, AQL, NO DELETE - 2 times" @pytest.mark.usefixtures("requests_repo_name_here") @@ -45,15 +43,11 @@ def test_destroy(capsys, shared_datadir, requests_mock): _, code = ArtifactoryCleanupCLI.run( [ "ArtifactoryCleanupCLI", - "--destroy", - "--user", - "user", - "--password", - "password", - "--artifactory-server", - "http://example.com/", "--config", - str(shared_datadir / "policies.py"), + str(shared_datadir / "cleanup.yaml"), + "--load-rules", + str(shared_datadir / "myrule.py"), + "--destroy", ], exit=False, ) @@ -62,11 +56,11 @@ def test_destroy(capsys, shared_datadir, requests_mock): assert "Destroy MODE" in stdout assert ( - requests_mock.call_count == 3 - ), "Requests: check repository exists, AQL, DELETE" + requests_mock.call_count == 6 + ), "Requests: check repository exists, AQL, DELETE - 2 times" last_request = requests_mock.last_request assert last_request.method == "DELETE" assert ( last_request.url - == "http://example.com/repo-name-here/path/to/file/filename1.json" + == "https://repo.example.com/artifactory/repo-name-here/path/to/file/filename1.json" ) diff --git a/tests/test_loaders.py b/tests/test_loaders.py new file mode 100644 index 0000000..d6d822b --- /dev/null +++ b/tests/test_loaders.py @@ -0,0 +1,19 @@ +from artifactory_cleanup.loaders import YamlConfigLoader + + +class TestYamlLoader: + def test_all_rules(self, shared_datadir): + loader = YamlConfigLoader(shared_datadir / "all-built-in-rules.yaml") + policies = loader.get_policies() + assert len(policies) == 6, "5 rules files + 1 special for repo-without-name" + + def test_load_env_variables(self, shared_datadir, monkeypatch): + monkeypatch.setenv("ARTIFACTORY_USERNAME", "UserName") + monkeypatch.setenv("ARTIFACTORY_PASSWORD", "P@ssw0rd") + + loader = YamlConfigLoader(shared_datadir / "all-built-in-rules.yaml") + server, user, password = loader.get_connection() + + assert server == "https://repo.example.com/artifactory" + assert user == "UserName" + assert password == "P@ssw0rd" diff --git a/tests/test_rules_delete.py b/tests/test_rules_delete.py index 9d653f2..d6cd431 100644 --- a/tests/test_rules_delete.py +++ b/tests/test_rules_delete.py @@ -2,7 +2,7 @@ import pytest -from artifactory_cleanup.rules import delete_empty_folder +from artifactory_cleanup.rules import DeleteEmptyFolders @pytest.fixture @@ -67,11 +67,11 @@ def artifacts_list(shared_datadir): return artifacts_list -def test_delete_empty_folder(artifacts_list): +def test_delete_empty_folders(artifacts_list): artifacts = artifacts_list - rule = delete_empty_folder() - artifacts_to_remove = rule.filter_result(artifacts) + rule = DeleteEmptyFolders() + artifacts_to_remove = rule.filter(artifacts) expected_empty_folders = [ # Simple empty folder without children in the list, at a deeper level diff --git a/tests/test_rules_docker.py b/tests/test_rules_docker.py new file mode 100644 index 0000000..b8315a7 --- /dev/null +++ b/tests/test_rules_docker.py @@ -0,0 +1,75 @@ +from artifactory_cleanup import CleanupPolicy +from artifactory_cleanup.rules import ( + KeepLatestNVersionImagesByProperty, + ArtifactsList, + RuleForDocker, + DeleteDockerImagesOlderThan, +) + + +class TestKeepLatestNVersionImagesByProperty: + def test_filter(self): + # Skip collecting docker size + RuleForDocker._collect_docker_size = lambda self, x: x + + data = [ + { + "properties": {"docker.manifest": "0.1.100"}, + "path": "foobar/0.1.100", + "name": "manifest.json", + }, + { + "properties": {"docker.manifest": "0.1.200"}, + "path": "foobar/0.1.200", + "name": "manifest.json", + }, + { + "properties": {"docker.manifest": "0.1.99"}, + "path": "foobar/0.1.99", + "name": "manifest.json", + }, + { + "properties": {"docker.manifest": "1.1.1"}, + "path": "foobar/1.1.1", + "name": "manifest.json", + }, + { + "properties": {"docker.manifest": "1.2.1"}, + "path": "foobar/1.2.1", + "name": "manifest.json", + }, + { + "properties": {"docker.manifest": "1.3.1"}, + "path": "foobar/1.3.1", + "name": "manifest.json", + }, + { + "properties": {"docker.manifest": "2.1.1"}, + "path": "foobar/2.1.1", + "name": "manifest.json", + }, + ] + artifacts = ArtifactsList.from_response(data) + policy = CleanupPolicy( + "test", + # DeleteDockerImagesOlderThan here just to test how KeepLatestNVersionImagesByProperty works together + DeleteDockerImagesOlderThan(days=1), + KeepLatestNVersionImagesByProperty( + count=2, + number_of_digits_in_version=1, + ), + ) + assert policy.filter(artifacts) == [ + { + "name": "0.1.99", + "path": "foobar", + "properties": {"docker.manifest": "0.1.99"}, + "stats": {}, + }, + { + "name": "1.1.1", + "path": "foobar", + "properties": {"docker.manifest": "1.1.1"}, + "stats": {}, + }, + ]