From f2de8085babbf151a0ae7e4ad8cea8f4c2706def Mon Sep 17 00:00:00 2001 From: Maciej Urbanski Date: Wed, 5 Jul 2023 11:03:36 +0200 Subject: [PATCH] add --lifecycleRule option with improved validation --- CHANGELOG.md | 4 ++ Dockerfile.template | 2 +- MANIFEST.in | 2 + README.md | 9 +++- b2/_cli/obj_loads.py | 57 ++++++++++++++++++++++++ b2/console_tool.py | 52 ++++++++++++++++----- requirements-full.txt | 1 + setup.py | 1 + test/integration/test_b2_command_line.py | 6 +-- test/unit/test_console_tool.py | 57 +++++++++++++++++++++++- 10 files changed, 173 insertions(+), 18 deletions(-) create mode 100644 b2/_cli/obj_loads.py create mode 100644 requirements-full.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ce391049..f00d8cba4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +* Add `--lifecycleRule` to `create-bucket` and `update-bucket` and deprecate `--lifecycleRules` argument +* Add extra dependencies for better UX, installable with `pip install b2[all]` + ### Infrastructure * Autocomplete integration tests will now work properly even if tested package has not been installed * Automatically set copyright date when generating the docs diff --git a/Dockerfile.template b/Dockerfile.template index a4900118b..4ad37a413 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -12,7 +12,7 @@ LABEL build-date-iso8601="${build_date}" WORKDIR ${homedir} COPY ${tar_path}/${tar_name} . -RUN ["pip", "install", "${tar_name}"] +RUN ["pip", "install", "${tar_name}[full]"] ENV PATH=${homedir}/.local/bin:$$PATH diff --git a/MANIFEST.in b/MANIFEST.in index a7da3fcb4..74581135d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,4 @@ include requirements.txt +include requirements-full.txt +include requirements-license.txt include LICENSE diff --git a/README.md b/README.md index 9bd5410ff..e85923530 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,14 @@ Stand-alone binaries are available for Linux and Windows; this is the most strai You can also install it in your Python environment ([virtualenv](https://pypi.org/project/virtualenv/) is recommended) from PyPI with: ```bash -pip install b2 +pip install b2[full] +``` + +The extra dependencies improve debugging experience and, potentially, performance of `b2` CLI, but are not strictly required. +You can install the `b2` without them: + +```bash + pip install b2 ``` ### Installing from source diff --git a/b2/_cli/obj_loads.py b/b2/_cli/obj_loads.py new file mode 100644 index 000000000..dfd12b0ec --- /dev/null +++ b/b2/_cli/obj_loads.py @@ -0,0 +1,57 @@ +###################################################################### +# +# File: b2/_cli/obj_loads.py +# +# Copyright 2023 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +from __future__ import annotations + +import argparse +import io +import json +from typing import TypeVar + +from b2sdk.v2 import get_b2sdk_doc_urls + +try: + import pydantic + from pydantic import TypeAdapter, ValidationError +except ImportError: + pydantic = None + + +def convert_error_to_human_readable(validation_exc: ValidationError) -> str: + buf = io.StringIO() + for error in validation_exc.errors(): + loc = '.'.join(str(loc) for loc in error['loc']) + buf.write(f' In field {loc!r} input was `{error["input"]!r}`, error: {error["msg"]}\n') + return buf.getvalue() + + +def describe_type(type_) -> str: + urls = get_b2sdk_doc_urls(type_) + if urls: + url_links = ', '.join(f'{name} <{url}>' for name, url in urls.items()) + return f'{type_.__name__} ({url_links})' + return type_.__name__ + + +T = TypeVar('T') + + +def validated_loads(data: str, expected_type: type[T] | None = None) -> T: + if expected_type is not None and pydantic is not None: + ta = TypeAdapter(expected_type) + try: + val = ta.validate_json(data) + except ValidationError as e: + errors = convert_error_to_human_readable(e) + raise argparse.ArgumentTypeError( + f'Invalid value inputted, expected {describe_type(expected_type)}, got {data!r}, more detail below:\n{errors}' + ) from e + else: + val = json.loads(data) + return val diff --git a/b2/console_tool.py b/b2/console_tool.py index 15d50425c..6aefd5ec0 100644 --- a/b2/console_tool.py +++ b/b2/console_tool.py @@ -42,6 +42,7 @@ import b2sdk import requests import rst2ansi +from b2sdk.raw_api import LifecycleRule from b2sdk.v2 import ( ALL_CAPABILITIES, B2_ACCOUNT_INFO_DEFAULT_FILE, @@ -115,6 +116,7 @@ B2_USER_AGENT_APPEND_ENV_VAR, CREATE_BUCKET_TYPES, ) +from b2._cli.obj_loads import validated_loads from b2._cli.shell import detect_shell from b2.arg_parser import ( ArgumentParser, @@ -513,6 +515,32 @@ def _get_upload_mode_from_args(args): return UploadMode.FULL +class LifecycleRulesMixin(Described): + """ + Use `--lifecycleRule` to set lifecycle rule for the bucket. + Multiple rules can be specified by repeating the option. + + `--lifecycleRules` option is deprecated and cannot be used together with --lifecycleRule. + """ + + @classmethod + def _setup_parser(cls, parser): + lifecycle_group = parser.add_mutually_exclusive_group() + lifecycle_group.add_argument( + '--lifecycleRule', + action='append', + default=[], + type=functools.partial(validated_loads, expected_type=LifecycleRule), + dest='lifecycleRules', + help="Lifecycle rule in JSON format. Can be supplied multiple times.", + ) + lifecycle_group.add_argument( + '--lifecycleRules', + type=functools.partial(validated_loads, expected_type=List[LifecycleRule]), + help="(deprecated; use --lifecycleRule instead) List of lifecycle rules in JSON format.", + ) + + class Command(Described): # Set to True for commands that receive sensitive information in arguments FORBID_LOGGING_ARGUMENTS = False @@ -1022,7 +1050,7 @@ def _determine_source_metadata( @B2.register_subcommand -class CreateBucket(DefaultSseMixin, Command): +class CreateBucket(DefaultSseMixin, LifecycleRulesMixin, Command): """ Creates a new bucket. Prints the ID of the bucket created. @@ -1030,6 +1058,7 @@ class CreateBucket(DefaultSseMixin, Command): These can be given as JSON on the command line. {DEFAULTSSEMIXIN} + {LIFECYCLERULESMIXIN} Requires capability: @@ -1041,16 +1070,15 @@ class CreateBucket(DefaultSseMixin, Command): @classmethod def _setup_parser(cls, parser): - parser.add_argument('--bucketInfo', type=json.loads) - parser.add_argument('--corsRules', type=json.loads) - parser.add_argument('--lifecycleRules', type=json.loads) + parser.add_argument('--bucketInfo', type=validated_loads) + parser.add_argument('--corsRules', type=validated_loads) parser.add_argument( '--fileLockEnabled', action='store_true', help= "If given, the bucket will have the file lock mechanism enabled. This parameter cannot be changed after bucket creation." ) - parser.add_argument('--replication', type=json.loads) + parser.add_argument('--replication', type=validated_loads) parser.add_argument('bucketName') parser.add_argument('bucketType', choices=CREATE_BUCKET_TYPES) @@ -2495,7 +2523,7 @@ def get_synchronizer_from_args( @B2.register_subcommand -class UpdateBucket(DefaultSseMixin, Command): +class UpdateBucket(DefaultSseMixin, LifecycleRulesMixin, Command): """ Updates the ``bucketType`` of an existing bucket. Prints the ID of the bucket updated. @@ -2504,6 +2532,7 @@ class UpdateBucket(DefaultSseMixin, Command): These can be given as JSON on the command line. {DEFAULTSSEMIXIN} + {LIFECYCLERULESMIXIN} To set a default retention for files in the bucket ``--defaultRetentionMode`` and ``--defaultRetentionPeriod`` have to be specified. The latter one is of the form "X days|years". @@ -2537,9 +2566,8 @@ class UpdateBucket(DefaultSseMixin, Command): @classmethod def _setup_parser(cls, parser): - parser.add_argument('--bucketInfo', type=json.loads) - parser.add_argument('--corsRules', type=json.loads) - parser.add_argument('--lifecycleRules', type=json.loads) + parser.add_argument('--bucketInfo', type=validated_loads) + parser.add_argument('--corsRules', type=validated_loads) parser.add_argument( '--defaultRetentionMode', choices=( @@ -2554,7 +2582,7 @@ def _setup_parser(cls, parser): type=parse_default_retention_period, metavar='period', ) - parser.add_argument('--replication', type=json.loads) + parser.add_argument('--replication', type=validated_loads) parser.add_argument( '--fileLockEnabled', action='store_true', @@ -3279,7 +3307,7 @@ def _get_licenses_dicts(cls) -> List[Dict]: ] ) licenses_output = piplicenses.create_output_string(args) - licenses = json.loads(licenses_output) + licenses = validated_loads(licenses_output) return licenses def _fetch_license_from_url(self, url: str) -> str: @@ -3364,7 +3392,7 @@ def __init__(self, b2_api: Optional[B2Api], stdout, stderr): def run_command(self, argv): signal.signal(signal.SIGINT, keyboard_interrupt_handler) parser = B2.get_parser() - argcomplete.autocomplete(parser) + argcomplete.autocomplete(parser, default_completer=None) args = parser.parse_args(argv[1:]) self._setup_logging(args, argv) diff --git a/requirements-full.txt b/requirements-full.txt new file mode 100644 index 000000000..593a887c8 --- /dev/null +++ b/requirements-full.txt @@ -0,0 +1 @@ +pydantic>=2.0.1 \ No newline at end of file diff --git a/setup.py b/setup.py index 899d1a179..ab03fb9f3 100644 --- a/setup.py +++ b/setup.py @@ -109,6 +109,7 @@ def read_requirements(extra=None): # for example: # $ pip install -e .[dev,test] extras_require={ + 'full': read_requirements('full'), 'doc': read_requirements('doc'), 'license': read_requirements('license'), }, diff --git a/test/integration/test_b2_command_line.py b/test/integration/test_b2_command_line.py index f604c8749..9302b48fe 100644 --- a/test/integration/test_b2_command_line.py +++ b/test/integration/test_b2_command_line.py @@ -244,13 +244,13 @@ def test_basic(b2_tool, bucket_name): def test_bucket(b2_tool, bucket_name): - rules = """[{ + rule = """{ "daysFromHidingToDeleting": 1, "daysFromUploadingToHiding": null, "fileNamePrefix": "" - }]""" + }""" output = b2_tool.should_succeed_json( - ['update-bucket', '--lifecycleRules', rules, bucket_name, 'allPublic', *get_bucketinfo()], + ['update-bucket', '--lifecycleRule', rule, bucket_name, 'allPublic', *get_bucketinfo()], ) assert output["lifecycleRules"] == [ { diff --git a/test/unit/test_console_tool.py b/test/unit/test_console_tool.py index 7907c9225..db25a2192 100644 --- a/test/unit/test_console_tool.py +++ b/test/unit/test_console_tool.py @@ -188,7 +188,10 @@ def _run_command( expected_stderr = self._normalize_expected_output(expected_stderr, format_vars) stdout, stderr = self._get_stdouterr() console_tool = ConsoleTool(self.b2_api, stdout, stderr) - actual_status = console_tool.run_command(['b2'] + argv) + try: + actual_status = console_tool.run_command(['b2'] + argv) + except SystemExit as e: + actual_status = e.code actual_stdout = self._trim_trailing_spaces(stdout.getvalue()) actual_stderr = self._trim_trailing_spaces(stderr.getvalue()) @@ -448,6 +451,58 @@ def test_authorize_key_without_list_buckets(self): 1, ) + def test_create_bucket__with_lifecycle_rule(self): + self._authorize_account() + + rule = json.dumps( + { + "daysFromHidingToDeleting": 1, + "daysFromUploadingToHiding": None, + "fileNamePrefix": "" + } + ) + + self._run_command( + ['create-bucket', 'my-bucket', 'allPrivate', '--lifecycleRule', rule], 'bucket_0\n', '', + 0 + ) + + def test_create_bucket__with_lifecycle_rules(self): + self._authorize_account() + + rules = json.dumps( + [ + { + "daysFromHidingToDeleting": 1, + "daysFromUploadingToHiding": None, + "fileNamePrefix": "" + } + ] + ) + + self._run_command( + ['create-bucket', 'my-bucket', 'allPrivate', '--lifecycleRules', rules], 'bucket_0\n', + '', 0 + ) + + def test_create_bucket__mutually_exclusive_lifecycle_rules_options(self): + self._authorize_account() + + rule = json.dumps( + { + "daysFromHidingToDeleting": 1, + "daysFromUploadingToHiding": None, + "fileNamePrefix": "" + } + ) + + self._run_command( + [ + 'create-bucket', 'my-bucket', 'allPrivate', '--lifecycleRule', rule, + '--lifecycleRules', f"[{rule}]" + ], '', '', 2 + ) + def test_create_bucket_key_and_authorize_with_it(self): # Start with authorizing with the master key self._authorize_account()