From de976fe1719882c1fc13f02950e82b4d894276aa Mon Sep 17 00:00:00 2001 From: Andre Luis Anastacio Date: Wed, 23 Oct 2024 10:16:00 -0300 Subject: [PATCH] Move snapshot history expire table properties to constants (#1217) --- pyiceberg/cli/console.py | 42 ++++++++++++++++++++++++++++++++----- pyiceberg/table/__init__.py | 6 ++++++ 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/pyiceberg/cli/console.py b/pyiceberg/cli/console.py index d1833df081..82c27a256b 100644 --- a/pyiceberg/cli/console.py +++ b/pyiceberg/cli/console.py @@ -32,10 +32,34 @@ from pyiceberg.catalog import Catalog, load_catalog from pyiceberg.cli.output import ConsoleOutput, JsonOutput, Output from pyiceberg.exceptions import NoSuchNamespaceError, NoSuchPropertyException, NoSuchTableError +from pyiceberg.table import TableProperties from pyiceberg.table.refs import SnapshotRef +from pyiceberg.utils.deprecated import deprecated +from pyiceberg.utils.properties import property_as_int -DEFAULT_MIN_SNAPSHOTS_TO_KEEP = 1 -DEFAULT_MAX_SNAPSHOT_AGE_MS = 432000000 + +class DeprecatedConstants: + @property + @deprecated( + deprecated_in="0.8.0", + removed_in="0.9.0", + help_message="DEFAULT_MAX_SNAPSHOT_AGE_MS is deprecated. Use TableProperties.MAX_SNAPSHOT_AGE_MS_DEFAULT instead.", + ) + def DEFAULT_MAX_SNAPSHOT_AGE_MS(self) -> int: + return 432000000 + + @property + @deprecated( + deprecated_in="0.8.0", + removed_in="0.9.0", + help_message="DEFAULT_MIN_SNAPSHOTS_TO_KEEP is deprecated. Use TableProperties.MIN_SNAPSHOTS_TO_KEEP_DEFAULT instead.", + ) + def DEFAULT_MIN_SNAPSHOTS_TO_KEEP(self) -> int: + return 1 + + +DEFAULT_MIN_SNAPSHOTS_TO_KEEP = DeprecatedConstants().DEFAULT_MIN_SNAPSHOTS_TO_KEEP +DEFAULT_MAX_SNAPSHOT_AGE_MS = DeprecatedConstants().DEFAULT_MAX_SNAPSHOT_AGE_MS def catch_exception() -> Callable: # type: ignore @@ -435,13 +459,21 @@ def list_refs(ctx: Context, identifier: str, type: str, verbose: bool) -> None: def _retention_properties(ref: SnapshotRef, table_properties: Dict[str, str]) -> Dict[str, str]: retention_properties = {} if ref.snapshot_ref_type == "branch": - default_min_snapshots_to_keep = table_properties.get( - "history.expire.min-snapshots-to-keep", DEFAULT_MIN_SNAPSHOTS_TO_KEEP + default_min_snapshots_to_keep = property_as_int( + table_properties, + TableProperties.MIN_SNAPSHOTS_TO_KEEP, + TableProperties.MIN_SNAPSHOTS_TO_KEEP_DEFAULT, ) + + default_max_snapshot_age_ms = property_as_int( + table_properties, + TableProperties.MAX_SNAPSHOT_AGE_MS, + TableProperties.MAX_SNAPSHOT_AGE_MS_DEFAULT, + ) + retention_properties["min_snapshots_to_keep"] = ( str(ref.min_snapshots_to_keep) if ref.min_snapshots_to_keep else str(default_min_snapshots_to_keep) ) - default_max_snapshot_age_ms = table_properties.get("history.expire.max-snapshot-age-ms", DEFAULT_MAX_SNAPSHOT_AGE_MS) retention_properties["max_snapshot_age_ms"] = ( str(ref.max_snapshot_age_ms) if ref.max_snapshot_age_ms else str(default_max_snapshot_age_ms) ) diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 66b22a7a79..f94d9c8a14 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -206,6 +206,12 @@ class TableProperties: METADATA_PREVIOUS_VERSIONS_MAX = "write.metadata.previous-versions-max" METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT = 100 + MAX_SNAPSHOT_AGE_MS = "history.expire.max-snapshot-age-ms" + MAX_SNAPSHOT_AGE_MS_DEFAULT = 5 * 24 * 60 * 60 * 1000 # 5 days + + MIN_SNAPSHOTS_TO_KEEP = "history.expire.min-snapshots-to-keep" + MIN_SNAPSHOTS_TO_KEEP_DEFAULT = 1 + class Transaction: _table: Table