[DPE-4401] Add retention config option (#35)

* add ability to remove backups after a number of days

* handle retention config

* fix linting

* fix integration test

* fix retention in test

* delete-older-than-days as int + refactor

* fix tests

* blocked status if invalid value for retention

---------

Co-authored-by: Mykola Marzhan <mykola.marzhan@canonical.com>

README.md

@@ -50,6 +50,7 @@ To configure the S3 integrator charm, you may provide the following configuratio
 - storage-class:the storage class for objects uploaded to the object storage.
 - tls-ca-chain: the complete CA chain, which can be used for HTTPS validation.
 - s3-api-version: the S3 protocol specific API signature.
+- experimental-delete-older-than-days: the amount of day after which backups going to be deleted. EXPERIMENTAL option.
 
 The only mandatory fields for the integrator are access-key secret-key and bucket.
 

config.yaml

@@ -36,3 +36,13 @@ options:
     type: string
     description: S3 protocol specific API signature.
     default: ''
+  experimental-delete-older-than-days:
+    type: int
+    description: |
+      Full backups can be retained for a number of days. The removal of expired
+      backups happens imediatelly after finishing the first successful backup after
+      retention period.
+      When full backup expires, the all differential and incremental backups which
+      depends on this full backup also expires.
+      This option is EXPRERIMENTAL.
+      Allowed values are: from 1 to 9999999.

lib/charms/data_platform_libs/v0/s3.py

@@ -137,7 +137,7 @@ def _on_credential_gone(self, event: CredentialsGoneEvent):
 
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version
-LIBPATCH = 4
+LIBPATCH = 5
 
 logger = logging.getLogger(__name__)
 
@@ -212,7 +212,7 @@ class S3CredentialEvents(CharmEvents):
 class S3Provider(Object):
     """A provider handler for communicating S3 credentials to consumers."""
 
-    on = S3CredentialEvents()  # pyright: ignore [reportGeneralTypeIssues]
+    on = S3CredentialEvents()  # pyright: ignore [reportAssignmentType]
 
     def __init__(
         self,
@@ -481,6 +481,18 @@ def set_s3_api_version(self, relation_id: int, s3_api_version: str) -> None:
         """
         self.update_connection_info(relation_id, {"s3-api-version": s3_api_version})
 
+    def set_delete_older_than_days(self, relation_id: int, days: int) -> None:
+        """Sets the retention days for full backups in application databag.
+
+        This function writes in the application data bag, therefore,
+        only the leader unit can call it.
+
+        Args:
+            relation_id: the identifier for a particular relation.
+            days: the value.
+        """
+        self.update_connection_info(relation_id, {"delete-older-than-days": str(days)})
+
     def set_attributes(self, relation_id: int, attributes: List[str]) -> None:
         """Sets the connection attributes in application databag.
 
@@ -580,6 +592,17 @@ def s3_api_version(self) -> Optional[str]:
 
         return self.relation.data[self.relation.app].get("s3-api-version")
 
+    @property
+    def delete_older_than_days(self) -> Optional[int]:
+        """Returns the retention days for full backups."""
+        if not self.relation.app:
+            return None
+
+        days = self.relation.data[self.relation.app].get("delete-older-than-days")
+        if days is None:
+            return None
+        return int(days)
+
     @property
     def attributes(self) -> Optional[List[str]]:
         """Returns the attributes."""
@@ -613,7 +636,7 @@ class S3CredentialRequiresEvents(ObjectEvents):
 class S3Requirer(Object):
     """Requires-side of the s3 relation."""
 
-    on = S3CredentialRequiresEvents()  # pyright: ignore[reportGeneralTypeIssues]
+    on = S3CredentialRequiresEvents()  # pyright: ignore[reportAssignmentType]
 
     def __init__(
         self, charm: ops.charm.CharmBase, relation_name: str, bucket_name: Optional[str] = None

src/charm.py

@@ -19,7 +19,14 @@
 from ops.charm import ActionEvent, ConfigChangedEvent, RelationChangedEvent, StartEvent
 from ops.model import ActiveStatus, BlockedStatus
 
-from constants import KEYS_LIST, PEER, S3_LIST_OPTIONS, S3_MANDATORY_OPTIONS, S3_OPTIONS
+from constants import (
+    KEYS_LIST,
+    MAX_RETENTION_DAYS,
+    PEER,
+    S3_LIST_OPTIONS,
+    S3_MANDATORY_OPTIONS,
+    S3_OPTIONS,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -68,7 +75,7 @@ def _on_start(self, _: StartEvent) -> None:
         if missing_options:
             self.unit.status = ops.model.BlockedStatus(f"Missing parameters: {missing_options}")
 
-    def _on_config_changed(self, _: ConfigChangedEvent) -> None:
+    def _on_config_changed(self, _: ConfigChangedEvent) -> None:  # noqa: C901
         """Event handler for configuration changed events."""
         # Only execute in the unit leader
         if not self.unit.is_leader():
@@ -79,12 +86,32 @@ def _on_config_changed(self, _: ConfigChangedEvent) -> None:
 
         # iterate over the option and check for updates
         for option in S3_OPTIONS:
-            if option not in self.config:
-                logger.warning(f"Option {option} is not valid option!")
+            # experimental config options should be handled with the "experimental-" prefix
+            if option == "delete-older-than-days" and f"experimental-{option}" in self.config:
+                config_value = self.config[f"experimental-{option}"]
+                # check if new config value is inside allowed range
+                if config_value > 0 and config_value <= MAX_RETENTION_DAYS:
+                    update_config.update({option: str(config_value)})
+                    self.set_secret("app", option, str(config_value))
+                    self.unit.status = ActiveStatus()
+                else:
+                    logger.warning(
+                        "Invalid value %s for config '%s'",
+                        config_value,
+                        option,
+                    )
+                    self.unit.status = BlockedStatus(
+                        f"Option {option} value {config_value} outside allowed range [1, {MAX_RETENTION_DAYS}]."
+                    )
                 continue
-            # skip in case of empty config
-            if self.config[option] == "":
-                # reset previous value if present (e.g., juju model-config --reset PARAMETER)
+
+            # option possibly removed from the config
+            # (e.g. 'juju config --reset <option>' or 'juju config <option>=""')
+            if option not in self.config or self.config[option] == "":
+                if option in KEYS_LIST:
+                    logger.debug("Secret parameter %s not stored inside config.", option)
+                    continue
+                # reset previous config value if present
                 if self.get_secret("app", option) is not None:
                     self.set_secret("app", option, None)
                     update_config.update({option: ""})
@@ -103,8 +130,8 @@ def _on_config_changed(self, _: ConfigChangedEvent) -> None:
                 update_config.update({option: ca_chain})
                 self.set_secret("app", option, json.dumps(ca_chain))
             else:
-                update_config.update({option: self.config[option]})
-                self.set_secret("app", option, self.config[option])
+                update_config.update({option: str(self.config[option])})
+                self.set_secret("app", option, str(self.config[option]))
 
         if len(self.s3_provider.relations) > 0:
             for relation in self.s3_provider.relations:

src/constants.py

@@ -16,10 +16,12 @@
     "s3-api-version",
     "s3-uri-style",
     "tls-ca-chain",
+    "delete-older-than-days",
 ]
 S3_MANDATORY_OPTIONS = [
     "access-key",
     "secret-key",
 ]
 S3_LIST_OPTIONS = ["attributes", "tls-ca-chain"]
 KEYS_LIST = ["access-key", "secret-key"]
+MAX_RETENTION_DAYS = 9999999

tests/integration/application-charm/lib/charms/data_platform_libs/v0/s3.py

@@ -137,7 +137,7 @@ def _on_credential_gone(self, event: CredentialsGoneEvent):
 
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version
-LIBPATCH = 4
+LIBPATCH = 5
 
 logger = logging.getLogger(__name__)
 
@@ -212,7 +212,7 @@ class S3CredentialEvents(CharmEvents):
 class S3Provider(Object):
     """A provider handler for communicating S3 credentials to consumers."""
 
-    on = S3CredentialEvents()  # pyright: ignore [reportGeneralTypeIssues]
+    on = S3CredentialEvents()  # pyright: ignore [reportAssignmentType]
 
     def __init__(
         self,
@@ -481,6 +481,18 @@ def set_s3_api_version(self, relation_id: int, s3_api_version: str) -> None:
         """
         self.update_connection_info(relation_id, {"s3-api-version": s3_api_version})
 
+    def set_delete_older_than_days(self, relation_id: int, days: int) -> None:
+        """Sets the retention days for full backups in application databag.
+
+        This function writes in the application data bag, therefore,
+        only the leader unit can call it.
+
+        Args:
+            relation_id: the identifier for a particular relation.
+            days: the value.
+        """
+        self.update_connection_info(relation_id, {"delete-older-than-days": str(days)})
+
     def set_attributes(self, relation_id: int, attributes: List[str]) -> None:
         """Sets the connection attributes in application databag.
 
@@ -580,6 +592,17 @@ def s3_api_version(self) -> Optional[str]:
 
         return self.relation.data[self.relation.app].get("s3-api-version")
 
+    @property
+    def delete_older_than_days(self) -> Optional[int]:
+        """Returns the retention days for full backups."""
+        if not self.relation.app:
+            return None
+
+        days = self.relation.data[self.relation.app].get("delete-older-than-days")
+        if days is None:
+            return None
+        return int(days)
+
     @property
     def attributes(self) -> Optional[List[str]]:
         """Returns the attributes."""
@@ -613,7 +636,7 @@ class S3CredentialRequiresEvents(ObjectEvents):
 class S3Requirer(Object):
     """Requires-side of the s3 relation."""
 
-    on = S3CredentialRequiresEvents()  # pyright: ignore[reportGeneralTypeIssues]
+    on = S3CredentialRequiresEvents()  # pyright: ignore[reportAssignmentType]
 
     def __init__(
         self, charm: ops.charm.CharmBase, relation_name: str, bucket_name: Optional[str] = None

tests/integration/test_s3_charm.py

@@ -138,6 +138,7 @@ async def test_config_options(ops_test: OpsTest):
         "path": "/test/path_1/",
         "region": "us-east-2",
         "endpoint": "s3.amazonaws.com",
+        "experimental-delete-older-than-days": "7",
     }
     # apply new configuration options
     await ops_test.model.applications[S3_APP_NAME].set_config(configuration_parameters)
@@ -151,6 +152,7 @@ async def test_config_options(ops_test: OpsTest):
     # test the correctness of the configuration fields
     assert configured_options["storage-class"] == "cinder"
     assert configured_options["s3-api-version"] == "1.0"
+    assert configured_options["delete-older-than-days"] == "7"
     assert len(json.loads(configured_options["attributes"])) == 3
     assert len(json.loads(configured_options["tls-ca-chain"])) == 2
     assert configured_options["region"] == "us-east-2"
@@ -187,6 +189,7 @@ async def test_relation_creation(ops_test: OpsTest):
     assert application_data["secret-key"] == "new-test-secret-key"
     assert application_data["storage-class"] == "cinder"
     assert application_data["s3-api-version"] == "1.0"
+    assert application_data["delete-older-than-days"] == "7"
     assert len(json.loads(application_data["attributes"])) == 3
     assert len(json.loads(application_data["tls-ca-chain"])) == 2
     assert application_data["region"] == "us-east-2"
@@ -223,6 +226,7 @@ async def test_relation_creation(ops_test: OpsTest):
     assert application_data["secret-key"] == "new-test-secret-key"
     assert application_data["storage-class"] == "cinder"
     assert application_data["s3-api-version"] == "1.0"
+    assert application_data["delete-older-than-days"] == "7"
     assert len(json.loads(application_data["attributes"])) == 3
     assert len(json.loads(application_data["tls-ca-chain"])) == 2
     assert application_data["region"] == "us-east-2"