[DPE-5227][DPE-5228]: Enable Storage reuse test (#468)

## Issue

* This test is disabled and also needs some fixes
* There is no test for storage reuse on a new cluster

## Solution
* Enable the test and fix the small bugs
* Develop the Storage reuse on different cluster test, and skip it as it's not a supported feature for now.

lib/charms/mongodb/v1/helpers.py

@@ -8,7 +8,7 @@
 import secrets
 import string
 import subprocess
-from typing import List
+from typing import List, Mapping
 
 from charms.mongodb.v1.mongodb import MongoConfiguration
 from ops.model import ActiveStatus, MaintenanceStatus, StatusBase, WaitingStatus
@@ -23,7 +23,7 @@
 
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version
-LIBPATCH = 8
+LIBPATCH = 9
 
 # path to store mongodb ketFile
 KEY_FILE = "keyFile"
@@ -320,3 +320,25 @@ def add_args_to_env(var: str, args: str):
 
     with open(Config.ENV_VAR_PATH, "w") as service_file:
         service_file.writelines(env_vars)
+
+
+def safe_exec(
+    command: list[str] | str,
+    env: Mapping[str, str] | None = None,
+    working_dir: str | None = None,
+) -> str:
+    """Execs a command on the workload in a safe way."""
+    try:
+        output = subprocess.check_output(
+            command,
+            stderr=subprocess.PIPE,
+            universal_newlines=True,
+            shell=isinstance(command, str),
+            env=env,
+            cwd=working_dir,
+        )
+        logger.debug(f"{output=}")
+        return output
+    except subprocess.CalledProcessError as err:
+        logger.error(f"cmd failed - {err.cmd = }, {err.stdout = }, {err.stderr = }")
+        raise

src/charm.py

@@ -26,6 +26,7 @@
     generate_keyfile,
     generate_password,
     get_create_user_cmd,
+    safe_exec,
 )
 from charms.mongodb.v1.mongodb import MongoDBConnection, NotReadyError
 from charms.mongodb.v1.mongodb_backups import MongoDBBackups
@@ -46,6 +47,7 @@
     NoVersionError,
     get_charm_revision,
 )
+from ops import StorageAttachedEvent
 from ops.charm import (
     ActionEvent,
     CharmBase,
@@ -108,6 +110,7 @@ def __init__(self, *args):
         self.framework.observe(self.on.install, self._on_install)
         self.framework.observe(self.on.start, self._on_start)
         self.framework.observe(self.on.update_status, self._on_update_status)
+        self.framework.observe(self.on.mongodb_storage_attached, self._on_storage_attached)
         self.framework.observe(
             self.on[Config.Relations.PEERS].relation_joined, self._on_relation_joined
         )
@@ -148,7 +151,7 @@ def __init__(self, *args):
             ],
         )
         self.upgrade = MongoDBUpgrade(self)
-        self.config_server = ShardingProvider(self, substrate="vm")
+        self.config_server = ShardingProvider(self, substrate=Config.SUBSTRATE)
         self.cluster = ClusterProvider(self)
         self.shard = ConfigServerRequirer(self)
         self.status = MongoDBStatusHandler(self)
@@ -588,6 +591,14 @@ def _on_relation_departed(self, event: RelationDepartedEvent) -> None:
 
         self._update_hosts(event)
 
+    def _on_storage_attached(self, event: StorageAttachedEvent) -> None:
+        """Handler for `storage_attached` event.
+
+        This should handle fixing the permissions for the data dir.
+        """
+        safe_exec(f"chmod -R 770 {Config.MONGODB_COMMON_PATH}".split())
+        safe_exec(f"chown -R {Config.SNAP_USER}:root  {Config.MONGODB_COMMON_PATH}".split())
+
     def _on_storage_detaching(self, event: StorageDetachingEvent) -> None:
         """Before storage detaches, allow removing unit to remove itself from the set.
 

src/config.py

@@ -3,6 +3,7 @@
 # Copyright 2024 Canonical Ltd.
 # See LICENSE file for licensing details.
 
+from pathlib import Path
 from typing import Literal, TypeAlias
 
 from ops.model import BlockedStatus
@@ -21,7 +22,13 @@ class Config:
     MONGOD_CONF_DIR = f"{MONGODB_SNAP_DATA_DIR}/etc/mongod"
     MONGOD_CONF_FILE_PATH = f"{MONGOD_CONF_DIR}/mongod.conf"
     CHARM_INTERNAL_VERSION_FILE = "charm_internal_version"
-    SNAP_PACKAGES = [("charmed-mongodb", "6/edge", 118)]
+    SNAP_PACKAGES = [("charmed-mongodb", "6/edge", 121)]
+
+    MONGODB_COMMON_PATH = Path("/var/snap/charmed-mongodb/common")
+
+    # This is the snap_daemon user, which does not exist on the VM before the
+    # snap install so creating it by UID
+    SNAP_USER = 584788
 
     # Keep these alphabetically sorted
     class Actions:

src/upgrades/upgrade.py

@@ -178,7 +178,7 @@ def app_status(self) -> typing.Optional[ops.StatusBase]:
             resume_string = ""
             if len(self._sorted_units) > 1:
                 resume_string = (
-                    "Verify highest unit is healthy & run `{RESUME_ACTION_NAME}` action. "
+                    f"Verify highest unit is healthy & run `{RESUME_ACTION_NAME}` action. "
                 )
             return ops.BlockedStatus(
                 f"Upgrading. {resume_string}To rollback, `juju refresh` to last revision"

tests/integration/ha_tests/helpers.py

@@ -1,10 +1,10 @@
 # Copyright 2024 Canonical Ltd.
 # See LICENSE file for licensing details.
 
+import calendar
 import json
 import logging
 import subprocess
-import time
 from datetime import datetime
 from pathlib import Path
 from subprocess import PIPE, check_output
@@ -400,34 +400,7 @@ def storage_id(ops_test, unit_name):
             return line.split()[1]
 
 
-async def add_unit_with_storage(ops_test, app_name, storage):
-    """Adds unit with storage.
-
-    Note: this function exists as a temporary solution until this issue is resolved:
-    https://github.com/juju/python-libjuju/issues/695
-    """
-    expected_units = len(ops_test.model.applications[app_name].units) + 1
-    prev_units = [unit.name for unit in ops_test.model.applications[app_name].units]
-    model_name = ops_test.model.info.name
-    add_unit_cmd = f"add-unit {app_name} --model={model_name} --attach-storage={storage}".split()
-    await ops_test.juju(*add_unit_cmd)
-    await ops_test.model.wait_for_idle(apps=[app_name], status="active", timeout=1000)
-    assert (
-        len(ops_test.model.applications[app_name].units) == expected_units
-    ), "New unit not added to model"
-
-    # verify storage attached
-    curr_units = [unit.name for unit in ops_test.model.applications[app_name].units]
-    new_unit = list(set(curr_units) - set(prev_units))[0]
-    assert storage_id(ops_test, new_unit) == storage, "unit added with incorrect storage"
-
-    # return a reference to newly added unit
-    for unit in ops_test.model.applications[app_name].units:
-        if unit.name == new_unit:
-            return unit
-
-
-async def reused_storage(ops_test: OpsTest, unit_name, removal_time) -> bool:
+async def reused_storage(ops_test: OpsTest, unit_name: str, removal_time: float) -> bool:
     """Returns True if storage provided to mongod has been reused.
 
     MongoDB startup message indicates storage reuse:
@@ -450,11 +423,17 @@ async def reused_storage(ops_test: OpsTest, unit_name, removal_time) -> bool:
 
         item = json.loads(line)
 
-        if "msg" not in item:
+        # "attr" is needed and stores the state information and changes of mongodb
+        if "attr" not in item:
             continue
 
+        # Compute reuse time
         re_use_time = convert_time(item["t"]["$date"])
-        if '"newState": "STARTUP2", "oldState": "REMOVED"' in line and re_use_time > removal_time:
+
+        # Get newstate and oldstate if present
+        newstate = item["attr"].get("newState", "")
+        oldstate = item["attr"].get("oldState", "")
+        if newstate == "STARTUP2" and oldstate == "REMOVED" and re_use_time > removal_time:
             return True
 
     return False
@@ -641,10 +620,10 @@ async def verify_replica_set_configuration(ops_test: OpsTest, app_name=None) ->
 
 
 def convert_time(time_as_str: str) -> int:
-    """Converts a string time representation to an integer time representation."""
+    """Converts a string time representation to an integer time representation, in UTC."""
     # parse time representation, provided in this format: 'YYYY-MM-DDTHH:MM:SS.MMM+00:00'
     d = datetime.strptime(time_as_str, "%Y-%m-%dT%H:%M:%S.%f%z")
-    return time.mktime(d.timetuple())
+    return calendar.timegm(d.timetuple())
 
 
 def cut_network_from_unit(machine_name: str) -> None: