Add config option enable-rgw

.github/workflows/build-and-test.yml

@@ -112,6 +112,28 @@ jobs:
           ~/actionutils.sh wait_for_osds 3
           sudo microceph.ceph -s
 
+      - name: Enable RGW
+        run: |
+          set -eux
+          juju config microceph enable-rgw="*"
+          ~/actionutils.sh wait_for_rgw 1
+
+      - name: Collect logs
+        if: always()
+        run: ./tests/scripts/ci_helpers.sh collect_sunbeam_and_microceph_logs
+
+      - name: Upload logs
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: microceph_juju_functional_test_logs
+          path: logs
+          retention-days: 30
+
+      - name: Setup tmate session
+        if: ${{ failure() && runner.debug }}
+        uses: canonical/action-tmate@main
+
   juju-cluster-test:
     needs:
       - lint
@@ -182,6 +204,22 @@ jobs:
             exit 1
           fi
 
+      - name: Collect logs
+        if: always()
+        run: ./tests/scripts/ci_helpers.sh collect_microceph_logs
+
+      - name: Upload logs
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: microceph_juju_cluster_test_logs
+          path: logs
+          retention-days: 30
+
+      - name: Setup tmate session
+        if: ${{ failure() && runner.debug }}
+        uses: canonical/action-tmate@main
+
   juju-single-test:
     needs:
       - lint
@@ -364,6 +402,22 @@ jobs:
           juju status
           juju status blocked | egrep '^microceph/.*Cannot upgrade.*to quincy/stable'
 
+      - name: Collect logs
+        if: always()
+        run: ./tests/scripts/ci_helpers.sh collect_microceph_logs
+
+      - name: Upload logs
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: microceph_juju_upgrade_test_logs
+          path: logs
+          retention-days: 30
+
+      - name: Setup tmate session
+        if: ${{ failure() && runner.debug }}
+        uses: canonical/action-tmate@main
+
   juju-network-spaces-test:
     needs:
       - lint

config.yaml

@@ -14,3 +14,10 @@ options:
       The default replication factor for pools. Note that changing
       this value only sets the default value; it doesn't change
       the replication factor for existing pools.
+  enable-rgw:
+    default: ""
+    type: string
+    description: |
+      Supported values for this option are
+      "*" - Enable RGW on all storage nodes
+      ""  - Disable RGW on all storage nodes

docs/how-to-guides/enable-rgw.md

@@ -0,0 +1,60 @@
+Enable/Disable Ceph RADOS Gateway by setting the config option `enable-rgw`.
+
+1. Enable RGW service
+
+    juju config microceph enable-rgw="*"
+
+2. Check microceph status
+
+    juju ssh microceph/leader sudo microceph status
+
+The output of the above command should list rgw as part of services
+running on each node.
+Sample output is:
+
+```
+MicroCeph deployment summary:
+- microceph2 (10.121.193.184)
+  Services: mds, mgr, mon, rgw, osd
+  Disks: 1
+- microceph3 (10.121.193.185)
+  Services: mds, mgr, mon, rgw, osd
+  Disks: 1
+- microceph4 (10.121.193.186)
+  Services: mds, mgr, mon, rgw, osd
+  Disks: 1
+```
+
+3. Run ceph cluster status to check if rgw daemon is running
+
+    juju ssh microceph/leader sudo microceph.ceph status
+
+The output of the above command should list rgw under services.
+Sample output is:
+
+```
+  cluster:
+    id:     edd914f5-fdf8-4b56-bdd7-95d6c5e10d81
+    health: HEALTH_OK
+
+  services:
+    mon: 3 daemons, quorum microceph2,microceph3,microceph4 (age 12m)
+    mgr: microceph2(active, since 13m), standbys: microceph3, microceph4
+    osd: 3 osds: 3 up (since 34s), 3 in (since 56s)
+    rgw: 3 daemons, quorum microceph2,microceph3,microceph4 (age 30s)
+
+  data:
+    pools:   5 pools, 5 pgs
+    objects: 2 objects, 577 KiB
+    usage:   66 MiB used, 30 GiB / 30 GiB avail
+    pgs:     5 active+clean
+
+  io:
+    client:   938 B/s rd, 43 KiB/s wr, 0 op/s rd, 1 op/s wr
+```
+
+Now the ceph cluster is healthy and ready to use.
+
+4. Disable RGW service
+
+    juju config microceph enable-rgw=""

pyproject.toml

@@ -27,7 +27,7 @@ exclude = [".git", "__pycache__", ".tox", "build", "dist", "*.egg_info", "venv"]
 select = ["E", "W", "F", "C", "N", "R", "D", "H"]
 # Ignore W503, E501 because using black creates errors with this
 # Ignore D107 Missing docstring in __init__
-ignore = ["W503", "E501", "D107"]
+ignore = ["W503", "E501", "D107", "N818"]
 # D100, D101, D102, D103: Ignore missing docstrings in tests
 per-file-ignores = ["tests/*:D100,D101,D102,D103,D104"]
 docstring-convention = "google"

requirements.txt

@@ -5,5 +5,10 @@ netifaces
 jsonschema
 tenacity
 jinja2
-requests
+requests<2.32 # https://github.com/psf/requests/issues/6707 (similar issue with http+unix)
 git+https://opendev.org/openstack/sunbeam-charms/#egg=ops-sunbeam&subdirectory=ops-sunbeam
+
+# Used for communication with snapd socket
+requests-unixsocket # Apache 2
+urllib3<2 # https://github.com/psf/requests/issues/6432
+

src/charm.py

@@ -35,6 +35,7 @@
 import cluster
 import microceph
 from ceph_broker import get_named_key
+from microceph_client import ClusterServiceUnavailableException
 from relation_handlers import (
     CephClientProviderHandler,
     CephMdsProviderHandler,
@@ -113,7 +114,16 @@ def configure_charm(self, event: ops.framework.EventBase) -> None:
         self.configure_ceph(event)
 
     def _on_config_changed(self, event: ops.framework.EventBase) -> None:
-        self.configure_charm(event)
+        with sunbeam_guard.guard(self, "Checking configs"):
+            if not self.is_valid_placement_directive(self.model.config.get("enable-rgw")):
+                raise sunbeam_guard.BlockedExceptionError("Improper value for config enable-rgw")
+
+            self.configure_charm(event)
+
+    def is_valid_placement_directive(self, directive: str) -> bool:
+        """Check if placement directive is valid or not."""
+        supported_directives = ["*", ""]
+        return directive in supported_directives
 
     def _set_pool_size_action(self, event: ops.framework.EventBase) -> None:
         """Set the size for one or more pools."""
@@ -242,6 +252,7 @@ def configure_app_leader(self, event: ops.framework.EventBase) -> None:
             self.peers.interface.state.joined = True
 
         self.set_leader_ready()
+        self.manage_rgw_service(event)
         snap_chan = self.model.config.get("snap-channel")
 
         if self.cluster_upgrades.upgrade_requested(snap_chan):
@@ -259,6 +270,9 @@ def configure_app_non_leader(self, event: ops.framework.EventBase) -> None:
         if isinstance(event, MicroClusterNodeAddedEvent):
             self.cluster_nodes.join_node_to_cluster(event)
 
+        if self.peers.interface.state.joined:
+            self.manage_rgw_service(event)
+
     def _get_space_subnet(self, space: str):
         """Get the first available subnet in the network space."""
         space_nets = self.model.get_binding(binding_key=space).network.interfaces
@@ -317,6 +331,23 @@ def bootstrap_cluster(self, event: ops.framework.EventBase) -> None:
             if error_already_exists not in e.stderr:
                 raise e
 
+    def manage_rgw_service(self, event: ops.framework.EventBase) -> None:
+        """Enable/Disable RGW service."""
+        try:
+            enabled = microceph.is_rgw_enabled(gethostname())
+            if self.model.config.get("enable-rgw") == "*":
+                if not enabled:
+                    microceph.enable_rgw()
+            else:
+                if enabled:
+                    microceph.disable_rgw()
+        except ClusterServiceUnavailableException as e:
+            logger.warning(str(e))
+            event.defer()
+        except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e:
+            logger.warning(e.stderr)
+            raise e
+
     def configure_ceph(self, event) -> None:
         """Configure Ceph."""
         if not self.ready_for_service():

src/microceph.py

@@ -24,6 +24,8 @@
 
 import requests
 
+from microceph_client import Client
+
 logger = logging.getLogger(__name__)
 
 MAJOR_VERSIONS = {
@@ -83,6 +85,20 @@ def is_cluster_member(hostname: str) -> bool:
             raise e
 
 
+def is_rgw_enabled(hostname: str) -> bool:
+    """Check if RGW service is enabled on host.
+
+    Raises ClusterServiceUnavailableException if cluster is not available.
+    """
+    client = Client.from_socket()
+    services = client.cluster.list_services()
+    for service in services:
+        if service["service"] == "rgw" and service["location"] == hostname:
+            return True
+
+    return False
+
+
 def bootstrap_cluster(micro_ip: str = None, public_net: str = None, cluster_net: str = None):
     """Bootstrap MicroCeph cluster."""
     cmd = ["microceph", "cluster", "bootstrap"]
@@ -109,6 +125,18 @@ def join_cluster(token: str, micro_ip: str = None, **kwargs):
     _run_cmd(cmd=cmd)
 
 
+def enable_rgw() -> None:
+    """Enable RGW service."""
+    cmd = ["microceph", "enable", "rgw"]
+    _run_cmd(cmd)
+
+
+def disable_rgw() -> None:
+    """Disable RGW service."""
+    cmd = ["microceph", "disable", "rgw"]
+    _run_cmd(cmd)
+
+
 # Disk CMDs and Helpers
 def add_osd_cmd(spec: str, wal_dev: str = None, db_dev: str = None) -> None:
     """Executes MicroCeph add osd cmd with provided spec."""