Cleaning enhancements (#552)

* Reordoring clean steps; adding flags to disable cleaning parts.
* Bump version to 0.14.19.

biggraphite/accessor.py

@@ -390,6 +390,8 @@ def clean(
         shard=0,
         nshards=1,
         callback_on_progress=None,
+        disable_clean_directories=False,
+        disable_clean_metrics=False,
     ):
         """Remove metrics older than @max_age, considered to have expired (not used anymore)."""
         self._check_connected()

biggraphite/cli/command.py

@@ -103,3 +103,11 @@ def add_sharding_arguments(parser):
     parser.add_argument("--end-key", help="End key.")
     parser.add_argument("--shard", help="Shard number.", type=int, default=0)
     parser.add_argument("--nshards", help="Number of shards.", type=int, default=1)
+
+def add_clean_arguments(parser):
+    """Add cleaning arguments to a parser.
+
+    clear() will use this arguments
+    """
+    parser.add_argument("--disable-clean-directories", help="Disable cleaning directories", action="store_true", default=False)
+    parser.add_argument("--disable-clean-metrics", help="Disable cleaning outdated metrics", action="store_true", default=False)

biggraphite/cli/command_clean.py

@@ -63,6 +63,7 @@ def add_arguments(self, parser):
             action="store",
         )
         command.add_sharding_arguments(parser)
+        command.add_clean_arguments(parser)
 
     def run(self, accessor, opts, on_progress=None):
         """Run some cleanups.
@@ -106,6 +107,8 @@ def _on_progress(done, total):
                 start_key=opts.start_key,
                 end_key=opts.end_key,
                 callback_on_progress=on_progress,
+                disable_clean_directories=opts.disable_clean_directories,
+                disable_clean_metrics=opts.disable_clean_metrics,
             )
 
         if opts.clean_corrupted:

biggraphite/drivers/cassandra.py

@@ -2443,6 +2443,8 @@ def clean(
         shard=1,
         nshards=0,
         callback_on_progress=None,
+        disable_clean_directories=False,
+        disable_clean_metrics=False,
     ):
         """See bg_accessor.Accessor.
 
@@ -2456,23 +2458,32 @@ def clean(
         CLEAN_CURRENT_STEP.set(1)
 
         first_exception = None
-        try:
-            self._clean_empty_dir(
-                start_key, end_key, shard, nshards, callback_on_progress
-            )
-        except Exception as e:
-            first_exception = e
-            log.exception("Failed to clean directories.")
+
+        # First, clean metrics...
+        if not disable_clean_metrics:
+            try:
+                self._clean_expired_metrics(
+                    max_age, start_key, end_key, shard, nshards, callback_on_progress
+                )
+            except Exception as e:
+                first_exception = e
+                log.exception("Failed to clean metrics.")
+        else:
+            log.info("Cleaning metrics was disabled")
 
         CLEAN_CURRENT_STEP.set(2)
 
-        try:
-            self._clean_expired_metrics(
-                max_age, start_key, end_key, shard, nshards, callback_on_progress
-            )
-        except Exception as e:
-            first_exception = e
-            log.exception("Failed to clean metrics.")
+        # Then, clean directories...
+        if not disable_clean_directories:
+            try:
+                self._clean_empty_dir(
+                    start_key, end_key, shard, nshards, callback_on_progress
+                )
+            except Exception as e:
+                first_exception = e
+                log.exception("Failed to clean directories.")
+        else:
+            log.info("Cleaning directories was disabled")
 
         CLEAN_CURRENT_STEP.set(0)
         CLEAN_CURRENT_OFFSET.set(0)

biggraphite/drivers/elasticsearch.py

@@ -842,6 +842,8 @@ def clean(
         shard=1,
         nshards=0,
         callback_on_progress=None,
+        disable_clean_directories=False,
+        disable_clean_metrics=False,
     ):
         """See bg_accessor.Accessor."""
         super(_ElasticSearchAccessor, self).clean(

biggraphite/drivers/hybrid.py

@@ -164,6 +164,8 @@ def clean(
         shard=1,
         nshards=0,
         callback_on_progress=None,
+        disable_clean_directories=False,
+        disable_clean_metrics=False,
     ):
         """See the real Accessor for a description."""
         super(HybridAccessor, self).clean(

biggraphite/drivers/memory.py

@@ -246,6 +246,8 @@ def map(
         nshards=1,
         errback=None,
         callback_on_progress=None,
+        disable_clean_directories=False,
+        disable_clean_metrics=False,
     ):
         """See bg_accessor.Accessor."""
         super(_MemoryAccessor, self).map(

setup.py

@@ -40,7 +40,7 @@ def _read_reqs(relpath):
 
 setuptools.setup(
     name="biggraphite",
-    version="0.14.17",
+    version="0.14.19",
     maintainer="Criteo Graphite Team",
     maintainer_email="github@criteo.com",
     description="Simple Scalable Time Series Database.",