Remove Junk metadata when script runs (#1083)

* Remove Junk metadata when script runs

* Apply code review suggestions

src/neptune/management/exceptions.py

@@ -163,3 +163,8 @@ class BadRequestException(ManagementOperationFailure):
 class IncorrectIdentifierException(ManagementOperationFailure):
     code = 21
     description = "Can not parse '{identifier}' as identifier."
+
+
+class ObjectNotFound(ManagementOperationFailure):
+    code = 22
+    description = "Object not found."

src/neptune/new/internal/backends/hosted_neptune_backend.py

@@ -41,6 +41,7 @@
     NeptuneException,
 )
 from neptune.common.patterns import PROJECT_QUALIFIED_NAME_PATTERN
+from neptune.management.exceptions import ObjectNotFound
 from neptune.new.envs import NEPTUNE_FETCH_TABLE_STEP_SIZE
 from neptune.new.exceptions import (
     AmbiguousProjectName,
@@ -303,7 +304,7 @@ def get_metadata_container(
                 )
 
             return ApiExperiment.from_experiment(experiment)
-        except HTTPNotFound:
+        except ObjectNotFound:
             raise MetadataContainerNotFound.of_container_type(
                 container_type=expected_container_type, container_id=container_id
             )

src/neptune/new/internal/backends/swagger_client_wrapper.py

@@ -33,6 +33,7 @@ class ApiMethodWrapper:
     PROJECT_NAME_COLLISION = "PROJECT_NAME_COLLISION"
     PROJECT_KEY_INVALID = "PROJECT_KEY_INVALID"
     PROJECT_NAME_INVALID = "PROJECT_NAME_INVALID"
+    EXPERIMENT_NOT_FOUND = "EXPERIMENT_NOT_FOUND"
 
     def __init__(self, api_method):
         self._api_method = api_method
@@ -78,6 +79,10 @@ def handle_neptune_http_errors(response, exception: Optional[HTTPError] = None):
             raise ProjectNameInvalid(
                 name=body.get("name", "<unknown name>"), reason=body.get("reason", "Unknown reason")
             ) from exception
+        elif error_type == ApiMethodWrapper.EXPERIMENT_NOT_FOUND:
+            from neptune.management.exceptions import ObjectNotFound
+
+            raise ObjectNotFound() from exception
         elif exception:
             raise exception
 

src/neptune/new/internal/disk_queue.py

@@ -16,6 +16,7 @@
 import json
 import logging
 import os
+import shutil
 import threading
 from glob import glob
 from pathlib import Path
@@ -152,6 +153,22 @@ def close(self):
         self._last_ack_file.close()
         self._last_put_file.close()
 
+        if self.is_empty():
+            self._remove_data()
+
+    def _remove_data(self):
+        path = self._dir_path
+        shutil.rmtree(path, ignore_errors=True)
+
+        parent = path.parent
+
+        files = os.listdir(parent)
+        if len(files) == 0:
+            try:
+                os.rmdir(parent)
+            except OSError:
+                _logger.info(f"Cannot remove directory: {parent}")
+
     def wait_for_empty(self, seconds: Optional[float] = None) -> bool:
         with self._empty_cond:
             return self._empty_cond.wait_for(self.is_empty, timeout=seconds)

tests/neptune/new/client/abstract_experiment_test_mixin.py

@@ -129,6 +129,17 @@ def test_wrong_per_type_function(self):
         with self.assertRaises(TypeDoesNotSupportAttributeException):
             exp["some/path"].download()
 
+    def test_clean_data_on_stop(self):
+        exp = self.call_init(mode="async", flush_period=0.5)
+        container_path = exp._op_processor._queue._dir_path
+
+        assert os.path.exists(container_path)
+
+        exp.stop()
+
+        assert not os.path.exists(container_path)
+        assert not os.path.exists(container_path.parent)
+
     @abstractmethod
     def test_read_only_mode(self):
         pass