Upgrade_watchdog (#401)

* bump all product versions to 2

* bump version

* fix version label

* pin common CONF version for end2end test

* ignore VERS_CFG in end2end testing

* add test for TM folder observing

* bump watchdog==6.0.0

* integrate review fixes

setup.cfg

@@ -28,7 +28,7 @@ install_requires =
     scp
     spiceypy
     sunpy>=4.0.1
-    watchdog
+    watchdog==6.0.0
     requests
 
 

stixcore/processing/tests/test_pipline.py

@@ -0,0 +1,62 @@
+import re
+import time
+import shutil
+
+import pytest
+from watchdog.observers import Observer
+
+from stixcore.processing.pipeline import GFTSFileHandler
+from stixcore.util.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+@pytest.fixture
+def out_dir(tmp_path):
+    return tmp_path
+
+
+def processing_tm_file(path, **args):
+    logger.info(f"start processing tm file: {path}")
+    time.sleep(2)
+    logger.info(f"stop processing tm file: {path}")
+
+
+@pytest.fixture
+def gfts_manager():
+    return GFTSFileHandler(processing_tm_file, re.compile(r'.*test_[0-9]*.tm$'), name="w-dog-test")
+
+
+def test_gfts_manager_watchdog(out_dir, gfts_manager):
+    (out_dir / "test_1.tm").touch()
+    (out_dir / "test_2.tm").touch()
+    (out_dir / "test_3.tm").touch()
+    (out_dir / "test_4.tmp").touch()
+    (out_dir / "test_5.tmp").touch()
+    (out_dir / "test_6.tmp").touch()
+
+    observer = Observer()
+    observer.schedule(gfts_manager, out_dir,  recursive=False)
+    observer.start()
+
+    assert gfts_manager.queue.qsize() == 0
+
+    # 3 files are in the base dir
+    # simulate start with unprocessed
+    gfts_manager.add_to_queue(out_dir.glob("*.tm"))
+    assert gfts_manager.queue.qsize() == 3
+
+    # now some tm in coming in
+    time.sleep(1)
+    shutil.move((out_dir / "test_4.tmp"), (out_dir / "test_4.tm"))
+    time.sleep(1)
+    shutil.move((out_dir / "test_5.tmp"), (out_dir / "test_5.tm"))
+    time.sleep(1)
+    shutil.move((out_dir / "test_6.tmp"), (out_dir / "test_6.tm"))
+    time.sleep(1)
+
+    openfiles = gfts_manager.queue.qsize()
+    assert openfiles > 2
+    time.sleep(20)
+    assert gfts_manager.queue.qsize() < openfiles
+    observer.stop()

stixcore/processing/tests/test_processing.py

@@ -247,7 +247,8 @@ def test_single_vs_batch(out_dir):
 
         for i, f_b in enumerate(files_b):
             f_s = files_s[i]
-            diff = FITSDiff(f_b, f_s, ignore_keywords=['CHECKSUM', 'DATASUM', 'DATE', 'VERS_SW'])
+            diff = FITSDiff(f_b, f_s, ignore_keywords=['CHECKSUM', 'DATASUM', 'DATE',
+                                                       'VERS_SW', 'VERS_CFG'])
             assert diff.identical
     finally:
         CONFIG.set('Logging', 'stop_on_error', str(CONTINUE_ON_ERROR))

stixcore/soop/tests/test_soop_manager.py

@@ -2,7 +2,6 @@
 import sys
 import time
 import shutil
-import platform
 
 import dateutil.parser
 import pytest
@@ -61,12 +60,9 @@ def test_soop_manager_watchdog(soop_manager):
 
     time.sleep(3)
 
-    # currently not triggered on mac see: https://github.com/i4Ds/STIXCore/issues/149
-    if platform.system() != "Darwin":
-        # the new data should be integrated now
-        assert soop_manager.filecounter == 4
-        assert len(soop_manager.soops) == 129
-        assert len(soop_manager.observations) == 132
+    assert soop_manager.filecounter == 4
+    assert len(soop_manager.soops) == 129
+    assert len(soop_manager.observations) == 132
 
     observer.stop()
 

stixcore/version_conf.py

@@ -6,7 +6,7 @@ def get_conf_version():
         with open(Path(__file__).parent / "config" / "data" / "common" / "VERSION.TXT") as f:
             return f.readline().strip()
     except Exception:
-        return 'v0.1.3'
+        return 'v0.1.5code'
 
 
 __version_conf__ = get_conf_version()