new: check if the importer should stop and quit accordingly

bin/cisa_known_exploited.py

@@ -19,7 +19,7 @@ def __init__(self, loglevel: int | None=None) -> None:
         self.fetcher = CISAKnownExploited()
 
     def _to_run_forever(self) -> None:
-        self.fetcher.update()
+        self.fetcher.update(self.shutdown_requested)
 
 
 def main() -> None:

bin/csaf_certbund.py

@@ -19,7 +19,7 @@ def __init__(self, loglevel: int | None=None) -> None:
         self.csaf = CSAFCertBund()
 
     def _to_run_forever(self) -> None:
-        self.csaf.update()
+        self.csaf.update(self.shutdown_requested)
 
 
 def main() -> None:

bin/csaf_cisa.py

@@ -19,7 +19,7 @@ def __init__(self, loglevel: int | None=None) -> None:
         self.csaf = CSAFCisa()
 
     def _to_run_forever(self) -> None:
-        self.csaf.update()
+        self.csaf.update(self.shutdown_requested)
 
 
 def main() -> None:

bin/csaf_cisco.py

@@ -19,7 +19,7 @@ def __init__(self, loglevel: int | None=None) -> None:
         self.csaf = CSAFCisco()
 
     def _to_run_forever(self) -> None:
-        self.csaf.update()
+        self.csaf.update(self.shutdown_requested)
 
 
 def main() -> None:

bin/csaf_nozominetworks.py

@@ -19,7 +19,7 @@ def __init__(self, loglevel: int | None=None) -> None:
         self.csaf = CSAFNozomiNetworks()
 
     def _to_run_forever(self) -> None:
-        self.csaf.update()
+        self.csaf.update(self.shutdown_requested)
 
 
 def main() -> None:

bin/csaf_ox.py

@@ -19,7 +19,7 @@ def __init__(self, loglevel: int | None=None) -> None:
         self.csaf = CSAFOpenXchange()
 
     def _to_run_forever(self) -> None:
-        self.csaf.update()
+        self.csaf.update(self.shutdown_requested)
 
 
 def main() -> None:

bin/csaf_redhat.py

@@ -19,7 +19,7 @@ def __init__(self, loglevel: int | None=None) -> None:
         self.csaf = CSAFRedHat()
 
     def _to_run_forever(self) -> None:
-        self.csaf.update()
+        self.csaf.update(self.shutdown_requested)
 
 
 def main() -> None:

bin/csaf_sick.py

@@ -19,7 +19,7 @@ def __init__(self, loglevel: int | None=None) -> None:
         self.csaf = CSAFSick()
 
     def _to_run_forever(self) -> None:
-        self.csaf.update()
+        self.csaf.update(self.shutdown_requested)
 
 
 def main() -> None:

bin/csaf_siemens.py

@@ -19,7 +19,7 @@ def __init__(self, loglevel: int | None=None) -> None:
         self.csaf = CSAFSiemens()
 
     def _to_run_forever(self) -> None:
-        self.csaf.update()
+        self.csaf.update(self.shutdown_requested)
 
 
 def main() -> None:

bin/cvelist.py

@@ -19,7 +19,7 @@ def __init__(self, loglevel: int | None=None) -> None:
         self.fetcher = CVEListV5()
 
     def _to_run_forever(self) -> None:
-        self.fetcher.update()
+        self.fetcher.update(self.shutdown_requested)
 
 
 def main() -> None:

bin/github.py

@@ -19,7 +19,7 @@ def __init__(self, loglevel: int | None=None) -> None:
         self.gad = Github()
 
     def _to_run_forever(self) -> None:
-        self.gad.update()
+        self.gad.update(self.shutdown_requested)
 
 
 def main() -> None:

bin/gsd.py

@@ -19,7 +19,7 @@ def __init__(self, loglevel: int | None=None) -> None:
         self.gsd = GSD()
 
     def _to_run_forever(self) -> None:
-        self.gsd.update()
+        self.gsd.update(self.shutdown_requested)
 
 
 def main() -> None:

bin/nvd_fetcher.py

@@ -19,7 +19,7 @@ def __init__(self, loglevel: int | None=None) -> None:
         self.fetcher = NVD()
 
     def _to_run_forever(self) -> None:
-        self.fetcher.update()
+        self.fetcher.update(self.shutdown_requested)
 
 
 def main() -> None:

bin/ossf_malicious.py

@@ -19,7 +19,7 @@ def __init__(self, loglevel: int | None=None) -> None:
         self.fetcher = OSSFMaliciousPackages()
 
     def _to_run_forever(self) -> None:
-        self.fetcher.update()
+        self.fetcher.update(self.shutdown_requested)
 
 
 def main() -> None:

bin/pysec.py

@@ -19,7 +19,7 @@ def __init__(self, loglevel: int | None=None) -> None:
         self.pysec = PySec()
 
     def _to_run_forever(self) -> None:
-        self.pysec.update()
+        self.pysec.update(self.shutdown_requested)
 
 
 def main() -> None:

bin/variot.py

@@ -19,7 +19,7 @@ def __init__(self, loglevel: int | None=None) -> None:
         self.variotdb = VARIoTDB()
 
     def _to_run_forever(self) -> None:
-        self.variotdb.update()
+        self.variotdb.update(self.shutdown_requested)
 
 
 def main() -> None:

vulnerabilitylookup/feeders/abstract_feeder.py

@@ -7,6 +7,7 @@
 
 from abc import ABC, abstractmethod
 from pathlib import Path
+from typing import Callable
 
 from git import Repo
 import orjson
@@ -48,5 +49,5 @@ def init_git_repo(self) -> None:
         self.git = Repo(self.path_to_repo)
 
     @abstractmethod
-    def update(self) -> bool:
+    def update(self, stop: Callable[..., bool]) -> bool:
         ...

vulnerabilitylookup/feeders/cisa_known_exploited.py

@@ -4,10 +4,12 @@
 import requests
 
 from datetime import datetime
-import orjson
 from pathlib import Path
+from typing import Callable
 from uuid import uuid4
 
+import orjson
+
 from ..helpers import fromisoformat_wrapper
 
 from .abstract_feeder import AbstractFeeder
@@ -26,14 +28,16 @@ def _load_logging_config(self) -> None:
             log_config = orjson.loads(f.read())
         logging.config.dictConfig(log_config)
 
-    def update(self) -> bool:
+    def update(self, stop: Callable[..., bool]) -> bool:
+        # NOTE: Not checking stop in this method as the main thing that will lock is the request
+        # And this one has a timeout set to 300 seconds
         last_update: datetime | None
         if _last_update_str := self.storage.hget('last_updates', self.name):
             last_update = fromisoformat_wrapper(_last_update_str.decode())
         else:
             last_update = None
 
-        r = requests.get(self.url)
+        r = requests.get(self.url, timeout=300)
         try:
             r.raise_for_status()
         except Exception as e:

vulnerabilitylookup/feeders/csaf_generic.py

@@ -10,6 +10,7 @@
 from datetime import datetime
 from pathlib import Path
 from subprocess import Popen
+from typing import Callable
 
 from ..default import get_config, get_homedir
 from ..helpers import fromisoformat_wrapper
@@ -69,7 +70,7 @@ def pull_csaf_repo(self, last_update: str | None = None) -> None:
                 time.sleep(10)
             self.logger.info('Downloading CSAF data finished')
 
-    def update(self) -> bool:
+    def update(self, stop: Callable[..., bool]) -> bool:
         # TODO: try to only get the updates. Not sure it is possible
         # but this link might help: https://github.com/csaf-poc/csaf_distribution/blob/main/docs/csaf_downloader.md#timerange-option
         self.init_csaf_repo()
@@ -105,6 +106,7 @@ def update(self) -> bool:
         if not paths_to_import:
             return False
 
+        import_complete: bool = True
         p = self.storage.pipeline()
         csafids: dict[str, float] = {}
         for path in paths_to_import:
@@ -131,16 +133,24 @@ def update(self) -> bool:
                 p = self.storage.pipeline()
                 csafids = {}
 
+            if stop():
+                self.logger.info('Shutdown requested. Stopping import.')
+                import_complete = False
+                break
+
         if csafids:
             # remaining entries
             p.zadd(f'index:{self.name}', csafids)  # type: ignore
             p.zadd('index', csafids)  # type: ignore
             p.execute()
-        self.storage.hset('last_updates', mapping={self.name: update_time})
-        self.logger.info('Import done.')
-        last_line = self.get_download_stats()
-        self.logger.info(f'Download Statistics: {last_line}')
-        return True
+
+        if import_complete:
+            self.storage.hset('last_updates', mapping={self.name: update_time})
+            self.logger.info('Import done.')
+            last_line = self.get_download_stats()
+            self.logger.info(f'Download Statistics: {last_line}')
+            return True
+        return False
 
     def get_download_stats(self) -> str:
         log_file = self.path_to_repo / 'downloader.log'

vulnerabilitylookup/feeders/cvelistv5.py

@@ -6,6 +6,7 @@
 
 import orjson
 from pathlib import Path
+from typing import Callable
 
 from ..helpers import fromisoformat_wrapper
 
@@ -18,7 +19,7 @@ def __init__(self) -> None:
 
         self.init_git_repo()
 
-    def update(self) -> bool:
+    def update(self, stop: Callable[..., bool]) -> bool:
         self.git.remotes.origin.pull('main')
 
         paths_to_import: set[Path] = set()
@@ -45,6 +46,7 @@ def update(self) -> bool:
             self.logger.info('Nothing new to import.')
             return False
 
+        import_complete: bool = True
         p = self.storage.pipeline()
         cvelistv5ids: dict[str, float] = {}
         for path in paths_to_import:
@@ -102,12 +104,20 @@ def update(self) -> bool:
                 # reset pipeline
                 p = self.storage.pipeline()
                 cvelistv5ids = {}
+
+            if stop():
+                self.logger.info('Shutdown requested. Stopping import.')
+                import_complete = False
+                break
+
         if cvelistv5ids:
             # remaining entries
             p.zadd(f'index:{self.name}', cvelistv5ids)  # type: ignore
             p.zadd('index', cvelistv5ids)  # type: ignore
             p.execute()
 
-        self.storage.hset('last_updates', mapping={self.name: self.git.head.commit.hexsha})
-        self.logger.info('Import done.')
-        return True
+        if import_complete:
+            self.storage.hset('last_updates', mapping={self.name: self.git.head.commit.hexsha})
+            self.logger.info('Import done.')
+            return True
+        return False

vulnerabilitylookup/feeders/github.py

@@ -4,6 +4,7 @@
 
 import orjson
 from pathlib import Path
+from typing import Callable
 
 from ..helpers import fromisoformat_wrapper
 
@@ -16,7 +17,7 @@ def __init__(self) -> None:
 
         self.init_git_repo()
 
-    def update(self) -> bool:
+    def update(self, stop: Callable[..., bool]) -> bool:
         self.git.remotes.origin.pull('main')
 
         paths_to_import: set[Path] = set()
@@ -52,6 +53,7 @@ def update(self) -> bool:
         if not paths_to_import:
             return False
 
+        import_complete: bool = True
         p = self.storage.pipeline()
         gsids: dict[str, float] = {}
         for path in paths_to_import:
@@ -77,11 +79,20 @@ def update(self) -> bool:
                 p = self.storage.pipeline()
                 gsids = {}
 
+            if stop():
+                import_complete = False
+                self.logger.info('Shutdown requested. Stopping import.')
+                break
+
         if gsids:
             # remaining entries
             p.zadd(f'index:{self.name}', gsids)  # type: ignore
             p.zadd('index', gsids)  # type: ignore
             p.execute()
-        self.storage.hset('last_updates', mapping={self.name: self.git.head.commit.hexsha})
-        self.logger.info('Import done.')
-        return True
+
+        if import_complete:
+            self.storage.hset('last_updates', mapping={self.name: self.git.head.commit.hexsha})
+            self.logger.info('Import done.')
+            return True
+
+        return False

vulnerabilitylookup/feeders/gsd.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from pathlib import Path
+from typing import Callable
 
 import orjson
 
@@ -15,7 +16,7 @@ def __init__(self) -> None:
 
         self.init_git_repo()
 
-    def update(self) -> bool:
+    def update(self, stop: Callable[..., bool]) -> bool:
 
         self.git.remotes.origin.pull('main')
 
@@ -50,6 +51,7 @@ def update(self) -> bool:
             self.logger.info('Nothing new to import.')
             return False
 
+        import_complete: bool = True
         p = self.storage.pipeline()
         gsids: dict[str, float] = {}
         for path in paths_to_import:
@@ -164,11 +166,19 @@ def update(self) -> bool:
                 p = self.storage.pipeline()
                 gsids = {}
 
+            if stop():
+                self.logger.info('Shutdown requested. Stopping import.')
+                import_complete = False
+
         if gsids:
             # remaining entries
             p.zadd(f'index:{self.name}', gsids)  # type: ignore
             p.zadd('index', gsids)  # type: ignore
             p.execute()
-        self.storage.hset('last_updates', mapping={self.name: self.git.head.commit.hexsha})
-        self.logger.info('Import done.')
-        return True
+
+        if import_complete:
+            self.storage.hset('last_updates', mapping={self.name: self.git.head.commit.hexsha})
+            self.logger.info('Import done.')
+            return True
+
+        return False