chg: use scan to generate dump

bin/dump.py

@@ -1,39 +1,42 @@
 #!/usr/bin/env python3
 
 import argparse
+import json
 import logging
-from typing import Optional
 
 import logging.config
-import redis
 
-from vulnerabilitylookup.default import AbstractManager
-from vulnerabilitylookup.default import get_config, get_homedir
+from vulnerabilitylookup import VulnerabilityLookup
+from vulnerabilitylookup.default import get_config, get_homedir, safe_create_dir
 
 logging.config.dictConfig(get_config('logging'))
 
-host = get_config('generic', 'storage_db_hostname')
-port = get_config('generic', 'storage_db_port')
 
-r = redis.Redis(host=host, port=port, decode_responses=True)
+class Dump():
 
-parser = argparse.ArgumentParser(
-    description="Dump vulnerability-lookup storage in NDJSON."
-)
+    def __init__(self):
+        self.vl = VulnerabilityLookup()
+        self.root_dumps = get_homedir() / 'dumps'
+        safe_create_dir(self.root_dumps)
 
-parser.add_argument(
-    "--feed", help="Feed to dump. Default is set to nvd.", default="nvd"
-)
-
-args = parser.parse_args()
-
-index = f"index:{args.feed}"
+    def dump(self, feed: str, /) -> None:
+        dest_file = self.root_dumps / f'{feed}.json'
+        dest_file.unlink(missing_ok=True)
+        for vuln in self.vl.get_all(feed):
+            with dest_file.open('a') as f:
+                json.dump(vuln, f)
 
 
 def main():
-    for cve in r.smembers(index):
-        cve_json = r.get(str(cve))
-        print(f'{cve_json}')
+    parser = argparse.ArgumentParser(
+        description="Dump vulnerability-lookup storage in NDJSON."
+    )
+    parser.add_argument(
+        "--feed", help="Feed to dump. Default is set to nvd.", default="nvd"
+    )
+    args = parser.parse_args()
+    d = Dump()
+    d.dump(args.feed)
 
 
 if __name__ == '__main__':

pyproject.toml

@@ -16,6 +16,7 @@ start_website = "bin.start_website:main"
 nvd = "bin.nvd_fetcher:main"
 gsd_importer = "bin.gsd:main"
 github_importer = "bin.github:main"
+dump = "bin.dump:main"
 
 [tool.poetry.dependencies]
 python = "^3.10"

storage/kvrocks.conf

@@ -300,7 +300,7 @@ max-bitmap-to-string-mb 16
 # If enabled, the cursor will be unsigned 64-bit integers.
 # If disabled, the cursor will be a string.
 # Default: no
-redis-cursor-compatible no
+redis-cursor-compatible yes
 
 ################################## TLS ###################################
 

vulnerabilitylookup/vulnerabilitylookup.py

@@ -86,6 +86,13 @@ def get_last(self, source: Optional[str]=None, number: Optional[int]=30):
                 to_return.append(vuln)
         return to_return
 
+    def get_all(self, source: str, /):
+        """This method will scan a complete source and yield the vulnerabilities.
+        It is up to the caller to handle the yielded entries as it will be a lot"""
+        for vuln_id, _ in self.storage.zscan_iter(f'index:{source}'):
+            if vuln := self.get_vulnerability(vuln_id):
+                yield vuln
+
     def get_vendors(self):
         return self.storage.smembers('vendors')