update database

Junyi-99 · Sep 6, 2024 · 74792c5 · 74792c5
1 parent cab2590
commit 74792c5
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 0 deletions.
diff --git a/github.db b/github.db
diff --git a/main.py b/main.py
@@ -135,7 +135,10 @@ def _process_url(self, url: str):
                     retry += 1
                     time.sleep(3)
                     continue
+
                 new_apis = [api for api in matches if not db_key_exists(self.cur, api)]
+                new_apis = list(set(new_apis))
+
                 log.info(f"    🟢 Found {len(matches)} matches in the expanded page, leaving {len(new_apis)} new APIs to check")
                 with ThreadPoolExecutor(max_workers=10) as executor:
                     results = list(executor.map(check_key, new_apis))