Update index.js

indexnetwork · May 27, 2024 · d31baac · d31baac
1 parent 4feadb5
commit d31baac
Showing 1 changed file with 7 additions and 2 deletions.
diff --git a/index.js b/index.js
@@ -36,7 +36,12 @@ const initializeCrawler = async () => {
         }
       });
 
-      //sleep(60000);
+      await Promise.race([
+        page.waitForNetworkIdle({ idleTime: 500, concurrency: 3 }),
+        new Promise((_, reject) =>
+          setTimeout(() => reject(new Error("Timeout")), 10000),
+        ),
+      ]);
 
       await page.evaluate(() => {
         return window.scrollBy(0, window.innerHeight);
@@ -53,7 +58,7 @@ const initializeCrawler = async () => {
       console.log(`Title: ${await page.title()}`);
       console.log(`Content: ${content}`);
       contentMap.set(request.uniqueKey, content); // Store content with uniqueKey
-      //await requestQueue.markRequestHandled(request);
+      await requestQueue.markRequestHandled(request);
     },
   });