From d31baace0da3ea7fefb96ac971a39b7113d08a9c Mon Sep 17 00:00:00 2001
From: serafettin <mail@serafettin.net>
Date: Mon, 27 May 2024 16:38:29 +0300
Subject: [PATCH] Update index.js

---
 index.js | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/index.js b/index.js
index fc96429..89dd1df 100644
--- a/index.js
+++ b/index.js
@@ -36,7 +36,12 @@ const initializeCrawler = async () => {
         }
       });
 
-      //sleep(60000);
+      await Promise.race([
+        page.waitForNetworkIdle({ idleTime: 500, concurrency: 3 }),
+        new Promise((_, reject) =>
+          setTimeout(() => reject(new Error("Timeout")), 10000),
+        ),
+      ]);
 
       await page.evaluate(() => {
         return window.scrollBy(0, window.innerHeight);
@@ -53,7 +58,7 @@ const initializeCrawler = async () => {
       console.log(`Title: ${await page.title()}`);
       console.log(`Content: ${content}`);
       contentMap.set(request.uniqueKey, content); // Store content with uniqueKey
-      //await requestQueue.markRequestHandled(request);
+      await requestQueue.markRequestHandled(request);
     },
   });