Product scraping Captcha handling (#95)

* make requests use desktop and show error message when captcha response * format
cmintey · Jan 20, 2024 · 769c350 · 769c350
1 parent 8f0af35
commit 769c350
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 3 deletions.
diff --git a/src/lib/components/wishlists/ItemForm.svelte b/src/lib/components/wishlists/ItemForm.svelte
@@ -3,13 +3,15 @@
     import type { Item } from "@prisma/client";
     import Backdrop from "$lib/components/Backdrop.svelte";
     import { env } from "$env/dynamic/public";
+    import { getToastStore } from "@skeletonlabs/skeleton";
 
     export let data: Item;
     export let buttonText: string;
 
     $: form = $page.form;
     let loading = false;
     let urlChanged = false;
+    const toastStore = getToastStore();
 
     const formatPrice = (price: number | null, currency: string | null) => {
         if (!price) return null;
@@ -28,6 +30,15 @@
         return null;
     };
 
+    const triggerToast = () => {
+        toastStore.trigger({
+            message: `Unable to find product information. You can still fill in the details manually.`,
+            background: "variant-filled-warning",
+            autohide: true,
+            timeout: 5000
+        });
+    };
+
     const getInfo = async () => {
         if (data.url && urlChanged) {
             loading = true;
@@ -40,7 +51,7 @@
                 data.image_url = productData.image;
                 data.price = formatPrice(productData.price, productData.currency);
             } else {
-                console.log("invalid url");
+                triggerToast();
             }
             loading = false;
             urlChanged = false;

diff --git a/src/routes/api/product/+server.ts b/src/routes/api/product/+server.ts
@@ -18,8 +18,13 @@ const scraper = metascraper([
 ]);
 
 const goShopping = async (targetUrl: string) => {
-    const { body: html, url } = await gotScraping(targetUrl);
-    const metadata = await scraper({ html, url });
+    const resp = await gotScraping({
+        url: targetUrl,
+        headerGeneratorOptions: {
+            devices: ["desktop"]
+        }
+    });
+    const metadata = await scraper({ html: resp.body, url: resp.url });
     return metadata;
 };
 
@@ -44,6 +49,9 @@ export const GET: RequestHandler = async ({ request }) => {
             // retry with the resolved URL
             metadata = await goShopping(metadata.url);
         }
+        if (isCaptchaResponse(metadata)) {
+            error(424, "product information not available");
+        }
 
         return new Response(JSON.stringify(metadata));
     } else {