Merge pull request #615 from rnadigital/firecrawl-use-fetch

Firecrawl use fetch

agent-backend/src/tools/builtins/firecrawl_loader.py

@@ -1,22 +1,32 @@
 from langchain_community.document_loaders import AsyncChromiumLoader
 from langchain_community.document_loaders import FireCrawlLoader
+from pydantic import PrivateAttr
 from tools.builtins.base import BaseBuiltinTool
+import requests
 
 class FireCrawlLoader(BaseBuiltinTool):
-    loader = FireCrawlLoader
 
     def __init__(self, *args, **kwargs):
-        print(f"KWARGS: {kwargs}")
+        super().__init__(**kwargs)
         parameters: dict = kwargs.get("parameters")
         if parameters is not None:
-            print(f"PARAMETERS: {parameters}")
-            api_key:str = parameters.get("firecrawl_api_key")
-            print(f"API KEY: {api_key}")
-            kwargs["loader"] = FireCrawlLoader(api_key)
-            super().__init__(**kwargs)
+            self.__dict__['_api_key'] = parameters.get("api_key", "")
         else:
             print("Parameters was None!")
 
     def run_tool(self, query: str) -> str:
-        data = FireCrawlLoader(url = query, mode = "scrape")
-        return data
+        if not getattr(self, '_api_key', None):
+            raise ValueError("API key is not set!") #type saftey
+
+        # Use the API key for running the tool logic
+        url = "https://api.firecrawl.dev/v1/scrape"
+        payload = {
+        "url": query
+        }
+        headers = {
+            "Authorization": f"Bearer {self._api_key}",
+            "Content-Type": "application/json"
+        }
+        response = requests.request('POST', url, json=payload, headers=headers)
+        responseJson = response.json()
+        return responseJson