forked from rsrohan99/Llama-Researcher
-
Notifications
You must be signed in to change notification settings - Fork 0
/
tavily.py
45 lines (41 loc) · 1.44 KB
/
tavily.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
import os
from typing import List
import requests
from dotenv import load_dotenv
from llama_index.core.schema import Document
async def get_docs_from_tavily_search(sub_query: str, visited_urls: set[str]):
load_dotenv()
api_key = os.getenv("TAVILY_API_KEY")
base_url = "https://api.tavily.com/search"
headers = {
"Content-Type": "application/json",
}
data = {
"query": sub_query,
"api_key": api_key,
"include_raw_content": True,
}
docs = []
print(f"\n> Searching Tavily for sub query: {sub_query}\n")
response = requests.post(base_url, headers=headers, json=data)
if response.status_code == 200:
search_results = response.json().get("results", [])
for search_result in search_results:
url = search_result.get("url")
if not search_result.get("raw_content"):
continue
if url not in visited_urls:
visited_urls.add(url)
docs.append(
Document(
text=search_result.get("raw_content"),
metadata={
"source": url,
"title": search_result.get("title"),
},
)
)
print(f"\n> Found {len(docs)} docs from Tavily search on {sub_query}\n")
return docs, visited_urls
else:
response.raise_for_status()