From 3b55ac4bb498e1f84f6fb2f4923123edc9d77e7a Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 16 Oct 2023 17:54:34 -0400 Subject: [PATCH 1/3] feat: add types --- src/lib/server/websearch/searchWeb.ts | 6 ++---- src/lib/types/WebSearch.ts | 12 ++++++++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/lib/server/websearch/searchWeb.ts b/src/lib/server/websearch/searchWeb.ts index f7d2f6a4f8d..db601d85ecc 100644 --- a/src/lib/server/websearch/searchWeb.ts +++ b/src/lib/server/websearch/searchWeb.ts @@ -2,6 +2,7 @@ import { SERPAPI_KEY, SERPER_API_KEY, YDC_API_KEY } from "$env/static/private"; import { getJson } from "serpapi"; import type { GoogleParameters } from "serpapi"; +import { YouWebSearch } from "../../types/WebSearch"; // Show result as JSON export async function searchWeb(query: string) { @@ -72,16 +73,13 @@ export async function searchWebYouApi(query: string) { }, }); - /* eslint-disable @typescript-eslint/no-explicit-any */ - const data = (await response.json()) as Record; - if (!response.ok) { throw new Error( - data["message"] ?? `You.com API returned error code ${response.status} - ${response.statusText}` ); } + const data: YouWebSearch = await response.json(); return { organic_results: data.hits.map((hit) => hit.snippets.join("\n")) ?? [], diff --git a/src/lib/types/WebSearch.ts b/src/lib/types/WebSearch.ts index ef5ef41344c..29aefc0259b 100644 --- a/src/lib/types/WebSearch.ts +++ b/src/lib/types/WebSearch.ts @@ -24,3 +24,15 @@ export type WebSearchMessageSources = { type: "sources"; sources: WebSearchSource[]; }; + +export interface YouWebSearch { + hits: YouSearchHit[]; + latency: number; +} + +interface YouSearchHit { + url: string; + title: string; + description: string; + snippets: string[]; +} From 467e54d902837b4944fe7aba1fc29ec61a6f07ce Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 16 Oct 2023 18:03:13 -0400 Subject: [PATCH 2/3] feat: specify search provider --- src/lib/server/websearch/runWebSearch.ts | 4 +++- src/lib/server/websearch/searchWeb.ts | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/lib/server/websearch/runWebSearch.ts b/src/lib/server/websearch/runWebSearch.ts index c6745dbe8e1..8cd7dd2371a 100644 --- a/src/lib/server/websearch/runWebSearch.ts +++ b/src/lib/server/websearch/runWebSearch.ts @@ -10,6 +10,7 @@ import { } from "$lib/server/websearch/sentenceSimilarity"; import type { Conversation } from "$lib/types/Conversation"; import type { MessageUpdate } from "$lib/types/MessageUpdate"; +import { getWebSearchProvider } from "./searchWeb"; const MAX_N_PAGES_SCRAPE = 10 as const; const MAX_N_PAGES_EMBED = 5 as const; @@ -39,7 +40,8 @@ export async function runWebSearch( try { webSearch.searchQuery = await generateQuery(messages); - appendUpdate("Searching Google", [webSearch.searchQuery]); + const searchProvider = getWebSearchProvider(); + appendUpdate(`Searching ${searchProvider}`, [webSearch.searchQuery]); const results = await searchWeb(webSearch.searchQuery); webSearch.results = (results.organic_results && diff --git a/src/lib/server/websearch/searchWeb.ts b/src/lib/server/websearch/searchWeb.ts index db601d85ecc..fcc71cfa88c 100644 --- a/src/lib/server/websearch/searchWeb.ts +++ b/src/lib/server/websearch/searchWeb.ts @@ -4,6 +4,15 @@ import { getJson } from "serpapi"; import type { GoogleParameters } from "serpapi"; import { YouWebSearch } from "../../types/WebSearch"; +// get which SERP api is providing web results +export function getWebSearchProvider() { + if (YDC_API_KEY) { + return "You.com" + } else { + return "Google" + } +} + // Show result as JSON export async function searchWeb(query: string) { if (YDC_API_KEY) { From 3896d2ccec409d368ccb83477772bec5136e9b47 Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 16 Oct 2023 21:32:23 -0400 Subject: [PATCH 3/3] add values for ui --- src/lib/server/websearch/searchWeb.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/lib/server/websearch/searchWeb.ts b/src/lib/server/websearch/searchWeb.ts index fcc71cfa88c..2b1f2863774 100644 --- a/src/lib/server/websearch/searchWeb.ts +++ b/src/lib/server/websearch/searchWeb.ts @@ -89,8 +89,14 @@ export async function searchWebYouApi(query: string) { } const data: YouWebSearch = await response.json(); + const formattedResultsWithSnippets = data.hits.map(({ title, url, snippets }) => ({ + title, + link: url, + text: snippets?.join("\n") || "", + hostname: new URL(url).hostname, + })) return { - organic_results: data.hits.map((hit) => hit.snippets.join("\n")) ?? [], + organic_results: formattedResultsWithSnippets, }; }