add you.com integration

README.md

@@ -76,8 +76,8 @@ npm run dev
 
 Chat UI features a powerful Web Search feature. It works by:
 
-1. Generating an appropriate Google query from the user prompt.
-2. Performing Google search and extracting content from webpages.
+1. Generating an appropriate search query from the user prompt.
+2. Performing web search and extracting content from webpages.
 3. Creating embeddings from texts using [transformers.js](https://huggingface.co/docs/transformers.js). Specifically, using [Xenova/gte-small](https://huggingface.co/Xenova/gte-small) model.
 4. From these embeddings, find the ones that are closest to the user query using vector similarity search. Specifically, we use `inner product` distance.
 5. Get the corresponding texts to those closest embeddings and perform [Retrieval-Augmented Generation](https://huggingface.co/papers/2005.11401) (i.e. expand user prompt by adding those texts so that a LLM can use this information).
@@ -122,7 +122,7 @@ PUBLIC_APP_DISCLAIMER=
 
 ### Web Search config
 
-You can enable the web search by adding either `SERPER_API_KEY` ([serper.dev](https://serper.dev/)) or `SERPAPI_KEY` ([serpapi.com](https://serpapi.com/)) to your `.env.local`.
+You can enable the web search by adding any of `YDC_API_KEY` ([docs.you.com](https://docs.you.com)) or `SERPER_API_KEY` ([serper.dev](https://serper.dev/)) or `SERPAPI_KEY` ([serpapi.com](https://serpapi.com/)) to your `.env.local`.
 
 ### Custom models
 
@@ -209,7 +209,7 @@ The following is the default `webSearchQueryPromptTemplate`.
 ```prompt
 {{userMessageToken}}
   My question is: {{message.content}}.
-  Based on the conversation history (my previous questions are: {{previousMessages}}), give me an appropriate query to answer my question for google search. You should not say more than query. You should not say any words except the query. For the context, today is {{currentDate}}
+  Based on the conversation history (my previous questions are: {{previousMessages}}), give me an appropriate query to answer my question for web search. You should not say more than query. You should not say any words except the query. For the context, today is {{currentDate}}
 {{userMessageEndToken}}
 {{assistantMessageToken}}
 ```

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "chat-ui",
-	"version": "0.6.0",
+	"version": "0.7.0",
 	"private": true,
 	"packageManager": "npm@9.5.0",
 	"scripts": {

src/lib/server/websearch/runWebSearch.ts

@@ -10,6 +10,7 @@ import {
 } from "$lib/server/websearch/sentenceSimilarity";
 import type { Conversation } from "$lib/types/Conversation";
 import type { MessageUpdate } from "$lib/types/MessageUpdate";
+import { getWebSearchProvider } from "./searchWeb";
 
 const MAX_N_PAGES_SCRAPE = 10 as const;
 const MAX_N_PAGES_EMBED = 5 as const;
@@ -39,7 +40,8 @@ export async function runWebSearch(
 
 	try {
 		webSearch.searchQuery = await generateQuery(messages);
-		appendUpdate("Searching Google", [webSearch.searchQuery]);
+		const searchProvider = getWebSearchProvider();
+		appendUpdate(`Searching ${searchProvider}`, [webSearch.searchQuery]);
 		const results = await searchWeb(webSearch.searchQuery);
 		webSearch.results =
 			(results.organic_results &&

src/lib/server/websearch/searchWeb.ts

@@ -1,17 +1,30 @@
-import { SERPAPI_KEY, SERPER_API_KEY } from "$env/static/private";
+import { SERPAPI_KEY, SERPER_API_KEY, YDC_API_KEY } from "$env/static/private";
 
 import { getJson } from "serpapi";
 import type { GoogleParameters } from "serpapi";
+import { YouWebSearch } from "../../types/WebSearch";
+
+// get which SERP api is providing web results
+export function getWebSearchProvider() {
+	if (YDC_API_KEY) {
+		return "You.com"
+	} else {
+		return "Google"
+	}
+}
 
 // Show result as JSON
 export async function searchWeb(query: string) {
+	if (YDC_API_KEY) {
+		return await searchWebYouApi(query);
+	}
 	if (SERPER_API_KEY) {
 		return await searchWebSerper(query);
 	}
 	if (SERPAPI_KEY) {
 		return await searchWebSerpApi(query);
 	}
-	throw new Error("No Serper.dev or SerpAPI key found");
+	throw new Error("No You.com or Serper.dev or SerpAPI key found");
 }
 
 export async function searchWebSerper(query: string) {
@@ -59,3 +72,31 @@ export async function searchWebSerpApi(query: string) {
 
 	return response;
 }
+
+export async function searchWebYouApi(query: string) {
+	const response = await fetch(`https://api.ydc-index.io/search?query=${query}`, {
+		method: "GET",
+		headers: {
+			"X-API-Key": YDC_API_KEY,
+			"Content-type": "application/json; charset=UTF-8",
+		},
+	});
+
+	if (!response.ok) {
+		throw new Error(
+				`You.com API returned error code ${response.status} - ${response.statusText}`
+		);
+	}
+
+	const data: YouWebSearch = await response.json();
+	const formattedResultsWithSnippets = data.hits.map(({ title, url, snippets }) => ({
+		title,
+		link: url,
+		text: snippets?.join("\n") || "",
+		hostname: new URL(url).hostname,
+	}))
+
+	return {
+		organic_results: formattedResultsWithSnippets,
+	};
+}

src/lib/types/WebSearch.ts

@@ -24,3 +24,15 @@ export type WebSearchMessageSources = {
 	type: "sources";
 	sources: WebSearchSource[];
 };
+
+export interface YouWebSearch {
+  hits: YouSearchHit[];
+  latency: number;
+}
+
+interface YouSearchHit {
+  url: string;
+  title: string;
+  description: string;
+  snippets: string[];
+}