From da979e574531e1e9c943001087ddb5156039d434 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Thu, 24 Oct 2024 14:27:34 -0700
Subject: [PATCH] More intuitive search settings interfaces (#2899)

* clearer search settings interfaces

* nits
---
 .../admin/embeddings/RerankingFormPage.tsx    |  7 ++--
 web/src/app/admin/embeddings/interfaces.ts    |  5 ---
 .../pages/AdvancedEmbeddingFormPage.tsx       | 12 ++++---
 .../embeddings/pages/EmbeddingFormPage.tsx    | 32 ++++++++++++-------
 web/src/app/chat/ChatPage.tsx                 |  1 -
 .../embedding/CustomEmbeddingModelForm.tsx    |  9 ------
 web/src/components/embedding/interfaces.tsx   | 31 ------------------
 7 files changed, 32 insertions(+), 65 deletions(-)

diff --git a/web/src/app/admin/embeddings/RerankingFormPage.tsx b/web/src/app/admin/embeddings/RerankingFormPage.tsx
index 1e0aae06594..67f5fc1614b 100644
--- a/web/src/app/admin/embeddings/RerankingFormPage.tsx
+++ b/web/src/app/admin/embeddings/RerankingFormPage.tsx
@@ -90,9 +90,10 @@ const RerankingDetailsForm = forwardRef<
 
           return (
             <div className="p-2 rounded-lg max-w-4xl mx-auto">
-              <h2 className="text-2xl font-bold mb-4 text-text-800">
-                Post-processing
-              </h2>
+              <p className="mb-4">
+                Select from cloud, self-hosted models, or use no reranking
+                model.
+              </p>
               <div className="text-sm mr-auto mb-6 divide-x-2 flex">
                 {originalRerankingDetails.rerank_model_name && (
                   <button
diff --git a/web/src/app/admin/embeddings/interfaces.ts b/web/src/app/admin/embeddings/interfaces.ts
index 2fc328eab86..3f7549fa741 100644
--- a/web/src/app/admin/embeddings/interfaces.ts
+++ b/web/src/app/admin/embeddings/interfaces.ts
@@ -21,11 +21,6 @@ export enum RerankerProvider {
 }
 
 export interface AdvancedSearchConfiguration {
-  model_name: string;
-  model_dim: number;
-  normalize: boolean;
-  query_prefix: string;
-  passage_prefix: string;
   index_name: string | null;
   multipass_indexing: boolean;
   multilingual_expansion: string[];
diff --git a/web/src/app/admin/embeddings/pages/AdvancedEmbeddingFormPage.tsx b/web/src/app/admin/embeddings/pages/AdvancedEmbeddingFormPage.tsx
index b708675116b..bea80322e1a 100644
--- a/web/src/app/admin/embeddings/pages/AdvancedEmbeddingFormPage.tsx
+++ b/web/src/app/admin/embeddings/pages/AdvancedEmbeddingFormPage.tsx
@@ -4,7 +4,11 @@ import * as Yup from "yup";
 import { TrashIcon } from "@/components/icons/icons";
 import { FaPlus } from "react-icons/fa";
 import { AdvancedSearchConfiguration } from "../interfaces";
-import { BooleanFormField } from "@/components/admin/connectors/Field";
+import {
+  BooleanFormField,
+  Label,
+  SubLabel,
+} from "@/components/admin/connectors/Field";
 import NumberInput from "../../connectors/[connector]/pages/ConnectorInput/NumberInput";
 
 interface AdvancedEmbeddingFormPageProps {
@@ -21,9 +25,6 @@ const AdvancedEmbeddingFormPage = forwardRef<
 >(({ updateAdvancedEmbeddingDetails, advancedEmbeddingDetails }, ref) => {
   return (
     <div className="py-4 rounded-lg max-w-4xl px-4 mx-auto">
-      <h2 className="text-2xl font-bold mb-4 text-text-800">
-        Advanced Configuration
-      </h2>
       <Formik
         innerRef={ref}
         initialValues={advancedEmbeddingDetails}
@@ -52,6 +53,9 @@ const AdvancedEmbeddingFormPage = forwardRef<
             <FieldArray name="multilingual_expansion">
               {({ push, remove }) => (
                 <div className="w-full">
+                  <Label>Multi-lingual Expansion</Label>
+
+                  <SubLabel>Add additional languages to the search.</SubLabel>
                   {values.multilingual_expansion.map(
                     (_: any, index: number) => (
                       <div key={index} className="w-full flex mb-4">
diff --git a/web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx b/web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx
index 90f1b70fa04..4947e8759d5 100644
--- a/web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx
+++ b/web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx
@@ -33,11 +33,6 @@ export default function EmbeddingForm() {
 
   const [advancedEmbeddingDetails, setAdvancedEmbeddingDetails] =
     useState<AdvancedSearchConfiguration>({
-      model_name: "",
-      model_dim: 0,
-      normalize: false,
-      query_prefix: "",
-      passage_prefix: "",
       index_name: "",
       multipass_indexing: true,
       multilingual_expansion: [],
@@ -109,11 +104,6 @@ export default function EmbeddingForm() {
   useEffect(() => {
     if (searchSettings) {
       setAdvancedEmbeddingDetails({
-        model_name: searchSettings.model_name,
-        model_dim: searchSettings.model_dim,
-        normalize: searchSettings.normalize,
-        query_prefix: searchSettings.query_prefix,
-        passage_prefix: searchSettings.passage_prefix,
         index_name: searchSettings.index_name,
         multipass_indexing: searchSettings.multipass_indexing,
         multilingual_expansion: searchSettings.multilingual_expansion,
@@ -263,8 +253,8 @@ export default function EmbeddingForm() {
       // This is a cloud model
       newModel = {
         ...selectedProvider,
-        ...rerankingDetails,
         ...advancedEmbeddingDetails,
+        ...rerankingDetails,
         provider_type:
           (selectedProvider.provider_type
             ?.toLowerCase()
@@ -274,8 +264,8 @@ export default function EmbeddingForm() {
       // This is a locally hosted model
       newModel = {
         ...selectedProvider,
-        ...rerankingDetails,
         ...advancedEmbeddingDetails,
+        ...rerankingDetails,
         provider_type: null,
       };
     }
@@ -388,6 +378,16 @@ export default function EmbeddingForm() {
 
         {formStep == 1 && (
           <>
+            <h2 className="text-2xl font-bold mb-4 text-text-800">
+              Select a Reranking Model
+            </h2>
+            <Text className="mb-4">
+              Updating the reranking model does not require re-indexing
+              documents. The reranker helps improve search quality by reordering
+              results after the initial embedding search. Changes will take
+              effect immediately for all new searches.
+            </Text>
+
             <Card>
               <RerankingDetailsForm
                 setModelTab={setModelTab}
@@ -429,6 +429,14 @@ export default function EmbeddingForm() {
         )}
         {formStep == 2 && (
           <>
+            <h2 className="text-2xl font-bold mb-4 text-text-800">
+              Advanced Search Configuration
+            </h2>
+            <Text className="mb-4">
+              Configure advanced embedding and search settings. Changes will
+              require re-indexing documents.
+            </Text>
+
             <Card>
               <AdvancedEmbeddingFormPage
                 advancedEmbeddingDetails={advancedEmbeddingDetails}
diff --git a/web/src/app/chat/ChatPage.tsx b/web/src/app/chat/ChatPage.tsx
index b67ec473026..2ce657b2caf 100644
--- a/web/src/app/chat/ChatPage.tsx
+++ b/web/src/app/chat/ChatPage.tsx
@@ -809,7 +809,6 @@ export function ChatPage({
 
       // Check if all messages are currently rendered
       if (currentVisibleRange.end < messageHistory.length) {
-        console.log("Updating visible range");
         // Update visible range to include the last messages
         updateCurrentVisibleRange({
           start: Math.max(
diff --git a/web/src/components/embedding/CustomEmbeddingModelForm.tsx b/web/src/components/embedding/CustomEmbeddingModelForm.tsx
index 8dcd34f0ba6..7ba29531863 100644
--- a/web/src/components/embedding/CustomEmbeddingModelForm.tsx
+++ b/web/src/components/embedding/CustomEmbeddingModelForm.tsx
@@ -33,10 +33,6 @@ export function CustomEmbeddingModelForm({
             api_url: provider.api_url,
             description: "",
             index_name: "",
-            pricePerMillion: 0,
-            mtebScore: 0,
-            maxContext: 4096,
-            max_tokens: 1024,
           }
         }
         validationSchema={Yup.object().shape({
@@ -51,13 +47,8 @@ export function CustomEmbeddingModelForm({
           api_url: Yup.string().required("API base URL is required"),
           description: Yup.string(),
           index_name: Yup.string().nullable(),
-          pricePerMillion: Yup.number(),
-          mtebScore: Yup.number(),
-          maxContext: Yup.number(),
-          max_tokens: Yup.number(),
         })}
         onSubmit={async (values) => {
-          console.log(values);
           setShowTentativeModel(values as CloudEmbeddingModel);
         }}
       >
diff --git a/web/src/components/embedding/interfaces.tsx b/web/src/components/embedding/interfaces.tsx
index ff80ffd6ace..87a16959864 100644
--- a/web/src/components/embedding/interfaces.tsx
+++ b/web/src/components/embedding/interfaces.tsx
@@ -58,17 +58,10 @@ export interface EmbeddingModelDescriptor {
 
 export interface CloudEmbeddingModel extends EmbeddingModelDescriptor {
   pricePerMillion: number;
-  enabled?: boolean;
-  mtebScore: number;
-  maxContext: number;
 }
 
 export interface HostedEmbeddingModel extends EmbeddingModelDescriptor {
   link?: string;
-  model_dim: number;
-  normalize: boolean;
-  query_prefix: string;
-  passage_prefix: string;
   isDefault?: boolean;
 }
 
@@ -197,9 +190,6 @@ export const AVAILABLE_CLOUD_PROVIDERS: CloudEmbeddingProvider[] = [
         description:
           "Cohere's English embedding model. Good performance for English-language tasks.",
         pricePerMillion: 0.1,
-        mtebScore: 64.5,
-        maxContext: 512,
-        enabled: false,
         model_dim: 1024,
         normalize: false,
         query_prefix: "",
@@ -214,9 +204,6 @@ export const AVAILABLE_CLOUD_PROVIDERS: CloudEmbeddingProvider[] = [
         description:
           "Cohere's lightweight English embedding model. Faster and more efficient for simpler tasks.",
         pricePerMillion: 0.1,
-        mtebScore: 62,
-        maxContext: 512,
-        enabled: false,
         model_dim: 384,
         normalize: false,
         query_prefix: "",
@@ -247,9 +234,6 @@ export const AVAILABLE_CLOUD_PROVIDERS: CloudEmbeddingProvider[] = [
         normalize: false,
         query_prefix: "",
         passage_prefix: "",
-        mtebScore: 64.6,
-        maxContext: 8191,
-        enabled: false,
         index_name: "",
         api_key: null,
         api_url: null,
@@ -264,9 +248,6 @@ export const AVAILABLE_CLOUD_PROVIDERS: CloudEmbeddingProvider[] = [
         description:
           "OpenAI's newer, more efficient embedding model. Good balance of performance and cost.",
         pricePerMillion: 0.02,
-        enabled: false,
-        mtebScore: 62.3,
-        maxContext: 8191,
         index_name: "",
         api_key: null,
         api_url: null,
@@ -290,9 +271,6 @@ export const AVAILABLE_CLOUD_PROVIDERS: CloudEmbeddingProvider[] = [
         model_name: "text-embedding-004",
         description: "Google's most recent text embedding model.",
         pricePerMillion: 0.025,
-        mtebScore: 66.31,
-        maxContext: 2048,
-        enabled: false,
         model_dim: 768,
         normalize: false,
         query_prefix: "",
@@ -306,9 +284,6 @@ export const AVAILABLE_CLOUD_PROVIDERS: CloudEmbeddingProvider[] = [
         model_name: "textembedding-gecko@003",
         description: "Google's Gecko embedding model. Powerful and efficient.",
         pricePerMillion: 0.025,
-        mtebScore: 66.31,
-        maxContext: 2048,
-        enabled: false,
         model_dim: 768,
         normalize: false,
         query_prefix: "",
@@ -335,9 +310,6 @@ export const AVAILABLE_CLOUD_PROVIDERS: CloudEmbeddingProvider[] = [
         description:
           "Voyage's large embedding model. High performance with instruction fine-tuning.",
         pricePerMillion: 0.12,
-        mtebScore: 68.28,
-        maxContext: 4000,
-        enabled: false,
         model_dim: 1024,
         normalize: false,
         query_prefix: "",
@@ -352,9 +324,6 @@ export const AVAILABLE_CLOUD_PROVIDERS: CloudEmbeddingProvider[] = [
         description:
           "Voyage's lightweight embedding model. Good balance of performance and efficiency.",
         pricePerMillion: 0.12,
-        mtebScore: 67.13,
-        maxContext: 16000,
-        enabled: false,
         model_dim: 1024,
         normalize: false,
         query_prefix: "",