diff --git a/backend/danswer/search/pipeline.py b/backend/danswer/search/pipeline.py index ad3e19e149d..4ca6ba27038 100644 --- a/backend/danswer/search/pipeline.py +++ b/backend/danswer/search/pipeline.py @@ -360,10 +360,10 @@ def section_relevance(self) -> list[SectionRelevancePiece] | None: try: results = run_functions_in_parallel(function_calls=functions) self._section_relevance = list(results.values()) - except Exception: + except Exception as e: raise ValueError( - "An issue occured during the agentic evaluation proecss." - ) + "An issue occured during the agentic evaluation process." + ) from e elif self.search_query.evaluation_type == LLMEvaluationType.BASIC: if DISABLE_LLM_DOC_RELEVANCE: diff --git a/backend/danswer/secondary_llm_flows/agentic_evaluation.py b/backend/danswer/secondary_llm_flows/agentic_evaluation.py index 3de9db00be6..03121e3cf1d 100644 --- a/backend/danswer/secondary_llm_flows/agentic_evaluation.py +++ b/backend/danswer/secondary_llm_flows/agentic_evaluation.py @@ -58,25 +58,30 @@ def _get_metadata_str(metadata: dict[str, str | list[str]]) -> str: center_metadata=center_metadata_str, ) filled_llm_prompt = dict_based_prompt_to_langchain_prompt(messages) - model_output = message_to_string(llm.invoke(filled_llm_prompt)) + try: + model_output = message_to_string(llm.invoke(filled_llm_prompt)) - # Search for the "Useful Analysis" section in the model output - # This regex looks for "2. Useful Analysis" (case-insensitive) followed by an optional colon, - # then any text up to "3. Final Relevance" - # The (?i) flag makes it case-insensitive, and re.DOTALL allows the dot to match newlines - # If no match is found, the entire model output is used as the analysis - analysis_match = re.search( - r"(?i)2\.\s*useful analysis:?\s*(.+?)\n\n3\.\s*final relevance", - model_output, - re.DOTALL, - ) - analysis = analysis_match.group(1).strip() if analysis_match else model_output + # Search for the "Useful Analysis" section in the model output + # This regex looks for "2. Useful Analysis" (case-insensitive) followed by an optional colon, + # then any text up to "3. Final Relevance" + # The (?i) flag makes it case-insensitive, and re.DOTALL allows the dot to match newlines + # If no match is found, the entire model output is used as the analysis + analysis_match = re.search( + r"(?i)2\.\s*useful analysis:?\s*(.+?)\n\n3\.\s*final relevance", + model_output, + re.DOTALL, + ) + analysis = analysis_match.group(1).strip() if analysis_match else model_output - # Get the last non-empty line - last_line = next( - (line for line in reversed(model_output.split("\n")) if line.strip()), "" - ) - relevant = last_line.strip().lower().startswith("true") + # Get the last non-empty line + last_line = next( + (line for line in reversed(model_output.split("\n")) if line.strip()), "" + ) + relevant = last_line.strip().lower().startswith("true") + except Exception as e: + logger.exception(f"An issue occured during the agentic evaluation process. {e}") + relevant = False + analysis = "" return SectionRelevancePiece( document_id=document_id, diff --git a/web/src/components/search/DocumentDisplay.tsx b/web/src/components/search/DocumentDisplay.tsx index 5fb1c255b4e..2b2258bf6c3 100644 --- a/web/src/components/search/DocumentDisplay.tsx +++ b/web/src/components/search/DocumentDisplay.tsx @@ -19,6 +19,7 @@ import { FiTag } from "react-icons/fi"; import { DISABLE_LLM_DOC_RELEVANCE } from "@/lib/constants"; import { SettingsContext } from "../settings/SettingsProvider"; import { CustomTooltip, TooltipGroup } from "../tooltip/CustomTooltip"; +import { WarningCircle } from "@phosphor-icons/react"; export const buildDocumentSummaryDisplay = ( matchHighlights: string[], @@ -230,7 +231,7 @@ export const DocumentDisplay = ({ {document.semantic_identifier || document.document_id}

-
+
{isHovered && messageId && ( @@ -326,31 +327,32 @@ export const AgenticDocumentDisplay = ({

-
- {isHovered && messageId && ( - - )} - - {(contentEnriched || additional_relevance) && - relevance_explanation && - (isHovered || alternativeToggled) && ( - +
+ + {isHovered && messageId && ( + )} + + {(contentEnriched || additional_relevance) && + (isHovered || alternativeToggled) && ( + + )} +
@@ -367,7 +369,13 @@ export const AgenticDocumentDisplay = ({ document.match_highlights, document.blurb ) - : relevance_explanation} + : relevance_explanation || ( + + {" "} + + Model failed to produce an analysis of the document + + )}

diff --git a/web/src/components/search/SearchSection.tsx b/web/src/components/search/SearchSection.tsx index 5711854217f..feae79cdf18 100644 --- a/web/src/components/search/SearchSection.tsx +++ b/web/src/components/search/SearchSection.tsx @@ -579,6 +579,14 @@ export const SearchSection = ({ const { popup, setPopup } = usePopup(); + const shouldUseAgenticDisplay = + agenticResults && + (searchResponse.documents || []).some( + (document) => + searchResponse.additional_relevance && + searchResponse.additional_relevance[document.document_id] !== undefined + ); + return ( <>
@@ -756,7 +764,9 @@ export const SearchSection = ({ contentEnriched={contentEnriched} comments={comments} sweep={sweep} - agenticResults={agenticResults && !disabledAgentic} + agenticResults={ + shouldUseAgenticDisplay && !disabledAgentic + } performSweep={performSweep} searchResponse={searchResponse} isFetching={isFetching} diff --git a/web/src/lib/search/streamingQa.ts b/web/src/lib/search/streamingQa.ts index 1f9b595c125..e30063e2fb9 100644 --- a/web/src/lib/search/streamingQa.ts +++ b/web/src/lib/search/streamingQa.ts @@ -98,7 +98,7 @@ export const searchRequestStreamed = async ({ } previousPartialChunk = partialChunk as string | null; completedChunks.forEach((chunk) => { - // check for answer peice / end of answer + // check for answer piece / end of answer if (Object.hasOwn(chunk, "relevance_summaries")) { const relevanceChunk = chunk as RelevanceChunk;