Rag typo for Arabic based context query search

[rupeshKumarAmt]

Checked other resources

• I added a very descriptive title to this question.
• I searched the LangChain documentation with the integrated search.
• I used the GitHub search to find a similar question and didn't find it.

Commit to Help

• I commit to help with one of those options 👆

Example Code

# Code for embedding
====================

import os
import json
import io
import requests
from bs4 import BeautifulSoup
from dotenv import load_dotenv
from chromadb import HttpClient
from chromadb.config import Settings
from langchain.vectorstores.chroma import Chroma
from langchain.embeddings import OpenAIEmbeddings
from openai._exceptions import RateLimitError
from langchain.document_loaders.base import Document
from langchain.text_splitter import RecursiveCharacterTextSplitter
import re
import pdfplumber


def embed_data_from_pdf_url(url, uuid, open_api_key):
    try:
        chroma_client = get_chroma_client()

        if url.lower().endswith(".pdf") or (requests.get(url).status_code == 200 and requests.get(url).headers.get('Content-Type')== 'application/pdf'):
            pdf_file = pdfplumber.open(io.BytesIO(requests.get(url).content))
            if pdf_file:
                combined_text = "\n".join(page.extract_text() or "" for page in pdf_file.pages)
                if not combined_text:
                    return {'status': 400, "info": "Uploaded PDF Empty"}, 200
                
                # Normalize Arabic text
                # combined_text = normalize_arabic(combined_text)

                chunks = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=0).split_text(combined_text)
                metadata = {"source": url.split('?')[0]}
                data_with_metadata = [Document(page_content=chunk, metadata=metadata) for chunk in chunks]
                print('data_with_metadata ----------- ', data_with_metadata)
                Chroma.from_documents(
                    documents=data_with_metadata,
                    embedding=OpenAIEmbeddings(api_key=open_api_key, model="text-embedding-ada-002"),
                    client=chroma_client,
                    collection_name=uuid,
                )
                return {"status": 200, "info": "OK"}, 200
        else:
            return {'status': 400, "info": "Unsupported file type"}, 200

    except requests.exceptions.RequestException as e:
        return {"status": 400, "info": e}, 200
    except RateLimitError as e:
        raise ValueError(json.loads(e.response.text)["error"]["message"])
    except Exception as e:
        return {"status": 500, "info": e}, 200

==========================================================================
Code for querying data
---------------------
import os
import json
from dotenv import load_dotenv
from chromadb import HttpClient
from chromadb.config import Settings
from langchain.prompts import PromptTemplate
from langchain.chat_models import ChatOpenAI
from langchain.vectorstores.chroma import Chroma
from langchain.embeddings import OpenAIEmbeddings
import config
from openai._exceptions import RateLimitError
from langchain.schema.messages import HumanMessage, SystemMessage
import re
import unicodedata


def retrieve_answer(question: str, uuid, open_api_key):
    question = clean_user_query(question=question)

    try:

        llm = get_llm_object(open_api_key)

        chroma_client = get_chroma_client()
        
        
        # Normalize Arabic text for query
        question = normalize_arabic(question)

        # Check collection exist or not
        try:
            uuid = str(uuid)
            chroma_client.get_collection(name=uuid)
        except Exception as e:
            raise ValueError(json.loads(e.args[0]).get('error'))

        chroma_initializer = Chroma(
            embedding_function=OpenAIEmbeddings(api_key=open_api_key),
            client=chroma_client,
            collection_name=uuid,
        )

        vector_similarity_search = chroma_initializer.similarity_search(k=6, query=question)
        print('Vector Similarity search results ...... ', vector_similarity_search)
        
         # Ensure proper formatting for the retrieved context
        context = "\n".join(
            [doc.page_content for doc in vector_similarity_search]
        )

        print('context recieved here is ------- \n\n', context, '\n\n\n')

        template = """
            You are a helpful assistant who provides data based on the provided context.
            Use strictly the context to search on question provided below.
            If you don't know the answer, Just say that "It's out current context".
            Avoid the cached data.
            Avoid any data from other sources apart from the context.
            Strictly convert the question to english and then query the context.
            Keep the answer concise and short and simple 250 words.
            The confidence level is value of how much the question is related to the context.
            The default confidence level is 0 
            Higher the question matching to context greater the confidence level which is in between 0 and 1.
            The output should be in json format with message, confidence_level and source in any condition
            Context: {context} 
            Question: {question}
            Helpful Answer: 
            """

        prompt = PromptTemplate(template=template, input_variables=["question", "context"]).format(question=question,
                                                                                                   context=vector_similarity_search)

        message = [
            SystemMessage(content=prompt),
            HumanMessage(content=question),
        ]

        res = llm(message)

        embeddings_sources = chroma_initializer.get().get("metadatas")

        result_to_pass = res.content.strip('"')

        try:
            if result_to_pass is not None:
                result_new = json.loads(result_to_pass)
                source_dict = {"source": result_new.get("source")}

                if source_dict not in embeddings_sources:
                    result_new = config.api["gpt-out-of-context-message"]
            else:
                raise ValueError("No answer found.")
        except json.JSONDecodeError:
            result_new = config.api["gpt-out-of-context-message"]
        return result_new
    except RateLimitError as e:
        raise ValueError(json.loads(e.response.text)["error"]["message"])
    except ValueError as e:
        raise e
    except Exception as e:
        raise Exception(str(e))

Description

We are trying to do RAG on a Arabic document.
The issue we are facing here is that when we try to query data from the retrieved context from chroma DB we are getting typo issue for this specific language Arabic LLM.

Please Guide us on how we can resolve the same.

This is a production issue and we needed it to be solved ASAP.

System Info

langchain==0.0.346
langchain-core==0.0.10
langchainhub==0.1.14
langsmith==0.0.69