Merge pull request #188 from amosproj/refactoring/156_final_code_clean

Refactoring/156 final code clean

QAChat/Common/bucket_managing.py

@@ -1,12 +1,12 @@
-import os
+# SPDX-License-Identifier: MIT
+# SPDX-FileCopyrightText: 2023 Felix Nützel
+# SPDX-FileCopyrightText: 2023 Jesse Palarus
 
 from dotenv import load_dotenv
-from google.cloud import storage
 from weaviate.embedded import EmbeddedOptions
 import weaviate
 from QAChat.Common.init_db import clear_db
 from get_tokens import get_tokens_path
-import shutil
 
 load_dotenv(get_tokens_path())
 bucket_name = "qabot_db_data"

QAChat/Common/db_info.py

@@ -1,3 +1,4 @@
+# SPDX-License-Identifier: MIT
 # SPDX-FileCopyrightText: 2023 Felix Nützel
 
 from weaviate.embedded import EmbeddedOptions

QAChat/Common/deepL_translator.py

@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: MIT
 # SPDX-FileCopyrightText: 2023 Emanuel Erben
 # SPDX-FileCopyrightText: 2023 Felix Nützel
+# SPDX-FileCopyrightText: 2023 Jesse Palarus
 
 import os
 
@@ -39,10 +40,16 @@ def __init__(self):
         if "language_detector" not in self.muulti_lang_nlp.pipe_names:
             self.muulti_lang_nlp.add_pipe("language_detector", last=True)
 
-    def translate_to(self, text, target_lang):
-        doc = self.muulti_lang_nlp(text)
-        if doc._.language["language"] == "en" and doc._.language["score"] > 0.8:
-            return Result(text, "EN_US")
+    def translate_to(self, text, target_lang, use_spacy_to_detect_lang_if_needed=True):
+        if use_spacy_to_detect_lang_if_needed:
+            doc = self.muulti_lang_nlp(text)
+            if (
+                doc._.language["language"] == "en"
+                and doc._.language["score"] > 0.8
+                and target_lang == "EN-US"
+            ):
+                return Result(text, "EN_US")
+
         result = self.translator.translate_text(
             text, target_lang=target_lang, ignore_tags="name"
         )

QAChat/Common/init_db.py

@@ -1,3 +1,4 @@
+# SPDX-License-Identifier: MIT
 # SPDX-FileCopyrightText: 2023 Felix Nützel
 def init_db(weaviate_client):
     global client

QAChat/Data_Processing/data_preprocessor.py

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: MIT
-# SPDX-FileCopyrightText: 2023 Jesse Tim Palarus
+# SPDX-FileCopyrightText: 2023 Jesse Palarus
 # SPDX-FileCopyrightText: 2023 Amela Pucic
 
 from abc import ABC, abstractmethod

QAChat/Data_Processing/document_embedder.py

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: MIT
-# SPDX-FileCopyrightText: 2023 Jesse Tim Palarus
+# SPDX-FileCopyrightText: 2023 Jesse Palarus
 # SPDX-FileCopyrightText: 2023 Amela Pucic
 # SPDX-FileCopyrightText: 2023 Felix Nützel
 # SPDX-FileCopyrightText: 2023 Emanuel Erben

QAChat/Data_Processing/dummy_preprocessor.py

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: MIT
-# SPDX-FileCopyrightText: 2023 Jesse Tim Palarus
+# SPDX-FileCopyrightText: 2023 Jesse Palarus
 # SPDX-FileCopyrightText: 2023 Amela Pucic
 
 from datetime import datetime

QAChat/Data_Processing/google_doc_preprocessor.py

@@ -5,12 +5,9 @@
 
 from __future__ import print_function
 import io
-from pdf_reader import PDFReader
 from googleapiclient.discovery import build
 from googleapiclient.errors import HttpError
 from googleapiclient.http import MediaIoBaseDownload
-from google_auth_oauthlib.flow import InstalledAppFlow
-from QAChat.Common.init_db import init_db
 from QAChat.Data_Processing.pdf_reader import PDFReader
 from google.oauth2 import service_account
 import os

QAChat/Data_Processing/main.py

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: MIT
-# SPDX-FileCopyrightText: 2023 Jesse Tim Palarus
+# SPDX-FileCopyrightText: 2023 Jesse Palarus
 # SPDX-FileCopyrightText: 2023 Amela Pucic
 
 from document_embedder import DocumentEmbedder, DataSource

QAChat/Data_Processing/requirements.txt

@@ -2,7 +2,6 @@ langchain
 nltk
 deepl
 python-dateutil
-supabase
 pytesseract
 pdf2image
 pdfminer.six

QAChat/QA_Bot/qa_bot.py

@@ -4,9 +4,7 @@
 # SPDX-FileCopyrightText: 2023 Jesse Palarus
 # SPDX-FileCopyrightText: 2023 Amela Pucic
 
-import os
 from time import time
-from typing import List
 
 from huggingface_hub import hf_hub_download
 from langchain import LlamaCpp, PromptTemplate
@@ -152,7 +150,9 @@ def __sim_search(self, question: str) -> List[str]:
         ]
 
     def translate_text(self, question, language="EN-US"):
-        return self.translator.translate_to(question, language)
+        return self.translator.translate_to(
+            question, language, use_spacy_to_detect_lang_if_needed=False
+        )
 
     def answer_question(self, question: str, handler: StreamLLMCallbackHandler | None):
         """
@@ -183,7 +183,9 @@ def answer_question(self, question: str, handler: StreamLLMCallbackHandler | Non
             translated_question, context, handler
         )
         print(f"Answer: {answer}")
-        answer = self.translate_text(answer, translation.detected_source_lang).text
+        if translation.detected_source_lang != "EN-US":
+            answer = self.translate_text(answer, translation.detected_source_lang).text
+
         print(f"Translated answer: {answer}")
         return {
             "answer": answer,

QAChat/QA_Bot/requirements.txt

@@ -1,6 +1,5 @@
 flask
 langchain
-supabase
 InstructorEmbedding
 sentence-transformers
 deepl

QAChat/QA_Bot/setup_server.py

@@ -1,7 +1,6 @@
 # SPDX-License-Identifier: MIT
 # SPDX-FileCopyrightText: 2023 Jesse Palarus
 
-import json
 import threading
 
 from flask import Flask, request, stream_with_context, Response

QAChat/QA_Bot/stream_LLM_callback_handler.py

@@ -22,5 +22,7 @@ def on_llm_new_token(self, token: str, **kwargs):
 
     def send_response(self, text):
         if self.lang != "EN-US":
-            text = self.translator.translate_to(text, self.lang).text
+            text = self.translator.translate_to(
+                text, self.lang, use_spacy_to_detect_lang_if_needed=False
+            ).text
         return json.dumps({"text": text}) + "\n"

QAChat/Slack_Bot/qa_agent.py

@@ -3,10 +3,7 @@
 # SPDX-FileCopyrightText: 2023 Felix Nützel
 # SPDX-FileCopyrightText: 2023 Jesse Palarus
 import os
-import queue
 import re
-import threading
-import time
 
 from threading import Thread
 
@@ -44,8 +41,11 @@ def receive_question(self, question, say, channel_id):
             ),
         )
 
-        for answer in self.api_interface.request(question):
-            asynchronous_processor.add(answer)
+        try:
+            for answer in self.api_interface.request(question):
+                asynchronous_processor.add(answer)
+        except Exception as e:
+            asynchronous_processor.add(f"Ohh there is was an error...\n{e}")
         asynchronous_processor.end()
 
     def process_question(self, body, say):

QAChat/Translation_Testing/translation_tests.py

@@ -1,3 +1,7 @@
+# SPDX-License-Identifier: MIT
+# SPDX-FileCopyrightText: 2023 Emanuel Erben
+# SPDX-FileCopyrightText: 2023 Felix Nützel
+
 import csv
 from datetime import datetime
 

Testing/Performace/qa_bot_performance.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: MIT
+# SPDX-FileCopyrightText: 2023 Jesse Palarus
+
 import random
 import time
 from unittest.mock import Mock, create_autospec
@@ -86,7 +89,7 @@ def test_embedding_speed():
     print(f"Embedding test time: {elapsed_time} seconds.")
 
 
-def test_supabase_speed():
+def test_database_speed():
     qa_bot = QABot(
         translator=mock,
         model=mock,
@@ -99,7 +102,7 @@ def test_supabase_speed():
         qa_bot.database.similarity_search_by_vector(random_embedding, 3)
     end_time = time.time()
     elapsed_time = end_time - start_time
-    print(f"Supabase reading time: {elapsed_time} seconds.")
+    print(f"LLM reading time: {elapsed_time} seconds.")
 
 
 def time_this(original_method):
@@ -141,7 +144,7 @@ def test_overall_performance():
     sum_of_chars = 0
 
     for question in questions_en:
-        output = qa_bot.answer_question(question)
+        output = qa_bot.answer_question(question, None)
         sum_of_chars += len(output["answer"])
     end_time = time.time()
     elapsed_time = end_time - start_time
@@ -155,5 +158,5 @@ def test_overall_performance():
     test_llm_speed()
     test_deepl_speed()
     test_embedding_speed()
-    test_supabase_speed()
+    test_database_speed()
     test_overall_performance()

Testing/confluence_integration_test.py

@@ -13,18 +13,13 @@
 
 from QAChat.Data_Processing.pdf_reader import PDFReader
 
-
 load_dotenv("../QAChat/tokens.env")
 
 # Get Confluence API credentials from environment variables
 CONFLUENCE_ADDRESS = os.getenv("CONFLUENCE_ADDRESS")
 CONFLUENCE_USERNAME = os.getenv("CONFLUENCE_USERNAME")
 CONFLUENCE_TOKEN = os.getenv("CONFLUENCE_TOKEN")
 
-# Get Supabase API credentials from environment variables
-SUPABASE_URL = os.getenv("SUPABASE_URL")
-SUPABASE_SERVICE_KEY = os.getenv("SUPABASE_SERVICE_KEY")
-
 
 class ConfluenceIntegrationTest(unittest.TestCase):
     def setUp(self):

Testing/llm_test.py

@@ -1,3 +1,7 @@
+# SPDX-License-Identifier: MIT
+# SPDX-FileCopyrightText: 2023 Amela Pucic
+# SPDX-FileCopyrightText: 2023 Jesse Palarus
+
 from unittest.mock import Mock
 
 from QAChat.QA_Bot.qa_bot import QABot

get_tokens.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: MIT
+# SPDX-FileCopyrightText: 2023 Felix Nützel
+
 import os