From 37b97fbd9124b8b66afb9a791dd503b2f3e344ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Ja=C5=82oszy=C5=84ski?= Date: Mon, 23 Dec 2024 22:44:28 +0100 Subject: [PATCH] 280: Rename route, add models supported langages endpoint --- nlp/docs/__pycache__/conf.cpython-310.pyc | Bin 0 -> 794 bytes nlp/src/base_models.py | 26 +++++++++++++++ nlp/src/main.py | 38 ++++++++++++++++++---- nlp/src/models_loading.py | 4 +-- nlp/version_models.json | 7 ++-- 5 files changed, 65 insertions(+), 10 deletions(-) create mode 100644 nlp/docs/__pycache__/conf.cpython-310.pyc diff --git a/nlp/docs/__pycache__/conf.cpython-310.pyc b/nlp/docs/__pycache__/conf.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6c2450dded17992585b4f268194af5a2c0844ce6 GIT binary patch literal 794 zcmYjPv2N5r5Z%4AeYSI#L_xZWB0{1#3E3f{M2Lb02_i&NAUD?9J2@NHyJmL{IrB4w zj*9=;mI{e4pki!`aBF$S?@2TB{C3o86%#zaznpJ>1cdw?!2UswJjd5?44iN(NH%4f zQK0W?;DkFKh&DNMqPK1yC(=9gA!a5IqZc&f{_<+-h(fvZjI zVy=W>x?#3*3)YzFgTu9`jde$rZA8v^qhxJSi;G91ZrB#Zmr5v^UxO?WV5PPnJB4c8 zQUmHLafGypOjj6N+ivU-T4p{B%{Rwxri`1LopD|}%nxdbK3LRIn2}htTD^zTx~Np` zPS@MYx&R%nly(aZ0+<08-#KO6;hiIBDt0`Eo6*eyp>fi{ZRSs@_*KjS;m4{%Gy@aJJ#sRmL51xsIErSHa-Nu_90+yRr8N?s@J&V^4K h#XAAdC{~*L7(x=#HT5Eo4o~O@QAoX+=X?Ir{|l5#30wdG literal 0 HcmV?d00001 diff --git a/nlp/src/base_models.py b/nlp/src/base_models.py index 6ab4b03..31ff682 100644 --- a/nlp/src/base_models.py +++ b/nlp/src/base_models.py @@ -46,3 +46,29 @@ def json(self): "metadata": self.metadata, "results": self.results } + + +class ModelSupportedLanguages(BaseModel): + """Response model for used language""" + sentiment: List[str] + language: List[str] + sarcasm: List[str] + keywords: List[str] + spam: List[str] + political: List[str] + hateSpeech: List[str] + clickbait: List[str] + llmDetection: List[str] + + def json(self): + return { + "sentiment": self.sentiment, + "language": self.language, + "sarcasm": self.sarcasm, + "keywords": self.keywords, + "spam": self.spam, + "political": self.political, + "hateSpeech": self.hateSpeech, + "clickbait": self.clickbait, + "llmDetection": self.llmDetection + } diff --git a/nlp/src/main.py b/nlp/src/main.py index 0ffc769..3bd9fa7 100644 --- a/nlp/src/main.py +++ b/nlp/src/main.py @@ -5,6 +5,8 @@ from contextlib import asynccontextmanager from fastapi import FastAPI, HTTPException from fastapi.params import Depends + +from docs.conf import language from src.base_models import * from src.logger_config import logger from src.models_loading import * @@ -31,7 +33,7 @@ async def lifespan(application: FastAPI): raise models = [ - "language", "sentiment", "ai_detector", "sarcasm", "spam", + "language", "sentiment", "llm_detection", "sarcasm", "spam", "political", "hate_speech", "clickbait", "keywords" ] @@ -388,7 +390,7 @@ def process_fn(text): ).json() -@app.post("/ai-detector", response_model=TextResponse, +@app.post("/llm-detection", response_model=TextResponse, dependencies=[Depends(auth.get_api_key)]) async def get_ai_bots(request: TextRequest): """ @@ -408,11 +410,11 @@ def process_fn(text): prediction = gl.ai_detector_pipeline_english(text) confidence = prediction[0]["score"] if confidence > 0.5: - label="AI" + label = "AI" confidence = confidence_output(confidence) else: - label="Human" - confidence = confidence_output(1-confidence) + label = "Human" + confidence = confidence_output(1 - confidence) result = AnalysisResult( labels=[label], @@ -423,9 +425,33 @@ def process_fn(text): results, generation_time = measure_execution_time( lambda: process_inputs(process_fn, request.text)) - return TextResponse( kind="aiDetector", metadata=Metadata(generated_in=generation_time), results=results ).json() + + +@app.post("/info/langs", response_model=ModelSupportedLanguages, + dependencies=[Depends(auth.get_api_key)]) +async def get_languages(): + """ + Get languages used in the application. + + Returns: + ModelSupportedLanguages: Languages used in the application. + """ + with open("version_models.json", "r") as file: + data = json.load(file) + + return ModelSupportedLanguages( + sentiment=data.get("sentiment", []), + language=data.get("language", []), + sarcasm=data.get("sarcasm", []), + keywords=data.get("keywords", []), + spam=data.get("spam", []), + political=data.get("political", []), + hateSpeech=data.get("hate_speech", []), + clickbait=data.get("clickbait", []), + llmDetection=data.get("llm_detection", []) + ).json() diff --git a/nlp/src/models_loading.py b/nlp/src/models_loading.py index 633151a..57102ba 100644 --- a/nlp/src/models_loading.py +++ b/nlp/src/models_loading.py @@ -66,6 +66,6 @@ def load_model(model_name: str): elif model_name == "clickbait": g.clickbait_pipeline = pipeline("text-classification", model=model, tokenizer=tokenizer) - elif model_name == "ai_detector": + elif model_name == "llm_detection": g.ai_detector_pipeline_english = pipeline("text-classification", - model=model, tokenizer=tokenizer) + model=model, tokenizer=tokenizer) diff --git a/nlp/version_models.json b/nlp/version_models.json index d1ad799..7e602fe 100644 --- a/nlp/version_models.json +++ b/nlp/version_models.json @@ -1,6 +1,6 @@ { "language": [ - "english" + "all" ], "sentiment": [ "english", @@ -22,7 +22,10 @@ "clickbait": [ "english" ], - "ai_detector": [ + "llm_detection": [ "english" + ], + "keywords": [ + "all" ] } \ No newline at end of file