chore: remove tsne unused code

api/pyproject.toml

@@ -163,7 +163,6 @@ redis = { version = "~5.0.3", extras = ["hiredis"] }
 replicate = "~0.22.0"
 resend = "~0.7.0"
 safetensors = "~0.4.3"
-scikit-learn = "1.2.2"
 sentry-sdk = { version = "~1.39.2", extras = ["flask"] }
 sqlalchemy = "~2.0.29"
 tencentcloud-sdk-python-hunyuan = "~3.0.1158"

api/services/hit_testing_service.py

@@ -1,9 +1,6 @@
 import logging
 import time
 
-import numpy as np
-from sklearn.manifold import TSNE
-
 from core.rag.datasource.retrieval_service import RetrievalService
 from core.rag.models.document import Document
 from core.rag.retrieval.retrival_methods import RetrievalMethod
@@ -101,29 +98,6 @@ def compact_retrieve_response(cls, dataset: Dataset, query: str, documents: list
             "records": records
         }
 
-    @classmethod
-    def get_tsne_positions_from_embeddings(cls, embeddings: list):
-        embedding_length = len(embeddings)
-        if embedding_length <= 1:
-            return [{'x': 0, 'y': 0}]
-
-        noise = np.random.normal(0, 1e-4, np.array(embeddings).shape)
-        concatenate_data = np.array(embeddings) + noise
-        concatenate_data = concatenate_data.reshape(embedding_length, -1)
-
-        perplexity = embedding_length / 2 + 1
-        if perplexity >= embedding_length:
-            perplexity = max(embedding_length - 1, 1)
-
-        tsne = TSNE(n_components=2, perplexity=perplexity, early_exaggeration=12.0)
-        data_tsne = tsne.fit_transform(concatenate_data)
-
-        tsne_position_data = []
-        for i in range(len(data_tsne)):
-            tsne_position_data.append({'x': float(data_tsne[i][0]), 'y': float(data_tsne[i][1])})
-
-        return tsne_position_data
-
     @classmethod
     def hit_testing_args_check(cls, args):
         query = args['query']