From 0a9760326193cfeabba12a607523efacda98a8e0 Mon Sep 17 00:00:00 2001
From: Mathieu <mathieu@fellicht.nl>
Date: Fri, 1 Dec 2023 16:54:01 +0100
Subject: [PATCH] Skip existing embeddings

---
 src/Engines/OpenAiEngine.php | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/Engines/OpenAiEngine.php b/src/Engines/OpenAiEngine.php
index 4bf57e7..fb2728a 100644
--- a/src/Engines/OpenAiEngine.php
+++ b/src/Engines/OpenAiEngine.php
@@ -60,6 +60,16 @@ private function saveObjects($objects)
     {
         foreach ($objects as $object) {
 
+            $exists = DB::connection(config('embed.database.connection'))
+                ->table(config('embed.database.table'))
+                ->where('foreign_id', $object['objectID'])
+                ->where('content', $object['content'])
+                ->exists();
+
+            if ($exists) {
+                continue;
+            }
+
             $embed = $this->embed($object['content']);
 
             DB::connection(config('embed.database.connection'))
@@ -68,7 +78,7 @@ private function saveObjects($objects)
                     'foreign_id' => $object['objectID'],
                 ], [
                     'content' => $object['content'],
-                    'embedding' => '['.implode(',', $embed).']',
+                    'embedding' => '['.implode(',', $embed).']'
                 ]);
         }
     }