Skip existing embeddings

vormkracht10 · Dec 1, 2023 · 0a97603 · 0a97603
1 parent b000d54
commit 0a97603
Showing 1 changed file with 11 additions and 1 deletion.
diff --git a/src/Engines/OpenAiEngine.php b/src/Engines/OpenAiEngine.php
@@ -60,6 +60,16 @@ private function saveObjects($objects)
     {
         foreach ($objects as $object) {
 
+            $exists = DB::connection(config('embed.database.connection'))
+                ->table(config('embed.database.table'))
+                ->where('foreign_id', $object['objectID'])
+                ->where('content', $object['content'])
+                ->exists();
+
+            if ($exists) {
+                continue;
+            }
+
             $embed = $this->embed($object['content']);
 
             DB::connection(config('embed.database.connection'))
@@ -68,7 +78,7 @@ private function saveObjects($objects)
                     'foreign_id' => $object['objectID'],
                 ], [
                     'content' => $object['content'],
-                    'embedding' => '['.implode(',', $embed).']',
+                    'embedding' => '['.implode(',', $embed).']'
                 ]);
         }
     }