vectorization on demand

src/main/java/io/stargate/sgv2/jsonapi/api/model/command/clause/update/AddToSetOperation.java

@@ -91,7 +91,7 @@ private static boolean hasModifier(ObjectNode node) {
   }
 
   @Override
-  public boolean updateDocument(ObjectNode doc) {
+  public UpdateOperationResult updateDocument(ObjectNode doc) {
     boolean modified = false;
     for (Action action : actions) {
       PathMatch target = action.locator().findOrCreate(doc);
@@ -120,7 +120,7 @@ public boolean updateDocument(ObjectNode doc) {
       }
     }
 
-    return modified;
+    return new UpdateOperationResult(modified, List.of());
   }
 
   private boolean addToSet(ArrayNode set, JsonNode elementToAdd) {

src/main/java/io/stargate/sgv2/jsonapi/api/model/command/clause/update/CurrentDateOperation.java

@@ -48,7 +48,7 @@ private static void verifyIsTrueOrDate(JsonNode value) {
   }
 
   @Override
-  public boolean updateDocument(ObjectNode doc) {
+  public UpdateOperationResult updateDocument(ObjectNode doc) {
     boolean modified = false;
     final long now = System.currentTimeMillis();
     ObjectNode newValue = JsonUtil.createEJSonDate(doc, now);
@@ -62,7 +62,7 @@ public boolean updateDocument(ObjectNode doc) {
         modified = true;
       }
     }
-    return modified;
+    return new UpdateOperationResult(modified, List.of());
   }
 
   record Action(PathMatchLocator locator) implements ActionWithLocator {}

src/main/java/io/stargate/sgv2/jsonapi/api/model/command/clause/update/EmbeddingUpdateOperation.java

@@ -0,0 +1,35 @@
+package io.stargate.sgv2.jsonapi.api.model.command.clause.update;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.stargate.sgv2.jsonapi.config.constants.DocumentConstants;
+
+/**
+ * This operation will be used to pass along $vectorize content and provides a method to update a
+ * document with new vector
+ *
+ * @param vectorizeContent, $vectorize content needed to be vectorized
+ */
+public record EmbeddingUpdateOperation(String vectorizeContent) {
+
+  /**
+   * update the document with corresponding vector
+   *
+   * @param doc Document to update
+   * @param dataVectorizer dataVectorizer
+   * @return Uni<Boolean> modified
+   */
+  public void updateDocument(JsonNode doc, float[] vector) {
+    ObjectMapper objectMapper = new ObjectMapper();
+    JsonNodeFactory nodeFactory = objectMapper.getNodeFactory();
+    final JsonNode vectorJsonNode = doc.get(DocumentConstants.Fields.VECTOR_EMBEDDING_FIELD);
+    final ArrayNode arrayNode = nodeFactory.arrayNode(vector.length);
+    for (float listValue : vector) {
+      arrayNode.add(nodeFactory.numberNode(listValue));
+    }
+    ((ObjectNode) doc).put(DocumentConstants.Fields.VECTOR_EMBEDDING_FIELD, arrayNode);
+  }
+}

src/main/java/io/stargate/sgv2/jsonapi/api/model/command/clause/update/IncOperation.java

@@ -42,7 +42,7 @@ public static IncOperation construct(ObjectNode args) {
   }
 
   @Override
-  public boolean updateDocument(ObjectNode doc) {
+  public UpdateOperationResult updateDocument(ObjectNode doc) {
     // Almost always changes, except if adding zero; need to track
     boolean modified = false;
     for (Action action : actions) {
@@ -69,7 +69,7 @@ public boolean updateDocument(ObjectNode doc) {
       }
     }
 
-    return modified;
+    return new UpdateOperationResult(modified, List.of());
   }
 
   private JsonNode addNumbers(ObjectNode doc, NumericNode nr1, NumericNode nr2) {

src/main/java/io/stargate/sgv2/jsonapi/api/model/command/clause/update/MinMaxOperation.java

@@ -46,7 +46,7 @@ private static MinMaxOperation construct(ObjectNode args, UpdateOperator oper, b
   }
 
   @Override
-  public boolean updateDocument(ObjectNode doc) {
+  public UpdateOperationResult updateDocument(ObjectNode doc) {
     // Almost always changes, except if adding zero; need to track
     boolean modified = false;
     for (Action action : actions) {
@@ -66,7 +66,7 @@ public boolean updateDocument(ObjectNode doc) {
       }
     }
 
-    return modified;
+    return new UpdateOperationResult(modified, List.of());
   }
 
   private boolean shouldReplace(JsonNode oldValue, JsonNode newValue) {

src/main/java/io/stargate/sgv2/jsonapi/api/model/command/clause/update/MulOperation.java

@@ -43,7 +43,7 @@ public static MulOperation construct(ObjectNode args) {
   }
 
   @Override
-  public boolean updateDocument(ObjectNode doc) {
+  public UpdateOperationResult updateDocument(ObjectNode doc) {
     boolean modified = false;
     for (Action action : actions) {
       final NumericNode multiplier = action.value;
@@ -68,7 +68,7 @@ public boolean updateDocument(ObjectNode doc) {
       }
     }
 
-    return modified;
+    return new UpdateOperationResult(modified, List.of());
   }
 
   private JsonNode multiply(ObjectNode doc, JsonNode oldValue, JsonNode multiplierValue) {

src/main/java/io/stargate/sgv2/jsonapi/api/model/command/clause/update/PopOperation.java

@@ -50,7 +50,7 @@ public static PopOperation construct(ObjectNode args) {
   }
 
   @Override
-  public boolean updateDocument(ObjectNode doc) {
+  public UpdateOperationResult updateDocument(ObjectNode doc) {
     boolean changes = false;
     for (Action action : actions) {
       PathMatch target = action.locator().findIfExists(doc);
@@ -78,7 +78,7 @@ public boolean updateDocument(ObjectNode doc) {
             target.fullPath(), value.getNodeType());
       }
     }
-    return changes;
+    return new UpdateOperationResult(changes, List.of());
   }
 
   /** Value class for per-field Pop operation definitions. */

src/main/java/io/stargate/sgv2/jsonapi/api/model/command/clause/update/PushOperation.java

@@ -108,7 +108,7 @@ private static boolean hasModifier(ObjectNode node) {
   }
 
   @Override
-  public boolean updateDocument(ObjectNode doc) {
+  public UpdateOperationResult updateDocument(ObjectNode doc) {
     for (Action action : actions) {
       final JsonNode toAdd = action.value;
 
@@ -153,7 +153,7 @@ public boolean updateDocument(ObjectNode doc) {
     }
 
     // Every valid update operation modifies document so need just one:
-    return !actions.isEmpty();
+    return new UpdateOperationResult(!actions.isEmpty(), List.of());
   }
 
   // Just needed for tests

src/main/java/io/stargate/sgv2/jsonapi/api/model/command/clause/update/RenameOperation.java

@@ -36,7 +36,7 @@ public static RenameOperation construct(ObjectNode args) {
   }
 
   @Override
-  public boolean updateDocument(ObjectNode doc) {
+  public UpdateOperationResult updateDocument(ObjectNode doc) {
     boolean modified = false;
     for (Action action : actions) {
       PathMatch src = action.sourceLocator().findIfExists(doc);
@@ -62,7 +62,7 @@ public boolean updateDocument(ObjectNode doc) {
         dst.replaceValue(value);
       }
     }
-    return modified;
+    return new UpdateOperationResult(modified, List.of());
   }
 
   // Unlike most operations, we have 2 locators (src, dest), use explicit names

src/main/java/io/stargate/sgv2/jsonapi/api/model/command/clause/update/SetOperation.java

@@ -3,14 +3,11 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import io.stargate.sgv2.jsonapi.config.constants.DocumentConstants;
+import io.stargate.sgv2.jsonapi.exception.ErrorCode;
 import io.stargate.sgv2.jsonapi.util.JsonUtil;
 import io.stargate.sgv2.jsonapi.util.PathMatch;
 import io.stargate.sgv2.jsonapi.util.PathMatchLocator;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Objects;
-import java.util.Set;
+import java.util.*;
 
 /**
  * Implementation of {@code $set} update operation used to assign values to document fields; also
@@ -89,27 +86,48 @@ public boolean shouldApplyIf(boolean isInsert) {
   }
 
   @Override
-  public boolean updateDocument(ObjectNode doc) {
+  public UpdateOperationResult updateDocument(ObjectNode doc) {
     boolean modified = false;
     Set<String> setPaths = new HashSet<>();
     actions.stream().forEach(action -> setPaths.add(action.locator().path()));
+    List<EmbeddingUpdateOperation> embeddingUpdateOperationList = new ArrayList<>();
+
     for (Action action : actions) {
       PathMatch target = action.locator().findOrCreate(doc);
       JsonNode newValue = action.value();
       JsonNode oldValue = target.valueNode();
       // Modify if no old value OR new value differs, as per Mongo-equality rules
       if ((oldValue == null) || !JsonUtil.equalsOrdered(oldValue, newValue)) {
+        // replace old value with matched path new value
         target.replaceValue(newValue);
-        // $vector is updated and $vectorize is not updated, remove the $vectorize field in the
-        // document
+
+        // $vector is updated and $vectorize is not updated, remove $vectorize in the document
         if (DocumentConstants.Fields.VECTOR_EMBEDDING_FIELD.equals(action.locator().path())
             && !setPaths.contains(DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD)) {
           doc.remove(DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD);
         }
+
+        // $vectorize
+        if (DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD.equals(action.locator().path())) {
+          if (newValue.isNull()) {
+            // if $vectorize is null value, update $vector as null
+            doc.put(DocumentConstants.Fields.VECTOR_EMBEDDING_FIELD, (String) null);
+            doc.putNull(DocumentConstants.Fields.VECTOR_EMBEDDING_FIELD);
+          } else if (!newValue.isTextual()) {
+            // if $vectorize is not textual value
+            throw ErrorCode.INVALID_VECTORIZE_VALUE_TYPE.toApiException();
+          } else if (newValue.asText().isBlank()) {
+            // $vectorize is blank text value, set $vector as null value, no need to vectorize
+            doc.putNull(DocumentConstants.Fields.VECTOR_EMBEDDING_FIELD);
+          } else {
+            // if $vectorize is textual and not blank, create embeddingUpdateOperation
+            embeddingUpdateOperationList.add(new EmbeddingUpdateOperation(newValue.asText()));
+          }
+        }
         modified = true;
       }
     }
-    return modified;
+    return new UpdateOperationResult(modified, embeddingUpdateOperationList);
   }
 
   // Needed because some unit tests check for equality

src/main/java/io/stargate/sgv2/jsonapi/api/model/command/clause/update/UnsetOperation.java

@@ -28,7 +28,7 @@ public static UnsetOperation construct(ObjectNode args) {
   }
 
   @Override
-  public boolean updateDocument(ObjectNode doc) {
+  public UpdateOperationResult updateDocument(ObjectNode doc) {
     boolean modified = false;
     Set<String> unsetPaths = new HashSet<>();
     actions.stream().forEach(action -> unsetPaths.add(action.locator().path()));
@@ -40,7 +40,11 @@ public boolean updateDocument(ObjectNode doc) {
     if (modified && unsetPaths.contains(DocumentConstants.Fields.VECTOR_EMBEDDING_FIELD)) {
       doc.remove(DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD);
     }
-    return modified;
+    // $vectorize field is unset, remove $vector field value
+    if (modified && unsetPaths.contains(DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD)) {
+      doc.remove(DocumentConstants.Fields.VECTOR_EMBEDDING_FIELD);
+    }
+    return new UpdateOperationResult(modified, List.of());
   }
 
   record Action(PathMatchLocator locator) implements ActionWithLocator {}

src/main/java/io/stargate/sgv2/jsonapi/api/model/command/clause/update/UpdateOperation.java

@@ -27,9 +27,9 @@ public List<A> actions() {
    * Method called to apply operation to given document.
    *
    * @param doc Document to apply operation to
-   * @return True if document was modified by operation; false if not.
+   * @return UpdateOperationResult
    */
-  public abstract boolean updateDocument(ObjectNode doc);
+  public abstract UpdateOperationResult updateDocument(ObjectNode doc);
 
   /**
    * Method called to see if update operator should be applied for specific kind of update:
@@ -99,4 +99,12 @@ public int compare(ActionWithLocator o1, ActionWithLocator o2) {
       return o1.path().compareTo(o2.path());
     }
   }
+
+  /**
+   * Abstract method updateDocument will return a UpdateOperationResult. UpdateOperationResult
+   * indicated the doc is modified or not, also a List of embeddingUpdateOperation, empty is there
+   * is not any embeddingUpdateOperations
+   */
+  public record UpdateOperationResult(
+      boolean modified, List<EmbeddingUpdateOperation> embeddingUpdateOperations) {}
 }

src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/UpdateClauseDeserializer.java

@@ -7,11 +7,10 @@
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import io.stargate.sgv2.jsonapi.api.model.command.clause.update.UpdateClause;
 import io.stargate.sgv2.jsonapi.api.model.command.clause.update.UpdateOperator;
+import io.stargate.sgv2.jsonapi.config.constants.DocumentConstants;
 import io.stargate.sgv2.jsonapi.exception.ErrorCode;
 import java.io.IOException;
-import java.util.EnumMap;
-import java.util.Iterator;
-import java.util.Map;
+import java.util.*;
 
 /** {@link StdDeserializer} for the {@link UpdateClause}. */
 public class UpdateClauseDeserializer extends StdDeserializer<UpdateClause> {
@@ -52,6 +51,22 @@ public UpdateClause deserialize(
       }
       updateDefs.put(oper, (ObjectNode) operationArg);
     }
+    validateUpdateDefs(updateDefs);
     return new UpdateClause(updateDefs);
   }
+
+  public void validateUpdateDefs(EnumMap<UpdateOperator, ObjectNode> updateDefs) {
+    // check1: can not unset $vectorize and $vector at the same time
+    List<ObjectNode> checkUpdateOperationNodes = new ArrayList<>();
+    checkUpdateOperationNodes.add(updateDefs.get(UpdateOperator.UNSET));
+    checkUpdateOperationNodes.add(updateDefs.get(UpdateOperator.SET));
+    for (ObjectNode checkUpdateOperationNode : checkUpdateOperationNodes) {
+      if (checkUpdateOperationNode != null
+          && checkUpdateOperationNode.has(DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD)) {
+        if (checkUpdateOperationNode.has(DocumentConstants.Fields.VECTOR_EMBEDDING_FIELD)) {
+          throw ErrorCode.INVALID_USAGE_OF_VECTORIZE.toApiException();
+        }
+      }
+    }
+  }
 }