Update Embedding Providers Config

src/main/java/io/stargate/sgv2/jsonapi/service/embedding/configuration/PropertyBasedEmbeddingProviderConfig.java

@@ -29,8 +29,46 @@ interface EmbeddingProviderConfig {
     @JsonProperty
     String apiKey();
 
+    /**
+     * A map of supported authentications. HEADER, SHARED_SECRET and NONE are the only techniques
+     * the DataAPI supports (i.e. the key of map can only be HEADER, SHARED_SECRET or NONE).
+     *
+     * @return
+     */
     @JsonProperty
-    List<String> supportedAuthentication();
+    Map<String, AuthenticationConfig> supportedAuthentication();
+
+    /**
+     * enabled() is a JSON boolean to flag if this technique is supported. If false the rest of the
+     * object has no impact. Any technique not listed is also not supported for the provider.
+     *
+     * <p>tokens() is a list of token mappings, that map from the name accepted by the Data API to
+     * how they are forwarded to the provider. The provider information is included for the code,
+     * and to allow users to see what we do with the values.
+     */
+    interface AuthenticationConfig {
+      @JsonProperty
+      boolean enabled();
+
+      @JsonProperty
+      List<TokenConfig> tokens();
+    }
+
+    /**
+     * For the HEADER technique the `accepted` value is the name of the header the client should
+     * send, and `forwarded` is the name of the header the Data API will send to the provider.
+     *
+     * <p>For the SHARED_SECRET technique the `accepted` value is the name used in the
+     * authentication option with createCollection that maps to the name of a shared secret, and
+     * `forwarded` is the name of the header the Data API will send to the provider.
+     */
+    interface TokenConfig {
+      @JsonProperty
+      String accepted();
+
+      @JsonProperty
+      String forwarded();
+    }
 
     /**
      * A list of parameters for user customization. Parameters are used to construct the URL or to
@@ -60,8 +98,15 @@ interface ModelConfig {
       @JsonProperty
       String name();
 
+      /**
+       * vectorDimension is not null if the model supports a single dimension value. It will be null
+       * if the model supports different dimensions. A parameter called vectorDimension is included.
+       *
+       * @return
+       */
+      @Nullable
       @JsonProperty
-      Integer vectorDimension();
+      Optional<Integer> vectorDimension();
 
       @JsonProperty
       List<ParameterConfig> parameters();
@@ -84,6 +129,25 @@ interface ParameterConfig {
       @JsonProperty
       Optional<String> defaultValue();
 
+      /**
+       * validation is an object that describes how the Data API will validate the parameters, and
+       * how the UI may want to provide data entry hints. Only one of the validation methods will be
+       * specified for each parameter.
+       *
+       * <p>`numericRange` if present is an array of two numbers that represent the inclusive value
+       * range for a number parameter. E.g. the dimensions for the text-embedding-3
+       *
+       * <p>`options` if present is an array of valid options the user must select from, for example
+       * if a model supports 3 different dimensions. If options are present the only allowed values
+       * for the parameter are those in the options list. If not present, null, or an empty array
+       * any value of the correct type is accepted.
+       *
+       * @return
+       */
+      @Nullable
+      @JsonProperty
+      Map<String, List<Integer>> validation();
+
       @Nullable
       @JsonProperty
       Optional<String> help();

src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindEmbeddingProvidersOperation.java

@@ -10,11 +10,12 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
 /**
- * Operation that list all available vector providers into the {@link
+ * Operation that list all available and enabled vector providers into the {@link
  * CommandStatus#EXISTING_VECTOR_PROVIDERS} command status.
  */
 public record FindEmbeddingProvidersOperation(PropertyBasedEmbeddingProviderConfig config)
@@ -60,7 +61,8 @@ public CommandResult get() {
    */
   private record EmbeddingProviderResponse(
       String url,
-      List<String> supportedAuthentication,
+      Map<String, PropertyBasedEmbeddingProviderConfig.EmbeddingProviderConfig.AuthenticationConfig>
+          supportedAuthentication,
       List<PropertyBasedEmbeddingProviderConfig.EmbeddingProviderConfig.ParameterConfig> parameters,
       List<ModelConfigResponse> models) {
     private static EmbeddingProviderResponse provider(
@@ -91,7 +93,7 @@ private static EmbeddingProviderResponse provider(
    */
   private record ModelConfigResponse(
       String name,
-      Integer vectorDimension,
+      Optional<Integer> vectorDimension,
       List<PropertyBasedEmbeddingProviderConfig.EmbeddingProviderConfig.ParameterConfig>
           parameters) {}
 }

src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/CreateCollectionCommandResolver.java

@@ -260,22 +260,24 @@ private PropertyBasedEmbeddingProviderConfig.EmbeddingProviderConfig getAndValid
   }
 
   // TODO: 1. remove the first if statement when fully support validateAuthentication
-  //  2. validate the 'secretName' in the future
+  //  2. Check if user authentication type is support
+  //  3. Check if required token is provided
+  //  4. Check if token is valid
   private void validateAuthentication(
       CreateCollectionCommand.Options.VectorSearchConfig.VectorizeConfig userConfig,
       PropertyBasedEmbeddingProviderConfig.EmbeddingProviderConfig providerConfig) {
     if (userConfig.vectorizeServiceAuthentication() == null) {
       return;
     }
     // Check if user authentication type is support
-    userConfig.vectorizeServiceAuthentication().type().stream()
-        .filter(type -> !providerConfig.supportedAuthentication().contains(type))
-        .findFirst()
-        .ifPresent(
-            type -> {
-              throw ErrorCode.INVALID_CREATE_COLLECTION_OPTIONS.toApiException(
-                  "Authentication type '%s' is not supported", type);
-            });
+    //    userConfig.vectorizeServiceAuthentication().type().stream()
+    //        .filter(type -> !providerConfig.supportedAuthentication().contains(type))
+    //        .findFirst()
+    //        .ifPresent(
+    //            type -> {
+    //              throw ErrorCode.INVALID_CREATE_COLLECTION_OPTIONS.toApiException(
+    //                  "Authentication type '%s' is not supported", type);
+    //            });
     // Check if 'secretName' is provided if authentication type is 'SHARED_SECRET'
     if (userConfig.vectorizeServiceAuthentication().type().contains("SHARED_SECRET")
         && (userConfig.vectorizeServiceAuthentication().secretName() == null
@@ -376,6 +378,7 @@ private void validateParameterType(
   }
 
   // TODO: check model parameters provided by the user, will support in the future
+  // TODO: fix code 396-408
   private Integer validateModelAndDimension(
       CreateCollectionCommand.Options.VectorSearchConfig.VectorizeConfig userConfig,
       PropertyBasedEmbeddingProviderConfig.EmbeddingProviderConfig providerConfig,
@@ -390,14 +393,18 @@ private Integer validateModelAndDimension(
                         "Model name '%s' for provider '%s' is not supported",
                         userConfig.modelName(), userConfig.provider()));
 
-    Integer configVectorDimension = model.vectorDimension();
-    if (userVectorDimension == null) {
-      return configVectorDimension; // Use config dimension if user didn't provide one
-    } else if (!configVectorDimension.equals(userVectorDimension)) {
-      throw ErrorCode.INVALID_CREATE_COLLECTION_OPTIONS.toApiException(
-          "The provided dimension value '%s' doesn't match the model supports dimension value '%s'",
-          userVectorDimension, configVectorDimension);
+    // TODO: is dimension required? do we still auto populate the dimension?
+    if (model.vectorDimension().isPresent()) {
+      Integer configVectorDimension = model.vectorDimension().get();
+      if (userVectorDimension == null) {
+        return configVectorDimension; // Use config dimension if user didn't provide one
+      } else if (!configVectorDimension.equals(userVectorDimension)) {
+        throw ErrorCode.INVALID_CREATE_COLLECTION_OPTIONS.toApiException(
+            "The provided dimension value '%s' doesn't match the model supports dimension value '%s'",
+            userVectorDimension, configVectorDimension);
+      }
+      return configVectorDimension;
     }
-    return configVectorDimension;
+    return 0;
   }
 }

src/main/resources/embedding-providers-config.yaml

@@ -9,12 +9,37 @@ stargate:
           url: https://api.openai.com/v1/
           api-key: YOUR_API_KEY
           supported-authentication:
-            - "HEADER"
+            NONE:
+              enabled: false
+            HEADER:
+              enabled: true
+              tokens:
+                - accepted: x-embedding-provider-key
+                  forwarded: Authorization
+            SHARED_SECRET:
+              enabled: false
+              tokens:
+                - accepted: providerKey
+                  forwarded: Authorization
           models:
             - name: text-embedding-3-small
-              vector-dimension: 1536
+              parameters:
+                - name: "vectorDimension"
+                  type: NUMBER
+                  required: true
+                  default-value: 1536
+                  validation:
+                    numeric-range: [512, 1536]
+                  help: "The vector dimension could be any number between 512 and 1536."
             - name: text-embedding-3-large
-              vector-dimension: 3072
+              parameters:
+                - name: "vectorDimension"
+                  type: NUMBER
+                  required: true
+                  default-value: 1024
+                  validation:
+                    numeric-range: [256, 3072]
+                  help: "The vector dimension could be any number between 256 and 3072."
             - name: text-embedding-ada-002
               vector-dimension: 1536
 
@@ -24,7 +49,18 @@ stargate:
           url: https://api-inference.huggingface.co/pipeline/feature-extraction/
           api-key: YOUR_API_KEY
           supported-authentication:
-            - "HEADER"
+            NONE:
+              enabled: false
+            HEADER:
+                enabled: true
+                tokens:
+                  - accepted: x-embedding-provider-key
+                    forwarded: Authorization
+            SHARED_SECRET:
+                enabled: false
+                tokens:
+                  - accepted: providerKey
+                    forwarded: Authorization
           models:
             - name: sentence-transformers/all-MiniLM-L6-v2
               vector-dimension: 384
@@ -35,11 +71,23 @@ stargate:
           url: "https://us-central1-aiplatform.googleapis.com/v1/projects/{PROJECT_ID}/locations/us-central1/publishers/google/models/{MODEL}:predict"
           api-key: YOUR_API_KEY
           supported-authentication:
-            - "HEADER"
+            NONE:
+              enabled: false
+            HEADER:
+              enabled: true
+              tokens:
+                - accepted: x-embedding-provider-key
+                  forwarded: Authorization
+            SHARED_SECRET:
+              enabled: false
+              tokens:
+                - accepted: providerKey
+                  forwarded: Authorization
           parameters:
             - name: PROJECT_ID
               type: STRING
               required: true
+              help: "The Google Cloud Project ID to use when calling"
           properties:
             max-retries: 3
             request-timeout-millis: 1000
@@ -52,18 +100,28 @@ stargate:
                   type: BOOLEAN
                   required: false
                   default-value: true
+                  help: "If set to false, text that exceeds the token limit causes the request to fail. The default value is true."
               properties:
                 max-tokens: 3072
 
-
-
         # Cohere embedding service configuration
         cohere:
           enabled: true
           url: https://api.cohere.ai/v1/
           api-key: YOUR_API_KEY
           supported-authentication:
-            - "HEADER"
+            NONE:
+              enabled: false
+            HEADER:
+              enabled: true
+              tokens:
+                - accepted: x-embedding-provider-key
+                  forwarded: Authorization
+            SHARED_SECRET:
+              enabled: false
+              tokens:
+                - accepted: providerKey
+                  forwarded: Authorization
           models:
             - name: embed-english-v3.0
               vector-dimension: 1024
@@ -76,12 +134,19 @@ stargate:
           url: https://api.nvcf.nvidia.com/v2/nvcf/pexec/functions/091a03bb-7364-4087-8090-bd71e9277520
           api-key: YOUR_API_KEY
           supported-authentication:
-            - "HEADER"
+            NONE:
+              enabled: true
+            HEADER:
+              enabled: false
+              tokens:
+                - accepted: x-embedding-provider-key
+                  forwarded: Authorization
+            SHARED_SECRET:
+              enabled: false
+              tokens:
+                - accepted: providerKey
+                  forwarded: Authorization
           models:
-            - name: NVIDIA Retrieval QA Embedding Model-1.0
-              vector-dimension: 1024
-              properties:
-                max-tokens: 512
             - name: NV-Embed-QA
               vector-dimension: 1024
               properties: