Generate documentation for resources (AI services, datasource, vector…

…-database) (#513)

langstream-agents/langstream-ai-agents/src/main/java/ai/langstream/ai/agents/services/impl/HuggingFaceProvider.java

@@ -25,7 +25,6 @@
 import com.datastax.oss.streaming.ai.embeddings.HuggingFaceEmbeddingService;
 import com.datastax.oss.streaming.ai.embeddings.HuggingFaceRestEmbeddingService;
 import com.datastax.oss.streaming.ai.model.config.ComputeProvider;
-import com.datastax.oss.streaming.ai.model.config.TransformStepConfig;
 import com.datastax.oss.streaming.ai.services.ServiceProvider;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.core.type.TypeReference;
@@ -41,8 +40,7 @@
 import java.util.concurrent.CompletableFuture;
 import java.util.stream.Collectors;
 import lombok.SneakyThrows;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import lombok.extern.slf4j.Slf4j;
 
 public class HuggingFaceProvider implements ServiceProviderProvider {
 
@@ -60,24 +58,14 @@ public ServiceProvider createImplementation(Map<String, Object> agentConfigurati
                 (Map<String, Object>) agentConfiguration.get("huggingface"));
     }
 
+    @Slf4j
     static class HuggingFaceServiceProvider implements ServiceProvider {
-        private static final Logger log =
-                LoggerFactory.getLogger(
-                        com.datastax.oss.streaming.ai.services.HuggingFaceServiceProvider.class);
         private final Map<String, Object> providerConfiguration;
 
         public HuggingFaceServiceProvider(Map<String, Object> providerConfiguration) {
             this.providerConfiguration = providerConfiguration;
         }
 
-        public HuggingFaceServiceProvider(TransformStepConfig tranformConfiguration) {
-            this.providerConfiguration =
-                    (Map)
-                            (new ObjectMapper())
-                                    .convertValue(
-                                            tranformConfiguration.getHuggingface(), Map.class);
-        }
-
         public CompletionsService getCompletionsService(
                 Map<String, Object> additionalConfiguration) {
             String accessKey = (String) providerConfiguration.get("access-key");

langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/pulsar/functions/transforms/TransformFunction.java

@@ -17,6 +17,7 @@
 
 import static com.datastax.oss.streaming.ai.util.TransformFunctionUtil.buildStep;
 
+import ai.langstream.ai.agents.services.impl.HuggingFaceProvider;
 import com.datastax.oss.streaming.ai.JsonNodeSchema;
 import com.datastax.oss.streaming.ai.TransformContext;
 import com.datastax.oss.streaming.ai.TransformStep;
@@ -25,7 +26,6 @@
 import com.datastax.oss.streaming.ai.model.TransformSchemaType;
 import com.datastax.oss.streaming.ai.model.config.StepConfig;
 import com.datastax.oss.streaming.ai.model.config.TransformStepConfig;
-import com.datastax.oss.streaming.ai.services.HuggingFaceServiceProvider;
 import com.datastax.oss.streaming.ai.services.OpenAIServiceProvider;
 import com.datastax.oss.streaming.ai.services.ServiceProvider;
 import com.datastax.oss.streaming.ai.util.TransformFunctionUtil;
@@ -495,7 +495,9 @@ protected ServiceProvider buildServiceProvider(TransformStepConfig config) {
                 return new OpenAIServiceProvider(config);
             }
             if (config.getHuggingface() != null) {
-                return new HuggingFaceServiceProvider(config);
+                return new HuggingFaceProvider()
+                        .createImplementation(
+                                TransformFunctionUtil.convertToMap(config.getHuggingface()));
             }
         }
         return new ServiceProvider.NoopServiceProvider();

langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/milvus/MilvusDataSource.java

@@ -59,16 +59,16 @@ public static final class MilvusConfig {
         @JsonProperty(value = "host")
         private String host;
 
+        @JsonProperty(value = "port")
+        private int port = 19530;
+
         // Zillis service
 
         @JsonProperty(value = "url")
         private String url;
 
         @JsonProperty(value = "token")
         private String token;
-
-        @JsonProperty(value = "port")
-        private int port = 19530;
     }
 
     @Override

langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/pinecone/PineconeDataSource.java

@@ -87,15 +87,15 @@ public QueryStepDataSource createDataSourceImplementation(
 
         PineconeConfig clientConfig = MAPPER.convertValue(dataSourceConfig, PineconeConfig.class);
 
-        return new PinecodeQueryStepDataSource(clientConfig);
+        return new PineconeQueryStepDataSource(clientConfig);
     }
 
-    private static class PinecodeQueryStepDataSource implements QueryStepDataSource {
+    private static class PineconeQueryStepDataSource implements QueryStepDataSource {
 
         private final PineconeConfig clientConfig;
         private PineconeConnection connection;
 
-        public PinecodeQueryStepDataSource(PineconeConfig clientConfig) {
+        public PineconeQueryStepDataSource(PineconeConfig clientConfig) {
             this.clientConfig = clientConfig;
         }
 
@@ -258,7 +258,7 @@ public static Object valueToObject(Value value) {
                 case STRING_VALUE -> value.getStringValue();
                 case BOOL_VALUE -> value.getBoolValue();
                 case LIST_VALUE -> value.getListValue().getValuesList().stream()
-                        .map(PinecodeQueryStepDataSource::valueToObject)
+                        .map(PineconeQueryStepDataSource::valueToObject)
                         .toList();
                 case STRUCT_VALUE -> value.getStructValue().getFieldsMap().entrySet().stream()
                         .collect(

langstream-api/src/main/java/ai/langstream/api/doc/AgentConfigurationModel.java

@@ -17,7 +17,6 @@
 
 import java.util.Map;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
@@ -26,20 +25,7 @@
 @AllArgsConstructor
 public class AgentConfigurationModel {
 
-    @Data
-    @NoArgsConstructor
-    @AllArgsConstructor
-    @Builder
-    public static class AgentConfigurationProperty {
-        private String description;
-        boolean required;
-        private String type;
-        private Map<String, AgentConfigurationProperty> properties;
-        private AgentConfigurationProperty items;
-        private Object defaultValue;
-    }
-
     private String name;
     private String description;
-    private Map<String, AgentConfigurationProperty> properties;
+    private Map<String, ConfigPropertyModel> properties;
 }

langstream-api/src/main/java/ai/langstream/api/doc/ApiConfigurationModel.java

@@ -17,4 +17,7 @@
 
 import java.util.Map;
 
-public record ApiConfigurationModel(String version, Map<String, AgentConfigurationModel> agents) {}
+public record ApiConfigurationModel(
+        String version,
+        Map<String, AgentConfigurationModel> agents,
+        Map<String, ResourceConfigurationModel> resources) {}

langstream-api/src/main/java/ai/langstream/api/doc/ConfigPropertyModel.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ai.langstream.api.doc;
+
+import java.util.Map;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+@Builder
+public class ConfigPropertyModel {
+    private String description;
+    boolean required;
+    private String type;
+    private Map<String, ConfigPropertyModel> properties;
+    private ConfigPropertyModel items;
+    private Object defaultValue;
+}

langstream-api/src/main/java/ai/langstream/api/doc/ResourceConfig.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ai.langstream.api.doc;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Target(ElementType.TYPE)
+@Retention(RetentionPolicy.RUNTIME)
+public @interface ResourceConfig {
+    String name() default "";
+
+    String description() default "";
+}

langstream-api/src/main/java/ai/langstream/api/doc/ResourceConfigurationModel.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ai.langstream.api.doc;
+
+import java.util.Map;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+public class ResourceConfigurationModel {
+
+    private String type;
+    private String name;
+    private String description;
+    private Map<String, ConfigPropertyModel> properties;
+}

langstream-api/src/main/java/ai/langstream/api/runtime/PluginsRegistry.java

@@ -47,9 +47,7 @@ public AgentNodeProvider lookupAgentImplementation(
         return agentRuntimeProviderProvider.get();
     }
 
-    public List<AgentNodeProvider> lookupAvailableAgentImplementations(
-            ComputeClusterRuntime clusterRuntime) {
-        // TODO: cluster runtime '
+    public List<AgentNodeProvider> lookupAvailableAgentImplementations() {
         ServiceLoader<AgentNodeProvider> loader = ServiceLoader.load(AgentNodeProvider.class);
         return loader.stream().map(p -> p.get()).collect(Collectors.toList());
     }
@@ -79,6 +77,11 @@ public AssetNodeProvider lookupAssetImplementation(
         return assetRuntimeProviderProvider.get();
     }
 
+    public List<ResourceNodeProvider> lookupAvailableResourceImplementations() {
+        ServiceLoader<ResourceNodeProvider> loader = ServiceLoader.load(ResourceNodeProvider.class);
+        return loader.stream().map(p -> p.get()).collect(Collectors.toList());
+    }
+
     public ResourceNodeProvider lookupResourceImplementation(
             String type, ComputeClusterRuntime clusterRuntime) {
         log.info(