From 90dba205ada12227947701596747001ccb6d2137 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Tue, 3 Oct 2023 11:30:14 +0200 Subject: [PATCH] Generate documentation for resources (AI services, datasource, vector-database) (#513) --- .../services/impl/HuggingFaceProvider.java | 16 +- .../services/HuggingFaceServiceProvider.java | 114 ---------- .../transforms/TransformFunction.java | 6 +- .../vector/milvus/MilvusDataSource.java | 6 +- .../vector/pinecone/PineconeDataSource.java | 8 +- .../api/doc/AgentConfigurationModel.java | 16 +- .../api/doc/ApiConfigurationModel.java | 5 +- .../api/doc/ConfigPropertyModel.java | 35 +++ .../ai/langstream/api/doc/ResourceConfig.java | 29 +++ .../api/doc/ResourceConfigurationModel.java | 32 +++ .../api/runtime/PluginsRegistry.java | 9 +- .../api/runtime/ResourceNodeProvider.java | 18 +- .../ai/GenAIToolKitFunctionAgentProvider.java | 9 - .../ComputeAIEmbeddingsConfiguration.java | 37 +++- .../impl/common/BasicClusterRuntime.java | 5 +- .../AIProvidersResourceProvider.java | 204 ++++++++++++++---- .../resources/AbstractResourceProvider.java | 88 ++++++++ .../BaseDataSourceResourceProvider.java | 88 ++++++++ .../resources/DataSourceResourceProvider.java | 125 ++--------- .../VectorDatabaseResourceProvider.java | 86 ++------ .../datasource/AstraDatasourceConfig.java | 125 +++++++++++ .../datasource/BaseDatasourceConfig.java | 26 +++ .../datasource/CassandraDatasourceConfig.java | 96 +++++++++ .../datasource/JDBCDatasourceConfig.java | 61 ++++++ .../datasource/MilvusDatasourceConfig.java | 99 +++++++++ .../datasource/PineconeDatasourceConfig.java | 105 +++++++++ .../impl/uti/ClassConfigValidator.java | 127 ++++++++--- .../resources/ResourceNodeProviderTest.java | 46 +--- .../k8s/agents/PythonCodeAgentProvider.java | 6 +- .../agents/WebCrawlerSourceAgentProvider.java | 5 + .../runtime/impl/k8s/GenAIAgentsTest.java | 35 ++- .../KafkaConnectAgentsProviderTest.java | 4 +- ...GenAIToolKitFunctionAgentProviderTest.java | 21 +- .../agents/PythonCodeAgentProviderTest.java | 4 +- .../QueryVectorDBAgentProviderTest.java | 4 +- .../k8s/agents/S3SourceAgentProviderTest.java | 4 +- .../langstream/assets/DeployAssetsTest.java | 6 +- .../MockAssetManagerCodeProvider.java | 8 +- .../runtime/LoadAssertManagerCodeTest.java | 2 +- .../doc/DocumentationGenerator.java | 15 +- 40 files changed, 1216 insertions(+), 519 deletions(-) delete mode 100644 langstream-agents/langstream-ai-agents/src/main/java/com/datastax/oss/streaming/ai/services/HuggingFaceServiceProvider.java create mode 100644 langstream-api/src/main/java/ai/langstream/api/doc/ConfigPropertyModel.java create mode 100644 langstream-api/src/main/java/ai/langstream/api/doc/ResourceConfig.java create mode 100644 langstream-api/src/main/java/ai/langstream/api/doc/ResourceConfigurationModel.java create mode 100644 langstream-core/src/main/java/ai/langstream/impl/resources/AbstractResourceProvider.java create mode 100644 langstream-core/src/main/java/ai/langstream/impl/resources/BaseDataSourceResourceProvider.java create mode 100644 langstream-core/src/main/java/ai/langstream/impl/resources/datasource/AstraDatasourceConfig.java create mode 100644 langstream-core/src/main/java/ai/langstream/impl/resources/datasource/BaseDatasourceConfig.java create mode 100644 langstream-core/src/main/java/ai/langstream/impl/resources/datasource/CassandraDatasourceConfig.java create mode 100644 langstream-core/src/main/java/ai/langstream/impl/resources/datasource/JDBCDatasourceConfig.java create mode 100644 langstream-core/src/main/java/ai/langstream/impl/resources/datasource/MilvusDatasourceConfig.java create mode 100644 langstream-core/src/main/java/ai/langstream/impl/resources/datasource/PineconeDatasourceConfig.java diff --git a/langstream-agents/langstream-ai-agents/src/main/java/ai/langstream/ai/agents/services/impl/HuggingFaceProvider.java b/langstream-agents/langstream-ai-agents/src/main/java/ai/langstream/ai/agents/services/impl/HuggingFaceProvider.java index 6ed01e62e..d4e28fec3 100644 --- a/langstream-agents/langstream-ai-agents/src/main/java/ai/langstream/ai/agents/services/impl/HuggingFaceProvider.java +++ b/langstream-agents/langstream-ai-agents/src/main/java/ai/langstream/ai/agents/services/impl/HuggingFaceProvider.java @@ -25,7 +25,6 @@ import com.datastax.oss.streaming.ai.embeddings.HuggingFaceEmbeddingService; import com.datastax.oss.streaming.ai.embeddings.HuggingFaceRestEmbeddingService; import com.datastax.oss.streaming.ai.model.config.ComputeProvider; -import com.datastax.oss.streaming.ai.model.config.TransformStepConfig; import com.datastax.oss.streaming.ai.services.ServiceProvider; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; @@ -41,8 +40,7 @@ import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; import lombok.SneakyThrows; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; public class HuggingFaceProvider implements ServiceProviderProvider { @@ -60,24 +58,14 @@ public ServiceProvider createImplementation(Map agentConfigurati (Map) agentConfiguration.get("huggingface")); } + @Slf4j static class HuggingFaceServiceProvider implements ServiceProvider { - private static final Logger log = - LoggerFactory.getLogger( - com.datastax.oss.streaming.ai.services.HuggingFaceServiceProvider.class); private final Map providerConfiguration; public HuggingFaceServiceProvider(Map providerConfiguration) { this.providerConfiguration = providerConfiguration; } - public HuggingFaceServiceProvider(TransformStepConfig tranformConfiguration) { - this.providerConfiguration = - (Map) - (new ObjectMapper()) - .convertValue( - tranformConfiguration.getHuggingface(), Map.class); - } - public CompletionsService getCompletionsService( Map additionalConfiguration) { String accessKey = (String) providerConfiguration.get("access-key"); diff --git a/langstream-agents/langstream-ai-agents/src/main/java/com/datastax/oss/streaming/ai/services/HuggingFaceServiceProvider.java b/langstream-agents/langstream-ai-agents/src/main/java/com/datastax/oss/streaming/ai/services/HuggingFaceServiceProvider.java deleted file mode 100644 index 1e97de39c..000000000 --- a/langstream-agents/langstream-ai-agents/src/main/java/com/datastax/oss/streaming/ai/services/HuggingFaceServiceProvider.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright DataStax, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datastax.oss.streaming.ai.services; - -import static com.datastax.oss.streaming.ai.embeddings.AbstractHuggingFaceEmbeddingService.DLJ_BASE_URL; - -import com.datastax.oss.streaming.ai.completions.CompletionsService; -import com.datastax.oss.streaming.ai.embeddings.AbstractHuggingFaceEmbeddingService; -import com.datastax.oss.streaming.ai.embeddings.EmbeddingsService; -import com.datastax.oss.streaming.ai.embeddings.HuggingFaceEmbeddingService; -import com.datastax.oss.streaming.ai.embeddings.HuggingFaceRestEmbeddingService; -import com.datastax.oss.streaming.ai.model.config.ComputeProvider; -import com.datastax.oss.streaming.ai.model.config.TransformStepConfig; -import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.Map; -import java.util.Objects; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class HuggingFaceServiceProvider implements ServiceProvider { - - private final Map providerConfiguration; - - public HuggingFaceServiceProvider(Map providerConfiguration) { - this.providerConfiguration = providerConfiguration; - } - - public HuggingFaceServiceProvider(TransformStepConfig tranformConfiguration) { - this.providerConfiguration = - new ObjectMapper().convertValue(tranformConfiguration.getHuggingface(), Map.class); - } - - @Override - public CompletionsService getCompletionsService(Map additionalConfiguration) { - throw new IllegalArgumentException("Completions is still not available for HuggingFace"); - } - - @Override - public EmbeddingsService getEmbeddingsService(Map additionalConfiguration) - throws Exception { - String provider = - additionalConfiguration - .getOrDefault("provider", ComputeProvider.API.name()) - .toString() - .toUpperCase(); - String modelUrl = (String) additionalConfiguration.get("modelUrl"); - String model = (String) additionalConfiguration.get("model"); - Map options = (Map) additionalConfiguration.get("options"); - Map arguments = - (Map) additionalConfiguration.get("arguments"); - switch (provider) { - case "LOCAL": - AbstractHuggingFaceEmbeddingService.HuggingFaceConfig.HuggingFaceConfigBuilder - builder = - AbstractHuggingFaceEmbeddingService.HuggingFaceConfig.builder() - .options(options) - .arguments(arguments); - if (model != null && !model.isEmpty()) { - builder.modelName(model); - - // automatically build the model URL if not provided - if (modelUrl == null || modelUrl.isEmpty()) { - modelUrl = DLJ_BASE_URL + model; - log.info("Automatically computed model URL {}", modelUrl); - } - } - builder.modelUrl(modelUrl); - - return new HuggingFaceEmbeddingService(builder.build()); - case "API": - Objects.requireNonNull(model, "model name is required"); - HuggingFaceRestEmbeddingService.HuggingFaceApiConfig.HuggingFaceApiConfigBuilder - apiBuilder = - HuggingFaceRestEmbeddingService.HuggingFaceApiConfig.builder() - .accessKey((String) providerConfiguration.get("access-key")) - .model(model); - - String apiUurl = (String) providerConfiguration.get("api-url"); - if (apiUurl != null && !apiUurl.isEmpty()) { - apiBuilder.hfUrl(apiUurl); - } - String modelCheckUrl = (String) providerConfiguration.get("model-check-url"); - if (modelCheckUrl != null && !modelCheckUrl.isEmpty()) { - apiBuilder.hfCheckUrl(modelCheckUrl); - } - if (options != null && !options.isEmpty()) { - apiBuilder.options(options); - } else { - apiBuilder.options(Map.of("wait_for_model", "true")); - } - - return new HuggingFaceRestEmbeddingService(apiBuilder.build()); - default: - throw new IllegalArgumentException( - "Unsupported HuggingFace service type: " + provider); - } - } - - @Override - public void close() {} -} diff --git a/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/pulsar/functions/transforms/TransformFunction.java b/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/pulsar/functions/transforms/TransformFunction.java index 8951b43e4..67469cb61 100644 --- a/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/pulsar/functions/transforms/TransformFunction.java +++ b/langstream-agents/langstream-ai-agents/src/test/java/com/datastax/oss/pulsar/functions/transforms/TransformFunction.java @@ -17,6 +17,7 @@ import static com.datastax.oss.streaming.ai.util.TransformFunctionUtil.buildStep; +import ai.langstream.ai.agents.services.impl.HuggingFaceProvider; import com.datastax.oss.streaming.ai.JsonNodeSchema; import com.datastax.oss.streaming.ai.TransformContext; import com.datastax.oss.streaming.ai.TransformStep; @@ -25,7 +26,6 @@ import com.datastax.oss.streaming.ai.model.TransformSchemaType; import com.datastax.oss.streaming.ai.model.config.StepConfig; import com.datastax.oss.streaming.ai.model.config.TransformStepConfig; -import com.datastax.oss.streaming.ai.services.HuggingFaceServiceProvider; import com.datastax.oss.streaming.ai.services.OpenAIServiceProvider; import com.datastax.oss.streaming.ai.services.ServiceProvider; import com.datastax.oss.streaming.ai.util.TransformFunctionUtil; @@ -495,7 +495,9 @@ protected ServiceProvider buildServiceProvider(TransformStepConfig config) { return new OpenAIServiceProvider(config); } if (config.getHuggingface() != null) { - return new HuggingFaceServiceProvider(config); + return new HuggingFaceProvider() + .createImplementation( + TransformFunctionUtil.convertToMap(config.getHuggingface())); } } return new ServiceProvider.NoopServiceProvider(); diff --git a/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/milvus/MilvusDataSource.java b/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/milvus/MilvusDataSource.java index aa6b48a34..0d627cbe2 100644 --- a/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/milvus/MilvusDataSource.java +++ b/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/milvus/MilvusDataSource.java @@ -59,6 +59,9 @@ public static final class MilvusConfig { @JsonProperty(value = "host") private String host; + @JsonProperty(value = "port") + private int port = 19530; + // Zillis service @JsonProperty(value = "url") @@ -66,9 +69,6 @@ public static final class MilvusConfig { @JsonProperty(value = "token") private String token; - - @JsonProperty(value = "port") - private int port = 19530; } @Override diff --git a/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/pinecone/PineconeDataSource.java b/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/pinecone/PineconeDataSource.java index 0e5611a9d..8d6e6f569 100644 --- a/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/pinecone/PineconeDataSource.java +++ b/langstream-agents/langstream-vector-agents/src/main/java/ai/langstream/agents/vector/pinecone/PineconeDataSource.java @@ -87,15 +87,15 @@ public QueryStepDataSource createDataSourceImplementation( PineconeConfig clientConfig = MAPPER.convertValue(dataSourceConfig, PineconeConfig.class); - return new PinecodeQueryStepDataSource(clientConfig); + return new PineconeQueryStepDataSource(clientConfig); } - private static class PinecodeQueryStepDataSource implements QueryStepDataSource { + private static class PineconeQueryStepDataSource implements QueryStepDataSource { private final PineconeConfig clientConfig; private PineconeConnection connection; - public PinecodeQueryStepDataSource(PineconeConfig clientConfig) { + public PineconeQueryStepDataSource(PineconeConfig clientConfig) { this.clientConfig = clientConfig; } @@ -258,7 +258,7 @@ public static Object valueToObject(Value value) { case STRING_VALUE -> value.getStringValue(); case BOOL_VALUE -> value.getBoolValue(); case LIST_VALUE -> value.getListValue().getValuesList().stream() - .map(PinecodeQueryStepDataSource::valueToObject) + .map(PineconeQueryStepDataSource::valueToObject) .toList(); case STRUCT_VALUE -> value.getStructValue().getFieldsMap().entrySet().stream() .collect( diff --git a/langstream-api/src/main/java/ai/langstream/api/doc/AgentConfigurationModel.java b/langstream-api/src/main/java/ai/langstream/api/doc/AgentConfigurationModel.java index cbbbe59df..56471f7df 100644 --- a/langstream-api/src/main/java/ai/langstream/api/doc/AgentConfigurationModel.java +++ b/langstream-api/src/main/java/ai/langstream/api/doc/AgentConfigurationModel.java @@ -17,7 +17,6 @@ import java.util.Map; import lombok.AllArgsConstructor; -import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; @@ -26,20 +25,7 @@ @AllArgsConstructor public class AgentConfigurationModel { - @Data - @NoArgsConstructor - @AllArgsConstructor - @Builder - public static class AgentConfigurationProperty { - private String description; - boolean required; - private String type; - private Map properties; - private AgentConfigurationProperty items; - private Object defaultValue; - } - private String name; private String description; - private Map properties; + private Map properties; } diff --git a/langstream-api/src/main/java/ai/langstream/api/doc/ApiConfigurationModel.java b/langstream-api/src/main/java/ai/langstream/api/doc/ApiConfigurationModel.java index 8b4597ddd..d2d534eb4 100644 --- a/langstream-api/src/main/java/ai/langstream/api/doc/ApiConfigurationModel.java +++ b/langstream-api/src/main/java/ai/langstream/api/doc/ApiConfigurationModel.java @@ -17,4 +17,7 @@ import java.util.Map; -public record ApiConfigurationModel(String version, Map agents) {} +public record ApiConfigurationModel( + String version, + Map agents, + Map resources) {} diff --git a/langstream-api/src/main/java/ai/langstream/api/doc/ConfigPropertyModel.java b/langstream-api/src/main/java/ai/langstream/api/doc/ConfigPropertyModel.java new file mode 100644 index 000000000..c3a050322 --- /dev/null +++ b/langstream-api/src/main/java/ai/langstream/api/doc/ConfigPropertyModel.java @@ -0,0 +1,35 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.langstream.api.doc; + +import java.util.Map; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class ConfigPropertyModel { + private String description; + boolean required; + private String type; + private Map properties; + private ConfigPropertyModel items; + private Object defaultValue; +} diff --git a/langstream-api/src/main/java/ai/langstream/api/doc/ResourceConfig.java b/langstream-api/src/main/java/ai/langstream/api/doc/ResourceConfig.java new file mode 100644 index 000000000..e20d8e921 --- /dev/null +++ b/langstream-api/src/main/java/ai/langstream/api/doc/ResourceConfig.java @@ -0,0 +1,29 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.langstream.api.doc; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +public @interface ResourceConfig { + String name() default ""; + + String description() default ""; +} diff --git a/langstream-api/src/main/java/ai/langstream/api/doc/ResourceConfigurationModel.java b/langstream-api/src/main/java/ai/langstream/api/doc/ResourceConfigurationModel.java new file mode 100644 index 000000000..254825b7e --- /dev/null +++ b/langstream-api/src/main/java/ai/langstream/api/doc/ResourceConfigurationModel.java @@ -0,0 +1,32 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.langstream.api.doc; + +import java.util.Map; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class ResourceConfigurationModel { + + private String type; + private String name; + private String description; + private Map properties; +} diff --git a/langstream-api/src/main/java/ai/langstream/api/runtime/PluginsRegistry.java b/langstream-api/src/main/java/ai/langstream/api/runtime/PluginsRegistry.java index 3d72f3c75..d51397e49 100644 --- a/langstream-api/src/main/java/ai/langstream/api/runtime/PluginsRegistry.java +++ b/langstream-api/src/main/java/ai/langstream/api/runtime/PluginsRegistry.java @@ -47,9 +47,7 @@ public AgentNodeProvider lookupAgentImplementation( return agentRuntimeProviderProvider.get(); } - public List lookupAvailableAgentImplementations( - ComputeClusterRuntime clusterRuntime) { - // TODO: cluster runtime ' + public List lookupAvailableAgentImplementations() { ServiceLoader loader = ServiceLoader.load(AgentNodeProvider.class); return loader.stream().map(p -> p.get()).collect(Collectors.toList()); } @@ -79,6 +77,11 @@ public AssetNodeProvider lookupAssetImplementation( return assetRuntimeProviderProvider.get(); } + public List lookupAvailableResourceImplementations() { + ServiceLoader loader = ServiceLoader.load(ResourceNodeProvider.class); + return loader.stream().map(p -> p.get()).collect(Collectors.toList()); + } + public ResourceNodeProvider lookupResourceImplementation( String type, ComputeClusterRuntime clusterRuntime) { log.info( diff --git a/langstream-api/src/main/java/ai/langstream/api/runtime/ResourceNodeProvider.java b/langstream-api/src/main/java/ai/langstream/api/runtime/ResourceNodeProvider.java index 235869dcd..16a1ca3ea 100644 --- a/langstream-api/src/main/java/ai/langstream/api/runtime/ResourceNodeProvider.java +++ b/langstream-api/src/main/java/ai/langstream/api/runtime/ResourceNodeProvider.java @@ -15,7 +15,7 @@ */ package ai.langstream.api.runtime; -import ai.langstream.api.model.Module; +import ai.langstream.api.doc.ResourceConfigurationModel; import ai.langstream.api.model.Resource; import java.util.Map; @@ -24,18 +24,10 @@ public interface ResourceNodeProvider { /** * Create an Implementation of a Resource. * - * @param module the module - * @param executionPlan the physical application instance - * @param clusterRuntime the cluster runtime * @param pluginsRegistry the plugins registry - * @return the Agent + * @return the implementation */ - Map createImplementation( - Resource resource, - Module module, - ExecutionPlan executionPlan, - ComputeClusterRuntime clusterRuntime, - PluginsRegistry pluginsRegistry); + Map createImplementation(Resource resource, PluginsRegistry pluginsRegistry); /** * Returns the ability of a Resource to be deployed on the give runtimes. @@ -45,4 +37,8 @@ Map createImplementation( * @return true if this provider can create the implementation */ boolean supports(String type, ComputeClusterRuntime clusterRuntime); + + default Map generateSupportedTypesDocumentation() { + return Map.of(); + } } diff --git a/langstream-core/src/main/java/ai/langstream/impl/agents/ai/GenAIToolKitFunctionAgentProvider.java b/langstream-core/src/main/java/ai/langstream/impl/agents/ai/GenAIToolKitFunctionAgentProvider.java index 3d7228434..e0a0ce74a 100644 --- a/langstream-core/src/main/java/ai/langstream/impl/agents/ai/GenAIToolKitFunctionAgentProvider.java +++ b/langstream-core/src/main/java/ai/langstream/impl/agents/ai/GenAIToolKitFunctionAgentProvider.java @@ -27,7 +27,6 @@ import ai.langstream.api.runtime.PluginsRegistry; import ai.langstream.impl.agents.ai.steps.AIChatCompletionsConfiguration; import ai.langstream.impl.agents.ai.steps.AITextCompletionsConfiguration; -import ai.langstream.impl.agents.ai.steps.BaseGenAIStepConfiguration; import ai.langstream.impl.agents.ai.steps.CastConfiguration; import ai.langstream.impl.agents.ai.steps.ComputeAIEmbeddingsConfiguration; import ai.langstream.impl.agents.ai.steps.ComputeConfiguration; @@ -58,14 +57,6 @@ public class GenAIToolKitFunctionAgentProvider extends AbstractAgentProvider { List.of(SERVICE_VERTEX, SERVICE_HUGGING_FACE, SERVICE_OPEN_AI); static { - final StepConfigurationInitializer baseConfig = - new StepConfigurationInitializer() { - @Override - public Class getAgentConfigurationModelClass() { - return BaseGenAIStepConfiguration.class; - } - }; - final Map steps = new HashMap<>(); steps.put("drop-fields", DropFieldsConfiguration.STEP); steps.put("merge-key-value", MergeKeyValueConfiguration.STEP); diff --git a/langstream-core/src/main/java/ai/langstream/impl/agents/ai/steps/ComputeAIEmbeddingsConfiguration.java b/langstream-core/src/main/java/ai/langstream/impl/agents/ai/steps/ComputeAIEmbeddingsConfiguration.java index 80906be02..634a9a708 100644 --- a/langstream-core/src/main/java/ai/langstream/impl/agents/ai/steps/ComputeAIEmbeddingsConfiguration.java +++ b/langstream-core/src/main/java/ai/langstream/impl/agents/ai/steps/ComputeAIEmbeddingsConfiguration.java @@ -27,8 +27,8 @@ name = "Compute embeddings of the record", description = """ - Compute embeddings of the record. The embeddings are stored in the record under a specific field. - """) + Compute embeddings of the record. The embeddings are stored in the record under a specific field. + """) @Data public class ComputeAIEmbeddingsConfiguration extends BaseGenAIStepConfiguration { public static final GenAIToolKitFunctionAgentProvider.StepConfigurationInitializer STEP = @@ -59,6 +59,12 @@ public void generateSteps( aiServiceConfigurationGenerator); aiServiceConfigurationGenerator.generateAIServiceConfiguration( (String) step.remove("ai-service")); + + // in the user config we use the pascal but the downstream impl requires snake + final Object modelUrl = step.remove("modelUrl"); + if (modelUrl != null) { + step.put("model-url", modelUrl); + } } }; @@ -73,9 +79,9 @@ public void generateSteps( @ConfigProperty( description = """ - Text to create embeddings from. You can use Mustache syntax to compose multiple fields into a single text. Example: - text: "{{{ value.field1 }}} {{{ value.field2 }}}" - """, + Text to create embeddings from. You can use Mustache syntax to compose multiple fields into a single text. Example: + text: "{{{ value.field1 }}} {{{ value.field2 }}}" + """, required = true) private String text; @@ -121,4 +127,25 @@ public void generateSteps( """) @JsonProperty(value = "ai-service") private String aiService; + + @ConfigProperty( + description = + """ + Additional options to pass to the AI Service. (HuggingFace only) + """) + private Map options; + + @ConfigProperty( + description = + """ + Additional arguments to pass to the AI Service. (HuggingFace only) + """) + private Map arguments; + + @ConfigProperty( + description = + """ + URL of the model to use. (HuggingFace only). The default is computed from the model: "djl://ai.djl.huggingface.pytorch{model}" + """) + private String modelUrl; } diff --git a/langstream-core/src/main/java/ai/langstream/impl/common/BasicClusterRuntime.java b/langstream-core/src/main/java/ai/langstream/impl/common/BasicClusterRuntime.java index 88d3338cc..9e3b5b45d 100644 --- a/langstream-core/src/main/java/ai/langstream/impl/common/BasicClusterRuntime.java +++ b/langstream-core/src/main/java/ai/langstream/impl/common/BasicClusterRuntime.java @@ -150,8 +150,9 @@ public Map getResourceImplementation( Resource resource, PluginsRegistry pluginsRegistry) { ResourceNodeProvider nodeProvider = pluginsRegistry.lookupResourceImplementation(resource.type(), this); - // TODO: validate resource - return new HashMap<>(resource.configuration()); + final Map newConfig = + nodeProvider.createImplementation(resource, pluginsRegistry); + return new HashMap<>(newConfig); } protected AgentNode buildAgent( diff --git a/langstream-core/src/main/java/ai/langstream/impl/resources/AIProvidersResourceProvider.java b/langstream-core/src/main/java/ai/langstream/impl/resources/AIProvidersResourceProvider.java index 2b068def0..ce22cd178 100644 --- a/langstream-core/src/main/java/ai/langstream/impl/resources/AIProvidersResourceProvider.java +++ b/langstream-core/src/main/java/ai/langstream/impl/resources/AIProvidersResourceProvider.java @@ -15,57 +15,53 @@ */ package ai.langstream.impl.resources; -import static ai.langstream.api.util.ConfigurationUtils.getMap; import static ai.langstream.api.util.ConfigurationUtils.getString; import static ai.langstream.api.util.ConfigurationUtils.requiredField; import static ai.langstream.api.util.ConfigurationUtils.requiredNonEmptyField; -import static ai.langstream.api.util.ConfigurationUtils.validateEnumField; -import ai.langstream.api.model.Module; +import ai.langstream.api.doc.ConfigProperty; +import ai.langstream.api.doc.ResourceConfig; import ai.langstream.api.model.Resource; -import ai.langstream.api.runtime.ComputeClusterRuntime; -import ai.langstream.api.runtime.ExecutionPlan; import ai.langstream.api.runtime.PluginsRegistry; -import ai.langstream.api.runtime.ResourceNodeProvider; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; import java.util.Map; import java.util.Set; import java.util.function.Supplier; +import lombok.Data; -public class AIProvidersResourceProvider implements ResourceNodeProvider { +public class AIProvidersResourceProvider extends AbstractResourceProvider { + protected static final String OPEN_AI_CONFIGURATION = "open-ai-configuration"; + protected static final String HUGGING_FACE_CONFIGURATION = "hugging-face-configuration"; + protected static final String VERTEX_CONFIGURATION = "vertex-configuration"; private static final Set SUPPORTED_TYPES = - Set.of("open-ai-configuration", "hugging-face-configuration", "vertex-configuration"); + Set.of(OPEN_AI_CONFIGURATION, HUGGING_FACE_CONFIGURATION, VERTEX_CONFIGURATION); protected static final ObjectMapper MAPPER = new ObjectMapper(); + public AIProvidersResourceProvider() { + super(SUPPORTED_TYPES); + } + @Override - public Map createImplementation( - Resource resource, - Module module, - ExecutionPlan executionPlan, - ComputeClusterRuntime clusterRuntime, - PluginsRegistry pluginsRegistry) { - switch (resource.type()) { - case "open-ai-configuration" -> { - validateOpenAIConfigurationResource(resource); - } - case "hugging-face-configuration" -> { - validateHuggingFaceConfigurationResource(resource); + protected Map computeResourceConfiguration( + Resource resource, PluginsRegistry pluginsRegistry) { + final Map copy = + super.computeResourceConfiguration(resource, pluginsRegistry); + // only dynamic checks, the rest is done in AbstractResourceProvider + if (resource.type().equals(VERTEX_CONFIGURATION)) { + validateVertexConfigurationResource(resource); + } else if (resource.type().equals(OPEN_AI_CONFIGURATION)) { + String provider = getString("provider", "openai", resource.configuration()); + if (provider.equals("azure")) { + requiredField(resource.configuration(), "url", describe(resource)); } - case "vertex-configuration" -> { - validateVertexConfigurationResource(resource); - } - default -> throw new IllegalStateException(); } - return resource.configuration(); + return copy; } private void validateVertexConfigurationResource(Resource resource) { Map configuration = resource.configuration(); - requiredNonEmptyField(configuration, "url", describe(resource)); - requiredNonEmptyField(configuration, "region", describe(resource)); - requiredNonEmptyField(configuration, "project", describe(resource)); - String token = getString("token", "", configuration); String serviceAccountJson = getString("serviceAccountJson", "", configuration); if (!token.isEmpty() && !serviceAccountJson.isEmpty()) { @@ -87,30 +83,144 @@ private void validateVertexConfigurationResource(Resource resource) { } } - private void validateHuggingFaceConfigurationResource(Resource resource) { - Map configuration = resource.configuration(); - validateEnumField(configuration, "provider", Set.of("local", "api"), describe(resource)); - requiredField(configuration, "model", describe(resource)); - getMap("options", Map.of(), configuration); - getMap("arguments", Map.of(), configuration); + protected static Supplier describe(Resource resource) { + return () -> "resource with id = " + resource.id() + " of type " + resource.type(); } - private void validateOpenAIConfigurationResource(Resource resource) { - Map configuration = resource.configuration(); - validateEnumField(configuration, "provider", Set.of("azure", "openai"), describe(resource)); - String provider = getString("provider", "openai", configuration); - if (provider.equals("azure")) { - requiredField(configuration, "url", describe(resource)); + @Override + protected Class getResourceConfigModelClass(String type) { + switch (type) { + case OPEN_AI_CONFIGURATION -> { + return OpenAIConfig.class; + } + case HUGGING_FACE_CONFIGURATION -> { + return HuggingFaceConfig.class; + } + case VERTEX_CONFIGURATION -> { + return VertexAIConfig.class; + } + default -> throw new IllegalStateException(); } - requiredField(configuration, "access-key", describe(resource)); } - @Override - public boolean supports(String type, ComputeClusterRuntime clusterRuntime) { - return SUPPORTED_TYPES.contains(type); + @Data + @ResourceConfig(name = "Open AI", description = "Connect to OpenAI API or Azure OpenAI API.") + public static class OpenAIConfig { + + public enum Provider { + openai, + azure + } + + @ConfigProperty( + description = + """ + The provider to use. Either "openai" or "azure". + """, + defaultValue = "openai") + private Provider provider; + + @ConfigProperty( + description = + """ + The access key to use. + """, + required = true) + @JsonProperty("access-key") + private String accessKey; + + @ConfigProperty( + description = + """ + Url for Azure OpenAI API. Required only if provider is "azure". + """) + private String url; } - protected static Supplier describe(Resource resource) { - return () -> "resource with id = " + resource.id() + " of type " + resource.type(); + @Data + @ResourceConfig(name = "Vertex AI", description = "Connect to VertexAI API.") + public static class VertexAIConfig { + + @ConfigProperty( + description = + """ + URL connection for the Vertex API. + """, + required = true) + private String url; + + @ConfigProperty( + description = + """ + GCP region for the Vertex API. + """, + required = true) + private String region; + + @ConfigProperty( + description = + """ + GCP project name for the Vertex API. + """, + required = true) + private String project; + + @ConfigProperty( + description = + """ + Access key for the Vertex API. + """) + private String token; + + @ConfigProperty( + description = + """ + Specify service account credentials. Refer to the GCP documentation on how to download it + """) + private String serviceAccountJson; + } + + @Data + @ResourceConfig(name = "Hugging Face", description = "Connect to Hugging Face service.") + public static class HuggingFaceConfig { + + public enum Provider { + local, + api + } + + @ConfigProperty( + description = + """ + The provider to use. Either "local" or "api". + """, + defaultValue = "api") + private Provider provider; + + @JsonProperty("api-url") + @ConfigProperty( + description = + """ + The URL of the Hugging Face API. Relevant only if provider is "api". + """, + defaultValue = "https://api-inference.huggingface.co/pipeline/feature-extraction/") + private String apiUrl; + + @ConfigProperty( + description = + """ + The model url to use. Relevant only if provider is "api". + """, + defaultValue = "https://huggingface.co/api/models/") + @JsonProperty("model-check-url") + private String modelUrl; + + @ConfigProperty( + description = + """ + The access key to use for "api" provider. + """) + @JsonProperty("access-key") + private String accessKey; } } diff --git a/langstream-core/src/main/java/ai/langstream/impl/resources/AbstractResourceProvider.java b/langstream-core/src/main/java/ai/langstream/impl/resources/AbstractResourceProvider.java new file mode 100644 index 000000000..27f182a0d --- /dev/null +++ b/langstream-core/src/main/java/ai/langstream/impl/resources/AbstractResourceProvider.java @@ -0,0 +1,88 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.langstream.impl.resources; + +import ai.langstream.api.doc.ResourceConfigurationModel; +import ai.langstream.api.model.Resource; +import ai.langstream.api.runtime.ComputeClusterRuntime; +import ai.langstream.api.runtime.PluginsRegistry; +import ai.langstream.api.runtime.ResourceNodeProvider; +import ai.langstream.impl.uti.ClassConfigValidator; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; + +public abstract class AbstractResourceProvider implements ResourceNodeProvider { + + private final Set supportedTypes; + + public AbstractResourceProvider(Set supportedTypes) { + this.supportedTypes = supportedTypes; + } + + protected Class getResourceConfigModelClass(String type) { + return null; + } + + protected boolean isResourceConfigModelAllowUnknownProperties(String type) { + return false; + } + + @Override + public Map createImplementation( + Resource resource, PluginsRegistry pluginsRegistry) { + return computeResourceConfiguration(resource, pluginsRegistry); + } + + protected Map computeResourceConfiguration( + Resource resource, PluginsRegistry pluginsRegistry) { + final String type = resource.type(); + final Class modelClass = getResourceConfigModelClass(type); + if (modelClass != null) { + ClassConfigValidator.validateResourceModelFromClass( + resource, + modelClass, + resource.configuration(), + isResourceConfigModelAllowUnknownProperties(type)); + } + return new HashMap<>(resource.configuration()); + } + + @Override + public boolean supports(String type, ComputeClusterRuntime clusterRuntime) { + return supportedTypes.contains(type); + } + + @Override + public Map generateSupportedTypesDocumentation() { + Map result = new LinkedHashMap<>(); + for (String supportedType : supportedTypes) { + final Class modelClass = getResourceConfigModelClass(supportedType); + if (modelClass == null) { + final ResourceConfigurationModel model = new ResourceConfigurationModel(); + model.setType(supportedType); + result.put(supportedType, model); + } else { + final ResourceConfigurationModel value = + ClassConfigValidator.generateResourceModelFromClass(modelClass); + value.setType(supportedType); + result.put(supportedType, value); + } + } + return result; + } +} diff --git a/langstream-core/src/main/java/ai/langstream/impl/resources/BaseDataSourceResourceProvider.java b/langstream-core/src/main/java/ai/langstream/impl/resources/BaseDataSourceResourceProvider.java new file mode 100644 index 000000000..166f9f6c3 --- /dev/null +++ b/langstream-core/src/main/java/ai/langstream/impl/resources/BaseDataSourceResourceProvider.java @@ -0,0 +1,88 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.langstream.impl.resources; + +import static ai.langstream.api.util.ConfigurationUtils.getString; + +import ai.langstream.api.doc.ResourceConfigurationModel; +import ai.langstream.api.model.Resource; +import ai.langstream.api.runtime.ComputeClusterRuntime; +import ai.langstream.api.runtime.PluginsRegistry; +import ai.langstream.api.runtime.ResourceNodeProvider; +import ai.langstream.impl.uti.ClassConfigValidator; +import java.util.LinkedHashMap; +import java.util.Map; +import lombok.AllArgsConstructor; + +@AllArgsConstructor +public class BaseDataSourceResourceProvider implements ResourceNodeProvider { + + private final String resourceType; + private final Map supportedServices; + + public interface DatasourceConfig { + void validate(Resource resource); + + Class getResourceConfigModelClass(); + } + + @Override + public Map createImplementation( + Resource resource, PluginsRegistry pluginsRegistry) { + Map configuration = resource.configuration(); + + final String service = getString("service", null, configuration); + if (service == null) { + throw new IllegalArgumentException( + ClassConfigValidator.formatErrString( + new ClassConfigValidator.ResourceEntityRef(resource), + "service", + "service must be set to one of: " + supportedServices.keySet())); + } + if (!supportedServices.keySet().contains(service)) { + throw new IllegalArgumentException( + ClassConfigValidator.formatErrString( + new ClassConfigValidator.ResourceEntityRef(resource), + "service", + "service must be set to one of: " + supportedServices.keySet())); + } + supportedServices.get(service).validate(resource); + return resource.configuration(); + } + + @Override + public boolean supports(String type, ComputeClusterRuntime clusterRuntime) { + return resourceType.equals(type); + } + + @Override + public Map generateSupportedTypesDocumentation() { + Map result = new LinkedHashMap<>(); + for (Map.Entry datasource : supportedServices.entrySet()) { + final String service = datasource.getKey(); + final ResourceConfigurationModel value = + ClassConfigValidator.generateResourceModelFromClass( + datasource.getValue().getResourceConfigModelClass()); + value.getProperties() + .get("service") + .setDescription("Service type. Set to '" + service + "'"); + + value.setType(resourceType); + result.put(resourceType + "_" + service, value); + } + return result; + } +} diff --git a/langstream-core/src/main/java/ai/langstream/impl/resources/DataSourceResourceProvider.java b/langstream-core/src/main/java/ai/langstream/impl/resources/DataSourceResourceProvider.java index 4d8f782cf..3ff5636ad 100644 --- a/langstream-core/src/main/java/ai/langstream/impl/resources/DataSourceResourceProvider.java +++ b/langstream-core/src/main/java/ai/langstream/impl/resources/DataSourceResourceProvider.java @@ -15,120 +15,23 @@ */ package ai.langstream.impl.resources; -import static ai.langstream.api.util.ConfigurationUtils.requiredField; -import static ai.langstream.api.util.ConfigurationUtils.requiredNonEmptyField; -import static ai.langstream.api.util.ConfigurationUtils.validateEnumField; -import static ai.langstream.api.util.ConfigurationUtils.validateInteger; - -import ai.langstream.api.model.Module; -import ai.langstream.api.model.Resource; -import ai.langstream.api.runtime.ComputeClusterRuntime; -import ai.langstream.api.runtime.ExecutionPlan; -import ai.langstream.api.runtime.PluginsRegistry; -import ai.langstream.api.runtime.ResourceNodeProvider; -import ai.langstream.api.util.ConfigurationUtils; -import java.util.Base64; +import ai.langstream.impl.resources.datasource.AstraDatasourceConfig; +import ai.langstream.impl.resources.datasource.CassandraDatasourceConfig; +import ai.langstream.impl.resources.datasource.JDBCDatasourceConfig; import java.util.Map; -import java.util.Set; -import java.util.function.Supplier; - -public class DataSourceResourceProvider implements ResourceNodeProvider { - @Override - public Map createImplementation( - Resource resource, - Module module, - ExecutionPlan executionPlan, - ComputeClusterRuntime clusterRuntime, - PluginsRegistry pluginsRegistry) { - Map configuration = resource.configuration(); - - String service = requiredField(configuration, "service", describe(resource)); - validateEnumField( - configuration, "service", Set.of("astra", "cassandra"), describe(resource)); - - switch (service) { - case "astra": - validateAstraDatabaseResource(resource); - break; - case "cassandra": - validateCassandraDatabaseResource(resource); - break; - case "jdbc": - validateJDBCDatabaseResource(resource); - default: - throw new IllegalStateException(); - } - return resource.configuration(); - } - - private void validateJDBCDatabaseResource(Resource resource) { - Map configuration = resource.configuration(); - requiredNonEmptyField(configuration, "url", describe(resource)); - requiredNonEmptyField(configuration, "driverClass", describe(resource)); - } - - @Override - public boolean supports(String type, ComputeClusterRuntime clusterRuntime) { - return "datasource".equals(type); - } - protected void validateAstraDatabaseResource(Resource resource) { - Map configuration = resource.configuration(); +public class DataSourceResourceProvider extends BaseDataSourceResourceProvider { - String secureBundle = ConfigurationUtils.getString("secureBundle", "", configuration); - if (secureBundle.isEmpty()) { - requiredNonEmptyField(configuration, "token", describe(resource)); - requiredNonEmptyField(configuration, "database", describe(resource)); - } else { - if (secureBundle.startsWith("base64:")) { - secureBundle = secureBundle.substring("base64:".length()); - } - try { - Base64.getDecoder().decode(secureBundle); - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException( - "Invalid base64 encoding for secureBundle in " + describe(resource), e); - } - } - - String username = - ConfigurationUtils.getString( - "clientId", - ConfigurationUtils.getString("username", "", configuration), - configuration); - if (username.isEmpty()) { - requiredNonEmptyField(configuration, "clientId", describe(resource)); - } - - String password = - ConfigurationUtils.getString( - "secret", - ConfigurationUtils.getString("password", "", configuration), - configuration); - if (password.isEmpty()) { - requiredNonEmptyField(configuration, "secret", describe(resource)); - } - } - - protected void validateCassandraDatabaseResource(Resource resource) { - Map configuration = resource.configuration(); - - String secureBundle = ConfigurationUtils.getString("secureBundle", "", configuration); - if (!secureBundle.isEmpty()) { - throw new IllegalArgumentException( - "secureBundle is not supported for Cassandra services, use service=astra instead"); - } - - // in Cassandra testes you can use a Cassandra service without authentication - requiredField(configuration, "username", describe(resource)); - requiredField(configuration, "password", describe(resource)); - - requiredNonEmptyField(configuration, "contact-points", describe(resource)); - requiredNonEmptyField(configuration, "loadBalancing-localDc", describe(resource)); - validateInteger(configuration, "port", 1, 65535, describe(resource)); - } + protected static final String SERVICE_ASTRA = "astra"; + protected static final String SERVICE_CASSANDRA = "cassandra"; + protected static final String SERVICE_JDBC = "jdbc"; - protected static Supplier describe(Resource resource) { - return () -> "resource with id = " + resource.id() + " of type " + resource.type(); + public DataSourceResourceProvider() { + super( + "datasource", + Map.of( + SERVICE_ASTRA, AstraDatasourceConfig.CONFIG, + SERVICE_CASSANDRA, CassandraDatasourceConfig.CONFIG, + SERVICE_JDBC, JDBCDatasourceConfig.CONFIG)); } } diff --git a/langstream-core/src/main/java/ai/langstream/impl/resources/VectorDatabaseResourceProvider.java b/langstream-core/src/main/java/ai/langstream/impl/resources/VectorDatabaseResourceProvider.java index f3fa1a3be..f95e90859 100644 --- a/langstream-core/src/main/java/ai/langstream/impl/resources/VectorDatabaseResourceProvider.java +++ b/langstream-core/src/main/java/ai/langstream/impl/resources/VectorDatabaseResourceProvider.java @@ -15,81 +15,21 @@ */ package ai.langstream.impl.resources; -import static ai.langstream.api.util.ConfigurationUtils.requiredField; -import static ai.langstream.api.util.ConfigurationUtils.requiredNonEmptyField; -import static ai.langstream.api.util.ConfigurationUtils.validateEnumField; - -import ai.langstream.api.model.Module; -import ai.langstream.api.model.Resource; -import ai.langstream.api.runtime.ComputeClusterRuntime; -import ai.langstream.api.runtime.ExecutionPlan; -import ai.langstream.api.runtime.PluginsRegistry; -import ai.langstream.api.runtime.ResourceNodeProvider; -import ai.langstream.api.util.ConfigurationUtils; +import ai.langstream.impl.resources.datasource.AstraDatasourceConfig; +import ai.langstream.impl.resources.datasource.CassandraDatasourceConfig; +import ai.langstream.impl.resources.datasource.MilvusDatasourceConfig; +import ai.langstream.impl.resources.datasource.PineconeDatasourceConfig; import java.util.Map; -import java.util.Set; - -public class VectorDatabaseResourceProvider extends DataSourceResourceProvider - implements ResourceNodeProvider { - @Override - public Map createImplementation( - Resource resource, - Module module, - ExecutionPlan executionPlan, - ComputeClusterRuntime clusterRuntime, - PluginsRegistry pluginsRegistry) { - Map configuration = resource.configuration(); - - String service = requiredField(configuration, "service", describe(resource)); - validateEnumField( - configuration, - "service", - Set.of("astra", "cassandra", "pinecone", "milvus"), - describe(resource)); - - switch (service) { - case "astra": - validateAstraDatabaseResource(resource); - break; - case "cassandra": - validateCassandraDatabaseResource(resource); - break; - case "pinecone": - validatePineconeDatabaseResource(resource); - break; - case "milvus": - validateMilvusDatabaseResource(resource); - break; - default: - throw new IllegalStateException(); - } - - return resource.configuration(); - } - private void validateMilvusDatabaseResource(Resource resource) { - Map configuration = resource.configuration(); - - requiredNonEmptyField(configuration, "user", describe(resource)); - requiredNonEmptyField(configuration, "host", describe(resource)); - requiredNonEmptyField(configuration, "password", describe(resource)); - requiredNonEmptyField(configuration, "index-name", describe(resource)); - ConfigurationUtils.validateInteger(configuration, "port", 1, 300000, describe(resource)); - } - - protected void validatePineconeDatabaseResource(Resource resource) { - Map configuration = resource.configuration(); - - requiredNonEmptyField(configuration, "api-key", describe(resource)); - requiredNonEmptyField(configuration, "environment", describe(resource)); - requiredNonEmptyField(configuration, "project-name", describe(resource)); - requiredNonEmptyField(configuration, "index-name", describe(resource)); - ConfigurationUtils.validateInteger( - configuration, "server-side-timeout-sec", 1, 300000, describe(resource)); - } +public class VectorDatabaseResourceProvider extends BaseDataSourceResourceProvider { - @Override - public boolean supports(String type, ComputeClusterRuntime clusterRuntime) { - return "vector-database".equals(type); + public VectorDatabaseResourceProvider() { + super( + "vector-database", + Map.of( + "astra", AstraDatasourceConfig.CONFIG, + "cassandra", CassandraDatasourceConfig.CONFIG, + "pinecone", PineconeDatasourceConfig.CONFIG, + "milvus", MilvusDatasourceConfig.CONFIG)); } } diff --git a/langstream-core/src/main/java/ai/langstream/impl/resources/datasource/AstraDatasourceConfig.java b/langstream-core/src/main/java/ai/langstream/impl/resources/datasource/AstraDatasourceConfig.java new file mode 100644 index 000000000..69bcf6dbc --- /dev/null +++ b/langstream-core/src/main/java/ai/langstream/impl/resources/datasource/AstraDatasourceConfig.java @@ -0,0 +1,125 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.langstream.impl.resources.datasource; + +import ai.langstream.api.doc.ConfigProperty; +import ai.langstream.api.doc.ResourceConfig; +import ai.langstream.api.model.Resource; +import ai.langstream.api.util.ConfigurationUtils; +import ai.langstream.impl.resources.BaseDataSourceResourceProvider; +import ai.langstream.impl.uti.ClassConfigValidator; +import java.util.Base64; +import java.util.Map; +import lombok.Data; + +@Data +@ResourceConfig(name = "Astra", description = "Connect to DataStax Astra Database service.") +public class AstraDatasourceConfig extends BaseDatasourceConfig { + + public static final BaseDataSourceResourceProvider.DatasourceConfig CONFIG = + new BaseDataSourceResourceProvider.DatasourceConfig() { + + @Override + public Class getResourceConfigModelClass() { + return AstraDatasourceConfig.class; + } + + @Override + public void validate(Resource resource) { + ClassConfigValidator.validateResourceModelFromClass( + resource, AstraDatasourceConfig.class, resource.configuration(), false); + Map configuration = resource.configuration(); + + String secureBundle = + ConfigurationUtils.getString("secureBundle", "", configuration); + if (secureBundle.isEmpty()) { + if (configuration.get("token") == null + || configuration.get("database") == null) { + throw new IllegalArgumentException( + ClassConfigValidator.formatErrString( + new ClassConfigValidator.ResourceEntityRef(resource), + "token", + "token and database are required for Astra service if secureBundle is not" + + " configured.")); + } + } else { + if (secureBundle.startsWith("base64:")) { + secureBundle = secureBundle.substring("base64:".length()); + } + try { + Base64.getDecoder().decode(secureBundle); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException( + ClassConfigValidator.formatErrString( + new ClassConfigValidator.ResourceEntityRef(resource), + "secureBundle", + "secureBundle must be a valid base64 string.")); + } + } + } + }; + + @ConfigProperty( + description = + """ + Secure bundle of the database. Must be encoded in base64. + """) + private String secureBundle; + + @ConfigProperty( + description = + """ + Astra Token (AstraCS:xxx) for connecting to the database. If secureBundle is provided, this field is ignored. + """) + private String token; + + @ConfigProperty( + description = + """ + Astra Database name to connect to. If secureBundle is provided, this field is ignored. + """) + private String database; + + @ConfigProperty( + description = + """ + Astra Token clientId to use. + """, + required = true) + private String clientId; + + @ConfigProperty( + description = + """ + Astra Token secret to use. + """, + required = true) + private String secret; + + public enum Environments { + DEV, + PROD, + TEST; + } + + @ConfigProperty( + description = + """ + Astra environment. + """, + defaultValue = "PROD") + private Environments environment; +} diff --git a/langstream-core/src/main/java/ai/langstream/impl/resources/datasource/BaseDatasourceConfig.java b/langstream-core/src/main/java/ai/langstream/impl/resources/datasource/BaseDatasourceConfig.java new file mode 100644 index 000000000..c96fa9ee5 --- /dev/null +++ b/langstream-core/src/main/java/ai/langstream/impl/resources/datasource/BaseDatasourceConfig.java @@ -0,0 +1,26 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.langstream.impl.resources.datasource; + +import ai.langstream.api.doc.ConfigProperty; +import lombok.Data; + +@Data +public class BaseDatasourceConfig { + + @ConfigProperty(required = true) + private String service; +} diff --git a/langstream-core/src/main/java/ai/langstream/impl/resources/datasource/CassandraDatasourceConfig.java b/langstream-core/src/main/java/ai/langstream/impl/resources/datasource/CassandraDatasourceConfig.java new file mode 100644 index 000000000..b11498ba8 --- /dev/null +++ b/langstream-core/src/main/java/ai/langstream/impl/resources/datasource/CassandraDatasourceConfig.java @@ -0,0 +1,96 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.langstream.impl.resources.datasource; + +import static ai.langstream.api.util.ConfigurationUtils.validateInteger; + +import ai.langstream.api.doc.ConfigProperty; +import ai.langstream.api.doc.ResourceConfig; +import ai.langstream.api.model.Resource; +import ai.langstream.impl.resources.BaseDataSourceResourceProvider; +import ai.langstream.impl.uti.ClassConfigValidator; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Map; +import lombok.Data; + +@Data +@ResourceConfig(name = "Cassandra", description = "Connect to Apache cassandra.") +public class CassandraDatasourceConfig extends BaseDatasourceConfig { + + public static final BaseDataSourceResourceProvider.DatasourceConfig CONFIG = + new BaseDataSourceResourceProvider.DatasourceConfig() { + @Override + public Class getResourceConfigModelClass() { + return CassandraDatasourceConfig.class; + } + + @Override + public void validate(Resource resource) { + ClassConfigValidator.validateResourceModelFromClass( + resource, + CassandraDatasourceConfig.class, + resource.configuration(), + false); + Map configuration = resource.configuration(); + validateInteger( + configuration, + "port", + 1, + 65535, + () -> new ClassConfigValidator.ResourceEntityRef(resource).ref()); + } + }; + + @ConfigProperty( + description = + """ + Contact points of the cassandra cluster. + """, + required = true) + @JsonProperty("contact-points") + private String contactPoints; + + @ConfigProperty( + description = + """ + Load balancing local datacenter. + """, + required = true) + @JsonProperty("loadBalancing-localDc") + private String loadBalancingLocalDc; + + @ConfigProperty( + description = + """ + Cassandra port. + """, + defaultValue = "9042") + private int port; + + @ConfigProperty( + description = + """ + User username. + """) + private String username; + + @ConfigProperty( + description = + """ + User password. + """) + private String password; +} diff --git a/langstream-core/src/main/java/ai/langstream/impl/resources/datasource/JDBCDatasourceConfig.java b/langstream-core/src/main/java/ai/langstream/impl/resources/datasource/JDBCDatasourceConfig.java new file mode 100644 index 000000000..0b08b7ae8 --- /dev/null +++ b/langstream-core/src/main/java/ai/langstream/impl/resources/datasource/JDBCDatasourceConfig.java @@ -0,0 +1,61 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.langstream.impl.resources.datasource; + +import ai.langstream.api.doc.ConfigProperty; +import ai.langstream.api.doc.ResourceConfig; +import ai.langstream.api.model.Resource; +import ai.langstream.impl.resources.BaseDataSourceResourceProvider; +import ai.langstream.impl.uti.ClassConfigValidator; +import lombok.Data; + +@Data +@ResourceConfig( + name = "JDBC", + description = + "Connect to any JDBC compatible database. The driver must be provided as dependency. All the extra configuration properties are passed as is to the JDBC driver.") +public class JDBCDatasourceConfig extends BaseDatasourceConfig { + + public static final BaseDataSourceResourceProvider.DatasourceConfig CONFIG = + new BaseDataSourceResourceProvider.DatasourceConfig() { + @Override + public Class getResourceConfigModelClass() { + return JDBCDatasourceConfig.class; + } + + @Override + public void validate(Resource resource) { + ClassConfigValidator.validateResourceModelFromClass( + resource, JDBCDatasourceConfig.class, resource.configuration(), true); + } + }; + + @ConfigProperty( + description = + """ + JDBC entry-point driver class. + """, + required = true) + private String driverClass; + + @ConfigProperty( + description = + """ + JDBC connection url. + """, + required = true) + private String url; +} diff --git a/langstream-core/src/main/java/ai/langstream/impl/resources/datasource/MilvusDatasourceConfig.java b/langstream-core/src/main/java/ai/langstream/impl/resources/datasource/MilvusDatasourceConfig.java new file mode 100644 index 000000000..9f776f64c --- /dev/null +++ b/langstream-core/src/main/java/ai/langstream/impl/resources/datasource/MilvusDatasourceConfig.java @@ -0,0 +1,99 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.langstream.impl.resources.datasource; + +import ai.langstream.api.doc.ConfigProperty; +import ai.langstream.api.doc.ResourceConfig; +import ai.langstream.api.model.Resource; +import ai.langstream.api.util.ConfigurationUtils; +import ai.langstream.impl.resources.BaseDataSourceResourceProvider; +import ai.langstream.impl.uti.ClassConfigValidator; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +@ResourceConfig(name = "Milvus", description = "Connect to Milvus/Zillis service.") +public class MilvusDatasourceConfig extends BaseDatasourceConfig { + + public static final BaseDataSourceResourceProvider.DatasourceConfig CONFIG = + new BaseDataSourceResourceProvider.DatasourceConfig() { + + @Override + public Class getResourceConfigModelClass() { + return MilvusDatasourceConfig.class; + } + + @Override + public void validate(Resource resource) { + ClassConfigValidator.validateResourceModelFromClass( + resource, + MilvusDatasourceConfig.class, + resource.configuration(), + false); + ConfigurationUtils.validateInteger( + resource.configuration(), + "port", + 1, + 300000, + () -> new ClassConfigValidator.ResourceEntityRef(resource).ref()); + } + }; + + @ConfigProperty( + description = + """ + User parameter for connecting to Milvus. + """, + defaultValue = "default") + private String user; + + @ConfigProperty( + description = + """ + Host parameter for connecting to Milvus. + """) + private String host; + + @ConfigProperty( + description = + """ + Password parameter for connecting to Milvus. + """) + private String password; + + @ConfigProperty( + description = + """ + Port parameter for connecting to Milvus. + """, + defaultValue = "19530") + private int port; + + @ConfigProperty( + description = + """ + Url parameter for connecting to Zillis service. + """) + @JsonProperty("index-name") + private String url; + + @ConfigProperty( + description = + """ + Token parameter for connecting to Zillis service. + """) + private String token; +} diff --git a/langstream-core/src/main/java/ai/langstream/impl/resources/datasource/PineconeDatasourceConfig.java b/langstream-core/src/main/java/ai/langstream/impl/resources/datasource/PineconeDatasourceConfig.java new file mode 100644 index 000000000..2739f6c7d --- /dev/null +++ b/langstream-core/src/main/java/ai/langstream/impl/resources/datasource/PineconeDatasourceConfig.java @@ -0,0 +1,105 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.langstream.impl.resources.datasource; + +import ai.langstream.api.doc.ConfigProperty; +import ai.langstream.api.doc.ResourceConfig; +import ai.langstream.api.model.Resource; +import ai.langstream.api.util.ConfigurationUtils; +import ai.langstream.impl.resources.BaseDataSourceResourceProvider; +import ai.langstream.impl.uti.ClassConfigValidator; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +@ResourceConfig(name = "Pinecone", description = "Connect to Pinecone service.") +public class PineconeDatasourceConfig extends BaseDatasourceConfig { + + public static final BaseDataSourceResourceProvider.DatasourceConfig CONFIG = + new BaseDataSourceResourceProvider.DatasourceConfig() { + + @Override + public Class getResourceConfigModelClass() { + return PineconeDatasourceConfig.class; + } + + @Override + public void validate(Resource resource) { + ClassConfigValidator.validateResourceModelFromClass( + resource, + PineconeDatasourceConfig.class, + resource.configuration(), + false); + ConfigurationUtils.validateInteger( + resource.configuration(), + "server-side-timeout-sec", + 1, + 300000, + () -> new ClassConfigValidator.ResourceEntityRef(resource).ref()); + } + }; + + @ConfigProperty( + description = + """ + Api key for connecting to the Pinecone service. + """, + required = true) + @JsonProperty("api-key") + private String apiKey; + + @ConfigProperty( + description = + """ + Environment parameter for connecting to the Pinecone service. + """, + required = true) + private String environment; + + @ConfigProperty( + description = + """ + Project name parameter for connecting to the Pinecone service. + """, + required = true) + @JsonProperty("project-name") + private String project; + + @ConfigProperty( + description = + """ + Index name parameter for connecting to the Pinecone service. + """, + required = true) + @JsonProperty("index-name") + private String index; + + @ConfigProperty( + description = + """ + Server side timeout parameter for connecting to the Pinecone service. + """, + defaultValue = "10") + @JsonProperty("server-side-timeout-sec") + private int serverSideTimeoutSec; + + @ConfigProperty( + description = + """ + Endpoint of the Pinecone service. + """) + private String endpoint; +} diff --git a/langstream-core/src/main/java/ai/langstream/impl/uti/ClassConfigValidator.java b/langstream-core/src/main/java/ai/langstream/impl/uti/ClassConfigValidator.java index 62d75a4d5..b3e9a9738 100644 --- a/langstream-core/src/main/java/ai/langstream/impl/uti/ClassConfigValidator.java +++ b/langstream-core/src/main/java/ai/langstream/impl/uti/ClassConfigValidator.java @@ -19,7 +19,10 @@ import ai.langstream.api.doc.AgentConfigurationModel; import ai.langstream.api.doc.ConfigProperty; import ai.langstream.api.doc.ConfigPropertyIgnore; +import ai.langstream.api.doc.ResourceConfig; +import ai.langstream.api.doc.ResourceConfigurationModel; import ai.langstream.api.model.AgentConfiguration; +import ai.langstream.api.model.Resource; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.DeserializationFeature; @@ -40,6 +43,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; +import lombok.AllArgsConstructor; import lombok.Data; import lombok.SneakyThrows; import org.apache.commons.lang3.tuple.Pair; @@ -55,6 +59,7 @@ public class ClassConfigValidator { new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); static final Map agentModels = new ConcurrentHashMap<>(); + static final Map resourceModels = new ConcurrentHashMap<>(); public static AgentConfigurationModel generateAgentModelFromClass(Class clazz) { return agentModels.computeIfAbsent( @@ -77,11 +82,30 @@ private static AgentConfigurationModel generateModelFromClassNoCache(Class clazz return model; } - @SneakyThrows - public static void validateAgentModelFromClass( - AgentConfiguration agentConfiguration, Class modelClazz) { - validateAgentModelFromClass( - agentConfiguration, modelClazz, agentConfiguration.getConfiguration()); + public static ResourceConfigurationModel generateResourceModelFromClass(Class clazz) { + return resourceModels.computeIfAbsent( + clazz.getName(), k -> generateResourceModelFromClassNoCache(clazz)); + } + + private static ResourceConfigurationModel generateResourceModelFromClassNoCache(Class clazz) { + ResourceConfigurationModel model = new ResourceConfigurationModel(); + + final ResourceConfig resourceConfig = + (ResourceConfig) clazz.getAnnotation(ResourceConfig.class); + if (resourceConfig != null) { + if (resourceConfig.description() != null && !resourceConfig.description().isBlank()) { + model.setDescription(resourceConfig.description().strip()); + } + if (resourceConfig.name() != null && !resourceConfig.name().isBlank()) { + model.setName(resourceConfig.name()); + } + } + model.setProperties(readPropertiesFromClass(clazz)); + return model; + } + + public interface EntityRef { + String ref(); } @SneakyThrows @@ -96,13 +120,54 @@ public static void validateAgentModelFromClass( Class modelClazz, Map asMap, boolean allowUnknownProperties) { + final EntityRef ref = + () -> + "agent configuration (agent: '%s', type: '%s')" + .formatted( + agentConfiguration.getName() == null + ? agentConfiguration.getId() + : agentConfiguration.getName(), + agentConfiguration.getType()); + validateModelFromClass(ref, modelClazz, asMap, allowUnknownProperties); + } + + @AllArgsConstructor + public static class ResourceEntityRef implements EntityRef { + + private final Resource resource; + + @Override + public String ref() { + return "resource configuration (resource: '%s', type: '%s')" + .formatted( + resource.name() == null ? resource.id() : resource.name(), + resource.type()); + } + } + + @SneakyThrows + public static void validateResourceModelFromClass( + Resource resource, + Class modelClazz, + Map asMap, + boolean allowUnknownProperties) { + validateModelFromClass( + new ResourceEntityRef(resource), modelClazz, asMap, allowUnknownProperties); + } + + @SneakyThrows + private static void validateModelFromClass( + EntityRef entityRef, + Class modelClazz, + Map asMap, + boolean allowUnknownProperties) { asMap = validatorMapper.readValue(validatorMapper.writeValueAsBytes(asMap), Map.class); final AgentConfigurationModel agentConfigurationModel = generateAgentModelFromClass(modelClazz); validateProperties( - agentConfiguration, + entityRef, null, asMap, agentConfigurationModel.getProperties(), @@ -118,7 +183,7 @@ public static void validateAgentModelFromClass( .collect(Collectors.joining(".")); throw new IllegalArgumentException( formatErrString( - agentConfiguration, + entityRef, property, "has a wrong data type. Expected type: " + mismatchedInputException.getTargetType().getName())); @@ -128,21 +193,15 @@ public static void validateAgentModelFromClass( } } - private static String formatErrString( - AgentConfiguration agent, String property, String message) { - return "Found error on an agent configuration (agent: '%s', type: '%s'). Property '%s' %s" - .formatted( - agent.getName() == null ? agent.getId() : agent.getName(), - agent.getType(), - property, - message); + public static String formatErrString(EntityRef entityRef, String property, String message) { + return "Found error on %s. Property '%s' %s".formatted(entityRef.ref(), property, message); } private static void validateProperties( - AgentConfiguration agentConfiguration, + EntityRef entityRef, String parentProp, Map asMap, - Map properties, + Map properties, boolean allowUnknownProperties) { if (!allowUnknownProperties && asMap != null) { for (String key : asMap.keySet()) { @@ -150,18 +209,17 @@ private static void validateProperties( final String fullPropertyKey = parentProp == null ? key : parentProp + "." + key; throw new IllegalArgumentException( - formatErrString(agentConfiguration, fullPropertyKey, "is unknown")); + formatErrString(entityRef, fullPropertyKey, "is unknown")); } } } - for (Map.Entry property : + for (Map.Entry property : properties.entrySet()) { - final AgentConfigurationModel.AgentConfigurationProperty propertyValue = - property.getValue(); + final ai.langstream.api.doc.ConfigPropertyModel propertyValue = property.getValue(); final String propertyKey = property.getKey(); validateProperty( - agentConfiguration, + entityRef, parentProp, asMap == null ? null : asMap.get(propertyKey), propertyValue, @@ -170,10 +228,10 @@ private static void validateProperties( } private static void validateProperty( - AgentConfiguration agentConfiguration, + EntityRef entityRef, String parentProp, Object actualValue, - AgentConfigurationModel.AgentConfigurationProperty propertyValue, + ai.langstream.api.doc.ConfigPropertyModel propertyValue, String propertyKey) { final String fullPropertyKey = @@ -182,12 +240,12 @@ private static void validateProperty( if (propertyValue.isRequired()) { if (actualValue == null) { throw new IllegalArgumentException( - formatErrString(agentConfiguration, fullPropertyKey, "is required")); + formatErrString(entityRef, fullPropertyKey, "is required")); } } if (propertyValue.getProperties() != null) { validateProperties( - agentConfiguration, + entityRef, fullPropertyKey, actualValue == null ? null : (Map) actualValue, propertyValue.getProperties(), @@ -217,20 +275,19 @@ public static class Prop { Map properties; } - public static Map - readPropertiesFromClass(Class clazz) { + public static Map readPropertiesFromClass( + Class clazz) { JsonNode jsonSchema = getJsonSchema(clazz); final ObjectMapper mapper = new ObjectMapper() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); final ParsedJsonSchema parsed = mapper.convertValue(jsonSchema, ParsedJsonSchema.class); - Map props = - new LinkedHashMap<>(); + Map props = new LinkedHashMap<>(); if (parsed.getProperties() != null) { for (Map.Entry schema : parsed.getProperties().entrySet()) { - final AgentConfigurationModel.AgentConfigurationProperty parsedProp = + final ai.langstream.api.doc.ConfigPropertyModel parsedProp = parseProp(mapper, schema.getValue()); if (parsedProp != null) { props.put(schema.getKey(), parsedProp); @@ -241,10 +298,10 @@ public static class Prop { } @SneakyThrows - private static AgentConfigurationModel.AgentConfigurationProperty parseProp( + private static ai.langstream.api.doc.ConfigPropertyModel parseProp( ObjectMapper mapper, ParsedJsonSchema.Prop value) { - AgentConfigurationModel.AgentConfigurationProperty newProp = - new AgentConfigurationModel.AgentConfigurationProperty(); + ai.langstream.api.doc.ConfigPropertyModel newProp = + new ai.langstream.api.doc.ConfigPropertyModel(); newProp.setType(value.getType()); final String jsonDesc = value.getDescription(); @@ -268,7 +325,7 @@ private static AgentConfigurationModel.AgentConfigurationProperty parseProp( .collect(Collectors.toMap(Pair::getLeft, Pair::getRight))); } if (value.getItems() != null) { - final AgentConfigurationModel.AgentConfigurationProperty items = + final ai.langstream.api.doc.ConfigPropertyModel items = parseProp(mapper, value.getItems()); if (items != null) { diff --git a/langstream-core/src/test/java/ai/langstream/impl/resources/ResourceNodeProviderTest.java b/langstream-core/src/test/java/ai/langstream/impl/resources/ResourceNodeProviderTest.java index 8de27bf4c..7e7551da7 100644 --- a/langstream-core/src/test/java/ai/langstream/impl/resources/ResourceNodeProviderTest.java +++ b/langstream-core/src/test/java/ai/langstream/impl/resources/ResourceNodeProviderTest.java @@ -47,16 +47,12 @@ private void test( switch (outcome) { case VALID -> { assertDoesNotThrow( - () -> - providersResourceProvider.createImplementation( - resource, null, null, null, null)); + () -> providersResourceProvider.createImplementation(resource, null)); } case NON_VALID -> { assertThrows( IllegalArgumentException.class, - () -> - providersResourceProvider.createImplementation( - resource, null, null, null, null)); + () -> providersResourceProvider.createImplementation(resource, null)); } } } @@ -65,7 +61,7 @@ private static List configurationsAIProviders() { return Arrays.asList( Arguments.of(NON_VALID, "open-ai-configuration", Map.of()), Arguments.of(NON_VALID, "vertex-configuration", Map.of()), - Arguments.of(NON_VALID, "hugging-face-configuration", Map.of()), + Arguments.of(VALID, "hugging-face-configuration", Map.of()), Arguments.of( VALID, "open-ai-configuration", @@ -84,38 +80,12 @@ private static List configurationsAIProviders() { "azure", "url", "http://some-url")), - Arguments.of( - VALID, - "hugging-face-configuration", - Map.of("provider", "api", "model", "some-model")), - Arguments.of( - VALID, - "hugging-face-configuration", - Map.of("model", "some-model", "options", Map.of("wait-for-model", "true"))), + Arguments.of(VALID, "hugging-face-configuration", Map.of("provider", "api")), + Arguments.of(VALID, "hugging-face-configuration", Map.of("provider", "local")), Arguments.of( NON_VALID, "hugging-face-configuration", - Map.of("model", "some-model", "options", "this-is-not-a-map")), - Arguments.of( - VALID, - "hugging-face-configuration", - Map.of( - "model", - "some-model", - "arguments", - Map.of("some-argument", "true"))), - Arguments.of( - NON_VALID, - "hugging-face-configuration", - Map.of("model", "some-model", "arguments", "this-is-not-a-map")), - Arguments.of( - VALID, - "hugging-face-configuration", - Map.of("provider", "local", "model", "some-model")), - Arguments.of( - NON_VALID, - "hugging-face-configuration", - Map.of("provider", "bad-provider", "model", "some-model")), + Map.of("provider", "bad-provider")), Arguments.of( VALID, "vertex-configuration", @@ -285,7 +255,7 @@ private static List configurationsDataSourceProviders() { "secret", "pwd")), Arguments.of( - VALID, + NON_VALID, "astra", Map.of( "secureBundle", @@ -295,7 +265,7 @@ private static List configurationsDataSourceProviders() { "password", "pwd")), Arguments.of( - VALID, + NON_VALID, "astra", Map.of( "secureBundle", diff --git a/langstream-k8s-runtime/langstream-k8s-runtime-core/src/main/java/ai/langstream/runtime/impl/k8s/agents/PythonCodeAgentProvider.java b/langstream-k8s-runtime/langstream-k8s-runtime-core/src/main/java/ai/langstream/runtime/impl/k8s/agents/PythonCodeAgentProvider.java index a634bc0be..f9eb61d56 100644 --- a/langstream-k8s-runtime/langstream-k8s-runtime-core/src/main/java/ai/langstream/runtime/impl/k8s/agents/PythonCodeAgentProvider.java +++ b/langstream-k8s-runtime/langstream-k8s-runtime-core/src/main/java/ai/langstream/runtime/impl/k8s/agents/PythonCodeAgentProvider.java @@ -66,7 +66,7 @@ protected boolean isAgentConfigModelAllowUnknownProperties(String type) { description = """ Run a your own Python source. - All the configuration properties are available to in the class init method. + All the configuration properties are available in the class init method. """) public static class PythonSourceConfig extends PythonConfig {} @@ -75,7 +75,7 @@ public static class PythonSourceConfig extends PythonConfig {} description = """ Run a your own Python sink. - All the configuration properties are available to in the class init method. + All the configuration properties are available in the class init method. """) public static class PythonSinkConfig extends PythonConfig {} @@ -84,7 +84,7 @@ public static class PythonSinkConfig extends PythonConfig {} description = """ Run a your own Python processor. - All the configuration properties are available to in the class init method. + All the configuration properties are available the class init method. """) public static class PythonProcessorConfig extends PythonConfig {} diff --git a/langstream-k8s-runtime/langstream-k8s-runtime-core/src/main/java/ai/langstream/runtime/impl/k8s/agents/WebCrawlerSourceAgentProvider.java b/langstream-k8s-runtime/langstream-k8s-runtime-core/src/main/java/ai/langstream/runtime/impl/k8s/agents/WebCrawlerSourceAgentProvider.java index b6f4eac03..85d7afc5a 100644 --- a/langstream-k8s-runtime/langstream-k8s-runtime-core/src/main/java/ai/langstream/runtime/impl/k8s/agents/WebCrawlerSourceAgentProvider.java +++ b/langstream-k8s-runtime/langstream-k8s-runtime-core/src/main/java/ai/langstream/runtime/impl/k8s/agents/WebCrawlerSourceAgentProvider.java @@ -40,6 +40,11 @@ protected final ComponentType getComponentType(AgentConfiguration agentConfigura return ComponentType.SOURCE; } + @Override + protected Class getAgentConfigModelClass(String type) { + return Config.class; + } + @Data @AgentConfig( name = "Web crawler source", diff --git a/langstream-k8s-runtime/langstream-k8s-runtime-core/src/test/java/ai/langstream/runtime/impl/k8s/GenAIAgentsTest.java b/langstream-k8s-runtime/langstream-k8s-runtime-core/src/test/java/ai/langstream/runtime/impl/k8s/GenAIAgentsTest.java index cb2a9fb38..7bc72f5b2 100644 --- a/langstream-k8s-runtime/langstream-k8s-runtime-core/src/test/java/ai/langstream/runtime/impl/k8s/GenAIAgentsTest.java +++ b/langstream-k8s-runtime/langstream-k8s-runtime-core/src/test/java/ai/langstream/runtime/impl/k8s/GenAIAgentsTest.java @@ -395,7 +395,9 @@ public void testMapAllGenAIToolKitAgents() throws Exception { - name: my-database type: datasource configuration: - connectionUrl: localhost:1544 + service: jdbc + url: "jdbc:postgresql://localhost:5432/postgres" + driverClass: "org.postgresql.Driver" """, "module.yaml", """ @@ -540,11 +542,15 @@ public void testMultipleQuerySteps() throws Exception { - name: my-database-1 type: datasource configuration: - connectionUrl: localhost:1544 + service: jdbc + url: "jdbc:postgresql://localhost:5432/postgres" + driverClass: "org.postgresql.Driver" - name: my-database-2 type: datasource configuration: - connectionUrl: localhost:1545 + service: jdbc + url: "jdbc:postgresql://localhost:5432/postgres" + driverClass: "org.postgresql.Driver" """, "module.yaml", """ @@ -606,7 +612,11 @@ public void testMultipleQuerySteps() throws Exception { log.info("Configuration: {}", configuration); Map datasourceConfiguration1 = (Map) configuration.get("datasource"); - assertEquals("localhost:1544", datasourceConfiguration1.get("connectionUrl")); + assertEquals("jdbc", datasourceConfiguration1.get("service")); + assertEquals( + "jdbc:postgresql://localhost:5432/postgres", + datasourceConfiguration1.get("url")); + assertEquals("org.postgresql.Driver", datasourceConfiguration1.get("driverClass")); List> steps = (List>) configuration.get("steps"); assertEquals(1, steps.size()); @@ -622,7 +632,11 @@ public void testMultipleQuerySteps() throws Exception { log.info("Configuration: {}", configuration); Map datasourceConfiguration1 = (Map) configuration.get("datasource"); - assertEquals("localhost:1545", datasourceConfiguration1.get("connectionUrl")); + assertEquals("jdbc", datasourceConfiguration1.get("service")); + assertEquals( + "jdbc:postgresql://localhost:5432/postgres", + datasourceConfiguration1.get("url")); + assertEquals("org.postgresql.Driver", datasourceConfiguration1.get("driverClass")); List> steps = (List>) configuration.get("steps"); // query + cast @@ -675,7 +689,9 @@ public void testEmbeddingsThanQuery() throws Exception { - name: my-database-1 type: datasource configuration: - connectionUrl: localhost:1544 + service: jdbc + url: "jdbc:postgresql://localhost:5432/postgres" + driverClass: "org.postgresql.Driver" """, "module.yaml", """ @@ -743,7 +759,11 @@ public void testEmbeddingsThanQuery() throws Exception { log.info("Configuration: {}", configuration); Map datasourceConfiguration1 = (Map) configuration.get("datasource"); - assertEquals("localhost:1544", datasourceConfiguration1.get("connectionUrl")); + assertEquals("jdbc", datasourceConfiguration1.get("service")); + assertEquals( + "jdbc:postgresql://localhost:5432/postgres", + datasourceConfiguration1.get("url")); + assertEquals("org.postgresql.Driver", datasourceConfiguration1.get("driverClass")); List> steps = (List>) configuration.get("steps"); assertEquals(1, steps.size()); @@ -773,6 +793,7 @@ public void testForceAiService() throws Exception { url: "http://something" token: xx project: yy + region: us-central1 """, "module.yaml", """ diff --git a/langstream-k8s-runtime/langstream-k8s-runtime-core/src/test/java/ai/langstream/runtime/impl/k8s/agents/KafkaConnectAgentsProviderTest.java b/langstream-k8s-runtime/langstream-k8s-runtime-core/src/test/java/ai/langstream/runtime/impl/k8s/agents/KafkaConnectAgentsProviderTest.java index 206efb5c0..841f9247c 100644 --- a/langstream-k8s-runtime/langstream-k8s-runtime-core/src/test/java/ai/langstream/runtime/impl/k8s/agents/KafkaConnectAgentsProviderTest.java +++ b/langstream-k8s-runtime/langstream-k8s-runtime-core/src/test/java/ai/langstream/runtime/impl/k8s/agents/KafkaConnectAgentsProviderTest.java @@ -55,7 +55,7 @@ public void testValidationSource() { type: "source" configuration: {} """, - "Found error on an agent configuration (agent: 'my-source', type: 'source'). Property 'connector.class' is required"); + "Found error on agent configuration (agent: 'my-source', type: 'source'). Property 'connector.class' is required"); validate( """ @@ -105,7 +105,7 @@ public void testValidationSink() { type: "sink" configuration: {} """, - "Found error on an agent configuration (agent: 'my-source', type: 'sink'). Property 'connector.class' is required"); + "Found error on agent configuration (agent: 'my-source', type: 'sink'). Property 'connector.class' is required"); validate( """ diff --git a/langstream-k8s-runtime/langstream-k8s-runtime-core/src/test/java/ai/langstream/runtime/impl/k8s/agents/KubernetesGenAIToolKitFunctionAgentProviderTest.java b/langstream-k8s-runtime/langstream-k8s-runtime-core/src/test/java/ai/langstream/runtime/impl/k8s/agents/KubernetesGenAIToolKitFunctionAgentProviderTest.java index 6e3c8ada7..05daad3f2 100644 --- a/langstream-k8s-runtime/langstream-k8s-runtime-core/src/test/java/ai/langstream/runtime/impl/k8s/agents/KubernetesGenAIToolKitFunctionAgentProviderTest.java +++ b/langstream-k8s-runtime/langstream-k8s-runtime-core/src/test/java/ai/langstream/runtime/impl/k8s/agents/KubernetesGenAIToolKitFunctionAgentProviderTest.java @@ -40,7 +40,7 @@ public void testValidationDropFields() { configuration: a-field: "val" """, - "Found error on an agent configuration (agent: 'drop-my-field', type: 'drop-fields'). Property 'a-field' is unknown"); + "Found error on agent configuration (agent: 'drop-my-field', type: 'drop-fields'). Property 'a-field' is unknown"); validate( """ topics: [] @@ -49,7 +49,7 @@ public void testValidationDropFields() { type: "drop-fields" configuration: {} """, - "Found error on an agent configuration (agent: 'drop-my-field', type: 'drop-fields'). Property 'fields' is required"); + "Found error on agent configuration (agent: 'drop-my-field', type: 'drop-fields'). Property 'fields' is required"); validate( """ topics: [] @@ -59,7 +59,7 @@ public void testValidationDropFields() { configuration: fields: {} """, - "Found error on an agent configuration (agent: 'drop-my-field', type: 'drop-fields'). Property 'fields' has a wrong data type. Expected type: java.util.ArrayList"); + "Found error on agent configuration (agent: 'drop-my-field', type: 'drop-fields'). Property 'fields' has a wrong data type. Expected type: java.util.ArrayList"); validate( """ topics: [] @@ -417,6 +417,11 @@ public void testStepsDoc() { "required" : false, "type" : "string" }, + "arguments" : { + "description" : "Additional arguments to pass to the AI Service. (HuggingFace only)", + "required" : false, + "type" : "object" + }, "batch-size" : { "description" : "Batch size for submitting the embeddings requests.", "required" : false, @@ -452,6 +457,16 @@ public void testStepsDoc() { "type" : "string", "defaultValue" : "text-embedding-ada-002" }, + "modelUrl" : { + "description" : "URL of the model to use. (HuggingFace only). The default is computed from the model: \\"djl://ai.djl.huggingface.pytorch{model}\\"", + "required" : false, + "type" : "string" + }, + "options" : { + "description" : "Additional options to pass to the AI Service. (HuggingFace only)", + "required" : false, + "type" : "object" + }, "text" : { "description" : "Text to create embeddings from. You can use Mustache syntax to compose multiple fields into a single text. Example:\\ntext: \\"{{{ value.field1 }}} {{{ value.field2 }}}\\"", "required" : true, diff --git a/langstream-k8s-runtime/langstream-k8s-runtime-core/src/test/java/ai/langstream/runtime/impl/k8s/agents/PythonCodeAgentProviderTest.java b/langstream-k8s-runtime/langstream-k8s-runtime-core/src/test/java/ai/langstream/runtime/impl/k8s/agents/PythonCodeAgentProviderTest.java index 893d0afa8..4c48c8fd3 100644 --- a/langstream-k8s-runtime/langstream-k8s-runtime-core/src/test/java/ai/langstream/runtime/impl/k8s/agents/PythonCodeAgentProviderTest.java +++ b/langstream-k8s-runtime/langstream-k8s-runtime-core/src/test/java/ai/langstream/runtime/impl/k8s/agents/PythonCodeAgentProviderTest.java @@ -36,7 +36,7 @@ public void testValidation(String type) { a-field: "val" """ .formatted(type), - "Found error on an agent configuration (agent: 'python1', type: '%s'). Property 'className' is required" + "Found error on agent configuration (agent: 'python1', type: '%s'). Property 'className' is required" .formatted(type)); validate( """ @@ -47,7 +47,7 @@ public void testValidation(String type) { configuration: {} """ .formatted(type), - "Found error on an agent configuration (agent: 'python1', type: '%s'). Property 'className' is required" + "Found error on agent configuration (agent: 'python1', type: '%s'). Property 'className' is required" .formatted(type)); validate( """ diff --git a/langstream-k8s-runtime/langstream-k8s-runtime-core/src/test/java/ai/langstream/runtime/impl/k8s/agents/QueryVectorDBAgentProviderTest.java b/langstream-k8s-runtime/langstream-k8s-runtime-core/src/test/java/ai/langstream/runtime/impl/k8s/agents/QueryVectorDBAgentProviderTest.java index 6ff333789..9de14dc65 100644 --- a/langstream-k8s-runtime/langstream-k8s-runtime-core/src/test/java/ai/langstream/runtime/impl/k8s/agents/QueryVectorDBAgentProviderTest.java +++ b/langstream-k8s-runtime/langstream-k8s-runtime-core/src/test/java/ai/langstream/runtime/impl/k8s/agents/QueryVectorDBAgentProviderTest.java @@ -42,7 +42,7 @@ public void testValidationQueryDb() { datasource: "cassandra" unknown-field: "..." """, - "Found error on an agent configuration (agent: 'db', type: 'query-vector-db'). Property 'unknown-field' is unknown"); + "Found error on agent configuration (agent: 'db', type: 'query-vector-db'). Property 'unknown-field' is unknown"); validate( """ @@ -91,7 +91,7 @@ public void testWriteDb() { configuration: unknown-field: "..." """, - "Found error on an agent configuration (agent: 'db', type: 'vector-db-sink'). Property 'datasource' is required"); + "Found error on agent configuration (agent: 'db', type: 'vector-db-sink'). Property 'datasource' is required"); validate( """ diff --git a/langstream-k8s-runtime/langstream-k8s-runtime-core/src/test/java/ai/langstream/runtime/impl/k8s/agents/S3SourceAgentProviderTest.java b/langstream-k8s-runtime/langstream-k8s-runtime-core/src/test/java/ai/langstream/runtime/impl/k8s/agents/S3SourceAgentProviderTest.java index 05ae34c5e..3ac911cd9 100644 --- a/langstream-k8s-runtime/langstream-k8s-runtime-core/src/test/java/ai/langstream/runtime/impl/k8s/agents/S3SourceAgentProviderTest.java +++ b/langstream-k8s-runtime/langstream-k8s-runtime-core/src/test/java/ai/langstream/runtime/impl/k8s/agents/S3SourceAgentProviderTest.java @@ -39,7 +39,7 @@ public void testValidation() { configuration: a-field: "val" """, - "Found error on an agent configuration (agent: 's3-source', type: 's3-source'). Property 'a-field' is unknown"); + "Found error on agent configuration (agent: 's3-source', type: 's3-source'). Property 'a-field' is unknown"); validate( """ topics: [] @@ -84,7 +84,7 @@ public void testValidation() { configuration: bucketName: {object: true} """, - "Found error on an agent configuration (agent: 's3-source', type: 's3-source'). Property 'bucketName' has a wrong data type. Expected type: java.lang.String"); + "Found error on agent configuration (agent: 's3-source', type: 's3-source'). Property 'bucketName' has a wrong data type. Expected type: java.lang.String"); } private void validate(String pipeline, String expectErrMessage) throws Exception { diff --git a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/assets/DeployAssetsTest.java b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/assets/DeployAssetsTest.java index 46e8ea833..da085fa95 100644 --- a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/assets/DeployAssetsTest.java +++ b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/assets/DeployAssetsTest.java @@ -50,7 +50,9 @@ public void testDeployAsset() throws Exception { - type: "datasource" name: "the-resource" configuration: - foo: "${secrets.the-secret.password}" + service: jdbc + url: "${secrets.the-secret.password}" + driverClass: "org.postgresql.Driver" """, "module.yaml", """ @@ -94,7 +96,7 @@ public void testDeployAsset() throws Exception { (Map) deployedAsset.getConfig().get("datasource"); Map datasourceConfiguration = (Map) datasource.get("configuration"); - assertEquals("bar", datasourceConfiguration.get("foo")); + assertEquals("bar", datasourceConfiguration.get("url")); final ExecutionPlan plan = applicationRuntime.implementation(); applicationDeployer.cleanup(tenant, plan); diff --git a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/mockagents/MockAssetManagerCodeProvider.java b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/mockagents/MockAssetManagerCodeProvider.java index e94a8b0e9..278f92e2c 100644 --- a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/mockagents/MockAssetManagerCodeProvider.java +++ b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/mockagents/MockAssetManagerCodeProvider.java @@ -15,12 +15,13 @@ */ package ai.langstream.mockagents; +import static org.junit.jupiter.api.Assertions.assertEquals; + import ai.langstream.api.model.AssetDefinition; import ai.langstream.api.runner.assets.AssetManager; import ai.langstream.api.runner.assets.AssetManagerProvider; import ai.langstream.api.util.ConfigurationUtils; import java.util.Map; -import java.util.Objects; import java.util.concurrent.CopyOnWriteArrayList; import lombok.extern.slf4j.Slf4j; @@ -72,10 +73,7 @@ public synchronized void deployAsset() throws Exception { if (configuration == null) { throw new IllegalStateException("Datasource configuration is required"); } - String foo = (String) configuration.get("foo"); - if (!Objects.equals(foo, "bar")) { - throw new IllegalStateException("Datasource configuration is not configured well"); - } + assertEquals("bar", configuration.get("url")); DEPLOYED_ASSETS.add(assetDefinition); } diff --git a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/runtime/LoadAssertManagerCodeTest.java b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/runtime/LoadAssertManagerCodeTest.java index 2522521b1..884f39503 100644 --- a/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/runtime/LoadAssertManagerCodeTest.java +++ b/langstream-runtime/langstream-runtime-impl/src/test/java/ai/langstream/runtime/LoadAssertManagerCodeTest.java @@ -32,7 +32,7 @@ public void testLoadMockAsset() throws Exception { AssetManagerRegistry registry = new AssetManagerRegistry(); AssetDefinition assetDefinition = new AssetDefinition(); assetDefinition.setConfig( - Map.of("datasource", Map.of("configuration", Map.of("foo", "bar")))); + Map.of("datasource", Map.of("configuration", Map.of("url", "bar")))); assetDefinition.setAssetType("mock-database-resource"); AssetManager assetManager = registry.getAssetManager(assetDefinition.getAssetType()).agentCode(); diff --git a/langstream-webservice/src/main/java/ai/langstream/webservice/doc/DocumentationGenerator.java b/langstream-webservice/src/main/java/ai/langstream/webservice/doc/DocumentationGenerator.java index 5e2c2302c..3882b7371 100644 --- a/langstream-webservice/src/main/java/ai/langstream/webservice/doc/DocumentationGenerator.java +++ b/langstream-webservice/src/main/java/ai/langstream/webservice/doc/DocumentationGenerator.java @@ -17,8 +17,10 @@ import ai.langstream.api.doc.AgentConfigurationModel; import ai.langstream.api.doc.ApiConfigurationModel; +import ai.langstream.api.doc.ResourceConfigurationModel; import ai.langstream.api.runtime.AgentNodeProvider; import ai.langstream.api.runtime.PluginsRegistry; +import ai.langstream.api.runtime.ResourceNodeProvider; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -29,11 +31,18 @@ public class DocumentationGenerator { public static ApiConfigurationModel generateDocs(String version) { Map agents = new TreeMap<>(); - final List nodes = - new PluginsRegistry().lookupAvailableAgentImplementations(null); + Map resources = new TreeMap<>(); + final PluginsRegistry registry = new PluginsRegistry(); + final List nodes = registry.lookupAvailableAgentImplementations(); for (AgentNodeProvider node : nodes) { agents.putAll(node.generateSupportedTypesDocumentation()); } - return new ApiConfigurationModel(version, agents); + + final List resourceNodeProviders = + registry.lookupAvailableResourceImplementations(); + for (ResourceNodeProvider resourceNodeProvider : resourceNodeProviders) { + resources.putAll(resourceNodeProvider.generateSupportedTypesDocumentation()); + } + return new ApiConfigurationModel(version, agents, resources); } }