Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Generate documentation for resources (AI services, datasource, vector-database) #513

Merged
merged 16 commits into from
Oct 3, 2023
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
import com.datastax.oss.streaming.ai.embeddings.HuggingFaceEmbeddingService;
import com.datastax.oss.streaming.ai.embeddings.HuggingFaceRestEmbeddingService;
import com.datastax.oss.streaming.ai.model.config.ComputeProvider;
import com.datastax.oss.streaming.ai.model.config.TransformStepConfig;
import com.datastax.oss.streaming.ai.services.ServiceProvider;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
Expand All @@ -41,8 +40,7 @@
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
import lombok.SneakyThrows;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import lombok.extern.slf4j.Slf4j;

public class HuggingFaceProvider implements ServiceProviderProvider {

Expand All @@ -60,24 +58,14 @@ public ServiceProvider createImplementation(Map<String, Object> agentConfigurati
(Map<String, Object>) agentConfiguration.get("huggingface"));
}

@Slf4j
static class HuggingFaceServiceProvider implements ServiceProvider {
private static final Logger log =
LoggerFactory.getLogger(
com.datastax.oss.streaming.ai.services.HuggingFaceServiceProvider.class);
private final Map<String, Object> providerConfiguration;

public HuggingFaceServiceProvider(Map<String, Object> providerConfiguration) {
this.providerConfiguration = providerConfiguration;
}

public HuggingFaceServiceProvider(TransformStepConfig tranformConfiguration) {
this.providerConfiguration =
(Map)
(new ObjectMapper())
.convertValue(
tranformConfiguration.getHuggingface(), Map.class);
}

public CompletionsService getCompletionsService(
Map<String, Object> additionalConfiguration) {
String accessKey = (String) providerConfiguration.get("access-key");
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

import static com.datastax.oss.streaming.ai.util.TransformFunctionUtil.buildStep;

import ai.langstream.ai.agents.services.impl.HuggingFaceProvider;
import com.datastax.oss.streaming.ai.JsonNodeSchema;
import com.datastax.oss.streaming.ai.TransformContext;
import com.datastax.oss.streaming.ai.TransformStep;
Expand All @@ -25,7 +26,6 @@
import com.datastax.oss.streaming.ai.model.TransformSchemaType;
import com.datastax.oss.streaming.ai.model.config.StepConfig;
import com.datastax.oss.streaming.ai.model.config.TransformStepConfig;
import com.datastax.oss.streaming.ai.services.HuggingFaceServiceProvider;
import com.datastax.oss.streaming.ai.services.OpenAIServiceProvider;
import com.datastax.oss.streaming.ai.services.ServiceProvider;
import com.datastax.oss.streaming.ai.util.TransformFunctionUtil;
Expand Down Expand Up @@ -495,7 +495,9 @@ protected ServiceProvider buildServiceProvider(TransformStepConfig config) {
return new OpenAIServiceProvider(config);
}
if (config.getHuggingface() != null) {
return new HuggingFaceServiceProvider(config);
return new HuggingFaceProvider()
.createImplementation(
TransformFunctionUtil.convertToMap(config.getHuggingface()));
}
}
return new ServiceProvider.NoopServiceProvider();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,16 +59,16 @@ public static final class MilvusConfig {
@JsonProperty(value = "host")
private String host;

@JsonProperty(value = "port")
private int port = 19530;

// Zillis service

@JsonProperty(value = "url")
private String url;

@JsonProperty(value = "token")
private String token;

@JsonProperty(value = "port")
private int port = 19530;
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,15 +87,15 @@ public QueryStepDataSource createDataSourceImplementation(

PineconeConfig clientConfig = MAPPER.convertValue(dataSourceConfig, PineconeConfig.class);

return new PinecodeQueryStepDataSource(clientConfig);
return new PineconeQueryStepDataSource(clientConfig);
}

private static class PinecodeQueryStepDataSource implements QueryStepDataSource {
private static class PineconeQueryStepDataSource implements QueryStepDataSource {

private final PineconeConfig clientConfig;
private PineconeConnection connection;

public PinecodeQueryStepDataSource(PineconeConfig clientConfig) {
public PineconeQueryStepDataSource(PineconeConfig clientConfig) {
this.clientConfig = clientConfig;
}

Expand Down Expand Up @@ -258,7 +258,7 @@ public static Object valueToObject(Value value) {
case STRING_VALUE -> value.getStringValue();
case BOOL_VALUE -> value.getBoolValue();
case LIST_VALUE -> value.getListValue().getValuesList().stream()
.map(PinecodeQueryStepDataSource::valueToObject)
.map(PineconeQueryStepDataSource::valueToObject)
.toList();
case STRUCT_VALUE -> value.getStructValue().getFieldsMap().entrySet().stream()
.collect(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@

import java.util.Map;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;

Expand All @@ -26,20 +25,7 @@
@AllArgsConstructor
public class AgentConfigurationModel {

@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
public static class AgentConfigurationProperty {
private String description;
boolean required;
private String type;
private Map<String, AgentConfigurationProperty> properties;
private AgentConfigurationProperty items;
private Object defaultValue;
}

private String name;
private String description;
private Map<String, AgentConfigurationProperty> properties;
private Map<String, ConfigPropertyModel> properties;
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,4 +17,7 @@

import java.util.Map;

public record ApiConfigurationModel(String version, Map<String, AgentConfigurationModel> agents) {}
public record ApiConfigurationModel(
String version,
Map<String, AgentConfigurationModel> agents,
Map<String, ResourceConfigurationModel> resources) {}
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
/*
* Copyright DataStax, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ai.langstream.api.doc;

import java.util.Map;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;

@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
public class ConfigPropertyModel {
private String description;
boolean required;
private String type;
private Map<String, ConfigPropertyModel> properties;
private ConfigPropertyModel items;
private Object defaultValue;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/*
* Copyright DataStax, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ai.langstream.api.doc;

import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;

@Target(ElementType.TYPE)
@Retention(RetentionPolicy.RUNTIME)
public @interface ResourceConfig {
String name() default "";

String description() default "";
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
/*
* Copyright DataStax, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ai.langstream.api.doc;

import java.util.Map;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;

@Data
@NoArgsConstructor
@AllArgsConstructor
public class ResourceConfigurationModel {

private String type;
private String name;
private String description;
private Map<String, ConfigPropertyModel> properties;
}
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,7 @@ public AgentNodeProvider lookupAgentImplementation(
return agentRuntimeProviderProvider.get();
}

public List<AgentNodeProvider> lookupAvailableAgentImplementations(
ComputeClusterRuntime clusterRuntime) {
// TODO: cluster runtime '
public List<AgentNodeProvider> lookupAvailableAgentImplementations() {
ServiceLoader<AgentNodeProvider> loader = ServiceLoader.load(AgentNodeProvider.class);
return loader.stream().map(p -> p.get()).collect(Collectors.toList());
}
Expand Down Expand Up @@ -79,6 +77,11 @@ public AssetNodeProvider lookupAssetImplementation(
return assetRuntimeProviderProvider.get();
}

public List<ResourceNodeProvider> lookupAvailableResourceImplementations() {
ServiceLoader<ResourceNodeProvider> loader = ServiceLoader.load(ResourceNodeProvider.class);
return loader.stream().map(p -> p.get()).collect(Collectors.toList());
}

public ResourceNodeProvider lookupResourceImplementation(
String type, ComputeClusterRuntime clusterRuntime) {
log.info(
Expand Down
Loading
Loading