From 4846012120e215bad2a2d130e0e862e448d84f0e Mon Sep 17 00:00:00 2001 From: Willem Pienaar Date: Sun, 17 Nov 2019 00:20:25 +0800 Subject: [PATCH 1/7] Refactored GetFeatureSets and GetStores to ListFeatureSets, ListStores, and GetFeatureSet * Updated and implemented in Core * Updated and implemented in Python SDK * Updated serving --- core/README.md | 2 +- .../feast/core/dao/FeatureSetRepository.java | 4 +- .../java/feast/core/grpc/CoreServiceImpl.java | 54 +- .../java/feast/core/service/SpecService.java | 94 ++- .../java/feast/core/validators/Matchers.java | 14 +- .../core/service/JobStatusServiceTest.java | 108 ---- .../feast/core/service/SpecServiceTest.java | 100 ++-- docs/reference/proto.md | 16 +- protos/feast/core/CoreService.proto | 29 +- sdk/go/protos/feast/core/CoreService.pb.go | 543 +++++++++++------- sdk/go/protos/feast/core/FeatureSet.pb.go | 3 +- sdk/go/protos/feast/core/Store.pb.go | 4 +- sdk/python/feast/client.py | 61 +- sdk/python/feast/core/CoreService_pb2.py | 291 ++++++---- sdk/python/feast/core/CoreService_pb2.pyi | 68 ++- sdk/python/feast/core/CoreService_pb2_grpc.py | 53 +- sdk/python/tests/feast_core_server.py | 8 +- sdk/python/tests/feast_serving_server.py | 10 +- sdk/python/tests/test_client.py | 118 ++-- .../serving/service/CachedSpecService.java | 16 +- .../serving/service/CoreSpecService.java | 8 +- .../service/CachedSpecServiceTest.java | 16 +- 22 files changed, 951 insertions(+), 669 deletions(-) delete mode 100644 core/src/test/java/feast/core/service/JobStatusServiceTest.java diff --git a/core/README.md b/core/README.md index bfa588e1fb..8f5b6f03bf 100644 --- a/core/README.md +++ b/core/README.md @@ -28,5 +28,5 @@ If you have [grpc_cli](https://github.com/grpc/grpc/blob/master/doc/command_line ``` grpc_cli ls localhost:6565 grpc_cli call localhost:6565 GetFeastCoreVersion "" -grpc_cli call localhost:6565 GetStores "" +grpc_cli call localhost:6565 ListStores "" ``` \ No newline at end of file diff --git a/core/src/main/java/feast/core/dao/FeatureSetRepository.java b/core/src/main/java/feast/core/dao/FeatureSetRepository.java index 1d24e78f26..cf94bbb3c1 100644 --- a/core/src/main/java/feast/core/dao/FeatureSetRepository.java +++ b/core/src/main/java/feast/core/dao/FeatureSetRepository.java @@ -11,6 +11,6 @@ public interface FeatureSetRepository extends JpaRepository List findByName(String name); // find all versions of featureSets with names matching the regex - @Query(nativeQuery=true, value="SELECT * FROM feature_sets WHERE name ~ ?1") - List findByNameRegex(String regex); + @Query(nativeQuery=true, value="SELECT * FROM feature_sets WHERE name LIKE ?1") + List findByNameWithWildcard(String name); } diff --git a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java b/core/src/main/java/feast/core/grpc/CoreServiceImpl.java index 18482b5e6d..dbeb5c1381 100644 --- a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java +++ b/core/src/main/java/feast/core/grpc/CoreServiceImpl.java @@ -24,11 +24,13 @@ import feast.core.CoreServiceProto.ApplyFeatureSetResponse; import feast.core.CoreServiceProto.GetFeastCoreVersionRequest; import feast.core.CoreServiceProto.GetFeastCoreVersionResponse; -import feast.core.CoreServiceProto.GetFeatureSetsRequest; -import feast.core.CoreServiceProto.GetFeatureSetsResponse; -import feast.core.CoreServiceProto.GetStoresRequest; -import feast.core.CoreServiceProto.GetStoresRequest.Filter; -import feast.core.CoreServiceProto.GetStoresResponse; +import feast.core.CoreServiceProto.GetFeatureSetRequest; +import feast.core.CoreServiceProto.GetFeatureSetResponse; +import feast.core.CoreServiceProto.ListFeatureSetsRequest; +import feast.core.CoreServiceProto.ListFeatureSetsResponse; +import feast.core.CoreServiceProto.ListStoresRequest; +import feast.core.CoreServiceProto.ListStoresRequest.Filter; +import feast.core.CoreServiceProto.ListStoresResponse; import feast.core.CoreServiceProto.UpdateStoreRequest; import feast.core.CoreServiceProto.UpdateStoreResponse; import feast.core.CoreServiceProto.UpdateStoreResponse.Status; @@ -37,11 +39,9 @@ import feast.core.StoreProto.Store; import feast.core.StoreProto.Store.Subscription; import feast.core.exception.RetrievalException; -import feast.core.model.Source; import feast.core.service.JobCoordinatorService; import feast.core.service.SpecService; import io.grpc.stub.StreamObserver; -import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -73,28 +73,42 @@ public void getFeastCoreVersion( @Override @Transactional - public void getFeatureSets( - GetFeatureSetsRequest request, StreamObserver responseObserver) { + public void getFeatureSet( + GetFeatureSetRequest request, StreamObserver responseObserver) { try { - GetFeatureSetsResponse response = specService.getFeatureSets(request.getFilter()); + GetFeatureSetResponse response = specService.getFeatureSet(request); responseObserver.onNext(response); responseObserver.onCompleted(); } catch (RetrievalException | InvalidProtocolBufferException e) { - log.error("Exception has occurred in GetFeatureSets method: ", e); + log.error("Exception has occurred in GetFeatureSet method: ", e); responseObserver.onError(e); } } @Override @Transactional - public void getStores( - GetStoresRequest request, StreamObserver responseObserver) { + public void listFeatureSets( + ListFeatureSetsRequest request, StreamObserver responseObserver) { try { - GetStoresResponse response = specService.getStores(request.getFilter()); + ListFeatureSetsResponse response = specService.listFeatureSets(request.getFilter()); + responseObserver.onNext(response); + responseObserver.onCompleted(); + } catch (RetrievalException | InvalidProtocolBufferException e) { + log.error("Exception has occurred in ListFeatureSet method: ", e); + responseObserver.onError(e); + } + } + + @Override + @Transactional + public void listStores( + ListStoresRequest request, StreamObserver responseObserver) { + try { + ListStoresResponse response = specService.listStores(request.getFilter()); responseObserver.onNext(response); responseObserver.onCompleted(); } catch (RetrievalException e) { - log.error("Exception has occurred in GetStores method: ", e); + log.error("Exception has occurred in ListStores method: ", e); responseObserver.onError(e); } } @@ -106,7 +120,7 @@ public void applyFeatureSet( try { ApplyFeatureSetResponse response = specService.applyFeatureSet(request.getFeatureSet()); String featureSetName = response.getFeatureSet().getName(); - GetStoresResponse stores = specService.getStores(Filter.newBuilder().build()); + ListStoresResponse stores = specService.listStores(Filter.newBuilder().build()); for (Store store : stores.getStoreList()) { List relevantSubscriptions = store.getSubscriptionsList().stream() @@ -120,8 +134,8 @@ public void applyFeatureSet( for (Subscription subscription : relevantSubscriptions) { featureSetSpecs.addAll( specService - .getFeatureSets( - GetFeatureSetsRequest.Filter.newBuilder() + .listFeatureSets( + ListFeatureSetsRequest.Filter.newBuilder() .setFeatureSetName(subscription.getName()) .setFeatureSetVersion(subscription.getVersion()) .build()) @@ -157,8 +171,8 @@ public void updateStore(UpdateStoreRequest request, Store store = response.getStore(); for (Subscription subscription : store.getSubscriptionsList()) { featureSetSpecs.addAll( - specService.getFeatureSets( - GetFeatureSetsRequest.Filter.newBuilder() + specService.listFeatureSets( + ListFeatureSetsRequest.Filter.newBuilder() .setFeatureSetName(subscription.getName()) .setFeatureSetVersion(subscription.getVersion()) .build()) diff --git a/core/src/main/java/feast/core/service/SpecService.java b/core/src/main/java/feast/core/service/SpecService.java index b7862c2dd9..98c75a5587 100644 --- a/core/src/main/java/feast/core/service/SpecService.java +++ b/core/src/main/java/feast/core/service/SpecService.java @@ -17,15 +17,20 @@ package feast.core.service; +import static feast.core.validators.Matchers.checkValidCharacters; +import static feast.core.validators.Matchers.checkValidFeatureSetFilterName; + import com.google.common.collect.Ordering; import com.google.protobuf.InvalidProtocolBufferException; import feast.core.CoreServiceProto.ApplyFeatureSetResponse; import feast.core.CoreServiceProto.ApplyFeatureSetResponse.Status; -import feast.core.CoreServiceProto.GetFeatureSetsRequest; -import feast.core.CoreServiceProto.GetFeatureSetsResponse; -import feast.core.CoreServiceProto.GetStoresRequest; -import feast.core.CoreServiceProto.GetStoresResponse; -import feast.core.CoreServiceProto.GetStoresResponse.Builder; +import feast.core.CoreServiceProto.GetFeatureSetRequest; +import feast.core.CoreServiceProto.GetFeatureSetResponse; +import feast.core.CoreServiceProto.ListFeatureSetsRequest; +import feast.core.CoreServiceProto.ListFeatureSetsResponse; +import feast.core.CoreServiceProto.ListStoresRequest; +import feast.core.CoreServiceProto.ListStoresResponse; +import feast.core.CoreServiceProto.ListStoresResponse.Builder; import feast.core.CoreServiceProto.UpdateStoreRequest; import feast.core.CoreServiceProto.UpdateStoreResponse; import feast.core.FeatureSetProto.FeatureSetSpec; @@ -72,6 +77,59 @@ public SpecService( this.defaultSource = defaultSource; } + /** + * Get a feature set matching the feature name and version provided in the filter. The name + * is required. If the version is provided then it will be used for the lookup. If the version + * is omitted then the latest version will be returned. + * + * @param GetFeatureSetRequest containing the name and version of the feature set + * @return GetFeatureSetResponse containing a single feature set + */ + public GetFeatureSetResponse getFeatureSet(GetFeatureSetRequest request) + throws InvalidProtocolBufferException { + + // Validate input arguments + checkValidCharacters(request.getName(), "featureSetName"); + if (request.getName().isEmpty()) { + throw io.grpc.Status.INVALID_ARGUMENT + .withDescription("No feature set name provided") + .asRuntimeException(); + } + if (request.getVersion() < 0){ + throw new IllegalArgumentException("Version number cannot be less than 0"); + } + + // Find a list of feature sets with the requested name + List featureSets = featureSetRepository.findByNameWithWildcard(request.getName()); + + // Filter the list based on version + if (request.getVersion() == 0){ + // Version is not set, filter list to latest version + featureSets = Ordering.natural().reverse() + .sortedCopy(featureSets).subList(0, featureSets.size() == 0 ? 0 : 1); + } else if(request.getVersion() > 0) { + // Version is set, find specific version + featureSets = featureSets.stream() + .filter(fs -> request.getVersion() == fs.getVersion()).collect(Collectors.toList()); + } + + // Validate remaining items + if (featureSets.size() == 0){ + throw io.grpc.Status.NOT_FOUND + .withDescription("Feature set could not be found") + .asRuntimeException(); + } + if (featureSets.size() > 1){ + throw io.grpc.Status.INTERNAL + .withDescription(String.format("Multiple feature sets found with the name %s and " + + "version %s", request.getName(), request.getVersion())) + .asRuntimeException(); + } + + // Only a single item in list, return successfully + return GetFeatureSetResponse.newBuilder().setFeatureSet(featureSets.get(0).toProto()).build(); + } + /** * Get featureSets matching the feature name and version provided in the filter. If the feature * name is not provided, the method will return all featureSets currently registered to Feast. @@ -84,25 +142,21 @@ public SpecService( * comparator (<, <=, >, etc) and a version number, e.g. 10, <10, >=1 * * @param filter filter containing the desired featureSet name and version filter - * @return GetFeatureSetsResponse with list of featureSets found matching the filter + * @return ListFeatureSetsResponse with list of featureSets found matching the filter */ - public GetFeatureSetsResponse getFeatureSets(GetFeatureSetsRequest.Filter filter) + public ListFeatureSetsResponse listFeatureSets(ListFeatureSetsRequest.Filter filter) throws InvalidProtocolBufferException { String name = filter.getFeatureSetName(); + checkValidFeatureSetFilterName(name, "featureSetName"); List featureSets; if (name.equals("")) { featureSets = featureSetRepository.findAll(); } else { - featureSets = featureSetRepository.findByNameRegex(name); - if (filter.getFeatureSetVersion().equals("latest")) { - featureSets = Ordering.natural().reverse() - .sortedCopy(featureSets).subList(0, featureSets.size() == 0 ? 0 : 1); - } else { - featureSets = featureSets.stream().filter(getVersionFilter(filter.getFeatureSetVersion())) - .collect(Collectors.toList()); - } + featureSets = featureSetRepository.findByNameWithWildcard(name.replace('*', '%')); + featureSets = featureSets.stream().filter(getVersionFilter(filter.getFeatureSetVersion())) + .collect(Collectors.toList()); } - GetFeatureSetsResponse.Builder response = GetFeatureSetsResponse.newBuilder(); + ListFeatureSetsResponse.Builder response = ListFeatureSetsResponse.newBuilder(); for (FeatureSet featureSet : featureSets) { response.addFeatureSets(featureSet.toProto()); } @@ -114,13 +168,13 @@ public GetFeatureSetsResponse getFeatureSets(GetFeatureSetsRequest.Filter filter * the method will return all stores currently registered to Feast. * * @param filter filter containing the desired store name - * @return GetStoresResponse containing list of stores found matching the filter + * @return ListStoresResponse containing list of stores found matching the filter */ - public GetStoresResponse getStores(GetStoresRequest.Filter filter) { + public ListStoresResponse listStores(ListStoresRequest.Filter filter) { try { String name = filter.getName(); if (name.equals("")) { - Builder responseBuilder = GetStoresResponse.newBuilder(); + Builder responseBuilder = ListStoresResponse.newBuilder(); for (Store store : storeRepository.findAll()) { responseBuilder.addStore(store.toProto()); } @@ -129,7 +183,7 @@ public GetStoresResponse getStores(GetStoresRequest.Filter filter) { Store store = storeRepository.findById(name) .orElseThrow(() -> new RetrievalException(String.format("Store with name '%s' not found", name))); - return GetStoresResponse.newBuilder() + return ListStoresResponse.newBuilder() .addStore(store.toProto()) .build(); } catch (InvalidProtocolBufferException e) { diff --git a/core/src/main/java/feast/core/validators/Matchers.java b/core/src/main/java/feast/core/validators/Matchers.java index 78e6c72a03..f67a5bb380 100644 --- a/core/src/main/java/feast/core/validators/Matchers.java +++ b/core/src/main/java/feast/core/validators/Matchers.java @@ -17,8 +17,6 @@ package feast.core.validators; -import com.google.common.base.Strings; - import java.util.regex.Pattern; public class Matchers { @@ -26,6 +24,7 @@ public class Matchers { private static Pattern UPPER_SNAKE_CASE_REGEX = Pattern.compile("^[A-Z0-9]+(_[A-Z0-9]+)*$"); private static Pattern LOWER_SNAKE_CASE_REGEX = Pattern.compile("^[a-z0-9]+(_[a-z0-9]+)*$"); private static Pattern VALID_CHARACTERS_REGEX = Pattern.compile("^[a-zA-Z0-9\\-_]*$"); + private static Pattern VALID_CHARACTERS_FSET_FILTER_REGEX = Pattern.compile("^[a-zA-Z0-9\\-_*]*$"); private static String ERROR_MESSAGE_TEMPLATE = "invalid value for field %s: %s"; @@ -61,4 +60,15 @@ public static void checkValidCharacters(String input, String fieldName) "argument must only contain alphanumeric characters, dashes and underscores.")); } } + + public static void checkValidFeatureSetFilterName(String input, String fieldName) + throws IllegalArgumentException { + if (!VALID_CHARACTERS_FSET_FILTER_REGEX.matcher(input).matches()) { + throw new IllegalArgumentException( + String.format( + ERROR_MESSAGE_TEMPLATE, + fieldName, + "argument must only contain alphanumeric characters, dashes, underscores, or an asterisk.")); + } + } } diff --git a/core/src/test/java/feast/core/service/JobStatusServiceTest.java b/core/src/test/java/feast/core/service/JobStatusServiceTest.java deleted file mode 100644 index 1e9e9c6e61..0000000000 --- a/core/src/test/java/feast/core/service/JobStatusServiceTest.java +++ /dev/null @@ -1,108 +0,0 @@ -package feast.core.service; - -public class JobStatusServiceTest { - -// @Rule -// public final ExpectedException exception = ExpectedException.none(); -// @Mock -// private JobInfoRepository jobInfoRepository; -// @Mock -// private MetricsRepository metricsRepository; -// -// @Before -// public void setUp() { -// initMocks(this); -// } -// -// @Test -// public void shouldListAllJobDetails() { -// JobInfo jobInfo1 = -// new JobInfo( -// "job1", -// "", -// "", -// "", -// "", -// Collections.emptyList(), -// Collections.emptyList(), -// Collections.emptyList(), -// JobStatus.PENDING, -// ""); -// jobInfo1.setCreated(Date.from(Instant.ofEpochSecond(1))); -// jobInfo1.setLastUpdated(Date.from(Instant.ofEpochSecond(1))); -// JobInfo jobInfo2 = -// new JobInfo( -// "job2", -// "", -// "", -// "", -// "", -// Collections.emptyList(), -// Collections.emptyList(), -// Collections.emptyList(), -// JobStatus.PENDING, -// ""); -// jobInfo2.setCreated(Date.from(Instant.ofEpochSecond(1))); -// jobInfo2.setLastUpdated(Date.from(Instant.ofEpochSecond(1))); -// when(jobInfoRepository.findAll(any(Sort.class))) -// .thenReturn(Lists.newArrayList(jobInfo1, jobInfo2)); -// JobStatusService jobStatusService = -// new JobStatusService(jobInfoRepository, metricsRepository); -// List actual = jobStatusService.listJobs(); -// List expected = -// Lists.newArrayList( -// JobDetail.newBuilder() -// .setId("job1") -// .setStatus("PENDING") -// .setCreated(Timestamp.newBuilder().setSeconds(1).build()) -// .setLastUpdated(Timestamp.newBuilder().setSeconds(1).build()) -// .build(), -// JobDetail.newBuilder() -// .setId("job2") -// .setStatus("PENDING") -// .setCreated(Timestamp.newBuilder().setSeconds(1).build()) -// .setLastUpdated(Timestamp.newBuilder().setSeconds(1).build()) -// .build()); -// assertThat(actual, equalTo(expected)); -// } -// -// @Test -// public void shouldReturnDetailOfRequestedJobId() { -// JobInfo jobInfo1 = -// new JobInfo( -// "job1", -// "", -// "", -// "", -// "", -// Collections.emptyList(), -// Collections.emptyList(), -// Collections.emptyList(), -// JobStatus.PENDING, -// ""); -// jobInfo1.setCreated(Date.from(Instant.ofEpochSecond(1))); -// jobInfo1.setLastUpdated(Date.from(Instant.ofEpochSecond(1))); -// when(jobInfoRepository.findById("job1")).thenReturn(Optional.of(jobInfo1)); -// JobStatusService jobStatusService = -// new JobStatusService(jobInfoRepository, metricsRepository); -// JobDetail actual = jobStatusService.getJob("job1"); -// JobDetail expected = -// JobDetail.newBuilder() -// .setId("job1") -// .setStatus("PENDING") -// .setCreated(Timestamp.newBuilder().setSeconds(1).build()) -// .setLastUpdated(Timestamp.newBuilder().setSeconds(1).build()) -// .build(); -// assertThat(actual, equalTo(expected)); -// } -// -// @Test -// public void shouldThrowErrorIfJobIdNotFoundWhenGettingJob() { -// when(jobInfoRepository.findById("job1")).thenReturn(Optional.empty()); -// JobStatusService jobStatusService = -// new JobStatusService(jobInfoRepository, metricsRepository); -// exception.expect(RetrievalException.class); -// exception.expectMessage("Unable to retrieve job with id job1"); -// jobStatusService.getJob("job1"); -// } -} \ No newline at end of file diff --git a/core/src/test/java/feast/core/service/SpecServiceTest.java b/core/src/test/java/feast/core/service/SpecServiceTest.java index 63f15c8bd2..4a2c6e5774 100644 --- a/core/src/test/java/feast/core/service/SpecServiceTest.java +++ b/core/src/test/java/feast/core/service/SpecServiceTest.java @@ -28,10 +28,12 @@ import com.google.protobuf.InvalidProtocolBufferException; import feast.core.CoreServiceProto.ApplyFeatureSetResponse; import feast.core.CoreServiceProto.ApplyFeatureSetResponse.Status; -import feast.core.CoreServiceProto.GetFeatureSetsRequest.Filter; -import feast.core.CoreServiceProto.GetFeatureSetsResponse; -import feast.core.CoreServiceProto.GetStoresRequest; -import feast.core.CoreServiceProto.GetStoresResponse; +import feast.core.CoreServiceProto.GetFeatureSetRequest; +import feast.core.CoreServiceProto.GetFeatureSetResponse; +import feast.core.CoreServiceProto.ListFeatureSetsRequest.Filter; +import feast.core.CoreServiceProto.ListFeatureSetsResponse; +import feast.core.CoreServiceProto.ListStoresRequest; +import feast.core.CoreServiceProto.ListStoresResponse; import feast.core.CoreServiceProto.UpdateStoreRequest; import feast.core.CoreServiceProto.UpdateStoreResponse; import feast.core.FeatureSetProto.FeatureSetSpec; @@ -97,11 +99,11 @@ public void setUp() { .thenReturn(featureSets); when(featureSetRepository.findByName("f1")) .thenReturn(featureSets.subList(0, 3)); - when(featureSetRepository.findByNameRegex("f1")) + when(featureSetRepository.findByNameWithWildcard("f1")) .thenReturn(featureSets.subList(0, 3)); when(featureSetRepository.findByName("asd")) .thenReturn(Lists.newArrayList()); - when(featureSetRepository.findByNameRegex("asd")) + when(featureSetRepository.findByNameWithWildcard("asd")) .thenReturn(Lists.newArrayList()); Store store1 = newDummyStore("SERVING"); @@ -118,14 +120,14 @@ public void setUp() { @Test public void shouldGetAllFeatureSetsIfNoFilterProvided() throws InvalidProtocolBufferException { - GetFeatureSetsResponse actual = specService - .getFeatureSets(Filter.newBuilder().setFeatureSetName("").build()); + ListFeatureSetsResponse actual = specService + .listFeatureSets(Filter.newBuilder().setFeatureSetName("").build()); List list = new ArrayList<>(); for (FeatureSet featureSet : featureSets) { FeatureSetSpec toProto = featureSet.toProto(); list.add(toProto); } - GetFeatureSetsResponse expected = GetFeatureSetsResponse + ListFeatureSetsResponse expected = ListFeatureSetsResponse .newBuilder() .addAllFeatureSets( list) @@ -136,8 +138,8 @@ public void shouldGetAllFeatureSetsIfNoFilterProvided() throws InvalidProtocolBu @Test public void shouldGetAllFeatureSetsMatchingNameIfNoVersionProvided() throws InvalidProtocolBufferException { - GetFeatureSetsResponse actual = specService - .getFeatureSets(Filter.newBuilder().setFeatureSetName("f1").build()); + ListFeatureSetsResponse actual = specService + .listFeatureSets(Filter.newBuilder().setFeatureSetName("f1").build()); List expectedFeatureSets = featureSets.stream() .filter(fs -> fs.getName().equals("f1")) .collect(Collectors.toList()); @@ -146,7 +148,7 @@ public void shouldGetAllFeatureSetsMatchingNameIfNoVersionProvided() FeatureSetSpec toProto = expectedFeatureSet.toProto(); list.add(toProto); } - GetFeatureSetsResponse expected = GetFeatureSetsResponse + ListFeatureSetsResponse expected = ListFeatureSetsResponse .newBuilder() .addAllFeatureSets( list) @@ -157,8 +159,8 @@ public void shouldGetAllFeatureSetsMatchingNameIfNoVersionProvided() @Test public void shouldGetAllFeatureSetsMatchingVersionIfNoComparator() throws InvalidProtocolBufferException { - GetFeatureSetsResponse actual = specService - .getFeatureSets( + ListFeatureSetsResponse actual = specService + .listFeatureSets( Filter.newBuilder().setFeatureSetName("f1").setFeatureSetVersion("1").build()); List expectedFeatureSets = featureSets.stream() .filter(fs -> fs.getName().equals("f1")) @@ -169,7 +171,7 @@ public void shouldGetAllFeatureSetsMatchingVersionIfNoComparator() FeatureSetSpec toProto = expectedFeatureSet.toProto(); list.add(toProto); } - GetFeatureSetsResponse expected = GetFeatureSetsResponse + ListFeatureSetsResponse expected = ListFeatureSetsResponse .newBuilder() .addAllFeatureSets( list) @@ -180,8 +182,8 @@ public void shouldGetAllFeatureSetsMatchingVersionIfNoComparator() @Test public void shouldGetAllFeatureSetsGivenVersionWithComparator() throws InvalidProtocolBufferException { - GetFeatureSetsResponse actual = specService - .getFeatureSets( + ListFeatureSetsResponse actual = specService + .listFeatureSets( Filter.newBuilder().setFeatureSetName("f1").setFeatureSetVersion(">1").build()); List expectedFeatureSets = featureSets.stream() .filter(fs -> fs.getName().equals("f1")) @@ -192,7 +194,7 @@ public void shouldGetAllFeatureSetsGivenVersionWithComparator() FeatureSetSpec toProto = expectedFeatureSet.toProto(); list.add(toProto); } - GetFeatureSetsResponse expected = GetFeatureSetsResponse + ListFeatureSetsResponse expected = ListFeatureSetsResponse .newBuilder() .addAllFeatureSets( list) @@ -201,23 +203,37 @@ public void shouldGetAllFeatureSetsGivenVersionWithComparator() } @Test - public void shouldGetLatestFeatureSetGivenLatestVersionFilter() + public void shouldGetLatestFeatureSetGivenMissingVersionFilter() throws InvalidProtocolBufferException { - GetFeatureSetsResponse actual = specService - .getFeatureSets( - Filter.newBuilder().setFeatureSetName("f1").setFeatureSetVersion("latest").build()); - List expectedFeatureSets = featureSets.subList(2, 3); - List list = new ArrayList<>(); - for (FeatureSet expectedFeatureSet : expectedFeatureSets) { - FeatureSetSpec toProto = expectedFeatureSet.toProto(); - list.add(toProto); - } - GetFeatureSetsResponse expected = GetFeatureSetsResponse - .newBuilder() - .addAllFeatureSets( - list) - .build(); - assertThat(actual, equalTo(expected)); + GetFeatureSetResponse actual = specService + .getFeatureSet(GetFeatureSetRequest.newBuilder().setName("f1").build()); + FeatureSet expected = featureSets.get(2); + assertThat(actual.getFeatureSet(), equalTo(expected.toProto())); + } + + @Test + public void shouldGetSpecificFeatureSetGivenSpecificVersionFilter() + throws InvalidProtocolBufferException { + GetFeatureSetResponse actual = specService + .getFeatureSet(GetFeatureSetRequest.newBuilder().setName("f1").setVersion(2).build()); + FeatureSet expected = featureSets.get(1); + assertThat(actual.getFeatureSet(), equalTo(expected.toProto())); + } + + @Test + public void shouldThrowExceptionGivenMissingFeatureSetName() + throws InvalidProtocolBufferException { + expectedException.expect(StatusRuntimeException.class); + expectedException.expectMessage("INVALID_ARGUMENT: No feature set name provided"); + specService.getFeatureSet(GetFeatureSetRequest.newBuilder().setVersion(2).build()); + } + + @Test + public void shouldThrowExceptionGivenMissingFeatureSet() + throws InvalidProtocolBufferException { + expectedException.expect(StatusRuntimeException.class); + expectedException.expectMessage("NOT_FOUND: Feature set could not be found"); + specService.getFeatureSet(GetFeatureSetRequest.newBuilder().setName("f1000").setVersion(2).build()); } @Test @@ -225,15 +241,15 @@ public void shouldThrowRetrievalExceptionGivenInvalidFeatureSetVersionComparator throws InvalidProtocolBufferException { expectedException.expect(StatusRuntimeException.class); expectedException.expectMessage("Invalid comparator '=<' provided."); - specService.getFeatureSets( + specService.listFeatureSets( Filter.newBuilder().setFeatureSetName("f1").setFeatureSetVersion("=<1").build()); } @Test public void shouldReturnAllStoresIfNoNameProvided() throws InvalidProtocolBufferException { - GetStoresResponse actual = specService - .getStores(GetStoresRequest.Filter.newBuilder().build()); - GetStoresResponse.Builder expected = GetStoresResponse.newBuilder(); + ListStoresResponse actual = specService + .listStores(ListStoresRequest.Filter.newBuilder().build()); + ListStoresResponse.Builder expected = ListStoresResponse.newBuilder(); for (Store expectedStore : stores) { expected.addStore(expectedStore.toProto()); } @@ -242,11 +258,11 @@ public void shouldReturnAllStoresIfNoNameProvided() throws InvalidProtocolBuffer @Test public void shouldReturnStoreWithName() throws InvalidProtocolBufferException { - GetStoresResponse actual = specService - .getStores(GetStoresRequest.Filter.newBuilder().setName("SERVING").build()); + ListStoresResponse actual = specService + .listStores(ListStoresRequest.Filter.newBuilder().setName("SERVING").build()); List expectedStores = stores.stream().filter(s -> s.getName().equals("SERVING")) .collect(Collectors.toList()); - GetStoresResponse.Builder expected = GetStoresResponse.newBuilder(); + ListStoresResponse.Builder expected = ListStoresResponse.newBuilder(); for (Store expectedStore : expectedStores) { expected.addStore(expectedStore.toProto()); } @@ -258,7 +274,7 @@ public void shouldThrowRetrievalExceptionIfNoStoresFoundWithName() { expectedException.expect(RetrievalException.class); expectedException.expectMessage("Store with name 'NOTFOUND' not found"); specService - .getStores(GetStoresRequest.Filter.newBuilder().setName("NOTFOUND").build()); + .listStores(ListStoresRequest.Filter.newBuilder().setName("NOTFOUND").build()); } @Test diff --git a/docs/reference/proto.md b/docs/reference/proto.md index 9835861123..3cf8874a86 100644 --- a/docs/reference/proto.md +++ b/docs/reference/proto.md @@ -8,9 +8,9 @@ * [ApplyFeatureSetResponse](proto.md#applyfeaturesetresponse) * [GetFeastCoreVersionRequest](proto.md#getfeastcoreversionrequest) * [GetFeastCoreVersionResponse](proto.md#getfeastcoreversionresponse) - * [GetFeatureSetsRequest](proto.md#getfeaturesetsrequest) - * [GetFeatureSetsRequest.Filter](proto.md#filter) - * [GetFeatureSetsResponse](proto.md#getfeaturesetsresponse) + * [ListFeatureSetsRequest](proto.md#ListFeatureSetsRequest) + * [ListFeatureSetsRequest.Filter](proto.md#filter) + * [ListFeatureSetsResponse](proto.md#ListFeatureSetsResponse) * [GetStoresRequest](proto.md#getstoresrequest) * [GetStoresRequest.Filter](proto.md#filter) * [GetStoresResponse](proto.md#getstoresresponse) @@ -87,7 +87,7 @@ | Method Name | Request Type | Response Type | Description | | :--- | :--- | :--- | :--- | | GetFeastCoreVersion | [GetFeastCoreVersionRequest](proto.md#GetFeastCoreVersionRequest) | [GetFeastCoreVersionResponse](proto.md#GetFeastCoreVersionResponse) | Retrieve version information about this Feast deployment | -| GetFeatureSets | [GetFeatureSetsRequest](proto.md#GetFeatureSetsRequest) | [GetFeatureSetsResponse](proto.md#GetFeatureSetsResponse) | Retrieve feature set details given a filter. | +| GetFeatureSets | [ListFeatureSetsRequest](proto.md#ListFeatureSetsRequest) | [ListFeatureSetsResponse](proto.md#ListFeatureSetsResponse) | Retrieve feature set details given a filter. | Returns all feature sets matching that filter. If none are found, an empty list will be returned. If no filter is provided in the request, the response will contain all the feature sets currently stored in the registry. \| \| GetStores \| [GetStoresRequest](proto.md#GetStoresRequest) \| [GetStoresResponse](proto.md#GetStoresResponse) \| Retrieve store details given a filter. @@ -118,22 +118,22 @@ If the changes are valid, core will return the given store configuration in resp | :--- | :--- | :--- | :--- | | version | [string](proto.md#string) | | | -### GetFeatureSetsRequest +### ListFeatureSetsRequest Retrieves details for all versions of a specific feature set | Field | Type | Label | Description | | :--- | :--- | :--- | :--- | -| filter | [GetFeatureSetsRequest.Filter](proto.md#feast.core.GetFeatureSetsRequest.Filter) | | | +| filter | [ListFeatureSetsRequest.Filter](proto.md#feast.core.ListFeatureSetsRequest.Filter) | | | -### GetFeatureSetsRequest.Filter +### ListFeatureSetsRequest.Filter | Field | Type | Label | Description | | :--- | :--- | :--- | :--- | | feature\_set\_name | [string](proto.md#string) | | Name of the desired feature set. Valid regex strings are allowed. e.g. - . _can be used to match all feature sets - my-project-._ can be used to match all features prefixed by "my-project" | | feature\_set\_version | [string](proto.md#string) | | Version of the desired feature set. Either a number or valid expression can be provided. e.g. - 1 will match version 1 exactly - >=1 will match all versions greater or equal to 1 - <10 will match all versions less than 10 | -### GetFeatureSetsResponse +### ListFeatureSetsResponse | Field | Type | Label | Description | | :--- | :--- | :--- | :--- | diff --git a/protos/feast/core/CoreService.proto b/protos/feast/core/CoreService.proto index d9f9e2af59..2e0646e973 100644 --- a/protos/feast/core/CoreService.proto +++ b/protos/feast/core/CoreService.proto @@ -29,20 +29,23 @@ service CoreService { // Retrieve version information about this Feast deployment rpc GetFeastCoreVersion (GetFeastCoreVersionRequest) returns (GetFeastCoreVersionResponse); + // Returns a specific feature set + rpc GetFeatureSet (GetFeatureSetRequest) returns (GetFeatureSetResponse); + // Retrieve feature set details given a filter. // // Returns all feature sets matching that filter. If none are found, // an empty list will be returned. // If no filter is provided in the request, the response will contain all the feature // sets currently stored in the registry. - rpc GetFeatureSets (GetFeatureSetsRequest) returns (GetFeatureSetsResponse); + rpc ListFeatureSets (ListFeatureSetsRequest) returns (ListFeatureSetsResponse); // Retrieve store details given a filter. // // Returns all stores matching that filter. If none are found, an empty list will be returned. // If no filter is provided in the request, the response will contain all the stores currently // stored in the registry. - rpc GetStores (GetStoresRequest) returns (GetStoresResponse); + rpc ListStores (ListStoresRequest) returns (ListStoresResponse); // Create or update and existing feature set. // @@ -58,8 +61,22 @@ service CoreService { rpc UpdateStore(UpdateStoreRequest) returns (UpdateStoreResponse); } +// Request for a single feature set +message GetFeatureSetRequest { + // Name of feature set (required). + string name = 1; + + // Version of feature set (optional). If omitted then latest feature set will be returned. + int32 version = 2; +} + +// Response containing a single feature set +message GetFeatureSetResponse { + feast.core.FeatureSetSpec feature_set = 1; +} + // Retrieves details for all versions of a specific feature set -message GetFeatureSetsRequest { +message ListFeatureSetsRequest { message Filter { // Name of the desired feature set. Valid regex strings are allowed. // e.g. @@ -77,11 +94,11 @@ message GetFeatureSetsRequest { Filter filter = 1; } -message GetFeatureSetsResponse { +message ListFeatureSetsResponse { repeated feast.core.FeatureSetSpec feature_sets = 1; } -message GetStoresRequest { +message ListStoresRequest { message Filter { // Name of desired store. Regex is not supported in this query. string name = 1; @@ -90,7 +107,7 @@ message GetStoresRequest { Filter filter = 1; } -message GetStoresResponse { +message ListStoresResponse { repeated feast.core.Store store = 1; } diff --git a/sdk/go/protos/feast/core/CoreService.pb.go b/sdk/go/protos/feast/core/CoreService.pb.go index aa36367892..c7b3ec2bab 100644 --- a/sdk/go/protos/feast/core/CoreService.pb.go +++ b/sdk/go/protos/feast/core/CoreService.pb.go @@ -52,7 +52,7 @@ func (x ApplyFeatureSetResponse_Status) String() string { } func (ApplyFeatureSetResponse_Status) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_d9be266444105411, []int{5, 0} + return fileDescriptor_d9be266444105411, []int{7, 0} } type UpdateStoreResponse_Status int32 @@ -79,57 +79,150 @@ func (x UpdateStoreResponse_Status) String() string { } func (UpdateStoreResponse_Status) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_d9be266444105411, []int{9, 0} + return fileDescriptor_d9be266444105411, []int{11, 0} } -// Retrieves details for all versions of a specific feature set -type GetFeatureSetsRequest struct { - Filter *GetFeatureSetsRequest_Filter `protobuf:"bytes,1,opt,name=filter,proto3" json:"filter,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` +// Retrieves specific feature set +type GetFeatureSetRequest struct { + // Name of the desired feature set. + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // Version of the desired feature set (optional). If this is omitted then the latest version + // will be returned. + Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (m *GetFeatureSetsRequest) Reset() { *m = GetFeatureSetsRequest{} } -func (m *GetFeatureSetsRequest) String() string { return proto.CompactTextString(m) } -func (*GetFeatureSetsRequest) ProtoMessage() {} -func (*GetFeatureSetsRequest) Descriptor() ([]byte, []int) { +func (m *GetFeatureSetRequest) Reset() { *m = GetFeatureSetRequest{} } +func (m *GetFeatureSetRequest) String() string { return proto.CompactTextString(m) } +func (*GetFeatureSetRequest) ProtoMessage() {} +func (*GetFeatureSetRequest) Descriptor() ([]byte, []int) { return fileDescriptor_d9be266444105411, []int{0} } -func (m *GetFeatureSetsRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GetFeatureSetsRequest.Unmarshal(m, b) +func (m *GetFeatureSetRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetFeatureSetRequest.Unmarshal(m, b) +} +func (m *GetFeatureSetRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetFeatureSetRequest.Marshal(b, m, deterministic) +} +func (m *GetFeatureSetRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetFeatureSetRequest.Merge(m, src) +} +func (m *GetFeatureSetRequest) XXX_Size() int { + return xxx_messageInfo_GetFeatureSetRequest.Size(m) +} +func (m *GetFeatureSetRequest) XXX_DiscardUnknown() { + xxx_messageInfo_GetFeatureSetRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_GetFeatureSetRequest proto.InternalMessageInfo + +func (m *GetFeatureSetRequest) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *GetFeatureSetRequest) GetVersion() string { + if m != nil { + return m.Version + } + return "" +} + +// Response message containing a single feature set +type GetFeatureSetResponse struct { + // Feature set corresponding to request + FeatureSet *FeatureSetSpec `protobuf:"bytes,1,opt,name=feature_set,json=featureSet,proto3" json:"feature_set,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetFeatureSetResponse) Reset() { *m = GetFeatureSetResponse{} } +func (m *GetFeatureSetResponse) String() string { return proto.CompactTextString(m) } +func (*GetFeatureSetResponse) ProtoMessage() {} +func (*GetFeatureSetResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_d9be266444105411, []int{1} +} + +func (m *GetFeatureSetResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetFeatureSetResponse.Unmarshal(m, b) +} +func (m *GetFeatureSetResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetFeatureSetResponse.Marshal(b, m, deterministic) +} +func (m *GetFeatureSetResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetFeatureSetResponse.Merge(m, src) +} +func (m *GetFeatureSetResponse) XXX_Size() int { + return xxx_messageInfo_GetFeatureSetResponse.Size(m) +} +func (m *GetFeatureSetResponse) XXX_DiscardUnknown() { + xxx_messageInfo_GetFeatureSetResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_GetFeatureSetResponse proto.InternalMessageInfo + +func (m *GetFeatureSetResponse) GetFeatureSet() *FeatureSetSpec { + if m != nil { + return m.FeatureSet + } + return nil +} + +// Retrieves a list of feature sets +type ListFeatureSetsRequest struct { + Filter *ListFeatureSetsRequest_Filter `protobuf:"bytes,1,opt,name=filter,proto3" json:"filter,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ListFeatureSetsRequest) Reset() { *m = ListFeatureSetsRequest{} } +func (m *ListFeatureSetsRequest) String() string { return proto.CompactTextString(m) } +func (*ListFeatureSetsRequest) ProtoMessage() {} +func (*ListFeatureSetsRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_d9be266444105411, []int{2} +} + +func (m *ListFeatureSetsRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ListFeatureSetsRequest.Unmarshal(m, b) } -func (m *GetFeatureSetsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GetFeatureSetsRequest.Marshal(b, m, deterministic) +func (m *ListFeatureSetsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ListFeatureSetsRequest.Marshal(b, m, deterministic) } -func (m *GetFeatureSetsRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetFeatureSetsRequest.Merge(m, src) +func (m *ListFeatureSetsRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ListFeatureSetsRequest.Merge(m, src) } -func (m *GetFeatureSetsRequest) XXX_Size() int { - return xxx_messageInfo_GetFeatureSetsRequest.Size(m) +func (m *ListFeatureSetsRequest) XXX_Size() int { + return xxx_messageInfo_ListFeatureSetsRequest.Size(m) } -func (m *GetFeatureSetsRequest) XXX_DiscardUnknown() { - xxx_messageInfo_GetFeatureSetsRequest.DiscardUnknown(m) +func (m *ListFeatureSetsRequest) XXX_DiscardUnknown() { + xxx_messageInfo_ListFeatureSetsRequest.DiscardUnknown(m) } -var xxx_messageInfo_GetFeatureSetsRequest proto.InternalMessageInfo +var xxx_messageInfo_ListFeatureSetsRequest proto.InternalMessageInfo -func (m *GetFeatureSetsRequest) GetFilter() *GetFeatureSetsRequest_Filter { +func (m *ListFeatureSetsRequest) GetFilter() *ListFeatureSetsRequest_Filter { if m != nil { return m.Filter } return nil } -type GetFeatureSetsRequest_Filter struct { +type ListFeatureSetsRequest_Filter struct { // Name of the desired feature set. Valid regex strings are allowed. // e.g. - // - .* can be used to match all feature sets - // - my-project-.* can be used to match all features prefixed by "my-project" + // - * can be used to match all feature sets + // - my-project* can be used to match all feature sets prefixed by "my-project" FeatureSetName string `protobuf:"bytes,1,opt,name=feature_set_name,json=featureSetName,proto3" json:"feature_set_name,omitempty"` // Version of the desired feature set. Either a number or valid expression can be provided. // e.g. + // - Not specifying the version will match all versions // - 1 will match version 1 exactly // - >=1 will match all versions greater or equal to 1 // - <10 will match all versions less than 10 @@ -139,124 +232,124 @@ type GetFeatureSetsRequest_Filter struct { XXX_sizecache int32 `json:"-"` } -func (m *GetFeatureSetsRequest_Filter) Reset() { *m = GetFeatureSetsRequest_Filter{} } -func (m *GetFeatureSetsRequest_Filter) String() string { return proto.CompactTextString(m) } -func (*GetFeatureSetsRequest_Filter) ProtoMessage() {} -func (*GetFeatureSetsRequest_Filter) Descriptor() ([]byte, []int) { - return fileDescriptor_d9be266444105411, []int{0, 0} +func (m *ListFeatureSetsRequest_Filter) Reset() { *m = ListFeatureSetsRequest_Filter{} } +func (m *ListFeatureSetsRequest_Filter) String() string { return proto.CompactTextString(m) } +func (*ListFeatureSetsRequest_Filter) ProtoMessage() {} +func (*ListFeatureSetsRequest_Filter) Descriptor() ([]byte, []int) { + return fileDescriptor_d9be266444105411, []int{2, 0} } -func (m *GetFeatureSetsRequest_Filter) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GetFeatureSetsRequest_Filter.Unmarshal(m, b) +func (m *ListFeatureSetsRequest_Filter) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ListFeatureSetsRequest_Filter.Unmarshal(m, b) } -func (m *GetFeatureSetsRequest_Filter) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GetFeatureSetsRequest_Filter.Marshal(b, m, deterministic) +func (m *ListFeatureSetsRequest_Filter) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ListFeatureSetsRequest_Filter.Marshal(b, m, deterministic) } -func (m *GetFeatureSetsRequest_Filter) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetFeatureSetsRequest_Filter.Merge(m, src) +func (m *ListFeatureSetsRequest_Filter) XXX_Merge(src proto.Message) { + xxx_messageInfo_ListFeatureSetsRequest_Filter.Merge(m, src) } -func (m *GetFeatureSetsRequest_Filter) XXX_Size() int { - return xxx_messageInfo_GetFeatureSetsRequest_Filter.Size(m) +func (m *ListFeatureSetsRequest_Filter) XXX_Size() int { + return xxx_messageInfo_ListFeatureSetsRequest_Filter.Size(m) } -func (m *GetFeatureSetsRequest_Filter) XXX_DiscardUnknown() { - xxx_messageInfo_GetFeatureSetsRequest_Filter.DiscardUnknown(m) +func (m *ListFeatureSetsRequest_Filter) XXX_DiscardUnknown() { + xxx_messageInfo_ListFeatureSetsRequest_Filter.DiscardUnknown(m) } -var xxx_messageInfo_GetFeatureSetsRequest_Filter proto.InternalMessageInfo +var xxx_messageInfo_ListFeatureSetsRequest_Filter proto.InternalMessageInfo -func (m *GetFeatureSetsRequest_Filter) GetFeatureSetName() string { +func (m *ListFeatureSetsRequest_Filter) GetFeatureSetName() string { if m != nil { return m.FeatureSetName } return "" } -func (m *GetFeatureSetsRequest_Filter) GetFeatureSetVersion() string { +func (m *ListFeatureSetsRequest_Filter) GetFeatureSetVersion() string { if m != nil { return m.FeatureSetVersion } return "" } -type GetFeatureSetsResponse struct { +type ListFeatureSetsResponse struct { FeatureSets []*FeatureSetSpec `protobuf:"bytes,1,rep,name=feature_sets,json=featureSets,proto3" json:"feature_sets,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } -func (m *GetFeatureSetsResponse) Reset() { *m = GetFeatureSetsResponse{} } -func (m *GetFeatureSetsResponse) String() string { return proto.CompactTextString(m) } -func (*GetFeatureSetsResponse) ProtoMessage() {} -func (*GetFeatureSetsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_d9be266444105411, []int{1} +func (m *ListFeatureSetsResponse) Reset() { *m = ListFeatureSetsResponse{} } +func (m *ListFeatureSetsResponse) String() string { return proto.CompactTextString(m) } +func (*ListFeatureSetsResponse) ProtoMessage() {} +func (*ListFeatureSetsResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_d9be266444105411, []int{3} } -func (m *GetFeatureSetsResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GetFeatureSetsResponse.Unmarshal(m, b) +func (m *ListFeatureSetsResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ListFeatureSetsResponse.Unmarshal(m, b) } -func (m *GetFeatureSetsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GetFeatureSetsResponse.Marshal(b, m, deterministic) +func (m *ListFeatureSetsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ListFeatureSetsResponse.Marshal(b, m, deterministic) } -func (m *GetFeatureSetsResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetFeatureSetsResponse.Merge(m, src) +func (m *ListFeatureSetsResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_ListFeatureSetsResponse.Merge(m, src) } -func (m *GetFeatureSetsResponse) XXX_Size() int { - return xxx_messageInfo_GetFeatureSetsResponse.Size(m) +func (m *ListFeatureSetsResponse) XXX_Size() int { + return xxx_messageInfo_ListFeatureSetsResponse.Size(m) } -func (m *GetFeatureSetsResponse) XXX_DiscardUnknown() { - xxx_messageInfo_GetFeatureSetsResponse.DiscardUnknown(m) +func (m *ListFeatureSetsResponse) XXX_DiscardUnknown() { + xxx_messageInfo_ListFeatureSetsResponse.DiscardUnknown(m) } -var xxx_messageInfo_GetFeatureSetsResponse proto.InternalMessageInfo +var xxx_messageInfo_ListFeatureSetsResponse proto.InternalMessageInfo -func (m *GetFeatureSetsResponse) GetFeatureSets() []*FeatureSetSpec { +func (m *ListFeatureSetsResponse) GetFeatureSets() []*FeatureSetSpec { if m != nil { return m.FeatureSets } return nil } -type GetStoresRequest struct { - Filter *GetStoresRequest_Filter `protobuf:"bytes,1,opt,name=filter,proto3" json:"filter,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` +type ListStoresRequest struct { + Filter *ListStoresRequest_Filter `protobuf:"bytes,1,opt,name=filter,proto3" json:"filter,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (m *GetStoresRequest) Reset() { *m = GetStoresRequest{} } -func (m *GetStoresRequest) String() string { return proto.CompactTextString(m) } -func (*GetStoresRequest) ProtoMessage() {} -func (*GetStoresRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_d9be266444105411, []int{2} +func (m *ListStoresRequest) Reset() { *m = ListStoresRequest{} } +func (m *ListStoresRequest) String() string { return proto.CompactTextString(m) } +func (*ListStoresRequest) ProtoMessage() {} +func (*ListStoresRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_d9be266444105411, []int{4} } -func (m *GetStoresRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GetStoresRequest.Unmarshal(m, b) +func (m *ListStoresRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ListStoresRequest.Unmarshal(m, b) } -func (m *GetStoresRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GetStoresRequest.Marshal(b, m, deterministic) +func (m *ListStoresRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ListStoresRequest.Marshal(b, m, deterministic) } -func (m *GetStoresRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetStoresRequest.Merge(m, src) +func (m *ListStoresRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ListStoresRequest.Merge(m, src) } -func (m *GetStoresRequest) XXX_Size() int { - return xxx_messageInfo_GetStoresRequest.Size(m) +func (m *ListStoresRequest) XXX_Size() int { + return xxx_messageInfo_ListStoresRequest.Size(m) } -func (m *GetStoresRequest) XXX_DiscardUnknown() { - xxx_messageInfo_GetStoresRequest.DiscardUnknown(m) +func (m *ListStoresRequest) XXX_DiscardUnknown() { + xxx_messageInfo_ListStoresRequest.DiscardUnknown(m) } -var xxx_messageInfo_GetStoresRequest proto.InternalMessageInfo +var xxx_messageInfo_ListStoresRequest proto.InternalMessageInfo -func (m *GetStoresRequest) GetFilter() *GetStoresRequest_Filter { +func (m *ListStoresRequest) GetFilter() *ListStoresRequest_Filter { if m != nil { return m.Filter } return nil } -type GetStoresRequest_Filter struct { +type ListStoresRequest_Filter struct { // Name of desired store. Regex is not supported in this query. Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -264,71 +357,71 @@ type GetStoresRequest_Filter struct { XXX_sizecache int32 `json:"-"` } -func (m *GetStoresRequest_Filter) Reset() { *m = GetStoresRequest_Filter{} } -func (m *GetStoresRequest_Filter) String() string { return proto.CompactTextString(m) } -func (*GetStoresRequest_Filter) ProtoMessage() {} -func (*GetStoresRequest_Filter) Descriptor() ([]byte, []int) { - return fileDescriptor_d9be266444105411, []int{2, 0} +func (m *ListStoresRequest_Filter) Reset() { *m = ListStoresRequest_Filter{} } +func (m *ListStoresRequest_Filter) String() string { return proto.CompactTextString(m) } +func (*ListStoresRequest_Filter) ProtoMessage() {} +func (*ListStoresRequest_Filter) Descriptor() ([]byte, []int) { + return fileDescriptor_d9be266444105411, []int{4, 0} } -func (m *GetStoresRequest_Filter) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GetStoresRequest_Filter.Unmarshal(m, b) +func (m *ListStoresRequest_Filter) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ListStoresRequest_Filter.Unmarshal(m, b) } -func (m *GetStoresRequest_Filter) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GetStoresRequest_Filter.Marshal(b, m, deterministic) +func (m *ListStoresRequest_Filter) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ListStoresRequest_Filter.Marshal(b, m, deterministic) } -func (m *GetStoresRequest_Filter) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetStoresRequest_Filter.Merge(m, src) +func (m *ListStoresRequest_Filter) XXX_Merge(src proto.Message) { + xxx_messageInfo_ListStoresRequest_Filter.Merge(m, src) } -func (m *GetStoresRequest_Filter) XXX_Size() int { - return xxx_messageInfo_GetStoresRequest_Filter.Size(m) +func (m *ListStoresRequest_Filter) XXX_Size() int { + return xxx_messageInfo_ListStoresRequest_Filter.Size(m) } -func (m *GetStoresRequest_Filter) XXX_DiscardUnknown() { - xxx_messageInfo_GetStoresRequest_Filter.DiscardUnknown(m) +func (m *ListStoresRequest_Filter) XXX_DiscardUnknown() { + xxx_messageInfo_ListStoresRequest_Filter.DiscardUnknown(m) } -var xxx_messageInfo_GetStoresRequest_Filter proto.InternalMessageInfo +var xxx_messageInfo_ListStoresRequest_Filter proto.InternalMessageInfo -func (m *GetStoresRequest_Filter) GetName() string { +func (m *ListStoresRequest_Filter) GetName() string { if m != nil { return m.Name } return "" } -type GetStoresResponse struct { +type ListStoresResponse struct { Store []*Store `protobuf:"bytes,1,rep,name=store,proto3" json:"store,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } -func (m *GetStoresResponse) Reset() { *m = GetStoresResponse{} } -func (m *GetStoresResponse) String() string { return proto.CompactTextString(m) } -func (*GetStoresResponse) ProtoMessage() {} -func (*GetStoresResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_d9be266444105411, []int{3} +func (m *ListStoresResponse) Reset() { *m = ListStoresResponse{} } +func (m *ListStoresResponse) String() string { return proto.CompactTextString(m) } +func (*ListStoresResponse) ProtoMessage() {} +func (*ListStoresResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_d9be266444105411, []int{5} } -func (m *GetStoresResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GetStoresResponse.Unmarshal(m, b) +func (m *ListStoresResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ListStoresResponse.Unmarshal(m, b) } -func (m *GetStoresResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GetStoresResponse.Marshal(b, m, deterministic) +func (m *ListStoresResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ListStoresResponse.Marshal(b, m, deterministic) } -func (m *GetStoresResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetStoresResponse.Merge(m, src) +func (m *ListStoresResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_ListStoresResponse.Merge(m, src) } -func (m *GetStoresResponse) XXX_Size() int { - return xxx_messageInfo_GetStoresResponse.Size(m) +func (m *ListStoresResponse) XXX_Size() int { + return xxx_messageInfo_ListStoresResponse.Size(m) } -func (m *GetStoresResponse) XXX_DiscardUnknown() { - xxx_messageInfo_GetStoresResponse.DiscardUnknown(m) +func (m *ListStoresResponse) XXX_DiscardUnknown() { + xxx_messageInfo_ListStoresResponse.DiscardUnknown(m) } -var xxx_messageInfo_GetStoresResponse proto.InternalMessageInfo +var xxx_messageInfo_ListStoresResponse proto.InternalMessageInfo -func (m *GetStoresResponse) GetStore() []*Store { +func (m *ListStoresResponse) GetStore() []*Store { if m != nil { return m.Store } @@ -347,7 +440,7 @@ func (m *ApplyFeatureSetRequest) Reset() { *m = ApplyFeatureSetRequest{} func (m *ApplyFeatureSetRequest) String() string { return proto.CompactTextString(m) } func (*ApplyFeatureSetRequest) ProtoMessage() {} func (*ApplyFeatureSetRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_d9be266444105411, []int{4} + return fileDescriptor_d9be266444105411, []int{6} } func (m *ApplyFeatureSetRequest) XXX_Unmarshal(b []byte) error { @@ -388,7 +481,7 @@ func (m *ApplyFeatureSetResponse) Reset() { *m = ApplyFeatureSetResponse func (m *ApplyFeatureSetResponse) String() string { return proto.CompactTextString(m) } func (*ApplyFeatureSetResponse) ProtoMessage() {} func (*ApplyFeatureSetResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_d9be266444105411, []int{5} + return fileDescriptor_d9be266444105411, []int{7} } func (m *ApplyFeatureSetResponse) XXX_Unmarshal(b []byte) error { @@ -433,7 +526,7 @@ func (m *GetFeastCoreVersionRequest) Reset() { *m = GetFeastCoreVersionR func (m *GetFeastCoreVersionRequest) String() string { return proto.CompactTextString(m) } func (*GetFeastCoreVersionRequest) ProtoMessage() {} func (*GetFeastCoreVersionRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_d9be266444105411, []int{6} + return fileDescriptor_d9be266444105411, []int{8} } func (m *GetFeastCoreVersionRequest) XXX_Unmarshal(b []byte) error { @@ -465,7 +558,7 @@ func (m *GetFeastCoreVersionResponse) Reset() { *m = GetFeastCoreVersion func (m *GetFeastCoreVersionResponse) String() string { return proto.CompactTextString(m) } func (*GetFeastCoreVersionResponse) ProtoMessage() {} func (*GetFeastCoreVersionResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_d9be266444105411, []int{7} + return fileDescriptor_d9be266444105411, []int{9} } func (m *GetFeastCoreVersionResponse) XXX_Unmarshal(b []byte) error { @@ -504,7 +597,7 @@ func (m *UpdateStoreRequest) Reset() { *m = UpdateStoreRequest{} } func (m *UpdateStoreRequest) String() string { return proto.CompactTextString(m) } func (*UpdateStoreRequest) ProtoMessage() {} func (*UpdateStoreRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_d9be266444105411, []int{8} + return fileDescriptor_d9be266444105411, []int{10} } func (m *UpdateStoreRequest) XXX_Unmarshal(b []byte) error { @@ -544,7 +637,7 @@ func (m *UpdateStoreResponse) Reset() { *m = UpdateStoreResponse{} } func (m *UpdateStoreResponse) String() string { return proto.CompactTextString(m) } func (*UpdateStoreResponse) ProtoMessage() {} func (*UpdateStoreResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_d9be266444105411, []int{9} + return fileDescriptor_d9be266444105411, []int{11} } func (m *UpdateStoreResponse) XXX_Unmarshal(b []byte) error { @@ -582,12 +675,14 @@ func (m *UpdateStoreResponse) GetStatus() UpdateStoreResponse_Status { func init() { proto.RegisterEnum("feast.core.ApplyFeatureSetResponse_Status", ApplyFeatureSetResponse_Status_name, ApplyFeatureSetResponse_Status_value) proto.RegisterEnum("feast.core.UpdateStoreResponse_Status", UpdateStoreResponse_Status_name, UpdateStoreResponse_Status_value) - proto.RegisterType((*GetFeatureSetsRequest)(nil), "feast.core.GetFeatureSetsRequest") - proto.RegisterType((*GetFeatureSetsRequest_Filter)(nil), "feast.core.GetFeatureSetsRequest.Filter") - proto.RegisterType((*GetFeatureSetsResponse)(nil), "feast.core.GetFeatureSetsResponse") - proto.RegisterType((*GetStoresRequest)(nil), "feast.core.GetStoresRequest") - proto.RegisterType((*GetStoresRequest_Filter)(nil), "feast.core.GetStoresRequest.Filter") - proto.RegisterType((*GetStoresResponse)(nil), "feast.core.GetStoresResponse") + proto.RegisterType((*GetFeatureSetRequest)(nil), "feast.core.GetFeatureSetRequest") + proto.RegisterType((*GetFeatureSetResponse)(nil), "feast.core.GetFeatureSetResponse") + proto.RegisterType((*ListFeatureSetsRequest)(nil), "feast.core.ListFeatureSetsRequest") + proto.RegisterType((*ListFeatureSetsRequest_Filter)(nil), "feast.core.ListFeatureSetsRequest.Filter") + proto.RegisterType((*ListFeatureSetsResponse)(nil), "feast.core.ListFeatureSetsResponse") + proto.RegisterType((*ListStoresRequest)(nil), "feast.core.ListStoresRequest") + proto.RegisterType((*ListStoresRequest_Filter)(nil), "feast.core.ListStoresRequest.Filter") + proto.RegisterType((*ListStoresResponse)(nil), "feast.core.ListStoresResponse") proto.RegisterType((*ApplyFeatureSetRequest)(nil), "feast.core.ApplyFeatureSetRequest") proto.RegisterType((*ApplyFeatureSetResponse)(nil), "feast.core.ApplyFeatureSetResponse") proto.RegisterType((*GetFeastCoreVersionRequest)(nil), "feast.core.GetFeastCoreVersionRequest") @@ -599,45 +694,47 @@ func init() { func init() { proto.RegisterFile("feast/core/CoreService.proto", fileDescriptor_d9be266444105411) } var fileDescriptor_d9be266444105411 = []byte{ - // 598 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x55, 0x41, 0x6f, 0x12, 0x41, - 0x14, 0x76, 0x5b, 0x4b, 0xc3, 0x5b, 0xad, 0xcb, 0x10, 0x91, 0x6c, 0x51, 0xeb, 0xd4, 0xb4, 0xc4, - 0xc3, 0x6e, 0x82, 0x07, 0x0f, 0x5a, 0x23, 0x50, 0x4a, 0x4f, 0xd0, 0x2c, 0x62, 0x93, 0x5e, 0xc8, - 0x42, 0x07, 0xc4, 0x16, 0x66, 0xdd, 0x19, 0x9a, 0xf8, 0x7f, 0xfc, 0x1b, 0xde, 0xbc, 0xf9, 0xa7, - 0x0c, 0x33, 0x53, 0x76, 0x58, 0x96, 0xc5, 0xc4, 0xdb, 0xee, 0xbc, 0xef, 0x7d, 0xf3, 0xe6, 0xfb, - 0xde, 0xcc, 0x83, 0xd2, 0x90, 0xf8, 0x8c, 0xbb, 0x03, 0x1a, 0x12, 0xb7, 0x4e, 0x43, 0xd2, 0x21, - 0xe1, 0xdd, 0x78, 0x40, 0x9c, 0x20, 0xa4, 0x9c, 0x22, 0x10, 0x51, 0x67, 0x1e, 0xb5, 0xf7, 0x35, - 0xe4, 0x19, 0xf1, 0xf9, 0x6c, 0x0e, 0xe6, 0x12, 0x68, 0x17, 0xb4, 0x60, 0x87, 0xd3, 0x50, 0x11, - 0xe0, 0x5f, 0x06, 0x3c, 0x6d, 0x12, 0x1e, 0xe1, 0x99, 0x47, 0xbe, 0xcf, 0x08, 0xe3, 0xe8, 0x13, - 0x64, 0x86, 0xe3, 0x5b, 0x4e, 0xc2, 0xa2, 0x71, 0x60, 0x94, 0xcd, 0x4a, 0xd9, 0x89, 0xf6, 0x72, - 0x12, 0x53, 0x9c, 0x33, 0x81, 0xf7, 0x54, 0x9e, 0xdd, 0x87, 0x8c, 0x5c, 0x41, 0x65, 0xb0, 0x86, - 0x12, 0xde, 0x63, 0x84, 0xf7, 0xa6, 0xfe, 0x84, 0x08, 0xd6, 0xac, 0xb7, 0x37, 0x5c, 0xd0, 0xb4, - 0xfc, 0x09, 0x41, 0x0e, 0xe4, 0x75, 0xe4, 0x1d, 0x09, 0xd9, 0x98, 0x4e, 0x8b, 0x5b, 0x02, 0x9c, - 0x8b, 0xc0, 0x5f, 0x64, 0x00, 0x5f, 0x42, 0x21, 0x5e, 0x0b, 0x0b, 0xe8, 0x94, 0x11, 0x74, 0x02, - 0x8f, 0x34, 0x26, 0x56, 0x34, 0x0e, 0xb6, 0xcb, 0x66, 0xc5, 0xd6, 0x4f, 0x11, 0xa5, 0x75, 0x02, - 0x32, 0xf0, 0xcc, 0x88, 0x9e, 0xe1, 0x09, 0x58, 0x4d, 0xc2, 0x85, 0x54, 0x0b, 0x49, 0xde, 0xc7, - 0x24, 0x39, 0x8c, 0x49, 0xb2, 0x84, 0x8e, 0xab, 0x51, 0x5a, 0xa8, 0x81, 0xe0, 0xa1, 0xa6, 0x80, - 0xf8, 0xc6, 0x1f, 0x20, 0xa7, 0x11, 0xa8, 0x23, 0x1c, 0xc3, 0x0e, 0x9b, 0xaf, 0xa8, 0xda, 0x73, - 0xfa, 0x76, 0x02, 0xea, 0xc9, 0x38, 0xee, 0x42, 0xa1, 0x1a, 0x04, 0xb7, 0x3f, 0xa2, 0x03, 0x45, - 0x25, 0x9b, 0x9a, 0x0a, 0xaa, 0xee, 0x34, 0x11, 0x20, 0x12, 0x01, 0xff, 0x31, 0xe0, 0xd9, 0x0a, - 0xaf, 0xaa, 0xed, 0x7f, 0x88, 0x51, 0x0d, 0x32, 0x8c, 0xfb, 0x7c, 0xc6, 0x84, 0xb1, 0x7b, 0x95, - 0x37, 0x7a, 0xde, 0x9a, 0x1d, 0x9d, 0x8e, 0xc8, 0xf0, 0x54, 0x26, 0x76, 0x21, 0x23, 0x57, 0xd0, - 0x63, 0xc8, 0xb6, 0xda, 0xbd, 0xfa, 0x79, 0xb5, 0xd5, 0x6c, 0x58, 0x0f, 0x90, 0x09, 0xbb, 0x75, - 0xaf, 0x51, 0xfd, 0xdc, 0x38, 0xb5, 0x0c, 0x94, 0x85, 0x9d, 0x86, 0xe7, 0xb5, 0x3d, 0x6b, 0x0b, - 0x97, 0xc0, 0x96, 0xad, 0xc2, 0xf8, 0xfc, 0x22, 0xa9, 0x0e, 0x52, 0x42, 0xe1, 0x77, 0xb0, 0x9f, - 0x18, 0x55, 0xc7, 0x2d, 0xc2, 0xee, 0x7d, 0x2f, 0x4a, 0xdb, 0xee, 0x7f, 0xf1, 0x09, 0xa0, 0x6e, - 0x70, 0xed, 0x73, 0x22, 0x1d, 0x51, 0xba, 0x6b, 0xd6, 0x19, 0xa9, 0xd6, 0xfd, 0x34, 0x20, 0xbf, - 0x94, 0xbf, 0xea, 0x7d, 0x2a, 0x01, 0xfa, 0x18, 0xd3, 0xf2, 0x48, 0x47, 0x26, 0x30, 0xc7, 0x75, - 0x7c, 0x9d, 0xa2, 0x63, 0xf7, 0xe2, 0x54, 0xea, 0x58, 0xf9, 0xbd, 0x0d, 0xa6, 0xf6, 0xfc, 0xa0, - 0x21, 0xe4, 0x13, 0xe4, 0x42, 0x47, 0xab, 0x8f, 0x44, 0x92, 0xda, 0xf6, 0xf1, 0x46, 0x9c, 0x92, - 0xe1, 0x12, 0xf6, 0x96, 0xef, 0x37, 0x7a, 0xb5, 0xf1, 0x1d, 0xb2, 0x71, 0x1a, 0x44, 0x11, 0x9f, - 0x43, 0x76, 0x71, 0xe1, 0x50, 0x29, 0xed, 0x22, 0xdb, 0xcf, 0xd7, 0x44, 0x15, 0xd3, 0x15, 0x3c, - 0x89, 0xb5, 0x2c, 0xc2, 0xa9, 0xfd, 0x2c, 0x59, 0x0f, 0xff, 0xa1, 0xe7, 0x51, 0x0b, 0x4c, 0xcd, - 0x42, 0xf4, 0x62, 0xad, 0xb7, 0x92, 0xf3, 0xe5, 0x06, 0xef, 0x6b, 0x6d, 0xd0, 0x26, 0x46, 0xcd, - 0xd2, 0x1c, 0xbd, 0x98, 0x8f, 0x83, 0x2b, 0x77, 0x34, 0xe6, 0x5f, 0x67, 0x7d, 0x67, 0x40, 0x27, - 0xee, 0x88, 0x7e, 0x23, 0x37, 0xae, 0x9c, 0x1b, 0xec, 0xfa, 0xc6, 0x1d, 0x51, 0x57, 0xcc, 0x0c, - 0xe6, 0x46, 0xb3, 0xa4, 0x9f, 0x11, 0x4b, 0x6f, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0x9c, 0xcc, - 0xb9, 0x51, 0xa7, 0x06, 0x00, 0x00, + // 632 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcd, 0x6e, 0xd3, 0x40, + 0x10, 0xc6, 0xa5, 0x4d, 0xd5, 0x31, 0x2d, 0xc9, 0x06, 0xd2, 0xc8, 0x0d, 0x25, 0x98, 0x8a, 0x16, + 0x0e, 0xb6, 0x14, 0x0e, 0x1c, 0x20, 0x48, 0xf9, 0x6b, 0x91, 0x40, 0x49, 0xe5, 0x24, 0x08, 0xf5, + 0x12, 0x25, 0xe9, 0x26, 0x84, 0xb6, 0x59, 0xe3, 0xdd, 0x54, 0xe2, 0x6d, 0x38, 0xf0, 0x1e, 0x3c, + 0x00, 0x2f, 0x85, 0xbc, 0xbb, 0x8d, 0xd7, 0x8e, 0xed, 0x1c, 0xe0, 0x66, 0xcf, 0x7c, 0xf3, 0xed, + 0xec, 0x37, 0x33, 0x3b, 0x50, 0x9a, 0xe0, 0x21, 0x65, 0xf6, 0x98, 0x78, 0xd8, 0x6e, 0x10, 0x0f, + 0x77, 0xb1, 0x77, 0x3b, 0x1b, 0x63, 0xcb, 0xf5, 0x08, 0x23, 0x08, 0xb8, 0xd7, 0xf2, 0xbd, 0xc6, + 0x81, 0x82, 0x3c, 0xc5, 0x43, 0xb6, 0xf0, 0xc1, 0x4c, 0x00, 0x8d, 0x82, 0xe2, 0xec, 0x32, 0xe2, + 0x49, 0x02, 0xb3, 0x09, 0x8f, 0xce, 0x30, 0x0b, 0xe0, 0x0e, 0xfe, 0xbe, 0xc0, 0x94, 0x21, 0x04, + 0x9b, 0xf3, 0xe1, 0x0d, 0x2e, 0x6a, 0x65, 0xed, 0x64, 0xc7, 0xe1, 0xdf, 0xa8, 0x08, 0xdb, 0xb7, + 0xd8, 0xa3, 0x33, 0x32, 0x2f, 0x6e, 0x70, 0xf3, 0xdd, 0xaf, 0xd9, 0x83, 0xc7, 0x11, 0x16, 0xea, + 0x92, 0x39, 0xc5, 0xe8, 0x2d, 0xe8, 0x13, 0x61, 0x1d, 0x50, 0xcc, 0x38, 0x9b, 0x5e, 0x31, 0xac, + 0x20, 0x6b, 0x2b, 0x08, 0xea, 0xba, 0x78, 0xec, 0xc0, 0x64, 0xf9, 0x6f, 0xfe, 0xd6, 0xa0, 0xf0, + 0x69, 0x46, 0x15, 0x5e, 0x7a, 0x97, 0x5e, 0x0d, 0x32, 0x93, 0xd9, 0x35, 0xc3, 0x9e, 0xa4, 0x7c, + 0xa9, 0x52, 0xc6, 0xc7, 0x58, 0xa7, 0x3c, 0xc0, 0x91, 0x81, 0xc6, 0x08, 0x32, 0xc2, 0x82, 0x4e, + 0x20, 0xab, 0x24, 0x39, 0x50, 0xee, 0xbd, 0x17, 0x64, 0xd3, 0xf6, 0x15, 0xb0, 0x20, 0xaf, 0x22, + 0xc3, 0x6a, 0xe4, 0x02, 0xf0, 0x67, 0xa9, 0xcb, 0x17, 0xd8, 0x5f, 0x49, 0x46, 0x2a, 0x53, 0x85, + 0x07, 0x0a, 0x15, 0x2d, 0x6a, 0xe5, 0xfb, 0x6b, 0xa4, 0xd1, 0x03, 0x7e, 0x6a, 0x12, 0xc8, 0xf9, + 0xcc, 0xbc, 0x94, 0x4b, 0x55, 0xde, 0x45, 0x54, 0x39, 0x8a, 0xaa, 0x12, 0x82, 0x47, 0x05, 0x29, + 0x2d, 0x05, 0x89, 0x29, 0xbe, 0x59, 0x05, 0xa4, 0x32, 0xc8, 0x5b, 0x1c, 0xc3, 0x16, 0xf5, 0x2d, + 0x32, 0xfd, 0x9c, 0x7a, 0x20, 0x87, 0x3a, 0xc2, 0x6f, 0xf6, 0xa1, 0x50, 0x73, 0xdd, 0xeb, 0x1f, + 0xab, 0x9d, 0xf6, 0x4f, 0x2d, 0xf2, 0x47, 0x83, 0xfd, 0x15, 0xde, 0xff, 0xd0, 0x7b, 0xa8, 0x0e, + 0x19, 0xca, 0x86, 0x6c, 0x41, 0x79, 0x71, 0xf7, 0x2a, 0xaf, 0xd4, 0xb8, 0x84, 0x13, 0xad, 0x2e, + 0x8f, 0x70, 0x64, 0xa4, 0x69, 0x43, 0x46, 0x58, 0xd0, 0x2e, 0xec, 0xb4, 0x3b, 0x83, 0xc6, 0x87, + 0x5a, 0xfb, 0xac, 0x95, 0xbd, 0x87, 0x74, 0xd8, 0x6e, 0x38, 0xad, 0x5a, 0xaf, 0xd5, 0xcc, 0x6a, + 0x68, 0x07, 0xb6, 0x5a, 0x8e, 0xd3, 0x71, 0xb2, 0x1b, 0x66, 0x09, 0x0c, 0x31, 0x46, 0x94, 0xf9, + 0xa3, 0x2e, 0xbb, 0x48, 0x0a, 0x65, 0xbe, 0x81, 0x83, 0x58, 0xaf, 0xbc, 0xae, 0x32, 0x9d, 0x5a, + 0x78, 0x3a, 0xab, 0x80, 0xfa, 0xee, 0xe5, 0x90, 0x61, 0x51, 0x11, 0xa9, 0xbb, 0x52, 0x3a, 0x2d, + 0xb5, 0x74, 0xbf, 0x34, 0xc8, 0x87, 0xe2, 0x57, 0x6b, 0x9f, 0x4a, 0x80, 0xde, 0x47, 0xb4, 0x7c, + 0xa1, 0x22, 0x63, 0x98, 0xa3, 0x3a, 0x1e, 0xa5, 0xe8, 0xd8, 0x3f, 0x6f, 0x0a, 0x1d, 0x2b, 0x3f, + 0x37, 0x41, 0x57, 0x1e, 0x48, 0x34, 0x81, 0x7c, 0x8c, 0x5c, 0x28, 0x74, 0x78, 0xb2, 0xda, 0xc6, + 0xf1, 0x5a, 0x9c, 0x94, 0xa1, 0x07, 0xbb, 0xa1, 0xb7, 0x0f, 0x95, 0x57, 0x23, 0xc3, 0x2d, 0x6f, + 0x3c, 0x4b, 0x41, 0x48, 0xd6, 0x0b, 0x78, 0x18, 0x79, 0x39, 0x90, 0xb9, 0xfe, 0x8d, 0x33, 0x9e, + 0xa7, 0x62, 0x24, 0xf7, 0x47, 0x80, 0x60, 0x94, 0xd1, 0x93, 0xd4, 0x47, 0xc2, 0x38, 0x4c, 0x72, + 0x07, 0x89, 0x46, 0xc6, 0x21, 0x9c, 0x68, 0xfc, 0xd4, 0x87, 0x13, 0x4d, 0x9a, 0xe0, 0x36, 0xe8, + 0x4a, 0x7b, 0xa0, 0xc3, 0xc4, 0xbe, 0x11, 0x9c, 0x4f, 0xd7, 0xf4, 0x55, 0xbd, 0x03, 0xca, 0xbe, + 0xac, 0x67, 0x95, 0x6e, 0x39, 0xf7, 0x97, 0xe1, 0x85, 0x3d, 0x9d, 0xb1, 0xaf, 0x8b, 0x91, 0x35, + 0x26, 0x37, 0xf6, 0x94, 0x7c, 0xc3, 0x57, 0xb6, 0xd8, 0x9a, 0xf4, 0xf2, 0xca, 0x9e, 0x12, 0x9b, + 0x6f, 0x4c, 0x6a, 0x07, 0x9b, 0x74, 0x94, 0xe1, 0xa6, 0xd7, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, + 0x9b, 0xcc, 0x55, 0x65, 0xa5, 0x07, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -654,19 +751,25 @@ const _ = grpc.SupportPackageIsVersion4 type CoreServiceClient interface { // Retrieve version information about this Feast deployment GetFeastCoreVersion(ctx context.Context, in *GetFeastCoreVersionRequest, opts ...grpc.CallOption) (*GetFeastCoreVersionResponse, error) + // Retrieve feature set details for a specific feature set + // + // Request must contain a feature set name. + // If a version is provided it will be used. + // if the version is omitted then the latest feature set will be returned + GetFeatureSet(ctx context.Context, in *GetFeatureSetRequest, opts ...grpc.CallOption) (*GetFeatureSetResponse, error) // Retrieve feature set details given a filter. // // Returns all feature sets matching that filter. If none are found, // an empty list will be returned. // If no filter is provided in the request, the response will contain all the feature // sets currently stored in the registry. - GetFeatureSets(ctx context.Context, in *GetFeatureSetsRequest, opts ...grpc.CallOption) (*GetFeatureSetsResponse, error) + ListFeatureSets(ctx context.Context, in *ListFeatureSetsRequest, opts ...grpc.CallOption) (*ListFeatureSetsResponse, error) // Retrieve store details given a filter. // // Returns all stores matching that filter. If none are found, an empty list will be returned. // If no filter is provided in the request, the response will contain all the stores currently // stored in the registry. - GetStores(ctx context.Context, in *GetStoresRequest, opts ...grpc.CallOption) (*GetStoresResponse, error) + ListStores(ctx context.Context, in *ListStoresRequest, opts ...grpc.CallOption) (*ListStoresResponse, error) // Create or update and existing feature set. // // This function is idempotent - it will not create a new feature set if schema does not change. @@ -697,18 +800,27 @@ func (c *coreServiceClient) GetFeastCoreVersion(ctx context.Context, in *GetFeas return out, nil } -func (c *coreServiceClient) GetFeatureSets(ctx context.Context, in *GetFeatureSetsRequest, opts ...grpc.CallOption) (*GetFeatureSetsResponse, error) { - out := new(GetFeatureSetsResponse) - err := c.cc.Invoke(ctx, "/feast.core.CoreService/GetFeatureSets", in, out, opts...) +func (c *coreServiceClient) GetFeatureSet(ctx context.Context, in *GetFeatureSetRequest, opts ...grpc.CallOption) (*GetFeatureSetResponse, error) { + out := new(GetFeatureSetResponse) + err := c.cc.Invoke(ctx, "/feast.core.CoreService/GetFeatureSet", in, out, opts...) if err != nil { return nil, err } return out, nil } -func (c *coreServiceClient) GetStores(ctx context.Context, in *GetStoresRequest, opts ...grpc.CallOption) (*GetStoresResponse, error) { - out := new(GetStoresResponse) - err := c.cc.Invoke(ctx, "/feast.core.CoreService/GetStores", in, out, opts...) +func (c *coreServiceClient) ListFeatureSets(ctx context.Context, in *ListFeatureSetsRequest, opts ...grpc.CallOption) (*ListFeatureSetsResponse, error) { + out := new(ListFeatureSetsResponse) + err := c.cc.Invoke(ctx, "/feast.core.CoreService/ListFeatureSets", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *coreServiceClient) ListStores(ctx context.Context, in *ListStoresRequest, opts ...grpc.CallOption) (*ListStoresResponse, error) { + out := new(ListStoresResponse) + err := c.cc.Invoke(ctx, "/feast.core.CoreService/ListStores", in, out, opts...) if err != nil { return nil, err } @@ -737,19 +849,25 @@ func (c *coreServiceClient) UpdateStore(ctx context.Context, in *UpdateStoreRequ type CoreServiceServer interface { // Retrieve version information about this Feast deployment GetFeastCoreVersion(context.Context, *GetFeastCoreVersionRequest) (*GetFeastCoreVersionResponse, error) + // Retrieve feature set details for a specific feature set + // + // Request must contain a feature set name. + // If a version is provided it will be used. + // if the version is omitted then the latest feature set will be returned + GetFeatureSet(context.Context, *GetFeatureSetRequest) (*GetFeatureSetResponse, error) // Retrieve feature set details given a filter. // // Returns all feature sets matching that filter. If none are found, // an empty list will be returned. // If no filter is provided in the request, the response will contain all the feature // sets currently stored in the registry. - GetFeatureSets(context.Context, *GetFeatureSetsRequest) (*GetFeatureSetsResponse, error) + ListFeatureSets(context.Context, *ListFeatureSetsRequest) (*ListFeatureSetsResponse, error) // Retrieve store details given a filter. // // Returns all stores matching that filter. If none are found, an empty list will be returned. // If no filter is provided in the request, the response will contain all the stores currently // stored in the registry. - GetStores(context.Context, *GetStoresRequest) (*GetStoresResponse, error) + ListStores(context.Context, *ListStoresRequest) (*ListStoresResponse, error) // Create or update and existing feature set. // // This function is idempotent - it will not create a new feature set if schema does not change. @@ -770,11 +888,14 @@ type UnimplementedCoreServiceServer struct { func (*UnimplementedCoreServiceServer) GetFeastCoreVersion(ctx context.Context, req *GetFeastCoreVersionRequest) (*GetFeastCoreVersionResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method GetFeastCoreVersion not implemented") } -func (*UnimplementedCoreServiceServer) GetFeatureSets(ctx context.Context, req *GetFeatureSetsRequest) (*GetFeatureSetsResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method GetFeatureSets not implemented") +func (*UnimplementedCoreServiceServer) GetFeatureSet(ctx context.Context, req *GetFeatureSetRequest) (*GetFeatureSetResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetFeatureSet not implemented") +} +func (*UnimplementedCoreServiceServer) ListFeatureSets(ctx context.Context, req *ListFeatureSetsRequest) (*ListFeatureSetsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ListFeatureSets not implemented") } -func (*UnimplementedCoreServiceServer) GetStores(ctx context.Context, req *GetStoresRequest) (*GetStoresResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method GetStores not implemented") +func (*UnimplementedCoreServiceServer) ListStores(ctx context.Context, req *ListStoresRequest) (*ListStoresResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ListStores not implemented") } func (*UnimplementedCoreServiceServer) ApplyFeatureSet(ctx context.Context, req *ApplyFeatureSetRequest) (*ApplyFeatureSetResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method ApplyFeatureSet not implemented") @@ -805,38 +926,56 @@ func _CoreService_GetFeastCoreVersion_Handler(srv interface{}, ctx context.Conte return interceptor(ctx, in, info, handler) } -func _CoreService_GetFeatureSets_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(GetFeatureSetsRequest) +func _CoreService_GetFeatureSet_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetFeatureSetRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return srv.(CoreServiceServer).GetFeatureSets(ctx, in) + return srv.(CoreServiceServer).GetFeatureSet(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/feast.core.CoreService/GetFeatureSets", + FullMethod: "/feast.core.CoreService/GetFeatureSet", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(CoreServiceServer).GetFeatureSets(ctx, req.(*GetFeatureSetsRequest)) + return srv.(CoreServiceServer).GetFeatureSet(ctx, req.(*GetFeatureSetRequest)) } return interceptor(ctx, in, info, handler) } -func _CoreService_GetStores_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(GetStoresRequest) +func _CoreService_ListFeatureSets_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListFeatureSetsRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return srv.(CoreServiceServer).GetStores(ctx, in) + return srv.(CoreServiceServer).ListFeatureSets(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/feast.core.CoreService/GetStores", + FullMethod: "/feast.core.CoreService/ListFeatureSets", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(CoreServiceServer).GetStores(ctx, req.(*GetStoresRequest)) + return srv.(CoreServiceServer).ListFeatureSets(ctx, req.(*ListFeatureSetsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _CoreService_ListStores_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListStoresRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CoreServiceServer).ListStores(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/feast.core.CoreService/ListStores", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CoreServiceServer).ListStores(ctx, req.(*ListStoresRequest)) } return interceptor(ctx, in, info, handler) } @@ -886,12 +1025,16 @@ var _CoreService_serviceDesc = grpc.ServiceDesc{ Handler: _CoreService_GetFeastCoreVersion_Handler, }, { - MethodName: "GetFeatureSets", - Handler: _CoreService_GetFeatureSets_Handler, + MethodName: "GetFeatureSet", + Handler: _CoreService_GetFeatureSet_Handler, + }, + { + MethodName: "ListFeatureSets", + Handler: _CoreService_ListFeatureSets_Handler, }, { - MethodName: "GetStores", - Handler: _CoreService_GetStores_Handler, + MethodName: "ListStores", + Handler: _CoreService_ListStores_Handler, }, { MethodName: "ApplyFeatureSet", diff --git a/sdk/go/protos/feast/core/FeatureSet.pb.go b/sdk/go/protos/feast/core/FeatureSet.pb.go index b3af4c948f..79cbcaa94b 100644 --- a/sdk/go/protos/feast/core/FeatureSet.pb.go +++ b/sdk/go/protos/feast/core/FeatureSet.pb.go @@ -38,7 +38,8 @@ type FeatureSetSpec struct { // after [time - max_age]. Missing or older feature values will be returned // as nulls and indicated to end user MaxAge *duration.Duration `protobuf:"bytes,5,opt,name=max_age,json=maxAge,proto3" json:"max_age,omitempty"` - // Source on which feature rows can be found + // Optional. Source on which feature rows can be found. + // If not set, source will be set to the default value configured in Feast Core. Source *Source `protobuf:"bytes,6,opt,name=source,proto3" json:"source,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` diff --git a/sdk/go/protos/feast/core/Store.pb.go b/sdk/go/protos/feast/core/Store.pb.go index 4e06f8cad1..5dafb63d94 100644 --- a/sdk/go/protos/feast/core/Store.pb.go +++ b/sdk/go/protos/feast/core/Store.pb.go @@ -391,7 +391,9 @@ func (m *Store_CassandraConfig) GetPort() int32 { } type Store_Subscription struct { - // Name of featureSet to subscribe to. + // Name of featureSet to subscribe to. This field supports any valid basic POSIX regex, + // e.g. customer_.* or .* + // https://www.regular-expressions.info/posix.html Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` // Versions of the given featureSet that will be ingested into this store. // Valid options for version: diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py index 41589f700e..2466aa46b8 100644 --- a/sdk/python/feast/client.py +++ b/sdk/python/feast/client.py @@ -24,10 +24,12 @@ from feast.exceptions import format_grpc_exception from feast.core.CoreService_pb2 import ( GetFeastCoreVersionRequest, - GetFeatureSetsResponse, + ListFeatureSetsResponse, ApplyFeatureSetRequest, - GetFeatureSetsRequest, + ListFeatureSetsRequest, ApplyFeatureSetResponse, + GetFeatureSetRequest, + GetFeatureSetResponse, ) from feast.core.CoreService_pb2_grpc import CoreServiceStub from feast.feature_set import FeatureSet, Entity @@ -240,12 +242,12 @@ def list_feature_sets(self) -> List[FeatureSet]: try: # Get latest feature sets from Feast Core - feature_set_protos = self._core_service_stub.GetFeatureSets( - GetFeatureSetsRequest() - ) # type: GetFeatureSetsResponse + feature_set_protos = self._core_service_stub.ListFeatureSets( + ListFeatureSetsRequest() + ) # type: ListFeatureSetsResponse except grpc.RpcError as e: raise Exception( - format_grpc_exception("GetFeatureSets", e.code(), e.details()) + format_grpc_exception("ListFeatureSets", e.code(), e.details()) ) # Store list of feature sets @@ -261,45 +263,30 @@ def get_feature_set( ) -> Union[FeatureSet, None]: """ Retrieve a single feature set from Feast Core - :param name: Name of feature set - :param version: Version of feature set (int) + :param name: (str) Name of feature set + :param version: (int) Version of feature set + :param fail_if_missing: (bool) Throws an exception if the feature set is not + found :return: Returns a single feature set + """ self._connect_core() - - # TODO: Version should only be integer or unset. Core should handle 'latest' - if version is None: - version = "latest" - elif isinstance(version, int): - version = str(version) - else: - raise ValueError(f"Version {version} is not of type integer.") - try: - get_feature_set_response = self._core_service_stub.GetFeatureSets( - GetFeatureSetsRequest( - filter=GetFeatureSetsRequest.Filter( - feature_set_name=name.strip(), feature_set_version=version - ) - ) - ) # type: GetFeatureSetsResponse + get_feature_set_response = self._core_service_stub.GetFeatureSet( + GetFeatureSetRequest(name=name.strip(), version=str(version)) + ) # type: GetFeatureSetResponse + feature_set = get_feature_set_response.feature_set except grpc.RpcError as e: - print(format_grpc_exception("GetFeatureSets", e.code(), e.details())) + print(format_grpc_exception("GetFeatureSet", e.code(), e.details())) else: - num_feature_sets_found = len(list(get_feature_set_response.feature_sets)) - if num_feature_sets_found == 0: - if fail_if_missing: - raise Exception( - f'Could not find feature set with name "{name}" and ' - f'version "{version}" ' - ) - return None - if num_feature_sets_found > 1: + if feature_set is not None: + return FeatureSet.from_proto(feature_set) + + if fail_if_missing: raise Exception( - f'Found {num_feature_sets_found} feature sets with name "{name}"' - f' and version "{version}": {list(get_feature_set_response.feature_sets)}' + f'Could not find feature set with name "{name}" and ' + f'version "{version}"' ) - return FeatureSet.from_proto(get_feature_set_response.feature_sets[0]) def list_entities(self) -> Dict[str, Entity]: """ diff --git a/sdk/python/feast/core/CoreService_pb2.py b/sdk/python/feast/core/CoreService_pb2.py index ffc29db20c..61781edbad 100644 --- a/sdk/python/feast/core/CoreService_pb2.py +++ b/sdk/python/feast/core/CoreService_pb2.py @@ -22,7 +22,7 @@ package='feast.core', syntax='proto3', serialized_options=_b('\n\nfeast.coreB\020CoreServiceProtoZ/github.com/gojek/feast/sdk/go/protos/feast/core'), - serialized_pb=_b('\n\x1c\x66\x65\x61st/core/CoreService.proto\x12\nfeast.core\x1a\x1b\x66\x65\x61st/core/FeatureSet.proto\x1a\x16\x66\x65\x61st/core/Store.proto\"\x92\x01\n\x15GetFeatureSetsRequest\x12\x38\n\x06\x66ilter\x18\x01 \x01(\x0b\x32(.feast.core.GetFeatureSetsRequest.Filter\x1a?\n\x06\x46ilter\x12\x18\n\x10\x66\x65\x61ture_set_name\x18\x01 \x01(\t\x12\x1b\n\x13\x66\x65\x61ture_set_version\x18\x02 \x01(\t\"J\n\x16GetFeatureSetsResponse\x12\x30\n\x0c\x66\x65\x61ture_sets\x18\x01 \x03(\x0b\x32\x1a.feast.core.FeatureSetSpec\"_\n\x10GetStoresRequest\x12\x33\n\x06\x66ilter\x18\x01 \x01(\x0b\x32#.feast.core.GetStoresRequest.Filter\x1a\x16\n\x06\x46ilter\x12\x0c\n\x04name\x18\x01 \x01(\t\"5\n\x11GetStoresResponse\x12 \n\x05store\x18\x01 \x03(\x0b\x32\x11.feast.core.Store\"I\n\x16\x41pplyFeatureSetRequest\x12/\n\x0b\x66\x65\x61ture_set\x18\x01 \x01(\x0b\x32\x1a.feast.core.FeatureSetSpec\"\xb7\x01\n\x17\x41pplyFeatureSetResponse\x12/\n\x0b\x66\x65\x61ture_set\x18\x01 \x01(\x0b\x32\x1a.feast.core.FeatureSetSpec\x12:\n\x06status\x18\x02 \x01(\x0e\x32*.feast.core.ApplyFeatureSetResponse.Status\"/\n\x06Status\x12\r\n\tNO_CHANGE\x10\x00\x12\x0b\n\x07\x43REATED\x10\x01\x12\t\n\x05\x45RROR\x10\x02\"\x1c\n\x1aGetFeastCoreVersionRequest\".\n\x1bGetFeastCoreVersionResponse\x12\x0f\n\x07version\x18\x01 \x01(\t\"6\n\x12UpdateStoreRequest\x12 \n\x05store\x18\x01 \x01(\x0b\x32\x11.feast.core.Store\"\x95\x01\n\x13UpdateStoreResponse\x12 \n\x05store\x18\x01 \x01(\x0b\x32\x11.feast.core.Store\x12\x36\n\x06status\x18\x02 \x01(\x0e\x32&.feast.core.UpdateStoreResponse.Status\"$\n\x06Status\x12\r\n\tNO_CHANGE\x10\x00\x12\x0b\n\x07UPDATED\x10\x01\x32\xc4\x03\n\x0b\x43oreService\x12\x66\n\x13GetFeastCoreVersion\x12&.feast.core.GetFeastCoreVersionRequest\x1a\'.feast.core.GetFeastCoreVersionResponse\x12W\n\x0eGetFeatureSets\x12!.feast.core.GetFeatureSetsRequest\x1a\".feast.core.GetFeatureSetsResponse\x12H\n\tGetStores\x12\x1c.feast.core.GetStoresRequest\x1a\x1d.feast.core.GetStoresResponse\x12Z\n\x0f\x41pplyFeatureSet\x12\".feast.core.ApplyFeatureSetRequest\x1a#.feast.core.ApplyFeatureSetResponse\x12N\n\x0bUpdateStore\x12\x1e.feast.core.UpdateStoreRequest\x1a\x1f.feast.core.UpdateStoreResponseBO\n\nfeast.coreB\x10\x43oreServiceProtoZ/github.com/gojek/feast/sdk/go/protos/feast/coreb\x06proto3') + serialized_pb=_b('\n\x1c\x66\x65\x61st/core/CoreService.proto\x12\nfeast.core\x1a\x1b\x66\x65\x61st/core/FeatureSet.proto\x1a\x16\x66\x65\x61st/core/Store.proto\"5\n\x14GetFeatureSetRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\"H\n\x15GetFeatureSetResponse\x12/\n\x0b\x66\x65\x61ture_set\x18\x01 \x01(\x0b\x32\x1a.feast.core.FeatureSetSpec\"\x94\x01\n\x16ListFeatureSetsRequest\x12\x39\n\x06\x66ilter\x18\x01 \x01(\x0b\x32).feast.core.ListFeatureSetsRequest.Filter\x1a?\n\x06\x46ilter\x12\x18\n\x10\x66\x65\x61ture_set_name\x18\x01 \x01(\t\x12\x1b\n\x13\x66\x65\x61ture_set_version\x18\x02 \x01(\t\"K\n\x17ListFeatureSetsResponse\x12\x30\n\x0c\x66\x65\x61ture_sets\x18\x01 \x03(\x0b\x32\x1a.feast.core.FeatureSetSpec\"a\n\x11ListStoresRequest\x12\x34\n\x06\x66ilter\x18\x01 \x01(\x0b\x32$.feast.core.ListStoresRequest.Filter\x1a\x16\n\x06\x46ilter\x12\x0c\n\x04name\x18\x01 \x01(\t\"6\n\x12ListStoresResponse\x12 \n\x05store\x18\x01 \x03(\x0b\x32\x11.feast.core.Store\"I\n\x16\x41pplyFeatureSetRequest\x12/\n\x0b\x66\x65\x61ture_set\x18\x01 \x01(\x0b\x32\x1a.feast.core.FeatureSetSpec\"\xb7\x01\n\x17\x41pplyFeatureSetResponse\x12/\n\x0b\x66\x65\x61ture_set\x18\x01 \x01(\x0b\x32\x1a.feast.core.FeatureSetSpec\x12:\n\x06status\x18\x02 \x01(\x0e\x32*.feast.core.ApplyFeatureSetResponse.Status\"/\n\x06Status\x12\r\n\tNO_CHANGE\x10\x00\x12\x0b\n\x07\x43REATED\x10\x01\x12\t\n\x05\x45RROR\x10\x02\"\x1c\n\x1aGetFeastCoreVersionRequest\".\n\x1bGetFeastCoreVersionResponse\x12\x0f\n\x07version\x18\x01 \x01(\t\"6\n\x12UpdateStoreRequest\x12 \n\x05store\x18\x01 \x01(\x0b\x32\x11.feast.core.Store\"\x95\x01\n\x13UpdateStoreResponse\x12 \n\x05store\x18\x01 \x01(\x0b\x32\x11.feast.core.Store\x12\x36\n\x06status\x18\x02 \x01(\x0e\x32&.feast.core.UpdateStoreResponse.Status\"$\n\x06Status\x12\r\n\tNO_CHANGE\x10\x00\x12\x0b\n\x07UPDATED\x10\x01\x32\xa0\x04\n\x0b\x43oreService\x12\x66\n\x13GetFeastCoreVersion\x12&.feast.core.GetFeastCoreVersionRequest\x1a\'.feast.core.GetFeastCoreVersionResponse\x12T\n\rGetFeatureSet\x12 .feast.core.GetFeatureSetRequest\x1a!.feast.core.GetFeatureSetResponse\x12Z\n\x0fListFeatureSets\x12\".feast.core.ListFeatureSetsRequest\x1a#.feast.core.ListFeatureSetsResponse\x12K\n\nListStores\x12\x1d.feast.core.ListStoresRequest\x1a\x1e.feast.core.ListStoresResponse\x12Z\n\x0f\x41pplyFeatureSet\x12\".feast.core.ApplyFeatureSetRequest\x1a#.feast.core.ApplyFeatureSetResponse\x12N\n\x0bUpdateStore\x12\x1e.feast.core.UpdateStoreRequest\x1a\x1f.feast.core.UpdateStoreResponseBO\n\nfeast.coreB\x10\x43oreServiceProtoZ/github.com/gojek/feast/sdk/go/protos/feast/coreb\x06proto3') , dependencies=[feast_dot_core_dot_FeatureSet__pb2.DESCRIPTOR,feast_dot_core_dot_Store__pb2.DESCRIPTOR,]) @@ -49,8 +49,8 @@ ], containing_type=None, serialized_options=None, - serialized_start=686, - serialized_end=733, + serialized_start=821, + serialized_end=868, ) _sym_db.RegisterEnumDescriptor(_APPLYFEATURESETRESPONSE_STATUS) @@ -71,28 +71,97 @@ ], containing_type=None, serialized_options=None, - serialized_start=983, - serialized_end=1019, + serialized_start=1118, + serialized_end=1154, ) _sym_db.RegisterEnumDescriptor(_UPDATESTORERESPONSE_STATUS) -_GETFEATURESETSREQUEST_FILTER = _descriptor.Descriptor( +_GETFEATURESETREQUEST = _descriptor.Descriptor( + name='GetFeatureSetRequest', + full_name='feast.core.GetFeatureSetRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='feast.core.GetFeatureSetRequest.name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='version', full_name='feast.core.GetFeatureSetRequest.version', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=97, + serialized_end=150, +) + + +_GETFEATURESETRESPONSE = _descriptor.Descriptor( + name='GetFeatureSetResponse', + full_name='feast.core.GetFeatureSetResponse', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='feature_set', full_name='feast.core.GetFeatureSetResponse.feature_set', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=152, + serialized_end=224, +) + + +_LISTFEATURESETSREQUEST_FILTER = _descriptor.Descriptor( name='Filter', - full_name='feast.core.GetFeatureSetsRequest.Filter', + full_name='feast.core.ListFeatureSetsRequest.Filter', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='feature_set_name', full_name='feast.core.GetFeatureSetsRequest.Filter.feature_set_name', index=0, + name='feature_set_name', full_name='feast.core.ListFeatureSetsRequest.Filter.feature_set_name', index=0, number=1, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='feature_set_version', full_name='feast.core.GetFeatureSetsRequest.Filter.feature_set_version', index=1, + name='feature_set_version', full_name='feast.core.ListFeatureSetsRequest.Filter.feature_set_version', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, @@ -110,19 +179,19 @@ extension_ranges=[], oneofs=[ ], - serialized_start=181, - serialized_end=244, + serialized_start=312, + serialized_end=375, ) -_GETFEATURESETSREQUEST = _descriptor.Descriptor( - name='GetFeatureSetsRequest', - full_name='feast.core.GetFeatureSetsRequest', +_LISTFEATURESETSREQUEST = _descriptor.Descriptor( + name='ListFeatureSetsRequest', + full_name='feast.core.ListFeatureSetsRequest', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='filter', full_name='feast.core.GetFeatureSetsRequest.filter', index=0, + name='filter', full_name='feast.core.ListFeatureSetsRequest.filter', index=0, number=1, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, @@ -131,7 +200,7 @@ ], extensions=[ ], - nested_types=[_GETFEATURESETSREQUEST_FILTER, ], + nested_types=[_LISTFEATURESETSREQUEST_FILTER, ], enum_types=[ ], serialized_options=None, @@ -140,20 +209,20 @@ extension_ranges=[], oneofs=[ ], - serialized_start=98, - serialized_end=244, + serialized_start=227, + serialized_end=375, ) -_GETFEATURESETSRESPONSE = _descriptor.Descriptor( - name='GetFeatureSetsResponse', - full_name='feast.core.GetFeatureSetsResponse', +_LISTFEATURESETSRESPONSE = _descriptor.Descriptor( + name='ListFeatureSetsResponse', + full_name='feast.core.ListFeatureSetsResponse', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='feature_sets', full_name='feast.core.GetFeatureSetsResponse.feature_sets', index=0, + name='feature_sets', full_name='feast.core.ListFeatureSetsResponse.feature_sets', index=0, number=1, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, @@ -171,20 +240,20 @@ extension_ranges=[], oneofs=[ ], - serialized_start=246, - serialized_end=320, + serialized_start=377, + serialized_end=452, ) -_GETSTORESREQUEST_FILTER = _descriptor.Descriptor( +_LISTSTORESREQUEST_FILTER = _descriptor.Descriptor( name='Filter', - full_name='feast.core.GetStoresRequest.Filter', + full_name='feast.core.ListStoresRequest.Filter', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='name', full_name='feast.core.GetStoresRequest.Filter.name', index=0, + name='name', full_name='feast.core.ListStoresRequest.Filter.name', index=0, number=1, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, @@ -202,19 +271,19 @@ extension_ranges=[], oneofs=[ ], - serialized_start=395, - serialized_end=417, + serialized_start=529, + serialized_end=551, ) -_GETSTORESREQUEST = _descriptor.Descriptor( - name='GetStoresRequest', - full_name='feast.core.GetStoresRequest', +_LISTSTORESREQUEST = _descriptor.Descriptor( + name='ListStoresRequest', + full_name='feast.core.ListStoresRequest', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='filter', full_name='feast.core.GetStoresRequest.filter', index=0, + name='filter', full_name='feast.core.ListStoresRequest.filter', index=0, number=1, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, @@ -223,7 +292,7 @@ ], extensions=[ ], - nested_types=[_GETSTORESREQUEST_FILTER, ], + nested_types=[_LISTSTORESREQUEST_FILTER, ], enum_types=[ ], serialized_options=None, @@ -232,20 +301,20 @@ extension_ranges=[], oneofs=[ ], - serialized_start=322, - serialized_end=417, + serialized_start=454, + serialized_end=551, ) -_GETSTORESRESPONSE = _descriptor.Descriptor( - name='GetStoresResponse', - full_name='feast.core.GetStoresResponse', +_LISTSTORESRESPONSE = _descriptor.Descriptor( + name='ListStoresResponse', + full_name='feast.core.ListStoresResponse', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='store', full_name='feast.core.GetStoresResponse.store', index=0, + name='store', full_name='feast.core.ListStoresResponse.store', index=0, number=1, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, @@ -263,8 +332,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=419, - serialized_end=472, + serialized_start=553, + serialized_end=607, ) @@ -294,8 +363,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=474, - serialized_end=547, + serialized_start=609, + serialized_end=682, ) @@ -333,8 +402,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=550, - serialized_end=733, + serialized_start=685, + serialized_end=868, ) @@ -357,8 +426,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=735, - serialized_end=763, + serialized_start=870, + serialized_end=898, ) @@ -388,8 +457,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=765, - serialized_end=811, + serialized_start=900, + serialized_end=946, ) @@ -419,8 +488,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=813, - serialized_end=867, + serialized_start=948, + serialized_end=1002, ) @@ -458,16 +527,17 @@ extension_ranges=[], oneofs=[ ], - serialized_start=870, - serialized_end=1019, + serialized_start=1005, + serialized_end=1154, ) -_GETFEATURESETSREQUEST_FILTER.containing_type = _GETFEATURESETSREQUEST -_GETFEATURESETSREQUEST.fields_by_name['filter'].message_type = _GETFEATURESETSREQUEST_FILTER -_GETFEATURESETSRESPONSE.fields_by_name['feature_sets'].message_type = feast_dot_core_dot_FeatureSet__pb2._FEATURESETSPEC -_GETSTORESREQUEST_FILTER.containing_type = _GETSTORESREQUEST -_GETSTORESREQUEST.fields_by_name['filter'].message_type = _GETSTORESREQUEST_FILTER -_GETSTORESRESPONSE.fields_by_name['store'].message_type = feast_dot_core_dot_Store__pb2._STORE +_GETFEATURESETRESPONSE.fields_by_name['feature_set'].message_type = feast_dot_core_dot_FeatureSet__pb2._FEATURESETSPEC +_LISTFEATURESETSREQUEST_FILTER.containing_type = _LISTFEATURESETSREQUEST +_LISTFEATURESETSREQUEST.fields_by_name['filter'].message_type = _LISTFEATURESETSREQUEST_FILTER +_LISTFEATURESETSRESPONSE.fields_by_name['feature_sets'].message_type = feast_dot_core_dot_FeatureSet__pb2._FEATURESETSPEC +_LISTSTORESREQUEST_FILTER.containing_type = _LISTSTORESREQUEST +_LISTSTORESREQUEST.fields_by_name['filter'].message_type = _LISTSTORESREQUEST_FILTER +_LISTSTORESRESPONSE.fields_by_name['store'].message_type = feast_dot_core_dot_Store__pb2._STORE _APPLYFEATURESETREQUEST.fields_by_name['feature_set'].message_type = feast_dot_core_dot_FeatureSet__pb2._FEATURESETSPEC _APPLYFEATURESETRESPONSE.fields_by_name['feature_set'].message_type = feast_dot_core_dot_FeatureSet__pb2._FEATURESETSPEC _APPLYFEATURESETRESPONSE.fields_by_name['status'].enum_type = _APPLYFEATURESETRESPONSE_STATUS @@ -476,10 +546,12 @@ _UPDATESTORERESPONSE.fields_by_name['store'].message_type = feast_dot_core_dot_Store__pb2._STORE _UPDATESTORERESPONSE.fields_by_name['status'].enum_type = _UPDATESTORERESPONSE_STATUS _UPDATESTORERESPONSE_STATUS.containing_type = _UPDATESTORERESPONSE -DESCRIPTOR.message_types_by_name['GetFeatureSetsRequest'] = _GETFEATURESETSREQUEST -DESCRIPTOR.message_types_by_name['GetFeatureSetsResponse'] = _GETFEATURESETSRESPONSE -DESCRIPTOR.message_types_by_name['GetStoresRequest'] = _GETSTORESREQUEST -DESCRIPTOR.message_types_by_name['GetStoresResponse'] = _GETSTORESRESPONSE +DESCRIPTOR.message_types_by_name['GetFeatureSetRequest'] = _GETFEATURESETREQUEST +DESCRIPTOR.message_types_by_name['GetFeatureSetResponse'] = _GETFEATURESETRESPONSE +DESCRIPTOR.message_types_by_name['ListFeatureSetsRequest'] = _LISTFEATURESETSREQUEST +DESCRIPTOR.message_types_by_name['ListFeatureSetsResponse'] = _LISTFEATURESETSRESPONSE +DESCRIPTOR.message_types_by_name['ListStoresRequest'] = _LISTSTORESREQUEST +DESCRIPTOR.message_types_by_name['ListStoresResponse'] = _LISTSTORESRESPONSE DESCRIPTOR.message_types_by_name['ApplyFeatureSetRequest'] = _APPLYFEATURESETREQUEST DESCRIPTOR.message_types_by_name['ApplyFeatureSetResponse'] = _APPLYFEATURESETRESPONSE DESCRIPTOR.message_types_by_name['GetFeastCoreVersionRequest'] = _GETFEASTCOREVERSIONREQUEST @@ -488,49 +560,63 @@ DESCRIPTOR.message_types_by_name['UpdateStoreResponse'] = _UPDATESTORERESPONSE _sym_db.RegisterFileDescriptor(DESCRIPTOR) -GetFeatureSetsRequest = _reflection.GeneratedProtocolMessageType('GetFeatureSetsRequest', (_message.Message,), { +GetFeatureSetRequest = _reflection.GeneratedProtocolMessageType('GetFeatureSetRequest', (_message.Message,), { + 'DESCRIPTOR' : _GETFEATURESETREQUEST, + '__module__' : 'feast.core.CoreService_pb2' + # @@protoc_insertion_point(class_scope:feast.core.GetFeatureSetRequest) + }) +_sym_db.RegisterMessage(GetFeatureSetRequest) + +GetFeatureSetResponse = _reflection.GeneratedProtocolMessageType('GetFeatureSetResponse', (_message.Message,), { + 'DESCRIPTOR' : _GETFEATURESETRESPONSE, + '__module__' : 'feast.core.CoreService_pb2' + # @@protoc_insertion_point(class_scope:feast.core.GetFeatureSetResponse) + }) +_sym_db.RegisterMessage(GetFeatureSetResponse) + +ListFeatureSetsRequest = _reflection.GeneratedProtocolMessageType('ListFeatureSetsRequest', (_message.Message,), { 'Filter' : _reflection.GeneratedProtocolMessageType('Filter', (_message.Message,), { - 'DESCRIPTOR' : _GETFEATURESETSREQUEST_FILTER, + 'DESCRIPTOR' : _LISTFEATURESETSREQUEST_FILTER, '__module__' : 'feast.core.CoreService_pb2' - # @@protoc_insertion_point(class_scope:feast.core.GetFeatureSetsRequest.Filter) + # @@protoc_insertion_point(class_scope:feast.core.ListFeatureSetsRequest.Filter) }) , - 'DESCRIPTOR' : _GETFEATURESETSREQUEST, + 'DESCRIPTOR' : _LISTFEATURESETSREQUEST, '__module__' : 'feast.core.CoreService_pb2' - # @@protoc_insertion_point(class_scope:feast.core.GetFeatureSetsRequest) + # @@protoc_insertion_point(class_scope:feast.core.ListFeatureSetsRequest) }) -_sym_db.RegisterMessage(GetFeatureSetsRequest) -_sym_db.RegisterMessage(GetFeatureSetsRequest.Filter) +_sym_db.RegisterMessage(ListFeatureSetsRequest) +_sym_db.RegisterMessage(ListFeatureSetsRequest.Filter) -GetFeatureSetsResponse = _reflection.GeneratedProtocolMessageType('GetFeatureSetsResponse', (_message.Message,), { - 'DESCRIPTOR' : _GETFEATURESETSRESPONSE, +ListFeatureSetsResponse = _reflection.GeneratedProtocolMessageType('ListFeatureSetsResponse', (_message.Message,), { + 'DESCRIPTOR' : _LISTFEATURESETSRESPONSE, '__module__' : 'feast.core.CoreService_pb2' - # @@protoc_insertion_point(class_scope:feast.core.GetFeatureSetsResponse) + # @@protoc_insertion_point(class_scope:feast.core.ListFeatureSetsResponse) }) -_sym_db.RegisterMessage(GetFeatureSetsResponse) +_sym_db.RegisterMessage(ListFeatureSetsResponse) -GetStoresRequest = _reflection.GeneratedProtocolMessageType('GetStoresRequest', (_message.Message,), { +ListStoresRequest = _reflection.GeneratedProtocolMessageType('ListStoresRequest', (_message.Message,), { 'Filter' : _reflection.GeneratedProtocolMessageType('Filter', (_message.Message,), { - 'DESCRIPTOR' : _GETSTORESREQUEST_FILTER, + 'DESCRIPTOR' : _LISTSTORESREQUEST_FILTER, '__module__' : 'feast.core.CoreService_pb2' - # @@protoc_insertion_point(class_scope:feast.core.GetStoresRequest.Filter) + # @@protoc_insertion_point(class_scope:feast.core.ListStoresRequest.Filter) }) , - 'DESCRIPTOR' : _GETSTORESREQUEST, + 'DESCRIPTOR' : _LISTSTORESREQUEST, '__module__' : 'feast.core.CoreService_pb2' - # @@protoc_insertion_point(class_scope:feast.core.GetStoresRequest) + # @@protoc_insertion_point(class_scope:feast.core.ListStoresRequest) }) -_sym_db.RegisterMessage(GetStoresRequest) -_sym_db.RegisterMessage(GetStoresRequest.Filter) +_sym_db.RegisterMessage(ListStoresRequest) +_sym_db.RegisterMessage(ListStoresRequest.Filter) -GetStoresResponse = _reflection.GeneratedProtocolMessageType('GetStoresResponse', (_message.Message,), { - 'DESCRIPTOR' : _GETSTORESRESPONSE, +ListStoresResponse = _reflection.GeneratedProtocolMessageType('ListStoresResponse', (_message.Message,), { + 'DESCRIPTOR' : _LISTSTORESRESPONSE, '__module__' : 'feast.core.CoreService_pb2' - # @@protoc_insertion_point(class_scope:feast.core.GetStoresResponse) + # @@protoc_insertion_point(class_scope:feast.core.ListStoresResponse) }) -_sym_db.RegisterMessage(GetStoresResponse) +_sym_db.RegisterMessage(ListStoresResponse) ApplyFeatureSetRequest = _reflection.GeneratedProtocolMessageType('ApplyFeatureSetRequest', (_message.Message,), { 'DESCRIPTOR' : _APPLYFEATURESETREQUEST, @@ -583,8 +669,8 @@ file=DESCRIPTOR, index=0, serialized_options=None, - serialized_start=1022, - serialized_end=1474, + serialized_start=1157, + serialized_end=1701, methods=[ _descriptor.MethodDescriptor( name='GetFeastCoreVersion', @@ -596,27 +682,36 @@ serialized_options=None, ), _descriptor.MethodDescriptor( - name='GetFeatureSets', - full_name='feast.core.CoreService.GetFeatureSets', + name='GetFeatureSet', + full_name='feast.core.CoreService.GetFeatureSet', index=1, containing_service=None, - input_type=_GETFEATURESETSREQUEST, - output_type=_GETFEATURESETSRESPONSE, + input_type=_GETFEATURESETREQUEST, + output_type=_GETFEATURESETRESPONSE, serialized_options=None, ), _descriptor.MethodDescriptor( - name='GetStores', - full_name='feast.core.CoreService.GetStores', + name='ListFeatureSets', + full_name='feast.core.CoreService.ListFeatureSets', index=2, containing_service=None, - input_type=_GETSTORESREQUEST, - output_type=_GETSTORESRESPONSE, + input_type=_LISTFEATURESETSREQUEST, + output_type=_LISTFEATURESETSRESPONSE, + serialized_options=None, + ), + _descriptor.MethodDescriptor( + name='ListStores', + full_name='feast.core.CoreService.ListStores', + index=3, + containing_service=None, + input_type=_LISTSTORESREQUEST, + output_type=_LISTSTORESRESPONSE, serialized_options=None, ), _descriptor.MethodDescriptor( name='ApplyFeatureSet', full_name='feast.core.CoreService.ApplyFeatureSet', - index=3, + index=4, containing_service=None, input_type=_APPLYFEATURESETREQUEST, output_type=_APPLYFEATURESETRESPONSE, @@ -625,7 +720,7 @@ _descriptor.MethodDescriptor( name='UpdateStore', full_name='feast.core.CoreService.UpdateStore', - index=4, + index=5, containing_service=None, input_type=_UPDATESTOREREQUEST, output_type=_UPDATESTORERESPONSE, diff --git a/sdk/python/feast/core/CoreService_pb2.pyi b/sdk/python/feast/core/CoreService_pb2.pyi index c81ae3e11c..0f5d4b288e 100644 --- a/sdk/python/feast/core/CoreService_pb2.pyi +++ b/sdk/python/feast/core/CoreService_pb2.pyi @@ -35,7 +35,47 @@ from typing_extensions import ( ) -class GetFeatureSetsRequest(google___protobuf___message___Message): +class GetFeatureSetRequest(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... + name = ... # type: typing___Text + version = ... # type: typing___Text + + def __init__(self, + *, + name : typing___Optional[typing___Text] = None, + version : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> GetFeatureSetRequest: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"name",u"version"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[u"name",b"name",u"version",b"version"]) -> None: ... + +class GetFeatureSetResponse(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... + + @property + def feature_set(self) -> feast___core___FeatureSet_pb2___FeatureSetSpec: ... + + def __init__(self, + *, + feature_set : typing___Optional[feast___core___FeatureSet_pb2___FeatureSetSpec] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> GetFeatureSetResponse: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"feature_set"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"feature_set"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"feature_set",b"feature_set"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"feature_set",b"feature_set"]) -> None: ... + +class ListFeatureSetsRequest(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... class Filter(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @@ -48,7 +88,7 @@ class GetFeatureSetsRequest(google___protobuf___message___Message): feature_set_version : typing___Optional[typing___Text] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> GetFeatureSetsRequest.Filter: ... + def FromString(cls, s: bytes) -> ListFeatureSetsRequest.Filter: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): @@ -58,14 +98,14 @@ class GetFeatureSetsRequest(google___protobuf___message___Message): @property - def filter(self) -> GetFeatureSetsRequest.Filter: ... + def filter(self) -> ListFeatureSetsRequest.Filter: ... def __init__(self, *, - filter : typing___Optional[GetFeatureSetsRequest.Filter] = None, + filter : typing___Optional[ListFeatureSetsRequest.Filter] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> GetFeatureSetsRequest: ... + def FromString(cls, s: bytes) -> ListFeatureSetsRequest: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): @@ -75,7 +115,7 @@ class GetFeatureSetsRequest(google___protobuf___message___Message): def HasField(self, field_name: typing_extensions___Literal[u"filter",b"filter"]) -> bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"filter",b"filter"]) -> None: ... -class GetFeatureSetsResponse(google___protobuf___message___Message): +class ListFeatureSetsResponse(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @property @@ -86,7 +126,7 @@ class GetFeatureSetsResponse(google___protobuf___message___Message): feature_sets : typing___Optional[typing___Iterable[feast___core___FeatureSet_pb2___FeatureSetSpec]] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> GetFeatureSetsResponse: ... + def FromString(cls, s: bytes) -> ListFeatureSetsResponse: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): @@ -94,7 +134,7 @@ class GetFeatureSetsResponse(google___protobuf___message___Message): else: def ClearField(self, field_name: typing_extensions___Literal[u"feature_sets",b"feature_sets"]) -> None: ... -class GetStoresRequest(google___protobuf___message___Message): +class ListStoresRequest(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... class Filter(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @@ -105,7 +145,7 @@ class GetStoresRequest(google___protobuf___message___Message): name : typing___Optional[typing___Text] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> GetStoresRequest.Filter: ... + def FromString(cls, s: bytes) -> ListStoresRequest.Filter: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): @@ -115,14 +155,14 @@ class GetStoresRequest(google___protobuf___message___Message): @property - def filter(self) -> GetStoresRequest.Filter: ... + def filter(self) -> ListStoresRequest.Filter: ... def __init__(self, *, - filter : typing___Optional[GetStoresRequest.Filter] = None, + filter : typing___Optional[ListStoresRequest.Filter] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> GetStoresRequest: ... + def FromString(cls, s: bytes) -> ListStoresRequest: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): @@ -132,7 +172,7 @@ class GetStoresRequest(google___protobuf___message___Message): def HasField(self, field_name: typing_extensions___Literal[u"filter",b"filter"]) -> bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"filter",b"filter"]) -> None: ... -class GetStoresResponse(google___protobuf___message___Message): +class ListStoresResponse(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @property @@ -143,7 +183,7 @@ class GetStoresResponse(google___protobuf___message___Message): store : typing___Optional[typing___Iterable[feast___core___Store_pb2___Store]] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> GetStoresResponse: ... + def FromString(cls, s: bytes) -> ListStoresResponse: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): diff --git a/sdk/python/feast/core/CoreService_pb2_grpc.py b/sdk/python/feast/core/CoreService_pb2_grpc.py index e6e6a98c1e..c4d2808779 100644 --- a/sdk/python/feast/core/CoreService_pb2_grpc.py +++ b/sdk/python/feast/core/CoreService_pb2_grpc.py @@ -19,15 +19,20 @@ def __init__(self, channel): request_serializer=feast_dot_core_dot_CoreService__pb2.GetFeastCoreVersionRequest.SerializeToString, response_deserializer=feast_dot_core_dot_CoreService__pb2.GetFeastCoreVersionResponse.FromString, ) - self.GetFeatureSets = channel.unary_unary( - '/feast.core.CoreService/GetFeatureSets', - request_serializer=feast_dot_core_dot_CoreService__pb2.GetFeatureSetsRequest.SerializeToString, - response_deserializer=feast_dot_core_dot_CoreService__pb2.GetFeatureSetsResponse.FromString, + self.GetFeatureSet = channel.unary_unary( + '/feast.core.CoreService/GetFeatureSet', + request_serializer=feast_dot_core_dot_CoreService__pb2.GetFeatureSetRequest.SerializeToString, + response_deserializer=feast_dot_core_dot_CoreService__pb2.GetFeatureSetResponse.FromString, ) - self.GetStores = channel.unary_unary( - '/feast.core.CoreService/GetStores', - request_serializer=feast_dot_core_dot_CoreService__pb2.GetStoresRequest.SerializeToString, - response_deserializer=feast_dot_core_dot_CoreService__pb2.GetStoresResponse.FromString, + self.ListFeatureSets = channel.unary_unary( + '/feast.core.CoreService/ListFeatureSets', + request_serializer=feast_dot_core_dot_CoreService__pb2.ListFeatureSetsRequest.SerializeToString, + response_deserializer=feast_dot_core_dot_CoreService__pb2.ListFeatureSetsResponse.FromString, + ) + self.ListStores = channel.unary_unary( + '/feast.core.CoreService/ListStores', + request_serializer=feast_dot_core_dot_CoreService__pb2.ListStoresRequest.SerializeToString, + response_deserializer=feast_dot_core_dot_CoreService__pb2.ListStoresResponse.FromString, ) self.ApplyFeatureSet = channel.unary_unary( '/feast.core.CoreService/ApplyFeatureSet', @@ -52,7 +57,14 @@ def GetFeastCoreVersion(self, request, context): context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') - def GetFeatureSets(self, request, context): + def GetFeatureSet(self, request, context): + """Returns a specific feature set + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def ListFeatureSets(self, request, context): """Retrieve feature set details given a filter. Returns all feature sets matching that filter. If none are found, @@ -64,7 +76,7 @@ def GetFeatureSets(self, request, context): context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') - def GetStores(self, request, context): + def ListStores(self, request, context): """Retrieve store details given a filter. Returns all stores matching that filter. If none are found, an empty list will be returned. @@ -104,15 +116,20 @@ def add_CoreServiceServicer_to_server(servicer, server): request_deserializer=feast_dot_core_dot_CoreService__pb2.GetFeastCoreVersionRequest.FromString, response_serializer=feast_dot_core_dot_CoreService__pb2.GetFeastCoreVersionResponse.SerializeToString, ), - 'GetFeatureSets': grpc.unary_unary_rpc_method_handler( - servicer.GetFeatureSets, - request_deserializer=feast_dot_core_dot_CoreService__pb2.GetFeatureSetsRequest.FromString, - response_serializer=feast_dot_core_dot_CoreService__pb2.GetFeatureSetsResponse.SerializeToString, + 'GetFeatureSet': grpc.unary_unary_rpc_method_handler( + servicer.GetFeatureSet, + request_deserializer=feast_dot_core_dot_CoreService__pb2.GetFeatureSetRequest.FromString, + response_serializer=feast_dot_core_dot_CoreService__pb2.GetFeatureSetResponse.SerializeToString, + ), + 'ListFeatureSets': grpc.unary_unary_rpc_method_handler( + servicer.ListFeatureSets, + request_deserializer=feast_dot_core_dot_CoreService__pb2.ListFeatureSetsRequest.FromString, + response_serializer=feast_dot_core_dot_CoreService__pb2.ListFeatureSetsResponse.SerializeToString, ), - 'GetStores': grpc.unary_unary_rpc_method_handler( - servicer.GetStores, - request_deserializer=feast_dot_core_dot_CoreService__pb2.GetStoresRequest.FromString, - response_serializer=feast_dot_core_dot_CoreService__pb2.GetStoresResponse.SerializeToString, + 'ListStores': grpc.unary_unary_rpc_method_handler( + servicer.ListStores, + request_deserializer=feast_dot_core_dot_CoreService__pb2.ListStoresRequest.FromString, + response_serializer=feast_dot_core_dot_CoreService__pb2.ListStoresResponse.SerializeToString, ), 'ApplyFeatureSet': grpc.unary_unary_rpc_method_handler( servicer.ApplyFeatureSet, diff --git a/sdk/python/tests/feast_core_server.py b/sdk/python/tests/feast_core_server.py index 0530d46ad8..45ffc0a0a9 100644 --- a/sdk/python/tests/feast_core_server.py +++ b/sdk/python/tests/feast_core_server.py @@ -7,8 +7,8 @@ GetFeastCoreVersionResponse, ApplyFeatureSetResponse, ApplyFeatureSetRequest, - GetFeatureSetsResponse, - GetFeatureSetsRequest, + ListFeatureSetsResponse, + ListFeatureSetsRequest, ) from feast.core.FeatureSet_pb2 import FeatureSetSpec as FeatureSetSpec from feast.core.Source_pb2 import ( @@ -29,7 +29,7 @@ def __init__(self): def GetFeastCoreVersion(self, request, context): return GetFeastCoreVersionResponse(version="0.3.0") - def GetFeatureSets(self, request: GetFeatureSetsRequest, context): + def ListFeatureSets(self, request: ListFeatureSetsRequest, context): filtered_feature_set_response = [ fs @@ -44,7 +44,7 @@ def GetFeatureSets(self, request: GetFeatureSetsRequest, context): ) ] - return GetFeatureSetsResponse(feature_sets=filtered_feature_set_response) + return ListFeatureSetsResponse(feature_sets=filtered_feature_set_response) def ApplyFeatureSet(self, request: ApplyFeatureSetRequest, context): feature_set = request.feature_set diff --git a/sdk/python/tests/feast_serving_server.py b/sdk/python/tests/feast_serving_server.py index fc23c0a19a..bf40e7edaf 100644 --- a/sdk/python/tests/feast_serving_server.py +++ b/sdk/python/tests/feast_serving_server.py @@ -16,9 +16,9 @@ import sqlite3 from feast.core.CoreService_pb2_grpc import CoreServiceStub from feast.core.CoreService_pb2 import ( - GetFeatureSetsResponse, - GetStoresRequest, - GetStoresResponse, + ListFeatureSetsResponse, + ListStoresRequest, + ListStoresResponse, ) from feast.core import FeatureSet_pb2 as FeatureSetProto import stores @@ -67,8 +67,8 @@ def __connect_core(self, core_url: str): def __get_feature_sets_from_core(self): # Get updated list of feature sets feature_sets = ( - self._core_service_stub.GetFeatureSets - ) # type: GetFeatureSetsResponse + self._core_service_stub.ListFeatureSets + ) # type: ListFeatureSetsResponse # Store each feature set locally for feature_set in list(feature_sets.feature_sets): diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index 1f5f7004cf..a2457a5092 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -30,7 +30,9 @@ from feast.core.Source_pb2 import SourceType, KafkaSourceConfig, Source from feast.core.CoreService_pb2 import ( GetFeastCoreVersionResponse, - GetFeatureSetsResponse, + ListFeatureSetsResponse, + GetFeatureSetResponse, + GetFeatureSetRequest, ) from feast.serving.ServingService_pb2 import ( GetFeastServingInfoResponse, @@ -172,37 +174,32 @@ def test_get_feature_set(self, mock_client, mocker): mocker.patch.object( mock_client._core_service_stub, - "GetFeatureSets", - return_value=GetFeatureSetsResponse( - feature_sets=[ - FeatureSetSpec( - name="my_feature_set", - version=2, - max_age=Duration(seconds=3600), - features=[ - FeatureSpec( - name="my_feature_1", - value_type=ValueProto.ValueType.FLOAT, - ), - FeatureSpec( - name="my_feature_2", - value_type=ValueProto.ValueType.FLOAT, - ), - ], - entities=[ - EntitySpec( - name="my_entity_1", - value_type=ValueProto.ValueType.INT64, - ) - ], - source=Source( - type=SourceType.KAFKA, - kafka_source_config=KafkaSourceConfig( - bootstrap_servers="localhost:9092", topic="topic" - ), + "GetFeatureSet", + return_value=GetFeatureSetResponse( + feature_set=FeatureSetSpec( + name="my_feature_set", + version=2, + max_age=Duration(seconds=3600), + features=[ + FeatureSpec( + name="my_feature_1", value_type=ValueProto.ValueType.FLOAT ), - ) - ] + FeatureSpec( + name="my_feature_2", value_type=ValueProto.ValueType.FLOAT + ), + ], + entities=[ + EntitySpec( + name="my_entity_1", value_type=ValueProto.ValueType.INT64 + ) + ], + source=Source( + type=SourceType.KAFKA, + kafka_source_config=KafkaSourceConfig( + bootstrap_servers="localhost:9092", topic="topic" + ), + ), + ) ), ) @@ -228,33 +225,30 @@ def test_get_batch_features(self, mock_client, mocker): mocker.patch.object( mock_client._core_service_stub, - "GetFeatureSets", - return_value=GetFeatureSetsResponse( - feature_sets=[ - FeatureSetSpec( - name="customer_fs", - version=1, - entities=[ - EntitySpec( - name="customer", value_type=ValueProto.ValueType.INT64 - ), - EntitySpec( - name="transaction", - value_type=ValueProto.ValueType.INT64, - ), - ], - features=[ - FeatureSpec( - name="customer_feature_1", - value_type=ValueProto.ValueType.FLOAT, - ), - FeatureSpec( - name="customer_feature_2", - value_type=ValueProto.ValueType.STRING, - ), - ], - ) - ] + "GetFeatureSet", + return_value=GetFeatureSetResponse( + feature_set=FeatureSetSpec( + name="customer_fs", + version=1, + entities=[ + EntitySpec( + name="customer", value_type=ValueProto.ValueType.INT64 + ), + EntitySpec( + name="transaction", value_type=ValueProto.ValueType.INT64 + ), + ], + features=[ + FeatureSpec( + name="customer_feature_1", + value_type=ValueProto.ValueType.FLOAT, + ), + FeatureSpec( + name="customer_feature_2", + value_type=ValueProto.ValueType.STRING, + ), + ], + ) ), ) @@ -383,8 +377,8 @@ def test_feature_set_ingest_success(self, dataframe, client, mocker): mocker.patch.object( client._core_service_stub, - "GetFeatureSets", - return_value=GetFeatureSetsResponse(feature_sets=[driver_fs.to_proto()]), + "GetFeatureSet", + return_value=GetFeatureSetResponse(feature_set=driver_fs.to_proto()), ) # Ingest data into Feast @@ -452,8 +446,8 @@ def test_feature_set_types_success(self, client, dataframe, mocker): mocker.patch.object( client._core_service_stub, - "GetFeatureSets", - return_value=GetFeatureSetsResponse(feature_sets=[all_types_fs.to_proto()]), + "GetFeatureSet", + return_value=GetFeatureSetResponse(feature_set=all_types_fs.to_proto()), ) # Ingest data into Feast diff --git a/serving/src/main/java/feast/serving/service/CachedSpecService.java b/serving/src/main/java/feast/serving/service/CachedSpecService.java index 4fdf3e147a..df08fb7b38 100644 --- a/serving/src/main/java/feast/serving/service/CachedSpecService.java +++ b/serving/src/main/java/feast/serving/service/CachedSpecService.java @@ -6,9 +6,9 @@ import com.google.common.cache.CacheLoader; import com.google.common.cache.CacheLoader.InvalidCacheLoadException; import com.google.common.cache.LoadingCache; -import feast.core.CoreServiceProto.GetFeatureSetsRequest; -import feast.core.CoreServiceProto.GetFeatureSetsRequest.Filter; -import feast.core.CoreServiceProto.GetFeatureSetsResponse; +import feast.core.CoreServiceProto.ListFeatureSetsRequest; +import feast.core.CoreServiceProto.ListFeatureSetsRequest.Filter; +import feast.core.CoreServiceProto.ListFeatureSetsResponse; import feast.core.CoreServiceProto.UpdateStoreRequest; import feast.core.CoreServiceProto.UpdateStoreResponse; import feast.core.FeatureSetProto.FeatureSetSpec; @@ -75,12 +75,12 @@ public FeatureSetSpec getFeatureSet(String name, int version) { return featureSetSpecCache.get(id); } catch (InvalidCacheLoadException e) { // if not found, try to retrieve from core - GetFeatureSetsRequest request = GetFeatureSetsRequest.newBuilder() + ListFeatureSetsRequest request = ListFeatureSetsRequest.newBuilder() .setFilter(Filter.newBuilder() .setFeatureSetName(name) .setFeatureSetVersion(String.valueOf(version))) .build(); - GetFeatureSetsResponse featureSets = coreService.getFeatureSets(request); + ListFeatureSetsResponse featureSets = coreService.listFeatureSets(request); if (featureSets.getFeatureSetsList().size() == 0) { throw new SpecRetrievalException( String.format( @@ -117,10 +117,10 @@ private Map getFeatureSetSpecMap() { for (Subscription subscription : this.store.getSubscriptionsList()) { try { - GetFeatureSetsResponse featureSetsResponse = coreService - .getFeatureSets(GetFeatureSetsRequest.newBuilder() + ListFeatureSetsResponse featureSetsResponse = coreService + .listFeatureSets(ListFeatureSetsRequest.newBuilder() .setFilter( - GetFeatureSetsRequest.Filter.newBuilder() + ListFeatureSetsRequest.Filter.newBuilder() .setFeatureSetName(subscription.getName()) .setFeatureSetVersion(subscription.getVersion()) ).build()); diff --git a/serving/src/main/java/feast/serving/service/CoreSpecService.java b/serving/src/main/java/feast/serving/service/CoreSpecService.java index 0a6a66b1f4..f37ae61963 100644 --- a/serving/src/main/java/feast/serving/service/CoreSpecService.java +++ b/serving/src/main/java/feast/serving/service/CoreSpecService.java @@ -1,8 +1,8 @@ package feast.serving.service; import feast.core.CoreServiceGrpc; -import feast.core.CoreServiceProto.GetFeatureSetsRequest; -import feast.core.CoreServiceProto.GetFeatureSetsResponse; +import feast.core.CoreServiceProto.ListFeatureSetsRequest; +import feast.core.CoreServiceProto.ListFeatureSetsResponse; import feast.core.CoreServiceProto.UpdateStoreRequest; import feast.core.CoreServiceProto.UpdateStoreResponse; import io.grpc.ManagedChannel; @@ -22,8 +22,8 @@ public CoreSpecService(String feastCoreHost, int feastCorePort) { blockingStub = CoreServiceGrpc.newBlockingStub(channel); } - public GetFeatureSetsResponse getFeatureSets(GetFeatureSetsRequest getFeatureSetsRequest) { - return blockingStub.getFeatureSets(getFeatureSetsRequest); + public ListFeatureSetsResponse listFeatureSets(ListFeatureSetsRequest ListFeatureSetsRequest) { + return blockingStub.listFeatureSets(ListFeatureSetsRequest); } public UpdateStoreResponse updateStore(UpdateStoreRequest updateStoreRequest) { diff --git a/serving/src/test/java/feast/serving/service/CachedSpecServiceTest.java b/serving/src/test/java/feast/serving/service/CachedSpecServiceTest.java index 6b27f1a50e..2b422b609c 100644 --- a/serving/src/test/java/feast/serving/service/CachedSpecServiceTest.java +++ b/serving/src/test/java/feast/serving/service/CachedSpecServiceTest.java @@ -6,8 +6,8 @@ import static org.mockito.MockitoAnnotations.initMocks; import com.google.common.collect.Lists; -import feast.core.CoreServiceProto.GetFeatureSetsRequest; -import feast.core.CoreServiceProto.GetFeatureSetsResponse; +import feast.core.CoreServiceProto.ListFeatureSetsRequest; +import feast.core.CoreServiceProto.ListFeatureSetsResponse; import feast.core.CoreServiceProto.UpdateStoreRequest; import feast.core.CoreServiceProto.UpdateStoreResponse; import feast.core.FeatureSetProto.FeatureSetSpec; @@ -80,18 +80,18 @@ public void setUp() throws IOException { List fs1FeatureSets = Lists .newArrayList(featureSetSpecs.get("fs1:1"), featureSetSpecs.get("fs1:2")); List fs2FeatureSets = Lists.newArrayList(featureSetSpecs.get("fs2:1")); - when(coreService.getFeatureSets(GetFeatureSetsRequest + when(coreService.listFeatureSets(ListFeatureSetsRequest .newBuilder() - .setFilter(GetFeatureSetsRequest.Filter.newBuilder().setFeatureSetName("fs1") + .setFilter(ListFeatureSetsRequest.Filter.newBuilder().setFeatureSetName("fs1") .setFeatureSetVersion(">0").build()) .build())) - .thenReturn(GetFeatureSetsResponse.newBuilder().addAllFeatureSets(fs1FeatureSets).build()); - when(coreService.getFeatureSets(GetFeatureSetsRequest + .thenReturn(ListFeatureSetsResponse.newBuilder().addAllFeatureSets(fs1FeatureSets).build()); + when(coreService.listFeatureSets(ListFeatureSetsRequest .newBuilder() - .setFilter(GetFeatureSetsRequest.Filter.newBuilder().setFeatureSetName("fs2") + .setFilter(ListFeatureSetsRequest.Filter.newBuilder().setFeatureSetName("fs2") .setFeatureSetVersion(">0").build()) .build())) - .thenReturn(GetFeatureSetsResponse.newBuilder().addAllFeatureSets(fs2FeatureSets).build()); + .thenReturn(ListFeatureSetsResponse.newBuilder().addAllFeatureSets(fs2FeatureSets).build()); cachedSpecService = new CachedSpecService(coreService, configFile.toPath()); } From 2cca4376061fb1ddb5333fce52e3adee833880f6 Mon Sep 17 00:00:00 2001 From: Willem Pienaar Date: Sun, 17 Nov 2019 00:29:54 +0800 Subject: [PATCH 2/7] Added basic test for wildcard feature list filter --- .prow/scripts/test-end-to-end.sh | 2 +- CONTRIBUTING.md | 4 +-- .../feast/core/service/SpecServiceTest.java | 25 +++++++++++++++++-- .../feast/charts/feast-serving/values.yaml | 4 +-- infra/charts/feast/values-demo.yaml | 2 +- infra/charts/feast/values.yaml | 4 +-- serving/sample_redis_config.yml | 2 +- .../util/mappers/YamlToProtoMapperTest.java | 4 +-- 8 files changed, 34 insertions(+), 13 deletions(-) diff --git a/.prow/scripts/test-end-to-end.sh b/.prow/scripts/test-end-to-end.sh index 2a4cb9e710..de427879c5 100755 --- a/.prow/scripts/test-end-to-end.sh +++ b/.prow/scripts/test-end-to-end.sh @@ -140,7 +140,7 @@ redis_config: host: localhost port: 6379 subscriptions: - - name: .* + - name: * version: ">0" EOF diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3be8f3fb60..fcee6f3a39 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,7 +64,7 @@ store { name: "SERVING" type: REDIS subscriptions { - name: ".*" + name: "*" version: ">0" } redis_config { @@ -76,7 +76,7 @@ store { name: "WAREHOUSE" type: BIGQUERY subscriptions { - name: ".*" + name: "*" version: ">0" } bigquery_config { diff --git a/core/src/test/java/feast/core/service/SpecServiceTest.java b/core/src/test/java/feast/core/service/SpecServiceTest.java index 4a2c6e5774..85f6a4689c 100644 --- a/core/src/test/java/feast/core/service/SpecServiceTest.java +++ b/core/src/test/java/feast/core/service/SpecServiceTest.java @@ -103,8 +103,8 @@ public void setUp() { .thenReturn(featureSets.subList(0, 3)); when(featureSetRepository.findByName("asd")) .thenReturn(Lists.newArrayList()); - when(featureSetRepository.findByNameWithWildcard("asd")) - .thenReturn(Lists.newArrayList()); + when(featureSetRepository.findByNameWithWildcard("f%")) + .thenReturn(featureSets); Store store1 = newDummyStore("SERVING"); Store store2 = newDummyStore("WAREHOUSE"); @@ -156,6 +156,27 @@ public void shouldGetAllFeatureSetsMatchingNameIfNoVersionProvided() assertThat(actual, equalTo(expected)); } + @Test + public void shouldGetAllFeatureSetsMatchingNameWithWildcardSearch() + throws InvalidProtocolBufferException { + ListFeatureSetsResponse actual = specService + .listFeatureSets(Filter.newBuilder().setFeatureSetName("f*").build()); + List expectedFeatureSets = featureSets.stream() + .filter(fs -> fs.getName().startsWith("f")) + .collect(Collectors.toList()); + List list = new ArrayList<>(); + for (FeatureSet expectedFeatureSet : expectedFeatureSets) { + FeatureSetSpec toProto = expectedFeatureSet.toProto(); + list.add(toProto); + } + ListFeatureSetsResponse expected = ListFeatureSetsResponse + .newBuilder() + .addAllFeatureSets( + list) + .build(); + assertThat(actual, equalTo(expected)); + } + @Test public void shouldGetAllFeatureSetsMatchingVersionIfNoComparator() throws InvalidProtocolBufferException { diff --git a/infra/charts/feast/charts/feast-serving/values.yaml b/infra/charts/feast/charts/feast-serving/values.yaml index ffe8684c6c..d1ea85a9f5 100644 --- a/infra/charts/feast/charts/feast-serving/values.yaml +++ b/infra/charts/feast/charts/feast-serving/values.yaml @@ -91,7 +91,7 @@ application.yaml: # host: localhost # port: 6379 # subscriptions: -# - name: ".*" +# - name: "*" # version: ">0" # # store.yaml: @@ -101,7 +101,7 @@ application.yaml: # project_id: PROJECT_ID # dataset_id: DATASET_ID # subscriptions: -# - name: ".*" +# - name: "*" # version: ">0" # springConfigMountPath is the directory path where application.yaml and diff --git a/infra/charts/feast/values-demo.yaml b/infra/charts/feast/values-demo.yaml index 2c01c9ece1..9212070eb5 100644 --- a/infra/charts/feast/values-demo.yaml +++ b/infra/charts/feast/values-demo.yaml @@ -63,7 +63,7 @@ feast-serving-online: name: redis type: REDIS subscriptions: - - name: ".*" + - name: "*" version: ">0" feast-serving-batch: diff --git a/infra/charts/feast/values.yaml b/infra/charts/feast/values.yaml index d6a5004606..8a0e2ff066 100644 --- a/infra/charts/feast/values.yaml +++ b/infra/charts/feast/values.yaml @@ -68,7 +68,7 @@ feast-serving-online: redis_config: port: 6379 subscriptions: - - name: ".*" + - name: "*" version: ">0" feast-serving-batch: @@ -96,5 +96,5 @@ feast-serving-batch: project_id: PROJECT_ID dataset_id: DATASET_ID subscriptions: - - name: ".*" + - name: "*" version: ">0" diff --git a/serving/sample_redis_config.yml b/serving/sample_redis_config.yml index 8f5f939c9a..1e3864cf0c 100644 --- a/serving/sample_redis_config.yml +++ b/serving/sample_redis_config.yml @@ -4,5 +4,5 @@ redis_config: host: localhost port: 6379 subscriptions: - - name: .* + - name: * version: ">0" diff --git a/serving/src/test/java/feast/serving/util/mappers/YamlToProtoMapperTest.java b/serving/src/test/java/feast/serving/util/mappers/YamlToProtoMapperTest.java index 16ea5f8bb6..88d227d76a 100644 --- a/serving/src/test/java/feast/serving/util/mappers/YamlToProtoMapperTest.java +++ b/serving/src/test/java/feast/serving/util/mappers/YamlToProtoMapperTest.java @@ -20,7 +20,7 @@ public void shouldConvertYamlToProto() throws IOException { + " host: localhost\n" + " port: 6379\n" + "subscriptions:\n" - + "- name: \".*\"\n" + + "- name: \"*\"\n" + " version: \">0\"\n"; Store store = YamlToProtoMapper.yamlToStoreProto(yaml); Store expected = Store.newBuilder() @@ -28,7 +28,7 @@ public void shouldConvertYamlToProto() throws IOException { .setType(StoreType.REDIS) .setRedisConfig(RedisConfig.newBuilder().setHost("localhost").setPort(6379)) .addSubscriptions(Subscription.newBuilder() - .setName(".*") + .setName("*") .setVersion(">0")) .build(); assertThat(store, equalTo(expected)); From 1a9629e5fe1a4befec39ed17816ddbfa04c27677 Mon Sep 17 00:00:00 2001 From: Willem Pienaar Date: Sun, 17 Nov 2019 01:05:34 +0800 Subject: [PATCH 3/7] Remove unnecessary asserts from e2e tests --- tests/e2e/test_e2e.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/e2e/test_e2e.py b/tests/e2e/test_e2e.py index 7de640389c..a24bb4c03d 100644 --- a/tests/e2e/test_e2e.py +++ b/tests/e2e/test_e2e.py @@ -91,8 +91,6 @@ def test_basic_register_feature_set_success(client): "to be committed." ) - assert cust_trans_fs_applied == cust_trans_fs - @pytest.mark.timeout(30) def test_basic_ingest_success(client, basic_dataframe): @@ -241,8 +239,6 @@ def test_all_types_register_feature_set_success(client): "to be committed." ) - assert all_types_fs == all_types_fs_applied - @pytest.mark.timeout(300) def test_all_types_ingest_success(client, all_types_dataframe): @@ -342,8 +338,6 @@ def test_large_volume_register_feature_set_success(client): "to be committed." ) - assert cust_trans_fs_applied == cust_trans_fs - @pytest.mark.timeout(30) @pytest.mark.run(order=2) From da4d00d82885b82ed52d12590e832f2dcd6ea400 Mon Sep 17 00:00:00 2001 From: Willem Pienaar Date: Sun, 17 Nov 2019 10:56:21 +0800 Subject: [PATCH 4/7] Add handling for wildcard subscriptions in ingestion --- .../java/feast/ingestion/utils/SpecUtil.java | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/ingestion/src/main/java/feast/ingestion/utils/SpecUtil.java b/ingestion/src/main/java/feast/ingestion/utils/SpecUtil.java index 5636157ca3..b847e3d420 100644 --- a/ingestion/src/main/java/feast/ingestion/utils/SpecUtil.java +++ b/ingestion/src/main/java/feast/ingestion/utils/SpecUtil.java @@ -15,31 +15,40 @@ import java.util.regex.Pattern; public class SpecUtil { + /** * Get only feature set specs that matches the subscription - * - * @param subscriptions - * @param featureSetSpecs - * @return */ public static List getSubscribedFeatureSets( List subscriptions, List featureSetSpecs) { List subscribed = new ArrayList<>(); for (FeatureSetSpec featureSet : featureSetSpecs) { for (Subscription sub : subscriptions) { - Pattern pattern = Pattern.compile(sub.getName()); + // Convert wildcard to regex + String subName = sub.getName(); + if (!sub.getName().contains(".*")) { + subName = subName.replace("*", ".*"); + } + + // Match feature set name to pattern + Pattern pattern = Pattern.compile(subName); if (!pattern.matcher(featureSet.getName()).matches()) { continue; } - // Supports only subscription version with number only or number with ">" sign - if (sub.getVersion().startsWith(">") && sub.getVersion().length() > 1) { + // If version is empty, match all + if (sub.getVersion().isEmpty()) { + subscribed.add(featureSet); + break; + } else if (sub.getVersion().startsWith(">") && sub.getVersion().length() > 1) { + // if version starts with >, match only those greater than the version number int lowerBoundIncl = Integer.parseInt(sub.getVersion().substring(1)); if (featureSet.getVersion() >= lowerBoundIncl) { subscribed.add(featureSet); break; } } else { + // If a specific version, match that version alone int version = Integer.parseInt(sub.getVersion()); if (featureSet.getVersion() == version) { subscribed.add(featureSet); From a6265a4c9c80c90b9cb744245f0a1ecf942cf4f2 Mon Sep 17 00:00:00 2001 From: Willem Pienaar Date: Sun, 17 Nov 2019 12:49:19 +0800 Subject: [PATCH 5/7] * Fix handling of wildcards in core and update prow * Fix Python Kafka producer bug * Fix comparison between feature sets failing * Fix e2e tests failing due to recreating dataframes --- .prow/scripts/test-end-to-end.sh | 2 +- .../java/feast/core/grpc/CoreServiceImpl.java | 7 ++- sdk/python/feast/client.py | 3 - sdk/python/feast/feature_set.py | 21 ++----- sdk/python/feast/loaders/ingest.py | 14 +++-- serving/sample_redis_config.yml | 2 +- tests/e2e/test_e2e.py | 59 +++++++++++-------- 7 files changed, 56 insertions(+), 52 deletions(-) diff --git a/.prow/scripts/test-end-to-end.sh b/.prow/scripts/test-end-to-end.sh index de427879c5..fde251097f 100755 --- a/.prow/scripts/test-end-to-end.sh +++ b/.prow/scripts/test-end-to-end.sh @@ -140,7 +140,7 @@ redis_config: host: localhost port: 6379 subscriptions: - - name: * + - name: "*" version: ">0" EOF diff --git a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java b/core/src/main/java/feast/core/grpc/CoreServiceImpl.java index dbeb5c1381..81a77a6f45 100644 --- a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java +++ b/core/src/main/java/feast/core/grpc/CoreServiceImpl.java @@ -126,7 +126,12 @@ public void applyFeatureSet( store.getSubscriptionsList().stream() .filter( sub -> { - Pattern p = Pattern.compile(sub.getName()); + String subString = sub.getName(); + if (!subString.contains(".*")) + { + subString = subString.replace("*", ".*"); + } + Pattern p = Pattern.compile(subString); return p.matcher(featureSetName).matches(); }) .collect(Collectors.toList()); diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py index 2466aa46b8..abc9fc25aa 100644 --- a/sdk/python/feast/client.py +++ b/sdk/python/feast/client.py @@ -508,9 +508,6 @@ def ingest( max_workers=max_workers, disable_progress_bar=disable_progress_bar, chunk_size=chunk_size, - producer=Producer( - {"bootstrap.servers": feature_set.get_kafka_source_brokers()} - ), ) else: raise Exception( diff --git a/sdk/python/feast/feature_set.py b/sdk/python/feast/feature_set.py index 061cacb815..550779070f 100644 --- a/sdk/python/feast/feature_set.py +++ b/sdk/python/feast/feature_set.py @@ -77,22 +77,11 @@ def __eq__(self, other): if not isinstance(other, FeatureSet): return NotImplemented - for self_feature in self.features: - for other_feature in other.features: - if self_feature != other_feature: - return False - - for self_entity in self.entities: - for other_entity in other.entities: - if self_entity != other_entity: - return False - - if ( - self.name != other.name - or self.version != other.version - or self.max_age != other.max_age - or self.source != other.source - ): + for key in self.fields.keys(): + if key not in other.fields.keys() or self.fields[key] != other.fields[key]: + return False + + if self.name != other.name or self.max_age != other.max_age: return False return True diff --git a/sdk/python/feast/loaders/ingest.py b/sdk/python/feast/loaders/ingest.py index 21f483955c..74aa22a710 100644 --- a/sdk/python/feast/loaders/ingest.py +++ b/sdk/python/feast/loaders/ingest.py @@ -7,6 +7,7 @@ from multiprocessing import Process, Queue, Pool import pandas as pd from confluent_kafka import Producer +from kafka import KafkaProducer from tqdm import tqdm from feast.type_map import convert_df_to_feature_rows @@ -23,19 +24,22 @@ def _kafka_feature_row_chunk_producer( - feature_row_chunks: Queue, chunk_count: int, producer, topic, progress_bar: tqdm + feature_row_chunk_queue: Queue, chunk_count: int, brokers, topic, progress_bar: tqdm ): processed_chunks = 0 rows_processed = 0 + producer = Producer({"bootstrap.servers": brokers}) while processed_chunks < chunk_count: - if feature_row_chunks.empty(): + if feature_row_chunk_queue.empty(): time.sleep(0.1) else: - feature_rows = feature_row_chunks.get() + feature_rows = feature_row_chunk_queue.get() rows_processed += len(feature_rows) for row in feature_rows: progress_bar.update() producer.produce(topic=topic, value=row.SerializeToString()) + + producer.flush() progress_bar.refresh() processed_chunks += 1 @@ -48,7 +52,6 @@ def _encode_chunk(df: pd.DataFrame, feature_set: FeatureSet): def ingest_kafka( feature_set: FeatureSet, dataframe: pd.DataFrame, - producer: Producer, max_workers: int, chunk_size: int = 5000, disable_progress_bar: bool = False, @@ -73,7 +76,7 @@ def ingest_kafka( args=( chunk_queue, num_chunks, - producer, + feature_set.get_kafka_source_brokers(), feature_set.get_kafka_source_topic(), progress_bar, ), @@ -99,7 +102,6 @@ def ingest_kafka( except Exception as ex: _logger.error(f"Exception occurred: {ex}") finally: - producer.flush() ingestion_process.join() rows_ingested = progress_bar.total progress_bar.close() diff --git a/serving/sample_redis_config.yml b/serving/sample_redis_config.yml index 1e3864cf0c..d6008365e0 100644 --- a/serving/sample_redis_config.yml +++ b/serving/sample_redis_config.yml @@ -4,5 +4,5 @@ redis_config: host: localhost port: 6379 subscriptions: - - name: * + - name: "*" version: ">0" diff --git a/tests/e2e/test_e2e.py b/tests/e2e/test_e2e.py index a24bb4c03d..bcfdafa754 100644 --- a/tests/e2e/test_e2e.py +++ b/tests/e2e/test_e2e.py @@ -55,7 +55,7 @@ def client(core_url, serving_url, allow_dirty): return client -@pytest.fixture() +@pytest.fixture(scope='session') def basic_dataframe(): offset = random.randint(1000, 100000) # ensure a unique key space is used return pd.DataFrame( @@ -69,21 +69,24 @@ def basic_dataframe(): ) -@pytest.mark.timeout(30) +@pytest.mark.timeout(45) +@pytest.mark.run(order=10) def test_basic_register_feature_set_success(client): # Load feature set from file - cust_trans_fs = FeatureSet.from_yaml("basic/cust_trans_fs.yaml") + cust_trans_fs_expected = FeatureSet.from_yaml("basic/cust_trans_fs.yaml") # Register feature set - client.apply(cust_trans_fs) + client.apply(cust_trans_fs_expected) # Feast Core needs some time to fully commit the FeatureSet applied # when there is no existing job yet for the Featureset time.sleep(15) - cust_trans_fs_applied = client.get_feature_set(name="customer_transactions") + cust_trans_fs_actual = client.get_feature_set(name="customer_transactions") + + assert cust_trans_fs_actual == cust_trans_fs_expected - if cust_trans_fs_applied is None: + if cust_trans_fs_actual is None: raise Exception( "Client cannot retrieve 'customer_transactions' FeatureSet " "after registration. Either Feast Core does not save the " @@ -93,6 +96,7 @@ def test_basic_register_feature_set_success(client): @pytest.mark.timeout(30) +@pytest.mark.run(order=11) def test_basic_ingest_success(client, basic_dataframe): cust_trans_fs = client.get_feature_set(name="customer_transactions") @@ -100,7 +104,8 @@ def test_basic_ingest_success(client, basic_dataframe): client.ingest(cust_trans_fs, dataframe=basic_dataframe) -@pytest.mark.timeout(30) +@pytest.mark.timeout(45) +@pytest.mark.run(order=12) def test_basic_retrieve_online_success(client, basic_dataframe): # Poll serving for feature values until the correct values are returned while True: @@ -141,7 +146,7 @@ def test_basic_retrieve_online_success(client, basic_dataframe): break -@pytest.fixture() +@pytest.fixture(scope='session') def all_types_dataframe(): return pd.DataFrame( { @@ -195,10 +200,10 @@ def all_types_dataframe(): ) -@pytest.mark.timeout(30) -@pytest.mark.run(order=1) +@pytest.mark.timeout(45) +@pytest.mark.run(order=20) def test_all_types_register_feature_set_success(client): - all_types_fs = FeatureSet( + all_types_fs_expected = FeatureSet( name="all_types", entities=[Entity(name="user_id", dtype=ValueType.INT64)], features=[ @@ -223,15 +228,17 @@ def test_all_types_register_feature_set_success(client): ) # Register feature set - client.apply(all_types_fs) + client.apply(all_types_fs_expected) # Feast Core needs some time to fully commit the FeatureSet applied # when there is no existing job yet for the Featureset time.sleep(10) - all_types_fs_applied = client.get_feature_set(name="all_types") + all_types_fs_actual = client.get_feature_set(name="all_types") + + assert all_types_fs_actual == all_types_fs_expected - if all_types_fs is None: + if all_types_fs_actual is None: raise Exception( "Client cannot retrieve 'all_types_fs' FeatureSet " "after registration. Either Feast Core does not save the " @@ -241,6 +248,7 @@ def test_all_types_register_feature_set_success(client): @pytest.mark.timeout(300) +@pytest.mark.run(order=21) def test_all_types_ingest_success(client, all_types_dataframe): # Get all_types feature set all_types_fs = client.get_feature_set(name="all_types") @@ -250,6 +258,7 @@ def test_all_types_ingest_success(client, all_types_dataframe): @pytest.mark.timeout(300) +@pytest.mark.run(order=22) def test_all_types_retrieve_online_success(client, all_types_dataframe): # Poll serving for feature values until the correct values are returned @@ -298,9 +307,9 @@ def test_all_types_retrieve_online_success(client, all_types_dataframe): break -@pytest.fixture() +@pytest.fixture(scope='session') def large_volume_dataframe(): - ROW_COUNT = 50000 + ROW_COUNT = 100000 offset = random.randint(1000000, 10000000) # ensure a unique key space customer_data = pd.DataFrame( { @@ -317,20 +326,22 @@ def large_volume_dataframe(): @pytest.mark.timeout(30) -@pytest.mark.run(order=1) +@pytest.mark.run(order=30) def test_large_volume_register_feature_set_success(client): - cust_trans_fs = FeatureSet.from_yaml( + cust_trans_fs_expected = FeatureSet.from_yaml( "large_volume/cust_trans_large_fs.yaml") # Register feature set - client.apply(cust_trans_fs) + client.apply(cust_trans_fs_expected) # Feast Core needs some time to fully commit the FeatureSet applied # when there is no existing job yet for the Featureset time.sleep(10) - cust_trans_fs_applied = client.get_feature_set(name="customer_transactions_large") + cust_trans_fs_actual = client.get_feature_set(name="customer_transactions_large") - if cust_trans_fs is None: + assert cust_trans_fs_actual == cust_trans_fs_expected + + if cust_trans_fs_actual is None: raise Exception( "Client cannot retrieve 'customer_transactions' FeatureSet " "after registration. Either Feast Core does not save the " @@ -339,8 +350,8 @@ def test_large_volume_register_feature_set_success(client): ) -@pytest.mark.timeout(30) -@pytest.mark.run(order=2) +@pytest.mark.timeout(90) +@pytest.mark.run(order=31) def test_large_volume_ingest_success(client, large_volume_dataframe): # Get large volume feature set @@ -351,7 +362,7 @@ def test_large_volume_ingest_success(client, large_volume_dataframe): @pytest.mark.timeout(20) -@pytest.mark.run(order=3) +@pytest.mark.run(order=32) def test_large_volume_retrieve_online_success(client, large_volume_dataframe): # Poll serving for feature values until the correct values are returned while True: From 556c07fbb6b33e49d9fce9e97ee660b7936452be Mon Sep 17 00:00:00 2001 From: Willem Pienaar <6728866+woop@users.noreply.github.com> Date: Sun, 17 Nov 2019 16:57:42 +0800 Subject: [PATCH 6/7] Replace IllegalArgumentException with gRPC RuntimeException Co-Authored-By: David Heryanto --- core/src/main/java/feast/core/service/SpecService.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/feast/core/service/SpecService.java b/core/src/main/java/feast/core/service/SpecService.java index 98c75a5587..2b3403eaba 100644 --- a/core/src/main/java/feast/core/service/SpecService.java +++ b/core/src/main/java/feast/core/service/SpecService.java @@ -96,7 +96,9 @@ public GetFeatureSetResponse getFeatureSet(GetFeatureSetRequest request) .asRuntimeException(); } if (request.getVersion() < 0){ - throw new IllegalArgumentException("Version number cannot be less than 0"); + throw io.grpc.Status.INVALID_ARGUMENT + .withDescription("Version number cannot be less than 0") + .asRuntimeException(); } // Find a list of feature sets with the requested name From f7a15e40893a1219ea78cb7dd1499fd025a0a530 Mon Sep 17 00:00:00 2001 From: Willem Pienaar Date: Sun, 17 Nov 2019 17:08:32 +0800 Subject: [PATCH 7/7] Remove confluent-kafka and replace with kafka-python --- sdk/python/feast/client.py | 1 - sdk/python/feast/loaders/ingest.py | 5 ++--- sdk/python/setup.py | 1 - 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py index abc9fc25aa..277d413689 100644 --- a/sdk/python/feast/client.py +++ b/sdk/python/feast/client.py @@ -19,7 +19,6 @@ import grpc import pandas as pd from feast.loaders.ingest import ingest_kafka -from confluent_kafka import Producer from feast.exceptions import format_grpc_exception from feast.core.CoreService_pb2 import ( diff --git a/sdk/python/feast/loaders/ingest.py b/sdk/python/feast/loaders/ingest.py index 74aa22a710..55cc432098 100644 --- a/sdk/python/feast/loaders/ingest.py +++ b/sdk/python/feast/loaders/ingest.py @@ -6,7 +6,6 @@ from itertools import repeat from multiprocessing import Process, Queue, Pool import pandas as pd -from confluent_kafka import Producer from kafka import KafkaProducer from tqdm import tqdm @@ -28,7 +27,7 @@ def _kafka_feature_row_chunk_producer( ): processed_chunks = 0 rows_processed = 0 - producer = Producer({"bootstrap.servers": brokers}) + producer = KafkaProducer(bootstrap_servers=brokers) while processed_chunks < chunk_count: if feature_row_chunk_queue.empty(): time.sleep(0.1) @@ -37,7 +36,7 @@ def _kafka_feature_row_chunk_producer( rows_processed += len(feature_rows) for row in feature_rows: progress_bar.update() - producer.produce(topic=topic, value=row.SerializeToString()) + producer.send(topic, row.SerializeToString()) producer.flush() progress_bar.refresh() diff --git a/sdk/python/setup.py b/sdk/python/setup.py index 52df8021d9..f7ec4b1e51 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -26,7 +26,6 @@ REQUIRED = [ "click>=7.0", - "confluent-kafka==1.2.0", "google-api-core==1.*", "google-auth==1.*", "google-cloud-bigquery==1.*",