From 12015331d216ffa8973c3f958ceda08c2778c12b Mon Sep 17 00:00:00 2001 From: pyalex Date: Fri, 12 Nov 2021 18:51:11 +0800 Subject: [PATCH 1/3] remove core Signed-off-by: pyalex --- java/README.md | 13 +- java/common-test/pom.xml | 172 ---- .../src/main/java/feast/common/it/BaseIT.java | 182 ---- .../java/feast/common/it/ExternalApp.java | 92 -- .../feast/common/it/SimpleCoreClient.java | 154 --- .../main/java/feast/common/it/SimpleIT.java | 28 - .../main/java/feast/common/util/TestUtil.java | 79 -- java/common/.openapi-generator-ignore | 20 - java/common/pom.xml | 78 +- .../DefaultJwtAuthenticationProvider.java | 61 -- .../authorization/AuthorizationProvider.java | 35 - .../authorization/AuthorizationResult.java | 66 -- .../auth/config/CacheConfiguration.java | 108 --- .../common/auth/config/SecurityConfig.java | 152 --- .../auth/config/SecurityProperties.java | 62 -- .../CoreAuthenticationProperties.java | 56 -- .../credentials/GoogleAuthCredentials.java | 87 -- .../auth/credentials/JwtCallCredentials.java | 49 - .../auth/credentials/OAuthCredentials.java | 130 --- .../http/HttpAuthorizationProvider.java | 125 --- .../keto/KetoAuthorizationProvider.java | 164 ---- .../auth/service/AuthorizationService.java | 63 -- .../feast/common/auth/utils/AuthUtils.java | 53 - .../interceptors/GrpcMessageInterceptor.java | 24 +- .../main/java/feast/common/models/Store.java | 163 ---- .../feast/common/util/KafkaSerialization.java | 68 -- java/common/src/main/resources/api.yaml | 117 --- .../HttpAuthorizationProviderCachingTest.java | 121 --- .../java/feast/common/models/StoreTest.java | 95 -- java/core/.gitignore | 40 - java/core/README.md | 32 - java/core/lombok.config | 1 - java/core/pom.xml | 325 ------- .../main/java/feast/core/CoreApplication.java | 36 - .../feast/core/config/CoreSecurityConfig.java | 60 -- .../feast/core/config/FeastProperties.java | 164 ---- .../java/feast/core/config/JPAConfig.java | 37 - .../feast/core/config/MonitoringConfig.java | 77 -- .../java/feast/core/config/WebMvcConfig.java | 44 - .../feast/core/config/WebSecurityConfig.java | 67 -- .../controller/CoreServiceRestController.java | 139 --- .../RestResponseEntityExceptionHandler.java | 115 --- .../java/feast/core/dao/EntityRepository.java | 33 - .../core/dao/FeatureTableRepository.java | 31 - .../feast/core/dao/ProjectRepository.java | 27 - .../java/feast/core/dao/StoreRepository.java | 25 - .../core/exception/RegistrationException.java | 32 - .../core/exception/RetrievalException.java | 32 - .../java/feast/core/grpc/CoreServiceImpl.java | 410 -------- .../feast/core/grpc/HealthServiceImpl.java | 54 -- .../interceptors/MonitoringInterceptor.java | 56 -- .../java/feast/core/metrics/GrpcMetrics.java | 29 - .../collector/FeastResourceCollector.java | 57 -- .../collector/JVMResourceCollector.java | 76 -- .../core/model/AbstractTimestampEntity.java | 54 -- .../java/feast/core/model/DataSource.java | 239 ----- .../main/java/feast/core/model/EntityV2.java | 150 --- .../java/feast/core/model/FeatureTable.java | 426 --------- .../main/java/feast/core/model/FeatureV2.java | 149 --- .../main/java/feast/core/model/Project.java | 93 -- .../src/main/java/feast/core/model/Store.java | 144 --- .../feast/core/service/ProjectService.java | 81 -- .../java/feast/core/service/SpecService.java | 497 ---------- .../main/java/feast/core/util/StreamUtil.java | 46 - .../java/feast/core/util/TypeConversion.java | 98 -- .../core/validators/DataSourceValidator.java | 89 -- .../core/validators/EntityValidator.java | 37 - .../validators/FeatureTableValidator.java | 84 -- .../java/feast/core/validators/Matchers.java | 108 --- java/core/src/main/resources/application.yml | 103 -- java/core/src/main/resources/banner.txt | 14 - .../resources/db/migration/V1__Baseline.sql | 185 ---- .../migration/V2.1__Many_Stores_Per_Job.sql | 16 - .../V2.2__Subscription_Migration.sql | 3 - .../db/migration/V2.3__Fix_Primary_Keys.sql | 3 - .../db/migration/V2.4__Store_proto.sql | 1 - .../V2.5__Fix_Subscription_MIgration.sql | 14 - .../V2.6__Create_Project_If_Not_Exist.sql | 7 - .../V2.7__Entities_Higher_Level_Concept.sql | 21 - .../db/migration/V2.8__Feature_Tables_API.sql | 66 -- ...__Data_Source_Created_Timestamp_Column.sql | 1 - ...ureSetJobStatus_AND_Feature_Statistics.sql | 124 --- .../V3.0__Feature_Table_Deletion.sql | 1 - java/core/src/main/resources/log4j2.xml | 48 - .../main/resources/templates/bq_training.tmpl | 12 - .../src/main/resources/templates/bq_view.tmpl | 23 - .../java/feast/core/CoreApplicationTest.java | 19 - .../core/annotation/IntegrationTest.java | 19 - .../feast/core/auth/CoreServiceAuthTest.java | 126 --- .../auth/CoreServiceAuthenticationIT.java | 185 ---- .../core/auth/CoreServiceAuthorizationIT.java | 354 ------- .../auth/CoreServiceKetoAuthorizationIT.java | 352 ------- .../java/feast/core/auth/infra/JwtHelper.java | 91 -- .../core/controller/CoreServiceRestIT.java | 214 ----- .../feast/core/logging/CoreLoggingIT.java | 229 ----- .../feast/core/logging/TestLogAppender.java | 68 -- .../feast/core/metrics/CoreMetricsIT.java | 59 -- .../java/feast/core/model/DataSourceTest.java | 60 -- .../core/service/ProjectServiceTest.java | 93 -- .../feast/core/service/SpecServiceIT.java | 903 ------------------ .../feast/core/util/TypeConversionTest.java | 77 -- .../validators/DataSourceValidatorTest.java | 92 -- .../validators/FeatureTableValidatorTest.java | 69 -- .../feast/core/validators/MatchersTest.java | 88 -- .../test/resources/application-it.properties | 26 - .../src/test/resources/application.properties | 53 - .../test/resources/keto/docker-compose.yml | 43 - java/core/src/test/resources/log4j2.xml | 53 - .../org.mockito.plugins.MockMaker | 1 - .../core/src/test/resources/sql/expQuery1.sql | 11 - .../core/src/test/resources/sql/expQuery2.sql | 9 - .../resources/sql/expQueryWithJobIdFilter.sql | 8 - .../sql/expQueryWithNumberAndStringFilter.sql | 8 - .../sql/expQueryWithNumberFilter.sql | 8 - .../sql/expQueryWithStringFilter.sql | 8 - java/docs/architecture.png | Bin 28202 -> 0 bytes java/docs/coverage/java/pom.xml | 6 - java/infra/architecture.png | Bin 30647 -> 0 bytes java/infra/charts/feast-core/Chart.yaml | 10 - java/infra/charts/feast-core/README.md | 68 -- .../charts/feast-core/templates/_helpers.tpl | 45 - .../charts/feast-core/templates/_ingress.yaml | 68 -- .../feast-core/templates/configmap.yaml | 26 - .../feast-core/templates/deployment.yaml | 147 --- .../charts/feast-core/templates/ingress.yaml | 7 - .../charts/feast-core/templates/secret.yaml | 15 - .../charts/feast-core/templates/service.yaml | 41 - java/infra/charts/feast-core/values.yaml | 147 --- java/infra/charts/feast-serving/Chart.yaml | 10 - java/infra/charts/feast-serving/README.md | 71 -- .../feast-serving/templates/_helpers.tpl | 45 - .../feast-serving/templates/_ingress.yaml | 68 -- .../feast-serving/templates/configmap.yaml | 40 - .../feast-serving/templates/deployment.yaml | 148 --- .../feast-serving/templates/ingress.yaml | 7 - .../feast-serving/templates/secret.yaml | 15 - .../feast-serving/templates/service.yaml | 40 - java/infra/charts/feast-serving/values.yaml | 154 --- java/infra/docker/core/Dockerfile | 56 -- java/infra/docker/core/Dockerfile.debug | 0 java/infra/docker/core/Dockerfile.dev | 8 - java/infra/scripts/download-maven-cache.sh | 27 - java/infra/scripts/install-helm.sh | 10 - java/infra/scripts/publish-java-sdk.sh | 72 -- java/infra/scripts/push-helm-charts.sh | 21 - java/infra/scripts/setup-common-functions.sh | 43 - .../scripts/validate-helm-chart-versions.sh | 26 - java/pom.xml | 56 +- java/serving/pom.xml | 75 +- .../serving/config/ContextClosedHandler.java | 36 - .../feast/serving/config/CoreCondition.java | 34 - .../feast/serving/config/FeastProperties.java | 144 +-- .../serving/config/RegistryCondition.java | 36 - .../serving/config/ServingSecurityConfig.java | 105 -- .../config/ServingServiceConfigV2.java | 105 -- .../serving/config/SpecServiceConfig.java | 75 -- .../ServingServiceGRpcController.java | 13 +- .../service/OnlineServingServiceV2.java | 2 +- .../serving/specs/CachedSpecService.java | 219 ----- .../specs/CoreFeatureSpecRetriever.java | 69 -- .../feast/serving/specs/CoreSpecService.java | 68 -- .../specs/RegistryFeatureSpecRetriever.java | 24 +- .../src/main/resources/application.yml | 46 - .../ServingServiceGRpcControllerTest.java | 41 +- .../java/feast/serving/it/AuthTestUtils.java | 195 ---- .../java/feast/serving/it/BaseAuthIT.java | 124 --- .../feast/serving/it/CoreSimpleAPIClient.java | 65 -- .../serving/it/ServingServiceBigTableIT.java | 873 ----------------- .../serving/it/ServingServiceCassandraIT.java | 728 -------------- .../feast/serving/it/ServingServiceIT.java | 505 ---------- .../ServingServiceOauthAuthenticationIT.java | 190 ---- .../ServingServiceOauthAuthorizationIT.java | 227 ----- ...st10IT.java => ServingServiceRedisIT.java} | 109 ++- .../test/java/feast/serving/it/TestUtils.java | 63 +- .../service/CachedSpecServiceTest.java | 145 --- .../service/OnlineServingServiceTest.java | 55 +- .../feast/serving/util}/DataGenerator.java | 2 +- .../test/resources/application-it.properties | 4 - .../docker-compose/core/application-it.yml | 13 - .../docker-compose-bigtable-it.yml | 38 - .../docker-compose-cassandra-it.yml | 31 - .../docker-compose-it-hydra.yml | 54 -- .../docker-compose/docker-compose-it-keto.yml | 44 - .../docker-compose/docker-compose-it.yml | 53 - ...t10-it.yml => docker-compose-redis-it.yml} | 6 - .../feast10/driver_stats.parquet | Bin 34708 -> 0 bytes .../docker-compose/feast10/feature_store.yaml | 1 + .../docker-compose/feast10/materialize.py | 27 +- .../docker-compose/feast10/registry.db | Bin 997 -> 374 bytes .../docker-compose/feast10/requirements.txt | 7 + java/storage/api/pom.xml | 10 +- 191 files changed, 175 insertions(+), 16862 deletions(-) delete mode 100644 java/common-test/pom.xml delete mode 100644 java/common-test/src/main/java/feast/common/it/BaseIT.java delete mode 100644 java/common-test/src/main/java/feast/common/it/ExternalApp.java delete mode 100644 java/common-test/src/main/java/feast/common/it/SimpleCoreClient.java delete mode 100644 java/common-test/src/main/java/feast/common/it/SimpleIT.java delete mode 100644 java/common-test/src/main/java/feast/common/util/TestUtil.java delete mode 100644 java/common/.openapi-generator-ignore delete mode 100644 java/common/src/main/java/feast/common/auth/authentication/DefaultJwtAuthenticationProvider.java delete mode 100644 java/common/src/main/java/feast/common/auth/authorization/AuthorizationProvider.java delete mode 100644 java/common/src/main/java/feast/common/auth/authorization/AuthorizationResult.java delete mode 100644 java/common/src/main/java/feast/common/auth/config/CacheConfiguration.java delete mode 100644 java/common/src/main/java/feast/common/auth/config/SecurityConfig.java delete mode 100644 java/common/src/main/java/feast/common/auth/config/SecurityProperties.java delete mode 100644 java/common/src/main/java/feast/common/auth/credentials/CoreAuthenticationProperties.java delete mode 100644 java/common/src/main/java/feast/common/auth/credentials/GoogleAuthCredentials.java delete mode 100644 java/common/src/main/java/feast/common/auth/credentials/JwtCallCredentials.java delete mode 100644 java/common/src/main/java/feast/common/auth/credentials/OAuthCredentials.java delete mode 100644 java/common/src/main/java/feast/common/auth/providers/http/HttpAuthorizationProvider.java delete mode 100644 java/common/src/main/java/feast/common/auth/providers/keto/KetoAuthorizationProvider.java delete mode 100644 java/common/src/main/java/feast/common/auth/service/AuthorizationService.java delete mode 100644 java/common/src/main/java/feast/common/auth/utils/AuthUtils.java delete mode 100644 java/common/src/main/java/feast/common/models/Store.java delete mode 100644 java/common/src/main/java/feast/common/util/KafkaSerialization.java delete mode 100644 java/common/src/main/resources/api.yaml delete mode 100644 java/common/src/test/java/feast/common/auth/authorization/HttpAuthorizationProviderCachingTest.java delete mode 100644 java/common/src/test/java/feast/common/models/StoreTest.java delete mode 100644 java/core/.gitignore delete mode 100644 java/core/README.md delete mode 100644 java/core/lombok.config delete mode 100644 java/core/pom.xml delete mode 100644 java/core/src/main/java/feast/core/CoreApplication.java delete mode 100644 java/core/src/main/java/feast/core/config/CoreSecurityConfig.java delete mode 100644 java/core/src/main/java/feast/core/config/FeastProperties.java delete mode 100644 java/core/src/main/java/feast/core/config/JPAConfig.java delete mode 100644 java/core/src/main/java/feast/core/config/MonitoringConfig.java delete mode 100644 java/core/src/main/java/feast/core/config/WebMvcConfig.java delete mode 100644 java/core/src/main/java/feast/core/config/WebSecurityConfig.java delete mode 100644 java/core/src/main/java/feast/core/controller/CoreServiceRestController.java delete mode 100644 java/core/src/main/java/feast/core/controller/exception/handler/RestResponseEntityExceptionHandler.java delete mode 100644 java/core/src/main/java/feast/core/dao/EntityRepository.java delete mode 100644 java/core/src/main/java/feast/core/dao/FeatureTableRepository.java delete mode 100644 java/core/src/main/java/feast/core/dao/ProjectRepository.java delete mode 100644 java/core/src/main/java/feast/core/dao/StoreRepository.java delete mode 100644 java/core/src/main/java/feast/core/exception/RegistrationException.java delete mode 100644 java/core/src/main/java/feast/core/exception/RetrievalException.java delete mode 100644 java/core/src/main/java/feast/core/grpc/CoreServiceImpl.java delete mode 100644 java/core/src/main/java/feast/core/grpc/HealthServiceImpl.java delete mode 100644 java/core/src/main/java/feast/core/grpc/interceptors/MonitoringInterceptor.java delete mode 100644 java/core/src/main/java/feast/core/metrics/GrpcMetrics.java delete mode 100644 java/core/src/main/java/feast/core/metrics/collector/FeastResourceCollector.java delete mode 100644 java/core/src/main/java/feast/core/metrics/collector/JVMResourceCollector.java delete mode 100644 java/core/src/main/java/feast/core/model/AbstractTimestampEntity.java delete mode 100644 java/core/src/main/java/feast/core/model/DataSource.java delete mode 100644 java/core/src/main/java/feast/core/model/EntityV2.java delete mode 100644 java/core/src/main/java/feast/core/model/FeatureTable.java delete mode 100644 java/core/src/main/java/feast/core/model/FeatureV2.java delete mode 100644 java/core/src/main/java/feast/core/model/Project.java delete mode 100644 java/core/src/main/java/feast/core/model/Store.java delete mode 100644 java/core/src/main/java/feast/core/service/ProjectService.java delete mode 100644 java/core/src/main/java/feast/core/service/SpecService.java delete mode 100644 java/core/src/main/java/feast/core/util/StreamUtil.java delete mode 100644 java/core/src/main/java/feast/core/util/TypeConversion.java delete mode 100644 java/core/src/main/java/feast/core/validators/DataSourceValidator.java delete mode 100644 java/core/src/main/java/feast/core/validators/EntityValidator.java delete mode 100644 java/core/src/main/java/feast/core/validators/FeatureTableValidator.java delete mode 100644 java/core/src/main/java/feast/core/validators/Matchers.java delete mode 100644 java/core/src/main/resources/application.yml delete mode 100644 java/core/src/main/resources/banner.txt delete mode 100644 java/core/src/main/resources/db/migration/V1__Baseline.sql delete mode 100644 java/core/src/main/resources/db/migration/V2.1__Many_Stores_Per_Job.sql delete mode 100644 java/core/src/main/resources/db/migration/V2.2__Subscription_Migration.sql delete mode 100644 java/core/src/main/resources/db/migration/V2.3__Fix_Primary_Keys.sql delete mode 100644 java/core/src/main/resources/db/migration/V2.4__Store_proto.sql delete mode 100644 java/core/src/main/resources/db/migration/V2.5__Fix_Subscription_MIgration.sql delete mode 100644 java/core/src/main/resources/db/migration/V2.6__Create_Project_If_Not_Exist.sql delete mode 100644 java/core/src/main/resources/db/migration/V2.7__Entities_Higher_Level_Concept.sql delete mode 100644 java/core/src/main/resources/db/migration/V2.8__Feature_Tables_API.sql delete mode 100644 java/core/src/main/resources/db/migration/V2.9__Data_Source_Created_Timestamp_Column.sql delete mode 100644 java/core/src/main/resources/db/migration/V2__RELEASE_0.6_Generalizing_Source_AND_Extending_FeatureSetJobStatus_AND_Feature_Statistics.sql delete mode 100644 java/core/src/main/resources/db/migration/V3.0__Feature_Table_Deletion.sql delete mode 100644 java/core/src/main/resources/log4j2.xml delete mode 100644 java/core/src/main/resources/templates/bq_training.tmpl delete mode 100644 java/core/src/main/resources/templates/bq_view.tmpl delete mode 100644 java/core/src/test/java/feast/core/CoreApplicationTest.java delete mode 100644 java/core/src/test/java/feast/core/annotation/IntegrationTest.java delete mode 100644 java/core/src/test/java/feast/core/auth/CoreServiceAuthTest.java delete mode 100644 java/core/src/test/java/feast/core/auth/CoreServiceAuthenticationIT.java delete mode 100644 java/core/src/test/java/feast/core/auth/CoreServiceAuthorizationIT.java delete mode 100644 java/core/src/test/java/feast/core/auth/CoreServiceKetoAuthorizationIT.java delete mode 100644 java/core/src/test/java/feast/core/auth/infra/JwtHelper.java delete mode 100644 java/core/src/test/java/feast/core/controller/CoreServiceRestIT.java delete mode 100644 java/core/src/test/java/feast/core/logging/CoreLoggingIT.java delete mode 100644 java/core/src/test/java/feast/core/logging/TestLogAppender.java delete mode 100644 java/core/src/test/java/feast/core/metrics/CoreMetricsIT.java delete mode 100644 java/core/src/test/java/feast/core/model/DataSourceTest.java delete mode 100644 java/core/src/test/java/feast/core/service/ProjectServiceTest.java delete mode 100644 java/core/src/test/java/feast/core/service/SpecServiceIT.java delete mode 100644 java/core/src/test/java/feast/core/util/TypeConversionTest.java delete mode 100644 java/core/src/test/java/feast/core/validators/DataSourceValidatorTest.java delete mode 100644 java/core/src/test/java/feast/core/validators/FeatureTableValidatorTest.java delete mode 100644 java/core/src/test/java/feast/core/validators/MatchersTest.java delete mode 100644 java/core/src/test/resources/application-it.properties delete mode 100644 java/core/src/test/resources/application.properties delete mode 100644 java/core/src/test/resources/keto/docker-compose.yml delete mode 100644 java/core/src/test/resources/log4j2.xml delete mode 100644 java/core/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker delete mode 100644 java/core/src/test/resources/sql/expQuery1.sql delete mode 100644 java/core/src/test/resources/sql/expQuery2.sql delete mode 100644 java/core/src/test/resources/sql/expQueryWithJobIdFilter.sql delete mode 100644 java/core/src/test/resources/sql/expQueryWithNumberAndStringFilter.sql delete mode 100644 java/core/src/test/resources/sql/expQueryWithNumberFilter.sql delete mode 100644 java/core/src/test/resources/sql/expQueryWithStringFilter.sql delete mode 100644 java/docs/architecture.png delete mode 100644 java/infra/architecture.png delete mode 100644 java/infra/charts/feast-core/Chart.yaml delete mode 100644 java/infra/charts/feast-core/README.md delete mode 100644 java/infra/charts/feast-core/templates/_helpers.tpl delete mode 100644 java/infra/charts/feast-core/templates/_ingress.yaml delete mode 100644 java/infra/charts/feast-core/templates/configmap.yaml delete mode 100644 java/infra/charts/feast-core/templates/deployment.yaml delete mode 100644 java/infra/charts/feast-core/templates/ingress.yaml delete mode 100644 java/infra/charts/feast-core/templates/secret.yaml delete mode 100644 java/infra/charts/feast-core/templates/service.yaml delete mode 100644 java/infra/charts/feast-core/values.yaml delete mode 100644 java/infra/charts/feast-serving/Chart.yaml delete mode 100644 java/infra/charts/feast-serving/README.md delete mode 100644 java/infra/charts/feast-serving/templates/_helpers.tpl delete mode 100644 java/infra/charts/feast-serving/templates/_ingress.yaml delete mode 100644 java/infra/charts/feast-serving/templates/configmap.yaml delete mode 100644 java/infra/charts/feast-serving/templates/deployment.yaml delete mode 100644 java/infra/charts/feast-serving/templates/ingress.yaml delete mode 100644 java/infra/charts/feast-serving/templates/secret.yaml delete mode 100644 java/infra/charts/feast-serving/templates/service.yaml delete mode 100644 java/infra/charts/feast-serving/values.yaml delete mode 100644 java/infra/docker/core/Dockerfile delete mode 100644 java/infra/docker/core/Dockerfile.debug delete mode 100644 java/infra/docker/core/Dockerfile.dev delete mode 100755 java/infra/scripts/download-maven-cache.sh delete mode 100755 java/infra/scripts/install-helm.sh delete mode 100755 java/infra/scripts/publish-java-sdk.sh delete mode 100755 java/infra/scripts/push-helm-charts.sh delete mode 100755 java/infra/scripts/setup-common-functions.sh delete mode 100755 java/infra/scripts/validate-helm-chart-versions.sh delete mode 100644 java/serving/src/main/java/feast/serving/config/ContextClosedHandler.java delete mode 100644 java/serving/src/main/java/feast/serving/config/CoreCondition.java delete mode 100644 java/serving/src/main/java/feast/serving/config/RegistryCondition.java delete mode 100644 java/serving/src/main/java/feast/serving/config/ServingSecurityConfig.java delete mode 100644 java/serving/src/main/java/feast/serving/config/SpecServiceConfig.java delete mode 100644 java/serving/src/main/java/feast/serving/specs/CachedSpecService.java delete mode 100644 java/serving/src/main/java/feast/serving/specs/CoreFeatureSpecRetriever.java delete mode 100644 java/serving/src/main/java/feast/serving/specs/CoreSpecService.java delete mode 100644 java/serving/src/test/java/feast/serving/it/AuthTestUtils.java delete mode 100644 java/serving/src/test/java/feast/serving/it/BaseAuthIT.java delete mode 100644 java/serving/src/test/java/feast/serving/it/CoreSimpleAPIClient.java delete mode 100644 java/serving/src/test/java/feast/serving/it/ServingServiceBigTableIT.java delete mode 100644 java/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java delete mode 100644 java/serving/src/test/java/feast/serving/it/ServingServiceIT.java delete mode 100644 java/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthenticationIT.java delete mode 100644 java/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthorizationIT.java rename java/serving/src/test/java/feast/serving/it/{ServingServiceFeast10IT.java => ServingServiceRedisIT.java} (55%) delete mode 100644 java/serving/src/test/java/feast/serving/service/CachedSpecServiceTest.java rename java/{common-test/src/main/java/feast/common/it => serving/src/test/java/feast/serving/util}/DataGenerator.java (99%) delete mode 100644 java/serving/src/test/resources/docker-compose/core/application-it.yml delete mode 100644 java/serving/src/test/resources/docker-compose/docker-compose-bigtable-it.yml delete mode 100644 java/serving/src/test/resources/docker-compose/docker-compose-cassandra-it.yml delete mode 100644 java/serving/src/test/resources/docker-compose/docker-compose-it-hydra.yml delete mode 100644 java/serving/src/test/resources/docker-compose/docker-compose-it-keto.yml delete mode 100644 java/serving/src/test/resources/docker-compose/docker-compose-it.yml rename java/serving/src/test/resources/docker-compose/{docker-compose-feast10-it.yml => docker-compose-redis-it.yml} (55%) delete mode 100644 java/serving/src/test/resources/docker-compose/feast10/driver_stats.parquet diff --git a/java/README.md b/java/README.md index 69fca28182..8d6141faa8 100644 --- a/java/README.md +++ b/java/README.md @@ -4,20 +4,13 @@ ### Overview This repository contains the following Feast components. -* Feast Core: The central feature registry used to define and manage entities and features. * Feast Serving: A service used to serve the latest feature values to models. * Feast Java SDK: A client used to retrieve features from Feast Serving. -* Helm Charts: The repository also contains Helm charts to deploy Feast Core and Feast Serving into a Kubernetes cluster. ### Architecture -![](docs/architecture.png) - - -* Feast Core has a dependency on Postgres. -* Feast Serving has a dependency on an online store (Redis) for retrieving features. The process of ingesting data into the online store (Redis) is decoupled from the process of reading from it. Please see [Feast Spark](https://github.com/feast-dev/feast-spark) for more details about ingesting data into the online store. -* Feast Serving has a dependency on Feast Core. -* The Go and Python Clients are not a part of this repository. +Feast Serving has a dependency on an online store (Redis) for retrieving features. +The process of ingesting data into the online store (Redis) is decoupled from the process of reading from it. ### Contributing Guides on Contributing: @@ -26,4 +19,4 @@ Guides on Contributing: - [Development Guide for feast-java (this repository)](CONTRIBUTING.md) ### Installing using Helm -Please see the Helm charts in [charts](infra/charts). +Please see the Helm charts in [charts](https://github.com/feast-dev/feast-helm-charts). diff --git a/java/common-test/pom.xml b/java/common-test/pom.xml deleted file mode 100644 index 0232b4dc7f..0000000000 --- a/java/common-test/pom.xml +++ /dev/null @@ -1,172 +0,0 @@ - - - - 4.0.0 - - - feast-parent - dev.feast - ${revision} - - - Feast Common Test - Feast common module with test utilities - feast-common-test - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.0.0-M4 - - -Xms2048m -Xmx2048m -Djdk.net.URLClassPath.disableClassPathURLCheck=true - - - - - - - - dev.feast - datatypes-java - ${project.version} - compile - - - dev.feast - feast-common - ${project.version} - compile - - - com.google.protobuf - protobuf-java-util - - - - org.projectlombok - lombok - ${lombok.version} - - - javax.validation - validation-api - - - com.google.auto.value - auto-value-annotations - - - com.google.code.gson - gson - - - - - org.slf4j - slf4j-api - - - - org.hamcrest - hamcrest-library - - - org.springframework.boot - spring-boot-test - ${spring.boot.version} - - - org.springframework.boot - spring-boot-test-autoconfigure - - - org.springframework - spring-test - ${spring.version} - - - org.testcontainers - junit-jupiter - 1.15.1 - - - org.testcontainers - postgresql - 1.15.1 - - - org.testcontainers - kafka - 1.15.1 - - - org.junit.jupiter - junit-jupiter-api - 5.6.2 - - - org.springframework.kafka - spring-kafka - - - io.prometheus - simpleclient - 0.8.0 - - - org.apache.commons - commons-lang3 - 3.4 - - - org.awaitility - awaitility - 3.0.0 - - - org.awaitility - awaitility-proxy - 3.0.0 - - - org.mockito - mockito-core - ${mockito.version} - compile - - - io.rest-assured - rest-assured - 4.2.0 - - - io.rest-assured - json-path - 4.2.0 - - - io.rest-assured - xml-path - 4.2.0 - - - diff --git a/java/common-test/src/main/java/feast/common/it/BaseIT.java b/java/common-test/src/main/java/feast/common/it/BaseIT.java deleted file mode 100644 index 8d49b383ba..0000000000 --- a/java/common-test/src/main/java/feast/common/it/BaseIT.java +++ /dev/null @@ -1,182 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.it; - -import io.prometheus.client.CollectorRegistry; -import java.sql.*; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.common.serialization.ByteArrayDeserializer; -import org.apache.kafka.common.serialization.StringDeserializer; -import org.junit.jupiter.api.*; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.context.annotation.Bean; -import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; -import org.springframework.kafka.config.KafkaListenerContainerFactory; -import org.springframework.kafka.core.ConsumerFactory; -import org.springframework.kafka.core.DefaultKafkaConsumerFactory; -import org.springframework.kafka.listener.ConcurrentMessageListenerContainer; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.DynamicPropertyRegistry; -import org.springframework.test.context.DynamicPropertySource; -import org.testcontainers.containers.KafkaContainer; -import org.testcontainers.containers.PostgreSQLContainer; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; - -/** - * Base Integration Test class. Setups postgres and kafka containers. Configures related properties - * and beans. Provides DB related clean up between tests. - */ -@SpringBootTest -@ActiveProfiles("it") -@Testcontainers -@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) -public class BaseIT { - - @Container public static PostgreSQLContainer postgreSQLContainer = new PostgreSQLContainer<>(); - - @Container public static KafkaContainer kafka = new KafkaContainer(); - - /** - * Configure Spring Application to use postgres and kafka rolled out in containers - * - * @param registry - */ - @DynamicPropertySource - static void properties(DynamicPropertyRegistry registry) { - - registry.add("spring.datasource.url", postgreSQLContainer::getJdbcUrl); - registry.add("spring.datasource.username", postgreSQLContainer::getUsername); - registry.add("spring.datasource.password", postgreSQLContainer::getPassword); - registry.add("spring.jpa.hibernate.ddl-auto", () -> "none"); - - registry.add("feast.stream.options.bootstrapServers", kafka::getBootstrapServers); - } - - /** - * SequentialFlow is base class that is supposed to be inherited by @Nested test classes that - * wants to preserve context between test cases. For SequentialFlow databases is being truncated - * only once after all tests passed. - */ - @TestInstance(TestInstance.Lifecycle.PER_CLASS) - public class SequentialFlow { - @AfterAll - public void tearDown() throws Exception { - cleanTables(); - } - } - - /** - * This class must be inherited inside IT Class and annotated with {@link - * org.springframework.boot.test.context.TestConfiguration}. It provides configuration needed to - * communicate with Feast via Kafka - */ - public static class BaseTestConfig { - @Bean - public KafkaListenerContainerFactory> - testListenerContainerFactory(ConsumerFactory consumerFactory) { - ConcurrentKafkaListenerContainerFactory factory = - new ConcurrentKafkaListenerContainerFactory<>(); - factory.setConsumerFactory(consumerFactory); - return factory; - } - - @Bean - public ConsumerFactory testConsumerFactory() { - Map props = new HashMap<>(); - - props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka.getBootstrapServers()); - props.put(ConsumerConfig.GROUP_ID_CONFIG, this.getClass().getName()); - - return new DefaultKafkaConsumerFactory<>( - props, new StringDeserializer(), new ByteArrayDeserializer()); - } - } - - /** - * Truncates all tables in Database (between tests or flows). Retries on deadlock - * - * @throws SQLException when a SQL exception occurs - */ - public static void cleanTables() throws SQLException { - Connection connection = - DriverManager.getConnection( - postgreSQLContainer.getJdbcUrl(), - postgreSQLContainer.getUsername(), - postgreSQLContainer.getPassword()); - - List tableNames = new ArrayList<>(); - Statement statement = connection.createStatement(); - ResultSet rs = - statement.executeQuery( - "SELECT table_name FROM information_schema.tables WHERE table_schema='public'"); - while (rs.next()) { - tableNames.add(rs.getString(1)); - } - - if (tableNames.isEmpty()) { - return; - } - - // retries are needed since truncate require exclusive lock - // and that often leads to Deadlock - // since SpringApp is still running in another thread - int num_retries = 5; - for (int i = 1; i <= num_retries; i++) { - try { - statement = connection.createStatement(); - statement.execute(String.format("truncate %s cascade", String.join(", ", tableNames))); - } catch (SQLException e) { - if (i == num_retries) { - throw e; - } - continue; - } - - break; - } - } - - /** - * Used to determine SequentialFlows - * - * @param testInfo test info - * @return true if test is sequential - */ - public Boolean isSequentialTest(TestInfo testInfo) { - try { - testInfo.getTestClass().get().asSubclass(SequentialFlow.class); - } catch (ClassCastException e) { - return false; - } - return true; - } - - @AfterEach - public void tearDown(TestInfo testInfo) throws Exception { - CollectorRegistry.defaultRegistry.clear(); - - if (!isSequentialTest(testInfo)) { - cleanTables(); - } - } -} diff --git a/java/common-test/src/main/java/feast/common/it/ExternalApp.java b/java/common-test/src/main/java/feast/common/it/ExternalApp.java deleted file mode 100644 index 3fa98b0d50..0000000000 --- a/java/common-test/src/main/java/feast/common/it/ExternalApp.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.it; - -import com.google.auto.value.AutoValue; -import java.util.HashMap; -import java.util.Map; -import org.springframework.boot.Banner; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.builder.SpringApplicationBuilder; -import org.springframework.context.ConfigurableApplicationContext; -import org.springframework.core.env.MapPropertySource; -import org.springframework.core.env.StandardEnvironment; -import org.springframework.util.SocketUtils; -import org.testcontainers.containers.PostgreSQLContainer; - -@AutoValue -public abstract class ExternalApp { - private ConfigurableApplicationContext appContext; - - abstract PostgreSQLContainer getPostgreSQL(); - - abstract String getName(); - - abstract int getGRPCPort(); - - abstract int getWebPort(); - - abstract Map getProperties(); - - abstract Class getSpringApplication(); - - public static Builder builder() { - return new AutoValue_ExternalApp.Builder() - .setProperties(new HashMap<>()) - .setWebPort(SocketUtils.findAvailableTcpPort()); - } - - @AutoValue.Builder - public interface Builder { - Builder setSpringApplication(Class app); - - Builder setName(String name); - - Builder setPostgreSQL(PostgreSQLContainer psql); - - Builder setGRPCPort(int port); - - Builder setWebPort(int port); - - Builder setProperties(Map properties); - - ExternalApp build(); - } - - public void start() { - HashMap properties = new HashMap<>(getProperties()); - properties.put("spring.datasource.url", getPostgreSQL().getJdbcUrl()); - properties.put("spring.datasource.username", getPostgreSQL().getUsername()); - properties.put("spring.datasource.password", getPostgreSQL().getPassword()); - properties.put("grpc.server.port", getGRPCPort()); - properties.put("server.port", getWebPort()); - - StandardEnvironment env = new StandardEnvironment(); - env.setDefaultProfiles(getName()); - env.getPropertySources().addFirst(new MapPropertySource("primary", properties)); - - appContext = - new SpringApplicationBuilder(getSpringApplication()) - .environment(env) - .bannerMode(Banner.Mode.OFF) - .run(); - } - - public void stop() { - SpringApplication.exit(appContext); - } -} diff --git a/java/common-test/src/main/java/feast/common/it/SimpleCoreClient.java b/java/common-test/src/main/java/feast/common/it/SimpleCoreClient.java deleted file mode 100644 index 11fa6715dc..0000000000 --- a/java/common-test/src/main/java/feast/common/it/SimpleCoreClient.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.it; - -import feast.proto.core.*; -import feast.proto.core.CoreServiceProto.ApplyFeatureTableRequest; -import feast.proto.core.FeatureTableProto.FeatureTableSpec; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -public class SimpleCoreClient { - private CoreServiceGrpc.CoreServiceBlockingStub stub; - - public SimpleCoreClient(CoreServiceGrpc.CoreServiceBlockingStub stub) { - this.stub = stub; - } - - public CoreServiceProto.ApplyEntityResponse simpleApplyEntity( - String projectName, EntityProto.EntitySpecV2 spec) { - return stub.applyEntity( - CoreServiceProto.ApplyEntityRequest.newBuilder() - .setProject(projectName) - .setSpec(spec) - .build()); - } - - public List simpleListEntities(String projectName) { - return stub.listEntities( - CoreServiceProto.ListEntitiesRequest.newBuilder() - .setFilter( - CoreServiceProto.ListEntitiesRequest.Filter.newBuilder() - .setProject(projectName) - .build()) - .build()) - .getEntitiesList(); - } - - public List simpleListEntities( - String projectName, Map labels) { - return stub.listEntities( - CoreServiceProto.ListEntitiesRequest.newBuilder() - .setFilter( - CoreServiceProto.ListEntitiesRequest.Filter.newBuilder() - .setProject(projectName) - .putAllLabels(labels) - .build()) - .build()) - .getEntitiesList(); - } - - public List simpleListEntities( - CoreServiceProto.ListEntitiesRequest.Filter filter) { - return stub.listEntities( - CoreServiceProto.ListEntitiesRequest.newBuilder().setFilter(filter).build()) - .getEntitiesList(); - } - - public List simpleListFeatureTables( - CoreServiceProto.ListFeatureTablesRequest.Filter filter) { - return stub.listFeatureTables( - CoreServiceProto.ListFeatureTablesRequest.newBuilder().setFilter(filter).build()) - .getTablesList(); - } - - public EntityProto.Entity simpleGetEntity(String projectName, String name) { - return stub.getEntity( - CoreServiceProto.GetEntityRequest.newBuilder() - .setName(name) - .setProject(projectName) - .build()) - .getEntity(); - } - - public FeatureTableProto.FeatureTable simpleGetFeatureTable(String projectName, String name) { - return stub.getFeatureTable( - CoreServiceProto.GetFeatureTableRequest.newBuilder() - .setName(name) - .setProject(projectName) - .build()) - .getTable(); - } - - public Map simpleListFeatures( - String projectName, Map labels, List entities) { - return stub.listFeatures( - CoreServiceProto.ListFeaturesRequest.newBuilder() - .setFilter( - CoreServiceProto.ListFeaturesRequest.Filter.newBuilder() - .setProject(projectName) - .addAllEntities(entities) - .putAllLabels(labels) - .build()) - .build()) - .getFeaturesMap(); - } - - public Map simpleListFeatures( - String projectName, String... entities) { - return simpleListFeatures(projectName, Collections.emptyMap(), Arrays.asList(entities)); - } - - public CoreServiceProto.UpdateStoreResponse updateStore(StoreProto.Store store) { - return stub.updateStore( - CoreServiceProto.UpdateStoreRequest.newBuilder().setStore(store).build()); - } - - public void createProject(String name) { - stub.createProject(CoreServiceProto.CreateProjectRequest.newBuilder().setName(name).build()); - } - - public void archiveProject(String name) { - stub.archiveProject(CoreServiceProto.ArchiveProjectRequest.newBuilder().setName(name).build()); - } - - public String getFeastCoreVersion() { - return stub.getFeastCoreVersion( - CoreServiceProto.GetFeastCoreVersionRequest.getDefaultInstance()) - .getVersion(); - } - - public FeatureTableProto.FeatureTable applyFeatureTable( - String projectName, FeatureTableSpec spec) { - return stub.applyFeatureTable( - ApplyFeatureTableRequest.newBuilder() - .setProject(projectName) - .setTableSpec(spec) - .build()) - .getTable(); - } - - public void deleteFeatureTable(String projectName, String featureTableName) { - stub.deleteFeatureTable( - CoreServiceProto.DeleteFeatureTableRequest.newBuilder() - .setProject(projectName) - .setName(featureTableName) - .build()); - } -} diff --git a/java/common-test/src/main/java/feast/common/it/SimpleIT.java b/java/common-test/src/main/java/feast/common/it/SimpleIT.java deleted file mode 100644 index e8c5dff38c..0000000000 --- a/java/common-test/src/main/java/feast/common/it/SimpleIT.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.it; - -import org.junit.jupiter.api.Test; -import org.springframework.boot.test.context.SpringBootTest; - -@SpringBootTest() -public class SimpleIT extends BaseIT { - @Test - public void test() { - assert true; - } -} diff --git a/java/common-test/src/main/java/feast/common/util/TestUtil.java b/java/common-test/src/main/java/feast/common/util/TestUtil.java deleted file mode 100644 index 49e6cc7f86..0000000000 --- a/java/common-test/src/main/java/feast/common/util/TestUtil.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.util; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import feast.common.logging.AuditLogger; -import feast.common.logging.config.LoggingProperties; -import feast.proto.core.FeatureProto.FeatureSpecV2; -import feast.proto.core.FeatureTableProto.FeatureTableSpec; -import java.util.Comparator; -import java.util.stream.Collectors; -import org.springframework.boot.info.BuildProperties; - -public class TestUtil { - /** Setup the audit logger. This call is required to use the audit logger when testing. */ - public static void setupAuditLogger() { - LoggingProperties.AuditLogProperties properties = new LoggingProperties.AuditLogProperties(); - properties.setEnabled(true); - LoggingProperties loggingProperties = new LoggingProperties(); - loggingProperties.setAudit(properties); - - BuildProperties buildProperties = mock(BuildProperties.class); - when(buildProperties.getArtifact()).thenReturn("feast-core"); - when(buildProperties.getVersion()).thenReturn("0.6"); - - new AuditLogger(loggingProperties, buildProperties); - } - - /** - * Compare if two Feature Table specs are equal. Disregards order of features/entities in spec. - * - * @param spec one spec - * @param otherSpec the other spec - * @return true if specs equal - */ - public static boolean compareFeatureTableSpec(FeatureTableSpec spec, FeatureTableSpec otherSpec) { - spec = - spec.toBuilder() - .clearFeatures() - .addAllFeatures( - spec.getFeaturesList().stream() - .sorted(Comparator.comparing(FeatureSpecV2::getName)) - .collect(Collectors.toSet())) - .clearEntities() - .addAllEntities(spec.getEntitiesList().stream().sorted().collect(Collectors.toSet())) - .build(); - - otherSpec = - otherSpec - .toBuilder() - .clearFeatures() - .addAllFeatures( - otherSpec.getFeaturesList().stream() - .sorted(Comparator.comparing(FeatureSpecV2::getName)) - .collect(Collectors.toSet())) - .clearEntities() - .addAllEntities( - otherSpec.getEntitiesList().stream().sorted().collect(Collectors.toSet())) - .build(); - - return spec.equals(otherSpec); - } -} diff --git a/java/common/.openapi-generator-ignore b/java/common/.openapi-generator-ignore deleted file mode 100644 index 6b177032ba..0000000000 --- a/java/common/.openapi-generator-ignore +++ /dev/null @@ -1,20 +0,0 @@ -settings.gradle -README.md -pom.xml -gradle -git_push.sh -build.sbt -build.gradle -.travis* -.gitignore -src/main/resources/api.yaml -gradle* -gradle/* -gradle-wrapper.* -gradle** -gradle/ -src/main/java/feast/auth/providers/http/HttpAuthorizationProvider.java -src/main/java/feast/auth/providers/http/ketoadaptor/api/CheckAccessApiController.java -src/main/java/feast/auth/providers/http/ketoadaptor/api/KetoAuth.java -src/main/AndroidManifest.xml -.openapi-generator/ \ No newline at end of file diff --git a/java/common/pom.xml b/java/common/pom.xml index 99008e139f..f0d79b1732 100644 --- a/java/common/pom.xml +++ b/java/common/pom.xml @@ -102,7 +102,7 @@ org.hibernate.validator hibernate-validator - + @@ -121,42 +121,6 @@ org.springframework.security spring-security-oauth2-jose - - com.google.auth - google-auth-library-oauth2-http - - - sh.ory.keto - keto-client - 0.5.7-alpha.1.pre.0 - - - - - org.openapitools - jackson-databind-nullable - 0.1.0 - - - io.swagger - swagger-annotations - - - com.squareup.okhttp3 - okhttp - - - com.squareup.okhttp3 - logging-interceptor - - - io.springfox - springfox-swagger2 - - - io.springfox - springfox-swagger-ui - @@ -192,15 +156,11 @@ hamcrest-library test - - org.apache.kafka - kafka-clients - 2.5.0 - + junit junit - 4.12 + 4.13.2 org.springframework @@ -227,42 +187,10 @@ - - org.openapitools - openapi-generator-maven-plugin - 4.3.1 - - - client - - generate - - - ${project.basedir}/src/main/resources/api.yaml - java - ${feast.auth.providers.http.client.package.name} - ${feast.auth.providers.http.client.package.name}.model - ${feast.auth.providers.http.client.package.name}.api - ${feast.auth.providers.http.client.package.name}.invoker - - ${project.groupId} - ${project.artifactId} - ${project.version} - true - java8 - Apache 2.0 - https://www.apache.org/licenses/LICENSE-2.0 - ${project.build.directory}/generated-sources - - - - - org.apache.maven.plugins maven-javadoc-plugin - ${feast.auth.providers.http.client.package.name}.* diff --git a/java/common/src/main/java/feast/common/auth/authentication/DefaultJwtAuthenticationProvider.java b/java/common/src/main/java/feast/common/auth/authentication/DefaultJwtAuthenticationProvider.java deleted file mode 100644 index 2b5c89f66e..0000000000 --- a/java/common/src/main/java/feast/common/auth/authentication/DefaultJwtAuthenticationProvider.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.auth.authentication; - -import java.util.Map; -import org.springframework.security.authentication.AuthenticationProvider; -import org.springframework.security.core.Authentication; -import org.springframework.security.core.AuthenticationException; -import org.springframework.security.oauth2.jwt.NimbusJwtDecoder; -import org.springframework.security.oauth2.server.resource.authentication.JwtAuthenticationConverter; -import org.springframework.security.oauth2.server.resource.authentication.JwtAuthenticationProvider; - -/** Json Web Token Authentication Provider used to validate incoming requests to Feast Core. */ -public class DefaultJwtAuthenticationProvider implements AuthenticationProvider { - - private JwtAuthenticationProvider authProvider; - - /** - * @param options String K/V pair of options to initialize the AuthenticationProvider with. Only - * one option is currently configurable, the jwkEndpointURI. - */ - public DefaultJwtAuthenticationProvider(Map options) { - // Endpoint used to retrieve certificates to validate JWT token - String jwkEndpointURI = options.get("jwkEndpointURI"); - - // Provide a custom endpoint to retrieve certificates - authProvider = - new JwtAuthenticationProvider(NimbusJwtDecoder.withJwkSetUri(jwkEndpointURI).build()); - authProvider.setJwtAuthenticationConverter(new JwtAuthenticationConverter()); - } - - /** - * Authenticate a request based on its Spring Security Authentication object - * - * @param authentication Authentication object which contains a JWT to validate - * @return Returns the same authentication object after authentication - */ - @Override - public Authentication authenticate(Authentication authentication) throws AuthenticationException { - return authProvider.authenticate(authentication); - } - - @Override - public boolean supports(Class aClass) { - return authProvider.supports(aClass); - } -} diff --git a/java/common/src/main/java/feast/common/auth/authorization/AuthorizationProvider.java b/java/common/src/main/java/feast/common/auth/authorization/AuthorizationProvider.java deleted file mode 100644 index e4e398883e..0000000000 --- a/java/common/src/main/java/feast/common/auth/authorization/AuthorizationProvider.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.auth.authorization; - -import org.springframework.security.core.Authentication; - -/** - * AuthorizationProvider is the base interface that each AuthorizationProvider needs to implement in - * order to authorize requests to Feast Core - */ -public interface AuthorizationProvider { - - /** - * Validates whether a user is allowed access to a project - * - * @param projectId Id of the Feast project - * @param authentication Spring Security Authentication object - * @return AuthorizationResult result of authorization query - */ - AuthorizationResult checkAccessToProject(String projectId, Authentication authentication); -} diff --git a/java/common/src/main/java/feast/common/auth/authorization/AuthorizationResult.java b/java/common/src/main/java/feast/common/auth/authorization/AuthorizationResult.java deleted file mode 100644 index 897cef6d37..0000000000 --- a/java/common/src/main/java/feast/common/auth/authorization/AuthorizationResult.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.auth.authorization; - -import java.util.Optional; -import javax.annotation.Nullable; -import lombok.AllArgsConstructor; -import lombok.Getter; - -/** - * Implementation of AuthorizationProvider will return AuthorizationResult after validating incoming - * requests to Feast Core. AuthorizationResult provides methods to check if user is authorized to - * perform an action or not. - */ -@Getter -@AllArgsConstructor -public class AuthorizationResult { - - /** - * Method to create AuthorizationResult Object. - * - * @param allowed True If user is authorized, False otherwise. - * @param failureReason Reason for authorization failure, if any - * @return AuthorizationResult Object. - */ - public static AuthorizationResult create( - @Nullable boolean allowed, @Nullable String failureReason) { - return new AuthorizationResult(allowed, Optional.ofNullable(failureReason)); - } - - /** - * Method to create failed AuthorizationResult Object. - * - * @param failureReason Reason for authorization failure, if any or Null - * @return AuthorizationResult Object. - */ - public static AuthorizationResult failed(@Nullable String failureReason) { - return new AuthorizationResult(false, Optional.ofNullable(failureReason)); - } - - /** - * Method to create Success AuthorizationResult Object. - * - * @return AuthorizationResult Object. - */ - public static AuthorizationResult success() { - return new AuthorizationResult(true, Optional.empty()); - } - - private boolean allowed; - private Optional failureReason; -} diff --git a/java/common/src/main/java/feast/common/auth/config/CacheConfiguration.java b/java/common/src/main/java/feast/common/auth/config/CacheConfiguration.java deleted file mode 100644 index 7731906b88..0000000000 --- a/java/common/src/main/java/feast/common/auth/config/CacheConfiguration.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.auth.config; - -import com.google.common.cache.CacheBuilder; -import feast.common.auth.utils.AuthUtils; -import java.lang.reflect.Method; -import java.util.concurrent.TimeUnit; -import lombok.Getter; -import lombok.Setter; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.cache.Cache; -import org.springframework.cache.CacheManager; -import org.springframework.cache.annotation.CachingConfigurer; -import org.springframework.cache.annotation.EnableCaching; -import org.springframework.cache.concurrent.ConcurrentMapCache; -import org.springframework.cache.concurrent.ConcurrentMapCacheManager; -import org.springframework.cache.interceptor.CacheErrorHandler; -import org.springframework.cache.interceptor.CacheResolver; -import org.springframework.cache.interceptor.KeyGenerator; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.security.core.Authentication; - -/** CacheConfiguration class defines Cache settings for HttpAuthorizationProvider class. */ -@Configuration -@EnableCaching -@Setter -@Getter -public class CacheConfiguration implements CachingConfigurer { - - private static final int CACHE_SIZE = 10000; - - public static int TTL = 60; - - public static final String AUTHORIZATION_CACHE = "authorization"; - - @Autowired SecurityProperties securityProperties; - - @Bean - public CacheManager cacheManager() { - ConcurrentMapCacheManager cacheManager = - new ConcurrentMapCacheManager(AUTHORIZATION_CACHE) { - - @Override - protected Cache createConcurrentMapCache(final String name) { - return new ConcurrentMapCache( - name, - CacheBuilder.newBuilder() - .expireAfterWrite(TTL, TimeUnit.SECONDS) - .maximumSize(CACHE_SIZE) - .build() - .asMap(), - false); - } - }; - - return cacheManager; - } - - /* - * KeyGenerator used by {@link Cacheable} for caching authorization requests. - * Key format : checkAccessToProject-- - */ - @Bean - public KeyGenerator authKeyGenerator() { - return (Object target, Method method, Object... params) -> { - String projectId = (String) params[0]; - Authentication authentication = (Authentication) params[1]; - String subject = - AuthUtils.getSubjectFromAuth( - authentication, - securityProperties.getAuthorization().getOptions().get("subjectClaim")); - return String.format("%s-%s-%s", method.getName(), projectId, subject); - }; - } - - @Override - public CacheResolver cacheResolver() { - // TODO Auto-generated method stub - return null; - } - - @Override - public KeyGenerator keyGenerator() { - return null; - } - - @Override - public CacheErrorHandler errorHandler() { - // TODO Auto-generated method stub - return null; - } -} diff --git a/java/common/src/main/java/feast/common/auth/config/SecurityConfig.java b/java/common/src/main/java/feast/common/auth/config/SecurityConfig.java deleted file mode 100644 index 94a4018105..0000000000 --- a/java/common/src/main/java/feast/common/auth/config/SecurityConfig.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.auth.config; - -import feast.common.auth.authentication.DefaultJwtAuthenticationProvider; -import feast.common.auth.authorization.AuthorizationProvider; -import feast.common.auth.providers.http.HttpAuthorizationProvider; -import feast.common.auth.providers.keto.KetoAuthorizationProvider; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import net.devh.boot.grpc.server.security.authentication.BearerAuthenticationReader; -import net.devh.boot.grpc.server.security.authentication.GrpcAuthenticationReader; -import net.devh.boot.grpc.server.security.check.AccessPredicateVoter; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.security.access.AccessDecisionManager; -import org.springframework.security.access.AccessDecisionVoter; -import org.springframework.security.access.vote.UnanimousBased; -import org.springframework.security.authentication.AuthenticationManager; -import org.springframework.security.authentication.AuthenticationProvider; -import org.springframework.security.authentication.ProviderManager; -import org.springframework.security.oauth2.server.resource.BearerTokenAuthenticationToken; - -@Configuration -public class SecurityConfig { - - private final SecurityProperties securityProperties; - - public SecurityConfig(SecurityProperties securityProperties) { - this.securityProperties = securityProperties; - } - - /** - * Initializes an AuthenticationManager if authentication has been enabled. - * - * @return AuthenticationManager - */ - @Bean - @ConditionalOnProperty(prefix = "feast.security.authentication", name = "enabled") - AuthenticationManager authenticationManager() { - final List providers = new ArrayList<>(); - - if (securityProperties.getAuthentication().isEnabled()) { - switch (securityProperties.getAuthentication().getProvider()) { - case "jwt": - providers.add( - new DefaultJwtAuthenticationProvider( - securityProperties.getAuthentication().getOptions())); - break; - default: - throw new IllegalArgumentException( - "Please configure an Authentication Provider if you have enabled authentication."); - } - } - return new ProviderManager(providers); - } - - /** - * Creates an AuthenticationReader that the AuthenticationManager will use to authenticate - * requests - * - * @return GrpcAuthenticationReader - */ - @Bean - @ConditionalOnProperty(prefix = "feast.security.authentication", name = "enabled") - GrpcAuthenticationReader authenticationReader() { - return new BearerAuthenticationReader(BearerTokenAuthenticationToken::new); - } - - /** - * Creates an AccessDecisionManager if authorization is enabled. This object determines the policy - * used to make authorization decisions. - * - * @return AccessDecisionManager - */ - @Bean - @ConditionalOnProperty(prefix = "feast.security.authorization", name = "enabled") - AccessDecisionManager accessDecisionManager() { - final List> voters = new ArrayList<>(); - voters.add(new AccessPredicateVoter()); - return new UnanimousBased(voters); - } - - /** - * Creates an AuthorizationProvider based on Feast configuration. This provider is available - * through the security service. - * - * @return AuthorizationProvider used to validate access to Feast resources. - */ - @Bean - @ConditionalOnProperty(prefix = "feast.security.authorization", name = "enabled") - AuthorizationProvider authorizationProvider() { - if (securityProperties.getAuthentication().isEnabled() - && securityProperties.getAuthorization().isEnabled()) { - // Merge authentication and authorization options to create HttpAuthorizationProvider. - Map options = securityProperties.getAuthorization().getOptions(); - - options.putAll(securityProperties.getAuthentication().getOptions()); - switch (securityProperties.getAuthorization().getProvider()) { - case "http": - return new HttpAuthorizationProvider(options); - case "keto": - String subjectClaim = - options.get(SecurityProperties.AuthenticationProperties.SUBJECT_CLAIM); - String flavor = options.get("flavor"); - String action = options.get("action"); - String subjectPrefix = options.get("subjectPrefix"); - String resourcePrefix = options.get("resourcePrefix"); - - KetoAuthorizationProvider.Builder builder = - new KetoAuthorizationProvider.Builder(options.get("authorizationUrl")); - if (subjectClaim != null) { - builder = builder.withSubjectClaim(subjectClaim); - } - if (flavor != null) { - builder = builder.withFlavor(flavor); - } - if (action != null) { - builder = builder.withAction(action); - } - if (subjectPrefix != null) { - builder = builder.withSubjectPrefix(subjectPrefix); - } - if (resourcePrefix != null) { - builder = builder.withResourcePrefix(resourcePrefix); - } - - return builder.build(); - default: - throw new IllegalArgumentException( - "Please configure an Authorization Provider if you have enabled authorization."); - } - } - return null; - } -} diff --git a/java/common/src/main/java/feast/common/auth/config/SecurityProperties.java b/java/common/src/main/java/feast/common/auth/config/SecurityProperties.java deleted file mode 100644 index f48d734b23..0000000000 --- a/java/common/src/main/java/feast/common/auth/config/SecurityProperties.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.auth.config; - -import feast.common.validators.OneOfStrings; -import java.util.Map; -import lombok.Getter; -import lombok.Setter; - -@Getter -@Setter -public class SecurityProperties { - private AuthenticationProperties authentication; - private AuthorizationProperties authorization; - - // Bypass Authentication and Authorization at all HTTP endpoints at /api/v1 - private boolean disableRestControllerAuth; - - @Getter - @Setter - public static class AuthenticationProperties { - // Enable authentication - private boolean enabled; - - // Named authentication provider to use - @OneOfStrings({"jwt"}) - private String provider; - - // K/V options to initialize the provider with - private Map options; - // Key for Subject Claim option which sets the name of the subject claim field in tokens. - public static final String SUBJECT_CLAIM = "subjectClaim"; - } - - @Getter - @Setter - public static class AuthorizationProperties { - // Enable authorization. Authentication must be enabled if authorization is enabled. - private boolean enabled; - - // Named authorization provider to use. - @OneOfStrings({"none", "http", "keto"}) - private String provider; - - // K/V options to initialize the provider with - private Map options; - } -} diff --git a/java/common/src/main/java/feast/common/auth/credentials/CoreAuthenticationProperties.java b/java/common/src/main/java/feast/common/auth/credentials/CoreAuthenticationProperties.java deleted file mode 100644 index aa317cf8b1..0000000000 --- a/java/common/src/main/java/feast/common/auth/credentials/CoreAuthenticationProperties.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.auth.credentials; - -import feast.common.validators.OneOfStrings; -import java.util.Map; - -public class CoreAuthenticationProperties { - // needs to be set to true if authentication is enabled on core - private boolean enabled; - - // authentication provider to use - @OneOfStrings({"google", "oauth"}) - private String provider; - - // K/V options to initialize the provider. - Map options; - - public boolean isEnabled() { - return enabled; - } - - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } - - public String getProvider() { - return provider; - } - - public void setProvider(String provider) { - this.provider = provider; - } - - public Map getOptions() { - return options; - } - - public void setOptions(Map options) { - this.options = options; - } -} diff --git a/java/common/src/main/java/feast/common/auth/credentials/GoogleAuthCredentials.java b/java/common/src/main/java/feast/common/auth/credentials/GoogleAuthCredentials.java deleted file mode 100644 index 57aafa2724..0000000000 --- a/java/common/src/main/java/feast/common/auth/credentials/GoogleAuthCredentials.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.auth.credentials; - -import static io.grpc.Metadata.ASCII_STRING_MARSHALLER; - -import com.google.auth.oauth2.IdTokenCredentials; -import com.google.auth.oauth2.ServiceAccountCredentials; -import io.grpc.CallCredentials; -import io.grpc.Metadata; -import io.grpc.Status; -import java.io.IOException; -import java.util.Arrays; -import java.util.Map; -import java.util.concurrent.Executor; - -/** - * GoogleAuthCredentials provides a Google OIDC ID token for making authenticated gRPC calls. Uses - * Google Application - * Default credentials to obtain the OIDC token used for authentication. The given token will be - * passed as authorization bearer token when making calls. - */ -public class GoogleAuthCredentials extends CallCredentials { - private final IdTokenCredentials credentials; - private static final String BEARER_TYPE = "Bearer"; - private static final Metadata.Key AUTHORIZATION_METADATA_KEY = - Metadata.Key.of("Authorization", ASCII_STRING_MARSHALLER); - - /** - * Construct a new GoogleAuthCredentials with given options. - * - * @param options a map of options, Required unless specified: audience - Optional, Sets the - * target audience of the token obtained. - * @throws IOException if credentials are not available - */ - public GoogleAuthCredentials(Map options) throws IOException { - String targetAudience = options.getOrDefault("audience", "https://localhost"); - ServiceAccountCredentials serviceCreds = - (ServiceAccountCredentials) - ServiceAccountCredentials.getApplicationDefault() - .createScoped(Arrays.asList("openid", "email")); - - credentials = - IdTokenCredentials.newBuilder() - .setIdTokenProvider(serviceCreds) - .setTargetAudience(targetAudience) - .build(); - } - - @Override - public void applyRequestMetadata( - RequestInfo requestInfo, Executor appExecutor, MetadataApplier applier) { - appExecutor.execute( - () -> { - try { - credentials.refreshIfExpired(); - Metadata headers = new Metadata(); - headers.put( - AUTHORIZATION_METADATA_KEY, - String.format("%s %s", BEARER_TYPE, credentials.getIdToken().getTokenValue())); - applier.apply(headers); - } catch (Throwable e) { - applier.fail(Status.UNAUTHENTICATED.withCause(e)); - } - }); - } - - @Override - public void thisUsesUnstableApi() { - // TODO Auto-generated method stub - - } -} diff --git a/java/common/src/main/java/feast/common/auth/credentials/JwtCallCredentials.java b/java/common/src/main/java/feast/common/auth/credentials/JwtCallCredentials.java deleted file mode 100644 index f09f4ec940..0000000000 --- a/java/common/src/main/java/feast/common/auth/credentials/JwtCallCredentials.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.auth.credentials; - -import io.grpc.CallCredentials; -import io.grpc.Metadata; -import java.util.concurrent.Executor; - -/** - * JWTCallCredentials provides/attaches a static JWT token for making authenticated gRPC calls. The - * given token will be passed as authorization bearer token when making calls. - */ -public final class JwtCallCredentials extends CallCredentials { - - private String jwt; - - public JwtCallCredentials(String jwt) { - this.jwt = jwt; - } - - @Override - public void applyRequestMetadata( - RequestInfo requestInfo, Executor executor, MetadataApplier metadataApplier) { - Metadata metadata = new Metadata(); - metadata.put( - Metadata.Key.of("Authorization", Metadata.ASCII_STRING_MARSHALLER), - String.format("Bearer %s", jwt)); - metadataApplier.apply(metadata); - } - - @Override - public void thisUsesUnstableApi() { - // does nothing - } -} diff --git a/java/common/src/main/java/feast/common/auth/credentials/OAuthCredentials.java b/java/common/src/main/java/feast/common/auth/credentials/OAuthCredentials.java deleted file mode 100644 index 429fd7fdad..0000000000 --- a/java/common/src/main/java/feast/common/auth/credentials/OAuthCredentials.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.auth.credentials; - -import static io.grpc.Metadata.ASCII_STRING_MARSHALLER; - -import com.nimbusds.jose.util.JSONObjectUtils; -import io.grpc.CallCredentials; -import io.grpc.Metadata; -import io.grpc.Status; -import java.time.Instant; -import java.util.Map; -import java.util.concurrent.Executor; -import javax.security.sasl.AuthenticationException; -import net.minidev.json.JSONObject; -import okhttp3.FormBody; -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.RequestBody; -import okhttp3.Response; -import org.springframework.security.oauth2.jwt.NimbusJwtDecoder; - -/** - * OAuthCredentials uses a OAuth OIDC ID token making authenticated gRPC calls. Makes an OAuth - * request to obtain the OIDC token used for authentication. The given token will be passed as - * authorization bearer token when making calls. - */ -public class OAuthCredentials extends CallCredentials { - - private static final String JWK_ENDPOINT_URI = "jwkEndpointURI"; - static final String APPLICATION_JSON = "application/json"; - static final String CONTENT_TYPE = "content-type"; - static final String BEARER_TYPE = "Bearer"; - static final String GRANT_TYPE = "grant_type"; - static final String CLIENT_ID = "client_id"; - static final String CLIENT_SECRET = "client_secret"; - static final String AUDIENCE = "audience"; - static final String OAUTH_URL = "oauth_url"; - static final Metadata.Key AUTHORIZATION_METADATA_KEY = - Metadata.Key.of("Authorization", ASCII_STRING_MARSHALLER); - - private OkHttpClient httpClient; - private Request request; - private String accessToken; - private Instant tokenExpiryTime; - private NimbusJwtDecoder jwtDecoder; - - /** - * Constructs a new OAuthCredentials with given options. - * - * @param options a map of options, Required unless specified: grant_type - OAuth grant type. - * Should be set as client_credentials audience - Sets the target audience of the token - * obtained. client_id - Client id to use in the OAuth request. client_secret - Client securet - * to use in the OAuth request. jwtEndpointURI - HTTPS URL used to retrieve a JWK that can be - * used to decode the credential. - */ - public OAuthCredentials(Map options) { - this.httpClient = new OkHttpClient(); - if (!(options.containsKey(GRANT_TYPE) - && options.containsKey(CLIENT_ID) - && options.containsKey(AUDIENCE) - && options.containsKey(CLIENT_SECRET) - && options.containsKey(OAUTH_URL) - && options.containsKey(JWK_ENDPOINT_URI))) { - throw new AssertionError( - "please configure the properties:" - + " grant_type, client_id, client_secret, audience, oauth_url, jwkEndpointURI"); - } - RequestBody requestBody = - new FormBody.Builder() - .add(GRANT_TYPE, options.get(GRANT_TYPE)) - .add(CLIENT_ID, options.get(CLIENT_ID)) - .add(CLIENT_SECRET, options.get(CLIENT_SECRET)) - .add(AUDIENCE, options.get(AUDIENCE)) - .build(); - this.request = - new Request.Builder() - .url(options.get(OAUTH_URL)) - .addHeader(CONTENT_TYPE, APPLICATION_JSON) - .post(requestBody) - .build(); - this.jwtDecoder = NimbusJwtDecoder.withJwkSetUri(options.get(JWK_ENDPOINT_URI)).build(); - } - - @Override - public void thisUsesUnstableApi() { - // TODO Auto-generated method stub - - } - - @Override - public void applyRequestMetadata( - RequestInfo requestInfo, Executor appExecutor, MetadataApplier applier) { - appExecutor.execute( - () -> { - try { - // Fetches new token if it is not available or if token has expired. - if (this.accessToken == null || Instant.now().isAfter(this.tokenExpiryTime)) { - Response response = httpClient.newCall(request).execute(); - if (!response.isSuccessful()) { - throw new AuthenticationException(response.message()); - } - JSONObject json = JSONObjectUtils.parse(response.body().string()); - this.accessToken = json.getAsString("access_token"); - this.tokenExpiryTime = jwtDecoder.decode(this.accessToken).getExpiresAt(); - } - Metadata headers = new Metadata(); - headers.put( - AUTHORIZATION_METADATA_KEY, String.format("%s %s", BEARER_TYPE, this.accessToken)); - applier.apply(headers); - } catch (Throwable e) { - applier.fail(Status.UNAUTHENTICATED.withCause(e)); - } - }); - } -} diff --git a/java/common/src/main/java/feast/common/auth/providers/http/HttpAuthorizationProvider.java b/java/common/src/main/java/feast/common/auth/providers/http/HttpAuthorizationProvider.java deleted file mode 100644 index 041a6b8e1b..0000000000 --- a/java/common/src/main/java/feast/common/auth/providers/http/HttpAuthorizationProvider.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.auth.providers.http; - -import feast.common.auth.authorization.AuthorizationProvider; -import feast.common.auth.authorization.AuthorizationResult; -import feast.common.auth.config.CacheConfiguration; -import feast.common.auth.config.SecurityProperties.AuthenticationProperties; -import feast.common.auth.providers.http.client.api.DefaultApi; -import feast.common.auth.providers.http.client.invoker.ApiClient; -import feast.common.auth.providers.http.client.invoker.ApiException; -import feast.common.auth.providers.http.client.model.CheckAccessRequest; -import feast.common.auth.utils.AuthUtils; -import java.util.Map; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.cache.annotation.Cacheable; -import org.springframework.security.core.Authentication; -import org.springframework.security.oauth2.jwt.Jwt; - -/** - * HTTPAuthorizationProvider uses an external HTTP service for authorizing requests. Please see - * auth/src/main/resources/api.yaml for the API specification of this external service. - */ -public class HttpAuthorizationProvider implements AuthorizationProvider { - - private static final Logger log = LoggerFactory.getLogger(HttpAuthorizationProvider.class); - - private final DefaultApi defaultApiClient; - - /** - * The default subject claim is the key within the Authentication object where the user's identity - * can be found - */ - private final String subjectClaim; - - /** - * Initializes the HTTPAuthorizationProvider - * - * @param options String K/V pair of options to initialize the provider with. Expects at least a - * "basePath" for the provider URL - */ - public HttpAuthorizationProvider(Map options) { - if (options == null) { - throw new IllegalArgumentException( - "Cannot pass empty or null options to HTTPAuthorizationProvider"); - } - - ApiClient apiClient = new ApiClient(); - apiClient.setBasePath(options.get("authorizationUrl")); - this.defaultApiClient = new DefaultApi(apiClient); - subjectClaim = options.get(AuthenticationProperties.SUBJECT_CLAIM); - } - - /** - * Validates whether a user has access to a project. @Cacheable is using {@link - * CacheConfiguration} settings to cache output of the method {@link AuthorizationResult} for a - * specified duration set in cache settings. - * - * @param projectId Name of the Feast project - * @param authentication Spring Security Authentication object - * @return AuthorizationResult result of authorization query - */ - @Cacheable(value = CacheConfiguration.AUTHORIZATION_CACHE, keyGenerator = "authKeyGenerator") - public AuthorizationResult checkAccessToProject(String projectId, Authentication authentication) { - - CheckAccessRequest checkAccessRequest = new CheckAccessRequest(); - Object context = getContext(authentication); - String subject = AuthUtils.getSubjectFromAuth(authentication, subjectClaim); - String resource = "projects:" + projectId; - checkAccessRequest.setAction("ALL"); - checkAccessRequest.setContext(context); - checkAccessRequest.setResource(resource); - checkAccessRequest.setSubject(subject); - try { - Jwt credentials = ((Jwt) authentication.getCredentials()); - // Make authorization request to external service - feast.common.auth.providers.http.client.model.AuthorizationResult authResult = - this.defaultApiClient.checkAccessPost( - checkAccessRequest, "Bearer " + credentials.getTokenValue()); - if (authResult == null) { - throw new RuntimeException( - String.format( - "Empty response returned for access to project %s for subject %s", - projectId, subject)); - } - if (authResult.getAllowed()) { - // Successfully authenticated - return AuthorizationResult.success(); - } - } catch (ApiException e) { - log.error("API exception has occurred during authorization: {}", e.getMessage(), e); - } - - // Could not determine project membership, deny access. - return AuthorizationResult.failed( - String.format("Access denied to project %s for subject %s", projectId, subject)); - } - - /** - * Extract a context object to send as metadata to the authorization service - * - * @param authentication Spring Security Authentication object - * @return Returns a context object that will be serialized and sent as metadata to the - * authorization service - */ - private Object getContext(Authentication authentication) { - // Not implemented yet, left empty - return new Object(); - } -} diff --git a/java/common/src/main/java/feast/common/auth/providers/keto/KetoAuthorizationProvider.java b/java/common/src/main/java/feast/common/auth/providers/keto/KetoAuthorizationProvider.java deleted file mode 100644 index 05fcd3c7c1..0000000000 --- a/java/common/src/main/java/feast/common/auth/providers/keto/KetoAuthorizationProvider.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2021 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.auth.providers.keto; - -import feast.common.auth.authorization.AuthorizationProvider; -import feast.common.auth.authorization.AuthorizationResult; -import feast.common.auth.utils.AuthUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.security.core.Authentication; -import sh.ory.keto.ApiClient; -import sh.ory.keto.ApiException; -import sh.ory.keto.Configuration; -import sh.ory.keto.api.EnginesApi; -import sh.ory.keto.model.OryAccessControlPolicyAllowedInput; - -public class KetoAuthorizationProvider implements AuthorizationProvider { - - /** Builder for KetoAuthorizationProvider */ - public static class Builder { - private final String url; - private String subjectClaim = "email"; - private String flavor = "glob"; - private String action = "edit"; - private String subjectPrefix = ""; - private String resourcePrefix = ""; - - /** - * Initialized builder for Keto authorization provider. - * - * @param url Url string for Keto server. - */ - public Builder(String url) { - this.url = url; - } - - /** - * Set subject claim for authentication - * - * @param subjectClaim Subject claim. Default: email. - * @return Returns Builder - */ - public Builder withSubjectClaim(String subjectClaim) { - this.subjectClaim = subjectClaim; - return this; - } - - /** - * Set flavor for Keto authorization. One of [exact, glob regex] - * - * @param flavor Keto authorization flavor. Default: glob. - * @return Returns Builder - */ - public Builder withFlavor(String flavor) { - this.flavor = flavor; - return this; - } - - /** - * Set action that corresponds to the permission to edit a Feast project resource. - * - * @param action Keto action. Default: edit. - * @return Returns Builder - */ - public Builder withAction(String action) { - this.action = action; - return this; - } - - /** - * If set, The subject will be prefixed before sending the request to Keto. Example: - * users:someuser@email.com - * - * @param prefix Subject prefix. Default: Empty string. - * @return Returns Builder - */ - public Builder withSubjectPrefix(String prefix) { - this.subjectPrefix = prefix; - return this; - } - - /** - * If set, The resource will be prefixed before sending the request to Keto. Example: - * projects:somefeastproject - * - * @param prefix Resource prefix. Default: Empty string. - * @return Returns Builder - */ - public Builder withResourcePrefix(String prefix) { - this.resourcePrefix = prefix; - return this; - } - - /** - * Build KetoAuthorizationProvider - * - * @return Returns KetoAuthorizationProvider - */ - public KetoAuthorizationProvider build() { - return new KetoAuthorizationProvider(this); - } - } - - private static final Logger log = LoggerFactory.getLogger(KetoAuthorizationProvider.class); - - private final EnginesApi apiInstance; - private final String subjectClaim; - private final String flavor; - private final String action; - private final String subjectPrefix; - private final String resourcePrefix; - - private KetoAuthorizationProvider(Builder builder) { - ApiClient defaultClient = Configuration.getDefaultApiClient(); - defaultClient.setBasePath(builder.url); - apiInstance = new EnginesApi(defaultClient); - subjectClaim = builder.subjectClaim; - flavor = builder.flavor; - action = builder.action; - subjectPrefix = builder.subjectPrefix; - resourcePrefix = builder.resourcePrefix; - } - - @Override - public AuthorizationResult checkAccessToProject(String projectId, Authentication authentication) { - String subject = AuthUtils.getSubjectFromAuth(authentication, subjectClaim); - OryAccessControlPolicyAllowedInput body = new OryAccessControlPolicyAllowedInput(); - body.setAction(action); - body.setSubject(String.format("%s%s", subjectPrefix, subject)); - body.setResource(String.format("%s%s", resourcePrefix, projectId)); - try { - sh.ory.keto.model.AuthorizationResult authResult = - apiInstance.doOryAccessControlPoliciesAllow(flavor, body); - if (authResult == null) { - throw new RuntimeException( - String.format( - "Empty response returned for access to project %s for subject %s", - projectId, subject)); - } - if (authResult.getAllowed()) { - return AuthorizationResult.success(); - } - } catch (ApiException e) { - log.error("API exception has occurred during authorization: {}", e.getMessage(), e); - } - - return AuthorizationResult.failed( - String.format("Access denied to project %s for subject %s", projectId, subject)); - } -} diff --git a/java/common/src/main/java/feast/common/auth/service/AuthorizationService.java b/java/common/src/main/java/feast/common/auth/service/AuthorizationService.java deleted file mode 100644 index 7d325e880d..0000000000 --- a/java/common/src/main/java/feast/common/auth/service/AuthorizationService.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.auth.service; - -import feast.common.auth.authorization.AuthorizationProvider; -import feast.common.auth.authorization.AuthorizationResult; -import feast.common.auth.config.SecurityProperties; -import lombok.AllArgsConstructor; -import org.springframework.beans.factory.ObjectProvider; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.security.access.AccessDeniedException; -import org.springframework.security.core.Authentication; -import org.springframework.security.core.context.SecurityContext; -import org.springframework.stereotype.Service; - -@AllArgsConstructor -@Service -public class AuthorizationService { - - private final SecurityProperties securityProperties; - private final AuthorizationProvider authorizationProvider; - - @Autowired - public AuthorizationService( - SecurityProperties securityProperties, - ObjectProvider authorizationProvider) { - this.securityProperties = securityProperties; - this.authorizationProvider = authorizationProvider.getIfAvailable(); - } - - /** - * Determine whether a user has access to a project. - * - * @param securityContext Spring Security Context used to identify a user or service. - * @param project Name of the project for which membership should be tested. - */ - public void authorizeRequest(SecurityContext securityContext, String project) { - Authentication authentication = securityContext.getAuthentication(); - if (!this.securityProperties.getAuthorization().isEnabled()) { - return; - } - - AuthorizationResult result = - this.authorizationProvider.checkAccessToProject(project, authentication); - if (!result.isAllowed()) { - throw new AccessDeniedException(result.getFailureReason().orElse("Access Denied")); - } - } -} diff --git a/java/common/src/main/java/feast/common/auth/utils/AuthUtils.java b/java/common/src/main/java/feast/common/auth/utils/AuthUtils.java deleted file mode 100644 index e05fc70a18..0000000000 --- a/java/common/src/main/java/feast/common/auth/utils/AuthUtils.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.auth.utils; - -import java.util.Map; -import org.hibernate.validator.internal.constraintvalidators.bv.EmailValidator; -import org.springframework.security.core.Authentication; -import org.springframework.security.oauth2.jwt.Jwt; - -public class AuthUtils { - // Suppresses default constructor, ensuring non-instantiability. - private AuthUtils() {} - - /** - * Get user email from their authentication object. - * - * @param authentication Spring Security Authentication object, used to extract user details - * @param subjectClaim Indicates the claim where the subject can be found - * @return String user email - */ - public static String getSubjectFromAuth(Authentication authentication, String subjectClaim) { - Jwt principle = ((Jwt) authentication.getPrincipal()); - Map claims = principle.getClaims(); - String subjectValue = (String) claims.getOrDefault(subjectClaim, ""); - - if (subjectValue.isEmpty()) { - throw new IllegalStateException( - String.format("JWT does not have a valid claim %s.", subjectClaim)); - } - - if (subjectClaim.equals("email")) { - boolean validEmail = (new EmailValidator()).isValid(subjectValue, null); - if (!validEmail) { - throw new IllegalStateException("JWT contains an invalid email address"); - } - } - return subjectValue; - } -} diff --git a/java/common/src/main/java/feast/common/logging/interceptors/GrpcMessageInterceptor.java b/java/common/src/main/java/feast/common/logging/interceptors/GrpcMessageInterceptor.java index da895f0093..ffd7c6b954 100644 --- a/java/common/src/main/java/feast/common/logging/interceptors/GrpcMessageInterceptor.java +++ b/java/common/src/main/java/feast/common/logging/interceptors/GrpcMessageInterceptor.java @@ -18,9 +18,6 @@ import com.google.protobuf.Empty; import com.google.protobuf.Message; -import feast.common.auth.config.SecurityProperties; -import feast.common.auth.config.SecurityProperties.AuthenticationProperties; -import feast.common.auth.utils.AuthUtils; import feast.common.logging.AuditLogger; import feast.common.logging.config.LoggingProperties; import feast.common.logging.entry.MessageAuditLogEntry; @@ -32,10 +29,8 @@ import io.grpc.ServerCallHandler; import io.grpc.ServerInterceptor; import io.grpc.Status; -import java.util.Map; import org.slf4j.event.Level; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.lang.Nullable; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.stereotype.Component; @@ -48,20 +43,15 @@ */ @Component public class GrpcMessageInterceptor implements ServerInterceptor { - private SecurityProperties securityProperties; private LoggingProperties loggingProperties; /** * Construct GrpcMessageIntercetor. * * @param loggingProperties properties used to configure logging interceptor. - * @param securityProperties If provided, will output the subject claim specified in - * securityProperties as identity in {@link MessageAuditLogEntry} instead. */ @Autowired - public GrpcMessageInterceptor( - LoggingProperties loggingProperties, @Nullable SecurityProperties securityProperties) { - this.securityProperties = securityProperties; + public GrpcMessageInterceptor(LoggingProperties loggingProperties) { this.loggingProperties = loggingProperties; } @@ -132,18 +122,6 @@ public void onMessage(ReqT message) { */ private String getIdentity(Authentication authentication) { // use subject claim as identity if set in security authorization properties - if (securityProperties != null) { - Map options = securityProperties.getAuthentication().getOptions(); - if (options.containsKey(AuthenticationProperties.SUBJECT_CLAIM)) { - try { - return AuthUtils.getSubjectFromAuth( - authentication, options.get(AuthenticationProperties.SUBJECT_CLAIM)); - } catch (IllegalStateException e) { - // could not extract claim, revert to authenticated name. - return authentication.getName(); - } - } - } return authentication.getName(); } } diff --git a/java/common/src/main/java/feast/common/models/Store.java b/java/common/src/main/java/feast/common/models/Store.java deleted file mode 100644 index 1701b0bb3a..0000000000 --- a/java/common/src/main/java/feast/common/models/Store.java +++ /dev/null @@ -1,163 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.models; - -import feast.proto.core.StoreProto.Store.Subscription; -import java.util.Arrays; -import java.util.List; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -public class Store { - - /** - * Accepts a comma-delimited string and converts it to a list of Subscription class objects. - * - * @param subscriptions String formatted Subscriptions, comma delimited. - * @return List of Subscription class objects - */ - public static List parseSubFromStr(String subscriptions) { - List allSubscriptions = - Arrays.stream(subscriptions.split(",")) - .map(subscriptionStr -> convertStringToSubscription(subscriptionStr)) - .collect(Collectors.toList()); - - return allSubscriptions; - } - - /** - * Accepts a comma-delimited string and converts it to a list of Subscription class objects, with - * exclusions filtered out. - * - * @param subscriptions String formatted Subscriptions, comma delimited. - * @return List of Subscription class objects - */ - public static List parseSubFromStrWithoutExclusions(String subscriptions) { - List allSubscriptions = - Arrays.stream(subscriptions.split(",")) - .map(subscriptionStr -> convertStringToSubscription(subscriptionStr)) - .collect(Collectors.toList()); - - allSubscriptions = - allSubscriptions.stream().filter(sub -> !sub.getExclude()).collect(Collectors.toList()); - - return allSubscriptions; - } - - /** - * Accepts a Subscription class object and returns it in string format - * - * @param subscription Subscription class to be converted to string format - * @return String formatted Subscription class - */ - public static String parseSubscriptionFrom(Subscription subscription) { - if (subscription.getName().isEmpty() || subscription.getProject().isEmpty()) { - throw new IllegalArgumentException( - String.format("Missing arguments in subscription string: %s", subscription.toString())); - } - - return String.format( - "%s:%s:%s", subscription.getProject(), subscription.getName(), subscription.getExclude()); - } - - /** - * Accepts a exclude parameter to determine whether to return subscriptions that are excluded. - * - * @param subscription String formatted Subscription to be converted to Subscription class - * @return Subscription class with its respective attributes - */ - public static Subscription convertStringToSubscription(String subscription) { - if (subscription.equals("")) { - return Subscription.newBuilder().build(); - } - String[] split = subscription.split(":"); - if (split.length == 2) { - // Backward compatibility check - return Subscription.newBuilder().setProject(split[0]).setName(split[1]).build(); - } - return Subscription.newBuilder() - .setProject(split[0]) - .setName(split[1]) - .setExclude(Boolean.parseBoolean(split[2])) - .build(); - } - - /** - * The current use of this function is to determine whether a FeatureRow is subscribed to a - * Featureset. - * - * @param subscriptions List of Subscriptions available in Store - * @param projectName Project name used for matching Subscription's Project - * @param featureSetName Featureset name used for matching Subscription's Featureset - * @return boolean flag to signify if FeatureRow is subscribed to Featureset - */ - public static boolean isSubscribedToFeatureSet( - List subscriptions, String projectName, String featureSetName) { - // Case 1: Highest priority check, to exclude all matching subscriptions with excluded flag = - // true - for (Subscription sub : subscriptions) { - // If configuration missing, fail - if (sub.getProject().isEmpty() || sub.getName().isEmpty()) { - throw new IllegalArgumentException( - String.format("Subscription is missing arguments: %s", sub.toString())); - } - // Match feature set name to pattern - Pattern patternName = getNamePattern(sub); - Pattern patternProject = getProjectPattern(sub); - // SubCase: Project name and feature set name matches and excluded flag is true - if (patternProject.matcher(projectName).matches() - && patternName.matcher(featureSetName).matches() - && sub.getExclude()) { - return false; - } - } - // Case 2: Featureset is not excluded, check if it is included in the current subscriptions - // filteredSubscriptions only contain subscriptions with excluded flag = false - List filteredSubscriptions = - subscriptions.stream().filter(sub -> !sub.getExclude()).collect(Collectors.toList()); - - for (Subscription filteredSub : filteredSubscriptions) { - // Match feature set name to pattern - Pattern patternName = getNamePattern(filteredSub); - Pattern patternProject = getProjectPattern(filteredSub); - // SubCase: Project name and feature set name matches - if (patternProject.matcher(projectName).matches() - && patternName.matcher(featureSetName).matches()) { - return true; - } - } - return false; - } - - private static Pattern getProjectPattern(Subscription subscription) { - String subProject = subscription.getProject(); - if (!subscription.getProject().contains(".*")) { - subProject = subProject.replace("*", ".*"); - } - - return Pattern.compile(subProject); - } - - private static Pattern getNamePattern(Subscription subscription) { - String subName = subscription.getName(); - if (!subscription.getProject().contains(".*")) { - subName = subName.replace("*", ".*"); - } - - return Pattern.compile(subName); - } -} diff --git a/java/common/src/main/java/feast/common/util/KafkaSerialization.java b/java/common/src/main/java/feast/common/util/KafkaSerialization.java deleted file mode 100644 index 62c31a63bb..0000000000 --- a/java/common/src/main/java/feast/common/util/KafkaSerialization.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.util; - -import com.google.protobuf.GeneratedMessageV3; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.Message; -import com.google.protobuf.Parser; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import org.apache.kafka.common.serialization.Deserializer; -import org.apache.kafka.common.serialization.Serializer; - -/* -Serializer & Deserializer implementation to write & read protobuf object from/to kafka - */ -public class KafkaSerialization { - public static class ProtoSerializer implements Serializer { - @Override - public byte[] serialize(String topic, T data) { - ByteArrayOutputStream stream = new ByteArrayOutputStream(); - try { - data.writeTo(stream); - } catch (IOException e) { - throw new RuntimeException( - String.format( - "Unable to serialize object of type %s. Reason: %s", - data.getClass().getName(), e.getCause().getMessage())); - } - - return stream.toByteArray(); - } - } - - public static class ProtoDeserializer implements Deserializer { - private Parser parser; - - public ProtoDeserializer(Parser parser) { - this.parser = parser; - } - - @Override - public T deserialize(String topic, byte[] data) { - try { - return parser.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException( - String.format( - "Unable to deserialize object from topic %s. Reason: %s", - topic, e.getCause().getMessage())); - } - } - } -} diff --git a/java/common/src/main/resources/api.yaml b/java/common/src/main/resources/api.yaml deleted file mode 100644 index 73ddaf2c2d..0000000000 --- a/java/common/src/main/resources/api.yaml +++ /dev/null @@ -1,117 +0,0 @@ -openapi: 3.0.1 -info: - description: 'Feast Authorization Server' - license: - name: Apache 2.0 - url: http://www.apache.org/licenses/LICENSE-2.0.html - title: Feast Authorization Server - version: 1.0.0 -servers: - - url: / -paths: - /healthz: - get: - responses: - "200": - description: Online - "500": - description: Offline - /readiness: - get: - responses: - "200": - description: Ready - "500": - description: Not Ready - /checkAccess: - post: - parameters: - - name: Authorization - in: header - description: Auth token - schema: - type: string - operationId: check_access_post - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/checkAccessRequest' - description: Request containing user, resource, and action information. Used to make an authorization decision. - required: true - responses: - "200": - content: - application/json: - schema: - $ref: '#/components/schemas/authorizationResult' - description: Authorization passed response - "403": - content: - application/json: - schema: - $ref: '#/components/schemas/authorizationResult' - description: Authorization failed response - "500": - content: - application/json: - schema: - $ref: '#/components/schemas/inline_response_500' - description: The standard error format - summary: Check whether request is authorized to access a specific resource - x-codegen-request-body-name: body -components: - schemas: - checkAccessRequest: - example: - action: 'read' - context: '{}' - resource: 'feast:project' - subject: 'me@example.com' - properties: - action: - description: Action is the action that is being taken on the requested resource. - type: string - context: - description: Context is the request's environmental context. - properties: {} - type: object - resource: - description: Resource is the resource that access is requested to. - type: string - subject: - description: Subject is the subject that is requesting access, typically the user. - type: string - title: Input for checking if a request is allowed or not. - type: object - authorizationResult: - example: - allowed: true - properties: - allowed: - description: Allowed is true if the request should be allowed and false - otherwise. - type: boolean - required: - - allowed - title: AuthorizationResult is the result of an access control decision. It contains - the decision outcome. - type: object - inline_response_500: - properties: - code: - format: int64 - type: integer - details: - items: - properties: {} - type: object - type: array - message: - type: string - reason: - type: string - request: - type: string - status: - type: string \ No newline at end of file diff --git a/java/common/src/test/java/feast/common/auth/authorization/HttpAuthorizationProviderCachingTest.java b/java/common/src/test/java/feast/common/auth/authorization/HttpAuthorizationProviderCachingTest.java deleted file mode 100644 index f303801c30..0000000000 --- a/java/common/src/test/java/feast/common/auth/authorization/HttpAuthorizationProviderCachingTest.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.auth.authorization; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -import com.google.common.collect.ImmutableMap; -import feast.common.auth.config.CacheConfiguration; -import feast.common.auth.config.SecurityProperties; -import feast.common.auth.config.SecurityProperties.AuthenticationProperties; -import feast.common.auth.config.SecurityProperties.AuthorizationProperties; -import feast.common.auth.providers.http.HttpAuthorizationProvider; -import feast.common.auth.providers.http.client.api.DefaultApi; -import feast.common.auth.providers.http.client.model.AuthorizationResult; -import feast.common.auth.providers.http.client.model.CheckAccessRequest; -import java.util.HashMap; -import java.util.Map; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mockito; -import org.mockito.internal.util.reflection.FieldSetter; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.security.core.Authentication; -import org.springframework.security.oauth2.jwt.Jwt; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringRunner; - -@RunWith(SpringRunner.class) -@ContextConfiguration( - classes = {CacheConfiguration.class, HttpAuthorizationProviderCachingTest.Config.class}) -public class HttpAuthorizationProviderCachingTest { - - // static since field needs to updated in provider() bean - private static DefaultApi api = Mockito.mock(DefaultApi.class); - - @Autowired AuthorizationProvider provider; - - @Configuration - static class Config { - @Bean - SecurityProperties securityProps() { - // setting TTL static variable in SecurityProperties bean, since CacheConfiguration bean is - // dependent on SecurityProperties. - CacheConfiguration.TTL = 1; - AuthenticationProperties authentication = Mockito.mock(AuthenticationProperties.class); - AuthorizationProperties authorization = new AuthorizationProperties(); - authorization.setEnabled(true); - authorization.setProvider("http"); - authorization.setOptions(ImmutableMap.of("authorizationUrl", "localhost")); - - authentication.setOptions(ImmutableMap.of("subjectClaim", "email")); - - SecurityProperties sp = new SecurityProperties(); - sp.setAuthentication(authentication); - sp.setAuthorization(authorization); - return sp; - } - - @Bean - AuthorizationProvider provider() throws NoSuchFieldException, SecurityException { - Map options = new HashMap<>(); - options.put("authorizationUrl", "localhost"); - options.put("subjectClaim", "email"); - HttpAuthorizationProvider provider = new HttpAuthorizationProvider(options); - FieldSetter.setField(provider, provider.getClass().getDeclaredField("defaultApiClient"), api); - return provider; - } - } - - @Test - public void testCheckAccessToProjectShouldReadFromCacheWhenAvailable() throws Exception { - Authentication auth = Mockito.mock(Authentication.class); - Jwt jwt = Mockito.mock(Jwt.class); - Map claims = new HashMap<>(); - claims.put("email", "test@test.com"); - doReturn(jwt).when(auth).getCredentials(); - doReturn(jwt).when(auth).getPrincipal(); - doReturn(claims).when(jwt).getClaims(); - doReturn("test_token").when(jwt).getTokenValue(); - AuthorizationResult authResult = new AuthorizationResult(); - authResult.setAllowed(true); - doReturn(authResult) - .when(api) - .checkAccessPost(any(CheckAccessRequest.class), any(String.class)); - - // Should save the result in cache - provider.checkAccessToProject("test", auth); - // Should read from cache - provider.checkAccessToProject("test", auth); - verify(api, times(1)).checkAccessPost(any(CheckAccessRequest.class), any(String.class)); - - // cache ttl is set to 1 second for testing. - Thread.sleep(1100); - - // Should make an invocation to external service - provider.checkAccessToProject("test", auth); - verify(api, times(2)).checkAccessPost(any(CheckAccessRequest.class), any(String.class)); - // Should read from cache - provider.checkAccessToProject("test", auth); - verify(api, times(2)).checkAccessPost(any(CheckAccessRequest.class), any(String.class)); - } -} diff --git a/java/common/src/test/java/feast/common/models/StoreTest.java b/java/common/src/test/java/feast/common/models/StoreTest.java deleted file mode 100644 index 1acba12d2b..0000000000 --- a/java/common/src/test/java/feast/common/models/StoreTest.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.models; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.core.IsEqual.equalTo; -import static org.junit.Assert.assertTrue; - -import feast.proto.core.StoreProto.Store.Subscription; -import java.util.Arrays; -import java.util.List; -import org.junit.Before; -import org.junit.Test; - -public class StoreTest { - - private List allSubscriptions; - - @Before - public void setUp() { - - Subscription emptySubscription = Subscription.newBuilder().build(); - Subscription subscription1 = Subscription.newBuilder().setProject("*").setName("*").build(); - Subscription subscription2 = - Subscription.newBuilder().setProject("project1").setName("fs_2").build(); - Subscription subscription3 = - Subscription.newBuilder().setProject("project1").setName("fs_1").setExclude(true).build(); - allSubscriptions = - Arrays.asList(emptySubscription, subscription1, subscription2, subscription3); - } - - @Test - public void shouldReturnSubscriptionsBasedOnStr() { - String subscriptions = "project1:fs_1:true,project1:fs_2"; - List actual1 = Store.parseSubFromStr(subscriptions); - List expected1 = Arrays.asList(allSubscriptions.get(2), allSubscriptions.get(3)); - - List actual2 = Store.parseSubFromStrWithoutExclusions(subscriptions); - List expected2 = Arrays.asList(allSubscriptions.get(2)); - - assertTrue(actual1.containsAll(expected1) && expected1.containsAll(actual1)); - assertTrue(actual2.containsAll(expected2) && expected2.containsAll(actual2)); - } - - @Test - public void shouldReturnStringBasedOnSubscription() { - // Case: default exclude should be false - String actual1 = Store.parseSubscriptionFrom(allSubscriptions.get(2)); - Subscription sub1 = allSubscriptions.get(2); - String expected1 = sub1.getProject() + ":" + sub1.getName() + ":" + sub1.getExclude(); - - // Case: explicit setting of exclude to true - String actual2 = Store.parseSubscriptionFrom(allSubscriptions.get(3)); - Subscription sub2 = allSubscriptions.get(3); - String expected2 = sub2.getProject() + ":" + sub2.getName() + ":" + sub2.getExclude(); - - assertThat(actual1, equalTo(expected1)); - assertThat(actual2, equalTo(expected2)); - } - - @Test - public void shouldSubscribeToFeatureSet() { - allSubscriptions = allSubscriptions.subList(2, 4); - // Case: excluded flag = true - boolean actual1 = Store.isSubscribedToFeatureSet(allSubscriptions, "project1", "fs_1"); - boolean expected1 = false; - - // Case: excluded flag = false - boolean actual2 = Store.isSubscribedToFeatureSet(allSubscriptions, "project1", "fs_2"); - boolean expected2 = true; - - // Case: featureset does not exist - boolean actual3 = - Store.isSubscribedToFeatureSet(allSubscriptions, "project1", "fs_nonexistent"); - boolean expected3 = false; - - assertThat(actual1, equalTo(expected1)); - assertThat(actual2, equalTo(expected2)); - assertThat(actual3, equalTo(expected3)); - } -} diff --git a/java/core/.gitignore b/java/core/.gitignore deleted file mode 100644 index fc240cb6e9..0000000000 --- a/java/core/.gitignore +++ /dev/null @@ -1,40 +0,0 @@ -### Scratch files ### -scratch_redis* - -### Local Environment ### -*local*.env - -### Gradle ### -.gradle -**/build/ -!gradle/wrapper/gradle-wrapper.jar -feast-serving.jar - -### STS ### -.apt_generated -.classpath -.factorypath -.project -.settings -.springBeans -.sts4-cache - -### IntelliJ IDEA ### -.idea -*.iws -*.iml -*.ipr -/out/ - -### NetBeans ### -/nbproject/private/ -/nbbuild/ -/dist/ -/nbdist/ -/.nb-gradle/ - -## Static site ## -static - -## Feast Temporary Files ## -/temp/ \ No newline at end of file diff --git a/java/core/README.md b/java/core/README.md deleted file mode 100644 index 8f5b6f03bf..0000000000 --- a/java/core/README.md +++ /dev/null @@ -1,32 +0,0 @@ -### Getting Started Guide for Feast Core Developers - -Pre-requisites: -- [Maven](https://maven.apache.org/install.html) build tool version 3.6.x -- A running Postgres instance. For easier to get started, please configure the database like so - ``` - database: postgres - user: postgres - password: password - ``` -- A running Redis instance - ``` - host: localhost - port: 6379 - ``` -- Access to Google Cloud BigQuery (optional) -- Access to Kafka brokers (to test starting ingestion jobs from Feast Core) - -Run the following maven command to start Feast Core GRPC service running on port 6565 locally -```bash -# Using configuration from src/main/resources/application.yml -mvn spring-boot:run -# Using configuration from custom location e.g. /tmp/config.application.yml -mvn spring-boot:run -Dspring.config.location=/tmp/config.application.yml -``` - -If you have [grpc_cli](https://github.com/grpc/grpc/blob/master/doc/command_line_tool.md) installed, you can check that Feast Core is running -``` -grpc_cli ls localhost:6565 -grpc_cli call localhost:6565 GetFeastCoreVersion "" -grpc_cli call localhost:6565 ListStores "" -``` \ No newline at end of file diff --git a/java/core/lombok.config b/java/core/lombok.config deleted file mode 100644 index 8f7e8aa1ac..0000000000 --- a/java/core/lombok.config +++ /dev/null @@ -1 +0,0 @@ -lombok.addLombokGeneratedAnnotation = true \ No newline at end of file diff --git a/java/core/pom.xml b/java/core/pom.xml deleted file mode 100644 index 1c33e15386..0000000000 --- a/java/core/pom.xml +++ /dev/null @@ -1,325 +0,0 @@ - - - - 4.0.0 - - dev.feast - feast-parent - ${revision} - - - Feast Core - Feature registry - feast-core - - - - - org.apache.maven.plugins - maven-compiler-plugin - - 11 - - - - - org.jacoco - jacoco-maven-plugin - - - - org.springframework.boot - spring-boot-maven-plugin - - false - - - - build-info - - build-info - - - - - - - org.flywaydb - flyway-maven-plugin - ${flyway.version} - - - - - - - dev.feast - feast-common - ${project.version} - - - dev.feast - feast-common-test - ${project.version} - test - - - - - org.springframework.boot - spring-boot-devtools - true - - - - javax.inject - javax.inject - 1 - - - - org.springframework.boot - spring-boot-starter-web - - - - org.springframework.boot - spring-boot-starter-log4j2 - - - org.apache.logging.log4j - log4j-web - - - org.springframework.security - spring-security-core - ${spring.security.version} - - - org.springframework.security - spring-security-config - ${spring.security.version} - - - org.springframework.security.oauth - spring-security-oauth2 - ${spring.security.oauth2.version} - - - org.springframework.security - spring-security-oauth2-client - ${spring.security.version} - - - org.springframework.security - spring-security-web - ${spring.security.version} - - - org.springframework.security - spring-security-oauth2-resource-server - ${spring.security.version} - - - org.springframework.security - spring-security-oauth2-jose - ${spring.security.version} - - - net.devh - grpc-server-spring-boot-starter - ${grpc.spring.boot.starter.version} - - - com.nimbusds - nimbus-jose-jwt - 8.2.1 - - - org.springframework.security - spring-security-oauth2-core - ${spring.security.version} - - - - org.springframework.boot - spring-boot-starter-data-jpa - - - - org.springframework.boot - spring-boot-starter-actuator - - - - - org.springframework.boot - spring-boot-configuration-processor - - - - io.grpc - grpc-services - - - - io.grpc - grpc-stub - - - - com.google.protobuf - protobuf-java-util - - - - com.google.guava - guava - - - - com.google.code.gson - gson - 2.8.5 - - - com.google.api-client - google-api-client - 1.30.9 - - - com.google.apis - google-api-services-dataflow - v1b3-rev20200305-1.30.9 - - - org.hibernate - hibernate-core - - - - - org.postgresql - postgresql - provided - true - - - - org.springframework.kafka - spring-kafka - - - - - org.projectlombok - lombok - ${lombok.version} - - - - io.prometheus - simpleclient - - - - io.prometheus - simpleclient_servlet - - - com.google.api.client - google-api-client-googleapis-auth-oauth - 1.2.3-alpha - - - com.auth0 - jwks-rsa - 0.11.0 - - - - com.auth0 - java-jwt - 3.10.0 - - - - org.apache.commons - commons-lang3 - - - - joda-time - joda-time - - - - - com.jayway.jsonpath - json-path-assert - 2.2.0 - test - - - - javax.xml.bind - jaxb-api - - - org.flywaydb - flyway-core - ${flyway.version} - - - org.hibernate.validator - hibernate-validator-annotation-processor - 6.1.2.Final - - - - sh.ory.keto - keto-client - 0.4.4-alpha.1 - - - com.github.tomakehurst - wiremock - 2.27.0 - test - - - org.apache.avro - avro - 1.8.2 - test - - - com.squareup.okhttp - okhttp - 2.7.4 - test - - - io.grpc - grpc-testing - - - diff --git a/java/core/src/main/java/feast/core/CoreApplication.java b/java/core/src/main/java/feast/core/CoreApplication.java deleted file mode 100644 index 957fdf5015..0000000000 --- a/java/core/src/main/java/feast/core/CoreApplication.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core; - -import feast.core.config.FeastProperties; -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.data.jpa.repository.config.EnableJpaRepositories; -import org.springframework.scheduling.annotation.EnableScheduling; - -@EnableScheduling -@SpringBootApplication -@EnableJpaRepositories(basePackages = "feast.core.dao") -@EnableConfigurationProperties(FeastProperties.class) -@Slf4j -public class CoreApplication { - public static void main(String[] args) { - SpringApplication.run(CoreApplication.class, args); - } -} diff --git a/java/core/src/main/java/feast/core/config/CoreSecurityConfig.java b/java/core/src/main/java/feast/core/config/CoreSecurityConfig.java deleted file mode 100644 index 52911c3b22..0000000000 --- a/java/core/src/main/java/feast/core/config/CoreSecurityConfig.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.config; - -import feast.proto.core.CoreServiceGrpc; -import io.grpc.health.v1.HealthGrpc; -import lombok.extern.slf4j.Slf4j; -import net.devh.boot.grpc.server.security.check.AccessPredicate; -import net.devh.boot.grpc.server.security.check.GrpcSecurityMetadataSource; -import net.devh.boot.grpc.server.security.check.ManualGrpcSecurityMetadataSource; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.Configuration; - -@Configuration -@Slf4j -@ComponentScan( - basePackages = { - "feast.common.auth.config", - "feast.common.auth.service", - "feast.common.logging.interceptors" - }) -public class CoreSecurityConfig { - - /** - * Creates a SecurityMetadataSource when authentication is enabled. This allows for the - * configuration of endpoint level security rules. - * - * @return GrpcSecurityMetadataSource - */ - @Bean - @ConditionalOnProperty(prefix = "feast.security.authentication", name = "enabled") - GrpcSecurityMetadataSource grpcSecurityMetadataSource() { - final ManualGrpcSecurityMetadataSource source = new ManualGrpcSecurityMetadataSource(); - - // Authentication is enabled for all gRPC endpoints - source.setDefault(AccessPredicate.authenticated()); - - // The following endpoints allow unauthenticated access - source.set(CoreServiceGrpc.getGetFeastCoreVersionMethod(), AccessPredicate.permitAll()); - source.set(CoreServiceGrpc.getUpdateStoreMethod(), AccessPredicate.permitAll()); - source.set(HealthGrpc.getCheckMethod(), AccessPredicate.permitAll()); - return source; - } -} diff --git a/java/core/src/main/java/feast/core/config/FeastProperties.java b/java/core/src/main/java/feast/core/config/FeastProperties.java deleted file mode 100644 index fd926ea5da..0000000000 --- a/java/core/src/main/java/feast/core/config/FeastProperties.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.config; - -import feast.common.auth.config.SecurityProperties; -import feast.common.auth.config.SecurityProperties.AuthenticationProperties; -import feast.common.auth.config.SecurityProperties.AuthorizationProperties; -import feast.common.logging.config.LoggingProperties; -import feast.common.validators.OneOfStrings; -import feast.core.config.FeastProperties.StreamProperties.FeatureStreamOptions; -import java.util.Set; -import javax.annotation.PostConstruct; -import javax.validation.ConstraintViolation; -import javax.validation.ConstraintViolationException; -import javax.validation.Validation; -import javax.validation.Validator; -import javax.validation.ValidatorFactory; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; -import javax.validation.constraints.Positive; -import lombok.Getter; -import lombok.Setter; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.boot.info.BuildProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.Configuration; - -@Getter -@Setter -@Configuration -@ComponentScan("feast.common.logging") -@ConfigurationProperties(prefix = "feast", ignoreInvalidFields = true) -public class FeastProperties { - - /** - * Instantiates a new Feast properties. - * - * @param buildProperties Feast build properties - */ - @Autowired - public FeastProperties(BuildProperties buildProperties) { - setVersion(buildProperties.getVersion()); - } - - /** Instantiates a new Feast properties. */ - public FeastProperties() {} - - /* Feast Core Build Version */ - @NotBlank private String version = "unknown"; - - @NotNull - /* Feast Kafka stream properties */ - private StreamProperties stream; - - @NotNull private SecurityProperties security; - - @Bean - SecurityProperties securityProperties() { - return getSecurity(); - } - - /* Feast Audit Logging properties */ - @NotNull private LoggingProperties logging; - - @Bean - LoggingProperties loggingProperties() { - return getLogging(); - } - - /** Properties used to configure Feast's managed Kafka feature stream. */ - @Getter - @Setter - public static class StreamProperties { - - /* Feature stream type. Only "kafka" is supported. */ - @OneOfStrings({"kafka"}) - @NotBlank - private String type; - - /* Feature stream options */ - @NotNull private FeatureStreamOptions options; - - /** Feature stream options */ - @Getter - @Setter - public static class FeatureStreamOptions { - - /* Kafka topic to use for feature sets without source topics. */ - @NotBlank private String topic = "feast-features"; - - /** - * Comma separated list of Kafka bootstrap servers. Used for feature sets without a defined - * source. - */ - @NotBlank private String bootstrapServers = "localhost:9092"; - - /* Defines the number of copies of managed feature stream Kafka. */ - @Positive private short replicationFactor = 1; - - /* Number of Kafka partitions to to use for managed feature stream. */ - @Positive private int partitions = 1; - } - } - - /** - * Validates all FeastProperties. This method runs after properties have been initialized and - * individually and conditionally validates each class. - */ - @PostConstruct - public void validate() { - ValidatorFactory factory = Validation.buildDefaultValidatorFactory(); - Validator validator = factory.getValidator(); - - // Validate root fields in FeastProperties - Set> violations = validator.validate(this); - if (!violations.isEmpty()) { - throw new ConstraintViolationException(violations); - } - - // Validate Stream properties - Set> streamPropertyViolations = - validator.validate(getStream()); - if (!streamPropertyViolations.isEmpty()) { - throw new ConstraintViolationException(streamPropertyViolations); - } - - // Validate Stream Options - Set> featureStreamOptionsViolations = - validator.validate(getStream().getOptions()); - if (!featureStreamOptionsViolations.isEmpty()) { - throw new ConstraintViolationException(featureStreamOptionsViolations); - } - - // Validate AuthenticationProperties - Set> authenticationPropsViolations = - validator.validate(getSecurity().getAuthentication()); - if (!authenticationPropsViolations.isEmpty()) { - throw new ConstraintViolationException(authenticationPropsViolations); - } - - // Validate AuthorizationProperties - Set> authorizationPropsViolations = - validator.validate(getSecurity().getAuthorization()); - if (!authorizationPropsViolations.isEmpty()) { - throw new ConstraintViolationException(authorizationPropsViolations); - } - } -} diff --git a/java/core/src/main/java/feast/core/config/JPAConfig.java b/java/core/src/main/java/feast/core/config/JPAConfig.java deleted file mode 100644 index dada6c9d3a..0000000000 --- a/java/core/src/main/java/feast/core/config/JPAConfig.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.config; - -import javax.persistence.EntityManagerFactory; -import lombok.extern.slf4j.Slf4j; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.orm.jpa.JpaTransactionManager; -import org.springframework.transaction.PlatformTransactionManager; - -/** Configuration of JPA related services and beans for the core application. */ -@Configuration -@Slf4j -public class JPAConfig { - @Bean - public PlatformTransactionManager transactionManager(EntityManagerFactory emf) { - JpaTransactionManager transactionManager = new JpaTransactionManager(); - transactionManager.setEntityManagerFactory(emf); - - return transactionManager; - } -} diff --git a/java/core/src/main/java/feast/core/config/MonitoringConfig.java b/java/core/src/main/java/feast/core/config/MonitoringConfig.java deleted file mode 100644 index 5fc6b8280e..0000000000 --- a/java/core/src/main/java/feast/core/config/MonitoringConfig.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.config; - -import feast.core.dao.FeatureTableRepository; -import feast.core.dao.StoreRepository; -import feast.core.metrics.collector.FeastResourceCollector; -import feast.core.metrics.collector.JVMResourceCollector; -import io.prometheus.client.exporter.MetricsServlet; -import javax.servlet.http.HttpServlet; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.web.servlet.ServletRegistrationBean; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -public class MonitoringConfig { - - private static final String PROMETHEUS_METRICS_PATH = "/metrics"; - - /** - * Add Prometheus exposition to an existing HTTP server using servlets. - * - *

https://github.com/prometheus/client_java/tree/b61dd232a504e20dad404a2bf3e2c0b8661c212a#http - * - * @return HTTP servlet for returning metrics data - */ - @Bean - public ServletRegistrationBean metricsServlet() { - return new ServletRegistrationBean<>(new MetricsServlet(), PROMETHEUS_METRICS_PATH); - } - - /** - * Register custom Prometheus collector that exports metrics about Feast Resources. - * - *

For example: total number of registered feature tables and stores. - * - * @param featureTableRepository {@link FeatureTableRepository} - * @param storeRepository {@link StoreRepository} - * @return {@link FeastResourceCollector} - */ - @Bean - @Autowired - public FeastResourceCollector feastResourceCollector( - FeatureTableRepository featureTableRepository, StoreRepository storeRepository) { - FeastResourceCollector collector = - new FeastResourceCollector(featureTableRepository, storeRepository); - collector.register(); - return collector; - } - - /** - * Register custom Prometheus collector that exports metrics about JVM resource usage. - * - * @return {@link JVMResourceCollector} - */ - @Bean - public JVMResourceCollector jvmResourceCollector() { - JVMResourceCollector jvmResourceCollector = new JVMResourceCollector(); - jvmResourceCollector.register(); - return jvmResourceCollector; - } -} diff --git a/java/core/src/main/java/feast/core/config/WebMvcConfig.java b/java/core/src/main/java/feast/core/config/WebMvcConfig.java deleted file mode 100644 index 5d6a6b8ece..0000000000 --- a/java/core/src/main/java/feast/core/config/WebMvcConfig.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.config; - -import java.util.List; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.http.converter.HttpMessageConverter; -import org.springframework.http.converter.protobuf.ProtobufJsonFormatHttpMessageConverter; -import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; - -/** Configuration for the spring web MVC layer */ -@Configuration -public class WebMvcConfig implements WebMvcConfigurer { - /** - * Get a json-protobuf converter. - * - * @return ProtobufJsonFormatHttpMessageConverter - */ - @Bean - ProtobufJsonFormatHttpMessageConverter getProtobufHttpMessageConverter() { - return new ProtobufJsonFormatHttpMessageConverter(); - } - - /** Register json-protobuf converter. */ - @Override - public void configureMessageConverters(List> converters) { - converters.add(getProtobufHttpMessageConverter()); - } -} diff --git a/java/core/src/main/java/feast/core/config/WebSecurityConfig.java b/java/core/src/main/java/feast/core/config/WebSecurityConfig.java deleted file mode 100644 index 5c66730d6a..0000000000 --- a/java/core/src/main/java/feast/core/config/WebSecurityConfig.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.config; - -import java.util.ArrayList; -import java.util.List; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Configuration; -import org.springframework.security.config.annotation.web.builders.HttpSecurity; -import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; -import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; - -/** - * WebSecurityConfig disables auto configuration of Spring HTTP Security and allows security methods - * to be overridden - */ -@Configuration -@EnableWebSecurity -public class WebSecurityConfig extends WebSecurityConfigurerAdapter { - - private final FeastProperties feastProperties; - - @Autowired - public WebSecurityConfig(FeastProperties feastProperties) { - this.feastProperties = feastProperties; - } - - /** - * Allows for custom web security rules to be applied. - * - * @param http {@link HttpSecurity} for configuring web based security - * @throws Exception unexpected exception - */ - @Override - protected void configure(HttpSecurity http) throws Exception { - List matchersToBypass = new ArrayList<>(List.of("/actuator/**", "/metrics/**")); - - if (feastProperties.securityProperties().isDisableRestControllerAuth()) { - matchersToBypass.add("/api/v1/**"); - matchersToBypass.add("/api/v2/**"); - } - - // Bypasses security/authentication for the following paths - http.authorizeRequests() - .antMatchers(matchersToBypass.toArray(new String[0])) - .permitAll() - .anyRequest() - .authenticated() - .and() - .csrf() - .disable(); - } -} diff --git a/java/core/src/main/java/feast/core/controller/CoreServiceRestController.java b/java/core/src/main/java/feast/core/controller/CoreServiceRestController.java deleted file mode 100644 index f782c172cd..0000000000 --- a/java/core/src/main/java/feast/core/controller/CoreServiceRestController.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.controller; - -import feast.core.config.FeastProperties; -import feast.core.model.Project; -import feast.core.service.ProjectService; -import feast.core.service.SpecService; -import feast.proto.core.CoreServiceProto.GetFeastCoreVersionResponse; -import feast.proto.core.CoreServiceProto.ListEntitiesRequest; -import feast.proto.core.CoreServiceProto.ListEntitiesResponse; -import feast.proto.core.CoreServiceProto.ListFeatureTablesRequest; -import feast.proto.core.CoreServiceProto.ListFeatureTablesResponse; -import feast.proto.core.CoreServiceProto.ListFeaturesRequest; -import feast.proto.core.CoreServiceProto.ListFeaturesResponse; -import feast.proto.core.CoreServiceProto.ListProjectsResponse; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.CrossOrigin; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; - -/** - * EXPERIMENTAL: Controller for HTTP endpoints to Feast Core. These endpoints are subject to change. - */ -@RestController -@CrossOrigin -@Slf4j -@RequestMapping(value = "/api", produces = "application/json") -public class CoreServiceRestController { - - private final FeastProperties feastProperties; - private SpecService specService; - private ProjectService projectService; - - @Autowired - public CoreServiceRestController( - FeastProperties feastProperties, SpecService specService, ProjectService projectService) { - this.feastProperties = feastProperties; - this.specService = specService; - this.projectService = projectService; - } - - /** - * GET /version : Fetches the version of Feast Core. - * - * @return (200 OK) Returns {@link GetFeastCoreVersionResponse} in JSON. - */ - @RequestMapping(value = "/v2/version", method = RequestMethod.GET) - public GetFeastCoreVersionResponse getVersion() { - GetFeastCoreVersionResponse response = - GetFeastCoreVersionResponse.newBuilder().setVersion(feastProperties.getVersion()).build(); - return response; - } - - /** - * GET /features : List Features based on project and entities. - * - * @param entities Request Parameter: List of all entities every returned feature should belong - * to. At least one entity is required. For example, if entity1 and entity2 - * are given, then all features returned (if any) will belong to BOTH - * entities. - * @param project (Optional) Request Parameter: A single project where the feature table of all - * features returned is under. If not provided, the default project will be used, usually - * default. - * @return (200 OK) Return {@link ListFeaturesResponse} in JSON. - */ - @RequestMapping(value = "/v2/features", method = RequestMethod.GET) - public ListFeaturesResponse listFeatures( - @RequestParam String[] entities, @RequestParam(required = false) Optional project) { - ListFeaturesRequest.Filter.Builder filterBuilder = - ListFeaturesRequest.Filter.newBuilder().addAllEntities(Arrays.asList(entities)); - project.ifPresent(filterBuilder::setProject); - return specService.listFeatures(filterBuilder.build()); - } - - /** - * GET /projects : Get the list of existing feast projects. - * - * @return (200 OK) Returns {@link ListProjectsResponse} in JSON. - */ - @RequestMapping(value = "/v2/projects", method = RequestMethod.GET) - public ListProjectsResponse listProjects() { - List projects = projectService.listProjects(); - return ListProjectsResponse.newBuilder() - .addAllProjects(projects.stream().map(Project::getName).collect(Collectors.toList())) - .build(); - } - - /** - * GET /entities : Retrieve a list of Entities according to filtering parameters of Feast project - * name. If none matches, an empty JSON response is returned. - * - * @param project Request Parameter: Name of feast project to search in. - * @return (200 OK) Return {@link ListEntitiesResponse} in JSON. - */ - @RequestMapping(value = "/v2/entities", method = RequestMethod.GET) - public ListEntitiesResponse listEntities( - @RequestParam(defaultValue = Project.DEFAULT_NAME) String project) { - ListEntitiesRequest.Filter.Builder filterBuilder = - ListEntitiesRequest.Filter.newBuilder().setProject(project); - return specService.listEntities(filterBuilder.build()); - } - - /** - * GET /feature-tables : Retrieve a list of Feature Tables according to filtering parameters of - * Feast project name. If none matches, an empty JSON response is returned. - * - * @param project Request Parameter: Name of feast project to search in. - * @return (200 OK) Return {@link ListFeatureTablesResponse} in JSON. - */ - @RequestMapping(value = "/v2/feature-tables", method = RequestMethod.GET) - public ListFeatureTablesResponse listFeatureTables( - @RequestParam(defaultValue = Project.DEFAULT_NAME) String project) { - ListFeatureTablesRequest.Filter.Builder filterBuilder = - ListFeatureTablesRequest.Filter.newBuilder().setProject(project); - return specService.listFeatureTables(filterBuilder.build()); - } -} diff --git a/java/core/src/main/java/feast/core/controller/exception/handler/RestResponseEntityExceptionHandler.java b/java/core/src/main/java/feast/core/controller/exception/handler/RestResponseEntityExceptionHandler.java deleted file mode 100644 index ef27e4aee8..0000000000 --- a/java/core/src/main/java/feast/core/controller/exception/handler/RestResponseEntityExceptionHandler.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.controller.exception.handler; - -import com.google.protobuf.InvalidProtocolBufferException; -import feast.core.exception.RetrievalException; -import java.util.Map; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.MissingServletRequestParameterException; -import org.springframework.web.bind.UnsatisfiedServletRequestParameterException; -import org.springframework.web.bind.annotation.ControllerAdvice; -import org.springframework.web.bind.annotation.ExceptionHandler; -import org.springframework.web.context.request.WebRequest; -import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler; - -/** A exception handler for some common exceptions while accessing Feast Core via HTTP. */ -@ControllerAdvice -public class RestResponseEntityExceptionHandler extends ResponseEntityExceptionHandler { - - /** - * Handles the case when a request object (such as {@link - * feast.proto.core.CoreServiceProto.GetFeatureTableRequest}) or a response object (such as {@link - * feast.proto.core.CoreServiceProto.GetFeatureTableResponse} is malformed. - * - * @param ex the {@link InvalidProtocolBufferException} that occurred. - * @param request the {@link WebRequest} that caused this exception. - * @return (500 Internal Server Error) - */ - @ExceptionHandler({InvalidProtocolBufferException.class}) - protected ResponseEntity handleInvalidProtocolBuffer( - InvalidProtocolBufferException ex, WebRequest request) { - Map bodyOfResponse = - Map.of("error", "An unexpected error occurred in Feast Core."); - return handleExceptionInternal( - ex, bodyOfResponse, new HttpHeaders(), HttpStatus.INTERNAL_SERVER_ERROR, request); - } - - /** - * Handles the case that retrieval of information from the services triggered and exception. - * Instead of returning 500 with no error message, returns 500 with a body describing the error - * message. - * - * @param ex the {@link RetrievalException} that occurred. - * @param request the {@link WebRequest} that caused this exception. - * @return (500 Internal Server Error) - */ - @ExceptionHandler({RetrievalException.class}) - protected ResponseEntity handleRetrieval(RetrievalException ex, WebRequest request) { - Map bodyOfResponse = Map.of("error", ex.getMessage()); - return handleExceptionInternal( - ex, bodyOfResponse, new HttpHeaders(), HttpStatus.INTERNAL_SERVER_ERROR, request); - } - - /** - * Handles various exceptions that are due to malformed or invalid requests, such as - * - *
    - *
  • {@link UnsatisfiedServletRequestParameterException} where a parameter is requested in - * {@link org.springframework.web.bind.annotation.RequestMapping} but not supplied. - *
  • {@link IllegalArgumentException} where unsupported parameters are provided. - *
- * - * @param ex the {@link UnsatisfiedServletRequestParameterException} that occurred. - * @param request the {@link WebRequest} that caused this exception. - * @return (400 Bad Request) - */ - @ExceptionHandler({ - UnsatisfiedServletRequestParameterException.class, - IllegalArgumentException.class - }) - protected ResponseEntity handleBadRequest(Exception ex, WebRequest request) { - ex.printStackTrace(); - Map bodyOfResponse = Map.of("error", ex.getMessage()); - return handleExceptionInternal( - ex, bodyOfResponse, new HttpHeaders(), HttpStatus.BAD_REQUEST, request); - } - - /** - * Handles {@link MissingServletRequestParameterException} which occurs when a controller method - * expects a certain parameter but is not supplied by the request. The original implementation - * returns an empty body with (400 Bad Request), we add in the error and stacktrace. - * - * @param ex the {@link MissingServletRequestParameterException} that occurred. - * @param request the {@link WebRequest} that caused this exception. - * @param headers the {@link HttpHeaders} from the request. - * @param status the {@link HttpStatus} generated for the response, (400 Bad Request) - * @return (400 Bad Request) - */ - @Override - protected ResponseEntity handleMissingServletRequestParameter( - MissingServletRequestParameterException ex, - HttpHeaders headers, - HttpStatus status, - WebRequest request) { - ex.printStackTrace(); - Map bodyOfResponse = Map.of("error", ex.getMessage()); - return this.handleExceptionInternal(ex, bodyOfResponse, headers, status, request); - } -} diff --git a/java/core/src/main/java/feast/core/dao/EntityRepository.java b/java/core/src/main/java/feast/core/dao/EntityRepository.java deleted file mode 100644 index d7d7dcb5b9..0000000000 --- a/java/core/src/main/java/feast/core/dao/EntityRepository.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.dao; - -import feast.core.model.EntityV2; -import java.util.List; -import org.springframework.data.jpa.repository.JpaRepository; - -/** JPA repository supplying EntityV2 objects keyed by id. */ -public interface EntityRepository extends JpaRepository { - - long count(); - - // Find all EntityV2s by project - List findAllByProject_Name(String project); - - // Find single EntityV2 by project and name - EntityV2 findEntityByNameAndProject_Name(String name, String project); -} diff --git a/java/core/src/main/java/feast/core/dao/FeatureTableRepository.java b/java/core/src/main/java/feast/core/dao/FeatureTableRepository.java deleted file mode 100644 index 2d125345a9..0000000000 --- a/java/core/src/main/java/feast/core/dao/FeatureTableRepository.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.dao; - -import feast.core.model.FeatureTable; -import java.util.List; -import java.util.Optional; -import org.springframework.data.jpa.repository.JpaRepository; - -/** JPA repository for querying FeatureTables stored. */ -public interface FeatureTableRepository extends JpaRepository { - // Find single FeatureTable by project and name - Optional findFeatureTableByNameAndProject_Name(String name, String projectName); - - // Find FeatureTables by project - List findAllByProject_Name(String projectName); -} diff --git a/java/core/src/main/java/feast/core/dao/ProjectRepository.java b/java/core/src/main/java/feast/core/dao/ProjectRepository.java deleted file mode 100644 index 5adb7d44c2..0000000000 --- a/java/core/src/main/java/feast/core/dao/ProjectRepository.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.dao; - -import feast.core.model.Project; -import java.util.List; -import org.springframework.data.jpa.repository.JpaRepository; - -/** JPA repository supplying Project objects keyed by id. */ -public interface ProjectRepository extends JpaRepository { - - List findAllByArchivedIsFalse(); -} diff --git a/java/core/src/main/java/feast/core/dao/StoreRepository.java b/java/core/src/main/java/feast/core/dao/StoreRepository.java deleted file mode 100644 index 2cc0070471..0000000000 --- a/java/core/src/main/java/feast/core/dao/StoreRepository.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.dao; - -import feast.core.model.Store; -import org.springframework.data.jpa.repository.JpaRepository; - -/** JPA repository supplying Store objects keyed by id. */ -public interface StoreRepository extends JpaRepository { - long count(); -} diff --git a/java/core/src/main/java/feast/core/exception/RegistrationException.java b/java/core/src/main/java/feast/core/exception/RegistrationException.java deleted file mode 100644 index ccb4d55cae..0000000000 --- a/java/core/src/main/java/feast/core/exception/RegistrationException.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.exception; - -/** Exception thrown when a spec is fails to be registered to the Feast metadata registry. */ -public class RegistrationException extends RuntimeException { - public RegistrationException() { - super(); - } - - public RegistrationException(String message) { - super(message); - } - - public RegistrationException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/java/core/src/main/java/feast/core/exception/RetrievalException.java b/java/core/src/main/java/feast/core/exception/RetrievalException.java deleted file mode 100644 index bd543048a9..0000000000 --- a/java/core/src/main/java/feast/core/exception/RetrievalException.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.exception; - -/** Exception thrown when retrieval of a spec from the registry fails. */ -public class RetrievalException extends RuntimeException { - public RetrievalException() { - super(); - } - - public RetrievalException(String message) { - super(message); - } - - public RetrievalException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/java/core/src/main/java/feast/core/grpc/CoreServiceImpl.java b/java/core/src/main/java/feast/core/grpc/CoreServiceImpl.java deleted file mode 100644 index 648195a88c..0000000000 --- a/java/core/src/main/java/feast/core/grpc/CoreServiceImpl.java +++ /dev/null @@ -1,410 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.grpc; - -import feast.common.auth.service.AuthorizationService; -import feast.common.logging.interceptors.GrpcMessageInterceptor; -import feast.core.config.FeastProperties; -import feast.core.exception.RetrievalException; -import feast.core.grpc.interceptors.MonitoringInterceptor; -import feast.core.model.Project; -import feast.core.service.ProjectService; -import feast.core.service.SpecService; -import feast.proto.core.CoreServiceGrpc.CoreServiceImplBase; -import feast.proto.core.CoreServiceProto.*; -import feast.proto.core.EntityProto.EntitySpecV2; -import io.grpc.Status; -import io.grpc.StatusRuntimeException; -import io.grpc.stub.StreamObserver; -import java.util.List; -import java.util.NoSuchElementException; -import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import net.devh.boot.grpc.server.service.GrpcService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.security.access.AccessDeniedException; -import org.springframework.security.core.context.SecurityContextHolder; - -/** Implementation of the feast core GRPC service. */ -@Slf4j -@GrpcService(interceptors = {GrpcMessageInterceptor.class, MonitoringInterceptor.class}) -public class CoreServiceImpl extends CoreServiceImplBase { - - private final FeastProperties feastProperties; - private SpecService specService; - private ProjectService projectService; - private final AuthorizationService authorizationService; - - @Autowired - public CoreServiceImpl( - SpecService specService, - ProjectService projectService, - FeastProperties feastProperties, - AuthorizationService authorizationService) { - this.specService = specService; - this.projectService = projectService; - this.feastProperties = feastProperties; - this.authorizationService = authorizationService; - } - - @Override - public void getFeastCoreVersion( - GetFeastCoreVersionRequest request, - StreamObserver responseObserver) { - try { - GetFeastCoreVersionResponse response = - GetFeastCoreVersionResponse.newBuilder().setVersion(feastProperties.getVersion()).build(); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (RetrievalException | StatusRuntimeException e) { - log.error("Could not determine Feast Core version: ", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } - - @Override - public void getEntity( - GetEntityRequest request, StreamObserver responseObserver) { - try { - GetEntityResponse response = specService.getEntity(request); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (RetrievalException e) { - log.error("Unable to fetch entity requested in GetEntity method: ", e); - responseObserver.onError( - Status.NOT_FOUND.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } catch (IllegalArgumentException e) { - log.error("Illegal arguments provided to GetEntity method: ", e); - responseObserver.onError( - Status.INVALID_ARGUMENT - .withDescription(e.getMessage()) - .withCause(e) - .asRuntimeException()); - } catch (Exception e) { - log.error("Exception has occurred in GetEntity method: ", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } - - /** Retrieve a list of features */ - @Override - public void listFeatures( - ListFeaturesRequest request, StreamObserver responseObserver) { - try { - ListFeaturesResponse response = specService.listFeatures(request.getFilter()); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (IllegalArgumentException e) { - log.error("Illegal arguments provided to ListFeatures method: ", e); - responseObserver.onError( - Status.INVALID_ARGUMENT - .withDescription(e.getMessage()) - .withCause(e) - .asRuntimeException()); - } catch (RetrievalException e) { - log.error("Unable to fetch entities requested in ListFeatures method: ", e); - responseObserver.onError( - Status.NOT_FOUND.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } catch (Exception e) { - log.error("Exception has occurred in ListFeatures method: ", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } - - /** Retrieve a list of entities */ - @Override - public void listEntities( - ListEntitiesRequest request, StreamObserver responseObserver) { - try { - ListEntitiesResponse response = specService.listEntities(request.getFilter()); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (IllegalArgumentException e) { - log.error("Illegal arguments provided to ListEntities method: ", e); - responseObserver.onError( - Status.INVALID_ARGUMENT - .withDescription(e.getMessage()) - .withCause(e) - .asRuntimeException()); - } catch (RetrievalException e) { - log.error("Unable to fetch entities requested in ListEntities method: ", e); - responseObserver.onError( - Status.NOT_FOUND.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } catch (Exception e) { - log.error("Exception has occurred in ListEntities method: ", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } - - @Override - public void listStores( - ListStoresRequest request, StreamObserver responseObserver) { - try { - ListStoresResponse response = specService.listStores(request.getFilter()); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (RetrievalException e) { - log.error("Exception has occurred in ListStores method: ", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } - - /* Registers an entity to Feast Core */ - @Override - public void applyEntity( - ApplyEntityRequest request, StreamObserver responseObserver) { - - String projectId = null; - - try { - EntitySpecV2 spec = request.getSpec(); - projectId = request.getProject(); - authorizationService.authorizeRequest(SecurityContextHolder.getContext(), projectId); - ApplyEntityResponse response = specService.applyEntity(spec, projectId); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (org.hibernate.exception.ConstraintViolationException e) { - log.error( - "Unable to persist this entity due to a constraint violation. Please ensure that" - + " field names are unique within the project namespace: ", - e); - responseObserver.onError( - Status.ALREADY_EXISTS.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } catch (AccessDeniedException e) { - log.info(String.format("User prevented from accessing project: %s", projectId)); - responseObserver.onError( - Status.PERMISSION_DENIED - .withDescription(e.getMessage()) - .withCause(e) - .asRuntimeException()); - } catch (Exception e) { - log.error("Exception has occurred in ApplyEntity method: ", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } - - @Override - public void updateStore( - UpdateStoreRequest request, StreamObserver responseObserver) { - try { - UpdateStoreResponse response = specService.updateStore(request); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (Exception e) { - log.error("Exception has occurred in UpdateStore method: ", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } - - @Override - public void createProject( - CreateProjectRequest request, StreamObserver responseObserver) { - try { - projectService.createProject(request.getName()); - responseObserver.onNext(CreateProjectResponse.getDefaultInstance()); - responseObserver.onCompleted(); - } catch (Exception e) { - log.error("Exception has occurred in the createProject method: ", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } - - @Override - public void archiveProject( - ArchiveProjectRequest request, StreamObserver responseObserver) { - String projectId = null; - try { - projectId = request.getName(); - authorizationService.authorizeRequest(SecurityContextHolder.getContext(), projectId); - projectService.archiveProject(projectId); - responseObserver.onNext(ArchiveProjectResponse.getDefaultInstance()); - responseObserver.onCompleted(); - } catch (IllegalArgumentException e) { - log.error("Recieved an invalid request on calling archiveProject method:", e); - responseObserver.onError( - Status.INVALID_ARGUMENT - .withDescription(e.getMessage()) - .withCause(e) - .asRuntimeException()); - } catch (UnsupportedOperationException e) { - log.error("Attempted to archive an unsupported project:", e); - responseObserver.onError( - Status.UNIMPLEMENTED.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } catch (AccessDeniedException e) { - log.info(String.format("User prevented from accessing project: %s", projectId)); - responseObserver.onError( - Status.PERMISSION_DENIED - .withDescription(e.getMessage()) - .withCause(e) - .asRuntimeException()); - } catch (Exception e) { - log.error("Exception has occurred in the createProject method: ", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } - - @Override - public void listProjects( - ListProjectsRequest request, StreamObserver responseObserver) { - try { - List projects = projectService.listProjects(); - responseObserver.onNext( - ListProjectsResponse.newBuilder() - .addAllProjects(projects.stream().map(Project::getName).collect(Collectors.toList())) - .build()); - responseObserver.onCompleted(); - } catch (Exception e) { - log.error("Exception has occurred in the listProjects method: ", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } - - @Override - public void applyFeatureTable( - ApplyFeatureTableRequest request, - StreamObserver responseObserver) { - String projectName = SpecService.resolveProjectName(request.getProject()); - String tableName = request.getTableSpec().getName(); - - try { - // Check if user has authorization to apply feature table - authorizationService.authorizeRequest(SecurityContextHolder.getContext(), projectName); - - ApplyFeatureTableResponse response = specService.applyFeatureTable(request); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (AccessDeniedException e) { - log.info( - String.format( - "ApplyFeatureTable: Not authorized to access project to apply: %s", projectName)); - responseObserver.onError( - Status.PERMISSION_DENIED - .withDescription(e.getMessage()) - .withCause(e) - .asRuntimeException()); - } catch (org.hibernate.exception.ConstraintViolationException e) { - log.error( - String.format( - "ApplyFeatureTable: Unable to apply Feature Table due to a conflict: " - + "Ensure that name is unique within Project: (name: %s, project: %s)", - tableName, projectName)); - responseObserver.onError( - Status.ALREADY_EXISTS.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } catch (IllegalArgumentException e) { - log.error( - String.format( - "ApplyFeatureTable: Invalid apply Feature Table Request: (name: %s, project: %s)", - tableName, projectName)); - responseObserver.onError( - Status.INVALID_ARGUMENT - .withDescription(e.getMessage()) - .withCause(e) - .asRuntimeException()); - } catch (UnsupportedOperationException e) { - log.error( - String.format( - "ApplyFeatureTable: Unsupported apply Feature Table Request: (name: %s, project: %s)", - tableName, projectName)); - responseObserver.onError( - Status.UNIMPLEMENTED.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } catch (Exception e) { - log.error("ApplyFeatureTable Exception has occurred:", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } - - @Override - public void listFeatureTables( - ListFeatureTablesRequest request, - StreamObserver responseObserver) { - try { - ListFeatureTablesResponse response = specService.listFeatureTables(request.getFilter()); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (IllegalArgumentException e) { - log.error(String.format("ListFeatureTable: Invalid list Feature Table Request")); - responseObserver.onError( - Status.INVALID_ARGUMENT - .withDescription(e.getMessage()) - .withCause(e) - .asRuntimeException()); - } catch (Exception e) { - log.error("ListFeatureTable: Exception has occurred: ", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } - - @Override - public void getFeatureTable( - GetFeatureTableRequest request, StreamObserver responseObserver) { - try { - GetFeatureTableResponse response = specService.getFeatureTable(request); - - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (NoSuchElementException e) { - log.error( - String.format( - "GetFeatureTable: No such Feature Table: (project: %s, name: %s)", - request.getProject(), request.getName())); - responseObserver.onError( - Status.NOT_FOUND.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } catch (Exception e) { - log.error("GetFeatureTable: Exception has occurred: ", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } - - @Override - public void deleteFeatureTable( - DeleteFeatureTableRequest request, - StreamObserver responseObserver) { - String projectName = request.getProject(); - try { - // Check if user has authorization to delete feature table - authorizationService.authorizeRequest(SecurityContextHolder.getContext(), projectName); - specService.deleteFeatureTable(request); - - responseObserver.onNext(DeleteFeatureTableResponse.getDefaultInstance()); - responseObserver.onCompleted(); - } catch (NoSuchElementException e) { - log.error( - String.format( - "DeleteFeatureTable: No such Feature Table: (project: %s, name: %s)", - request.getProject(), request.getName())); - responseObserver.onError( - Status.NOT_FOUND.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } catch (Exception e) { - log.error("DeleteFeatureTable: Exception has occurred: ", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } -} diff --git a/java/core/src/main/java/feast/core/grpc/HealthServiceImpl.java b/java/core/src/main/java/feast/core/grpc/HealthServiceImpl.java deleted file mode 100644 index ed6e0a1c72..0000000000 --- a/java/core/src/main/java/feast/core/grpc/HealthServiceImpl.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.grpc; - -import feast.core.service.ProjectService; -import io.grpc.Status; -import io.grpc.health.v1.HealthGrpc.HealthImplBase; -import io.grpc.health.v1.HealthProto.HealthCheckRequest; -import io.grpc.health.v1.HealthProto.HealthCheckResponse; -import io.grpc.health.v1.HealthProto.ServingStatus; -import io.grpc.stub.StreamObserver; -import lombok.extern.slf4j.Slf4j; -import net.devh.boot.grpc.server.service.GrpcService; -import org.springframework.beans.factory.annotation.Autowired; - -@Slf4j -@GrpcService -public class HealthServiceImpl extends HealthImplBase { - private final ProjectService projectService; - - @Autowired - public HealthServiceImpl(ProjectService projectService) { - this.projectService = projectService; - } - - @Override - public void check( - HealthCheckRequest request, StreamObserver responseObserver) { - try { - projectService.listProjects(); - responseObserver.onNext( - HealthCheckResponse.newBuilder().setStatus(ServingStatus.SERVING).build()); - responseObserver.onCompleted(); - } catch (Exception e) { - log.error("Health Check: unable to retrieve projects.\nError: %s", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } -} diff --git a/java/core/src/main/java/feast/core/grpc/interceptors/MonitoringInterceptor.java b/java/core/src/main/java/feast/core/grpc/interceptors/MonitoringInterceptor.java deleted file mode 100644 index 870675594a..0000000000 --- a/java/core/src/main/java/feast/core/grpc/interceptors/MonitoringInterceptor.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.grpc.interceptors; - -import feast.core.metrics.GrpcMetrics; -import io.grpc.ForwardingServerCall.SimpleForwardingServerCall; -import io.grpc.Metadata; -import io.grpc.MethodDescriptor; -import io.grpc.ServerCall; -import io.grpc.ServerCall.Listener; -import io.grpc.ServerCallHandler; -import io.grpc.ServerInterceptor; -import io.grpc.Status; - -/** - * MonitoringInterceptor intercepts a GRPC call to provide a request latency historgram metrics in - * the Prometheus client. - */ -public class MonitoringInterceptor implements ServerInterceptor { - - @Override - public Listener interceptCall( - ServerCall call, Metadata headers, ServerCallHandler next) { - - long startCallMillis = System.currentTimeMillis(); - String fullMethodName = call.getMethodDescriptor().getFullMethodName(); - String serviceName = MethodDescriptor.extractFullServiceName(fullMethodName); - String methodName = fullMethodName.substring(fullMethodName.indexOf("/") + 1); - - return next.startCall( - new SimpleForwardingServerCall(call) { - @Override - public void close(Status status, Metadata trailers) { - GrpcMetrics.requestLatency - .labels(serviceName, methodName, status.getCode().name()) - .observe((System.currentTimeMillis() - startCallMillis) / 1000f); - super.close(status, trailers); - } - }, - headers); - } -} diff --git a/java/core/src/main/java/feast/core/metrics/GrpcMetrics.java b/java/core/src/main/java/feast/core/metrics/GrpcMetrics.java deleted file mode 100644 index 7c20e63250..0000000000 --- a/java/core/src/main/java/feast/core/metrics/GrpcMetrics.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.metrics; - -import io.prometheus.client.Histogram; - -public class GrpcMetrics { - - public static final Histogram requestLatency = - Histogram.build() - .name("feast_core_request_latency_seconds") - .labelNames("service", "method", "status_code") - .help("Request latency in seconds") - .register(); -} diff --git a/java/core/src/main/java/feast/core/metrics/collector/FeastResourceCollector.java b/java/core/src/main/java/feast/core/metrics/collector/FeastResourceCollector.java deleted file mode 100644 index 3064a25b8f..0000000000 --- a/java/core/src/main/java/feast/core/metrics/collector/FeastResourceCollector.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.metrics.collector; - -import feast.core.dao.FeatureTableRepository; -import feast.core.dao.StoreRepository; -import io.prometheus.client.Collector; -import io.prometheus.client.GaugeMetricFamily; -import java.util.ArrayList; -import java.util.List; - -/** - * FeastResourceCollector exports metrics about Feast Resources. - * - *

For example: total number of registered feature tables and stores. - */ -public class FeastResourceCollector extends Collector { - - private final FeatureTableRepository featureTableRepository; - private final StoreRepository storeRepository; - - public FeastResourceCollector( - FeatureTableRepository featureTableRepository, StoreRepository storeRepository) { - this.featureTableRepository = featureTableRepository; - this.storeRepository = storeRepository; - } - - @Override - public List collect() { - List samples = new ArrayList<>(); - samples.add( - new GaugeMetricFamily( - "feast_core_feature_set_total", - "Total number of registered feature tables", - featureTableRepository.count())); - samples.add( - new GaugeMetricFamily( - "feast_core_store_total", - "Total number of registered stores", - storeRepository.count())); - return samples; - } -} diff --git a/java/core/src/main/java/feast/core/metrics/collector/JVMResourceCollector.java b/java/core/src/main/java/feast/core/metrics/collector/JVMResourceCollector.java deleted file mode 100644 index 8602f6c249..0000000000 --- a/java/core/src/main/java/feast/core/metrics/collector/JVMResourceCollector.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.metrics.collector; - -import io.prometheus.client.Collector; -import io.prometheus.client.GaugeMetricFamily; -import io.prometheus.client.SummaryMetricFamily; -import java.lang.management.GarbageCollectorMXBean; -import java.lang.management.ManagementFactory; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -/** - * JVMResourceCollector exports metrics about Java virtual machine memory and garbage collection. - */ -public class JVMResourceCollector extends Collector { - - private final List garbageCollectors; - private final Runtime runtime; - - public JVMResourceCollector() { - garbageCollectors = ManagementFactory.getGarbageCollectorMXBeans(); - runtime = Runtime.getRuntime(); - } - - @Override - public List collect() { - List samples = new ArrayList<>(); - - samples.add( - new GaugeMetricFamily( - "feast_core_max_memory_bytes", - "Max amount of memory the Java virtual machine will attempt to use", - runtime.maxMemory())); - samples.add( - new GaugeMetricFamily( - "feast_core_total_memory_bytes", - "Total amount of memory in the Java virtual machine", - runtime.totalMemory())); - samples.add( - new GaugeMetricFamily( - "feast_core_free_memory_bytes", - "Total amount of free memory in the Java virtual machine", - runtime.freeMemory())); - - SummaryMetricFamily gcMetricFamily = - new SummaryMetricFamily( - "feast_core_gc_collection_seconds", - "Time spent in a given JVM garbage collector in seconds", - Collections.singletonList("gc")); - for (final GarbageCollectorMXBean gc : garbageCollectors) { - gcMetricFamily.addMetric( - Collections.singletonList(gc.getName()), - gc.getCollectionCount(), - gc.getCollectionTime() / MILLISECONDS_PER_SECOND); - } - samples.add(gcMetricFamily); - - return samples; - } -} diff --git a/java/core/src/main/java/feast/core/model/AbstractTimestampEntity.java b/java/core/src/main/java/feast/core/model/AbstractTimestampEntity.java deleted file mode 100644 index 188f19b5bc..0000000000 --- a/java/core/src/main/java/feast/core/model/AbstractTimestampEntity.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.model; - -import java.time.Instant; -import java.util.Date; -import javax.persistence.*; -import lombok.Data; - -/** - * Base object class ensuring that all objects stored in the registry have an auto-generated - * creation and last updated time. - */ -@MappedSuperclass -@Data -public abstract class AbstractTimestampEntity { - @Temporal(TemporalType.TIMESTAMP) - @Column(name = "created", nullable = false) - private Date created; - - @Temporal(TemporalType.TIMESTAMP) - @Column(name = "last_updated", nullable = false) - private Date lastUpdated; - - @PrePersist - protected void onCreate() { - lastUpdated = created = new Date(); - } - - @PreUpdate - protected void onUpdate() { - lastUpdated = new Date(); - } - - // This constructor is used for testing. - public AbstractTimestampEntity() { - this.created = Date.from(Instant.ofEpochMilli(0L)); - this.lastUpdated = Date.from(Instant.ofEpochMilli(0L)); - } -} diff --git a/java/core/src/main/java/feast/core/model/DataSource.java b/java/core/src/main/java/feast/core/model/DataSource.java deleted file mode 100644 index 2bfe20f809..0000000000 --- a/java/core/src/main/java/feast/core/model/DataSource.java +++ /dev/null @@ -1,239 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.model; - -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.Message; -import com.google.protobuf.MessageOrBuilder; -import com.google.protobuf.util.JsonFormat; -import feast.core.util.TypeConversion; -import feast.proto.core.DataFormatProto.FileFormat; -import feast.proto.core.DataFormatProto.StreamFormat; -import feast.proto.core.DataSourceProto; -import feast.proto.core.DataSourceProto.DataSource.BigQueryOptions; -import feast.proto.core.DataSourceProto.DataSource.FileOptions; -import feast.proto.core.DataSourceProto.DataSource.KafkaOptions; -import feast.proto.core.DataSourceProto.DataSource.KinesisOptions; -import feast.proto.core.DataSourceProto.DataSource.SourceType; -import java.util.HashMap; -import java.util.Map; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.EnumType; -import javax.persistence.Enumerated; -import javax.persistence.GeneratedValue; -import javax.persistence.Id; -import javax.persistence.Table; -import lombok.AccessLevel; -import lombok.Getter; -import lombok.Setter; - -@Entity -@Getter -@Setter(AccessLevel.PRIVATE) -@Table(name = "data_sources") -public class DataSource { - @Column(name = "id") - @Id - @GeneratedValue - private long id; - - // Type of this Data Source - @Enumerated(EnumType.STRING) - @Column(name = "type", nullable = false) - private SourceType type; - - // DataSource Options - @Column(name = "config") - private String configJSON; - - // Field mapping between sourced fields (key) and feature fields (value). - // Stored as serialized JSON string. - @Column(name = "field_mapping", columnDefinition = "text") - private String fieldMapJSON; - - @Column(name = "timestamp_column") - private String eventTimestampColumn; - - @Column(name = "created_timestamp_column") - private String createdTimestampColumn; - - @Column(name = "date_partition_column") - private String datePartitionColumn; - - public DataSource() {}; - - public DataSource(SourceType type) { - this.type = type; - } - - /** - * Construct a DataSource from the given Protobuf representation spec - * - * @param spec Protobuf representation of DataSource to construct from. - * @throws IllegalArgumentException when provided with a invalid Protobuf spec - * @throws UnsupportedOperationException if source type is unsupported. - * @return data source - */ - public static DataSource fromProto(DataSourceProto.DataSource spec) { - DataSource source = new DataSource(spec.getType()); - // Copy source type specific options - Map dataSourceConfigMap = new HashMap<>(); - switch (spec.getType()) { - case BATCH_FILE: - dataSourceConfigMap.put("file_url", spec.getFileOptions().getFileUrl()); - dataSourceConfigMap.put("file_format", printJSON(spec.getFileOptions().getFileFormat())); - break; - case BATCH_BIGQUERY: - dataSourceConfigMap.put("table_ref", spec.getBigqueryOptions().getTableRef()); - break; - case STREAM_KAFKA: - dataSourceConfigMap.put("bootstrap_servers", spec.getKafkaOptions().getBootstrapServers()); - dataSourceConfigMap.put( - "message_format", printJSON(spec.getKafkaOptions().getMessageFormat())); - dataSourceConfigMap.put("topic", spec.getKafkaOptions().getTopic()); - break; - case STREAM_KINESIS: - dataSourceConfigMap.put( - "record_format", printJSON(spec.getKinesisOptions().getRecordFormat())); - dataSourceConfigMap.put("region", spec.getKinesisOptions().getRegion()); - dataSourceConfigMap.put("stream_name", spec.getKinesisOptions().getStreamName()); - - break; - default: - throw new UnsupportedOperationException( - String.format("Unsupported Feature Store Type: %s", spec.getType())); - } - - // Store DataSource mapping as serialised JSON - source.setConfigJSON(TypeConversion.convertMapToJsonString(dataSourceConfigMap)); - - // Store field mapping as serialised JSON - source.setFieldMapJSON(TypeConversion.convertMapToJsonString(spec.getFieldMappingMap())); - - // Set timestamp mapping columns - source.setEventTimestampColumn(spec.getEventTimestampColumn()); - source.setCreatedTimestampColumn(spec.getCreatedTimestampColumn()); - source.setDatePartitionColumn(spec.getDatePartitionColumn()); - - return source; - } - - /** - * Convert this DataSource to its Protobuf representation. - * - * @return protobuf representation - */ - public DataSourceProto.DataSource toProto() { - DataSourceProto.DataSource.Builder spec = DataSourceProto.DataSource.newBuilder(); - spec.setType(getType()); - - // Extract source type specific options - Map dataSourceConfigMap = - TypeConversion.convertJsonStringToMap(getConfigJSON()); - switch (getType()) { - case BATCH_FILE: - FileOptions.Builder fileOptions = FileOptions.newBuilder(); - fileOptions.setFileUrl(dataSourceConfigMap.get("file_url")); - - FileFormat.Builder fileFormat = FileFormat.newBuilder(); - parseMessage(dataSourceConfigMap.get("file_format"), fileFormat); - fileOptions.setFileFormat(fileFormat.build()); - - spec.setFileOptions(fileOptions.build()); - break; - case BATCH_BIGQUERY: - BigQueryOptions.Builder bigQueryOptions = BigQueryOptions.newBuilder(); - bigQueryOptions.setTableRef(dataSourceConfigMap.get("table_ref")); - spec.setBigqueryOptions(bigQueryOptions.build()); - break; - case STREAM_KAFKA: - KafkaOptions.Builder kafkaOptions = KafkaOptions.newBuilder(); - kafkaOptions.setBootstrapServers(dataSourceConfigMap.get("bootstrap_servers")); - kafkaOptions.setTopic(dataSourceConfigMap.get("topic")); - - StreamFormat.Builder messageFormat = StreamFormat.newBuilder(); - parseMessage(dataSourceConfigMap.get("message_format"), messageFormat); - kafkaOptions.setMessageFormat(messageFormat.build()); - - spec.setKafkaOptions(kafkaOptions.build()); - break; - case STREAM_KINESIS: - KinesisOptions.Builder kinesisOptions = KinesisOptions.newBuilder(); - kinesisOptions.setRegion(dataSourceConfigMap.get("region")); - kinesisOptions.setStreamName(dataSourceConfigMap.get("stream_name")); - - StreamFormat.Builder recordFormat = StreamFormat.newBuilder(); - parseMessage(dataSourceConfigMap.get("record_format"), recordFormat); - kinesisOptions.setRecordFormat(recordFormat.build()); - - spec.setKinesisOptions(kinesisOptions.build()); - break; - default: - throw new UnsupportedOperationException( - String.format("Unsupported Feature Store Type: %s", getType())); - } - - // Parse field mapping and options from JSON - spec.putAllFieldMapping(TypeConversion.convertJsonStringToMap(getFieldMapJSON())); - - spec.setEventTimestampColumn(getEventTimestampColumn()); - spec.setCreatedTimestampColumn(getCreatedTimestampColumn()); - spec.setDatePartitionColumn(getDatePartitionColumn()); - - return spec.build(); - } - - public Map getFieldsMap() { - return TypeConversion.convertJsonStringToMap(getFieldMapJSON()); - } - - @Override - public int hashCode() { - return toProto().hashCode(); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - DataSource other = (DataSource) o; - return this.toProto().equals(other.toProto()); - } - - /** Print the given Message into its JSON string representation */ - private static String printJSON(MessageOrBuilder message) { - try { - return JsonFormat.printer().print(message); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException("Unexpected exception convering Proto to JSON", e); - } - } - - /** Parse the given Message in JSON representation into the given Message Builder */ - private static void parseMessage(String json, Message.Builder message) { - try { - JsonFormat.parser().merge(json, message); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException("Unexpected exception convering JSON to Proto", e); - } - } -} diff --git a/java/core/src/main/java/feast/core/model/EntityV2.java b/java/core/src/main/java/feast/core/model/EntityV2.java deleted file mode 100644 index d72bef1ebf..0000000000 --- a/java/core/src/main/java/feast/core/model/EntityV2.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.model; - -import com.google.protobuf.Timestamp; -import feast.core.util.TypeConversion; -import feast.proto.core.EntityProto; -import feast.proto.core.EntityProto.*; -import feast.proto.types.ValueProto.ValueType; -import java.util.Map; -import javax.persistence.*; -import lombok.Getter; -import lombok.Setter; - -@Getter -@Setter -@javax.persistence.Entity -@Table( - name = "entities_v2", - uniqueConstraints = @UniqueConstraint(columnNames = {"name", "project_name"})) -public class EntityV2 extends AbstractTimestampEntity { - @Id @GeneratedValue private long id; - - // Name of the Entity - @Column(name = "name", nullable = false) - private String name; - - // Project that this Entity belongs to - @ManyToOne(fetch = FetchType.LAZY) - @JoinColumn(name = "project_name") - private Project project; - - // Description of entity - @Column(name = "description", columnDefinition = "text") - private String description; - - // Columns of entities - /** Data type of each entity column: String representation of {@link ValueType} * */ - private String type; - - // User defined metadata - @Column(name = "labels", columnDefinition = "text") - private String labels; - - public EntityV2() { - super(); - } - - /** - * EntityV2 object supports Entity registration in FeatureTable. - * - *

This data model supports Scalar Entity and would allow ease of discovery of entities and - * reasoning when used in association with FeatureTable. - * - * @param name name - * @param description description - * @param type type - * @param labels labels - */ - public EntityV2( - String name, String description, ValueType.Enum type, Map labels) { - this.name = name; - this.description = description; - this.type = type.toString(); - this.labels = TypeConversion.convertMapToJsonString(labels); - } - - public static EntityV2 fromProto(EntityProto.Entity entityProto) { - EntitySpecV2 spec = entityProto.getSpec(); - - return new EntityV2( - spec.getName(), spec.getDescription(), spec.getValueType(), spec.getLabelsMap()); - } - - public EntityProto.Entity toProto() { - EntityMeta.Builder meta = - EntityMeta.newBuilder() - .setCreatedTimestamp( - Timestamp.newBuilder().setSeconds(super.getCreated().getTime() / 1000L)) - .setLastUpdatedTimestamp( - Timestamp.newBuilder().setSeconds(super.getLastUpdated().getTime() / 1000L)); - - EntitySpecV2.Builder spec = - EntitySpecV2.newBuilder() - .setName(getName()) - .setDescription(getDescription()) - .setValueType(ValueType.Enum.valueOf(getType())) - .putAllLabels(TypeConversion.convertJsonStringToMap(labels)); - - // Build Entity - EntityProto.Entity entity = EntityProto.Entity.newBuilder().setMeta(meta).setSpec(spec).build(); - return entity; - } - - /** - * Updates the existing entity from a proto. - * - * @param entityProto EntityProto with updated spec - * @param projectName Project namespace of Entity which is to be created/updated - */ - public void updateFromProto(EntityProto.Entity entityProto, String projectName) { - EntitySpecV2 spec = entityProto.getSpec(); - - // Validate no change to type - if (!spec.getValueType().equals(ValueType.Enum.valueOf(getType()))) { - throw new IllegalArgumentException( - String.format( - "You are attempting to change the type of this entity in %s project from %s to %s. This isn't allowed. Please create a new entity.", - projectName, getType(), spec.getValueType().toString())); - } - - // 2. Update description, labels - this.setDescription(spec.getDescription()); - this.setLabels(TypeConversion.convertMapToJsonString(spec.getLabelsMap())); - } - - /** - * Determine whether an entity has all the specified labels. - * - * @param labelsFilter labels contain key-value mapping for labels attached to the Entity - * @return boolean True if Entity contains all labels in the labelsFilter - */ - public boolean hasAllLabels(Map labelsFilter) { - Map LabelsMap = this.getLabelsMap(); - for (String key : labelsFilter.keySet()) { - if (!LabelsMap.containsKey(key) || !LabelsMap.get(key).equals(labelsFilter.get(key))) { - return false; - } - } - return true; - } - - public Map getLabelsMap() { - return TypeConversion.convertJsonStringToMap(this.getLabels()); - } -} diff --git a/java/core/src/main/java/feast/core/model/FeatureTable.java b/java/core/src/main/java/feast/core/model/FeatureTable.java deleted file mode 100644 index 9b9bdeff3b..0000000000 --- a/java/core/src/main/java/feast/core/model/FeatureTable.java +++ /dev/null @@ -1,426 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.model; - -import static feast.common.models.FeatureV2.getFeatureStringRef; - -import com.google.common.hash.Hashing; -import com.google.protobuf.Duration; -import com.google.protobuf.Timestamp; -import feast.core.dao.EntityRepository; -import feast.core.util.TypeConversion; -import feast.proto.core.DataSourceProto; -import feast.proto.core.FeatureProto.FeatureSpecV2; -import feast.proto.core.FeatureTableProto; -import feast.proto.core.FeatureTableProto.FeatureTableSpec; -import feast.proto.serving.ServingAPIProto; -import java.util.*; -import java.util.stream.Collectors; -import javax.persistence.CascadeType; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.FetchType; -import javax.persistence.GeneratedValue; -import javax.persistence.Id; -import javax.persistence.JoinColumn; -import javax.persistence.JoinTable; -import javax.persistence.ManyToMany; -import javax.persistence.ManyToOne; -import javax.persistence.OneToMany; -import javax.persistence.OneToOne; -import javax.persistence.Table; -import javax.persistence.UniqueConstraint; -import lombok.AccessLevel; -import lombok.Getter; -import lombok.Setter; - -@Getter -@Entity -@Setter(AccessLevel.PRIVATE) -@Table( - name = "feature_tables", - uniqueConstraints = @UniqueConstraint(columnNames = {"name", "project_name"})) -public class FeatureTable extends AbstractTimestampEntity { - - @Id @GeneratedValue private long id; - - // Name of Feature Table - @Column(name = "name", nullable = false) - private String name; - - // Name of the Project that this FeatureTable belongs to - @ManyToOne(fetch = FetchType.LAZY) - @JoinColumn(name = "project_name") - private Project project; - - // Features defined in this Feature Table - @OneToMany( - mappedBy = "featureTable", - cascade = CascadeType.ALL, - fetch = FetchType.EAGER, - orphanRemoval = true) - private Set features; - - // Entites to associate the features defined in this FeatureTable with - @ManyToMany(fetch = FetchType.EAGER) - @JoinTable( - name = "feature_tables_entities_v2", - joinColumns = @JoinColumn(name = "feature_table_id"), - inverseJoinColumns = @JoinColumn(name = "entity_v2_id")) - private Set entities; - - // User defined metadata labels serialized as JSON string. - @Column(name = "labels", columnDefinition = "text") - private String labelsJSON; - - // Max Age of the Features defined in this Feature Table in seconds - @Column(name = "max_age", nullable = false) - private long maxAgeSecs; - - // Streaming DataSource used to obtain data for features from a stream - @OneToOne(cascade = CascadeType.ALL) - @JoinColumn(name = "stream_source_id", nullable = true) - private DataSource streamSource; - - // Batched DataSource used to obtain data for features from a batch of data - @OneToOne(cascade = CascadeType.ALL) - @JoinColumn(name = "batch_source_id", nullable = false) - private DataSource batchSource; - - // Autoincrementing version no. of this FeatureTable. - // Autoincrements every update made to the FeatureTable. - @Column(name = "revision", nullable = false) - private int revision; - - @Column(name = "is_deleted", nullable = false) - private boolean isDeleted; - - public FeatureTable() {}; - - /** - * Construct FeatureTable from Protobuf spec representation in the given project with entities - * registered in entity repository. - * - * @param projectName the name of the project that the constructed FeatureTable belongs. - * @param spec the Protobuf spec to construct the Feature from. - * @param entityRepo {@link EntityRepository} used to resolve entity names. - * @throws IllegalArgumentException if the Protobuf spec provided is invalid. - * @return constructed FeatureTable from the given Protobuf spec. - */ - public static FeatureTable fromProto( - String projectName, FeatureTableSpec spec, EntityRepository entityRepo) { - FeatureTable table = new FeatureTable(); - table.setName(spec.getName()); - table.setProject(new Project(projectName)); - - Set features = - spec.getFeaturesList().stream() - .map(featureSpec -> FeatureV2.fromProto(table, featureSpec)) - .collect(Collectors.toSet()); - table.setFeatures(features); - - Set entities = - FeatureTable.resolveEntities( - projectName, spec.getName(), entityRepo, spec.getEntitiesList()); - table.setEntities(entities); - - String labelsJSON = TypeConversion.convertMapToJsonString(spec.getLabelsMap()); - table.setLabelsJSON(labelsJSON); - - table.setMaxAgeSecs(spec.getMaxAge().getSeconds()); - table.setBatchSource(DataSource.fromProto(spec.getBatchSource())); - - // Configure stream source only if set - if (!spec.getStreamSource().equals(DataSourceProto.DataSource.getDefaultInstance())) { - table.setStreamSource(DataSource.fromProto(spec.getStreamSource())); - } - - return table; - } - - /** - * Update the FeatureTable from the given Protobuf representation. - * - * @param projectName project name - * @param spec the Protobuf spec to update the FeatureTable from. - * @param entityRepo repository - * @throws IllegalArgumentException if the update will make prohibited changes. - */ - public void updateFromProto( - String projectName, FeatureTableSpec spec, EntityRepository entityRepo) { - // Check for prohibited changes made in spec: - // - Name cannot be changed - if (!getName().equals(spec.getName())) { - throw new IllegalArgumentException( - String.format( - "Updating the name of a registered FeatureTable is not allowed: %s to %s", - getName(), spec.getName())); - } - // Update Entities if changed - Set entities = - FeatureTable.resolveEntities( - projectName, spec.getName(), entityRepo, spec.getEntitiesList()); - this.setEntities(entities); - - // Update FeatureTable based on spec - // Update existing features, create new feature, drop missing features - Map existingFeatures = - getFeatures().stream().collect(Collectors.toMap(FeatureV2::getName, feature -> feature)); - this.features.clear(); - this.features.addAll( - spec.getFeaturesList().stream() - .map( - featureSpec -> { - if (!existingFeatures.containsKey(featureSpec.getName())) { - // Create new Feature based on spec - return FeatureV2.fromProto(this, featureSpec); - } - // Update existing feature based on spec - FeatureV2 feature = existingFeatures.get(featureSpec.getName()); - feature.updateFromProto(featureSpec); - return feature; - }) - .collect(Collectors.toSet())); - - this.maxAgeSecs = spec.getMaxAge().getSeconds(); - this.labelsJSON = TypeConversion.convertMapToJsonString(spec.getLabelsMap()); - - this.batchSource = DataSource.fromProto(spec.getBatchSource()); - if (!spec.getStreamSource().equals(DataSourceProto.DataSource.getDefaultInstance())) { - this.streamSource = DataSource.fromProto(spec.getStreamSource()); - } else { - this.streamSource = null; - } - - // Set isDeleted to false - this.setDeleted(false); - - // Bump revision no. - this.revision++; - } - - /** - * Convert this Feature Table to its Protobuf representation - * - * @return protobuf representation - */ - public FeatureTableProto.FeatureTable toProto() { - // Convert field types to Protobuf compatible types - Timestamp creationTime = TypeConversion.convertTimestamp(getCreated()); - Timestamp updatedTime = TypeConversion.convertTimestamp(getLastUpdated()); - String metadataHashBytes = this.protoHash(); - - List featureSpecs = - getFeatures().stream().map(FeatureV2::toProto).collect(Collectors.toList()); - List entityNames = - getEntities().stream().map(EntityV2::getName).collect(Collectors.toList()); - Map labels = TypeConversion.convertJsonStringToMap(getLabelsJSON()); - - FeatureTableSpec.Builder spec = - FeatureTableSpec.newBuilder() - .setName(getName()) - .setMaxAge(Duration.newBuilder().setSeconds(getMaxAgeSecs()).build()) - .setBatchSource(getBatchSource().toProto()) - .addAllEntities(entityNames) - .addAllFeatures(featureSpecs) - .putAllLabels(labels); - if (getStreamSource() != null) { - spec.setStreamSource(getStreamSource().toProto()); - } - - return FeatureTableProto.FeatureTable.newBuilder() - .setMeta( - FeatureTableProto.FeatureTableMeta.newBuilder() - .setRevision(getRevision()) - .setCreatedTimestamp(creationTime) - .setLastUpdatedTimestamp(updatedTime) - .setHash(metadataHashBytes) - .build()) - .setSpec(spec.build()) - .build(); - } - - /** Use given entity repository to resolve entity names to entity native objects */ - private static Set resolveEntities( - String projectName, String tableName, EntityRepository repo, Collection names) { - return names.stream() - .map( - entityName -> { - EntityV2 entity = repo.findEntityByNameAndProject_Name(entityName, projectName); - if (entity == null) { - throw new IllegalArgumentException( - String.format( - "Feature Table refers to no existent Entity: (table: %s, entity: %s, project: %s)", - tableName, entityName, projectName)); - } - return entity; - }) - .collect(Collectors.toSet()); - } - - /** - * Return a boolean to indicate if FeatureTable contains all specified entities. - * - * @param entitiesFilter contain entities that should be attached to the FeatureTable - * @return boolean True if FeatureTable contains all entities in the entitiesFilter - */ - public boolean hasAllEntities(List entitiesFilter) { - Set allEntitiesName = - this.getEntities().stream().map(entity -> entity.getName()).collect(Collectors.toSet()); - return allEntitiesName.equals(new HashSet<>(entitiesFilter)); - } - - /** - * Returns a map of Feature references and Features if FeatureTable's Feature contains all labels - * in the labelsFilter - * - * @param labelsFilter contain labels that should be attached to FeatureTable's features - * @return Map of Feature references and Features - */ - public Map getFeaturesByLabels(Map labelsFilter) { - Map validFeaturesMap; - List validFeatures; - if (labelsFilter.size() > 0) { - validFeatures = filterFeaturesByAllLabels(this.getFeatures(), labelsFilter); - validFeaturesMap = getFeaturesRefToFeaturesMap(validFeatures); - return validFeaturesMap; - } - validFeaturesMap = getFeaturesRefToFeaturesMap(List.copyOf(this.getFeatures())); - return validFeaturesMap; - } - - /** - * Returns map for accessing features using their respective feature reference. - * - * @param features List of features to insert to map. - * @return Map of featureRef:feature. - */ - private Map getFeaturesRefToFeaturesMap(List features) { - Map validFeaturesMap = new HashMap<>(); - for (FeatureV2 feature : features) { - ServingAPIProto.FeatureReferenceV2 featureRef = - ServingAPIProto.FeatureReferenceV2.newBuilder() - .setFeatureTable(this.getName()) - .setName(feature.getName()) - .build(); - validFeaturesMap.put(getFeatureStringRef(featureRef), feature); - } - return validFeaturesMap; - } - - /** - * Returns a list of Features if FeatureTable's Feature contains all labels in labelsFilter - * - * @param features features - * @param labelsFilter contain labels that should be attached to FeatureTable's features - * @return List of Features - */ - public static List filterFeaturesByAllLabels( - Set features, Map labelsFilter) { - List validFeatures = - features.stream() - .filter(feature -> feature.hasAllLabels(labelsFilter)) - .collect(Collectors.toList()); - - return validFeatures; - } - - /** - * Determine whether a FeatureTable has all the specified labels. - * - * @param labelsFilter labels contain key-value mapping for labels attached to the FeatureTable - * @return boolean True if Entity contains all labels in the labelsFilter - */ - public boolean hasAllLabels(Map labelsFilter) { - Map LabelsMap = this.getLabelsMap(); - for (String key : labelsFilter.keySet()) { - if (!LabelsMap.containsKey(key) || !LabelsMap.get(key).equals(labelsFilter.get(key))) { - return false; - } - } - return true; - } - - public Map getLabelsMap() { - return TypeConversion.convertJsonStringToMap(getLabelsJSON()); - } - - public void delete() { - this.setDeleted(true); - this.setRevision(0); - } - - public String protoHash() { - List sortedEntities = - this.getEntities().stream().map(EntityV2::getName).sorted().collect(Collectors.toList()); - - List sortedFeatureSpecs = - this.getFeatures().stream() - .sorted(Comparator.comparing(FeatureV2::getName)) - .map(FeatureV2::toProto) - .collect(Collectors.toList()); - - DataSourceProto.DataSource streamSource = DataSourceProto.DataSource.getDefaultInstance(); - if (getStreamSource() != null) { - streamSource = getStreamSource().toProto(); - } - - FeatureTableSpec featureTableSpec = - FeatureTableSpec.newBuilder() - .addAllEntities(sortedEntities) - .addAllFeatures(sortedFeatureSpecs) - .setBatchSource(getBatchSource().toProto()) - .setStreamSource(streamSource) - .setMaxAge(Duration.newBuilder().setSeconds(getMaxAgeSecs()).build()) - .build(); - return Hashing.murmur3_32().hashBytes(featureTableSpec.toByteArray()).toString(); - } - - @Override - public int hashCode() { - return Objects.hash( - getName(), - getProject(), - getFeatures(), - getEntities(), - getMaxAgeSecs(), - getBatchSource(), - getStreamSource()); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof FeatureTable)) { - return false; - } - - FeatureTable other = (FeatureTable) o; - - return getName().equals(other.getName()) - && getProject().equals(other.getProject()) - && getLabelsJSON().equals(other.getLabelsJSON()) - && getFeatures().equals(other.getFeatures()) - && getEntities().equals(other.getEntities()) - && getMaxAgeSecs() == other.getMaxAgeSecs() - && Optional.ofNullable(getBatchSource()).equals(Optional.ofNullable(other.getBatchSource())) - && Optional.ofNullable(getStreamSource()) - .equals(Optional.ofNullable(other.getStreamSource())); - } -} diff --git a/java/core/src/main/java/feast/core/model/FeatureV2.java b/java/core/src/main/java/feast/core/model/FeatureV2.java deleted file mode 100644 index d0a6082dc8..0000000000 --- a/java/core/src/main/java/feast/core/model/FeatureV2.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.model; - -import feast.core.util.TypeConversion; -import feast.proto.core.FeatureProto.FeatureSpecV2; -import feast.proto.types.ValueProto.ValueType; -import java.util.Map; -import java.util.Objects; -import javax.persistence.*; -import javax.persistence.Entity; -import lombok.AccessLevel; -import lombok.Getter; -import lombok.Setter; - -/** Defines a single Feature defined in a {@link FeatureTable} */ -@Getter -@Entity -@Setter(AccessLevel.PRIVATE) -@Table( - name = "features_v2", - uniqueConstraints = @UniqueConstraint(columnNames = {"name", "feature_table_id"})) -public class FeatureV2 { - @Id @GeneratedValue private long id; - - // Name of the Feature - private String name; - - // Feature Table where this Feature is defined in. - @ManyToOne(fetch = FetchType.LAZY) - @JoinColumn(name = "feature_table_id", nullable = false) - private FeatureTable featureTable; - - // Value type of the feature. String representation of ValueType. - @Enumerated(EnumType.STRING) - @Column(name = "type") - private ValueType.Enum type; - - // User defined metadata labels for this feature encoded a JSON string. - @Column(name = "labels", columnDefinition = "text") - private String labelsJSON; - - public FeatureV2() {}; - - public FeatureV2(FeatureTable table, String name, ValueType.Enum type, String labelsJSON) { - this.featureTable = table; - this.name = name; - this.type = type; - this.labelsJSON = labelsJSON; - } - - /** - * Construct Feature from Protobuf spec representation. - * - * @param table the FeatureTable to associate the constructed feature with. - * @param spec the Protobuf spec to contruct the Feature from. - * @return constructed Feature from the given Protobuf spec. - */ - public static FeatureV2 fromProto(FeatureTable table, FeatureSpecV2 spec) { - String labelsJSON = TypeConversion.convertMapToJsonString(spec.getLabelsMap()); - return new FeatureV2(table, spec.getName(), spec.getValueType(), labelsJSON); - } - - /** - * Convert this Feature to its Protobuf representation. - * - * @return protobuf representation - */ - public FeatureSpecV2 toProto() { - Map labels = TypeConversion.convertJsonStringToMap(getLabelsJSON()); - return FeatureSpecV2.newBuilder() - .setName(getName()) - .setValueType(getType()) - .putAllLabels(labels) - .build(); - } - - /** - * Update the Feature from the given Protobuf representation. - * - * @param spec the Protobuf spec to update the Feature from. - * @throws IllegalArgumentException if the update will make prohibited changes. - */ - public void updateFromProto(FeatureSpecV2 spec) { - // Check for prohibited changes made in spec - if (!getName().equals(spec.getName())) { - throw new IllegalArgumentException( - String.format( - "Updating the name of a registered Feature is not allowed: %s to %s", - getName(), spec.getName())); - } - // Update feature type - this.setType(spec.getValueType()); - - // Update Feature based on spec - this.labelsJSON = TypeConversion.convertMapToJsonString(spec.getLabelsMap()); - } - - /** - * Return a boolean to indicate if Feature contains all specified labels. - * - * @param labelsFilter contain labels that should be attached to Feature - * @return boolean True if Feature contains all labels in the labelsFilter - */ - public boolean hasAllLabels(Map labelsFilter) { - Map featureLabelsMap = TypeConversion.convertJsonStringToMap(getLabelsJSON()); - for (String key : labelsFilter.keySet()) { - if (!featureLabelsMap.containsKey(key) - || !featureLabelsMap.get(key).equals(labelsFilter.get(key))) { - return false; - } - } - return true; - } - - @Override - public int hashCode() { - return Objects.hash(getName(), getType(), getLabelsJSON()); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - FeatureV2 feature = (FeatureV2) o; - return getName().equals(feature.getName()) - && getType().equals(feature.getType()) - && getLabelsJSON().equals(feature.getLabelsJSON()); - } -} diff --git a/java/core/src/main/java/feast/core/model/Project.java b/java/core/src/main/java/feast/core/model/Project.java deleted file mode 100644 index 2d60d5e0e0..0000000000 --- a/java/core/src/main/java/feast/core/model/Project.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.model; - -import java.util.HashSet; -import java.util.Objects; -import java.util.Set; -import javax.persistence.CascadeType; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.FetchType; -import javax.persistence.Id; -import javax.persistence.OneToMany; -import javax.persistence.Table; -import lombok.Getter; -import lombok.Setter; - -@Getter -@Setter -@Entity -@Table(name = "projects") -public class Project { - public static final String DEFAULT_NAME = "default"; - - // Name of the project - @Id - @Column(name = "name", nullable = false, unique = true) - private String name; - - // Flag to set whether the project has been archived - @Column(name = "archived", nullable = false) - private boolean archived; - - @OneToMany( - cascade = CascadeType.ALL, - fetch = FetchType.EAGER, - orphanRemoval = true, - mappedBy = "project") - private Set entities; - - @OneToMany( - cascade = CascadeType.ALL, - fetch = FetchType.EAGER, - orphanRemoval = true, - mappedBy = "project") - private Set featureTables; - - public Project() { - super(); - } - - public Project(String name) { - this.name = name; - this.entities = new HashSet<>(); - this.featureTables = new HashSet<>(); - } - - public void addEntity(EntityV2 entity) { - entity.setProject(this); - entities.add(entity); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - Project field = (Project) o; - return name.equals(field.getName()); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), name); - } -} diff --git a/java/core/src/main/java/feast/core/model/Store.java b/java/core/src/main/java/feast/core/model/Store.java deleted file mode 100644 index 7a8f9a6d61..0000000000 --- a/java/core/src/main/java/feast/core/model/Store.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.model; - -import static feast.common.models.Store.convertStringToSubscription; -import static feast.common.models.Store.parseSubscriptionFrom; - -import com.google.protobuf.InvalidProtocolBufferException; -import feast.proto.core.StoreProto; -import feast.proto.core.StoreProto.Store.Builder; -import feast.proto.core.StoreProto.Store.RedisClusterConfig; -import feast.proto.core.StoreProto.Store.RedisConfig; -import feast.proto.core.StoreProto.Store.StoreType; -import feast.proto.core.StoreProto.Store.Subscription; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.Id; -import javax.persistence.Lob; -import javax.persistence.Table; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.Setter; - -@Getter -@Setter -@AllArgsConstructor -@Entity -@Table(name = "stores") -public class Store { - - // Name of the store. Must be unique - @Id - @Column(name = "name", nullable = false, unique = true) - private String name; - - // Type of the store, should map to feast.core.Store.StoreType - @Column(name = "type", nullable = false) - private String type; - - // Connection string to the database - @Column(name = "config", nullable = false) - @Lob - private byte[] config; - - // FeatureSets this store is subscribed to, comma delimited. - @Column(name = "subscriptions") - private String subscriptions; - - public Store() { - super(); - } - - public static Store fromProto(StoreProto.Store storeProto) throws IllegalArgumentException { - List subs = new ArrayList<>(); - for (Subscription s : storeProto.getSubscriptionsList()) { - subs.add(parseSubscriptionFrom(s)); - } - byte[] config; - switch (storeProto.getType()) { - case REDIS: - config = storeProto.getRedisConfig().toByteArray(); - break; - case REDIS_CLUSTER: - config = storeProto.getRedisClusterConfig().toByteArray(); - break; - default: - throw new IllegalArgumentException("Invalid store provided"); - } - return new Store( - storeProto.getName(), storeProto.getType().toString(), config, String.join(",", subs)); - } - - public StoreProto.Store toProto() throws InvalidProtocolBufferException { - List subscriptionProtos = getSubscriptions(); - Builder storeProtoBuilder = - StoreProto.Store.newBuilder() - .setName(name) - .setType(StoreType.valueOf(type)) - .addAllSubscriptions(subscriptionProtos); - switch (StoreType.valueOf(type)) { - case REDIS: - RedisConfig redisConfig = RedisConfig.parseFrom(config); - return storeProtoBuilder.setRedisConfig(redisConfig).build(); - case REDIS_CLUSTER: - RedisClusterConfig redisClusterConfig = RedisClusterConfig.parseFrom(config); - return storeProtoBuilder.setRedisClusterConfig(redisClusterConfig).build(); - default: - throw new InvalidProtocolBufferException("Invalid store set"); - } - } - - /** - * Returns a List of Subscriptions. - * - * @return List of Subscription - */ - public List getSubscriptions() { - return Arrays.stream(subscriptions.split(",")) - .map(s -> convertStringToSubscription(s)) - .collect(Collectors.toList()); - } - - @Override - public int hashCode() { - return Objects.hash(this.name, this.type, this.subscriptions) ^ Arrays.hashCode(this.config); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) return true; - if (getClass() != obj.getClass()) return false; - Store other = (Store) obj; - - if (!name.equals(other.name)) { - return false; - } else if (!type.equals(other.type)) { - return false; - } else if (!Arrays.equals(config, config)) { - return false; - } else if (!subscriptions.equals(other.subscriptions)) { - return false; - } - return true; - } -} diff --git a/java/core/src/main/java/feast/core/service/ProjectService.java b/java/core/src/main/java/feast/core/service/ProjectService.java deleted file mode 100644 index 308c79bccf..0000000000 --- a/java/core/src/main/java/feast/core/service/ProjectService.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.service; - -import feast.core.dao.ProjectRepository; -import feast.core.model.Project; -import java.util.List; -import java.util.Optional; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; - -@Slf4j -@Service -public class ProjectService { - - private ProjectRepository projectRepository; - - @Autowired - public ProjectService(ProjectRepository projectRepository) { - this.projectRepository = projectRepository; - } - - /** - * Creates a project - * - * @param name Name of project to be created - */ - @Transactional - public void createProject(String name) { - if (projectRepository.existsById(name)) { - throw new IllegalArgumentException(String.format("Project already exists: %s", name)); - } - Project project = new Project(name); - projectRepository.saveAndFlush(project); - } - - /** - * Archives a project - * - * @param name Name of the project to be archived - */ - @Transactional - public void archiveProject(String name) { - Optional project = projectRepository.findById(name); - if (!project.isPresent()) { - throw new IllegalArgumentException(String.format("Could not find project: \"%s\"", name)); - } - if (name.equals(Project.DEFAULT_NAME)) { - throw new UnsupportedOperationException("Archiving the default project is not allowed."); - } - Project p = project.get(); - p.setArchived(true); - projectRepository.saveAndFlush(p); - } - - /** - * List all active projects - * - * @return List of active projects - */ - @Transactional - public List listProjects() { - return projectRepository.findAllByArchivedIsFalse(); - } -} diff --git a/java/core/src/main/java/feast/core/service/SpecService.java b/java/core/src/main/java/feast/core/service/SpecService.java deleted file mode 100644 index ff45dcd7ed..0000000000 --- a/java/core/src/main/java/feast/core/service/SpecService.java +++ /dev/null @@ -1,497 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.service; - -import static feast.core.validators.Matchers.checkValidCharacters; -import static feast.core.validators.Matchers.checkValidCharactersAllowDash; - -import com.google.protobuf.InvalidProtocolBufferException; -import feast.core.dao.EntityRepository; -import feast.core.dao.FeatureTableRepository; -import feast.core.dao.ProjectRepository; -import feast.core.dao.StoreRepository; -import feast.core.exception.RetrievalException; -import feast.core.model.*; -import feast.core.validators.EntityValidator; -import feast.core.validators.FeatureTableValidator; -import feast.proto.core.CoreServiceProto.ApplyEntityResponse; -import feast.proto.core.CoreServiceProto.ApplyFeatureTableRequest; -import feast.proto.core.CoreServiceProto.ApplyFeatureTableResponse; -import feast.proto.core.CoreServiceProto.DeleteFeatureTableRequest; -import feast.proto.core.CoreServiceProto.GetEntityRequest; -import feast.proto.core.CoreServiceProto.GetEntityResponse; -import feast.proto.core.CoreServiceProto.GetFeatureTableRequest; -import feast.proto.core.CoreServiceProto.GetFeatureTableResponse; -import feast.proto.core.CoreServiceProto.ListEntitiesRequest; -import feast.proto.core.CoreServiceProto.ListEntitiesResponse; -import feast.proto.core.CoreServiceProto.ListFeatureTablesRequest; -import feast.proto.core.CoreServiceProto.ListFeatureTablesResponse; -import feast.proto.core.CoreServiceProto.ListFeaturesRequest; -import feast.proto.core.CoreServiceProto.ListFeaturesResponse; -import feast.proto.core.CoreServiceProto.ListStoresRequest; -import feast.proto.core.CoreServiceProto.ListStoresResponse; -import feast.proto.core.CoreServiceProto.ListStoresResponse.Builder; -import feast.proto.core.CoreServiceProto.UpdateStoreRequest; -import feast.proto.core.CoreServiceProto.UpdateStoreResponse; -import feast.proto.core.EntityProto; -import feast.proto.core.FeatureTableProto.FeatureTableSpec; -import feast.proto.core.StoreProto; -import feast.proto.core.StoreProto.Store.Subscription; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Optional; -import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; - -/** - * Facilitates management of specs within the Feast registry. This includes getting existing specs - * and registering new specs. - */ -@Slf4j -@Service -public class SpecService { - - private final EntityRepository entityRepository; - private final FeatureTableRepository tableRepository; - private final ProjectRepository projectRepository; - private final StoreRepository storeRepository; - - @Autowired - public SpecService( - EntityRepository entityRepository, - FeatureTableRepository tableRepository, - StoreRepository storeRepository, - ProjectRepository projectRepository) { - this.entityRepository = entityRepository; - this.tableRepository = tableRepository; - this.storeRepository = storeRepository; - this.projectRepository = projectRepository; - } - - /** - * Get an entity matching the entity name and set project. The entity name and project are - * required. If the project is omitted, the default would be used. - * - * @param request GetEntityRequest Request - * @return Returns a GetEntityResponse containing an entity - */ - public GetEntityResponse getEntity(GetEntityRequest request) { - String projectName = request.getProject(); - String entityName = request.getName(); - - if (entityName.isEmpty()) { - throw new IllegalArgumentException("No entity name provided"); - } - // Autofill default project if project is not specified - if (projectName.isEmpty()) { - projectName = Project.DEFAULT_NAME; - } - - checkValidCharactersAllowDash(projectName, "project"); - checkValidCharacters(entityName, "entity"); - - EntityV2 entity = entityRepository.findEntityByNameAndProject_Name(entityName, projectName); - - if (entity == null) { - throw new RetrievalException( - String.format("Entity with name \"%s\" could not be found.", entityName)); - } - - // Build GetEntityResponse - GetEntityResponse response = GetEntityResponse.newBuilder().setEntity(entity.toProto()).build(); - - return response; - } - - /** - * Return a map of feature references and features matching the project, labels and entities - * provided in the filter. All fields are required. - * - *

Project name must be explicitly provided or if the project name is omitted, the default - * project would be used. A combination of asterisks/wildcards and text is not allowed. - * - *

The entities in the filter accepts a list. All matching features will be returned. Regex is - * not supported. If no entities are provided, features will not be filtered by entities. - * - *

The labels in the filter accepts a map. All matching features will be returned. Regex is not - * supported. If no labels are provided, features will not be filtered by labels. - * - * @param filter filter containing the desired project name, entities and labels - * @return ListEntitiesResponse with map of feature references and features found matching the - * filter - */ - public ListFeaturesResponse listFeatures(ListFeaturesRequest.Filter filter) { - String project = filter.getProject(); - List entities = filter.getEntitiesList(); - Map labels = filter.getLabelsMap(); - - // Autofill default project if project not specified - if (project.isEmpty()) { - project = Project.DEFAULT_NAME; - } - - checkValidCharactersAllowDash(project, "project"); - - // Currently defaults to all FeatureTables - List featureTables = tableRepository.findAllByProject_Name(project); - - ListFeaturesResponse.Builder response = ListFeaturesResponse.newBuilder(); - if (entities.size() > 0) { - featureTables = - featureTables.stream() - .filter(featureTable -> featureTable.hasAllEntities(entities)) - .collect(Collectors.toList()); - } - - Map featuresMap; - for (FeatureTable featureTable : featureTables) { - featuresMap = featureTable.getFeaturesByLabels(labels); - for (Map.Entry entry : featuresMap.entrySet()) { - response.putFeatures(entry.getKey(), entry.getValue().toProto()); - } - } - - return response.build(); - } - - /** - * Return a list of entities matching the entity name, project and labels provided in the filter. - * All fields are required. Use '*' in entity name and project, and empty map in labels in order - * to return all entities in all projects. - * - *

Project name can be explicitly provided, or an asterisk can be provided to match all - * projects. It is not possible to provide a combination of asterisks/wildcards and text. If the - * project name is omitted, the default project would be used. - * - *

The entity name in the filter accepts an asterisk as a wildcard. All matching entities will - * be returned. Regex is not supported. Explicitly defining an entity name is not possible if a - * project name is not set explicitly. - * - *

The labels in the filter accepts a map. All entities which contain every provided label will - * be returned. - * - * @param filter Filter containing the desired entity name, project and labels - * @return ListEntitiesResponse with list of entities found matching the filter - */ - public ListEntitiesResponse listEntities(ListEntitiesRequest.Filter filter) { - String project = filter.getProject(); - Map labelsFilter = filter.getLabelsMap(); - - // Autofill default project if project not specified - if (project.isEmpty()) { - project = Project.DEFAULT_NAME; - } - - checkValidCharactersAllowDash(project, "project"); - - List entities = entityRepository.findAllByProject_Name(project); - - ListEntitiesResponse.Builder response = ListEntitiesResponse.newBuilder(); - if (entities.size() > 0) { - entities = - entities.stream() - .filter(entity -> entity.hasAllLabels(labelsFilter)) - .collect(Collectors.toList()); - for (EntityV2 entity : entities) { - response.addEntities(entity.toProto()); - } - } - - return response.build(); - } - - /** - * Get stores matching the store name provided in the filter. If the store name is not provided, - * the method will return all stores currently registered to Feast. - * - * @param filter filter containing the desired store name - * @return ListStoresResponse containing list of stores found matching the filter - */ - @Transactional - public ListStoresResponse listStores(ListStoresRequest.Filter filter) { - try { - String name = filter.getName(); - if (name.equals("")) { - Builder responseBuilder = ListStoresResponse.newBuilder(); - for (Store store : storeRepository.findAll()) { - responseBuilder.addStore(store.toProto()); - } - return responseBuilder.build(); - } - Store store = - storeRepository - .findById(name) - .orElseThrow( - () -> - new RetrievalException( - String.format("Store with name '%s' not found", name))); - return ListStoresResponse.newBuilder().addStore(store.toProto()).build(); - } catch (InvalidProtocolBufferException e) { - - throw io.grpc.Status.NOT_FOUND - .withDescription("Unable to retrieve stores") - .withCause(e) - .asRuntimeException(); - } - } - - /** - * Creates or updates an entity in the repository. - * - *

This function is idempotent. If no changes are detected in the incoming entity's schema, - * this method will return the existing entity stored in the repository. If project is not - * specified, the entity will be assigned to the 'default' project. - * - * @param newEntitySpec EntitySpecV2 that will be used to create or update an Entity. - * @param projectName Project namespace of Entity which is to be created/updated - * @return response of the operation - */ - @Transactional - public ApplyEntityResponse applyEntity( - EntityProto.EntitySpecV2 newEntitySpec, String projectName) { - // Autofill default project if not specified - if (projectName == null || projectName.isEmpty()) { - projectName = Project.DEFAULT_NAME; - } - - checkValidCharactersAllowDash(projectName, "project"); - - // Validate incoming entity - EntityValidator.validateSpec(newEntitySpec); - - // Find project or create new one if it does not exist - Project project = projectRepository.findById(projectName).orElse(new Project(projectName)); - - // Ensure that the project retrieved from repository is not archived - if (project.isArchived()) { - throw new IllegalArgumentException(String.format("Project is archived: %s", projectName)); - } - - // Retrieve existing Entity - EntityV2 entity = - entityRepository.findEntityByNameAndProject_Name(newEntitySpec.getName(), projectName); - - EntityProto.Entity newEntity = EntityProto.Entity.newBuilder().setSpec(newEntitySpec).build(); - if (entity == null) { - // Create new entity since it doesn't exist - entity = EntityV2.fromProto(newEntity); - } else { - // If the entity remains unchanged, we do nothing. - if (entity.toProto().getSpec().equals(newEntitySpec)) { - return ApplyEntityResponse.newBuilder().setEntity(entity.toProto()).build(); - } - entity.updateFromProto(newEntity, projectName); - } - - // Persist the EntityV2 object - project.addEntity(entity); - projectRepository.saveAndFlush(project); - - // Build ApplyEntityResponse - ApplyEntityResponse response = - ApplyEntityResponse.newBuilder().setEntity(entity.toProto()).build(); - return response; - } - - /** - * Resolves the project name by returning name if given, autofilling default project otherwise. - * - * @param projectName name of the project to resolve. - * @return project name - */ - public static String resolveProjectName(String projectName) { - return (projectName.isEmpty()) ? Project.DEFAULT_NAME : projectName; - } - - /** - * UpdateStore updates the repository with the new given store. - * - * @param updateStoreRequest containing the new store definition - * @return UpdateStoreResponse containing the new store definition - * @throws InvalidProtocolBufferException if protobuf exception occurs - */ - @Transactional - public UpdateStoreResponse updateStore(UpdateStoreRequest updateStoreRequest) - throws InvalidProtocolBufferException { - StoreProto.Store newStoreProto = updateStoreRequest.getStore(); - - List subs = newStoreProto.getSubscriptionsList(); - for (Subscription sub : subs) { - // Ensure that all fields in a subscription contain values - if ((sub.getName().isEmpty()) || sub.getProject().isEmpty()) { - throw new IllegalArgumentException( - String.format("Missing parameter in subscription: %s", sub)); - } - } - Store existingStore = storeRepository.findById(newStoreProto.getName()).orElse(null); - - // Do nothing if no change - if (existingStore != null && existingStore.toProto().equals(newStoreProto)) { - return UpdateStoreResponse.newBuilder() - .setStatus(UpdateStoreResponse.Status.NO_CHANGE) - .setStore(updateStoreRequest.getStore()) - .build(); - } - - Store newStore = Store.fromProto(newStoreProto); - storeRepository.save(newStore); - return UpdateStoreResponse.newBuilder() - .setStatus(UpdateStoreResponse.Status.UPDATED) - .setStore(updateStoreRequest.getStore()) - .build(); - } - - /** - * Applies the given FeatureTable to the FeatureTable registry. Creates the FeatureTable if does - * not exist, otherwise updates the existing FeatureTable. Applies FeatureTable in project if - * specified, otherwise in default project. - * - * @param request Contains FeatureTable spec and project parameters used to create or update a - * FeatureTable. - * @throws NoSuchElementException projects and entities referenced in request do not exist. - * @return response containing the applied FeatureTable spec. - */ - @Transactional - public ApplyFeatureTableResponse applyFeatureTable(ApplyFeatureTableRequest request) { - String projectName = resolveProjectName(request.getProject()); - - checkValidCharactersAllowDash(projectName, "project"); - - // Check that specification provided is valid - FeatureTableSpec applySpec = request.getTableSpec(); - FeatureTableValidator.validateSpec(applySpec); - - // Prevent apply if the project is archived. - Project project = projectRepository.findById(projectName).orElse(new Project(projectName)); - if (project.isArchived()) { - throw new IllegalArgumentException( - String.format( - "Cannot apply Feature Table to archived Project: (table: %s, project: %s)", - applySpec.getName(), projectName)); - } - - // Create or update depending on whether there is an existing Feature Table - Optional existingTable = - tableRepository.findFeatureTableByNameAndProject_Name(applySpec.getName(), projectName); - FeatureTable table = FeatureTable.fromProto(projectName, applySpec, entityRepository); - if (existingTable.isPresent() && table.equals(existingTable.get())) { - // Skip update if no change is detected - return ApplyFeatureTableResponse.newBuilder().setTable(existingTable.get().toProto()).build(); - } - if (existingTable.isPresent()) { - existingTable.get().updateFromProto(projectName, applySpec, entityRepository); - table = existingTable.get(); - } - - // Commit FeatureTable to database and return applied FeatureTable - tableRepository.saveAndFlush(table); - return ApplyFeatureTableResponse.newBuilder().setTable(table.toProto()).build(); - } - - /** - * List the FeatureTables matching the filter in the given filter. Scopes down listing to project - * if specified, the default project otherwise. - * - * @param filter Filter containing the desired project and labels - * @return ListFeatureTablesResponse with list of FeatureTables found matching the filter - */ - @Transactional - public ListFeatureTablesResponse listFeatureTables(ListFeatureTablesRequest.Filter filter) { - String projectName = resolveProjectName(filter.getProject()); - Map labelsFilter = filter.getLabelsMap(); - - checkValidCharactersAllowDash(projectName, "project"); - - List matchingTables = tableRepository.findAllByProject_Name(projectName); - - ListFeatureTablesResponse.Builder response = ListFeatureTablesResponse.newBuilder(); - - if (matchingTables.size() > 0) { - matchingTables = - matchingTables.stream() - .filter(table -> table.hasAllLabels(labelsFilter)) - .filter(table -> !table.isDeleted()) - .collect(Collectors.toList()); - } - for (FeatureTable table : matchingTables) { - response.addTables(table.toProto()); - } - - return response.build(); - } - - /** - * Get the FeatureTable with the name and project specified in the request. Gets FeatureTable in - * project if specified, otherwise in default project. - * - * @param request containing the retrieval parameters. - * @throws NoSuchElementException if no FeatureTable matches given request. - * @return response containing the requested FeatureTable. - */ - @Transactional - public GetFeatureTableResponse getFeatureTable(GetFeatureTableRequest request) { - String projectName = resolveProjectName(request.getProject()); - String featureTableName = request.getName(); - - checkValidCharactersAllowDash(projectName, "project"); - checkValidCharacters(featureTableName, "featureTable"); - - Optional retrieveTable = - tableRepository.findFeatureTableByNameAndProject_Name(featureTableName, projectName); - if (retrieveTable.isEmpty()) { - throw new NoSuchElementException( - String.format( - "No such Feature Table: (project: %s, name: %s)", projectName, featureTableName)); - } - - if (retrieveTable.get().isDeleted()) { - throw new NoSuchElementException( - String.format( - "Feature Table has been deleted: (project: %s, name: %s)", - projectName, featureTableName)); - } - - // Build GetFeatureTableResponse - GetFeatureTableResponse response = - GetFeatureTableResponse.newBuilder().setTable(retrieveTable.get().toProto()).build(); - - return response; - } - - @Transactional - public void deleteFeatureTable(DeleteFeatureTableRequest request) { - String projectName = resolveProjectName(request.getProject()); - String featureTableName = request.getName(); - - checkValidCharactersAllowDash(projectName, "project"); - checkValidCharacters(featureTableName, "featureTable"); - - Optional existingTable = - tableRepository.findFeatureTableByNameAndProject_Name(featureTableName, projectName); - if (existingTable.isEmpty()) { - throw new NoSuchElementException( - String.format( - "No such Feature Table: (project: %s, name: %s)", projectName, featureTableName)); - } - - existingTable.get().delete(); - } -} diff --git a/java/core/src/main/java/feast/core/util/StreamUtil.java b/java/core/src/main/java/feast/core/util/StreamUtil.java deleted file mode 100644 index 4925f02310..0000000000 --- a/java/core/src/main/java/feast/core/util/StreamUtil.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.util; - -import java.util.function.Function; - -/** Collection of functions useful for stream-style programming */ -public class StreamUtil { - - @FunctionalInterface - public interface CheckedFunction { - R apply(T t) throws Exception; - } - - /** - * Wrap function to convert its checked exceptions into RuntimeException - * - * @param checkedFunction function that throws checked exception - * @param input - * @param output - * @return wrapped function that doesn't throw checked exceptions - */ - public static Function wrapException(CheckedFunction checkedFunction) { - return t -> { - try { - return checkedFunction.apply(t); - } catch (Exception e) { - throw new RuntimeException(e); - } - }; - } -} diff --git a/java/core/src/main/java/feast/core/util/TypeConversion.java b/java/core/src/main/java/feast/core/util/TypeConversion.java deleted file mode 100644 index d2a2d0a691..0000000000 --- a/java/core/src/main/java/feast/core/util/TypeConversion.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.util; - -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; -import feast.proto.types.ValueProto.ValueType.Enum; -import java.lang.reflect.Type; -import java.util.*; - -public class TypeConversion { - private static Gson gson = new Gson(); - - /** - * Convert a java data object to protobuf Timestamp object - * - * @param ts timestamp - * @return protobuf.Timestamp object of the given timestamp - */ - public static com.google.protobuf.Timestamp convertTimestamp(Date ts) { - return com.google.protobuf.Timestamp.newBuilder().setSeconds(ts.getTime() / 1000).build(); - } - - /** - * Convert a string of comma-separated strings to list of strings - * - * @param tags comma separated tags - * @return list of tags - */ - public static List convertTagStringToList(String tags) { - if (tags == null || tags.isEmpty()) { - return Collections.emptyList(); - } - return Arrays.asList(tags.split(",")); - } - - /** - * Unmarshals a given json string to map - * - * @param jsonString valid json formatted string - * @return map of keys to values in json - */ - public static Map convertJsonStringToMap(String jsonString) { - if (jsonString == null || jsonString.equals("") || jsonString.equals("{}")) { - return Collections.emptyMap(); - } - Type stringMapType = new TypeToken>() {}.getType(); - return gson.fromJson(jsonString, stringMapType); - } - - /** - * Unmarshals a given json string to Enum map - * - * @param jsonString valid json formatted string - * @return map of keys to Enum values in json string - */ - public static Map convertJsonStringToEnumMap(String jsonString) { - if (jsonString == null || jsonString.equals("") || jsonString.equals("{}")) { - return Collections.emptyMap(); - } - Type stringMapType = new TypeToken>() {}.getType(); - return gson.fromJson(jsonString, stringMapType); - } - - /** - * Marshals a given map into its corresponding json string - * - * @param map map to be converted - * @return json string corresponding to given map - */ - public static String convertMapToJsonString(Map map) { - return gson.toJson(map); - } - - /** - * Marshals a given Enum map into its corresponding json string - * - * @param map map to be converted - * @return json string corresponding to given Enum map - */ - public static String convertEnumMapToJsonString(Map map) { - return gson.toJson(map); - } -} diff --git a/java/core/src/main/java/feast/core/validators/DataSourceValidator.java b/java/core/src/main/java/feast/core/validators/DataSourceValidator.java deleted file mode 100644 index 548d18fb2c..0000000000 --- a/java/core/src/main/java/feast/core/validators/DataSourceValidator.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.validators; - -import static feast.core.validators.Matchers.*; -import static feast.proto.core.DataSourceProto.DataSource.SourceType.*; - -import feast.proto.core.DataFormatProto.FileFormat; -import feast.proto.core.DataFormatProto.StreamFormat; -import feast.proto.core.DataSourceProto.DataSource; - -public class DataSourceValidator { - /** - * Validate if the given DataSource protobuf spec is valid. - * - * @param spec spec to be validated - */ - public static void validate(DataSource spec) { - switch (spec.getType()) { - case BATCH_FILE: - FileFormat.FormatCase fileFormat = spec.getFileOptions().getFileFormat().getFormatCase(); - switch (fileFormat) { - case PARQUET_FORMAT: - break; - default: - throw new UnsupportedOperationException( - String.format("Unsupported File Format: %s", fileFormat)); - } - break; - - case BATCH_BIGQUERY: - checkValidBigQueryTableRef(spec.getBigqueryOptions().getTableRef(), "FeatureTable"); - break; - - case STREAM_KAFKA: - StreamFormat.FormatCase messageFormat = - spec.getKafkaOptions().getMessageFormat().getFormatCase(); - switch (messageFormat) { - case PROTO_FORMAT: - checkValidClassPath( - spec.getKafkaOptions().getMessageFormat().getProtoFormat().getClassPath(), - "FeatureTable"); - break; - case AVRO_FORMAT: - break; - default: - throw new UnsupportedOperationException( - String.format( - "Unsupported Stream Format for Kafka Source Type: %s", messageFormat)); - } - break; - - case STREAM_KINESIS: - // Verify tht DataFormat is supported by kinesis data source - StreamFormat.FormatCase recordFormat = - spec.getKinesisOptions().getRecordFormat().getFormatCase(); - switch (recordFormat) { - case PROTO_FORMAT: - checkValidClassPath( - spec.getKinesisOptions().getRecordFormat().getProtoFormat().getClassPath(), - "FeatureTable"); - break; - case AVRO_FORMAT: - break; - default: - throw new UnsupportedOperationException( - String.format("Unsupported Stream Format for Kafka Source Type: %s", recordFormat)); - } - break; - default: - throw new UnsupportedOperationException( - String.format("Unsupported Feature Store Type: %s", spec.getType())); - } - } -} diff --git a/java/core/src/main/java/feast/core/validators/EntityValidator.java b/java/core/src/main/java/feast/core/validators/EntityValidator.java deleted file mode 100644 index 743a044690..0000000000 --- a/java/core/src/main/java/feast/core/validators/EntityValidator.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.validators; - -import static feast.core.validators.Matchers.checkValidCharacters; - -import feast.proto.core.EntityProto; - -public class EntityValidator { - public static void validateSpec(EntityProto.EntitySpecV2 entitySpec) { - if (entitySpec.getName().isEmpty()) { - throw new IllegalArgumentException("Entity name must be provided"); - } - if (entitySpec.getValueType().toString().isEmpty()) { - throw new IllegalArgumentException("Entity type must not be empty"); - } - if (entitySpec.getLabelsMap().containsKey("")) { - throw new IllegalArgumentException("Entity label keys must not be empty"); - } - - checkValidCharacters(entitySpec.getName(), "entity"); - } -} diff --git a/java/core/src/main/java/feast/core/validators/FeatureTableValidator.java b/java/core/src/main/java/feast/core/validators/FeatureTableValidator.java deleted file mode 100644 index 863c9442f7..0000000000 --- a/java/core/src/main/java/feast/core/validators/FeatureTableValidator.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.validators; - -import static feast.core.validators.Matchers.*; - -import feast.proto.core.DataSourceProto.DataSource.SourceType; -import feast.proto.core.FeatureProto.FeatureSpecV2; -import feast.proto.core.FeatureTableProto.FeatureTableSpec; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Set; -import java.util.stream.Collectors; - -public class FeatureTableValidator { - protected static final Set RESERVED_NAMES = - Set.of("created_timestamp", "event_timestamp"); - - public static void validateSpec(FeatureTableSpec spec) { - if (spec.getName().isEmpty()) { - throw new IllegalArgumentException("FeatureTable name must be provided"); - } - if (spec.getLabelsMap().containsKey("")) { - throw new IllegalArgumentException("FeatureTable cannot have labels with empty key."); - } - if (spec.getEntitiesCount() == 0) { - throw new IllegalArgumentException("FeatureTable entities list cannot be empty."); - } - if (spec.getFeaturesCount() == 0) { - throw new IllegalArgumentException("FeatureTable features list cannot be empty."); - } - if (!spec.hasBatchSource()) { - throw new IllegalArgumentException("FeatureTable batch source cannot be empty."); - } - - checkValidCharacters(spec.getName(), "FeatureTable"); - spec.getFeaturesList().forEach(FeatureTableValidator::validateFeatureSpec); - - // Check that features and entities defined in FeatureTable do not use reserved names - ArrayList fieldNames = new ArrayList<>(spec.getEntitiesList()); - fieldNames.addAll( - spec.getFeaturesList().stream().map(FeatureSpecV2::getName).collect(Collectors.toList())); - if (!Collections.disjoint(fieldNames, RESERVED_NAMES)) { - throw new IllegalArgumentException( - String.format( - "Reserved names has been used as Feature(s) names. Reserved: %s", RESERVED_NAMES)); - } - - // Check that Feature and Entity names in FeatureTable do not collide with each other - if (hasDuplicates(fieldNames)) { - throw new IllegalArgumentException( - String.format("Entity and Feature names within a Feature Table should be unique.")); - } - - // Check that the data sources defined in the feature table are valid - if (!spec.getBatchSource().getType().equals(SourceType.INVALID)) { - DataSourceValidator.validate(spec.getBatchSource()); - } - if (!spec.getStreamSource().getType().equals(SourceType.INVALID)) { - DataSourceValidator.validate(spec.getStreamSource()); - } - } - - private static void validateFeatureSpec(FeatureSpecV2 spec) { - checkValidCharacters(spec.getName(), "Feature"); - if (spec.getLabelsMap().containsKey("")) { - throw new IllegalArgumentException("Features cannot have labels with empty key."); - } - } -} diff --git a/java/core/src/main/java/feast/core/validators/Matchers.java b/java/core/src/main/java/feast/core/validators/Matchers.java deleted file mode 100644 index b6d7fccca2..0000000000 --- a/java/core/src/main/java/feast/core/validators/Matchers.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.validators; - -import java.util.Collection; -import java.util.HashSet; -import java.util.regex.Pattern; - -public class Matchers { - - private static Pattern BIGQUERY_TABLE_REF_REGEX = - Pattern.compile("[a-zA-Z0-9-]+[:]+[a-zA-Z0-9_]+[.]+[a-zA-Z0-9_]*"); - private static Pattern CLASS_PATH_REGEX = - Pattern.compile("[a-zA-Z_][a-zA-Z0-9_]*(\\.[a-zA-Z_][a-zA-Z0-9_]*)*$"); - private static Pattern UPPER_SNAKE_CASE_REGEX = Pattern.compile("^[A-Z0-9]+(_[A-Z0-9]+)*$"); - private static Pattern LOWER_SNAKE_CASE_REGEX = Pattern.compile("^[a-z0-9]+(_[a-z0-9]+)*$"); - private static Pattern VALID_CHARACTERS_REGEX = Pattern.compile("^[a-zA-Z_][a-zA-Z0-9_]*$"); - private static Pattern VALID_CHARACTERS_REGEX_WITH_DASH = - Pattern.compile("^[a-zA-Z_][a-zA-Z0-9_-]*$"); - - private static String ERROR_MESSAGE_TEMPLATE = "invalid value for %s resource, %s: %s"; - - public static void checkUpperSnakeCase(String input, String resource) - throws IllegalArgumentException { - if (!UPPER_SNAKE_CASE_REGEX.matcher(input).matches()) { - throw new IllegalArgumentException( - String.format( - ERROR_MESSAGE_TEMPLATE, - resource, - input, - "argument must be in upper snake case, and cannot include any special characters.")); - } - } - - public static void checkLowerSnakeCase(String input, String resource) - throws IllegalArgumentException { - if (!LOWER_SNAKE_CASE_REGEX.matcher(input).matches()) { - throw new IllegalArgumentException( - String.format( - ERROR_MESSAGE_TEMPLATE, - resource, - input, - "argument must be in lower snake case, and cannot include any special characters.")); - } - } - - public static void checkValidCharacters(String input, String resource) - throws IllegalArgumentException { - if (!VALID_CHARACTERS_REGEX.matcher(input).matches()) { - throw new IllegalArgumentException( - String.format( - ERROR_MESSAGE_TEMPLATE, - resource, - input, - "argument must only contain alphanumeric characters and underscores.")); - } - } - - public static void checkValidCharactersAllowDash(String input, String resource) - throws IllegalArgumentException { - if (!VALID_CHARACTERS_REGEX_WITH_DASH.matcher(input).matches()) { - throw new IllegalArgumentException( - String.format( - ERROR_MESSAGE_TEMPLATE, - resource, - input, - "argument must only contain alphanumeric characters, dashes, or underscores.")); - } - } - - public static void checkValidBigQueryTableRef(String input, String resource) - throws IllegalArgumentException { - if (!BIGQUERY_TABLE_REF_REGEX.matcher(input).matches()) { - throw new IllegalArgumentException( - String.format( - ERROR_MESSAGE_TEMPLATE, - resource, - input, - "argument must be in the form of .")); - } - } - - public static void checkValidClassPath(String input, String resource) { - if (!CLASS_PATH_REGEX.matcher(input).matches()) { - throw new IllegalArgumentException( - String.format( - ERROR_MESSAGE_TEMPLATE, resource, input, "argument must be a valid Java Classpath")); - } - } - - public static boolean hasDuplicates(Collection strings) { - return (new HashSet<>(strings)).size() < strings.size(); - } -} diff --git a/java/core/src/main/resources/application.yml b/java/core/src/main/resources/application.yml deleted file mode 100644 index 1c7b14f415..0000000000 --- a/java/core/src/main/resources/application.yml +++ /dev/null @@ -1,103 +0,0 @@ -# -# Copyright 2018 The Feast Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -feast: - stream: - # Feature stream type. Only kafka is supported. - type: kafka - # Feature stream options. - # See the following for options https://api.docs.feast.dev/grpc/feast.core.pb.html#KafkaSourceConfig - options: - topic: feast-features - bootstrapServers: localhost:9092 - replicationFactor: 1 - partitions: 1 - specsOptions: - specsTopic: feast-specs - specsAckTopic: feast-specs-ack - notifyIntervalMilliseconds: 1000 - - security: - authentication: - enabled: false - provider: jwt - options: - jwkEndpointURI: "https://www.googleapis.com/oauth2/v3/certs" - subjectClaim: email - - authorization: - enabled: false - provider: http - options: - authorizationUrl: http://localhost:8082 - - # If set to true, HTTP REST endpoints at /api/v1 implemented by - # CoreServiceRestController will be accessible in Feast Core WITHOUT - # authentication. - disableRestControllerAuth: false - - logging: - # Audit logging provides a machine readable structured JSON log that can give better - # insight into what is happening in Feast. - audit: - # Whether audit logging is enabled. - enabled: true - # Whether to enable message level (ie request/response) audit logging - messageLogging: - enabled: false - # Logging forwarder currently provides a machine readable structured JSON log to an - # external fluentd service that can give better insight into what is happening in Feast. - # Accepts console / fluentd as destination - destination: console - fluentdHost: localhost - fluentdPort: 24224 - -grpc: - server: - # The port that Feast Core gRPC service listens on - port: 6565 - security: - enabled: false - certificateChain: server.crt - privateKey: server.key - -spring: - jpa: - properties.hibernate: - format_sql: true - event: - merge: - entity_copy_observer: allow - hibernate.naming.physical-strategy=org.hibernate.boot.model.naming: PhysicalNamingStrategyStandardImpl - hibernate.ddl-auto: validate - datasource: - driverClassName: org.postgresql.Driver - url: jdbc:postgresql://${DB_HOST:127.0.0.1}:${DB_PORT:5432}/${DB_DATABASE:postgres} - username: ${DB_USERNAME:postgres} - password: ${DB_PASSWORD:password} - flyway: - baseline-on-migrate: true - -management: - metrics: - export: - simple: - enabled: false - statsd: - enabled: true - host: ${STATSD_HOST:localhost} - port: ${STATSD_PORT:8125} diff --git a/java/core/src/main/resources/banner.txt b/java/core/src/main/resources/banner.txt deleted file mode 100644 index d0f1c033eb..0000000000 --- a/java/core/src/main/resources/banner.txt +++ /dev/null @@ -1,14 +0,0 @@ - -███████╗███████╗ █████╗ ███████╗████████╗ -██╔════╝██╔════╝██╔══██╗██╔════╝╚══██╔══╝ -█████╗ █████╗ ███████║███████╗ ██║ -██╔══╝ ██╔══╝ ██╔══██║╚════██║ ██║ -██║ ███████╗██║ ██║███████║ ██║ -╚═╝ ╚══════╝╚═╝ ╚═╝╚══════╝ ╚═╝ - - ██████╗ ██████╗ ██████╗ ███████╗ -██╔════╝██╔═══██╗██╔══██╗██╔════╝ -██║ ██║ ██║██████╔╝█████╗ -██║ ██║ ██║██╔══██╗██╔══╝ -╚██████╗╚██████╔╝██║ ██║███████╗ - ╚═════╝ ╚═════╝ ╚═╝ ╚═╝╚══════╝ diff --git a/java/core/src/main/resources/db/migration/V1__Baseline.sql b/java/core/src/main/resources/db/migration/V1__Baseline.sql deleted file mode 100644 index 262e0586ef..0000000000 --- a/java/core/src/main/resources/db/migration/V1__Baseline.sql +++ /dev/null @@ -1,185 +0,0 @@ --- --- Dump of Feast Database (as of RELEASE 0.5) --- Baseline dump for migrating to flyway --- - - -SET statement_timeout = 0; -SET lock_timeout = 0; -SET idle_in_transaction_session_timeout = 0; -SET client_encoding = 'UTF8'; -SET default_tablespace = ''; - - -CREATE TABLE entities ( - id bigint NOT NULL, - name character varying(255), - type character varying(255), - feature_set_id bigint -); - - -CREATE TABLE feature_sets ( - id bigint NOT NULL, - created timestamp without time zone NOT NULL, - last_updated timestamp without time zone NOT NULL, - labels text, - max_age bigint, - name character varying(255) NOT NULL, - status character varying(255), - project_name character varying(255), - source character varying(255) -); - - -CREATE TABLE features ( - id bigint NOT NULL, - archived boolean NOT NULL, - bool_domain bytea, - domain character varying(255), - float_domain bytea, - group_presence bytea, - image_domain bytea, - int_domain bytea, - labels text, - mid_domain bytea, - name character varying(255), - natural_language_domain bytea, - presence bytea, - shape bytea, - string_domain bytea, - struct_domain bytea, - time_domain bytea, - time_of_day_domain bytea, - type character varying(255), - url_domain bytea, - value_count bytea, - feature_set_id bigint -); - - - -CREATE SEQUENCE hibernate_sequence - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - -CREATE TABLE jobs ( - id character varying(255) NOT NULL, - created timestamp without time zone NOT NULL, - last_updated timestamp without time zone NOT NULL, - ext_id character varying(255), - runner character varying(255), - status character varying(16), - source_id character varying(255), - store_name character varying(255) -); - - -CREATE TABLE jobs_feature_sets ( - job_id character varying(255) NOT NULL, - feature_sets_id bigint NOT NULL -); - - -CREATE TABLE projects ( - name character varying(255) NOT NULL, - archived boolean NOT NULL -); - - -CREATE TABLE sources ( - id character varying(255) NOT NULL, - bootstrap_servers character varying(255), - is_default boolean, - topics character varying(255), - type character varying(255) NOT NULL -); - - -CREATE TABLE stores ( - name character varying(255) NOT NULL, - config oid NOT NULL, - subscriptions character varying(255), - type character varying(255) NOT NULL -); - - -ALTER TABLE ONLY entities - ADD CONSTRAINT entities_pkey PRIMARY KEY (id); - - -ALTER TABLE ONLY feature_sets - ADD CONSTRAINT feature_sets_pkey PRIMARY KEY (id); - - -ALTER TABLE ONLY features - ADD CONSTRAINT features_pkey PRIMARY KEY (id); - - -ALTER TABLE ONLY jobs - ADD CONSTRAINT jobs_pkey PRIMARY KEY (id); - - -ALTER TABLE ONLY projects - ADD CONSTRAINT projects_pkey PRIMARY KEY (name); - - -ALTER TABLE ONLY sources - ADD CONSTRAINT sources_pkey PRIMARY KEY (id); - - -ALTER TABLE ONLY stores - ADD CONSTRAINT stores_pkey PRIMARY KEY (name); - - -ALTER TABLE ONLY entities - ADD CONSTRAINT uk4hredqqfh86prhp1hf08nofvk UNIQUE (name, feature_set_id); - - -ALTER TABLE ONLY features - ADD CONSTRAINT ukedouxmpcoev743cmstfwq25yp UNIQUE (name, feature_set_id); - - -ALTER TABLE ONLY feature_sets - ADD CONSTRAINT ukoajkc7tn9nwhodjrbcjri5jix UNIQUE (name, project_name); - - -CREATE INDEX idx_jobs_feature_sets_feature_sets_id ON jobs_feature_sets USING btree (feature_sets_id); - -CREATE INDEX idx_jobs_feature_sets_job_id ON jobs_feature_sets USING btree (job_id); - - -ALTER TABLE ONLY feature_sets - ADD CONSTRAINT fk2di8f74x6wir076hrfbyi1qfh FOREIGN KEY (source) REFERENCES sources(id); - - -ALTER TABLE ONLY jobs_feature_sets - ADD CONSTRAINT fk2qt5yj45cr02spdhp59h4wpeg FOREIGN KEY (job_id) REFERENCES jobs(id); - - -ALTER TABLE ONLY jobs - ADD CONSTRAINT fk3dwuno3phk8j3iwdl4cckdqqd FOREIGN KEY (store_name) REFERENCES stores(name); - - -ALTER TABLE ONLY features - ADD CONSTRAINT fkfxcpsscvj0g89o4p5dx4insb1 FOREIGN KEY (feature_set_id) REFERENCES feature_sets(id); - - -ALTER TABLE ONLY jobs - ADD CONSTRAINT fkhkfwvhc2gei0wqw5h4mfvsy9f FOREIGN KEY (source_id) REFERENCES sources(id); - - -ALTER TABLE ONLY entities - ADD CONSTRAINT fkhyblh5sfunv00a8ums8ms9otq FOREIGN KEY (feature_set_id) REFERENCES feature_sets(id); - - -ALTER TABLE ONLY feature_sets - ADD CONSTRAINT fkiiqcdeuuq9mf0tmt7jtnln3oa FOREIGN KEY (project_name) REFERENCES projects(name); - - -ALTER TABLE ONLY jobs_feature_sets - ADD CONSTRAINT fkroca9etjw89c48e8jays6jl4l FOREIGN KEY (feature_sets_id) REFERENCES feature_sets(id); diff --git a/java/core/src/main/resources/db/migration/V2.1__Many_Stores_Per_Job.sql b/java/core/src/main/resources/db/migration/V2.1__Many_Stores_Per_Job.sql deleted file mode 100644 index c70df67039..0000000000 --- a/java/core/src/main/resources/db/migration/V2.1__Many_Stores_Per_Job.sql +++ /dev/null @@ -1,16 +0,0 @@ --- Migrating to Many2Many relationship between Job and Store - -CREATE TABLE jobs_stores( - job_id character varying(255) NOT NULL, - store_name character varying(255) NOT NULL -); - -ALTER TABLE ONLY jobs_stores - ADD CONSTRAINT jobs_stores_job_fkey FOREIGN KEY (job_id) REFERENCES jobs(id); - -ALTER TABLE ONLY jobs_stores - ADD CONSTRAINT jobs_stores_store_fkey FOREIGN KEY (store_name) REFERENCES stores(name); - - -CREATE INDEX idx_jobs_stores_job_id ON jobs_stores USING btree (job_id); -CREATE INDEX idx_jobs_stores_store_name ON jobs_stores USING btree (store_name); diff --git a/java/core/src/main/resources/db/migration/V2.2__Subscription_Migration.sql b/java/core/src/main/resources/db/migration/V2.2__Subscription_Migration.sql deleted file mode 100644 index 624bf77377..0000000000 --- a/java/core/src/main/resources/db/migration/V2.2__Subscription_Migration.sql +++ /dev/null @@ -1,3 +0,0 @@ -UPDATE stores - SET subscriptions = array_to_string(string_to_array(subscriptions, ':') || '{false}', ':') - WHERE array_length(string_to_array(subscriptions, ':'), 1) = 2; \ No newline at end of file diff --git a/java/core/src/main/resources/db/migration/V2.3__Fix_Primary_Keys.sql b/java/core/src/main/resources/db/migration/V2.3__Fix_Primary_Keys.sql deleted file mode 100644 index 98cb8c2aaa..0000000000 --- a/java/core/src/main/resources/db/migration/V2.3__Fix_Primary_Keys.sql +++ /dev/null @@ -1,3 +0,0 @@ -ALTER TABLE jobs_feature_sets ADD CONSTRAINT jobs_feature_sets_pkey PRIMARY KEY (job_id, feature_sets_id); - -ALTER TABLE jobs_stores ADD CONSTRAINT jobs_stores_pkey PRIMARY KEY (job_id, store_name); \ No newline at end of file diff --git a/java/core/src/main/resources/db/migration/V2.4__Store_proto.sql b/java/core/src/main/resources/db/migration/V2.4__Store_proto.sql deleted file mode 100644 index 9e18313e95..0000000000 --- a/java/core/src/main/resources/db/migration/V2.4__Store_proto.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE jobs_stores ADD COLUMN store_proto oid not null; \ No newline at end of file diff --git a/java/core/src/main/resources/db/migration/V2.5__Fix_Subscription_MIgration.sql b/java/core/src/main/resources/db/migration/V2.5__Fix_Subscription_MIgration.sql deleted file mode 100644 index 4ef3013c2c..0000000000 --- a/java/core/src/main/resources/db/migration/V2.5__Fix_Subscription_MIgration.sql +++ /dev/null @@ -1,14 +0,0 @@ -WITH updates as ( - select name, - array_to_string(array_agg(b.array_to_string), ',') as subscriptions - from ( - select name, array_to_string(string_to_array(subscriptions, ':') || '{false}', ':') - from ( - select name, unnest(string_to_array(subscriptions, ',')) as subscriptions from stores) a - WHERE array_length(string_to_array(subscriptions, ':'), 1) = 2) b - group by name -) -UPDATE stores -SET subscriptions = updates.subscriptions -FROM updates -WHERE stores.name = updates.name \ No newline at end of file diff --git a/java/core/src/main/resources/db/migration/V2.6__Create_Project_If_Not_Exist.sql b/java/core/src/main/resources/db/migration/V2.6__Create_Project_If_Not_Exist.sql deleted file mode 100644 index 0d824aed70..0000000000 --- a/java/core/src/main/resources/db/migration/V2.6__Create_Project_If_Not_Exist.sql +++ /dev/null @@ -1,7 +0,0 @@ --- SQL migration to migrate create 'default' project if it does not already exist. -INSERT INTO projects - (name, archived) - SELECT 'default', false - WHERE NOT EXISTS ( - SELECT name FROM projects WHERE name='default' - ); diff --git a/java/core/src/main/resources/db/migration/V2.7__Entities_Higher_Level_Concept.sql b/java/core/src/main/resources/db/migration/V2.7__Entities_Higher_Level_Concept.sql deleted file mode 100644 index 183664ad65..0000000000 --- a/java/core/src/main/resources/db/migration/V2.7__Entities_Higher_Level_Concept.sql +++ /dev/null @@ -1,21 +0,0 @@ --- Migrating to Entities as a higher-level concept - -CREATE TABLE entities_v2( - id bigint NOT NULL, - created timestamp without time zone NOT NULL, - last_updated timestamp without time zone NOT NULL, - name character varying(255), - project_name character varying(255), - type character varying(255), - description text, - labels text -); - -ALTER TABLE ONLY entities_v2 - ADD CONSTRAINT entities_v2_pkey PRIMARY KEY (id); - -ALTER TABLE ONLY entities_v2 - ADD CONSTRAINT entities_v2_project_ukey UNIQUE (name, project_name); - -ALTER TABLE ONLY entities_v2 - ADD CONSTRAINT entities_v2_project_fkey FOREIGN KEY (project_name) REFERENCES projects(name); \ No newline at end of file diff --git a/java/core/src/main/resources/db/migration/V2.8__Feature_Tables_API.sql b/java/core/src/main/resources/db/migration/V2.8__Feature_Tables_API.sql deleted file mode 100644 index a083dfd1ae..0000000000 --- a/java/core/src/main/resources/db/migration/V2.8__Feature_Tables_API.sql +++ /dev/null @@ -1,66 +0,0 @@ --- Data Sources SQL table used to Store Feature project -CREATE TABLE data_sources ( - id bigint NOT NULL, - type character varying(255) NOT NULL, - field_mapping text NOT NULL, - timestamp_column character varying(255), - date_partition_column character varying(255), - -- Only the options corresponding to type should set & non-null - -- DataSource Options - config text, - - CONSTRAINT data_sources_pkey PRIMARY KEY (id) -); - --- Feature Table SQL table used to store FeatureTables protobuf -CREATE TABLE feature_tables ( - id bigint NOT NULL, - project_name character varying(255), - name character varying(255) NOT NULL, - created timestamp without time zone NOT NULL, - last_updated timestamp without time zone NOT NULL, - labels text NOT NULL, - max_age bigint NOT NULL, - stream_source_id bigint, - batch_source_id bigint, - revision int NOT NULL , - - CONSTRAINT feature_tables_pkey PRIMARY KEY (id), - CONSTRAINT feature_tables_project_fkey FOREIGN KEY (project_name) - REFERENCES projects(name), - CONSTRAINT feature_tables_stream_data_source_fkey FOREIGN KEY (stream_source_id) - REFERENCES data_sources(id), - CONSTRAINT feature_tables_batch_data_source_fkey FOREIGN KEY (batch_source_id) - REFERENCES data_sources(id), - -- Feature Tables must be unique within a project - CONSTRAINT feature_tables_unique_project UNIQUE (name, project_name) -); - --- Join table between feature tables and entities V2 -CREATE TABLE feature_tables_entities_v2 ( - feature_table_id bigint NOT NULL, - entity_v2_id bigint NOT NULL, - - CONSTRAINT feature_tables_entities_v2_pkey PRIMARY KEY (feature_table_id, entity_v2_id), - CONSTRAINT feature_tables_entities_v2_join_feature_tables_fkey - FOREIGN KEY (feature_table_id) REFERENCES feature_tables(id), - CONSTRAINT feature_tables_entities_v2_join_entities_v2_fkey - FOREIGN KEY (entity_v2_id) REFERENCES entities_v2 (id) -); - - --- Feature v2 SQL table used to store FeatureSpecV2 protobuf -CREATE TABLE features_v2 ( - id bigint NOT NULL, - feature_table_id bigint NOT NULL, - name character varying(255), - type character varying(255), - labels text NOT NULL, - - CONSTRAINT features_v2_pkey PRIMARY KEY (id), - CONSTRAINT features_v2_feature_table_fkey FOREIGN KEY (feature_table_id) - REFERENCES feature_tables(id), - -- Features should be unique within a feature set - CONSTRAINT feature_v2_feature_table_unique UNIQUE (name, feature_table_id) -); - diff --git a/java/core/src/main/resources/db/migration/V2.9__Data_Source_Created_Timestamp_Column.sql b/java/core/src/main/resources/db/migration/V2.9__Data_Source_Created_Timestamp_Column.sql deleted file mode 100644 index ddbeffc584..0000000000 --- a/java/core/src/main/resources/db/migration/V2.9__Data_Source_Created_Timestamp_Column.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE data_sources ADD COLUMN created_timestamp_column character varying(255); \ No newline at end of file diff --git a/java/core/src/main/resources/db/migration/V2__RELEASE_0.6_Generalizing_Source_AND_Extending_FeatureSetJobStatus_AND_Feature_Statistics.sql b/java/core/src/main/resources/db/migration/V2__RELEASE_0.6_Generalizing_Source_AND_Extending_FeatureSetJobStatus_AND_Feature_Statistics.sql deleted file mode 100644 index 230a5f5c8c..0000000000 --- a/java/core/src/main/resources/db/migration/V2__RELEASE_0.6_Generalizing_Source_AND_Extending_FeatureSetJobStatus_AND_Feature_Statistics.sql +++ /dev/null @@ -1,124 +0,0 @@ ---- Feast Release 0.6 - ---- New fields from FeatureSetJobStatus (version & deliveryStatus) - -ALTER TABLE jobs_feature_sets - ADD column version int4 default 0; - -ALTER TABLE jobs_feature_sets - ADD column delivery_status varchar(255); - -ALTER TABLE feature_sets - ADD column version int4 default 0; - -UPDATE feature_sets SET version = 1; - - ---- FeatureStatistics Creation - -CREATE TABLE feature_statistics -( - id integer NOT NULL, - average_length real NOT NULL, - avg_bytes real NOT NULL, - avg_num_values real NOT NULL, - count bigint NOT NULL, - dataset_id character varying(255), - date timestamp without time zone, - feature_type character varying(255), - max double precision NOT NULL, - max_bytes real NOT NULL, - max_num_values bigint NOT NULL, - mean double precision NOT NULL, - median double precision NOT NULL, - min double precision NOT NULL, - min_bytes real NOT NULL, - min_num_values bigint NOT NULL, - num_missing bigint NOT NULL, - num_values_histogram bytea, - numeric_value_histogram bytea, - numeric_value_quantiles bytea, - rank_histogram bytea, - stdev double precision NOT NULL, - top_values bytea, - total_num_values bigint NOT NULL, - n_unique bigint, - zeroes bigint NOT NULL, - feature_id bigint -); - - -ALTER TABLE ONLY feature_statistics - ADD CONSTRAINT feature_statistics_pkey PRIMARY KEY (id); - -CREATE INDEX idx_feature_statistics_dataset_id ON public.feature_statistics USING btree (dataset_id); - -CREATE INDEX idx_feature_statistics_date ON public.feature_statistics USING btree (date); - -CREATE INDEX idx_feature_statistics_feature ON public.feature_statistics USING btree (feature_id); - -ALTER TABLE ONLY feature_statistics - ADD CONSTRAINT feature_statistics_feature_fkey FOREIGN KEY (feature_id) REFERENCES public.features (id); - - ---- Releasing previous PK in Source - -ALTER TABLE feature_sets - DROP CONSTRAINT fk2di8f74x6wir076hrfbyi1qfh; -ALTER TABLE jobs - DROP CONSTRAINT fkhkfwvhc2gei0wqw5h4mfvsy9f; -ALTER TABLE sources - DROP CONSTRAINT sources_pkey; -ALTER TABLE sources - ALTER COLUMN id DROP NOT NULL; - ---- Migrating to auto-incremental Source primary key - -ALTER TABLE sources - ADD column pk SERIAL PRIMARY KEY; - --- ALTER TABLE sources --- ALTER column type Type int4 USING ('{"KAFKA": 1}'::json ->> type)::INTEGER; -ALTER TABLE sources - ADD column config varchar(255); - - ---- Update all related to Source tables - -ALTER TABLE feature_sets - ADD COLUMN source_id int4; - --- foreign key on source(id) -> source(pk) - -ALTER TABLE feature_sets - ADD CONSTRAINT feature_sets_to_sources_fkey - FOREIGN KEY (source_id) REFERENCES sources (pk); - - -ALTER TABLE jobs - ADD COLUMN source_type varchar(255); -- Enum SourceType -ALTER TABLE jobs - ADD COLUMN source_config varchar(255); - - - ---- Migrate Data ---- creating sources with identical primary keys as in feature_sets - -SELECT feature_sets.id, - sources.bootstrap_servers, - sources.topics, - sources.is_default, - sources.type -INTO TEMP TABLE converted_sources -FROM feature_sets, - sources -WHERE feature_sets.source = sources.id; - -DELETE FROM sources; - -INSERT INTO sources (pk, bootstrap_servers, topics, is_default, type) - (SELECT * FROM converted_sources); - -UPDATE feature_sets -SET source_id = id; diff --git a/java/core/src/main/resources/db/migration/V3.0__Feature_Table_Deletion.sql b/java/core/src/main/resources/db/migration/V3.0__Feature_Table_Deletion.sql deleted file mode 100644 index e915ec07d4..0000000000 --- a/java/core/src/main/resources/db/migration/V3.0__Feature_Table_Deletion.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE feature_tables ADD COLUMN is_deleted boolean NOT NULL; \ No newline at end of file diff --git a/java/core/src/main/resources/log4j2.xml b/java/core/src/main/resources/log4j2.xml deleted file mode 100644 index 8781d668a8..0000000000 --- a/java/core/src/main/resources/log4j2.xml +++ /dev/null @@ -1,48 +0,0 @@ - - - - - - - %d{yyyy-MM-dd HH:mm:ss.SSS} %5p ${hostName} --- [%15.15t] %-40.40c{1.} : %m%n%ex - - - {"time":"%d{yyyy-MM-dd'T'HH:mm:ssXXX}","hostname":"${hostName}","severity":"%p","message":%m}%n%ex - - - - - - - - - - - - - - - - - - - - - - - diff --git a/java/core/src/main/resources/templates/bq_training.tmpl b/java/core/src/main/resources/templates/bq_training.tmpl deleted file mode 100644 index a92666e91b..0000000000 --- a/java/core/src/main/resources/templates/bq_training.tmpl +++ /dev/null @@ -1,12 +0,0 @@ -SELECT - id, - event_timestamp{%- if features | length > 0 %},{%- endif %} - {{ features | join(',') }} -FROM - `{{ table_id }}` -WHERE event_timestamp >= TIMESTAMP("{{ start_date }}") AND event_timestamp <= TIMESTAMP(DATETIME_ADD("{{ end_date }}", INTERVAL 1 DAY)) -{%- for key, val in number_filters.items() %} AND {{ key }} = {{ val }} {%- endfor %} -{%- for key, val in string_filters.items() %} AND {{ key }} = "{{ val }}" {%- endfor %} -{% if limit is not none -%} -LIMIT {{ limit }} -{%- endif %} \ No newline at end of file diff --git a/java/core/src/main/resources/templates/bq_view.tmpl b/java/core/src/main/resources/templates/bq_view.tmpl deleted file mode 100644 index 861e5c1658..0000000000 --- a/java/core/src/main/resources/templates/bq_view.tmpl +++ /dev/null @@ -1,23 +0,0 @@ -WITH {{tableName}} AS ( - SELECT - ROW_NUMBER() OVER (PARTITION BY id, event_timestamp ORDER BY created_timestamp ASC) rownum, - id, - event_timestamp, - created_timestamp - {{#features}} - ,FIRST_VALUE({{name}} IGNORE NULLS) OVER w as {{name}} - {{/features}} - FROM - `{{project}}.{{dataset}}.{{tableName}}` - WINDOW w AS ( - PARTITION BY id, event_timestamp ORDER BY event_timestamp DESC - ) -) -SELECT - id, - event_timestamp, - created_timestamp - {{#features}} - ,{{name}} - {{/features}} -FROM {{tableName}} WHERE rownum = 1 \ No newline at end of file diff --git a/java/core/src/test/java/feast/core/CoreApplicationTest.java b/java/core/src/test/java/feast/core/CoreApplicationTest.java deleted file mode 100644 index 7a35fc4369..0000000000 --- a/java/core/src/test/java/feast/core/CoreApplicationTest.java +++ /dev/null @@ -1,19 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core; - -public class CoreApplicationTest {} diff --git a/java/core/src/test/java/feast/core/annotation/IntegrationTest.java b/java/core/src/test/java/feast/core/annotation/IntegrationTest.java deleted file mode 100644 index f1617e32a6..0000000000 --- a/java/core/src/test/java/feast/core/annotation/IntegrationTest.java +++ /dev/null @@ -1,19 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.annotation; - -public interface IntegrationTest {} diff --git a/java/core/src/test/java/feast/core/auth/CoreServiceAuthTest.java b/java/core/src/test/java/feast/core/auth/CoreServiceAuthTest.java deleted file mode 100644 index 4bfa084e15..0000000000 --- a/java/core/src/test/java/feast/core/auth/CoreServiceAuthTest.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.auth; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import avro.shaded.com.google.common.collect.ImmutableMap; -import com.google.protobuf.InvalidProtocolBufferException; -import feast.common.auth.authorization.AuthorizationProvider; -import feast.common.auth.authorization.AuthorizationResult; -import feast.common.auth.config.SecurityProperties; -import feast.common.auth.service.AuthorizationService; -import feast.common.it.DataGenerator; -import feast.core.config.FeastProperties; -import feast.core.dao.ProjectRepository; -import feast.core.grpc.CoreServiceImpl; -import feast.core.service.ProjectService; -import feast.core.service.SpecService; -import feast.proto.core.CoreServiceProto.ApplyEntityRequest; -import feast.proto.core.CoreServiceProto.ApplyEntityResponse; -import feast.proto.core.EntityProto; -import feast.proto.types.ValueProto; -import io.grpc.internal.testing.StreamRecorder; -import org.junit.Test; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; -import org.springframework.security.core.Authentication; -import org.springframework.security.core.context.SecurityContext; -import org.springframework.security.core.context.SecurityContextHolder; - -public class CoreServiceAuthTest { - - private CoreServiceImpl coreService; - private ProjectService projectService; - - @Mock private SpecService specService; - @Mock private ProjectRepository projectRepository; - @Mock private AuthorizationProvider authProvider; - - public CoreServiceAuthTest() { - MockitoAnnotations.initMocks(this); - SecurityProperties.AuthorizationProperties authProp = - new SecurityProperties.AuthorizationProperties(); - authProp.setEnabled(true); - SecurityProperties sp = new SecurityProperties(); - sp.setAuthorization(authProp); - FeastProperties feastProperties = new FeastProperties(); - feastProperties.setSecurity(sp); - projectService = new ProjectService(projectRepository); - AuthorizationService authService = - new AuthorizationService(feastProperties.getSecurity(), authProvider); - coreService = new CoreServiceImpl(specService, projectService, feastProperties, authService); - } - - @Test - public void shouldNotApplyEntityIfNotProjectMember() throws InvalidProtocolBufferException { - - String project = "project1"; - Authentication auth = mock(Authentication.class); - SecurityContext context = mock(SecurityContext.class); - SecurityContextHolder.setContext(context); - when(context.getAuthentication()).thenReturn(auth); - - doReturn(AuthorizationResult.failed(null)) - .when(authProvider) - .checkAccessToProject(anyString(), any(Authentication.class)); - - StreamRecorder responseObserver = StreamRecorder.create(); - EntityProto.EntitySpecV2 incomingEntitySpec = - DataGenerator.createEntitySpecV2( - "entity1", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - - ApplyEntityRequest request = - ApplyEntityRequest.newBuilder().setProject(project).setSpec(incomingEntitySpec).build(); - - coreService.applyEntity(request, responseObserver); - assertEquals("PERMISSION_DENIED: Access Denied", responseObserver.getError().getMessage()); - } - - @Test - public void shouldApplyEntityIfProjectMember() throws InvalidProtocolBufferException { - - String project = "project1"; - Authentication auth = mock(Authentication.class); - SecurityContext context = mock(SecurityContext.class); - SecurityContextHolder.setContext(context); - when(context.getAuthentication()).thenReturn(auth); - doReturn(AuthorizationResult.success()) - .when(authProvider) - .checkAccessToProject(anyString(), any(Authentication.class)); - - StreamRecorder responseObserver = StreamRecorder.create(); - EntityProto.EntitySpecV2 incomingEntitySpec = - DataGenerator.createEntitySpecV2( - "entity1", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - ApplyEntityRequest request = - ApplyEntityRequest.newBuilder().setProject(project).setSpec(incomingEntitySpec).build(); - - coreService.applyEntity(request, responseObserver); - } -} diff --git a/java/core/src/test/java/feast/core/auth/CoreServiceAuthenticationIT.java b/java/core/src/test/java/feast/core/auth/CoreServiceAuthenticationIT.java deleted file mode 100644 index 77cae43cd8..0000000000 --- a/java/core/src/test/java/feast/core/auth/CoreServiceAuthenticationIT.java +++ /dev/null @@ -1,185 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.auth; - -import static org.junit.jupiter.api.Assertions.*; - -import avro.shaded.com.google.common.collect.ImmutableMap; -import com.github.tomakehurst.wiremock.client.WireMock; -import com.github.tomakehurst.wiremock.junit.WireMockClassRule; -import com.nimbusds.jose.JOSEException; -import com.nimbusds.jose.jwk.JWKSet; -import feast.common.it.BaseIT; -import feast.common.it.DataGenerator; -import feast.common.it.SimpleCoreClient; -import feast.core.auth.infra.JwtHelper; -import feast.core.config.FeastProperties; -import feast.proto.core.*; -import feast.proto.types.ValueProto; -import io.grpc.CallCredentials; -import io.grpc.Channel; -import io.grpc.ManagedChannelBuilder; -import java.util.*; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.jupiter.api.*; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.context.TestConfiguration; -import org.springframework.test.context.DynamicPropertyRegistry; -import org.springframework.test.context.DynamicPropertySource; -import org.springframework.util.SocketUtils; - -@SpringBootTest( - properties = { - "feast.security.authentication.enabled=true", - "feast.security.authorization.enabled=false", - }) -public class CoreServiceAuthenticationIT extends BaseIT { - - @Autowired FeastProperties feastProperties; - - private static int feast_core_port; - private static int JWKS_PORT = SocketUtils.findAvailableTcpPort(); - - private static JwtHelper jwtHelper = new JwtHelper(); - - static String subjectClaim = "sub"; - - @ClassRule public static WireMockClassRule wireMockRule = new WireMockClassRule(JWKS_PORT); - - @Rule public WireMockClassRule instanceRule = wireMockRule; - - static SimpleCoreClient insecureApiClient; - - @DynamicPropertySource - static void initialize(DynamicPropertyRegistry registry) { - - // Start Wiremock Server to act as fake JWKS server - wireMockRule.start(); - JWKSet keySet = jwtHelper.getKeySet(); - String jwksJson = String.valueOf(keySet.toPublicJWKSet().toJSONObject()); - - // When Feast Core looks up a Json Web Token Key Set, we provide our self-signed public key - wireMockRule.stubFor( - WireMock.get(WireMock.urlPathEqualTo("/.well-known/jwks.json")) - .willReturn( - WireMock.aResponse() - .withStatus(200) - .withHeader("Content-Type", "application/json") - .withBody(jwksJson))); - - String jwkEndpointURI = - String.format("http://localhost:%s/.well-known/jwks.json", wireMockRule.port()); - - // Initialize dynamic properties - registry.add("feast.security.authentication.options.subjectClaim", () -> subjectClaim); - registry.add("feast.security.authentication.options.jwkEndpointURI", () -> jwkEndpointURI); - } - - @BeforeAll - public static void globalSetUp(@Value("${grpc.server.port}") int port) { - feast_core_port = port; - - // Create insecure Feast Core gRPC client - Channel insecureChannel = - ManagedChannelBuilder.forAddress("localhost", feast_core_port).usePlaintext().build(); - CoreServiceGrpc.CoreServiceBlockingStub insecureCoreService = - CoreServiceGrpc.newBlockingStub(insecureChannel); - insecureApiClient = new SimpleCoreClient(insecureCoreService); - } - - @AfterAll - static void tearDown() { - wireMockRule.stop(); - } - - @Test - public void shouldGetVersionFromFeastCoreAlways() { - SimpleCoreClient secureApiClient = - getSecureApiClient("fakeUserThatIsAuthenticated@example.com"); - - String feastCoreVersionSecure = secureApiClient.getFeastCoreVersion(); - String feastCoreVersionInsecure = insecureApiClient.getFeastCoreVersion(); - - assertEquals(feastCoreVersionSecure, feastCoreVersionInsecure); - assertEquals(feastProperties.getVersion(), feastCoreVersionSecure); - } - - /** - * If authentication is enabled but authorization is disabled, users can still connect to Feast - * Core as anonymous users. They are not forced to authenticate. - */ - @Test - public void shouldAllowUnauthenticatedEntityApplyAndListing() { - String project = "default"; - EntityProto.EntitySpecV2 expectedEntitySpec = - DataGenerator.createEntitySpecV2( - "entity1", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - insecureApiClient.simpleApplyEntity(project, expectedEntitySpec); - - List listEntitiesResponse = insecureApiClient.simpleListEntities(project); - EntityProto.Entity actualEntity = listEntitiesResponse.get(0); - - assert listEntitiesResponse.size() == 1; - assertEquals(actualEntity.getSpec().getName(), expectedEntitySpec.getName()); - } - - @Test - public void shouldAllowAuthenticatedEntityApplyAndListing() { - SimpleCoreClient secureApiClient = - getSecureApiClient("AuthenticatedUserWithoutAuthorization@example.com"); - String project = "default"; - EntityProto.EntitySpecV2 expectedEntitySpec = - DataGenerator.createEntitySpecV2( - "entity1", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - secureApiClient.simpleApplyEntity(project, expectedEntitySpec); - List listEntitiesResponse = insecureApiClient.simpleListEntities(project); - EntityProto.Entity actualEntity = listEntitiesResponse.get(0); - - assert listEntitiesResponse.size() == 1; - assertEquals(actualEntity.getSpec().getName(), expectedEntitySpec.getName()); - } - - @TestConfiguration - public static class TestConfig extends BaseTestConfig {} - - // Create secure Feast Core gRPC client for a specific user - private static SimpleCoreClient getSecureApiClient(String subjectEmail) { - CallCredentials callCredentials = null; - try { - callCredentials = jwtHelper.getCallCredentials(subjectEmail); - } catch (JOSEException e) { - throw new RuntimeException( - String.format("Could not build call credentials: %s", e.getMessage())); - } - Channel secureChannel = - ManagedChannelBuilder.forAddress("localhost", feast_core_port).usePlaintext().build(); - - CoreServiceGrpc.CoreServiceBlockingStub secureCoreService = - CoreServiceGrpc.newBlockingStub(secureChannel).withCallCredentials(callCredentials); - - return new SimpleCoreClient(secureCoreService); - } -} diff --git a/java/core/src/test/java/feast/core/auth/CoreServiceAuthorizationIT.java b/java/core/src/test/java/feast/core/auth/CoreServiceAuthorizationIT.java deleted file mode 100644 index 41faee7f71..0000000000 --- a/java/core/src/test/java/feast/core/auth/CoreServiceAuthorizationIT.java +++ /dev/null @@ -1,354 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.auth; - -import static org.junit.jupiter.api.Assertions.*; -import static org.testcontainers.containers.wait.strategy.Wait.forHttp; - -import avro.shaded.com.google.common.collect.ImmutableMap; -import com.github.tomakehurst.wiremock.client.WireMock; -import com.github.tomakehurst.wiremock.junit.WireMockClassRule; -import com.google.protobuf.InvalidProtocolBufferException; -import com.nimbusds.jose.JOSEException; -import com.nimbusds.jose.jwk.JWKSet; -import feast.common.it.BaseIT; -import feast.common.it.DataGenerator; -import feast.common.it.SimpleCoreClient; -import feast.core.auth.infra.JwtHelper; -import feast.core.config.FeastProperties; -import feast.proto.core.CoreServiceGrpc; -import feast.proto.core.EntityProto; -import feast.proto.types.ValueProto; -import io.grpc.CallCredentials; -import io.grpc.Channel; -import io.grpc.ManagedChannelBuilder; -import io.grpc.StatusRuntimeException; -import java.io.File; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.context.TestConfiguration; -import org.springframework.test.context.DynamicPropertyRegistry; -import org.springframework.test.context.DynamicPropertySource; -import org.springframework.util.SocketUtils; -import org.testcontainers.containers.DockerComposeContainer; -import sh.ory.keto.ApiClient; -import sh.ory.keto.ApiException; -import sh.ory.keto.Configuration; -import sh.ory.keto.api.EnginesApi; -import sh.ory.keto.model.OryAccessControlPolicy; -import sh.ory.keto.model.OryAccessControlPolicyRole; - -@SpringBootTest( - properties = { - "feast.security.authentication.enabled=true", - "feast.security.authorization.enabled=true", - "feast.security.authorization.provider=http", - }) -public class CoreServiceAuthorizationIT extends BaseIT { - - @Autowired FeastProperties feastProperties; - - private static final String DEFAULT_FLAVOR = "glob"; - private static int KETO_PORT = 4466; - private static int KETO_ADAPTOR_PORT = 8080; - private static int feast_core_port; - private static int JWKS_PORT = SocketUtils.findAvailableTcpPort(); - - private static JwtHelper jwtHelper = new JwtHelper(); - - static String project = "myproject"; - static String subjectInProject = "good_member@example.com"; - static String subjectIsAdmin = "bossman@example.com"; - static String subjectClaim = "sub"; - - static SimpleCoreClient insecureApiClient; - - @ClassRule public static WireMockClassRule wireMockRule = new WireMockClassRule(JWKS_PORT); - - @Rule public WireMockClassRule instanceRule = wireMockRule; - - @ClassRule - public static DockerComposeContainer environment = - new DockerComposeContainer(new File("src/test/resources/keto/docker-compose.yml")) - .withExposedService("adaptor_1", KETO_ADAPTOR_PORT) - .withExposedService("keto_1", KETO_PORT, forHttp("/health/ready").forStatusCode(200)); - - @DynamicPropertySource - static void initialize(DynamicPropertyRegistry registry) { - - // Start Keto and with Docker Compose - environment.start(); - - // Seed Keto with data - String ketoExternalHost = environment.getServiceHost("keto_1", KETO_PORT); - Integer ketoExternalPort = environment.getServicePort("keto_1", KETO_PORT); - String ketoExternalUrl = String.format("http://%s:%s", ketoExternalHost, ketoExternalPort); - try { - seedKeto(ketoExternalUrl); - } catch (ApiException e) { - throw new RuntimeException(String.format("Could not seed Keto store %s", ketoExternalUrl)); - } - - // Start Wiremock Server to act as fake JWKS server - wireMockRule.start(); - JWKSet keySet = jwtHelper.getKeySet(); - String jwksJson = String.valueOf(keySet.toPublicJWKSet().toJSONObject()); - - // When Feast Core looks up a Json Web Token Key Set, we provide our self-signed public key - wireMockRule.stubFor( - WireMock.get(WireMock.urlPathEqualTo("/.well-known/jwks.json")) - .willReturn( - WireMock.aResponse() - .withStatus(200) - .withHeader("Content-Type", "application/json") - .withBody(jwksJson))); - - String jwkEndpointURI = - String.format("http://localhost:%s/.well-known/jwks.json", wireMockRule.port()); - - // Get Keto Authorization Server (Adaptor) url - String ketoAdaptorHost = environment.getServiceHost("adaptor_1", KETO_ADAPTOR_PORT); - Integer ketoAdaptorPort = environment.getServicePort("adaptor_1", KETO_ADAPTOR_PORT); - String ketoAdaptorUrl = String.format("http://%s:%s", ketoAdaptorHost, ketoAdaptorPort); - - // Initialize dynamic properties - registry.add("feast.security.authentication.options.subjectClaim", () -> subjectClaim); - registry.add("feast.security.authentication.options.jwkEndpointURI", () -> jwkEndpointURI); - registry.add("feast.security.authorization.options.authorizationUrl", () -> ketoAdaptorUrl); - } - - @BeforeAll - public static void globalSetUp(@Value("${grpc.server.port}") int port) { - feast_core_port = port; - // Create insecure Feast Core gRPC client - Channel insecureChannel = - ManagedChannelBuilder.forAddress("localhost", feast_core_port).usePlaintext().build(); - CoreServiceGrpc.CoreServiceBlockingStub insecureCoreService = - CoreServiceGrpc.newBlockingStub(insecureChannel); - insecureApiClient = new SimpleCoreClient(insecureCoreService); - } - - @BeforeEach - public void setUp() { - SimpleCoreClient secureApiClient = getSecureApiClient(subjectIsAdmin); - EntityProto.EntitySpecV2 expectedEntitySpec = - DataGenerator.createEntitySpecV2( - "entity1", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - secureApiClient.simpleApplyEntity(project, expectedEntitySpec); - } - - @AfterAll - static void tearDown() { - environment.stop(); - wireMockRule.stop(); - } - - @Test - public void shouldGetVersionFromFeastCoreAlways() { - SimpleCoreClient secureApiClient = - getSecureApiClient("fakeUserThatIsAuthenticated@example.com"); - - String feastCoreVersionSecure = secureApiClient.getFeastCoreVersion(); - String feastCoreVersionInsecure = insecureApiClient.getFeastCoreVersion(); - - assertEquals(feastCoreVersionSecure, feastCoreVersionInsecure); - assertEquals(feastProperties.getVersion(), feastCoreVersionSecure); - } - - @Test - public void shouldNotAllowUnauthenticatedEntityListing() { - Exception exception = - assertThrows( - StatusRuntimeException.class, - () -> { - insecureApiClient.simpleListEntities("8"); - }); - - String expectedMessage = "UNAUTHENTICATED: Authentication failed"; - String actualMessage = exception.getMessage(); - assertEquals(actualMessage, expectedMessage); - } - - @Test - public void shouldAllowAuthenticatedEntityListing() { - SimpleCoreClient secureApiClient = - getSecureApiClient("AuthenticatedUserWithoutAuthorization@example.com"); - EntityProto.EntitySpecV2 expectedEntitySpec = - DataGenerator.createEntitySpecV2( - "entity1", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - List listEntitiesResponse = secureApiClient.simpleListEntities("myproject"); - EntityProto.Entity actualEntity = listEntitiesResponse.get(0); - - assert listEntitiesResponse.size() == 1; - assertEquals(actualEntity.getSpec().getName(), expectedEntitySpec.getName()); - } - - @Test - void cantApplyEntityIfNotProjectMember() throws InvalidProtocolBufferException { - String userName = "random_user@example.com"; - SimpleCoreClient secureApiClient = getSecureApiClient(userName); - EntityProto.EntitySpecV2 expectedEntitySpec = - DataGenerator.createEntitySpecV2( - "entity1", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - - StatusRuntimeException exception = - assertThrows( - StatusRuntimeException.class, - () -> secureApiClient.simpleApplyEntity(project, expectedEntitySpec)); - - String expectedMessage = - String.format( - "PERMISSION_DENIED: Access denied to project %s for subject %s", project, userName); - String actualMessage = exception.getMessage(); - assertEquals(actualMessage, expectedMessage); - } - - @Test - void canApplyEntityIfProjectMember() { - SimpleCoreClient secureApiClient = getSecureApiClient(subjectInProject); - EntityProto.EntitySpecV2 expectedEntitySpec = - DataGenerator.createEntitySpecV2( - "entity_6", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - - secureApiClient.simpleApplyEntity(project, expectedEntitySpec); - - EntityProto.Entity actualEntity = secureApiClient.simpleGetEntity(project, "entity_6"); - - assertEquals(expectedEntitySpec.getName(), actualEntity.getSpec().getName()); - assertEquals(expectedEntitySpec.getValueType(), actualEntity.getSpec().getValueType()); - } - - @Test - void canApplyEntityIfAdmin() { - SimpleCoreClient secureApiClient = getSecureApiClient(subjectIsAdmin); - EntityProto.EntitySpecV2 expectedEntitySpec = - DataGenerator.createEntitySpecV2( - "entity_7", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - - secureApiClient.simpleApplyEntity(project, expectedEntitySpec); - - EntityProto.Entity actualEntity = secureApiClient.simpleGetEntity(project, "entity_7"); - - assertEquals(expectedEntitySpec.getName(), actualEntity.getSpec().getName()); - assertEquals(expectedEntitySpec.getValueType(), actualEntity.getSpec().getValueType()); - } - - @TestConfiguration - public static class TestConfig extends BaseTestConfig {} - - private static void seedKeto(String url) throws ApiException { - ApiClient ketoClient = Configuration.getDefaultApiClient(); - ketoClient.setBasePath(url); - EnginesApi enginesApi = new EnginesApi(ketoClient); - - // Add policies - OryAccessControlPolicy adminPolicy = getAdminPolicy(); - enginesApi.upsertOryAccessControlPolicy(DEFAULT_FLAVOR, adminPolicy); - - OryAccessControlPolicy projectPolicy = getMyProjectMemberPolicy(); - enginesApi.upsertOryAccessControlPolicy(DEFAULT_FLAVOR, projectPolicy); - - // Add policy roles - OryAccessControlPolicyRole adminPolicyRole = getAdminPolicyRole(); - enginesApi.upsertOryAccessControlPolicyRole(DEFAULT_FLAVOR, adminPolicyRole); - - OryAccessControlPolicyRole myProjectMemberPolicyRole = getMyProjectMemberPolicyRole(); - enginesApi.upsertOryAccessControlPolicyRole(DEFAULT_FLAVOR, myProjectMemberPolicyRole); - } - - private static OryAccessControlPolicyRole getMyProjectMemberPolicyRole() { - OryAccessControlPolicyRole role = new OryAccessControlPolicyRole(); - role.setId(String.format("roles:%s-project-members", project)); - role.setMembers(Collections.singletonList("users:" + subjectInProject)); - return role; - } - - private static OryAccessControlPolicyRole getAdminPolicyRole() { - OryAccessControlPolicyRole role = new OryAccessControlPolicyRole(); - role.setId("roles:admin"); - role.setMembers(Collections.singletonList("users:" + subjectIsAdmin)); - return role; - } - - private static OryAccessControlPolicy getAdminPolicy() { - OryAccessControlPolicy policy = new OryAccessControlPolicy(); - policy.setId("policies:admin"); - policy.subjects(Collections.singletonList("roles:admin")); - policy.resources(Collections.singletonList("resources:**")); - policy.actions(Collections.singletonList("actions:**")); - policy.effect("allow"); - policy.conditions(null); - return policy; - } - - private static OryAccessControlPolicy getMyProjectMemberPolicy() { - OryAccessControlPolicy policy = new OryAccessControlPolicy(); - policy.setId(String.format("policies:%s-project-members-policy", project)); - policy.subjects(Collections.singletonList(String.format("roles:%s-project-members", project))); - policy.resources( - Arrays.asList( - String.format("resources:projects:%s", project), - String.format("resources:projects:%s:**", project))); - policy.actions(Collections.singletonList("actions:**")); - policy.effect("allow"); - policy.conditions(null); - return policy; - } - - // Create secure Feast Core gRPC client for a specific user - private static SimpleCoreClient getSecureApiClient(String subjectEmail) { - CallCredentials callCredentials = null; - try { - callCredentials = jwtHelper.getCallCredentials(subjectEmail); - } catch (JOSEException e) { - throw new RuntimeException( - String.format("Could not build call credentials: %s", e.getMessage())); - } - Channel secureChannel = - ManagedChannelBuilder.forAddress("localhost", feast_core_port).usePlaintext().build(); - - CoreServiceGrpc.CoreServiceBlockingStub secureCoreService = - CoreServiceGrpc.newBlockingStub(secureChannel).withCallCredentials(callCredentials); - - return new SimpleCoreClient(secureCoreService); - } -} diff --git a/java/core/src/test/java/feast/core/auth/CoreServiceKetoAuthorizationIT.java b/java/core/src/test/java/feast/core/auth/CoreServiceKetoAuthorizationIT.java deleted file mode 100644 index 0a09cced2c..0000000000 --- a/java/core/src/test/java/feast/core/auth/CoreServiceKetoAuthorizationIT.java +++ /dev/null @@ -1,352 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.auth; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.testcontainers.containers.wait.strategy.Wait.forHttp; - -import avro.shaded.com.google.common.collect.ImmutableMap; -import com.github.tomakehurst.wiremock.client.WireMock; -import com.github.tomakehurst.wiremock.junit.WireMockClassRule; -import com.google.protobuf.InvalidProtocolBufferException; -import com.nimbusds.jose.JOSEException; -import com.nimbusds.jose.jwk.JWKSet; -import feast.common.it.BaseIT; -import feast.common.it.DataGenerator; -import feast.common.it.SimpleCoreClient; -import feast.core.auth.infra.JwtHelper; -import feast.core.config.FeastProperties; -import feast.proto.core.CoreServiceGrpc; -import feast.proto.core.EntityProto; -import feast.proto.types.ValueProto; -import io.grpc.CallCredentials; -import io.grpc.Channel; -import io.grpc.ManagedChannelBuilder; -import io.grpc.StatusRuntimeException; -import java.io.File; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.context.TestConfiguration; -import org.springframework.test.context.DynamicPropertyRegistry; -import org.springframework.test.context.DynamicPropertySource; -import org.springframework.util.SocketUtils; -import org.testcontainers.containers.DockerComposeContainer; -import sh.ory.keto.ApiClient; -import sh.ory.keto.ApiException; -import sh.ory.keto.Configuration; -import sh.ory.keto.api.EnginesApi; -import sh.ory.keto.model.OryAccessControlPolicy; -import sh.ory.keto.model.OryAccessControlPolicyRole; - -@SpringBootTest( - properties = { - "feast.security.authentication.enabled=true", - "feast.security.authorization.enabled=true", - "feast.security.authorization.provider=keto", - "feast.security.authorization.options.action=actions:any", - "feast.security.authorization.options.subjectPrefix=users:", - "feast.security.authorization.options.resourcePrefix=resources:projects:", - }) -public class CoreServiceKetoAuthorizationIT extends BaseIT { - - @Autowired FeastProperties feastProperties; - - private static final String DEFAULT_FLAVOR = "glob"; - private static int KETO_PORT = 4466; - private static int feast_core_port; - private static int JWKS_PORT = SocketUtils.findAvailableTcpPort(); - - private static JwtHelper jwtHelper = new JwtHelper(); - - static String project = "myproject"; - static String subjectInProject = "good_member@example.com"; - static String subjectIsAdmin = "bossman@example.com"; - static String subjectClaim = "sub"; - - static SimpleCoreClient insecureApiClient; - - @ClassRule public static WireMockClassRule wireMockRule = new WireMockClassRule(JWKS_PORT); - - @Rule public WireMockClassRule instanceRule = wireMockRule; - - @ClassRule - public static DockerComposeContainer environment = - new DockerComposeContainer(new File("src/test/resources/keto/docker-compose.yml")) - .withExposedService("keto_1", KETO_PORT, forHttp("/health/ready").forStatusCode(200)); - - @DynamicPropertySource - static void initialize(DynamicPropertyRegistry registry) { - - // Start Keto and with Docker Compose - environment.start(); - - // Seed Keto with data - String ketoExternalHost = environment.getServiceHost("keto_1", KETO_PORT); - Integer ketoExternalPort = environment.getServicePort("keto_1", KETO_PORT); - String ketoExternalUrl = String.format("http://%s:%s", ketoExternalHost, ketoExternalPort); - try { - seedKeto(ketoExternalUrl); - } catch (ApiException e) { - throw new RuntimeException(String.format("Could not seed Keto store %s", ketoExternalUrl)); - } - - // Start Wiremock Server to act as fake JWKS server - wireMockRule.start(); - JWKSet keySet = jwtHelper.getKeySet(); - String jwksJson = String.valueOf(keySet.toPublicJWKSet().toJSONObject()); - - // When Feast Core looks up a Json Web Token Key Set, we provide our self-signed public key - wireMockRule.stubFor( - WireMock.get(WireMock.urlPathEqualTo("/.well-known/jwks.json")) - .willReturn( - WireMock.aResponse() - .withStatus(200) - .withHeader("Content-Type", "application/json") - .withBody(jwksJson))); - - String jwkEndpointURI = - String.format("http://localhost:%s/.well-known/jwks.json", wireMockRule.port()); - - // Initialize dynamic properties - registry.add("feast.security.authentication.options.subjectClaim", () -> subjectClaim); - registry.add("feast.security.authentication.options.jwkEndpointURI", () -> jwkEndpointURI); - registry.add("feast.security.authorization.options.authorizationUrl", () -> ketoExternalUrl); - registry.add("feast.security.authorization.options.flavor", () -> DEFAULT_FLAVOR); - } - - @BeforeAll - public static void globalSetUp(@Value("${grpc.server.port}") int port) { - feast_core_port = port; - // Create insecure Feast Core gRPC client - Channel insecureChannel = - ManagedChannelBuilder.forAddress("localhost", feast_core_port).usePlaintext().build(); - CoreServiceGrpc.CoreServiceBlockingStub insecureCoreService = - CoreServiceGrpc.newBlockingStub(insecureChannel); - insecureApiClient = new SimpleCoreClient(insecureCoreService); - } - - @BeforeEach - public void setUp() { - SimpleCoreClient secureApiClient = getSecureApiClient(subjectIsAdmin); - EntityProto.EntitySpecV2 expectedEntitySpec = - DataGenerator.createEntitySpecV2( - "entity1", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - secureApiClient.simpleApplyEntity(project, expectedEntitySpec); - } - - @AfterAll - static void tearDown() { - environment.stop(); - wireMockRule.stop(); - } - - @Test - public void shouldGetVersionFromFeastCoreAlways() { - SimpleCoreClient secureApiClient = - getSecureApiClient("fakeUserThatIsAuthenticated@example.com"); - - String feastCoreVersionSecure = secureApiClient.getFeastCoreVersion(); - String feastCoreVersionInsecure = insecureApiClient.getFeastCoreVersion(); - - assertEquals(feastCoreVersionSecure, feastCoreVersionInsecure); - assertEquals(feastProperties.getVersion(), feastCoreVersionSecure); - } - - @Test - public void shouldNotAllowUnauthenticatedEntityListing() { - Exception exception = - assertThrows( - StatusRuntimeException.class, - () -> { - insecureApiClient.simpleListEntities("8"); - }); - - String expectedMessage = "UNAUTHENTICATED: Authentication failed"; - String actualMessage = exception.getMessage(); - assertEquals(actualMessage, expectedMessage); - } - - @Test - public void shouldAllowAuthenticatedEntityListing() { - SimpleCoreClient secureApiClient = - getSecureApiClient("AuthenticatedUserWithoutAuthorization@example.com"); - EntityProto.EntitySpecV2 expectedEntitySpec = - DataGenerator.createEntitySpecV2( - "entity1", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - List listEntitiesResponse = secureApiClient.simpleListEntities("myproject"); - EntityProto.Entity actualEntity = listEntitiesResponse.get(0); - - assert listEntitiesResponse.size() == 1; - assertEquals(actualEntity.getSpec().getName(), expectedEntitySpec.getName()); - } - - @Test - void cantApplyEntityIfNotProjectMember() throws InvalidProtocolBufferException { - String userName = "random_user@example.com"; - SimpleCoreClient secureApiClient = getSecureApiClient(userName); - EntityProto.EntitySpecV2 expectedEntitySpec = - DataGenerator.createEntitySpecV2( - "entity1", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - - StatusRuntimeException exception = - assertThrows( - StatusRuntimeException.class, - () -> secureApiClient.simpleApplyEntity(project, expectedEntitySpec)); - - String expectedMessage = - String.format( - "PERMISSION_DENIED: Access denied to project %s for subject %s", project, userName); - String actualMessage = exception.getMessage(); - assertEquals(actualMessage, expectedMessage); - } - - @Test - void canApplyEntityIfProjectMember() { - SimpleCoreClient secureApiClient = getSecureApiClient(subjectInProject); - EntityProto.EntitySpecV2 expectedEntitySpec = - DataGenerator.createEntitySpecV2( - "entity_6", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - - secureApiClient.simpleApplyEntity(project, expectedEntitySpec); - - EntityProto.Entity actualEntity = secureApiClient.simpleGetEntity(project, "entity_6"); - - assertEquals(expectedEntitySpec.getName(), actualEntity.getSpec().getName()); - assertEquals(expectedEntitySpec.getValueType(), actualEntity.getSpec().getValueType()); - } - - @Test - void canApplyEntityIfAdmin() { - SimpleCoreClient secureApiClient = getSecureApiClient(subjectIsAdmin); - EntityProto.EntitySpecV2 expectedEntitySpec = - DataGenerator.createEntitySpecV2( - "entity_7", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - - secureApiClient.simpleApplyEntity(project, expectedEntitySpec); - - EntityProto.Entity actualEntity = secureApiClient.simpleGetEntity(project, "entity_7"); - - assertEquals(expectedEntitySpec.getName(), actualEntity.getSpec().getName()); - assertEquals(expectedEntitySpec.getValueType(), actualEntity.getSpec().getValueType()); - } - - @TestConfiguration - public static class TestConfig extends BaseTestConfig {} - - private static void seedKeto(String url) throws ApiException { - ApiClient ketoClient = Configuration.getDefaultApiClient(); - ketoClient.setBasePath(url); - EnginesApi enginesApi = new EnginesApi(ketoClient); - - // Add policies - OryAccessControlPolicy adminPolicy = getAdminPolicy(); - enginesApi.upsertOryAccessControlPolicy(DEFAULT_FLAVOR, adminPolicy); - - OryAccessControlPolicy projectPolicy = getMyProjectMemberPolicy(); - enginesApi.upsertOryAccessControlPolicy(DEFAULT_FLAVOR, projectPolicy); - - // Add policy roles - OryAccessControlPolicyRole adminPolicyRole = getAdminPolicyRole(); - enginesApi.upsertOryAccessControlPolicyRole(DEFAULT_FLAVOR, adminPolicyRole); - - OryAccessControlPolicyRole myProjectMemberPolicyRole = getMyProjectMemberPolicyRole(); - enginesApi.upsertOryAccessControlPolicyRole(DEFAULT_FLAVOR, myProjectMemberPolicyRole); - } - - private static OryAccessControlPolicyRole getMyProjectMemberPolicyRole() { - OryAccessControlPolicyRole role = new OryAccessControlPolicyRole(); - role.setId(String.format("roles:%s-project-members", project)); - role.setMembers(Collections.singletonList("users:" + subjectInProject)); - return role; - } - - private static OryAccessControlPolicyRole getAdminPolicyRole() { - OryAccessControlPolicyRole role = new OryAccessControlPolicyRole(); - role.setId("roles:admin"); - role.setMembers(Collections.singletonList("users:" + subjectIsAdmin)); - return role; - } - - private static OryAccessControlPolicy getAdminPolicy() { - OryAccessControlPolicy policy = new OryAccessControlPolicy(); - policy.setId("policies:admin"); - policy.subjects(Collections.singletonList("roles:admin")); - policy.resources(Collections.singletonList("resources:**")); - policy.actions(Collections.singletonList("actions:**")); - policy.effect("allow"); - policy.conditions(null); - return policy; - } - - private static OryAccessControlPolicy getMyProjectMemberPolicy() { - OryAccessControlPolicy policy = new OryAccessControlPolicy(); - policy.setId(String.format("policies:%s-project-members-policy", project)); - policy.subjects(Collections.singletonList(String.format("roles:%s-project-members", project))); - policy.resources( - Arrays.asList( - String.format("resources:projects:%s", project), - String.format("resources:projects:%s:**", project))); - policy.actions(Collections.singletonList("actions:**")); - policy.effect("allow"); - policy.conditions(null); - return policy; - } - - // Create secure Feast Core gRPC client for a specific user - private static SimpleCoreClient getSecureApiClient(String subjectEmail) { - CallCredentials callCredentials = null; - try { - callCredentials = jwtHelper.getCallCredentials(subjectEmail); - } catch (JOSEException e) { - throw new RuntimeException( - String.format("Could not build call credentials: %s", e.getMessage())); - } - Channel secureChannel = - ManagedChannelBuilder.forAddress("localhost", feast_core_port).usePlaintext().build(); - - CoreServiceGrpc.CoreServiceBlockingStub secureCoreService = - CoreServiceGrpc.newBlockingStub(secureChannel).withCallCredentials(callCredentials); - - return new SimpleCoreClient(secureCoreService); - } -} diff --git a/java/core/src/test/java/feast/core/auth/infra/JwtHelper.java b/java/core/src/test/java/feast/core/auth/infra/JwtHelper.java deleted file mode 100644 index 091956fb88..0000000000 --- a/java/core/src/test/java/feast/core/auth/infra/JwtHelper.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.auth.infra; - -import com.nimbusds.jose.JOSEException; -import com.nimbusds.jose.JOSEObjectType; -import com.nimbusds.jose.JWSAlgorithm; -import com.nimbusds.jose.JWSHeader; -import com.nimbusds.jose.crypto.RSASSASigner; -import com.nimbusds.jose.jwk.*; -import com.nimbusds.jose.jwk.gen.RSAKeyGenerator; -import com.nimbusds.jwt.JWTClaimsSet; -import com.nimbusds.jwt.SignedJWT; -import feast.common.auth.credentials.JwtCallCredentials; -import io.grpc.*; -import java.security.interfaces.RSAPublicKey; -import java.time.Instant; -import java.util.Date; - -public final class JwtHelper { - - private static RSAKey key = null; - private JWKSet keySet; - - public JwtHelper() { - try { - key = new RSAKeyGenerator(2048).keyID("123").generate(); - } catch (JOSEException e) { - throw new RuntimeException("Could not generate RSA key"); - } - RSAKey.Builder builder = null; - try { - builder = - new RSAKey.Builder((RSAPublicKey) this.getKey().toKeyPair().getPublic()) - .keyUse(KeyUse.SIGNATURE) - .algorithm(JWSAlgorithm.RS256) - .keyID(this.getKey().getKeyID()); - } catch (JOSEException e) { - throw new RuntimeException("Could not create RSAKey builder"); - } - keySet = new JWKSet(builder.build()); - } - - public CallCredentials getCallCredentials(String email) throws JOSEException { - String jwt = createToken(email); - return new JwtCallCredentials(jwt); - } - - public String createToken(String email) throws JOSEException { - assert key != null; - JWSHeader header = - new JWSHeader.Builder(JWSAlgorithm.RS256) - .type(JOSEObjectType.JWT) - .keyID(key.getKeyID()) - .build(); - - JWTClaimsSet payload = - new JWTClaimsSet.Builder() - .issuer("me") - .audience("you") - .subject(email) - .expirationTime(Date.from(Instant.now().plusSeconds(120))) - .build(); - - SignedJWT signedJWT = new SignedJWT(header, payload); - signedJWT.sign(new RSASSASigner(key.toRSAPrivateKey())); - return signedJWT.serialize(); - } - - public RSAKey getKey() { - return key; - } - - public JWKSet getKeySet() { - return this.keySet; - } -} diff --git a/java/core/src/test/java/feast/core/controller/CoreServiceRestIT.java b/java/core/src/test/java/feast/core/controller/CoreServiceRestIT.java deleted file mode 100644 index f26ce8a343..0000000000 --- a/java/core/src/test/java/feast/core/controller/CoreServiceRestIT.java +++ /dev/null @@ -1,214 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.controller; - -import static io.restassured.RestAssured.get; -import static org.hamcrest.Matchers.aMapWithSize; -import static org.hamcrest.Matchers.notNullValue; -import static org.junit.jupiter.api.Assertions.assertEquals; - -import com.google.common.collect.ImmutableMap; -import feast.common.it.BaseIT; -import feast.common.it.DataGenerator; -import feast.common.it.SimpleCoreClient; -import feast.proto.core.CoreServiceGrpc; -import feast.proto.core.EntityProto; -import feast.proto.core.FeatureTableProto; -import feast.proto.types.ValueProto; -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; -import io.restassured.RestAssured; -import io.restassured.http.ContentType; -import io.restassured.path.json.JsonPath; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.test.autoconfigure.web.reactive.AutoConfigureWebTestClient; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.context.TestConfiguration; -import org.springframework.boot.web.server.LocalServerPort; -import org.springframework.web.util.UriComponentsBuilder; - -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@AutoConfigureWebTestClient -public class CoreServiceRestIT extends BaseIT { - - static CoreServiceGrpc.CoreServiceBlockingStub stub; - static SimpleCoreClient apiClient; - @LocalServerPort private int port; - - @TestConfiguration - public static class TestConfig extends BaseTestConfig {} - - @BeforeAll - public static void globalSetUp(@Value("${grpc.server.port}") int port) { - ManagedChannel channel = - ManagedChannelBuilder.forAddress("localhost", port).usePlaintext().build(); - stub = CoreServiceGrpc.newBlockingStub(channel); - apiClient = new SimpleCoreClient(stub); - } - - @Test - public void getVersion() { - String uriString = UriComponentsBuilder.fromPath("/api/v2/version").toUriString(); - get(uriString) - .then() - .log() - .everything() - .assertThat() - .contentType(ContentType.JSON) - .body("version", notNullValue()); - } - - // list projects - @Test - public void listProjects() { - // should get 2 projects - String uriString = UriComponentsBuilder.fromPath("/api/v2/projects").toUriString(); - String responseBody = - get(uriString) - .then() - .log() - .everything() - .assertThat() - .contentType(ContentType.JSON) - .extract() - .response() - .getBody() - .asString(); - List projectList = JsonPath.from(responseBody).getList("projects"); - assertEquals(projectList, List.of("default")); - } - - @Test - public void listFeatures() { - String uri1 = - UriComponentsBuilder.fromPath("/api/v2/features") - .queryParam("entities", "entity1", "entity2") - .buildAndExpand() - .toString(); - get(uri1) - .then() - .log() - .everything() - .assertThat() - .contentType(ContentType.JSON) - .body("features", aMapWithSize(2)); - - String uri2 = - UriComponentsBuilder.fromPath("/api/v2/features") - .queryParam("entities", "entity1", "entity2") - .queryParam("project", "default") - .buildAndExpand() - .toString(); - get(uri2) - .then() - .log() - .everything() - .assertThat() - .contentType(ContentType.JSON) - .body("features", aMapWithSize(2)); - } - - @Test - public void listEntities() { - String uri1 = - UriComponentsBuilder.fromPath("/api/v2/entities") - .queryParam("project", "default") - .buildAndExpand() - .toString(); - String responseBody = - get(uri1) - .then() - .log() - .everything() - .assertThat() - .contentType(ContentType.JSON) - .extract() - .response() - .getBody() - .asString(); - List entityList = JsonPath.from(responseBody).getList("entities"); - assertEquals(entityList.size(), 2); - } - - @Test - public void listFeatureTables() { - String uri1 = - UriComponentsBuilder.fromPath("/api/v2/feature-tables") - .queryParam("project", "default") - .buildAndExpand() - .toString(); - String responseBody = - get(uri1) - .then() - .log() - .everything() - .assertThat() - .contentType(ContentType.JSON) - .extract() - .response() - .getBody() - .asString(); - List featureTableList = JsonPath.from(responseBody).getList("tables"); - assertEquals(featureTableList.size(), 1); - } - - @BeforeEach - private void createSpecs() { - // Apply entities - EntityProto.EntitySpecV2 entitySpec1 = - DataGenerator.createEntitySpecV2( - "entity1", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - avro.shaded.com.google.common.collect.ImmutableMap.of("label_key", "label_value")); - EntityProto.EntitySpecV2 entitySpec2 = - DataGenerator.createEntitySpecV2( - "entity2", - "Entity 2 description", - ValueProto.ValueType.Enum.STRING, - avro.shaded.com.google.common.collect.ImmutableMap.of("label_key2", "label_value2")); - apiClient.simpleApplyEntity("default", entitySpec1); - apiClient.simpleApplyEntity("default", entitySpec2); - - // Apply feature table - FeatureTableProto.FeatureTableSpec featureTableSpec = - DataGenerator.createFeatureTableSpec( - "featuretable1", - Arrays.asList("entity1", "entity2"), - new HashMap<>() { - { - put("feature1", ValueProto.ValueType.Enum.STRING); - put("feature2", ValueProto.ValueType.Enum.FLOAT); - } - }, - 7200, - ImmutableMap.of("feat_key2", "feat_value2")) - .toBuilder() - .setBatchSource( - DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", "")) - .build(); - apiClient.applyFeatureTable("default", featureTableSpec); - - RestAssured.port = port; - } -} diff --git a/java/core/src/test/java/feast/core/logging/CoreLoggingIT.java b/java/core/src/test/java/feast/core/logging/CoreLoggingIT.java deleted file mode 100644 index 0f137b4639..0000000000 --- a/java/core/src/test/java/feast/core/logging/CoreLoggingIT.java +++ /dev/null @@ -1,229 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.logging; - -import static org.hamcrest.CoreMatchers.*; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import com.google.common.collect.Streams; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.util.JsonFormat; -import feast.common.it.BaseIT; -import feast.common.it.DataGenerator; -import feast.common.logging.entry.AuditLogEntryKind; -import feast.proto.core.CoreServiceGrpc; -import feast.proto.core.CoreServiceGrpc.CoreServiceBlockingStub; -import feast.proto.core.CoreServiceGrpc.CoreServiceFutureStub; -import feast.proto.core.CoreServiceProto.GetFeastCoreVersionRequest; -import feast.proto.core.CoreServiceProto.ListFeatureTablesRequest; -import feast.proto.core.CoreServiceProto.ListStoresRequest; -import feast.proto.core.CoreServiceProto.ListStoresResponse; -import feast.proto.core.CoreServiceProto.UpdateStoreRequest; -import feast.proto.core.CoreServiceProto.UpdateStoreResponse; -import io.grpc.Channel; -import io.grpc.ManagedChannelBuilder; -import io.grpc.Status.Code; -import io.grpc.StatusRuntimeException; -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.core.LoggerContext; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.test.context.SpringBootTest; - -@SpringBootTest( - properties = { - "feast.logging.audit.enabled=true", - "feast.logging.audit.messageLogging.enabled=true", - "feast.logging.audit.messageLogging.destination=console" - }) -public class CoreLoggingIT extends BaseIT { - private static TestLogAppender testAuditLogAppender; - private static CoreServiceBlockingStub coreService; - private static CoreServiceFutureStub asyncCoreService; - - @BeforeAll - public static void globalSetUp(@Value("${grpc.server.port}") int coreGrpcPort) - throws InterruptedException, ExecutionException { - LoggerContext logContext = (LoggerContext) LogManager.getContext(false); - // NOTE: As log appender state is shared across tests use a different method - // for each test and filter by method name to ensure that you only get logs - // for a specific test. - testAuditLogAppender = logContext.getConfiguration().getAppender("TestAuditLogAppender"); - - // Connect to core service. - Channel channel = - ManagedChannelBuilder.forAddress("localhost", coreGrpcPort).usePlaintext().build(); - coreService = CoreServiceGrpc.newBlockingStub(channel); - asyncCoreService = CoreServiceGrpc.newFutureStub(channel); - - // Preflight a request to core service stubs to verify connection - coreService.getFeastCoreVersion(GetFeastCoreVersionRequest.getDefaultInstance()); - asyncCoreService.getFeastCoreVersion(GetFeastCoreVersionRequest.getDefaultInstance()).get(); - } - - /** Check that messsage audit log are produced on service call */ - @Test - public void shouldProduceMessageAuditLogsOnCall() - throws InterruptedException, InvalidProtocolBufferException { - // Generate artifical load on feast core. - UpdateStoreRequest request = - UpdateStoreRequest.newBuilder().setStore(DataGenerator.getDefaultStore()).build(); - UpdateStoreResponse response = coreService.updateStore(request); - - // Wait required to ensure audit logs are flushed into test audit log appender - Thread.sleep(1000); - // Check message audit logs are produced for each audit log. - JsonFormat.Parser protoJSONParser = JsonFormat.parser(); - // Pull message audit logs logs from test log appender - List logJsonObjects = - parseMessageJsonLogObjects(testAuditLogAppender.getLogs(), "UpdateStore"); - assertEquals(1, logJsonObjects.size()); - JsonObject logObj = logJsonObjects.get(0); - - // Extract & Check that request/response are returned correctly - String requestJson = logObj.getAsJsonObject("request").toString(); - UpdateStoreRequest.Builder gotRequest = UpdateStoreRequest.newBuilder(); - protoJSONParser.merge(requestJson, gotRequest); - - String responseJson = logObj.getAsJsonObject("response").toString(); - UpdateStoreResponse.Builder gotResponse = UpdateStoreResponse.newBuilder(); - protoJSONParser.merge(responseJson, gotResponse); - - assertThat(gotRequest.build(), equalTo(request)); - assertThat(gotResponse.build(), equalTo(response)); - } - - /** Check that message audit logs are produced when server encounters an error */ - @Test - public void shouldProduceMessageAuditLogsOnError() throws InterruptedException { - // Send a bad request which should cause Core to error - ListFeatureTablesRequest request = - ListFeatureTablesRequest.newBuilder() - .setFilter(ListFeatureTablesRequest.Filter.newBuilder().setProject("*").build()) - .build(); - - boolean hasExpectedException = false; - Code statusCode = null; - try { - coreService.listFeatureTables(request); - } catch (StatusRuntimeException e) { - hasExpectedException = true; - statusCode = e.getStatus().getCode(); - } - assertTrue(hasExpectedException); - - // Wait required to ensure audit logs are flushed into test audit log appender - Thread.sleep(1000); - // Pull message audit logs logs from test log appender - List logJsonObjects = - parseMessageJsonLogObjects(testAuditLogAppender.getLogs(), "ListFeatureTables"); - - assertEquals(1, logJsonObjects.size()); - JsonObject logJsonObject = logJsonObjects.get(0); - // Check correct status code is tracked on error. - assertEquals(logJsonObject.get("statusCode").getAsString(), statusCode.toString()); - } - - /** Check that expected message audit logs are produced when under load. */ - @Test - public void shouldProduceExpectedAuditLogsUnderLoad() - throws InterruptedException, ExecutionException { - // Generate artifical requests on core to simulate load. - int LOAD_SIZE = 40; // Total number of requests to send. - int BURST_SIZE = 5; // Number of requests to send at once. - - ListStoresRequest request = ListStoresRequest.getDefaultInstance(); - List responses = new LinkedList<>(); - for (int i = 0; i < LOAD_SIZE; i += 5) { - List> futures = new LinkedList<>(); - for (int j = 0; j < BURST_SIZE; j++) { - futures.add(asyncCoreService.listStores(request)); - } - - responses.addAll(Futures.allAsList(futures).get()); - } - // Wait required to ensure audit logs are flushed into test audit log appender - Thread.sleep(1000); - - // Pull message audit logs from test log appender - List logJsonObjects = - parseMessageJsonLogObjects(testAuditLogAppender.getLogs(), "ListStores"); - assertEquals(responses.size(), logJsonObjects.size()); - - // Extract & Check that request/response are returned correctly - JsonFormat.Parser protoJSONParser = JsonFormat.parser(); - Streams.zip( - responses.stream(), - logJsonObjects.stream(), - (response, logObj) -> Pair.of(response, logObj)) - .forEach( - responseLogJsonPair -> { - ListStoresResponse response = responseLogJsonPair.getLeft(); - JsonObject logObj = responseLogJsonPair.getRight(); - - ListStoresRequest.Builder gotRequest = null; - ListStoresResponse.Builder gotResponse = null; - try { - String requestJson = logObj.getAsJsonObject("request").toString(); - gotRequest = ListStoresRequest.newBuilder(); - protoJSONParser.merge(requestJson, gotRequest); - - String responseJson = logObj.getAsJsonObject("response").toString(); - gotResponse = ListStoresResponse.newBuilder(); - protoJSONParser.merge(responseJson, gotResponse); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - - assertThat(gotRequest.build(), equalTo(request)); - assertThat(gotResponse.build(), equalTo(response)); - }); - } - - /** - * Filter and Parse out Message Audit Logs from the given logsStrings for the given method name - */ - private List parseMessageJsonLogObjects(List logsStrings, String methodName) { - JsonParser jsonParser = new JsonParser(); - // copy to prevent concurrent modification. - return logsStrings.stream() - .map(logJSON -> jsonParser.parse(logJSON).getAsJsonObject()) - // Filter to only include message audit logs - .filter( - logObj -> - logObj - .getAsJsonPrimitive("kind") - .getAsString() - .equals(AuditLogEntryKind.MESSAGE.toString()) - // filter by method name to ensure logs from other tests do not interfere with - // test - && logObj.get("method").getAsString().equals(methodName)) - .collect(Collectors.toList()); - } -} diff --git a/java/core/src/test/java/feast/core/logging/TestLogAppender.java b/java/core/src/test/java/feast/core/logging/TestLogAppender.java deleted file mode 100644 index b1908c0db2..0000000000 --- a/java/core/src/test/java/feast/core/logging/TestLogAppender.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.logging; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; -import lombok.Getter; -import org.apache.logging.log4j.core.Appender; -import org.apache.logging.log4j.core.Core; -import org.apache.logging.log4j.core.Filter; -import org.apache.logging.log4j.core.Layout; -import org.apache.logging.log4j.core.LogEvent; -import org.apache.logging.log4j.core.appender.AbstractAppender; -import org.apache.logging.log4j.core.config.Property; -import org.apache.logging.log4j.core.config.plugins.Plugin; -import org.apache.logging.log4j.core.config.plugins.PluginAttribute; -import org.apache.logging.log4j.core.config.plugins.PluginElement; -import org.apache.logging.log4j.core.config.plugins.PluginFactory; -import org.apache.logging.log4j.core.layout.PatternLayout; - -/** Test Log Appender used for collecting logs for testing logging. */ -@Plugin( - name = "TestLogAppender", - category = Core.CATEGORY_NAME, - elementType = Appender.ELEMENT_TYPE) -@Getter -public class TestLogAppender extends AbstractAppender { - private List logs; - - protected TestLogAppender(String name, Filter filter, Layout layout) { - super(name, filter, layout, false, new Property[] {}); - logs = new ArrayList<>(); - } - - @Override - public void append(LogEvent event) { - getLogs().add(event.getMessage().toString()); - } - - @PluginFactory - public static TestLogAppender createAppender( - @PluginAttribute("name") String name, - @PluginElement("Layout") Layout layout, - @PluginElement("Filter") final Filter filter) { - if (name == null) { - return null; - } - if (layout == null) { - layout = PatternLayout.createDefaultLayout(); - } - return new TestLogAppender(name, filter, layout); - } -} diff --git a/java/core/src/test/java/feast/core/metrics/CoreMetricsIT.java b/java/core/src/test/java/feast/core/metrics/CoreMetricsIT.java deleted file mode 100644 index f38dc75e81..0000000000 --- a/java/core/src/test/java/feast/core/metrics/CoreMetricsIT.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.metrics; - -import static org.junit.Assert.assertTrue; - -import com.squareup.okhttp.OkHttpClient; -import com.squareup.okhttp.Request; -import com.squareup.okhttp.Response; -import feast.common.it.BaseIT; -import java.io.IOException; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.context.SpringBootTest.WebEnvironment; -import org.springframework.boot.web.server.LocalServerPort; - -@SpringBootTest( - webEnvironment = WebEnvironment.RANDOM_PORT, - properties = { - "feast.security.authentication.enabled=true", - }) -public class CoreMetricsIT extends BaseIT { - private static final String METRIC_ENDPOINT = "/metrics"; - private static OkHttpClient httpClient; - @LocalServerPort private int metricsPort; - - @BeforeAll - public static void globalSetUp() { - httpClient = new OkHttpClient(); - } - - /** Test that Feast Core metrics endpoint can be accessed with authentication enabled */ - @Test - public void shouldAllowUnauthenticatedAccessToMetricsEndpoint() throws IOException { - Request request = - new Request.Builder() - .url(String.format("http://localhost:%d%s", metricsPort, METRIC_ENDPOINT)) - .get() - .build(); - Response response = httpClient.newCall(request).execute(); - assertTrue(response.isSuccessful()); - assertTrue(!response.body().string().isEmpty()); - } -} diff --git a/java/core/src/test/java/feast/core/model/DataSourceTest.java b/java/core/src/test/java/feast/core/model/DataSourceTest.java deleted file mode 100644 index 2dcbaa93cd..0000000000 --- a/java/core/src/test/java/feast/core/model/DataSourceTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.model; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.core.IsEqual.equalTo; - -import feast.common.it.DataGenerator; -import feast.proto.core.DataSourceProto; -import java.util.List; -import java.util.Map; -import org.junit.Test; - -public class DataSourceTest { - @Test - public void shouldSerializeFieldMappingAsJSON() { - Map expectedMap = Map.of("test", "value"); - - getTestSpecs() - .forEach( - spec -> { - DataSource source = - DataSource.fromProto(spec.toBuilder().putAllFieldMapping(expectedMap).build()); - Map actualMap = source.getFieldsMap(); - assertThat(actualMap, equalTo(actualMap)); - }); - } - - @Test - public void shouldFromProtoBeReversableWithToProto() { - getTestSpecs() - .forEach( - expectedSpec -> { - DataSourceProto.DataSource actualSpec = DataSource.fromProto(expectedSpec).toProto(); - assertThat(actualSpec, equalTo(expectedSpec)); - }); - } - - private List getTestSpecs() { - return List.of( - DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", ""), - DataGenerator.createKafkaDataSourceSpec("localhost:9092", "topic", "class.path", "ts_col"), - DataGenerator.createBigQueryDataSourceSpec("project:dataset.table", "ts_col", "dt_col"), - DataGenerator.createKinesisDataSourceSpec("ap-nowhere1", "stream", "class.path", "ts_col")); - } -} diff --git a/java/core/src/test/java/feast/core/service/ProjectServiceTest.java b/java/core/src/test/java/feast/core/service/ProjectServiceTest.java deleted file mode 100644 index afff1e55cb..0000000000 --- a/java/core/src/test/java/feast/core/service/ProjectServiceTest.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.service; - -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.*; -import static org.mockito.MockitoAnnotations.initMocks; - -import feast.core.dao.ProjectRepository; -import feast.core.model.Project; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mock; - -public class ProjectServiceTest { - - @Mock private ProjectRepository projectRepository; - private ProjectService projectService; - - @Before - public void setUp() { - initMocks(this); - projectRepository = mock(ProjectRepository.class); - projectService = new ProjectService(projectRepository); - } - - @Test - public void shouldCreateProjectIfItDoesntExist() { - String projectName = "project1"; - Project project = new Project(projectName); - when(projectRepository.saveAndFlush(project)).thenReturn(project); - projectService.createProject(projectName); - verify(projectRepository, times(1)).saveAndFlush(project); - } - - @Test(expected = IllegalArgumentException.class) - public void shouldNotCreateProjectIfItExist() { - String projectName = "project1"; - when(projectRepository.existsById(projectName)).thenReturn(true); - projectService.createProject(projectName); - } - - @Test - public void shouldArchiveProjectIfItExists() { - String projectName = "project1"; - Project project = new Project(projectName); - when(projectRepository.findById(projectName)).thenReturn(Optional.of(project)); - projectService.archiveProject(projectName); - verify(projectRepository, times(1)).saveAndFlush(project); - } - - @Test - public void shouldNotArchiveDefaultProject() { - assertThrows( - IllegalArgumentException.class, - () -> this.projectService.archiveProject(Project.DEFAULT_NAME)); - } - - @Test(expected = IllegalArgumentException.class) - public void shouldNotArchiveProjectIfItIsAlreadyArchived() { - String projectName = "project1"; - when(projectRepository.findById(projectName)).thenReturn(Optional.empty()); - projectService.archiveProject(projectName); - } - - @Test - public void shouldListProjects() { - String projectName = "project1"; - Project project = new Project(projectName); - List expected = Arrays.asList(project); - when(projectRepository.findAllByArchivedIsFalse()).thenReturn(expected); - List actual = projectService.listProjects(); - Assert.assertEquals(expected, actual); - } -} diff --git a/java/core/src/test/java/feast/core/service/SpecServiceIT.java b/java/core/src/test/java/feast/core/service/SpecServiceIT.java deleted file mode 100644 index d6d0f74fa6..0000000000 --- a/java/core/src/test/java/feast/core/service/SpecServiceIT.java +++ /dev/null @@ -1,903 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.service; - -import static com.jayway.jsonassert.impl.matcher.IsMapContainingKey.hasKey; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.beans.HasPropertyWithValue.hasProperty; -import static org.hamcrest.collection.IsCollectionWithSize.hasSize; -import static org.hamcrest.collection.IsMapWithSize.aMapWithSize; -import static org.hamcrest.core.IsEqual.equalTo; -import static org.hamcrest.core.IsIterableContaining.hasItem; -import static org.junit.Assert.assertTrue; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import avro.shaded.com.google.common.collect.ImmutableMap; -import com.google.protobuf.Duration; -import feast.common.it.BaseIT; -import feast.common.it.DataGenerator; -import feast.common.it.SimpleCoreClient; -import feast.common.util.TestUtil; -import feast.proto.core.*; -import feast.proto.core.FeatureTableProto.FeatureTableSpec; -import feast.proto.types.ValueProto; -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; -import io.grpc.StatusRuntimeException; -import java.util.*; -import java.util.stream.IntStream; -import org.apache.commons.lang3.tuple.Triple; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.test.context.SpringBootTest; -import org.testcontainers.shaded.com.google.common.collect.ImmutableList; - -@SpringBootTest -public class SpecServiceIT extends BaseIT { - - static CoreServiceGrpc.CoreServiceBlockingStub stub; - static SimpleCoreClient apiClient; - - @BeforeAll - public static void globalSetUp(@Value("${grpc.server.port}") int port) { - ManagedChannel channel = - ManagedChannelBuilder.forAddress("localhost", port).usePlaintext().build(); - stub = CoreServiceGrpc.newBlockingStub(channel); - apiClient = new SimpleCoreClient(stub); - } - - private FeatureTableProto.FeatureTableSpec example1; - private FeatureTableProto.FeatureTableSpec example2; - - @BeforeEach - public void initState() { - - EntityProto.EntitySpecV2 entitySpec1 = - DataGenerator.createEntitySpecV2( - "entity1", - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - EntityProto.EntitySpecV2 entitySpec2 = - DataGenerator.createEntitySpecV2( - "entity2", - "Entity 2 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key2", "label_value2")); - apiClient.simpleApplyEntity("default", entitySpec1); - apiClient.simpleApplyEntity("default", entitySpec2); - - example1 = - DataGenerator.createFeatureTableSpec( - "featuretable1", - Arrays.asList("entity1", "entity2"), - new HashMap<>() { - { - put("feature1", ValueProto.ValueType.Enum.STRING); - put("feature2", ValueProto.ValueType.Enum.FLOAT); - } - }, - 7200, - ImmutableMap.of("feat_key2", "feat_value2")) - .toBuilder() - .setBatchSource( - DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", "")) - .build(); - - example2 = - DataGenerator.createFeatureTableSpec( - "featuretable2", - Arrays.asList("entity1", "entity2"), - new HashMap<>() { - { - put("feature3", ValueProto.ValueType.Enum.STRING); - put("feature4", ValueProto.ValueType.Enum.FLOAT); - } - }, - 7200, - ImmutableMap.of("feat_key4", "feat_value4")) - .toBuilder() - .setBatchSource( - DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", "")) - .build(); - - apiClient.applyFeatureTable("default", example1); - apiClient.applyFeatureTable("default", example2); - apiClient.simpleApplyEntity( - "project1", - DataGenerator.createEntitySpecV2( - "entity3", - "Entity 3 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key2", "label_value2"))); - apiClient.updateStore(DataGenerator.getDefaultStore()); - } - - @Nested - class ListEntities { - @Test - public void shouldFilterEntitiesByLabels() { - List entities = - apiClient.simpleListEntities("", ImmutableMap.of("label_key2", "label_value2")); - - assertThat(entities, hasSize(1)); - assertThat(entities, hasItem(hasProperty("spec", hasProperty("name", equalTo("entity2"))))); - } - - @Test - public void shouldUseDefaultProjectIfProjectUnspecified() { - List entities = apiClient.simpleListEntities(""); - - assertThat(entities, hasSize(2)); - assertThat(entities, hasItem(hasProperty("spec", hasProperty("name", equalTo("entity1"))))); - } - - @Test - public void shouldFilterEntitiesByProjectAndLabels() { - List entities = - apiClient.simpleListEntities("project1", ImmutableMap.of("label_key2", "label_value2")); - - assertThat(entities, hasSize(1)); - assertThat(entities, hasItem(hasProperty("spec", hasProperty("name", equalTo("entity3"))))); - } - - @Test - public void shouldThrowExceptionGivenWildcardProject() { - CoreServiceProto.ListEntitiesRequest.Filter filter = - CoreServiceProto.ListEntitiesRequest.Filter.newBuilder().setProject("default*").build(); - StatusRuntimeException exc = - assertThrows(StatusRuntimeException.class, () -> apiClient.simpleListEntities(filter)); - - assertThat( - exc.getMessage(), - equalTo( - String.format( - "INVALID_ARGUMENT: invalid value for project resource, %s: " - + "argument must only contain alphanumeric characters, dashes, or underscores.", - filter.getProject()))); - } - } - - @Nested - class ListFeatureTables { - @Test - public void shouldFilterFeatureTablesByProjectAndLabels() { - CoreServiceProto.ListFeatureTablesRequest.Filter filter = - CoreServiceProto.ListFeatureTablesRequest.Filter.newBuilder() - .setProject("default") - .putAllLabels(ImmutableMap.of("feat_key2", "feat_value2")) - .build(); - List featureTables = - apiClient.simpleListFeatureTables(filter); - - assertThat(featureTables, hasSize(1)); - assertThat( - featureTables, - hasItem(hasProperty("spec", hasProperty("name", equalTo("featuretable1"))))); - } - - @Test - public void shouldUseDefaultProjectIfProjectUnspecified() { - CoreServiceProto.ListFeatureTablesRequest.Filter filter = - CoreServiceProto.ListFeatureTablesRequest.Filter.newBuilder() - .setProject("default") - .build(); - List featureTables = - apiClient.simpleListFeatureTables(filter); - - assertThat(featureTables, hasSize(2)); - assertThat( - featureTables, - hasItem(hasProperty("spec", hasProperty("name", equalTo("featuretable1"))))); - assertThat( - featureTables, - hasItem(hasProperty("spec", hasProperty("name", equalTo("featuretable2"))))); - } - - @Test - public void shouldThrowExceptionGivenWildcardProject() { - CoreServiceProto.ListFeatureTablesRequest.Filter filter = - CoreServiceProto.ListFeatureTablesRequest.Filter.newBuilder() - .setProject("default*") - .build(); - StatusRuntimeException exc = - assertThrows( - StatusRuntimeException.class, () -> apiClient.simpleListFeatureTables(filter)); - - assertThat( - exc.getMessage(), - equalTo( - String.format( - "INVALID_ARGUMENT: invalid value for project resource, %s: " - + "argument must only contain alphanumeric characters, dashes, or underscores.", - filter.getProject()))); - } - } - - @Nested - class ApplyEntity { - @Test - public void shouldThrowExceptionGivenEntityWithDash() { - StatusRuntimeException exc = - assertThrows( - StatusRuntimeException.class, - () -> - apiClient.simpleApplyEntity( - "default", - DataGenerator.createEntitySpecV2( - "dash-entity", - "Dash Entity description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("test_key", "test_value")))); - - assertThat( - exc.getMessage(), - equalTo( - String.format( - "INTERNAL: invalid value for %s resource, %s: %s", - "entity", - "dash-entity", - "argument must only contain alphanumeric characters and underscores."))); - } - - @Test - public void shouldThrowExceptionIfTypeChanged() { - String projectName = "default"; - - EntityProto.EntitySpecV2 spec = - DataGenerator.createEntitySpecV2( - "entity1", - "Entity description", - ValueProto.ValueType.Enum.FLOAT, - ImmutableMap.of("label_key", "label_value")); - - StatusRuntimeException exc = - assertThrows( - StatusRuntimeException.class, () -> apiClient.simpleApplyEntity("default", spec)); - - assertThat( - exc.getMessage(), - equalTo( - String.format( - "INTERNAL: You are attempting to change the type of this entity in %s project from %s to %s. This isn't allowed. Please create a new entity.", - "default", "STRING", spec.getValueType()))); - } - - @Test - public void shouldReturnEntityIfEntityHasNotChanged() { - String projectName = "default"; - EntityProto.EntitySpecV2 spec = apiClient.simpleGetEntity(projectName, "entity1").getSpec(); - - CoreServiceProto.ApplyEntityResponse response = - apiClient.simpleApplyEntity(projectName, spec); - - assertThat(response.getEntity().getSpec().getName(), equalTo(spec.getName())); - assertThat(response.getEntity().getSpec().getDescription(), equalTo(spec.getDescription())); - assertThat(response.getEntity().getSpec().getLabelsMap(), equalTo(spec.getLabelsMap())); - assertThat(response.getEntity().getSpec().getValueType(), equalTo(spec.getValueType())); - } - - @Test - public void shouldApplyEntityIfNotExists() { - String projectName = "default"; - EntityProto.EntitySpecV2 spec = - DataGenerator.createEntitySpecV2( - "new_entity", - "Entity description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - - CoreServiceProto.ApplyEntityResponse response = - apiClient.simpleApplyEntity(projectName, spec); - - assertThat(response.getEntity().getSpec().getName(), equalTo(spec.getName())); - assertThat(response.getEntity().getSpec().getDescription(), equalTo(spec.getDescription())); - assertThat(response.getEntity().getSpec().getLabelsMap(), equalTo(spec.getLabelsMap())); - assertThat(response.getEntity().getSpec().getValueType(), equalTo(spec.getValueType())); - } - - @Test - public void shouldCreateProjectWhenNotAlreadyExists() { - EntityProto.EntitySpecV2 spec = - DataGenerator.createEntitySpecV2( - "new_entity2", - "Entity description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("key1", "val1")); - CoreServiceProto.ApplyEntityResponse response = - apiClient.simpleApplyEntity("new_project", spec); - - assertThat(response.getEntity().getSpec().getName(), equalTo(spec.getName())); - assertThat(response.getEntity().getSpec().getDescription(), equalTo(spec.getDescription())); - assertThat(response.getEntity().getSpec().getLabelsMap(), equalTo(spec.getLabelsMap())); - assertThat(response.getEntity().getSpec().getValueType(), equalTo(spec.getValueType())); - } - - @Test - public void shouldFailWhenProjectIsArchived() { - apiClient.createProject("archived"); - apiClient.archiveProject("archived"); - - StatusRuntimeException exc = - assertThrows( - StatusRuntimeException.class, - () -> - apiClient.simpleApplyEntity( - "archived", - DataGenerator.createEntitySpecV2( - "new_entity3", - "Entity description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("key1", "val1")))); - assertThat(exc.getMessage(), equalTo("INTERNAL: Project is archived: archived")); - } - - @Test - public void shouldUpdateLabels() { - EntityProto.EntitySpecV2 spec = - DataGenerator.createEntitySpecV2( - "entity1", - "Entity description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value", "label_key2", "label_value2")); - - CoreServiceProto.ApplyEntityResponse response = apiClient.simpleApplyEntity("default", spec); - - assertThat(response.getEntity().getSpec().getLabelsMap(), equalTo(spec.getLabelsMap())); - } - } - - @Nested - class UpdateStore { - @Test - public void shouldUpdateStoreIfConfigChanges() { - StoreProto.Store defaultStore = DataGenerator.getDefaultStore(); - - StoreProto.Store updatedStore = - DataGenerator.createStore( - defaultStore.getName(), - defaultStore.getType(), - ImmutableList.of(Triple.of("project1", "*", false))); - - CoreServiceProto.UpdateStoreResponse response = apiClient.updateStore(updatedStore); - assertThat( - response.getStatus(), equalTo(CoreServiceProto.UpdateStoreResponse.Status.UPDATED)); - } - - @Test - public void shouldDoNothingIfNoChange() { - CoreServiceProto.UpdateStoreResponse response = - apiClient.updateStore(DataGenerator.getDefaultStore()); - assertThat( - response.getStatus(), equalTo(CoreServiceProto.UpdateStoreResponse.Status.NO_CHANGE)); - } - } - - @Nested - class GetEntity { - @Test - public void shouldThrowExceptionGivenMissingEntity() { - StatusRuntimeException exc = - assertThrows( - StatusRuntimeException.class, () -> apiClient.simpleGetEntity("default", "")); - - assertThat(exc.getMessage(), equalTo("INVALID_ARGUMENT: No entity name provided")); - } - - public void shouldRetrieveFromDefaultIfProjectNotSpecified() { - String entityName = "entity1"; - EntityProto.Entity entity = apiClient.simpleGetEntity("", entityName); - - assertThat(entity.getSpec().getName(), equalTo(entityName)); - } - } - - @Nested - class GetFeatureTable { - @Test - public void shouldThrowExceptionGivenNoSuchFeatureTable() { - String projectName = "default"; - String featureTableName = "invalid_table"; - StatusRuntimeException exc = - assertThrows( - StatusRuntimeException.class, - () -> apiClient.simpleGetFeatureTable(projectName, featureTableName)); - - assertThat( - exc.getMessage(), - equalTo( - String.format( - "NOT_FOUND: No such Feature Table: (project: %s, name: %s)", - projectName, featureTableName))); - } - - @Test - public void shouldReturnFeatureTableIfExists() { - FeatureTableProto.FeatureTable featureTable = - apiClient.simpleGetFeatureTable("default", "featuretable1"); - - assertTrue(TestUtil.compareFeatureTableSpec(featureTable.getSpec(), example1)); - } - } - - @Nested - class ListStores { - @Test - public void shouldReturnAllStoresIfNoNameProvided() { - apiClient.updateStore(DataGenerator.getDefaultStore()); - apiClient.updateStore( - DataGenerator.createStore( - "data", StoreProto.Store.StoreType.REDIS, Collections.emptyList())); - - List actual = - stub.listStores( - CoreServiceProto.ListStoresRequest.newBuilder() - .setFilter(CoreServiceProto.ListStoresRequest.Filter.newBuilder().build()) - .build()) - .getStoreList(); - - assertThat(actual, hasSize(2)); - assertThat(actual, hasItem(hasProperty("name", equalTo("test-store")))); - assertThat(actual, hasItem(hasProperty("name", equalTo("data")))); - } - - @Test - public void shouldThrowRetrievalExceptionIfNoStoresFoundWithName() { - StatusRuntimeException exc = - assertThrows( - StatusRuntimeException.class, - () -> - stub.listStores( - CoreServiceProto.ListStoresRequest.newBuilder() - .setFilter( - CoreServiceProto.ListStoresRequest.Filter.newBuilder() - .setName("unknown") - .build()) - .build())); - - assertThat(exc.getMessage(), equalTo("INTERNAL: Store with name 'unknown' not found")); - } - } - - @Nested - class ListFeatures { - @Test - public void shouldFilterFeaturesByEntitiesAndLabels() { - // Case 1: Only filter by entities - Map result1 = - apiClient.simpleListFeatures("default", "entity1", "entity2"); - - assertThat(result1, aMapWithSize(4)); - assertThat(result1, hasKey(equalTo("featuretable1:feature1"))); - assertThat(result1, hasKey(equalTo("featuretable1:feature2"))); - assertThat(result1, hasKey(equalTo("featuretable2:feature3"))); - assertThat(result1, hasKey(equalTo("featuretable2:feature4"))); - - // Case 2: Filter by entities and labels - Map result2 = - apiClient.simpleListFeatures( - "default", - ImmutableMap.of("feat_key2", "feat_value2"), - ImmutableList.of("entity1", "entity2")); - - assertThat(result2, aMapWithSize(2)); - assertThat(result2, hasKey(equalTo("featuretable1:feature1"))); - assertThat(result2, hasKey(equalTo("featuretable1:feature2"))); - - // Case 3: Filter by labels - Map result3 = - apiClient.simpleListFeatures( - "default", ImmutableMap.of("feat_key4", "feat_value4"), Collections.emptyList()); - - assertThat(result3, aMapWithSize(2)); - assertThat(result3, hasKey(equalTo("featuretable2:feature3"))); - assertThat(result3, hasKey(equalTo("featuretable2:feature4"))); - - // Case 4: Filter by nothing, except project - Map result4 = - apiClient.simpleListFeatures("project1", ImmutableMap.of(), Collections.emptyList()); - - assertThat(result4, aMapWithSize(0)); - - // Case 5: Filter by nothing; will use default project - Map result5 = - apiClient.simpleListFeatures("", ImmutableMap.of(), Collections.emptyList()); - - assertThat(result5, aMapWithSize(4)); - assertThat(result5, hasKey(equalTo("featuretable1:feature1"))); - assertThat(result5, hasKey(equalTo("featuretable1:feature2"))); - assertThat(result5, hasKey(equalTo("featuretable2:feature3"))); - assertThat(result5, hasKey(equalTo("featuretable2:feature4"))); - - // Case 6: Filter by mismatched entity - Map result6 = - apiClient.simpleListFeatures("default", ImmutableMap.of(), ImmutableList.of("entity1")); - assertThat(result6, aMapWithSize(0)); - } - } - - @Nested - public class ApplyFeatureTable { - private FeatureTableSpec getTestSpec() { - return example1 - .toBuilder() - .setName("apply_test") - .setStreamSource( - DataGenerator.createKafkaDataSourceSpec( - "localhost:9092", "topic", "class.path", "ts_col")) - .build(); - } - - @Test - public void shouldApplyNewValidTable() { - FeatureTableProto.FeatureTable table = apiClient.applyFeatureTable("default", getTestSpec()); - - assertTrue(TestUtil.compareFeatureTableSpec(table.getSpec(), getTestSpec())); - assertThat(table.getMeta().getRevision(), equalTo(0L)); - } - - @Test - public void shouldUpdateExistingTableWithValidSpec() { - FeatureTableProto.FeatureTable table = apiClient.applyFeatureTable("default", getTestSpec()); - - FeatureTableSpec updatedSpec = - getTestSpec() - .toBuilder() - .clearFeatures() - .addFeatures( - DataGenerator.createFeatureSpecV2( - "feature5", ValueProto.ValueType.Enum.FLOAT, ImmutableMap.of())) - .setStreamSource( - DataGenerator.createKafkaDataSourceSpec( - "localhost:9092", "new_topic", "new.class", "ts_col")) - .build(); - - FeatureTableProto.FeatureTable updatedTable = - apiClient.applyFeatureTable("default", updatedSpec); - - assertTrue(TestUtil.compareFeatureTableSpec(updatedTable.getSpec(), updatedSpec)); - assertThat(updatedTable.getMeta().getRevision(), equalTo(table.getMeta().getRevision() + 1L)); - } - - @Test - public void shouldUpdateFeatureTableOnEntityChange() { - FeatureTableProto.FeatureTableSpec updatedSpec = - getTestSpec().toBuilder().clearEntities().addEntities("entity1").build(); - - FeatureTableProto.FeatureTable updatedTable = - apiClient.applyFeatureTable("default", updatedSpec); - - assertTrue(TestUtil.compareFeatureTableSpec(updatedTable.getSpec(), updatedSpec)); - } - - @Test - public void shouldUpdateFeatureTableOnMaxAgeChange() { - FeatureTableProto.FeatureTableSpec updatedSpec = - getTestSpec() - .toBuilder() - .setMaxAge(Duration.newBuilder().setSeconds(600).build()) - .build(); - - FeatureTableProto.FeatureTable updatedTable = - apiClient.applyFeatureTable("default", updatedSpec); - - assertTrue(TestUtil.compareFeatureTableSpec(updatedTable.getSpec(), updatedSpec)); - } - - @Test - public void shouldUpdateFeatureTableOnFeatureTypeChange() { - int featureIdx = - IntStream.range(0, getTestSpec().getFeaturesCount()) - .filter(i -> getTestSpec().getFeatures(i).getName().equals("feature2")) - .findFirst() - .orElse(-1); - - FeatureTableProto.FeatureTableSpec updatedSpec = - getTestSpec() - .toBuilder() - .setFeatures( - featureIdx, - DataGenerator.createFeatureSpecV2( - "feature2", ValueProto.ValueType.Enum.STRING_LIST, ImmutableMap.of())) - .build(); - - FeatureTableProto.FeatureTable updatedTable = - apiClient.applyFeatureTable("default", updatedSpec); - - assertTrue(TestUtil.compareFeatureTableSpec(updatedTable.getSpec(), updatedSpec)); - } - - @Test - public void shouldUpdateFeatureTableOnFeatureAddition() { - FeatureTableProto.FeatureTableSpec updatedSpec = - getTestSpec() - .toBuilder() - .addFeatures( - DataGenerator.createFeatureSpecV2( - "feature6", ValueProto.ValueType.Enum.FLOAT, ImmutableMap.of())) - .build(); - - FeatureTableProto.FeatureTable updatedTable = - apiClient.applyFeatureTable("default", updatedSpec); - - assertTrue(TestUtil.compareFeatureTableSpec(updatedTable.getSpec(), updatedSpec)); - } - - @Test - public void shouldNotUpdateIfNoChanges() { - FeatureTableProto.FeatureTable table = apiClient.applyFeatureTable("default", getTestSpec()); - FeatureTableProto.FeatureTable updatedTable = - apiClient.applyFeatureTable("default", getTestSpec()); - - assertThat(updatedTable.getMeta().getRevision(), equalTo(table.getMeta().getRevision())); - } - - @Test - public void shouldErrorOnMissingBatchSource() { - FeatureTableProto.FeatureTableSpec spec = - DataGenerator.createFeatureTableSpec( - "ft", - List.of("entity1"), - Map.of("event_timestamp", ValueProto.ValueType.Enum.INT64), - 3600, - Map.of()) - .toBuilder() - .clearBatchSource() - .build(); - - StatusRuntimeException exc = - assertThrows( - StatusRuntimeException.class, () -> apiClient.applyFeatureTable("default", spec)); - - assertThat( - exc.getMessage(), - equalTo("INVALID_ARGUMENT: FeatureTable batch source cannot be empty.")); - } - - @Test - public void shouldErrorOnInvalidBigQueryTableRef() { - String invalidTableRef = "invalid.bq:path"; - FeatureTableProto.FeatureTableSpec spec = - DataGenerator.createFeatureTableSpec( - "ft", - List.of("entity1"), - Map.of("feature", ValueProto.ValueType.Enum.INT64), - 3600, - Map.of()) - .toBuilder() - .setBatchSource( - DataGenerator.createBigQueryDataSourceSpec(invalidTableRef, "ts_col", "")) - .build(); - - StatusRuntimeException exc = - assertThrows( - StatusRuntimeException.class, () -> apiClient.applyFeatureTable("default", spec)); - - assertThat( - exc.getMessage(), - equalTo( - String.format( - "INVALID_ARGUMENT: invalid value for FeatureTable resource, %s: argument must be in the form of .", - invalidTableRef))); - } - - @Test - public void shouldErrorOnReservedNames() { - // Reserved name used as feature name - assertThrows( - StatusRuntimeException.class, - () -> - apiClient.applyFeatureTable( - "default", - DataGenerator.createFeatureTableSpec( - "ft", - List.of("entity1"), - Map.of("event_timestamp", ValueProto.ValueType.Enum.INT64), - 3600, - Map.of()) - .toBuilder() - .setBatchSource( - DataGenerator.createFileDataSourceSpec( - "file:///path/to/file", "ts_col", "")) - .build())); - - // Reserved name used in as entity name - assertThrows( - StatusRuntimeException.class, - () -> - apiClient.applyFeatureTable( - "default", - DataGenerator.createFeatureTableSpec( - "ft", - List.of("created_timestamp"), - Map.of("feature1", ValueProto.ValueType.Enum.INT64), - 3600, - Map.of()) - .toBuilder() - .setBatchSource( - DataGenerator.createFileDataSourceSpec( - "file:///path/to/file", "ts_col", "")) - .build())); - } - - @Test - public void shouldErrorOnInvalidName() { - // Invalid feature table name - assertThrows( - StatusRuntimeException.class, - () -> - apiClient.applyFeatureTable( - "default", - DataGenerator.createFeatureTableSpec( - "f-t", - List.of("entity1"), - Map.of("feature1", ValueProto.ValueType.Enum.INT64), - 3600, - Map.of()) - .toBuilder() - .setBatchSource( - DataGenerator.createFileDataSourceSpec( - "file:///path/to/file", "ts_col", "")) - .build())); - - // Invalid feature name - assertThrows( - StatusRuntimeException.class, - () -> - apiClient.applyFeatureTable( - "default", - DataGenerator.createFeatureTableSpec( - "ft", - List.of("entity1"), - Map.of("feature-1", ValueProto.ValueType.Enum.INT64), - 3600, - Map.of()) - .toBuilder() - .setBatchSource( - DataGenerator.createFileDataSourceSpec( - "file:///path/to/file", "ts_col", "")) - .build())); - } - - @Test - public void shouldErrorOnNotFoundEntityName() { - assertThrows( - StatusRuntimeException.class, - () -> - apiClient.applyFeatureTable( - "default", - DataGenerator.createFeatureTableSpec( - "ft1", - List.of("entity_not_found"), - Map.of("feature1", ValueProto.ValueType.Enum.INT64), - 3600, - Map.of()) - .toBuilder() - .setBatchSource( - DataGenerator.createFileDataSourceSpec( - "file:///path/to/file", "ts_col", "")) - .build())); - } - - @Test - public void shouldErrorOnArchivedProject() { - apiClient.createProject("archived"); - apiClient.archiveProject("archived"); - - assertThrows( - StatusRuntimeException.class, - () -> - apiClient.applyFeatureTable( - "archived", - DataGenerator.createFeatureTableSpec( - "ft1", - List.of("entity1", "entity2"), - Map.of("feature1", ValueProto.ValueType.Enum.INT64), - 3600, - Map.of()) - .toBuilder() - .setBatchSource( - DataGenerator.createFileDataSourceSpec( - "file:///path/to/file", "ts_col", "")) - .build())); - } - } - - @Nested - public class DeleteFeatureTable { - - @Test - public void shouldReturnNoTables() { - String projectName = "default"; - String featureTableName = "featuretable1"; - - apiClient.deleteFeatureTable(projectName, featureTableName); - - CoreServiceProto.ListFeatureTablesRequest.Filter filter = - CoreServiceProto.ListFeatureTablesRequest.Filter.newBuilder() - .setProject("default") - .putLabels("feat_key2", "feat_value2") - .build(); - List featureTables = - apiClient.simpleListFeatureTables(filter); - - StatusRuntimeException exc = - assertThrows( - StatusRuntimeException.class, - () -> apiClient.simpleGetFeatureTable(projectName, featureTableName)); - - assertThat(featureTables.size(), equalTo(0)); - assertThat( - exc.getMessage(), - equalTo( - String.format( - "NOT_FOUND: Feature Table has been deleted: (project: %s, name: %s)", - projectName, featureTableName))); - } - - @Test - public void shouldUpdateDeletedTable() { - String projectName = "default"; - String featureTableName = "featuretable1"; - - apiClient.deleteFeatureTable(projectName, featureTableName); - - FeatureTableSpec featureTableSpec = - DataGenerator.createFeatureTableSpec( - featureTableName, - Arrays.asList("entity1", "entity2"), - new HashMap<>() { - { - put("feature3", ValueProto.ValueType.Enum.INT64); - } - }, - 7200, - ImmutableMap.of("feat_key3", "feat_value3")) - .toBuilder() - .setBatchSource( - DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", "")) - .build(); - - apiClient.applyFeatureTable(projectName, featureTableSpec); - - FeatureTableProto.FeatureTable featureTable = - apiClient.simpleGetFeatureTable(projectName, featureTableName); - - assertTrue(TestUtil.compareFeatureTableSpec(featureTable.getSpec(), featureTableSpec)); - } - - @Test - public void shouldErrorIfTableNotExist() { - String projectName = "default"; - String featureTableName = "nonexistent_table"; - StatusRuntimeException exc = - assertThrows( - StatusRuntimeException.class, - () -> apiClient.deleteFeatureTable(projectName, featureTableName)); - - assertThat( - exc.getMessage(), - equalTo( - String.format( - "NOT_FOUND: No such Feature Table: (project: %s, name: %s)", - projectName, featureTableName))); - } - } -} diff --git a/java/core/src/test/java/feast/core/util/TypeConversionTest.java b/java/core/src/test/java/feast/core/util/TypeConversionTest.java deleted file mode 100644 index ba965a76c9..0000000000 --- a/java/core/src/test/java/feast/core/util/TypeConversionTest.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.util; - -import static com.jayway.jsonpath.matchers.JsonPathMatchers.hasJsonPath; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; - -import com.google.protobuf.Timestamp; -import java.util.*; -import org.junit.Test; - -public class TypeConversionTest { - @Test - public void convertTimeStampShouldCorrectlyConvertDateToProtobufTimestamp() { - Date date = new Date(1000); - Timestamp expected = Timestamp.newBuilder().setSeconds(1).build(); - assertThat(TypeConversion.convertTimestamp(date), equalTo(expected)); - } - - @Test - public void convertTagStringToListShouldConvertTagStringToList() { - String input = "value1,value2"; - List expected = Arrays.asList("value1", "value2"); - assertThat(TypeConversion.convertTagStringToList(input), equalTo(expected)); - } - - @Test - public void convertTagStringToListShouldReturnEmptyListForEmptyString() { - String input = ""; - List expected = Collections.emptyList(); - assertThat(TypeConversion.convertTagStringToList(input), equalTo(expected)); - } - - @Test - public void convertJsonStringToMapShouldConvertJsonStringToMap() { - String input = "{\"key\": \"value\"}"; - Map expected = new HashMap<>(); - expected.put("key", "value"); - assertThat(TypeConversion.convertJsonStringToMap(input), equalTo(expected)); - } - - @Test - public void convertJsonStringToMapShouldReturnEmptyMapForEmptyJson() { - String input = "{}"; - Map expected = Collections.emptyMap(); - assertThat(TypeConversion.convertJsonStringToMap(input), equalTo(expected)); - } - - @Test - public void convertMapToJsonStringShouldReturnJsonStringForGivenMap() { - Map input = new HashMap<>(); - input.put("key", "value"); - assertThat( - TypeConversion.convertMapToJsonString(input), hasJsonPath("$.key", equalTo("value"))); - } - - @Test - public void convertMapToJsonStringShouldReturnEmptyJsonForAnEmptyMap() { - Map input = new HashMap<>(); - assertThat(TypeConversion.convertMapToJsonString(input), equalTo("{}")); - } -} diff --git a/java/core/src/test/java/feast/core/validators/DataSourceValidatorTest.java b/java/core/src/test/java/feast/core/validators/DataSourceValidatorTest.java deleted file mode 100644 index 842e4c657d..0000000000 --- a/java/core/src/test/java/feast/core/validators/DataSourceValidatorTest.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.validators; - -import static feast.proto.core.DataSourceProto.DataSource.SourceType.*; - -import feast.common.it.DataGenerator; -import feast.proto.core.DataSourceProto; -import feast.proto.core.DataSourceProto.DataSource.BigQueryOptions; -import feast.proto.core.DataSourceProto.DataSource.KafkaOptions; -import feast.proto.core.DataSourceProto.DataSource.KinesisOptions; -import feast.proto.core.DataSourceProto.DataSource.SourceType; -import java.util.Map; -import org.junit.Test; - -public class DataSourceValidatorTest { - - @Test(expected = UnsupportedOperationException.class) - public void shouldErrorIfSourceTypeUnsupported() { - DataSourceProto.DataSource badSpec = - getTestSpecsMap().get(BATCH_FILE).toBuilder().setType(SourceType.INVALID).build(); - DataSourceValidator.validate(badSpec); - } - - @Test - public void shouldPassValidSpecs() { - getTestSpecsMap().values().forEach(DataSourceValidator::validate); - } - - @Test(expected = IllegalArgumentException.class) - public void shouldErrorIfBadBigQueryTableRef() { - DataSourceProto.DataSource badSpec = - getTestSpecsMap() - .get(BATCH_BIGQUERY) - .toBuilder() - .setBigqueryOptions(BigQueryOptions.newBuilder().setTableRef("bad:/ref").build()) - .build(); - DataSourceValidator.validate(badSpec); - } - - @Test(expected = IllegalArgumentException.class) - public void shouldErrorIfBadClassPath() { - DataSourceProto.DataSource badSpec = - getTestSpecsMap() - .get(STREAM_KAFKA) - .toBuilder() - .setKafkaOptions( - KafkaOptions.newBuilder() - .setMessageFormat(DataGenerator.createProtoFormat(".bad^path")) - .build()) - .build(); - DataSourceValidator.validate(badSpec); - - badSpec = - getTestSpecsMap() - .get(STREAM_KINESIS) - .toBuilder() - .setKinesisOptions( - KinesisOptions.newBuilder() - .setRecordFormat(DataGenerator.createProtoFormat(".bad^path")) - .build()) - .build(); - DataSourceValidator.validate(badSpec); - } - - private Map getTestSpecsMap() { - return Map.of( - BATCH_FILE, DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", ""), - BATCH_BIGQUERY, - DataGenerator.createBigQueryDataSourceSpec("project:dataset.table", "ts_col", "dt_col"), - STREAM_KINESIS, - DataGenerator.createKinesisDataSourceSpec( - "ap-nowhere1", "stream", "class.path", "ts_col"), - STREAM_KAFKA, - DataGenerator.createKafkaDataSourceSpec( - "localhost:9092", "topic", "class.path", "ts_col")); - } -} diff --git a/java/core/src/test/java/feast/core/validators/FeatureTableValidatorTest.java b/java/core/src/test/java/feast/core/validators/FeatureTableValidatorTest.java deleted file mode 100644 index 8410799dc6..0000000000 --- a/java/core/src/test/java/feast/core/validators/FeatureTableValidatorTest.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.validators; - -import static feast.proto.types.ValueProto.ValueType.Enum.*; - -import feast.common.it.DataGenerator; -import feast.proto.core.FeatureProto.FeatureSpecV2; -import feast.proto.core.FeatureTableProto.FeatureTableSpec; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import org.junit.Test; - -public class FeatureTableValidatorTest { - - @Test(expected = IllegalArgumentException.class) - public void shouldErrorIfLabelsHasEmptyKey() { - Map badLabels = Map.of("", "empty"); - FeatureTableSpec badSpec = getTestSpec().toBuilder().putAllLabels(badLabels).build(); - FeatureTableValidator.validateSpec(badSpec); - } - - @Test(expected = IllegalArgumentException.class) - public void shouldErrorIfFeaturesLabelsHasEmptyKey() { - Map badLabels = Map.of("", "empty"); - - List badFeatureSpecs = - getTestSpec().getFeaturesList().stream() - .map(featureSpec -> featureSpec.toBuilder().putAllLabels(badLabels).build()) - .collect(Collectors.toList()); - FeatureTableSpec badSpec = getTestSpec().toBuilder().addAllFeatures(badFeatureSpecs).build(); - FeatureTableValidator.validateSpec(badSpec); - } - - @Test(expected = IllegalArgumentException.class) - public void shouldErrorIfUsedReservedName() { - FeatureTableSpec badSpec = - getTestSpec().toBuilder().addAllEntities(FeatureTableValidator.RESERVED_NAMES).build(); - FeatureTableValidator.validateSpec(badSpec); - } - - @Test(expected = IllegalArgumentException.class) - public void shouldErrorIfNamesUsedNotUnique() { - FeatureTableSpec badSpec = - DataGenerator.createFeatureTableSpec( - "driver", List.of("region"), Map.of("region", STRING), 3600, Map.of()); - FeatureTableValidator.validateSpec(badSpec); - } - - private FeatureTableSpec getTestSpec() { - return DataGenerator.createFeatureTableSpec( - "driver", List.of("driver_id"), Map.of("n_drivers", INT64), 3600, Map.of()); - } -} diff --git a/java/core/src/test/java/feast/core/validators/MatchersTest.java b/java/core/src/test/java/feast/core/validators/MatchersTest.java deleted file mode 100644 index 559330ae3c..0000000000 --- a/java/core/src/test/java/feast/core/validators/MatchersTest.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.validators; - -import static feast.core.validators.Matchers.checkLowerSnakeCase; -import static feast.core.validators.Matchers.checkUpperSnakeCase; -import static feast.core.validators.Matchers.checkValidClassPath; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import com.google.common.base.Strings; -import org.junit.Test; - -public class MatchersTest { - - @Test - public void checkUpperSnakeCaseShouldPassForLegitUpperSnakeCase() { - String in = "REDIS_DB"; - checkUpperSnakeCase(in, "featuretable"); - } - - @Test - public void checkUpperSnakeCaseShouldPassForLegitUpperSnakeCaseWithNumbers() { - String in = "REDIS1"; - checkUpperSnakeCase(in, "featuretable"); - } - - @Test - public void checkUpperSnakeCaseShouldThrowIllegalArgumentExceptionWithFieldForInvalidString() { - String in = "redis"; - assertThrows( - IllegalArgumentException.class, - () -> checkUpperSnakeCase(in, "featuretable"), - Strings.lenientFormat( - "invalid value for %s resource, %s: %s", - "featuretable", - "redis", - "argument must be in upper snake case, and cannot include any special characters.")); - } - - @Test - public void checkLowerSnakeCaseShouldPassForLegitLowerSnakeCase() { - String in = "feature_name_v1"; - checkLowerSnakeCase(in, "feature"); - } - - @Test - public void checkLowerSnakeCaseShouldThrowIllegalArgumentExceptionWithFieldForInvalidString() { - String in = "Invalid_feature name"; - assertThrows( - IllegalArgumentException.class, - () -> checkLowerSnakeCase(in, "feature"), - Strings.lenientFormat( - "invalid value for %s resource, %s: %s", - "feature", - "Invalid_feature name", - "argument must be in lower snake case, and cannot include any special characters.")); - } - - @Test - public void checkValidClassPathSuccess() { - checkValidClassPath("com.example.foo", "FeatureTable"); - checkValidClassPath("com.example", "FeatureTable"); - } - - @Test - public void checkValidClassPathEmpty() { - assertThrows(IllegalArgumentException.class, () -> checkValidClassPath("", "FeatureTable")); - } - - @Test - public void checkValidClassPathDigits() { - assertThrows(IllegalArgumentException.class, () -> checkValidClassPath("123", "FeatureTable")); - } -} diff --git a/java/core/src/test/resources/application-it.properties b/java/core/src/test/resources/application-it.properties deleted file mode 100644 index 7e71d8293f..0000000000 --- a/java/core/src/test/resources/application-it.properties +++ /dev/null @@ -1,26 +0,0 @@ -# -# Copyright 2018 The Feast Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -grpc.server.port=6666 - -feast.security.authentication.enabled = false -feast.security.authorization.enabled = false - -feast.jobs.enabled=false - -spring.datasource.hikari.maximum-pool-size=40 -spring.main.allow-bean-definition-overriding=true - diff --git a/java/core/src/test/resources/application.properties b/java/core/src/test/resources/application.properties deleted file mode 100644 index cbc2e8ef0d..0000000000 --- a/java/core/src/test/resources/application.properties +++ /dev/null @@ -1,53 +0,0 @@ -# -# Copyright 2018 The Feast Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -grpc.port=${GRPC_PORT:6565} - -feast.security.authentication.enabled = true -feast.security.disableRestControllerAuth = true -feast.security.authorization.enabled = false - -feast.core.projectId = ${PROJECT_ID:} -feast.core.datasetPrefix = ${DATASET_PREFIX:fs} - -feast.jobs.workspace=${JOB_WORKSPACE} -feast.jobs.runner=${JOB_RUNNER:DirectRunner} -feast.jobs.options=${JOB_OPTIONS:{}} -feast.jobs.executable=${JOB_EXECUTABLE:feast-ingestion.jar} - -feast.jobs.dataflow.projectId = ${DATAFLOW_PROJECT_ID:} -feast.jobs.dataflow.location = ${DATAFLOW_LOCATION:} - -feast.jobs.flink.configDir = ${FLINK_CONF_DIR:/etc/flink/flink-1.5.5/conf} -feast.jobs.flink.masterUrl = ${FLINK_MASTER_URL:localhost:8081} - -feast.jobs.monitor.period = ${JOB_MONITOR_PERIOD_MS:5000} -feast.jobs.monitor.initialDelay = ${JOB_MONITOR_INITIAL_DELAY_MS:60000} - -feast.store.serving.type = ${STORE_SERVING_TYPE:} -feast.store.serving.options = ${STORE_SERVING_OPTIONS:{}} -feast.store.warehouse.type = ${STORE_WAREHOUSE_TYPE:} -feast.store.warehouse.options = ${STORE_WAREHOUSE_OPTIONS:{}} -feast.store.errors.type = ${STORE_ERRORS_TYPE:} -feast.store.errors.options = ${STORE_ERRORS_OPTIONS:{}} - -statsd.host= ${STATSD_HOST:localhost} -statsd.port= ${STATSD_PORT:8125} - -management.metrics.export.simple.enabled=false -management.metrics.export.statsd.enabled=true -management.metrics.export.statsd.host=${STATSD_HOST:localhost} -management.metrics.export.statsd.port=${STATSD_PORT:8125} diff --git a/java/core/src/test/resources/keto/docker-compose.yml b/java/core/src/test/resources/keto/docker-compose.yml deleted file mode 100644 index 714f83ea64..0000000000 --- a/java/core/src/test/resources/keto/docker-compose.yml +++ /dev/null @@ -1,43 +0,0 @@ -version: '3' -services: - keto: - depends_on: - - db - - migrations - image: oryd/keto:v0.4.3-alpha.2 - environment: - - DSN=postgres://keto:keto@db:5432/keto?sslmode=disable - - SERVE_CORS_ENABLED=true - command: - - serve - ports: - - 4466 - - db: - image: bitnami/postgresql:9.6 - environment: - - POSTGRESQL_USERNAME=keto - - POSTGRESQL_PASSWORD=keto - - POSTGRESQL_DATABASE=keto - - migrations: - depends_on: - - db - image: oryd/keto:v0.4.3-alpha.2 - environment: - - DSN=postgres://keto:keto@db:5432/keto?sslmode=disable - command: - - migrate - - sql - - -e - - adaptor: - depends_on: - - keto - image: gcr.io/kf-feast/feast-keto-auth-server:latest - environment: - SERVER_PORT: 8080 - KETO_URL: http://keto:4466 - ports: - - 8080 - restart: on-failure diff --git a/java/core/src/test/resources/log4j2.xml b/java/core/src/test/resources/log4j2.xml deleted file mode 100644 index 9f785952e0..0000000000 --- a/java/core/src/test/resources/log4j2.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - - - - - %d{yyyy-MM-dd HH:mm:ss.SSS} %5p ${hostName} --- [%15.15t] %-40.40c{1.} : %m%n%ex - - - {"time":"%d{yyyy-MM-dd'T'HH:mm:ssXXX}","hostname":"${hostName}","severity":"%p","message":%m}%n%ex - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/java/core/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/java/core/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker deleted file mode 100644 index ca6ee9cea8..0000000000 --- a/java/core/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker +++ /dev/null @@ -1 +0,0 @@ -mock-maker-inline \ No newline at end of file diff --git a/java/core/src/test/resources/sql/expQuery1.sql b/java/core/src/test/resources/sql/expQuery1.sql deleted file mode 100644 index 019f5d9cbd..0000000000 --- a/java/core/src/test/resources/sql/expQuery1.sql +++ /dev/null @@ -1,11 +0,0 @@ -SELECT - id, - event_timestamp, - feature1, - feature2, - feature3 -FROM - `project.dataset.myentity` -WHERE - event_timestamp >= TIMESTAMP("2018-01-02") - AND event_timestamp <= TIMESTAMP(DATETIME_ADD("2018-01-30", INTERVAL 1 DAY)) LIMIT 100 \ No newline at end of file diff --git a/java/core/src/test/resources/sql/expQuery2.sql b/java/core/src/test/resources/sql/expQuery2.sql deleted file mode 100644 index e9b212d20a..0000000000 --- a/java/core/src/test/resources/sql/expQuery2.sql +++ /dev/null @@ -1,9 +0,0 @@ -SELECT - id, - event_timestamp, - feature1 -FROM - `project.dataset.myentity` -WHERE - event_timestamp >= TIMESTAMP("2018-01-02") - AND event_timestamp <= TIMESTAMP(DATETIME_ADD("2018-01-30", INTERVAL 1 DAY)) LIMIT 1000 \ No newline at end of file diff --git a/java/core/src/test/resources/sql/expQueryWithJobIdFilter.sql b/java/core/src/test/resources/sql/expQueryWithJobIdFilter.sql deleted file mode 100644 index 96c076b6a6..0000000000 --- a/java/core/src/test/resources/sql/expQueryWithJobIdFilter.sql +++ /dev/null @@ -1,8 +0,0 @@ -SELECT - id, - event_timestamp, - feature1,feature2 -FROM - `project.dataset.myentity` -WHERE event_timestamp >= TIMESTAMP("2018-01-02") AND event_timestamp <= TIMESTAMP(DATETIME_ADD("2018-01-30", INTERVAL 1 DAY)) AND feature1 = 10 AND feature2 = "HELLO" AND job_id = "1234567890" -LIMIT 1000 \ No newline at end of file diff --git a/java/core/src/test/resources/sql/expQueryWithNumberAndStringFilter.sql b/java/core/src/test/resources/sql/expQueryWithNumberAndStringFilter.sql deleted file mode 100644 index 8769fe44af..0000000000 --- a/java/core/src/test/resources/sql/expQueryWithNumberAndStringFilter.sql +++ /dev/null @@ -1,8 +0,0 @@ -SELECT - id, - event_timestamp, - feature1,feature2 -FROM - `project.dataset.myentity` -WHERE event_timestamp >= TIMESTAMP("2018-01-02") AND event_timestamp <= TIMESTAMP(DATETIME_ADD("2018-01-30", INTERVAL 1 DAY)) AND feature1 = 10 AND feature2 = "HELLO" -LIMIT 1000 \ No newline at end of file diff --git a/java/core/src/test/resources/sql/expQueryWithNumberFilter.sql b/java/core/src/test/resources/sql/expQueryWithNumberFilter.sql deleted file mode 100644 index 6b199b7c4b..0000000000 --- a/java/core/src/test/resources/sql/expQueryWithNumberFilter.sql +++ /dev/null @@ -1,8 +0,0 @@ -SELECT - id, - event_timestamp, - feature1,feature2 -FROM - `project.dataset.myentity` -WHERE event_timestamp >= TIMESTAMP("2018-01-02") AND event_timestamp <= TIMESTAMP(DATETIME_ADD("2018-01-30", INTERVAL 1 DAY)) AND feature1 = 10 -LIMIT 1000 \ No newline at end of file diff --git a/java/core/src/test/resources/sql/expQueryWithStringFilter.sql b/java/core/src/test/resources/sql/expQueryWithStringFilter.sql deleted file mode 100644 index 8c0a380504..0000000000 --- a/java/core/src/test/resources/sql/expQueryWithStringFilter.sql +++ /dev/null @@ -1,8 +0,0 @@ -SELECT - id, - event_timestamp, - feature1,feature2 -FROM - `project.dataset.myentity` -WHERE event_timestamp >= TIMESTAMP("2018-01-02") AND event_timestamp <= TIMESTAMP(DATETIME_ADD("2018-01-30", INTERVAL 1 DAY)) AND feature1 = "10" -LIMIT 1000 \ No newline at end of file diff --git a/java/docs/architecture.png b/java/docs/architecture.png deleted file mode 100644 index 3bc348b73b4db80bbc1fe64df8aa7e6fb7aef2d3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 28202 zcmeEuX*iVoA9vd++K5WBg^+CFAp2>Qtx#m&D%*rHA!8Xs=Y&X1vSmL*C1ECvb<9ki zFc=JxWyVrj1~Wsp8I0w*)mi@AyXVdG;<>M`F6v%>%Xj^JKi^-%bxV^Y2Tvc|wQJWA zGt~9I=X*84CHl?KDyN`#^SxicczQ zZq_Rcb@HL`LGRweK;5d!+ZSq#3x!2CpT9qOG3lrMw|~7p#1cMx=%JVHjJt=Q>Pa44 zik^eG@rLFsmFEc_y#I37EBt+xYpFz?nTs050VDB3^ONKb4t*)s$eSRf^0g%?Q_C%K zyfJuwcsDSh|9|~ou7GCT0X}ij#+80&m4S-}1_qH_RxNU>rN{(1n&2>{QIQ10^_(=> z_x&rw2cg7Gb}av$z*9>`vJSuasxweRBRr2>9~?cEQS zF#LbekL-QJzyFj!FO;x064^w{<0^V6v( zWJ1#*;jeG1tkDJ7O^(o2{i!W887rvH`t#)1w6cOMY z*#1^*5U&(JT{K-fT{VrH{@~S~)}$&`yIZIa#82Io8~qdN2qB?+|E&i@SCf&*c}iBD zs@fBycLdG=FG<%W*)PG>n%%-`!XVuxKZt%nSz%VrUfdBlqg*l`R@Ugi*u7_7c_dt11!$S|F0YJzVsBC z5JBMir0%Wx(S-+xd?k4xfFuHg8oLbM3rKHO_ZFXdGSi|hg@Yge9}6>8F<<{oS~hmL z(@gTw#iir}F4qrT0`I-Ibs|P{RTd<2;oYs`7??thmp|0Rm>tt9i~S!jQ9^CTZZFrP zc{YcIqGrBwwpMFZ2!J>(|LXhU`x;*~#(QuIqQ~ul?fa>kKSpyG-Te~yAyf*xd+%YG z71*S6@`@ut(vrBIG}ah4kBJ`J_pJ8k9*<>&=tI>6*DA*LG~8Y8e|_2l<;u7aJwtZS zJ@3A7E%(*uEpkC5?EqTCWcltv=1-HCm8Fv%%cCv9|6?DPK)T~Q0xLAOu{4$?V^ezb z$ZioejTry34+kn{;td`X9vdVQa)Q-Sqc`I{`WQZ-|FK8-W7P8dN&a6P0?_`DX1_+w z3BUOM*W023p%SurPY_QKNr+?w(jcDTFlBrxDGVZ-ctz{bKg%$-`GmNRgSu}l3Rx+8 z%&Pt^Bvl}s{ZVQ4$HlYSlq3feEXsU7{v7HUt$lFc?)=|$q#3sMj$#$)s1o6MGB&k) z|Brnej9r?aCvcpFFvP9(vO!r#TwlyLD?-Q+Uqy61#)P;!l!)1Qk8#~-<0C{Z)OLng zD;AI~^o{@83$`(sBdgePc%ojTVBm!OQ;b4@uVJNZxc6K4kgpaXku%sGVdkgi2)b{S zQ_9b#Mvwau*Z+tzSgqZjswWx+*q4l?4s4GXRgno#>Sho~Haf_ERfoX)T{)_GS=uWo zb$EEz=^ot4ig}Ggn>vgqg4A889Sy%V5X06)mdm5;(}#NaBwzaRra(8F?eojEV%B86 zJxp@!@k8$h7A$}P>^&Ud0;VX99MeEm`I~emyO|6nGm@Q@Wu!RL>>16gF9B1t^G-yF zTlD5iQAOCwC;NFr*NJ!W4RshbxA4jI%;mJDd5c4=r#Q(d5B&x8t$Zf~wY;h(6GeBK zMF;$>OZ4T95rb_X!*vXWE!mHMeS{Tdr6eI^5>$WglA0_wnAp;OG>?=e>z`K``5`=pB4e_0Xg%4xW;E$xoQWBbw>}- zD+(CXrS|TKl}QG891rJ=)_1LV=*GfYKRE?Qyt!;*%=2Y*aeJ$3d(PUtmO)XI!aqAB z{|AZs_Vs2@i-U3{R!B+Yp7rK=t|fTs$CAN7 z;6s1zAk-0D;5Cd3oi~&Aou8lHl9)13?y;v=V|*LgRKJ3HS=44sw=9-S1l;k)R1P%CTZHuZd^wE{ z%C{RD(L?Y*pAjKDq#Pc8KV)~CyAT^_Gc4?ymw zu2Wy-spz?h!!5sNiU`s3H1&ja#UW#*2COpsC~c#spuBr79n@w}!jS&2!3$7R5Ml)iP6>E22oA(acpd7M_?9J; zq(*#NpOH5#y_0i5G9u~(O24c)!&JMB!Q>exL--uqC{%qU{Erw>`Zne!<1~AnQD=^h`3nYS|d0$*Bb4ESJObKsfe?n})XG`Wb5m zvKZb!o%C*$5)b4-_sOsQu`pAn2~r%tbU#bZLjVpKpv$?%*8|iF_!1|!es3Q=f5mU@ zasGyzG1wd_G+8 zeB3UyBh6l3d$CsGnyH~Zn1a7?oqP99jnF|X!SRi=fr-J;SbOYh8fu6RuHe!8s zO*8>72f`^y2VqU4_{e#%7Xdl9H-GC6p!N$=6RGgNV{a+u;HD)H8N+_bZ5l#CUT*Ql z46-8OqJ-O{ZZxT=om0^|nUg)MQMF-Efj2 zWhzELm>vUL>FFnibJo2A6&UT zB2oLbUOjM9{r&Lrr@!;UA0EU{ks zK=;{ah6#}LmE3fXNaFCdFrQYcAy~FTZ?If-@&yn55lnecTUv0?u9L{R%U)Ol&X%>uMLZ8f)Flg#xfHNww8+z zWhQwAtsy?c$GD>{1gpar{y!wsUCVTfhA2|LX!!t_e@b#S8b6hexlV zMjrQ_S9(V?QsLWoO+l=ZbdUDzngSo*ev%@{egkEvU| zDNWW_2p1stFP^tQM*Xn(D5M~6j87zMs}9S0o5J)N%D?1R;FHHtyD`2wOp^StejEC% z^U6!xJDW!PxoZyYo^VD|+!e&shAuJEl&8nm_kzj(Ak+RZi04HTWfkWwS{tO*1oKpQ zbQZ0qMLmWo7app}U)6OJw7yt#ESlp9#mZ#(4XDbgdFV_;jAVj6p!z-N8cGDR8qkwF zV?}wAAK{y6jA{K$(wh^^3V*CV2O$y*YckwkGPF_#{@PvwbO@^tf+zp(`iMssTU&5y zqejVNY?sl_8hy{i@bOP+AND;%hMIvfzdrQJxZCa{T3$zZSvzLV%<*ig;S0DW;>YT} z4L{IT(?z_4pch&2Q*F%RZ~RfDd=-efbY-yPN5sjt8KYUSN2B;0voTERx zXQhAO;J8EdR#BIHMdeYHD{Y9z!9=Z;sxU*By)bzTVdx@=&C$z{rih5b2&O~xdX>Hn zDYz^ShV?5$(h%i{{!!s4@fbqWE-+xgEWjQsOf>(IMA_!wfUK773a$kLMTx1HduOm; zHW(WUH5OnJMNv*I<6737Y@U-s&WGM7+Kpk2^x->V<${mBdL652rcKGPSyNPR_p6sUNIcqgx#$oM6dmQ44k?|4ob7beLf zeOOfydT012Lc`3P1BP>@{R4$#Sztq{u!v__FbE z0rk;sL(@P2Bw;3Xw?xM2bofYX)b8HNlE{-rOzcM_8=ym5e6x}x?91`G8JLoXzbKE@ z$f?)*$*wqzEsJ9c8};368|cBT&$eSGEBdN-b1i_@aNk`9L-$yCQyp^0jQgoBC|9|R z?D!%--*Hr#!9Um!-VK%K_VNVfldabfwO^!qGKREaAMtYbp3|>x1L>9(HK>(K<-uox z9ZM7zDlLL0pN|>ud~SVzHp#V3SJ8GfIeiLDqbVG)z7lwK!$@gKR>C=IM3}0ik%M<7 z)KY8@`sAEs`9b+xasK#A9&ffj7LtPTB*Y9%6LP-|}YMC!~??1JZg4CYFQq-?1 zeFYK>tJ(UMS8{KiZh#9`zBW|O<)gW2-LQXPCq_Y}s+f(%@v=p2atwCP>&B6}Wexe) zrP%z%OCbmwL3_4o#1_xPjMjOj3hMkr>$2(gCq*ghi zz=(5zz14C77g(50!Z~wzn*tGE7emn|&Rwf8x|&r%^v3;qzWuu+FMiD6eo!+3oP`Wq zw>=H;K^$`4m7{T9mX9{!dZr5x!zv@MY(`G!I7}&uOSpw*L~GF^=N8r-{HjIM2rVBK!6=uu4vdyByE!O@749 zX7+gyyo^ZQ9oTd0GWa1d#n4M&J~f?!XqU5#Gp%6=CwD`h**1YQl0h{8Z^xS$C#esA zF__2FwJTh`B6`t$I-6|8I1+Mp3?DKz-U$;Qpu;zxWi1D~{D4DtFi$U3o_LAf5mTYG zNl*j;s(grO=S`|FQr+!_=q{W_kPCpX+OL!9%;R95H?2O+p|cunSYirJhW&BbuA}{% z;#}V*2oK8*wNaw;d5CYJ!HZEi@MISX-pztn*@Uo2G7HHB8EZ&bY$H*yp`N$(fy$3AD@{V#5 z_kKblRI&35Pluw2@-KrNJU8lp0D1r#hf~BdB7G#^_9}e3HaqI~>dk?^%0J4x(ZNF& zUSNA-H8`I*s^X zgDA^XB_6Jp2y!j$>lPRJM%9gs(d2s9c~#T1gA0JVOza$}rrY{cT07zBRT zq8(8Zih*_EPj<h~uR#lE9_SZhR}drNNd+$D6l5fl;C^1bH1t7+Kgy!sM(7gze$r9gr(=-e9di zc)>oJ!^)HB^&oCf9B?PD_Pa-VG1#Gkv9J}l8TK1E1OG~gC{G+zIPg&iyY8YQUn$r5 zp)(eDusA(G8IV4qS4%zVt>%OPo^ z2@sNIknF&!ZViomk$41^7uw}(*c?=S)|B=t8zD|>`&GU_otysf8WWCCMEaDgx*bS` zU2h*B5~PLWQu}h4UHbn!H-3JiHm%~z+Fr>ao=beh+6qN7R??D5j<8i8Cg>T z(#5+mzO}gZ28YeeTVJl_MXxM4*qhH(gfIE7jfFMJ2C(1g@l#{iOw(VrWnrYshcqpJ z0{URT8+P*Q6c4LDq>&Nxm~}bc(WGJX0PYBob#J}Cu|4FM1G-iF2hP};@B&?&cf6{W z2!VfHyYkXBB1KD(2f|OiZo;pPj>b%fCA<4ti=cg{TILd+yT5=SkBvKy-hRWt+ zLc`Q(T5dM#aZ3f~#1jXaa@5^~7+{9e0Gpkk7@Zb|mcuRuSaOj9v}8!^MiSjOA+_Ud za%=Ql35zn~&34C#whsDPZ;|}`wr|Rh`PnE6k?C8_1nk~xVmyl%n?66=>~V%J-+=2C zuN#uHCseyaYF~Q4Vn^Q?6TSuLY)<#+e$Z4m63p6ao~0%{bz_914Z*fj;0!(=oJfY7k04%UdGWL;jddFLE6^P~kG{Ntk&OKuCt}LE$s| zPE%Lr9ELBE1w?YV-Gn6-$M5;|~m0^BK zz48nELfmi^ah5;|8hSMtpeTspc5QRU@>#tO`@`6_;yDf`xV%V%8bOAc*)djjJAX$A zaA!?}0Z3#Q3h!bU6cv@lEzeg)+OW}IxMRA6B5zGr!GKkd4&OO(3XH{fr(YR2AeiTbad zs`~`eRnA`m)rX>n+`8=(hD+^7+6PKjNLK>jUm831Irn5V{*GZ}iv*?^^w&IX3mH>~ z>v1neZ?6wQl^driiE9q@!HPGG>FSaJp$bC&*%DA(_x@kN6r|`_rgrgi3{2m!`3Om` zpy^Zrav<0^U(Rfv8Yu=q zJ~@2>%A$q)dmJqMPud()YK|@86&3Hv$V%-}NKB;W}ITQ94^A#P-!% z76|7Z+uVHL<(+p66u*n*n%lA%n39)84*zvDfM}VK9#S+OvH~(x$!9c-&uhEqBp>=x zQG*U1V>BapV!+Wn+hKa@WH@c!YZS-}9YQbb36brEud6pOqD-9c424+`@^>aU)=`g@ z!;WvHj~(@e9_D0FRPl0!OM2I4bI1JDt(w~%4UdNcu3$7x9W}D>HXL7}*!jq3b7og%X_4GxJU=^7Ng{!`H zXHZ8TbQ8o-`W$1X)G=4$360P{nq9WedljE#*(kmIWSl<_FOZI)lX7(I7eXQdC-H7q zRT0R?w*LFVDkn!1AWig{Ri`ptO8j|H@p!HWJ{9$^mrF5u4 z*l9k}#8Dt;-pke5f)sY=3zi>0GD8&{^Kf(vL`et@hYX!hC7F9Ptp^FTQ%?gDdhtZZe6F%343ovG4)vo4a6Fz??3R(hSiO z`tN$R6Z@L*syalmn-qKfwvqg$C1Y3=v z>1?BTWV&Pt{ec5v^g=ns^$K_ZLdnBc=&P)lmp;iWJ&AY-Ji1dN7g2o|KN9~`7b&hE zL7$;)n}X+&-J=pz@2~$V|nx(z-KzZO-cC z{u@T{tRThGIj*Q#uH&pc$ogw7t?o^?o2(JezBX+~IaETG^RlrOTKxmODWr_~JS*#% zN6TH18m%jgQ)3ykujwdPT%&u1;+U5T)gG@82=up@eprGi|O7Y0;i7 zjlBe@c=-DfZ=_PTsjAu^)WqrRKuvn-hpZVfM*E>juUME%A1Q*0xnq^Es;BbAt7~WO ziui09EzLF$eY>pm+Olr9n?_t&LJJ@+gBqxjeESxHu#KD{+5=(EV2j#NWbjKyXq7HX zc)tJD8=r!m5D08VRN|YFM;jUa!~W9^lnO(S5VS|$ig5_5K(DqMk^t1asli*7th2!I zpijHE=+RsB!~$-80d9l7#U*ZYzr(JS{f}O71~~){o=hYzzf>8S7NHkx%@iop)aAAo zB(?+Pw!^fgiRl4`2xEv*cqKJe%r+e%le;6l67~;B(B%Jv6$YrI-;F%?xTw@rr>x zrnt2@W(bw6paCnNS9c+QK3%kpw=^OI#Rz-;Ec^z(Q+x7q1UE&fuX=<$$IV^PRoU`@ z57p#fm=`OJ^aqN3TXSNG6%hakzY7E(4SyLb{zmJNhaNh*mORiV>XEk)iciR;zg)$i zXlCsr=~p;Y{1zU9LWbUZ4@G}=FhJU? z5Y|6o0)`)G*n@3%2oW@B=aoM$-Konsk>EAH zQh`5ZB|Y05(fG<6-5eC&<2K+P3&m||dw8McRK8ZZYTvdMHZz1v) zh}IUw%f?P#c*jI`10FRz61cYCKRj?iy}v3D1ZOg47j~;CuP+&m+WL`QR~R!f?745t z7Yj1@N+0FXDT{0M!a#n)N{+S>fZNBw%qQ4|b#DGOAZ)FF#+ke7^-c@aME0IK{MtpI zuWvMS%H>2NgojrmyWtiioO+K4g>1TbDWJJIN`v05S;qx{SjTBX+lv{QJs`LcnQK%W zAUXx`Vuq+!gJ#;kMFyN-dsS1UAKIit`zRJ3;9z@vAg8(B%EQYv!dh?ryt}Tg^~4)i zBLpQjH24V_yEl>a)hB&8Czojk*baq}HeS`!(7C6D#K;+%ZYTT?J`v0XvMYvx(Q#^88oL-L*a z3dAY%T+t(}@S)XEoKPRRW$+4XII(Yi<;xYJnYERv>|-M}MajY1!k%cuN=i$=1bK>) z*{Mn4m17LzUpL)^j=_bfHshi5kxvm9IS!K^_&`GnA0AOcPaeJO@?A5lU!*a%H0c&5 zZrj6bt{T&041b7!NtXFawE5P?L~MkOtKyD@p9pqz-r|3`_=dAsK772i}w!i7_x* z=N28B#;Xw;WOxt3$32M!pR?)rZHIU|rdNz>unpRD6Dq{_c337k(t#peF^NQaRBD4{9W~EpO4|pg+a@^1L|K!JrfM=)n2v?`!7N^I$$Rf@Cg7WvvN zu|6`g-R7l=IWhI<>w&3U1!Z@2yNt6}%Wu{K!Mx37k4?LQu>b`Qrk!~@@pWi_Mv2BvWba_jxJ0xfoG1iw zpNuF<3Lyqd{WfaQPok}SQiK51uuEVh0B`Y%Wks#o;w!RnjDZG^tT43YI#6Pz>-dh~ z_>Zq$YT%?t1Le?Iwyt#LTA+b}r)lk;wbv;9=zd$|`6`i#u z-E^lOMJ!d>Vs7dWSw$_Tnw5w%zTYRZwE132#% zAk;#dwXY*jLK``nHxOlsp2&cIyONg{*>WM1c-UD*>&*%#P z!URDlfgK)ay7&;MuH>2YCr*rW7Q40r7G7T-ywDJ1ml^AiJW0-~CE!bz(r*>8{(A z;1M_IFT(J7#b-khoL%{nCy>TNt-+)Nl*T$tkL9YZVXpEUZKvi-x`#MvyWG!W869JZ zzIXv@j#c{?$*Q%k>(X598?Ki*w~x_~&f)m|s7*x%{CV19nWbgcq?JzAa79NCIzvPf z+l)46>1{W!`PlfrSpQDpWu|t|A#el>HD-#XcUhw;<4|$q$388lx_o6rL4UC=rLU&e z?ty7RpWUTtnQY7SUnH@$E?}!qM7Jwom1k**qc_(!axrFo(J!1z8-(PImMl95*M;;A zO&$!ulmwQ51#*(Z*$bQ%_p5cKeZpN$SBsb<&>JGSZu1Q*Y>oZJjZ2y`Rer#scXe=k<3@7V+y=mL=MqJFaY<%PQ#G)(#}E(D7? zj}_Wf@oZUrTmC39WhvVlP>bhrPy7iu|ELs;;7MnSB>xCx!ZVKfQKW7x@uSZ*!L0Iw*?jk*oM___D*DK2*PoTp)anhEajo5l=oIowTeRPR6d4fe4O zxItTRk0R9$`t9}Rr#Y@|47=||wud9P^yISEJD~Wagk?-9f@KPQHVbC8A-p5*J;PKX#=)lJ}qI4PHG4}2q+IA#)YZ!V(PtGFMPQt zVvfIHy!NOvKf>}N`}Se3VYxO$P1SV$+#p?V~>rReW6JLvhUX8dpBr4n+T%%=cr2As+JP23Igmj@T!I-mH zznYap!XLdVTdvNG^fg4uzJ-IhT}#5Bh42Hfx&wSY3ba1z$vx0JmkM%2L&0O=cX4Kt zu74Ae*{55f?ZudqWbgL2j*$3rxkJ#KBR3eRjLt2>8@`=lVQea>f4!9@Qqev~Y3E^6Ud4zAh?QOnC`W^Urt2(Y$OM619fMZPw))iwrZb-tE{oZd9D{c@T(0OD%j zYhr-9*!v(#KdEW?HE^tk739=39VZ<$!xVBE3++)QSJ*Tr)9LXGfa;j>wgjP;QC-0O z2qyv%A`2owUAV2%9HLO@m-W=aB=S@uUqZ-lB&Xg61Mxk&o}pz$<+1wdfOXfO2x&Ah z*l5;doJ}`R_Lq<>H&Wyas?YhN=NTlycC&B))dgK-Lt4M}ymjI2hJFM7l9NwXit@`i z_Kozfe^K8hVH><=qMvbzSj(88piWGu3kD=jL*{(E<#ZU(8NX z$l%p_kv2%T)|NiRSgW-&wKwdO{kdj{7BZCIiX`0`sxQp%9t8nxyF)X!Ky=+q<34Hw zP;z`%(`jf!33I$Os`>ZYO|W|aF}=sOuB9@(R$iPkHm;#P<1vC9#7tL(1(Xe&9i)Zn z&1-fl$q7K>S_D^9wGDME{m(oFVI$4LQH# z*cMQ=g{Q{CfTe2*i64KS8(21H&%LCYLE(cyVpj8|v3`(wZQ9vCY3=X)=e9a18yytm zMj%|?o_(bi6&ALN3FaPc4#@5!D~wvz-7eG zMe$K=AX=_IZlU|xQ2JpR=4B4XrTXHnoRe-jqFlwSFlSlMyU{XNo%lYHo5XWO z)iUP2Vg;$XvRIqb{fo!tqsEhL$6WBFuM$#Qa|zQ8kzAx8nIxh(`d}e==7Es%%QJLHrJo3N=?HUR|S!0K6 z5ZB#vK1O|7AKGCe_(T69t1l_GuZr$taqQE|pu-uTU5%$h*_E}xcgaYmiSfZDR6M^xauTD^NEJyF=WMB9Q;NpVR7|qX165q-5 z`XJrQ0WG~m!xBpQvuaVP$lh2O(|(Aa9?O{E14N#p4EClBd@helm`nZYGsxuFfj^Pv{hP_ZnQbBWh-EOcT zkUyrIV7``|Q`iq9&OWLBLD2Zo&7!D+rO~s?#_bkIj|gup*CUNw2B@7kDB(|=3XWp7 z&u%4?N`FZv!tZEHV31c_XHjQcO+KuTQRtQWR?*w{;maC4B%1nk9RpyD^~n7hJA{C6(E` zZg+*S#WBd(*8%-D>Tkx*E$XSaTMbBJA63d1JW2CdI~yB}nJ`5tQf+81Zqdd5jG`+t z?uF61K^?@1AB+rH5zN<4$*ID;SU>QXyndwt*;P{m># z@2TD(NufqcDXwy5{+SYf(9W|^35JDl7Pr4NU0(~H2f(;k>770$fY1$y?oO*3z(~0u zKT#~7l~cyjYL74o@VWFDy9V!2p6Y`vtw~OC??NXxC^kBuU%XH^3-hOw$Dhxgh%JZBMH7#MS^o}h!Kcem?3yqHbPH{ z`z_GMWFtUVbsUgEkt98$E>=Mu>&`#6=FoXo4p8z6(;1O0s$;+lxtsx@t{^-ZEEnry z2c~cJnaTefJN$do8as-YWtSG^67!8i@?Fz(9GKS%`Q7~-=EgwuLm88~% zYAfrKj&ItZpspBSswpbD8dZyc&;oXZ@_5#Xk(|RpDe-Q0-~rNC4M&=8O%{!(-pwxx z2;_`9+^eDg$)*3Tdo&28ueTkj5**&zj`lJ$m`yvw8@OTC+P@y@eZM$9Y31e(T^lel zy7@FU)Xxsq`6NK-@Oyp#`BZQBc}ufduUPh(Zo`5~1MaKx1bu+kFvhhn5lh}YM$Kx| ztIj`P=#ka&)$Qek$zw*^z{>n+gB{R~FzfdpO$Xom9F6S{W7hx3=EW-w-^!VvyR)sm zJ&Nv5_Mg6zs?i@o3xSvRzdb6MFKF!@^B_R3$i)`yqyMeqN$|hv#E%9t{|)UWfM)$+ ze0+R+6=0|0>|F=YxzR>aJ~hi0P+iYptDEi`)&wZ1{un3+C~gB?lfMA2X%`PuPY!zR}nUqUHU1-XVbU1Jp>rTLOUl3iL?;eif~V#YolGZf=Nt)Y=DY zXBN3{FoOkE1N@o{{HP_k=a-$nN`Q|7=)|Ajumw|6Xtb!Uq&A5n7(1j7L+E5OpNp#9u`y5Lr~arT?ga$k zd-qp-`o7$fcScz$2k*p-^fL7fAa)ux`Cba4LxZ>3llbz9_?_O} ze{Dhn$nK#}a}@*5znYk{J#~2=H&DHGf;*l%JZ}&5$GKNi3?cI(Ya6{|hPeuDWWUiB zG;Sz(iePn6{=a2Z7CZ0`Xk8-^?>^I_KI3ju{V;jS)!|@{NDP||WWoZpf0}pyYX91Y z!hZq56Ea7R4B>u{Y#HGvNZvcJ;5_-ot0c7;8LeYFI`4Bgwe|5_`UY5>=wj~~2B?e= z#-d{LlBLS==M{M$wBSGxVq_W z2zpH{*wdiM*~~u|@M6Wp%Qdw06o!#S&5Vq?21c^z4w{_qVhJtU>gyBR)t=^6Oc5@u ztMPFY?4K^$@80RVfI;iK(S=$~cQVlPOjd2`WTKz@oFGq*mX=z_KfRMfIG=&vOHRn+0zN|JntG5d!Y`A`X z^Ir@7J%(Z(5u!fag^=8TETUGZut}YZS!P>i z2gkBq3BNnfv1Owa7uQoOx95+(G**s{il58Y034X>l@Mj;s?Svgj4dz2fdnm%8Tdcf z_J`XW2T};@R#`4*T(We%C?)%2brc(Ci+%MX56F9Tg?L%sdq_Bl?AluLYL+%zztmGVOa?=DI`1$skHY$ zHSaryQtJj*kq+}w4$aaY?yJgI!seW&N&wskOI-uV)wr^5Ek`bwTVL)JjgwbdyiKpT zff`A>t<7RZs~Ew(d)56mxEA_LT8-m*fAzZWxXtG5?`_e%Kl-29aR*ttl!5fED1%dK zHT_S~sb{B}64cF;^=b)r@%CnoG2Kxp0H~YSFV4T8!B=8kmvwgWgPpk+DB>nK)N^&@ z_9DsZl`ptB4ZVVUM4FqbIOP@^%JkmzcYpoi3w1xC#1^!+x%v{>u(Mws`aR%5K|!-( zeBW*P=sDCTNQv&0s`K)`jnR7Vv&oIS(suKjnC)|CS!Y5D6w-T2XYi(AEh$d#D^|g) zZ+V5i8Hwu+#)N1ms?E#gM;=4dA@!(!PO-GtjX6zZv_q!S=EChN0>^JV=lI8e)cv8z z)J_w9M5OdpZuHhNLL#?D@9U|S3jY}P)ju27S)Bq@BfD>0v4!;;k4I6X>|#I^Ud)-) zGvi-J5s$SMD850B+?^Y_}tf2>7vu) z$s{+(VwdN02@XK+BftOGfuvE3S59ruMu*eC>Kd-c@mV>a4rG2k)!y}|ARU!-R?i4` zZ5>2y7;E*0^S`Bh@;{m)N-F_rONi7&*B<*={L(d)Hkmz;EI@8<$T&7+`<1TQqLw>e z4`3&??t+Xu-FTMPT-wKT9<)SKc9bw~fq!#*u99@(Upn&3+`%Rp?9_tG4_#*nJg9=ziKcq~YJX>V-%@Czpp_{I??-zMFYn z(!#>Rrdtgxo#wvT0(ENy4p@NhX>-n=R$Vtpejn3B=x9&&m=QbjT^|1J3 zvN1DNm1pJawio5b*uVO99z$gBfxpN#bync+mHO{ac;HlqtD})iofN%@<>|bVGL|M` zc~Zsg;Gp69=zvNrlW*&b6@uTHzsID|Fmjm1x$03So^-43-Zl+6p!uMm)c-##Uu(u(jFx>u1*QSOTrN5az{`XMf}X z|8ZP}_qQFUz{awE;CCJPE|O^?o1!-*#DD$~y+>V81F$ zcDPQD2*~9_L>}KcN#oxZ>3ah@y0xvX?dnbu?@ExN@fH1)A=7@xQjbAwXA9FSq0tS7tZ`T>j_o7He|S zIo&;f{_mja&dD6z?fzaSf88(gU2;`XVw4%lzrT48*tRe!BW{KGUr!F}*i0I$-4WLR z>Cc<$Y+tqi>zhyg`>#AxX(BiN9^*YmhrCuHljqN8Klf+Z{Ocgg)$P3%{vKqVEzqG| zo=Gze-vq+8Lxwa16Y$ zR$Imw9#iAxbG`TJ2t*~pmEL}9h|E-t*Fr*Qev_VV868W18PPZ25%D;StE2H_*q|$( z>BEHtz!d9DrX^SuO8YD!5^-MhU@PQmx6J7K9iiyROKxIS1W;96#?|3Kogh&6;A^x7 zWcS25^mTP@4I6%xB0=xuP_6r6lF94-fvW7vq}p`qaky>=<6YQ9E#+qB;Cp&GOYOqf=;;8${JlpxX)M@?N#EJY*; zF0Uen8lmp}L9m|b1}e%p*h%;}_?KuQLN11GHC3;o26a3-;_lwktXx>Gvy!|mR^!kh zdlnAB4umv+o;#db8@z~4sGIV3b*^{qI(xZA4M@4ohDx-O?;=WW^zDd$xJ47 znUt^Hav{D;8Qf7abuufB7L|pf1f~0Ih%;DnFkBiG0u^X#KUEDwz zPJQa4%r(^G#1sKi^vfo_%K2~A&}4*`2g6-s&kXf|Sp2P>4^GX6gi4cC(>OCO8_7)E z#O>?8hQ19a-WE(!05okwTm`}59(m}Ak6y>Ig}zOjb8f-Jc|z`ri1z6yQbpzsn}eh- zPfZ~@qrJC!uCYV{gQG*{?g8L#)%rSWe1Nbv0`~qLfmEq2Z>-NIumdhTk(YhMG6~dS zH0-m@{*JusI!bw0(>sU{C+AF5FoYwRnTSfcs0}HVppg_fx0r)38J(8^+?kdbQUN|jHv+rdL8v)V$YL9_BdxK&#%fknRz2x-$K{*7 zuFcd!TGsr>*jj|9r|qE-LIT`PrMklhZ+Wf}NEBZ*fJp$f%R&mo#6G1GU@HA-YQO;-YHKx^v+HNkfBaU!-C>mIzYc z%16xa&?g^1^nV6`B#HDoR!_2W`j^Jflr_Y?{Kf^H>fu5dqBm~+D0WFY51eCA|NR;I zR8G3AsFXCZ1E1H{{KfQkcZ6;c_NlcxEAjuOdbBPbhNtajeB?D!6 z+t+*h{BR$laqgR#XZxvr)(=}}%q1IRo_gyhQFX3RaizPII!cY)KplFb#mqEjf3>s< zQTxmCJ4s`Z#dC1vqAyTfSkYywP@{g-W#uYLlpb%xqMmEb@Js-CquRFNcWi)%apE~^Qvz%l=hFHio(->)p^F@0ZaPJjvf+k1 zhF39^7XvObE3Kz>+{sU-0+_5vv1xnx2~8sl&|x#9Eny5Zo#IAk8js>CHsy!k+je9t ze64j<>u)S<6uiK)hpyX*t^^qsv5V44=S=NY-9^J2+v3WP+*Hrv> z{x<))L?4D`Tx?jMPRZ#0hp`h9Jw>e{H%9tVN^5NT|MhFc|JXgd{+ouN&kHu0QuypU zPoQ{p4VZJWu4p#34Sz+ayMqwbCr1&TpX99I#F%JWQfm*xNGWve{P)hFUrGRbgmr{tc2crgSiLglepH|tZ{Gk zH>WeU@E^qxG)mI8M>e4pyhL?f)C=C&{J6ED7wl7f2=Wf{Ld^x9LuJ!7Qsi7;y-Jz< zqr79sa<+q9Fn^haUmZH-KsVd!&?dCc(KgpW>X&YkU+;=Rlzq|OaJtwo;%q5;%7nu% zX~&vwPT;oJ>J3P@)}6d&yE*aEtZ6YNX+lkEf$gyR+T62M&5p@NWfQV#*`TD8cldCT zBYVoJ{}@vGe?el{{(be*%Ka6{$|B&nYPfjLH-!D+A-QA6-+K|8b!TYs7QYr3o&1x= zxH9t_b)kpQ&U0@*hV2FY`RRew9c@(D$iE>N;zsHoh7TB?{f~i?X*7AJ`8O_+|4U^F6!&fan+o( zt9m{afKjlQK!_)Rp;(a*r;l+{w-!V|cJVu|P_q<(uGsiD8bXh=|4=elQPbge+i0qQ zSELHfZtFXw+=9pT5M}ZPdW;C$x;6Iv zM6q9W`?mv88Zw?96USDOqrscewtxg4J(>XDB)iXD-|l9Ks3skCUVo~=no|#2p1?M3 zBKWV2eYp_R&G6j+Ny?|IY%7O`0FsL3Ml-qSNOf}HvP!@&Z>>RJ9^)=Dt%O*Qx-%u8 z{=7$EBzQd@8jp8J&h`il@_v49bx9a)v6TumcAm-YSTy|QREL~){NM5nhNY$>|3Q_h zsm?!V6ag?L2FYMTBOJf}A}igNJ4FH}1JsdNOIm6qT247=wEiE{4BCw!yJ8B&O}AC3 zzB1hjR2zF8dwWaAgy8?MRV({|Nx+-YBPYYKo zf3#mpo=y43mcvpCW(u_D;Z@=Ef7mtSeR1C;n6fBD6;Go%Rm$QKb{ zYjpVZ&d++kroLZzH+5UWOV1(WA^PsVtOE+doIKkpAV8YtBkie{he>9o*4DBXsX}pr z*6xo9&r_SddLH-#Ft~)@`*tm6ll+9L#)NKXSJ>*WRY!1s&HjsX5 zq8^~HT-4LmlH}or*u{VPZN<2j8ZcJ}Y;BJuGuAaMw6VPGx(`@6J4&JMby$@kcUcTD{#s(@7SxM{v@mnD6M?{tt}Ssya@pQ#jI~A)EJA$okbYB**>#tf`o1_ z?dH-WBv|9&_=-Yi`JB3&L+2(N5I*sD6H7gZDNNw+`E>@`)i~j!_faDc?ytChW!lvE zf{(Sl6f$TLJ-Qn4(=7>9aqC)gD<$Anj*I-$dohqnSa(^c0(AJTgN};mvUe4tqA_@D zD46jcY`D)$_;7L-_A@a4e91STh2WPdw(DWG@NrGA zU@Ox`{=MtHIi3DB^TCF*PMpd4qs zy4xowpO;CFSE_4o@&-U=Ca$=GyPQ^z)>R~;h*vmn0j|H zP6kl(z1EXR;l4UD_PDqM2csVNP;pxv3njJ=3RCV)3;h*!VKqA?$UnY(6lGKUY7H#v z9`P7Su0S4J&H*_Lk!Gm=l1x{_Tp;JnzDi_Z53jq`JjrpfD!4m~P;D@bv4WP#hM$uc zbI_(bVtqc+mlxGqlr|UI6HY0R269&-)mArTRQBC@)C<8(UCCxq-iaH#IbOzP;fIn-28|nJo*+`{$INwuP!qubD`DDv-zar!F5zzUW+R zswmAj6M3pKhecHnFOow!!V%VVe@snTG0O$7mZY=uwM~(G3arn9H*Oa%DPnI$v}dCt zY6t`K(}Sj6uhZob5BgVn-2 zJGm)+erNo>ho6YOeFE%Bj@lr(-Ov_yVpatj2>IEM)E$d*o}xGMP&c|_$K1WFO4O+n zs*8R%P7L^SS=j3UqR>B$o@L%dH<`t?t_Jqhhgx*JUBRnD#CoUT>L#dUBVb*yU*wPM z&f3M}q>g=w0Epa6BHTbb)G6g0SQ>+yy9?A{d~+>4Gt$un;X?jP(~fbPY0`J5xqp~s z?<**FSv^D=?DnKM3>}LfX3AjTWDmh8HU3)cxEj4HUdyyUGQ|fg6wr~b4iGz0*0j_) z1Lvk)OBHnwW6zrG*jB~Qvw-RY^BRoo3%DRG?hlRXXp4@3B5Q96lp24XOSIYZIES1r z0Vo%K&}?)#pt3uCY@qk3^HqRi(CkpC!MsKINYKh`xAr#uIlXl9%3+dsdYFyV{aCH) zkN48ZYCM`FDMFb@|3#8X<2jYXj^x3&{N5uO!v%MM$~Q^d-i#eVv;mlfS)KbHTkbdk znUm)Xb`|WRIeRc<_IGrCIHmtU#wJ0%n{ELD_*2Vl(NI}zNqj-X?xI)kWVP~v8d<>j zFcX0l6VsBf)TrlI$6x86ITCH6tlJMIPHMdWtc#fmzbIXr4XIrS2*arJ(C)U!jp@=K zAZfRE7M=k-+v>`TxYwLisVI|>-u-iJ{k(rzsr+GCdo#knMBUk!S+y33P#EaYN#!F^ zHylL7A3cD~TNp^uYpd1MqgMM_HdFGehDbZmCq?PYroyn|@>0hKB1xnaPe)A=eM&p9=Wm%;MRwh>;=#9ci5<=_+M`* zIVF+*!085cSqr_+4Vj;PuPR*ukg$o7gV}rol@!uZfadGcb@7q0!QyP~CjVDttwMD^ zcb<5^)3}M`+BDS~$_7P49oU}E&=7j#P!A=A)Yy3hg>q*E8v;brq?bXFZ9;o1$ivBb z02-OSLDg>UhvX6)3%PIg@XJ*-`k?T0)w{EzMMP@Kq$+UzRZ0fh(9Aw)>B=;}8^v}2 z`YJrP%MuNCWPW7Ta@+-z5jO59VR0DOS0L5R`O^KeCsW#OpH00ibDl69c-I48s6 zU)H42^gNb`M@6}&4Mb|HS^-oCvcFq3&xm8=fUO$w|Jd9_V~CsE7FJS|pVwkNjn!{s zMcnEmBbZ*nk`L<@yJxzQ8USBid7^SGkld7L*obp`cl;v8%rAVgCY)BZ%W%z_p8oPY^9Kmg``)L+M-h{r?tqnM>rsUzvfP2GgqJKQ4yX>;3FgSLb&#TMIgrW8o5f@1R=?Y?PCWWXQud4+n3) zL7yTT`+C-)T*o~@b<4}zP;BFS<1xzcX|Bq3A`BPVw}GAxJN{<<7#H${7SlaU`*OK6Z7CBGOk^ns|&-^ zeshDs6Sy8+!Aih4CMsw`xU>2@=(f6p((=7Q&qpjK%kRmE zd8Rc=dPJnj^?vImtQzuMA?-LWqjxi!BK5V5wUE{~l<#XF1M*9aV0a6E(|iZ~=LsVEwtMT??nZ4h zTMqA0GC}x?oj=R_)HO8d!_rtS_O3ez)dRh9qH~B-$~DJiLkzT=^r-B&<$b#0eEqW; zo+$Y<#%pyn6>vG;_q0g{3e1b-S7V&(0LRk@`qR#Mf)f*#pRt+0Z48n0m1eh3gsK zS_6&rZ>YVRc+uBG+BG09668LNsmPDCrgcZ54v|(LwadX%lDz53qzbI;d@t;f3S%Ashc9K5#--~plL4Bqx-+@ z8sYtzJE}1ObYiQy2G~bA=5N}(iersebocCX5pGfRFdw7226abdfG5y>SSzvd26z@v zF?UoO4w%=cG2;jLUYf<~AvwO|i09XpuM+~i7v``-KbJ)P2%E$c#HS{|jm6%fOL!L@ z8ZU zs-1QuyYiip^}{oHVCYL66P{9+K{zy|NN-lzeFi-=Q!;RVwIK;|r_bIx?JEl|uV1#x zWK6^S^(eA|3(PlF4_HAjy+;al1EiRVh)#?3FSd5slL)_2?uLCf}XZlPe*^hnSuIEeb za2R2rX_18X71I;g55;&hv|r)>*4a_sWu z%SuyOq48%{hKg;sNZ(tywpLUvR)@Aq0O39Dd1AYAj1gF z-ZY&J{5t_=@dy2bMx3W9o%zILyI<2ffC$xG`U*oWVaa_Vk(Y^f6e8=Cm*ze-8gKxr~xGG+2AYV0-M4!|c+HTDA*M^Qq6+q|Nlq z?03;q&d?Tzg0OdJxB|6`g_o@z*N|RI%r!1E~-zx zHoygu0TpM&^K!$?Ot{*TOw7E6HC+$e%vepQZgg#%s_!KPqi&6kxGujJEA{_6X;g}K z41Vl$+PJ${project.version} - - dev.feast - feast-core - ${project.version} - - dev.feast feast-serving diff --git a/java/infra/architecture.png b/java/infra/architecture.png deleted file mode 100644 index 2667fd1cf2eacb0e01f76517487b790d48cf03dc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 30647 zcmd43bySsY_cgld4h5xKK~!40K_o;Z1*Abj>6Qj56%Y`SE(r~gnht( zqb9FR#yZcc_Lio-DAzHh;VRHeqJK#x#=t0k$Bxizy&kq>e_A8nzi;XN21B!J zVo$rG($0Ap!bG>s74qJZB8uMbEq)iGXLu&BpzsYVb2UIA)A+jk6?dP*mc!ZT>*0&3 zqLLEhYpx&p`4eN48`y56gR2~`pC6pU2c^RY={`S?8(K%M3?dv2H{K7sx9PD5t=jqO ziUlTn>@{oEe^*y-Y`;zNgwxt>ev{z{zi`vCLUulJkQ-eW4m}LG&wOszg#VOyjmc?uPXp2;tiRW9eRWiOUX^8o;!}ghnk}i9MdmjbUtberxpEq6g$x2Isb1aA0>4L^@g=P z&)Xr@_~wr-iQEW@K1pXWf~{1H)T>lHeEhb9zr$0xlm898x`BIgl3jQTnW{#|+cRB1 ze=a^;jid70vAI-4XPk^*aU-PIX}vPg97g=XxMeds{qCO>@0Yvb!QmNMS&Y2Aw}!Kx zwXO}OvkE=`PwaN7A>)yQJX#?zT{gU=9tB}z*z%w$hs1J>cdsXm)*wb!GNfjsQd#Aj zvByi!=8sKH5|3qNU5%}A{uAJCE-Df4BFM0#27oLTF9F=E6zt#U&!2^AsFTcF;pOg1*GYxsm*vx;K5TQwTUpcZzR3Lv-M;|1bSEi=rZ+@!% z=RzrL3|2Ud&-QveTwQMc_o%v={pWrM`oE4C zqYgE3qJ*QSCLZ&*USnWQ{FD$HiLcP^5}Olhe+&6_rGpR4@6Z2pb9C@XZj!JwobbeT zXo)JZYM6fl-&9~BNXihuTczHvJejZ6@i(_YN8Z%KP}q|7p7lz>EeUi4e0pZ3CVKxg zHmvhotSY-K`~1AuYY}HR|GTat-;nDnz^vsbE6Tk7hSCX>Ep`@7P^e<{qVPu3nu*PY zh2GAS4IzIQ`fqS-5Egs8_{>@+J7&E&ivPL2N3xM~RV-P2&ka@yv%7~q?5X zRqj)*sVtS+)$HCRBA!V%Zd2gPcvBqNbhj;c&6}@W$gLuOb3poCNxWx@{U{Y!k`E{^ zdagzAO!`X1Ryq=!Fg*N9AJ6=bEgi&DqZ-h|7>YqiB+jwU!b zBX46lr)yuKnB&j;*~2KtcS3bHeAspTxC3t*MK-hDZ!~|(%+7jW<-`MLd#08Mu{JVA zZ1SCGbyGUh05hCWAVyS&8zr^f zYyN$G0xuPDnglxX986BNv{fhRVZ4Zw`mlKWU>AYdBNJ!bd3+0zO>Qmt3cGh z7`hMZLcLgJmDQX{Y%kdA23?XH`nFv9%5#Ncwf81X=Q8${Bi<1Y-H&hOeaYkGPoPkQ z63T*SvnK44K1M0Z87uMFoQNM;e7qkss-&BGiqSDW3H6PR)?oRaZT+F$IO*5c<|Z-i zY?YB*O~KcLuNK+YXy)N$-y~k*TMFa7JrPPz8AwuFIm#EVvTLBm5G1q9Ye3;U`ih-+ zCm<&f=R5WA6arC)S>2M}NK$+8FYv);KdLxcz4KJ*T@p8zkDTY1oajpb%wvv8YT%P- zcs2M{(rf%2zuzL~Y>O(*XKcj(Sf~2-&DQntLkd#2b;O77DLj~1XOAY$9A0ti^o^JC zfA4HdW{bR-VkzO-+4p)9 z1xx)ce+vn_Y&PX; z%-%$NN4F7(xC2z(y3M^J8FR_xTcjEE0Zy+;W9b7T3S=`?(~63S{rvor^llNRDR=!< ziTwICC?Ej+)vH%_??@>qD5~rhP^V{RZctMGoappJ!%9`nncvwUGYRr!j9TGZ+W_0@@JlxqpGrHg0J=w4K{0mHTW zjgDS&zoITfzD}$^8E+eq#}b(&ABEs3cD>?;DuW^sgQsd?Lb>?o&!5c_`6%Wz%|iX6 z@^Tpm2W}~8>B!hvY{dKUaM>qMFs-btlr=O0*VZiVu&|W+g^!PGDbd4UgG=@A++={t zCzVDV9UZNFKl=TrPfx6@9@@;*1b!f4DRF*T@6OA}$XH}C$oRh9m`{M=cE}A9lHqm7 zTOsYl@P&<@4a=_eFi7ssJ6P(+jFP&#)SH`|O-p<9w?nwh`=}r4*OAY8pR>Ux!nW;h zRnAF0ylv7O&)e18YwF<)8`##-!N|;v$;HLBc{V?9_{z)-`K^nK%iGl?SG)a{XV~bd z@XG@N0u1482;cYbv0>}rhp_WZtgJYeqdz3&7g?_3`{M4Ld6v87&J*3wx-rBt(B_luI)fXwD!I zu~P@HM_kU}s5C(kP^BUensiuNlkrPE#ZpQUEtaVd%yk^8)DnNaFq{s(VXaCw#0xe3 zKijj`hR*znR`g>_OG{sYQi#Qojhuo4Dnc=VkHA+5TSZSVY$Q+9q$if^Zh-Q8UY*rgx>svz5jQ_R)Aq}F#BcrQJtgipkSoagLdtyI_s{ZWY{`0f}V$sD(>|} z*iYSh&#+5DlU4g=+USM#%s11gOZ`twpDBiMvNB>1_3U?veZhJl?1)eKH_VsVV7Dif zDtfGQcI39Nki4mmvR;m)n|)^AzDZW0xj@zNpE^FywLxZQdOa#6m_ovt4Q+fy&J0yn zV-u5!vC+{{tLK;!qGip4gEyustXnIrr^X#1n&0luqaY562@vF*hKMI&l!0F7N9Akd za9Xkp3WB1dNWK%^y?Zx1H}_p_t+4misrh3W83q9X^1i-4a&d1FU0vOmp56}Cbt)Vo zv9aV8t{u@2G{;TG#>O0jK0)M3Ge$}YQNBtwbH8wVuUR0FEp_-b3fgzEBg(E)8tS0+jH7+6idK}yICeyhU|4%S3V;pDreSo{JFv)+(E+jbh&EBGxZ*1(0Zkh{eT4 z`e+4YDu9$;QE&#!QI;L!zfx{&K^uef1hIZW4~rQ1n9&H0!f~ZXQcH{6WoLF~vsj<> zHeVLP*CFuc@EqZYPJzVB&}Kg|Y_G$;qTsqHha^)1cS;HJ-$Zb{?RHiJ=teh#7#n|R zw2EPVI_PNl!UV~9fI_!x@`23RwLGWL!_ism4|IeD zsRW#j#EaXers1_)Qxg-#Q#|z)q_`L&qICRgA|fK=aRS`j+{AQrM3HiBf87R2Nk~YN z>WS7jHYU!vBq^&e?EN+FN5AV|HQ8HihjiLHny-@}LP&A~`FMW+G(zr9?`Rl~7UaJo zH?G)ybMIsxb3$WdV`WXvV914XnW}ZCD+z2u#{~-ewuNj5d~P^?Q2_#M-Lj_DAA6sw z(AWEJ&26?1*n3+M&re2oic-BP99d`zU)SX_{fArSE6dSkW!p=e} zk;qYJyn7b|GMCkmirJdHsp(xQDJdi~l6n0pK3z43^3cZZrAZcnw9H?P3)?@>cMqH; zP2Fyho|=+VueI{ccRr*0WDOf#bMkEMwT16HS+(%1DkHi(jjxt2{uAb@w(>-@Pw}iv`3Nv-RsO`9y@C)?v zo1wXS!GdA^w8re*?K5}2AZ?UIW#+m2XKNwpT1-}`Jqgd6hU3|7UcY`#j2o!;rsmtv zp9+0+34AM*Wh)}W@dY^r@f(|@yVaGySX`Xwf&bzp$_R}6=_hZ3NXmYEqE85x3C2=J7?lF_)^U>bO2pk0NzX4X@>BDgb7HF@TDo>PyMUU?2?JKi-m)mpAh1 zgwzIs5&*FRsq{ZJ_c3EdCAJz-?*c8vC@U)~@;Wqz2QPjgQu?t{`h(@}NWFYXd9L_F zyOAsqKn<#RjM=XVAuZ#DpVg4teAjHEcHTHy4@n*B>P$gJqB5iR|(?>yJq{lO z+)YSTo8v=pYCm|tceNTR;95iOYBOhDvQ5Ll!h%&@U7cN6*!h)B$LXtfr43Du^F~l4 zlWc{2L~4$Ug^tbepwO!-SsgE?l!ypKq}=b!cclGjTkpL9oSge9Ah(_@rocB4n7;4y zTM7rFwr@CW?;9;O`Jr*y%}dlzVPazP#k=r6`XXG=kdsE0mX;RwlP4DOQH(9ljfs&F z^~Q}GVP7}mtNr6n(Upk?2a<8W#05%^?HwLc!#;&onACNB;HJnw{W_OwPJ~bSXXX_J z3z4vZo85OB4J>Y2EFSkmzJR=NH1nG0)@ZVPUewlKE6j?@c%`oqMD?bR`ZeaakAsu(Xj?Ei1L8qaWlP?N1RGn0|bv2lfWxFIMqG7^ezQE913czQJvfVi@DhjKo$^QR(XL_G@eucW`hZzIChJjqm1%$67^(C_jJwGBPs@ zf?o{q>_aIXu%nf;?s$89d)YZTNCdLDv?OTVYAzFekO7U}&99%j#&vN%}b6+1JB%Mt$X(=gUGO}O@o}yN% zSi#Kl&U*PfH#0Ib?cHsyW@@6NZV3tqRH^vum-a*%-^4*#=W<(ESde@AR9a3BdwqQ! z`n-u(vcB$CsYk^4#af}kM08Nc0!4$N{z5V}?n@LzCd8*t{oy&kZzM_38(!p4NT%vh4k3QXt@-Mm0(?DY?}F#gnWL z{i{s(@8j~A_if%hd>z8xl)z`*Qf$(BZ(b7nH4n$jK6I(B;ev{mNCpk7S~SgyQ!x!Xd}Ld2?Ui>rg{WTYI5$&KsHWC)%nIyUkd1 zcAsh;SCtZlT?s#Y_+Yj3;Re%_`9hz|2QFKaffPJuWvj0%a|m%%*J_(BF$vL8Q2>)g zKoYCgCCpTv*!0hWko_hTBJeAgfn>6ku`Jjgv5r}9<_k#AvEgt`!*+}NxkBw=-@$JL z?Q@Du!3+P+rDADT@*crAENsT^S#Zb!F)y#INZpSv2}9_a+&VP+%>3FXgc!E6+;WVs zrZ4{7EXa-1(Un9>C|LR4^7QlJp&_^h-_THer3i>|G%|Kw-`ZM_^a4NbID9uZHwcm4 zy*Xz#LXk%dS?Oa{WcU&js{mOaXSQePCw~EkP=!XC;Uw1I{v^SaTKRw#8jquxDda4XK zNU^+@x2>$L?RREHD>mbJ%$1=F_P(rgQP2^yHHbi3ULNP2f3>_c7CyNz zG$ciZr6ncvYinI@LHqmrt=TH+5y{C)lDO7vNfStdKu}N+7znKfPr>hd`C)EVx)3@n z-!j4rym-rfBxWR@D=7s%W_-QRfLHU`hkGJ1E{$UHaCrKO$M{FStcDOW4zpnM-BV9k zWhAl|WRGt}CokKWnl5JwaMDvuudl01)Z=?Sbmw4w1h1l^0tA3378dtzl8~Sv;23sw zb@>AJP*6}nWK->#nVD^VaqsBp;N<3R`Nh$Aq3&#Q#6pTYI52>SQw)O5V!U?`8z?g% zWP+=!1(8K?W314m`5i{Auq%6;G1nW*k^T0NcH{Stkwgn%`aW}IAY#6yr%y~jR?Q*B zyZulCh#e}sk%rs)`ubXp-l9Fdy(#O{$d>q@F*Y7vYm9;S2iG|-GGF)XPFXrHbVMRO zMpTNF@HP5=O>XW->u@dhCFxgTXly*RSkiO)JdRstRTI{fr5A&pP~964Y^Zuz=vX#w36 zvJ-xm)u^DP)X^2mw0AgWfTldRG z+I%P@v0?t@vqI*0qmTHHA3qNC@wW6?{-cqcA@2mxSJU27r~UPn7XbNUQE6U(QntZ- z9e%|0bf~X;1@SZZq+lQq0M8te^6pU9s7Ji(_9CXD3IS3LJ5VYK1$%x^8I?ACjD47b zOIqm>8fnuKE&i4tjw)yU?J^Hr3tAl>?7`3b<}2BRL`3FjY$3PV-c-nX;LIVMSV|~H{r;a&$Fhowl~6C(>%jDMCg3)Ltfwl9{{TGTa$Z*lT@K41lc1$9 zNq8b4VPs@v^IEa;31WA5K%6G7;S3?+I zA|?X%)DM;o;jE9>OPe=ounw)#(&j{If4h349qSSn&M>$_ z4PU(qpyIb#(=8M=s4lQ<_6*r$95|Htk)J0{ulY64?t0d1S24`c_)RD+CJlew1oU?Z z-@p%!!ke+s7qu$wp+MtSih_&jXn^92Q;J%9A^LXdXNoekW|Sk(T;hPLP82yM7k4C8 zJUg&4-dOE~jLT^``jDKQ+_3W#&89;yy^OLlv8Sh}va)g$jw(EJ)5HV?3JS{FNFEt9 ztV$_jy#*YstP6FRzoixXl0`$H!0>SdDl$u}tCO1b#QJ4Ejj0&Rsi;PlLrSt%Lt4JL zy(UEyVP@u_q_0ju+X9;Mhs<*G=1oI$bF4_Y03gNyQ!#LJf4=N6V2rdgJ9?`MauP(6 zAEXgoc3I%;0bN9V`4Rvp3ob1`wZp*#<_SoqNGNeRYTrxbc~^Vl{PT4xOa8=qk^pQ= zY_kJ;bMp{MV@pY)LfqT{Mihw5506`NY5{-1PZ(l;O-wv~)ZX511boY;GZJvCsNA^C zp|4mC(&iJ+=bN!{am_6) zsNxs9Xoy2m?R%t)Dc2V)=8c!<$DKfq+5a87D*gl-iB5cWnJY9bY+?(P5W9ml6@-OKljKSM~>8zI!e&)s_xa|lc+HT)Zch_g9OuA8AS1n!%x@lm_)Q918ylM?~*qb`4{?Y zefVC^MO*2fl6d!jHk!OpKA>B~5oURC@82(FW=+OJABuY-YVU}~`hq3<+4uF_YVGap zd?DYsAFQG1=;*jqACYF*HeT{IT>N`Vdh1s6uU~l3APcx|Q-6gPnumu6(FFa-4LZ8; zfdS>J+FEJmsDz3vykny%FE0s=P%S~JBH$mQN@%RD(CTx9-ar2K#O*=ZuL8&gK&V2i zfJEvZ1duy`Me*%i;`R76xq*R-0=&TYPrgFLsUPCrL<{;bcPFgu;xf-YR$J}ggtj9U zm;qEj1f<_ZbbDwZBEEjbl(;+}&@f*S;u$kO{r4x)t87IGpjCgWq(7kZz}^IYlbIT4 zt&&$bkp6?TOU;aC>)c`m9htq)xA+kf#>R|O?RJ2Wkirw3jTA92q)b(;g-=IEmnh~% zxl?~k8-!2sEAu=qH@5|-d?&lDL|iu00`Pu5o#1fstE$2BlcjKQB>7yRH8eCl z_q6H%R`8o7RWa?cJ&&l!eRlo!h2K%+v_jY*JGQj_#`c-5N7AXDm2X&ZGuq#JbX#ue zj};#z%*4<8}3cj}PX{kd7&F#JkdWZe45svF(+ ziC_4~&v^ol-y%v2)_PFP%3F=;`TG2V?dXOBA0W z1*KJI=sz~Mw(J*LFup`b8?^-BocLVtT-)C?4GIb}21HrqdCGlu@E1*7Tzq_d{L$G> zQ`l5;ej942o_j-2IY3RRP-*EkH_AP7AyDOXBYyd^qI&Dpv!Rc@MJi3%HdVra#wmJC zgYV9r3>AHRKZ&O94}-r2-|VOv5aMj525LPtGhZ|W1uJQa&-qG`CO+QfOXrgP=!OuB zSx2m)hQYi}Hg{xG{gD1;aP2Mj{!0QgOv_d z3W?6B=)IT1_#u(|{`=<;5Ey85ak3`>JY)D!_Oq{0KiPkA>z5YJq+g0%M?rfrjAFXMS@}ic~Jd$l4r*$iV!7q{i{*bb~V% zSzk;#{VJ->2>+Vbi79OsN|R;81CcY~;$IxV2wy$ibsW^LoO4MEh+?6OVtmrtrk>PZ z2SRs%|nF=@F!nydb{0)ZJ*8vc!kOhr%I-@|9tKQo^KMDTbK z1F5ckCjLAGzokuc=Un|n$?^8Axz8*(3C>kN8QP^1ffFQ>NiaAxsdMj&EOM826O#vz zwxVFpU%pf}-2+O^tP-^dS~thU$+_!#|8Cd=pO#EK%>}L|98vcJi4~5C2zd@eqeb0= zM^Kj%|0B)&qj_-(L$=kc@K3mwa+xA^CpoQN*Ptk>bIGV9lp?XZygW*u+wbGbr-^AHuXFQITG4nO zeG-H)9(L;0S4l@3I|(oSQuF>%H%~9;*X+CJWyvweJ8-w~@MxfgH@O!3N%qr@?`q>_ zHSpdhG5civ>x1~*r)|-i*2zAp)mGc`5=@-IjRnzh0+&9%DvdGUFV269T+59-#4ZNy zzE7NCW7g20E7K~OC0dg{9`MfBc9nfDtgc)45zHr)BciengkuGxjK^JWRc|L+a|l6cGNqx57uY!z$YWHYjWFs21Fd}6k$v(7i$0OwW}(R)#rEER4cI^S={{a=vBRg% zVUH@;{0Ibl;6axgJ`v6MH^*#^k+iEPXqfmn9vNu07GGA|-PqlmU5(u2Wb!DG^lwQm3~@B#7y#w5LcyF^;2%tJSVObxgPRW-GRtt|pu+nv>K&C*>Vi*9b$ zwEu@CfknOi|J{-h8$37k4fpMtP`X4e0?~6ykN>q|U7zsd1vmAi*UiFoy z(NV^4*CaNd3cf?ye>Cg~^wDPKdAn|Xt;k}_^*BoNv{Fzo-5^Aa{M@i}+6nn{?M1ip z!Yjg6^-~mux!sbx=oHCv$DzA!VU6EzwaLNgv#%Z~^5R}S4OccJN?&ek6cvVNqy&XG zQmryvP6b%g2h@?<=oSX2%g(Lfy`##|`nS^|+VhOfO<5z?gY;idCcobHDk>tmZ zA)5B9M|${lg?+;$0^N-&{aCWk4@Gw;M<#ElNFSz~#7`Y8Jk(r{kxbT{%=QGndENFA z{q+}W5;Ob8Q`aN)<34uHdq!q|Ml5vxU8K{?klnsD1VB@VC zBASg5FP-orXY%lgS3#jCAbpehOxS{a^YMy!dd1Ds*n@=+8hj4#=8~RN7ZH60 zS-(s1myP<$`X9^9{$A0G_*hCK)yKpcajTJD_lRD&tdKt%qC#P4n2)2V%?eJ^mTS+oqp= zy0-OY`_B&s)`s*n(!`RFq=n`J&*O=u5c9L?ZE=t_j6tb7IjLnZmK0EAQ#3TO{XmeC zY4`IyO!@dRQWoK?5-%b8!}Mx>sSY#e`kVlk!UPa+)v@DaIy@4vaV%UG_3LPGFp(S% zG;U~2&yoQE(P~^(1GAD-?o2H||I}y*&n5o?SOWj5LJJ^Vi`$Q(M?kw(e35bKyfRlB z9eoRU8R4yy{&b+Pvek24#FaGNuK9SM%E)wUAeyQum^clyuO1r+H(cI=rRx>D2$I2E z0@2j&D1iFoh3clF*!#Muf8XO-m+mr2^I51cYl`J|3OM;Z!mxI99y6cE=Ac#*#TdDD z=zk(JVcK(kGQv{8k`T6^HoAyKM0M+KfJllW+gmWPMR~DB=}Y(>pk6$ATU<}L{$@lz z$C_qDw23qvwR^$#)1QN@Av41PG$=_CDQv-}20Xg;Ka7Lj5fah=C^r*-+&EjucA1De zTBPLbBdX;$o7b)QCUdMwG)q4qequ?hLm) zNm@#8?9R~hiv$JQ@7W6!?)d_a%yeG#FbmF>Cc zbyF&tE$cROm0NwX*A|nL>||DJmCp{!Ln}q2*Oo#1g6#-j$?(rQvmTE&m-@D9H(fA> z@J{x(>z@pN`Q$l4H^*!hsA1rELP*jSJC>m5T6;w_-FWTmVdn2ox=mgF1_iO@WXQMT>$yoc-1V!E;u%dqyU< zIQ_C8{NcPSFGsgTnF+faIcY_FtEwW;(=6<;EUXU#g0jSM0xg`U+Opy3_;@T~;Zj~c zbI9zM)8T7vpdIaiPK`}SXq}#>#>AwdG9S9qdp%idPUL<1hpZ=op9p}R&$mliS=rW~ zKXXstaa8fod8+LQfyxfauz6~V3SggEp}8!d15`MxM^_O>J7#z+L}H9&O0T0#SpWT+ zACwUm5 zz25IScIyM$u?Ji&<$2|?q&LvPbBXwk!%>+(=kL6ifV3$Ies(0rck&A#prH__=>*DU zGlmab5XkuHOJVOQJ6#g!yp4I-Sz&v`XfMfq!p_%q#YP0)hk<%o*LzP3OU?)R(Uo-; zM+K}dU4<&(gRM7?o1oh%A6vI8&7yt6%wg1rr8d6cfZ$w-HX5(-CPJ$1x^*;mr!BXj zpc?j_8*vVvJRC=QJZsmiWRoVuEOV`o9&Qvpi2d>f9n6oRbmFwe#Z}$De(LuF8ymMz zol~ADDJezG6sALZx=1?(k10}j3i z)Tgon0|OV1=X^R>Q+z%-Z;Up#wx*Zn9kduy@btz*Pv5Qf{7_QzrN)A zZ&6{o+KWuzUzf}NXw7UxjFmB4<~TBvgXqa3mNBAfk#|4&?4{Bt{O0QyR{f=L5XXTqH#U+aHJpw6R%G=rUz)6&3NdKd=q+2h+29G_MsQ z9U_*RBDCK9=6y@6tAEv4 zdehE}?SE#XppKpJJ}NCQM%R4UDg`G{YWdHf=IQCWr{(?08ZSkv(Hw zoru+B8II5O`J8aAP(Sa=)C)Sg6!w`SE^ap*h~;L=6}!C>*~>6Y(q(;#g`K;J^vGHW zJXWM@2{uHC-Cl*PI0OHHhKj?j)y(!PbzPjD=Iul|k<{7;-&THW@_9aei*NU?nm5ee zQx(*oHVB{H{eE%BccdOin<$7-8sXc%XAK%!TG-$(KG>O~gGd)T?nM6)hbT5fBxo~7 zb=&PN%IG$F(*ReOc!5tqKoHUW3iMS;2?+%Bvq+k?px{BH4~93p5S?B5|Fe^V$IL?f z;>nfseZUq-OK^5$8m)o#8>4qXxTZbKmuhd%LmHwO)PkEv2jge@^9D)g2j_73->e&r zj47IXK}{vDbP)r)Ok{!vLJf}=uw~114N)S-1@p!>Zx==+UaMd?M?wuDp-sgmT4Np& zfAPwzeEQ&m1@p=C3dfR`ZzHynkp@BM$8_0f55t2z$Y1y3OJTcR9gw&iw)~I)SpS(o z6|sIEawPgBpUmUc$1v^_MZ zo=D<6gT3gQWC=`)Dxc-1c~x>FGp|c>!hfhikg~#7#06w zE0Oqb9Z=0=)l54+(cnFQ{(QR5jRaKQ4v3L~N?U`2jWHq;WglViw_Q;a6Q`Ei zfI|DE(fg8^hNc(u9>NefovOEcj3T0~Lf!LW zXicpm7CQ;f?O*l3hC|rg^OLk%wtp#@w|1}Z-^BAToZ*P8t1FPS1?=V#4JV6uPCK(S zK>k#DUkWilPsUeMQ)}z&WCG65#>NJ0g(wKm{a*gWR7|9;0Zc6dZvXC>Qw*2pXJnwm z!*AB_tbYS+udzD zIPqx8d}U?bP^nPA-l?J=VKX%}GIGn?+nbzg(Y}E>OY|0q)?jCF+88AOQz^+k%gW~1 z5;QErHc(`!qG7@cGE#ec2pO37Z%|W*ih0rcNPgEQ1NIpXdt>|7+NYuVdZLKPNIx(@ z^TwrvpRTfd7Tv&e594TOE?g!3N%Qb92~b(>8-f(vB0eAw<=ZuSE^Y`HmPNUZ6M_V8(W2W?*UsuU~qQ;SyZZ-3GAHgyfO(oE) zvK*84ZTWuUqlO8Q3UOqU^7cAJuI_NbsYmYumstjVVa!@~V);3Y*TCT?-0`CQ0K7(< z_mPmIVghYta)L8$t2CC;Y-geI*;Ly5%S~r&|MT=U{IElBO0}%){STbSfj=HPfAWKS zVdUT-;^X5Z7jn7>MmO0b|3{*tdOKHOb3G^sff)i|vWq-V9e`;>=3y*CY6%eQkG5w* zpf1#1U!4BZIEt@N~QKHy`TJEFyn`oAh#=`@ldJ2FY5 z{ui^i!KUHHc4SJr8YAhx>}jr<8QC4#45-hd!BBXYa(Oa4=B2sivvb8#>xn!~j(eoI zp5Vy{oBVoyyaU#_79#Nr-^4^}-D(G!)qOP(3Xl>p5UKocWS;_OE+b>KGg}X~Q{0C0 zZQ+MnMZVT4w8Ad&qRV32w-&ZIF)ax2FX`AEp-frbZN5O>pm?u4Bf5Lj;rrr1S$$nG%)4EcZw- z=pfnt?(Xh;92}hy4ATE`Vx&w0^hTcffS}ycuXJLb7%DP z@+5h=xd+>YIpCWVc)mr{A0JP7)IbW_5tm6vQ)??)`jasC^?rnu&}38O?0X$yIvD)2 z+frbAlWS{u!D@57vbaEILnL=$cv~Eb?CD zzXQvQZ|BxDxB*<_^mn&tGhl4Mdl9SS%NNoMSWP)aMX*pH5QqMMkwJ2{QCQmeN<#Et zLHzoZh>%)fU~}&vYz7A#8*CyvAxGk1H|~jTw%)X$bA>Ff1#>kn+tVM2#7+?4*XWMA zv#JZS(!YQIkiK%yy>>FhZ)sWJ=s8V)lJ;>%Kut60ieT8ZWXAam2@^)HK7x_ecw2Xy zAmYP;#)}sYv${NHJp&Y}UH$!?kiLoyn^BQdYlH64@pZi_d6B~6)?m?J6M6cS&sZ!D zqw$75u#~LBFrDVKy)>!e%yjHNLU4k=;=1suS!zh;LCC9CBlo-I6dcN z(>gxi-`i{M2%{=C@27?NC2ELZ2=V^q9)F9tp)?PsuQLK)4v7tqxg>#S4yYHZ zay{^H8d~OEseYo!vWIi>UyHaVCL*boM=4q417zVot5@vo?Bfky!oqhl!5Iw7j|mzn zFq8W!*YqqbtL-?du>D#(;@Y31kWQS34?~A}kJdE~$q%-sDor2>hgGmXu=IAP_G%@D zC4X6d{Vaix0qLIN&~JlFsG@oK`DMXMfmCL|vw#H%2k9+gR!SUyb!?{hB$V8*UOrCf ze|Z7GCq@rNywYYy3MvR3+=i<&OBfkJ+I_6F!Fe1TA74`V?Af!0^>wg`;<`+LAIR?C zvOLH*r87QGgBNI{WuO=VunOPGPwSnWjQ1jmB#SwU{mIjbyLZk-!6G+U51y2os9&$w zG3N&dL`Hk#-IGiyFhvP=V={SEvXsAp0!NziptCXNlI}KhU#43{$`DxQ?Ka1`T(Yr9 zA`IOlpyy)%9~C^mrGxm&$_nHga|52^Fq`lzb_BT5OXjeVjRaKHdXK!|GtL>(I1?XW zA~#9D|Hm-1m3eoobhJXx%d47a?pYi+2H3c*A{fEn&m{XPz+ZZ4wyme3tWm%f{9--@ zKYj%0vX5!UfQE59(YbT2N?uoo6%rj->yW0X(o&w4{uCI8oz+Wv0s_|T{HwRNVOg~L ze;nLGjvB!zge=f_WbAF={53K)#SlyBKRPi)8Vo^$1(#WbLMGBY2)Hn19c6NSJl0DM z6gNY7x!j{R9%05 zfAr>yP#SAg%sU(l?!jwOkE7Iziret}kvcF;z<_5|pcxB+O+e88iAI>yWm5~EoHKRs zA$`xP#H&|#;2#FqgMNnzd}1j+SAt530)8haZ`<11=Aoz|K`{8G_P~9ZSN}ZaKci-u zgry`z-pP1F*X|vDbR}axuSh_Dz{LhfHWV1KDIoC#nH479Lq z-$=y^@3h>z;thVTuA#9|yVHP%NEG*>1+&1(a@2E}-TDAVhtfZff{n`H5t+anh6x$l z1vgH=-W^5x8l^zD`X-pp7DMfC;sy$Mojp45?8dhp-29WKcMQ{wXal0Ox+zE`0HzOc zXd=h&%<+eZ@%I_}D>c(u=6kp(RIgo$eYA7?%(=}bIWOik8x1d6bdY?@FS7e-T^y~y z`8$jDEs5}G-owUfm0B=?vYPrZza?A#49yE9tM^*OpS~0h+Z|Qmdy?s>Yv@YD;KSiJ z3nn0#EXzVWmlmQm^_w3Q+R+J>eU~)jbaOZBPwr(O-hQ2xbS*m!t!Zt~i98tHGeo=p zGpa9_{s*`l5P&QX@yR_ZUP;Zb4lPItwuf0>xP-?-?Ne@w!S?CSGrU|pST=t8PL5g314sP+c z409B+P7V$ZJ|FpH^Av!^?%|;f&@kJx_2+Foo}Nb&=E!-ccwS2+^aLw6DtMKU?ipVc zG#IfSpGC2@H{w0!AjMTtRYe5@+{87Q^8H}Yj}fX)z58JcU>3+W$cE5rs)7KpV5|7m z9?`{?txxOk95pWY0}G_Nk;jKKlAVKt1JVbwTtUL7Ii0u1ov1;i4Y!d0?SR$`MJD&GfRqR%cq#3ZHlz#Fb2Hr zAo!wCKMa5>ir3jeGoa_ll$3CoA7r__6=9zEpNST-qz!jU%K5Fkg(%8CO!I#?@O7}B zB1fd5A6M3^CIoxQjnnI?+T*I+bXElA`>k5qBSp-Sv#F#nA)}^3Z;!YB;&mcA+Wx)L6F9sO_Rf{*|Fou1ftU!F+AwXo%t z!gn33ca-D6IL24+^Gg?Un9s(S_Z6%snj9s6@3GY-zqg0pIs+1fm_^@E(!`wodC zf1}I~TC+rdPTI>|pO0m_zi~f&!6mG1_mZmdtf3`kte||m5`)$F#QN+kieP<$<1s;f z;ih15#qZ_QAK%P>tp}k3lIBj%6T8@LRXC>MF$Jri3lCV^okLeA?M;DoWfzPfx_>yi&v&O;cJ&Ncn@b$v8HVaf#gv=df**3hw= z3J$H0vR9|nw$AV$YwEknxYX)tYioB5<%wTklyaeo+lXo0{kL7=!IxdAfB9f1i*8`{ z4gR5>E4%AzIp#?><@|S@ombefxtr6^{;&4FGbqaR`?4KJ9Tj9|6jY*)4G0JrK*^{m z+MtqiGJrsnl^~f>k)H(7mY_&d$x%UaQb7<9P$Wnx>rB;d?We8(hppY( zrDh7Nd3nS0KKI^p&pr1NYoFWk0=5MOob}vjhfeGzXfEib?pKSR^?C>`^QJB4>S}9; zJ$dpZONTP`?VGw&M@Od9*&h|h$ET`|gOn&C~o}HjkGV9Ns zk1Z2dldWKg4EmGWEXU8;D-Pv2_?-LO)vlp=uEA5nxpP;6qM3JGL~6zuf2G=W zkM(KuerAF;@32b@7;My`!eiAwrmS2;-KQ4n8Oe{I_sFyL2r(8+7A|)rUCE0wIUCzl zr(iTU#?V6xh4aIuoSeZHwC(g4WsE*qZxPo36mryv}C_CGtrDomt+FxwXi13LX|jHrDSK$66g z@#6GmZ+MCNhFC+Hw|7+p_7fS5f4ceUw7JEh&Z6A<(F~o9n>VL_d!*+=b*t%;XJKK{ zK7YP8K<~nZ$3)J{$|Dmd_K8G#>Aa64NcH8nYA%PN+$M#O=m<5L^Mq3hfA}yJd%5iU z=OCJj$3YCJ*kKm=b@Z`%vAm27eP9;No#FTI_fO@T!Dr_=4{PGp>HNsGbas9LE-)na zkHmp5U#y@8Rc;8Cn>y`ZLAu)ZB}07A9wmp;$y@abAesCLY5Bn5pa_|K0uM}H4b+2XlPk4!C-!h$=c1XKfv9HbUp8U)=zKQ$V&1?7ci3ho?j5cjrvgr6C z&XF|qDDja*->UqRqy{e)`kz_@Hg;)=d|rtu8g4_q<9imGPOWAoTgsXL%IVW4u!-ya z*i=Fjz!!DmC%`!sWAe;|;Z!9I?Yr%s7rh|1wdXVF+wLhvv!Zfm4xFqYi)tegor zjM9u#vTxg_Ew+?$$9?+iFhMt?K4tM--#oV?EG#Sm!ORAcRhbpS%0na5Gczqy1NGr? zapD1$j{!GCcJDrhK2KdLHSMPt*~dZQuUBH&9)43(!$htSrQlsGW?UGL>sf&anVOj) zC!|;`y7A~T&Ska~D{amje}7z6QUB>gdB^xc_iK1TE)!e{5q6J63N1Heo;%|rEBGcQ zd42jNt6GO2T6*@aQT$xNcmJ|;DW1v+9ZoD#<0>B#f<-X*tVzC$U*d> zed)?{|5X=9wL7ZWLEUSW!i%Zt2Y%ggH^HNwd0f0~hGu*C5pej`GO^oC+txH4OkQ|) zxiLDG$WUCRAuP8?wQi?xtmn8iMH8$lnSz!3h^3K)+>>8WFdg7R#$zUBg;}vupSVF8 zEo32I-9F`7hN4G>9WUb+t7>s67DCh@=LBp$jEve&Z(w&*jzzsOmG_8!w3u0W2E3)t zwRhlMThZizvvSEhK&gyCaLyWF8Q$xCXVKO0=;(c@H;#RJuB2JZy#q}}m_zp9upK=C zUR}5N%kHdl`Bk^0Jq~YMZnYkUJCi7zp86$o_2MIs&Pise@bdU@Vu;xaopHG*%a)>f z?ZmuFQ_R)glt0u@^H*M0`@~aS_g6{2d8$hQhghYV2pc1VKH@@myPT28LUxy9us?4`%RzY zjZ+G>vebW$7nR&*>-rU_;0fL?Ezi{=?iPLZ+S}Xx$-HUGSIX#o^u(T zlGHCbO?crpa`&RU>?2#LIvN`H7-NYwcVv|Un^ix0y1Sb=oz^fm9?c0f1)dIZ#aT~F z4eqf4YIxqzA@Ug++z7qZrFj^TjtM@5Mm96ktxFR> z7lkDyb3H4AQtED0R2qR&1{(tf$<9-Ab6FZ?sENpUFTCLO`gW`gLsVpZmI}!` z@5djZww#7{Cp;5ERT8vcPrG931d2zy%gm)-f-?*5<|lmhi4!+7bV2hOCW1i%$D>A@ z>9CADz~@itk|v)q6Yn^`uZ)-1sop`sGALhnfLqEiYgxO`+Q8^Xk@S-*g1$Sy_AyFp z9kN)D++iLTT!%+GF91sV88$@H=g$}A7e{kx<{4KGxKEkCsj9MqEpfu)CVMA_hlSv{ z;3x^}7Ktvy=mt*dD+z#5>7`kR625&)>2^r03EsJfvVd$1V*A9=QQD7ns5tyZ;gm=* z&v{+3G$0;$`}T*P)fKn?JlWV-_SM*fyLP>N@bQW7>I&z-I8Xk^m-{pylV6C+#Jz2{oB@h=ceIS4W8qj*>jp(y(dOeJ8%4`tIy}MbMJ5^Hs>ey zo4W)zbJG$8Dp(dtWDjC9^$PSEXFhoms<3dHSbx{i2cV$0{qe_RU|H*dx^TcXO~5h9 zD_>5ZJaOVQZRLB7B6Vfy2XSNq&2$TLOnR|PI&8$$eSrOJtXxqu^RRvACp*z`I(Zmv zv3dbpcwS&24Lb)%9W{>%?1=x15@_cWl>)Xb%rP9mBPXJr)B5|jKII0Brm^8+gK@(| zvkHWM;k|p`W2Bmnwsry-I4k&8S^~OShkzQ1fCImpuz~lQpR~}Kj8*RohKs`jAu)0- ziSqGf6=dtl#q@!XEk;^jwH++V*USeD_l|RWOD%4_w6R&q^Gb_gPQZG_YnCC*MIY}Z z!xz?#b-a$pPOl-6{2m=2Zj~0c3B7YC%W>ciGN9%m9d(b@z{WxDgBtirT7xyRXwCx{ zPxv$9d={+bmuEiO7lda%mYHbpcv{6XNF|>`(@UyjPq8_e5KegyXS+y^$Gx#U+d6ZV zLhe|1=y(K|OmC;F)kfs2{!zKmG9=$ueTukWQg2cLDmkKN<;Er@t~$U&XSeGcQygxn zZ7tXANiiB2`}nLgf6nBWmU7jjiG$fgQ5H(KN4T}}mzn%r7S?oqw~%qYz9Pq~e9Q7>VYgtXSNHZBE$?snor;DXV#~9^?6<^i+q}2|M$i}i0#<<1^qgg&{HxQd8O(4CcGK$1pgJgj-_D~ z8s}=o@0u7JwVSw@{ief@8th8mT>Q~6xDvYhooyLS8#-eA9+DCc-NwK~GLF4|3JXJf zA_JJ~Hgj0vrEtnP4@qYCD-W^=tlz?&0zwCv;N{tWohNJ18%^uG&p`15rc-DrMb1=h zt+*4@Q{t9gzC34yKz|er&B5z_nD}J(>De(fo{M`dNl zbRTAN>0Z8^1dL8V`P%mO0^=AICgQt2n4$m=Rs0Fy^zLdm%PI>G8$5UFSC+JTe-zzT z#r=0=JIg21M#2-s%fBB-D-@9lvj6$#6HFu9xrkDiF`D%>u3X7MF64~nmoxx8_HN7E zD;W64%F6$MMBVsvf#PX>yMPiof=&Wjyayn3`TB0>@?@xQ3uazrqDMImL0ni&tPL8r zr#CKzGief!=4;q`b6A~d55ouY|MQYYI~UJ}6p z*NO9*x=oDFavA;fA+ZhR&k?@W?*VAh7c?oorh&n6b#LFQJ2^cEtMe4S-qxT>382L( zVHFPodg#8I!Ur(56R!G7wze5$9esy?vFFqFTC_ChVbo2gagioX0zAv|(sUEGCFNfE z>WU`-sSSMLCKi^`z*YaO%#B+Gem_GJcvM#TN7zAf?=%3h!E0Q9bR5u!lRLkX5OU?JBGrLg$^9hK=qpQuTib$2Ov(qJmf-BI$vN31isd(@=#p+~(OB8661V-$SlLbrz>`tyy<| z%M9TfUY@Na0NDCyFbRla{FGwnVcKL~K|ulcmH_fFn&pxezW1(0GYSfEcAB^-QsFl} z#!VoS(~yu-GyzK>{2&=?788d_hH(T%2yJggbmfuDlsi6*CBsJw;v z(~SH@M_v7V>1yCL58WjzN8m8zBDl}iM0r8MQy8O=uCP3#C2Cx#j@@~{!)d?xdOwkW zo%yXd7gL?XLouYGU<|wm9#r$%wZiK+ZXh4g*4Nh`m>qR3fth2UciFpbnRYBjuqxsN zk>1J6NL1Yp%5xY=M_q00huZssAs2QV<=cJ$?I#SZ)8)a;)M*o<%#JM6Yw0Fsm_DNk zQ?JisI8gumGgk*t7(sgvreHKl+&RyliC2Q$)If$ulX zdw;oi3VMZPxSYb$BZJTG#L9cZ$j)I>jfRJZhyP%zUUn=xi!DoC^(F}9^jMHUPUhwb zpe6+cY*XWe}B|c7%CJpsm^QE+dTTB<$DJ2XcB&$-1!vkSD1d z8%G~#Ax7`f5@6I6TE0V17Hppiyn-}mKXo5hg!4!X1qGXrb_HEY@b78p0~KS`gm5Jj zE(q9)+L4o(lbg$qMMI~ZKA-Q>a}F`%6u20CB!bNqDQ?z=rW~84o;9QgQdaqZjrvdL;L;vB?};Pz7s3yDio zKPH95dQDNf;Dy86SV`m8w#l}urmZjOvVX_I&BayE)>|n2-OXkB)0ivfe^_CXG>se- zS8#mc*O7eB<2FS(#q}Ec37yNXW8JRmMT&I(@EOC02QIB&jc{rWTZ#7as`j3_`C6xM z@4n`m+={TOd9z-X$se{pUTr3PdeaZ@^zpttTLlD(TDxI|Pg#Vxnb@fd_wv1CQso(2 zgc8)^>CQQ6E|#lwv5zy+uTC_E*1WJ}?2fb}d>KSm)YOCUVe$5Ru%Qp?rt3fj+XlO? z_h)DLbkuZI9{;m!pZ1?BQkxpzYFulx+rK;f=2l28oMo9r=I%GL8^5P%Q5lE~iL3Hx z^kaXr-J*;ZhoVE?_^3FZ6IZWm&X{Cvk9ltU`|j{F*N3@ZYeckC!&|AXxDH^HAJae> zr*fvhqer%oO_ zVSw8updIQ`uYM)Yz-sJt`a*%;1XTYnsxHTanz{W?iutBTu`MN5va04*i`kuB;w<@< zt2$3bbYgg~oY8o6@}{Q3wyF99cV7P$nO`U6d0IbH*ZKHc-QL=<=_7< zep!THe`);P>#$<%1?Ku&o}`p}&Gj2w&8;6dzw)1+5ZQP{xY*SqJLcHUX*S$G>FU&8 zl=ri0*ErvMl~=WvTqnHzu(E=!^WFCqBkxAfHgUzaP4Mjoy_)?A32S*JM*Zby zH2Qz#M7*fHy~#6y8NF=1SirC-V6;L(LKg$7Z~2{A2?3}%^s+J`nICD-j?2(x>QtB_ zIe;>RhzbSmV8bPxK3AoiHsx{f@l8z0fIdbQHb;M7t3nNE=~=WP3mrM4hvs&AzTG2G zf*6;HI*IAPBtFXNu*IhgcDoIvUgbUm1}^AV2X|R$=g?K0=L0b|voC<}@Jabr_Z``F zP&U2*22#0kH$Bt#^ll1J0g8+Wo2hNmIBVFWN9lToxX@_~tg~CR$NbZY{cM$;vDDX@DEk z#^~kVC58p|nV{e!O?UKo&&NQKDW2Y%27-+Es~9kRSb9!4Rz4YMEt zyabm)9HMIQ+1uNj021i&2cfGE#Tw^9^S2IZ4esRxo?vGeI&?@2%Rw+zZCNJiD5x7i zPD|92jvf2k^jhCK`N^uSfuW)85Mcm>*+b#1g{8o{ncR{-6D_4dfF3O5^JV|-rs~I! zU;i-uc(m-djT=9pzMX;zMl&B*Pv5-4G##pgJmihkxl7R@uc#uqJUt<)!4KWFQ0VxmA@=TkN z!_G=+Y=LC_?8o=-L?IBj>(((|s)#ZufX3Z77d3Fls_I&;vzX-@9u{Uky7gxj;Q@7? zdkv}A@PZ5BUjKu+W&KU6;-J96UWO&me*gUG`K73X7J=B-Go4qPh*|`3**xT?c~&2e zW72Oc#)QlcDy|yPW{}>h&p}nFdO-IBV zpbOQ`MMbagjt$Y$T1XM@A!mU~Q=rx*D=2EqJcc{8<7K3R!Yi1+*c2;|;-g(&)8MF1 zFALv4X=pGnN2B9K=ue{#+eJOEI8^FbgMtatNmxk?3K8Uh%D(19Rka;YD|g-sBg_Cx zy>7z>5oB@DLj~TsqYEd3Im;Tuvq*LX@#jud4mQR;c#wjb6SXxp$@KXRBtIMfSh(<( zHI23VVo>~)1cULk;c{*&fk0A{j^qN1p~{hO|18NPJK|r8EQh23G(?Vk7fY zOp2XTo#bf}IKk}j$5S-tl_mE}p2*6A116!V)WvX`7Hl$S6#kKXpMmthoM!a-GuXHz zx3Ym;i_5m7qeRbZrTKzC`I66n;0<*=AKVNL4gIiIKu*pWW9H~!CY*ZGZLlV&?pjF+ z&SSK$wx%ZSz-U#idl~q;z5M)dJ}IDKC@~?y>dV`E)#_}6&oH2#`ZND=@w@ugk6eao zBaxeyV??f`S%Qi*+uMCzGoN@=824T+ap6 zN!=h>lAIQ1tSMRdZzwA(w^`-9Z@!@1;bHT7=Ytk=qmFPDV)NqV;JJxBoEB`l6YoWU zh_oZW4X&L;W-^NeY%eNRZ1OXAiQC$-BLzH?6+{)(_N*8}bNypsAq{W0TAi3S?H>@p zbnsn#RPSB_J{f3x1Y_xlBkSe1x27gbSccvPmq>wS(#W8Ig!b+|O?%ESM@c~$i9;!a^sWj;gibBoMnQ@ijq*26Uo{LJmn8i;3t!W z0?GoE&d{bY%{*XE5|FxVH^H1dljojUI3tHxA*EZL=_PbbJQR1F_@@QY01O;dPinq~7PXBQbl-QAz?+yP!U% z@1S1H?a=R9`!5iC5M9E%88onlvhuAhymBc^OI|KWMv>JME(5RtMb|{x@j|5-85bJJ zn3bhLMJ9L!8jOt4Ie5dff&c!Q^&&J-n!(3>85_$)@stTdL^3V;h4j_00Ep*0IywkK z>T*Mrg`Ausrh*U#9o2$FCroF;1~O~{f-?w#)+aaTFOisxp;Z)Sppz!c?4DM3Hs98uAX+0NFB0B^8X(Y6Knck#rnPf0@aSOIl9>6#^;Q&&H~P7hM9RLVA^TqPz3GI zbTRx}`<;D(4WeS6$$IVq||2qy~!gal0e289V`Loq{Q+dRB_OJ-#`3D@}pO|UBp^kA}T5)RV|wedRFzTT-bzM?Q5GKXBe|` z9Z9M4))PJRp)W&jY-QyXXlx2?rbOAa*k*>ORfZQ!!}T)_4LtjB`h;ENd8x4Qbnn)o zzCTtY-_XmgLzP4LGB#AR=|b~%uz8;Oa(s_r%!9PoOIjXFZ~kb1OkA9UNC*lqwhoJG zI4?x|d@3~BIlTCV@BJAQk6%Y3zBP(1M$2C{U39Y$xy0~^kv2)-u#2*Jx!gykzpJsR z(AlgNoUP>Hq8({J7BaVI(f45amhp~Pc6R9;Nmh$V@|=sE`&kPW-&s}gm*(1UoXEhn zg>83|E(jD7sr_A}sDX-yTX-qg5pCuc(JvkG<=wuL8hch9 zBgef9E1sJzajoi>EzA;+1TUvoH)s_0u-r`RtK8%Drm->+Mk(!rrRiB^vO>F_`8P@h zhJ9~T$s7K3Tn!pQnse5WSNQCbnPC#$ZKLDHy(%J%8?Oy~k9zv%jd90EuRS&U<5X?2 zXj(w5mvtC<_~_R^Op32j7Tqp0ts0%Wq!S7fXx}cnTueKT6`I!dp+9%erbYTD4-MW* zwXC=+QXB^#HAHOqmy)@Cs*mstE#G&t3B{M`&{}e0DpuyR@tZRv0=rrz$=r(9Wpu9?c&32vPv!K ztD<B+I-;~r;%e67jsvn7q{~J&hZdi=O4aBFJtCgQ1f+9S9Pq- z*mwKv^&vwxH`!?0c%6cDS40f*TTGO*Z{^5y(bugjEZq`1l*Kjs1Nrlb_OIP&Acy>uv5a?=i;1`j@NC+8h`DB2P_L1#a6YY+0K)%PwM`;hV+uPToyP+MEApq@K^G(Derw$ zXkj@QgEVz*g$kNJ?5PRKZucqH$e(>E4y;)pRck$};-x8BYnLCv_ zb#v8;d&`^i{cx6UW@FK=bDwxx={*L%kM4*X(-|&88`badiH4Eg4}WVZdJ;F#LvQ*t zSS4PgaAcIVU{K+L+gr}wL02viO^1GRNpFOWy7mm#Jf$U+)XZqjmmgyvFEH=cQ!#jc z@+s_e(1@N&ImCzeXf(dY_fz39&Z*PB%g?N{b6mMwk~f;!EPgF(HN8;9NH#YOzP_|I zd>nW;zu4I(+hbNPP3lsu>%9D5f&4mNzMIC9iGAhxv$ANswZTv#T>Id@i!&c@h;)kY z@s9Ig`>nNi!{Samn3hXPw&hwi|5Ke$zWv9jC9gNsl z@QQ*LX82=LhLyXm=E0{!gGb9>?usc$aGL;v>!ZkTXRUemY^Sew-fN_od-kfBZ zhtQj9`qLxYp9%5eq{4U9bw?(b+LnBXy;p9QTfJIop%B!sxJ)nUt8xlx+js6hnTP)9 zY0!4D;TuOHha9vN9Ho8x1wyFCAN>|9TVE8NRQ>9mdWI1y#%R`Mynr3F;4>xO;cQ%U z)5sqwm5WzQ=49at;i^PZEH_YS$$HCI+J~JypTUgZXW1QSQo$zHw@{Z&Ynt%hMZEIn&{J{dciklDd6`o-;g#Gp4nbo@>R`ES7R+ zhCa{h*>nvWy|7^_EcYqa>>Zyy86~;GH`0AX-?AzCknW1(+!HHCQ8;1sZlD~^U3L~fk2gr)iCYM3mn_Ujho>%a#pqDzltum zq?J|FQ~79N5U*~S|B~|1S1XVCXFP<8VUc$25HSZcqu~p#uzeL7e5uhopOJl-$0)@@ z)^|CpFS}Bdh$MoW)#~YWdIdM#QeR~CpHdr2u5=|HAi^#x{p6#|)YbYroe^vBnK72^CK!;5~)7{iksot56y*y6g8h;uB&v%0)RmC2?A&+cb) zs}J!XU;iUyR+ld6GG*}huD_gT`13IGPdx>Wy8qwt*Z&Ttc**x+&X*`}^HPGLCs9tG LJCSw#{Pq6^ko_$R diff --git a/java/infra/charts/feast-core/Chart.yaml b/java/infra/charts/feast-core/Chart.yaml deleted file mode 100644 index ac52a3259c..0000000000 --- a/java/infra/charts/feast-core/Chart.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -description: "Feast Core: Feature registry for Feast." -name: feast-core -version: 0.26.2 -appVersion: 0.26.2 -keywords: -- machine learning -- big data -- mlops -home: https://github.com/feast-dev/feast-java diff --git a/java/infra/charts/feast-core/README.md b/java/infra/charts/feast-core/README.md deleted file mode 100644 index bae6786028..0000000000 --- a/java/infra/charts/feast-core/README.md +++ /dev/null @@ -1,68 +0,0 @@ -feast-core -========== -Feast Core: Feature registry for Feast. - -Current chart version is `0.26.2` - -Source code can be found [here](https://github.com/feast-dev/feast-java) - - - -## Chart Values - -| Key | Type | Default | Description | -|-----|------|---------|-------------| -| "application-generated.yaml".enabled | bool | `true` | Flag to include Helm generated configuration for http port, Feast database URL, Kafka bootstrap servers and jobs metrics host. This is useful for deployment that uses default configuration for Kafka, Postgres and StatsD exporter. Please set `application-override.yaml` to override this configuration. | -| "application-override.yaml" | object | `{"enabled":true}` | Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/core/src/main/resources/application.yml). Will be created as a ConfigMap. `application-override.yaml` has a higher precedence than `application-secret.yaml` | -| "application-secret.yaml" | object | `{"enabled":true}` | Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/core/src/main/resources/application.yml). Will be created as a Secret. `application-override.yaml` has a higher precedence than `application-secret.yaml`. It is recommended to either set `application-override.yaml` or `application-secret.yaml` only to simplify config management. | -| "application.yaml".enabled | bool | `true` | Flag to include the default [configuration](https://github.com/feast-dev/feast/blob/master/core/src/main/resources/application.yml). Please set `application-override.yaml` to override this configuration. | -| envOverrides | object | `{}` | Extra environment variables to set | -| image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | -| image.repository | string | `"gcr.io/kf-feast/feast-core"` | Docker image repository | -| image.tag | string | `"develop"` | Image tag | -| ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress | -| ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth | -| ingress.grpc.class | string | `"nginx"` | Which ingress controller to use | -| ingress.grpc.enabled | bool | `false` | Flag to create an ingress resource for the service | -| ingress.grpc.hosts | list | `[]` | List of hostnames to match when routing requests | -| ingress.grpc.https.enabled | bool | `true` | Flag to enable HTTPS | -| ingress.grpc.https.secretNames | object | `{}` | Map of hostname to TLS secret name | -| ingress.grpc.whitelist | string | `""` | Allowed client IP source ranges | -| ingress.http.annotations | object | `{}` | Extra annotations for the ingress | -| ingress.http.auth.authUrl | string | `"http://auth-server.auth-ns.svc.cluster.local/auth"` | URL to an existing authentication service | -| ingress.http.auth.enabled | bool | `false` | Flag to enable auth | -| ingress.http.class | string | `"nginx"` | Which ingress controller to use | -| ingress.http.enabled | bool | `false` | Flag to create an ingress resource for the service | -| ingress.http.hosts | list | `[]` | List of hostnames to match when routing requests | -| ingress.http.https.enabled | bool | `true` | Flag to enable HTTPS | -| ingress.http.https.secretNames | object | `{}` | Map of hostname to TLS secret name | -| ingress.http.whitelist | string | `""` | Allowed client IP source ranges | -| javaOpts | string | `nil` | [JVM options](https://docs.oracle.com/cd/E22289_01/html/821-1274/configuring-the-default-jvm-and-java-arguments.html). For better performance, it is advised to set the min and max heap:
`-Xms2048m -Xmx2048m` | -| livenessProbe.enabled | bool | `false` | Flag to enabled the probe | -| livenessProbe.failureThreshold | int | `5` | Min consecutive failures for the probe to be considered failed | -| livenessProbe.initialDelaySeconds | int | `60` | Delay before the probe is initiated | -| livenessProbe.periodSeconds | int | `10` | How often to perform the probe | -| livenessProbe.successThreshold | int | `1` | Min consecutive success for the probe to be considered successful | -| livenessProbe.timeoutSeconds | int | `5` | When the probe times out | -| logLevel | string | `"WARN"` | Default log level, use either one of `DEBUG`, `INFO`, `WARN` or `ERROR` | -| logType | string | `"Console"` | Log format, either `JSON` or `Console` | -| nodeSelector | object | `{}` | Node labels for pod assignment | -| podLabels | object | `{}` | Labels to be added to Feast Core pods | -| postgresql.existingSecret | string | `""` | Existing secret to use for authenticating to Postgres | -| prometheus.enabled | bool | `true` | Flag to enable scraping of Feast Core metrics | -| readinessProbe.enabled | bool | `true` | Flag to enabled the probe | -| readinessProbe.failureThreshold | int | `5` | Min consecutive failures for the probe to be considered failed | -| readinessProbe.initialDelaySeconds | int | `20` | Delay before the probe is initiated | -| readinessProbe.periodSeconds | int | `10` | How often to perform the probe | -| readinessProbe.successThreshold | int | `1` | Min consecutive success for the probe to be considered successful | -| readinessProbe.timeoutSeconds | int | `10` | When the probe times out | -| replicaCount | int | `1` | Number of pods that will be created | -| resources | object | `{}` | CPU/memory [resource requests/limit](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container) | -| secrets | list | `[]` | List of Kubernetes secrets to be mounted on Feast Core pods. These secrets will be mounted on /etc/secrets/. | -| service.grpc.nodePort | string | `nil` | Port number that each cluster node will listen to | -| service.grpc.port | int | `6565` | Service port for GRPC requests | -| service.grpc.targetPort | int | `6565` | Container port serving GRPC requests | -| service.http.nodePort | string | `nil` | Port number that each cluster node will listen to | -| service.http.port | int | `80` | Service port for HTTP requests | -| service.http.targetPort | int | `8080` | Container port serving HTTP requests and Prometheus metrics | -| service.type | string | `"ClusterIP"` | Kubernetes service type | diff --git a/java/infra/charts/feast-core/templates/_helpers.tpl b/java/infra/charts/feast-core/templates/_helpers.tpl deleted file mode 100644 index 7c095e11c4..0000000000 --- a/java/infra/charts/feast-core/templates/_helpers.tpl +++ /dev/null @@ -1,45 +0,0 @@ -{{/* vim: set filetype=mustache: */}} -{{/* -Expand the name of the chart. -*/}} -{{- define "feast-core.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "feast-core.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "feast-core.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Common labels -*/}} -{{- define "feast-core.labels" -}} -app.kubernetes.io/name: {{ include "feast-core.name" . }} -helm.sh/chart: {{ include "feast-core.chart" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end -}} diff --git a/java/infra/charts/feast-core/templates/_ingress.yaml b/java/infra/charts/feast-core/templates/_ingress.yaml deleted file mode 100644 index 5bed6df047..0000000000 --- a/java/infra/charts/feast-core/templates/_ingress.yaml +++ /dev/null @@ -1,68 +0,0 @@ -{{- /* -This takes an array of three values: -- the top context -- the feast component -- the service protocol -- the ingress context -*/ -}} -{{- define "feast.ingress" -}} -{{- $top := (index . 0) -}} -{{- $component := (index . 1) -}} -{{- $protocol := (index . 2) -}} -{{- $ingressValues := (index . 3) -}} -apiVersion: extensions/v1beta1 -kind: Ingress -{{ include "feast.ingress.metadata" . }} -spec: - rules: - {{- range $host := $ingressValues.hosts }} - - host: {{ $host }} - http: - paths: - - path: / - backend: - serviceName: {{ include (printf "feast-%s.fullname" $component) $top }} - servicePort: {{ index $top.Values "service" $protocol "port" }} - {{- end }} -{{- if $ingressValues.https.enabled }} - tls: - {{- range $host := $ingressValues.hosts }} - - secretName: {{ index $ingressValues.https.secretNames $host | default (splitList "." $host | rest | join "-" | printf "%s-tls") }} - hosts: - - {{ $host }} - {{- end }} -{{- end -}} -{{- end -}} - -{{- define "feast.ingress.metadata" -}} -{{- $commonMetadata := fromYaml (include "common.metadata" (first .)) }} -{{- $overrides := fromYaml (include "feast.ingress.metadata-overrides" .) -}} -{{- toYaml (merge $overrides $commonMetadata) -}} -{{- end -}} - -{{- define "feast.ingress.metadata-overrides" -}} -{{- $top := (index . 0) -}} -{{- $component := (index . 1) -}} -{{- $protocol := (index . 2) -}} -{{- $ingressValues := (index . 3) -}} -{{- $commonFullname := include "common.fullname" $top }} -metadata: - name: {{ $commonFullname }}-{{ $component }}-{{ $protocol }} - annotations: - kubernetes.io/ingress.class: {{ $ingressValues.class | quote }} - {{- if (and (eq $ingressValues.class "nginx") $ingressValues.auth.enabled) }} - nginx.ingress.kubernetes.io/auth-url: {{ $ingressValues.auth.authUrl | quote }} - nginx.ingress.kubernetes.io/auth-response-headers: "x-auth-request-email, x-auth-request-user" - nginx.ingress.kubernetes.io/auth-signin: "https://{{ $ingressValues.auth.signinHost | default (splitList "." (index $ingressValues.hosts 0) | rest | join "." | printf "auth.%s")}}/oauth2/start?rd=/r/$host/$request_uri" - {{- end }} - {{- if (and (eq $ingressValues.class "nginx") $ingressValues.whitelist) }} - nginx.ingress.kubernetes.io/whitelist-source-range: {{ $ingressValues.whitelist | quote -}} - {{- end }} - {{- if (and (eq $ingressValues.class "nginx") (eq $protocol "grpc") ) }} - # TODO: Allow choice of GRPC/GRPCS - nginx.ingress.kubernetes.io/backend-protocol: "GRPC" - {{- end }} - {{- if $ingressValues.annotations -}} - {{ include "common.annote" $ingressValues.annotations | indent 4 }} - {{- end }} -{{- end -}} diff --git a/java/infra/charts/feast-core/templates/configmap.yaml b/java/infra/charts/feast-core/templates/configmap.yaml deleted file mode 100644 index 7e0185ae6a..0000000000 --- a/java/infra/charts/feast-core/templates/configmap.yaml +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ template "feast-core.fullname" . }} - namespace: {{ .Release.Namespace }} - labels: - app: {{ template "feast-core.name" . }} - component: core - chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -data: - application-generated.yaml: | -{{- if index .Values "application-generated.yaml" "enabled" }} - spring: - datasource: - url: jdbc:postgresql://{{ .Release.Name }}-postgresql:5432/postgres - - server: - port: {{ .Values.service.http.targetPort }} -{{- end }} - - application-override.yaml: | -{{- if index .Values "application-override.yaml" "enabled" }} -{{- toYaml (index .Values "application-override.yaml") | nindent 4 }} -{{- end }} \ No newline at end of file diff --git a/java/infra/charts/feast-core/templates/deployment.yaml b/java/infra/charts/feast-core/templates/deployment.yaml deleted file mode 100644 index a24672d940..0000000000 --- a/java/infra/charts/feast-core/templates/deployment.yaml +++ /dev/null @@ -1,147 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ template "feast-core.fullname" . }} - namespace: {{ .Release.Namespace }} - labels: - app: {{ template "feast-core.name" . }} - component: core - chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -spec: - replicas: {{ .Values.replicaCount }} - selector: - matchLabels: - app: {{ template "feast-core.name" . }} - component: core - release: {{ .Release.Name }} - template: - metadata: - annotations: - checksum/configmap: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} - checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} - {{- if .Values.prometheus.enabled }} - prometheus.io/path: /metrics - prometheus.io/port: "{{ .Values.service.http.targetPort }}" - prometheus.io/scrape: "true" - {{- end }} - labels: - app: {{ template "feast-core.name" . }} - component: core - release: {{ .Release.Name }} - {{- if .Values.podLabels }} - {{ toYaml .Values.podLabels | nindent 8 }} - {{- end }} - spec: - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - - volumes: - - name: {{ template "feast-core.fullname" . }}-config - configMap: - name: {{ template "feast-core.fullname" . }} - - name: {{ template "feast-core.fullname" . }}-secret - secret: - secretName: {{ template "feast-core.fullname" . }} - {{- range $secret := .Values.secrets }} - - name: {{ $secret }} - secret: - secretName: {{ $secret }} - {{- end }} - - containers: - - name: {{ .Chart.Name }} - image: {{ .Values.image.repository }}:{{ .Values.image.tag }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - - volumeMounts: - - name: {{ template "feast-core.fullname" . }}-config - mountPath: /etc/feast - - name: {{ template "feast-core.fullname" . }}-secret - mountPath: /etc/secrets/feast - readOnly: true - {{- range $secret := .Values.secrets }} - - name: {{ $secret }} - mountPath: "/etc/secrets/{{ $secret }}" - readOnly: true - {{- end }} - - env: - - name: LOG_TYPE - value: {{ .Values.logType | quote }} - - name: LOG_LEVEL - value: {{ .Values.logLevel | quote }} - - {{- if .Values.postgresql.existingSecret }} - - name: SPRING_DATASOURCE_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .Values.postgresql.existingSecret }} - key: postgresql-password - {{- end }} - - {{- if .Values.javaOpts }} - - name: JAVA_TOOL_OPTIONS - value: {{ .Values.javaOpts }} - {{- end }} - - {{- range $key, $value := .Values.envOverrides }} - - name: {{ printf "%s" $key | replace "." "_" | upper | quote }} - {{- if eq (kindOf $value) "map" }} - valueFrom: - {{- toYaml $value | nindent 12}} - {{- else }} - value: {{ $value | quote }} - {{- end}} - {{- end }} - - command: - - java - - -jar - - /opt/feast/feast-core.jar - - --spring.config.location= - {{- if index .Values "application.yaml" "enabled" -}} - classpath:/application.yml - {{- end }} - {{- if index .Values "application-generated.yaml" "enabled" -}} - ,file:/etc/feast/application-generated.yaml - {{- end }} - {{- if index .Values "application-secret.yaml" "enabled" -}} - ,file:/etc/secrets/feast/application-secret.yaml - {{- end }} - {{- if index .Values "application-override.yaml" "enabled" -}} - ,file:/etc/feast/application-override.yaml - {{- end }} - ports: - - name: http - containerPort: {{ .Values.service.http.targetPort }} - - name: grpc - containerPort: {{ .Values.service.grpc.targetPort }} - - {{- if .Values.livenessProbe.enabled }} - livenessProbe: - exec: - command: ["/usr/bin/grpc-health-probe", "-addr=:{{ .Values.service.grpc.targetPort }}"] - initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} - periodSeconds: {{ .Values.livenessProbe.periodSeconds }} - successThreshold: {{ .Values.livenessProbe.successThreshold }} - timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} - failureThreshold: {{ .Values.livenessProbe.failureThreshold }} - {{- end }} - - {{- if .Values.readinessProbe.enabled }} - readinessProbe: - exec: - command: ["/usr/bin/grpc-health-probe", "-addr=:{{ .Values.service.grpc.targetPort }}"] - initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} - periodSeconds: {{ .Values.readinessProbe.periodSeconds }} - successThreshold: {{ .Values.readinessProbe.successThreshold }} - timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} - failureThreshold: {{ .Values.readinessProbe.failureThreshold }} - {{- end }} - - resources: - {{- toYaml .Values.resources | nindent 10 }} diff --git a/java/infra/charts/feast-core/templates/ingress.yaml b/java/infra/charts/feast-core/templates/ingress.yaml deleted file mode 100644 index 7f453e1a75..0000000000 --- a/java/infra/charts/feast-core/templates/ingress.yaml +++ /dev/null @@ -1,7 +0,0 @@ -{{- if .Values.ingress.http.enabled -}} -{{ template "feast.ingress" (list . "core" "http" .Values.ingress.http) }} -{{- end }} ---- -{{ if .Values.ingress.grpc.enabled -}} -{{ template "feast.ingress" (list . "core" "grpc" .Values.ingress.grpc) }} -{{- end }} diff --git a/java/infra/charts/feast-core/templates/secret.yaml b/java/infra/charts/feast-core/templates/secret.yaml deleted file mode 100644 index dd33e2dd48..0000000000 --- a/java/infra/charts/feast-core/templates/secret.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: {{ template "feast-core.fullname" . }} - namespace: {{ .Release.Namespace }} - labels: - app: {{ template "feast-core.name" . }} - component: core - chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -type: Opaque -stringData: - application-secret.yaml: | -{{- toYaml (index .Values "application-secret.yaml") | nindent 4 }} diff --git a/java/infra/charts/feast-core/templates/service.yaml b/java/infra/charts/feast-core/templates/service.yaml deleted file mode 100644 index 1561398337..0000000000 --- a/java/infra/charts/feast-core/templates/service.yaml +++ /dev/null @@ -1,41 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ template "feast-core.fullname" . }} - namespace: {{ .Release.Namespace }} - labels: - app: {{ template "feast-core.name" . }} - chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} - {{- with .Values.service.annotations }} - annotations: - {{ toYaml . | nindent 4 }} - {{- end }} -spec: - type: {{ .Values.service.type }} - {{- if .Values.service.loadBalancerIP }} - loadBalancerIP: {{ .Values.service.loadBalancerIP }} - {{- end }} - {{- if .Values.service.loadBalancerSourceRanges }} - loadBalancerSourceRanges: - {{ toYaml .Values.service.loadBalancerSourceRanges | nindent 2 }} - {{- end }} - ports: - - name: http - port: {{ .Values.service.http.port }} - targetPort: {{ .Values.service.http.targetPort }} - {{- if .Values.service.http.nodePort }} - nodePort: {{ .Values.service.http.nodePort }} - {{- end }} - - name: grpc - port: {{ .Values.service.grpc.port }} - targetPort: {{ .Values.service.grpc.targetPort }} - {{- if .Values.service.grpc.nodePort }} - nodePort: {{ .Values.service.grpc.nodePort }} - {{- end }} - selector: - app: {{ template "feast-core.name" . }} - component: core - release: {{ .Release.Name }} - diff --git a/java/infra/charts/feast-core/values.yaml b/java/infra/charts/feast-core/values.yaml deleted file mode 100644 index 423373e61c..0000000000 --- a/java/infra/charts/feast-core/values.yaml +++ /dev/null @@ -1,147 +0,0 @@ -# replicaCount -- Number of pods that will be created -replicaCount: 1 - -image: - # image.repository -- Docker image repository - repository: gcr.io/kf-feast/feast-core - # image.tag -- Image tag - tag: develop - # image.pullPolicy -- Image pull policy - pullPolicy: IfNotPresent - -application.yaml: - # "application.yaml".enabled -- Flag to include the default [configuration](https://github.com/feast-dev/feast/blob/master/core/src/main/resources/application.yml). Please set `application-override.yaml` to override this configuration. - enabled: true - -application-generated.yaml: - # "application-generated.yaml".enabled -- Flag to include Helm generated configuration for http port, Feast database URL, Kafka bootstrap servers and jobs metrics host. This is useful for deployment that uses default configuration for Kafka, Postgres and StatsD exporter. Please set `application-override.yaml` to override this configuration. - enabled: true - -# "application-secret.yaml" -- Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/core/src/main/resources/application.yml). Will be created as a Secret. `application-override.yaml` has a higher precedence than `application-secret.yaml`. It is recommended to either set `application-override.yaml` or `application-secret.yaml` only to simplify config management. -application-secret.yaml: - enabled: true - -# "application-override.yaml" -- Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/core/src/main/resources/application.yml). Will be created as a ConfigMap. `application-override.yaml` has a higher precedence than `application-secret.yaml` -application-override.yaml: - enabled: true - -postgresql: - # postgresql.existingSecret -- Existing secret to use for authenticating to Postgres - existingSecret: "" - -# javaOpts -- [JVM options](https://docs.oracle.com/cd/E22289_01/html/821-1274/configuring-the-default-jvm-and-java-arguments.html). For better performance, it is advised to set the min and max heap:
`-Xms2048m -Xmx2048m` -javaOpts: - -# logType -- Log format, either `JSON` or `Console` -logType: Console -# logLevel -- Default log level, use either one of `DEBUG`, `INFO`, `WARN` or `ERROR` -logLevel: WARN - -prometheus: - # prometheus.enabled -- Flag to enable scraping of Feast Core metrics - enabled: true - -# By default we disable the liveness probe, since if the DB fails restarting core will not result -# in application healing. -livenessProbe: - # livenessProbe.enabled -- Flag to enabled the probe - enabled: false - # livenessProbe.initialDelaySeconds -- Delay before the probe is initiated - initialDelaySeconds: 60 - # livenessProbe.periodSeconds -- How often to perform the probe - periodSeconds: 10 - # livenessProbe.timeoutSeconds -- When the probe times out - timeoutSeconds: 5 - # livenessProbe.successThreshold -- Min consecutive success for the probe to be considered successful - successThreshold: 1 - # livenessProbe.failureThreshold -- Min consecutive failures for the probe to be considered failed - failureThreshold: 5 - -readinessProbe: - # readinessProbe.enabled -- Flag to enabled the probe - enabled: true - # readinessProbe.initialDelaySeconds -- Delay before the probe is initiated - initialDelaySeconds: 20 - # readinessProbe.periodSeconds -- How often to perform the probe - periodSeconds: 10 - # readinessProbe.timeoutSeconds -- When the probe times out - timeoutSeconds: 10 - # readinessProbe.successThreshold -- Min consecutive success for the probe to be considered successful - successThreshold: 1 - # readinessProbe.failureThreshold -- Min consecutive failures for the probe to be considered failed - failureThreshold: 5 - -service: - # service.type -- Kubernetes service type - type: ClusterIP - http: - # service.http.port -- Service port for HTTP requests - port: 80 - # service.http.targetPort -- Container port serving HTTP requests and Prometheus metrics - targetPort: 8080 - # service.http.nodePort -- Port number that each cluster node will listen to - nodePort: - grpc: - # service.grpc.port -- Service port for GRPC requests - port: 6565 - # service.grpc.targetPort -- Container port serving GRPC requests - targetPort: 6565 - # service.grpc.nodePort -- Port number that each cluster node will listen to - nodePort: - -ingress: - grpc: - # ingress.grpc.enabled -- Flag to create an ingress resource for the service - enabled: false - # ingress.grpc.class -- Which ingress controller to use - class: nginx - # ingress.grpc.hosts -- List of hostnames to match when routing requests - hosts: [] - # ingress.grpc.annotations -- Extra annotations for the ingress - annotations: {} - https: - # ingress.grpc.https.enabled -- Flag to enable HTTPS - enabled: true - # ingress.grpc.https.secretNames -- Map of hostname to TLS secret name - secretNames: {} - # ingress.grpc.whitelist -- Allowed client IP source ranges - whitelist: "" - auth: - # ingress.grpc.auth.enabled -- Flag to enable auth - enabled: false - http: - # ingress.http.enabled -- Flag to create an ingress resource for the service - enabled: false - # ingress.http.class -- Which ingress controller to use - class: nginx - # ingress.http.hosts -- List of hostnames to match when routing requests - hosts: [] - # ingress.http.annotations -- Extra annotations for the ingress - annotations: {} - https: - # ingress.http.https.enabled -- Flag to enable HTTPS - enabled: true - # ingress.http.https.secretNames -- Map of hostname to TLS secret name - secretNames: {} - # ingress.http.whitelist -- Allowed client IP source ranges - whitelist: "" - auth: - # ingress.http.auth.enabled -- Flag to enable auth - enabled: false - # ingress.http.auth.authUrl -- URL to an existing authentication service - authUrl: http://auth-server.auth-ns.svc.cluster.local/auth - -# resources -- CPU/memory [resource requests/limit](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container) -resources: {} - -# nodeSelector -- Node labels for pod assignment -nodeSelector: {} - -# envOverrides -- Extra environment variables to set -envOverrides: {} - -# secrets -- List of Kubernetes secrets to be mounted on Feast Core pods. These secrets will be mounted on /etc/secrets/. -secrets: [] - -# podLabels -- Labels to be added to Feast Core pods -podLabels: {} diff --git a/java/infra/charts/feast-serving/Chart.yaml b/java/infra/charts/feast-serving/Chart.yaml deleted file mode 100644 index e9a4c417b3..0000000000 --- a/java/infra/charts/feast-serving/Chart.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -description: "Feast Serving: Online feature serving service for Feast" -name: feast-serving -version: 0.26.2 -appVersion: 0.26.2 -keywords: -- machine learning -- big data -- mlops -home: https://github.com/feast-dev/feast-java diff --git a/java/infra/charts/feast-serving/README.md b/java/infra/charts/feast-serving/README.md deleted file mode 100644 index 66a982c529..0000000000 --- a/java/infra/charts/feast-serving/README.md +++ /dev/null @@ -1,71 +0,0 @@ -feast-serving -============= -Feast Serving: Online feature serving service for Feast - -Current chart version is `0.26.2` - -Source code can be found [here](https://github.com/feast-dev/feast-java) - - - -## Chart Values - -| Key | Type | Default | Description | -|-----|------|---------|-------------| -| "application-generated.yaml".enabled | bool | `true` | Flag to include Helm generated configuration for http port, Feast Core host, Redis store and job store. This is useful for deployment that uses default configuration for Redis. Please set `application-override.yaml` to override this configuration. | -| "application-override.yaml" | object | `{"enabled":true}` | Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/serving/src/main/resources/application.yml). Will be created as a ConfigMap. `application-override.yaml` has a higher precedence than `application-secret.yaml` | -| "application-secret.yaml" | object | `{"enabled":true}` | Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/serving/src/main/resources/application.yml). Will be created as a Secret. `application-override.yaml` has a higher precedence than `application-secret.yaml`. It is recommended to either set `application-override.yaml` or `application-secret.yaml` only to simplify config management. | -| "application.yaml".enabled | bool | `true` | Flag to include the default [configuration](https://github.com/feast-dev/feast/blob/master/serving/src/main/resources/application.yml). Please set `application-override.yaml` to override this configuration. | -| envOverrides | object | `{}` | Extra environment variables to set | -| gcpProjectId | string | `""` | Project ID to use when using Google Cloud services such as BigQuery, Cloud Storage and Dataflow | -| gcpServiceAccount.enabled | bool | `false` | Flag to use [service account](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) JSON key Cloud service account JSON key file. | -| gcpServiceAccount.existingSecret.key | string | `"credentials.json"` | Key in the secret data (file name of the service account) | -| gcpServiceAccount.existingSecret.name | string | `"feast-gcp-service-account"` | Name of the existing secret containing the service account | -| image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | -| image.repository | string | `"gcr.io/kf-feast/feast-serving"` | Docker image repository | -| image.tag | string | `"develop"` | Image tag | -| ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress | -| ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth | -| ingress.grpc.class | string | `"nginx"` | Which ingress controller to use | -| ingress.grpc.enabled | bool | `false` | Flag to create an ingress resource for the service | -| ingress.grpc.hosts | list | `[]` | List of hostnames to match when routing requests | -| ingress.grpc.https.enabled | bool | `true` | Flag to enable HTTPS | -| ingress.grpc.https.secretNames | object | `{}` | Map of hostname to TLS secret name | -| ingress.grpc.whitelist | string | `""` | Allowed client IP source ranges | -| ingress.http.annotations | object | `{}` | Extra annotations for the ingress | -| ingress.http.auth.authUrl | string | `"http://auth-server.auth-ns.svc.cluster.local/auth"` | URL to an existing authentication service | -| ingress.http.auth.enabled | bool | `false` | Flag to enable auth | -| ingress.http.class | string | `"nginx"` | Which ingress controller to use | -| ingress.http.enabled | bool | `false` | Flag to create an ingress resource for the service | -| ingress.http.hosts | list | `[]` | List of hostnames to match when routing requests | -| ingress.http.https.enabled | bool | `true` | Flag to enable HTTPS | -| ingress.http.https.secretNames | object | `{}` | Map of hostname to TLS secret name | -| ingress.http.whitelist | string | `""` | Allowed client IP source ranges | -| javaOpts | string | `nil` | [JVM options](https://docs.oracle.com/cd/E22289_01/html/821-1274/configuring-the-default-jvm-and-java-arguments.html). For better performance, it is advised to set the min and max heap:
`-Xms2048m -Xmx2048m` | -| livenessProbe.enabled | bool | `true` | Flag to enabled the probe | -| livenessProbe.failureThreshold | int | `5` | Min consecutive failures for the probe to be considered failed | -| livenessProbe.initialDelaySeconds | int | `60` | Delay before the probe is initiated | -| livenessProbe.periodSeconds | int | `10` | How often to perform the probe | -| livenessProbe.successThreshold | int | `1` | Min consecutive success for the probe to be considered successful | -| livenessProbe.timeoutSeconds | int | `5` | When the probe times out | -| logLevel | string | `"WARN"` | Default log level, use either one of `DEBUG`, `INFO`, `WARN` or `ERROR` | -| logType | string | `"Console"` | Log format, either `JSON` or `Console` | -| nodeSelector | object | `{}` | Node labels for pod assignment | -| podLabels | object | `{}` | Labels to be added to Feast Serving pods | -| prometheus.enabled | bool | `true` | Flag to enable scraping of Feast Core metrics | -| readinessProbe.enabled | bool | `true` | Flag to enabled the probe | -| readinessProbe.failureThreshold | int | `5` | Min consecutive failures for the probe to be considered failed | -| readinessProbe.initialDelaySeconds | int | `15` | Delay before the probe is initiated | -| readinessProbe.periodSeconds | int | `10` | How often to perform the probe | -| readinessProbe.successThreshold | int | `1` | Min consecutive success for the probe to be considered successful | -| readinessProbe.timeoutSeconds | int | `10` | When the probe times out | -| replicaCount | int | `1` | Number of pods that will be created | -| resources | object | `{}` | CPU/memory [resource requests/limit](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container) | -| secrets | list | `[]` | List of Kubernetes secrets to be mounted on Feast Core pods. These secrets will be mounted on /etc/secrets/. | -| service.grpc.nodePort | string | `nil` | Port number that each cluster node will listen to | -| service.grpc.port | int | `6566` | Service port for GRPC requests | -| service.grpc.targetPort | int | `6566` | Container port serving GRPC requests | -| service.http.nodePort | string | `nil` | Port number that each cluster node will listen to | -| service.http.port | int | `80` | Service port for HTTP requests | -| service.http.targetPort | int | `8080` | Container port serving HTTP requests and Prometheus metrics | -| service.type | string | `"ClusterIP"` | Kubernetes service type | diff --git a/java/infra/charts/feast-serving/templates/_helpers.tpl b/java/infra/charts/feast-serving/templates/_helpers.tpl deleted file mode 100644 index 49abb6b8e5..0000000000 --- a/java/infra/charts/feast-serving/templates/_helpers.tpl +++ /dev/null @@ -1,45 +0,0 @@ -{{/* vim: set filetype=mustache: */}} -{{/* -Expand the name of the chart. -*/}} -{{- define "feast-serving.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "feast-serving.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "feast-serving.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Common labels -*/}} -{{- define "feast-serving.labels" -}} -app.kubernetes.io/name: {{ include "feast-serving.name" . }} -helm.sh/chart: {{ include "feast-serving.chart" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end -}} diff --git a/java/infra/charts/feast-serving/templates/_ingress.yaml b/java/infra/charts/feast-serving/templates/_ingress.yaml deleted file mode 100644 index 5bed6df047..0000000000 --- a/java/infra/charts/feast-serving/templates/_ingress.yaml +++ /dev/null @@ -1,68 +0,0 @@ -{{- /* -This takes an array of three values: -- the top context -- the feast component -- the service protocol -- the ingress context -*/ -}} -{{- define "feast.ingress" -}} -{{- $top := (index . 0) -}} -{{- $component := (index . 1) -}} -{{- $protocol := (index . 2) -}} -{{- $ingressValues := (index . 3) -}} -apiVersion: extensions/v1beta1 -kind: Ingress -{{ include "feast.ingress.metadata" . }} -spec: - rules: - {{- range $host := $ingressValues.hosts }} - - host: {{ $host }} - http: - paths: - - path: / - backend: - serviceName: {{ include (printf "feast-%s.fullname" $component) $top }} - servicePort: {{ index $top.Values "service" $protocol "port" }} - {{- end }} -{{- if $ingressValues.https.enabled }} - tls: - {{- range $host := $ingressValues.hosts }} - - secretName: {{ index $ingressValues.https.secretNames $host | default (splitList "." $host | rest | join "-" | printf "%s-tls") }} - hosts: - - {{ $host }} - {{- end }} -{{- end -}} -{{- end -}} - -{{- define "feast.ingress.metadata" -}} -{{- $commonMetadata := fromYaml (include "common.metadata" (first .)) }} -{{- $overrides := fromYaml (include "feast.ingress.metadata-overrides" .) -}} -{{- toYaml (merge $overrides $commonMetadata) -}} -{{- end -}} - -{{- define "feast.ingress.metadata-overrides" -}} -{{- $top := (index . 0) -}} -{{- $component := (index . 1) -}} -{{- $protocol := (index . 2) -}} -{{- $ingressValues := (index . 3) -}} -{{- $commonFullname := include "common.fullname" $top }} -metadata: - name: {{ $commonFullname }}-{{ $component }}-{{ $protocol }} - annotations: - kubernetes.io/ingress.class: {{ $ingressValues.class | quote }} - {{- if (and (eq $ingressValues.class "nginx") $ingressValues.auth.enabled) }} - nginx.ingress.kubernetes.io/auth-url: {{ $ingressValues.auth.authUrl | quote }} - nginx.ingress.kubernetes.io/auth-response-headers: "x-auth-request-email, x-auth-request-user" - nginx.ingress.kubernetes.io/auth-signin: "https://{{ $ingressValues.auth.signinHost | default (splitList "." (index $ingressValues.hosts 0) | rest | join "." | printf "auth.%s")}}/oauth2/start?rd=/r/$host/$request_uri" - {{- end }} - {{- if (and (eq $ingressValues.class "nginx") $ingressValues.whitelist) }} - nginx.ingress.kubernetes.io/whitelist-source-range: {{ $ingressValues.whitelist | quote -}} - {{- end }} - {{- if (and (eq $ingressValues.class "nginx") (eq $protocol "grpc") ) }} - # TODO: Allow choice of GRPC/GRPCS - nginx.ingress.kubernetes.io/backend-protocol: "GRPC" - {{- end }} - {{- if $ingressValues.annotations -}} - {{ include "common.annote" $ingressValues.annotations | indent 4 }} - {{- end }} -{{- end -}} diff --git a/java/infra/charts/feast-serving/templates/configmap.yaml b/java/infra/charts/feast-serving/templates/configmap.yaml deleted file mode 100644 index 011b4cbcca..0000000000 --- a/java/infra/charts/feast-serving/templates/configmap.yaml +++ /dev/null @@ -1,40 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ template "feast-serving.fullname" . }} - namespace: {{ .Release.Namespace }} - labels: - app: {{ template "feast-serving.name" . }} - component: serving - chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -data: - application-generated.yaml: | -{{- if index .Values "application-generated.yaml" "enabled" }} - feast: - core-host: {{ .Release.Name }}-feast-core - - stores: - - name: online - type: REDIS - config: - host: {{ .Release.Name }}-redis-master - port: 6379 - subscriptions: - - name: "*" - project: "*" - version: "*" - - job_store: - redis_host: {{ .Release.Name }}-redis-master - redis_port: 6379 - - server: - port: {{ .Values.service.http.targetPort }} -{{- end }} - - application-override.yaml: | -{{- if index .Values "application-override.yaml" "enabled" }} -{{- toYaml (index .Values "application-override.yaml") | nindent 4 }} -{{- end }} diff --git a/java/infra/charts/feast-serving/templates/deployment.yaml b/java/infra/charts/feast-serving/templates/deployment.yaml deleted file mode 100644 index 1d6e3aadc4..0000000000 --- a/java/infra/charts/feast-serving/templates/deployment.yaml +++ /dev/null @@ -1,148 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ template "feast-serving.fullname" . }} - namespace: {{ .Release.Namespace }} - labels: - app: {{ template "feast-serving.name" . }} - component: serving - chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -spec: - replicas: {{ .Values.replicaCount }} - selector: - matchLabels: - app: {{ template "feast-serving.name" . }} - component: serving - release: {{ .Release.Name }} - template: - metadata: - annotations: - checksum/configmap: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} - checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} - {{- if .Values.prometheus.enabled }} - prometheus.io/path: /metrics - prometheus.io/port: "{{ .Values.service.http.targetPort }}" - prometheus.io/scrape: "true" - {{- end }} - labels: - app: {{ template "feast-serving.name" . }} - component: serving - release: {{ .Release.Name }} - {{- if .Values.podLabels }} - {{ toYaml .Values.podLabels | nindent 8 }} - {{- end }} - spec: - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - - volumes: - - name: {{ template "feast-serving.fullname" . }}-config - configMap: - name: {{ template "feast-serving.fullname" . }} - - name: {{ template "feast-serving.fullname" . }}-secret - secret: - secretName: {{ template "feast-serving.fullname" . }} - {{- range $secret := .Values.secrets }} - - name: {{ $secret }} - secret: - secretName: {{ $secret }} - {{- end }} - - containers: - - name: {{ .Chart.Name }} - image: {{ .Values.image.repository }}:{{ .Values.image.tag }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - - volumeMounts: - - name: {{ template "feast-serving.fullname" . }}-config - mountPath: /etc/feast - - name: {{ template "feast-serving.fullname" . }}-secret - mountPath: /etc/secrets/feast - readOnly: true - {{- range $secret := .Values.secrets }} - - name: {{ $secret }} - mountPath: "/etc/secrets/{{ $secret }}" - readOnly: true - {{- end }} - - env: - - name: LOG_TYPE - value: {{ .Values.logType | quote }} - - name: LOG_LEVEL - value: {{ .Values.logLevel | quote }} - - {{- if .Values.javaOpts }} - - name: JAVA_TOOL_OPTIONS - value: {{ .Values.javaOpts }} - {{- end }} - - {{- range $key, $value := .Values.envOverrides }} - - name: {{ printf "%s" $key | replace "." "_" | upper | quote }} - {{- if eq (kindOf $value) "map" }} - valueFrom: - {{- toYaml $value | nindent 12 }} - {{- else }} - value: {{ $value | quote }} - {{- end }} - {{- end }} - - command: - - java - - -jar - - /opt/feast/feast-serving.jar - - --spring.config.location= - {{- if index .Values "application.yaml" "enabled" -}} - classpath:/application.yml - {{- end }} - {{- if index .Values "application-generated.yaml" "enabled" -}} - ,file:/etc/feast/application-generated.yaml - {{- end }} - {{- if index .Values "application-secret.yaml" "enabled" -}} - ,file:/etc/secrets/feast/application-secret.yaml - {{- end }} - {{- if index .Values "application-override.yaml" "enabled" -}} - ,file:/etc/feast/application-override.yaml - {{- end }} - - ports: - - name: http - containerPort: {{ .Values.service.http.targetPort }} - - name: grpc - containerPort: {{ .Values.service.grpc.targetPort }} - - {{- if .Values.livenessProbe.enabled }} - livenessProbe: - exec: - command: - - "grpc-health-probe" - - "-addr=:{{ .Values.service.grpc.targetPort }}" - - "-connect-timeout={{ .Values.livenessProbe.timeoutSeconds }}s" - - "-rpc-timeout={{ .Values.livenessProbe.timeoutSeconds }}s" - initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} - periodSeconds: {{ .Values.livenessProbe.periodSeconds }} - successThreshold: {{ .Values.livenessProbe.successThreshold }} - timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} - failureThreshold: {{ .Values.livenessProbe.failureThreshold }} - {{- end }} - - {{- if .Values.readinessProbe.enabled }} - readinessProbe: - exec: - command: - - "grpc-health-probe" - - "-addr=:{{ .Values.service.grpc.targetPort }}" - - "-connect-timeout={{ .Values.readinessProbe.timeoutSeconds }}s" - - "-rpc-timeout={{ .Values.readinessProbe.timeoutSeconds }}s" - initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} - periodSeconds: {{ .Values.readinessProbe.periodSeconds }} - successThreshold: {{ .Values.readinessProbe.successThreshold }} - timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} - failureThreshold: {{ .Values.readinessProbe.failureThreshold }} - {{- end }} - - resources: - {{- toYaml .Values.resources | nindent 10 }} diff --git a/java/infra/charts/feast-serving/templates/ingress.yaml b/java/infra/charts/feast-serving/templates/ingress.yaml deleted file mode 100644 index 1bcd176147..0000000000 --- a/java/infra/charts/feast-serving/templates/ingress.yaml +++ /dev/null @@ -1,7 +0,0 @@ -{{- if .Values.ingress.http.enabled -}} -{{ template "feast.ingress" (list . "serving" "http" .Values.ingress.http) }} -{{- end }} ---- -{{ if .Values.ingress.grpc.enabled -}} -{{ template "feast.ingress" (list . "serving" "grpc" .Values.ingress.grpc) }} -{{- end }} diff --git a/java/infra/charts/feast-serving/templates/secret.yaml b/java/infra/charts/feast-serving/templates/secret.yaml deleted file mode 100644 index 2ccbccfcf7..0000000000 --- a/java/infra/charts/feast-serving/templates/secret.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: {{ template "feast-serving.fullname" . }} - namespace: {{ .Release.Namespace }} - labels: - app: {{ template "feast-serving.name" . }} - component: serving - chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -type: Opaque -stringData: - application-secret.yaml: | -{{- toYaml (index .Values "application-secret.yaml") | nindent 4 }} diff --git a/java/infra/charts/feast-serving/templates/service.yaml b/java/infra/charts/feast-serving/templates/service.yaml deleted file mode 100644 index 7afdd31b45..0000000000 --- a/java/infra/charts/feast-serving/templates/service.yaml +++ /dev/null @@ -1,40 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ template "feast-serving.fullname" . }} - namespace: {{ .Release.Namespace }} - labels: - app: {{ template "feast-serving.name" . }} - chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} - {{- with .Values.service.annotations }} - annotations: -{{ toYaml . | indent 4 }} - {{- end }} -spec: - type: {{ .Values.service.type }} - {{- if .Values.service.loadBalancerIP }} - loadBalancerIP: {{ .Values.service.loadBalancerIP }} - {{- end }} - {{- if .Values.service.loadBalancerSourceRanges }} - loadBalancerSourceRanges: -{{ toYaml .Values.service.loadBalancerSourceRanges | indent 2 }} - {{- end }} - ports: - - name: http - port: {{ .Values.service.http.port }} - targetPort: {{ .Values.service.http.targetPort }} - {{- if .Values.service.http.nodePort }} - nodePort: {{ .Values.service.http.nodePort }} - {{- end }} - - name: grpc - port: {{ .Values.service.grpc.port }} - targetPort: {{ .Values.service.grpc.targetPort }} - {{- if .Values.service.grpc.nodePort }} - nodePort: {{ .Values.service.grpc.nodePort }} - {{- end }} - selector: - app: {{ template "feast-serving.name" . }} - component: serving - release: {{ .Release.Name }} diff --git a/java/infra/charts/feast-serving/values.yaml b/java/infra/charts/feast-serving/values.yaml deleted file mode 100644 index 06dbb85cb9..0000000000 --- a/java/infra/charts/feast-serving/values.yaml +++ /dev/null @@ -1,154 +0,0 @@ -# replicaCount -- Number of pods that will be created -replicaCount: 1 - -image: - # image.repository -- Docker image repository - repository: gcr.io/kf-feast/feast-serving - # image.tag -- Image tag - tag: develop - # image.pullPolicy -- Image pull policy - pullPolicy: IfNotPresent - -application.yaml: - # "application.yaml".enabled -- Flag to include the default [configuration](https://github.com/feast-dev/feast/blob/master/serving/src/main/resources/application.yml). Please set `application-override.yaml` to override this configuration. - enabled: true - -application-generated.yaml: - # "application-generated.yaml".enabled -- Flag to include Helm generated configuration for http port, Feast Core host, Redis store and job store. This is useful for deployment that uses default configuration for Redis. Please set `application-override.yaml` to override this configuration. - enabled: true - -# "application-secret.yaml" -- Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/serving/src/main/resources/application.yml). Will be created as a Secret. `application-override.yaml` has a higher precedence than `application-secret.yaml`. It is recommended to either set `application-override.yaml` or `application-secret.yaml` only to simplify config management. -application-secret.yaml: - enabled: true - -# "application-override.yaml" -- Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/serving/src/main/resources/application.yml). Will be created as a ConfigMap. `application-override.yaml` has a higher precedence than `application-secret.yaml` -application-override.yaml: - enabled: true - -gcpServiceAccount: - # gcpServiceAccount.enabled -- Flag to use [service account](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) JSON key - # Cloud service account JSON key file. - enabled: false - existingSecret: - # gcpServiceAccount.existingSecret.name -- Name of the existing secret containing the service account - name: feast-gcp-service-account - # gcpServiceAccount.existingSecret.key -- Key in the secret data (file name of the service account) - key: credentials.json - -# gcpProjectId -- Project ID to use when using Google Cloud services such as BigQuery, Cloud Storage and Dataflow -gcpProjectId: "" - -# javaOpts -- [JVM options](https://docs.oracle.com/cd/E22289_01/html/821-1274/configuring-the-default-jvm-and-java-arguments.html). For better performance, it is advised to set the min and max heap:
`-Xms2048m -Xmx2048m` -javaOpts: - -# logType -- Log format, either `JSON` or `Console` -logType: Console -# logLevel -- Default log level, use either one of `DEBUG`, `INFO`, `WARN` or `ERROR` -logLevel: WARN - -prometheus: - # prometheus.enabled -- Flag to enable scraping of Feast Core metrics - enabled: true - -livenessProbe: - # livenessProbe.enabled -- Flag to enabled the probe - enabled: true - # livenessProbe.initialDelaySeconds -- Delay before the probe is initiated - initialDelaySeconds: 60 - # livenessProbe.periodSeconds -- How often to perform the probe - periodSeconds: 10 - # livenessProbe.timeoutSeconds -- When the probe times out - timeoutSeconds: 5 - # livenessProbe.successThreshold -- Min consecutive success for the probe to be considered successful - successThreshold: 1 - # livenessProbe.failureThreshold -- Min consecutive failures for the probe to be considered failed - failureThreshold: 5 - -readinessProbe: - # readinessProbe.enabled -- Flag to enabled the probe - enabled: true - # readinessProbe.initialDelaySeconds -- Delay before the probe is initiated - initialDelaySeconds: 15 - # readinessProbe.periodSeconds -- How often to perform the probe - periodSeconds: 10 - # readinessProbe.timeoutSeconds -- When the probe times out - timeoutSeconds: 10 - # readinessProbe.successThreshold -- Min consecutive success for the probe to be considered successful - successThreshold: 1 - # readinessProbe.failureThreshold -- Min consecutive failures for the probe to be considered failed - failureThreshold: 5 - -service: - # service.type -- Kubernetes service type - type: ClusterIP - http: - # service.http.port -- Service port for HTTP requests - port: 80 - # service.http.targetPort -- Container port serving HTTP requests and Prometheus metrics - targetPort: 8080 - # service.http.nodePort -- Port number that each cluster node will listen to - nodePort: - grpc: - # service.grpc.port -- Service port for GRPC requests - port: 6566 - # service.grpc.targetPort -- Container port serving GRPC requests - targetPort: 6566 - # service.grpc.nodePort -- Port number that each cluster node will listen to - nodePort: - -ingress: - grpc: - # ingress.grpc.enabled -- Flag to create an ingress resource for the service - enabled: false - # ingress.grpc.class -- Which ingress controller to use - class: nginx - # ingress.grpc.hosts -- List of hostnames to match when routing requests - hosts: [] - # ingress.grpc.annotations -- Extra annotations for the ingress - annotations: {} - https: - # ingress.grpc.https.enabled -- Flag to enable HTTPS - enabled: true - # ingress.grpc.https.secretNames -- Map of hostname to TLS secret name - secretNames: {} - # ingress.grpc.whitelist -- Allowed client IP source ranges - whitelist: "" - auth: - # ingress.grpc.auth.enabled -- Flag to enable auth - enabled: false - http: - # ingress.http.enabled -- Flag to create an ingress resource for the service - enabled: false - # ingress.http.class -- Which ingress controller to use - class: nginx - # ingress.http.hosts -- List of hostnames to match when routing requests - hosts: [] - # ingress.http.annotations -- Extra annotations for the ingress - annotations: {} - https: - # ingress.http.https.enabled -- Flag to enable HTTPS - enabled: true - # ingress.http.https.secretNames -- Map of hostname to TLS secret name - secretNames: {} - # ingress.http.whitelist -- Allowed client IP source ranges - whitelist: "" - auth: - # ingress.http.auth.enabled -- Flag to enable auth - enabled: false - # ingress.http.auth.authUrl -- URL to an existing authentication service - authUrl: http://auth-server.auth-ns.svc.cluster.local/auth - -# resources -- CPU/memory [resource requests/limit](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container) -resources: {} - -# nodeSelector -- Node labels for pod assignment -nodeSelector: {} - -# envOverrides -- Extra environment variables to set -envOverrides: {} - -# secrets -- List of Kubernetes secrets to be mounted on Feast Core pods. These secrets will be mounted on /etc/secrets/. -secrets: [] - -# podLabels -- Labels to be added to Feast Serving pods -podLabels: {} diff --git a/java/infra/docker/core/Dockerfile b/java/infra/docker/core/Dockerfile deleted file mode 100644 index 35d58ce750..0000000000 --- a/java/infra/docker/core/Dockerfile +++ /dev/null @@ -1,56 +0,0 @@ -# ============================================================ -# Build stage 1: Builder -# ============================================================ - -FROM maven:3.6-jdk-11 as builder - -WORKDIR /build - -COPY pom.xml . -COPY datatypes/java/pom.xml datatypes/java/pom.xml -COPY common/pom.xml common/pom.xml -COPY core/pom.xml core/pom.xml -COPY serving/pom.xml serving/pom.xml -COPY storage/api/pom.xml storage/api/pom.xml -COPY storage/connectors/pom.xml storage/connectors/pom.xml -COPY storage/connectors/redis/pom.xml storage/connectors/redis/pom.xml -COPY sdk/java/pom.xml sdk/java/pom.xml -COPY docs/coverage/java/pom.xml docs/coverage/java/pom.xml -COPY deps/feast/protos/ deps/feast/protos/ - -# Setting Maven repository .m2 directory relative to /build folder gives the -# user to optionally use cached repository when building the image by copying -# the existing .m2 directory to $FEAST_REPO_ROOT/.m2 -ENV MAVEN_OPTS="-Dmaven.repo.local=/build/.m2/repository -DdependencyLocationsEnabled=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3" -COPY pom.xml .m2/* .m2/ -RUN mvn dependency:go-offline -DexcludeGroupIds:dev.feast 2>/dev/null || true - -COPY . . - -ARG VERSION=dev -RUN mvn --also-make --projects core -Drevision=$VERSION \ - -DskipUTs=true --batch-mode clean package - -# -# Download grpc_health_probe to run health check for Feast Serving -# https://kubernetes.io/blog/2018/10/01/health-checking-grpc-servers-on-kubernetes/ -# -RUN wget -q https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/v0.3.1/grpc_health_probe-linux-amd64 \ - -O /usr/bin/grpc-health-probe && \ - chmod +x /usr/bin/grpc-health-probe - -# ============================================================ -# Build stage 2: Production -# ============================================================ - -FROM openjdk:11-jre as production -ARG VERSION=dev - -COPY --from=builder /build/core/target/feast-core-$VERSION-exec.jar /opt/feast/feast-core.jar -COPY --from=builder /usr/bin/grpc-health-probe /usr/bin/grpc-health-probe - -CMD ["java",\ - "-Xms2048m",\ - "-Xmx2048m",\ - "-jar",\ - "/opt/feast/feast-core.jar"] diff --git a/java/infra/docker/core/Dockerfile.debug b/java/infra/docker/core/Dockerfile.debug deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/java/infra/docker/core/Dockerfile.dev b/java/infra/docker/core/Dockerfile.dev deleted file mode 100644 index 70de4d9339..0000000000 --- a/java/infra/docker/core/Dockerfile.dev +++ /dev/null @@ -1,8 +0,0 @@ -FROM openjdk:11-jre -ARG REVISION=dev -ADD $PWD/core/target/feast-core-$REVISION-exec.jar /opt/feast/feast-core.jar -CMD ["java",\ - "-Xms2048m",\ - "-Xmx2048m",\ - "-jar",\ - "/opt/feast/feast-core.jar"] diff --git a/java/infra/scripts/download-maven-cache.sh b/java/infra/scripts/download-maven-cache.sh deleted file mode 100755 index e511338942..0000000000 --- a/java/infra/scripts/download-maven-cache.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash -set -e - -# This script downloads previous maven packages that have been downloaded -# from Google Cloud Storage to local path for faster build - -usage() -{ - echo "usage: prepare_maven_cache.sh - --archive-uri gcs uri to retrieve maven .m2 archive - --output-dir output directory for .m2 directory" -} - -while [ "$1" != "" ]; do - case "$1" in - --archive-uri ) ARCHIVE_URI="$2"; shift;; - --output-dir ) OUTPUT_DIR="$2"; shift;; - * ) usage; exit 1 - esac - shift -done - -if [[ ! ${ARCHIVE_URI} ]]; then usage; exit 1; fi -if [[ ! ${OUTPUT_DIR} ]]; then usage; exit 1; fi - -gsutil -q cp ${ARCHIVE_URI} /tmp/.m2.tar -tar xf /tmp/.m2.tar -C ${OUTPUT_DIR} diff --git a/java/infra/scripts/install-helm.sh b/java/infra/scripts/install-helm.sh deleted file mode 100755 index 3686f9dfdb..0000000000 --- a/java/infra/scripts/install-helm.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash -set -e -readonly HELM_URL=https://storage.googleapis.com/kubernetes-helm -readonly HELM_TARBALL="helm-${HELM_VERSION}-linux-amd64.tar.gz" -readonly STABLE_REPO_URL=https://charts.helm.sh/stable -readonly INCUBATOR_REPO_URL=https://charts.helm.sh/incubator -curl -s "https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz" | tar -C /tmp -xz -sudo mv /tmp/linux-amd64/helm /usr/bin/helm -helm init --client-only -helm repo add incubator "$INCUBATOR_REPO_URL" \ No newline at end of file diff --git a/java/infra/scripts/publish-java-sdk.sh b/java/infra/scripts/publish-java-sdk.sh deleted file mode 100755 index bd064108e3..0000000000 --- a/java/infra/scripts/publish-java-sdk.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash - -set -e -set -o pipefail - -GPG_KEY_IMPORT_DIR=/etc/gpg - -usage() -{ - echo "usage: publish-java-sdk.sh - - --revision Value for the revision e.g. '0.2.3' - --gpg-key-import-dir Directory containing existing GPG keys to import. - The directory should contain these 2 files: - - public-key - - private-key - The default value is '/etc/gpg' - - This script assumes the GPG private key is protected by a passphrase. - The passphrase can be specified in \$HOME/.m2/settings.xml. In the same xml - file, credentials to upload releases to Sonatype must also be provided. - - # Example settings: ~/.m2/settings.xml - - - - ossrh - SONATYPE_USER - SONATYPE_PASSWORD - - - - - ossrh - - GPG_PASSPHRASE - - - - -" -} - -while [ "$1" != "" ]; do - case "$1" in - --revision ) REVISION="$2"; shift;; - --gpg-key-import-dir ) GPG_KEY_IMPORT_DIR="$2"; shift;; - -h | --help ) usage; exit;; - * ) usage; exit 1 - esac - shift -done - -if [ -z $REVISION ]; then usage; exit 1; fi - -echo "============================================================" -echo "Checking Maven and GPG versions" -echo "============================================================" -mvn --version -echo "" -gpg --version - -echo "============================================================" -echo "Importing GPG keys" -echo "============================================================" -gpg --import --batch --yes $GPG_KEY_IMPORT_DIR/public-key -gpg --import --batch --yes $GPG_KEY_IMPORT_DIR/private-key - -echo "============================================================" -echo "Deploying Java SDK with revision: $REVISION" -echo "============================================================" -mvn --projects .,datatypes/java,common,sdk/java -Drevision=$REVISION --batch-mode clean deploy \ No newline at end of file diff --git a/java/infra/scripts/push-helm-charts.sh b/java/infra/scripts/push-helm-charts.sh deleted file mode 100755 index 4f45a9ac00..0000000000 --- a/java/infra/scripts/push-helm-charts.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash - -set -e - -if [ $# -ne 1 ]; then - echo "Please provide a single semver version (without a \"v\" prefix) to test the repository against, e.g 0.99.0" - exit 1 -fi - -bucket=gs://feast-helm-charts/ -repo_url=https://feast-helm-charts.storage.googleapis.com/ - -helm plugin install https://github.com/hayorov/helm-gcs.git --version 0.2.2 || true - -helm repo add feast-helm-chart-repo $bucket - -helm package infra/charts/feast-core -helm package infra/charts/feast-serving - -helm gcs push --public --force feast-core-${1}.tgz feast-helm-chart-repo -helm gcs push --public --force feast-serving-${1}.tgz feast-helm-chart-repo \ No newline at end of file diff --git a/java/infra/scripts/setup-common-functions.sh b/java/infra/scripts/setup-common-functions.sh deleted file mode 100755 index 376c376c53..0000000000 --- a/java/infra/scripts/setup-common-functions.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash - -# Usage: TAG=$(get_tag_release [-ms]) -# Parses the last release from git tags. -# Options: -# -m - Use only tags that are tagged on the current branch -# -s - Use only stable version tags. (ie no prerelease tags). -get_tag_release() { - local GIT_TAG_CMD="git tag -l" - # Match only Semver tags - # Regular expression should match MAJOR.MINOR.PATCH[-PRERELEASE[.IDENTIFIER]] - # eg. v0.7.1 v0.7.2-alpha v0.7.2-rc.1 - local TAG_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$' - local OPTIND opt - while getopts "ms" opt; do - case "${opt}" in - m) - GIT_TAG_CMD="$GIT_TAG_CMD --merged" - ;; - s) - # Match only stable version tags. - TAG_REGEX="^v[0-9]+\.[0-9]+\.[0-9]+$" - ;; - *) - echo "get_tag_release(): Error: Bad arguments: $@" - return 1 - ;; - esac - done - shift $((OPTIND-1)) - - # Retrieve tags from git and filter as per regex. - local FILTERED_TAGS=$(bash -c "$GIT_TAG_CMD" | grep -P "$TAG_REGEX") - - # Sort version tags in highest semver version first. - # To make sure that prerelease versions (ie versions vMAJOR.MINOR.PATCH-PRERELEASE suffix) - # are sorted after stable versions (ie vMAJOR.MINOR.PATCH), we append '_' after - # eachustable version as '_' is after '-' found in prerelease version - # alphanumerically and remove after sorting. - local SEMVER_SORTED_TAGS=$(echo "$FILTERED_TAGS" | sed -e '/-/!{s/$/_/}' | sort -rV \ - | sed -e 's/_$//') - echo $(echo "$SEMVER_SORTED_TAGS" | head -n 1) -} diff --git a/java/infra/scripts/validate-helm-chart-versions.sh b/java/infra/scripts/validate-helm-chart-versions.sh deleted file mode 100755 index 826966534c..0000000000 --- a/java/infra/scripts/validate-helm-chart-versions.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -function finish { - echo "Please ensure the Chart.yaml have the version ${1}" - exit -} - -trap "finish $1" ERR - -set -e - -if [ $# -ne 1 ]; then - echo "Please provide a single semver version (without a \"v\" prefix) to test the repository against, e.g 0.99.0" - exit 1 -fi - -# Get project root -PROJECT_ROOT_DIR=$(git rev-parse --show-toplevel) - -echo "Trying to find version ${1} in the feast-core Chart.yaml. Exiting if not found." -grep "version: ${1}" "${PROJECT_ROOT_DIR}/infra/charts/feast-core/Chart.yaml" - -echo "Trying to find version ${1} in the feast-serving Chart.yaml. Exiting if not found." -grep "version: ${1}" "${PROJECT_ROOT_DIR}/infra/charts/feast-serving/Chart.yaml" - -echo "Success! All versions found!" \ No newline at end of file diff --git a/java/pom.xml b/java/pom.xml index 50d9b95d49..f7169cec54 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -31,12 +31,10 @@ datatypes/java storage/api storage/connectors - core serving sdk/java docs/coverage/java common - common-test @@ -57,14 +55,11 @@ 0.8.0 1.9.10 1.3 - 5.4.18.Final - 2.5.0 2.28.2 0.26.0 2.12.1 - 6.0.8 2.9.9 2.0.2 2.5.0.RELEASE @@ -79,7 +74,6 @@ 2.0.1.Final 2.8.0 0.20.0 - 6.1.2.Final 1.6.6 runtime - - - io.swagger - swagger-annotations - ${swagger.core.version} - - - com.squareup.okhttp3 - okhttp - ${okhttp.version} - - - com.squareup.okhttp3 - logging-interceptor - ${okhttp.version} - - - io.springfox - springfox-swagger2 - ${springfox.version} - - - io.springfox - springfox-swagger-ui - ${springfox.version} - - net.devh @@ -388,7 +355,7 @@ com.google.guava guava - 26.0-jre + 29.0-jre com.google.protobuf @@ -435,26 +402,7 @@ - - org.apache.kafka - kafka_2.12 - ${kafka.version} - - - org.apache.kafka - kafka-clients - ${kafka.version} - - - org.hibernate - hibernate-core - ${hibernate.version} - - - org.hibernate.validator - hibernate-validator - ${org.hibernate.validator.version} - + net.bytebuddy byte-buddy diff --git a/java/serving/pom.xml b/java/serving/pom.xml index 981c23c001..8fdc090b37 100644 --- a/java/serving/pom.xml +++ b/java/serving/pom.xml @@ -96,19 +96,6 @@ ${project.version} - - dev.feast - feast-common - ${project.version} - - - - com.google.cloud - google-cloud-bigtable-emulator - 0.130.2 - test - - @@ -316,67 +303,21 @@ jakarta.validation-api ${jakarta.validation.api.version} - - org.springframework.security - spring-security-core - ${spring.security.version} - - - org.springframework.security - spring-security-config - ${spring.security.version} - - - org.springframework.security.oauth - spring-security-oauth2 - ${spring.security.oauth2.version} - - - org.springframework.security - spring-security-oauth2-client - ${spring.security.version} - - - org.springframework.security - spring-security-web - ${spring.security.version} - - - org.springframework.security - spring-security-oauth2-jose - ${spring.security.version} - net.devh grpc-server-spring-boot-starter ${grpc.spring.boot.starter.version} - - com.nimbusds - nimbus-jose-jwt - 8.2.1 - - - org.springframework.security - spring-security-oauth2-core - ${spring.security.version} - org.testcontainers testcontainers - 1.15.1 + 1.16.2 test org.testcontainers junit-jupiter - 1.15.1 - test - - - org.testcontainers - gcloud - 1.15.2 + 1.16.2 test @@ -385,18 +326,6 @@ 3.0.0 test - - sh.ory.keto - keto-client - 0.4.4-alpha.1 - test - - - dev.feast - feast-common-test - ${project.version} - test - com.squareup.okhttp okhttp diff --git a/java/serving/src/main/java/feast/serving/config/ContextClosedHandler.java b/java/serving/src/main/java/feast/serving/config/ContextClosedHandler.java deleted file mode 100644 index cdf791cb8d..0000000000 --- a/java/serving/src/main/java/feast/serving/config/ContextClosedHandler.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.config; - -import java.util.concurrent.ScheduledExecutorService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; -import org.springframework.context.ApplicationListener; -import org.springframework.context.event.ContextClosedEvent; -import org.springframework.stereotype.Component; - -@Component -@ConditionalOnBean(CoreCondition.class) -public class ContextClosedHandler implements ApplicationListener { - - @Autowired ScheduledExecutorService executor; - - @Override - public void onApplicationEvent(ContextClosedEvent event) { - executor.shutdown(); - } -} diff --git a/java/serving/src/main/java/feast/serving/config/CoreCondition.java b/java/serving/src/main/java/feast/serving/config/CoreCondition.java deleted file mode 100644 index 10dabfaf29..0000000000 --- a/java/serving/src/main/java/feast/serving/config/CoreCondition.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2021 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.config; - -import org.springframework.context.annotation.Condition; -import org.springframework.context.annotation.ConditionContext; -import org.springframework.core.env.Environment; -import org.springframework.core.type.AnnotatedTypeMetadata; - -/** - * A {@link Condition} to signal that the ServingService should get feature definitions and metadata - * from Core service. - */ -public class CoreCondition implements Condition { - @Override - public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { - final Environment env = context.getEnvironment(); - return env.getProperty("feast.registry") == null; - } -} diff --git a/java/serving/src/main/java/feast/serving/config/FeastProperties.java b/java/serving/src/main/java/feast/serving/config/FeastProperties.java index 88d9f537ab..493f8f9ed6 100644 --- a/java/serving/src/main/java/feast/serving/config/FeastProperties.java +++ b/java/serving/src/main/java/feast/serving/config/FeastProperties.java @@ -21,13 +21,7 @@ // https://www.baeldung.com/configuration-properties-in-spring-boot // https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-external-config.html#boot-features-external-config-typesafe-configuration-properties -import feast.common.auth.config.SecurityProperties; -import feast.common.auth.config.SecurityProperties.AuthenticationProperties; -import feast.common.auth.config.SecurityProperties.AuthorizationProperties; -import feast.common.auth.credentials.CoreAuthenticationProperties; import feast.common.logging.config.LoggingProperties; -import feast.storage.connectors.bigtable.retriever.BigTableStoreConfig; -import feast.storage.connectors.cassandra.retriever.CassandraStoreConfig; import feast.storage.connectors.redis.retriever.RedisClusterStoreConfig; import feast.storage.connectors.redis.retriever.RedisStoreConfig; import io.lettuce.core.ReadFrom; @@ -37,8 +31,6 @@ import javax.validation.*; import javax.validation.constraints.NotBlank; import javax.validation.constraints.NotNull; -import javax.validation.constraints.Positive; -import org.apache.logging.log4j.core.config.plugins.validation.constraints.ValidHost; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.info.BuildProperties; @@ -66,13 +58,7 @@ public FeastProperties() {} /* Feast Serving build version */ @NotBlank private String version = "unknown"; - /* Feast Core host to connect to. */ - @ValidHost @NotBlank private String coreHost; - - /* Feast Core port to connect to. */ - @Positive private int coreGrpcPort; - - private String registry; + @NotBlank private String registry; public String getRegistry() { return registry; @@ -92,43 +78,6 @@ public void setTransformationServiceEndpoint(final String transformationServiceE this.transformationServiceEndpoint = transformationServiceEndpoint; } - private CoreAuthenticationProperties coreAuthentication; - - public CoreAuthenticationProperties getCoreAuthentication() { - return coreAuthentication; - } - - public void setCoreAuthentication(CoreAuthenticationProperties coreAuthentication) { - this.coreAuthentication = coreAuthentication; - } - - @Positive private int coreCacheRefreshInterval; - - private SecurityProperties security; - - @Bean - SecurityProperties securityProperties() { - return this.getSecurity(); - } - - /** - * Getter for SecurityProperties - * - * @return Returns the {@link SecurityProperties} object. - */ - public SecurityProperties getSecurity() { - return security; - } - - /** - * Setter for SecurityProperties - * - * @param security :input {@link SecurityProperties} object - */ - public void setSecurity(SecurityProperties security) { - this.security = security; - } - /** * Finds and returns the active store * @@ -199,60 +148,6 @@ public void setVersion(String version) { this.version = version; } - /** - * Gets Feast Core host. - * - * @return Feast Core host - */ - public String getCoreHost() { - return coreHost; - } - - /** - * Sets Feast Core host to connect to. - * - * @param coreHost Feast Core host - */ - public void setCoreHost(String coreHost) { - this.coreHost = coreHost; - } - - /** - * Gets Feast Core gRPC port. - * - * @return Feast Core gRPC port - */ - public int getCoreGrpcPort() { - return coreGrpcPort; - } - - /** - * Sets Feast Core gRPC port. - * - * @param coreGrpcPort gRPC port of Feast Core - */ - public void setCoreGrpcPort(int coreGrpcPort) { - this.coreGrpcPort = coreGrpcPort; - } - - /** - * Gets CachedSpecService refresh interval. - * - * @return CachedSpecService refresh interval - */ - public int getCoreCacheRefreshInterval() { - return coreCacheRefreshInterval; - } - - /** - * Sets CachedSpecService refresh interval. - * - * @param coreCacheRefreshInterval CachedSpecService refresh interval - */ - public void setCoreCacheRefreshInterval(int coreCacheRefreshInterval) { - this.coreCacheRefreshInterval = coreCacheRefreshInterval; - } - /** * Sets the collection of configured stores. * @@ -333,20 +228,6 @@ public RedisStoreConfig getRedisConfig() { this.config.getOrDefault("password", "")); } - public BigTableStoreConfig getBigtableConfig() { - return new BigTableStoreConfig( - this.config.get("project_id"), - this.config.get("instance_id"), - this.config.get("app_profile_id")); - } - - public CassandraStoreConfig getCassandraConfig() { - return new CassandraStoreConfig( - this.config.get("connection_string"), - this.config.get("data_center"), - this.config.get("keyspace")); - } - /** * Sets the store config. Please protos/feast/core/Store.proto for the specific options for each * store. @@ -359,8 +240,6 @@ public void setConfig(Map config) { } public enum StoreType { - BIGTABLE, - CASSANDRA, REDIS, REDIS_CLUSTER; } @@ -482,26 +361,5 @@ public void validate() { if (!violations.isEmpty()) { throw new ConstraintViolationException(violations); } - - // Validate CoreAuthenticationProperties - Set> coreAuthenticationPropsViolations = - validator.validate(getCoreAuthentication()); - if (!coreAuthenticationPropsViolations.isEmpty()) { - throw new ConstraintViolationException(coreAuthenticationPropsViolations); - } - - // Validate AuthenticationProperties - Set> authenticationPropsViolations = - validator.validate(getSecurity().getAuthentication()); - if (!authenticationPropsViolations.isEmpty()) { - throw new ConstraintViolationException(authenticationPropsViolations); - } - - // Validate AuthorizationProperties - Set> authorizationPropsViolations = - validator.validate(getSecurity().getAuthorization()); - if (!authorizationPropsViolations.isEmpty()) { - throw new ConstraintViolationException(authorizationPropsViolations); - } } } diff --git a/java/serving/src/main/java/feast/serving/config/RegistryCondition.java b/java/serving/src/main/java/feast/serving/config/RegistryCondition.java deleted file mode 100644 index 621d124ed5..0000000000 --- a/java/serving/src/main/java/feast/serving/config/RegistryCondition.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2021 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.config; - -import org.springframework.context.annotation.Condition; -import org.springframework.context.annotation.ConditionContext; -import org.springframework.core.env.Environment; -import org.springframework.core.type.AnnotatedTypeMetadata; - -/** - * A {@link Condition} to signal that the ServingService should get feature definitions and metadata - * from the Registry object. This is needed for versions of the feature store written by feast - * 0.10+. - */ -public class RegistryCondition implements Condition { - - @Override - public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { - final Environment env = context.getEnvironment(); - return env.getProperty("feast.registry") != null; - } -} diff --git a/java/serving/src/main/java/feast/serving/config/ServingSecurityConfig.java b/java/serving/src/main/java/feast/serving/config/ServingSecurityConfig.java deleted file mode 100644 index f51e06292d..0000000000 --- a/java/serving/src/main/java/feast/serving/config/ServingSecurityConfig.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.config; - -import feast.common.auth.credentials.GoogleAuthCredentials; -import feast.common.auth.credentials.OAuthCredentials; -import feast.proto.serving.ServingServiceGrpc; -import io.grpc.CallCredentials; -import io.grpc.health.v1.HealthGrpc; -import java.io.IOException; -import net.devh.boot.grpc.server.security.check.AccessPredicate; -import net.devh.boot.grpc.server.security.check.GrpcSecurityMetadataSource; -import net.devh.boot.grpc.server.security.check.ManualGrpcSecurityMetadataSource; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.Configuration; - -/* - * Copyright 2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -@Configuration -@ComponentScan( - basePackages = { - "feast.common.auth.config", - "feast.common.auth.service", - "feast.common.logging.interceptors" - }) -public class ServingSecurityConfig { - - private final FeastProperties feastProperties; - - public ServingSecurityConfig(FeastProperties feastProperties) { - this.feastProperties = feastProperties; - } - - /** - * Creates a SecurityMetadataSource when authentication is enabled. This allows for the - * configuration of endpoint level security rules. - * - * @return GrpcSecurityMetadataSource - */ - @Bean - @ConditionalOnProperty(prefix = "feast.security.authentication", name = "enabled") - GrpcSecurityMetadataSource grpcSecurityMetadataSource() { - final ManualGrpcSecurityMetadataSource source = new ManualGrpcSecurityMetadataSource(); - - // Authentication is enabled for all gRPC endpoints - source.setDefault(AccessPredicate.authenticated()); - - // The following endpoints allow unauthenticated access - source.set(ServingServiceGrpc.getGetFeastServingInfoMethod(), AccessPredicate.permitAll()); - source.set(HealthGrpc.getCheckMethod(), AccessPredicate.permitAll()); - return source; - } - - /** - * Creates a CallCredentials when authentication is enabled on core. This allows serving to - * connect to core with CallCredentials - * - * @return CallCredentials - */ - @Bean - @ConditionalOnProperty(prefix = "feast.core-authentication", name = "enabled") - CallCredentials CoreGrpcAuthenticationCredentials() throws IOException { - switch (feastProperties.getCoreAuthentication().getProvider()) { - case "google": - return new GoogleAuthCredentials(feastProperties.getCoreAuthentication().getOptions()); - case "oauth": - return new OAuthCredentials(feastProperties.getCoreAuthentication().getOptions()); - default: - throw new IllegalArgumentException( - "Please configure an Core Authentication Provider " - + "if you have enabled Authentication on core. " - + "Currently `google` and `oauth` are supported"); - } - } -} diff --git a/java/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java b/java/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java index ce2aabf4de..3a7936cb57 100644 --- a/java/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java +++ b/java/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java @@ -16,130 +16,25 @@ */ package feast.serving.config; -import com.datastax.oss.driver.api.core.CqlSession; -import com.datastax.oss.driver.api.core.CqlSessionBuilder; -import com.google.cloud.bigtable.data.v2.BigtableDataClient; -import com.google.cloud.bigtable.data.v2.BigtableDataSettings; -import com.google.protobuf.AbstractMessageLite; import feast.serving.registry.LocalRegistryRepo; import feast.serving.service.OnlineServingServiceV2; import feast.serving.service.OnlineTransformationService; import feast.serving.service.ServingServiceV2; -import feast.serving.specs.CachedSpecService; -import feast.serving.specs.CoreFeatureSpecRetriever; import feast.serving.specs.FeatureSpecRetriever; import feast.serving.specs.RegistryFeatureSpecRetriever; import feast.storage.api.retriever.OnlineRetrieverV2; -import feast.storage.connectors.bigtable.retriever.BigTableOnlineRetriever; -import feast.storage.connectors.bigtable.retriever.BigTableStoreConfig; -import feast.storage.connectors.cassandra.retriever.CassandraOnlineRetriever; -import feast.storage.connectors.cassandra.retriever.CassandraStoreConfig; import feast.storage.connectors.redis.retriever.*; import io.opentracing.Tracer; -import java.io.IOException; -import java.net.InetSocketAddress; import java.nio.file.Paths; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; import org.slf4j.Logger; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Lazy; @Configuration public class ServingServiceConfigV2 { private static final Logger log = org.slf4j.LoggerFactory.getLogger(ServingServiceConfigV2.class); - @Autowired private ApplicationContext context; - - @Bean - @Lazy(true) - public BigtableDataClient bigtableClient(FeastProperties feastProperties) throws IOException { - BigTableStoreConfig config = feastProperties.getActiveStore().getBigtableConfig(); - String projectId = config.getProjectId(); - String instanceId = config.getInstanceId(); - - return BigtableDataClient.create( - BigtableDataSettings.newBuilder() - .setProjectId(projectId) - .setInstanceId(instanceId) - .setAppProfileId(config.getAppProfileId()) - .build()); - } - - @Bean - @Conditional(CoreCondition.class) - public ServingServiceV2 servingServiceV2( - FeastProperties feastProperties, CachedSpecService specService, Tracer tracer) { - final ServingServiceV2 servingService; - final FeastProperties.Store store = feastProperties.getActiveStore(); - - OnlineRetrieverV2 retrieverV2; - switch (store.getType()) { - case REDIS_CLUSTER: - RedisClientAdapter redisClusterClient = - RedisClusterClient.create(store.getRedisClusterConfig()); - retrieverV2 = new OnlineRetriever(redisClusterClient, (AbstractMessageLite::toByteArray)); - break; - case REDIS: - RedisClientAdapter redisClient = RedisClient.create(store.getRedisConfig()); - retrieverV2 = new OnlineRetriever(redisClient, (AbstractMessageLite::toByteArray)); - break; - case BIGTABLE: - BigtableDataClient bigtableClient = context.getBean(BigtableDataClient.class); - retrieverV2 = new BigTableOnlineRetriever(bigtableClient); - break; - case CASSANDRA: - CassandraStoreConfig config = feastProperties.getActiveStore().getCassandraConfig(); - String connectionString = config.getConnectionString(); - String dataCenter = config.getDataCenter(); - String keySpace = config.getKeySpace(); - - List contactPoints = - Arrays.stream(connectionString.split(",")) - .map(String::trim) - .map(cs -> cs.split(":")) - .map( - hostPort -> { - int port = hostPort.length > 1 ? Integer.parseInt(hostPort[1]) : 9042; - return new InetSocketAddress(hostPort[0], port); - }) - .collect(Collectors.toList()); - - CqlSession session = - new CqlSessionBuilder() - .addContactPoints(contactPoints) - .withLocalDatacenter(dataCenter) - .withKeyspace(keySpace) - .build(); - retrieverV2 = new CassandraOnlineRetriever(session); - break; - default: - throw new RuntimeException( - String.format("Unable to identify online store type: %s", store.getType())); - } - - final FeatureSpecRetriever featureSpecRetriever; - log.info("Created CoreFeatureSpecRetriever"); - featureSpecRetriever = new CoreFeatureSpecRetriever(specService); - - final String transformationServiceEndpoint = feastProperties.getTransformationServiceEndpoint(); - final OnlineTransformationService onlineTransformationService = - new OnlineTransformationService(transformationServiceEndpoint, featureSpecRetriever); - - servingService = - new OnlineServingServiceV2( - retrieverV2, tracer, featureSpecRetriever, onlineTransformationService); - - return servingService; - } - @Bean - @Conditional(RegistryCondition.class) public ServingServiceV2 registryBasedServingServiceV2( FeastProperties feastProperties, Tracer tracer) { final ServingServiceV2 servingService; diff --git a/java/serving/src/main/java/feast/serving/config/SpecServiceConfig.java b/java/serving/src/main/java/feast/serving/config/SpecServiceConfig.java deleted file mode 100644 index 29d3bf0986..0000000000 --- a/java/serving/src/main/java/feast/serving/config/SpecServiceConfig.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.config; - -import feast.serving.specs.CachedSpecService; -import feast.serving.specs.CoreSpecService; -import io.grpc.CallCredentials; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import org.slf4j.Logger; -import org.springframework.beans.factory.ObjectProvider; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Conditional; -import org.springframework.context.annotation.Configuration; - -@Configuration -public class SpecServiceConfig { - - private static final Logger log = org.slf4j.LoggerFactory.getLogger(SpecServiceConfig.class); - private final String feastCoreHost; - private final int feastCorePort; - private final int feastCachedSpecServiceRefreshInterval; - - @Autowired - public SpecServiceConfig(FeastProperties feastProperties) { - this.feastCoreHost = feastProperties.getCoreHost(); - this.feastCorePort = feastProperties.getCoreGrpcPort(); - this.feastCachedSpecServiceRefreshInterval = feastProperties.getCoreCacheRefreshInterval(); - } - - @Bean - @Conditional(CoreCondition.class) - public ScheduledExecutorService cachedSpecServiceScheduledExecutorService( - CachedSpecService cachedSpecStorage) { - ScheduledExecutorService scheduledExecutorService = - Executors.newSingleThreadScheduledExecutor(); - // reload all specs including new ones periodically - scheduledExecutorService.scheduleAtFixedRate( - cachedSpecStorage::scheduledPopulateCache, - feastCachedSpecServiceRefreshInterval, - feastCachedSpecServiceRefreshInterval, - TimeUnit.SECONDS); - return scheduledExecutorService; - } - - @Bean - @Conditional(CoreCondition.class) - public CachedSpecService specService(ObjectProvider callCredentials) { - CoreSpecService coreService = - new CoreSpecService(feastCoreHost, feastCorePort, callCredentials); - CachedSpecService cachedSpecStorage = new CachedSpecService(coreService); - try { - cachedSpecStorage.populateCache(); - } catch (Exception e) { - log.error("Unable to preload feast's spec"); - } - return cachedSpecStorage; - } -} diff --git a/java/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java b/java/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java index 81bbfd02fd..048a53fad1 100644 --- a/java/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java +++ b/java/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java @@ -16,7 +16,6 @@ */ package feast.serving.controller; -import feast.common.auth.service.AuthorizationService; import feast.common.logging.interceptors.GrpcMessageInterceptor; import feast.proto.serving.ServingAPIProto; import feast.proto.serving.ServingAPIProto.GetFeastServingInfoRequest; @@ -38,7 +37,6 @@ import org.slf4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.security.access.AccessDeniedException; -import org.springframework.security.core.context.SecurityContextHolder; @GrpcService( interceptors = { @@ -53,15 +51,10 @@ public class ServingServiceGRpcController extends ServingServiceImplBase { private final ServingServiceV2 servingServiceV2; private final String version; private final Tracer tracer; - private final AuthorizationService authorizationService; @Autowired public ServingServiceGRpcController( - AuthorizationService authorizationService, - ServingServiceV2 servingServiceV2, - FeastProperties feastProperties, - Tracer tracer) { - this.authorizationService = authorizationService; + ServingServiceV2 servingServiceV2, FeastProperties feastProperties, Tracer tracer) { this.servingServiceV2 = servingServiceV2; this.version = feastProperties.getVersion(); this.tracer = tracer; @@ -84,10 +77,6 @@ public void getOnlineFeaturesV2( try { // authorize for the project in request object. if (request.getProject() != null && !request.getProject().isEmpty()) { - // project set at root level overrides the project set at feature table level - this.authorizationService.authorizeRequest( - SecurityContextHolder.getContext(), request.getProject()); - // update monitoring context GrpcMonitoringContext.getInstance().setProject(request.getProject()); } diff --git a/java/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java b/java/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java index 2cd810cff5..2992c5022d 100644 --- a/java/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java +++ b/java/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java @@ -208,7 +208,7 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequestV2 re checkOutsideMaxAge( feature, entityRow, featureMaxAges.get(feature.getFeatureReference())); - if (!isOutsideMaxAge && value != null) { + if (value != null) { rowValues.put(FeatureV2.getFeatureStringRef(feature.getFeatureReference()), value); } else { rowValues.put( diff --git a/java/serving/src/main/java/feast/serving/specs/CachedSpecService.java b/java/serving/src/main/java/feast/serving/specs/CachedSpecService.java deleted file mode 100644 index 440b2245ac..0000000000 --- a/java/serving/src/main/java/feast/serving/specs/CachedSpecService.java +++ /dev/null @@ -1,219 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.specs; - -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import feast.proto.core.CoreServiceProto; -import feast.proto.core.CoreServiceProto.ListFeatureTablesRequest; -import feast.proto.core.CoreServiceProto.ListFeatureTablesResponse; -import feast.proto.core.CoreServiceProto.ListProjectsRequest; -import feast.proto.core.FeatureProto; -import feast.proto.core.FeatureTableProto.FeatureTable; -import feast.proto.core.FeatureTableProto.FeatureTableSpec; -import feast.proto.serving.ServingAPIProto; -import feast.serving.exception.SpecRetrievalException; -import io.grpc.StatusRuntimeException; -import io.prometheus.client.Gauge; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.slf4j.Logger; - -/** In-memory cache of specs hosted in Feast Core. */ -public class CachedSpecService { - - private static final int MAX_SPEC_COUNT = 1000; - private static final Logger log = org.slf4j.LoggerFactory.getLogger(CachedSpecService.class); - private static final String DEFAULT_PROJECT_NAME = "default"; - - private final CoreSpecService coreService; - - private static Gauge cacheLastUpdated = - Gauge.build() - .name("cache_last_updated") - .subsystem("feast_serving") - .help("epoch time of the last time the cache was updated") - .register(); - - private static Gauge featureTablesCount = - Gauge.build() - .name("feature_table_count") - .subsystem("feast_serving") - .help("number of feature tables served by this instance") - .register(); - - private final LoadingCache, FeatureTableSpec> featureTableCache; - - private final LoadingCache< - ImmutablePair, FeatureProto.FeatureSpecV2> - featureCache; - - public CachedSpecService(CoreSpecService coreService) { - this.coreService = coreService; - - CacheLoader, FeatureTableSpec> featureTableCacheLoader = - CacheLoader.from(k -> retrieveSingleFeatureTable(k.getLeft(), k.getRight())); - featureTableCache = - CacheBuilder.newBuilder().maximumSize(MAX_SPEC_COUNT).build(featureTableCacheLoader); - - CacheLoader< - ImmutablePair, FeatureProto.FeatureSpecV2> - featureCacheLoader = - CacheLoader.from(k -> retrieveSingleFeature(k.getLeft(), k.getRight())); - featureCache = CacheBuilder.newBuilder().build(featureCacheLoader); - } - - /** - * Reload the store configuration from the given config path, then retrieve the necessary specs - * from core to preload the cache. - */ - public void populateCache() { - ImmutablePair< - HashMap, FeatureTableSpec>, - HashMap< - ImmutablePair, - FeatureProto.FeatureSpecV2>> - specs = getFeatureTableMap(); - - featureTableCache.invalidateAll(); - featureTableCache.putAll(specs.getLeft()); - - featureTablesCount.set(featureTableCache.size()); - - featureCache.invalidateAll(); - featureCache.putAll(specs.getRight()); - - cacheLastUpdated.set(System.currentTimeMillis()); - } - - public void scheduledPopulateCache() { - try { - populateCache(); - } catch (Exception e) { - log.warn("Error updating store configuration and specs: {}", e.getMessage()); - } - } - - /** - * Provides a map for easy retrieval of FeatureTable spec using FeatureTable reference - * - * @return Map in the format of - */ - private ImmutablePair< - HashMap, FeatureTableSpec>, - HashMap< - ImmutablePair, - FeatureProto.FeatureSpecV2>> - getFeatureTableMap() { - HashMap, FeatureTableSpec> featureTables = new HashMap<>(); - HashMap, FeatureProto.FeatureSpecV2> - features = new HashMap<>(); - - List projects = - coreService.listProjects(ListProjectsRequest.newBuilder().build()).getProjectsList(); - - for (String project : projects) { - try { - ListFeatureTablesResponse featureTablesResponse = - coreService.listFeatureTables( - ListFeatureTablesRequest.newBuilder() - .setFilter(ListFeatureTablesRequest.Filter.newBuilder().setProject(project)) - .build()); - Map featureRefSpecMap = - new HashMap<>(); - for (FeatureTable featureTable : featureTablesResponse.getTablesList()) { - FeatureTableSpec spec = featureTable.getSpec(); - featureTables.put(ImmutablePair.of(project, spec.getName()), spec); - - String featureTableName = spec.getName(); - List featureSpecs = spec.getFeaturesList(); - for (FeatureProto.FeatureSpecV2 featureSpec : featureSpecs) { - ServingAPIProto.FeatureReferenceV2 featureReference = - ServingAPIProto.FeatureReferenceV2.newBuilder() - .setFeatureTable(featureTableName) - .setName(featureSpec.getName()) - .build(); - features.put(ImmutablePair.of(project, featureReference), featureSpec); - } - } - - } catch (StatusRuntimeException e) { - throw new RuntimeException( - String.format("Unable to retrieve specs matching project %s", project), e); - } - } - return ImmutablePair.of(featureTables, features); - } - - private FeatureTableSpec retrieveSingleFeatureTable(String projectName, String tableName) { - FeatureTable table = - coreService - .getFeatureTable( - CoreServiceProto.GetFeatureTableRequest.newBuilder() - .setProject(projectName) - .setName(tableName) - .build()) - .getTable(); - return table.getSpec(); - } - - private FeatureProto.FeatureSpecV2 retrieveSingleFeature( - String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { - FeatureTableSpec featureTableSpec = - getFeatureTableSpec(projectName, featureReference); // don't stress core too much - if (featureTableSpec == null) { - return null; - } - return featureTableSpec.getFeaturesList().stream() - .filter(f -> f.getName().equals(featureReference.getName())) - .findFirst() - .orElse(null); - } - - public FeatureTableSpec getFeatureTableSpec( - String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { - FeatureTableSpec featureTableSpec; - try { - featureTableSpec = - featureTableCache.get(ImmutablePair.of(projectName, featureReference.getFeatureTable())); - } catch (ExecutionException | CacheLoader.InvalidCacheLoadException e) { - throw new SpecRetrievalException( - String.format( - "Unable to find FeatureTable %s/%s", projectName, featureReference.getFeatureTable()), - e); - } - - return featureTableSpec; - } - - public FeatureProto.FeatureSpecV2 getFeatureSpec( - String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { - FeatureProto.FeatureSpecV2 featureSpec; - try { - featureSpec = featureCache.get(ImmutablePair.of(projectName, featureReference)); - } catch (ExecutionException | CacheLoader.InvalidCacheLoadException e) { - throw new SpecRetrievalException( - String.format("Unable to find Feature with name: %s", featureReference.getName()), e); - } - - return featureSpec; - } -} diff --git a/java/serving/src/main/java/feast/serving/specs/CoreFeatureSpecRetriever.java b/java/serving/src/main/java/feast/serving/specs/CoreFeatureSpecRetriever.java deleted file mode 100644 index 2a88659cba..0000000000 --- a/java/serving/src/main/java/feast/serving/specs/CoreFeatureSpecRetriever.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2021 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.specs; - -import com.google.protobuf.Duration; -import feast.proto.core.FeatureProto; -import feast.proto.core.FeatureViewProto; -import feast.proto.core.OnDemandFeatureViewProto; -import feast.proto.serving.ServingAPIProto; -import java.util.List; - -public class CoreFeatureSpecRetriever implements FeatureSpecRetriever { - private final CachedSpecService specService; - - public CoreFeatureSpecRetriever(CachedSpecService specService) { - this.specService = specService; - } - - @Override - public Duration getMaxAge( - String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { - return this.specService.getFeatureTableSpec(projectName, featureReference).getMaxAge(); - } - - @Override - public List getEntitiesList( - String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { - return this.specService.getFeatureTableSpec(projectName, featureReference).getEntitiesList(); - } - - @Override - public FeatureProto.FeatureSpecV2 getFeatureSpec( - String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { - return this.specService.getFeatureSpec(projectName, featureReference); - } - - @Override - public FeatureViewProto.FeatureViewSpec getBatchFeatureViewSpec( - String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { - throw new UnsupportedOperationException( - String.format("Feast Core does not support getting feature view specs.")); - } - - @Override - public OnDemandFeatureViewProto.OnDemandFeatureViewSpec getOnDemandFeatureViewSpec( - String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { - throw new UnsupportedOperationException( - String.format("Feast Core does not support on demand feature views.")); - } - - @Override - public boolean isOnDemandFeatureReference(ServingAPIProto.FeatureReferenceV2 featureReference) { - return false; - } -} diff --git a/java/serving/src/main/java/feast/serving/specs/CoreSpecService.java b/java/serving/src/main/java/feast/serving/specs/CoreSpecService.java deleted file mode 100644 index eee50d85da..0000000000 --- a/java/serving/src/main/java/feast/serving/specs/CoreSpecService.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.specs; - -import feast.proto.core.CoreServiceGrpc; -import feast.proto.core.CoreServiceProto; -import feast.proto.core.CoreServiceProto.ListFeatureTablesRequest; -import feast.proto.core.CoreServiceProto.ListFeatureTablesResponse; -import feast.proto.core.CoreServiceProto.ListProjectsRequest; -import feast.proto.core.CoreServiceProto.ListProjectsResponse; -import feast.proto.core.CoreServiceProto.UpdateStoreRequest; -import feast.proto.core.CoreServiceProto.UpdateStoreResponse; -import io.grpc.CallCredentials; -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; -import org.slf4j.Logger; -import org.springframework.beans.factory.ObjectProvider; - -/** Client for interfacing with specs in Feast Core. */ -public class CoreSpecService { - - private static final Logger log = org.slf4j.LoggerFactory.getLogger(CoreSpecService.class); - private final CoreServiceGrpc.CoreServiceBlockingStub blockingStub; - - public CoreSpecService( - String feastCoreHost, int feastCorePort, ObjectProvider callCredentials) { - ManagedChannel channel = - ManagedChannelBuilder.forAddress(feastCoreHost, feastCorePort).usePlaintext().build(); - CallCredentials creds = callCredentials.getIfAvailable(); - if (creds != null) { - blockingStub = CoreServiceGrpc.newBlockingStub(channel).withCallCredentials(creds); - } else { - blockingStub = CoreServiceGrpc.newBlockingStub(channel); - } - } - - public UpdateStoreResponse updateStore(UpdateStoreRequest updateStoreRequest) { - return blockingStub.updateStore(updateStoreRequest); - } - - public ListProjectsResponse listProjects(ListProjectsRequest listProjectsRequest) { - return blockingStub.listProjects(listProjectsRequest); - } - - public ListFeatureTablesResponse listFeatureTables( - ListFeatureTablesRequest listFeatureTablesRequest) { - return blockingStub.listFeatureTables(listFeatureTablesRequest); - } - - public CoreServiceProto.GetFeatureTableResponse getFeatureTable( - CoreServiceProto.GetFeatureTableRequest getFeatureTableRequest) { - return blockingStub.getFeatureTable(getFeatureTableRequest); - } -} diff --git a/java/serving/src/main/java/feast/serving/specs/RegistryFeatureSpecRetriever.java b/java/serving/src/main/java/feast/serving/specs/RegistryFeatureSpecRetriever.java index 0cd851ed87..540e7f504e 100644 --- a/java/serving/src/main/java/feast/serving/specs/RegistryFeatureSpecRetriever.java +++ b/java/serving/src/main/java/feast/serving/specs/RegistryFeatureSpecRetriever.java @@ -20,9 +20,7 @@ import feast.proto.core.FeatureProto; import feast.proto.core.FeatureViewProto; import feast.proto.core.OnDemandFeatureViewProto; -import feast.proto.core.RegistryProto; import feast.proto.serving.ServingAPIProto; -import feast.serving.exception.SpecRetrievalException; import feast.serving.registry.RegistryRepository; import java.util.List; @@ -36,29 +34,15 @@ public RegistryFeatureSpecRetriever(RegistryRepository registryRepository) { @Override public Duration getMaxAge( String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { - final RegistryProto.Registry registry = this.registryRepository.getRegistry(); - for (final FeatureViewProto.FeatureView featureView : registry.getFeatureViewsList()) { - if (featureView.getSpec().getName().equals(featureReference.getFeatureTable())) { - return featureView.getSpec().getTtl(); - } - } - throw new SpecRetrievalException( - String.format( - "Unable to find feature view with name: %s", featureReference.getFeatureTable())); + return this.registryRepository.getFeatureViewSpec(projectName, featureReference).getTtl(); } @Override public List getEntitiesList( String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { - final RegistryProto.Registry registry = this.registryRepository.getRegistry(); - for (final FeatureViewProto.FeatureView featureView : registry.getFeatureViewsList()) { - if (featureView.getSpec().getName().equals(featureReference.getFeatureTable())) { - return featureView.getSpec().getEntitiesList(); - } - } - throw new SpecRetrievalException( - String.format( - "Unable to find feature view with name: %s", featureReference.getFeatureTable())); + return this.registryRepository + .getFeatureViewSpec(projectName, featureReference) + .getEntitiesList(); } @Override diff --git a/java/serving/src/main/resources/application.yml b/java/serving/src/main/resources/application.yml index 3e4e07be6d..087bb46f2f 100644 --- a/java/serving/src/main/resources/application.yml +++ b/java/serving/src/main/resources/application.yml @@ -1,42 +1,8 @@ feast: - # GRPC service address for Feast Core - # Feast Serving requires connection to Feast Core to retrieve and reload Feast metadata (e.g. FeatureSpecs, Store information) - core-host: ${FEAST_CORE_HOST:localhost} - core-grpc-port: ${FEAST_CORE_GRPC_PORT:6565} - - core-authentication: - enabled: false # should be set to true if authentication is enabled on core. - provider: google # can be set to `oauth` or `google` - # if google, GOOGLE_APPLICATION_CREDENTIALS environment variable should be set. - options: - #if provider is oauth following properties need to be set, else serving boot up will fail. - oauth_url: https://localhost/oauth/token #oauth token request url - grant_type: client_credentials #oauth grant type - client_id: #oauth client id which will be used for jwt token token request - client_secret: #oauth client secret which will be used for jwt token token request - audience: https://localhost #token audience. - jwkEndpointURI: #jwk enpoint uri, used for caching token till expiry. - - core-cache-refresh-interval: 10 - # Indicates the active store. Only a single store in the last can be active at one time. In the future this key # will be deprecated in order to allow multiple stores to be served from a single serving instance active_store: online - security: - authentication: - enabled: false - provider: jwt - options: - jwkEndpointURI: "https://www.googleapis.com/oauth2/v3/certs" - subjectClaim: email - - authorization: - enabled: false - provider: http - options: - basePath: http://localhost:3000 - # List of store configurations stores: # Please see https://api.docs.feast.dev/grpc/feast.core.pb.html#Store for configuration options @@ -54,18 +20,6 @@ feast: read_from: MASTER # Redis operation timeout in ISO-8601 format timeout: PT0.5S - - name: bigtable - type: BIGTABLE - config: - project_id: - instance_id: - app_profile_id: - - name: cassandra - type: CASSANDRA - config: - connection_string: localhost:9042 - data_center: datacenter1 - keyspace: feast tracing: # If true, Feast will provide tracing data (using OpenTracing API) for various RPC method calls # which can be useful to debug performance issues and perform benchmarking diff --git a/java/serving/src/test/java/feast/serving/controller/ServingServiceGRpcControllerTest.java b/java/serving/src/test/java/feast/serving/controller/ServingServiceGRpcControllerTest.java index c0901c48bc..b7f0231521 100644 --- a/java/serving/src/test/java/feast/serving/controller/ServingServiceGRpcControllerTest.java +++ b/java/serving/src/test/java/feast/serving/controller/ServingServiceGRpcControllerTest.java @@ -16,20 +16,9 @@ */ package feast.serving.controller; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; import static org.mockito.MockitoAnnotations.initMocks; import com.google.protobuf.Timestamp; -import feast.common.auth.authorization.AuthorizationProvider; -import feast.common.auth.authorization.AuthorizationResult; -import feast.common.auth.config.SecurityProperties; -import feast.common.auth.config.SecurityProperties.AuthenticationProperties; -import feast.common.auth.config.SecurityProperties.AuthorizationProperties; -import feast.common.auth.service.AuthorizationService; import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow; @@ -45,8 +34,6 @@ import org.mockito.Mock; import org.mockito.Mockito; import org.springframework.security.core.Authentication; -import org.springframework.security.core.context.SecurityContext; -import org.springframework.security.core.context.SecurityContextHolder; public class ServingServiceGRpcControllerTest { @@ -60,8 +47,6 @@ public class ServingServiceGRpcControllerTest { @Mock private Authentication authentication; - @Mock private AuthorizationProvider authProvider; - @Before public void setUp() { initMocks(this); @@ -90,18 +75,7 @@ private ServingServiceGRpcController getServingServiceGRpcController(boolean ena Tracer tracer = Configuration.fromEnv("dummy").getTracer(); FeastProperties feastProperties = new FeastProperties(); - AuthorizationProperties authorizationProps = new AuthorizationProperties(); - authorizationProps.setEnabled(enableAuth); - AuthenticationProperties authenticationProps = new AuthenticationProperties(); - authenticationProps.setEnabled(enableAuth); - SecurityProperties securityProperties = new SecurityProperties(); - securityProperties.setAuthentication(authenticationProps); - securityProperties.setAuthorization(authorizationProps); - feastProperties.setSecurity(securityProperties); - AuthorizationService authorizationservice = - new AuthorizationService(feastProperties.getSecurity(), authProvider); - return new ServingServiceGRpcController( - authorizationservice, mockServingServiceV2, feastProperties, tracer); + return new ServingServiceGRpcController(mockServingServiceV2, feastProperties, tracer); } @Test @@ -119,17 +93,4 @@ public void shouldCallOnErrorIfEntityDatasetIsNotSet() { service.getOnlineFeaturesV2(missingEntityName, mockStreamObserver); Mockito.verify(mockStreamObserver).onError(Mockito.any(IllegalArgumentException.class)); } - - @Test - public void shouldPassValidRequestAsIsIfRequestIsAuthorized() { - service = getServingServiceGRpcController(true); - SecurityContext context = mock(SecurityContext.class); - SecurityContextHolder.setContext(context); - when(context.getAuthentication()).thenReturn(authentication); - doReturn(AuthorizationResult.success()) - .when(authProvider) - .checkAccessToProject(anyString(), any(Authentication.class)); - service.getOnlineFeaturesV2(validRequest, mockStreamObserver); - Mockito.verify(mockServingServiceV2).getOnlineFeatures(validRequest); - } } diff --git a/java/serving/src/test/java/feast/serving/it/AuthTestUtils.java b/java/serving/src/test/java/feast/serving/it/AuthTestUtils.java deleted file mode 100644 index a4c3db7def..0000000000 --- a/java/serving/src/test/java/feast/serving/it/AuthTestUtils.java +++ /dev/null @@ -1,195 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.it; - -import com.google.gson.JsonArray; -import com.google.gson.JsonObject; -import com.google.protobuf.Timestamp; -import feast.common.auth.credentials.OAuthCredentials; -import feast.proto.core.CoreServiceGrpc; -import feast.proto.serving.ServingAPIProto; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; -import feast.proto.serving.ServingServiceGrpc; -import feast.proto.types.ValueProto.Value; -import io.grpc.CallCredentials; -import io.grpc.Channel; -import io.grpc.ManagedChannelBuilder; -import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; -import java.util.Map; -import okhttp3.MediaType; -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.RequestBody; -import okhttp3.Response; -import org.junit.runners.model.InitializationError; -import sh.ory.keto.ApiClient; -import sh.ory.keto.ApiException; -import sh.ory.keto.Configuration; -import sh.ory.keto.api.EnginesApi; -import sh.ory.keto.model.OryAccessControlPolicy; -import sh.ory.keto.model.OryAccessControlPolicyRole; - -public class AuthTestUtils { - - private static final String DEFAULT_FLAVOR = "glob"; - - public static GetOnlineFeaturesRequestV2 createOnlineFeatureRequest( - String projectName, - String featureTableName, - String featureName, - String entityId, - int entityValue) { - return GetOnlineFeaturesRequestV2.newBuilder() - .setProject(projectName) - .addFeatures( - ServingAPIProto.FeatureReferenceV2.newBuilder() - .setFeatureTable(featureTableName) - .setName(featureName) - .build()) - .addEntityRows( - GetOnlineFeaturesRequestV2.EntityRow.newBuilder() - .setTimestamp(Timestamp.newBuilder().setSeconds(100)) - .putFields(entityId, Value.newBuilder().setInt64Val(entityValue).build())) - .build(); - } - - public static CoreSimpleAPIClient getSecureApiClientForCore( - int feastCorePort, Map options) { - CallCredentials callCredentials = null; - callCredentials = new OAuthCredentials(options); - Channel secureChannel = - ManagedChannelBuilder.forAddress("localhost", feastCorePort).usePlaintext().build(); - - CoreServiceGrpc.CoreServiceBlockingStub secureCoreService = - CoreServiceGrpc.newBlockingStub(secureChannel).withCallCredentials(callCredentials); - - return new CoreSimpleAPIClient(secureCoreService); - } - - public static ServingServiceGrpc.ServingServiceBlockingStub getServingServiceStub( - boolean isSecure, int feastServingPort, Map options) { - Channel secureChannel = - ManagedChannelBuilder.forAddress("localhost", feastServingPort).usePlaintext().build(); - - if (isSecure) { - CallCredentials callCredentials = null; - callCredentials = new OAuthCredentials(options); - return ServingServiceGrpc.newBlockingStub(secureChannel).withCallCredentials(callCredentials); - } else { - return ServingServiceGrpc.newBlockingStub(secureChannel); - } - } - - public static void seedHydra( - String hydraExternalUrl, - String clientId, - String clientSecrret, - String audience, - String grantType) - throws IOException, InitializationError { - - OkHttpClient httpClient = new OkHttpClient(); - String createClientEndpoint = String.format("%s/%s", hydraExternalUrl, "clients"); - JsonObject jsonObject = new JsonObject(); - JsonArray audienceArrray = new JsonArray(); - audienceArrray.add(audience); - JsonArray grantTypes = new JsonArray(); - grantTypes.add(grantType); - jsonObject.addProperty("client_id", clientId); - jsonObject.addProperty("client_secret", clientSecrret); - jsonObject.addProperty("token_endpoint_auth_method", "client_secret_post"); - jsonObject.add("audience", audienceArrray); - jsonObject.add("grant_types", grantTypes); - MediaType JSON = MediaType.parse("application/json; charset=utf-8"); - - RequestBody requestBody = RequestBody.create(JSON, jsonObject.toString()); - Request request = - new Request.Builder() - .url(createClientEndpoint) - .addHeader("Content-Type", "application/json") - .post(requestBody) - .build(); - Response response = httpClient.newCall(request).execute(); - if (!response.isSuccessful()) { - throw new InitializationError(response.message()); - } - } - - public static void seedKeto(String url, String project, String subjectInProject, String admin) - throws ApiException { - ApiClient ketoClient = Configuration.getDefaultApiClient(); - ketoClient.setBasePath(url); - EnginesApi enginesApi = new EnginesApi(ketoClient); - - // Add policies - OryAccessControlPolicy adminPolicy = getAdminPolicy(); - enginesApi.upsertOryAccessControlPolicy(DEFAULT_FLAVOR, adminPolicy); - - OryAccessControlPolicy projectPolicy = getMyProjectMemberPolicy(project); - enginesApi.upsertOryAccessControlPolicy(DEFAULT_FLAVOR, projectPolicy); - - // Add policy roles - OryAccessControlPolicyRole adminPolicyRole = getAdminPolicyRole(admin); - enginesApi.upsertOryAccessControlPolicyRole(DEFAULT_FLAVOR, adminPolicyRole); - - OryAccessControlPolicyRole myProjectMemberPolicyRole = - getMyProjectMemberPolicyRole(project, subjectInProject); - enginesApi.upsertOryAccessControlPolicyRole(DEFAULT_FLAVOR, myProjectMemberPolicyRole); - } - - private static OryAccessControlPolicyRole getMyProjectMemberPolicyRole( - String project, String subjectInProject) { - OryAccessControlPolicyRole role = new OryAccessControlPolicyRole(); - role.setId(String.format("roles:%s-project-members", project)); - role.setMembers(Collections.singletonList("users:" + subjectInProject)); - return role; - } - - private static OryAccessControlPolicyRole getAdminPolicyRole(String subjectIsAdmin) { - OryAccessControlPolicyRole role = new OryAccessControlPolicyRole(); - role.setId("roles:admin"); - role.setMembers(Collections.singletonList("users:" + subjectIsAdmin)); - return role; - } - - private static OryAccessControlPolicy getAdminPolicy() { - OryAccessControlPolicy policy = new OryAccessControlPolicy(); - policy.setId("policies:admin"); - policy.subjects(Collections.singletonList("roles:admin")); - policy.resources(Collections.singletonList("resources:**")); - policy.actions(Collections.singletonList("actions:**")); - policy.effect("allow"); - policy.conditions(null); - return policy; - } - - private static OryAccessControlPolicy getMyProjectMemberPolicy(String project) { - OryAccessControlPolicy policy = new OryAccessControlPolicy(); - policy.setId(String.format("policies:%s-project-members-policy", project)); - policy.subjects(Collections.singletonList(String.format("roles:%s-project-members", project))); - policy.resources( - Arrays.asList( - String.format("resources:projects:%s", project), - String.format("resources:projects:%s:**", project))); - policy.actions(Collections.singletonList("actions:**")); - policy.effect("allow"); - policy.conditions(null); - return policy; - } -} diff --git a/java/serving/src/test/java/feast/serving/it/BaseAuthIT.java b/java/serving/src/test/java/feast/serving/it/BaseAuthIT.java deleted file mode 100644 index d49ac41fad..0000000000 --- a/java/serving/src/test/java/feast/serving/it/BaseAuthIT.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.it; - -import java.net.InetAddress; -import java.net.UnknownHostException; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.DynamicPropertyRegistry; -import org.springframework.test.context.DynamicPropertySource; - -@ActiveProfiles("it") -@SpringBootTest -public class BaseAuthIT { - - static final String FEATURE_TABLE_NAME = "featuretable_1"; - static final String FEATURE_NAME = "feature_1"; - static final String ENTITY_ID = "entity_id"; - static final String PROJECT_NAME = "project_1"; - static final int SERVICE_START_MAX_WAIT_TIME_IN_MINUTES = 3; - static final String CLIENT_ID = "client_id"; - static final String CLIENT_SECRET = "client_secret"; - static final String TOKEN_URL = "http://localhost:4444/oauth2/token"; - static final String JWK_URI = "http://localhost:4444/.well-known/jwks.json"; - - static final String GRANT_TYPE = "client_credentials"; - - static final String AUDIENCE = "https://localhost"; - - static final String CORE = "core_1"; - - static final String HYDRA = "hydra_1"; - static final int HYDRA_PORT = 4445; - - static CoreSimpleAPIClient insecureApiClient; - - static final String REDIS = "redis_1"; - static final int REDIS_PORT = 6379; - - static final String BIGTABLE = "bigtable_1"; - static final int BIGTABLE_PORT = 8086; - - static final String CASSANDRA = "cassandra_1"; - static final int CASSANDRA_PORT = 9042; - static final String CASSANDRA_DATACENTER = "datacenter1"; - static final String CASSANDRA_KEYSPACE = "feast"; - static final String CASSANDRA_SCHEMA_TABLE = "feast_schema_reference"; - static final String CASSANDRA_ENTITY_KEY = "key"; - - static final int FEAST_CORE_PORT = 6565; - - @DynamicPropertySource - static void properties(DynamicPropertyRegistry registry) { - registry.add("feast.stores[0].name", () -> "online"); - registry.add("feast.stores[0].type", () -> "REDIS"); - // Redis needs to accessible by both core and serving, hence using host address - registry.add( - "feast.stores[0].config.host", - () -> { - try { - return InetAddress.getLocalHost().getHostAddress(); - } catch (UnknownHostException e) { - e.printStackTrace(); - return ""; - } - }); - registry.add("feast.stores[0].config.port", () -> REDIS_PORT); - - registry.add("feast.stores[1].name", () -> "bigtable"); - registry.add("feast.stores[1].type", () -> "BIGTABLE"); - registry.add("feast.stores[1].config.project_id", () -> "test-project"); - registry.add("feast.stores[1].config.instance_id", () -> "test-instance"); - - registry.add("feast.stores[2].name", () -> "cassandra"); - registry.add("feast.stores[2].type", () -> "CASSANDRA"); - registry.add( - "feast.stores[2].config.host", - () -> { - try { - return InetAddress.getLocalHost().getHostAddress(); - } catch (UnknownHostException e) { - e.printStackTrace(); - return ""; - } - }); - - registry.add( - "feast.stores[2].config.connection_string", - () -> { - String hostAddress = ""; - try { - hostAddress = InetAddress.getLocalHost().getHostAddress(); - } catch (UnknownHostException e) { - e.printStackTrace(); - } - - return String.format("%s:%s", hostAddress, CASSANDRA_PORT); - }); - registry.add("feast.stores[2].config.data_center", () -> CASSANDRA_DATACENTER); - registry.add("feast.stores[2].config.keyspace", () -> CASSANDRA_KEYSPACE); - - registry.add("feast.core-authentication.options.oauth_url", () -> TOKEN_URL); - registry.add("feast.core-authentication.options.grant_type", () -> GRANT_TYPE); - registry.add("feast.core-authentication.options.client_id", () -> CLIENT_ID); - registry.add("feast.core-authentication.options.client_secret", () -> CLIENT_SECRET); - registry.add("feast.core-authentication.options.audience", () -> AUDIENCE); - registry.add("feast.core-authentication.options.jwkEndpointURI", () -> JWK_URI); - registry.add("feast.security.authentication.options.jwkEndpointURI", () -> JWK_URI); - } -} diff --git a/java/serving/src/test/java/feast/serving/it/CoreSimpleAPIClient.java b/java/serving/src/test/java/feast/serving/it/CoreSimpleAPIClient.java deleted file mode 100644 index f7bc12f5fc..0000000000 --- a/java/serving/src/test/java/feast/serving/it/CoreSimpleAPIClient.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.it; - -import feast.proto.core.CoreServiceGrpc; -import feast.proto.core.CoreServiceProto; -import feast.proto.core.EntityProto; -import feast.proto.core.FeatureTableProto; - -public class CoreSimpleAPIClient { - private CoreServiceGrpc.CoreServiceBlockingStub stub; - - public CoreSimpleAPIClient(CoreServiceGrpc.CoreServiceBlockingStub stub) { - this.stub = stub; - } - - public void simpleApplyEntity(String projectName, EntityProto.EntitySpecV2 entitySpec) { - stub.applyEntity( - CoreServiceProto.ApplyEntityRequest.newBuilder() - .setProject(projectName) - .setSpec(entitySpec) - .build()); - } - - public EntityProto.Entity getEntity(String projectName, String name) { - return stub.getEntity( - CoreServiceProto.GetEntityRequest.newBuilder() - .setProject(projectName) - .setName(name) - .build()) - .getEntity(); - } - - public void simpleApplyFeatureTable( - String projectName, FeatureTableProto.FeatureTableSpec featureTable) { - stub.applyFeatureTable( - CoreServiceProto.ApplyFeatureTableRequest.newBuilder() - .setProject(projectName) - .setTableSpec(featureTable) - .build()); - } - - public FeatureTableProto.FeatureTable simpleGetFeatureTable(String projectName, String name) { - return stub.getFeatureTable( - CoreServiceProto.GetFeatureTableRequest.newBuilder() - .setName(name) - .setProject(projectName) - .build()) - .getTable(); - } -} diff --git a/java/serving/src/test/java/feast/serving/it/ServingServiceBigTableIT.java b/java/serving/src/test/java/feast/serving/it/ServingServiceBigTableIT.java deleted file mode 100644 index 21bddb5b82..0000000000 --- a/java/serving/src/test/java/feast/serving/it/ServingServiceBigTableIT.java +++ /dev/null @@ -1,873 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2021 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.it; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import com.google.api.gax.core.CredentialsProvider; -import com.google.api.gax.core.NoCredentialsProvider; -import com.google.api.gax.grpc.GrpcTransportChannel; -import com.google.api.gax.rpc.FixedTransportChannelProvider; -import com.google.api.gax.rpc.TransportChannelProvider; -import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; -import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; -import com.google.cloud.bigtable.admin.v2.stub.BigtableTableAdminStubSettings; -import com.google.cloud.bigtable.admin.v2.stub.EnhancedBigtableTableAdminStub; -import com.google.cloud.bigtable.data.v2.BigtableDataClient; -import com.google.cloud.bigtable.data.v2.BigtableDataSettings; -import com.google.cloud.bigtable.data.v2.models.RowMutation; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.hash.Hashing; -import com.google.protobuf.ByteString; -import feast.common.it.DataGenerator; -import feast.common.models.FeatureV2; -import feast.proto.core.EntityProto; -import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; -import feast.proto.serving.ServingServiceGrpc; -import feast.proto.types.ValueProto; -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.time.Duration; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import org.apache.avro.Schema; -import org.apache.avro.SchemaBuilder; -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericDatumWriter; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.GenericRecordBuilder; -import org.apache.avro.io.Encoder; -import org.apache.avro.io.EncoderFactory; -import org.junit.ClassRule; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.context.TestConfiguration; -import org.springframework.context.annotation.Bean; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.DynamicPropertyRegistry; -import org.springframework.test.context.DynamicPropertySource; -import org.testcontainers.containers.DockerComposeContainer; -import org.testcontainers.containers.wait.strategy.Wait; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; - -@ActiveProfiles("it") -@SpringBootTest( - webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, - properties = { - "feast.core-cache-refresh-interval=1", - "feast.active_store=bigtable", - "spring.main.allow-bean-definition-overriding=true" - }) -@Testcontainers -public class ServingServiceBigTableIT extends BaseAuthIT { - - static final Map options = new HashMap<>(); - static CoreSimpleAPIClient coreClient; - static ServingServiceGrpc.ServingServiceBlockingStub servingStub; - - static BigtableDataClient client; - static final int FEAST_SERVING_PORT = 6569; - - static final String PROJECT_ID = "test-project"; - static final String INSTANCE_ID = "test-instance"; - static final String APP_PROFILE_ID = "default"; - static ManagedChannel channel; - - static final FeatureReferenceV2 feature1Reference = - DataGenerator.createFeatureReference("rides", "trip_cost"); - static final FeatureReferenceV2 feature2Reference = - DataGenerator.createFeatureReference("rides", "trip_distance"); - static final FeatureReferenceV2 feature3Reference = - DataGenerator.createFeatureReference("rides", "trip_empty"); - static final FeatureReferenceV2 feature4Reference = - DataGenerator.createFeatureReference("rides", "trip_wrong_type"); - - @ClassRule @Container - public static DockerComposeContainer environment = - new DockerComposeContainer( - new File("src/test/resources/docker-compose/docker-compose-bigtable-it.yml")) - .withExposedService( - CORE, - FEAST_CORE_PORT, - Wait.forLogMessage(".*gRPC Server started.*\\n", 1) - .withStartupTimeout(Duration.ofMinutes(SERVICE_START_MAX_WAIT_TIME_IN_MINUTES))) - .withExposedService(BIGTABLE, BIGTABLE_PORT); - - @DynamicPropertySource - static void initialize(DynamicPropertyRegistry registry) { - registry.add("grpc.server.port", () -> FEAST_SERVING_PORT); - } - - @BeforeAll - static void globalSetup() throws IOException { - coreClient = TestUtils.getApiClientForCore(FEAST_CORE_PORT); - servingStub = TestUtils.getServingServiceStub(false, FEAST_SERVING_PORT, null); - - // Initialize BigTable Client - client = - BigtableDataClient.create( - BigtableDataSettings.newBuilderForEmulator( - environment.getServiceHost("bigtable_1", BIGTABLE_PORT), - environment.getServicePort("bigtable_1", BIGTABLE_PORT)) - .setProjectId(PROJECT_ID) - .setInstanceId(INSTANCE_ID) - .build()); - - String endpoint = - environment.getServiceHost("bigtable_1", BIGTABLE_PORT) - + ":" - + environment.getServicePort("bigtable_1", BIGTABLE_PORT); - channel = ManagedChannelBuilder.forTarget(endpoint).usePlaintext().build(); - - /** Feast resource creation Workflow */ - String projectName = "default"; - // Apply Entity (driver_id) - String driverEntityName = "driver_id"; - String driverEntityDescription = "My driver id"; - ValueProto.ValueType.Enum driverEntityType = ValueProto.ValueType.Enum.INT64; - EntityProto.EntitySpecV2 driverEntitySpec = - EntityProto.EntitySpecV2.newBuilder() - .setName(driverEntityName) - .setDescription(driverEntityDescription) - .setValueType(driverEntityType) - .build(); - TestUtils.applyEntity(coreClient, projectName, driverEntitySpec); - - // Apply Entity (this_is_a_long_long_long_long_long_long_entity_id) - String superLongEntityName = "this_is_a_long_long_long_long_long_long_entity_id"; - String superLongEntityDescription = "My super long entity id"; - ValueProto.ValueType.Enum superLongEntityType = ValueProto.ValueType.Enum.INT64; - EntityProto.EntitySpecV2 superLongEntitySpec = - EntityProto.EntitySpecV2.newBuilder() - .setName(superLongEntityName) - .setDescription(superLongEntityDescription) - .setValueType(superLongEntityType) - .build(); - TestUtils.applyEntity(coreClient, projectName, superLongEntitySpec); - - // Apply Entity (merchant_id) - String merchantEntityName = "merchant_id"; - String merchantEntityDescription = "My merchant id"; - ValueProto.ValueType.Enum merchantEntityType = ValueProto.ValueType.Enum.INT64; - EntityProto.EntitySpecV2 merchantEntitySpec = - EntityProto.EntitySpecV2.newBuilder() - .setName(merchantEntityName) - .setDescription(merchantEntityDescription) - .setValueType(merchantEntityType) - .build(); - TestUtils.applyEntity(coreClient, projectName, merchantEntitySpec); - - // Apply FeatureTable (rides) - String ridesFeatureTableName = "rides"; - ImmutableList ridesEntities = ImmutableList.of(driverEntityName); - ImmutableMap ridesFeatures = - ImmutableMap.of( - "trip_cost", - ValueProto.ValueType.Enum.INT64, - "trip_distance", - ValueProto.ValueType.Enum.DOUBLE, - "trip_empty", - ValueProto.ValueType.Enum.DOUBLE, - "trip_wrong_type", - ValueProto.ValueType.Enum.STRING); - TestUtils.applyFeatureTable( - coreClient, projectName, ridesFeatureTableName, ridesEntities, ridesFeatures, 7200); - - // Apply FeatureTable (superLong) - String superLongFeatureTableName = "superlong"; - ImmutableList superLongEntities = ImmutableList.of(superLongEntityName); - ImmutableMap superLongFeatures = - ImmutableMap.of( - "trip_cost", - ValueProto.ValueType.Enum.INT64, - "trip_distance", - ValueProto.ValueType.Enum.DOUBLE, - "trip_empty", - ValueProto.ValueType.Enum.DOUBLE, - "trip_wrong_type", - ValueProto.ValueType.Enum.STRING); - TestUtils.applyFeatureTable( - coreClient, - projectName, - superLongFeatureTableName, - superLongEntities, - superLongFeatures, - 7200); - - // Apply FeatureTable (rides_merchant) - String rideMerchantFeatureTableName = "rides_merchant"; - ImmutableList ridesMerchantEntities = - ImmutableList.of(driverEntityName, merchantEntityName); - TestUtils.applyFeatureTable( - coreClient, - projectName, - rideMerchantFeatureTableName, - ridesMerchantEntities, - ridesFeatures, - 7200); - - // BigTable Table names - String superLongBtTableName = String.format("%s__%s", projectName, superLongEntityName); - String hashSuffix = - Hashing.murmur3_32().hashBytes(superLongBtTableName.substring(42).getBytes()).toString(); - superLongBtTableName = - superLongBtTableName - .substring(0, Math.min(superLongBtTableName.length(), 42)) - .concat(hashSuffix); - String btTableName = String.format("%s__%s", projectName, driverEntityName); - String compoundBtTableName = - String.format( - "%s__%s", - projectName, ridesMerchantEntities.stream().collect(Collectors.joining("__"))); - String featureTableName = "rides"; - String metadataColumnFamily = "metadata"; - ImmutableList columnFamilies = ImmutableList.of(featureTableName, metadataColumnFamily); - ImmutableList compoundColumnFamilies = - ImmutableList.of(rideMerchantFeatureTableName, metadataColumnFamily); - - /** Single Entity Ingestion Workflow */ - Schema ftSchema = - SchemaBuilder.record("DriverData") - .namespace(featureTableName) - .fields() - .requiredLong(feature1Reference.getName()) - .requiredDouble(feature2Reference.getName()) - .nullableString(feature3Reference.getName(), "null") - .requiredString(feature4Reference.getName()) - .endRecord(); - byte[] schemaReference = - Hashing.murmur3_32().hashBytes(ftSchema.toString().getBytes()).asBytes(); - - GenericRecord record = - new GenericRecordBuilder(ftSchema) - .set("trip_cost", 5L) - .set("trip_distance", 3.5) - .set("trip_empty", null) - .set("trip_wrong_type", "test") - .build(); - byte[] entityFeatureKey = - String.valueOf(DataGenerator.createInt64Value(1).getInt64Val()).getBytes(); - byte[] entityFeatureValue = createEntityValue(ftSchema, schemaReference, record); - byte[] schemaKey = createSchemaKey(schemaReference); - ingestData( - featureTableName, btTableName, entityFeatureKey, entityFeatureValue, schemaKey, ftSchema); - - /** SuperLong Entity Ingestion Workflow */ - Schema superLongFtSchema = - SchemaBuilder.record("SuperLongData") - .namespace(superLongFeatureTableName) - .fields() - .requiredLong(feature1Reference.getName()) - .requiredDouble(feature2Reference.getName()) - .nullableString(feature3Reference.getName(), "null") - .requiredString(feature4Reference.getName()) - .endRecord(); - byte[] superLongSchemaReference = - Hashing.murmur3_32().hashBytes(superLongFtSchema.toString().getBytes()).asBytes(); - - GenericRecord superLongRecord = - new GenericRecordBuilder(superLongFtSchema) - .set("trip_cost", 5L) - .set("trip_distance", 3.5) - .set("trip_empty", null) - .set("trip_wrong_type", "test") - .build(); - byte[] superLongEntityFeatureKey = - String.valueOf(DataGenerator.createInt64Value(1).getInt64Val()).getBytes(); - byte[] superLongEntityFeatureValue = - createEntityValue(superLongFtSchema, superLongSchemaReference, superLongRecord); - byte[] superLongSchemaKey = createSchemaKey(superLongSchemaReference); - ingestData( - superLongFeatureTableName, - superLongBtTableName, - superLongEntityFeatureKey, - superLongEntityFeatureValue, - superLongSchemaKey, - superLongFtSchema); - - /** Compound Entity Ingestion Workflow */ - Schema compoundFtSchema = - SchemaBuilder.record("DriverMerchantData") - .namespace(rideMerchantFeatureTableName) - .fields() - .requiredLong(feature1Reference.getName()) - .requiredDouble(feature2Reference.getName()) - .nullableString(feature3Reference.getName(), "null") - .requiredString(feature4Reference.getName()) - .endRecord(); - byte[] compoundSchemaReference = - Hashing.murmur3_32().hashBytes(compoundFtSchema.toString().getBytes()).asBytes(); - - // Entity-Feature Row - GenericRecord compoundEntityRecord = - new GenericRecordBuilder(compoundFtSchema) - .set("trip_cost", 10L) - .set("trip_distance", 5.5) - .set("trip_empty", null) - .set("trip_wrong_type", "wrong_type") - .build(); - - ValueProto.Value driverEntityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); - ValueProto.Value merchantEntityValue = ValueProto.Value.newBuilder().setInt64Val(1234).build(); - ImmutableMap compoundEntityMap = - ImmutableMap.of( - driverEntityName, driverEntityValue, merchantEntityName, merchantEntityValue); - GetOnlineFeaturesRequestV2.EntityRow entityRow = - DataGenerator.createCompoundEntityRow(compoundEntityMap, 100); - byte[] compoundEntityFeatureKey = - ridesMerchantEntities.stream() - .map(entity -> DataGenerator.valueToString(entityRow.getFieldsMap().get(entity))) - .collect(Collectors.joining("#")) - .getBytes(); - byte[] compoundEntityFeatureValue = - createEntityValue(compoundFtSchema, compoundSchemaReference, compoundEntityRecord); - byte[] compoundSchemaKey = createSchemaKey(compoundSchemaReference); - ingestData( - rideMerchantFeatureTableName, - compoundBtTableName, - compoundEntityFeatureKey, - compoundEntityFeatureValue, - compoundSchemaKey, - compoundFtSchema); - - // set up options for call credentials - options.put("oauth_url", TOKEN_URL); - options.put(CLIENT_ID, CLIENT_ID); - options.put(CLIENT_SECRET, CLIENT_SECRET); - options.put("jwkEndpointURI", JWK_URI); - options.put("audience", AUDIENCE); - options.put("grant_type", GRANT_TYPE); - } - - @AfterAll - static void tearDown() { - ((ManagedChannel) servingStub.getChannel()).shutdown(); - channel.shutdown(); - } - - private static void createTable( - TransportChannelProvider channelProvider, - CredentialsProvider credentialsProvider, - String tableName, - List columnFamilies) - throws IOException { - EnhancedBigtableTableAdminStub stub = - EnhancedBigtableTableAdminStub.createEnhanced( - BigtableTableAdminStubSettings.newBuilder() - .setTransportChannelProvider(channelProvider) - .setCredentialsProvider(credentialsProvider) - .build()); - - try (BigtableTableAdminClient client = - BigtableTableAdminClient.create(PROJECT_ID, INSTANCE_ID, stub)) { - CreateTableRequest createTableRequest = CreateTableRequest.of(tableName); - for (String columnFamily : columnFamilies) { - createTableRequest.addFamily(columnFamily); - } - if (!client.exists(tableName)) { - client.createTable(createTableRequest); - } - } - } - - private static byte[] createSchemaKey(byte[] schemaReference) throws IOException { - String schemaKeyPrefix = "schema#"; - - ByteArrayOutputStream concatOutputStream = new ByteArrayOutputStream(); - concatOutputStream.write(schemaKeyPrefix.getBytes()); - concatOutputStream.write(schemaReference); - byte[] schemaKey = concatOutputStream.toByteArray(); - - return schemaKey; - } - - private static byte[] createEntityValue( - Schema schema, byte[] schemaReference, GenericRecord record) throws IOException { - // Entity-Feature Row - byte[] avroSerializedFeatures = recordToAvro(record, schema); - - ByteArrayOutputStream concatOutputStream = new ByteArrayOutputStream(); - concatOutputStream.write(schemaReference); - concatOutputStream.write("".getBytes()); - concatOutputStream.write(avroSerializedFeatures); - byte[] entityFeatureValue = concatOutputStream.toByteArray(); - - return entityFeatureValue; - } - - private static byte[] schemaReference(Schema schema) { - return Hashing.murmur3_32().hashBytes(schema.toString().getBytes()).asBytes(); - } - - private static void ingestData( - String featureTableName, - String btTableName, - byte[] btEntityFeatureKey, - byte[] btEntityFeatureValue, - byte[] btSchemaKey, - Schema btSchema) - throws IOException { - String emptyQualifier = ""; - String avroQualifier = "avro"; - String metadataColumnFamily = "metadata"; - - TransportChannelProvider channelProvider = - FixedTransportChannelProvider.create(GrpcTransportChannel.create(channel)); - NoCredentialsProvider credentialsProvider = NoCredentialsProvider.create(); - createTable( - channelProvider, - credentialsProvider, - btTableName, - ImmutableList.of(featureTableName, metadataColumnFamily)); - - // Update Compound Entity-Feature Row - client.mutateRow( - RowMutation.create(btTableName, ByteString.copyFrom(btEntityFeatureKey)) - .setCell( - featureTableName, - ByteString.copyFrom(emptyQualifier.getBytes()), - ByteString.copyFrom(btEntityFeatureValue))); - - // Update Schema Row - client.mutateRow( - RowMutation.create(btTableName, ByteString.copyFrom(btSchemaKey)) - .setCell( - metadataColumnFamily, - ByteString.copyFrom(avroQualifier.getBytes()), - ByteString.copyFrom(btSchema.toString().getBytes()))); - } - - private static byte[] recordToAvro(GenericRecord datum, Schema schema) throws IOException { - GenericDatumWriter writer = new GenericDatumWriter<>(schema); - ByteArrayOutputStream output = new ByteArrayOutputStream(); - Encoder encoder = EncoderFactory.get().binaryEncoder(output, null); - writer.write(datum, encoder); - encoder.flush(); - - return output.toByteArray(); - } - - @Test - public void shouldRegisterSingleEntityAndGetOnlineFeatures() { - // getOnlineFeatures Information - String projectName = "default"; - String entityName = "driver_id"; - ValueProto.Value entityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); - - // Instantiate EntityRows - GetOnlineFeaturesRequestV2.EntityRow entityRow1 = - DataGenerator.createEntityRow(entityName, DataGenerator.createInt64Value(1), 100); - ImmutableList entityRows = ImmutableList.of(entityRow1); - - // Instantiate FeatureReferences - FeatureReferenceV2 featureReference = - DataGenerator.createFeatureReference("rides", "trip_cost"); - FeatureReferenceV2 notFoundFeatureReference = - DataGenerator.createFeatureReference("rides", "trip_transaction"); - - ImmutableList featureReferences = - ImmutableList.of(featureReference, notFoundFeatureReference); - - // Build GetOnlineFeaturesRequestV2 - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - - ImmutableMap expectedValueMap = - ImmutableMap.of( - entityName, - entityValue, - FeatureV2.getFeatureStringRef(featureReference), - DataGenerator.createInt64Value(5), - FeatureV2.getFeatureStringRef(notFoundFeatureReference), - DataGenerator.createEmptyValue()); - - ImmutableMap expectedStatusMap = - ImmutableMap.of( - entityName, - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(featureReference), - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(notFoundFeatureReference), - GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND); - - GetOnlineFeaturesResponse.FieldValues expectedFieldValues = - GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(expectedValueMap) - .putAllStatuses(expectedStatusMap) - .build(); - ImmutableList expectedFieldValuesList = - ImmutableList.of(expectedFieldValues); - - assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); - } - - @Test - public void shouldRegisterCompoundEntityAndGetOnlineFeatures() { - String projectName = "default"; - String driverEntityName = "driver_id"; - String merchantEntityName = "merchant_id"; - ValueProto.Value driverEntityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); - ValueProto.Value merchantEntityValue = ValueProto.Value.newBuilder().setInt64Val(1234).build(); - - ImmutableMap compoundEntityMap = - ImmutableMap.of( - driverEntityName, driverEntityValue, merchantEntityName, merchantEntityValue); - - // Instantiate EntityRows - GetOnlineFeaturesRequestV2.EntityRow entityRow = - DataGenerator.createCompoundEntityRow(compoundEntityMap, 100); - ImmutableList entityRows = ImmutableList.of(entityRow); - - // Instantiate FeatureReferences - FeatureReferenceV2 featureReference = - DataGenerator.createFeatureReference("rides", "trip_cost"); - FeatureReferenceV2 notFoundFeatureReference = - DataGenerator.createFeatureReference("rides", "trip_transaction"); - - ImmutableList featureReferences = - ImmutableList.of(featureReference, notFoundFeatureReference); - - // Build GetOnlineFeaturesRequestV2 - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - - ImmutableMap expectedValueMap = - ImmutableMap.of( - driverEntityName, - driverEntityValue, - merchantEntityName, - merchantEntityValue, - FeatureV2.getFeatureStringRef(featureReference), - DataGenerator.createInt64Value(5), - FeatureV2.getFeatureStringRef(notFoundFeatureReference), - DataGenerator.createEmptyValue()); - - ImmutableMap expectedStatusMap = - ImmutableMap.of( - driverEntityName, - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - merchantEntityName, - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(featureReference), - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(notFoundFeatureReference), - GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND); - - GetOnlineFeaturesResponse.FieldValues expectedFieldValues = - GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(expectedValueMap) - .putAllStatuses(expectedStatusMap) - .build(); - ImmutableList expectedFieldValuesList = - ImmutableList.of(expectedFieldValues); - - assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); - } - - @Test - public void shouldReturnCorrectRowCount() { - // getOnlineFeatures Information - String projectName = "default"; - String entityName = "driver_id"; - ValueProto.Value entityValue1 = ValueProto.Value.newBuilder().setInt64Val(1).build(); - ValueProto.Value entityValue2 = ValueProto.Value.newBuilder().setInt64Val(2).build(); - - // Instantiate EntityRows - GetOnlineFeaturesRequestV2.EntityRow entityRow1 = - DataGenerator.createEntityRow(entityName, entityValue1, 100); - GetOnlineFeaturesRequestV2.EntityRow entityRow2 = - DataGenerator.createEntityRow(entityName, entityValue2, 100); - ImmutableList entityRows = - ImmutableList.of(entityRow1, entityRow2); - - // Instantiate FeatureReferences - FeatureReferenceV2 featureReference = - DataGenerator.createFeatureReference("rides", "trip_cost"); - FeatureReferenceV2 notFoundFeatureReference = - DataGenerator.createFeatureReference("rides", "trip_transaction"); - FeatureReferenceV2 emptyFeatureReference = - DataGenerator.createFeatureReference("rides", "trip_empty"); - - ImmutableList featureReferences = - ImmutableList.of(featureReference, notFoundFeatureReference, emptyFeatureReference); - - // Build GetOnlineFeaturesRequestV2 - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - - ImmutableMap expectedValueMap = - ImmutableMap.of( - entityName, - entityValue1, - FeatureV2.getFeatureStringRef(featureReference), - DataGenerator.createInt64Value(5), - FeatureV2.getFeatureStringRef(notFoundFeatureReference), - DataGenerator.createEmptyValue(), - FeatureV2.getFeatureStringRef(emptyFeatureReference), - DataGenerator.createEmptyValue()); - - ImmutableMap expectedStatusMap = - ImmutableMap.of( - entityName, - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(featureReference), - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(notFoundFeatureReference), - GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND, - FeatureV2.getFeatureStringRef(emptyFeatureReference), - GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE); - - GetOnlineFeaturesResponse.FieldValues expectedFieldValues = - GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(expectedValueMap) - .putAllStatuses(expectedStatusMap) - .build(); - - ImmutableMap expectedValueMap2 = - ImmutableMap.of( - entityName, - entityValue2, - FeatureV2.getFeatureStringRef(featureReference), - DataGenerator.createEmptyValue(), - FeatureV2.getFeatureStringRef(notFoundFeatureReference), - DataGenerator.createEmptyValue(), - FeatureV2.getFeatureStringRef(emptyFeatureReference), - DataGenerator.createEmptyValue()); - - ImmutableMap expectedStatusMap2 = - ImmutableMap.of( - entityName, - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(featureReference), - GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND, - FeatureV2.getFeatureStringRef(notFoundFeatureReference), - GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND, - FeatureV2.getFeatureStringRef(emptyFeatureReference), - GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND); - - GetOnlineFeaturesResponse.FieldValues expectedFieldValues2 = - GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(expectedValueMap2) - .putAllStatuses(expectedStatusMap2) - .build(); - ImmutableList expectedFieldValuesList = - ImmutableList.of(expectedFieldValues, expectedFieldValues2); - - assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); - } - - @Test - public void shouldSupportAllFeastTypes() throws IOException { - EntityProto.EntitySpecV2 entitySpec = - EntityProto.EntitySpecV2.newBuilder() - .setName("entity") - .setDescription("") - .setValueType(ValueProto.ValueType.Enum.STRING) - .build(); - TestUtils.applyEntity(coreClient, "default", entitySpec); - - ImmutableMap allTypesFeatures = - new ImmutableMap.Builder() - .put("f_int64", ValueProto.ValueType.Enum.INT64) - .put("f_int32", ValueProto.ValueType.Enum.INT32) - .put("f_float", ValueProto.ValueType.Enum.FLOAT) - .put("f_double", ValueProto.ValueType.Enum.DOUBLE) - .put("f_string", ValueProto.ValueType.Enum.STRING) - .put("f_bytes", ValueProto.ValueType.Enum.BYTES) - .put("f_bool", ValueProto.ValueType.Enum.BOOL) - .put("f_int64_list", ValueProto.ValueType.Enum.INT64_LIST) - .put("f_int32_list", ValueProto.ValueType.Enum.INT32_LIST) - .put("f_float_list", ValueProto.ValueType.Enum.FLOAT_LIST) - .put("f_double_list", ValueProto.ValueType.Enum.DOUBLE_LIST) - .put("f_string_list", ValueProto.ValueType.Enum.STRING_LIST) - .put("f_bytes_list", ValueProto.ValueType.Enum.BYTES_LIST) - .put("f_bool_list", ValueProto.ValueType.Enum.BOOL_LIST) - .build(); - - TestUtils.applyFeatureTable( - coreClient, "default", "all_types", ImmutableList.of("entity"), allTypesFeatures, 7200); - - Schema schema = - SchemaBuilder.record("AllTypesRecord") - .namespace("") - .fields() - .requiredLong("f_int64") - .requiredInt("f_int32") - .requiredFloat("f_float") - .requiredDouble("f_double") - .requiredString("f_string") - .requiredBytes("f_bytes") - .requiredBoolean("f_bool") - .name("f_int64_list") - .type(SchemaBuilder.array().items(SchemaBuilder.builder().longType())) - .noDefault() - .name("f_int32_list") - .type(SchemaBuilder.array().items(SchemaBuilder.builder().intType())) - .noDefault() - .name("f_float_list") - .type(SchemaBuilder.array().items(SchemaBuilder.builder().floatType())) - .noDefault() - .name("f_double_list") - .type(SchemaBuilder.array().items(SchemaBuilder.builder().doubleType())) - .noDefault() - .name("f_string_list") - .type(SchemaBuilder.array().items(SchemaBuilder.builder().stringType())) - .noDefault() - .name("f_bytes_list") - .type(SchemaBuilder.array().items(SchemaBuilder.builder().bytesType())) - .noDefault() - .name("f_bool_list") - .type(SchemaBuilder.array().items(SchemaBuilder.builder().booleanType())) - .noDefault() - .endRecord(); - - GenericData.Record record = - new GenericRecordBuilder(schema) - .set("f_int64", 10L) - .set("f_int32", 10) - .set("f_float", 10.0) - .set("f_double", 10.0D) - .set("f_string", "test") - .set("f_bytes", ByteBuffer.wrap("test".getBytes())) - .set("f_bool", true) - .set("f_int64_list", ImmutableList.of(10L)) - .set("f_int32_list", ImmutableList.of(10)) - .set("f_float_list", ImmutableList.of(10.0)) - .set("f_double_list", ImmutableList.of(10.0D)) - .set("f_string_list", ImmutableList.of("test")) - .set("f_bytes_list", ImmutableList.of(ByteBuffer.wrap("test".getBytes()))) - .set("f_bool_list", ImmutableList.of(true)) - .build(); - - ValueProto.Value entity = DataGenerator.createStrValue("key"); - - ingestData( - "all_types", - "default__entity", - entity.getStringVal().getBytes(), - createEntityValue(schema, schemaReference(schema), record), - createSchemaKey(schemaReference(schema)), - schema); - - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - TestUtils.createOnlineFeatureRequest( - "default", - allTypesFeatures.keySet().stream() - .map( - f -> - FeatureReferenceV2.newBuilder() - .setFeatureTable("all_types") - .setName(f) - .build()) - .collect(Collectors.toList()), - ImmutableList.of(DataGenerator.createEntityRow("entity", entity, 100))); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - - assert featureResponse.getFieldValues(0).getStatusesMap().values().stream() - .allMatch(status -> status.equals(GetOnlineFeaturesResponse.FieldStatus.PRESENT)); - } - - @Test - public void shouldRegisterSuperLongEntityAndGetOnlineFeatures() { - // getOnlineFeatures Information - String projectName = "default"; - String entityName = "this_is_a_long_long_long_long_long_long_entity_id"; - ValueProto.Value entityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); - - // Instantiate EntityRows - GetOnlineFeaturesRequestV2.EntityRow entityRow1 = - DataGenerator.createEntityRow(entityName, DataGenerator.createInt64Value(1), 100); - ImmutableList entityRows = ImmutableList.of(entityRow1); - - // Instantiate FeatureReferences - FeatureReferenceV2 featureReference = - DataGenerator.createFeatureReference("superlong", "trip_cost"); - FeatureReferenceV2 notFoundFeatureReference = - DataGenerator.createFeatureReference("superlong", "trip_transaction"); - - ImmutableList featureReferences = - ImmutableList.of(featureReference, notFoundFeatureReference); - - // Build GetOnlineFeaturesRequestV2 - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - - ImmutableMap expectedValueMap = - ImmutableMap.of( - entityName, - entityValue, - FeatureV2.getFeatureStringRef(featureReference), - DataGenerator.createInt64Value(5), - FeatureV2.getFeatureStringRef(notFoundFeatureReference), - DataGenerator.createEmptyValue()); - - ImmutableMap expectedStatusMap = - ImmutableMap.of( - entityName, - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(featureReference), - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(notFoundFeatureReference), - GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND); - - GetOnlineFeaturesResponse.FieldValues expectedFieldValues = - GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(expectedValueMap) - .putAllStatuses(expectedStatusMap) - .build(); - ImmutableList expectedFieldValuesList = - ImmutableList.of(expectedFieldValues); - - assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); - } - - @TestConfiguration - public static class TestConfig { - @Bean - public BigtableDataClient bigtableClient() throws IOException { - return BigtableDataClient.create( - BigtableDataSettings.newBuilderForEmulator( - environment.getServiceHost("bigtable_1", BIGTABLE_PORT), - environment.getServicePort("bigtable_1", BIGTABLE_PORT)) - .setProjectId(PROJECT_ID) - .setInstanceId(INSTANCE_ID) - .setAppProfileId(APP_PROFILE_ID) - .build()); - } - } -} diff --git a/java/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java b/java/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java deleted file mode 100644 index 93ee5f5cb6..0000000000 --- a/java/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java +++ /dev/null @@ -1,728 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2021 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.it; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import com.datastax.oss.driver.api.core.CqlSession; -import com.datastax.oss.driver.api.core.cql.PreparedStatement; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.hash.Hashing; -import feast.common.it.DataGenerator; -import feast.common.models.FeatureV2; -import feast.proto.core.EntityProto; -import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; -import feast.proto.serving.ServingServiceGrpc; -import feast.proto.types.ValueProto; -import io.grpc.ManagedChannel; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.nio.ByteBuffer; -import java.time.Duration; -import java.util.HashMap; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import org.apache.avro.Schema; -import org.apache.avro.SchemaBuilder; -import org.apache.avro.generic.GenericDatumWriter; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.GenericRecordBuilder; -import org.apache.avro.io.Encoder; -import org.apache.avro.io.EncoderFactory; -import org.junit.ClassRule; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.DynamicPropertyRegistry; -import org.springframework.test.context.DynamicPropertySource; -import org.testcontainers.containers.DockerComposeContainer; -import org.testcontainers.containers.wait.strategy.Wait; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; - -@ActiveProfiles("it") -@SpringBootTest( - webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, - properties = { - "feast.core-cache-refresh-interval=1", - "feast.active_store=cassandra", - "spring.main.allow-bean-definition-overriding=true" - }) -@Testcontainers -public class ServingServiceCassandraIT extends BaseAuthIT { - - static final Map options = new HashMap<>(); - static CoreSimpleAPIClient coreClient; - static ServingServiceGrpc.ServingServiceBlockingStub servingStub; - - static CqlSession cqlSession; - static final int FEAST_SERVING_PORT = 6570; - - static final FeatureReferenceV2 feature1Reference = - DataGenerator.createFeatureReference("rides", "trip_cost"); - static final FeatureReferenceV2 feature2Reference = - DataGenerator.createFeatureReference("rides", "trip_distance"); - static final FeatureReferenceV2 feature3Reference = - DataGenerator.createFeatureReference("rides", "trip_empty"); - static final FeatureReferenceV2 feature4Reference = - DataGenerator.createFeatureReference("rides", "trip_wrong_type"); - - @ClassRule @Container - public static DockerComposeContainer environment = - new DockerComposeContainer( - new File("src/test/resources/docker-compose/docker-compose-cassandra-it.yml")) - .withExposedService( - CORE, - FEAST_CORE_PORT, - Wait.forLogMessage(".*gRPC Server started.*\\n", 1) - .withStartupTimeout(Duration.ofMinutes(SERVICE_START_MAX_WAIT_TIME_IN_MINUTES))) - .withExposedService(CASSANDRA, CASSANDRA_PORT); - - @DynamicPropertySource - static void initialize(DynamicPropertyRegistry registry) { - registry.add("grpc.server.port", () -> FEAST_SERVING_PORT); - } - - @BeforeAll - static void globalSetup() throws IOException { - coreClient = TestUtils.getApiClientForCore(FEAST_CORE_PORT); - servingStub = TestUtils.getServingServiceStub(false, FEAST_SERVING_PORT, null); - - cqlSession = - CqlSession.builder() - .addContactPoint( - new InetSocketAddress( - environment.getServiceHost("cassandra_1", CASSANDRA_PORT), - environment.getServicePort("cassandra_1", CASSANDRA_PORT))) - .withLocalDatacenter(CASSANDRA_DATACENTER) - .build(); - - /** Feast resource creation Workflow */ - String projectName = "default"; - // Apply Entity (driver_id) - String driverEntityName = "driver_id"; - String driverEntityDescription = "My driver id"; - ValueProto.ValueType.Enum driverEntityType = ValueProto.ValueType.Enum.INT64; - EntityProto.EntitySpecV2 driverEntitySpec = - EntityProto.EntitySpecV2.newBuilder() - .setName(driverEntityName) - .setDescription(driverEntityDescription) - .setValueType(driverEntityType) - .build(); - TestUtils.applyEntity(coreClient, projectName, driverEntitySpec); - - // Apply Entity (merchant_id) - String merchantEntityName = "merchant_id"; - String merchantEntityDescription = "My driver id"; - ValueProto.ValueType.Enum merchantEntityType = ValueProto.ValueType.Enum.INT64; - EntityProto.EntitySpecV2 merchantEntitySpec = - EntityProto.EntitySpecV2.newBuilder() - .setName(merchantEntityName) - .setDescription(merchantEntityDescription) - .setValueType(merchantEntityType) - .build(); - TestUtils.applyEntity(coreClient, projectName, merchantEntitySpec); - - // Apply FeatureTable (rides) - String ridesFeatureTableName = "rides"; - ImmutableList ridesEntities = ImmutableList.of(driverEntityName); - ImmutableMap ridesFeatures = - ImmutableMap.of( - "trip_cost", - ValueProto.ValueType.Enum.INT32, - "trip_distance", - ValueProto.ValueType.Enum.DOUBLE, - "trip_empty", - ValueProto.ValueType.Enum.DOUBLE, - "trip_wrong_type", - ValueProto.ValueType.Enum.STRING); - TestUtils.applyFeatureTable( - coreClient, projectName, ridesFeatureTableName, ridesEntities, ridesFeatures, 7200); - - // Apply FeatureTable (food) - String foodFeatureTableName = "food"; - ImmutableList foodEntities = ImmutableList.of(driverEntityName); - ImmutableMap foodFeatures = - ImmutableMap.of( - "trip_cost", - ValueProto.ValueType.Enum.INT32, - "trip_distance", - ValueProto.ValueType.Enum.DOUBLE); - TestUtils.applyFeatureTable( - coreClient, projectName, foodFeatureTableName, foodEntities, foodFeatures, 7200); - - // Apply FeatureTable (rides_merchant) - String rideMerchantFeatureTableName = "rides_merchant"; - ImmutableList ridesMerchantEntities = - ImmutableList.of(driverEntityName, merchantEntityName); - TestUtils.applyFeatureTable( - coreClient, - projectName, - rideMerchantFeatureTableName, - ridesMerchantEntities, - ridesFeatures, - 7200); - - /** Create Cassandra Tables Workflow */ - String cassandraTableName = String.format("%s__%s", projectName, driverEntityName); - String compoundCassandraTableName = - String.format("%s__%s", projectName, String.join("__", ridesMerchantEntities)); - - cqlSession.execute(String.format("DROP KEYSPACE IF EXISTS %s", CASSANDRA_KEYSPACE)); - cqlSession.execute( - String.format( - "CREATE KEYSPACE %s WITH replication = \n" - + "{'class':'SimpleStrategy','replication_factor':'1'};", - CASSANDRA_KEYSPACE)); - - // Create Cassandra Tables - createCassandraTable(cassandraTableName); - createCassandraTable(compoundCassandraTableName); - - // Add column families - addCassandraTableColumn(cassandraTableName, ridesFeatureTableName); - addCassandraTableColumn(cassandraTableName, foodFeatureTableName); - addCassandraTableColumn(compoundCassandraTableName, rideMerchantFeatureTableName); - - /** Single Entity Ingestion Workflow */ - Schema ftSchema = - SchemaBuilder.record("DriverData") - .namespace(ridesFeatureTableName) - .fields() - .requiredInt(feature1Reference.getName()) - .requiredDouble(feature2Reference.getName()) - .nullableString(feature3Reference.getName(), "null") - .requiredString(feature4Reference.getName()) - .endRecord(); - byte[] schemaReference = - Hashing.murmur3_32().hashBytes(ftSchema.toString().getBytes()).asBytes(); - byte[] schemaKey = createSchemaKey(schemaReference); - - ingestBulk(ridesFeatureTableName, cassandraTableName, ftSchema, 20); - - Schema foodFtSchema = - SchemaBuilder.record("FoodDriverData") - .namespace(foodFeatureTableName) - .fields() - .requiredInt(feature1Reference.getName()) - .requiredDouble(feature2Reference.getName()) - .nullableString(feature3Reference.getName(), "null") - .requiredString(feature4Reference.getName()) - .endRecord(); - byte[] foodSchemaReference = - Hashing.murmur3_32().hashBytes(foodFtSchema.toString().getBytes()).asBytes(); - byte[] foodSchemaKey = createSchemaKey(foodSchemaReference); - - ingestBulk(foodFeatureTableName, cassandraTableName, foodFtSchema, 20); - - /** Compound Entity Ingestion Workflow */ - Schema compoundFtSchema = - SchemaBuilder.record("DriverMerchantData") - .namespace(rideMerchantFeatureTableName) - .fields() - .requiredLong(feature1Reference.getName()) - .requiredDouble(feature2Reference.getName()) - .nullableString(feature3Reference.getName(), "null") - .requiredString(feature4Reference.getName()) - .endRecord(); - byte[] compoundSchemaReference = - Hashing.murmur3_32().hashBytes(compoundFtSchema.toString().getBytes()).asBytes(); - - GenericRecord compoundEntityRecord = - new GenericRecordBuilder(compoundFtSchema) - .set("trip_cost", 10L) - .set("trip_distance", 5.5) - .set("trip_empty", null) - .set("trip_wrong_type", "wrong_type") - .build(); - ValueProto.Value driverEntityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); - ValueProto.Value merchantEntityValue = ValueProto.Value.newBuilder().setInt64Val(1234).build(); - ImmutableMap compoundEntityMap = - ImmutableMap.of( - driverEntityName, driverEntityValue, merchantEntityName, merchantEntityValue); - GetOnlineFeaturesRequestV2.EntityRow entityRow = - DataGenerator.createCompoundEntityRow(compoundEntityMap, 100); - byte[] compoundEntityFeatureKey = - ridesMerchantEntities.stream() - .map(entity -> DataGenerator.valueToString(entityRow.getFieldsMap().get(entity))) - .collect(Collectors.joining("#")) - .getBytes(); - byte[] compoundEntityFeatureValue = createEntityValue(compoundFtSchema, compoundEntityRecord); - byte[] compoundSchemaKey = createSchemaKey(compoundSchemaReference); - - ingestData( - rideMerchantFeatureTableName, - compoundCassandraTableName, - compoundEntityFeatureKey, - compoundEntityFeatureValue, - compoundSchemaKey); - - /** Schema Ingestion Workflow */ - cqlSession.execute( - String.format( - "CREATE TABLE %s.%s (schema_ref BLOB PRIMARY KEY, avro_schema BLOB);", - CASSANDRA_KEYSPACE, CASSANDRA_SCHEMA_TABLE)); - - ingestSchema(schemaKey, ftSchema); - ingestSchema(foodSchemaKey, foodFtSchema); - ingestSchema(compoundSchemaKey, compoundFtSchema); - - // set up options for call credentials - options.put("oauth_url", TOKEN_URL); - options.put(CLIENT_ID, CLIENT_ID); - options.put(CLIENT_SECRET, CLIENT_SECRET); - options.put("jwkEndpointURI", JWK_URI); - options.put("audience", AUDIENCE); - options.put("grant_type", GRANT_TYPE); - } - - private static byte[] createSchemaKey(byte[] schemaReference) throws IOException { - ByteArrayOutputStream concatOutputStream = new ByteArrayOutputStream(); - concatOutputStream.write(schemaReference); - byte[] schemaKey = concatOutputStream.toByteArray(); - - return schemaKey; - } - - private static byte[] createEntityValue(Schema schema, GenericRecord record) throws IOException { - // Entity-Feature Row - byte[] avroSerializedFeatures = recordToAvro(record, schema); - - ByteArrayOutputStream concatOutputStream = new ByteArrayOutputStream(); - concatOutputStream.write(avroSerializedFeatures); - byte[] entityFeatureValue = concatOutputStream.toByteArray(); - - return entityFeatureValue; - } - - private static void createCassandraTable(String cassandraTableName) { - cqlSession.execute( - String.format( - "CREATE TABLE %s.%s (key BLOB PRIMARY KEY);", CASSANDRA_KEYSPACE, cassandraTableName)); - } - - private static void addCassandraTableColumn(String cassandraTableName, String featureTableName) { - cqlSession.execute( - String.format( - "ALTER TABLE %s.%s ADD (%s BLOB, %s__schema_ref BLOB);", - CASSANDRA_KEYSPACE, cassandraTableName, featureTableName, featureTableName)); - } - - private static void ingestData( - String featureTableName, - String cassandraTableName, - byte[] entityFeatureKey, - byte[] entityFeatureValue, - byte[] schemaKey) { - PreparedStatement statement = - cqlSession.prepare( - String.format( - "INSERT INTO %s.%s (%s, %s__schema_ref, %s) VALUES (?, ?, ?)", - CASSANDRA_KEYSPACE, - cassandraTableName, - CASSANDRA_ENTITY_KEY, - featureTableName, - featureTableName)); - - cqlSession.execute( - statement.bind( - ByteBuffer.wrap(entityFeatureKey), - ByteBuffer.wrap(schemaKey), - ByteBuffer.wrap(entityFeatureValue))); - } - - private static void ingestBulk( - String featureTableName, String cassandraTableName, Schema schema, Integer counts) { - - IntStream.range(0, counts) - .forEach( - i -> { - try { - GenericRecord record = - new GenericRecordBuilder(schema) - .set("trip_cost", i) - .set("trip_distance", (double) i) - .set("trip_empty", null) - .set("trip_wrong_type", "test") - .build(); - byte[] schemaReference = - Hashing.murmur3_32().hashBytes(schema.toString().getBytes()).asBytes(); - - byte[] entityFeatureKey = - String.valueOf(DataGenerator.createInt64Value(i).getInt64Val()).getBytes(); - byte[] entityFeatureValue = createEntityValue(schema, record); - - byte[] schemaKey = createSchemaKey(schemaReference); - ingestData( - featureTableName, - cassandraTableName, - entityFeatureKey, - entityFeatureValue, - schemaKey); - } catch (IOException e) { - e.printStackTrace(); - } - }); - } - - private static void ingestSchema(byte[] schemaKey, Schema schema) { - PreparedStatement schemaStatement = - cqlSession.prepare( - String.format( - "INSERT INTO %s.%s (schema_ref, avro_schema) VALUES (?, ?);", - CASSANDRA_KEYSPACE, CASSANDRA_SCHEMA_TABLE)); - cqlSession.execute( - schemaStatement.bind( - ByteBuffer.wrap(schemaKey), ByteBuffer.wrap(schema.toString().getBytes()))); - } - - private static byte[] recordToAvro(GenericRecord datum, Schema schema) throws IOException { - GenericDatumWriter writer = new GenericDatumWriter<>(schema); - ByteArrayOutputStream output = new ByteArrayOutputStream(); - Encoder encoder = EncoderFactory.get().binaryEncoder(output, null); - writer.write(datum, encoder); - encoder.flush(); - - return output.toByteArray(); - } - - @AfterAll - static void tearDown() { - ((ManagedChannel) servingStub.getChannel()).shutdown(); - } - - @Test - public void shouldRegisterSingleEntityAndGetOnlineFeatures() { - String projectName = "default"; - String entityName = "driver_id"; - ValueProto.Value entityValue = DataGenerator.createInt64Value(1); - - // Instantiate EntityRows - GetOnlineFeaturesRequestV2.EntityRow entityRow = - DataGenerator.createEntityRow(entityName, entityValue, 100); - ImmutableList entityRows = ImmutableList.of(entityRow); - - // Instantiate FeatureReferences - FeatureReferenceV2 featureReference = - DataGenerator.createFeatureReference("rides", "trip_cost"); - FeatureReferenceV2 notFoundFeatureReference = - DataGenerator.createFeatureReference("rides", "trip_transaction"); - - ImmutableList featureReferences = - ImmutableList.of(featureReference, notFoundFeatureReference); - - // Build GetOnlineFeaturesRequestV2 - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - - ImmutableMap expectedValueMap = - ImmutableMap.of( - entityName, - entityValue, - FeatureV2.getFeatureStringRef(featureReference), - DataGenerator.createInt32Value(1), - FeatureV2.getFeatureStringRef(notFoundFeatureReference), - DataGenerator.createEmptyValue()); - - ImmutableMap expectedStatusMap = - ImmutableMap.of( - entityName, - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(featureReference), - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(notFoundFeatureReference), - GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND); - - GetOnlineFeaturesResponse.FieldValues expectedFieldValues = - GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(expectedValueMap) - .putAllStatuses(expectedStatusMap) - .build(); - ImmutableList expectedFieldValuesList = - ImmutableList.of(expectedFieldValues); - - assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); - } - - @Test - public void shouldRegisterCompoundEntityAndGetOnlineFeatures() { - String projectName = "default"; - String driverEntityName = "driver_id"; - String merchantEntityName = "merchant_id"; - ValueProto.Value driverEntityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); - ValueProto.Value merchantEntityValue = ValueProto.Value.newBuilder().setInt64Val(1234).build(); - - ImmutableMap compoundEntityMap = - ImmutableMap.of( - driverEntityName, driverEntityValue, merchantEntityName, merchantEntityValue); - - // Instantiate EntityRows - GetOnlineFeaturesRequestV2.EntityRow entityRow = - DataGenerator.createCompoundEntityRow(compoundEntityMap, 100); - ImmutableList entityRows = ImmutableList.of(entityRow); - - // Instantiate FeatureReferences - FeatureReferenceV2 featureReference = - DataGenerator.createFeatureReference("rides", "trip_cost"); - FeatureReferenceV2 notFoundFeatureReference = - DataGenerator.createFeatureReference("rides", "trip_transaction"); - - ImmutableList featureReferences = - ImmutableList.of(featureReference, notFoundFeatureReference); - - // Build GetOnlineFeaturesRequestV2 - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - - ImmutableMap expectedValueMap = - ImmutableMap.of( - driverEntityName, - driverEntityValue, - merchantEntityName, - merchantEntityValue, - FeatureV2.getFeatureStringRef(featureReference), - DataGenerator.createInt32Value(1), - FeatureV2.getFeatureStringRef(notFoundFeatureReference), - DataGenerator.createEmptyValue()); - - ImmutableMap expectedStatusMap = - ImmutableMap.of( - driverEntityName, - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - merchantEntityName, - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(featureReference), - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(notFoundFeatureReference), - GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND); - - GetOnlineFeaturesResponse.FieldValues expectedFieldValues = - GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(expectedValueMap) - .putAllStatuses(expectedStatusMap) - .build(); - ImmutableList expectedFieldValuesList = - ImmutableList.of(expectedFieldValues); - - assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); - } - - @Test - public void shouldReturnCorrectRowCountAndOrder() { - // getOnlineFeatures Information - String projectName = "default"; - String entityName = "driver_id"; - ValueProto.Value entityValue1 = ValueProto.Value.newBuilder().setInt64Val(1).build(); - ValueProto.Value entityValue2 = ValueProto.Value.newBuilder().setInt64Val(2).build(); - ValueProto.Value entityValue3 = ValueProto.Value.newBuilder().setInt64Val(3).build(); - ValueProto.Value entityValue4 = ValueProto.Value.newBuilder().setInt64Val(4).build(); - - // Instantiate EntityRows - GetOnlineFeaturesRequestV2.EntityRow entityRow1 = - DataGenerator.createEntityRow(entityName, entityValue1, 100); - GetOnlineFeaturesRequestV2.EntityRow entityRow2 = - DataGenerator.createEntityRow(entityName, entityValue2, 100); - GetOnlineFeaturesRequestV2.EntityRow entityRow3 = - DataGenerator.createEntityRow(entityName, entityValue3, 100); - GetOnlineFeaturesRequestV2.EntityRow entityRow4 = - DataGenerator.createEntityRow(entityName, entityValue4, 100); - ImmutableList entityRows = - ImmutableList.of(entityRow1, entityRow2, entityRow4, entityRow3); - - // Instantiate FeatureReferences - FeatureReferenceV2 featureReference = - DataGenerator.createFeatureReference("rides", "trip_cost"); - FeatureReferenceV2 notFoundFeatureReference = - DataGenerator.createFeatureReference("rides", "trip_transaction"); - FeatureReferenceV2 emptyFeatureReference = - DataGenerator.createFeatureReference("rides", "trip_empty"); - - ImmutableList featureReferences = - ImmutableList.of(featureReference, notFoundFeatureReference, emptyFeatureReference); - - // Build GetOnlineFeaturesRequestV2 - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - - ImmutableMap expectedValueMap = - ImmutableMap.of( - entityName, - entityValue1, - FeatureV2.getFeatureStringRef(featureReference), - DataGenerator.createInt32Value(1), - FeatureV2.getFeatureStringRef(notFoundFeatureReference), - DataGenerator.createEmptyValue(), - FeatureV2.getFeatureStringRef(emptyFeatureReference), - DataGenerator.createEmptyValue()); - - ImmutableMap expectedStatusMap = - ImmutableMap.of( - entityName, - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(featureReference), - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(notFoundFeatureReference), - GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND, - FeatureV2.getFeatureStringRef(emptyFeatureReference), - GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE); - - GetOnlineFeaturesResponse.FieldValues expectedFieldValues = - GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(expectedValueMap) - .putAllStatuses(expectedStatusMap) - .build(); - - ImmutableMap expectedValueMap2 = - ImmutableMap.of( - entityName, - entityValue2, - FeatureV2.getFeatureStringRef(featureReference), - DataGenerator.createInt32Value(2), - FeatureV2.getFeatureStringRef(notFoundFeatureReference), - DataGenerator.createEmptyValue(), - FeatureV2.getFeatureStringRef(emptyFeatureReference), - DataGenerator.createEmptyValue()); - - ImmutableMap expectedValueMap3 = - ImmutableMap.of( - entityName, - entityValue3, - FeatureV2.getFeatureStringRef(featureReference), - DataGenerator.createInt32Value(3), - FeatureV2.getFeatureStringRef(notFoundFeatureReference), - DataGenerator.createEmptyValue(), - FeatureV2.getFeatureStringRef(emptyFeatureReference), - DataGenerator.createEmptyValue()); - - ImmutableMap expectedValueMap4 = - ImmutableMap.of( - entityName, - entityValue4, - FeatureV2.getFeatureStringRef(featureReference), - DataGenerator.createInt32Value(4), - FeatureV2.getFeatureStringRef(notFoundFeatureReference), - DataGenerator.createEmptyValue(), - FeatureV2.getFeatureStringRef(emptyFeatureReference), - DataGenerator.createEmptyValue()); - - GetOnlineFeaturesResponse.FieldValues expectedFieldValues2 = - GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(expectedValueMap2) - .putAllStatuses(expectedStatusMap) - .build(); - GetOnlineFeaturesResponse.FieldValues expectedFieldValues3 = - GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(expectedValueMap3) - .putAllStatuses(expectedStatusMap) - .build(); - GetOnlineFeaturesResponse.FieldValues expectedFieldValues4 = - GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(expectedValueMap4) - .putAllStatuses(expectedStatusMap) - .build(); - ImmutableList expectedFieldValuesList = - ImmutableList.of( - expectedFieldValues, expectedFieldValues2, expectedFieldValues4, expectedFieldValues3); - - assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); - } - - @Test - public void shouldReturnFeaturesFromDiffFeatureTable() { - String projectName = "default"; - String entityName = "driver_id"; - ValueProto.Value entityValue = DataGenerator.createInt64Value(1); - - // Instantiate EntityRows - GetOnlineFeaturesRequestV2.EntityRow entityRow = - DataGenerator.createEntityRow(entityName, entityValue, 100); - ImmutableList entityRows = ImmutableList.of(entityRow); - - // Instantiate FeatureReferences - FeatureReferenceV2 rideFeatureReference = - DataGenerator.createFeatureReference("rides", "trip_cost"); - FeatureReferenceV2 rideFeatureReference2 = - DataGenerator.createFeatureReference("rides", "trip_distance"); - FeatureReferenceV2 foodFeatureReference = - DataGenerator.createFeatureReference("food", "trip_cost"); - FeatureReferenceV2 foodFeatureReference2 = - DataGenerator.createFeatureReference("food", "trip_distance"); - - ImmutableList featureReferences = - ImmutableList.of( - rideFeatureReference, - rideFeatureReference2, - foodFeatureReference, - foodFeatureReference2); - - // Build GetOnlineFeaturesRequestV2 - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - - ImmutableMap expectedValueMap = - ImmutableMap.of( - entityName, - entityValue, - FeatureV2.getFeatureStringRef(rideFeatureReference), - DataGenerator.createInt32Value(1), - FeatureV2.getFeatureStringRef(rideFeatureReference2), - DataGenerator.createDoubleValue(1.0), - FeatureV2.getFeatureStringRef(foodFeatureReference), - DataGenerator.createInt32Value(1), - FeatureV2.getFeatureStringRef(foodFeatureReference2), - DataGenerator.createDoubleValue(1.0)); - - ImmutableMap expectedStatusMap = - ImmutableMap.of( - entityName, - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(rideFeatureReference), - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(rideFeatureReference2), - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(foodFeatureReference), - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(foodFeatureReference2), - GetOnlineFeaturesResponse.FieldStatus.PRESENT); - - GetOnlineFeaturesResponse.FieldValues expectedFieldValues = - GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(expectedValueMap) - .putAllStatuses(expectedStatusMap) - .build(); - ImmutableList expectedFieldValuesList = - ImmutableList.of(expectedFieldValues); - - assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); - } -} diff --git a/java/serving/src/test/java/feast/serving/it/ServingServiceIT.java b/java/serving/src/test/java/feast/serving/it/ServingServiceIT.java deleted file mode 100644 index c0be6c9b7c..0000000000 --- a/java/serving/src/test/java/feast/serving/it/ServingServiceIT.java +++ /dev/null @@ -1,505 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.it; - -import static org.junit.jupiter.api.Assertions.*; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.hash.Hashing; -import com.google.protobuf.Timestamp; -import com.squareup.okhttp.OkHttpClient; -import com.squareup.okhttp.Request; -import com.squareup.okhttp.Response; -import feast.common.it.DataGenerator; -import feast.common.models.FeatureV2; -import feast.proto.core.EntityProto; -import feast.proto.serving.ServingAPIProto; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; -import feast.proto.serving.ServingServiceGrpc; -import feast.proto.storage.RedisProto; -import feast.proto.types.ValueProto; -import io.grpc.ManagedChannel; -import io.lettuce.core.RedisClient; -import io.lettuce.core.RedisURI; -import io.lettuce.core.api.StatefulRedisConnection; -import io.lettuce.core.api.sync.RedisCommands; -import io.lettuce.core.codec.ByteArrayCodec; -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.util.*; -import org.junit.ClassRule; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.web.server.LocalServerPort; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.DynamicPropertyRegistry; -import org.springframework.test.context.DynamicPropertySource; -import org.testcontainers.containers.DockerComposeContainer; -import org.testcontainers.containers.wait.strategy.Wait; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; - -@ActiveProfiles("it") -@SpringBootTest( - webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, - properties = { - "feast.core-cache-refresh-interval=1", - }) -@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_CLASS) -@Testcontainers -public class ServingServiceIT extends BaseAuthIT { - - static final Map options = new HashMap<>(); - static final String timestampPrefix = "_ts"; - static CoreSimpleAPIClient coreClient; - static ServingServiceGrpc.ServingServiceBlockingStub servingStub; - static RedisCommands syncCommands; - - static final int FEAST_SERVING_PORT = 6568; - @LocalServerPort private int metricsPort; - - @ClassRule @Container - public static DockerComposeContainer environment = - new DockerComposeContainer( - new File("src/test/resources/docker-compose/docker-compose-it.yml")) - .withExposedService( - CORE, - FEAST_CORE_PORT, - Wait.forLogMessage(".*gRPC Server started.*\\n", 1) - .withStartupTimeout(Duration.ofMinutes(SERVICE_START_MAX_WAIT_TIME_IN_MINUTES))) - .withExposedService(REDIS, REDIS_PORT); - - @DynamicPropertySource - static void initialize(DynamicPropertyRegistry registry) { - registry.add("grpc.server.port", () -> FEAST_SERVING_PORT); - } - - @BeforeAll - static void globalSetup() { - coreClient = TestUtils.getApiClientForCore(FEAST_CORE_PORT); - servingStub = TestUtils.getServingServiceStub(false, FEAST_SERVING_PORT, null); - - RedisClient redisClient = - RedisClient.create( - new RedisURI( - environment.getServiceHost("redis_1", REDIS_PORT), - environment.getServicePort("redis_1", REDIS_PORT), - java.time.Duration.ofMillis(2000))); - StatefulRedisConnection connection = redisClient.connect(new ByteArrayCodec()); - syncCommands = connection.sync(); - - String projectName = "default"; - // Apply Entity - String entityName = "driver_id"; - ValueProto.Value entityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); - String description = "My driver id"; - ValueProto.ValueType.Enum entityType = ValueProto.ValueType.Enum.INT64; - EntityProto.EntitySpecV2 entitySpec = - EntityProto.EntitySpecV2.newBuilder() - .setName(entityName) - .setDescription(description) - .setValueType(entityType) - .build(); - TestUtils.applyEntity(coreClient, projectName, entitySpec); - - // Apply FeatureTable - String featureTableName = "rides"; - ImmutableList entities = ImmutableList.of(entityName); - - ServingAPIProto.FeatureReferenceV2 feature1Reference = - DataGenerator.createFeatureReference("rides", "trip_cost"); - ServingAPIProto.FeatureReferenceV2 feature2Reference = - DataGenerator.createFeatureReference("rides", "trip_distance"); - ServingAPIProto.FeatureReferenceV2 feature3Reference = - DataGenerator.createFeatureReference("rides", "trip_empty"); - ServingAPIProto.FeatureReferenceV2 feature4Reference = - DataGenerator.createFeatureReference("rides", "trip_wrong_type"); - - // Event Timestamp - String eventTimestampKey = timestampPrefix + ":" + featureTableName; - Timestamp eventTimestampValue = Timestamp.newBuilder().setSeconds(100).build(); - - ImmutableMap features = - ImmutableMap.of( - "trip_cost", - ValueProto.ValueType.Enum.INT64, - "trip_distance", - ValueProto.ValueType.Enum.DOUBLE, - "trip_empty", - ValueProto.ValueType.Enum.DOUBLE, - "trip_wrong_type", - ValueProto.ValueType.Enum.STRING); - - TestUtils.applyFeatureTable( - coreClient, projectName, featureTableName, entities, features, 7200); - - // Serialize Redis Key with Entity i.e - RedisProto.RedisKeyV2 redisKey = - RedisProto.RedisKeyV2.newBuilder() - .setProject(projectName) - .addEntityNames(entityName) - .addEntityValues(entityValue) - .build(); - - ImmutableMap featureReferenceValueMap = - ImmutableMap.of( - feature1Reference, - DataGenerator.createInt64Value(42), - feature2Reference, - DataGenerator.createDoubleValue(42.2), - feature3Reference, - DataGenerator.createEmptyValue(), - feature4Reference, - DataGenerator.createDoubleValue(42.2)); - - // Insert timestamp into Redis and isTimestampMap only once - syncCommands.hset( - redisKey.toByteArray(), eventTimestampKey.getBytes(), eventTimestampValue.toByteArray()); - featureReferenceValueMap.forEach( - (featureReference, featureValue) -> { - // Murmur hash Redis Feature Field i.e murmur() - String delimitedFeatureReference = - featureReference.getFeatureTable() + ":" + featureReference.getName(); - byte[] featureReferenceBytes = - Hashing.murmur3_32() - .hashString(delimitedFeatureReference, StandardCharsets.UTF_8) - .asBytes(); - // Insert features into Redis - syncCommands.hset( - redisKey.toByteArray(), featureReferenceBytes, featureValue.toByteArray()); - }); - - // set up options for call credentials - options.put("oauth_url", TOKEN_URL); - options.put(CLIENT_ID, CLIENT_ID); - options.put(CLIENT_SECRET, CLIENT_SECRET); - options.put("jwkEndpointURI", JWK_URI); - options.put("audience", AUDIENCE); - options.put("grant_type", GRANT_TYPE); - } - - @AfterAll - static void tearDown() { - ((ManagedChannel) servingStub.getChannel()).shutdown(); - } - - /** Test that Feast Serving metrics endpoint can be accessed with authentication enabled */ - @Test - @DirtiesContext(methodMode = DirtiesContext.MethodMode.AFTER_METHOD) - public void shouldAllowUnauthenticatedAccessToMetricsEndpoint() throws IOException { - Request request = - new Request.Builder() - .url(String.format("http://localhost:%d/metrics", metricsPort)) - .get() - .build(); - Response response = new OkHttpClient().newCall(request).execute(); - assertTrue(response.isSuccessful()); - assertFalse(response.body().string().isEmpty()); - } - - @Test - @DirtiesContext(methodMode = DirtiesContext.MethodMode.AFTER_METHOD) - public void shouldRegisterAndGetOnlineFeatures() { - // getOnlineFeatures Information - String projectName = "default"; - String entityName = "driver_id"; - ValueProto.Value entityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); - - // Instantiate EntityRows - GetOnlineFeaturesRequestV2.EntityRow entityRow1 = - DataGenerator.createEntityRow(entityName, DataGenerator.createInt64Value(1), 100); - ImmutableList entityRows = ImmutableList.of(entityRow1); - - // Instantiate FeatureReferences - ServingAPIProto.FeatureReferenceV2 feature1Reference = - DataGenerator.createFeatureReference("rides", "trip_cost"); - ImmutableList featureReferences = - ImmutableList.of(feature1Reference); - - // Build GetOnlineFeaturesRequestV2 - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - - ImmutableMap expectedValueMap = - ImmutableMap.of( - entityName, - entityValue, - FeatureV2.getFeatureStringRef(feature1Reference), - DataGenerator.createInt64Value(42)); - - ImmutableMap expectedStatusMap = - ImmutableMap.of( - entityName, - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(feature1Reference), - GetOnlineFeaturesResponse.FieldStatus.PRESENT); - - GetOnlineFeaturesResponse.FieldValues expectedFieldValues = - GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(expectedValueMap) - .putAllStatuses(expectedStatusMap) - .build(); - ImmutableList expectedFieldValuesList = - ImmutableList.of(expectedFieldValues); - - assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); - } - - @Test - @DirtiesContext(methodMode = DirtiesContext.MethodMode.AFTER_METHOD) - public void shouldRegisterAndGetOnlineFeaturesWithNotFound() { - // getOnlineFeatures Information - String projectName = "default"; - String entityName = "driver_id"; - ValueProto.Value entityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); - - // Instantiate EntityRows - GetOnlineFeaturesRequestV2.EntityRow entityRow1 = - DataGenerator.createEntityRow(entityName, DataGenerator.createInt64Value(1), 100); - ImmutableList entityRows = ImmutableList.of(entityRow1); - - // Instantiate FeatureReferences - ServingAPIProto.FeatureReferenceV2 featureReference = - DataGenerator.createFeatureReference("rides", "trip_cost"); - ServingAPIProto.FeatureReferenceV2 notFoundFeatureReference = - DataGenerator.createFeatureReference("rides", "trip_transaction"); - ServingAPIProto.FeatureReferenceV2 emptyFeatureReference = - DataGenerator.createFeatureReference("rides", "trip_empty"); - - ImmutableList featureReferences = - ImmutableList.of(featureReference, notFoundFeatureReference, emptyFeatureReference); - - // Build GetOnlineFeaturesRequestV2 - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - - ImmutableMap expectedValueMap = - ImmutableMap.of( - entityName, - entityValue, - FeatureV2.getFeatureStringRef(featureReference), - DataGenerator.createInt64Value(42), - FeatureV2.getFeatureStringRef(notFoundFeatureReference), - DataGenerator.createEmptyValue(), - FeatureV2.getFeatureStringRef(emptyFeatureReference), - DataGenerator.createEmptyValue()); - - ImmutableMap expectedStatusMap = - ImmutableMap.of( - entityName, - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(featureReference), - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(notFoundFeatureReference), - GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND, - FeatureV2.getFeatureStringRef(emptyFeatureReference), - GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND); - - GetOnlineFeaturesResponse.FieldValues expectedFieldValues = - GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(expectedValueMap) - .putAllStatuses(expectedStatusMap) - .build(); - ImmutableList expectedFieldValuesList = - ImmutableList.of(expectedFieldValues); - - assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); - } - - @Test - @DirtiesContext(methodMode = DirtiesContext.MethodMode.AFTER_METHOD) - public void shouldGetOnlineFeaturesOutsideMaxAge() { - String projectName = "default"; - String entityName = "driver_id"; - ValueProto.Value entityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); - - // Instantiate EntityRows - GetOnlineFeaturesRequestV2.EntityRow entityRow1 = - DataGenerator.createEntityRow(entityName, DataGenerator.createInt64Value(1), 7400); - ImmutableList entityRows = ImmutableList.of(entityRow1); - - // Instantiate FeatureReferences - ServingAPIProto.FeatureReferenceV2 featureReference = - DataGenerator.createFeatureReference("rides", "trip_cost"); - - ImmutableList featureReferences = - ImmutableList.of(featureReference); - - // Build GetOnlineFeaturesRequestV2 - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - - ImmutableMap expectedValueMap = - ImmutableMap.of( - entityName, - entityValue, - FeatureV2.getFeatureStringRef(featureReference), - DataGenerator.createEmptyValue()); - - ImmutableMap expectedStatusMap = - ImmutableMap.of( - entityName, - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(featureReference), - GetOnlineFeaturesResponse.FieldStatus.OUTSIDE_MAX_AGE); - - GetOnlineFeaturesResponse.FieldValues expectedFieldValues = - GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(expectedValueMap) - .putAllStatuses(expectedStatusMap) - .build(); - ImmutableList expectedFieldValuesList = - ImmutableList.of(expectedFieldValues); - - assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); - } - - @Test - @DirtiesContext(methodMode = DirtiesContext.MethodMode.AFTER_METHOD) - public void shouldReturnNotFoundForDiffType() { - String projectName = "default"; - String entityName = "driver_id"; - ValueProto.Value entityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); - - // Instantiate EntityRows - GetOnlineFeaturesRequestV2.EntityRow entityRow1 = - DataGenerator.createEntityRow(entityName, DataGenerator.createInt64Value(1), 100); - ImmutableList entityRows = ImmutableList.of(entityRow1); - - // Instantiate FeatureReferences - ServingAPIProto.FeatureReferenceV2 featureReference = - DataGenerator.createFeatureReference("rides", "trip_wrong_type"); - - ImmutableList featureReferences = - ImmutableList.of(featureReference); - - // Build GetOnlineFeaturesRequestV2 - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - - ImmutableMap expectedValueMap = - ImmutableMap.of( - entityName, - entityValue, - FeatureV2.getFeatureStringRef(featureReference), - DataGenerator.createEmptyValue()); - - ImmutableMap expectedStatusMap = - ImmutableMap.of( - entityName, - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(featureReference), - GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND); - - GetOnlineFeaturesResponse.FieldValues expectedFieldValues = - GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(expectedValueMap) - .putAllStatuses(expectedStatusMap) - .build(); - ImmutableList expectedFieldValuesList = - ImmutableList.of(expectedFieldValues); - - assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); - } - - @Test - @DirtiesContext(methodMode = DirtiesContext.MethodMode.AFTER_METHOD) - public void shouldReturnNotFoundForUpdatedType() { - String projectName = "default"; - String entityName = "driver_id"; - String featureTableName = "rides"; - - ImmutableList entities = ImmutableList.of(entityName); - ImmutableMap features = - ImmutableMap.of( - "trip_cost", - ValueProto.ValueType.Enum.INT64, - "trip_distance", - ValueProto.ValueType.Enum.STRING, - "trip_empty", - ValueProto.ValueType.Enum.DOUBLE, - "trip_wrong_type", - ValueProto.ValueType.Enum.STRING); - - TestUtils.applyFeatureTable( - coreClient, projectName, featureTableName, entities, features, 7200); - - // Sleep is necessary to ensure caching (every 1s) of updated FeatureTable is done - try { - Thread.sleep(2000); - } catch (InterruptedException e) { - } - - ValueProto.Value entityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); - // Instantiate EntityRows - GetOnlineFeaturesRequestV2.EntityRow entityRow1 = - DataGenerator.createEntityRow(entityName, DataGenerator.createInt64Value(1), 100); - ImmutableList entityRows = ImmutableList.of(entityRow1); - - // Instantiate FeatureReferences - ServingAPIProto.FeatureReferenceV2 featureReference = - DataGenerator.createFeatureReference("rides", "trip_distance"); - - ImmutableList featureReferences = - ImmutableList.of(featureReference); - - // Build GetOnlineFeaturesRequestV2 - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - - ImmutableMap expectedValueMap = - ImmutableMap.of( - entityName, - entityValue, - FeatureV2.getFeatureStringRef(featureReference), - DataGenerator.createEmptyValue()); - - ImmutableMap expectedStatusMap = - ImmutableMap.of( - entityName, - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(featureReference), - GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND); - - GetOnlineFeaturesResponse.FieldValues expectedFieldValues = - GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(expectedValueMap) - .putAllStatuses(expectedStatusMap) - .build(); - ImmutableList expectedFieldValuesList = - ImmutableList.of(expectedFieldValues); - - assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); - } -} diff --git a/java/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthenticationIT.java b/java/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthenticationIT.java deleted file mode 100644 index 8f2440d247..0000000000 --- a/java/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthenticationIT.java +++ /dev/null @@ -1,190 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.it; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.testcontainers.containers.wait.strategy.Wait.forHttp; - -import com.google.common.collect.ImmutableMap; -import com.squareup.okhttp.OkHttpClient; -import com.squareup.okhttp.Request; -import com.squareup.okhttp.Response; -import feast.common.it.DataGenerator; -import feast.proto.core.EntityProto; -import feast.proto.core.FeatureTableProto; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; -import feast.proto.serving.ServingServiceGrpc.ServingServiceBlockingStub; -import feast.proto.types.ValueProto; -import feast.proto.types.ValueProto.Value; -import io.grpc.ManagedChannel; -import java.io.File; -import java.io.IOException; -import java.time.Duration; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import org.junit.ClassRule; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.runners.model.InitializationError; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.context.SpringBootTest.WebEnvironment; -import org.springframework.boot.web.server.LocalServerPort; -import org.springframework.test.context.ActiveProfiles; -import org.testcontainers.containers.DockerComposeContainer; -import org.testcontainers.containers.wait.strategy.Wait; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; - -@ActiveProfiles("it") -@SpringBootTest( - webEnvironment = WebEnvironment.RANDOM_PORT, - properties = { - "feast.core-authentication.enabled=true", - "feast.core-authentication.provider=oauth", - "feast.security.authentication.enabled=true", - "feast.security.authorization.enabled=false" - }) -@Testcontainers -public class ServingServiceOauthAuthenticationIT extends BaseAuthIT { - - CoreSimpleAPIClient coreClient; - FeatureTableProto.FeatureTableSpec expectedFeatureTableSpec; - static final Map options = new HashMap<>(); - - static final int FEAST_SERVING_PORT = 6566; - @LocalServerPort private int metricsPort; - - @ClassRule @Container - public static DockerComposeContainer environment = - new DockerComposeContainer( - new File("src/test/resources/docker-compose/docker-compose-it-hydra.yml"), - new File("src/test/resources/docker-compose/docker-compose-it.yml")) - .withExposedService(HYDRA, HYDRA_PORT, forHttp("/health/alive").forStatusCode(200)) - .withExposedService( - CORE, - FEAST_CORE_PORT, - Wait.forLogMessage(".*gRPC Server started.*\\n", 1) - .withStartupTimeout(Duration.ofMinutes(SERVICE_START_MAX_WAIT_TIME_IN_MINUTES))); - - @BeforeAll - static void globalSetup() throws IOException, InitializationError, InterruptedException { - String hydraExternalHost = environment.getServiceHost(HYDRA, HYDRA_PORT); - Integer hydraExternalPort = environment.getServicePort(HYDRA, HYDRA_PORT); - String hydraExternalUrl = String.format("http://%s:%s", hydraExternalHost, hydraExternalPort); - AuthTestUtils.seedHydra(hydraExternalUrl, CLIENT_ID, CLIENT_SECRET, AUDIENCE, GRANT_TYPE); - - // set up options for call credentials - options.put("oauth_url", TOKEN_URL); - options.put(CLIENT_ID, CLIENT_ID); - options.put(CLIENT_SECRET, CLIENT_SECRET); - options.put("jwkEndpointURI", JWK_URI); - options.put("audience", AUDIENCE); - options.put("grant_type", GRANT_TYPE); - } - - @BeforeEach - public void initState() { - coreClient = AuthTestUtils.getSecureApiClientForCore(FEAST_CORE_PORT, options); - EntityProto.EntitySpecV2 entitySpec = - DataGenerator.createEntitySpecV2( - ENTITY_ID, - "Entity 1 description", - ValueProto.ValueType.Enum.STRING, - ImmutableMap.of("label_key", "label_value")); - coreClient.simpleApplyEntity(PROJECT_NAME, entitySpec); - - expectedFeatureTableSpec = - DataGenerator.createFeatureTableSpec( - FEATURE_TABLE_NAME, - Arrays.asList(ENTITY_ID), - new HashMap<>() { - { - put(FEATURE_NAME, ValueProto.ValueType.Enum.STRING); - } - }, - 7200, - ImmutableMap.of("feat_key2", "feat_value2")) - .toBuilder() - .setBatchSource( - DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", "")) - .build(); - coreClient.simpleApplyFeatureTable(PROJECT_NAME, expectedFeatureTableSpec); - } - - /** Test that Feast Serving metrics endpoint can be accessed with authentication enabled */ - @Test - public void shouldAllowUnauthenticatedAccessToMetricsEndpoint() throws IOException { - Request request = - new Request.Builder() - .url(String.format("http://localhost:%d/metrics", metricsPort)) - .get() - .build(); - Response response = new OkHttpClient().newCall(request).execute(); - assertTrue(response.isSuccessful()); - assertTrue(!response.body().string().isEmpty()); - } - - @Test - public void shouldAllowUnauthenticatedGetOnlineFeatures() { - FeatureTableProto.FeatureTable actualFeatureTable = - coreClient.simpleGetFeatureTable(PROJECT_NAME, FEATURE_TABLE_NAME); - assertEquals(expectedFeatureTableSpec.getName(), actualFeatureTable.getSpec().getName()); - assertEquals( - expectedFeatureTableSpec.getBatchSource(), actualFeatureTable.getSpec().getBatchSource()); - - ServingServiceBlockingStub servingStub = - AuthTestUtils.getServingServiceStub(false, FEAST_SERVING_PORT, null); - GetOnlineFeaturesRequestV2 onlineFeatureRequestV2 = - AuthTestUtils.createOnlineFeatureRequest( - PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequestV2); - - assertEquals(1, featureResponse.getFieldValuesCount()); - Map fieldsMap = featureResponse.getFieldValues(0).getFieldsMap(); - assertTrue(fieldsMap.containsKey(ENTITY_ID)); - assertTrue(fieldsMap.containsKey(FEATURE_TABLE_NAME + ":" + FEATURE_NAME)); - ((ManagedChannel) servingStub.getChannel()).shutdown(); - } - - @Test - void canGetOnlineFeaturesIfAuthenticated() { - FeatureTableProto.FeatureTable actualFeatureTable = - coreClient.simpleGetFeatureTable(PROJECT_NAME, FEATURE_TABLE_NAME); - assertEquals(expectedFeatureTableSpec.getName(), actualFeatureTable.getSpec().getName()); - assertEquals( - expectedFeatureTableSpec.getBatchSource(), actualFeatureTable.getSpec().getBatchSource()); - - ServingServiceBlockingStub servingStub = - AuthTestUtils.getServingServiceStub(true, FEAST_SERVING_PORT, options); - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - AuthTestUtils.createOnlineFeatureRequest( - PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); - - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - assertEquals(1, featureResponse.getFieldValuesCount()); - Map fieldsMap = featureResponse.getFieldValues(0).getFieldsMap(); - assertTrue(fieldsMap.containsKey(ENTITY_ID)); - assertTrue(fieldsMap.containsKey(FEATURE_TABLE_NAME + ":" + FEATURE_NAME)); - ((ManagedChannel) servingStub.getChannel()).shutdown(); - } -} diff --git a/java/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthorizationIT.java b/java/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthorizationIT.java deleted file mode 100644 index 64fe44b2dc..0000000000 --- a/java/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthorizationIT.java +++ /dev/null @@ -1,227 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.it; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.testcontainers.containers.wait.strategy.Wait.forHttp; - -import feast.common.it.DataGenerator; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; -import feast.proto.serving.ServingServiceGrpc.ServingServiceBlockingStub; -import feast.proto.types.ValueProto; -import feast.proto.types.ValueProto.Value; -import io.grpc.ManagedChannel; -import io.grpc.StatusRuntimeException; -import java.io.File; -import java.io.IOException; -import java.time.Duration; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import org.junit.ClassRule; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.junit.runners.model.InitializationError; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.DynamicPropertyRegistry; -import org.springframework.test.context.DynamicPropertySource; -import org.testcontainers.containers.DockerComposeContainer; -import org.testcontainers.containers.wait.strategy.Wait; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; -import org.testcontainers.shaded.com.google.common.collect.ImmutableList; -import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; -import sh.ory.keto.ApiException; - -@ActiveProfiles("it") -@SpringBootTest( - properties = { - "feast.core-authentication.enabled=true", - "feast.core-authentication.provider=oauth", - "feast.security.authentication.enabled=true", - "feast.security.authorization.enabled=true" - }) -@Testcontainers -public class ServingServiceOauthAuthorizationIT extends BaseAuthIT { - - static final Map adminCredentials = new HashMap<>(); - static final Map memberCredentials = new HashMap<>(); - static final String PROJECT_MEMBER_CLIENT_ID = "client_id_1"; - static final String NOT_PROJECT_MEMBER_CLIENT_ID = "client_id_2"; - private static int KETO_PORT = 4466; - private static int KETO_ADAPTOR_PORT = 8080; - static String subjectClaim = "sub"; - static CoreSimpleAPIClient coreClient; - static final int FEAST_SERVING_PORT = 6766; - - @ClassRule @Container - public static DockerComposeContainer environment = - new DockerComposeContainer( - new File("src/test/resources/docker-compose/docker-compose-it-hydra.yml"), - new File("src/test/resources/docker-compose/docker-compose-it.yml"), - new File("src/test/resources/docker-compose/docker-compose-it-keto.yml")) - .withExposedService(HYDRA, HYDRA_PORT, forHttp("/health/alive").forStatusCode(200)) - .withExposedService( - CORE, - FEAST_CORE_PORT, - Wait.forLogMessage(".*gRPC Server started.*\\n", 1) - .withStartupTimeout(Duration.ofMinutes(SERVICE_START_MAX_WAIT_TIME_IN_MINUTES))) - .withExposedService("adaptor_1", KETO_ADAPTOR_PORT) - .withExposedService("keto_1", KETO_PORT, forHttp("/health/ready").forStatusCode(200)); - - @DynamicPropertySource - static void initialize(DynamicPropertyRegistry registry) { - - // Seed Keto with data - String ketoExternalHost = environment.getServiceHost("keto_1", KETO_PORT); - Integer ketoExternalPort = environment.getServicePort("keto_1", KETO_PORT); - String ketoExternalUrl = String.format("http://%s:%s", ketoExternalHost, ketoExternalPort); - try { - AuthTestUtils.seedKeto(ketoExternalUrl, PROJECT_NAME, PROJECT_MEMBER_CLIENT_ID, CLIENT_ID); - } catch (ApiException e) { - throw new RuntimeException(String.format("Could not seed Keto store %s", ketoExternalUrl)); - } - - // Get Keto Authorization Server (Adaptor) url - String ketoAdaptorHost = environment.getServiceHost("adaptor_1", KETO_ADAPTOR_PORT); - Integer ketoAdaptorPort = environment.getServicePort("adaptor_1", KETO_ADAPTOR_PORT); - String ketoAdaptorUrl = String.format("http://%s:%s", ketoAdaptorHost, ketoAdaptorPort); - - // Initialize dynamic properties - registry.add("feast.security.authentication.options.subjectClaim", () -> subjectClaim); - registry.add("feast.security.authentication.options.jwkEndpointURI", () -> JWK_URI); - registry.add("feast.security.authorization.options.authorizationUrl", () -> ketoAdaptorUrl); - registry.add("grpc.server.port", () -> FEAST_SERVING_PORT); - } - - @BeforeAll - static void globalSetup() throws IOException, InitializationError, InterruptedException { - String hydraExternalHost = environment.getServiceHost(HYDRA, HYDRA_PORT); - Integer hydraExternalPort = environment.getServicePort(HYDRA, HYDRA_PORT); - String hydraExternalUrl = String.format("http://%s:%s", hydraExternalHost, hydraExternalPort); - AuthTestUtils.seedHydra(hydraExternalUrl, CLIENT_ID, CLIENT_SECRET, AUDIENCE, GRANT_TYPE); - AuthTestUtils.seedHydra( - hydraExternalUrl, PROJECT_MEMBER_CLIENT_ID, CLIENT_SECRET, AUDIENCE, GRANT_TYPE); - AuthTestUtils.seedHydra( - hydraExternalUrl, NOT_PROJECT_MEMBER_CLIENT_ID, CLIENT_SECRET, AUDIENCE, GRANT_TYPE); - // set up options for call credentials - adminCredentials.put("oauth_url", TOKEN_URL); - adminCredentials.put(CLIENT_ID, CLIENT_ID); - adminCredentials.put(CLIENT_SECRET, CLIENT_SECRET); - adminCredentials.put("jwkEndpointURI", JWK_URI); - adminCredentials.put("audience", AUDIENCE); - adminCredentials.put("grant_type", GRANT_TYPE); - - coreClient = AuthTestUtils.getSecureApiClientForCore(FEAST_CORE_PORT, adminCredentials); - coreClient.simpleApplyEntity( - PROJECT_NAME, - DataGenerator.createEntitySpecV2( - ENTITY_ID, "", ValueProto.ValueType.Enum.STRING, Collections.emptyMap())); - coreClient.simpleApplyFeatureTable( - PROJECT_NAME, - DataGenerator.createFeatureTableSpec( - FEATURE_TABLE_NAME, - ImmutableList.of(ENTITY_ID), - ImmutableMap.of(FEATURE_NAME, ValueProto.ValueType.Enum.STRING), - 0, - Collections.emptyMap())); - } - - @Test - public void shouldNotAllowUnauthenticatedGetOnlineFeatures() { - ServingServiceBlockingStub servingStub = - AuthTestUtils.getServingServiceStub(false, FEAST_SERVING_PORT, null); - - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - AuthTestUtils.createOnlineFeatureRequest( - PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); - Exception exception = - assertThrows( - StatusRuntimeException.class, - () -> { - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - }); - - String expectedMessage = "UNAUTHENTICATED: Authentication failed"; - String actualMessage = exception.getMessage(); - assertEquals(actualMessage, expectedMessage); - ((ManagedChannel) servingStub.getChannel()).shutdown(); - } - - @Test - void canGetOnlineFeaturesIfAdmin() { - ServingServiceBlockingStub servingStub = - AuthTestUtils.getServingServiceStub(true, FEAST_SERVING_PORT, adminCredentials); - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - AuthTestUtils.createOnlineFeatureRequest( - PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - assertEquals(1, featureResponse.getFieldValuesCount()); - Map fieldsMap = featureResponse.getFieldValues(0).getFieldsMap(); - assertTrue(fieldsMap.containsKey(ENTITY_ID)); - assertTrue(fieldsMap.containsKey(FEATURE_TABLE_NAME + ":" + FEATURE_NAME)); - ((ManagedChannel) servingStub.getChannel()).shutdown(); - } - - @Test - void canGetOnlineFeaturesIfProjectMember() { - Map memberCredsOptions = new HashMap<>(); - memberCredsOptions.putAll(adminCredentials); - memberCredsOptions.put(CLIENT_ID, PROJECT_MEMBER_CLIENT_ID); - ServingServiceBlockingStub servingStub = - AuthTestUtils.getServingServiceStub(true, FEAST_SERVING_PORT, memberCredsOptions); - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - AuthTestUtils.createOnlineFeatureRequest( - PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); - GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - assertEquals(1, featureResponse.getFieldValuesCount()); - Map fieldsMap = featureResponse.getFieldValues(0).getFieldsMap(); - assertTrue(fieldsMap.containsKey(ENTITY_ID)); - assertTrue(fieldsMap.containsKey(FEATURE_TABLE_NAME + ":" + FEATURE_NAME)); - ((ManagedChannel) servingStub.getChannel()).shutdown(); - } - - @Test - void cantGetOnlineFeaturesIfNotProjectMember() { - Map notMemberCredsOptions = new HashMap<>(); - notMemberCredsOptions.putAll(adminCredentials); - notMemberCredsOptions.put(CLIENT_ID, NOT_PROJECT_MEMBER_CLIENT_ID); - ServingServiceBlockingStub servingStub = - AuthTestUtils.getServingServiceStub(true, FEAST_SERVING_PORT, notMemberCredsOptions); - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - AuthTestUtils.createOnlineFeatureRequest( - PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); - StatusRuntimeException exception = - assertThrows( - StatusRuntimeException.class, - () -> servingStub.getOnlineFeaturesV2(onlineFeatureRequest)); - - String expectedMessage = - String.format( - "PERMISSION_DENIED: Access denied to project %s for subject %s", - PROJECT_NAME, NOT_PROJECT_MEMBER_CLIENT_ID); - String actualMessage = exception.getMessage(); - assertEquals(actualMessage, expectedMessage); - ((ManagedChannel) servingStub.getChannel()).shutdown(); - } -} diff --git a/java/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java b/java/serving/src/test/java/feast/serving/it/ServingServiceRedisIT.java similarity index 55% rename from java/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java rename to java/serving/src/test/java/feast/serving/it/ServingServiceRedisIT.java index c1e7a15137..6fd065d264 100644 --- a/java/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java +++ b/java/serving/src/test/java/feast/serving/it/ServingServiceRedisIT.java @@ -16,18 +16,22 @@ */ package feast.serving.it; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; import com.google.common.collect.ImmutableList; import com.google.protobuf.Timestamp; -import feast.common.it.DataGenerator; +import com.squareup.okhttp.OkHttpClient; +import com.squareup.okhttp.Request; +import com.squareup.okhttp.Response; import feast.proto.serving.ServingAPIProto; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; import feast.proto.serving.ServingServiceGrpc; +import feast.serving.util.DataGenerator; import io.grpc.ManagedChannel; import java.io.File; +import java.io.IOException; +import java.time.Duration; import java.util.concurrent.TimeUnit; import org.junit.ClassRule; import org.junit.jupiter.api.AfterAll; @@ -42,6 +46,7 @@ import org.springframework.test.context.DynamicPropertyRegistry; import org.springframework.test.context.DynamicPropertySource; import org.testcontainers.containers.DockerComposeContainer; +import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @@ -53,25 +58,35 @@ }) @DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_CLASS) @Testcontainers -public class ServingServiceFeast10IT extends BaseAuthIT { +public class ServingServiceRedisIT { - public static final Logger log = LoggerFactory.getLogger(ServingServiceFeast10IT.class); + public static final Logger log = LoggerFactory.getLogger(ServingServiceRedisIT.class); - static final String timestampPrefix = "_ts"; static ServingServiceGrpc.ServingServiceBlockingStub servingStub; static final int FEAST_SERVING_PORT = 6568; + @LocalServerPort private int metricsPort; @ClassRule @Container public static DockerComposeContainer environment = new DockerComposeContainer( - new File("src/test/resources/docker-compose/docker-compose-feast10-it.yml")) - .withExposedService(REDIS, REDIS_PORT); + new File("src/test/resources/docker-compose/docker-compose-redis-it.yml")) + .withExposedService("redis", 6379) + .withOptions() + .waitingFor( + "materialize", + Wait.forLogMessage(".*Materialization finished.*\\n", 1) + .withStartupTimeout(Duration.ofMinutes(5))); @DynamicPropertySource static void initialize(DynamicPropertyRegistry registry) { registry.add("grpc.server.port", () -> FEAST_SERVING_PORT); + + registry.add("feast.stores[0].name", () -> "online"); + registry.add("feast.stores[0].type", () -> "REDIS"); + registry.add("feast.stores[0].config.host", () -> environment.getServiceHost("redis", 6379)); + registry.add("feast.stores[0].config.port", () -> environment.getServicePort("redis", 6379)); } @BeforeAll @@ -84,9 +99,7 @@ static void tearDown() throws Exception { ((ManagedChannel) servingStub.getChannel()).shutdown().awaitTermination(10, TimeUnit.SECONDS); } - @Test - @DirtiesContext(methodMode = DirtiesContext.MethodMode.AFTER_METHOD) - public void shouldGetOnlineFeatures() { + private GetOnlineFeaturesRequestV2 buildOnlineRequest(int driverId) { // getOnlineFeatures Information String projectName = "feast_project"; String entityName = "driver_id"; @@ -95,7 +108,7 @@ public void shouldGetOnlineFeatures() { final Timestamp timestamp = Timestamp.getDefaultInstance(); GetOnlineFeaturesRequestV2.EntityRow entityRow1 = DataGenerator.createEntityRow( - entityName, DataGenerator.createInt64Value(1001), timestamp.getSeconds()); + entityName, DataGenerator.createInt64Value(driverId), timestamp.getSeconds()); ImmutableList entityRows = ImmutableList.of(entityRow1); // Instantiate FeatureReferences @@ -107,10 +120,14 @@ public void shouldGetOnlineFeatures() { ImmutableList.of(feature1Reference, feature2Reference); // Build GetOnlineFeaturesRequestV2 - GetOnlineFeaturesRequestV2 onlineFeatureRequest = - TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); + return TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); + } + + @Test + @DirtiesContext(methodMode = DirtiesContext.MethodMode.AFTER_METHOD) + public void shouldGetOnlineFeatures() { GetOnlineFeaturesResponse featureResponse = - servingStub.getOnlineFeaturesV2(onlineFeatureRequest); + servingStub.getOnlineFeaturesV2(buildOnlineRequest(1005)); assertEquals(1, featureResponse.getFieldValuesCount()); @@ -125,11 +142,65 @@ public void shouldGetOnlineFeatures() { } assertEquals( - 721, fieldValue.getFieldsOrThrow("driver_hourly_stats:avg_daily_trips").getInt64Val()); + 500, fieldValue.getFieldsOrThrow("driver_hourly_stats:avg_daily_trips").getInt64Val()); + assertEquals(1005, fieldValue.getFieldsOrThrow("driver_id").getInt64Val()); + assertEquals( + 0.5, fieldValue.getFieldsOrThrow("driver_hourly_stats:conv_rate").getDoubleVal(), 0.0001); + } + + @Test + @DirtiesContext(methodMode = DirtiesContext.MethodMode.AFTER_METHOD) + public void shouldGetOnlineFeaturesWithOutsideMaxAgeStatus() { + GetOnlineFeaturesResponse featureResponse = + servingStub.getOnlineFeaturesV2(buildOnlineRequest(1001)); + + assertEquals(1, featureResponse.getFieldValuesCount()); + + final GetOnlineFeaturesResponse.FieldValues fieldValue = featureResponse.getFieldValues(0); + for (final String key : + ImmutableList.of("driver_hourly_stats:avg_daily_trips", "driver_hourly_stats:conv_rate")) { + assertTrue(fieldValue.containsFields(key)); + assertTrue(fieldValue.containsStatuses(key)); + assertEquals( + GetOnlineFeaturesResponse.FieldStatus.OUTSIDE_MAX_AGE, + fieldValue.getStatusesOrThrow(key)); + } + + assertEquals( + 100, fieldValue.getFieldsOrThrow("driver_hourly_stats:avg_daily_trips").getInt64Val()); assertEquals(1001, fieldValue.getFieldsOrThrow("driver_id").getInt64Val()); assertEquals( - 0.74203354, - fieldValue.getFieldsOrThrow("driver_hourly_stats:conv_rate").getDoubleVal(), - 0.0001); + 0.1, fieldValue.getFieldsOrThrow("driver_hourly_stats:conv_rate").getDoubleVal(), 0.0001); + } + + @Test + @DirtiesContext(methodMode = DirtiesContext.MethodMode.AFTER_METHOD) + public void shouldGetOnlineFeaturesWithNotFoundStatus() { + GetOnlineFeaturesResponse featureResponse = + servingStub.getOnlineFeaturesV2(buildOnlineRequest(-1)); + + assertEquals(1, featureResponse.getFieldValuesCount()); + + final GetOnlineFeaturesResponse.FieldValues fieldValue = featureResponse.getFieldValues(0); + for (final String key : + ImmutableList.of("driver_hourly_stats:avg_daily_trips", "driver_hourly_stats:conv_rate")) { + assertTrue(fieldValue.containsFields(key)); + assertTrue(fieldValue.containsStatuses(key)); + assertEquals( + GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND, fieldValue.getStatusesOrThrow(key)); + } + } + + @Test + @DirtiesContext(methodMode = DirtiesContext.MethodMode.AFTER_METHOD) + public void shouldAllowUnauthenticatedAccessToMetricsEndpoint() throws IOException { + Request request = + new Request.Builder() + .url(String.format("http://localhost:%d/metrics", metricsPort)) + .get() + .build(); + Response response = new OkHttpClient().newCall(request).execute(); + assertTrue(response.isSuccessful()); + assertFalse(response.body().string().isEmpty()); } } diff --git a/java/serving/src/test/java/feast/serving/it/TestUtils.java b/java/serving/src/test/java/feast/serving/it/TestUtils.java index 6772dade9c..fb88b2fb37 100644 --- a/java/serving/src/test/java/feast/serving/it/TestUtils.java +++ b/java/serving/src/test/java/feast/serving/it/TestUtils.java @@ -16,22 +16,9 @@ */ package feast.serving.it; -import static org.junit.jupiter.api.Assertions.assertEquals; - -import com.google.common.collect.ImmutableMap; -import feast.common.auth.credentials.OAuthCredentials; -import feast.common.it.DataGenerator; -import feast.proto.core.CoreServiceGrpc; -import feast.proto.core.CoreServiceGrpc.CoreServiceBlockingStub; -import feast.proto.core.EntityProto.Entity; -import feast.proto.core.EntityProto.EntitySpecV2; -import feast.proto.core.FeatureTableProto.FeatureTable; -import feast.proto.core.FeatureTableProto.FeatureTableSpec; import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; import feast.proto.serving.ServingServiceGrpc; -import feast.proto.types.ValueProto; -import io.grpc.CallCredentials; import io.grpc.Channel; import io.grpc.ManagedChannelBuilder; import java.util.*; @@ -42,23 +29,7 @@ public static ServingServiceGrpc.ServingServiceBlockingStub getServingServiceStu boolean isSecure, int feastServingPort, Map options) { Channel secureChannel = ManagedChannelBuilder.forAddress("localhost", feastServingPort).usePlaintext().build(); - - if (isSecure) { - CallCredentials callCredentials = null; - callCredentials = new OAuthCredentials(options); - return ServingServiceGrpc.newBlockingStub(secureChannel).withCallCredentials(callCredentials); - } else { - return ServingServiceGrpc.newBlockingStub(secureChannel); - } - } - - public static CoreSimpleAPIClient getApiClientForCore(int feastCorePort) { - Channel channel = - ManagedChannelBuilder.forAddress("localhost", feastCorePort).usePlaintext().build(); - - CoreServiceBlockingStub coreService = CoreServiceGrpc.newBlockingStub(channel); - - return new CoreSimpleAPIClient(coreService); + return ServingServiceGrpc.newBlockingStub(secureChannel); } public static GetOnlineFeaturesRequestV2 createOnlineFeatureRequest( @@ -71,36 +42,4 @@ public static GetOnlineFeaturesRequestV2 createOnlineFeatureRequest( .addAllEntityRows(entityRows) .build(); } - - public static void applyFeatureTable( - CoreSimpleAPIClient secureApiClient, - String projectName, - String featureTableName, - List entities, - ImmutableMap features, - int maxAgeSecs) { - FeatureTableSpec expectedFeatureTableSpec = - DataGenerator.createFeatureTableSpec( - featureTableName, - entities, - features, - maxAgeSecs, - Map.of("feat_key2", "feat_value2")) - .toBuilder() - .setBatchSource( - DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", "dt_col")) - .build(); - secureApiClient.simpleApplyFeatureTable(projectName, expectedFeatureTableSpec); - FeatureTable actualFeatureTable = - secureApiClient.simpleGetFeatureTable(projectName, featureTableName); - assertEquals(expectedFeatureTableSpec.getName(), actualFeatureTable.getSpec().getName()); - } - - public static void applyEntity( - CoreSimpleAPIClient coreApiClient, String projectName, EntitySpecV2 entitySpec) { - coreApiClient.simpleApplyEntity(projectName, entitySpec); - String entityName = entitySpec.getName(); - Entity actualEntity = coreApiClient.getEntity(projectName, entityName); - assertEquals(entitySpec.getName(), actualEntity.getSpec().getName()); - } } diff --git a/java/serving/src/test/java/feast/serving/service/CachedSpecServiceTest.java b/java/serving/src/test/java/feast/serving/service/CachedSpecServiceTest.java deleted file mode 100644 index 4e48b64ab6..0000000000 --- a/java/serving/src/test/java/feast/serving/service/CachedSpecServiceTest.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.service; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.junit.Assert.assertThat; -import static org.mockito.Mockito.when; -import static org.mockito.MockitoAnnotations.initMocks; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import feast.common.it.DataGenerator; -import feast.proto.core.CoreServiceProto.ListFeatureTablesRequest; -import feast.proto.core.CoreServiceProto.ListFeatureTablesResponse; -import feast.proto.core.CoreServiceProto.ListProjectsRequest; -import feast.proto.core.CoreServiceProto.ListProjectsResponse; -import feast.proto.core.FeatureTableProto; -import feast.proto.core.FeatureTableProto.FeatureTableSpec; -import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; -import feast.proto.types.ValueProto; -import feast.serving.specs.CachedSpecService; -import feast.serving.specs.CoreSpecService; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.mockito.Mock; - -public class CachedSpecServiceTest { - - @Rule public final ExpectedException expectedException = ExpectedException.none(); - - @Mock CoreSpecService coreService; - - private CachedSpecService cachedSpecService; - - private ImmutableList featureTableEntities; - private ImmutableMap featureTable1Features; - private ImmutableMap featureTable2Features; - private FeatureTableSpec featureTable1Spec; - private FeatureTableSpec featureTable2Spec; - - @Before - public void setUp() { - initMocks(this); - - this.setupProject("default"); - this.featureTableEntities = ImmutableList.of("entity1"); - this.featureTable1Features = - ImmutableMap.of( - "trip_cost1", ValueProto.ValueType.Enum.INT64, - "trip_distance1", ValueProto.ValueType.Enum.DOUBLE, - "trip_empty1", ValueProto.ValueType.Enum.DOUBLE); - this.featureTable2Features = - ImmutableMap.of( - "trip_cost2", ValueProto.ValueType.Enum.INT64, - "trip_distance2", ValueProto.ValueType.Enum.DOUBLE, - "trip_empty2", ValueProto.ValueType.Enum.DOUBLE); - this.featureTable1Spec = - DataGenerator.createFeatureTableSpec( - "featuretable1", - this.featureTableEntities, - featureTable1Features, - 7200, - ImmutableMap.of()); - this.featureTable2Spec = - DataGenerator.createFeatureTableSpec( - "featuretable2", - this.featureTableEntities, - featureTable2Features, - 7200, - ImmutableMap.of()); - - this.setupFeatureTableAndProject("default"); - - cachedSpecService = new CachedSpecService(this.coreService); - } - - private void setupProject(String project) { - when(coreService.listProjects(ListProjectsRequest.newBuilder().build())) - .thenReturn(ListProjectsResponse.newBuilder().addProjects(project).build()); - } - - private void setupFeatureTableAndProject(String project) { - FeatureTableProto.FeatureTable featureTable1 = - FeatureTableProto.FeatureTable.newBuilder().setSpec(this.featureTable1Spec).build(); - FeatureTableProto.FeatureTable featureTable2 = - FeatureTableProto.FeatureTable.newBuilder().setSpec(this.featureTable2Spec).build(); - - when(coreService.listFeatureTables( - ListFeatureTablesRequest.newBuilder() - .setFilter(ListFeatureTablesRequest.Filter.newBuilder().setProject(project).build()) - .build())) - .thenReturn( - ListFeatureTablesResponse.newBuilder() - .addTables(featureTable1) - .addTables(featureTable2) - .build()); - } - - @Test - public void shouldPopulateAndReturnDifferentFeatureTables() { - // test that CachedSpecService can retrieve fully qualified feature references. - cachedSpecService.populateCache(); - FeatureReferenceV2 featureReference1 = - FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable1") - .setName("trip_cost1") - .build(); - FeatureReferenceV2 featureReference2 = - FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable1") - .setName("trip_distance1") - .build(); - FeatureReferenceV2 featureReference3 = - FeatureReferenceV2.newBuilder() - .setFeatureTable("featuretable2") - .setName("trip_empty2") - .build(); - - assertThat( - cachedSpecService.getFeatureTableSpec("default", featureReference1), - equalTo(this.featureTable1Spec)); - assertThat( - cachedSpecService.getFeatureTableSpec("default", featureReference2), - equalTo(this.featureTable1Spec)); - assertThat( - cachedSpecService.getFeatureTableSpec("default", featureReference3), - equalTo(this.featureTable2Spec)); - } -} diff --git a/java/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java b/java/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java index d3e62b4a49..d64cd07fb4 100644 --- a/java/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java +++ b/java/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java @@ -16,7 +16,7 @@ */ package feast.serving.service; -import static feast.common.it.DataGenerator.*; +import static feast.serving.util.DataGenerator.*; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.mockito.ArgumentMatchers.any; @@ -26,15 +26,16 @@ import com.google.protobuf.Duration; import com.google.protobuf.Timestamp; import feast.proto.core.FeatureProto; -import feast.proto.core.FeatureTableProto.FeatureTableSpec; +import feast.proto.core.FeatureViewProto; import feast.proto.serving.ServingAPIProto; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; import feast.proto.types.ValueProto; -import feast.serving.specs.CachedSpecService; -import feast.serving.specs.CoreFeatureSpecRetriever; +import feast.serving.registry.LocalRegistryRepo; +import feast.serving.specs.FeatureSpecRetriever; +import feast.serving.specs.RegistryFeatureSpecRetriever; import feast.storage.api.retriever.Feature; import feast.storage.api.retriever.ProtoFeature; import feast.storage.connectors.redis.retriever.OnlineRetriever; @@ -50,7 +51,7 @@ public class OnlineServingServiceTest { - @Mock CachedSpecService specService; + @Mock LocalRegistryRepo registryRepo; @Mock Tracer tracer; @Mock OnlineRetriever retrieverV2; private String transformationServiceEndpoint; @@ -63,12 +64,12 @@ public class OnlineServingServiceTest { @Before public void setUp() { initMocks(this); - CoreFeatureSpecRetriever coreFeatureSpecRetriever = new CoreFeatureSpecRetriever(specService); + FeatureSpecRetriever featureSpecRetriever = new RegistryFeatureSpecRetriever(registryRepo); OnlineTransformationService onlineTransformationService = - new OnlineTransformationService(transformationServiceEndpoint, coreFeatureSpecRetriever); + new OnlineTransformationService(transformationServiceEndpoint, featureSpecRetriever); onlineServingServiceV2 = new OnlineServingServiceV2( - retrieverV2, tracer, coreFeatureSpecRetriever, onlineTransformationService); + retrieverV2, tracer, featureSpecRetriever, onlineTransformationService); mockedFeatureRows = new ArrayList<>(); mockedFeatureRows.add( @@ -160,14 +161,14 @@ public void shouldReturnResponseWithValuesAndMetadataIfKeysPresent() { List> featureRows = List.of(entityKeyList1, entityKeyList2); when(retrieverV2.getOnlineFeatures(any(), any(), any(), any())).thenReturn(featureRows); - when(specService.getFeatureTableSpec(any(), any())).thenReturn(getFeatureTableSpec()); - when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(0).getFeatureReference())) + when(registryRepo.getFeatureViewSpec(any(), any())).thenReturn(getFeatureViewSpec()); + when(registryRepo.getFeatureSpec(projectName, mockedFeatureRows.get(0).getFeatureReference())) .thenReturn(featureSpecs.get(0)); - when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(1).getFeatureReference())) + when(registryRepo.getFeatureSpec(projectName, mockedFeatureRows.get(1).getFeatureReference())) .thenReturn(featureSpecs.get(1)); - when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(2).getFeatureReference())) + when(registryRepo.getFeatureSpec(projectName, mockedFeatureRows.get(2).getFeatureReference())) .thenReturn(featureSpecs.get(0)); - when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(3).getFeatureReference())) + when(registryRepo.getFeatureSpec(projectName, mockedFeatureRows.get(3).getFeatureReference())) .thenReturn(featureSpecs.get(1)); when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); @@ -227,10 +228,10 @@ public void shouldReturnResponseWithUnsetValuesAndMetadataIfKeysNotPresent() { List> featureRows = List.of(entityKeyList1, entityKeyList2); when(retrieverV2.getOnlineFeatures(any(), any(), any(), any())).thenReturn(featureRows); - when(specService.getFeatureTableSpec(any(), any())).thenReturn(getFeatureTableSpec()); - when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(0).getFeatureReference())) + when(registryRepo.getFeatureViewSpec(any(), any())).thenReturn(getFeatureViewSpec()); + when(registryRepo.getFeatureSpec(projectName, mockedFeatureRows.get(0).getFeatureReference())) .thenReturn(featureSpecs.get(0)); - when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(1).getFeatureReference())) + when(registryRepo.getFeatureSpec(projectName, mockedFeatureRows.get(1).getFeatureReference())) .thenReturn(featureSpecs.get(1)); when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); @@ -265,7 +266,7 @@ public void shouldReturnResponseWithUnsetValuesAndMetadataIfKeysNotPresent() { } @Test - public void shouldReturnResponseWithUnsetValuesAndMetadataIfMaxAgeIsExceeded() { + public void shouldReturnResponseWithValuesAndMetadataIfMaxAgeIsExceeded() { String projectName = "default"; ServingAPIProto.FeatureReferenceV2 featureReference1 = ServingAPIProto.FeatureReferenceV2.newBuilder() @@ -291,9 +292,9 @@ public void shouldReturnResponseWithUnsetValuesAndMetadataIfMaxAgeIsExceeded() { List> featureRows = List.of(entityKeyList1, entityKeyList2); when(retrieverV2.getOnlineFeatures(any(), any(), any(), any())).thenReturn(featureRows); - when(specService.getFeatureTableSpec(any(), any())) + when(registryRepo.getFeatureViewSpec(any(), any())) .thenReturn( - FeatureTableSpec.newBuilder() + FeatureViewProto.FeatureViewSpec.newBuilder() .setName("featuretable_1") .addEntities("entity1") .addEntities("entity2") @@ -307,11 +308,11 @@ public void shouldReturnResponseWithUnsetValuesAndMetadataIfMaxAgeIsExceeded() { .setName("feature_2") .setValueType(ValueProto.ValueType.Enum.STRING) .build()) - .setMaxAge(Duration.newBuilder().setSeconds(1)) + .setTtl(Duration.newBuilder().setSeconds(1)) .build()); - when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(1).getFeatureReference())) + when(registryRepo.getFeatureSpec(projectName, mockedFeatureRows.get(1).getFeatureReference())) .thenReturn(featureSpecs.get(1)); - when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(5).getFeatureReference())) + when(registryRepo.getFeatureSpec(projectName, mockedFeatureRows.get(5).getFeatureReference())) .thenReturn(featureSpecs.get(0)); when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); @@ -324,7 +325,7 @@ public void shouldReturnResponseWithUnsetValuesAndMetadataIfMaxAgeIsExceeded() { .putStatuses("entity1", FieldStatus.PRESENT) .putFields("entity2", createStrValue("a")) .putStatuses("entity2", FieldStatus.PRESENT) - .putFields("featuretable_1:feature_1", createEmptyValue()) + .putFields("featuretable_1:feature_1", createStrValue("6")) .putStatuses("featuretable_1:feature_1", FieldStatus.OUTSIDE_MAX_AGE) .putFields("featuretable_1:feature_2", createStrValue("2")) .putStatuses("featuretable_1:feature_2", FieldStatus.PRESENT) @@ -335,7 +336,7 @@ public void shouldReturnResponseWithUnsetValuesAndMetadataIfMaxAgeIsExceeded() { .putStatuses("entity1", FieldStatus.PRESENT) .putFields("entity2", createStrValue("b")) .putStatuses("entity2", FieldStatus.PRESENT) - .putFields("featuretable_1:feature_1", createEmptyValue()) + .putFields("featuretable_1:feature_1", createStrValue("6")) .putStatuses("featuretable_1:feature_1", FieldStatus.OUTSIDE_MAX_AGE) .putFields("featuretable_1:feature_2", createStrValue("2")) .putStatuses("featuretable_1:feature_2", FieldStatus.PRESENT) @@ -345,8 +346,8 @@ public void shouldReturnResponseWithUnsetValuesAndMetadataIfMaxAgeIsExceeded() { assertThat(actual, equalTo(expected)); } - private FeatureTableSpec getFeatureTableSpec() { - return FeatureTableSpec.newBuilder() + private FeatureViewProto.FeatureViewSpec getFeatureViewSpec() { + return FeatureViewProto.FeatureViewSpec.newBuilder() .setName("featuretable_1") .addEntities("entity1") .addEntities("entity2") @@ -360,7 +361,7 @@ private FeatureTableSpec getFeatureTableSpec() { .setName("feature_2") .setValueType(ValueProto.ValueType.Enum.STRING) .build()) - .setMaxAge(Duration.newBuilder().setSeconds(120)) + .setTtl(Duration.newBuilder().setSeconds(120)) .build(); } diff --git a/java/common-test/src/main/java/feast/common/it/DataGenerator.java b/java/serving/src/test/java/feast/serving/util/DataGenerator.java similarity index 99% rename from java/common-test/src/main/java/feast/common/it/DataGenerator.java rename to java/serving/src/test/java/feast/serving/util/DataGenerator.java index 8a0dbb02cd..ab537fa6f9 100644 --- a/java/common-test/src/main/java/feast/common/it/DataGenerator.java +++ b/java/serving/src/test/java/feast/serving/util/DataGenerator.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.common.it; +package feast.serving.util; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; diff --git a/java/serving/src/test/resources/application-it.properties b/java/serving/src/test/resources/application-it.properties index 000e512a68..6b56408d43 100644 --- a/java/serving/src/test/resources/application-it.properties +++ b/java/serving/src/test/resources/application-it.properties @@ -12,7 +12,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -feast.core-authentication.enabled=false -feast.security.authentication.enabled=false -feast.security.authorization.enabled=false \ No newline at end of file diff --git a/java/serving/src/test/resources/docker-compose/core/application-it.yml b/java/serving/src/test/resources/docker-compose/core/application-it.yml deleted file mode 100644 index 1298ff9905..0000000000 --- a/java/serving/src/test/resources/docker-compose/core/application-it.yml +++ /dev/null @@ -1,13 +0,0 @@ -feast: - stream: - type: kafka - options: - topic: feast-features - bootstrapServers: "kafka:9092,localhost:9094" - - security: - authentication: - enabled: true - provider: jwt - options: - jwkEndpointURI: http://hydra:4444/.well-known/jwks.json \ No newline at end of file diff --git a/java/serving/src/test/resources/docker-compose/docker-compose-bigtable-it.yml b/java/serving/src/test/resources/docker-compose/docker-compose-bigtable-it.yml deleted file mode 100644 index 28985efec9..0000000000 --- a/java/serving/src/test/resources/docker-compose/docker-compose-bigtable-it.yml +++ /dev/null @@ -1,38 +0,0 @@ -version: '3' - -services: - core: - image: gcr.io/kf-feast/feast-core:develop - volumes: - - ./core/application-it.yml:/etc/feast/application.yml - environment: - DB_HOST: db - restart: on-failure - depends_on: - - db - ports: - - 6565:6565 - command: - - java - - -jar - - /opt/feast/feast-core.jar - - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml - - db: - image: postgres:12-alpine - environment: - POSTGRES_PASSWORD: password - ports: - - "5432:5432" - - bigtable: - image: google/cloud-sdk:latest - environment: - GOOGLE_APPLICATION_CREDENTIALS: /Users/user/.config/gcloud/application_default_credentials.json - command: - - gcloud - - beta - - emulators - - bigtable - - start - - --host-port=0.0.0.0:8086 \ No newline at end of file diff --git a/java/serving/src/test/resources/docker-compose/docker-compose-cassandra-it.yml b/java/serving/src/test/resources/docker-compose/docker-compose-cassandra-it.yml deleted file mode 100644 index 15afad0541..0000000000 --- a/java/serving/src/test/resources/docker-compose/docker-compose-cassandra-it.yml +++ /dev/null @@ -1,31 +0,0 @@ -version: '3' - -services: - core: - image: gcr.io/kf-feast/feast-core:develop - volumes: - - ./core/application-it.yml:/etc/feast/application.yml - environment: - DB_HOST: db - restart: on-failure - depends_on: - - db - ports: - - 6565:6565 - command: - - java - - -jar - - /opt/feast/feast-core.jar - - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml - - db: - image: postgres:12-alpine - environment: - POSTGRES_PASSWORD: password - ports: - - "5432:5432" - - cassandra: - image: datastax/cassandra:4.0 - ports: - - "9042:9042" \ No newline at end of file diff --git a/java/serving/src/test/resources/docker-compose/docker-compose-it-hydra.yml b/java/serving/src/test/resources/docker-compose/docker-compose-it-hydra.yml deleted file mode 100644 index 1c20610cc7..0000000000 --- a/java/serving/src/test/resources/docker-compose/docker-compose-it-hydra.yml +++ /dev/null @@ -1,54 +0,0 @@ -version: '3' - -services: - hydra-migrate: - image: oryd/hydra:v1.6.0 - environment: - - DSN=postgres://hydra:secret@postgresd:5432/hydra?sslmode=disable&max_conns=20&max_idle_conns=4 - command: - migrate sql -e --yes - restart: on-failure - - hydra: - depends_on: - - hydra-migrate - environment: - - DSN=postgres://hydra:secret@postgresd:5432/hydra?sslmode=disable&max_conns=20&max_idle_conns=4 - - postgresd: - image: postgres:9.6 - ports: - - "54320:5432" - environment: - - POSTGRES_USER=hydra - - POSTGRES_PASSWORD=secret - - POSTGRES_DB=hydra - - hydra: - image: oryd/hydra:v1.6.0 - ports: - - "4444:4444" # Public port - - "4445:4445" # Admin port - #- "5555:5555" # Port for hydra token user - command: - serve all --dangerous-force-http - environment: - - URLS_SELF_ISSUER=http://hydra:4444 - - URLS_CONSENT=http://hydra:3000/consent - - URLS_LOGIN=http://hydra:3000/login - - URLS_LOGOUT=http://hydra:3000/logout - - DSN=memory - - SECRETS_SYSTEM=youReallyNeedToChangeThis - - OIDC_SUBJECT_IDENTIFIERS_SUPPORTED_TYPES=public,pairwise - - OIDC_SUBJECT_IDENTIFIERS_PAIRWISE_SALT=youReallyNeedToChangeThis - - OAUTH2_ACCESS_TOKEN_STRATEGY=jwt - - OIDC_SUBJECT_IDENTIFIERS_SUPPORTED_TYPES=public - restart: unless-stopped - - consent: - environment: - - HYDRA_ADMIN_URL=http://hydra:4445 - image: oryd/hydra-login-consent-node:v1.5.2 - ports: - - "3000:3000" - restart: unless-stopped diff --git a/java/serving/src/test/resources/docker-compose/docker-compose-it-keto.yml b/java/serving/src/test/resources/docker-compose/docker-compose-it-keto.yml deleted file mode 100644 index 8ebf7f225e..0000000000 --- a/java/serving/src/test/resources/docker-compose/docker-compose-it-keto.yml +++ /dev/null @@ -1,44 +0,0 @@ -version: '3' -services: - keto: - depends_on: - - ketodb - - migrations - image: oryd/keto:v0.4.3-alpha.2 - environment: - - DSN=postgres://keto:keto@ketodb:5432/keto?sslmode=disable - command: - - serve - ports: - - 4466 - - ketodb: - image: bitnami/postgresql:9.6 - environment: - - POSTGRESQL_USERNAME=keto - - POSTGRESQL_PASSWORD=keto - - POSTGRESQL_DATABASE=keto - ports: - - "54340:5432" - - migrations: - depends_on: - - ketodb - image: oryd/keto:v0.4.3-alpha.2 - environment: - - DSN=postgres://keto:keto@ketodb:5432/keto?sslmode=disable - command: - - migrate - - sql - - -e - - adaptor: - depends_on: - - keto - image: gcr.io/kf-feast/feast-keto-auth-server:latest - environment: - SERVER_PORT: 8080 - KETO_URL: http://keto:4466 - ports: - - 8080 - restart: on-failure \ No newline at end of file diff --git a/java/serving/src/test/resources/docker-compose/docker-compose-it.yml b/java/serving/src/test/resources/docker-compose/docker-compose-it.yml deleted file mode 100644 index fb7fb1f6ad..0000000000 --- a/java/serving/src/test/resources/docker-compose/docker-compose-it.yml +++ /dev/null @@ -1,53 +0,0 @@ -version: '3' - -services: - core: - image: gcr.io/kf-feast/feast-core:develop - volumes: - - ./core/application-it.yml:/etc/feast/application.yml - environment: - DB_HOST: db - restart: on-failure - depends_on: - - db - - kafka - ports: - - 6565:6565 - command: - - java - - -jar - - /opt/feast/feast-core.jar - - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml - - kafka: - image: confluentinc/cp-kafka:5.2.1 - environment: - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka:9092,OUTSIDE://localhost:9094 - KAFKA_LISTENERS: INSIDE://:9092,OUTSIDE://:9094 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE - ports: - - "9092:9092" - - "9094:9094" - - depends_on: - - zookeeper - - zookeeper: - image: confluentinc/cp-zookeeper:5.2.1 - environment: - ZOOKEEPER_CLIENT_PORT: 2181 - - db: - image: postgres:12-alpine - environment: - POSTGRES_PASSWORD: password - ports: - - "5432:5432" - - redis: - image: redis:5-alpine - ports: - - "6379:6379" \ No newline at end of file diff --git a/java/serving/src/test/resources/docker-compose/docker-compose-feast10-it.yml b/java/serving/src/test/resources/docker-compose/docker-compose-redis-it.yml similarity index 55% rename from java/serving/src/test/resources/docker-compose/docker-compose-feast10-it.yml rename to java/serving/src/test/resources/docker-compose/docker-compose-redis-it.yml index 33d65f465e..08a50233df 100644 --- a/java/serving/src/test/resources/docker-compose/docker-compose-feast10-it.yml +++ b/java/serving/src/test/resources/docker-compose/docker-compose-redis-it.yml @@ -1,12 +1,6 @@ version: '3' services: - db: - image: postgres:12-alpine - environment: - POSTGRES_PASSWORD: password - ports: - - "5432:5432" redis: image: redis:5-alpine ports: diff --git a/java/serving/src/test/resources/docker-compose/feast10/driver_stats.parquet b/java/serving/src/test/resources/docker-compose/feast10/driver_stats.parquet deleted file mode 100644 index df8cbba388827fe2e312921abf21e6322b3b8e27..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 34708 zcmb5Uc|29o`}b|0i8513G9^jKaQ1bXXEJ3%fG~KKF{@$;eYhQa^%Sy*)ivW{A)+T{5X>)-G%mPd+ z>F9a|mFQ?iH{^Sm>FB05Wo@KqRB2jBt zj5C1;x5G{XwlkYt2x?WBb`jsIcY^p|cV2NNe!<72-6WzNb9Ez_5>(wo5MRUTPDELl zCgR}zroBWA@I3B8BK{wqo&-(lG%o@diB;Z2q-%IX#Ir`NK14KrlE05c?+)zuCCFmX z_ajir7xgFNnB2bsjl9*fpNJCHr3XmVITLh{z$er!fPjljGLVRMrI0*_Z%h2-LD=)piaLcmWY$NuZS2U+Y(1asg~?xBG@r4r1&Tb)M4^q8+iRN@~< zC*o+;nG6zzYK3MJh~Kk3PSEPUA&ZC}i$96TmipiX5o=eU&nA)8m7_TXU-j*C2@1v( z@(5`DjQPZ$`a4ei_?)_v#Fv#xDj?Bd^R7aI085o30{&@^Vj?yjc|k-M_NG%rq$|lR zA<YXL%91=N4M4vqi zMCAI`T}i~cc;*9Uz3q(x*bGDX53fU1C35GY= zTp|d%A$ysK!WMst*fRC7j)-nSwO2^Q#CEKnpt{)UDuMZCr3Qk9c9v_z&$WL_eD$|i z8;L&|m3o~-F?@TP2&B$!yFt*a&eKdp|G^m|^19u;NyLUPCt674nB;Sd;Fl=gCMdfk zaEFNcy6=cM_prT{i0S(Z+eoDJ`(Qi4XqI6IL8#Q~P9lmoeIa72S$`K1J)W0!lZfqb zNDo0Ri)Al?RgqL55x;KwH=qS=_xp)R+fhA0qN%y)L4x?m9YX}Nyb5;-1}hlu5kFwt zzX9bRxI9e!CfCIKBy#!e@_>LYaq9@dSrLv=A{t+uBH}yk>ko;THJb5=L@GWWj|s+p zYK{>^9_Je;qQv@t1KQbmdxD5Qrg=|D#P!T?lAtb7_bGuLv&b_dE){$v;wgph=S0+Q zJvBw58Qa4z2$El$OcN+XNW3KCFwegM4Jsd+A)>I_*;gcL=?kAFaNA}5ntBx+ST_>;h+*Weccn~V5wBG!KVOhl_=eSe7fRj}+YJ)1&7O)yMWABM&P~Lzndd}|3~l5gq6AmQ3KDgm-pfnivsIIifUAdh zB@yeKUlY;p!>v_BT#C)*C($VZKLG;m^E!eAGc;i#A|?-iAfkdtmoO2BzZZ*;C@Afa zD1q==6ET98`nBRjbTeEeBGbg+Y9dzeuUJDObNcYL1Pi%V5(K%jGV6$_-twD>liNon zi5T;;W<80dLSm%|dN~|75croWN)zxZGs_Uaq3a3p9i8fBiT~?evK)!ZVs^_D=<};> zB$%t>QXpcw)=MHP-Mg`gh@d$$oW-tx3MiDajbXb=n@G1nvrU|+9AME;V0 z6-1NLFpY>Vos|@c=p3Vf;Ox9D5*Qzq*Cu$kl3s`SS(X1P2o=psy2KwFiq|7iR zAmT#QzX~FkZ_u8eS6%&Fxx>Hnc=Gj%wmZyp0(1hu)(Rv_CMjE zXLoJkRqx?)lVZb*Mc4es>i0eAzF2(y;8c5xn9-$EHxAD|IKQ*!QpwGbh3URxqsyhY zB9=bCdD3(F^qr%0j2z;|b!BbG*jCBA^wyo}Na9^@ddm1pc~_e7R^Q3qD-}J*CG=9o zP3q6~<;dFBxb)SZ8#t-7xBryM)ykn_^}}x``>s~qJFR_;W3_2R_5BLt9Qj@S4d+Ly ztSU@POs~~Eyx@4%_i6vN3y&|mb)>E~Ypfl=>O;@Os}!S6&&bLvWmOlW!^pZqLO)9> zR+pJmNXcWkE>@3q#aiun-Ys$Z?5kv~&RSiGGvE~3?ADpJ<(MJ2*tUR~;VZ|CR;0h)lSZ~%IciWwbUIO9WJ|dWviyyZ`yf1;MId`X%3rr--%&frIzlv zWp7W`2HVDTr>)-i$_#SUGMrWY9@lw}G-mAFcJO&?!m4eVE*ghl51+HWp1Dgak6Y2(mR@S z_t{o$KVz7undiIXg2NTBhs}9@4wrYgB?@Te`#W9rzR$oW;YJtBz``S8T;@i9jD=^7 zcCy4Ch6D~lMYrCvJ&Z{_YaoVgojX&?N@>e7<1_BeX@ZKbt;y^5vSf&=9hmApvzPVw z8i=N2m-JveA!(Q?VN&kFo+E8ps+}U~$&n{-f61+{+>`U9qU-G#_Vr#|h00!c%SeJZnu^@6A)98T?_Yuflu9X^8$q$06mzdqy{rXPxO;AHE91%r!cxQu|h(GtF1r z(|>l~sw&G;h~?Pe%YWXc((;VyIbVSb_Lp4SQaAVsUUY6e@S^{mpU`F3+tKu#(*D9% z+`BW^nN|9WT=lwJs*@(YU-X*qqf2`RD))RA* z%%f{Cj!W*4RA09A_Hy&{hc*o4EADPjmpd#yRHSxrdhq;VnR}&hl!045NcMiY;c-ce znjpE6O3Tx_8S+QuAJ*7kb|0!avhndH*E@0C8-o?b>%H!kGxKV<(lIb|iEC$Rw=uAA zi_>Cwb=sLY1Qc9avvfLGcvjOQ>G*Uz*;Z~aKc#&_w~Iq?lS@kspI$eYsH*>D>j}Lc zo;6hDFS?cby}XhJ>Eb%s`h6>TeQ`9j*iwPCqhqb#p@-3dI9eCOj#VBMsseb6yV2R#I%cq*b?A`l$k&TssS_Um9kv zrlOfdY-WtJH&~SF7uvis&EK@^cD#t~ta*{@{%4(qwy!Nqsi;3pqIPpu4SK!WIww{|r7B{4q^2eiP$)1E>ZPR zx^;2!?ELbMO)=|^C31^vhAz=b#wYVhnHH$WN+zTVC^$ET(yvcU7g6yYYl>Z;l)0J~ zzO+Oym7KNCOg~jvBTgzMdxLqAdKklo)Lc2c8pj)P8`APOxip9VV31BP*y7P&pm9t( zqe#_%qA83~Ci9d=(A%*a$7GI|QjyC`KNw}R%5>s6L^R`NPm~*^i)w^3$z`84$(46% zj+e`+v^b?1w#+1-TWx*Tv`{laKCfm+t@Dj==8gHaP7S`}%?TS%UfR_X{$rV0p`dP0 zXR3%+qC#Q4=U|aW1k0wPhJB+oPB#-b6*ummY!3U$qIjw)aHhXdD@n1W`N+G88xgFV zOIyOez8$}rwE6Vys9(!JezGc+wZ<}Wiqevm&a@|RiD^c%Z7J_e;os=olDws&J40M6 z{1=<@+1?XUW<|6V<#YXc3Ok!4*|$~>7OMD7w4`jUx?4hvSpLPXQayY|KTVWMRXP9Q zoO!Wk6o+cf=y|&f&bLxkFFd;Fax?rlhg$8}6_0@;DlK1d6PakJ6QbF33%T&k?hG9NHJ`d1&`OkN2xDn7xag)eThA@3&@i1{3a zE+R2hM-~k?oZ&!iD^8G^c7{KTPI&w1X)V>ap6GH{5NEFJ!#(~QsG=c*=el#?`J>&~ zpK=9`Zi&P-2ZkZ2_zkG7ipF*ImiWuy9P~TvLff;^c=+5PwbfGy?|)WAokM-VQ%i^2 zj{0HqHfL zcrlj-=0&eS$>;+W9?FZ7f^OJtWQ=o9D=2Aebr1~_!$F%efP^uye*Y3e!x^FL77fq6 zG@*VNYv3lQGnCbhjkuu2fX8!Ou(B`$f5=;cZm&CJoABb579~7-K!mE54#3A3SRq4g z11`<5;fRGGiaonUjpTAs%hGw^F&Tv?`+mVQc7NQxFB|fI@4=sTF;KPFADfgpsN=lN zz#XcB>GT#LelZ$(TWxSQu@gia1@ZIUZ&31S0k{`EF~>6kMnr_M=0Y%zv@_%DG*J}k z4MEA7{rGB)7-nAfLDh<<)Z%MCI6Z8FfA_nP@4^~YIW^%mGY9T{>xE{!!l^$g zp@v0lP_~$xy0PXDoIGudyLuXd&b|%yT{{gedSWbN48?9XsM1*qTSFx>Ni|Wj)OfS$yA$|EM7J6 zN7p7cYUGG8Hs^n)==48p-HO_PW+P10-TA#}YcvQiSmuGrvKDSl??tAX6o?UB2h$QC zXxq#_fl@6a{s=4s)p7}Vb1n!^=e(gJSO=K(g8Ga%>ch9;G-VRTpmrJ2HDMUFgt{YFE++ovG< z_Ac;Jv%{nL<#7GUPdIWQi;`E^j6ZKg;A$;52x07obFK22^tcjadZRTzu9=6i>w$PR zdNXxldn-&kv7*nU4pQ-|cqhvXI`{3jd9m+}V z#Ngeh;Q87P2+g2JznxD2X3xSlb1@M4$U>?6J_EjM2ax~6G9}k*h#LC$sXZRWu*ypj zOS*>OQFArKKoF*~?!@Silkg+g1|3E>qW}IlXpiN@Aigd5drdpga%SO3x*#q`-3Ke< z?HE>*0MCtcpz%I8?s~?9lDxL~MTrTYx>Zuz+SlRfJSVcAzXF}rrubp(C~h6^gD6=M z^gMY9@+SbF9J&tEyA{y(n-BIq3cxM0ir8kIO4V(VMCr%-@phdUIz*XJ{Ec!Hza~G1 z-fM+d+@@H~?TI7wN06Dm51g7~pmgyGBrjY6Frq{5s(Ar)ZqEb4KnJG`P&NAKeDsa5KFxoZ?@N?Qg1K zUR6@|JhJ$BnHlr6P4LK$Er@sA zk#UbVdbP8o+0_V?Bi8`=vr)L}h7Ni-xFO>$U38C|2amD|@N%C54!Jcbn132RsQ-pT z_UvdXCXDstI@opb8~l3o5e~2v(s-A2G1<%lpIy~KUwHt=}F~9=pSxWL`FwRPv;SNc2G}@to3c4ZqKH?N~ zS1;1;TRY)oCKK3SwuDQGbdV;w9X@Ir(+cPkxwi9I!8eoXIJbu|b272q1amsKM)YfK#?9v{5cg_%EnJr+GNEBqM zt-`nDeAf2bgqz-9fM*l3U?vp@Uw3Z7Z$GwR!9_9#^#`!PDT+FGC(C%o5q1Y!7 zsg_1b@`+&5BPH7N-P_RQiU`UFZp6(ypTox|6|j-38lEb~P_q7Swczyz5S#PC8u}27 zt-B5fG?*!BuPt1NAAtFmdf2#H0UzI%!BRU_xcM^;M%_bkUz!-UKQzR!ToycH!hr&e z4)}{=!i^^)(R`8>*avpb<#!(Zy#2`i_0ql;Jz`<@~&`8S%A)LoFCh!5sfeSlg3G?c~kzM{i;ZfShMkW6hE zLFk&|K@~?j+*{-cEMbSiHP{eugFM`rRS*jiFm5w{Ui(=^2M&)5qMVrhthaIO51Zo8?y|h@s71CSosIz z{Elo|%LP$%Y0*aq`8+sqdnJB!$%b>}d{P$WM>XwuaP<{LVfPHk6wpSS7;PA4ItD-J z5-BDzf9yE+gt}(?1dVg~kUYY?rttS-QKolP2b4Bgf(_nHr93S3T zk8fS>!zQf^sz1vQHJqDZ%?D-}v13A432khA6c0-q_M?5r5jT!Id=?J8PUfLZvoP7kpbpo(9 zwt{pqg-Q(uc&Ew}*J@3J=#wT$IUR&L4?HRQ!a)2o?g3Z(TVVhFWw@g*j(1NBVCIH6 zDrwsgj2{og9ckYnLM0L0_mQ=OzO~@IygZpgzAr>AnhK6kLVHC zP3w{QWDn-<3V_Ms2rL}FM@3&1MX|D#IQLKq^Y7}RxfvtYCG+A+sk$=!`mFKl~&>IKqEXn?}6iMoc; zscr#=_Bv3US&f!sfSr?}6l=c&4vgM~(`OZN>y`;Hib6DTazWO#NYJ_b3GU|Kg|Xcm zu(*{4ulCxZDZeO{bAbz^cC|8H4(v2xM$rLwWtY3~TQF0p@$|7@aZ&z_|kl8g`>m@>(1<5TyF< zjYGI<9cUb03syn*XeO2`Fr|wJ_r)Z_%tj%+-M1ID%eO^gmBZLB1+nh4!sMaNmv|O7@mg_h&e9(as;l_(maR_9pdhWdY6fj|UDabEmYJ|-ZVh$tfIbSc zXHvU3Hsjlh5@oVu6pUF5n*Y z2WA}qg4)A)a*bV&(pT@m3YA&P^kEpjE0V_3a`fOT!j5~cO2V@$AF84y2*p3~P!V+D zFrBp*%lL0lVdg=|ble+qs=S`9?(E~bLW96D+`L^*txz^mmVR4j)Bx^sKOsgN89 zS`0wlQ{}YJ=TAUXI1>(E_s666rr@9p1Gv{k0(S26uqK@rt zf1vMkI6T}bfI<=NkeMPun{Zl<)BCva;8_Py6Sl>dVjd`7O@{&>!eAg}6Ks1Yj(nV3 zv8U}YL|6-;ed${Kn{SFEsu!unD^=j8vC!W#@Rs^kd=7?EB2b= zra%ddpYej@ibfdslo9{>b-~oVr__9{KV>UK&Rg?5N`YpJY43 zntT7k<{lfc_i;uVS(`BN+mDT+W3YFc6ItZ_sIB+5;plq-{Hah5wMDa(L#Zqtim8IQ z7prhnr61;){2>>H7?|=Y0u9z$Sf@*2`d>+u%}SxJwH(A!?`C+ltq)cV|DsK-yit+I(uDhPm7mkRvgGUh7qlEL{a%kCk^VG+^9iVh-4d8xryxegQ z=zMfgYqbx~d}Bo4ympvF1LP^S#l&Abknzw#tOyPUL-Jn3>=}^ce;Ra+RFJ3gB*GPb(RINgZ3A$osu@f5UGGKRMG*%Bg;S!rPhD994yp{^8!D1~HW6}V*o<+d!_Y_$6 zO+fpfceL=T?P$40AC2A$VsTzMaKZx4dr)Vq;_nVhua08V4nPs&i~$y4}QJ^ z?$Vp^;Jgz4y0Hs_qYGeY6NTH-OyI$n3J~6H01hISz;G@Y-?Z@{cZV;Qy6?kIyEeG{ zb_q_Hcwpd?fmZznO)Qn$hc_B+@vlV>gjEVcWuP%$`%ntI(?_Tsr;V}RS_pMDxbPD* zxwgzLQ8F5O=(Ifq6`x1LuI6xD)cOoFpH?(U;U?VL~4?R7(-W@v_SWFGqAVv9>2)lf3G0%OViweisjuzjVD%RI5r zO74ZeRv4lV%NghmT8YcUDU{oeXxPLfiJNNwK(w0-UgWHYNU^8Xan}y$IuMF^L7Z^c z?IXl){RQIFrPPDNyP%WCh-%w}Fjr(2HmJx$&Y@r|`XP@r1;E|M_%PzgX}HO=1D`!q z#g4=lY8}r#P;Jn~>+xE^(6Rx?j<3bHYOEm1XNNXt$@Olh80B=@2(z{dLP!}GijQc6 zJI6J+blnZ3L^tC-z6P+o6NpXuitsYY1GiSM#doRmaKv98h8C|IDl?& z5I3^Wu;@tv+}p&DZU^YmICdi@io}4N>U&!F-)QPQBl!%m0+>FO2Puc8ky(KSFT9C? zcy&1}v^|Q0TO-l9t{RxC4`Jz-2AI1-k6wq(fx3}Mc{FA~-Y0pAvDplF_4A{jC@+>2 zj)VF+R%&Tj6;+3HP@K;LbNg8E#db|>=-Q0Q4G*EUgatJuHv{`8Ie7EkjS5WjKzm77 zTqkCP)U5;>*VBDik~R+yV@E)d%@tSwalrPt+n~Lz7c_r;fb2a@zYDu~@*YQSCdBZOU*0wpg_bin=CJ@3ctV99+KCF;Za6%Y1z`z?!TS`c5vtj6-aQO!8H<6n+ZTNk#6VEr3ZFk) zf_vo!kf2X$OjSMz1QSFpb5ex2mwHLN^I>9A#3Np@Lg$WlEH1hVv)0}oV>UJ93 zBi~RQoX=rcYzZ26B3_tq1eNNOAfI^TF=tr`n9wmt{X%ont_RlliM`q$vi_8Q=1 zoPt1yy;S)m1Nx7tgY!KyOY#VgwzMuRBuC;=rFZ&BQwzN3dIo)cg)USf_V8+%0YvHviazOA4}Qs zk<i>8Bs@o0j*yu;Y)1^+@rM{oP75qPu*%v8?r{ukM^j)aRO9s1>vCa zO-O%#k({GAM?Wo18t`G;I^IY0>T5(nHkB8!QEUdXcVAbz%UrVK^D zLEL&7xDwV0w?f#F^~7qFWju)X`y{pW)YBmpxz(5AqTr@|S{I&45wh4T>GNDyqJ>-3hfFWgF9DP0xGk+BEw%SqjzWNTn zZy$m~V#c({U*agmC66aph=IgKe;kriz>q{0O!BOPtE0cE!SR<6JEnjFne%Yv??pH- z<^mDNulB71lR!eeNPyA7aFp_v!KO91BjF=;K-TCaUWCJ>WmS0wsEuKc#@{`MFfT-U4Oe?F~jjcPTrDK%8p6 z1&j2u(E4y5uD|o6-IX&yhf^WQHK|LD&$HK%LNH##pI#nBTY^6J>3o?(<<(%(ulEt}u9gv>v|hs(?Lzs-RVe4G*sS z01Tyyv=^V0kR~OJYC8u?_&X?DQBw3Bde+b%G**+$9z{-)LAcn-|{F2ZWlOH}-p z0@!3(t^rDKbsW6_Rb5T2rAGmc>pY>i=2pQ7hYj$xA4j^%Ki zVi4|)VNq+1_G{vlx9LwdUeN1$0ibcIQmM6IN z*qWs&R&BO8-bC$qN#Pf4#>R0g^IPMk)nBk%CCBksYA47$zTj}|jT3BaO;8Md!Rf|! zOt@P+QMup+myhu=(MPR`YE3V=1Co!4&uS-Wj=kUs?LD?;u{8;nUaW{=iJtU)|O%!I=!llEkV{$C)K84n!nmOLEgSC z)xK$3pe{KTV- zl}eoqU&oiCW4(!LjcpkRLSKqau_dW@>tr4(cqu+-oTT}vEi<_3+RR z$PArXFTtK-Y^a-^T`(ghYm#DW-=3Y{G_ye|CB@uJH>YT9Mq0fu#WJ`(r*vsXMw>m= zDp5DLT=2P(W3LoL`_fz&+fUwJdbK%*J>88(ub^FcRw>ye-CeMw zpj&-*OIAv{hqPW{zvHZOL0`I;T1VmC(AlkJ>={0WdPNTkW>u<9GJNejiXJu1s@A1s z_YZL5dksgI`i@60 zcAWmR^cv|nvSL{D%NRuFwAoCv;siU(STyEzcvG|DrS;EnIL+w__h%)lb)Mk~o70ov zIFW3qU%s+%PG8paM5=vfx!{dC1Etgx>0bI3qT_Rh>is7&gF7qM{FpP+=E%-U)ITdJ z^2XTMG&?)L^Q^SS8xyP4?A%KIbMj7aOdb2P^BX(QDTci`PE_`F|W13U+ zsIyY-#v6-()SOeZ`c<0aZ!AOmb4nLGtKi3*?J*p=Wh@5Ox*~6_l1+2V1-q&ZHQrih zrRJWMHaKtU^wy@JKetk?>%3*yTiY^@ylO*(8k@qmcGaeNHTGRK_BY<{s7uYO^)k5N zJpR_csXy;haMuObA8#F6Ir8fg4Qky*<{f)Y^Xv1wYP~e(orY8M8!8Ph`Z~=!kM-v_ zHg;V+5H`PaisNKcx51@Dh4U_RrYD;pbzKU+F~4ge^<>Me!R7GrdDo@>leZVUE=T{E z-%ZC^(8^+17bo)0jm@l}U9h_@QRCen-n4>FX~Qe2PVd}>2MW5?y02u0z1u6nS=ei6 zSf5?^&O_F$u;0GBKL5r$Po=cNK`+CrMdRXs9@mKtXj{FyI~&=mvNTN85%V|DEtsqZC3KuzNh)ojSola(n{WW8Qq*1 z{}9|XQ1T(T=jO8?A3|C=OBWK2TBb!lhW473e$MY{nbr6hHk?-awbJO;Tc?lVV*{m& zjXk$Mgnf*d;yk_7ZFKu{;m621v(w9udTuY?_!xC`A?@_9S))74;~%4!22TH3?78#j z$H${|TxE2u#;pvZ3o&fwWeh^Stt^@gvApSJOftr89L@`I!h>Zj+j`r0!WWK7aGhZ@ zGHzd4v=A?Aeul%Lw_UJ#Awenq441cYhv>vYqWa(&o{-*-HOmW0+Fa$lNyeR$qMwqD z&C6Gw?Cq4+{FGvqUd~@-+$Hb)Db;bXT=05tmty#*G&ima;U42|<)TmNKIRpok9)h- znm=U(q*sW)Htx}!_>>tsSh41NZx1YgIv&GyR)W=}S6B3NRpN#Fx^g!7BLPcQ1PR%V|39 zYHe22;W*K+Wo#DJxp7*2^XA^BBSjNmE7gb2TZZ(HlrDd*(&nzQN-`ZS7yVXkY*Ax# zvVXKv^V@l=j2gQt(}y+A-)bC(YV5D~KfDzF?SdQk1;-xKNA*SDYJDs&I6v-x)Y$y( zVnD_Pm)E9`n{aoi`QGZF9jr-O-_q`Z|b$Ubm-*3Z$PH|rj?J;}yx#)X~*__4Y@W%ts7Ms7{TFAH@`P%IH^2GPsOGB5VzYjeBv;6%I z9Zy{htN9dz*itK-WnG-m;1rA2QX6k(UA&C>3yz&j?ZS8K61NS$;E7o3kl?wJY-B#Y zvUsUe*78cK!{D^w&804-%q!{M<}XE`EOo2jy^#R^u@A*8b-5!@A6CwcZx<&QF#f zHQl{-DP(Bg_2=^AR-VSXB#U?MVn4@vEgS1k4!!fz`Z+$F+1OBJ@!ogm&xx_Sjg8lb z-XDng`DBXcdQ*?ZheO3bC+94$H$NWw5Pb9J(}m3IEw3#;hCliFZ0YXx+uw&iM*sZz zoNh%^E34%~oY=1^w(U*rLU#i?M0n_=n`i081=#+xGmm}m!2`j&5A6x^{2%+&J`~Uk z#ud^_58kGiZn6*;{x2oq|56bCf8Cmmyk0a{aXSZ32j$-P*EKR6jdlEoW#T<@SN z2X*m*`%WDC`W_OGYvAnZ^~mvsY=cq)WZm3D%jmJeTIu~L$vOmLTItZRS&xdolL+-@ zS@7dp9UMMl4?G=xVCQlO>d!dBNbnGta%m#Jq6zFe_7^ICFyhywB+7H(2t1my!SaUH zD0YyQY=wJD-7WORR_M7^1v+rW0f%BZs9yaF z2YL!1e1*{xCv*UYU1k^m9(|rnyBM)Mwoxg7P-UH;N0pksBo1)zm^%= zXlgNZ)SLm!3{w<&8Ud3%hfs3^FBaTAfO&=CQ1VU_XCIuQO4~V6`KlETc^7LLznOyx zHU?^7F&c5Ul6vqk5ssB@AZ;B9oV((U2Apeg)npfC|7;507?@xRStg(74Z}y4AK*(v zB8>mK4DWs%*3<|R!1F>Z$SB1M0_yA(!s1`@l%J63x%Dg3j)GV7_< z_DjWPkS{DURvMJNw`*H7G8Ax(-zzy$`VcBa3w$=OLhQ<{TYhMd5550nK zhxfuXY4I7gm4R63R`~pr8Go+!!=LYe!k*j^JiKp_x-cS&rv)mhH;!!B!(~ObVQoPE zRnoZWxH^7|FQGK;`M`N}4UVS@Bd^#?+EiOC<<}SmGXqk{(7B2d?2ZFVM;ka6dkNI@ zoH1f66LO~+VfMzqT2o}3kZF)L?tbt98ckc_;A0+O|FZ|4>P~3wU3Zd-oIC<5hm7z? zMlimWy94?B7omA_7ba-dQV)$zK<_R_jDGM5*mv?^XM;9&M`yvsl?Q1ffpPHX-a+ht zNQYThB+y|+2fSJu1h$(l`0B6>CcK>mvHdpK%^l5^&ez=C$1y>2O;<}XG zNZGg{f72vf&*TEtoh6VP0pRs`6V_Bdf&7zE$l<1r#t$s1#7|tv^GtwtsDBP58O2fe z%{7?uDxv1I#GvVH2K9bi5m$s+*D9Z@d#!uk`_H_)hqPs({82R2k`}N6#l7-{!4Apd4drG z#j|1KBR{Z?jK+_QxlnehiegMM!>8dEXqx*48YTB5crk$>*)liobW8K7<+Rt?fXUX9@=o*g@#NP&}Zq4u#a4Kq)C7+D2?KiH92nZd!m; z@L{Z&8ltLq%Hg>D5qur3h7Rwyquh~IQ2fmvd>9|XGT37Q=TnM(w2I39Lx*0e0qB>f zi#z4HG3MGWXzyml*JSQf-p~g{**i%~g+g}VMA13)T@Z^QeQsl1wOK;zhf8 zOKR7fL+HFu9&a9Jhw=F_F#ehcC!|{7a<3fhfUw6)>qD0=BJHE zOLTBy(@u;N{SF`Niox8Mv=xLJq1z`AU-18ih^P-BaLgJ7_PXLy+&(n6md6xnMwC9Z z7FUpMa95J|pt#8_YDU?>T(41*XkhZF`&SAefXNMir!q;z?fbdJNWr< z|Le8LJDN;=KJyCJ8Er#dM;@r(&4q0MxvVTFhaO5IXL_x?w4{g^+- zF(=aa>GxrMN(ctsDTVtvDmeX$1J1APfrfpSIG$yTyKihkwHMYnXUziEyPg48sUv=h zUJHro;<#Cz9c9lbpkMk+D9z#D$fv7pOB)N5DZ{8#i5W zL{6_5=q4@FO@bkCSZ6n$CtC@{8-ZS-R;1qq%Nc)VsEZegE<&}#-*;UR>N8a1JRMI$&D zT&8}K?Vv49;iyDsg^Qkz;5H@%v@c?KqF06D&h!JR#dTO#xDAr`PC!@UFt8ow#O=r0 zVH1NL{*l*1y)+$g%QM6PS$?$Bj-g6lvtg-82(rvG;f&)C5Hyp+HLuv<3%dk-3T*=^ z`#W%8l7=>(gD{mQfbZXjp)Fae{h?Du6A|pCyk|$KRUf5b;9($cJ-DANpLW2X?_t=# z1F#@m2EW8-!-G@w$dynF=c_ZJK~NYAS$5{ZiYB~Uv~1tqx1R`H7^ z6z?q>>TTr4!0VsEPXcj|k`30+T!Nk6d|;QD2l~cufTt%LN@YaA_D}|Fqx6vTU?i&7 zKZU!~&)}S=G;Y4I1rMrFIACLqvq#2ZgkBOIm9Nu8@AUw?xdyJ%Dbt!%md7nW)zEJu z0Jg^I;`1>-T-r_MhSx_?G3y`>l9)wg zm_R)m{S6O|mGPlI4K;K&QUzLxluLjj-ZkHVC$wI}+Mjz-C1V3-y=Q|fKC&pD7J}Qg z1gODkCe&Xzg6DSyQs3NcQG;iI`fu%hXH*qQ7cEJ$1VID^1q4)rqzektb-d&(AUQ}* zatV^bgrK6BF@mUwm?P#aig^rRzfaeR5hnipCeK(-F@JT-XbB8l8WNg;>XNdDc97 zFiw%Bt`8@#s%AC}(_0=U*J}-%t)f9Kv6-A_C8m~sZeT74y3i_(BdjS`mP+JGcD~|%JB?)s*EotYVRT4bmVH@wl5OawO&`Cw zaGw`XVgbv?vc%W!v}~ahpgH9))!@o<0?6vZ zXSPiewk#i~>p;5yURz^}>6!o%0)Xn}t5rqLDKP(t}Am z*u^_1S%_Y_U0mPy?8`?HRvFBrvO_5>Y{V5NJB?3b^Hng-Qi-PT97zprfpk71jpi?X z#EvHSVv3m7z5eoRwlPhNj6?QuHRp}lvc%prze$H4CrZ=9%0x0%abVVa-RRhE6LQU6 z&z$5Jas4dK@s+O|JrWTi^A}wxLa`H9=4?R|wam#yH;KmC+0w{s@^q}ply2(mVM*)5 z>{_<_kRY@x1@C#ow94hE6W%wt>bztdmQLp)wwiDkmOWx3BO^&VyNReUiCf%1fMZV5+|IuHS%AzZu1i{Pl9+#$ z>D-Gat>e+`PQEz1X>Lgqo}6Mq!3m_CJ&DQBddN1^$kE_1Ia(d~np2$9krGasP^F0} zi*J%8g;Uy8G4M7Mi_N5?zlX4L6)iG+s6*v9)tQluIF*W?XKv}i)KqIkt)*FXSM!?P z5V0a=(_5LvaN&#GhuzOq}WH>Zk0MJLCF2Nn^Vj=CdY^fzL%<{z!HcwB2K5yzwaicBiM1Tf6vooCLU3fUa&*~o2 z+xM6mNe`x^mi}Z_tI4`9?nAaK<0&gif)pk`V;Vzk$bQc#%Fw>f&232~kGZ><^4%pY zQ?(lR>Tqf}`-M$zd&!c=M3AeB3u){m7GCs$t&i@<=ER7T%$_eS^GP0!z%;2r3p%o_ zT{2YXYeac(?sB{#uNd+5X!IEu+V^S-Gb1;e8e7I*gi5eY({$Jb<+W^zycTnu`G&a& zc5o}AM5)IM4O&0iogI-#B#Ck9RDZe)W#_%aRN(%UXXeG;NoLWAjTvzN@0iZF~v^gd1`; zuM)^_y$7AK5F-=s5~e)yDtpQtDLh4qw%Sc&1!W7_hZ$qo{YauMvxrT+bcv0BdWFpz zIGm2n%cZu7?khj#wshB_XXy`ZgHAnR zm)AdJqo!+MIg|nI_^d@6ch;~2N2O@dJUcFDm=rBuh-DEMZg9iBj&M7tDbe<|!ZkxzfUreNO(e=#njy{Fj zJYjAn`P3cnE0?_(K^vx2vAjt~S?{(?n%F~;*0-5Zhh0Ww>@T2{eF>Day(3-B>r3kf zA7(>uN3pg>Uv6oi6nfWp5!2h6$NENR+Rc}}!Db}~(#OjIY+83CI?syihQG2RNw+4Z zF=#I9wIhuw$lPHf2gFzpDM`AdW6MTOdd&o*bIAB~1sjxK!j0M zJtG#egije%C@M)`_S?{n!e{LB!>cS@CX62ZUe8_gox$vSN3s>K53|Kl{-mjI!nAoA zG;KjKim@5kv}#BUwhZy2Aietn(!{@%pe3{>dJ?E&=O{tKu1{s8MX z!;mw(eUWLD^d##kJDHNJ18t}Yq$gO)$?m969LQA@{KHm7e>^euOV{UL; z2J|QH?RnO`9G~Cp?nfJ)f~h32iFKm|+@U;&)Bs>fLzE z7|Tj|SQ?fU!{=Hm7P6r_hNO8tjt1!u;l|DzN!jyPv9k^l6mae#mKC}X3*BmWBiw@? zn8?zp>8~)gE0E59Nu*CY1yp$YF>}&&qYJmR$#BOqcBWjBo}HDUbp5k-zP=^Q;?gFj zhG`5X+wJJ4f+5{dSHtuXMUvG1#6rvuvyRUy*oeNe6oDnFuM`Z}l>xJvbJc8S@eX6C zE63O}dvj8pqeCVx6WJEe=Ulkser6GyO{wumSccyX_NJ2q!rcWY&v^MWX@Erl}X z&Edw}=2Oa=V9Gf%jd}JEp`{x+c4A;POE15S<-Rd=z9o$;yUEeLCBxXokDa(T?gnht z2}L$xwJA$)97^Y+`;(zyB*hjPkYU*n3b}NYGo2bk-EYRyn8BX35bxQhUDcvPwYZ3Ee4 zrH*zYFOxDmpDnqpbvEAnWEL@xxoYH=CQp;*45T~Oa`b7mBRh@d1yyN`;@4vNr2Bie zS2l|qqThwWUbS&z`TN+#W6qd5E=r|mR@=!=uVY6N$Ft(f2qyRPGE2`INnZ7L8SiKk zZSUesZ)2Y^|Ib^P#r!ZzaEPQgcKxuF(18|Cx1psQrZfMULuun|0qLE5&&H)}j7cyDzgFzhWiVQQ;M9n6jDM zC+I*^x8An1eym2G-S;t3&lWb&0n2>ShETVt912y?CXZYN60_AM{X@8REY9S5-AtkN z8orcmyMjsH89|*d`_cBLrEIn6E9T~rKuH>-X%oiF!{7hT{8V}5x>*ZT?s73!F{f^Z zgQ;-dZdN2~K zPae&De2^{e{EEE}97cDVcq~SYUl1u z220wbjwVyr6}9YC*GRUi_8r?9f0(sAXkaUX1SGGUNQQV%c9W+`JFyHVa8VGgoNP^2 z?yk(Uv3@|8vqPAqycQ?^DuYHCccOh0`Z90D-fV3ZF>d@e#%o-`##W25m#zbti^ob9 z#T!X&3G3K(dwDi^cnceqU`fx*RH?@LG?Q@c%z`|IuzTCIn0EP7wr=kLnx+^+)3gd{ zXIU8OV5z{H$~?AYJ(e3E(4!nl4X%@JG&!Bi#AloxXw!$=%(XI*mF`cX&`Luxcwxvy zI<>M*QLfYl(-JEcqG&SXN-p6J~%x6QIcW|CErc@pDh}C0ChB5=#h@4~0@=P&losGo?^_Yg*wD!Y+&qrmcR(@$MOLLJOFgrql&F#_1 z=9UFe;It45uei?m&8x$Q@F!!Z^ym!W_!QudTBcA7_CK(kNsG|hyp4Ye3{+5YDpUSPU!O0iQFd332b>@30w5x5O*fq zlb6`=hQUm=-Aj_%uFl3BxdmRo+zfK{W%2uX+ZePscCJNo!?q|cK#^e{$JDH zYBbIf6%nxpF5!R34Z$FCBcKQbLmLZ(Vy(kmpcwIT_!Z#MqVg~I%0t`%VIPF)u;0Tz zjDo#jUjW1aTf`PX6Gbc%)M>A(W`eUP^bb`9(qU}N~ous0(w9=-vv4*nMSClDJ0lmaQxC7?G0 zH{t&VECq^z@rWG*8d0VKJR6z?Y+dj>AOd&}e-&5{=zuzske3eb5B~&W8-YQ93jDLs zh5rgL9sU*gaeylFFM;KO9H2X(3#|(9guM{h4D1Iip>0L^a{vK+XwKl(U_R>X2TX-u z1hxf-K#u_5fqxUM0-i67p`Kod1w$)^FFZF*z)FAv&=v6m&s6kt%W@Yyc-w+90jzI-xWwhP6pT;+AQEca&G}i&}#rcXl`KqFKLlT*qy;; z;Pb#PNIj~Q{4n)o{ z_#I)Jfk%M1fL(zQU>E#D;4?rC&;~T4>=5Xa!2Q4vz*@j**qZ=1paprZs5={627fK= zonTkwX@Zvnd!P-0{}NaZTmUSg^MO7#aRe;@o;;K3-@0(%$qVX)r-(_r5~tPju~_y{CJ zj|9g7zQ`X0UJjfDqJ(9EFz7dscN*Fhuqq&coeg~!;?l6!1H<5d0y{z*fSi>;Ewtm% z4#DmMn-5zHTmh6I?+Ek;a18v9u=fgcVP6&I!|sR}4}K7AAK1&FcLkpW-T+sj&jL0g zCmsGiVID9MSOA=XCI_xVIeplXfD8P1FoSjqBSUF$-G{bcPS^#(WHpufwjuxN@m_mO5=tB<& zp8#h16=fgJ{jBBvX)F3^U?*Jg@*hL1g+C8)h4wqN0pLAA9B>V~BQOK711=)I24&vDo`d*# zAP;s&Xt9V*05pL{U>dYs#P$fc0pA{Mg#2dkD(IomZ2*u6 zZ3$2W-3$6~a3-LMm>l#?utmTf5c7q$4Lkwd4Vne88Aya*1N|Ow8MYp@Fz^bX6N=x3 zzY+E=*sEaoKwb`fCHUiD?}nWKdnvdH*b2=CNCZB>_e1PD$^`I2?#-K0JI#~<=}cC1@?UK zUF1Cit|Mn2w1Kdzp)ZE522B$7DL@Ur06u<24kJe!tOV8r#zI^Jv;jyH*#<}ea=;Q` zIC4HC)(c31-vB=eiR!Rt0M7tf;2iW1$hiyq4EQ2og**dbGVD*l8raT=7lBWMO;9El z_8Fij;_m=)*n7Z+h+hN7z~2kr08~TY1>n>5`ktMS_x#JkZ{ff~U3i7UFsQf0pB{u8 z$}g`=BP{&4*RrEUB(&>ABvhm7MWyws>&4`GH|oU|2P%z`Q1S5@)1gb$^f8j%G8)H7 z^(ay*mhL&-r&y*}_4Hy{qYaJ4a;Ei4CGwVyJ|!Ld-I!jYFrclmM3F?4$0|7}`Hod~ z@h@`h^Y@!_t5 zv5k^l(sqkVbscupTuLkF;;>TX{M(C6LrhyXnsfFLpDi6dZ8twQz)(d+(ab_OcosX6cUWv%I>z%y6;K23e!E+n>uC z*B!NxGud)6Q?Boh`}X_$#{njY)3tx=WCw4XyH&T`4e!@3@s@bBAR%l#qHv}>`v}|xW#AO zjkl=;i)d2wT{r`MZ{s*PS^H)hTB9`Nkk%t73zS_vP%jFqad zlT(x`E0Q1PFMhT6K~WV~{X;YSJj~Cl%6M}f0{s2FoB{#@BgO>;hNj->6!bj%&iSCI zar(`{akJyqtP&T`o)emKX_Hi#!j>&l!&GkHP|;7(e>KNBgHxFsp4AjEHzMz5{J{3ZhmxR}GLyr>v7^<`-@F(*-N~RL zV5VuqyrY6^V-$Yz&aI8>3d4G<)*0{ruSJftAUHO?qdN zR}7m`IdsHYNtu+jJ6>Pzx;sfMOt9=&!Ud(^t9mVhxC-$usWs*S^HZ;0aaK;#zqw?h z+KvZXyQS|*Z4HYIuYTDrW51WSrtfthp60MaE6oG=heNV8GmrJzR+U<7exWMsw3Bm( zevNg?my(+zw6A7MU2=}dX)egt%Kb2F*46%(Q;$UCb;~cU&s)C5S}FhL#D3iiN@lyv z4-}i1w@{%^^|lEy4?b1P#@ViTv%m21mInTC;~huN4{saO`MTcolP=eXd7MkUK5~t8 zY2>K)Ph4b2Kdh+1Qz-)cFBJBDb#GDDIk)FMit5{xs*A*r4~!baTN$ezC#5@2TVA&B z#iJtmW*4jb7>#VbQKFQlvRLzmn|kwD)rVUTcW_!6GWS$;67YO>|wL&pd3nlDbD((kZ}yj=gv@__i;2WR#ywUwM8S2*SI(ObG5UvH1< z%06$ukUdaRX36vr&$x!^sY)w5Yxn5vcDRePZkAqmSA#x9*>!zZN>28$D453+Uv<8A zrpeS^Ei-NH3od8)y4x%*@vre&I{RwGJB3-sF%|L3GD#ZO=Y+3nj`tdp(AsI%%?VDy z6I1%M-I2Ldpww|*bftlTzx|YBIu*vp{GBT9Rb0AVp?6}FY(~ z9!Xpjabn}V5uTA+K<+lopY30 zTX=4$QQWrHox#pK{A?DjbDy={Xnj|KYTSB*GKVSa=S94DG^zf8a{PuT@jSC-d3^WBi3!`&yj^_u;lR4}6|K&d*bL_aL9?s#7#8G{preR>bc=gCz^2akI_ z)g;jNgYo7jZ|$|?vNPS>C2gw9j%+>`xH?IySvM+4_fYsGhvCVt&bQnoh35}S51vvcZcrCkue(8nH+Ae&O^KBv7wRYY7_BPqb$j=-wH?pJL&V>zX z*V@`*_t>vVJJ>1Fer$ruqy3M34SfY?s3@R(vfqkizT4E!a^ud)nY144pPO~3%jIdj*l@Vp(U!?`yJd|~Gd!1F)upS3 z=;6{MSsd?FSr4OE`Z=G=#WS;Q#)se96u9pE(=ECxYi8~$o{`;jRJ^C#MtgnAn0l<& zqUvl2Zc*xUB=*k!;}^$?rAW#Z$rP#INGUr$FGBvz5~JlqZ!5pbwbYXu zG}-9EvVFy4w5*KZ)O(~!ovU{I?e6=|elZiC+{;xO z2bM(VCPv0ZCkqN9von)3vc4{k-WaTQFxX)4R&j&cVsU%`{pTp@r!mtnKY&*1tm@fW zto_qxY0ds23DzNa;u^+cYMs0OI1-coBc%S_Lhaw}KUECKc1TicN}7jqWUu z$FJy%{p|@a`l_Cjl{JmgI*d1k34cBiMWnN|(Eq;h{Hy={(|)`y6V~_B0r*+$IID0*sZ}+LtKto* zvASeNbXr_=mikWV;kwCbaq;<)vFU=G)U+&JTkWvnx+&0L>1IZ!CC2L-YwM;(r^Z7_ z%Ml3R;h{&F*>KI;?eq+o=Kan3wR`dH-pB|zU#pC4uRs1<&3~;ZAvsZw08a?$DlAr* zp4Psb0sanXs4;fnTW{!of8x^LTlJUFPwh!g%eLnIy(!=2A395TH2<>6@7J)uZ}Kmp zpEj8wNRQ68wD|i*e+c}i&HnMl=-;>cW90w!M*n(8{KdBap72u>(5JuJ>+kY^+UIY# zv%hclhrmyp`{Qi-dq4jX`Q>@mew5o^j>0==dS?5pM35Yl8J$@me34`nL}zBE=Lu`Y z6TY&N)6;}1udi8Ob6uPry5CyyL(IIdWnVMhks|8yd_TWILAGCSo$70pOoc!Bi7tFT z-xm=6c<}i=zI`I!na@w;J0!Mi_V#~?=Ltop?{Qb5-HvZr?Qu6gKaY=cU|}WpZm@+u zpYNECeEWWGse+tPmq1>yg?Da@YhZRvnpg6mWc!%-0{6`5AgdI&q3MZkY3@lesc{0g zaFN|{u z%oEn-6>J_S%n7lu$%+dy&k45py8Y1L@7v7@2{IQ5b7Rs1vtk^C+YHSJ4a!dmw)n-i zU7XQ==fYr1KdV?5VY~RraY0rYs6Qoiu$yq(Xs3Bze4ve4e6W3cy>6*C$r%*V_#HlJ~XWe%^=wkeg z=fn^D1-2fgr}k zFCiw?Is4nON(!;?{;~c4yS4>~rUhI0B}E723y(oTbg+HlKONhUf6D#GWBkwkC_Khq z*i!qs`+s+g3qpf%e1pE87r*Gkp|Pof!q>&ubMu#N`tx=3kH;6UeOv=Fa9y+TcFxX; zH8=a`HU^pruPNWQYf~V+PT{=FiVkuw2nlw#`g*;zw}M>|V;S3CkMQ*qYaz&uNfy3F z-ON0P@^icfyW^a}we$P+D^w7h=A98^!E3+fcn#(iA}=`scLdxA_J5@ZRlt+AW-{!jbUUg(GYaS-mBi|}=TV<51}2yHKz$aieNZ^jDi`}Tv|F@DgW z0mAz!jstpK*icmVy;0x#`|C}+z3YVue7>-6?FBd&a^jqA+OOln`({iqUs%7_w|0N+ zYIO0p{s!~;!fUgT@xzIz_sEFO9F`NGZ4#T2VbVSZ(f&5b&~6`nXbA9a;!nRK>lN{Z JJm1g!{{WF|k&FNU diff --git a/java/serving/src/test/resources/docker-compose/feast10/feature_store.yaml b/java/serving/src/test/resources/docker-compose/feast10/feature_store.yaml index 87e3310c3d..7554725004 100644 --- a/java/serving/src/test/resources/docker-compose/feast10/feature_store.yaml +++ b/java/serving/src/test/resources/docker-compose/feast10/feature_store.yaml @@ -1,4 +1,5 @@ project: feast_project +registry: registry.db provider: local online_store: type: redis diff --git a/java/serving/src/test/resources/docker-compose/feast10/materialize.py b/java/serving/src/test/resources/docker-compose/feast10/materialize.py index 35641a1c5e..c347728c68 100644 --- a/java/serving/src/test/resources/docker-compose/feast10/materialize.py +++ b/java/serving/src/test/resources/docker-compose/feast10/materialize.py @@ -1,12 +1,30 @@ # This is an example feature definition file +import pandas as pd +import numpy as np + from google.protobuf.duration_pb2 import Duration -from datetime import datetime +from datetime import datetime, timedelta from feast import Entity, Feature, FeatureView, FileSource, ValueType, FeatureService, FeatureStore print("Running materialize.py") +# Fill our temporary data source +start = datetime.now() - timedelta(days=10) + +df = pd.DataFrame() +df["driver_id"] = np.arange(1000, 1010) +df["created"] = datetime.now() +df["conv_rate"] = np.arange(0, 1, 0.1) +df["acc_rate"] = np.arange(0.5, 1, 0.05) +df["avg_daily_trips"] = np.arange(0, 1000, 100) + +# some of rows are beyond 7 days to test OUTSIDE_MAX_AGE status +df["event_timestamp"] = start + pd.Series(np.arange(0, 10)).map(lambda days: timedelta(days=days)) + +df.to_parquet("driver_stats.parquet") + # Read data from parquet files. Parquet is convenient for local development mode. For # production, you can use your favorite DWH, such as BigQuery. See Feast documentation # for more info. @@ -27,7 +45,7 @@ driver_hourly_stats_view = FeatureView( name="driver_hourly_stats", entities=["driver_id"], - ttl=Duration(seconds=86400 * 365), + ttl=Duration(seconds=86400 * 7), features=[ Feature(name="conv_rate", dtype=ValueType.DOUBLE), Feature(name="acc_rate", dtype=ValueType.FLOAT), @@ -38,8 +56,9 @@ tags={}, ) -fs = FeatureStore("") +fs = FeatureStore(".") fs.apply([driver_hourly_stats_view, driver]) now = datetime.now() -fs.materialize_incremental(now) +fs.materialize(start, now) +print("Materialization finished") diff --git a/java/serving/src/test/resources/docker-compose/feast10/registry.db b/java/serving/src/test/resources/docker-compose/feast10/registry.db index 774b4938e71d16cb5c9d598c26500e528c86a075..b9a19475af0789de930ed80b73e5d39b707c1fcf 100644 GIT binary patch delta 207 zcmaFL{*8%+%Z$rtB8y5rZ(3?%aY=kZQGQlxa)}TFmll^A7khksN@;Fxr2w-M7nB*F znc{`6R*KP3NyXeG)hIbF(L&cWEjdxwB+)WWH_5=#KsPbP($v(_$lT1>GEIwzoOyka7s~T*<@uVK~!TW?zdBC;b{1(VwJ(cD8-*zmYP=*Uy_-d dT3nKtTcE|BT$GwvlA01FCL}UBkkOlw0RYr}K(_z@ literal 997 zcmds#y>1gh5XZeva=8paK5ZllS8$|2mV9@1Y~oA9M*)Zy(a@}x-956>ePwU&gq0#L zj06#);VF0kT3!GOBp!l>h9)8iAvTVI7_OkE+L_Vp%zu6}Fn|EoxRRAnL~>`jJ$Eij z?*5M7J(sI2$u;4b8si3BflXK?gqKM&c9vY2o2J4anQhc_H+$iTvi^3^4To&h9c*0< zyOcg0b@^b}4~M~Mz{9Y!Zhv|5;m~s4KY#HK_1`w(JA#e5u-&Xw@t72v>H?pY@>8v8 z?4E};mZcR@R10UtT?d;ocIsK~2DE7Ph;S-R9j#iZek5q2{WDm6PBr!Cif2;2oT|N2=tI<- zWqUG>6!o$wie)OyIGGZK7s=SzW=0.13,<1 + +# for source generation +pyarrow==6.0.0 + +# temp fixes +proto-plus +Jinja2>=2.0.0 \ No newline at end of file diff --git a/java/storage/api/pom.xml b/java/storage/api/pom.xml index 4e7ad390ce..8b3532b4db 100644 --- a/java/storage/api/pom.xml +++ b/java/storage/api/pom.xml @@ -36,11 +36,11 @@ ${project.version} - - dev.feast - feast-common - ${project.version} - + + + + + com.google.auto.value From bf86ebb17dad3e43ab70e904936b7941fe261c4f Mon Sep 17 00:00:00 2001 From: pyalex Date: Fri, 12 Nov 2021 19:06:40 +0800 Subject: [PATCH 2/3] fix sdk Signed-off-by: pyalex --- .../java/com/gojek/feast/FeastClientTest.java | 29 ------------------- 1 file changed, 29 deletions(-) diff --git a/java/sdk/java/src/test/java/com/gojek/feast/FeastClientTest.java b/java/sdk/java/src/test/java/com/gojek/feast/FeastClientTest.java index c458a06425..29185cd153 100644 --- a/java/sdk/java/src/test/java/com/gojek/feast/FeastClientTest.java +++ b/java/sdk/java/src/test/java/com/gojek/feast/FeastClientTest.java @@ -21,7 +21,6 @@ import static org.mockito.Mockito.mock; import com.google.protobuf.Timestamp; -import feast.common.auth.credentials.JwtCallCredentials; import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow; @@ -31,7 +30,6 @@ import feast.proto.serving.ServingServiceGrpc.ServingServiceImplBase; import feast.proto.types.ValueProto.Value; import io.grpc.*; -import io.grpc.ServerCall.Listener; import io.grpc.inprocess.InProcessChannelBuilder; import io.grpc.inprocess.InProcessServerBuilder; import io.grpc.stub.StreamObserver; @@ -70,24 +68,7 @@ public void getOnlineFeaturesV2( } })); - // Mock Authentication interceptor will flag authenticated request by setting isAuthenticated to - // true. - private ServerInterceptor mockAuthInterceptor = - new ServerInterceptor() { - @Override - public Listener interceptCall( - ServerCall call, Metadata headers, ServerCallHandler next) { - final Metadata.Key authorizationKey = - Metadata.Key.of("Authorization", Metadata.ASCII_STRING_MARSHALLER); - if (headers.containsKey(authorizationKey)) { - isAuthenticated.set(true); - } - return next.startCall(call, headers); - } - }; - private FeastClient client; - private FeastClient authenticatedClient; @Before public void setup() throws Exception { @@ -99,7 +80,6 @@ public void setup() throws Exception { InProcessServerBuilder.forName(serverName) .directExecutor() .addService(this.servingMock) - .intercept(mockAuthInterceptor) .build() .start()); @@ -108,8 +88,6 @@ public void setup() throws Exception { this.grpcRule.register( InProcessChannelBuilder.forName(serverName).directExecutor().build()); this.client = new FeastClient(channel, Optional.empty()); - this.authenticatedClient = - new FeastClient(channel, Optional.of(new JwtCallCredentials(AUTH_TOKEN))); } @Test @@ -117,13 +95,6 @@ public void shouldGetOnlineFeatures() { shouldGetOnlineFeaturesWithClient(this.client); } - @Test - public void shouldAuthenticateAndGetOnlineFeatures() { - isAuthenticated.set(false); - shouldGetOnlineFeaturesWithClient(this.authenticatedClient); - assertEquals(isAuthenticated.get(), true); - } - private void shouldGetOnlineFeaturesWithClient(FeastClient client) { List rows = client.getOnlineFeatures( From 92973f6d0a4f3ac16c93095d8627ca76149c4c6d Mon Sep 17 00:00:00 2001 From: pyalex Date: Fri, 12 Nov 2021 19:15:10 +0800 Subject: [PATCH 3/3] update dependencies Signed-off-by: pyalex --- java/common/pom.xml | 1 + java/pom.xml | 2 +- java/storage/api/pom.xml | 2 +- java/storage/connectors/redis/pom.xml | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/java/common/pom.xml b/java/common/pom.xml index f0d79b1732..0c5651876e 100644 --- a/java/common/pom.xml +++ b/java/common/pom.xml @@ -102,6 +102,7 @@ org.hibernate.validator hibernate-validator + 6.1.5.Final diff --git a/java/pom.xml b/java/pom.xml index f7169cec54..073616b69a 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -59,7 +59,7 @@ 0.26.0 - 2.12.1 + 2.13.2 2.9.9 2.0.2 2.5.0.RELEASE diff --git a/java/storage/api/pom.xml b/java/storage/api/pom.xml index 8b3532b4db..6bce6cec4e 100644 --- a/java/storage/api/pom.xml +++ b/java/storage/api/pom.xml @@ -70,7 +70,7 @@ junit junit - 4.12 + 4.13.2 test diff --git a/java/storage/connectors/redis/pom.xml b/java/storage/connectors/redis/pom.xml index f65cbd0f96..7b0c944a66 100644 --- a/java/storage/connectors/redis/pom.xml +++ b/java/storage/connectors/redis/pom.xml @@ -87,7 +87,7 @@ junit junit - 4.12 + 4.13.2 test