From bc42133d7996d539d440481ca4e0aaf70b6b773d Mon Sep 17 00:00:00 2001 From: zhilingc Date: Tue, 17 Dec 2019 18:24:04 +0800 Subject: [PATCH 1/3] Catch errors thrown by BQ during entity table loading --- .../service/BigQueryServingService.java | 60 ++++++++++++------- 1 file changed, 39 insertions(+), 21 deletions(-) diff --git a/serving/src/main/java/feast/serving/service/BigQueryServingService.java b/serving/src/main/java/feast/serving/service/BigQueryServingService.java index 701e146ee5..e7fe917f5c 100644 --- a/serving/src/main/java/feast/serving/service/BigQueryServingService.java +++ b/serving/src/main/java/feast/serving/service/BigQueryServingService.java @@ -87,7 +87,9 @@ public BigQueryServingService( this.storage = storage; } - /** {@inheritDoc} */ + /** + * {@inheritDoc} + */ @Override public GetFeastServingInfoResponse getFeastServingInfo( GetFeastServingInfoRequest getFeastServingInfoRequest) { @@ -97,13 +99,17 @@ public GetFeastServingInfoResponse getFeastServingInfo( .build(); } - /** {@inheritDoc} */ + /** + * {@inheritDoc} + */ @Override public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest getFeaturesRequest) { throw Status.UNIMPLEMENTED.withDescription("Method not implemented").asRuntimeException(); } - /** {@inheritDoc} */ + /** + * {@inheritDoc} + */ @Override public GetBatchFeaturesResponse getBatchFeatures(GetBatchFeaturesRequest getFeaturesRequest) { long startTime = System.currentTimeMillis(); @@ -123,7 +129,19 @@ public GetBatchFeaturesResponse getBatchFeatures(GetBatchFeaturesRequest getFeat .asRuntimeException(); } - Table entityTable = loadEntities(getFeaturesRequest.getDatasetSource()); + Table entityTable; + String entityTableName; + try { + entityTable = loadEntities(getFeaturesRequest.getDatasetSource()); + + TableId entityTableWithUUIDs = generateUUIDs(entityTable); + entityTableName = generateFullTableName(entityTableWithUUIDs); + } catch (Exception e) { + throw Status.INTERNAL + .withDescription("Unable to load entity dataset to Bigquery") + .asRuntimeException(); + } + Schema entityTableSchema = entityTable.getDefinition().getSchema(); List entityNames = entityTableSchema.getFields().stream() @@ -131,11 +149,9 @@ public GetBatchFeaturesResponse getBatchFeatures(GetBatchFeaturesRequest getFeat .filter(name -> !name.equals("event_timestamp")) .collect(Collectors.toList()); - TableId entityTableWithUUIDs = generateUUIDs(entityTable); - String entityTableName = generateFullTableName(entityTableWithUUIDs); - List featureSetInfos = - QueryTemplater.getFeatureSetInfos(featureSetSpecs, getFeaturesRequest.getFeatureSetsList()); + QueryTemplater + .getFeatureSetInfos(featureSetSpecs, getFeaturesRequest.getFeatureSetsList()); String feastJobId = UUID.randomUUID().toString(); ServingAPIProto.Job feastJob = @@ -147,25 +163,27 @@ public GetBatchFeaturesResponse getBatchFeatures(GetBatchFeaturesRequest getFeat jobService.upsert(feastJob); new Thread( - BatchRetrievalQueryRunnable.builder() - .setEntityTableName(entityTableName) - .setBigquery(bigquery) - .setStorage(storage) - .setJobService(jobService) - .setProjectId(projectId) - .setDatasetId(datasetId) - .setFeastJobId(feastJobId) - .setEntityTableColumnNames(entityNames) - .setFeatureSetInfos(featureSetInfos) - .setJobStagingLocation(jobStagingLocation) - .build()) + BatchRetrievalQueryRunnable.builder() + .setEntityTableName(entityTableName) + .setBigquery(bigquery) + .setStorage(storage) + .setJobService(jobService) + .setProjectId(projectId) + .setDatasetId(datasetId) + .setFeastJobId(feastJobId) + .setEntityTableColumnNames(entityNames) + .setFeatureSetInfos(featureSetInfos) + .setJobStagingLocation(jobStagingLocation) + .build()) .start(); requestLatency.labels("getBatchFeatures").observe(System.currentTimeMillis() - startTime); return GetBatchFeaturesResponse.newBuilder().setJob(feastJob).build(); } - /** {@inheritDoc} */ + /** + * {@inheritDoc} + */ @Override public GetJobResponse getJob(GetJobRequest getJobRequest) { Optional job = jobService.get(getJobRequest.getJob().getId()); From c85f75a10640019662198c4e9906d5376fae2f80 Mon Sep 17 00:00:00 2001 From: zhilingc Date: Tue, 17 Dec 2019 18:28:10 +0800 Subject: [PATCH 2/3] Apply spotless --- .../service/BigQueryServingService.java | 43 ++++++++----------- 1 file changed, 17 insertions(+), 26 deletions(-) diff --git a/serving/src/main/java/feast/serving/service/BigQueryServingService.java b/serving/src/main/java/feast/serving/service/BigQueryServingService.java index e7fe917f5c..d1658bde54 100644 --- a/serving/src/main/java/feast/serving/service/BigQueryServingService.java +++ b/serving/src/main/java/feast/serving/service/BigQueryServingService.java @@ -87,9 +87,7 @@ public BigQueryServingService( this.storage = storage; } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override public GetFeastServingInfoResponse getFeastServingInfo( GetFeastServingInfoRequest getFeastServingInfoRequest) { @@ -99,17 +97,13 @@ public GetFeastServingInfoResponse getFeastServingInfo( .build(); } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest getFeaturesRequest) { throw Status.UNIMPLEMENTED.withDescription("Method not implemented").asRuntimeException(); } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override public GetBatchFeaturesResponse getBatchFeatures(GetBatchFeaturesRequest getFeaturesRequest) { long startTime = System.currentTimeMillis(); @@ -150,8 +144,7 @@ public GetBatchFeaturesResponse getBatchFeatures(GetBatchFeaturesRequest getFeat .collect(Collectors.toList()); List featureSetInfos = - QueryTemplater - .getFeatureSetInfos(featureSetSpecs, getFeaturesRequest.getFeatureSetsList()); + QueryTemplater.getFeatureSetInfos(featureSetSpecs, getFeaturesRequest.getFeatureSetsList()); String feastJobId = UUID.randomUUID().toString(); ServingAPIProto.Job feastJob = @@ -163,27 +156,25 @@ public GetBatchFeaturesResponse getBatchFeatures(GetBatchFeaturesRequest getFeat jobService.upsert(feastJob); new Thread( - BatchRetrievalQueryRunnable.builder() - .setEntityTableName(entityTableName) - .setBigquery(bigquery) - .setStorage(storage) - .setJobService(jobService) - .setProjectId(projectId) - .setDatasetId(datasetId) - .setFeastJobId(feastJobId) - .setEntityTableColumnNames(entityNames) - .setFeatureSetInfos(featureSetInfos) - .setJobStagingLocation(jobStagingLocation) - .build()) + BatchRetrievalQueryRunnable.builder() + .setEntityTableName(entityTableName) + .setBigquery(bigquery) + .setStorage(storage) + .setJobService(jobService) + .setProjectId(projectId) + .setDatasetId(datasetId) + .setFeastJobId(feastJobId) + .setEntityTableColumnNames(entityNames) + .setFeatureSetInfos(featureSetInfos) + .setJobStagingLocation(jobStagingLocation) + .build()) .start(); requestLatency.labels("getBatchFeatures").observe(System.currentTimeMillis() - startTime); return GetBatchFeaturesResponse.newBuilder().setJob(feastJob).build(); } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override public GetJobResponse getJob(GetJobRequest getJobRequest) { Optional job = jobService.get(getJobRequest.getJob().getId()); From 94290e7179f1cd527d474009fbd16a2ca054314d Mon Sep 17 00:00:00 2001 From: zhilingc Date: Tue, 17 Dec 2019 21:31:50 +0800 Subject: [PATCH 3/3] Add logging at grpc service level --- .../feast/serving/controller/ServingServiceGRpcController.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java b/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java index 0160150558..0eb9d1e345 100644 --- a/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java +++ b/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java @@ -74,6 +74,7 @@ public void getOnlineFeatures( responseObserver.onNext(onlineFeatures); responseObserver.onCompleted(); } catch (Exception e) { + log.warn("Failed to get Online Features", e); responseObserver.onError(e); } span.finish(); @@ -88,6 +89,7 @@ public void getBatchFeatures( responseObserver.onNext(batchFeatures); responseObserver.onCompleted(); } catch (Exception e) { + log.warn("Failed to get Batch Features", e); responseObserver.onError(e); } } @@ -99,6 +101,7 @@ public void getJob(GetJobRequest request, StreamObserver respons responseObserver.onNext(response); responseObserver.onCompleted(); } catch (Exception e) { + log.warn("Failed to get Job", e); responseObserver.onError(e); } }